From a2665937ee7ea08fb908a7778092bfcf8886295c Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 5 Apr 2018 14:38:29 -0600 Subject: [PATCH 0001/2309] grid-manager proposal --- docs/configuration.rst | 3 + docs/proposed/grid-manager/managed-grid.rst | 247 ++++++++++++++++++++ 2 files changed, 250 insertions(+) create mode 100644 docs/proposed/grid-manager/managed-grid.rst diff --git a/docs/configuration.rst b/docs/configuration.rst index 3d7d68ef3..1f5a862b0 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -958,6 +958,9 @@ If you omit the introducer definitions from both ``tahoe.cfg`` and "introducerless" clients must be configured with static servers (described below), or they will not be able to upload and download files. + +.. _server_list: + Static Server Definitions ========================= diff --git a/docs/proposed/grid-manager/managed-grid.rst b/docs/proposed/grid-manager/managed-grid.rst new file mode 100644 index 000000000..05844d9be --- /dev/null +++ b/docs/proposed/grid-manager/managed-grid.rst @@ -0,0 +1,247 @@ +(This document is "in-progress", with feedback and input from two +devchats with Brain Warner and exarkun as well as other input, +discussion and edits from exarkun. It is NOT done). Search for +"DECIDE" for open questions. + + +Managed Grid +============ + +In a grid using an Introducer, a client will use any storage-server +the Introducer announces. This means that anyone with the Introducer +fURL can connect storage to the grid. + +Sometimes, this is just what you want! + +For some use-cases, though, you want to have clients only use certain +servers. One case might be a "managed" grid, where some entity runs +the grid; clients of this grid don't want their uploads to go to +"unmanaged" storage if some other client decides to provide storage. + +One way to limit which storage servers a client connects to is via the +"server list" (:ref:`server_list`) (aka "Introducer-less" +mode). Clients are given static lists of storage-servers, and connect +only to those. This means manually updating these lists if the storage +servers change, however. + +Another method is for clients to use `[client] peers.preferred=` +configuration option (XXX link? appears undocumented), which suffers +from a similar disadvantage. + + +Grid Manager +------------ + +A "grid-manager" consists of some data defining a keypair along with +some other details and Tahoe sub-commands to manipulate the data and +produce certificates to give to storage-servers. Certificates assert +the statement: "Grid Manager X suggests you use storage-server Y to +upload shares to" (X and Y are public-keys). Such a certificate +consists of: + + - a version (currently 1) + - the public-key of a storage-server + - an expiry timestamp + - a signature of the above + +A client will always use any storage-server for downloads (expired +certificate, or no certificate) because we check the ciphertext and +re-assembled plaintext agains the keys in the capability; +"grid-manager" certificates only control uploads. + + +Grid Manager Data Storage +------------------------- + +The data defining the grid-manager is stored in an arbitrary +directory, which you indicate with the ``--config`` option (in the +future, we may add the ability to store the data directly in a grid, +at which time you may be able to pass a directory-capability to this +option). + +If you don't want to store the configuration on disk at all, you may +use ``--config -`` (that's a dash) and write a valid JSON (YAML? I'm +guessing JSON is easier to deal with here, more-interoperable?) +configuration to stdin. + +All commands take the ``--config`` option, and they all behave +similarly for "data from stdin" versus "data from disk". + +DECIDE: + - config is YAML or JSON? + + +tahoe grid-manager create +````````````````````````` + +Create a new grid-manager. + +If you specify ``--config -`` then a new grid-manager configuration is +written to stdout. Otherwise, a new grid-manager is created in the +directory specified by the ``--config`` option. It is an error if the +directory already exists. + + +tahoe grid-manager show-identity +```````````````````````````````` + +Print out a grid-manager's public key. This key is derived from the +private-key of the grid-manager, so a valid grid-manager config must +be given via ``--config`` + +This public key is what is put in clients' configuration to actually +validate and use grid-manager certificates. + + +tahoe grid-manager add +`````````````````````` + +Takes two args: ``name pubkey``. The ``name`` is an arbitrary local +identifier and the pubkey is the encoded key from a ``node.pubkey`` +file in the storage-server's node directory (with no whitespace). + +This adds a new storage-server to a Grid Manager's +configuration. (Since it mutates the configuration, if you used +``--config -`` the new configuration will be printed to stdout). + + +tahoe grid-manager sign +``````````````````````` + +Takes one arg: ``name``, the petname used previous in a ``tahoe +grid-manager add`` command. + +Note that this mutates the state of the grid-manager if it is on disk, +by adding this certificate to our collection of issued +certificates. If you used ``--config -``, the certificate isn't +persisted anywhere except to stdout (so if you wish to keep it +somewhere, that is up to you). + +This command creates a new "version 1" certificate for a +storage-server (identified by its public key). The new certificate is +printed to stdout. If you stored the config on disk, the new +certificate will (also) be in a file named like +``pub-v0-kioayfth3g7zaitcskln64ddx7tkd6xoe7dbr62uogmlwxtxudpq.cert``. + + +Enrolling a Storage Server +-------------------------- + +tahoe admin add-grid-manager-cert +````````````````````````````````` + +- `--filename`: the file to read the cert from (default: stdin) +- `--name`: the name of this certificate (default: "default") + +Import a "version 1" storage-certificate produced by a grid-manager +(probably: a storage server may have zero or more such certificates +installed; for now just one is sufficient). You will have to re-start +your node after this. Subsequent announcements to the Introducer will +include this certificate. + + +Enrolling a Client +------------------ + +tahoe add-grid-manager +`````````````````````` + +- ``--name``: a petname to call this Grid Manager (default: "default") + +For clients to start using a Grid Manager, they must add a +public-key. A client may have any number of grid-managers, so each one +has a name. If you don't supply ``--name`` then ``"default"`` is used. + +This command takes a single argument, which is the hex-encoded public +key of the Grid Manager. The client will have to be re-started once +this change is made. + + +Example Setup of a New Managed Grid +----------------------------------- + +We'll store our Grid Manager configuration on disk, in +``~/grid-manager``. To initialize this directory:: + + tahoe grid-manager create --config ~/grid-manager + +This example creates an actual grid, but it's all just on one machine +with different "node directories". Usually of course each one would be +on a separate computer. + +(If you already have a grid, you can :ref:`skip ahead `.) + +First of all, create an Introducer. Note that we actually have to run +it briefly before it creates the "Introducer fURL" we want for the +next steps. + + tahoe create-introducer --listen=tcp --port=5555 --location=tcp:localhost:5555 ./introducer + tahoe -d introducer run + (Ctrl-C to stop it after a bit) + +Next, we attach a couple of storage nodes:: + + tahoe create-node --introducer $(cat introducer/private/introducer.furl) --nickname storage0 --webport 6001 --webport 6002 --location tcp:localhost:6003 --port 6003 ./storage0 + tahoe create-node --introducer $(cat introducer/private/introducer.furl) --nickname storage1 --webport 6101 --webport 6102 --location tcp:localhost:6103 --port 6103 ./storage1 + daemonize tahoe -d storage0 run + daemonize tahoe -d storage1 run + +.. _skip_ahead: + +We can now ask the Grid Manager to create certificates for our new +storage servers:: + + tahoe grid-manager --config ~/grid-manager add-storage --pubkey $(cat storage0/node.pubkey) > storage0.cert + tahoe grid-manager --config ~/grid-manager add-storage --pubkey $(cat storage1/node.pubkey) > storage1.cert + + # enroll server0 (using file) + kill $(cat storage0/twistd.pid) + tahoe -d storage0 admin add-grid-manager-cert --filename storage0.cert + daemonize tahoe -d storage0 run + + # enroll server1 (using stdin) + kill $(cat storage1/twistd.pid) + cat storage1.cert | tahoe -d storage1 admin add-grid-manager-cert + daemonize tahoe -d storage1 run + +Now try adding a new storage server ``storage2``. This client can join +the grid just fine, and announce itself to the Introducer as providing +storage:: + + tahoe create-node --introducer $(cat introducer/private/introducer.furl) --nickname storage2 --webport 6301 --webport 6302 --location tcp:localhost:6303 --port 6303 ./storage2 + daemonize tahoe -d storage2 run + +At this point any client will upload to any of these three +storage-servers. Make a client "alice" and try! + +:: + + tahoe create-client --introducer $(cat introducer/private/introducer.furl) --nickname alice --webport 6301 --shares-total=3 --shares-needed=2 --shares-happy=3 ./alice + daemonize tahoe -d alice run + tahoe -d alice mkdir # prints out a dir-cap + find storage2/storage/shares # confirm storage2 has a share + +Now we want to make Alice only upload to the storage servers that the +grid-manager has given certificates to (``storage0`` and +``storage1``). We need the grid-manager's public key to put in Alice's +configuration:: + + kill $(cat alice/twistd.pid) + tahoe -d alice add-grid-manager --name work-grid $(tahoe grid-manager --config ~/grid-manager show-identity) + daemonize tahoe -d alice start + +DECIDE: + - should the grid-manager be identified by a certificate? exarkun + points out: --name seems like the hint of the beginning of a + use-case for certificates rather than bare public keys?). + +Since we made Alice's parameters require 3 storage servers to be +reachable (`--happy=3`), all their uploads should now fail (so `tahoe +mkdir` will fail) because they won't use storage2 and can't "achieve +happiness". + +You can check Alice's "Welcome" page (where the list of connected servers +is) at http://localhost:6301/ and should be able to see details about +the "work-grid" Grid Manager that you added. When any Grid Managers +are enabled, each storage-server line will show whether it has a valid +cerifiticate or not (and how much longer it's valid until). From a4be9494afd6419281a1841e1b4a934dff6aa80a Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 6 Apr 2018 13:42:51 -0600 Subject: [PATCH 0002/2309] first cut and implementing some grid-manager commands --- src/allmydata/scripts/runner.py | 5 +- src/allmydata/scripts/tahoe_grid_manager.py | 286 ++++++++++++++++++++ 2 files changed, 290 insertions(+), 1 deletion(-) create mode 100644 src/allmydata/scripts/tahoe_grid_manager.py diff --git a/src/allmydata/scripts/runner.py b/src/allmydata/scripts/runner.py index 327325e95..164b17334 100644 --- a/src/allmydata/scripts/runner.py +++ b/src/allmydata/scripts/runner.py @@ -9,7 +9,7 @@ from twisted.internet import defer, task, threads from allmydata.scripts.common import get_default_nodedir from allmydata.scripts import debug, create_node, cli, \ stats_gatherer, admin, magic_folder_cli, tahoe_daemonize, tahoe_start, \ - tahoe_stop, tahoe_restart, tahoe_run, tahoe_invite + tahoe_stop, tahoe_restart, tahoe_run, tahoe_invite, tahoe_grid_manager from allmydata.util.encodingutil import quote_output, quote_local_unicode_path, get_io_encoding from allmydata.util.eliotutil import ( opt_eliot_destination, @@ -62,6 +62,7 @@ class Options(usage.Options): + cli.subCommands + magic_folder_cli.subCommands + tahoe_invite.subCommands + + tahoe_grid_manager.subCommands ) optFlags = [ @@ -159,6 +160,8 @@ def dispatch(config, # same f0 = magic_folder_cli.dispatch[command] f = lambda so: threads.deferToThread(f0, so) + elif command in tahoe_grid_manager.dispatch: + f = tahoe_grid_manager.dispatch[command] elif command in tahoe_invite.dispatch: f = tahoe_invite.dispatch[command] else: diff --git a/src/allmydata/scripts/tahoe_grid_manager.py b/src/allmydata/scripts/tahoe_grid_manager.py new file mode 100644 index 000000000..3f62efc46 --- /dev/null +++ b/src/allmydata/scripts/tahoe_grid_manager.py @@ -0,0 +1,286 @@ + +import os +import sys +import json + +from pycryptopp.publickey import ed25519 # perhaps NaCl instead? other code uses this though + +from allmydata.scripts.common import BasedirOptions +from twisted.scripts import twistd +from twisted.python import usage +from twisted.python.reflect import namedAny +from twisted.python.filepath import FilePath +from allmydata.scripts.default_nodedir import _default_nodedir +from allmydata.util import fileutil +from allmydata.util import base32 +from allmydata.util import keyutil +from allmydata.node import read_config +from allmydata.util.encodingutil import listdir_unicode, quote_local_unicode_path +from twisted.application.service import Service +from twisted.internet.defer import inlineCallbacks, returnValue + + +class CreateOptions(BasedirOptions): + description = ( + "Create a new identity key and configuration of a Grid Manager" + ) + + +class ShowIdentityOptions(BasedirOptions): + description = ( + "Create a new identity key and configuration of a Grid Manager" + ) + + +class AddOptions(BasedirOptions): + description = ( + "Add a new storage-server's key to a Grid Manager configuration" + ) + + def parseArgs(self, *args, **kw): + BasedirOptions.parseArgs(self, **kw) + if len(args) != 2: + raise usage.UsageError( + "Requires two arguments: name pubkey" + ) + self['name'] = unicode(args[0]) + try: + # WTF?! why does it want 'str' and not six.text_type? + self['storage_pubkey'] = keyutil.parse_pubkey(args[1]) + except Exception as e: + raise usage.UsageError( + "Invalid pubkey argument: {}".format(e) + ) + + +class SignOptions(BasedirOptions): + description = ( + "Create and sign a new certificate for a storage-server" + ) + + def parseArgs(self, *args, **kw): + BasedirOptions.parseArgs(self, **kw) + if len(args) != 1: + raise usage.UsageError( + "Requires one argument: name" + ) + self['name'] = unicode(args[0]) + + +class GridManagerOptions(BasedirOptions): + subCommands = [ + ["create", None, CreateOptions, "Create a Grid Manager."], + ["show-identity", None, ShowIdentityOptions, "Show public-key for Grid Manager."], + ["add", None, AddOptions, "Add a storage server to a Grid Manager."], + ["sign", None, SignOptions, "Create and sign a new Storage Certificate."], + ] + + optParameters = [ + ("config", "c", None, "How to find the Grid Manager's configuration") + ] + + def postOptions(self): + if not hasattr(self, 'subOptions'): + raise usage.UsageError("must specify a subcommand") + if self['config'] is None: + raise usage.UsageError("Must supply configuration with --config") + + description = ( + "A Grid Manager creates certificates for Storage Servers certifying " + "them for use by clients to upload shares to. Configuration may be " + "passed in on stdin or stored in a directory." + ) + + +def _create_gridmanager(): + return { + "grid_manager_config_version": 0, + "privkey": ed25519.SigningKey(os.urandom(32)), + } + +def _create(gridoptions, options): + gm_config = gridoptions['config'] + assert gm_config is not None + + # pre-conditions check + fp = None + if gm_config.strip() != '-': + fp = FilePath(gm_config.strip()) + if fp.exists(): + raise usage.UsageError( + "The directory '{}' already exists.".format(gm_config) + ) + + gm = _create_gridmanager() + _save_gridmanager_config(fp, gm) + + +def _save_gridmanager_config(file_path, grid_manager): + """ + Writes a Grid Manager configuration to the place specified by + 'file_path' (if None, stdout is used). + """ + # FIXME probably want a GridManagerConfig class or something with + # .save and .load instead of this crap + raw_data = { + k: v + for k, v in grid_manager.items() + } + raw_data['privkey'] = base32.b2a(raw_data['privkey'].sk_and_vk[:32]) + data = json.dumps(raw_data, indent=4) + + if file_path is None: + print("{}\n".format(data)) + else: + fileutil.make_dirs(file_path.path, mode=0o700) + with file_path.child("config.json").open("w") as f: + f.write("{}\n".format(data)) + return 0 + + +# XXX should take a FilePath or None +def _load_gridmanager_config(gm_config): + """ + Loads a Grid Manager configuration and returns it (a dict) after + validating. Exceptions if the config can't be found, or has + problems. + """ + fp = None + if gm_config.strip() != '-': + fp = FilePath(gm_config.strip()) + if not fp.exists(): + raise RuntimeError( + "No such directory '{}'".format(gm_config) + ) + + if fp is None: + gm = json.load(sys.stdin) + else: + with fp.child("config.json").open("r") as f: + gm = json.load(f) + + if 'privkey' not in gm: + raise RuntimeError( + "Grid Manager config from '{}' requires a 'privkey'".format( + gm_config + ) + ) + + privkey_str = gm['privkey'] + try: + privkey_bytes = base32.a2b(privkey_str.encode('ascii')) # WTF?! why is a2b requiring "str", not "unicode"? + gm['privkey'] = ed25519.SigningKey(privkey_bytes) + except Exception as e: + raise RuntimeError( + "Invalid Grid Manager privkey: {}".format(e) + ) + + gm_version = gm.get('grid_manager_config_version', None) + if gm_version != 0: + raise RuntimeError( + "Missing or unknown version '{}' of Grid Manager config".format( + gm_version + ) + ) + return gm + + +def _show_identity(gridoptions, options): + """ + Output the public-key of a Grid Manager + """ + gm_config = gridoptions['config'].strip() + assert gm_config is not None + + gm = _load_gridmanager_config(gm_config) + verify_key_bytes = gm['privkey'].get_verifying_key_bytes() + print(base32.b2a(verify_key_bytes)) + + +def _add(gridoptions, options): + """ + Add a new storage-server by name to a Grid Manager + """ + gm_config = gridoptions['config'].strip() + assert gm_config is not None + fp = FilePath(gm_config) if gm_config.strip() != '-' else None + + gm = _load_gridmanager_config(gm_config) + if options['name'] in gm.get('storage_severs', set()): + raise usage.UsageError( + "A storage-server called '{}' already exists".format(options['name']) + ) + if 'storage_servers' not in gm: + gm['storage_servers'] = dict() + gm['storage_servers'][options['name']] = base32.b2a(options['storage_pubkey'].vk_bytes) + _save_gridmanager_config(fp, gm) + + +def _sign(gridoptions, options): + """ + sign a new certificate + """ + gm_config = gridoptions['config'].strip() + assert gm_config is not None + fp = FilePath(gm_config) if gm_config.strip() != '-' else None + gm = _load_gridmanager_config(gm_config) + + if options['name'] not in gm.get('storage_servers', dict()): + raise usage.UsageError( + "No storage-server called '{}' exists".format(options['name']) + ) + + pubkey = gm['storage_servers'][options['name']] + import time + cert_info = { + "expires": int(time.time() + 86400), # XXX FIXME + "pubkey": pubkey, + "version": 1, + } + cert_data = json.dumps(cert_info, separators=(',',':'), sort_keys=True) + sig = gm['privkey'].sign(cert_data) + certificate = { + "certificate": cert_data, + "signature": base32.b2a(sig), + } + certificate_data = json.dumps(certificate, indent=4) + print(certificate_data) + if fp is not None: + with fp.child('{}.cert'.format(options['name'])).open('w') as f: + f.write(certificate_data) + + +grid_manager_commands = { + CreateOptions: _create, + ShowIdentityOptions: _show_identity, + AddOptions: _add, + SignOptions: _sign, +} + +@inlineCallbacks +def gridmanager(config): + """ + Runs the 'tahoe grid-manager' command. + """ + if config.subCommand is None: + print(config) + returnValue(1) + + try: + f = grid_manager_commands[config.subOptions.__class__] + except KeyError: + print(config.subOptions, grid_manager_commands.keys()) + print("Unknown command 'tahoe grid-manager {}': no such grid-manager subcommand".format(config.subCommand)) + returnValue(2) + + x = yield f(config, config.subOptions) + returnValue(x) + +subCommands = [ + ["grid-manager", None, GridManagerOptions, + "Grid Manager subcommands: use 'tahoe grid-manager' for a list."], +] + +dispatch = { + "grid-manager": gridmanager, +} From 31246c84ab8769218e822f7b0e1085e5aa6666d5 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 18 May 2018 14:58:32 -0600 Subject: [PATCH 0003/2309] pubkey -> public_key, privkey -> private_key and doc fixups --- docs/proposed/grid-manager/managed-grid.rst | 8 ++-- src/allmydata/scripts/tahoe_grid_manager.py | 52 ++++++++++++--------- 2 files changed, 34 insertions(+), 26 deletions(-) diff --git a/docs/proposed/grid-manager/managed-grid.rst b/docs/proposed/grid-manager/managed-grid.rst index 05844d9be..342bcf29d 100644 --- a/docs/proposed/grid-manager/managed-grid.rst +++ b/docs/proposed/grid-manager/managed-grid.rst @@ -32,8 +32,8 @@ from a similar disadvantage. Grid Manager ------------ -A "grid-manager" consists of some data defining a keypair along with -some other details and Tahoe sub-commands to manipulate the data and +A "grid-manager" consists of some data defining a keypair (along with +some other details) and Tahoe sub-commands to manipulate the data and produce certificates to give to storage-servers. Certificates assert the statement: "Grid Manager X suggests you use storage-server Y to upload shares to" (X and Y are public-keys). Such a certificate @@ -45,8 +45,8 @@ consists of: - a signature of the above A client will always use any storage-server for downloads (expired -certificate, or no certificate) because we check the ciphertext and -re-assembled plaintext agains the keys in the capability; +certificate, or no certificate) because clients check the ciphertext +and re-assembled plaintext against the keys in the capability; "grid-manager" certificates only control uploads. diff --git a/src/allmydata/scripts/tahoe_grid_manager.py b/src/allmydata/scripts/tahoe_grid_manager.py index 3f62efc46..9f87adc98 100644 --- a/src/allmydata/scripts/tahoe_grid_manager.py +++ b/src/allmydata/scripts/tahoe_grid_manager.py @@ -28,7 +28,10 @@ class CreateOptions(BasedirOptions): class ShowIdentityOptions(BasedirOptions): description = ( - "Create a new identity key and configuration of a Grid Manager" + "Show the public identity key of a Grid Manager\n" + "\n" + "This is what you give to clients to add to their configuration" + " so they use announcements from this Grid Manager" ) @@ -41,15 +44,15 @@ class AddOptions(BasedirOptions): BasedirOptions.parseArgs(self, **kw) if len(args) != 2: raise usage.UsageError( - "Requires two arguments: name pubkey" + "Requires two arguments: name public_key" ) self['name'] = unicode(args[0]) try: # WTF?! why does it want 'str' and not six.text_type? - self['storage_pubkey'] = keyutil.parse_pubkey(args[1]) + self['storage_public_key'] = keyutil.parse_public_key(args[1]) except Exception as e: raise usage.UsageError( - "Invalid pubkey argument: {}".format(e) + "Invalid public_key argument: {}".format(e) ) @@ -70,8 +73,8 @@ class SignOptions(BasedirOptions): class GridManagerOptions(BasedirOptions): subCommands = [ ["create", None, CreateOptions, "Create a Grid Manager."], - ["show-identity", None, ShowIdentityOptions, "Show public-key for Grid Manager."], - ["add", None, AddOptions, "Add a storage server to a Grid Manager."], + ["public-identity", None, ShowIdentityOptions, "Get the public-key for this Grid Manager."], + ["add", None, AddOptions, "Add a storage server to this Grid Manager."], ["sign", None, SignOptions, "Create and sign a new Storage Certificate."], ] @@ -86,16 +89,21 @@ class GridManagerOptions(BasedirOptions): raise usage.UsageError("Must supply configuration with --config") description = ( - "A Grid Manager creates certificates for Storage Servers certifying " - "them for use by clients to upload shares to. Configuration may be " - "passed in on stdin or stored in a directory." + 'A "grid-manager" consists of some data defining a keypair (along with ' + 'some other details) and Tahoe sub-commands to manipulate the data and ' + 'produce certificates to give to storage-servers. Certificates assert ' + 'the statement: "Grid Manager X suggests you use storage-server Y to ' + 'upload shares to" (X and Y are public-keys).' + '\n\n' + 'Clients can use Grid Managers to decide which storage servers to ' + 'upload shares to.' ) def _create_gridmanager(): return { "grid_manager_config_version": 0, - "privkey": ed25519.SigningKey(os.urandom(32)), + "private_key": ed25519.SigningKey(os.urandom(32)), } def _create(gridoptions, options): @@ -126,7 +134,7 @@ def _save_gridmanager_config(file_path, grid_manager): k: v for k, v in grid_manager.items() } - raw_data['privkey'] = base32.b2a(raw_data['privkey'].sk_and_vk[:32]) + raw_data['private_key'] = base32.b2a(raw_data['private_key'].sk_and_vk[:32]) data = json.dumps(raw_data, indent=4) if file_path is None: @@ -159,20 +167,20 @@ def _load_gridmanager_config(gm_config): with fp.child("config.json").open("r") as f: gm = json.load(f) - if 'privkey' not in gm: + if 'private_key' not in gm: raise RuntimeError( - "Grid Manager config from '{}' requires a 'privkey'".format( + "Grid Manager config from '{}' requires a 'private_key'".format( gm_config ) ) - privkey_str = gm['privkey'] + private_key_str = gm['private_key'] try: - privkey_bytes = base32.a2b(privkey_str.encode('ascii')) # WTF?! why is a2b requiring "str", not "unicode"? - gm['privkey'] = ed25519.SigningKey(privkey_bytes) + private_key_bytes = base32.a2b(private_key_str.encode('ascii')) # WTF?! why is a2b requiring "str", not "unicode"? + gm['private_key'] = ed25519.SigningKey(private_key_bytes) except Exception as e: raise RuntimeError( - "Invalid Grid Manager privkey: {}".format(e) + "Invalid Grid Manager private_key: {}".format(e) ) gm_version = gm.get('grid_manager_config_version', None) @@ -193,7 +201,7 @@ def _show_identity(gridoptions, options): assert gm_config is not None gm = _load_gridmanager_config(gm_config) - verify_key_bytes = gm['privkey'].get_verifying_key_bytes() + verify_key_bytes = gm['private_key'].get_verifying_key_bytes() print(base32.b2a(verify_key_bytes)) @@ -212,7 +220,7 @@ def _add(gridoptions, options): ) if 'storage_servers' not in gm: gm['storage_servers'] = dict() - gm['storage_servers'][options['name']] = base32.b2a(options['storage_pubkey'].vk_bytes) + gm['storage_servers'][options['name']] = base32.b2a(options['storage_public_key'].vk_bytes) _save_gridmanager_config(fp, gm) @@ -230,15 +238,15 @@ def _sign(gridoptions, options): "No storage-server called '{}' exists".format(options['name']) ) - pubkey = gm['storage_servers'][options['name']] + public_key = gm['storage_servers'][options['name']] import time cert_info = { "expires": int(time.time() + 86400), # XXX FIXME - "pubkey": pubkey, + "public_key": public_key, "version": 1, } cert_data = json.dumps(cert_info, separators=(',',':'), sort_keys=True) - sig = gm['privkey'].sign(cert_data) + sig = gm['private_key'].sign(cert_data) certificate = { "certificate": cert_data, "signature": base32.b2a(sig), From c874a388ce6c7f49d3514727d30ba511873dcc62 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 18 May 2018 16:33:10 -0600 Subject: [PATCH 0004/2309] wip --- src/allmydata/scripts/tahoe_grid_manager.py | 140 ++++++++++++++------ 1 file changed, 100 insertions(+), 40 deletions(-) diff --git a/src/allmydata/scripts/tahoe_grid_manager.py b/src/allmydata/scripts/tahoe_grid_manager.py index 9f87adc98..9a39ab567 100644 --- a/src/allmydata/scripts/tahoe_grid_manager.py +++ b/src/allmydata/scripts/tahoe_grid_manager.py @@ -49,13 +49,19 @@ class AddOptions(BasedirOptions): self['name'] = unicode(args[0]) try: # WTF?! why does it want 'str' and not six.text_type? - self['storage_public_key'] = keyutil.parse_public_key(args[1]) + self['storage_public_key'] = keyutil.parse_pubkey(args[1]) except Exception as e: raise usage.UsageError( "Invalid public_key argument: {}".format(e) ) +class ListOptions(BasedirOptions): + description = ( + "List all storage servers in this Grid Manager" + ) + + class SignOptions(BasedirOptions): description = ( "Create and sign a new certificate for a storage-server" @@ -75,6 +81,7 @@ class GridManagerOptions(BasedirOptions): ["create", None, CreateOptions, "Create a Grid Manager."], ["public-identity", None, ShowIdentityOptions, "Get the public-key for this Grid Manager."], ["add", None, AddOptions, "Add a storage server to this Grid Manager."], + ["list", None, ListOptions, "List all storage servers in this Grid Manager."], ["sign", None, SignOptions, "Create and sign a new Storage Certificate."], ] @@ -107,8 +114,10 @@ def _create_gridmanager(): } def _create(gridoptions, options): + """ + Create a new Grid Manager + """ gm_config = gridoptions['config'] - assert gm_config is not None # pre-conditions check fp = None @@ -123,19 +132,78 @@ def _create(gridoptions, options): _save_gridmanager_config(fp, gm) +class _GridManager(object): + """ + A Grid Manager's configuration. + """ + + def __init__(self, config, config_location): + if 'private_key' not in config: + raise RuntimeError( + "Grid Manager config from '{}' requires a 'private_key'".format( + config_config + ) + ) + + private_key_str = config['private_key'] + try: + self._private_key_bytes = base32.a2b(private_key_str.encode('ascii')) + self._private_key = ed25519.SigningKey(self._private_key_bytes) + except Exception as e: + raise RuntimeError( + "Invalid Grid Manager private_key: {}".format(e) + ) + + gm_version = config.get('grid_manager_config_version', None) + if gm_version != 0: + raise RuntimeError( + "Missing or unknown version '{}' of Grid Manager config".format( + gm_version + ) + ) + self._version = 0 + self._storage_servers = dict() + + @property + def storage_servers(self): + return self._storage_servers + + def add_storage_server(self, name, public_key): + """ + :param name: a user-meaningful name for the server + :param public_key: ed25519.VerifyingKey the public-key of the + storage provider (e.g. from the contents of node.pubkey + for the client) + """ + if name in self._storage_servers: + raise KeyError( + "Already have a storage server called '{}'".format(name) + ) + assert public_key.vk_bytes + self._storage_servers[name] = public_key + + def marshal(self): + data = { + u"grid_manager_config_version": self._version, + u"private_key": base32.b2a(self._private_key.sk_and_vk[:32]), + } + if self._storage_servers: + data[u"storage_servers"] = { + name: base32.b2a(public_key.vk_bytes) + for name, public_key + in self._storage_servers.items() + } + + def _save_gridmanager_config(file_path, grid_manager): """ Writes a Grid Manager configuration to the place specified by 'file_path' (if None, stdout is used). """ - # FIXME probably want a GridManagerConfig class or something with - # .save and .load instead of this crap - raw_data = { - k: v - for k, v in grid_manager.items() - } - raw_data['private_key'] = base32.b2a(raw_data['private_key'].sk_and_vk[:32]) - data = json.dumps(raw_data, indent=4) + data = json.dumps( + grid_manager.marshal(), + indent=4, + ) if file_path is None: print("{}\n".format(data)) @@ -167,30 +235,7 @@ def _load_gridmanager_config(gm_config): with fp.child("config.json").open("r") as f: gm = json.load(f) - if 'private_key' not in gm: - raise RuntimeError( - "Grid Manager config from '{}' requires a 'private_key'".format( - gm_config - ) - ) - - private_key_str = gm['private_key'] - try: - private_key_bytes = base32.a2b(private_key_str.encode('ascii')) # WTF?! why is a2b requiring "str", not "unicode"? - gm['private_key'] = ed25519.SigningKey(private_key_bytes) - except Exception as e: - raise RuntimeError( - "Invalid Grid Manager private_key: {}".format(e) - ) - - gm_version = gm.get('grid_manager_config_version', None) - if gm_version != 0: - raise RuntimeError( - "Missing or unknown version '{}' of Grid Manager config".format( - gm_version - ) - ) - return gm + return _GridManager(gm, gm_config) def _show_identity(gridoptions, options): @@ -210,26 +255,40 @@ def _add(gridoptions, options): Add a new storage-server by name to a Grid Manager """ gm_config = gridoptions['config'].strip() - assert gm_config is not None fp = FilePath(gm_config) if gm_config.strip() != '-' else None gm = _load_gridmanager_config(gm_config) - if options['name'] in gm.get('storage_severs', set()): + try: + gm.add_storage_server( + options['name'], + options['storage_public_key'], + ) + except KeyError: raise usage.UsageError( "A storage-server called '{}' already exists".format(options['name']) ) - if 'storage_servers' not in gm: - gm['storage_servers'] = dict() - gm['storage_servers'][options['name']] = base32.b2a(options['storage_public_key'].vk_bytes) + _save_gridmanager_config(fp, gm) +def _list(gridoptions, options): + """ + List all storage-servers known to a Grid Manager + """ + gm_config = gridoptions['config'].strip() + fp = FilePath(gm_config) if gm_config.strip() != '-' else None + + gm = _load_gridmanager_config(gm_config) + for name in sorted(gm.storage_servers.keys()): + key = "pub-v0-" + gm.storage_servers[name].vk_bytes + print("{}: {}".format(name, key)) + + def _sign(gridoptions, options): """ sign a new certificate """ gm_config = gridoptions['config'].strip() - assert gm_config is not None fp = FilePath(gm_config) if gm_config.strip() != '-' else None gm = _load_gridmanager_config(gm_config) @@ -262,6 +321,7 @@ grid_manager_commands = { CreateOptions: _create, ShowIdentityOptions: _show_identity, AddOptions: _add, + ListOptions: _list, SignOptions: _sign, } From a40364adba45090a43ec192ad426e809534a0db4 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 18 May 2018 18:12:28 -0600 Subject: [PATCH 0005/2309] do something sensible on UsageError --- src/allmydata/scripts/runner.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/allmydata/scripts/runner.py b/src/allmydata/scripts/runner.py index 164b17334..421eca5d5 100644 --- a/src/allmydata/scripts/runner.py +++ b/src/allmydata/scripts/runner.py @@ -175,7 +175,13 @@ def dispatch(config, # 4: return a Deferred that does 1 or 2 or 3 def _raise_sys_exit(rc): sys.exit(rc) + + def _error(f): + f.trap(usage.UsageError) + print("Error: {}".format(f.value.message)) + sys.exit(1) d.addCallback(_raise_sys_exit) + d.addErrback(_error) return d def _maybe_enable_eliot_logging(options, reactor): From 4d409776650cd3454b7ec0202513f9fae61eaee7 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 18 May 2018 18:12:46 -0600 Subject: [PATCH 0006/2309] refactor .sign() into GridManager, better GM construction --- src/allmydata/scripts/tahoe_grid_manager.py | 76 +++++++++++++-------- 1 file changed, 47 insertions(+), 29 deletions(-) diff --git a/src/allmydata/scripts/tahoe_grid_manager.py b/src/allmydata/scripts/tahoe_grid_manager.py index 9a39ab567..6775b38f1 100644 --- a/src/allmydata/scripts/tahoe_grid_manager.py +++ b/src/allmydata/scripts/tahoe_grid_manager.py @@ -2,6 +2,7 @@ import os import sys import json +import time from pycryptopp.publickey import ed25519 # perhaps NaCl instead? other code uses this though @@ -108,10 +109,7 @@ class GridManagerOptions(BasedirOptions): def _create_gridmanager(): - return { - "grid_manager_config_version": 0, - "private_key": ed25519.SigningKey(os.urandom(32)), - } + return _GridManager(ed25519.SigningKey(os.urandom(32)), {}) def _create(gridoptions, options): """ @@ -137,9 +135,10 @@ class _GridManager(object): A Grid Manager's configuration. """ - def __init__(self, config, config_location): + @staticmethod + def from_config(config, config_location): if 'private_key' not in config: - raise RuntimeError( + raise ValueError( "Grid Manager config from '{}' requires a 'private_key'".format( config_config ) @@ -147,27 +146,56 @@ class _GridManager(object): private_key_str = config['private_key'] try: - self._private_key_bytes = base32.a2b(private_key_str.encode('ascii')) - self._private_key = ed25519.SigningKey(self._private_key_bytes) + private_key_bytes = base32.a2b(private_key_str.encode('ascii')) + private_key = ed25519.SigningKey(private_key_bytes) except Exception as e: - raise RuntimeError( + raise ValueError( "Invalid Grid Manager private_key: {}".format(e) ) - gm_version = config.get('grid_manager_config_version', None) + storage_servers = dict() + for name, pubkey in config.get(u'storage_servers', {}).items(): + pubkey_bytes = base32.a2b(pubkey.encode('ascii')) + storage_servers[name] = ed25519.VerifyingKey(pubkey_bytes) + + gm_version = config.get(u'grid_manager_config_version', None) if gm_version != 0: - raise RuntimeError( + raise ValueError( "Missing or unknown version '{}' of Grid Manager config".format( gm_version ) ) + return _GridManager(private_key, storage_servers) + + def __init__(self, private_key, storage_servers): + self._storage_servers = dict() if storage_servers is None else storage_servers + self._private_key = private_key self._version = 0 - self._storage_servers = dict() @property def storage_servers(self): return self._storage_servers + def sign(self, name): + try: + public_key = self._storage_servers[name] + except KeyError: + raise KeyError( + u"No storage server named '{}'".format(name) + ) + cert_info = { + "expires": int(time.time() + 86400), # XXX FIXME + "public_key": base32.b2a(public_key.vk_bytes), + "version": 1, + } + cert_data = json.dumps(cert_info, separators=(',',':'), sort_keys=True) + sig = self._private_key.sign(cert_data) + certificate = { + u"certificate": cert_data, + u"signature": base32.b2a(sig), + } + return certificate + def add_storage_server(self, name, public_key): """ :param name: a user-meaningful name for the server @@ -193,6 +221,7 @@ class _GridManager(object): for name, public_key in self._storage_servers.items() } + return data def _save_gridmanager_config(file_path, grid_manager): @@ -211,7 +240,6 @@ def _save_gridmanager_config(file_path, grid_manager): fileutil.make_dirs(file_path.path, mode=0o700) with file_path.child("config.json").open("w") as f: f.write("{}\n".format(data)) - return 0 # XXX should take a FilePath or None @@ -235,7 +263,7 @@ def _load_gridmanager_config(gm_config): with fp.child("config.json").open("r") as f: gm = json.load(f) - return _GridManager(gm, gm_config) + return _GridManager.from_config(gm, gm_config) def _show_identity(gridoptions, options): @@ -269,6 +297,7 @@ def _add(gridoptions, options): ) _save_gridmanager_config(fp, gm) + return 0 def _list(gridoptions, options): @@ -280,7 +309,7 @@ def _list(gridoptions, options): gm = _load_gridmanager_config(gm_config) for name in sorted(gm.storage_servers.keys()): - key = "pub-v0-" + gm.storage_servers[name].vk_bytes + key = "pub-v0-" + base32.b2a(gm.storage_servers[name].vk_bytes) print("{}: {}".format(name, key)) @@ -292,24 +321,13 @@ def _sign(gridoptions, options): fp = FilePath(gm_config) if gm_config.strip() != '-' else None gm = _load_gridmanager_config(gm_config) - if options['name'] not in gm.get('storage_servers', dict()): + try: + certificate = gm.sign(options['name']) + except KeyError: raise usage.UsageError( "No storage-server called '{}' exists".format(options['name']) ) - public_key = gm['storage_servers'][options['name']] - import time - cert_info = { - "expires": int(time.time() + 86400), # XXX FIXME - "public_key": public_key, - "version": 1, - } - cert_data = json.dumps(cert_info, separators=(',',':'), sort_keys=True) - sig = gm['private_key'].sign(cert_data) - certificate = { - "certificate": cert_data, - "signature": base32.b2a(sig), - } certificate_data = json.dumps(certificate, indent=4) print(certificate_data) if fp is not None: From 7803febf2da0ec1674d2dd46654619acccee7268 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 18 May 2018 18:17:20 -0600 Subject: [PATCH 0007/2309] refactor; make doc match code --- docs/proposed/grid-manager/managed-grid.rst | 11 +++++++++-- src/allmydata/scripts/tahoe_grid_manager.py | 7 +++++-- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/docs/proposed/grid-manager/managed-grid.rst b/docs/proposed/grid-manager/managed-grid.rst index 342bcf29d..304273fe2 100644 --- a/docs/proposed/grid-manager/managed-grid.rst +++ b/docs/proposed/grid-manager/managed-grid.rst @@ -82,8 +82,8 @@ directory specified by the ``--config`` option. It is an error if the directory already exists. -tahoe grid-manager show-identity -```````````````````````````````` +tahoe grid-manager public-identity +`````````````````````````````````` Print out a grid-manager's public key. This key is derived from the private-key of the grid-manager, so a valid grid-manager config must @@ -105,6 +105,13 @@ configuration. (Since it mutates the configuration, if you used ``--config -`` the new configuration will be printed to stdout). +tahoe grid-manager list +``````````````````````` + +Lists all storage-servers that have previously been added using +``tahoe grid-manager add``. + + tahoe grid-manager sign ``````````````````````` diff --git a/src/allmydata/scripts/tahoe_grid_manager.py b/src/allmydata/scripts/tahoe_grid_manager.py index 6775b38f1..1a8c627ac 100644 --- a/src/allmydata/scripts/tahoe_grid_manager.py +++ b/src/allmydata/scripts/tahoe_grid_manager.py @@ -176,6 +176,10 @@ class _GridManager(object): def storage_servers(self): return self._storage_servers + def public_identity(self): + verify_key_bytes = self._private_key.get_verifying_key_bytes() + return base32.b2a(verify_key_bytes) + def sign(self, name): try: public_key = self._storage_servers[name] @@ -274,8 +278,7 @@ def _show_identity(gridoptions, options): assert gm_config is not None gm = _load_gridmanager_config(gm_config) - verify_key_bytes = gm['private_key'].get_verifying_key_bytes() - print(base32.b2a(verify_key_bytes)) + print(gm.public_identity()) def _add(gridoptions, options): From 870b0c40a0a230d195b7fd0e497ee4916980fdbc Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 18 May 2018 18:49:24 -0600 Subject: [PATCH 0008/2309] tahoe grid-manager list and other fixups --- docs/proposed/grid-manager/managed-grid.rst | 3 +- src/allmydata/scripts/tahoe_grid_manager.py | 59 +++++++++++++++++---- src/allmydata/util/abbreviate.py | 2 +- 3 files changed, 52 insertions(+), 12 deletions(-) diff --git a/docs/proposed/grid-manager/managed-grid.rst b/docs/proposed/grid-manager/managed-grid.rst index 304273fe2..d13cfef33 100644 --- a/docs/proposed/grid-manager/managed-grid.rst +++ b/docs/proposed/grid-manager/managed-grid.rst @@ -127,8 +127,7 @@ somewhere, that is up to you). This command creates a new "version 1" certificate for a storage-server (identified by its public key). The new certificate is printed to stdout. If you stored the config on disk, the new -certificate will (also) be in a file named like -``pub-v0-kioayfth3g7zaitcskln64ddx7tkd6xoe7dbr62uogmlwxtxudpq.cert``. +certificate will (also) be in a file named like ``alice.cert.0``. Enrolling a Storage Server diff --git a/src/allmydata/scripts/tahoe_grid_manager.py b/src/allmydata/scripts/tahoe_grid_manager.py index 1a8c627ac..f9dc90280 100644 --- a/src/allmydata/scripts/tahoe_grid_manager.py +++ b/src/allmydata/scripts/tahoe_grid_manager.py @@ -3,10 +3,12 @@ import os import sys import json import time +from datetime import datetime from pycryptopp.publickey import ed25519 # perhaps NaCl instead? other code uses this though from allmydata.scripts.common import BasedirOptions +from allmydata.util.abbreviate import abbreviate_time from twisted.scripts import twistd from twisted.python import usage from twisted.python.reflect import namedAny @@ -130,6 +132,25 @@ def _create(gridoptions, options): _save_gridmanager_config(fp, gm) +class _GridManagerStorageServer(object): + """ + A Grid Manager's notion of a storage server + """ + + def __init__(self, name, public_key, certificates): + self.name = name + self._public_key = public_key + self._certificates = [] if certificates is None else certificates + + def add_certificate(self, certificate): + self._certificates.append(certificate) + + def public_key(self): + return "pub-v0-" + base32.b2a(self._public_key.vk_bytes) + + def marshal(self): + pass + class _GridManager(object): """ A Grid Manager's configuration. @@ -156,7 +177,9 @@ class _GridManager(object): storage_servers = dict() for name, pubkey in config.get(u'storage_servers', {}).items(): pubkey_bytes = base32.a2b(pubkey.encode('ascii')) - storage_servers[name] = ed25519.VerifyingKey(pubkey_bytes) + storage_servers[name] = _GridManagerStorageServer( + name, ed25519.VerifyingKey(pubkey_bytes), None + ) gm_version = config.get(u'grid_manager_config_version', None) if gm_version != 0: @@ -182,14 +205,14 @@ class _GridManager(object): def sign(self, name): try: - public_key = self._storage_servers[name] + srv = self._storage_servers[name] except KeyError: raise KeyError( u"No storage server named '{}'".format(name) ) cert_info = { "expires": int(time.time() + 86400), # XXX FIXME - "public_key": base32.b2a(public_key.vk_bytes), + "public_key": srv.public_key(), "version": 1, } cert_data = json.dumps(cert_info, separators=(',',':'), sort_keys=True) @@ -212,7 +235,9 @@ class _GridManager(object): "Already have a storage server called '{}'".format(name) ) assert public_key.vk_bytes - self._storage_servers[name] = public_key + ss = _GridManagerStorageServer(name, public_key, None) + self._storage_servers[name] = ss + return ss def marshal(self): data = { @@ -221,8 +246,8 @@ class _GridManager(object): } if self._storage_servers: data[u"storage_servers"] = { - name: base32.b2a(public_key.vk_bytes) - for name, public_key + name: srv.marshal() + for name, srv in self._storage_servers.items() } return data @@ -312,8 +337,20 @@ def _list(gridoptions, options): gm = _load_gridmanager_config(gm_config) for name in sorted(gm.storage_servers.keys()): - key = "pub-v0-" + base32.b2a(gm.storage_servers[name].vk_bytes) - print("{}: {}".format(name, key)) + print("{}: {}".format(name, gm.storage_servers[name].public_key())) + if fp: + cert_count = 0 + while fp.child('{}.cert.{}'.format(name, cert_count)).exists(): + container = json.load(fp.child('{}.cert.{}'.format(name, cert_count)).open('r')) + cert_data = json.loads(container['certificate']) + expires = datetime.fromtimestamp(cert_data['expires']) + delta = datetime.utcnow() - expires + if delta.total_seconds() < 0: + print(" {}: valid until {} ({})".format(cert_count, expires, abbreviate_time(delta))) + else: + print(" {}: expired ({})".format(cert_count, abbreviate_time(delta))) + cert_count += 1 + certs = ' ({} certificates)'.format(cert_count) def _sign(gridoptions, options): @@ -334,7 +371,11 @@ def _sign(gridoptions, options): certificate_data = json.dumps(certificate, indent=4) print(certificate_data) if fp is not None: - with fp.child('{}.cert'.format(options['name'])).open('w') as f: + next_serial = 0 + p = fp.child('{}.cert'.format(options['name'])).path + while fp.child("{}.cert.{}".format(options['name'], next_serial)).exists(): + next_serial += 1 + with fp.child('{}.cert.{}'.format(options['name'], next_serial)).open('w') as f: f.write(certificate_data) diff --git a/src/allmydata/util/abbreviate.py b/src/allmydata/util/abbreviate.py index 6fdf894ce..b3f481d5e 100644 --- a/src/allmydata/util/abbreviate.py +++ b/src/allmydata/util/abbreviate.py @@ -23,7 +23,7 @@ def abbreviate_time(s): if s >= 0.0: postfix = ' ago' else: - postfix = ' in the future' + postfix = ' from now' s = -s def _plural(count, unit): count = int(count) From c76d25ed8734096be5bc15d0394eac57f70bffb1 Mon Sep 17 00:00:00 2001 From: meejah Date: Sat, 19 May 2018 19:17:19 -0600 Subject: [PATCH 0009/2309] Add 'tahoe grid-manager remove' command --- src/allmydata/scripts/tahoe_grid_manager.py | 65 +++++++++++++++++++-- 1 file changed, 61 insertions(+), 4 deletions(-) diff --git a/src/allmydata/scripts/tahoe_grid_manager.py b/src/allmydata/scripts/tahoe_grid_manager.py index f9dc90280..d75b293cd 100644 --- a/src/allmydata/scripts/tahoe_grid_manager.py +++ b/src/allmydata/scripts/tahoe_grid_manager.py @@ -59,6 +59,20 @@ class AddOptions(BasedirOptions): ) +class RemoveOptions(BasedirOptions): + description = ( + "Remove a storage-server from a Grid Manager configuration" + ) + + def parseArgs(self, *args, **kw): + BasedirOptions.parseArgs(self, **kw) + if len(args) != 1: + raise usage.UsageError( + "Requires one arguments: name" + ) + self['name'] = unicode(args[0]) + + class ListOptions(BasedirOptions): description = ( "List all storage servers in this Grid Manager" @@ -84,6 +98,7 @@ class GridManagerOptions(BasedirOptions): ["create", None, CreateOptions, "Create a Grid Manager."], ["public-identity", None, ShowIdentityOptions, "Get the public-key for this Grid Manager."], ["add", None, AddOptions, "Add a storage server to this Grid Manager."], + ["remove", None, RemoveOptions, "Remove a storage server from this Grid Manager."], ["list", None, ListOptions, "List all storage servers in this Grid Manager."], ["sign", None, SignOptions, "Create and sign a new Storage Certificate."], ] @@ -149,7 +164,9 @@ class _GridManagerStorageServer(object): return "pub-v0-" + base32.b2a(self._public_key.vk_bytes) def marshal(self): - pass + return { + u"public_key": self.public_key(), + } class _GridManager(object): """ @@ -175,10 +192,15 @@ class _GridManager(object): ) storage_servers = dict() - for name, pubkey in config.get(u'storage_servers', {}).items(): - pubkey_bytes = base32.a2b(pubkey.encode('ascii')) + for name, srv_config in config.get(u'storage_servers', {}).items(): + if not 'public_key' in srv_config: + raise ValueError( + "No 'public_key' for storage server '{}'".format(name) + ) storage_servers[name] = _GridManagerStorageServer( - name, ed25519.VerifyingKey(pubkey_bytes), None + name, + keyutil.parse_pubkey(srv_config['public_key'].encode('ascii')), + None, ) gm_version = config.get(u'grid_manager_config_version', None) @@ -239,6 +261,17 @@ class _GridManager(object): self._storage_servers[name] = ss return ss + def remove_storage_server(self, name): + """ + :param name: a user-meaningful name for the server + """ + try: + del self._storage_servers[name] + except KeyError: + raise KeyError( + "No storage server called '{}'".format(name) + ) + def marshal(self): data = { u"grid_manager_config_version": self._version, @@ -328,6 +361,29 @@ def _add(gridoptions, options): return 0 +def _remove(gridoptions, options): + """ + Remove an existing storage-server by name from a Grid Manager + """ + gm_config = gridoptions['config'].strip() + fp = FilePath(gm_config) if gm_config.strip() != '-' else None + gm = _load_gridmanager_config(gm_config) + + try: + gm.remove_storage_server(options['name']) + except KeyError: + raise usage.UsageError( + "No storage-server called '{}' exists".format(options['name']) + ) + cert_count = 0 + while fp.child('{}.cert.{}'.format(options['name'], cert_count)).exists(): + fp.child('{}.cert.{}'.format(options['name'], cert_count)).remove() + cert_count += 1 + + _save_gridmanager_config(fp, gm) + return 0 + + def _list(gridoptions, options): """ List all storage-servers known to a Grid Manager @@ -383,6 +439,7 @@ grid_manager_commands = { CreateOptions: _create, ShowIdentityOptions: _show_identity, AddOptions: _add, + RemoveOptions: _remove, ListOptions: _list, SignOptions: _sign, } From 64eb9d7c30f752a6b2e0f2205e676cc6645d726e Mon Sep 17 00:00:00 2001 From: meejah Date: Sat, 19 May 2018 19:17:47 -0600 Subject: [PATCH 0010/2309] hack in grid-manager announcements to storage-servers --- src/allmydata/client.py | 66 +++++++++++++++++++++++++++++++---------- 1 file changed, 51 insertions(+), 15 deletions(-) diff --git a/src/allmydata/client.py b/src/allmydata/client.py index e521f08bc..4702436ed 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -1,4 +1,9 @@ -import os, stat, time, weakref +import os +import stat +import time +import weakref +import json +from allmydata import node from base64 import urlsafe_b64encode from functools import partial from errno import ENOENT, EPERM @@ -564,23 +569,54 @@ class _Client(node.Node, pollmixin.PollMixin): sharetypes.append("mutable") expiration_sharetypes = tuple(sharetypes) - ss = StorageServer(storedir, self.nodeid, - reserved_space=reserved, - discard_storage=discard, - readonly_storage=readonly, - stats_provider=self.stats_provider, - expiration_enabled=expire, - expiration_mode=mode, - expiration_override_lease_duration=o_l_d, - expiration_cutoff_date=cutoff_date, - expiration_sharetypes=expiration_sharetypes) + ss = StorageServer( + storedir, self.nodeid, + reserved_space=reserved, + discard_storage=discard, + readonly_storage=readonly, + stats_provider=self.stats_provider, + expiration_enabled=expire, + expiration_mode=mode, + expiration_override_lease_duration=o_l_d, + expiration_cutoff_date=cutoff_date, + expiration_sharetypes=expiration_sharetypes, + ) ss.setServiceParent(self) - furl_file = self.config.get_private_path("storage.furl").encode(get_filesystem_encoding()) + grid_manager_certificates = [] + cert_fnames = self.get_config("storage", "grid_manager_certificate_files", "") + for fname in cert_fnames.split(): + fname = abspath_expanduser_unicode(fname.decode('ascii'), base=self.basedir) + if not os.path.exists(fname): + raise ValueError( + "Grid Manager certificate file '{}' doesn't exist".format( + fname + ) + ) + with open(fname, 'r') as f: + cert = json.load(f) + if set(cert.keys()) != {"certificate", "signature"}: + raise ValueError( + "Unknown key in Grid Manager certificate '{}'".format( + fname + ) + ) + grid_manager_certificates.append(cert) + + # XXX we should probably verify that the certificates are + # valid and not expired, as that could be confusing for the + # storage-server operator -- but then we need the public key + # of the Grid Manager (should that go in the config too, + # then? How to handle multiple grid-managers?) + + + furl_file = os.path.join(self.basedir, "private", "storage.furl").encode(get_filesystem_encoding()) furl = self.tub.registerReference(ss, furlFile=furl_file) - ann = {"anonymous-storage-FURL": furl, - "permutation-seed-base32": self._init_permutation_seed(ss), - } + ann = { + "anonymous-storage-FURL": furl, + "permutation-seed-base32": self._init_permutation_seed(ss), + "grid-manager-certificates": grid_manager_certificates, + } for ic in self.introducer_clients: ic.publish("storage", ann, self._node_key) From ee3e1cbcf23a179d8d125a66d12dc7b382aa31d9 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 22 May 2018 10:42:34 -0600 Subject: [PATCH 0011/2309] wip; updates to grid-manager impl --- src/allmydata/client.py | 21 +++++++ src/allmydata/immutable/upload.py | 2 +- src/allmydata/storage_client.py | 96 ++++++++++++++++++++++++++++--- 3 files changed, 111 insertions(+), 8 deletions(-) diff --git a/src/allmydata/client.py b/src/allmydata/client.py index 4702436ed..b9225e169 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -15,6 +15,7 @@ from twisted.application.internet import TimerService from twisted.python.filepath import FilePath from twisted.python.failure import Failure from pycryptopp.publickey import rsa +from pycryptopp.publickey import ed25519 import allmydata from allmydata.storage.server import StorageServer @@ -31,6 +32,7 @@ from allmydata.util.abbreviate import parse_abbreviated_size from allmydata.util.time_format import parse_duration, parse_date from allmydata.util.i2p_provider import create as create_i2p_provider from allmydata.util.tor_provider import create as create_tor_provider +from allmydata.util.base32 import a2b, b2a from allmydata.stats import StatsProvider from allmydata.history import History from allmydata.interfaces import IStatsProducer, SDMF_VERSION, MDMF_VERSION, DEFAULT_MAX_SEGMENT_SIZE @@ -58,6 +60,7 @@ def _valid_config_sections(): "shares.needed", "shares.total", "stats_gatherer.furl", + "grid_managers", ), "drop_upload": ( # deprecated already? "enabled", @@ -380,10 +383,28 @@ def create_storage_farm_broker(config, default_connection_handlers, foolscap_con **kwargs ) + # grid manager setup + + grid_manager_keys = [] + gm_keydata = self.get_config('client', 'grid_manager_public_keys', '') + for gm_key in gm_keydata.strip().split(): + grid_manager_keys.append( + keyutil.parse_pubkey(a2b(gm_key)) + ) + + my_pubkey = keyutil.parse_pubkey( + self.get_config_from_file("node.pubkey") + ) + + # create the actual storage-broker + sb = storage_client.StorageFarmBroker( permute_peers=True, tub_maker=tub_creator, preferred_peers=preferred_peers, + grid_manager_keys=grid_manager_keys, + node_pubkey=my_pubkey, + ) for ic in introducer_clients: sb.use_introducer(ic) diff --git a/src/allmydata/immutable/upload.py b/src/allmydata/immutable/upload.py index 6720e4195..79c632c91 100644 --- a/src/allmydata/immutable/upload.py +++ b/src/allmydata/immutable/upload.py @@ -515,7 +515,7 @@ class Tahoe2ServerSelector(log.PrefixingLogMixin): # 0. Start with an ordered list of servers. Maybe *2N* of them. # - all_servers = storage_broker.get_servers_for_psi(storage_index) + all_servers = storage_broker.get_servers_for_psi(storage_index, for_upload=True) if not all_servers: raise NoServersError("client gave us zero servers") diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 65c65f535..34e96f38a 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -29,7 +29,11 @@ the foolscap-based server implemented in src/allmydata/storage/*.py . # 6: implement other sorts of IStorageClient classes: S3, etc -import re, time, hashlib +import re +import time +import hashlib +import json + from zope.interface import implementer from twisted.internet import defer from twisted.application import service @@ -67,12 +71,14 @@ class StorageFarmBroker(service.MultiService): I'm also responsible for subscribing to the IntroducerClient to find out about new servers as they are announced by the Introducer. """ - def __init__(self, permute_peers, tub_maker, preferred_peers=()): + def __init__(self, permute_peers, tub_maker, preferred_peers=(), grid_manager_keys=[], node_pubkey=None): service.MultiService.__init__(self) assert permute_peers # False not implemented yet self.permute_peers = permute_peers self._tub_maker = tub_maker self.preferred_peers = preferred_peers + self._grid_manager_keys = grid_manager_keys + self._node_pubkey = node_pubkey # self.servers maps serverid -> IServer, and keeps track of all the # storage servers that we've heard about. Each descriptor manages its @@ -91,7 +97,7 @@ class StorageFarmBroker(service.MultiService): self._static_server_ids.add(server_id) handler_overrides = server.get("connections", {}) s = NativeStorageServer(server_id, server["ann"], - self._tub_maker, handler_overrides) + self._tub_maker, handler_overrides, [], self._node_pubkey) s.on_status_changed(lambda _: self._got_connection()) s.setServiceParent(self) self.servers[server_id] = s @@ -110,7 +116,7 @@ class StorageFarmBroker(service.MultiService): # these two are used in unit tests def test_add_rref(self, serverid, rref, ann): - s = NativeStorageServer(serverid, ann.copy(), self._tub_maker, {}) + s = NativeStorageServer(serverid, ann.copy(), self._tub_maker, {}, [], self._node_pubkey) s.rref = rref s._is_connected = True self.servers[serverid] = s @@ -151,7 +157,7 @@ class StorageFarmBroker(service.MultiService): facility="tahoe.storage_broker", umid="AlxzqA", level=log.UNUSUAL) return - s = NativeStorageServer(server_id, ann, self._tub_maker, {}) + s = NativeStorageServer(server_id, ann, self._tub_maker, {}, self._grid_manager_keys, self._node_pubkey) s.on_status_changed(lambda _: self._got_connection()) server_id = s.get_serverid() old = self.servers.get(server_id) @@ -192,11 +198,26 @@ class StorageFarmBroker(service.MultiService): for dsc in self.servers.values(): dsc.try_to_connect() - def get_servers_for_psi(self, peer_selection_index): + def get_servers_for_psi(self, peer_selection_index, for_upload=False): + """ + :param for_upload: used to determine if we should include any + servers that are invalid according to Grid Manager + processing. When for_upload is True and we have any Grid + Manager keys configured, any storage servers with invalid or + missing certificates will be excluded. + """ # return a list of server objects (IServers) assert self.permute_peers == True connected_servers = self.get_connected_servers() preferred_servers = frozenset(s for s in connected_servers if s.get_longname() in self.preferred_peers) + if for_upload: + print("upload processing: {}".format([srv.upload_permitted() for srv in connected_servers])) + connected_servers = [ + srv + for srv in connected_servers + if srv.upload_permitted() + ] + def _permuted(server): seed = server.get_permutation_seed() is_unpreferred = server not in preferred_servers @@ -248,6 +269,38 @@ class StubServer(object): def get_nickname(self): return "?" + +def _validate_grid_manager_certificate(gm_key, alleged_cert, storage_pubkey): + """ + :param gm_key: a VerifyingKey instance, a Grid Manager's public + key. + + :param cert: dict with "certificate" and "signature" keys, where + "certificate" contains a JSON-serialized certificate for a Storage + Server (comes from a Grid Manager). + + :return: False if the signature is invalid or the certificate is + expired. + """ + try: + gm_key.verify( + alleged_cert['signature'], + alleged_cert['certificate'], + ) + except Exception: + return False + # signature is valid; now we can load the actual data + cert = json.loads(alleged_cert['certificate']) + now = datetime.utcnow() + expires = datetime.fromordinal(cert['expires']) + cert_pubkey = keyutil.parse_pubkey(cert['public_key']) + if cert_pubkey != storage_pubkey: + return False # certificate is for wrong server + if expires < now: + return False # certificate is expired + return True + + @implementer(IServer) class NativeStorageServer(service.MultiService): """I hold information about a storage server that we want to connect to. @@ -276,13 +329,15 @@ class NativeStorageServer(service.MultiService): "application-version": "unknown: no get_version()", } - def __init__(self, server_id, ann, tub_maker, handler_overrides): + def __init__(self, server_id, ann, tub_maker, handler_overrides, grid_manager_keys, node_pubkey): service.MultiService.__init__(self) assert isinstance(server_id, str) self._server_id = server_id self.announcement = ann self._tub_maker = tub_maker self._handler_overrides = handler_overrides + self._node_pubkey = node_pubkey + self._grid_manager_keys = grid_manager_keys assert "anonymous-storage-FURL" in ann, ann furl = str(ann["anonymous-storage-FURL"]) @@ -321,6 +376,33 @@ class NativeStorageServer(service.MultiService): self._trigger_cb = None self._on_status_changed = ObserverList() + def upload_permitted(self): + """ + If our client is configured with Grid Manager public-keys, we will + only upload to storage servers that have a currently-valid + certificate signed by at least one of the Grid Managers we + accept. + + :return: True if we should use this server for uploads, False + otherwise. + """ + # if we have no Grid Manager keys configured, choice is easy + if not self._grid_manager_keys: + return True + + # XXX probably want to cache the answer to this? (ignoring + # that for now because certificates expire, so .. slightly + # more complex) + certificates = self.announcements.get("grid-manager-certificates", None) + if not certificates: + return False + for gm_key in self._grid_manager_keys: + for cert in certificates: + if _validate_grid_manager_certificate(gm_key, cert, self._pubkey): + return True + return False + + def on_status_changed(self, status_changed): """ :param status_changed: a callable taking a single arg (the From 3f1a8d64d8ba79e3164c9fa8bd91fc81bd374867 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 22 May 2018 10:44:25 -0600 Subject: [PATCH 0012/2309] reformatting/whitespace --- src/allmydata/introducer/client.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/allmydata/introducer/client.py b/src/allmydata/introducer/client.py index a22c46103..4d41b5655 100644 --- a/src/allmydata/introducer/client.py +++ b/src/allmydata/introducer/client.py @@ -35,12 +35,13 @@ class IntroducerClient(service.Service, Referenceable): self._sequencer = sequencer self._cache_filepath = cache_filepath - self._my_subscriber_info = { "version": 0, - "nickname": self._nickname, - "app-versions": self._app_versions, - "my-version": self._my_version, - "oldest-supported": self._oldest_supported, - } + self._my_subscriber_info = { + "version": 0, + "nickname": self._nickname, + "app-versions": self._app_versions, + "my-version": self._my_version, + "oldest-supported": self._oldest_supported, + } self._outbound_announcements = {} # not signed self._published_announcements = {} # signed From 2031b723b8bf087ec31e980a5ae2ff78ce397b0d Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 22 May 2018 16:19:43 -0600 Subject: [PATCH 0013/2309] various fixes (flake8, errors) --- src/allmydata/client.py | 1 - src/allmydata/scripts/tahoe_grid_manager.py | 10 +--------- src/allmydata/storage_client.py | 2 ++ src/allmydata/test/no_network.py | 2 +- src/allmydata/test/test_checker.py | 2 +- src/allmydata/test/test_storage_client.py | 2 +- src/allmydata/test/test_util.py | 2 +- src/allmydata/test/web/test_root.py | 2 +- 8 files changed, 8 insertions(+), 15 deletions(-) diff --git a/src/allmydata/client.py b/src/allmydata/client.py index b9225e169..335d5b31d 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -15,7 +15,6 @@ from twisted.application.internet import TimerService from twisted.python.filepath import FilePath from twisted.python.failure import Failure from pycryptopp.publickey import rsa -from pycryptopp.publickey import ed25519 import allmydata from allmydata.storage.server import StorageServer diff --git a/src/allmydata/scripts/tahoe_grid_manager.py b/src/allmydata/scripts/tahoe_grid_manager.py index d75b293cd..7cbfcd22d 100644 --- a/src/allmydata/scripts/tahoe_grid_manager.py +++ b/src/allmydata/scripts/tahoe_grid_manager.py @@ -9,17 +9,11 @@ from pycryptopp.publickey import ed25519 # perhaps NaCl instead? other code use from allmydata.scripts.common import BasedirOptions from allmydata.util.abbreviate import abbreviate_time -from twisted.scripts import twistd from twisted.python import usage -from twisted.python.reflect import namedAny from twisted.python.filepath import FilePath -from allmydata.scripts.default_nodedir import _default_nodedir from allmydata.util import fileutil from allmydata.util import base32 from allmydata.util import keyutil -from allmydata.node import read_config -from allmydata.util.encodingutil import listdir_unicode, quote_local_unicode_path -from twisted.application.service import Service from twisted.internet.defer import inlineCallbacks, returnValue @@ -178,7 +172,7 @@ class _GridManager(object): if 'private_key' not in config: raise ValueError( "Grid Manager config from '{}' requires a 'private_key'".format( - config_config + config_location, ) ) @@ -406,7 +400,6 @@ def _list(gridoptions, options): else: print(" {}: expired ({})".format(cert_count, abbreviate_time(delta))) cert_count += 1 - certs = ' ({} certificates)'.format(cert_count) def _sign(gridoptions, options): @@ -428,7 +421,6 @@ def _sign(gridoptions, options): print(certificate_data) if fp is not None: next_serial = 0 - p = fp.child('{}.cert'.format(options['name'])).path while fp.child("{}.cert.{}".format(options['name'], next_serial)).exists(): next_serial += 1 with fp.child('{}.cert.{}'.format(options['name'], next_serial)).open('w') as f: diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 34e96f38a..f0928d46b 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -33,6 +33,7 @@ import re import time import hashlib import json +from datetime import datetime from zope.interface import implementer from twisted.internet import defer @@ -41,6 +42,7 @@ from twisted.application import service from foolscap.api import eventually from allmydata.interfaces import IStorageBroker, IDisplayableServer, IServer from allmydata.util import log, base32, connection_status +from allmydata.util import keyutil from allmydata.util.assertutil import precondition from allmydata.util.observer import ObserverList from allmydata.util.rrefutil import add_version_to_remote_reference diff --git a/src/allmydata/test/no_network.py b/src/allmydata/test/no_network.py index 1fcfab1f9..de15d32bf 100644 --- a/src/allmydata/test/no_network.py +++ b/src/allmydata/test/no_network.py @@ -171,7 +171,7 @@ class NoNetworkServer(object): @implementer(IStorageBroker) class NoNetworkStorageBroker(object): - def get_servers_for_psi(self, peer_selection_index): + def get_servers_for_psi(self, peer_selection_index, for_upload=True): def _permuted(server): seed = server.get_permutation_seed() return permute_server_hash(peer_selection_index, seed) diff --git a/src/allmydata/test/test_checker.py b/src/allmydata/test/test_checker.py index 46dec8c24..7f89e9e08 100644 --- a/src/allmydata/test/test_checker.py +++ b/src/allmydata/test/test_checker.py @@ -41,7 +41,7 @@ class WebResultsRendering(unittest.TestCase, WebRenderingMixin): "my-version": "ver", "oldest-supported": "oldest", } - s = NativeStorageServer(server_id, ann, None, None) + s = NativeStorageServer(server_id, ann, None, None, {}, None) sb.test_add_server(server_id, s) c = FakeClient() c.storage_broker = sb diff --git a/src/allmydata/test/test_storage_client.py b/src/allmydata/test/test_storage_client.py index a78f91acb..6a4ae5d06 100644 --- a/src/allmydata/test/test_storage_client.py +++ b/src/allmydata/test/test_storage_client.py @@ -40,7 +40,7 @@ class TestNativeStorageServer(unittest.TestCase): ann = {"anonymous-storage-FURL": "pb://w2hqnbaa25yw4qgcvghl5psa3srpfgw3@tcp:127.0.0.1:51309/vucto2z4fxment3vfxbqecblbf6zyp6x", "permutation-seed-base32": "w2hqnbaa25yw4qgcvghl5psa3srpfgw3", } - nss = NativeStorageServer("server_id", ann, None, {}) + nss = NativeStorageServer("server_id", ann, None, {}, {}, None) self.assertEqual(nss.get_nickname(), "") class TestStorageFarmBroker(unittest.TestCase): diff --git a/src/allmydata/test/test_util.py b/src/allmydata/test/test_util.py index 381add043..76e3f8155 100644 --- a/src/allmydata/test/test_util.py +++ b/src/allmydata/test/test_util.py @@ -96,7 +96,7 @@ class HumanReadable(unittest.TestCase): def test_abbrev_time_future_5_minutes(self): diff = timedelta(minutes=-5) s = abbreviate.abbreviate_time(diff) - self.assertEqual('5 minutes in the future', s) + self.assertEqual('5 minutes from now', s) def test_abbrev_time_hours(self): diff = timedelta(hours=4) diff --git a/src/allmydata/test/web/test_root.py b/src/allmydata/test/web/test_root.py index 727189421..81b531113 100644 --- a/src/allmydata/test/web/test_root.py +++ b/src/allmydata/test/web/test_root.py @@ -26,7 +26,7 @@ class RenderServiceRow(unittest.TestCase): ann = {"anonymous-storage-FURL": "pb://w2hqnbaa25yw4qgcvghl5psa3srpfgw3@tcp:127.0.0.1:51309/vucto2z4fxment3vfxbqecblbf6zyp6x", "permutation-seed-base32": "w2hqnbaa25yw4qgcvghl5psa3srpfgw3", } - s = NativeStorageServer("server_id", ann, None, {}) + s = NativeStorageServer("server_id", ann, None, {}, {}, None) cs = ConnectionStatus(False, "summary", {}, 0, 0) s.get_connection_status = lambda: cs From 4afebbd88cfd302131097d7684a854dff7719165 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 31 Jul 2018 15:23:58 -0600 Subject: [PATCH 0014/2309] better docs and make load consistent with save internal-API --- src/allmydata/scripts/tahoe_grid_manager.py | 43 ++++++++++++++------- 1 file changed, 30 insertions(+), 13 deletions(-) diff --git a/src/allmydata/scripts/tahoe_grid_manager.py b/src/allmydata/scripts/tahoe_grid_manager.py index 7cbfcd22d..60d8b0f63 100644 --- a/src/allmydata/scripts/tahoe_grid_manager.py +++ b/src/allmydata/scripts/tahoe_grid_manager.py @@ -282,8 +282,12 @@ class _GridManager(object): def _save_gridmanager_config(file_path, grid_manager): """ - Writes a Grid Manager configuration to the place specified by - 'file_path' (if None, stdout is used). + Writes a Grid Manager configuration. + + :param file_path: a FilePath specifying where to write the config + (if None, stdout is used) + + :param grid_manager: a _GridManager instance """ data = json.dumps( grid_manager.marshal(), @@ -298,20 +302,33 @@ def _save_gridmanager_config(file_path, grid_manager): f.write("{}\n".format(data)) -# XXX should take a FilePath or None -def _load_gridmanager_config(gm_config): +def _config_to_filepath(gm_config_location): """ - Loads a Grid Manager configuration and returns it (a dict) after - validating. Exceptions if the config can't be found, or has - problems. + Converts a command-line string specifying the GridManager + configuration's location into a readable file-like object. + + :param gm_config_location str: a valid GridManager directory or + '-' (a single dash) to use stdin. """ fp = None if gm_config.strip() != '-': - fp = FilePath(gm_config.strip()) + fp = FilePath(gm_config_location.strip()) if not fp.exists(): raise RuntimeError( "No such directory '{}'".format(gm_config) ) + return fp + + +def _load_gridmanager_config(file_path) + """ + Loads a Grid Manager configuration and returns it (a dict) after + validating. Exceptions if the config can't be found, or has + problems. + + :param file_path: a FilePath to a vlid GridManager directory or + None to load from stdin. + """ if fp is None: gm = json.load(sys.stdin) @@ -329,7 +346,7 @@ def _show_identity(gridoptions, options): gm_config = gridoptions['config'].strip() assert gm_config is not None - gm = _load_gridmanager_config(gm_config) + gm = _load_gridmanager_config(_config_to_filepath(gm_config)) print(gm.public_identity()) @@ -340,7 +357,7 @@ def _add(gridoptions, options): gm_config = gridoptions['config'].strip() fp = FilePath(gm_config) if gm_config.strip() != '-' else None - gm = _load_gridmanager_config(gm_config) + gm = _load_gridmanager_config(_config_to_filepath(gm_config)) try: gm.add_storage_server( options['name'], @@ -361,7 +378,7 @@ def _remove(gridoptions, options): """ gm_config = gridoptions['config'].strip() fp = FilePath(gm_config) if gm_config.strip() != '-' else None - gm = _load_gridmanager_config(gm_config) + gm = _load_gridmanager_config(_config_to_filepath(gm_config)) try: gm.remove_storage_server(options['name']) @@ -385,7 +402,7 @@ def _list(gridoptions, options): gm_config = gridoptions['config'].strip() fp = FilePath(gm_config) if gm_config.strip() != '-' else None - gm = _load_gridmanager_config(gm_config) + gm = _load_gridmanager_config(_config_to_filepath(gm_config)) for name in sorted(gm.storage_servers.keys()): print("{}: {}".format(name, gm.storage_servers[name].public_key())) if fp: @@ -408,7 +425,7 @@ def _sign(gridoptions, options): """ gm_config = gridoptions['config'].strip() fp = FilePath(gm_config) if gm_config.strip() != '-' else None - gm = _load_gridmanager_config(gm_config) + gm = _load_gridmanager_config(_config_to_filepath(gm_config)) try: certificate = gm.sign(options['name']) From e5f608f80e1f99d79b0317954271d418c25546f9 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 3 Aug 2018 13:09:39 -0600 Subject: [PATCH 0015/2309] fixes to Options use --- src/allmydata/client.py | 3 +- src/allmydata/scripts/tahoe_grid_manager.py | 77 ++++++++++++--------- 2 files changed, 47 insertions(+), 33 deletions(-) diff --git a/src/allmydata/client.py b/src/allmydata/client.py index 335d5b31d..478a3fc98 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -387,8 +387,9 @@ def create_storage_farm_broker(config, default_connection_handlers, foolscap_con grid_manager_keys = [] gm_keydata = self.get_config('client', 'grid_manager_public_keys', '') for gm_key in gm_keydata.strip().split(): + # XXX FIXME this needs pub-v0- prefix then ... grid_manager_keys.append( - keyutil.parse_pubkey(a2b(gm_key)) + keyutil.parse_pubkey(gm_key) ) my_pubkey = keyutil.parse_pubkey( diff --git a/src/allmydata/scripts/tahoe_grid_manager.py b/src/allmydata/scripts/tahoe_grid_manager.py index 60d8b0f63..5dd084c01 100644 --- a/src/allmydata/scripts/tahoe_grid_manager.py +++ b/src/allmydata/scripts/tahoe_grid_manager.py @@ -7,7 +7,7 @@ from datetime import datetime from pycryptopp.publickey import ed25519 # perhaps NaCl instead? other code uses this though -from allmydata.scripts.common import BasedirOptions +from allmydata.scripts.common import BaseOptions from allmydata.util.abbreviate import abbreviate_time from twisted.python import usage from twisted.python.filepath import FilePath @@ -17,13 +17,13 @@ from allmydata.util import keyutil from twisted.internet.defer import inlineCallbacks, returnValue -class CreateOptions(BasedirOptions): +class CreateOptions(BaseOptions): description = ( "Create a new identity key and configuration of a Grid Manager" ) -class ShowIdentityOptions(BasedirOptions): +class ShowIdentityOptions(BaseOptions): description = ( "Show the public identity key of a Grid Manager\n" "\n" @@ -32,13 +32,17 @@ class ShowIdentityOptions(BasedirOptions): ) -class AddOptions(BasedirOptions): +class AddOptions(BaseOptions): description = ( - "Add a new storage-server's key to a Grid Manager configuration" + "Add a new storage-server's key to a Grid Manager configuration\n" + "using NAME and PUBIC_KEY (comes from a node.pubkey file)" ) + def getSynopsis(self): + return "{} add NAME PUBLIC_KEY".format(BaseOptions.getSynopsis()) + def parseArgs(self, *args, **kw): - BasedirOptions.parseArgs(self, **kw) + BaseOptions.parseArgs(self, **kw) if len(args) != 2: raise usage.UsageError( "Requires two arguments: name public_key" @@ -53,13 +57,13 @@ class AddOptions(BasedirOptions): ) -class RemoveOptions(BasedirOptions): +class RemoveOptions(BaseOptions): description = ( "Remove a storage-server from a Grid Manager configuration" ) def parseArgs(self, *args, **kw): - BasedirOptions.parseArgs(self, **kw) + BaseOptions.parseArgs(self, **kw) if len(args) != 1: raise usage.UsageError( "Requires one arguments: name" @@ -67,19 +71,22 @@ class RemoveOptions(BasedirOptions): self['name'] = unicode(args[0]) -class ListOptions(BasedirOptions): +class ListOptions(BaseOptions): description = ( "List all storage servers in this Grid Manager" ) -class SignOptions(BasedirOptions): +class SignOptions(BaseOptions): description = ( "Create and sign a new certificate for a storage-server" ) + def getSynopsis(self): + return "{} NAME".format(super(SignOptions, self).getSynopsis()) + def parseArgs(self, *args, **kw): - BasedirOptions.parseArgs(self, **kw) + BaseOptions.parseArgs(self, **kw) if len(args) != 1: raise usage.UsageError( "Requires one argument: name" @@ -87,7 +94,7 @@ class SignOptions(BasedirOptions): self['name'] = unicode(args[0]) -class GridManagerOptions(BasedirOptions): +class GridManagerOptions(BaseOptions): subCommands = [ ["create", None, CreateOptions, "Create a Grid Manager."], ["public-identity", None, ShowIdentityOptions, "Get the public-key for this Grid Manager."], @@ -115,7 +122,8 @@ class GridManagerOptions(BasedirOptions): 'upload shares to" (X and Y are public-keys).' '\n\n' 'Clients can use Grid Managers to decide which storage servers to ' - 'upload shares to.' + 'upload shares to. They do this by adding one or more Grid Manager ' + 'public keys to their config.' ) @@ -169,6 +177,10 @@ class _GridManager(object): @staticmethod def from_config(config, config_location): + if not config: + raise ValueError( + "Invalid Grid Manager config in '{}'".format(config_location) + ) if 'private_key' not in config: raise ValueError( "Grid Manager config from '{}' requires a 'private_key'".format( @@ -307,28 +319,26 @@ def _config_to_filepath(gm_config_location): Converts a command-line string specifying the GridManager configuration's location into a readable file-like object. - :param gm_config_location str: a valid GridManager directory or - '-' (a single dash) to use stdin. + :param gm_config_location str: a valid path, or '-' (a single + dash) to use stdin. """ - fp = None - if gm_config.strip() != '-': - fp = FilePath(gm_config_location.strip()) - if not fp.exists(): - raise RuntimeError( - "No such directory '{}'".format(gm_config) - ) - return fp -def _load_gridmanager_config(file_path) +def _load_gridmanager_config(gm_config): """ Loads a Grid Manager configuration and returns it (a dict) after validating. Exceptions if the config can't be found, or has problems. - :param file_path: a FilePath to a vlid GridManager directory or - None to load from stdin. + :param gm_config str: "-" (a single dash) for stdin or a filename """ + fp = None + if gm_config.strip() != '-': + fp = FilePath(gm_config.strip()) + if not fp.exists(): + raise RuntimeError( + "No such directory '{}'".format(gm_config) + ) if fp is None: gm = json.load(sys.stdin) @@ -336,7 +346,10 @@ def _load_gridmanager_config(file_path) with fp.child("config.json").open("r") as f: gm = json.load(f) - return _GridManager.from_config(gm, gm_config) + try: + return _GridManager.from_config(gm, gm_config) + except ValueError as e: + raise usage.UsageError(str(e)) def _show_identity(gridoptions, options): @@ -346,7 +359,7 @@ def _show_identity(gridoptions, options): gm_config = gridoptions['config'].strip() assert gm_config is not None - gm = _load_gridmanager_config(_config_to_filepath(gm_config)) + gm = _load_gridmanager_config(gm_config) print(gm.public_identity()) @@ -357,7 +370,7 @@ def _add(gridoptions, options): gm_config = gridoptions['config'].strip() fp = FilePath(gm_config) if gm_config.strip() != '-' else None - gm = _load_gridmanager_config(_config_to_filepath(gm_config)) + gm = _load_gridmanager_config(gm_config) try: gm.add_storage_server( options['name'], @@ -378,7 +391,7 @@ def _remove(gridoptions, options): """ gm_config = gridoptions['config'].strip() fp = FilePath(gm_config) if gm_config.strip() != '-' else None - gm = _load_gridmanager_config(_config_to_filepath(gm_config)) + gm = _load_gridmanager_config(gm_config) try: gm.remove_storage_server(options['name']) @@ -402,7 +415,7 @@ def _list(gridoptions, options): gm_config = gridoptions['config'].strip() fp = FilePath(gm_config) if gm_config.strip() != '-' else None - gm = _load_gridmanager_config(_config_to_filepath(gm_config)) + gm = _load_gridmanager_config(gm_config) for name in sorted(gm.storage_servers.keys()): print("{}: {}".format(name, gm.storage_servers[name].public_key())) if fp: @@ -425,7 +438,7 @@ def _sign(gridoptions, options): """ gm_config = gridoptions['config'].strip() fp = FilePath(gm_config) if gm_config.strip() != '-' else None - gm = _load_gridmanager_config(_config_to_filepath(gm_config)) + gm = _load_gridmanager_config(gm_config) try: certificate = gm.sign(options['name']) From 2692c2caea89e786eaa7c3c937b47208bd8f2f36 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 3 Aug 2018 15:08:22 -0600 Subject: [PATCH 0016/2309] verify certs properly --- src/allmydata/scripts/tahoe_grid_manager.py | 2 +- src/allmydata/storage_client.py | 46 +++++++++++++-------- 2 files changed, 29 insertions(+), 19 deletions(-) diff --git a/src/allmydata/scripts/tahoe_grid_manager.py b/src/allmydata/scripts/tahoe_grid_manager.py index 5dd084c01..91dc4f890 100644 --- a/src/allmydata/scripts/tahoe_grid_manager.py +++ b/src/allmydata/scripts/tahoe_grid_manager.py @@ -243,7 +243,7 @@ class _GridManager(object): "public_key": srv.public_key(), "version": 1, } - cert_data = json.dumps(cert_info, separators=(',',':'), sort_keys=True) + cert_data = json.dumps(cert_info, separators=(',',':'), sort_keys=True).encode('utf8') sig = self._private_key.sign(cert_data) certificate = { u"certificate": cert_data, diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index f0928d46b..dcf92a61c 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -38,8 +38,9 @@ from datetime import datetime from zope.interface import implementer from twisted.internet import defer from twisted.application import service - from foolscap.api import eventually +from pycryptopp.publickey import ed25519 # perhaps NaCl instead? other code uses this though + from allmydata.interfaces import IStorageBroker, IDisplayableServer, IServer from allmydata.util import log, base32, connection_status from allmydata.util import keyutil @@ -99,7 +100,7 @@ class StorageFarmBroker(service.MultiService): self._static_server_ids.add(server_id) handler_overrides = server.get("connections", {}) s = NativeStorageServer(server_id, server["ann"], - self._tub_maker, handler_overrides, [], self._node_pubkey) + self._tub_maker, handler_overrides, []) s.on_status_changed(lambda _: self._got_connection()) s.setServiceParent(self) self.servers[server_id] = s @@ -118,7 +119,7 @@ class StorageFarmBroker(service.MultiService): # these two are used in unit tests def test_add_rref(self, serverid, rref, ann): - s = NativeStorageServer(serverid, ann.copy(), self._tub_maker, {}, [], self._node_pubkey) + s = NativeStorageServer(serverid, ann.copy(), self._tub_maker, {}, []) s.rref = rref s._is_connected = True self.servers[serverid] = s @@ -159,7 +160,9 @@ class StorageFarmBroker(service.MultiService): facility="tahoe.storage_broker", umid="AlxzqA", level=log.UNUSUAL) return - s = NativeStorageServer(server_id, ann, self._tub_maker, {}, self._grid_manager_keys, self._node_pubkey) + + grid_manager_certs = ann.get("grid-manager-certificates", []) + s = NativeStorageServer(server_id, ann, self._tub_maker, {}, self._grid_manager_keys, grid_manager_certs) s.on_status_changed(lambda _: self._got_connection()) server_id = s.get_serverid() old = self.servers.get(server_id) @@ -272,7 +275,7 @@ class StubServer(object): return "?" -def _validate_grid_manager_certificate(gm_key, alleged_cert, storage_pubkey): +def _validate_grid_manager_certificate(gm_key, alleged_cert): """ :param gm_key: a VerifyingKey instance, a Grid Manager's public key. @@ -286,18 +289,16 @@ def _validate_grid_manager_certificate(gm_key, alleged_cert, storage_pubkey): """ try: gm_key.verify( - alleged_cert['signature'], - alleged_cert['certificate'], + base32.a2b(alleged_cert['signature'].encode('ascii')), + alleged_cert['certificate'].encode('ascii'), ) - except Exception: + except ed25519.BadSignatureError: return False # signature is valid; now we can load the actual data cert = json.loads(alleged_cert['certificate']) now = datetime.utcnow() - expires = datetime.fromordinal(cert['expires']) - cert_pubkey = keyutil.parse_pubkey(cert['public_key']) - if cert_pubkey != storage_pubkey: - return False # certificate is for wrong server + expires = datetime.utcfromtimestamp(cert['expires']) + # cert_pubkey = keyutil.parse_pubkey(cert['public_key'].encode('ascii')) if expires < now: return False # certificate is expired return True @@ -331,15 +332,25 @@ class NativeStorageServer(service.MultiService): "application-version": "unknown: no get_version()", } - def __init__(self, server_id, ann, tub_maker, handler_overrides, grid_manager_keys, node_pubkey): + def __init__(self, server_id, ann, tub_maker, handler_overrides, grid_manager_keys, grid_manager_certs): service.MultiService.__init__(self) assert isinstance(server_id, str) self._server_id = server_id self.announcement = ann self._tub_maker = tub_maker self._handler_overrides = handler_overrides - self._node_pubkey = node_pubkey + + # XXX we should validate as much as we can about the + # certificates right now -- the only thing we HAVE to be lazy + # about is the expiry, which should be checked before any + # possible upload... + + # any public-keys which the user has configured (if none, it + # means use any storage servers) self._grid_manager_keys = grid_manager_keys + # any storage-certificates that this storage-server included + # in its announcement + self._grid_manager_certificates = grid_manager_certs assert "anonymous-storage-FURL" in ann, ann furl = str(ann["anonymous-storage-FURL"]) @@ -395,12 +406,11 @@ class NativeStorageServer(service.MultiService): # XXX probably want to cache the answer to this? (ignoring # that for now because certificates expire, so .. slightly # more complex) - certificates = self.announcements.get("grid-manager-certificates", None) - if not certificates: + if not self._grid_manager_certificates: return False for gm_key in self._grid_manager_keys: - for cert in certificates: - if _validate_grid_manager_certificate(gm_key, cert, self._pubkey): + for cert in self._grid_manager_certificates: + if _validate_grid_manager_certificate(gm_key, cert): return True return False From 36c471fdef25e67a9d0979bb87394ae6f0a5c742 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 3 Aug 2018 15:13:30 -0600 Subject: [PATCH 0017/2309] don't need our pubkey for anything --- src/allmydata/client.py | 8 +++++--- src/allmydata/scripts/tahoe_grid_manager.py | 6 ++++++ src/allmydata/storage_client.py | 3 +-- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/src/allmydata/client.py b/src/allmydata/client.py index 478a3fc98..7d7723b6d 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -392,9 +392,11 @@ def create_storage_farm_broker(config, default_connection_handlers, foolscap_con keyutil.parse_pubkey(gm_key) ) - my_pubkey = keyutil.parse_pubkey( - self.get_config_from_file("node.pubkey") - ) + # we don't actually use this keypair for anything (yet) as far + # as I can see. + # my_pubkey = keyutil.parse_pubkey( + # self.get_config_from_file("node.pubkey") + # ) # create the actual storage-broker diff --git a/src/allmydata/scripts/tahoe_grid_manager.py b/src/allmydata/scripts/tahoe_grid_manager.py index 91dc4f890..01cb72356 100644 --- a/src/allmydata/scripts/tahoe_grid_manager.py +++ b/src/allmydata/scripts/tahoe_grid_manager.py @@ -249,6 +249,12 @@ class _GridManager(object): u"certificate": cert_data, u"signature": base32.b2a(sig), } + + if True: + verify_key_bytes = self._private_key.get_verifying_key_bytes() + vk = ed25519.VerifyingKey(verify_key_bytes) + assert vk.verify(sig, cert_data) is None, "cert should verify" + return certificate def add_storage_server(self, name, public_key): diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index dcf92a61c..33c72e5e2 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -74,14 +74,13 @@ class StorageFarmBroker(service.MultiService): I'm also responsible for subscribing to the IntroducerClient to find out about new servers as they are announced by the Introducer. """ - def __init__(self, permute_peers, tub_maker, preferred_peers=(), grid_manager_keys=[], node_pubkey=None): + def __init__(self, permute_peers, tub_maker, preferred_peers=(), grid_manager_keys=[]): service.MultiService.__init__(self) assert permute_peers # False not implemented yet self.permute_peers = permute_peers self._tub_maker = tub_maker self.preferred_peers = preferred_peers self._grid_manager_keys = grid_manager_keys - self._node_pubkey = node_pubkey # self.servers maps serverid -> IServer, and keeps track of all the # storage servers that we've heard about. Each descriptor manages its From df88b9059f410a83e9e491d024d210a847ddd1c4 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 3 Aug 2018 15:24:12 -0600 Subject: [PATCH 0018/2309] put 'header' on pubkey --- src/allmydata/scripts/tahoe_grid_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/scripts/tahoe_grid_manager.py b/src/allmydata/scripts/tahoe_grid_manager.py index 01cb72356..8e1d1093c 100644 --- a/src/allmydata/scripts/tahoe_grid_manager.py +++ b/src/allmydata/scripts/tahoe_grid_manager.py @@ -366,7 +366,7 @@ def _show_identity(gridoptions, options): assert gm_config is not None gm = _load_gridmanager_config(gm_config) - print(gm.public_identity()) + print("pub-v0-{}".format(gm.public_identity())) def _add(gridoptions, options): From 62a6277429a51024d4002ba2a533654362832292 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 3 Aug 2018 15:24:25 -0600 Subject: [PATCH 0019/2309] bring tutorial/doc in line with implementation --- docs/proposed/grid-manager/managed-grid.rst | 56 ++++++++++++--------- 1 file changed, 33 insertions(+), 23 deletions(-) diff --git a/docs/proposed/grid-manager/managed-grid.rst b/docs/proposed/grid-manager/managed-grid.rst index d13cfef33..bcc7f7164 100644 --- a/docs/proposed/grid-manager/managed-grid.rst +++ b/docs/proposed/grid-manager/managed-grid.rst @@ -169,17 +169,17 @@ Example Setup of a New Managed Grid We'll store our Grid Manager configuration on disk, in ``~/grid-manager``. To initialize this directory:: - tahoe grid-manager create --config ~/grid-manager + tahoe grid-manager --config ~/grid-manager create This example creates an actual grid, but it's all just on one machine -with different "node directories". Usually of course each one would be -on a separate computer. +with different "node directories". Usually of course each storage +server would be on a separate computer. (If you already have a grid, you can :ref:`skip ahead `.) First of all, create an Introducer. Note that we actually have to run it briefly before it creates the "Introducer fURL" we want for the -next steps. +next steps:: tahoe create-introducer --listen=tcp --port=5555 --location=tcp:localhost:5555 ./introducer tahoe -d introducer run @@ -194,21 +194,26 @@ Next, we attach a couple of storage nodes:: .. _skip_ahead: -We can now ask the Grid Manager to create certificates for our new -storage servers:: +We can now tell the Grid Manager about our new storage servers:: - tahoe grid-manager --config ~/grid-manager add-storage --pubkey $(cat storage0/node.pubkey) > storage0.cert - tahoe grid-manager --config ~/grid-manager add-storage --pubkey $(cat storage1/node.pubkey) > storage1.cert + tahoe grid-manager --config ~/grid-manager add storage0 $(cat storage0/node.pubkey) + tahoe grid-manager --config ~/grid-manager add storage1 $(cat storage1/node.pubkey) - # enroll server0 (using file) - kill $(cat storage0/twistd.pid) - tahoe -d storage0 admin add-grid-manager-cert --filename storage0.cert - daemonize tahoe -d storage0 run - # enroll server1 (using stdin) - kill $(cat storage1/twistd.pid) - cat storage1.cert | tahoe -d storage1 admin add-grid-manager-cert - daemonize tahoe -d storage1 run +To produce a new certificate for each node, we do this:: + + tahoe grid-manager --config ~/grid-manager sign storage0 > ./storage0/gridmanager.cert + tahoe grid-manager --config ~/grid-manager sign storage1 > ./storage1/gridmanager.cert + +Now, we want our storage servers to actually announce these +certificates into the grid. We do this by adding some configuration +(in ``tahoe.cfg``):: + + [storage] + grid_manager_certificate_files = gridmanager.cert + +Add the above bit to each node's ``tahoe.cfg`` and re-start the +storage nodes. Now try adding a new storage server ``storage2``. This client can join the grid just fine, and announce itself to the Introducer as providing @@ -232,19 +237,24 @@ grid-manager has given certificates to (``storage0`` and ``storage1``). We need the grid-manager's public key to put in Alice's configuration:: - kill $(cat alice/twistd.pid) - tahoe -d alice add-grid-manager --name work-grid $(tahoe grid-manager --config ~/grid-manager show-identity) - daemonize tahoe -d alice start + tahoe grid-manager --config ~/grid-manager public-identity + +Put the key printed out above into Alice's ``tahoe.cfg`` in section +``client``:: + + [client] + grid_manager_keys = pub-v0-vqimc4s5eflwajttsofisp5st566dbq36xnpp4siz57ufdavpvlq + DECIDE: - should the grid-manager be identified by a certificate? exarkun points out: --name seems like the hint of the beginning of a use-case for certificates rather than bare public keys?). -Since we made Alice's parameters require 3 storage servers to be -reachable (`--happy=3`), all their uploads should now fail (so `tahoe -mkdir` will fail) because they won't use storage2 and can't "achieve -happiness". +Now, re-start the "alice" client. Since we made Alice's parameters +require 3 storage servers to be reachable (``--happy=3``), all their +uploads should now fail (so ``tahoe mkdir`` will fail) because they +won't use storage2 and thus can't "achieve happiness". You can check Alice's "Welcome" page (where the list of connected servers is) at http://localhost:6301/ and should be able to see details about From 82a74e898645526212de921684d86605818b71a1 Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 16 Aug 2018 17:26:49 -0600 Subject: [PATCH 0020/2309] updates and a bunch of debugging for cert/grid-manager handling --- src/allmydata/client.py | 2 +- src/allmydata/node.py | 13 +++++++++++++ src/allmydata/storage_client.py | 22 ++++++++++++++++++++-- 3 files changed, 34 insertions(+), 3 deletions(-) diff --git a/src/allmydata/client.py b/src/allmydata/client.py index 7d7723b6d..c45a20698 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -386,7 +386,7 @@ def create_storage_farm_broker(config, default_connection_handlers, foolscap_con grid_manager_keys = [] gm_keydata = self.get_config('client', 'grid_manager_public_keys', '') - for gm_key in gm_keydata.strip().split(): + for name, gm_key in self.config.enumerate_section('grid_managers').items(): # XXX FIXME this needs pub-v0- prefix then ... grid_manager_keys.append( keyutil.parse_pubkey(gm_key) diff --git a/src/allmydata/node.py b/src/allmydata/node.py index 426a0f796..c058fce4f 100644 --- a/src/allmydata/node.py +++ b/src/allmydata/node.py @@ -287,6 +287,19 @@ class _Config(object): "Unable to write config file '{}'".format(fn), ) + def enumerate_section(self, section): + """ + returns a dict containing all items in a configuration section. an + empty dict is returned if the section doesn't exist. + """ + answer = dict() + try: + for k in self.config.options(section): + answer[k] = self.config.get(section, k) + except ConfigParser.NoSectionError: + pass + return answer + def get_config(self, section, option, default=_None, boolean=False): try: if boolean: diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 33c72e5e2..1a928ea27 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -98,8 +98,15 @@ class StorageFarmBroker(service.MultiService): server_id = server_id.encode("ascii") self._static_server_ids.add(server_id) handler_overrides = server.get("connections", {}) - s = NativeStorageServer(server_id, server["ann"], - self._tub_maker, handler_overrides, []) + s = NativeStorageServer( + server_id, + server["ann"], + self._tub_maker, + handler_overrides, + self._grid_manager_keys, + self._grid_manager_certificates, + ) + print("SET STATIC {}".format(s)) s.on_status_changed(lambda _: self._got_connection()) s.setServiceParent(self) self.servers[server_id] = s @@ -161,6 +168,7 @@ class StorageFarmBroker(service.MultiService): return grid_manager_certs = ann.get("grid-manager-certificates", []) + print("certs for {}: {}".format(key_s, grid_manager_certs)) s = NativeStorageServer(server_id, ann, self._tub_maker, {}, self._grid_manager_keys, grid_manager_certs) s.on_status_changed(lambda _: self._got_connection()) server_id = s.get_serverid() @@ -332,6 +340,7 @@ class NativeStorageServer(service.MultiService): } def __init__(self, server_id, ann, tub_maker, handler_overrides, grid_manager_keys, grid_manager_certs): + print("CREATE {}: {}".format(server_id, grid_manager_certs)) service.MultiService.__init__(self) assert isinstance(server_id, str) self._server_id = server_id @@ -347,9 +356,11 @@ class NativeStorageServer(service.MultiService): # any public-keys which the user has configured (if none, it # means use any storage servers) self._grid_manager_keys = grid_manager_keys + print("keys: {}".format(self._grid_manager_keys)) # any storage-certificates that this storage-server included # in its announcement self._grid_manager_certificates = grid_manager_certs + print("certs: {}".format(self._grid_manager_certificates)) assert "anonymous-storage-FURL" in ann, ann furl = str(ann["anonymous-storage-FURL"]) @@ -398,19 +409,26 @@ class NativeStorageServer(service.MultiService): :return: True if we should use this server for uploads, False otherwise. """ + print("upload permitted? {}".format(self._server_id)) # if we have no Grid Manager keys configured, choice is easy if not self._grid_manager_keys: + print("{} no grid manager keys at all (so yes)".format(self._server_id)) return True # XXX probably want to cache the answer to this? (ignoring # that for now because certificates expire, so .. slightly # more complex) if not self._grid_manager_certificates: + print("{} no grid-manager certificates {} (so no)".format(self._server_id, self._grid_manager_certificates)) return False for gm_key in self._grid_manager_keys: for cert in self._grid_manager_certificates: if _validate_grid_manager_certificate(gm_key, cert): + print("valid: {}\n{}".format(gm_key, cert)) return True + else: + print("invalid: {}\n{}".format(gm_key, cert)) + print("didn't validate {} keys".format(len(self._grid_manager_keys))) return False From 5deb31ea2a0a698c4f838c993cca69e46aa079be Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 24 Sep 2018 15:04:48 -0600 Subject: [PATCH 0021/2309] different directory for example grid-manager --- docs/proposed/grid-manager/managed-grid.rst | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/docs/proposed/grid-manager/managed-grid.rst b/docs/proposed/grid-manager/managed-grid.rst index bcc7f7164..5df37aff1 100644 --- a/docs/proposed/grid-manager/managed-grid.rst +++ b/docs/proposed/grid-manager/managed-grid.rst @@ -167,9 +167,9 @@ Example Setup of a New Managed Grid ----------------------------------- We'll store our Grid Manager configuration on disk, in -``~/grid-manager``. To initialize this directory:: +``./gm0``. To initialize this directory:: - tahoe grid-manager --config ~/grid-manager create + tahoe grid-manager --config ./gm0 create This example creates an actual grid, but it's all just on one machine with different "node directories". Usually of course each storage @@ -196,14 +196,13 @@ Next, we attach a couple of storage nodes:: We can now tell the Grid Manager about our new storage servers:: - tahoe grid-manager --config ~/grid-manager add storage0 $(cat storage0/node.pubkey) - tahoe grid-manager --config ~/grid-manager add storage1 $(cat storage1/node.pubkey) - + tahoe grid-manager --config ./gm0 add storage0 $(cat storage0/node.pubkey) + tahoe grid-manager --config ./gm0 add storage1 $(cat storage1/node.pubkey) To produce a new certificate for each node, we do this:: - tahoe grid-manager --config ~/grid-manager sign storage0 > ./storage0/gridmanager.cert - tahoe grid-manager --config ~/grid-manager sign storage1 > ./storage1/gridmanager.cert + tahoe grid-manager --config ./gm0 sign storage0 > ./storage0/gridmanager.cert + tahoe grid-manager --config ./gm0 sign storage1 > ./storage1/gridmanager.cert Now, we want our storage servers to actually announce these certificates into the grid. We do this by adding some configuration From a884c75ef6aac29036b1d487cdb9605403336037 Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 24 Sep 2018 15:04:58 -0600 Subject: [PATCH 0022/2309] proviso --- docs/proposed/grid-manager/managed-grid.rst | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/proposed/grid-manager/managed-grid.rst b/docs/proposed/grid-manager/managed-grid.rst index 5df37aff1..9f392f292 100644 --- a/docs/proposed/grid-manager/managed-grid.rst +++ b/docs/proposed/grid-manager/managed-grid.rst @@ -236,19 +236,23 @@ grid-manager has given certificates to (``storage0`` and ``storage1``). We need the grid-manager's public key to put in Alice's configuration:: - tahoe grid-manager --config ~/grid-manager public-identity + tahoe grid-manager --config ./gm0 public-identity Put the key printed out above into Alice's ``tahoe.cfg`` in section ``client``:: [client] - grid_manager_keys = pub-v0-vqimc4s5eflwajttsofisp5st566dbq36xnpp4siz57ufdavpvlq + grid_manager_public_keys = pub-v0-vqimc4s5eflwajttsofisp5st566dbq36xnpp4siz57ufdavpvlq DECIDE: - should the grid-manager be identified by a certificate? exarkun points out: --name seems like the hint of the beginning of a use-case for certificates rather than bare public keys?). + - (note the "--name" thing came from a former version of this + proposal that used CLI commands to add the public-keys -- but the + point remains, if there's to be metadata associated with "grid + managers" maybe they should be certificates..) Now, re-start the "alice" client. Since we made Alice's parameters require 3 storage servers to be reachable (``--happy=3``), all their From e453db879a46905e3c6cad90ef0dc8fce82a7fa0 Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 24 Sep 2018 15:06:03 -0600 Subject: [PATCH 0023/2309] make options work, config API usage --- src/allmydata/client.py | 17 ++++++++--------- src/allmydata/util/configutil.py | 17 +++++++++-------- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/src/allmydata/client.py b/src/allmydata/client.py index c45a20698..d504f5fbe 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -59,8 +59,8 @@ def _valid_config_sections(): "shares.needed", "shares.total", "stats_gatherer.furl", - "grid_managers", ), + "grid_managers": None, # means "any options valid" "drop_upload": ( # deprecated already? "enabled", ), @@ -83,6 +83,7 @@ def _valid_config_sections(): "readonly", "reserved_space", "storage_dir", + "grid_manager_certificate_files", ), "sftpd": ( "accounts.file", @@ -385,9 +386,9 @@ def create_storage_farm_broker(config, default_connection_handlers, foolscap_con # grid manager setup grid_manager_keys = [] - gm_keydata = self.get_config('client', 'grid_manager_public_keys', '') - for name, gm_key in self.config.enumerate_section('grid_managers').items(): + for name, gm_key in config.enumerate_section('grid_managers').items(): # XXX FIXME this needs pub-v0- prefix then ... + print("KEY: {}".format(gm_key)) grid_manager_keys.append( keyutil.parse_pubkey(gm_key) ) @@ -399,14 +400,13 @@ def create_storage_farm_broker(config, default_connection_handlers, foolscap_con # ) # create the actual storage-broker - + sb = storage_client.StorageFarmBroker( permute_peers=True, tub_maker=tub_creator, preferred_peers=preferred_peers, grid_manager_keys=grid_manager_keys, - node_pubkey=my_pubkey, - +## node_pubkey=my_pubkey, ) for ic in introducer_clients: sb.use_introducer(ic) @@ -609,7 +609,7 @@ class _Client(node.Node, pollmixin.PollMixin): grid_manager_certificates = [] cert_fnames = self.get_config("storage", "grid_manager_certificate_files", "") for fname in cert_fnames.split(): - fname = abspath_expanduser_unicode(fname.decode('ascii'), base=self.basedir) + fname = self.config.get_config_path(fname.decode('ascii')) if not os.path.exists(fname): raise ValueError( "Grid Manager certificate file '{}' doesn't exist".format( @@ -632,8 +632,7 @@ class _Client(node.Node, pollmixin.PollMixin): # of the Grid Manager (should that go in the config too, # then? How to handle multiple grid-managers?) - - furl_file = os.path.join(self.basedir, "private", "storage.furl").encode(get_filesystem_encoding()) + furl_file = self.config.get_private_path("storage.furl").encode(get_filesystem_encoding()) furl = self.tub.registerReference(ss, furlFile=furl_file) ann = { "anonymous-storage-FURL": furl, diff --git a/src/allmydata/util/configutil.py b/src/allmydata/util/configutil.py index 78894e301..0e22767cb 100644 --- a/src/allmydata/util/configutil.py +++ b/src/allmydata/util/configutil.py @@ -51,12 +51,13 @@ def validate_config(fname, cfg, valid_sections): section=section, ) ) - for option in cfg.options(section): - if option not in valid_in_section: - raise UnknownConfigError( - "'{fname}' section [{section}] contains unknown option '{option}'".format( - fname=fname, - section=section, - option=option, + if valid_in_section is not None: + for option in cfg.options(section): + if option not in valid_in_section: + raise UnknownConfigError( + "'{fname}' section [{section}] contains unknown option '{option}'".format( + fname=fname, + section=section, + option=option, + ) ) - ) From 41233284047602d5033b41cc61e2711d1edbc817 Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 24 Sep 2018 15:07:57 -0600 Subject: [PATCH 0024/2309] for mutable publishes, ignore servers w/o valid grid-manager certiicates --- src/allmydata/mutable/publish.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/allmydata/mutable/publish.py b/src/allmydata/mutable/publish.py index 629d06bd5..8810bdf14 100644 --- a/src/allmydata/mutable/publish.py +++ b/src/allmydata/mutable/publish.py @@ -945,6 +945,17 @@ class Publish: serverid = server.get_serverid() if server in self.bad_servers: continue + # if we have >= 1 grid-managers, this checks that we have + # a valid certificate for this server + if not server.upload_permitted(): + self.log( + "No valid grid-manager certificates for '{}' while choosing slots for mutable".format( + server.get_serverid(), + ), + level=log.UNUSUAL, + ) + continue + entry = (len(old_assignments.get(server, [])), i, serverid, server) serverlist.append(entry) serverlist.sort() From 026c7735b6ef303c0140bf11a9e8e48d541e1a63 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 27 Nov 2018 14:45:01 -0700 Subject: [PATCH 0025/2309] configuration is JSON --- docs/proposed/grid-manager/managed-grid.rst | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/docs/proposed/grid-manager/managed-grid.rst b/docs/proposed/grid-manager/managed-grid.rst index 9f392f292..602d93611 100644 --- a/docs/proposed/grid-manager/managed-grid.rst +++ b/docs/proposed/grid-manager/managed-grid.rst @@ -60,16 +60,12 @@ at which time you may be able to pass a directory-capability to this option). If you don't want to store the configuration on disk at all, you may -use ``--config -`` (that's a dash) and write a valid JSON (YAML? I'm -guessing JSON is easier to deal with here, more-interoperable?) +use ``--config -`` (that's a dash) and write a valid JSON configuration to stdin. All commands take the ``--config`` option, and they all behave similarly for "data from stdin" versus "data from disk". -DECIDE: - - config is YAML or JSON? - tahoe grid-manager create ````````````````````````` From 02dfb51dba9333c0af5f22b8cba891ae2488e723 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 27 Nov 2018 14:46:28 -0700 Subject: [PATCH 0026/2309] spelling --- docs/proposed/grid-manager/managed-grid.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/proposed/grid-manager/managed-grid.rst b/docs/proposed/grid-manager/managed-grid.rst index 602d93611..4cec4a040 100644 --- a/docs/proposed/grid-manager/managed-grid.rst +++ b/docs/proposed/grid-manager/managed-grid.rst @@ -111,7 +111,7 @@ Lists all storage-servers that have previously been added using tahoe grid-manager sign ``````````````````````` -Takes one arg: ``name``, the petname used previous in a ``tahoe +Takes one arg: ``name``, the petname used previously in a ``tahoe grid-manager add`` command. Note that this mutates the state of the grid-manager if it is on disk, From 2415058dd5d39180ec761c65d666c4dd1917ac46 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 27 Nov 2018 14:54:05 -0700 Subject: [PATCH 0027/2309] more-clear output from 'tahoe grid-manager lsit' --- src/allmydata/scripts/tahoe_grid_manager.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/scripts/tahoe_grid_manager.py b/src/allmydata/scripts/tahoe_grid_manager.py index 8e1d1093c..ee6080174 100644 --- a/src/allmydata/scripts/tahoe_grid_manager.py +++ b/src/allmydata/scripts/tahoe_grid_manager.py @@ -432,9 +432,9 @@ def _list(gridoptions, options): expires = datetime.fromtimestamp(cert_data['expires']) delta = datetime.utcnow() - expires if delta.total_seconds() < 0: - print(" {}: valid until {} ({})".format(cert_count, expires, abbreviate_time(delta))) + print("{}: cert {}: valid until {} ({})".format(name, cert_count, expires, abbreviate_time(delta))) else: - print(" {}: expired ({})".format(cert_count, abbreviate_time(delta))) + print("{}: cert {}: expired ({})".format(name, cert_count, abbreviate_time(delta))) cert_count += 1 From 71f96c9c9e340f7dcf68a0c95525c3cf2872c104 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 27 Nov 2018 15:01:01 -0700 Subject: [PATCH 0028/2309] call super properly; docstring --- src/allmydata/scripts/tahoe_grid_manager.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/allmydata/scripts/tahoe_grid_manager.py b/src/allmydata/scripts/tahoe_grid_manager.py index ee6080174..f2c3aace6 100644 --- a/src/allmydata/scripts/tahoe_grid_manager.py +++ b/src/allmydata/scripts/tahoe_grid_manager.py @@ -39,7 +39,7 @@ class AddOptions(BaseOptions): ) def getSynopsis(self): - return "{} add NAME PUBLIC_KEY".format(BaseOptions.getSynopsis()) + return "{} add NAME PUBLIC_KEY".format(super(AddOptions, self).getSynopsis()) def parseArgs(self, *args, **kw): BaseOptions.parseArgs(self, **kw) @@ -128,7 +128,13 @@ class GridManagerOptions(BaseOptions): def _create_gridmanager(): - return _GridManager(ed25519.SigningKey(os.urandom(32)), {}) + """ + :return: an object providing the GridManager interface initialized + with a new random keypair + """ + private_key_bytes, public_key_bytes = keyutil.make_keypair() + secret_key, public_key_bytes = keyutil.parse_privkey(private_key_bytes) + return _GridManager(secret_key, {}) def _create(gridoptions, options): """ From 03258dcb9f4ce75979cca5c39d1368344b22b094 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 27 Nov 2018 19:45:50 -0700 Subject: [PATCH 0029/2309] use keyutil --- src/allmydata/scripts/tahoe_grid_manager.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/src/allmydata/scripts/tahoe_grid_manager.py b/src/allmydata/scripts/tahoe_grid_manager.py index f2c3aace6..6fbdd4f7b 100644 --- a/src/allmydata/scripts/tahoe_grid_manager.py +++ b/src/allmydata/scripts/tahoe_grid_manager.py @@ -133,8 +133,7 @@ def _create_gridmanager(): with a new random keypair """ private_key_bytes, public_key_bytes = keyutil.make_keypair() - secret_key, public_key_bytes = keyutil.parse_privkey(private_key_bytes) - return _GridManager(secret_key, {}) + return _GridManager(private_key_bytes, {}) def _create(gridoptions, options): """ @@ -194,10 +193,9 @@ class _GridManager(object): ) ) - private_key_str = config['private_key'] + private_key_bytes = config['private_key'].encode('ascii') try: - private_key_bytes = base32.a2b(private_key_str.encode('ascii')) - private_key = ed25519.SigningKey(private_key_bytes) + private_key, public_key_bytes = keyutil.parse_privkey(private_key_bytes) except Exception as e: raise ValueError( "Invalid Grid Manager private_key: {}".format(e) @@ -222,11 +220,12 @@ class _GridManager(object): gm_version ) ) - return _GridManager(private_key, storage_servers) + return _GridManager(private_key_bytes, storage_servers) - def __init__(self, private_key, storage_servers): + def __init__(self, private_key_bytes, storage_servers): self._storage_servers = dict() if storage_servers is None else storage_servers - self._private_key = private_key + self._private_key_bytes = private_key_bytes + self._private_key, _ = keyutil.parse_privkey(self._private_key_bytes) self._version = 0 @property @@ -293,7 +292,7 @@ class _GridManager(object): def marshal(self): data = { u"grid_manager_config_version": self._version, - u"private_key": base32.b2a(self._private_key.sk_and_vk[:32]), + u"private_key": self._private_key_bytes.decode('ascii'), } if self._storage_servers: data[u"storage_servers"] = { From 4f481bbb4c963b44d99852e71c6db6fa0ee5b766 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 27 Nov 2018 22:54:03 -0700 Subject: [PATCH 0030/2309] remove FIXME --- src/allmydata/client.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/allmydata/client.py b/src/allmydata/client.py index d504f5fbe..d30b85694 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -387,8 +387,6 @@ def create_storage_farm_broker(config, default_connection_handlers, foolscap_con grid_manager_keys = [] for name, gm_key in config.enumerate_section('grid_managers').items(): - # XXX FIXME this needs pub-v0- prefix then ... - print("KEY: {}".format(gm_key)) grid_manager_keys.append( keyutil.parse_pubkey(gm_key) ) From 43b446bacf53c953ba0ad6690e338395971c806e Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 27 Nov 2018 23:40:06 -0700 Subject: [PATCH 0031/2309] CLI vs. 'edit config' for server, client enrollment --- docs/proposed/grid-manager/managed-grid.rst | 56 +++++++++++++++++++-- 1 file changed, 52 insertions(+), 4 deletions(-) diff --git a/docs/proposed/grid-manager/managed-grid.rst b/docs/proposed/grid-manager/managed-grid.rst index 4cec4a040..3f2053814 100644 --- a/docs/proposed/grid-manager/managed-grid.rst +++ b/docs/proposed/grid-manager/managed-grid.rst @@ -126,8 +126,15 @@ printed to stdout. If you stored the config on disk, the new certificate will (also) be in a file named like ``alice.cert.0``. -Enrolling a Storage Server --------------------------- +Enrolling a Storage Server: CLI +------------------------------- + +DECIDE: is a command like this best, or should you have to edit the + config "by hand"? (below fits into warner's philosophy that "at some + point" it might be best to have all config in a database or similar + and the only way to view/edit it is via tahoe commands...) + if command: write it + if not: delete this section tahoe admin add-grid-manager-cert ````````````````````````````````` @@ -142,8 +149,31 @@ your node after this. Subsequent announcements to the Introducer will include this certificate. -Enrolling a Client ------------------- +Enrolling a Storage Server: Config +---------------------------------- + +You may edit the ``[storage]`` section of the ``tahoe.cfg`` file to +include an entry ``grid_manager_certificate_files = `` whose value is +a space-separated list of paths to valid certificate files. These +certificate files are issued by the ``tahoe grid-manager sign`` +command; these should be securely transmitted to the storage +server. Relative paths are relative to the node directory. Example:: + + [storage] + grid_manager_certificate_files = example_grid.cert + +This will cause us to give this certificate to any Introducers we +connect to (and subsequently, the Introducer will give the certificate +out to clients). + + +Enrolling a Client: CLI +----------------------- + +DECIDE: is a command like this best, or should you have to edit the + config "by hand"? (below fits into warner's philosophy that "at some + point" it might be best to have all config in a database or similar + and the only way to view/edit it is via tahoe commands...) tahoe add-grid-manager `````````````````````` @@ -159,6 +189,24 @@ key of the Grid Manager. The client will have to be re-started once this change is made. +Enrolling a Client: Config +-------------------------- + +You may instruct a Tahoe client to use only storage servers from given +Grid Managers. If there are no such keys, any servers are used. If +there are one or more keys, the client will only upload to a storage +server that has a valid certificate (from any of the keys). + +To specify public-keys, add a ``[grid_managers]`` section to the +config. This consists of ``name = value`` pairs where ``name`` is an +arbitrary name and ``value`` is a public-key of a Grid +Manager. Example:: + + [grid_managers] + example_grid = pub-v0-vqimc4s5eflwajttsofisp5st566dbq36xnpp4siz57ufdavpvlq + + + Example Setup of a New Managed Grid ----------------------------------- From 85142acf97b2ffb957fd3cff93d326778e0d4ae3 Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 28 Nov 2018 01:06:53 -0700 Subject: [PATCH 0032/2309] add some integration tests --- integration/test_grid_manager.py | 62 ++++++++++++++++++++++++++++++++ integration/util.py | 21 ++++++++++- 2 files changed, 82 insertions(+), 1 deletion(-) create mode 100644 integration/test_grid_manager.py diff --git a/integration/test_grid_manager.py b/integration/test_grid_manager.py new file mode 100644 index 000000000..1a6e86266 --- /dev/null +++ b/integration/test_grid_manager.py @@ -0,0 +1,62 @@ +import sys +import time +import json +import shutil +from os import mkdir, unlink, listdir, utime +from os.path import join, exists, getmtime + +from allmydata.util import keyutil +from allmydata.util import base32 + +import util + +import pytest + + +@pytest.inlineCallbacks +def test_create_certificate(reactor): + gm_config = yield util.run_tahoe( + reactor, "grid-manager", "--config", "-", "create", + ) + privkey_bytes = json.loads(gm_config)['private_key'].encode('ascii') + privkey, pubkey_bytes = keyutil.parse_privkey(privkey_bytes) + pubkey = keyutil.parse_pubkey(pubkey_bytes) + + gm_config = yield util.run_tahoe( + reactor, "grid-manager", "--config", "-", "add", + "alice", "pub-v0-kzug3ut2m7ziihf3ndpqlquuxeie4foyl36wn54myqc4wmiwe4ga", + stdin=gm_config, + ) + alice_cert_bytes = yield util.run_tahoe( + reactor, "grid-manager", "--config", "-", "sign", "alice", + stdin=gm_config, + ) + alice_cert = json.loads(alice_cert_bytes) + + # confirm that alice's certificate is made by the Grid Manager + assert pubkey.verify( + base32.a2b(alice_cert['signature'].encode('ascii')), + alice_cert['certificate'].encode('ascii'), + ) + + +@pytest.inlineCallbacks +def test_remove_client(reactor): + gm_config = yield util.run_tahoe( + reactor, "grid-manager", "--config", "-", "create", + ) + + gm_config = yield util.run_tahoe( + reactor, "grid-manager", "--config", "-", "add", + "alice", "pub-v0-kzug3ut2m7ziihf3ndpqlquuxeie4foyl36wn54myqc4wmiwe4ga", + stdin=gm_config, + ) + assert json.loads(gm_config)['storage_servers'].has_key("alice") + + gm_config = yield util.run_tahoe( + reactor, "grid-manager", "--config", "-", "remove", + "alice", + stdin=gm_config, + ) + # there are no storage servers left at all now + assert not json.loads(gm_config).has_key('storage_servers') diff --git a/integration/util.py b/integration/util.py index 4dd20aa49..e2a198d34 100644 --- a/integration/util.py +++ b/integration/util.py @@ -37,9 +37,15 @@ class _CollectOutputProtocol(ProcessProtocol): self.output, and callback's on done with all of it after the process exits (for any reason). """ - def __init__(self): + def __init__(self, stdin=None): self.done = Deferred() self.output = StringIO() + self._stdin = stdin + + def connectionMade(self): + if self._stdin is not None: + self.transport.write(self._stdin) + self.transport.closeStdin() def processEnded(self, reason): if not self.done.called: @@ -126,6 +132,19 @@ def _cleanup_twistd_process(twistd_process, exited): pass +def run_tahoe(reactor, *args, **kwargs): + stdin = kwargs.get('stdin', None) + protocol = _CollectOutputProtocol(stdin=stdin) + process = reactor.spawnProcess( + protocol, + sys.executable, + (sys.executable, '-m', 'allmydata.scripts.runner') + args + ) + process.exited = protocol.done + + return protocol.done + + def _run_node(reactor, node_dir, request, magic_text): if magic_text is None: magic_text = "client running" From c27964813f699730390ae9746cd674d850890654 Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 28 Nov 2018 01:06:58 -0700 Subject: [PATCH 0033/2309] fix a bug --- src/allmydata/scripts/tahoe_grid_manager.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/allmydata/scripts/tahoe_grid_manager.py b/src/allmydata/scripts/tahoe_grid_manager.py index 6fbdd4f7b..f7193cbf2 100644 --- a/src/allmydata/scripts/tahoe_grid_manager.py +++ b/src/allmydata/scripts/tahoe_grid_manager.py @@ -411,9 +411,10 @@ def _remove(gridoptions, options): "No storage-server called '{}' exists".format(options['name']) ) cert_count = 0 - while fp.child('{}.cert.{}'.format(options['name'], cert_count)).exists(): - fp.child('{}.cert.{}'.format(options['name'], cert_count)).remove() - cert_count += 1 + if fp is not None: + while fp.child('{}.cert.{}'.format(options['name'], cert_count)).exists(): + fp.child('{}.cert.{}'.format(options['name'], cert_count)).remove() + cert_count += 1 _save_gridmanager_config(fp, gm) return 0 From 992314471e1c205bb3a9ec54ae9d6099c104c427 Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 28 Nov 2018 01:20:51 -0700 Subject: [PATCH 0034/2309] more integration tests --- integration/test_grid_manager.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/integration/test_grid_manager.py b/integration/test_grid_manager.py index 1a6e86266..103c3c897 100644 --- a/integration/test_grid_manager.py +++ b/integration/test_grid_manager.py @@ -46,6 +46,35 @@ def test_remove_client(reactor): reactor, "grid-manager", "--config", "-", "create", ) + gm_config = yield util.run_tahoe( + reactor, "grid-manager", "--config", "-", "add", + "alice", "pub-v0-kzug3ut2m7ziihf3ndpqlquuxeie4foyl36wn54myqc4wmiwe4ga", + stdin=gm_config, + ) + gm_config = yield util.run_tahoe( + reactor, "grid-manager", "--config", "-", "add", + "bob", "pub-v0-kvxhb3nexybmipkrar2ztfrwp4uxxsmrjzkpzafit3ket4u5yldq", + stdin=gm_config, + ) + assert json.loads(gm_config)['storage_servers'].has_key("alice") + assert json.loads(gm_config)['storage_servers'].has_key("bob") + return + + gm_config = yield util.run_tahoe( + reactor, "grid-manager", "--config", "-", "remove", + "alice", + stdin=gm_config, + ) + assert not json.loads(gm_config)['storage_servers'].has_key('alice') + assert json.loads(gm_config)['storage_servers'].has_key('bob') + + +@pytest.inlineCallbacks +def test_remove_last_client(reactor): + gm_config = yield util.run_tahoe( + reactor, "grid-manager", "--config", "-", "create", + ) + gm_config = yield util.run_tahoe( reactor, "grid-manager", "--config", "-", "add", "alice", "pub-v0-kzug3ut2m7ziihf3ndpqlquuxeie4foyl36wn54myqc4wmiwe4ga", From 0432b7d42f3a280bfe7108a87c938ba4843c0f81 Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 28 Nov 2018 01:22:49 -0700 Subject: [PATCH 0035/2309] fix assert --- integration/test_grid_manager.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/integration/test_grid_manager.py b/integration/test_grid_manager.py index 103c3c897..886bf76e9 100644 --- a/integration/test_grid_manager.py +++ b/integration/test_grid_manager.py @@ -34,7 +34,8 @@ def test_create_certificate(reactor): alice_cert = json.loads(alice_cert_bytes) # confirm that alice's certificate is made by the Grid Manager - assert pubkey.verify( + # (.verify returns None on success, raises exception on error) + pubkey.verify( base32.a2b(alice_cert['signature'].encode('ascii')), alice_cert['certificate'].encode('ascii'), ) From 7f195c9c9a9b20b95e5f94267c83645de871a2e5 Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 28 Nov 2018 02:54:49 -0700 Subject: [PATCH 0036/2309] test of certificates on not-enough-servers causing uploads to fail --- integration/test_grid_manager.py | 79 ++++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) diff --git a/integration/test_grid_manager.py b/integration/test_grid_manager.py index 886bf76e9..39e2fbe9b 100644 --- a/integration/test_grid_manager.py +++ b/integration/test_grid_manager.py @@ -7,6 +7,7 @@ from os.path import join, exists, getmtime from allmydata.util import keyutil from allmydata.util import base32 +from allmydata.util import configutil import util @@ -90,3 +91,81 @@ def test_remove_last_client(reactor): ) # there are no storage servers left at all now assert not json.loads(gm_config).has_key('storage_servers') + + +@pytest.inlineCallbacks +def test_reject_storage_server(reactor, request, alice, storage_nodes): + gm_config = yield util.run_tahoe( + reactor, "grid-manager", "--config", "-", "create", + ) + privkey_bytes = json.loads(gm_config)['private_key'].encode('ascii') + privkey, pubkey_bytes = keyutil.parse_privkey(privkey_bytes) + pubkey = keyutil.parse_pubkey(pubkey_bytes) + + # create certificates for first 2 storage-servers + for idx, storage in enumerate(storage_nodes[:2]): + pubkey_fname = join(storage._node_dir, "node.pubkey") + with open(pubkey_fname, 'r') as f: + pubkey = f.read().strip() + + gm_config = yield util.run_tahoe( + reactor, "grid-manager", "--config", "-", "add", + "storage{}".format(idx), pubkey, + stdin=gm_config, + ) + assert sorted(json.loads(gm_config)['storage_servers'].keys()) == ['storage0', 'storage1'] + + print("inserting certificates") + # insert their certificates + for idx, storage in enumerate(storage_nodes[:2]): + print(idx, storage) + cert = yield util.run_tahoe( + reactor, "grid-manager", "--config", "-", "sign", + "storage{}".format(idx), + stdin=gm_config, + ) + with open(join(storage._node_dir, "gridmanager.cert"), "w") as f: + f.write(cert) + config = configutil.get_config(join(storage._node_dir, "tahoe.cfg")) + config.set("storage", "grid_manager_certificate_files", "gridmanager.cert") + config.write(open(join(storage._node_dir, "tahoe.cfg"), "w")) + + # re-start this storage server + storage.signalProcess('TERM') + yield storage._protocol.exited + time.sleep(1) + storage_nodes[idx] = yield util._run_node( + reactor, storage._node_dir, request, None, + ) + + # now only two storage-servers have certificates .. configure + # alice to have the grid-manager certificate + + config = configutil.get_config(join(alice._node_dir, "tahoe.cfg")) + print(dir(config)) + config.add_section("grid_managers") + config.set("grid_managers", "test", pubkey_bytes) + config.write(open(join(alice._node_dir, "tahoe.cfg"), "w")) + alice.signalProcess('TERM') + yield alice._protocol.exited + time.sleep(1) + alice = yield util._run_node( + reactor, alice._node_dir, request, None, + ) + time.sleep(5) + + # try to put something into the grid, which should fail (because + # alice has happy=3 but should only find storage0, storage1 to be + # acceptable to upload to) + + try: + yield util.run_tahoe( + reactor, "--node-directory", alice._node_dir, + "put", "-", + stdin="some content" * 200, + ) + assert False, "Should get a failure" + except Exception as e: + # XXX but how to tell if we failed for the right reason? + # (should get UploadUnhappinessError) + print("failure expected: {}".format(e)) From 3c679e92027a063d08f4213fdfb8ba867aa5ea77 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 7 Dec 2018 15:57:21 -0700 Subject: [PATCH 0037/2309] tweak tests/error output --- integration/test_grid_manager.py | 7 ++++--- integration/util.py | 8 +++++++- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/integration/test_grid_manager.py b/integration/test_grid_manager.py index 39e2fbe9b..33710a457 100644 --- a/integration/test_grid_manager.py +++ b/integration/test_grid_manager.py @@ -166,6 +166,7 @@ def test_reject_storage_server(reactor, request, alice, storage_nodes): ) assert False, "Should get a failure" except Exception as e: - # XXX but how to tell if we failed for the right reason? - # (should get UploadUnhappinessError) - print("failure expected: {}".format(e)) + # depending on the full output being in the error-message + # here; see util.py + assert 'UploadUnhappinessError' in str(e) + print("found expected UploadUnhappinessError") diff --git a/integration/util.py b/integration/util.py index e2a198d34..c5bc04ab2 100644 --- a/integration/util.py +++ b/integration/util.py @@ -53,7 +53,13 @@ class _CollectOutputProtocol(ProcessProtocol): def processExited(self, reason): if not isinstance(reason.value, ProcessDone): - self.done.errback(reason) + #self.done.errback(reason) + self.done.errback(RuntimeError( + "Process failed: {}\nOutput:\n{}".format( + reason, + self.output.getvalue(), + ) + )) def outReceived(self, data): self.output.write(data) From 7b5783536b9248e7adc17d023fb23074d1a62516 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 7 Dec 2018 16:01:06 -0700 Subject: [PATCH 0038/2309] remove prints --- src/allmydata/storage_client.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 1a928ea27..2a09efa8f 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -223,7 +223,7 @@ class StorageFarmBroker(service.MultiService): connected_servers = self.get_connected_servers() preferred_servers = frozenset(s for s in connected_servers if s.get_longname() in self.preferred_peers) if for_upload: - print("upload processing: {}".format([srv.upload_permitted() for srv in connected_servers])) + # print("upload processing: {}".format([srv.upload_permitted() for srv in connected_servers])) connected_servers = [ srv for srv in connected_servers @@ -409,26 +409,24 @@ class NativeStorageServer(service.MultiService): :return: True if we should use this server for uploads, False otherwise. """ - print("upload permitted? {}".format(self._server_id)) + # print("upload permitted? {}".format(self._server_id)) # if we have no Grid Manager keys configured, choice is easy if not self._grid_manager_keys: - print("{} no grid manager keys at all (so yes)".format(self._server_id)) + # print("{} no grid manager keys at all (so yes)".format(self._server_id)) return True # XXX probably want to cache the answer to this? (ignoring # that for now because certificates expire, so .. slightly # more complex) if not self._grid_manager_certificates: - print("{} no grid-manager certificates {} (so no)".format(self._server_id, self._grid_manager_certificates)) + # print("{} no grid-manager certificates {} (so no)".format(self._server_id, self._grid_manager_certificates)) return False for gm_key in self._grid_manager_keys: for cert in self._grid_manager_certificates: if _validate_grid_manager_certificate(gm_key, cert): - print("valid: {}\n{}".format(gm_key, cert)) + # print("valid: {}\n{}".format(gm_key, cert)) return True - else: - print("invalid: {}\n{}".format(gm_key, cert)) - print("didn't validate {} keys".format(len(self._grid_manager_keys))) + # print("didn't validate {} keys".format(len(self._grid_manager_keys))) return False From 75fee995b6777bec17e79e16c65ac63bc92693f0 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 11 Dec 2018 12:48:37 -0700 Subject: [PATCH 0039/2309] some tweaks and fixes for integration tests --- integration/test_grid_manager.py | 27 +++++++++++++++++------- integration/test_servers_of_happiness.py | 2 +- integration/test_tor.py | 3 +++ integration/util.py | 4 ++++ 4 files changed, 27 insertions(+), 9 deletions(-) diff --git a/integration/test_grid_manager.py b/integration/test_grid_manager.py index 33710a457..f68f97941 100644 --- a/integration/test_grid_manager.py +++ b/integration/test_grid_manager.py @@ -94,7 +94,7 @@ def test_remove_last_client(reactor): @pytest.inlineCallbacks -def test_reject_storage_server(reactor, request, alice, storage_nodes): +def test_reject_storage_server(reactor, request, storage_nodes, temp_dir, introducer_furl, flog_gatherer): gm_config = yield util.run_tahoe( reactor, "grid-manager", "--config", "-", "create", ) @@ -115,6 +115,15 @@ def test_reject_storage_server(reactor, request, alice, storage_nodes): ) assert sorted(json.loads(gm_config)['storage_servers'].keys()) == ['storage0', 'storage1'] + + # XXX FIXME need to shut-down and nuke carol when we're done this + # test (i.d. request.addfinalizer) + carol = yield util._create_node( + reactor, request, temp_dir, introducer_furl, flog_gatherer, "carol", + web_port="tcp:9982:interface=localhost", + storage=False, + ) + print("inserting certificates") # insert their certificates for idx, storage in enumerate(storage_nodes[:2]): @@ -141,26 +150,28 @@ def test_reject_storage_server(reactor, request, alice, storage_nodes): # now only two storage-servers have certificates .. configure # alice to have the grid-manager certificate - config = configutil.get_config(join(alice._node_dir, "tahoe.cfg")) + # XXX FIXME remove this cert when test ends (fail or not!) + + config = configutil.get_config(join(carol._node_dir, "tahoe.cfg")) print(dir(config)) config.add_section("grid_managers") config.set("grid_managers", "test", pubkey_bytes) - config.write(open(join(alice._node_dir, "tahoe.cfg"), "w")) - alice.signalProcess('TERM') - yield alice._protocol.exited + config.write(open(join(carol._node_dir, "tahoe.cfg"), "w")) + carol.signalProcess('TERM') + yield carol._protocol.exited time.sleep(1) alice = yield util._run_node( - reactor, alice._node_dir, request, None, + reactor, carol._node_dir, request, None, ) time.sleep(5) # try to put something into the grid, which should fail (because - # alice has happy=3 but should only find storage0, storage1 to be + # carol has happy=3 but should only find storage0, storage1 to be # acceptable to upload to) try: yield util.run_tahoe( - reactor, "--node-directory", alice._node_dir, + reactor, "--node-directory", carol._node_dir, "put", "-", stdin="some content" * 200, ) diff --git a/integration/test_servers_of_happiness.py b/integration/test_servers_of_happiness.py index 1f84094e2..1984dd507 100644 --- a/integration/test_servers_of_happiness.py +++ b/integration/test_servers_of_happiness.py @@ -43,7 +43,7 @@ def test_upload_immutable(reactor, temp_dir, introducer_furl, flog_gatherer, sto yield proto.done assert False, "should raise exception" except Exception as e: - assert isinstance(e, ProcessTerminated) + assert "UploadUnhappinessError" in str(e) output = proto.output.getvalue() assert "shares could be placed on only" in output diff --git a/integration/test_tor.py b/integration/test_tor.py index 187754f08..e0b991128 100644 --- a/integration/test_tor.py +++ b/integration/test_tor.py @@ -68,6 +68,9 @@ def _create_anonymous_node(reactor, name, control_port, request, temp_dir, flog_ web_port = "tcp:{}:interface=localhost".format(control_port + 2000) if True: + if exists(node_dir): + print("nuking '{}'".format(node_dir)) + shutil.rmtree(node_dir) print("creating", node_dir) mkdir(node_dir) proto = util._DumpOutputProtocol(None) diff --git a/integration/util.py b/integration/util.py index c5bc04ab2..703402aa8 100644 --- a/integration/util.py +++ b/integration/util.py @@ -4,6 +4,7 @@ from os import mkdir from os.path import exists, join from six.moves import StringIO from functools import partial +from shutil import rmtree from twisted.internet.defer import Deferred, succeed from twisted.internet.protocol import ProcessProtocol @@ -200,6 +201,9 @@ def _create_node(reactor, request, temp_dir, introducer_furl, flog_gatherer, nam if exists(node_dir): created_d = succeed(None) else: + if exists(node_dir): + print("nuking: {}".format(node_dir)) + rmtree(node_dir) print("creating", node_dir) mkdir(node_dir) done_proto = _ProcessExitedProtocol() From fea86d7e95e692d9b8c52c7fd7ed0c5c1dee54a3 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 11 Dec 2018 12:54:22 -0700 Subject: [PATCH 0040/2309] alice->carol --- integration/test_grid_manager.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/integration/test_grid_manager.py b/integration/test_grid_manager.py index f68f97941..3553ccc23 100644 --- a/integration/test_grid_manager.py +++ b/integration/test_grid_manager.py @@ -148,9 +148,7 @@ def test_reject_storage_server(reactor, request, storage_nodes, temp_dir, introd ) # now only two storage-servers have certificates .. configure - # alice to have the grid-manager certificate - - # XXX FIXME remove this cert when test ends (fail or not!) + # carol to have the grid-manager certificate config = configutil.get_config(join(carol._node_dir, "tahoe.cfg")) print(dir(config)) @@ -160,7 +158,7 @@ def test_reject_storage_server(reactor, request, storage_nodes, temp_dir, introd carol.signalProcess('TERM') yield carol._protocol.exited time.sleep(1) - alice = yield util._run_node( + carol = yield util._run_node( reactor, carol._node_dir, request, None, ) time.sleep(5) From 39a80f7939359ea1bc4bfe9ef90eaf0cc3c092e0 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 11 Dec 2018 13:40:58 -0700 Subject: [PATCH 0041/2309] utest fixes --- src/allmydata/storage_client.py | 2 +- src/allmydata/test/no_network.py | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 2a09efa8f..75a08866c 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -125,7 +125,7 @@ class StorageFarmBroker(service.MultiService): # these two are used in unit tests def test_add_rref(self, serverid, rref, ann): - s = NativeStorageServer(serverid, ann.copy(), self._tub_maker, {}, []) + s = NativeStorageServer(serverid, ann.copy(), self._tub_maker, {}, [], []) s.rref = rref s._is_connected = True self.servers[serverid] = s diff --git a/src/allmydata/test/no_network.py b/src/allmydata/test/no_network.py index de15d32bf..fb1da0635 100644 --- a/src/allmydata/test/no_network.py +++ b/src/allmydata/test/no_network.py @@ -149,6 +149,10 @@ class NoNetworkServer(object): return self def __deepcopy__(self, memodict): return self + + def upload_permitted(self): + return True + def get_serverid(self): return self.serverid def get_permutation_seed(self): From d0791eb2b5303da2be9f6008f92abe77dadf37e9 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 11 Dec 2018 18:45:42 -0700 Subject: [PATCH 0042/2309] fix storage-client --- src/allmydata/storage_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 75a08866c..34b2c0caf 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -104,7 +104,7 @@ class StorageFarmBroker(service.MultiService): self._tub_maker, handler_overrides, self._grid_manager_keys, - self._grid_manager_certificates, + [], # XXX FIXME? need grid_manager_certs too? ) print("SET STATIC {}".format(s)) s.on_status_changed(lambda _: self._got_connection()) From 633f36a33cc1468b8f268d1062208877149b6721 Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 17 Dec 2018 19:37:43 -0700 Subject: [PATCH 0043/2309] Made A Decision, and start of 'admin add-grid-manager-cert' command --- docs/proposed/grid-manager/managed-grid.rst | 32 +++++++++++++++++---- src/allmydata/scripts/admin.py | 10 ++++++- 2 files changed, 35 insertions(+), 7 deletions(-) diff --git a/docs/proposed/grid-manager/managed-grid.rst b/docs/proposed/grid-manager/managed-grid.rst index 3f2053814..59cfb3e50 100644 --- a/docs/proposed/grid-manager/managed-grid.rst +++ b/docs/proposed/grid-manager/managed-grid.rst @@ -129,12 +129,6 @@ certificate will (also) be in a file named like ``alice.cert.0``. Enrolling a Storage Server: CLI ------------------------------- -DECIDE: is a command like this best, or should you have to edit the - config "by hand"? (below fits into warner's philosophy that "at some - point" it might be best to have all config in a database or similar - and the only way to view/edit it is via tahoe commands...) - if command: write it - if not: delete this section tahoe admin add-grid-manager-cert ````````````````````````````````` @@ -148,6 +142,32 @@ installed; for now just one is sufficient). You will have to re-start your node after this. Subsequent announcements to the Introducer will include this certificate. +.. note:: + + This command will simply edit the `tahoe.cfg` file and direct you + to re-start. In the Future(tm), we should consider (in exarkun's + words): + + "A python program you run as a new process" might not be the + best abstraction to layer on top of the configuration + persistence system, though. It's a nice abstraction for users + (although most users would probably rather have a GUI) but it's + not a great abstraction for automation. So at some point it + may be better if there is CLI -> public API -> configuration + persistence system. And maybe "public API" is even a network + API for the storage server so it's equally easy to access from + an agent implemented in essentially any language and maybe if + the API is exposed by the storage node itself then this also + gives you live-configuration-updates, avoiding the need for + node restarts (not that this is the only way to accomplish + this, but I think it's a good way because it avoids the need + for messes like inotify and it supports the notion that the + storage node process is in charge of its own configuration + persistence system, not just one consumer among many ... which + has some nice things going for it ... though how this interacts + exactly with further node management automation might bear + closer scrutiny). + Enrolling a Storage Server: Config ---------------------------------- diff --git a/src/allmydata/scripts/admin.py b/src/allmydata/scripts/admin.py index d8e0b77f0..690059265 100644 --- a/src/allmydata/scripts/admin.py +++ b/src/allmydata/scripts/admin.py @@ -45,6 +45,13 @@ def derive_pubkey(options): print("public:", pubkey_vs, file=out) return 0 + +def add_grid_manager_cert(options): + """ + Add a new Grid Manager certificate to our config + """ + return 0 + class AdminCommand(BaseOptions): subCommands = [ ("generate-keypair", None, GenerateKeypairOptions, @@ -68,7 +75,8 @@ each subcommand. subDispatch = { "generate-keypair": print_keypair, "derive-pubkey": derive_pubkey, - } + "add-grid-manager-cert": add_grid_manager_cert, +} def do_admin(options): so = options.subOptions From 08937c6acfb6af167eaa45b3223528b1c27713c5 Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 17 Dec 2018 19:38:22 -0700 Subject: [PATCH 0044/2309] whitespace --- src/allmydata/scripts/admin.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/allmydata/scripts/admin.py b/src/allmydata/scripts/admin.py index 690059265..55ad224c2 100644 --- a/src/allmydata/scripts/admin.py +++ b/src/allmydata/scripts/admin.py @@ -52,6 +52,7 @@ def add_grid_manager_cert(options): """ return 0 + class AdminCommand(BaseOptions): subCommands = [ ("generate-keypair", None, GenerateKeypairOptions, @@ -72,12 +73,14 @@ each subcommand. """ return t + subDispatch = { "generate-keypair": print_keypair, "derive-pubkey": derive_pubkey, "add-grid-manager-cert": add_grid_manager_cert, } + def do_admin(options): so = options.subOptions so.stdout = options.stdout @@ -88,8 +91,8 @@ def do_admin(options): subCommands = [ ["admin", None, AdminCommand, "admin subcommands: use 'tahoe admin' for a list"], - ] +] dispatch = { "admin": do_admin, - } +} From d302a9867210fad4f5523efe58b57cb88888eefd Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 17 Dec 2018 22:38:06 -0700 Subject: [PATCH 0045/2309] 'tahoe admin add-grid-manager-cert' command --- src/allmydata/scripts/admin.py | 96 +++++++++++++++++++++++++++++++++ src/allmydata/storage_client.py | 36 ++++++++++++- 2 files changed, 130 insertions(+), 2 deletions(-) diff --git a/src/allmydata/scripts/admin.py b/src/allmydata/scripts/admin.py index 55ad224c2..0a641728c 100644 --- a/src/allmydata/scripts/admin.py +++ b/src/allmydata/scripts/admin.py @@ -1,7 +1,17 @@ from __future__ import print_function +import sys +import json +from os.path import exists, join + from twisted.python import usage +#from allmydata.node import read_config +from allmydata.client import read_config +from allmydata.scripts.cli import _default_nodedir from allmydata.scripts.common import BaseOptions +from allmydata.util.encodingutil import argv_to_abspath +from allmydata.util import fileutil + class GenerateKeypairOptions(BaseOptions): @@ -46,10 +56,93 @@ def derive_pubkey(options): return 0 +class AddGridManagerCertOptions(BaseOptions): + + optParameters = [ + ['filename', 'f', None, "Filename of the certificate ('-', a dash, for stdin)"], + ['name', 'n', "default", "Name to give this certificate"], + ] + + def getSynopsis(self): + return "Usage: tahoe [global-options] admin add-grid-manager-cert [options]" + + def postOptions(self): + if self['filename'] is None: + raise usage.UsageError( + "Must provide --filename option" + ) + if self['filename'] == '-': + print >>self.parent.parent.stderr, "reading certificate from stdin" + data = sys.stdin.read() + if len(data) == 0: + raise usage.UsageError( + "Reading certificate from stdin failed" + ) + from allmydata.storage_client import parse_grid_manager_data + try: + self.certificate_data = parse_grid_manager_data(data) + except ValueError as e: + print >>self.parent.parent.stderr, "Error parsing certificate: {}".format(e) + self.certificate_data = None + else: + with open(self['filename'], 'r') as f: + self.certificate_data = f.read() + + def getUsage(self, width=None): + t = BaseOptions.getUsage(self, width) + t += ( + "Adds a Grid Manager certificate to a Storage Server.\n\n" + "The certificate will be copied into the base-dir and config\n" + "will be added to 'tahoe.cfg', which will be re-written. A\n" + "restart is required for changes to take effect.\n\n" + "The human who operates a Grid Manager would produce such a\n" + "certificate and communicate it securely to you.\n" + ) + return t + + def add_grid_manager_cert(options): """ Add a new Grid Manager certificate to our config """ + if options.certificate_data is None: + return 1 + # XXX is there really not already a function for this? + if options.parent.parent['node-directory']: + nd = argv_to_abspath(options.parent.parent['node-directory']) + else: + nd = _default_nodedir + + config = read_config(nd, "portnum") + config_path = join(nd, "tahoe.cfg") + cert_fname = "{}.cert".format(options['name']) + cert_path = config.get_config_path(cert_fname) + cert_bytes = json.dumps(options.certificate_data, indent=4) + '\n' + cert_name = options['name'] + + if exists(cert_path): + print >>options.parent.parent.stderr, "Already have file '{}'".format(cert_path) + return 1 + + cfg = config.config # why aren't methods we call on cfg in _Config itself? + + gm_certs = config.get_config("storage", "grid_manager_certificate_files", "").split() + if cert_fname not in gm_certs: + gm_certs.append(cert_fname) + cfg.set("storage", "grid_manager_certificate_files", " ".join(gm_certs)) + + # print("grid_manager_certificate_files in {}: {}".format(config_path, len(gm_certs))) + + # write all the data out + + fileutil.write(cert_path, cert_bytes) + # print("created {}: {} bytes".format(cert_fname, len(cert_bytes))) + with open(config_path, "w") as f: + cfg.write(f) + # print("wrote {}".format(config_fname)) + + print >>options.parent.parent.stderr, "There are now {} certificates".format(len(gm_certs)) + return 0 @@ -59,6 +152,9 @@ class AdminCommand(BaseOptions): "Generate a public/private keypair, write to stdout."), ("derive-pubkey", None, DerivePubkeyOptions, "Derive a public key from a private key."), + ("add-grid-manager-cert", None, AddGridManagerCertOptions, + "Add a Grid Manager-provided certificate to a storage " + "server's config."), ] def postOptions(self): if not hasattr(self, 'subOptions'): diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 34b2c0caf..53b0d5c53 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -282,7 +282,39 @@ class StubServer(object): return "?" -def _validate_grid_manager_certificate(gm_key, alleged_cert): +def parse_grid_manager_data(gm_data): + """ + :param gm_data: some data that might be JSON that might be a valid + Grid Manager Certificate + + :returns: json data of a valid Grid Manager certificate, or an + exception if the data is not valid. + """ + + required_keys = allowed_keys = [ + 'certificate', + 'signature', + ] + + js = json.loads(gm_data) + for k in js.keys(): + if k not in allowed_keys: + raise ValueError( + "Grid Manager certificate JSON may not contain '{}'".format( + k, + ) + ) + for k in allowed_keys: + if k not in js: + raise ValueError( + "Grid Manager certificate JSON must contain '{}'".format( + k, + ) + ) + return js + + +def validate_grid_manager_certificate(gm_key, alleged_cert): """ :param gm_key: a VerifyingKey instance, a Grid Manager's public key. @@ -423,7 +455,7 @@ class NativeStorageServer(service.MultiService): return False for gm_key in self._grid_manager_keys: for cert in self._grid_manager_certificates: - if _validate_grid_manager_certificate(gm_key, cert): + if validate_grid_manager_certificate(gm_key, cert): # print("valid: {}\n{}".format(gm_key, cert)) return True # print("didn't validate {} keys".format(len(self._grid_manager_keys))) From c88409afc545f95bddc4758d6f99ad8a769e4ba8 Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 27 Mar 2019 21:04:48 -0600 Subject: [PATCH 0046/2309] add a note --- src/allmydata/client.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/allmydata/client.py b/src/allmydata/client.py index d30b85694..ad5e61c14 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -605,6 +605,10 @@ class _Client(node.Node, pollmixin.PollMixin): ss.setServiceParent(self) grid_manager_certificates = [] + + # XXX this is probably a bad idea for multiple fnames -- what + # about spaces in filenames? + cert_fnames = self.get_config("storage", "grid_manager_certificate_files", "") for fname in cert_fnames.split(): fname = self.config.get_config_path(fname.decode('ascii')) From 4665617818c37edf188a6a81ace3da24f62b3bfd Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 8 Apr 2019 18:53:25 -0600 Subject: [PATCH 0047/2309] newsfragment --- newsfragments/2916.feature | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/2916.feature diff --git a/newsfragments/2916.feature b/newsfragments/2916.feature new file mode 100644 index 000000000..292d429ea --- /dev/null +++ b/newsfragments/2916.feature @@ -0,0 +1 @@ +A new 'Grid Manager' specification and implementation \ No newline at end of file From 2cf36cf88460e7f6b5e322668e38fa7bb58f8dd6 Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 8 Apr 2019 18:58:39 -0600 Subject: [PATCH 0048/2309] flake8 --- src/allmydata/client.py | 2 -- src/allmydata/scripts/admin.py | 10 +++++----- src/allmydata/scripts/tahoe_grid_manager.py | 2 +- src/allmydata/storage_client.py | 3 +-- 4 files changed, 7 insertions(+), 10 deletions(-) diff --git a/src/allmydata/client.py b/src/allmydata/client.py index ad5e61c14..50ceb448f 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -31,13 +31,11 @@ from allmydata.util.abbreviate import parse_abbreviated_size from allmydata.util.time_format import parse_duration, parse_date from allmydata.util.i2p_provider import create as create_i2p_provider from allmydata.util.tor_provider import create as create_tor_provider -from allmydata.util.base32 import a2b, b2a from allmydata.stats import StatsProvider from allmydata.history import History from allmydata.interfaces import IStatsProducer, SDMF_VERSION, MDMF_VERSION, DEFAULT_MAX_SEGMENT_SIZE from allmydata.nodemaker import NodeMaker from allmydata.blacklist import Blacklist -from allmydata import node KiB=1024 diff --git a/src/allmydata/scripts/admin.py b/src/allmydata/scripts/admin.py index 0a641728c..ef1059a31 100644 --- a/src/allmydata/scripts/admin.py +++ b/src/allmydata/scripts/admin.py @@ -72,7 +72,7 @@ class AddGridManagerCertOptions(BaseOptions): "Must provide --filename option" ) if self['filename'] == '-': - print >>self.parent.parent.stderr, "reading certificate from stdin" + print("reading certificate from stdin", file=self.parent.parent.stderr) data = sys.stdin.read() if len(data) == 0: raise usage.UsageError( @@ -82,7 +82,7 @@ class AddGridManagerCertOptions(BaseOptions): try: self.certificate_data = parse_grid_manager_data(data) except ValueError as e: - print >>self.parent.parent.stderr, "Error parsing certificate: {}".format(e) + print("Error parsing certificate: {}".format(e), file=self.parent.parent.stderr) self.certificate_data = None else: with open(self['filename'], 'r') as f: @@ -118,10 +118,10 @@ def add_grid_manager_cert(options): cert_fname = "{}.cert".format(options['name']) cert_path = config.get_config_path(cert_fname) cert_bytes = json.dumps(options.certificate_data, indent=4) + '\n' - cert_name = options['name'] + # cert_name = options['name'] if exists(cert_path): - print >>options.parent.parent.stderr, "Already have file '{}'".format(cert_path) + print("Already have file '{}'".format(cert_path), file=options.parent.parent.stderr) return 1 cfg = config.config # why aren't methods we call on cfg in _Config itself? @@ -141,7 +141,7 @@ def add_grid_manager_cert(options): cfg.write(f) # print("wrote {}".format(config_fname)) - print >>options.parent.parent.stderr, "There are now {} certificates".format(len(gm_certs)) + print("There are now {} certificates".format(len(gm_certs)), file=options.parent.parent.stderr) return 0 diff --git a/src/allmydata/scripts/tahoe_grid_manager.py b/src/allmydata/scripts/tahoe_grid_manager.py index f7193cbf2..b33552861 100644 --- a/src/allmydata/scripts/tahoe_grid_manager.py +++ b/src/allmydata/scripts/tahoe_grid_manager.py @@ -1,5 +1,5 @@ +from __future__ import print_function -import os import sys import json import time diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 53b0d5c53..5f65c6cec 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -43,7 +43,6 @@ from pycryptopp.publickey import ed25519 # perhaps NaCl instead? other code use from allmydata.interfaces import IStorageBroker, IDisplayableServer, IServer from allmydata.util import log, base32, connection_status -from allmydata.util import keyutil from allmydata.util.assertutil import precondition from allmydata.util.observer import ObserverList from allmydata.util.rrefutil import add_version_to_remote_reference @@ -304,7 +303,7 @@ def parse_grid_manager_data(gm_data): k, ) ) - for k in allowed_keys: + for k in required_keys: if k not in js: raise ValueError( "Grid Manager certificate JSON must contain '{}'".format( From cfda360294c84273ca2842cc11deeef5d035b61e Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 8 Apr 2019 18:59:14 -0600 Subject: [PATCH 0049/2309] space-separated paths is a bad idea --- docs/proposed/grid-manager/managed-grid.rst | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/docs/proposed/grid-manager/managed-grid.rst b/docs/proposed/grid-manager/managed-grid.rst index 59cfb3e50..10d0c142a 100644 --- a/docs/proposed/grid-manager/managed-grid.rst +++ b/docs/proposed/grid-manager/managed-grid.rst @@ -173,14 +173,19 @@ Enrolling a Storage Server: Config ---------------------------------- You may edit the ``[storage]`` section of the ``tahoe.cfg`` file to -include an entry ``grid_manager_certificate_files = `` whose value is -a space-separated list of paths to valid certificate files. These -certificate files are issued by the ``tahoe grid-manager sign`` -command; these should be securely transmitted to the storage -server. Relative paths are relative to the node directory. Example:: +turn on grid-management with ``grid_management = true``. You then must +also provide a ``[grid_management_keys]]`` section in the config-file which +lists ``name = path/to/certificate`` pairs. + +These certificate files are issued by the ``tahoe grid-manager sign`` +command; these should be **securely transmitted** to the storage +server. Relative paths are based from the node directory. Example:: [storage] - grid_manager_certificate_files = example_grid.cert + grid_management = true + + [grid_management_keys] + default = example_grid.cert This will cause us to give this certificate to any Introducers we connect to (and subsequently, the Introducer will give the certificate From 16e7bc5e07a2284f13e49f3b7f1585b5244ec603 Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 8 Apr 2019 19:42:39 -0600 Subject: [PATCH 0050/2309] pytest API --- integration/test_grid_manager.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/integration/test_grid_manager.py b/integration/test_grid_manager.py index 3553ccc23..075a1004a 100644 --- a/integration/test_grid_manager.py +++ b/integration/test_grid_manager.py @@ -11,10 +11,10 @@ from allmydata.util import configutil import util -import pytest +import pytest_twisted -@pytest.inlineCallbacks +@pytest_twisted.inlineCallbacks def test_create_certificate(reactor): gm_config = yield util.run_tahoe( reactor, "grid-manager", "--config", "-", "create", @@ -42,7 +42,7 @@ def test_create_certificate(reactor): ) -@pytest.inlineCallbacks +@pytest_twisted.inlineCallbacks def test_remove_client(reactor): gm_config = yield util.run_tahoe( reactor, "grid-manager", "--config", "-", "create", @@ -71,7 +71,7 @@ def test_remove_client(reactor): assert json.loads(gm_config)['storage_servers'].has_key('bob') -@pytest.inlineCallbacks +@pytest_twisted.inlineCallbacks def test_remove_last_client(reactor): gm_config = yield util.run_tahoe( reactor, "grid-manager", "--config", "-", "create", @@ -93,7 +93,7 @@ def test_remove_last_client(reactor): assert not json.loads(gm_config).has_key('storage_servers') -@pytest.inlineCallbacks +@pytest_twisted.inlineCallbacks def test_reject_storage_server(reactor, request, storage_nodes, temp_dir, introducer_furl, flog_gatherer): gm_config = yield util.run_tahoe( reactor, "grid-manager", "--config", "-", "create", From 9220dd12efe2bcee0b1e320276e669e0179b19b8 Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 8 Apr 2019 19:42:50 -0600 Subject: [PATCH 0051/2309] get rid of debug --- src/allmydata/storage_client.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 5f65c6cec..39545b3a9 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -371,7 +371,7 @@ class NativeStorageServer(service.MultiService): } def __init__(self, server_id, ann, tub_maker, handler_overrides, grid_manager_keys, grid_manager_certs): - print("CREATE {}: {}".format(server_id, grid_manager_certs)) + # print("CREATE {}: {}".format(server_id, grid_manager_certs)) service.MultiService.__init__(self) assert isinstance(server_id, str) self._server_id = server_id @@ -387,11 +387,11 @@ class NativeStorageServer(service.MultiService): # any public-keys which the user has configured (if none, it # means use any storage servers) self._grid_manager_keys = grid_manager_keys - print("keys: {}".format(self._grid_manager_keys)) + # print("keys: {}".format(self._grid_manager_keys)) # any storage-certificates that this storage-server included # in its announcement self._grid_manager_certificates = grid_manager_certs - print("certs: {}".format(self._grid_manager_certificates)) + # print("certs: {}".format(self._grid_manager_certificates)) assert "anonymous-storage-FURL" in ann, ann furl = str(ann["anonymous-storage-FURL"]) From 3d7055711a6ecb8411528eb0fa6736d14cc6ec84 Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 8 Apr 2019 22:35:07 -0600 Subject: [PATCH 0052/2309] make and use set_config instead of internals --- src/allmydata/node.py | 15 +++++++++++++++ src/allmydata/scripts/admin.py | 4 +--- src/allmydata/test/test_node.py | 6 ++++++ 3 files changed, 22 insertions(+), 3 deletions(-) diff --git a/src/allmydata/node.py b/src/allmydata/node.py index c058fce4f..083cbef2c 100644 --- a/src/allmydata/node.py +++ b/src/allmydata/node.py @@ -321,6 +321,21 @@ class _Config(object): ) return default + def set_config(self, section, option, value): + """ + Set a config options in a section and re-write the tahoe.cfg file + """ + if option.endswith(".furl") and self._contains_unescaped_hash(value): + raise UnescapedHashError(section, option, item) + + try: + self.config.add_section(section) + except ConfigParser.DuplicateSectionError: + pass + self.config.set(section, option, value) + with open(self._config_fname, "w") as f: + self.config.write(f) + def get_config_from_file(self, name, required=False): """Get the (string) contents of a config file, or None if the file did not exist. If required=True, raise an exception rather than diff --git a/src/allmydata/scripts/admin.py b/src/allmydata/scripts/admin.py index ef1059a31..60d31cfb8 100644 --- a/src/allmydata/scripts/admin.py +++ b/src/allmydata/scripts/admin.py @@ -124,12 +124,10 @@ def add_grid_manager_cert(options): print("Already have file '{}'".format(cert_path), file=options.parent.parent.stderr) return 1 - cfg = config.config # why aren't methods we call on cfg in _Config itself? - gm_certs = config.get_config("storage", "grid_manager_certificate_files", "").split() if cert_fname not in gm_certs: gm_certs.append(cert_fname) - cfg.set("storage", "grid_manager_certificate_files", " ".join(gm_certs)) + config.set_config("storage", "grid_manager_certificate_files", " ".join(gm_certs)) # print("grid_manager_certificate_files in {}: {}".format(config_path, len(gm_certs))) diff --git a/src/allmydata/test/test_node.py b/src/allmydata/test/test_node.py index 096cd4f8f..a7ed6d528 100644 --- a/src/allmydata/test/test_node.py +++ b/src/allmydata/test/test_node.py @@ -316,6 +316,12 @@ class TestCase(testutil.SignalMixin, unittest.TestCase): yield client.create_client(basedir) self.failUnless(ns.called) + def test_set_config_new_section(self): + basedir = "test_node/test_set_config_new_section" + config = config_from_string(basedir, "", "") + config.set_config("foo", "bar", "value1") + config.set_config("foo", "bar", "value2") + class TestMissingPorts(unittest.TestCase): """ From f0e3b69f90278c71a45e978b833f64ae0b3dfcd9 Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 8 Apr 2019 23:33:40 -0600 Subject: [PATCH 0053/2309] switch around how we do config (avoid space-separated filenames) --- docs/proposed/grid-manager/managed-grid.rst | 7 ++- integration/test_grid_manager.py | 5 +- src/allmydata/client.py | 54 ++++++++++++--------- src/allmydata/scripts/admin.py | 11 ++--- 4 files changed, 44 insertions(+), 33 deletions(-) diff --git a/docs/proposed/grid-manager/managed-grid.rst b/docs/proposed/grid-manager/managed-grid.rst index 10d0c142a..5c388ffa1 100644 --- a/docs/proposed/grid-manager/managed-grid.rst +++ b/docs/proposed/grid-manager/managed-grid.rst @@ -174,7 +174,7 @@ Enrolling a Storage Server: Config You may edit the ``[storage]`` section of the ``tahoe.cfg`` file to turn on grid-management with ``grid_management = true``. You then must -also provide a ``[grid_management_keys]]`` section in the config-file which +also provide a ``[grid_management_keys]`` section in the config-file which lists ``name = path/to/certificate`` pairs. These certificate files are issued by the ``tahoe grid-manager sign`` @@ -278,7 +278,10 @@ certificates into the grid. We do this by adding some configuration (in ``tahoe.cfg``):: [storage] - grid_manager_certificate_files = gridmanager.cert + grid_management = true + + [grid_manager_certificates] + default = gridmanager.cert Add the above bit to each node's ``tahoe.cfg`` and re-start the storage nodes. diff --git a/integration/test_grid_manager.py b/integration/test_grid_manager.py index 075a1004a..f25819092 100644 --- a/integration/test_grid_manager.py +++ b/integration/test_grid_manager.py @@ -115,7 +115,6 @@ def test_reject_storage_server(reactor, request, storage_nodes, temp_dir, introd ) assert sorted(json.loads(gm_config)['storage_servers'].keys()) == ['storage0', 'storage1'] - # XXX FIXME need to shut-down and nuke carol when we're done this # test (i.d. request.addfinalizer) carol = yield util._create_node( @@ -136,7 +135,9 @@ def test_reject_storage_server(reactor, request, storage_nodes, temp_dir, introd with open(join(storage._node_dir, "gridmanager.cert"), "w") as f: f.write(cert) config = configutil.get_config(join(storage._node_dir, "tahoe.cfg")) - config.set("storage", "grid_manager_certificate_files", "gridmanager.cert") + config.set("storage", "grid_management", "True") + config.add_section("grid_manager_certificates") + config.set("grid_manager_certificates", "default", "gridmanager.cert") config.write(open(join(storage._node_dir, "tahoe.cfg"), "w")) # re-start this storage server diff --git a/src/allmydata/client.py b/src/allmydata/client.py index 50ceb448f..35512d8f5 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -59,6 +59,7 @@ def _valid_config_sections(): "stats_gatherer.furl", ), "grid_managers": None, # means "any options valid" + "grid_manager_certificates": None, "drop_upload": ( # deprecated already? "enabled", ), @@ -81,7 +82,7 @@ def _valid_config_sections(): "readonly", "reserved_space", "storage_dir", - "grid_manager_certificate_files", + "grid_management", ), "sftpd": ( "accounts.file", @@ -409,6 +410,34 @@ def create_storage_farm_broker(config, default_connection_handlers, foolscap_con return sb +def _load_grid_manager_certificates(config): + """ + Load all Grid Manager certificates in the config in a list. An + empty list is returned if there are none. + """ + grid_manager_certificates = [] + + cert_fnames = list(config.enumerate_section("grid_manager_certificates").values()) + for fname in cert_fnames: + fname = config.get_config_path(fname.decode('ascii')) + if not os.path.exists(fname): + raise ValueError( + "Grid Manager certificate file '{}' doesn't exist".format( + fname + ) + ) + with open(fname, 'r') as f: + cert = json.load(f) + if set(cert.keys()) != {"certificate", "signature"}: + raise ValueError( + "Unknown key in Grid Manager certificate '{}'".format( + fname + ) + ) + grid_manager_certificates.append(cert) + return grid_manager_certificates + + @implementer(IStatsProducer) class _Client(node.Node, pollmixin.PollMixin): @@ -604,27 +633,8 @@ class _Client(node.Node, pollmixin.PollMixin): grid_manager_certificates = [] - # XXX this is probably a bad idea for multiple fnames -- what - # about spaces in filenames? - - cert_fnames = self.get_config("storage", "grid_manager_certificate_files", "") - for fname in cert_fnames.split(): - fname = self.config.get_config_path(fname.decode('ascii')) - if not os.path.exists(fname): - raise ValueError( - "Grid Manager certificate file '{}' doesn't exist".format( - fname - ) - ) - with open(fname, 'r') as f: - cert = json.load(f) - if set(cert.keys()) != {"certificate", "signature"}: - raise ValueError( - "Unknown key in Grid Manager certificate '{}'".format( - fname - ) - ) - grid_manager_certificates.append(cert) + if self.config.get_config("storage", "grid_management", default=False, boolean=True): + grid_manager_certificates = _load_grid_manager_certificates(self.config) # XXX we should probably verify that the certificates are # valid and not expired, as that could be confusing for the diff --git a/src/allmydata/scripts/admin.py b/src/allmydata/scripts/admin.py index 60d31cfb8..3ddbe8d22 100644 --- a/src/allmydata/scripts/admin.py +++ b/src/allmydata/scripts/admin.py @@ -118,18 +118,15 @@ def add_grid_manager_cert(options): cert_fname = "{}.cert".format(options['name']) cert_path = config.get_config_path(cert_fname) cert_bytes = json.dumps(options.certificate_data, indent=4) + '\n' - # cert_name = options['name'] + cert_name = options['name'] if exists(cert_path): print("Already have file '{}'".format(cert_path), file=options.parent.parent.stderr) return 1 - gm_certs = config.get_config("storage", "grid_manager_certificate_files", "").split() - if cert_fname not in gm_certs: - gm_certs.append(cert_fname) - config.set_config("storage", "grid_manager_certificate_files", " ".join(gm_certs)) - - # print("grid_manager_certificate_files in {}: {}".format(config_path, len(gm_certs))) + config.set_config("storage", "grid_management", "True") + config.add_section("grid_manager_certificates") + config.set_config("grid_manager_certificates", cert_name, cert_fname) # write all the data out From f0a9240f5f25b88ec8795dd8839ac13632a0e952 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 9 Apr 2019 11:32:48 -0600 Subject: [PATCH 0054/2309] cleanup --- src/allmydata/scripts/admin.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/allmydata/scripts/admin.py b/src/allmydata/scripts/admin.py index 3ddbe8d22..47ca4a0eb 100644 --- a/src/allmydata/scripts/admin.py +++ b/src/allmydata/scripts/admin.py @@ -133,10 +133,12 @@ def add_grid_manager_cert(options): fileutil.write(cert_path, cert_bytes) # print("created {}: {} bytes".format(cert_fname, len(cert_bytes))) with open(config_path, "w") as f: - cfg.write(f) + # XXX probably want a _Config.write_tahoe_cfg() or something? or just set_config() does that automagically + config.config.write(f) # print("wrote {}".format(config_fname)) - print("There are now {} certificates".format(len(gm_certs)), file=options.parent.parent.stderr) + cert_count = len(config.enumerate_section("grid_manager_certificates")) + print("There are now {} certificates".format(cert_count), file=options.parent.parent.stderr) return 0 From 11993a230b750fd7caa386bdd7ab41bf5344e58f Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 9 Apr 2019 11:33:31 -0600 Subject: [PATCH 0055/2309] item -> value --- src/allmydata/node.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/node.py b/src/allmydata/node.py index 083cbef2c..da18c0720 100644 --- a/src/allmydata/node.py +++ b/src/allmydata/node.py @@ -326,7 +326,7 @@ class _Config(object): Set a config options in a section and re-write the tahoe.cfg file """ if option.endswith(".furl") and self._contains_unescaped_hash(value): - raise UnescapedHashError(section, option, item) + raise UnescapedHashError(section, option, value) try: self.config.add_section(section) From 8e363ca26b33c7b56ff73c15348997c3eb172734 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 9 Apr 2019 17:08:44 -0600 Subject: [PATCH 0056/2309] can't re-write a static string --- src/allmydata/node.py | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/src/allmydata/node.py b/src/allmydata/node.py index da18c0720..dcbc5088c 100644 --- a/src/allmydata/node.py +++ b/src/allmydata/node.py @@ -203,7 +203,13 @@ def config_from_string(basedir, portnumfile, config_str): # load configuration from in-memory string parser = ConfigParser.SafeConfigParser() parser.readfp(BytesIO(config_str)) - return _Config(parser, portnumfile, basedir, '') + + def write_new_config(cfg): + """ + We throw away any attempt to persist + """ + pass + return _Config(parser, portnumfile, basedir, '', write_new_config) def get_app_versions(): @@ -248,7 +254,8 @@ class _Config(object): have better names. """ - def __init__(self, configparser, portnum_fname, basedir, config_fname): + def __init__(self, configparser, portnum_fname, basedir, config_fname, + write_new_tahoecfg=None): """ :param configparser: a ConfigParser instance @@ -260,12 +267,25 @@ class _Config(object): :param config_fname: the pathname actually used to create the configparser (might be 'fake' if using in-memory data) + + :param write_new_tahoecfg: callable taking one argument which + is a ConfigParser instance """ self.portnum_fname = portnum_fname self._basedir = abspath_expanduser_unicode(unicode(basedir)) self._config_fname = config_fname self.config = configparser + if write_new_tahoecfg is None: + def write_new_tahoecfg(config): + """ + Write to the default place, /tahoe.cfg + """ + fn = os.path.join(self._basedir, "tahoe.cfg") + with open(fn, "w") as f: + config.write(f) + self._write_config = write_new_tahoecfg + nickname_utf8 = self.get_config("node", "nickname", "") self.nickname = nickname_utf8.decode("utf-8") assert type(self.nickname) is unicode @@ -333,8 +353,7 @@ class _Config(object): except ConfigParser.DuplicateSectionError: pass self.config.set(section, option, value) - with open(self._config_fname, "w") as f: - self.config.write(f) + self._write_config(self.config) def get_config_from_file(self, name, required=False): """Get the (string) contents of a config file, or None if the file From 1b3bfd53d4076cba7c9db2db73b9eebafcf79b28 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 14 May 2019 05:55:34 -0600 Subject: [PATCH 0057/2309] grid-manager client config is different --- docs/proposed/grid-manager/managed-grid.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/proposed/grid-manager/managed-grid.rst b/docs/proposed/grid-manager/managed-grid.rst index 5c388ffa1..8a655271d 100644 --- a/docs/proposed/grid-manager/managed-grid.rst +++ b/docs/proposed/grid-manager/managed-grid.rst @@ -313,8 +313,8 @@ configuration:: Put the key printed out above into Alice's ``tahoe.cfg`` in section ``client``:: - [client] - grid_manager_public_keys = pub-v0-vqimc4s5eflwajttsofisp5st566dbq36xnpp4siz57ufdavpvlq + [grid_managers] + example_name = pub-v0-vqimc4s5eflwajttsofisp5st566dbq36xnpp4siz57ufdavpvlq DECIDE: From 883a3ba12e63c74bbd6d91c86bbb1601aba6eb23 Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 6 May 2020 17:50:30 -0600 Subject: [PATCH 0058/2309] no mutable defaults --- src/allmydata/storage_client.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 39545b3a9..489892b7a 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -73,13 +73,13 @@ class StorageFarmBroker(service.MultiService): I'm also responsible for subscribing to the IntroducerClient to find out about new servers as they are announced by the Introducer. """ - def __init__(self, permute_peers, tub_maker, preferred_peers=(), grid_manager_keys=[]): + def __init__(self, permute_peers, tub_maker, preferred_peers=None, grid_manager_keys=None): service.MultiService.__init__(self) assert permute_peers # False not implemented yet self.permute_peers = permute_peers self._tub_maker = tub_maker - self.preferred_peers = preferred_peers - self._grid_manager_keys = grid_manager_keys + self.preferred_peers = preferred_peers if preferred_peers else tuple() + self._grid_manager_keys = grid_manager_keys if grid_manager_keys else list() # self.servers maps serverid -> IServer, and keeps track of all the # storage servers that we've heard about. Each descriptor manages its From 3af6d74ba828f2c8427d1882c36c275fc0311d7d Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 6 May 2020 17:52:48 -0600 Subject: [PATCH 0059/2309] clarify further --- docs/proposed/grid-manager/managed-grid.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/proposed/grid-manager/managed-grid.rst b/docs/proposed/grid-manager/managed-grid.rst index 8a655271d..b97e97c7b 100644 --- a/docs/proposed/grid-manager/managed-grid.rst +++ b/docs/proposed/grid-manager/managed-grid.rst @@ -8,8 +8,9 @@ Managed Grid ============ In a grid using an Introducer, a client will use any storage-server -the Introducer announces. This means that anyone with the Introducer -fURL can connect storage to the grid. +the Introducer announces (and the Introducer will annoucne any +storage-server that connects to it). This means that anyone with the +Introducer fURL can connect storage to the grid. Sometimes, this is just what you want! From 8e8b2f4d8f3916b5d1f778ae7d3fcdb72f22bb7a Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 6 May 2020 17:52:59 -0600 Subject: [PATCH 0060/2309] introducer-less -> introducerless --- docs/proposed/grid-manager/managed-grid.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/proposed/grid-manager/managed-grid.rst b/docs/proposed/grid-manager/managed-grid.rst index b97e97c7b..410172870 100644 --- a/docs/proposed/grid-manager/managed-grid.rst +++ b/docs/proposed/grid-manager/managed-grid.rst @@ -20,7 +20,7 @@ the grid; clients of this grid don't want their uploads to go to "unmanaged" storage if some other client decides to provide storage. One way to limit which storage servers a client connects to is via the -"server list" (:ref:`server_list`) (aka "Introducer-less" +"server list" (:ref:`server_list`) (aka "Introducerless" mode). Clients are given static lists of storage-servers, and connect only to those. This means manually updating these lists if the storage servers change, however. From 235d5debe1e281dafd559fa998e6007b9860f66d Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 6 May 2020 18:07:18 -0600 Subject: [PATCH 0061/2309] added example and more explanation for 'add' --- docs/proposed/grid-manager/managed-grid.rst | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/docs/proposed/grid-manager/managed-grid.rst b/docs/proposed/grid-manager/managed-grid.rst index 410172870..04832fd3f 100644 --- a/docs/proposed/grid-manager/managed-grid.rst +++ b/docs/proposed/grid-manager/managed-grid.rst @@ -94,12 +94,19 @@ tahoe grid-manager add `````````````````````` Takes two args: ``name pubkey``. The ``name`` is an arbitrary local -identifier and the pubkey is the encoded key from a ``node.pubkey`` -file in the storage-server's node directory (with no whitespace). +identifier for the new storage node (also sometimes called "a +petname"). The pubkey is the encoded key from a ``node.pubkey`` file +in the storage-server's node directory (with no whitespace). For +example, if ``~/storage0`` contains a storage-node, you might do +something like this: + + tahoe grid-manager --config ./gm0 add storage0 $(cat ~/storage0/node.pubkey) This adds a new storage-server to a Grid Manager's configuration. (Since it mutates the configuration, if you used -``--config -`` the new configuration will be printed to stdout). +``--config -`` the new configuration will be printed to stdout). The +usefulness of the ``name`` is solely for reference within this Grid +Manager. tahoe grid-manager list From 18fead08dd819f3b34125867759fbc95843aa45c Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 6 May 2020 18:08:18 -0600 Subject: [PATCH 0062/2309] consistently use 'nickname' --- docs/proposed/grid-manager/managed-grid.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/proposed/grid-manager/managed-grid.rst b/docs/proposed/grid-manager/managed-grid.rst index 04832fd3f..bae3bf990 100644 --- a/docs/proposed/grid-manager/managed-grid.rst +++ b/docs/proposed/grid-manager/managed-grid.rst @@ -94,9 +94,9 @@ tahoe grid-manager add `````````````````````` Takes two args: ``name pubkey``. The ``name`` is an arbitrary local -identifier for the new storage node (also sometimes called "a -petname"). The pubkey is the encoded key from a ``node.pubkey`` file -in the storage-server's node directory (with no whitespace). For +identifier for the new storage node (also sometimes called "a petname" +or "nickname"). The pubkey is the encoded key from a ``node.pubkey`` +file in the storage-server's node directory (with no whitespace). For example, if ``~/storage0`` contains a storage-node, you might do something like this: @@ -119,7 +119,7 @@ Lists all storage-servers that have previously been added using tahoe grid-manager sign ``````````````````````` -Takes one arg: ``name``, the petname used previously in a ``tahoe +Takes one arg: ``name``, the nickname used previously in a ``tahoe grid-manager add`` command. Note that this mutates the state of the grid-manager if it is on disk, @@ -211,7 +211,7 @@ DECIDE: is a command like this best, or should you have to edit the tahoe add-grid-manager `````````````````````` -- ``--name``: a petname to call this Grid Manager (default: "default") +- ``--name``: a nickname to call this Grid Manager (default: "default") For clients to start using a Grid Manager, they must add a public-key. A client may have any number of grid-managers, so each one From 4340cfa1b83f6aba6cba70e7b89f700b156127bc Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 6 May 2020 18:16:12 -0600 Subject: [PATCH 0063/2309] decision made --- docs/proposed/grid-manager/managed-grid.rst | 4 ---- 1 file changed, 4 deletions(-) diff --git a/docs/proposed/grid-manager/managed-grid.rst b/docs/proposed/grid-manager/managed-grid.rst index bae3bf990..88f210d34 100644 --- a/docs/proposed/grid-manager/managed-grid.rst +++ b/docs/proposed/grid-manager/managed-grid.rst @@ -203,10 +203,6 @@ out to clients). Enrolling a Client: CLI ----------------------- -DECIDE: is a command like this best, or should you have to edit the - config "by hand"? (below fits into warner's philosophy that "at some - point" it might be best to have all config in a database or similar - and the only way to view/edit it is via tahoe commands...) tahoe add-grid-manager `````````````````````` From a98eae3c7c92e3c8779c8e266d704bcdd189607a Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 6 May 2020 18:26:30 -0600 Subject: [PATCH 0064/2309] clarify --- docs/proposed/grid-manager/managed-grid.rst | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/docs/proposed/grid-manager/managed-grid.rst b/docs/proposed/grid-manager/managed-grid.rst index 88f210d34..6f7676b3c 100644 --- a/docs/proposed/grid-manager/managed-grid.rst +++ b/docs/proposed/grid-manager/managed-grid.rst @@ -207,15 +207,17 @@ Enrolling a Client: CLI tahoe add-grid-manager `````````````````````` -- ``--name``: a nickname to call this Grid Manager (default: "default") +This takes two arguments: ``name`` and ``public-identity``. -For clients to start using a Grid Manager, they must add a -public-key. A client may have any number of grid-managers, so each one -has a name. If you don't supply ``--name`` then ``"default"`` is used. +The ``name`` argument is a nickname to call this Grid Manager. A +client may have any number of grid-managers, so each one has a name. A +client with zero Grid Managers will accept any announcement from an +Introducer. -This command takes a single argument, which is the hex-encoded public -key of the Grid Manager. The client will have to be re-started once -this change is made. +The ``public-identity`` argument is the encoded public key of the Grid +Manager (that is, the output of ``tahoe grid-manager +public-identity``). The client will have to be re-started once this +change is made. Enrolling a Client: Config From fb3f0b79e1721b4f3d4f11e7450cd1f15c939490 Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 6 May 2020 18:34:56 -0600 Subject: [PATCH 0065/2309] more words about 'daemonize' --- docs/proposed/grid-manager/managed-grid.rst | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/proposed/grid-manager/managed-grid.rst b/docs/proposed/grid-manager/managed-grid.rst index 6f7676b3c..1ece84e42 100644 --- a/docs/proposed/grid-manager/managed-grid.rst +++ b/docs/proposed/grid-manager/managed-grid.rst @@ -241,6 +241,11 @@ Manager. Example:: Example Setup of a New Managed Grid ----------------------------------- +Note that we use the ``daemonize`` command in the following but that's +only one way to handle "running a command in the background". You +could instead run commands that start with ``daemonize ...`` in their +own shell/terminal window or via something like ``systemd`` + We'll store our Grid Manager configuration on disk, in ``./gm0``. To initialize this directory:: @@ -257,8 +262,7 @@ it briefly before it creates the "Introducer fURL" we want for the next steps:: tahoe create-introducer --listen=tcp --port=5555 --location=tcp:localhost:5555 ./introducer - tahoe -d introducer run - (Ctrl-C to stop it after a bit) + daemonize tahoe -d introducer run Next, we attach a couple of storage nodes:: From 7feed4146e74286854e1beb68db94ffd86e2f425 Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 7 May 2020 14:27:49 -0600 Subject: [PATCH 0066/2309] redundant webport --- docs/proposed/grid-manager/managed-grid.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/proposed/grid-manager/managed-grid.rst b/docs/proposed/grid-manager/managed-grid.rst index 1ece84e42..c3ac29409 100644 --- a/docs/proposed/grid-manager/managed-grid.rst +++ b/docs/proposed/grid-manager/managed-grid.rst @@ -266,8 +266,8 @@ next steps:: Next, we attach a couple of storage nodes:: - tahoe create-node --introducer $(cat introducer/private/introducer.furl) --nickname storage0 --webport 6001 --webport 6002 --location tcp:localhost:6003 --port 6003 ./storage0 - tahoe create-node --introducer $(cat introducer/private/introducer.furl) --nickname storage1 --webport 6101 --webport 6102 --location tcp:localhost:6103 --port 6103 ./storage1 + tahoe create-node --introducer $(cat introducer/private/introducer.furl) --nickname storage0 --webport 6001 --location tcp:localhost:6003 --port 6003 ./storage0 + tahoe create-node --introducer $(cat introducer/private/introducer.furl) --nickname storage1 --webport 6101 --location tcp:localhost:6103 --port 6103 ./storage1 daemonize tahoe -d storage0 run daemonize tahoe -d storage1 run @@ -300,7 +300,7 @@ Now try adding a new storage server ``storage2``. This client can join the grid just fine, and announce itself to the Introducer as providing storage:: - tahoe create-node --introducer $(cat introducer/private/introducer.furl) --nickname storage2 --webport 6301 --webport 6302 --location tcp:localhost:6303 --port 6303 ./storage2 + tahoe create-node --introducer $(cat introducer/private/introducer.furl) --nickname storage2 --webport 6301 --location tcp:localhost:6303 --port 6303 ./storage2 daemonize tahoe -d storage2 run At this point any client will upload to any of these three @@ -308,7 +308,7 @@ storage-servers. Make a client "alice" and try! :: - tahoe create-client --introducer $(cat introducer/private/introducer.furl) --nickname alice --webport 6301 --shares-total=3 --shares-needed=2 --shares-happy=3 ./alice + tahoe create-client --introducer $(cat introducer/private/introducer.furl) --nickname alice --webport 6401 --shares-total=3 --shares-needed=2 --shares-happy=3 ./alice daemonize tahoe -d alice run tahoe -d alice mkdir # prints out a dir-cap find storage2/storage/shares # confirm storage2 has a share From 6c15d67df64b8c22b37d22429db049122a047534 Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 7 May 2020 14:28:12 -0600 Subject: [PATCH 0067/2309] 'tahoe put' not 'tahoe mkdir' --- docs/proposed/grid-manager/managed-grid.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/proposed/grid-manager/managed-grid.rst b/docs/proposed/grid-manager/managed-grid.rst index c3ac29409..5dbb44e57 100644 --- a/docs/proposed/grid-manager/managed-grid.rst +++ b/docs/proposed/grid-manager/managed-grid.rst @@ -310,7 +310,7 @@ storage-servers. Make a client "alice" and try! tahoe create-client --introducer $(cat introducer/private/introducer.furl) --nickname alice --webport 6401 --shares-total=3 --shares-needed=2 --shares-happy=3 ./alice daemonize tahoe -d alice run - tahoe -d alice mkdir # prints out a dir-cap + tahoe -d alice put README.rst # prints out a read-cap find storage2/storage/shares # confirm storage2 has a share Now we want to make Alice only upload to the storage servers that the @@ -338,7 +338,7 @@ DECIDE: Now, re-start the "alice" client. Since we made Alice's parameters require 3 storage servers to be reachable (``--happy=3``), all their -uploads should now fail (so ``tahoe mkdir`` will fail) because they +uploads should now fail (so ``tahoe put`` will fail) because they won't use storage2 and thus can't "achieve happiness". You can check Alice's "Welcome" page (where the list of connected servers From f1ae20a6771a93c8c9e1bf6f97920f2525072dbc Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 7 May 2020 14:33:02 -0600 Subject: [PATCH 0068/2309] spelling --- docs/proposed/grid-manager/managed-grid.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/proposed/grid-manager/managed-grid.rst b/docs/proposed/grid-manager/managed-grid.rst index 5dbb44e57..845aff7cc 100644 --- a/docs/proposed/grid-manager/managed-grid.rst +++ b/docs/proposed/grid-manager/managed-grid.rst @@ -345,4 +345,4 @@ You can check Alice's "Welcome" page (where the list of connected servers is) at http://localhost:6301/ and should be able to see details about the "work-grid" Grid Manager that you added. When any Grid Managers are enabled, each storage-server line will show whether it has a valid -cerifiticate or not (and how much longer it's valid until). +certificate or not (and how much longer it's valid until). From 65403eb164c9e63d6c5de72832a8c4ee9932b4bb Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 7 May 2020 14:37:45 -0600 Subject: [PATCH 0069/2309] alice -> zara, bob -> yakov and docstrings --- integration/test_grid_manager.py | 43 +++++++++++++++++++------------- 1 file changed, 26 insertions(+), 17 deletions(-) diff --git a/integration/test_grid_manager.py b/integration/test_grid_manager.py index f25819092..2547b35be 100644 --- a/integration/test_grid_manager.py +++ b/integration/test_grid_manager.py @@ -16,6 +16,9 @@ import pytest_twisted @pytest_twisted.inlineCallbacks def test_create_certificate(reactor): + """ + The Grid Manager produces a valid, correctly-signed certificate. + """ gm_config = yield util.run_tahoe( reactor, "grid-manager", "--config", "-", "create", ) @@ -23,52 +26,58 @@ def test_create_certificate(reactor): privkey, pubkey_bytes = keyutil.parse_privkey(privkey_bytes) pubkey = keyutil.parse_pubkey(pubkey_bytes) + # Note that zara + her key here are arbitrary and don't match any + # "actual" clients in the test-grid; we're just checking that the + # Grid Manager signs this properly. gm_config = yield util.run_tahoe( reactor, "grid-manager", "--config", "-", "add", - "alice", "pub-v0-kzug3ut2m7ziihf3ndpqlquuxeie4foyl36wn54myqc4wmiwe4ga", + "zara", "pub-v0-kzug3ut2m7ziihf3ndpqlquuxeie4foyl36wn54myqc4wmiwe4ga", stdin=gm_config, ) - alice_cert_bytes = yield util.run_tahoe( - reactor, "grid-manager", "--config", "-", "sign", "alice", + zara_cert_bytes = yield util.run_tahoe( + reactor, "grid-manager", "--config", "-", "sign", "zara", stdin=gm_config, ) - alice_cert = json.loads(alice_cert_bytes) + zara_cert = json.loads(zara_cert_bytes) - # confirm that alice's certificate is made by the Grid Manager + # confirm that zara's certificate is made by the Grid Manager # (.verify returns None on success, raises exception on error) pubkey.verify( - base32.a2b(alice_cert['signature'].encode('ascii')), - alice_cert['certificate'].encode('ascii'), + base32.a2b(zara_cert['signature'].encode('ascii')), + zara_cert['certificate'].encode('ascii'), ) @pytest_twisted.inlineCallbacks def test_remove_client(reactor): + """ + A Grid Manager can add and successfully remove a client + """ gm_config = yield util.run_tahoe( reactor, "grid-manager", "--config", "-", "create", ) gm_config = yield util.run_tahoe( reactor, "grid-manager", "--config", "-", "add", - "alice", "pub-v0-kzug3ut2m7ziihf3ndpqlquuxeie4foyl36wn54myqc4wmiwe4ga", + "zara", "pub-v0-kzug3ut2m7ziihf3ndpqlquuxeie4foyl36wn54myqc4wmiwe4ga", stdin=gm_config, ) gm_config = yield util.run_tahoe( reactor, "grid-manager", "--config", "-", "add", - "bob", "pub-v0-kvxhb3nexybmipkrar2ztfrwp4uxxsmrjzkpzafit3ket4u5yldq", + "yakov", "pub-v0-kvxhb3nexybmipkrar2ztfrwp4uxxsmrjzkpzafit3ket4u5yldq", stdin=gm_config, ) - assert json.loads(gm_config)['storage_servers'].has_key("alice") - assert json.loads(gm_config)['storage_servers'].has_key("bob") + assert json.loads(gm_config)['storage_servers'].has_key("zara") + assert json.loads(gm_config)['storage_servers'].has_key("yakov") return gm_config = yield util.run_tahoe( reactor, "grid-manager", "--config", "-", "remove", - "alice", + "zara", stdin=gm_config, ) - assert not json.loads(gm_config)['storage_servers'].has_key('alice') - assert json.loads(gm_config)['storage_servers'].has_key('bob') + assert not json.loads(gm_config)['storage_servers'].has_key('zara') + assert json.loads(gm_config)['storage_servers'].has_key('yakov') @pytest_twisted.inlineCallbacks @@ -79,14 +88,14 @@ def test_remove_last_client(reactor): gm_config = yield util.run_tahoe( reactor, "grid-manager", "--config", "-", "add", - "alice", "pub-v0-kzug3ut2m7ziihf3ndpqlquuxeie4foyl36wn54myqc4wmiwe4ga", + "zara", "pub-v0-kzug3ut2m7ziihf3ndpqlquuxeie4foyl36wn54myqc4wmiwe4ga", stdin=gm_config, ) - assert json.loads(gm_config)['storage_servers'].has_key("alice") + assert json.loads(gm_config)['storage_servers'].has_key("zara") gm_config = yield util.run_tahoe( reactor, "grid-manager", "--config", "-", "remove", - "alice", + "zara", stdin=gm_config, ) # there are no storage servers left at all now From 97a7672226010ef5de2715c0fc40f7e7fa6a2f92 Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 7 May 2020 14:40:10 -0600 Subject: [PATCH 0070/2309] .has_key -> 'in' --- integration/test_grid_manager.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/integration/test_grid_manager.py b/integration/test_grid_manager.py index 2547b35be..6aa50b09e 100644 --- a/integration/test_grid_manager.py +++ b/integration/test_grid_manager.py @@ -67,8 +67,8 @@ def test_remove_client(reactor): "yakov", "pub-v0-kvxhb3nexybmipkrar2ztfrwp4uxxsmrjzkpzafit3ket4u5yldq", stdin=gm_config, ) - assert json.loads(gm_config)['storage_servers'].has_key("zara") - assert json.loads(gm_config)['storage_servers'].has_key("yakov") + assert "zara" in json.loads(gm_config)['storage_servers'] + assert "yakov" in json.loads(gm_config)['storage_servers'] return gm_config = yield util.run_tahoe( @@ -76,8 +76,8 @@ def test_remove_client(reactor): "zara", stdin=gm_config, ) - assert not json.loads(gm_config)['storage_servers'].has_key('zara') - assert json.loads(gm_config)['storage_servers'].has_key('yakov') + assert "zara" not in json.loads(gm_config)['storage_servers'] + assert "yakov" in json.loads(gm_config)['storage_servers'] @pytest_twisted.inlineCallbacks @@ -91,7 +91,7 @@ def test_remove_last_client(reactor): "zara", "pub-v0-kzug3ut2m7ziihf3ndpqlquuxeie4foyl36wn54myqc4wmiwe4ga", stdin=gm_config, ) - assert json.loads(gm_config)['storage_servers'].has_key("zara") + assert "zara" in json.loads(gm_config)['storage_servers'] gm_config = yield util.run_tahoe( reactor, "grid-manager", "--config", "-", "remove", @@ -99,7 +99,7 @@ def test_remove_last_client(reactor): stdin=gm_config, ) # there are no storage servers left at all now - assert not json.loads(gm_config).has_key('storage_servers') + assert "storage_servers" not in json.loads(gm_config) @pytest_twisted.inlineCallbacks From 5071da2c0560ffd2803b89415186e6c4271a0b4b Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 7 May 2020 14:47:18 -0600 Subject: [PATCH 0071/2309] remove early return --- integration/test_grid_manager.py | 1 - 1 file changed, 1 deletion(-) diff --git a/integration/test_grid_manager.py b/integration/test_grid_manager.py index 6aa50b09e..06d763b4f 100644 --- a/integration/test_grid_manager.py +++ b/integration/test_grid_manager.py @@ -69,7 +69,6 @@ def test_remove_client(reactor): ) assert "zara" in json.loads(gm_config)['storage_servers'] assert "yakov" in json.loads(gm_config)['storage_servers'] - return gm_config = yield util.run_tahoe( reactor, "grid-manager", "--config", "-", "remove", From 72a51e903a8a6951ff86b2be116f300aabd392bf Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 7 May 2020 14:50:10 -0600 Subject: [PATCH 0072/2309] docstring --- integration/test_grid_manager.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/integration/test_grid_manager.py b/integration/test_grid_manager.py index 06d763b4f..d9fac17a9 100644 --- a/integration/test_grid_manager.py +++ b/integration/test_grid_manager.py @@ -81,6 +81,9 @@ def test_remove_client(reactor): @pytest_twisted.inlineCallbacks def test_remove_last_client(reactor): + """ + A Grid Manager can remove all clients + """ gm_config = yield util.run_tahoe( reactor, "grid-manager", "--config", "-", "create", ) @@ -103,6 +106,10 @@ def test_remove_last_client(reactor): @pytest_twisted.inlineCallbacks def test_reject_storage_server(reactor, request, storage_nodes, temp_dir, introducer_furl, flog_gatherer): + """ + A client using grid-manager refuses to upload to a storage-server + without a valid certificate + """ gm_config = yield util.run_tahoe( reactor, "grid-manager", "--config", "-", "create", ) From 371fcd5b8600f7332fff777090acbeb054ff1a1c Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 7 May 2020 14:54:50 -0600 Subject: [PATCH 0073/2309] timeouts not required --- integration/test_grid_manager.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/integration/test_grid_manager.py b/integration/test_grid_manager.py index d9fac17a9..6ef1e053b 100644 --- a/integration/test_grid_manager.py +++ b/integration/test_grid_manager.py @@ -173,11 +173,10 @@ def test_reject_storage_server(reactor, request, storage_nodes, temp_dir, introd config.write(open(join(carol._node_dir, "tahoe.cfg"), "w")) carol.signalProcess('TERM') yield carol._protocol.exited - time.sleep(1) + carol = yield util._run_node( reactor, carol._node_dir, request, None, ) - time.sleep(5) # try to put something into the grid, which should fail (because # carol has happy=3 but should only find storage0, storage1 to be From c823ff1195334ba61f3c8ae1ff66b5fe9f3524d8 Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 7 May 2020 14:56:24 -0600 Subject: [PATCH 0074/2309] better docstring --- integration/test_grid_manager.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/integration/test_grid_manager.py b/integration/test_grid_manager.py index 6ef1e053b..0a825f14a 100644 --- a/integration/test_grid_manager.py +++ b/integration/test_grid_manager.py @@ -107,8 +107,9 @@ def test_remove_last_client(reactor): @pytest_twisted.inlineCallbacks def test_reject_storage_server(reactor, request, storage_nodes, temp_dir, introducer_furl, flog_gatherer): """ - A client using grid-manager refuses to upload to a storage-server - without a valid certificate + A client with happines=3 fails to upload to a Grid when it is + using Grid Manager and there are only two storage-servers with + valid certificates. """ gm_config = yield util.run_tahoe( reactor, "grid-manager", "--config", "-", "create", From 4ece4e9dce3cf62ce33fb245f507d070255be3aa Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 7 May 2020 15:24:07 -0600 Subject: [PATCH 0075/2309] specific exception for failing subprocess --- integration/test_grid_manager.py | 7 ++----- integration/test_servers_of_happiness.py | 4 ++-- integration/util.py | 22 +++++++++++++++------- 3 files changed, 19 insertions(+), 14 deletions(-) diff --git a/integration/test_grid_manager.py b/integration/test_grid_manager.py index 0a825f14a..d46bb6b10 100644 --- a/integration/test_grid_manager.py +++ b/integration/test_grid_manager.py @@ -190,8 +190,5 @@ def test_reject_storage_server(reactor, request, storage_nodes, temp_dir, introd stdin="some content" * 200, ) assert False, "Should get a failure" - except Exception as e: - # depending on the full output being in the error-message - # here; see util.py - assert 'UploadUnhappinessError' in str(e) - print("found expected UploadUnhappinessError") + except util.ProcessFailed as e: + assert 'UploadUnhappinessError' in e.output.getvalue() diff --git a/integration/test_servers_of_happiness.py b/integration/test_servers_of_happiness.py index 1984dd507..c9b654d9c 100644 --- a/integration/test_servers_of_happiness.py +++ b/integration/test_servers_of_happiness.py @@ -42,8 +42,8 @@ def test_upload_immutable(reactor, temp_dir, introducer_furl, flog_gatherer, sto try: yield proto.done assert False, "should raise exception" - except Exception as e: - assert "UploadUnhappinessError" in str(e) + except util.ProcessFailed as e: + assert "UploadUnhappinessError" in e.output.getvalue() output = proto.output.getvalue() assert "shares could be placed on only" in output diff --git a/integration/util.py b/integration/util.py index 703402aa8..ffdf5a317 100644 --- a/integration/util.py +++ b/integration/util.py @@ -32,6 +32,20 @@ class _ProcessExitedProtocol(ProcessProtocol): self.done.callback(None) +class ProcessFailed(Exception): + """ + A subprocess has failed. + + :ivar ProcessTerminated reason: the original reason from .processExited + + :ivar StringIO output: all stdout and stderr collected to this point. + """ + + def __init__(self, reason, output): + self.reason = reason + self.output = output + + class _CollectOutputProtocol(ProcessProtocol): """ Internal helper. Collects all output (stdout + stderr) into @@ -54,13 +68,7 @@ class _CollectOutputProtocol(ProcessProtocol): def processExited(self, reason): if not isinstance(reason.value, ProcessDone): - #self.done.errback(reason) - self.done.errback(RuntimeError( - "Process failed: {}\nOutput:\n{}".format( - reason, - self.output.getvalue(), - ) - )) + self.done.errback(ProcessFailed(reason, self.output)) def outReceived(self, data): self.output.write(data) From 4fc649fee16587846d0ccecb7f3fc81949fe30fe Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 7 May 2020 15:24:26 -0600 Subject: [PATCH 0076/2309] error to create a duplicate node --- integration/test_tor.py | 42 ++++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/integration/test_tor.py b/integration/test_tor.py index e0b991128..801efc3ca 100644 --- a/integration/test_tor.py +++ b/integration/test_tor.py @@ -67,28 +67,28 @@ def _create_anonymous_node(reactor, name, control_port, request, temp_dir, flog_ node_dir = join(temp_dir, name) web_port = "tcp:{}:interface=localhost".format(control_port + 2000) - if True: - if exists(node_dir): - print("nuking '{}'".format(node_dir)) - shutil.rmtree(node_dir) - print("creating", node_dir) - mkdir(node_dir) - proto = util._DumpOutputProtocol(None) - reactor.spawnProcess( - proto, - sys.executable, - ( - sys.executable, '-m', 'allmydata.scripts.runner', - 'create-node', - '--nickname', name, - '--introducer', introducer_furl, - '--hide-ip', - '--tor-control-port', 'tcp:localhost:{}'.format(control_port), - '--listen', 'tor', - node_dir, - ) + if exists(node_dir): + raise RuntimeError( + "A node already exists in '{}'".format(node_dir) ) - yield proto.done + print("creating", node_dir) + mkdir(node_dir) + proto = util._DumpOutputProtocol(None) + reactor.spawnProcess( + proto, + sys.executable, + ( + sys.executable, '-m', 'allmydata.scripts.runner', + 'create-node', + '--nickname', name, + '--introducer', introducer_furl, + '--hide-ip', + '--tor-control-port', 'tcp:localhost:{}'.format(control_port), + '--listen', 'tor', + node_dir, + ) + ) + yield proto.done with open(join(node_dir, 'tahoe.cfg'), 'w') as f: f.write(''' From dfa8b37a152246abc6150b1a697afe41ddbd4884 Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 7 May 2020 15:26:22 -0600 Subject: [PATCH 0077/2309] irrelevant code --- integration/util.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/integration/util.py b/integration/util.py index ffdf5a317..e0454533f 100644 --- a/integration/util.py +++ b/integration/util.py @@ -209,9 +209,6 @@ def _create_node(reactor, request, temp_dir, introducer_furl, flog_gatherer, nam if exists(node_dir): created_d = succeed(None) else: - if exists(node_dir): - print("nuking: {}".format(node_dir)) - rmtree(node_dir) print("creating", node_dir) mkdir(node_dir) done_proto = _ProcessExitedProtocol() From 85d8e24421bd55a2b096aa852889a2032837b7be Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 7 May 2020 15:27:47 -0600 Subject: [PATCH 0078/2309] ascii -> utf8 --- src/allmydata/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/client.py b/src/allmydata/client.py index 35512d8f5..901004db4 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -419,7 +419,7 @@ def _load_grid_manager_certificates(config): cert_fnames = list(config.enumerate_section("grid_manager_certificates").values()) for fname in cert_fnames: - fname = config.get_config_path(fname.decode('ascii')) + fname = config.get_config_path(fname.decode('utf8')) if not os.path.exists(fname): raise ValueError( "Grid Manager certificate file '{}' doesn't exist".format( From c9f5ed7d6442a4fe00d89adc4d53217641c6071b Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 7 May 2020 15:39:51 -0600 Subject: [PATCH 0079/2309] redundant --- src/allmydata/node.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/allmydata/node.py b/src/allmydata/node.py index dcbc5088c..f2f52d323 100644 --- a/src/allmydata/node.py +++ b/src/allmydata/node.py @@ -208,7 +208,6 @@ def config_from_string(basedir, portnumfile, config_str): """ We throw away any attempt to persist """ - pass return _Config(parser, portnumfile, basedir, '', write_new_config) From 7cb7cdfac90a45e9c3fcc5de35cbc93f7d96b805 Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 7 May 2020 15:44:25 -0600 Subject: [PATCH 0080/2309] method instead of nested function --- src/allmydata/node.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/src/allmydata/node.py b/src/allmydata/node.py index f2f52d323..ae62e2fc3 100644 --- a/src/allmydata/node.py +++ b/src/allmydata/node.py @@ -276,19 +276,21 @@ class _Config(object): self.config = configparser if write_new_tahoecfg is None: - def write_new_tahoecfg(config): - """ - Write to the default place, /tahoe.cfg - """ - fn = os.path.join(self._basedir, "tahoe.cfg") - with open(fn, "w") as f: - config.write(f) + write_new_tahoecfg = self._default_write_new_tahoecfg self._write_config = write_new_tahoecfg nickname_utf8 = self.get_config("node", "nickname", "") self.nickname = nickname_utf8.decode("utf-8") assert type(self.nickname) is unicode + def _default_write_new_tahoecfg(self, config): + """ + Write to the default place, /tahoe.cfg + """ + fn = os.path.join(self._basedir, "tahoe.cfg") + with open(fn, "w") as f: + config.write(f) + def validate(self, valid_config_sections): configutil.validate_config(self._config_fname, self.config, valid_config_sections) From c0f0d765633edaad55cd1cfb1ba5e1a0768c70b5 Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 7 May 2020 15:52:45 -0600 Subject: [PATCH 0081/2309] parametrize 'now' function --- src/allmydata/storage_client.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 489892b7a..e78470e88 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -313,18 +313,24 @@ def parse_grid_manager_data(gm_data): return js -def validate_grid_manager_certificate(gm_key, alleged_cert): +def validate_grid_manager_certificate(gm_key, alleged_cert, now_fn=None): """ :param gm_key: a VerifyingKey instance, a Grid Manager's public key. - :param cert: dict with "certificate" and "signature" keys, where + :param alleged_cert: dict with "certificate" and "signature" keys, where "certificate" contains a JSON-serialized certificate for a Storage Server (comes from a Grid Manager). + :param now_fn: a zero-argument callable that returns a UTC + timestamp (will use datetime.utcnow by default) + :return: False if the signature is invalid or the certificate is expired. """ + if now_fn is None: + now_fn = datetime.utcnow + try: gm_key.verify( base32.a2b(alleged_cert['signature'].encode('ascii')), @@ -334,7 +340,7 @@ def validate_grid_manager_certificate(gm_key, alleged_cert): return False # signature is valid; now we can load the actual data cert = json.loads(alleged_cert['certificate']) - now = datetime.utcnow() + now = now_fn() expires = datetime.utcfromtimestamp(cert['expires']) # cert_pubkey = keyutil.parse_pubkey(cert['public_key'].encode('ascii')) if expires < now: From 026bcca6c562b32540ce458c6e5ad8b3fa94b8dd Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 7 May 2020 15:54:23 -0600 Subject: [PATCH 0082/2309] upload_permitted in IServer --- src/allmydata/interfaces.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/allmydata/interfaces.py b/src/allmydata/interfaces.py index 36c3622d8..aced2b546 100644 --- a/src/allmydata/interfaces.py +++ b/src/allmydata/interfaces.py @@ -470,6 +470,12 @@ class IServer(IDisplayableServer): once the connection is lost. """ + def upload_permitted(): + """ + :return: True if we should use this server for uploads, False + otherwise. + """ + class IMutableSlotWriter(Interface): """ From a384df720a67e7deb765f482cae3c5d3805094be Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 7 May 2020 15:56:10 -0600 Subject: [PATCH 0083/2309] link to docs --- docs/proposed/grid-manager/managed-grid.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/proposed/grid-manager/managed-grid.rst b/docs/proposed/grid-manager/managed-grid.rst index 845aff7cc..15c728784 100644 --- a/docs/proposed/grid-manager/managed-grid.rst +++ b/docs/proposed/grid-manager/managed-grid.rst @@ -26,7 +26,7 @@ only to those. This means manually updating these lists if the storage servers change, however. Another method is for clients to use `[client] peers.preferred=` -configuration option (XXX link? appears undocumented), which suffers +configuration option (:ref:`Client Configuration`), which suffers from a similar disadvantage. From ac46fb24f97a7b7e619afdbdffc5dd0697c39f60 Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 7 May 2020 15:57:13 -0600 Subject: [PATCH 0084/2309] take -> require --- docs/proposed/grid-manager/managed-grid.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/proposed/grid-manager/managed-grid.rst b/docs/proposed/grid-manager/managed-grid.rst index 15c728784..08ab07f70 100644 --- a/docs/proposed/grid-manager/managed-grid.rst +++ b/docs/proposed/grid-manager/managed-grid.rst @@ -64,7 +64,7 @@ If you don't want to store the configuration on disk at all, you may use ``--config -`` (that's a dash) and write a valid JSON configuration to stdin. -All commands take the ``--config`` option, and they all behave +All commands require the ``--config`` option, and they all behave similarly for "data from stdin" versus "data from disk". From 92bd22f2c611ba4c220cfbeb0ce696bfccf01f3f Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 7 May 2020 16:02:25 -0600 Subject: [PATCH 0085/2309] reword --- docs/proposed/grid-manager/managed-grid.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/proposed/grid-manager/managed-grid.rst b/docs/proposed/grid-manager/managed-grid.rst index 08ab07f70..a3ad0d115 100644 --- a/docs/proposed/grid-manager/managed-grid.rst +++ b/docs/proposed/grid-manager/managed-grid.rst @@ -96,9 +96,9 @@ tahoe grid-manager add Takes two args: ``name pubkey``. The ``name`` is an arbitrary local identifier for the new storage node (also sometimes called "a petname" or "nickname"). The pubkey is the encoded key from a ``node.pubkey`` -file in the storage-server's node directory (with no whitespace). For -example, if ``~/storage0`` contains a storage-node, you might do -something like this: +file in the storage-server's node directory (minus any +whitespace). For example, if ``~/storage0`` contains a storage-node, +you might do something like this: tahoe grid-manager --config ./gm0 add storage0 $(cat ~/storage0/node.pubkey) From 91af588c9be3bcc230ce308087f2d5a5050d20f5 Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 7 May 2020 20:08:02 -0600 Subject: [PATCH 0086/2309] post-merge fixups (keyutil, preferred_peers) --- src/allmydata/client.py | 3 +-- src/allmydata/node.py | 2 +- src/allmydata/scripts/tahoe_grid_manager.py | 17 ++++++++++------- 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/src/allmydata/client.py b/src/allmydata/client.py index ced282900..24f217288 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -580,8 +580,7 @@ def create_storage_farm_broker(config, default_connection_handlers, foolscap_con tub_maker=tub_creator, node_config=config, storage_client_config=storage_client_config, - preferred_peers=preferred_peers, - grid_manager_keys=grid_manager_keys, + grid_manager_keys=grid_manager_keys, # XXX maybe roll into above storage_client_config? ) for ic in introducer_clients: sb.use_introducer(ic) diff --git a/src/allmydata/node.py b/src/allmydata/node.py index 404f8a125..676975728 100644 --- a/src/allmydata/node.py +++ b/src/allmydata/node.py @@ -205,7 +205,7 @@ def config_from_string(basedir, portnumfile, config_str, _valid_config=None): # load configuration from in-memory string parser = ConfigParser.SafeConfigParser() parser.readfp(BytesIO(config_str)) - configutil.validate_config(fname, parser, _valid_config) + configutil.validate_config('', parser, _valid_config) def write_new_config(cfg): """ diff --git a/src/allmydata/scripts/tahoe_grid_manager.py b/src/allmydata/scripts/tahoe_grid_manager.py index b33552861..6c40f0814 100644 --- a/src/allmydata/scripts/tahoe_grid_manager.py +++ b/src/allmydata/scripts/tahoe_grid_manager.py @@ -13,7 +13,7 @@ from twisted.python import usage from twisted.python.filepath import FilePath from allmydata.util import fileutil from allmydata.util import base32 -from allmydata.util import keyutil +from allmydata.crypto import ed25519 from twisted.internet.defer import inlineCallbacks, returnValue @@ -50,7 +50,7 @@ class AddOptions(BaseOptions): self['name'] = unicode(args[0]) try: # WTF?! why does it want 'str' and not six.text_type? - self['storage_public_key'] = keyutil.parse_pubkey(args[1]) + self['storage_public_key'] = ed25519.verifying_key_from_string(args[1]) except Exception as e: raise usage.UsageError( "Invalid public_key argument: {}".format(e) @@ -132,8 +132,11 @@ def _create_gridmanager(): :return: an object providing the GridManager interface initialized with a new random keypair """ - private_key_bytes, public_key_bytes = keyutil.make_keypair() - return _GridManager(private_key_bytes, {}) + private_key, public_key = ed25519.create_signing_keypair() + return _GridManager( + ed25519.string_from_signing_key(private_key), + {}, + ) def _create(gridoptions, options): """ @@ -195,7 +198,7 @@ class _GridManager(object): private_key_bytes = config['private_key'].encode('ascii') try: - private_key, public_key_bytes = keyutil.parse_privkey(private_key_bytes) + private_key, public_key = ed25519.signing_keypair_from_string(private_key_bytes) except Exception as e: raise ValueError( "Invalid Grid Manager private_key: {}".format(e) @@ -209,7 +212,7 @@ class _GridManager(object): ) storage_servers[name] = _GridManagerStorageServer( name, - keyutil.parse_pubkey(srv_config['public_key'].encode('ascii')), + ed25519.verifying_key_from_string(srv_config['public_key'].encode('ascii')), None, ) @@ -225,7 +228,7 @@ class _GridManager(object): def __init__(self, private_key_bytes, storage_servers): self._storage_servers = dict() if storage_servers is None else storage_servers self._private_key_bytes = private_key_bytes - self._private_key, _ = keyutil.parse_privkey(self._private_key_bytes) + self._private_key, _ = ed25519.signing_keypair_from_string(self._private_key_bytes) self._version = 0 @property From 38069e51fb65254418fd63f5c3e454b488e61e49 Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 7 May 2020 20:43:17 -0600 Subject: [PATCH 0087/2309] keyutil -> ed25519 --- src/allmydata/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/client.py b/src/allmydata/client.py index 24f217288..54a4e9ee2 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -564,7 +564,7 @@ def create_storage_farm_broker(config, default_connection_handlers, foolscap_con grid_manager_keys = [] for name, gm_key in config.enumerate_section('grid_managers').items(): grid_manager_keys.append( - keyutil.parse_pubkey(gm_key) + ed25519.verifying_key_from_string(gm_key) ) # we don't actually use this keypair for anything (yet) as far From e6cb700bcc2e33f743c0f100591ca6013a264243 Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 7 May 2020 20:43:37 -0600 Subject: [PATCH 0088/2309] incorrect merge conflict-resolution --- src/allmydata/client.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/allmydata/client.py b/src/allmydata/client.py index 54a4e9ee2..36f870a64 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -954,13 +954,6 @@ class _Client(node.Node, pollmixin.PollMixin): # of the Grid Manager (should that go in the config too, # then? How to handle multiple grid-managers?) - furl_file = self.config.get_private_path("storage.furl").encode(get_filesystem_encoding()) - furl = self.tub.registerReference(ss, furlFile=furl_file) - ann = { - "anonymous-storage-FURL": furl, - "permutation-seed-base32": self._init_permutation_seed(ss), - "grid-manager-certificates": grid_manager_certificates, - } for ic in self.introducer_clients: ic.publish("storage", announcement, self._node_private_key) From 02c3401e016e48f0a1a080168cde28c57e7a6bea Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 7 May 2020 20:43:49 -0600 Subject: [PATCH 0089/2309] make code more like master --- src/allmydata/node.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/allmydata/node.py b/src/allmydata/node.py index 676975728..32b78b094 100644 --- a/src/allmydata/node.py +++ b/src/allmydata/node.py @@ -205,13 +205,14 @@ def config_from_string(basedir, portnumfile, config_str, _valid_config=None): # load configuration from in-memory string parser = ConfigParser.SafeConfigParser() parser.readfp(BytesIO(config_str)) - configutil.validate_config('', parser, _valid_config) + fname = "" + configutil.validate_config(fname, parser, _valid_config) def write_new_config(cfg): """ We throw away any attempt to persist """ - return _Config(parser, portnumfile, basedir, '', write_new_config) + return _Config(parser, portnumfile, basedir, fname, write_new_config) def get_app_versions(): From 765d9daa8cfee588ee70167b32d44b5c9e412fd5 Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 7 May 2020 20:57:06 -0600 Subject: [PATCH 0090/2309] keyutil -> ed25519 --- integration/test_grid_manager.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/integration/test_grid_manager.py b/integration/test_grid_manager.py index d46bb6b10..ffe6733d1 100644 --- a/integration/test_grid_manager.py +++ b/integration/test_grid_manager.py @@ -5,7 +5,7 @@ import shutil from os import mkdir, unlink, listdir, utime from os.path import join, exists, getmtime -from allmydata.util import keyutil +from allmydata.crypto import ed25519 from allmydata.util import base32 from allmydata.util import configutil @@ -23,8 +23,7 @@ def test_create_certificate(reactor): reactor, "grid-manager", "--config", "-", "create", ) privkey_bytes = json.loads(gm_config)['private_key'].encode('ascii') - privkey, pubkey_bytes = keyutil.parse_privkey(privkey_bytes) - pubkey = keyutil.parse_pubkey(pubkey_bytes) + privkey, pubkey = ed25519.signing_keypair_from_string(privkey_bytes) # Note that zara + her key here are arbitrary and don't match any # "actual" clients in the test-grid; we're just checking that the @@ -115,8 +114,7 @@ def test_reject_storage_server(reactor, request, storage_nodes, temp_dir, introd reactor, "grid-manager", "--config", "-", "create", ) privkey_bytes = json.loads(gm_config)['private_key'].encode('ascii') - privkey, pubkey_bytes = keyutil.parse_privkey(privkey_bytes) - pubkey = keyutil.parse_pubkey(pubkey_bytes) + privkey, _ = ed25519.signing_keypair_from_string(privkey_bytes) # create certificates for first 2 storage-servers for idx, storage in enumerate(storage_nodes[:2]): @@ -170,7 +168,7 @@ def test_reject_storage_server(reactor, request, storage_nodes, temp_dir, introd config = configutil.get_config(join(carol._node_dir, "tahoe.cfg")) print(dir(config)) config.add_section("grid_managers") - config.set("grid_managers", "test", pubkey_bytes) + config.set("grid_managers", "test", ed25519.string_from_verifying_key(pubkey)) config.write(open(join(carol._node_dir, "tahoe.cfg"), "w")) carol.signalProcess('TERM') yield carol._protocol.exited From 37a23d8b5e551e0a194b2c5a96357ce8fee67209 Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 7 May 2020 21:02:39 -0600 Subject: [PATCH 0091/2309] debug --- src/allmydata/storage_client.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index c076624ab..7d8df34c7 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -233,7 +233,6 @@ class StorageFarmBroker(service.MultiService): assert isinstance(server_id, unicode) # from YAML server_id = server_id.encode("ascii") handler_overrides = server.get("connections", {}) - print("ANN", server["ann"]) s = NativeStorageServer( server_id, server["ann"], From 5e1e90df89655abdddd113e107c6fbe461d4fac5 Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 7 May 2020 21:02:50 -0600 Subject: [PATCH 0092/2309] pass on stdin= --- integration/util.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/integration/util.py b/integration/util.py index 0c41c6bd3..d85050d4e 100644 --- a/integration/util.py +++ b/integration/util.py @@ -151,11 +151,10 @@ def _cleanup_tahoe_process(tahoe_transport, exited): pass -def run_tahoe(reactor, request, *args): +def run_tahoe(reactor, request, stdin=None, *args): """ Helper to run tahoe with optional coverage """ - stdin = kwargs.get('stdin', None) protocol = _CollectOutputProtocol(stdin=stdin) process = _tahoe_runner_optional_coverage(protocol, reactor, request, args) process.exited = protocol.done From f21c1f4b4f4cfa8d23322c853903e20e0b458a98 Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 7 May 2020 22:29:04 -0600 Subject: [PATCH 0093/2309] pass on request arg --- integration/test_grid_manager.py | 34 ++++++++++++++++---------------- integration/util.py | 8 ++++++-- 2 files changed, 23 insertions(+), 19 deletions(-) diff --git a/integration/test_grid_manager.py b/integration/test_grid_manager.py index ffe6733d1..4910bbfa2 100644 --- a/integration/test_grid_manager.py +++ b/integration/test_grid_manager.py @@ -15,12 +15,12 @@ import pytest_twisted @pytest_twisted.inlineCallbacks -def test_create_certificate(reactor): +def test_create_certificate(reactor, request): """ The Grid Manager produces a valid, correctly-signed certificate. """ gm_config = yield util.run_tahoe( - reactor, "grid-manager", "--config", "-", "create", + reactor, request, "grid-manager", "--config", "-", "create", ) privkey_bytes = json.loads(gm_config)['private_key'].encode('ascii') privkey, pubkey = ed25519.signing_keypair_from_string(privkey_bytes) @@ -29,12 +29,12 @@ def test_create_certificate(reactor): # "actual" clients in the test-grid; we're just checking that the # Grid Manager signs this properly. gm_config = yield util.run_tahoe( - reactor, "grid-manager", "--config", "-", "add", + reactor, request, "grid-manager", "--config", "-", "add", "zara", "pub-v0-kzug3ut2m7ziihf3ndpqlquuxeie4foyl36wn54myqc4wmiwe4ga", stdin=gm_config, ) zara_cert_bytes = yield util.run_tahoe( - reactor, "grid-manager", "--config", "-", "sign", "zara", + reactor, request, "grid-manager", "--config", "-", "sign", "zara", stdin=gm_config, ) zara_cert = json.loads(zara_cert_bytes) @@ -48,21 +48,21 @@ def test_create_certificate(reactor): @pytest_twisted.inlineCallbacks -def test_remove_client(reactor): +def test_remove_client(reactor, request): """ A Grid Manager can add and successfully remove a client """ gm_config = yield util.run_tahoe( - reactor, "grid-manager", "--config", "-", "create", + reactor, request, "grid-manager", "--config", "-", "create", ) gm_config = yield util.run_tahoe( - reactor, "grid-manager", "--config", "-", "add", + reactor, request, "grid-manager", "--config", "-", "add", "zara", "pub-v0-kzug3ut2m7ziihf3ndpqlquuxeie4foyl36wn54myqc4wmiwe4ga", stdin=gm_config, ) gm_config = yield util.run_tahoe( - reactor, "grid-manager", "--config", "-", "add", + reactor, request, "grid-manager", "--config", "-", "add", "yakov", "pub-v0-kvxhb3nexybmipkrar2ztfrwp4uxxsmrjzkpzafit3ket4u5yldq", stdin=gm_config, ) @@ -70,7 +70,7 @@ def test_remove_client(reactor): assert "yakov" in json.loads(gm_config)['storage_servers'] gm_config = yield util.run_tahoe( - reactor, "grid-manager", "--config", "-", "remove", + reactor, request, "grid-manager", "--config", "-", "remove", "zara", stdin=gm_config, ) @@ -79,23 +79,23 @@ def test_remove_client(reactor): @pytest_twisted.inlineCallbacks -def test_remove_last_client(reactor): +def test_remove_last_client(reactor, request): """ A Grid Manager can remove all clients """ gm_config = yield util.run_tahoe( - reactor, "grid-manager", "--config", "-", "create", + reactor, request, "grid-manager", "--config", "-", "create", ) gm_config = yield util.run_tahoe( - reactor, "grid-manager", "--config", "-", "add", + reactor, request, "grid-manager", "--config", "-", "add", "zara", "pub-v0-kzug3ut2m7ziihf3ndpqlquuxeie4foyl36wn54myqc4wmiwe4ga", stdin=gm_config, ) assert "zara" in json.loads(gm_config)['storage_servers'] gm_config = yield util.run_tahoe( - reactor, "grid-manager", "--config", "-", "remove", + reactor, request, "grid-manager", "--config", "-", "remove", "zara", stdin=gm_config, ) @@ -111,7 +111,7 @@ def test_reject_storage_server(reactor, request, storage_nodes, temp_dir, introd valid certificates. """ gm_config = yield util.run_tahoe( - reactor, "grid-manager", "--config", "-", "create", + reactor, request, "grid-manager", "--config", "-", "create", ) privkey_bytes = json.loads(gm_config)['private_key'].encode('ascii') privkey, _ = ed25519.signing_keypair_from_string(privkey_bytes) @@ -142,7 +142,7 @@ def test_reject_storage_server(reactor, request, storage_nodes, temp_dir, introd for idx, storage in enumerate(storage_nodes[:2]): print(idx, storage) cert = yield util.run_tahoe( - reactor, "grid-manager", "--config", "-", "sign", + reactor, request, "grid-manager", "--config", "-", "sign", "storage{}".format(idx), stdin=gm_config, ) @@ -183,9 +183,9 @@ def test_reject_storage_server(reactor, request, storage_nodes, temp_dir, introd try: yield util.run_tahoe( - reactor, "--node-directory", carol._node_dir, + reactor, request, "--node-directory", carol._node_dir, "put", "-", - stdin="some content" * 200, + stdin="some content\n" * 200, ) assert False, "Should get a failure" except util.ProcessFailed as e: diff --git a/integration/util.py b/integration/util.py index d85050d4e..0ad02aed9 100644 --- a/integration/util.py +++ b/integration/util.py @@ -151,10 +151,14 @@ def _cleanup_tahoe_process(tahoe_transport, exited): pass -def run_tahoe(reactor, request, stdin=None, *args): +def run_tahoe(reactor, request, *args, **kwargs): """ - Helper to run tahoe with optional coverage + Helper to run tahoe with optional coverage. + + :returns: a Deferred that fires when the command is done (or a + ProcessFailed exception if it exits non-zero) """ + stdin = kwargs.get("stdin", None) protocol = _CollectOutputProtocol(stdin=stdin) process = _tahoe_runner_optional_coverage(protocol, reactor, request, args) process.exited = protocol.done From fea9fc03d0d73b24658f6334027f85a90bf071a8 Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 7 May 2020 22:29:31 -0600 Subject: [PATCH 0094/2309] fix more keyutil -> ed25519 --- integration/test_grid_manager.py | 6 +++--- src/allmydata/scripts/tahoe_grid_manager.py | 18 ++++++------------ 2 files changed, 9 insertions(+), 15 deletions(-) diff --git a/integration/test_grid_manager.py b/integration/test_grid_manager.py index 4910bbfa2..0f3070f91 100644 --- a/integration/test_grid_manager.py +++ b/integration/test_grid_manager.py @@ -120,11 +120,11 @@ def test_reject_storage_server(reactor, request, storage_nodes, temp_dir, introd for idx, storage in enumerate(storage_nodes[:2]): pubkey_fname = join(storage._node_dir, "node.pubkey") with open(pubkey_fname, 'r') as f: - pubkey = f.read().strip() + pubkey_str = f.read().strip() gm_config = yield util.run_tahoe( - reactor, "grid-manager", "--config", "-", "add", - "storage{}".format(idx), pubkey, + reactor, request, "grid-manager", "--config", "-", "add", + "storage{}".format(idx), pubkey_str, stdin=gm_config, ) assert sorted(json.loads(gm_config)['storage_servers'].keys()) == ['storage0', 'storage1'] diff --git a/src/allmydata/scripts/tahoe_grid_manager.py b/src/allmydata/scripts/tahoe_grid_manager.py index 6c40f0814..4403b9f48 100644 --- a/src/allmydata/scripts/tahoe_grid_manager.py +++ b/src/allmydata/scripts/tahoe_grid_manager.py @@ -5,8 +5,6 @@ import json import time from datetime import datetime -from pycryptopp.publickey import ed25519 # perhaps NaCl instead? other code uses this though - from allmydata.scripts.common import BaseOptions from allmydata.util.abbreviate import abbreviate_time from twisted.python import usage @@ -171,7 +169,7 @@ class _GridManagerStorageServer(object): self._certificates.append(certificate) def public_key(self): - return "pub-v0-" + base32.b2a(self._public_key.vk_bytes) + return ed25519.string_from_verifying_key(self._public_key) def marshal(self): return { @@ -228,7 +226,7 @@ class _GridManager(object): def __init__(self, private_key_bytes, storage_servers): self._storage_servers = dict() if storage_servers is None else storage_servers self._private_key_bytes = private_key_bytes - self._private_key, _ = ed25519.signing_keypair_from_string(self._private_key_bytes) + self._private_key, self._public_key = ed25519.signing_keypair_from_string(self._private_key_bytes) self._version = 0 @property @@ -236,8 +234,7 @@ class _GridManager(object): return self._storage_servers def public_identity(self): - verify_key_bytes = self._private_key.get_verifying_key_bytes() - return base32.b2a(verify_key_bytes) + return ed25519.string_from_verifying_key(self._public_key) def sign(self, name): try: @@ -258,10 +255,8 @@ class _GridManager(object): u"signature": base32.b2a(sig), } - if True: - verify_key_bytes = self._private_key.get_verifying_key_bytes() - vk = ed25519.VerifyingKey(verify_key_bytes) - assert vk.verify(sig, cert_data) is None, "cert should verify" + vk = ed25519.verifying_key_from_signing_key(self._private_key) + assert vk.verify(sig, cert_data) is None, "cert should verify" return certificate @@ -276,7 +271,6 @@ class _GridManager(object): raise KeyError( "Already have a storage server called '{}'".format(name) ) - assert public_key.vk_bytes ss = _GridManagerStorageServer(name, public_key, None) self._storage_servers[name] = ss return ss @@ -374,7 +368,7 @@ def _show_identity(gridoptions, options): assert gm_config is not None gm = _load_gridmanager_config(gm_config) - print("pub-v0-{}".format(gm.public_identity())) + print(gm.public_identity()) def _add(gridoptions, options): From 5bd5d8145e459bcf1d02c8d7ec18a82a01ed1af8 Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 7 May 2020 22:29:45 -0600 Subject: [PATCH 0095/2309] kill process --- integration/test_grid_manager.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/integration/test_grid_manager.py b/integration/test_grid_manager.py index 0f3070f91..448c641c0 100644 --- a/integration/test_grid_manager.py +++ b/integration/test_grid_manager.py @@ -155,8 +155,8 @@ def test_reject_storage_server(reactor, request, storage_nodes, temp_dir, introd config.write(open(join(storage._node_dir, "tahoe.cfg"), "w")) # re-start this storage server - storage.signalProcess('TERM') - yield storage._protocol.exited + storage.transport.signalProcess('TERM') + yield storage.transport._protocol.exited time.sleep(1) storage_nodes[idx] = yield util._run_node( reactor, storage._node_dir, request, None, @@ -166,12 +166,11 @@ def test_reject_storage_server(reactor, request, storage_nodes, temp_dir, introd # carol to have the grid-manager certificate config = configutil.get_config(join(carol._node_dir, "tahoe.cfg")) - print(dir(config)) config.add_section("grid_managers") - config.set("grid_managers", "test", ed25519.string_from_verifying_key(pubkey)) + config.set("grid_managers", "test", pubkey_str) config.write(open(join(carol._node_dir, "tahoe.cfg"), "w")) - carol.signalProcess('TERM') - yield carol._protocol.exited + carol.transport.signalProcess('TERM') + yield carol.transport._protocol.exited carol = yield util._run_node( reactor, carol._node_dir, request, None, From 0bdfae845ed623f5e926dbef1041a6606b4f6bb9 Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 7 May 2020 22:30:05 -0600 Subject: [PATCH 0096/2309] actually put grid-manager-certificates in announcement --- src/allmydata/client.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/allmydata/client.py b/src/allmydata/client.py index 36f870a64..8e063a1f5 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -943,10 +943,9 @@ class _Client(node.Node, pollmixin.PollMixin): announcement.update(plugins_announcement) - grid_manager_certificates = [] - if self.config.get_config("storage", "grid_management", default=False, boolean=True): grid_manager_certificates = _load_grid_manager_certificates(self.config) + announcement["grid-manager-certificates"] = grid_manager_certificates # XXX we should probably verify that the certificates are # valid and not expired, as that could be confusing for the From 38e6557f817db2bf6819964cc1b75ff73be42f20 Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 7 May 2020 22:30:23 -0600 Subject: [PATCH 0097/2309] fix imports --- src/allmydata/storage_client.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 7d8df34c7..09da110db 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -29,7 +29,11 @@ the foolscap-based server implemented in src/allmydata/storage/*.py . # 6: implement other sorts of IStorageClient classes: S3, etc -import re, time, hashlib +import re +import time +import json +import hashlib +from datetime import datetime from ConfigParser import ( NoSectionError, ) From 975491b519d0f3f8c7b840726daf4fadd73b64d0 Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 7 May 2020 22:30:35 -0600 Subject: [PATCH 0098/2309] allow 'anything valid' sections --- src/allmydata/util/configutil.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/allmydata/util/configutil.py b/src/allmydata/util/configutil.py index d58bc4217..ae2a568f5 100644 --- a/src/allmydata/util/configutil.py +++ b/src/allmydata/util/configutil.py @@ -95,8 +95,11 @@ class ValidConfiguration(object): :return: True if the given section name, ite name pair is valid, False otherwise. """ + valid_items = self._static_valid_sections.get(section_name, ()) + if valid_items is None: + return True return ( - item_name in self._static_valid_sections.get(section_name, ()) or + item_name in valid_items or self._is_valid_item(section_name, item_name) ) From c029698435ecb05129e6f5fa549a179ee08a3f00 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 8 May 2020 00:47:17 -0600 Subject: [PATCH 0099/2309] fix more keyutil things and key-handling in test --- integration/test_grid_manager.py | 15 ++++++++------- integration/util.py | 6 +++--- src/allmydata/scripts/tahoe_grid_manager.py | 4 ++-- src/allmydata/storage_client.py | 6 ++++-- 4 files changed, 17 insertions(+), 14 deletions(-) diff --git a/integration/test_grid_manager.py b/integration/test_grid_manager.py index 448c641c0..ba0debd5c 100644 --- a/integration/test_grid_manager.py +++ b/integration/test_grid_manager.py @@ -113,8 +113,8 @@ def test_reject_storage_server(reactor, request, storage_nodes, temp_dir, introd gm_config = yield util.run_tahoe( reactor, request, "grid-manager", "--config", "-", "create", ) - privkey_bytes = json.loads(gm_config)['private_key'].encode('ascii') - privkey, _ = ed25519.signing_keypair_from_string(privkey_bytes) + gm_privkey_bytes = json.loads(gm_config)['private_key'].encode('ascii') + gm_privkey, gm_pubkey = ed25519.signing_keypair_from_string(gm_privkey_bytes) # create certificates for first 2 storage-servers for idx, storage in enumerate(storage_nodes[:2]): @@ -140,7 +140,6 @@ def test_reject_storage_server(reactor, request, storage_nodes, temp_dir, introd print("inserting certificates") # insert their certificates for idx, storage in enumerate(storage_nodes[:2]): - print(idx, storage) cert = yield util.run_tahoe( reactor, request, "grid-manager", "--config", "-", "sign", "storage{}".format(idx), @@ -152,12 +151,12 @@ def test_reject_storage_server(reactor, request, storage_nodes, temp_dir, introd config.set("storage", "grid_management", "True") config.add_section("grid_manager_certificates") config.set("grid_manager_certificates", "default", "gridmanager.cert") - config.write(open(join(storage._node_dir, "tahoe.cfg"), "w")) + with open(join(storage._node_dir, "tahoe.cfg"), "w") as f: + config.write(f) # re-start this storage server storage.transport.signalProcess('TERM') yield storage.transport._protocol.exited - time.sleep(1) storage_nodes[idx] = yield util._run_node( reactor, storage._node_dir, request, None, ) @@ -167,14 +166,16 @@ def test_reject_storage_server(reactor, request, storage_nodes, temp_dir, introd config = configutil.get_config(join(carol._node_dir, "tahoe.cfg")) config.add_section("grid_managers") - config.set("grid_managers", "test", pubkey_str) - config.write(open(join(carol._node_dir, "tahoe.cfg"), "w")) + config.set("grid_managers", "test", ed25519.string_from_verifying_key(gm_pubkey)) + with open(join(carol._node_dir, "tahoe.cfg"), "w") as f: + config.write(f) carol.transport.signalProcess('TERM') yield carol.transport._protocol.exited carol = yield util._run_node( reactor, carol._node_dir, request, None, ) + yield util.await_client_ready(carol, servers=5) # try to put something into the grid, which should fail (because # carol has happy=3 but should only find storage0, storage1 to be diff --git a/integration/util.py b/integration/util.py index 0ad02aed9..7178e1d67 100644 --- a/integration/util.py +++ b/integration/util.py @@ -478,7 +478,7 @@ def web_post(tahoe, uri_fragment, **kwargs): return resp.content -def await_client_ready(tahoe, timeout=10, liveness=60*2): +def await_client_ready(tahoe, timeout=10, liveness=60*2, servers=1): """ Uses the status API to wait for a client-type node (in `tahoe`, a `TahoeProcess` instance usually from a fixture e.g. `alice`) to be @@ -502,8 +502,8 @@ def await_client_ready(tahoe, timeout=10, liveness=60*2): time.sleep(1) continue - if len(js['servers']) == 0: - print("waiting because no servers at all") + if len(js['servers']) < servers: + print("waiting because fewer than {} server(s)".format(servers)) time.sleep(1) continue server_times = [ diff --git a/src/allmydata/scripts/tahoe_grid_manager.py b/src/allmydata/scripts/tahoe_grid_manager.py index 4403b9f48..64561b95e 100644 --- a/src/allmydata/scripts/tahoe_grid_manager.py +++ b/src/allmydata/scripts/tahoe_grid_manager.py @@ -249,14 +249,14 @@ class _GridManager(object): "version": 1, } cert_data = json.dumps(cert_info, separators=(',',':'), sort_keys=True).encode('utf8') - sig = self._private_key.sign(cert_data) + sig = ed25519.sign_data(self._private_key, cert_data) certificate = { u"certificate": cert_data, u"signature": base32.b2a(sig), } vk = ed25519.verifying_key_from_signing_key(self._private_key) - assert vk.verify(sig, cert_data) is None, "cert should verify" + ed25519.verify_signature(vk, sig, cert_data) return certificate diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 09da110db..523dfa066 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -67,6 +67,7 @@ from allmydata.util.assertutil import precondition from allmydata.util.observer import ObserverList from allmydata.util.rrefutil import add_version_to_remote_reference from allmydata.util.hashutil import permute_server_hash +from allmydata.crypto import ed25519 # who is responsible for de-duplication? # both? @@ -473,11 +474,12 @@ def validate_grid_manager_certificate(gm_key, alleged_cert, now_fn=None): now_fn = datetime.utcnow try: - gm_key.verify( + ed25519.verify_signature( + gm_key, base32.a2b(alleged_cert['signature'].encode('ascii')), alleged_cert['certificate'].encode('ascii'), ) - except ed25519.BadSignatureError: + except ed25519.BadSignature: return False # signature is valid; now we can load the actual data cert = json.loads(alleged_cert['certificate']) From 0540df8ab1af6f04dd97b4b4cfbfede48b8e89cb Mon Sep 17 00:00:00 2001 From: meejah Date: Sun, 10 May 2020 00:56:52 -0600 Subject: [PATCH 0100/2309] Re-factor grid fixtures to be mostly helpers This lets us use them to create our own tiny grid to test the grid-manager certificates (instead of messing with "the" grid) --- integration/conftest.py | 121 ++------------ integration/grid.py | 262 +++++++++++++++++++++++++++++++ integration/test_grid_manager.py | 124 ++++++++------- integration/test_web.py | 10 +- integration/util.py | 9 +- 5 files changed, 352 insertions(+), 174 deletions(-) create mode 100644 integration/grid.py diff --git a/integration/conftest.py b/integration/conftest.py index 5395d7c5f..462ac8b60 100644 --- a/integration/conftest.py +++ b/integration/conftest.py @@ -36,6 +36,7 @@ from util import ( await_client_ready, TahoeProcess, ) +import grid # pytest customization hooks @@ -106,60 +107,10 @@ def flog_binary(): @pytest.fixture(scope='session') @log_call(action_type=u"integration:flog_gatherer", include_args=[]) def flog_gatherer(reactor, temp_dir, flog_binary, request): - out_protocol = _CollectOutputProtocol() - gather_dir = join(temp_dir, 'flog_gather') - reactor.spawnProcess( - out_protocol, - flog_binary, - ( - 'flogtool', 'create-gatherer', - '--location', 'tcp:localhost:3117', - '--port', '3117', - gather_dir, - ) + fg = pytest_twisted.blockon( + grid.create_flog_gatherer(reactor, request, temp_dir, flog_binary) ) - pytest_twisted.blockon(out_protocol.done) - - twistd_protocol = _MagicTextProtocol("Gatherer waiting at") - twistd_process = reactor.spawnProcess( - twistd_protocol, - which('twistd')[0], - ( - 'twistd', '--nodaemon', '--python', - join(gather_dir, 'gatherer.tac'), - ), - path=gather_dir, - ) - pytest_twisted.blockon(twistd_protocol.magic_seen) - - def cleanup(): - _cleanup_tahoe_process(twistd_process, twistd_protocol.exited) - - flog_file = mktemp('.flog_dump') - flog_protocol = _DumpOutputProtocol(open(flog_file, 'w')) - flog_dir = join(temp_dir, 'flog_gather') - flogs = [x for x in listdir(flog_dir) if x.endswith('.flog')] - - print("Dumping {} flogtool logfiles to '{}'".format(len(flogs), flog_file)) - reactor.spawnProcess( - flog_protocol, - flog_binary, - ( - 'flogtool', 'dump', join(temp_dir, 'flog_gather', flogs[0]) - ), - ) - print("Waiting for flogtool to complete") - try: - pytest_twisted.blockon(flog_protocol.done) - except ProcessTerminated as e: - print("flogtool exited unexpectedly: {}".format(str(e))) - print("Flogtool completed") - - request.addfinalizer(cleanup) - - with open(join(gather_dir, 'log_gatherer.furl'), 'r') as f: - furl = f.read().strip() - return furl + return fg @pytest.fixture(scope='session') @@ -169,64 +120,14 @@ def flog_gatherer(reactor, temp_dir, flog_binary, request): include_result=False, ) def introducer(reactor, temp_dir, flog_gatherer, request): - config = ''' -[node] -nickname = introducer0 -web.port = 4560 -log_gatherer.furl = {log_furl} -'''.format(log_furl=flog_gatherer) - - intro_dir = join(temp_dir, 'introducer') - print("making introducer", intro_dir) - - if not exists(intro_dir): - mkdir(intro_dir) - done_proto = _ProcessExitedProtocol() - _tahoe_runner_optional_coverage( - done_proto, - reactor, - request, - ( - 'create-introducer', - '--listen=tcp', - '--hostname=localhost', - intro_dir, - ), - ) - pytest_twisted.blockon(done_proto.done) - - # over-write the config file with our stuff - with open(join(intro_dir, 'tahoe.cfg'), 'w') as f: - f.write(config) - - # on windows, "tahoe start" means: run forever in the foreground, - # but on linux it means daemonize. "tahoe run" is consistent - # between platforms. - protocol = _MagicTextProtocol('introducer running') - transport = _tahoe_runner_optional_coverage( - protocol, - reactor, - request, - ( - 'run', - intro_dir, - ), - ) - request.addfinalizer(partial(_cleanup_tahoe_process, transport, protocol.exited)) - - pytest_twisted.blockon(protocol.magic_seen) - return TahoeProcess(transport, intro_dir) + intro = pytest_twisted.blockon(grid.create_introducer(reactor, request, temp_dir, flog_gatherer)) + return intro @pytest.fixture(scope='session') @log_call(action_type=u"integration:introducer:furl", include_args=["temp_dir"]) def introducer_furl(introducer, temp_dir): - furl_fname = join(temp_dir, 'introducer', 'private', 'introducer.furl') - while not exists(furl_fname): - print("Don't see {} yet".format(furl_fname)) - sleep(.1) - furl = open(furl_fname, 'r').read() - return furl + return introducer.furl @pytest.fixture(scope='session') @@ -313,12 +214,10 @@ def storage_nodes(reactor, temp_dir, introducer, introducer_furl, flog_gatherer, # start all 5 nodes in parallel for x in range(5): name = 'node{}'.format(x) - web_port= 9990 + x + web_port = 'tcp:{}:interface=localhost'.format(9990 + x) nodes_d.append( - _create_node( - reactor, request, temp_dir, introducer_furl, flog_gatherer, name, - web_port="tcp:{}:interface=localhost".format(web_port), - storage=True, + grid.create_storage_server( + reactor, request, temp_dir, introducer, flog_gatherer, name, web_port, ) ) nodes_status = pytest_twisted.blockon(DeferredList(nodes_d)) diff --git a/integration/grid.py b/integration/grid.py new file mode 100644 index 000000000..7df86d248 --- /dev/null +++ b/integration/grid.py @@ -0,0 +1,262 @@ +from os import mkdir, listdir, environ +from os.path import join, exists +from tempfile import mkdtemp, mktemp + +from twisted.python.procutils import which +from twisted.internet.defer import ( + inlineCallbacks, + returnValue, +) +from twisted.internet.task import ( + deferLater, +) +from twisted.internet.interfaces import ( + IProcessTransport, + IProcessProtocol, + IProtocol, +) + +from util import ( + _CollectOutputProtocol, + _MagicTextProtocol, + _DumpOutputProtocol, + _ProcessExitedProtocol, + _create_node, + _run_node, + _cleanup_tahoe_process, + _tahoe_runner_optional_coverage, + await_client_ready, + TahoeProcess, +) + +import attr +import pytest_twisted + + +@attr.s +class FlogGatherer(object): + """ + Flog Gatherer process. + """ + + process = attr.ib( + validator=attr.validators.provides(IProcessTransport) + ) + protocol = attr.ib( + validator=attr.validators.provides(IProcessProtocol) + ) + furl = attr.ib() + + +@inlineCallbacks +def create_flog_gatherer(reactor, request, temp_dir, flog_binary): + out_protocol = _CollectOutputProtocol() + gather_dir = join(temp_dir, 'flog_gather') + reactor.spawnProcess( + out_protocol, + flog_binary, + ( + 'flogtool', 'create-gatherer', + '--location', 'tcp:localhost:3117', + '--port', '3117', + gather_dir, + ) + ) + yield out_protocol.done + + twistd_protocol = _MagicTextProtocol("Gatherer waiting at") + twistd_process = reactor.spawnProcess( + twistd_protocol, + which('twistd')[0], + ( + 'twistd', '--nodaemon', '--python', + join(gather_dir, 'gatherer.tac'), + ), + path=gather_dir, + ) + yield twistd_protocol.magic_seen + + def cleanup(): + _cleanup_tahoe_process(twistd_process, twistd_protocol.exited) + + flog_file = mktemp('.flog_dump') + flog_protocol = _DumpOutputProtocol(open(flog_file, 'w')) + flog_dir = join(temp_dir, 'flog_gather') + flogs = [x for x in listdir(flog_dir) if x.endswith('.flog')] + + print("Dumping {} flogtool logfiles to '{}'".format(len(flogs), flog_file)) + reactor.spawnProcess( + flog_protocol, + flog_binary, + ( + 'flogtool', 'dump', join(temp_dir, 'flog_gather', flogs[0]) + ), + ) + print("Waiting for flogtool to complete") + try: + pytest_twisted.blockon(flog_protocol.done) + except ProcessTerminated as e: + print("flogtool exited unexpectedly: {}".format(str(e))) + print("Flogtool completed") + + request.addfinalizer(cleanup) + + with open(join(gather_dir, 'log_gatherer.furl'), 'r') as f: + furl = f.read().strip() + returnValue( + FlogGatherer( + protocol=twistd_protocol, + process=twistd_process, + furl=furl, + ) + ) + + +@attr.s +class StorageServer(object): + """ + Represents a Tahoe Storage Server + """ + + process = attr.ib( + validator=attr.validators.instance_of(TahoeProcess) + ) + protocol = attr.ib( + validator=attr.validators.provides(IProcessProtocol) + ) + + # XXX needs a restart() probably .. or at least a stop() and + # start() + + +@inlineCallbacks +def create_storage_server(reactor, request, temp_dir, introducer, flog_gatherer, name, web_port, + needed=2, happy=3, total=4): + """ + Create a new storage server + """ + from util import _create_node + node_process = yield _create_node( + reactor, request, temp_dir, introducer.furl, flog_gatherer, + name, web_port, storage=True, needed=needed, happy=happy, total=total, + ) + storage = StorageServer( + process=node_process, + protocol=node_process.transport._protocol, + ) + returnValue(storage) + +@attr.s +class Introducer(object): + """ + Reprsents a running introducer + """ + + process = attr.ib( + validator=attr.validators.instance_of(TahoeProcess) + ) + protocol = attr.ib( + validator=attr.validators.provides(IProcessProtocol) + ) + furl = attr.ib() + + +_introducer_num = 0 + + +@inlineCallbacks +def create_introducer(reactor, request, temp_dir, flog_gatherer): + """ + Run a new Introducer and return an Introducer instance. + """ + global _introducer_num + config = ( + '[node]\n' + 'nickname = introducer{num}\n' + 'web.port = {port}\n' + 'log_gatherer.furl = {log_furl}\n' + ).format( + num=_introducer_num, + log_furl=flog_gatherer.furl, + port=4560 + _introducer_num, + ) + _introducer_num += 1 + + intro_dir = join(temp_dir, 'introducer{}'.format(_introducer_num)) + print("making introducer", intro_dir, _introducer_num) + + if not exists(intro_dir): + mkdir(intro_dir) + done_proto = _ProcessExitedProtocol() + _tahoe_runner_optional_coverage( + done_proto, + reactor, + request, + ( + 'create-introducer', + '--listen=tcp', + '--hostname=localhost', + intro_dir, + ), + ) + yield done_proto.done + + # over-write the config file with our stuff + with open(join(intro_dir, 'tahoe.cfg'), 'w') as f: + f.write(config) + + # on windows, "tahoe start" means: run forever in the foreground, + # but on linux it means daemonize. "tahoe run" is consistent + # between platforms. + protocol = _MagicTextProtocol('introducer running') + transport = _tahoe_runner_optional_coverage( + protocol, + reactor, + request, + ( + 'run', + intro_dir, + ), + ) + + def clean(): + return _cleanup_tahoe_process(transport, protocol.exited) + request.addfinalizer(clean) + + yield protocol.magic_seen + + furl_fname = join(intro_dir, 'private', 'introducer.furl') + while not exists(furl_fname): + print("Don't see {} yet".format(furl_fname)) + yield deferLater(reactor, .1, lambda: None) + furl = open(furl_fname, 'r').read() + + returnValue( + Introducer( + process=TahoeProcess(transport, intro_dir), + protocol=protocol, + furl=furl, + ) + ) + + +@attr.s +class Grid(object): + """ + Represents an entire Tahoe Grid setup + + A Grid includes an Introducer, Flog Gatherer and some number of + Storage Servers. + """ + + introducer = attr.ib(default=None) + flog_gatherer = attr.ib(default=None) + storage_servers = attr.ib(factory=list) + + @storage_servers.validator + def check(self, attribute, value): + for server in value: + if not isinstance(server, StorageServer): + raise ValueError( + "storage_servers must be StorageServer" + ) diff --git a/integration/test_grid_manager.py b/integration/test_grid_manager.py index ba0debd5c..d0fdcdd31 100644 --- a/integration/test_grid_manager.py +++ b/integration/test_grid_manager.py @@ -104,86 +104,102 @@ def test_remove_last_client(reactor, request): @pytest_twisted.inlineCallbacks -def test_reject_storage_server(reactor, request, storage_nodes, temp_dir, introducer_furl, flog_gatherer): +def test_reject_storage_server(reactor, request, temp_dir, flog_gatherer): """ - A client with happines=3 fails to upload to a Grid when it is - using Grid Manager and there are only two storage-servers with - valid certificates. + A client with happines=2 fails to upload to a Grid when it is + using Grid Manager and there is only 1 storage server with a valid + certificate. """ + import grid + introducer = yield grid.create_introducer(reactor, request, temp_dir, flog_gatherer) + storage0 = yield grid.create_storage_server( + reactor, request, temp_dir, introducer, flog_gatherer, + name="gm_storage0", + web_port="tcp:9995:interface=localhost", + needed=2, + happy=2, + total=2, + ) + storage1 = yield grid.create_storage_server( + reactor, request, temp_dir, introducer, flog_gatherer, + name="gm_storage1", + web_port="tcp:9996:interface=localhost", + needed=2, + happy=2, + total=2, + ) + gm_config = yield util.run_tahoe( reactor, request, "grid-manager", "--config", "-", "create", ) gm_privkey_bytes = json.loads(gm_config)['private_key'].encode('ascii') gm_privkey, gm_pubkey = ed25519.signing_keypair_from_string(gm_privkey_bytes) - # create certificates for first 2 storage-servers - for idx, storage in enumerate(storage_nodes[:2]): - pubkey_fname = join(storage._node_dir, "node.pubkey") - with open(pubkey_fname, 'r') as f: - pubkey_str = f.read().strip() + # create certificate for the first storage-server + pubkey_fname = join(storage0.process.node_dir, "node.pubkey") + with open(pubkey_fname, 'r') as f: + pubkey_str = f.read().strip() - gm_config = yield util.run_tahoe( - reactor, request, "grid-manager", "--config", "-", "add", - "storage{}".format(idx), pubkey_str, - stdin=gm_config, - ) - assert sorted(json.loads(gm_config)['storage_servers'].keys()) == ['storage0', 'storage1'] + gm_config = yield util.run_tahoe( + reactor, request, "grid-manager", "--config", "-", "add", + "storage0", pubkey_str, + stdin=gm_config, + ) + assert json.loads(gm_config)['storage_servers'].keys() == ['storage0'] - # XXX FIXME need to shut-down and nuke carol when we're done this - # test (i.d. request.addfinalizer) - carol = yield util._create_node( - reactor, request, temp_dir, introducer_furl, flog_gatherer, "carol", - web_port="tcp:9982:interface=localhost", + # XXX FIXME want a grid.create_client() or similar + diana = yield util._create_node( + reactor, request, temp_dir, introducer.furl, flog_gatherer, "diana", + web_port="tcp:9984:interface=localhost", storage=False, ) - print("inserting certificates") - # insert their certificates - for idx, storage in enumerate(storage_nodes[:2]): - cert = yield util.run_tahoe( - reactor, request, "grid-manager", "--config", "-", "sign", - "storage{}".format(idx), - stdin=gm_config, - ) - with open(join(storage._node_dir, "gridmanager.cert"), "w") as f: - f.write(cert) - config = configutil.get_config(join(storage._node_dir, "tahoe.cfg")) - config.set("storage", "grid_management", "True") - config.add_section("grid_manager_certificates") - config.set("grid_manager_certificates", "default", "gridmanager.cert") - with open(join(storage._node_dir, "tahoe.cfg"), "w") as f: - config.write(f) + print("inserting certificate") + cert = yield util.run_tahoe( + reactor, request, "grid-manager", "--config", "-", "sign", "storage0", + stdin=gm_config, + ) + with open(join(storage0.process.node_dir, "gridmanager.cert"), "w") as f: + f.write(cert) + config = configutil.get_config(join(storage0.process.node_dir, "tahoe.cfg")) + config.set("storage", "grid_management", "True") + config.add_section("grid_manager_certificates") + config.set("grid_manager_certificates", "default", "gridmanager.cert") + with open(join(storage0.process.node_dir, "tahoe.cfg"), "w") as f: + config.write(f) - # re-start this storage server - storage.transport.signalProcess('TERM') - yield storage.transport._protocol.exited - storage_nodes[idx] = yield util._run_node( - reactor, storage._node_dir, request, None, - ) + # re-start this storage server + storage0.process.transport.signalProcess('TERM') + yield storage0.protocol.exited + yield util._run_node( + reactor, storage0.process.node_dir, request, None, cleanup=False, + ) - # now only two storage-servers have certificates .. configure - # carol to have the grid-manager certificate + yield util.await_client_ready(diana, servers=2) - config = configutil.get_config(join(carol._node_dir, "tahoe.cfg")) + # now only one storage-server has the certificate .. configure + # diana to have the grid-manager certificate + + config = configutil.get_config(join(diana.node_dir, "tahoe.cfg")) config.add_section("grid_managers") config.set("grid_managers", "test", ed25519.string_from_verifying_key(gm_pubkey)) - with open(join(carol._node_dir, "tahoe.cfg"), "w") as f: + with open(join(diana.node_dir, "tahoe.cfg"), "w") as f: config.write(f) - carol.transport.signalProcess('TERM') - yield carol.transport._protocol.exited + diana.transport.signalProcess('TERM') + yield diana.transport._protocol.exited - carol = yield util._run_node( - reactor, carol._node_dir, request, None, + diana = yield util._run_node( + reactor, diana._node_dir, request, None, cleanup=False, ) - yield util.await_client_ready(carol, servers=5) + yield util.await_client_ready(diana, servers=2) # try to put something into the grid, which should fail (because - # carol has happy=3 but should only find storage0, storage1 to be - # acceptable to upload to) + # diana has happy=2 but should only find storage0 to be acceptable + # to upload to) try: yield util.run_tahoe( - reactor, request, "--node-directory", carol._node_dir, + reactor, request, "--node-directory", diana._node_dir, "put", "-", stdin="some content\n" * 200, ) diff --git a/integration/test_web.py b/integration/test_web.py index 4ba0a6fd1..ac7f5f285 100644 --- a/integration/test_web.py +++ b/integration/test_web.py @@ -96,7 +96,7 @@ def test_helper_status(storage_nodes): successfully GET the /helper_status page """ - url = util.node_url(storage_nodes[0].node_dir, "helper_status") + url = util.node_url(storage_nodes[0].process.node_dir, "helper_status") resp = requests.get(url) assert resp.status_code >= 200 and resp.status_code < 300 dom = BeautifulSoup(resp.content, "html5lib") @@ -418,7 +418,7 @@ def test_storage_info(storage_nodes): storage0 = storage_nodes[0] requests.get( - util.node_url(storage0.node_dir, u"storage"), + util.node_url(storage0.process.node_dir, u"storage"), ) @@ -429,7 +429,7 @@ def test_storage_info_json(storage_nodes): storage0 = storage_nodes[0] resp = requests.get( - util.node_url(storage0.node_dir, u"storage"), + util.node_url(storage0.process.node_dir, u"storage"), params={u"t": u"json"}, ) data = json.loads(resp.content) @@ -441,12 +441,12 @@ def test_introducer_info(introducer): retrieve and confirm /introducer URI for the introducer """ resp = requests.get( - util.node_url(introducer.node_dir, u""), + util.node_url(introducer.process.node_dir, u""), ) assert "Introducer" in resp.content resp = requests.get( - util.node_url(introducer.node_dir, u""), + util.node_url(introducer.process.node_dir, u""), params={u"t": u"json"}, ) data = json.loads(resp.content) diff --git a/integration/util.py b/integration/util.py index 7178e1d67..f5c3f3462 100644 --- a/integration/util.py +++ b/integration/util.py @@ -144,7 +144,7 @@ def _cleanup_tahoe_process(tahoe_transport, exited): try: print("signaling {} with TERM".format(tahoe_transport.pid)) tahoe_transport.signalProcess('TERM') - print("signaled, blocking on exit") + print("signaled, blocking on exit {}".format(exited)) pytest_twisted.blockon(exited) print("exited, goodbye") except ProcessExitedAlready: @@ -210,7 +210,7 @@ class TahoeProcess(object): return "".format(self._node_dir) -def _run_node(reactor, node_dir, request, magic_text): +def _run_node(reactor, node_dir, request, magic_text, cleanup=True): """ Run a tahoe process from its node_dir. @@ -236,7 +236,8 @@ def _run_node(reactor, node_dir, request, magic_text): ) transport.exited = protocol.exited - request.addfinalizer(partial(_cleanup_tahoe_process, transport, protocol.exited)) + if cleanup: + request.addfinalizer(partial(_cleanup_tahoe_process, transport, protocol.exited)) # XXX abusing the Deferred; should use .when_magic_seen() pattern @@ -291,7 +292,7 @@ def _create_node(reactor, request, temp_dir, introducer_furl, flog_gatherer, nam def created(_): config_path = join(node_dir, 'tahoe.cfg') config = get_config(config_path) - set_config(config, 'node', 'log_gatherer.furl', flog_gatherer) + set_config(config, 'node', 'log_gatherer.furl', flog_gatherer.furl) write_config(config_path, config) created_d.addCallback(created) From 9a62b1f93f784e213a352a3f7df6cc19939f0792 Mon Sep 17 00:00:00 2001 From: meejah Date: Sun, 10 May 2020 22:05:47 -0600 Subject: [PATCH 0101/2309] cleanup is non-optional --- integration/test_grid_manager.py | 4 ++-- integration/util.py | 5 ++--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/integration/test_grid_manager.py b/integration/test_grid_manager.py index d0fdcdd31..dc25c3b29 100644 --- a/integration/test_grid_manager.py +++ b/integration/test_grid_manager.py @@ -172,7 +172,7 @@ def test_reject_storage_server(reactor, request, temp_dir, flog_gatherer): storage0.process.transport.signalProcess('TERM') yield storage0.protocol.exited yield util._run_node( - reactor, storage0.process.node_dir, request, None, cleanup=False, + reactor, storage0.process.node_dir, request, None, ) yield util.await_client_ready(diana, servers=2) @@ -189,7 +189,7 @@ def test_reject_storage_server(reactor, request, temp_dir, flog_gatherer): yield diana.transport._protocol.exited diana = yield util._run_node( - reactor, diana._node_dir, request, None, cleanup=False, + reactor, diana._node_dir, request, None, ) yield util.await_client_ready(diana, servers=2) diff --git a/integration/util.py b/integration/util.py index f5c3f3462..2cdcb2563 100644 --- a/integration/util.py +++ b/integration/util.py @@ -210,7 +210,7 @@ class TahoeProcess(object): return "".format(self._node_dir) -def _run_node(reactor, node_dir, request, magic_text, cleanup=True): +def _run_node(reactor, node_dir, request, magic_text): """ Run a tahoe process from its node_dir. @@ -236,8 +236,7 @@ def _run_node(reactor, node_dir, request, magic_text, cleanup=True): ) transport.exited = protocol.exited - if cleanup: - request.addfinalizer(partial(_cleanup_tahoe_process, transport, protocol.exited)) + request.addfinalizer(partial(_cleanup_tahoe_process, transport, protocol.exited)) # XXX abusing the Deferred; should use .when_magic_seen() pattern From a9fe12063a6463bd9395f03db5191abf0927de52 Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 11 May 2020 00:20:48 -0600 Subject: [PATCH 0102/2309] docs --- integration/grid.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/integration/grid.py b/integration/grid.py index 7df86d248..726c22e40 100644 --- a/integration/grid.py +++ b/integration/grid.py @@ -1,3 +1,13 @@ +""" +Classes which directly represent various kinds of Tahoe processes +that co-operate to for "a Grid". + +These methods and objects are used by conftest.py fixtures but may +also be used as direct helpers for tests that don't want to (or can't) +rely on 'the' global grid as provided by fixtures like 'alice' or +'storage_servers'. +""" + from os import mkdir, listdir, environ from os.path import join, exists from tempfile import mkdtemp, mktemp @@ -33,6 +43,15 @@ import attr import pytest_twisted +# further directions: +# - "Grid" is unused, basically -- tie into the rest? +# - could make a Grid instance mandatory for create_* calls +# - could instead make create_* calls methods of Grid +# - Bring more 'util' or 'conftest' code into here +# - stop()/start()/restart() methods on StorageServer etc +# - more-complex stuff like config changes (which imply a restart too)? + + @attr.s class FlogGatherer(object): """ From c5fb2b5a8b0f874e4992ca392bc234224d232371 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 12 May 2020 12:22:35 -0600 Subject: [PATCH 0103/2309] refactor more code into grid.py --- integration/conftest.py | 55 +++++---- integration/grid.py | 191 ++++++++++++++++++++++++++++--- integration/test_grid_manager.py | 47 +++----- integration/util.py | 2 +- 4 files changed, 223 insertions(+), 72 deletions(-) diff --git a/integration/conftest.py b/integration/conftest.py index 462ac8b60..11ee61d11 100644 --- a/integration/conftest.py +++ b/integration/conftest.py @@ -36,7 +36,11 @@ from util import ( await_client_ready, TahoeProcess, ) -import grid +from grid import ( + create_port_allocator, + create_flog_gatherer, + create_grid, +) # pytest customization hooks @@ -73,6 +77,12 @@ def reactor(): return _reactor +@pytest.fixture(scope='session') +@log_call(action_type=u"integration:port_allocator", include_result=False) +def port_allocator(reactor): + return create_port_allocator(start_port=45000) + + @pytest.fixture(scope='session') @log_call(action_type=u"integration:temp_dir", include_args=[]) def temp_dir(request): @@ -108,20 +118,23 @@ def flog_binary(): @log_call(action_type=u"integration:flog_gatherer", include_args=[]) def flog_gatherer(reactor, temp_dir, flog_binary, request): fg = pytest_twisted.blockon( - grid.create_flog_gatherer(reactor, request, temp_dir, flog_binary) + create_flog_gatherer(reactor, request, temp_dir, flog_binary) ) return fg @pytest.fixture(scope='session') -@log_call( - action_type=u"integration:introducer", - include_args=["temp_dir", "flog_gatherer"], - include_result=False, -) -def introducer(reactor, temp_dir, flog_gatherer, request): - intro = pytest_twisted.blockon(grid.create_introducer(reactor, request, temp_dir, flog_gatherer)) - return intro +@log_call(action_type=u"integration:grid", include_args=[]) +def grid(reactor, request, temp_dir, flog_gatherer, port_allocator): + g = pytest_twisted.blockon( + create_grid(reactor, request, temp_dir, flog_gatherer, port_allocator) + ) + return g + + +@pytest.fixture(scope='session') +def introducer(grid): + return grid.introducer @pytest.fixture(scope='session') @@ -206,26 +219,20 @@ def tor_introducer_furl(tor_introducer, temp_dir): @pytest.fixture(scope='session') @log_call( action_type=u"integration:storage_nodes", - include_args=["temp_dir", "introducer_furl", "flog_gatherer"], + include_args=["grid"], include_result=False, ) -def storage_nodes(reactor, temp_dir, introducer, introducer_furl, flog_gatherer, request): +def storage_nodes(grid): nodes_d = [] # start all 5 nodes in parallel for x in range(5): - name = 'node{}'.format(x) - web_port = 'tcp:{}:interface=localhost'.format(9990 + x) - nodes_d.append( - grid.create_storage_server( - reactor, request, temp_dir, introducer, flog_gatherer, name, web_port, - ) - ) + #nodes_d.append(grid.add_storage_node()) + pytest_twisted.blockon(grid.add_storage_node()) + nodes_status = pytest_twisted.blockon(DeferredList(nodes_d)) - nodes = [] - for ok, process in nodes_status: - assert ok, "Storage node creation failed: {}".format(process) - nodes.append(process) - return nodes + for ok, value in nodes_status: + assert ok, "Storage node creation failed: {}".format(value) + return grid.storage_servers @pytest.fixture(scope='session') diff --git a/integration/grid.py b/integration/grid.py index 726c22e40..166cf940d 100644 --- a/integration/grid.py +++ b/integration/grid.py @@ -12,10 +12,15 @@ from os import mkdir, listdir, environ from os.path import join, exists from tempfile import mkdtemp, mktemp +from eliot import ( + log_call, +) + from twisted.python.procutils import which from twisted.internet.defer import ( inlineCallbacks, returnValue, + maybeDeferred, ) from twisted.internet.task import ( deferLater, @@ -25,6 +30,13 @@ from twisted.internet.interfaces import ( IProcessProtocol, IProtocol, ) +from twisted.internet.endpoints import ( + TCP4ServerEndpoint, +) +from twisted.internet.protocol import ( + Factory, + Protocol, +) from util import ( _CollectOutputProtocol, @@ -35,7 +47,6 @@ from util import ( _run_node, _cleanup_tahoe_process, _tahoe_runner_optional_coverage, - await_client_ready, TahoeProcess, ) @@ -165,6 +176,43 @@ def create_storage_server(reactor, request, temp_dir, introducer, flog_gatherer, ) returnValue(storage) + +@attr.s +class Client(object): + """ + Represents a Tahoe client + """ + + process = attr.ib( + validator=attr.validators.instance_of(TahoeProcess) + ) + protocol = attr.ib( + validator=attr.validators.provides(IProcessProtocol) + ) + + # XXX add stop / start / restart + # ...maybe "reconfig" of some kind? + + +@inlineCallbacks +def create_client(reactor, request, temp_dir, introducer, flog_gatherer, name, web_port, + needed=2, happy=3, total=4): + """ + Create a new storage server + """ + from util import _create_node + node_process = yield _create_node( + reactor, request, temp_dir, introducer.furl, flog_gatherer, + name, web_port, storage=False, needed=needed, happy=happy, total=total, + ) + returnValue( + Client( + process=node_process, + protocol=node_process.transport._protocol, + ) + ) + + @attr.s class Introducer(object): """ @@ -180,29 +228,27 @@ class Introducer(object): furl = attr.ib() -_introducer_num = 0 - - @inlineCallbacks -def create_introducer(reactor, request, temp_dir, flog_gatherer): +@log_call( + action_type=u"integration:introducer", + include_args=["temp_dir", "flog_gatherer"], + include_result=False, +) +def create_introducer(reactor, request, temp_dir, flog_gatherer, port): """ Run a new Introducer and return an Introducer instance. """ - global _introducer_num config = ( '[node]\n' - 'nickname = introducer{num}\n' + 'nickname = introducer{port}\n' 'web.port = {port}\n' 'log_gatherer.furl = {log_furl}\n' ).format( - num=_introducer_num, + port=port, log_furl=flog_gatherer.furl, - port=4560 + _introducer_num, ) - _introducer_num += 1 - intro_dir = join(temp_dir, 'introducer{}'.format(_introducer_num)) - print("making introducer", intro_dir, _introducer_num) + intro_dir = join(temp_dir, 'introducer{}'.format(port)) if not exists(intro_dir): mkdir(intro_dir) @@ -268,9 +314,14 @@ class Grid(object): Storage Servers. """ - introducer = attr.ib(default=None) - flog_gatherer = attr.ib(default=None) + _reactor = attr.ib() + _request = attr.ib() + _temp_dir = attr.ib() + _port_allocator = attr.ib() + introducer = attr.ib() + flog_gatherer = attr.ib() storage_servers = attr.ib(factory=list) + clients = attr.ib(factory=dict) @storage_servers.validator def check(self, attribute, value): @@ -279,3 +330,115 @@ class Grid(object): raise ValueError( "storage_servers must be StorageServer" ) + + @inlineCallbacks + def add_storage_node(self): + """ + Creates a new storage node, returns a StorageServer instance + (which will already be added to our .storage_servers list) + """ + port = yield self._port_allocator() + print("make {}".format(port)) + name = 'node{}'.format(port) + web_port = 'tcp:{}:interface=localhost'.format(port) + server = yield create_storage_server( + self._reactor, + self._request, + self._temp_dir, + self.introducer, + self.flog_gatherer, + name, + web_port, + ) + self.storage_servers.append(server) + returnValue(server) + + @inlineCallbacks + def add_client(self, name, needed=2, happy=3, total=4): + """ + Create a new client node + """ + port = yield self._port_allocator() + web_port = 'tcp:{}:interface=localhost'.format(port) + client = yield create_client( + self._reactor, + self._request, + self._temp_dir, + self.introducer, + self.flog_gatherer, + name, + web_port, + needed=needed, + happy=happy, + total=total, + ) + self.clients[name] = client + returnValue(client) + + + +# XXX THINK can we tie a whole *grid* to a single request? (I think +# that's all that makes sense) +@inlineCallbacks +def create_grid(reactor, request, temp_dir, flog_gatherer, port_allocator): + """ + """ + intro_port = yield port_allocator() + introducer = yield create_introducer(reactor, request, temp_dir, flog_gatherer, intro_port) + grid = Grid( + reactor, + request, + temp_dir, + port_allocator, + introducer, + flog_gatherer, + ) + returnValue(grid) + + +def create_port_allocator(start_port): + """ + Returns a new port-allocator .. which is a zero-argument function + that returns Deferreds that fire with new, sequential ports + starting at `start_port` skipping any that already appear to have + a listener. + + There can still be a race against other processes allocating ports + -- between the time when we check the status of the port and when + our subprocess starts up. This *could* be mitigated by instructing + the OS to not randomly-allocate ports in some range, and then + using that range here (explicitly, ourselves). + + NB once we're Python3-only this could be an async-generator + """ + port = [start_port - 1] + + # import stays here to not interfere with reactor selection -- but + # maybe this function should be arranged to be called once from a + # fixture (with the reactor)? + from twisted.internet import reactor + + class NothingProtocol(Protocol): + """ + I do nothing. + """ + + def port_generator(): + print("Checking port {}".format(port)) + port[0] += 1 + ep = TCP4ServerEndpoint(reactor, port[0], interface="localhost") + d = ep.listen(Factory.forProtocol(NothingProtocol)) + + def good(listening_port): + unlisten_d = maybeDeferred(listening_port.stopListening) + def return_port(_): + return port[0] + unlisten_d.addBoth(return_port) + return unlisten_d + + def try_again(fail): + return port_generator() + + d.addCallbacks(good, try_again) + return d + return port_generator diff --git a/integration/test_grid_manager.py b/integration/test_grid_manager.py index dc25c3b29..863a806a6 100644 --- a/integration/test_grid_manager.py +++ b/integration/test_grid_manager.py @@ -104,30 +104,16 @@ def test_remove_last_client(reactor, request): @pytest_twisted.inlineCallbacks -def test_reject_storage_server(reactor, request, temp_dir, flog_gatherer): +def test_reject_storage_server(reactor, request, temp_dir, flog_gatherer, port_allocator): """ A client with happines=2 fails to upload to a Grid when it is using Grid Manager and there is only 1 storage server with a valid certificate. """ - import grid - introducer = yield grid.create_introducer(reactor, request, temp_dir, flog_gatherer) - storage0 = yield grid.create_storage_server( - reactor, request, temp_dir, introducer, flog_gatherer, - name="gm_storage0", - web_port="tcp:9995:interface=localhost", - needed=2, - happy=2, - total=2, - ) - storage1 = yield grid.create_storage_server( - reactor, request, temp_dir, introducer, flog_gatherer, - name="gm_storage1", - web_port="tcp:9996:interface=localhost", - needed=2, - happy=2, - total=2, - ) + from grid import create_grid + grid = yield create_grid(reactor, request, temp_dir, flog_gatherer, port_allocator) + storage0 = yield grid.add_storage_node() + storage1 = yield grid.add_storage_node() gm_config = yield util.run_tahoe( reactor, request, "grid-manager", "--config", "-", "create", @@ -147,12 +133,7 @@ def test_reject_storage_server(reactor, request, temp_dir, flog_gatherer): ) assert json.loads(gm_config)['storage_servers'].keys() == ['storage0'] - # XXX FIXME want a grid.create_client() or similar - diana = yield util._create_node( - reactor, request, temp_dir, introducer.furl, flog_gatherer, "diana", - web_port="tcp:9984:interface=localhost", - storage=False, - ) + diana = yield grid.add_client("diana", needed=2, happy=2, total=2) print("inserting certificate") cert = yield util.run_tahoe( @@ -175,23 +156,23 @@ def test_reject_storage_server(reactor, request, temp_dir, flog_gatherer): reactor, storage0.process.node_dir, request, None, ) - yield util.await_client_ready(diana, servers=2) + yield util.await_client_ready(diana.process, servers=2) # now only one storage-server has the certificate .. configure # diana to have the grid-manager certificate - config = configutil.get_config(join(diana.node_dir, "tahoe.cfg")) + config = configutil.get_config(join(diana.process.node_dir, "tahoe.cfg")) config.add_section("grid_managers") config.set("grid_managers", "test", ed25519.string_from_verifying_key(gm_pubkey)) - with open(join(diana.node_dir, "tahoe.cfg"), "w") as f: + with open(join(diana.process.node_dir, "tahoe.cfg"), "w") as f: config.write(f) - diana.transport.signalProcess('TERM') - yield diana.transport._protocol.exited + diana.process.transport.signalProcess('TERM') + yield diana.protocol.exited diana = yield util._run_node( - reactor, diana._node_dir, request, None, + reactor, diana.process.node_dir, request, None, ) - yield util.await_client_ready(diana, servers=2) + yield util.await_client_ready(diana.process, servers=2) # try to put something into the grid, which should fail (because # diana has happy=2 but should only find storage0 to be acceptable @@ -199,7 +180,7 @@ def test_reject_storage_server(reactor, request, temp_dir, flog_gatherer): try: yield util.run_tahoe( - reactor, request, "--node-directory", diana._node_dir, + reactor, request, "--node-directory", diana.process.node_dir, "put", "-", stdin="some content\n" * 200, ) diff --git a/integration/util.py b/integration/util.py index 2cdcb2563..72689d387 100644 --- a/integration/util.py +++ b/integration/util.py @@ -266,7 +266,7 @@ def _create_node(reactor, request, temp_dir, introducer_furl, flog_gatherer, nam if exists(node_dir): created_d = succeed(None) else: - print("creating", node_dir) + print("creating: {}".format(node_dir)) mkdir(node_dir) done_proto = _ProcessExitedProtocol() args = [ From c9de1ee757d7943bbd51231a7eec8f3e45b69467 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 12 May 2020 14:22:32 -0600 Subject: [PATCH 0104/2309] add a .restart() to Client and StorageServer --- integration/grid.py | 20 ++++++++++++++++++-- integration/test_grid_manager.py | 19 ++++--------------- 2 files changed, 22 insertions(+), 17 deletions(-) diff --git a/integration/grid.py b/integration/grid.py index 166cf940d..d7b104e16 100644 --- a/integration/grid.py +++ b/integration/grid.py @@ -48,6 +48,7 @@ from util import ( _cleanup_tahoe_process, _tahoe_runner_optional_coverage, TahoeProcess, + await_client_ready, ) import attr @@ -155,8 +156,14 @@ class StorageServer(object): validator=attr.validators.provides(IProcessProtocol) ) - # XXX needs a restart() probably .. or at least a stop() and - # start() + @inlineCallbacks + def restart(self, reactor, request): + self.process.transport.signalProcess('TERM') + yield self.protocol.exited + self.process = yield _run_node( + reactor, self.process.node_dir, request, None, + ) + self.protocol = self.process.transport._protocol @inlineCallbacks @@ -190,6 +197,14 @@ class Client(object): validator=attr.validators.provides(IProcessProtocol) ) + @inlineCallbacks + def restart(self, reactor, request): + self.process.transport.signalProcess('TERM') + yield self.protocol.exited + x = yield _run_node( + reactor, self.process.node_dir, request, None, + ) + # XXX add stop / start / restart # ...maybe "reconfig" of some kind? @@ -373,6 +388,7 @@ class Grid(object): total=total, ) self.clients[name] = client + yield await_client_ready(client.process) returnValue(client) diff --git a/integration/test_grid_manager.py b/integration/test_grid_manager.py index 863a806a6..38c2dd78a 100644 --- a/integration/test_grid_manager.py +++ b/integration/test_grid_manager.py @@ -133,8 +133,6 @@ def test_reject_storage_server(reactor, request, temp_dir, flog_gatherer, port_a ) assert json.loads(gm_config)['storage_servers'].keys() == ['storage0'] - diana = yield grid.add_client("diana", needed=2, happy=2, total=2) - print("inserting certificate") cert = yield util.run_tahoe( reactor, request, "grid-manager", "--config", "-", "sign", "storage0", @@ -150,29 +148,20 @@ def test_reject_storage_server(reactor, request, temp_dir, flog_gatherer, port_a config.write(f) # re-start this storage server - storage0.process.transport.signalProcess('TERM') - yield storage0.protocol.exited - yield util._run_node( - reactor, storage0.process.node_dir, request, None, - ) - - yield util.await_client_ready(diana.process, servers=2) + yield storage0.restart(reactor, request) # now only one storage-server has the certificate .. configure # diana to have the grid-manager certificate + diana = yield grid.add_client("diana", needed=2, happy=2, total=2) + config = configutil.get_config(join(diana.process.node_dir, "tahoe.cfg")) config.add_section("grid_managers") config.set("grid_managers", "test", ed25519.string_from_verifying_key(gm_pubkey)) with open(join(diana.process.node_dir, "tahoe.cfg"), "w") as f: config.write(f) - diana.process.transport.signalProcess('TERM') - yield diana.protocol.exited - diana = yield util._run_node( - reactor, diana.process.node_dir, request, None, - ) - yield util.await_client_ready(diana.process, servers=2) + yield diana.restart(reactor, request) # try to put something into the grid, which should fail (because # diana has happy=2 but should only find storage0 to be acceptable From f52cd4363ad9921ba8b6d85d64a486f6390b0cb9 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 12 May 2020 16:56:35 -0600 Subject: [PATCH 0105/2309] set the stage for the example in the intro --- docs/proposed/grid-manager/managed-grid.rst | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/docs/proposed/grid-manager/managed-grid.rst b/docs/proposed/grid-manager/managed-grid.rst index a3ad0d115..ff4b6e5c4 100644 --- a/docs/proposed/grid-manager/managed-grid.rst +++ b/docs/proposed/grid-manager/managed-grid.rst @@ -241,6 +241,11 @@ Manager. Example:: Example Setup of a New Managed Grid ----------------------------------- +This example creates an actual grid, but it's all just on one machine +with different "node directories" and a separate tahoe process for +each node. Usually of course each storage server would be on a +separate computer. + Note that we use the ``daemonize`` command in the following but that's only one way to handle "running a command in the background". You could instead run commands that start with ``daemonize ...`` in their @@ -251,10 +256,6 @@ We'll store our Grid Manager configuration on disk, in tahoe grid-manager --config ./gm0 create -This example creates an actual grid, but it's all just on one machine -with different "node directories". Usually of course each storage -server would be on a separate computer. - (If you already have a grid, you can :ref:`skip ahead `.) First of all, create an Introducer. Note that we actually have to run From 7c8a7f973a9eed2dd98de69af68665617884506b Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 12 May 2020 16:56:46 -0600 Subject: [PATCH 0106/2309] better re-start --- integration/grid.py | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/integration/grid.py b/integration/grid.py index d7b104e16..58eafe6b6 100644 --- a/integration/grid.py +++ b/integration/grid.py @@ -158,6 +158,13 @@ class StorageServer(object): @inlineCallbacks def restart(self, reactor, request): + """ + re-start our underlying process by issuing a TERM, waiting and + then running again. await_client_ready() will be done as well + + Note that self.process and self.protocol will be new instances + after this. + """ self.process.transport.signalProcess('TERM') yield self.protocol.exited self.process = yield _run_node( @@ -165,6 +172,9 @@ class StorageServer(object): ) self.protocol = self.process.transport._protocol + @inlineCallbacks + def run( + @inlineCallbacks def create_storage_server(reactor, request, temp_dir, introducer, flog_gatherer, name, web_port, @@ -198,12 +208,25 @@ class Client(object): ) @inlineCallbacks - def restart(self, reactor, request): + def restart(self, reactor, request, servers=1): + """ + re-start our underlying process by issuing a TERM, waiting and + then running again. + + :param int servers: number of server connections we will wait + for before being 'ready' + + Note that self.process and self.protocol will be new instances + after this. + """ self.process.transport.signalProcess('TERM') yield self.protocol.exited - x = yield _run_node( + process = yield _run_node( reactor, self.process.node_dir, request, None, ) + self.process = process + self.protocol = self.process.transport._protocol + # XXX add stop / start / restart # ...maybe "reconfig" of some kind? From aa2066e2214269d0cd9e0f8ffdba06d50a274b12 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 12 May 2020 16:57:37 -0600 Subject: [PATCH 0107/2309] move imports --- integration/test_grid_manager.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/integration/test_grid_manager.py b/integration/test_grid_manager.py index 38c2dd78a..07b8a9083 100644 --- a/integration/test_grid_manager.py +++ b/integration/test_grid_manager.py @@ -10,6 +10,9 @@ from allmydata.util import base32 from allmydata.util import configutil import util +from grid import ( + create_grid, +) import pytest_twisted @@ -110,7 +113,6 @@ def test_reject_storage_server(reactor, request, temp_dir, flog_gatherer, port_a using Grid Manager and there is only 1 storage server with a valid certificate. """ - from grid import create_grid grid = yield create_grid(reactor, request, temp_dir, flog_gatherer, port_allocator) storage0 = yield grid.add_storage_node() storage1 = yield grid.add_storage_node() @@ -161,7 +163,7 @@ def test_reject_storage_server(reactor, request, temp_dir, flog_gatherer, port_a with open(join(diana.process.node_dir, "tahoe.cfg"), "w") as f: config.write(f) - yield diana.restart(reactor, request) + yield diana.restart(reactor, request, servers=2) # try to put something into the grid, which should fail (because # diana has happy=2 but should only find storage0 to be acceptable From 114c2faf1bb66019cb57d327a79fa4910b8a3350 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 12 May 2020 17:12:51 -0600 Subject: [PATCH 0108/2309] oops --- integration/grid.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/integration/grid.py b/integration/grid.py index 58eafe6b6..f39f04d3c 100644 --- a/integration/grid.py +++ b/integration/grid.py @@ -172,9 +172,6 @@ class StorageServer(object): ) self.protocol = self.process.transport._protocol - @inlineCallbacks - def run( - @inlineCallbacks def create_storage_server(reactor, request, temp_dir, introducer, flog_gatherer, name, web_port, From d4e2c668432aa3b092a14f20a43bccfd72228c41 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 12 May 2020 18:17:24 -0600 Subject: [PATCH 0109/2309] use eliot logging --- src/allmydata/mutable/publish.py | 43 ++++++++++++++++++++------------ 1 file changed, 27 insertions(+), 16 deletions(-) diff --git a/src/allmydata/mutable/publish.py b/src/allmydata/mutable/publish.py index aa97c2242..5477e3077 100644 --- a/src/allmydata/mutable/publish.py +++ b/src/allmydata/mutable/publish.py @@ -23,6 +23,9 @@ from allmydata.mutable.layout import get_version_from_checkstring,\ MDMFSlotWriteProxy, \ SDMFSlotWriteProxy +import eliot + + KiB = 1024 DEFAULT_MAX_SEGMENT_SIZE = 128 * KiB PUSHING_BLOCKS_STATE = 0 @@ -942,23 +945,31 @@ class Publish(object): old_assignments.add(server, shnum) serverlist = [] - for i, server in enumerate(self.full_serverlist): - serverid = server.get_serverid() - if server in self.bad_servers: - continue - # if we have >= 1 grid-managers, this checks that we have - # a valid certificate for this server - if not server.upload_permitted(): - self.log( - "No valid grid-manager certificates for '{}' while choosing slots for mutable".format( - server.get_serverid(), - ), - level=log.UNUSUAL, - ) - continue - entry = (len(old_assignments.get(server, [])), i, serverid, server) - serverlist.append(entry) + action = eliot.start_action( + action_type=u"mutable:upload:update_goal", + homeless_shares=len(homeless_shares), + ) + with action: + for i, server in enumerate(self.full_serverlist): + serverid = server.get_serverid() + if server in self.bad_servers: + action.log( + server_id=server.get_server_id(), + message="Server is bad", + ) + continue + # if we have >= 1 grid-managers, this checks that we have + # a valid certificate for this server + if not server.upload_permitted(): + action.log( + server_id=server.get_server_id(), + message="No valid grid-manager certificates", + ) + continue + + entry = (len(old_assignments.get(server, [])), i, serverid, server) + serverlist.append(entry) serverlist.sort() if not serverlist: From b13a688ea816bd93519fd08043aab6b6859f21b9 Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 1 Oct 2020 19:21:09 -0600 Subject: [PATCH 0110/2309] better words --- docs/proposed/grid-manager/managed-grid.rst | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/proposed/grid-manager/managed-grid.rst b/docs/proposed/grid-manager/managed-grid.rst index ff4b6e5c4..8d584e71d 100644 --- a/docs/proposed/grid-manager/managed-grid.rst +++ b/docs/proposed/grid-manager/managed-grid.rst @@ -40,7 +40,7 @@ the statement: "Grid Manager X suggests you use storage-server Y to upload shares to" (X and Y are public-keys). Such a certificate consists of: - - a version (currently 1) + - a version - the public-key of a storage-server - an expiry timestamp - a signature of the above @@ -61,10 +61,10 @@ at which time you may be able to pass a directory-capability to this option). If you don't want to store the configuration on disk at all, you may -use ``--config -`` (that's a dash) and write a valid JSON -configuration to stdin. +use ``--config -`` (the last character is a dash) and write a valid +JSON configuration to stdin. -All commands require the ``--config`` option, and they all behave +All commands require the ``--config`` option and they all behave similarly for "data from stdin" versus "data from disk". @@ -144,11 +144,11 @@ tahoe admin add-grid-manager-cert - `--filename`: the file to read the cert from (default: stdin) - `--name`: the name of this certificate (default: "default") -Import a "version 1" storage-certificate produced by a grid-manager -(probably: a storage server may have zero or more such certificates -installed; for now just one is sufficient). You will have to re-start -your node after this. Subsequent announcements to the Introducer will -include this certificate. +Import a "version 1" storage-certificate produced by a grid-manager A +storage server may have zero or more such certificates installed; for +now just one is sufficient. You will have to re-start your node after +this. Subsequent announcements to the Introducer will include this +certificate. .. note:: From b0d48ddbc987c4919b47cc6686838eef9d5f6968 Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 1 Oct 2020 19:21:24 -0600 Subject: [PATCH 0111/2309] don't need to create section --- src/allmydata/scripts/admin.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/allmydata/scripts/admin.py b/src/allmydata/scripts/admin.py index 0ac3c6479..d9e45149b 100644 --- a/src/allmydata/scripts/admin.py +++ b/src/allmydata/scripts/admin.py @@ -125,7 +125,6 @@ def add_grid_manager_cert(options): return 1 config.set_config("storage", "grid_management", "True") - config.add_section("grid_manager_certificates") config.set_config("grid_manager_certificates", cert_name, cert_fname) # write all the data out From 6b791cb3b5b47708bf4602e07a2a5023c88f4969 Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 1 Oct 2020 19:23:42 -0600 Subject: [PATCH 0112/2309] nicer error --- src/allmydata/scripts/admin.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/allmydata/scripts/admin.py b/src/allmydata/scripts/admin.py index d9e45149b..e18b31672 100644 --- a/src/allmydata/scripts/admin.py +++ b/src/allmydata/scripts/admin.py @@ -121,7 +121,11 @@ def add_grid_manager_cert(options): cert_name = options['name'] if exists(cert_path): - print("Already have file '{}'".format(cert_path), file=options.parent.parent.stderr) + msg = "Already have certificate for '{}' (at {})".format( + options['name'], + cert_path, + ) + print(msg, file=options.parent.parent.stderr) return 1 config.set_config("storage", "grid_management", "True") From e65b6ba9274ed9bed1002a4652e58f5f1bf75525 Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 1 Oct 2020 19:24:17 -0600 Subject: [PATCH 0113/2309] no default named 'default' --- src/allmydata/scripts/admin.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/allmydata/scripts/admin.py b/src/allmydata/scripts/admin.py index e18b31672..96c1aa916 100644 --- a/src/allmydata/scripts/admin.py +++ b/src/allmydata/scripts/admin.py @@ -60,13 +60,17 @@ class AddGridManagerCertOptions(BaseOptions): optParameters = [ ['filename', 'f', None, "Filename of the certificate ('-', a dash, for stdin)"], - ['name', 'n', "default", "Name to give this certificate"], + ['name', 'n', None, "Name to give this certificate"], ] def getSynopsis(self): return "Usage: tahoe [global-options] admin add-grid-manager-cert [options]" def postOptions(self): + if self['name'] is None: + raise usage.UsageError( + "Must provide --name option" + ) if self['filename'] is None: raise usage.UsageError( "Must provide --filename option" From 8deddc62725863e5f9fa697fd0557d784d7c912c Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 1 Oct 2020 19:46:43 -0600 Subject: [PATCH 0114/2309] no defaults --- docs/proposed/grid-manager/managed-grid.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/proposed/grid-manager/managed-grid.rst b/docs/proposed/grid-manager/managed-grid.rst index 8d584e71d..d0284dd16 100644 --- a/docs/proposed/grid-manager/managed-grid.rst +++ b/docs/proposed/grid-manager/managed-grid.rst @@ -141,8 +141,8 @@ Enrolling a Storage Server: CLI tahoe admin add-grid-manager-cert ````````````````````````````````` -- `--filename`: the file to read the cert from (default: stdin) -- `--name`: the name of this certificate (default: "default") +- `--filename`: the file to read the cert from +- `--name`: the name of this certificate Import a "version 1" storage-certificate produced by a grid-manager A storage server may have zero or more such certificates installed; for From 2483e938dc235ec33892c1c79f3c78cea6a29c4a Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 1 Oct 2020 19:47:03 -0600 Subject: [PATCH 0115/2309] tahoe add-grid-manager is only 'idea' currently --- docs/proposed/grid-manager/managed-grid.rst | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/docs/proposed/grid-manager/managed-grid.rst b/docs/proposed/grid-manager/managed-grid.rst index d0284dd16..72eec7660 100644 --- a/docs/proposed/grid-manager/managed-grid.rst +++ b/docs/proposed/grid-manager/managed-grid.rst @@ -204,8 +204,10 @@ Enrolling a Client: CLI ----------------------- -tahoe add-grid-manager -`````````````````````` +tahoe add-grid-manager (PROPOSED) +````````````````````````````````` + +DECIDE: this command hasn't actually been written yet. This takes two arguments: ``name`` and ``public-identity``. @@ -295,7 +297,8 @@ certificates into the grid. We do this by adding some configuration default = gridmanager.cert Add the above bit to each node's ``tahoe.cfg`` and re-start the -storage nodes. +storage nodes. (Alternatively, use the ``tahoe add-grid-manager`` +command). Now try adding a new storage server ``storage2``. This client can join the grid just fine, and announce itself to the Introducer as providing From d46c35e953e22cc21bbb43729cc5fb58b59e09e5 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 2 Oct 2020 16:02:56 -0600 Subject: [PATCH 0116/2309] grid-manager unit-tests --- src/allmydata/test/test_grid_manager.py | 72 +++++++++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 src/allmydata/test/test_grid_manager.py diff --git a/src/allmydata/test/test_grid_manager.py b/src/allmydata/test/test_grid_manager.py new file mode 100644 index 000000000..459b360e5 --- /dev/null +++ b/src/allmydata/test/test_grid_manager.py @@ -0,0 +1,72 @@ + + +from allmydata.client import ( + _load_grid_manager_certificates, + create_storage_farm_broker, +) +from allmydata.node import ( + config_from_string, +) +from allmydata.client import ( + _valid_config as client_valid_config, +) + +from .common import SyncTestCase + + +class GridManagerUtilities(SyncTestCase): + """ + Confirm operation of utility functions used by GridManager + """ + + def test_client_grid_manager(self): + config_data = ( + "[grid_managers]\n" + "fluffy = pub-v0-vqimc4s5eflwajttsofisp5st566dbq36xnpp4siz57ufdavpvlq\n" + ) + config = config_from_string("/foo", "portnum", config_data, client_valid_config()) + sfb = create_storage_farm_broker(config, {}, {}, {}, []) + # could introspect sfb._grid_manager_certificates, but that's + # "cheating"? even though _make_storage_sever is also + # "private"? + + # ...but, okay, a "real" client will call set_static_servers() + # with any configured/cached servers (thus causing + # _make_storage_server to be called). The other way + # _make_storage_server is called is when _got_announcement + # runs, which is when an introducer client gets an + # announcement... + + invalid_cert = { + "certificate": "foo", + "signature": "43564356435643564356435643564356", + } + announcement = { + "anonymous-storage-FURL": b"pb://abcde@nowhere/fake", + "grid-manager-certificates": [ + invalid_cert, + ] + } + static_servers = { + "v0-4uazse3xb6uu5qpkb7tel2bm6bpea4jhuigdhqcuvvse7hugtsia": { + "ann": announcement, + } + } + sfb.set_static_servers(static_servers) + nss = sfb._make_storage_server(u"server0", {"ann": announcement}) + + # we have some grid-manager keys defined so the server should + # only upload if there's a valid certificate -- but the only + # one we have is invalid + self.assertFalse(nss.upload_permitted()) + + def test_load_certificates(self): + config_data = ( + "[grid_managers]\n" + "fluffy = pub-v0-vqimc4s5eflwajttsofisp5st566dbq36xnpp4siz57ufdavpvlq\n" + ) + config = config_from_string("/foo", "portnum", config_data, client_valid_config()) + self.assertEqual( + 1, + len(config.enumerate_section("grid_managers")) + ) From 6a2f1ae00307781effc7054207044fca3d25641b Mon Sep 17 00:00:00 2001 From: meejah Date: Sun, 4 Oct 2020 17:34:23 -0600 Subject: [PATCH 0117/2309] use 'tahoe admin add-grid-manager-cert' in tests --- integration/test_grid_manager.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/integration/test_grid_manager.py b/integration/test_grid_manager.py index 07b8a9083..8ecf72561 100644 --- a/integration/test_grid_manager.py +++ b/integration/test_grid_manager.py @@ -142,12 +142,13 @@ def test_reject_storage_server(reactor, request, temp_dir, flog_gatherer, port_a ) with open(join(storage0.process.node_dir, "gridmanager.cert"), "w") as f: f.write(cert) - config = configutil.get_config(join(storage0.process.node_dir, "tahoe.cfg")) - config.set("storage", "grid_management", "True") - config.add_section("grid_manager_certificates") - config.set("grid_manager_certificates", "default", "gridmanager.cert") - with open(join(storage0.process.node_dir, "tahoe.cfg"), "w") as f: - config.write(f) + + yield util.run_tahoe( + reactor, request, "--node-directory", storage0.process.node_dir, + "admin", "add-grid-manager-cert", + "--name", "default", + "--filename", join(storage0.process.node_dir, "gridmanager.cert"), + ) # re-start this storage server yield storage0.restart(reactor, request) From 5eb1ad953bab89d460240f04a46ef0d42dc55ec6 Mon Sep 17 00:00:00 2001 From: meejah Date: Sun, 4 Oct 2020 17:35:02 -0600 Subject: [PATCH 0118/2309] fix 'admin add-grid-manager-cert' --- src/allmydata/scripts/admin.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/src/allmydata/scripts/admin.py b/src/allmydata/scripts/admin.py index 96c1aa916..57cac39c7 100644 --- a/src/allmydata/scripts/admin.py +++ b/src/allmydata/scripts/admin.py @@ -7,6 +7,9 @@ from os.path import exists, join from twisted.python import usage #from allmydata.node import read_config from allmydata.client import read_config +from allmydata.storage_client import ( + parse_grid_manager_data, +) from allmydata.scripts.cli import _default_nodedir from allmydata.scripts.common import BaseOptions from allmydata.util.encodingutil import argv_to_abspath @@ -82,15 +85,16 @@ class AddGridManagerCertOptions(BaseOptions): raise usage.UsageError( "Reading certificate from stdin failed" ) - from allmydata.storage_client import parse_grid_manager_data - try: - self.certificate_data = parse_grid_manager_data(data) - except ValueError as e: - print("Error parsing certificate: {}".format(e), file=self.parent.parent.stderr) - self.certificate_data = None else: with open(self['filename'], 'r') as f: - self.certificate_data = f.read() + data = f.read() + + try: + self.certificate_data = parse_grid_manager_data(data) + except ValueError as e: + raise UsageError( + "Error parsing certificate: {}".format(e) + ) def getUsage(self, width=None): t = BaseOptions.getUsage(self, width) From 370d4b0f8b801ea52b9a33915a59b96c1b0bb4da Mon Sep 17 00:00:00 2001 From: meejah Date: Sun, 4 Oct 2020 17:35:50 -0600 Subject: [PATCH 0119/2309] load certs as well as keys in utest --- src/allmydata/test/test_grid_manager.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/allmydata/test/test_grid_manager.py b/src/allmydata/test/test_grid_manager.py index 459b360e5..d5c98f2e7 100644 --- a/src/allmydata/test/test_grid_manager.py +++ b/src/allmydata/test/test_grid_manager.py @@ -1,4 +1,5 @@ +import json from allmydata.client import ( _load_grid_manager_certificates, @@ -61,9 +62,17 @@ class GridManagerUtilities(SyncTestCase): self.assertFalse(nss.upload_permitted()) def test_load_certificates(self): + cert_path = self.mktemp() + with open(cert_path, "w") as f: + f.write(json.dumps({ + "certificate": "{\"expires\":1601687822,\"public_key\":\"pub-v0-cbq6hcf3pxcz6ouoafrbktmkixkeuywpcpbcomzd3lqbkq4nmfga\",\"version\":1}", + "signature": "fvjd3uvvupf2v6tnvkwjd473u3m3inyqkwiclhp7balmchkmn3px5pei3qyfjnhymq4cjcwvbpqmcwwnwswdtrfkpnlaxuih2zbdmda" + })) config_data = ( "[grid_managers]\n" "fluffy = pub-v0-vqimc4s5eflwajttsofisp5st566dbq36xnpp4siz57ufdavpvlq\n" + "[grid_manager_certificates]\n" + "ding = {}\n".format(cert_path) ) config = config_from_string("/foo", "portnum", config_data, client_valid_config()) self.assertEqual( From c43c84b6024733320150a63c59e6131c3ec0dbaa Mon Sep 17 00:00:00 2001 From: meejah Date: Sun, 4 Oct 2020 18:02:46 -0600 Subject: [PATCH 0120/2309] fewer files in test --- integration/test_grid_manager.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/integration/test_grid_manager.py b/integration/test_grid_manager.py index 8ecf72561..dca403d1d 100644 --- a/integration/test_grid_manager.py +++ b/integration/test_grid_manager.py @@ -140,14 +140,13 @@ def test_reject_storage_server(reactor, request, temp_dir, flog_gatherer, port_a reactor, request, "grid-manager", "--config", "-", "sign", "storage0", stdin=gm_config, ) - with open(join(storage0.process.node_dir, "gridmanager.cert"), "w") as f: - f.write(cert) yield util.run_tahoe( reactor, request, "--node-directory", storage0.process.node_dir, "admin", "add-grid-manager-cert", "--name", "default", - "--filename", join(storage0.process.node_dir, "gridmanager.cert"), + "--filename", "-", + stdin=cert, ) # re-start this storage server From 84c9da4e7bc1c0a45498772a35b1f1c691a027c1 Mon Sep 17 00:00:00 2001 From: meejah Date: Sun, 4 Oct 2020 18:09:27 -0600 Subject: [PATCH 0121/2309] unused --- src/allmydata/scripts/tahoe_grid_manager.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/src/allmydata/scripts/tahoe_grid_manager.py b/src/allmydata/scripts/tahoe_grid_manager.py index 64561b95e..4b0a24d76 100644 --- a/src/allmydata/scripts/tahoe_grid_manager.py +++ b/src/allmydata/scripts/tahoe_grid_manager.py @@ -322,16 +322,6 @@ def _save_gridmanager_config(file_path, grid_manager): f.write("{}\n".format(data)) -def _config_to_filepath(gm_config_location): - """ - Converts a command-line string specifying the GridManager - configuration's location into a readable file-like object. - - :param gm_config_location str: a valid path, or '-' (a single - dash) to use stdin. - """ - - def _load_gridmanager_config(gm_config): """ Loads a Grid Manager configuration and returns it (a dict) after From c43955573bd9c278a0d1ff28544eefb68fb4f2a4 Mon Sep 17 00:00:00 2001 From: meejah Date: Sun, 4 Oct 2020 20:40:26 -0600 Subject: [PATCH 0122/2309] more gridmanager tests --- integration/test_grid_manager.py | 36 +++++++++++++++++++++++- integration/test_servers_of_happiness.py | 3 +- integration/util.py | 5 +++- 3 files changed, 40 insertions(+), 4 deletions(-) diff --git a/integration/test_grid_manager.py b/integration/test_grid_manager.py index dca403d1d..d701172f4 100644 --- a/integration/test_grid_manager.py +++ b/integration/test_grid_manager.py @@ -5,6 +5,11 @@ import shutil from os import mkdir, unlink, listdir, utime from os.path import join, exists, getmtime +from cryptography.hazmat.primitives.serialization import ( + Encoding, + PublicFormat, +) + from allmydata.crypto import ed25519 from allmydata.util import base32 from allmydata.util import configutil @@ -177,4 +182,33 @@ def test_reject_storage_server(reactor, request, temp_dir, flog_gatherer, port_a ) assert False, "Should get a failure" except util.ProcessFailed as e: - assert 'UploadUnhappinessError' in e.output.getvalue() + assert 'UploadUnhappinessError' in e.output + + +@pytest_twisted.inlineCallbacks +def test_identity(reactor, request, temp_dir): + """ + Dump public key to CLI + """ + gm_config = join(temp_dir, "test_identity") + yield util.run_tahoe( + reactor, request, "grid-manager", "--config", gm_config, "create", + ) + + # ask the CLI for the grid-manager pubkey + pubkey = yield util.run_tahoe( + reactor, request, "grid-manager", "--config", gm_config, "public-identity", + ) + alleged_pubkey = ed25519.verifying_key_from_string(pubkey.strip()) + + # load the grid-manager pubkey "ourselves" + with open(join(gm_config, "config.json"), "r") as f: + real_config = json.load(f) + real_privkey, real_pubkey = ed25519.signing_keypair_from_string( + real_config["private_key"].encode("ascii"), + ) + + # confirm the CLI told us the correct thing + alleged_bytes = alleged_pubkey.public_bytes(Encoding.Raw, PublicFormat.Raw) + real_bytes = real_pubkey.public_bytes(Encoding.Raw, PublicFormat.Raw) + assert alleged_bytes == real_bytes, "Keys don't match" diff --git a/integration/test_servers_of_happiness.py b/integration/test_servers_of_happiness.py index fd8f75e39..dd0b3e32f 100644 --- a/integration/test_servers_of_happiness.py +++ b/integration/test_servers_of_happiness.py @@ -42,5 +42,4 @@ def test_upload_immutable(reactor, temp_dir, introducer_furl, flog_gatherer, sto except util.ProcessFailed as e: assert "UploadUnhappinessError" in e.output.getvalue() - output = proto.output.getvalue() - assert "shares could be placed on only" in output + assert "shares could be placed on only" in proto.output diff --git a/integration/util.py b/integration/util.py index 72689d387..5cad59c12 100644 --- a/integration/util.py +++ b/integration/util.py @@ -49,6 +49,9 @@ class ProcessFailed(Exception): self.reason = reason self.output = output + def __str__(self): + return ":\n{}".format(self.reason, self.output) + class _CollectOutputProtocol(ProcessProtocol): """ @@ -72,7 +75,7 @@ class _CollectOutputProtocol(ProcessProtocol): def processExited(self, reason): if not isinstance(reason.value, ProcessDone): - self.done.errback(ProcessFailed(reason, self.output)) + self.done.errback(ProcessFailed(reason, self.output.getvalue())) def outReceived(self, data): self.output.write(data) From 774ab72c9d4eb2d1d8ae41054c9b699a496c80b0 Mon Sep 17 00:00:00 2001 From: meejah Date: Sun, 4 Oct 2020 21:06:29 -0600 Subject: [PATCH 0123/2309] test user-management on files --- integration/test_grid_manager.py | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/integration/test_grid_manager.py b/integration/test_grid_manager.py index d701172f4..2c1f0d1c1 100644 --- a/integration/test_grid_manager.py +++ b/integration/test_grid_manager.py @@ -111,6 +111,37 @@ def test_remove_last_client(reactor, request): assert "storage_servers" not in json.loads(gm_config) +@pytest_twisted.inlineCallbacks +def test_add_remove_client_file(reactor, request, temp_dir): + """ + A Grid Manager can add and successfully remove a client (when + keeping data on disk) + """ + gmconfig = join(temp_dir, "gmtest") + gmconfig_file = join(temp_dir, "gmtest", "config.json") + yield util.run_tahoe( + reactor, request, "grid-manager", "--config", gmconfig, "create", + ) + + yield util.run_tahoe( + reactor, request, "grid-manager", "--config", gmconfig, "add", + "zara", "pub-v0-kzug3ut2m7ziihf3ndpqlquuxeie4foyl36wn54myqc4wmiwe4ga", + ) + yield util.run_tahoe( + reactor, request, "grid-manager", "--config", gmconfig, "add", + "yakov", "pub-v0-kvxhb3nexybmipkrar2ztfrwp4uxxsmrjzkpzafit3ket4u5yldq", + ) + assert "zara" in json.load(open(gmconfig_file, "r"))['storage_servers'] + assert "yakov" in json.load(open(gmconfig_file, "r"))['storage_servers'] + + yield util.run_tahoe( + reactor, request, "grid-manager", "--config", gmconfig, "remove", + "zara", + ) + assert "zara" not in json.load(open(gmconfig_file, "r"))['storage_servers'] + assert "yakov" in json.load(open(gmconfig_file, "r"))['storage_servers'] + + @pytest_twisted.inlineCallbacks def test_reject_storage_server(reactor, request, temp_dir, flog_gatherer, port_allocator): """ From bf45f57b603d0bc0098a1d86e880689ea2ab40df Mon Sep 17 00:00:00 2001 From: meejah Date: Sun, 4 Oct 2020 21:54:38 -0600 Subject: [PATCH 0124/2309] minor fixes --- integration/test_grid_manager.py | 4 +-- integration/test_servers_of_happiness.py | 4 +-- src/allmydata/scripts/tahoe_grid_manager.py | 27 +++++++++++++++------ 3 files changed, 23 insertions(+), 12 deletions(-) diff --git a/integration/test_grid_manager.py b/integration/test_grid_manager.py index 2c1f0d1c1..db93f17c7 100644 --- a/integration/test_grid_manager.py +++ b/integration/test_grid_manager.py @@ -42,7 +42,7 @@ def test_create_certificate(reactor, request): stdin=gm_config, ) zara_cert_bytes = yield util.run_tahoe( - reactor, request, "grid-manager", "--config", "-", "sign", "zara", + reactor, request, "grid-manager", "--config", "-", "sign", "zara", "1", stdin=gm_config, ) zara_cert = json.loads(zara_cert_bytes) @@ -173,7 +173,7 @@ def test_reject_storage_server(reactor, request, temp_dir, flog_gatherer, port_a print("inserting certificate") cert = yield util.run_tahoe( - reactor, request, "grid-manager", "--config", "-", "sign", "storage0", + reactor, request, "grid-manager", "--config", "-", "sign", "storage0", "1", stdin=gm_config, ) diff --git a/integration/test_servers_of_happiness.py b/integration/test_servers_of_happiness.py index dd0b3e32f..d98bfe0b8 100644 --- a/integration/test_servers_of_happiness.py +++ b/integration/test_servers_of_happiness.py @@ -40,6 +40,6 @@ def test_upload_immutable(reactor, temp_dir, introducer_furl, flog_gatherer, sto yield proto.done assert False, "should raise exception" except util.ProcessFailed as e: - assert "UploadUnhappinessError" in e.output.getvalue() + assert "UploadUnhappinessError" in e.output - assert "shares could be placed on only" in proto.output + assert "shares could be placed on only" in proto.output.getvalue() diff --git a/src/allmydata/scripts/tahoe_grid_manager.py b/src/allmydata/scripts/tahoe_grid_manager.py index 4b0a24d76..67c8b631b 100644 --- a/src/allmydata/scripts/tahoe_grid_manager.py +++ b/src/allmydata/scripts/tahoe_grid_manager.py @@ -3,7 +3,7 @@ from __future__ import print_function import sys import json import time -from datetime import datetime +from datetime import datetime, timedelta from allmydata.scripts.common import BaseOptions from allmydata.util.abbreviate import abbreviate_time @@ -81,15 +81,22 @@ class SignOptions(BaseOptions): ) def getSynopsis(self): - return "{} NAME".format(super(SignOptions, self).getSynopsis()) + return "{} NAME EXPIRY_DAYS".format(super(SignOptions, self).getSynopsis()) def parseArgs(self, *args, **kw): BaseOptions.parseArgs(self, **kw) - if len(args) != 1: + if len(args) != 2: raise usage.UsageError( - "Requires one argument: name" + "Requires two arguments: name expiry_days" ) self['name'] = unicode(args[0]) + self['expiry_days'] = int(args[1]) + if self['expiry_days'] < 1 or self['expiry_days'] > 20*365: + raise usage.UsageError( + "Certificate expires in an unreasonable number of days: {}".format( + self['expiry_days'], + ) + ) class GridManagerOptions(BaseOptions): @@ -236,15 +243,17 @@ class _GridManager(object): def public_identity(self): return ed25519.string_from_verifying_key(self._public_key) - def sign(self, name): + def sign(self, name, expiry_seconds): try: srv = self._storage_servers[name] except KeyError: raise KeyError( u"No storage server named '{}'".format(name) ) + expiration = datetime.utcnow() + timedelta(seconds=expiry_seconds) + epoch_offset = (expiration - datetime(1970, 1, 1)).total_seconds() cert_info = { - "expires": int(time.time() + 86400), # XXX FIXME + "expires": epoch_offset, "public_key": srv.public_key(), "version": 1, } @@ -422,7 +431,7 @@ def _list(gridoptions, options): while fp.child('{}.cert.{}'.format(name, cert_count)).exists(): container = json.load(fp.child('{}.cert.{}'.format(name, cert_count)).open('r')) cert_data = json.loads(container['certificate']) - expires = datetime.fromtimestamp(cert_data['expires']) + expires = datetime.utcfromtimestamp(cert_data['expires']) delta = datetime.utcnow() - expires if delta.total_seconds() < 0: print("{}: cert {}: valid until {} ({})".format(name, cert_count, expires, abbreviate_time(delta))) @@ -439,8 +448,10 @@ def _sign(gridoptions, options): fp = FilePath(gm_config) if gm_config.strip() != '-' else None gm = _load_gridmanager_config(gm_config) + expiry_seconds = int(options['expiry_days']) * 86400 + try: - certificate = gm.sign(options['name']) + certificate = gm.sign(options['name'], expiry_seconds) except KeyError: raise usage.UsageError( "No storage-server called '{}' exists".format(options['name']) From c65a01fbe0692fb6c9f784a2f5efc64807e520e8 Mon Sep 17 00:00:00 2001 From: meejah Date: Sun, 4 Oct 2020 21:56:53 -0600 Subject: [PATCH 0125/2309] flake8 --- src/allmydata/scripts/admin.py | 2 +- src/allmydata/scripts/tahoe_grid_manager.py | 1 - src/allmydata/test/test_grid_manager.py | 1 - 3 files changed, 1 insertion(+), 3 deletions(-) diff --git a/src/allmydata/scripts/admin.py b/src/allmydata/scripts/admin.py index 57cac39c7..e5b7dca2d 100644 --- a/src/allmydata/scripts/admin.py +++ b/src/allmydata/scripts/admin.py @@ -92,7 +92,7 @@ class AddGridManagerCertOptions(BaseOptions): try: self.certificate_data = parse_grid_manager_data(data) except ValueError as e: - raise UsageError( + raise usage.UsageError( "Error parsing certificate: {}".format(e) ) diff --git a/src/allmydata/scripts/tahoe_grid_manager.py b/src/allmydata/scripts/tahoe_grid_manager.py index 67c8b631b..f84f57766 100644 --- a/src/allmydata/scripts/tahoe_grid_manager.py +++ b/src/allmydata/scripts/tahoe_grid_manager.py @@ -2,7 +2,6 @@ from __future__ import print_function import sys import json -import time from datetime import datetime, timedelta from allmydata.scripts.common import BaseOptions diff --git a/src/allmydata/test/test_grid_manager.py b/src/allmydata/test/test_grid_manager.py index d5c98f2e7..a60739634 100644 --- a/src/allmydata/test/test_grid_manager.py +++ b/src/allmydata/test/test_grid_manager.py @@ -2,7 +2,6 @@ import json from allmydata.client import ( - _load_grid_manager_certificates, create_storage_farm_broker, ) from allmydata.node import ( From 817eef287d11cd14391986a201732c1b5ab23f62 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 20 Oct 2020 10:16:10 -0600 Subject: [PATCH 0126/2309] typo --- docs/proposed/grid-manager/managed-grid.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/proposed/grid-manager/managed-grid.rst b/docs/proposed/grid-manager/managed-grid.rst index 72eec7660..5720fd89d 100644 --- a/docs/proposed/grid-manager/managed-grid.rst +++ b/docs/proposed/grid-manager/managed-grid.rst @@ -8,7 +8,7 @@ Managed Grid ============ In a grid using an Introducer, a client will use any storage-server -the Introducer announces (and the Introducer will annoucne any +the Introducer announces (and the Introducer will announce any storage-server that connects to it). This means that anyone with the Introducer fURL can connect storage to the grid. From 91e1fa3c52fee6947867ad5a7a3650d9dbbf8b63 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 20 Oct 2020 10:16:23 -0600 Subject: [PATCH 0127/2309] correct config section --- docs/proposed/grid-manager/managed-grid.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/proposed/grid-manager/managed-grid.rst b/docs/proposed/grid-manager/managed-grid.rst index 5720fd89d..6583868cd 100644 --- a/docs/proposed/grid-manager/managed-grid.rst +++ b/docs/proposed/grid-manager/managed-grid.rst @@ -182,8 +182,8 @@ Enrolling a Storage Server: Config You may edit the ``[storage]`` section of the ``tahoe.cfg`` file to turn on grid-management with ``grid_management = true``. You then must -also provide a ``[grid_management_keys]`` section in the config-file which -lists ``name = path/to/certificate`` pairs. +also provide a ``[grid_management_certificates]`` section in the +config-file which lists ``name = path/to/certificate`` pairs. These certificate files are issued by the ``tahoe grid-manager sign`` command; these should be **securely transmitted** to the storage @@ -192,7 +192,7 @@ server. Relative paths are based from the node directory. Example:: [storage] grid_management = true - [grid_management_keys] + [grid_management_certificates] default = example_grid.cert This will cause us to give this certificate to any Introducers we From 6ea3c6842bca31782f77fe8c825beb88d07ddfe0 Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 29 Oct 2020 19:32:27 -0600 Subject: [PATCH 0128/2309] encoded -> tahoe-encoded --- docs/proposed/grid-manager/managed-grid.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/proposed/grid-manager/managed-grid.rst b/docs/proposed/grid-manager/managed-grid.rst index 6583868cd..01181d42a 100644 --- a/docs/proposed/grid-manager/managed-grid.rst +++ b/docs/proposed/grid-manager/managed-grid.rst @@ -95,7 +95,7 @@ tahoe grid-manager add Takes two args: ``name pubkey``. The ``name`` is an arbitrary local identifier for the new storage node (also sometimes called "a petname" -or "nickname"). The pubkey is the encoded key from a ``node.pubkey`` +or "nickname"). The pubkey is the tahoe-encoded key from a ``node.pubkey`` file in the storage-server's node directory (minus any whitespace). For example, if ``~/storage0`` contains a storage-node, you might do something like this: From f9231444784d56410ce84414b829ea3a3823c3ea Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 6 Nov 2020 17:28:49 -0700 Subject: [PATCH 0129/2309] re-factor; use a predicate instead of expanding NativeStorageServer --- src/allmydata/storage_client.py | 124 +++++++++++++++++++------------- 1 file changed, 76 insertions(+), 48 deletions(-) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 93df74319..5ffb44092 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -242,6 +242,11 @@ class StorageFarmBroker(service.MultiService): assert isinstance(server_id, unicode) # from YAML server_id = server_id.encode("ascii") handler_overrides = server.get("connections", {}) + gm_verifier = create_grid_manager_verifier( + self._grid_manager_keys, + server["ann"].get("grid-manager-certificates", []), + ) + s = NativeStorageServer( server_id, server["ann"], @@ -249,8 +254,7 @@ class StorageFarmBroker(service.MultiService): handler_overrides, self.node_config, self.storage_client_config, - self._grid_manager_keys, - server["ann"].get("grid-manager-certificates", []), + gm_verifier, ) s.on_status_changed(lambda _: self._got_connection()) return s @@ -459,7 +463,7 @@ def parse_grid_manager_data(gm_data): return js -def validate_grid_manager_certificate(gm_key, alleged_cert, now_fn=None): +def _validate_grid_manager_certificate(gm_key, alleged_cert): """ :param gm_key: a VerifyingKey instance, a Grid Manager's public key. @@ -468,15 +472,10 @@ def validate_grid_manager_certificate(gm_key, alleged_cert, now_fn=None): "certificate" contains a JSON-serialized certificate for a Storage Server (comes from a Grid Manager). - :param now_fn: a zero-argument callable that returns a UTC - timestamp (will use datetime.utcnow by default) - - :return: False if the signature is invalid or the certificate is - expired. + :return: a dict consisting of the deserialized certificate data or + None if the signature is invalid. Note we do NOT check the + expiry time in this function. """ - if now_fn is None: - now_fn = datetime.utcnow - try: ed25519.verify_signature( gm_key, @@ -484,15 +483,70 @@ def validate_grid_manager_certificate(gm_key, alleged_cert, now_fn=None): alleged_cert['certificate'].encode('ascii'), ) except ed25519.BadSignature: - return False + return None # signature is valid; now we can load the actual data cert = json.loads(alleged_cert['certificate']) - now = now_fn() - expires = datetime.utcfromtimestamp(cert['expires']) - # cert_pubkey = keyutil.parse_pubkey(cert['public_key'].encode('ascii')) - if expires < now: - return False # certificate is expired - return True + return cert + + +def create_grid_manager_verifier(keys, certs, now_fn=None): + """ + Creates a predicate for confirming some Grid Manager-issued + certificates against Grid Manager keys. A predicate is used + (instead of just returning True/False here) so that the + expiry-time can be tested on each call. + + :param list keys: 0 or more `VerifyingKey` instances + + :param list certs: 1 or more Grid Manager certificates each of + which is a `dict` containing 'signature' and 'certificate' keys. + + :param callable now_fn: a callable which returns the current UTC + timestamp (or datetime.utcnow if None). + + :returns: a callable which will return True only-if there is at + least one valid certificate in `certs` signed by one of the keys + in `keys`. + """ + + now_fn = datetime.utcnow if now_fn is None else now_fn + valid_certs = [] + + # if we have zero grid-manager keys then everything is valid + if not keys: + return lambda: True + + # validate the signatures on any certificates we have (not yet the expiry dates) + for alleged_cert in certs: + for key in keys: + cert = _validate_grid_manager_certificate(key, alleged_cert) + if cert is not None: + valid_certs.append(cert) + else: + # we might want to let the user know about this + # failed-to-verify certificate .. but also if you have + # multiple grid-managers then a bunch of these + # messages would appear + print( + "Grid Manager certificate signature failed. Certificate: " + "\"{cert}\" for key \"{key}\".".format( + key=key, + cert=alleged_cert, + ) + ) + + def validate(): + now = now_fn() + # if *any* certificate is still valid then we consider the server valid + for cert in valid_certs: + expires = datetime.utcfromtimestamp(cert['expires']) + # cert_pubkey = keyutil.parse_pubkey(cert['public_key'].encode('ascii')) + if expires > now: + # not-expired + return True + return False + + return validate class IFoolscapStorageServer(Interface): @@ -733,7 +787,7 @@ class NativeStorageServer(service.MultiService): } def __init__(self, server_id, ann, tub_maker, handler_overrides, node_config, config=None, - grid_manager_keys=None, grid_manager_certs=None): + grid_manager_verifier=None): service.MultiService.__init__(self) assert isinstance(server_id, bytes) self._server_id = server_id @@ -744,18 +798,7 @@ class NativeStorageServer(service.MultiService): if config is None: config = StorageClientConfig() - # XXX we should validate as much as we can about the - # certificates right now -- the only thing we HAVE to be lazy - # about is the expiry, which should be checked before any - # possible upload... - - # any public-keys which the user has configured (if none, it - # means use any storage servers) - self._grid_manager_keys = grid_manager_keys or [] - - # any storage-certificates that this storage-server included - # in its announcement - self._grid_manager_certificates = grid_manager_certs or [] + self._grid_manager_verifier = grid_manager_verifier self._storage = self._make_storage_system(node_config, config, ann) @@ -778,25 +821,10 @@ class NativeStorageServer(service.MultiService): :return: True if we should use this server for uploads, False otherwise. """ - # print("upload permitted? {}".format(self._server_id)) # if we have no Grid Manager keys configured, choice is easy - if not self._grid_manager_keys: - # print("{} no grid manager keys at all (so yes)".format(self._server_id)) + if self._grid_manager_verifier is None: return True - - # XXX probably want to cache the answer to this? (ignoring - # that for now because certificates expire, so .. slightly - # more complex) - if not self._grid_manager_certificates: - # print("{} no grid-manager certificates {} (so no)".format(self._server_id, self._grid_manager_certificates)) - return False - for gm_key in self._grid_manager_keys: - for cert in self._grid_manager_certificates: - if validate_grid_manager_certificate(gm_key, cert): - # print("valid: {}\n{}".format(gm_key, cert)) - return True - # print("didn't validate {} keys".format(len(self._grid_manager_keys))) - return False + return self._grid_manager_verifier() def _make_storage_system(self, node_config, config, ann): """ From 30b7be6f1d0b7fe27e3ff93558fc7387312f926a Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 6 Nov 2020 18:07:30 -0700 Subject: [PATCH 0130/2309] remove integration tests/refactoring --- integration/conftest.py | 158 ++++++-- integration/grid.py | 480 ----------------------- integration/test_grid_manager.py | 245 ------------ integration/test_servers_of_happiness.py | 7 +- integration/test_tor.py | 39 +- integration/test_web.py | 10 +- integration/util.py | 54 +-- 7 files changed, 161 insertions(+), 832 deletions(-) delete mode 100644 integration/grid.py delete mode 100644 integration/test_grid_manager.py diff --git a/integration/conftest.py b/integration/conftest.py index 1fb3fd761..04e3dcb52 100644 --- a/integration/conftest.py +++ b/integration/conftest.py @@ -36,11 +36,6 @@ from util import ( await_client_ready, TahoeProcess, ) -from grid import ( - create_port_allocator, - create_flog_gatherer, - create_grid, -) # pytest customization hooks @@ -77,12 +72,6 @@ def reactor(): return _reactor -@pytest.fixture(scope='session') -@log_call(action_type=u"integration:port_allocator", include_result=False) -def port_allocator(reactor): - return create_port_allocator(start_port=45000) - - @pytest.fixture(scope='session') @log_call(action_type=u"integration:temp_dir", include_args=[]) def temp_dir(request): @@ -117,30 +106,127 @@ def flog_binary(): @pytest.fixture(scope='session') @log_call(action_type=u"integration:flog_gatherer", include_args=[]) def flog_gatherer(reactor, temp_dir, flog_binary, request): - fg = pytest_twisted.blockon( - create_flog_gatherer(reactor, request, temp_dir, flog_binary) + out_protocol = _CollectOutputProtocol() + gather_dir = join(temp_dir, 'flog_gather') + reactor.spawnProcess( + out_protocol, + flog_binary, + ( + 'flogtool', 'create-gatherer', + '--location', 'tcp:localhost:3117', + '--port', '3117', + gather_dir, + ) ) - return fg + pytest_twisted.blockon(out_protocol.done) + + twistd_protocol = _MagicTextProtocol("Gatherer waiting at") + twistd_process = reactor.spawnProcess( + twistd_protocol, + which('twistd')[0], + ( + 'twistd', '--nodaemon', '--python', + join(gather_dir, 'gatherer.tac'), + ), + path=gather_dir, + ) + pytest_twisted.blockon(twistd_protocol.magic_seen) + + def cleanup(): + _cleanup_tahoe_process(twistd_process, twistd_protocol.exited) + + flog_file = mktemp('.flog_dump') + flog_protocol = _DumpOutputProtocol(open(flog_file, 'w')) + flog_dir = join(temp_dir, 'flog_gather') + flogs = [x for x in listdir(flog_dir) if x.endswith('.flog')] + + print("Dumping {} flogtool logfiles to '{}'".format(len(flogs), flog_file)) + reactor.spawnProcess( + flog_protocol, + flog_binary, + ( + 'flogtool', 'dump', join(temp_dir, 'flog_gather', flogs[0]) + ), + ) + print("Waiting for flogtool to complete") + try: + pytest_twisted.blockon(flog_protocol.done) + except ProcessTerminated as e: + print("flogtool exited unexpectedly: {}".format(str(e))) + print("Flogtool completed") + + request.addfinalizer(cleanup) + + with open(join(gather_dir, 'log_gatherer.furl'), 'r') as f: + furl = f.read().strip() + return furl @pytest.fixture(scope='session') -@log_call(action_type=u"integration:grid", include_args=[]) -def grid(reactor, request, temp_dir, flog_gatherer, port_allocator): - g = pytest_twisted.blockon( - create_grid(reactor, request, temp_dir, flog_gatherer, port_allocator) +@log_call( + action_type=u"integration:introducer", + include_args=["temp_dir", "flog_gatherer"], + include_result=False, +) +def introducer(reactor, temp_dir, flog_gatherer, request): + config = ''' +[node] +nickname = introducer0 +web.port = 4560 +log_gatherer.furl = {log_furl} +'''.format(log_furl=flog_gatherer) + + intro_dir = join(temp_dir, 'introducer') + print("making introducer", intro_dir) + + if not exists(intro_dir): + mkdir(intro_dir) + done_proto = _ProcessExitedProtocol() + _tahoe_runner_optional_coverage( + done_proto, + reactor, + request, + ( + 'create-introducer', + '--listen=tcp', + '--hostname=localhost', + intro_dir, + ), + ) + pytest_twisted.blockon(done_proto.done) + + # over-write the config file with our stuff + with open(join(intro_dir, 'tahoe.cfg'), 'w') as f: + f.write(config) + + # on windows, "tahoe start" means: run forever in the foreground, + # but on linux it means daemonize. "tahoe run" is consistent + # between platforms. + protocol = _MagicTextProtocol('introducer running') + transport = _tahoe_runner_optional_coverage( + protocol, + reactor, + request, + ( + 'run', + intro_dir, + ), ) - return g + request.addfinalizer(partial(_cleanup_tahoe_process, transport, protocol.exited)) - -@pytest.fixture(scope='session') -def introducer(grid): - return grid.introducer + pytest_twisted.blockon(protocol.magic_seen) + return TahoeProcess(transport, intro_dir) @pytest.fixture(scope='session') @log_call(action_type=u"integration:introducer:furl", include_args=["temp_dir"]) def introducer_furl(introducer, temp_dir): - return introducer.furl + furl_fname = join(temp_dir, 'introducer', 'private', 'introducer.furl') + while not exists(furl_fname): + print("Don't see {} yet".format(furl_fname)) + sleep(.1) + furl = open(furl_fname, 'r').read() + return furl @pytest.fixture(scope='session') @@ -219,20 +305,28 @@ def tor_introducer_furl(tor_introducer, temp_dir): @pytest.fixture(scope='session') @log_call( action_type=u"integration:storage_nodes", - include_args=["grid"], + include_args=["temp_dir", "introducer_furl", "flog_gatherer"], include_result=False, ) -def storage_nodes(grid): +def storage_nodes(reactor, temp_dir, introducer, introducer_furl, flog_gatherer, request): nodes_d = [] # start all 5 nodes in parallel for x in range(5): - #nodes_d.append(grid.add_storage_node()) - pytest_twisted.blockon(grid.add_storage_node()) - + name = 'node{}'.format(x) + web_port= 9990 + x + nodes_d.append( + _create_node( + reactor, request, temp_dir, introducer_furl, flog_gatherer, name, + web_port="tcp:{}:interface=localhost".format(web_port), + storage=True, + ) + ) nodes_status = pytest_twisted.blockon(DeferredList(nodes_d)) - for ok, value in nodes_status: - assert ok, "Storage node creation failed: {}".format(value) - return grid.storage_servers + nodes = [] + for ok, process in nodes_status: + assert ok, "Storage node creation failed: {}".format(process) + nodes.append(process) + return nodes @pytest.fixture(scope='session') diff --git a/integration/grid.py b/integration/grid.py deleted file mode 100644 index f39f04d3c..000000000 --- a/integration/grid.py +++ /dev/null @@ -1,480 +0,0 @@ -""" -Classes which directly represent various kinds of Tahoe processes -that co-operate to for "a Grid". - -These methods and objects are used by conftest.py fixtures but may -also be used as direct helpers for tests that don't want to (or can't) -rely on 'the' global grid as provided by fixtures like 'alice' or -'storage_servers'. -""" - -from os import mkdir, listdir, environ -from os.path import join, exists -from tempfile import mkdtemp, mktemp - -from eliot import ( - log_call, -) - -from twisted.python.procutils import which -from twisted.internet.defer import ( - inlineCallbacks, - returnValue, - maybeDeferred, -) -from twisted.internet.task import ( - deferLater, -) -from twisted.internet.interfaces import ( - IProcessTransport, - IProcessProtocol, - IProtocol, -) -from twisted.internet.endpoints import ( - TCP4ServerEndpoint, -) -from twisted.internet.protocol import ( - Factory, - Protocol, -) - -from util import ( - _CollectOutputProtocol, - _MagicTextProtocol, - _DumpOutputProtocol, - _ProcessExitedProtocol, - _create_node, - _run_node, - _cleanup_tahoe_process, - _tahoe_runner_optional_coverage, - TahoeProcess, - await_client_ready, -) - -import attr -import pytest_twisted - - -# further directions: -# - "Grid" is unused, basically -- tie into the rest? -# - could make a Grid instance mandatory for create_* calls -# - could instead make create_* calls methods of Grid -# - Bring more 'util' or 'conftest' code into here -# - stop()/start()/restart() methods on StorageServer etc -# - more-complex stuff like config changes (which imply a restart too)? - - -@attr.s -class FlogGatherer(object): - """ - Flog Gatherer process. - """ - - process = attr.ib( - validator=attr.validators.provides(IProcessTransport) - ) - protocol = attr.ib( - validator=attr.validators.provides(IProcessProtocol) - ) - furl = attr.ib() - - -@inlineCallbacks -def create_flog_gatherer(reactor, request, temp_dir, flog_binary): - out_protocol = _CollectOutputProtocol() - gather_dir = join(temp_dir, 'flog_gather') - reactor.spawnProcess( - out_protocol, - flog_binary, - ( - 'flogtool', 'create-gatherer', - '--location', 'tcp:localhost:3117', - '--port', '3117', - gather_dir, - ) - ) - yield out_protocol.done - - twistd_protocol = _MagicTextProtocol("Gatherer waiting at") - twistd_process = reactor.spawnProcess( - twistd_protocol, - which('twistd')[0], - ( - 'twistd', '--nodaemon', '--python', - join(gather_dir, 'gatherer.tac'), - ), - path=gather_dir, - ) - yield twistd_protocol.magic_seen - - def cleanup(): - _cleanup_tahoe_process(twistd_process, twistd_protocol.exited) - - flog_file = mktemp('.flog_dump') - flog_protocol = _DumpOutputProtocol(open(flog_file, 'w')) - flog_dir = join(temp_dir, 'flog_gather') - flogs = [x for x in listdir(flog_dir) if x.endswith('.flog')] - - print("Dumping {} flogtool logfiles to '{}'".format(len(flogs), flog_file)) - reactor.spawnProcess( - flog_protocol, - flog_binary, - ( - 'flogtool', 'dump', join(temp_dir, 'flog_gather', flogs[0]) - ), - ) - print("Waiting for flogtool to complete") - try: - pytest_twisted.blockon(flog_protocol.done) - except ProcessTerminated as e: - print("flogtool exited unexpectedly: {}".format(str(e))) - print("Flogtool completed") - - request.addfinalizer(cleanup) - - with open(join(gather_dir, 'log_gatherer.furl'), 'r') as f: - furl = f.read().strip() - returnValue( - FlogGatherer( - protocol=twistd_protocol, - process=twistd_process, - furl=furl, - ) - ) - - -@attr.s -class StorageServer(object): - """ - Represents a Tahoe Storage Server - """ - - process = attr.ib( - validator=attr.validators.instance_of(TahoeProcess) - ) - protocol = attr.ib( - validator=attr.validators.provides(IProcessProtocol) - ) - - @inlineCallbacks - def restart(self, reactor, request): - """ - re-start our underlying process by issuing a TERM, waiting and - then running again. await_client_ready() will be done as well - - Note that self.process and self.protocol will be new instances - after this. - """ - self.process.transport.signalProcess('TERM') - yield self.protocol.exited - self.process = yield _run_node( - reactor, self.process.node_dir, request, None, - ) - self.protocol = self.process.transport._protocol - - -@inlineCallbacks -def create_storage_server(reactor, request, temp_dir, introducer, flog_gatherer, name, web_port, - needed=2, happy=3, total=4): - """ - Create a new storage server - """ - from util import _create_node - node_process = yield _create_node( - reactor, request, temp_dir, introducer.furl, flog_gatherer, - name, web_port, storage=True, needed=needed, happy=happy, total=total, - ) - storage = StorageServer( - process=node_process, - protocol=node_process.transport._protocol, - ) - returnValue(storage) - - -@attr.s -class Client(object): - """ - Represents a Tahoe client - """ - - process = attr.ib( - validator=attr.validators.instance_of(TahoeProcess) - ) - protocol = attr.ib( - validator=attr.validators.provides(IProcessProtocol) - ) - - @inlineCallbacks - def restart(self, reactor, request, servers=1): - """ - re-start our underlying process by issuing a TERM, waiting and - then running again. - - :param int servers: number of server connections we will wait - for before being 'ready' - - Note that self.process and self.protocol will be new instances - after this. - """ - self.process.transport.signalProcess('TERM') - yield self.protocol.exited - process = yield _run_node( - reactor, self.process.node_dir, request, None, - ) - self.process = process - self.protocol = self.process.transport._protocol - - - # XXX add stop / start / restart - # ...maybe "reconfig" of some kind? - - -@inlineCallbacks -def create_client(reactor, request, temp_dir, introducer, flog_gatherer, name, web_port, - needed=2, happy=3, total=4): - """ - Create a new storage server - """ - from util import _create_node - node_process = yield _create_node( - reactor, request, temp_dir, introducer.furl, flog_gatherer, - name, web_port, storage=False, needed=needed, happy=happy, total=total, - ) - returnValue( - Client( - process=node_process, - protocol=node_process.transport._protocol, - ) - ) - - -@attr.s -class Introducer(object): - """ - Reprsents a running introducer - """ - - process = attr.ib( - validator=attr.validators.instance_of(TahoeProcess) - ) - protocol = attr.ib( - validator=attr.validators.provides(IProcessProtocol) - ) - furl = attr.ib() - - -@inlineCallbacks -@log_call( - action_type=u"integration:introducer", - include_args=["temp_dir", "flog_gatherer"], - include_result=False, -) -def create_introducer(reactor, request, temp_dir, flog_gatherer, port): - """ - Run a new Introducer and return an Introducer instance. - """ - config = ( - '[node]\n' - 'nickname = introducer{port}\n' - 'web.port = {port}\n' - 'log_gatherer.furl = {log_furl}\n' - ).format( - port=port, - log_furl=flog_gatherer.furl, - ) - - intro_dir = join(temp_dir, 'introducer{}'.format(port)) - - if not exists(intro_dir): - mkdir(intro_dir) - done_proto = _ProcessExitedProtocol() - _tahoe_runner_optional_coverage( - done_proto, - reactor, - request, - ( - 'create-introducer', - '--listen=tcp', - '--hostname=localhost', - intro_dir, - ), - ) - yield done_proto.done - - # over-write the config file with our stuff - with open(join(intro_dir, 'tahoe.cfg'), 'w') as f: - f.write(config) - - # on windows, "tahoe start" means: run forever in the foreground, - # but on linux it means daemonize. "tahoe run" is consistent - # between platforms. - protocol = _MagicTextProtocol('introducer running') - transport = _tahoe_runner_optional_coverage( - protocol, - reactor, - request, - ( - 'run', - intro_dir, - ), - ) - - def clean(): - return _cleanup_tahoe_process(transport, protocol.exited) - request.addfinalizer(clean) - - yield protocol.magic_seen - - furl_fname = join(intro_dir, 'private', 'introducer.furl') - while not exists(furl_fname): - print("Don't see {} yet".format(furl_fname)) - yield deferLater(reactor, .1, lambda: None) - furl = open(furl_fname, 'r').read() - - returnValue( - Introducer( - process=TahoeProcess(transport, intro_dir), - protocol=protocol, - furl=furl, - ) - ) - - -@attr.s -class Grid(object): - """ - Represents an entire Tahoe Grid setup - - A Grid includes an Introducer, Flog Gatherer and some number of - Storage Servers. - """ - - _reactor = attr.ib() - _request = attr.ib() - _temp_dir = attr.ib() - _port_allocator = attr.ib() - introducer = attr.ib() - flog_gatherer = attr.ib() - storage_servers = attr.ib(factory=list) - clients = attr.ib(factory=dict) - - @storage_servers.validator - def check(self, attribute, value): - for server in value: - if not isinstance(server, StorageServer): - raise ValueError( - "storage_servers must be StorageServer" - ) - - @inlineCallbacks - def add_storage_node(self): - """ - Creates a new storage node, returns a StorageServer instance - (which will already be added to our .storage_servers list) - """ - port = yield self._port_allocator() - print("make {}".format(port)) - name = 'node{}'.format(port) - web_port = 'tcp:{}:interface=localhost'.format(port) - server = yield create_storage_server( - self._reactor, - self._request, - self._temp_dir, - self.introducer, - self.flog_gatherer, - name, - web_port, - ) - self.storage_servers.append(server) - returnValue(server) - - @inlineCallbacks - def add_client(self, name, needed=2, happy=3, total=4): - """ - Create a new client node - """ - port = yield self._port_allocator() - web_port = 'tcp:{}:interface=localhost'.format(port) - client = yield create_client( - self._reactor, - self._request, - self._temp_dir, - self.introducer, - self.flog_gatherer, - name, - web_port, - needed=needed, - happy=happy, - total=total, - ) - self.clients[name] = client - yield await_client_ready(client.process) - returnValue(client) - - - -# XXX THINK can we tie a whole *grid* to a single request? (I think -# that's all that makes sense) -@inlineCallbacks -def create_grid(reactor, request, temp_dir, flog_gatherer, port_allocator): - """ - """ - intro_port = yield port_allocator() - introducer = yield create_introducer(reactor, request, temp_dir, flog_gatherer, intro_port) - grid = Grid( - reactor, - request, - temp_dir, - port_allocator, - introducer, - flog_gatherer, - ) - returnValue(grid) - - -def create_port_allocator(start_port): - """ - Returns a new port-allocator .. which is a zero-argument function - that returns Deferreds that fire with new, sequential ports - starting at `start_port` skipping any that already appear to have - a listener. - - There can still be a race against other processes allocating ports - -- between the time when we check the status of the port and when - our subprocess starts up. This *could* be mitigated by instructing - the OS to not randomly-allocate ports in some range, and then - using that range here (explicitly, ourselves). - - NB once we're Python3-only this could be an async-generator - """ - port = [start_port - 1] - - # import stays here to not interfere with reactor selection -- but - # maybe this function should be arranged to be called once from a - # fixture (with the reactor)? - from twisted.internet import reactor - - class NothingProtocol(Protocol): - """ - I do nothing. - """ - - def port_generator(): - print("Checking port {}".format(port)) - port[0] += 1 - ep = TCP4ServerEndpoint(reactor, port[0], interface="localhost") - d = ep.listen(Factory.forProtocol(NothingProtocol)) - - def good(listening_port): - unlisten_d = maybeDeferred(listening_port.stopListening) - def return_port(_): - return port[0] - unlisten_d.addBoth(return_port) - return unlisten_d - - def try_again(fail): - return port_generator() - - d.addCallbacks(good, try_again) - return d - return port_generator diff --git a/integration/test_grid_manager.py b/integration/test_grid_manager.py deleted file mode 100644 index db93f17c7..000000000 --- a/integration/test_grid_manager.py +++ /dev/null @@ -1,245 +0,0 @@ -import sys -import time -import json -import shutil -from os import mkdir, unlink, listdir, utime -from os.path import join, exists, getmtime - -from cryptography.hazmat.primitives.serialization import ( - Encoding, - PublicFormat, -) - -from allmydata.crypto import ed25519 -from allmydata.util import base32 -from allmydata.util import configutil - -import util -from grid import ( - create_grid, -) - -import pytest_twisted - - -@pytest_twisted.inlineCallbacks -def test_create_certificate(reactor, request): - """ - The Grid Manager produces a valid, correctly-signed certificate. - """ - gm_config = yield util.run_tahoe( - reactor, request, "grid-manager", "--config", "-", "create", - ) - privkey_bytes = json.loads(gm_config)['private_key'].encode('ascii') - privkey, pubkey = ed25519.signing_keypair_from_string(privkey_bytes) - - # Note that zara + her key here are arbitrary and don't match any - # "actual" clients in the test-grid; we're just checking that the - # Grid Manager signs this properly. - gm_config = yield util.run_tahoe( - reactor, request, "grid-manager", "--config", "-", "add", - "zara", "pub-v0-kzug3ut2m7ziihf3ndpqlquuxeie4foyl36wn54myqc4wmiwe4ga", - stdin=gm_config, - ) - zara_cert_bytes = yield util.run_tahoe( - reactor, request, "grid-manager", "--config", "-", "sign", "zara", "1", - stdin=gm_config, - ) - zara_cert = json.loads(zara_cert_bytes) - - # confirm that zara's certificate is made by the Grid Manager - # (.verify returns None on success, raises exception on error) - pubkey.verify( - base32.a2b(zara_cert['signature'].encode('ascii')), - zara_cert['certificate'].encode('ascii'), - ) - - -@pytest_twisted.inlineCallbacks -def test_remove_client(reactor, request): - """ - A Grid Manager can add and successfully remove a client - """ - gm_config = yield util.run_tahoe( - reactor, request, "grid-manager", "--config", "-", "create", - ) - - gm_config = yield util.run_tahoe( - reactor, request, "grid-manager", "--config", "-", "add", - "zara", "pub-v0-kzug3ut2m7ziihf3ndpqlquuxeie4foyl36wn54myqc4wmiwe4ga", - stdin=gm_config, - ) - gm_config = yield util.run_tahoe( - reactor, request, "grid-manager", "--config", "-", "add", - "yakov", "pub-v0-kvxhb3nexybmipkrar2ztfrwp4uxxsmrjzkpzafit3ket4u5yldq", - stdin=gm_config, - ) - assert "zara" in json.loads(gm_config)['storage_servers'] - assert "yakov" in json.loads(gm_config)['storage_servers'] - - gm_config = yield util.run_tahoe( - reactor, request, "grid-manager", "--config", "-", "remove", - "zara", - stdin=gm_config, - ) - assert "zara" not in json.loads(gm_config)['storage_servers'] - assert "yakov" in json.loads(gm_config)['storage_servers'] - - -@pytest_twisted.inlineCallbacks -def test_remove_last_client(reactor, request): - """ - A Grid Manager can remove all clients - """ - gm_config = yield util.run_tahoe( - reactor, request, "grid-manager", "--config", "-", "create", - ) - - gm_config = yield util.run_tahoe( - reactor, request, "grid-manager", "--config", "-", "add", - "zara", "pub-v0-kzug3ut2m7ziihf3ndpqlquuxeie4foyl36wn54myqc4wmiwe4ga", - stdin=gm_config, - ) - assert "zara" in json.loads(gm_config)['storage_servers'] - - gm_config = yield util.run_tahoe( - reactor, request, "grid-manager", "--config", "-", "remove", - "zara", - stdin=gm_config, - ) - # there are no storage servers left at all now - assert "storage_servers" not in json.loads(gm_config) - - -@pytest_twisted.inlineCallbacks -def test_add_remove_client_file(reactor, request, temp_dir): - """ - A Grid Manager can add and successfully remove a client (when - keeping data on disk) - """ - gmconfig = join(temp_dir, "gmtest") - gmconfig_file = join(temp_dir, "gmtest", "config.json") - yield util.run_tahoe( - reactor, request, "grid-manager", "--config", gmconfig, "create", - ) - - yield util.run_tahoe( - reactor, request, "grid-manager", "--config", gmconfig, "add", - "zara", "pub-v0-kzug3ut2m7ziihf3ndpqlquuxeie4foyl36wn54myqc4wmiwe4ga", - ) - yield util.run_tahoe( - reactor, request, "grid-manager", "--config", gmconfig, "add", - "yakov", "pub-v0-kvxhb3nexybmipkrar2ztfrwp4uxxsmrjzkpzafit3ket4u5yldq", - ) - assert "zara" in json.load(open(gmconfig_file, "r"))['storage_servers'] - assert "yakov" in json.load(open(gmconfig_file, "r"))['storage_servers'] - - yield util.run_tahoe( - reactor, request, "grid-manager", "--config", gmconfig, "remove", - "zara", - ) - assert "zara" not in json.load(open(gmconfig_file, "r"))['storage_servers'] - assert "yakov" in json.load(open(gmconfig_file, "r"))['storage_servers'] - - -@pytest_twisted.inlineCallbacks -def test_reject_storage_server(reactor, request, temp_dir, flog_gatherer, port_allocator): - """ - A client with happines=2 fails to upload to a Grid when it is - using Grid Manager and there is only 1 storage server with a valid - certificate. - """ - grid = yield create_grid(reactor, request, temp_dir, flog_gatherer, port_allocator) - storage0 = yield grid.add_storage_node() - storage1 = yield grid.add_storage_node() - - gm_config = yield util.run_tahoe( - reactor, request, "grid-manager", "--config", "-", "create", - ) - gm_privkey_bytes = json.loads(gm_config)['private_key'].encode('ascii') - gm_privkey, gm_pubkey = ed25519.signing_keypair_from_string(gm_privkey_bytes) - - # create certificate for the first storage-server - pubkey_fname = join(storage0.process.node_dir, "node.pubkey") - with open(pubkey_fname, 'r') as f: - pubkey_str = f.read().strip() - - gm_config = yield util.run_tahoe( - reactor, request, "grid-manager", "--config", "-", "add", - "storage0", pubkey_str, - stdin=gm_config, - ) - assert json.loads(gm_config)['storage_servers'].keys() == ['storage0'] - - print("inserting certificate") - cert = yield util.run_tahoe( - reactor, request, "grid-manager", "--config", "-", "sign", "storage0", "1", - stdin=gm_config, - ) - - yield util.run_tahoe( - reactor, request, "--node-directory", storage0.process.node_dir, - "admin", "add-grid-manager-cert", - "--name", "default", - "--filename", "-", - stdin=cert, - ) - - # re-start this storage server - yield storage0.restart(reactor, request) - - # now only one storage-server has the certificate .. configure - # diana to have the grid-manager certificate - - diana = yield grid.add_client("diana", needed=2, happy=2, total=2) - - config = configutil.get_config(join(diana.process.node_dir, "tahoe.cfg")) - config.add_section("grid_managers") - config.set("grid_managers", "test", ed25519.string_from_verifying_key(gm_pubkey)) - with open(join(diana.process.node_dir, "tahoe.cfg"), "w") as f: - config.write(f) - - yield diana.restart(reactor, request, servers=2) - - # try to put something into the grid, which should fail (because - # diana has happy=2 but should only find storage0 to be acceptable - # to upload to) - - try: - yield util.run_tahoe( - reactor, request, "--node-directory", diana.process.node_dir, - "put", "-", - stdin="some content\n" * 200, - ) - assert False, "Should get a failure" - except util.ProcessFailed as e: - assert 'UploadUnhappinessError' in e.output - - -@pytest_twisted.inlineCallbacks -def test_identity(reactor, request, temp_dir): - """ - Dump public key to CLI - """ - gm_config = join(temp_dir, "test_identity") - yield util.run_tahoe( - reactor, request, "grid-manager", "--config", gm_config, "create", - ) - - # ask the CLI for the grid-manager pubkey - pubkey = yield util.run_tahoe( - reactor, request, "grid-manager", "--config", gm_config, "public-identity", - ) - alleged_pubkey = ed25519.verifying_key_from_string(pubkey.strip()) - - # load the grid-manager pubkey "ourselves" - with open(join(gm_config, "config.json"), "r") as f: - real_config = json.load(f) - real_privkey, real_pubkey = ed25519.signing_keypair_from_string( - real_config["private_key"].encode("ascii"), - ) - - # confirm the CLI told us the correct thing - alleged_bytes = alleged_pubkey.public_bytes(Encoding.Raw, PublicFormat.Raw) - real_bytes = real_pubkey.public_bytes(Encoding.Raw, PublicFormat.Raw) - assert alleged_bytes == real_bytes, "Keys don't match" diff --git a/integration/test_servers_of_happiness.py b/integration/test_servers_of_happiness.py index d98bfe0b8..e5e4eb565 100644 --- a/integration/test_servers_of_happiness.py +++ b/integration/test_servers_of_happiness.py @@ -39,7 +39,8 @@ def test_upload_immutable(reactor, temp_dir, introducer_furl, flog_gatherer, sto try: yield proto.done assert False, "should raise exception" - except util.ProcessFailed as e: - assert "UploadUnhappinessError" in e.output + except Exception as e: + assert isinstance(e, ProcessTerminated) - assert "shares could be placed on only" in proto.output.getvalue() + output = proto.output.getvalue() + assert "shares could be placed on only" in output diff --git a/integration/test_tor.py b/integration/test_tor.py index 2423ef7d8..28360207a 100644 --- a/integration/test_tor.py +++ b/integration/test_tor.py @@ -76,28 +76,25 @@ def _create_anonymous_node(reactor, name, control_port, request, temp_dir, flog_ node_dir = join(temp_dir, name) web_port = "tcp:{}:interface=localhost".format(control_port + 2000) - if exists(node_dir): - raise RuntimeError( - "A node already exists in '{}'".format(node_dir) + if True: + print("creating", node_dir) + mkdir(node_dir) + proto = util._DumpOutputProtocol(None) + reactor.spawnProcess( + proto, + sys.executable, + ( + sys.executable, '-m', 'allmydata.scripts.runner', + 'create-node', + '--nickname', name, + '--introducer', introducer_furl, + '--hide-ip', + '--tor-control-port', 'tcp:localhost:{}'.format(control_port), + '--listen', 'tor', + node_dir, + ) ) - print("creating", node_dir) - mkdir(node_dir) - proto = util._DumpOutputProtocol(None) - reactor.spawnProcess( - proto, - sys.executable, - ( - sys.executable, '-m', 'allmydata.scripts.runner', - 'create-node', - '--nickname', name, - '--introducer', introducer_furl, - '--hide-ip', - '--tor-control-port', 'tcp:localhost:{}'.format(control_port), - '--listen', 'tor', - node_dir, - ) - ) - yield proto.done + yield proto.done with open(join(node_dir, 'tahoe.cfg'), 'w') as f: f.write(''' diff --git a/integration/test_web.py b/integration/test_web.py index 36a7d3757..575e4fc1a 100644 --- a/integration/test_web.py +++ b/integration/test_web.py @@ -96,7 +96,7 @@ def test_helper_status(storage_nodes): successfully GET the /helper_status page """ - url = util.node_url(storage_nodes[0].process.node_dir, "helper_status") + url = util.node_url(storage_nodes[0].node_dir, "helper_status") resp = requests.get(url) assert resp.status_code >= 200 and resp.status_code < 300 dom = BeautifulSoup(resp.content, "html5lib") @@ -416,7 +416,7 @@ def test_storage_info(storage_nodes): storage0 = storage_nodes[0] requests.get( - util.node_url(storage0.process.node_dir, u"storage"), + util.node_url(storage0.node_dir, u"storage"), ) @@ -427,7 +427,7 @@ def test_storage_info_json(storage_nodes): storage0 = storage_nodes[0] resp = requests.get( - util.node_url(storage0.process.node_dir, u"storage"), + util.node_url(storage0.node_dir, u"storage"), params={u"t": u"json"}, ) data = json.loads(resp.content) @@ -439,12 +439,12 @@ def test_introducer_info(introducer): retrieve and confirm /introducer URI for the introducer """ resp = requests.get( - util.node_url(introducer.process.node_dir, u""), + util.node_url(introducer.node_dir, u""), ) assert "Introducer" in resp.content resp = requests.get( - util.node_url(introducer.process.node_dir, u""), + util.node_url(introducer.node_dir, u""), params={u"t": u"json"}, ) data = json.loads(resp.content) diff --git a/integration/util.py b/integration/util.py index 5cad59c12..bbcf5efc6 100644 --- a/integration/util.py +++ b/integration/util.py @@ -5,7 +5,6 @@ from os import mkdir from os.path import exists, join from six.moves import StringIO from functools import partial -from shutil import rmtree from twisted.internet.defer import Deferred, succeed from twisted.internet.protocol import ProcessProtocol @@ -36,38 +35,15 @@ class _ProcessExitedProtocol(ProcessProtocol): self.done.callback(None) -class ProcessFailed(Exception): - """ - A subprocess has failed. - - :ivar ProcessTerminated reason: the original reason from .processExited - - :ivar StringIO output: all stdout and stderr collected to this point. - """ - - def __init__(self, reason, output): - self.reason = reason - self.output = output - - def __str__(self): - return ":\n{}".format(self.reason, self.output) - - class _CollectOutputProtocol(ProcessProtocol): """ Internal helper. Collects all output (stdout + stderr) into self.output, and callback's on done with all of it after the process exits (for any reason). """ - def __init__(self, stdin=None): + def __init__(self): self.done = Deferred() self.output = StringIO() - self._stdin = stdin - - def connectionMade(self): - if self._stdin is not None: - self.transport.write(self._stdin) - self.transport.closeStdin() def processEnded(self, reason): if not self.done.called: @@ -75,7 +51,7 @@ class _CollectOutputProtocol(ProcessProtocol): def processExited(self, reason): if not isinstance(reason.value, ProcessDone): - self.done.errback(ProcessFailed(reason, self.output.getvalue())) + self.done.errback(reason) def outReceived(self, data): self.output.write(data) @@ -147,27 +123,13 @@ def _cleanup_tahoe_process(tahoe_transport, exited): try: print("signaling {} with TERM".format(tahoe_transport.pid)) tahoe_transport.signalProcess('TERM') - print("signaled, blocking on exit {}".format(exited)) + print("signaled, blocking on exit") pytest_twisted.blockon(exited) print("exited, goodbye") except ProcessExitedAlready: pass -def run_tahoe(reactor, request, *args, **kwargs): - """ - Helper to run tahoe with optional coverage. - - :returns: a Deferred that fires when the command is done (or a - ProcessFailed exception if it exits non-zero) - """ - stdin = kwargs.get("stdin", None) - protocol = _CollectOutputProtocol(stdin=stdin) - process = _tahoe_runner_optional_coverage(protocol, reactor, request, args) - process.exited = protocol.done - return protocol.done - - def _tahoe_runner_optional_coverage(proto, reactor, request, other_args): """ Internal helper. Calls spawnProcess with `-m @@ -269,7 +231,7 @@ def _create_node(reactor, request, temp_dir, introducer_furl, flog_gatherer, nam if exists(node_dir): created_d = succeed(None) else: - print("creating: {}".format(node_dir)) + print("creating", node_dir) mkdir(node_dir) done_proto = _ProcessExitedProtocol() args = [ @@ -294,7 +256,7 @@ def _create_node(reactor, request, temp_dir, introducer_furl, flog_gatherer, nam def created(_): config_path = join(node_dir, 'tahoe.cfg') config = get_config(config_path) - set_config(config, 'node', 'log_gatherer.furl', flog_gatherer.furl) + set_config(config, 'node', 'log_gatherer.furl', flog_gatherer) write_config(config_path, config) created_d.addCallback(created) @@ -481,7 +443,7 @@ def web_post(tahoe, uri_fragment, **kwargs): return resp.content -def await_client_ready(tahoe, timeout=10, liveness=60*2, servers=1): +def await_client_ready(tahoe, timeout=10, liveness=60*2): """ Uses the status API to wait for a client-type node (in `tahoe`, a `TahoeProcess` instance usually from a fixture e.g. `alice`) to be @@ -505,8 +467,8 @@ def await_client_ready(tahoe, timeout=10, liveness=60*2, servers=1): time.sleep(1) continue - if len(js['servers']) < servers: - print("waiting because fewer than {} server(s)".format(servers)) + if len(js['servers']) == 0: + print("waiting because no servers at all") time.sleep(1) continue server_times = [ From c7f4a1a1574f712b0b9b4b9074b5a370e6da387d Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 6 Nov 2020 18:33:54 -0700 Subject: [PATCH 0131/2309] factor to use FilePath more --- src/allmydata/scripts/tahoe_grid_manager.py | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/src/allmydata/scripts/tahoe_grid_manager.py b/src/allmydata/scripts/tahoe_grid_manager.py index f84f57766..42ded3344 100644 --- a/src/allmydata/scripts/tahoe_grid_manager.py +++ b/src/allmydata/scripts/tahoe_grid_manager.py @@ -330,22 +330,15 @@ def _save_gridmanager_config(file_path, grid_manager): f.write("{}\n".format(data)) -def _load_gridmanager_config(gm_config): +def _load_gridmanager_config(fp): """ Loads a Grid Manager configuration and returns it (a dict) after validating. Exceptions if the config can't be found, or has problems. - :param gm_config str: "-" (a single dash) for stdin or a filename + :param FilePath fp: None for stdin or a path to a Grid Manager + configuration directory """ - fp = None - if gm_config.strip() != '-': - fp = FilePath(gm_config.strip()) - if not fp.exists(): - raise RuntimeError( - "No such directory '{}'".format(gm_config) - ) - if fp is None: gm = json.load(sys.stdin) else: @@ -353,7 +346,7 @@ def _load_gridmanager_config(gm_config): gm = json.load(f) try: - return _GridManager.from_config(gm, gm_config) + return _GridManager.from_config(gm, fp or "") except ValueError as e: raise usage.UsageError(str(e)) From 2118a2446e4ae36fd282a60955e9e268a81392dd Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 6 Nov 2020 22:23:53 -0700 Subject: [PATCH 0132/2309] grid-manager stand-alone, via Click --- setup.py | 7 +- src/allmydata/cli/__init__.py | 0 src/allmydata/cli/grid_manager.py | 207 ++++++++ src/allmydata/grid_manager.py | 217 +++++++++ src/allmydata/scripts/tahoe_grid_manager.py | 497 -------------------- 5 files changed, 430 insertions(+), 498 deletions(-) create mode 100644 src/allmydata/cli/__init__.py create mode 100644 src/allmydata/cli/grid_manager.py create mode 100644 src/allmydata/grid_manager.py delete mode 100644 src/allmydata/scripts/tahoe_grid_manager.py diff --git a/setup.py b/setup.py index 4151545f7..4c603147f 100644 --- a/setup.py +++ b/setup.py @@ -410,6 +410,11 @@ setup(name="tahoe-lafs", # also set in __init__.py }, include_package_data=True, setup_requires=setup_requires, - entry_points = { 'console_scripts': [ 'tahoe = allmydata.scripts.runner:run' ] }, + entry_points={ + 'console_scripts': [ + 'tahoe = allmydata.scripts.runner:run', + 'grid-manager = allmydata.cli.grid_manager:grid_manager', + ] + }, **setup_args ) diff --git a/src/allmydata/cli/__init__.py b/src/allmydata/cli/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/allmydata/cli/grid_manager.py b/src/allmydata/cli/grid_manager.py new file mode 100644 index 000000000..ce0197345 --- /dev/null +++ b/src/allmydata/cli/grid_manager.py @@ -0,0 +1,207 @@ +from datetime import ( + datetime, +) +import json + +import click + +from twisted.python.filepath import ( + FilePath, +) + +from allmydata.crypto import ( + ed25519, +) +from allmydata.util.abbreviate import ( + abbreviate_time, +) +from allmydata.grid_manager import ( + create_grid_manager, + save_grid_manager, + load_grid_manager, +) + + +@click.group() +@click.option( + '--config', '-c', + type=click.Path(), + help="Configuration directory (or - for stdin)", + required=True, +) +@click.pass_context +def grid_manager(ctx, config): + """ + A Tahoe Grid Manager issues certificates to storage-servers + + A Tahoe client with one or more Grid Manager public keys + configured will only upload to a Storage Server that presents a + valid certificate signed by one of the configured Grid + Manager keys. + + Grid Manager configuration can be in a local directory or given + via stdin. It contains long-term secret information (a private + signing key) and should be kept safe. + """ + + class Config(object): + """ + Availble to all sub-commands as Click's context.obj + """ + _grid_manager = None + + @property + def grid_manager(self): + if self._grid_manager is None: + config_path = _config_path_from_option(config) + self._grid_manager = load_grid_manager(config_path, config) + return self._grid_manager + + ctx.obj = Config() + + +@grid_manager.command() +@click.pass_context +def create(ctx): + """ + Make a new Grid Manager + """ + config_location = ctx.parent.params["config"] + fp = None + if config_location != '-': + fp = FilePath(config_location) + if fp.exists(): + raise click.ClickException( + "The directory '{}' already exists.".format(config_location) + ) + + gm = create_grid_manager() + save_grid_manager(fp, gm) + + +@grid_manager.command() +@click.pass_obj +def public_identity(config): + """ + Show the public identity key of a Grid Manager + + This is what you give to clients to add to their configuration so + they use announcements from this Grid Manager + """ + click.echo(config.grid_manager.public_identity()) + + +@grid_manager.command() +@click.argument("name") +@click.argument("public_key", type=click.UNPROCESSED) +@click.pass_context +def add(ctx, name, public_key): + """ + Add a new storage-server by name to a Grid Manager + + PUBLIC_KEY is the contents of a node.pubkey file from a Tahoe + node-directory. NAME is an arbitrary label. + """ + try: + ctx.obj.grid_manager.add_storage_server( + name, + ed25519.verifying_key_from_string(public_key), + ) + except KeyError: + raise click.ClickException( + "A storage-server called '{}' already exists".format(name) + ) + save_grid_manager( + _config_path_from_option(ctx.parent.params["config"]), + ctx.obj.grid_manager, + ) + return 0 + + +@grid_manager.command() +@click.argument("name") +@click.pass_context +def remove(ctx, name): + """ + Remove an existing storage-server by name from a Grid Manager + """ + fp = _config_path_from_option(ctx.parent.params["config"]) + try: + ctx.obj.grid_manager.remove_storage_server(name) + except KeyError: + raise click.ClickException( + "No storage-server called '{}' exists".format(name) + ) + cert_count = 0 + if fp is not None: + while fp.child('{}.cert.{}'.format(name, cert_count)).exists(): + fp.child('{}.cert.{}'.format(name, cert_count)).remove() + cert_count += 1 + + save_grid_manager(fp, ctx.obj.grid_manager) + + +@grid_manager.command() +@click.pass_context +def list(ctx): + """ + List all storage-servers known to a Grid Manager + """ + fp = _config_path_from_option(ctx.parent.params["config"]) + for name in sorted(ctx.obj.grid_manager.storage_servers.keys()): + blank_name = " " * len(name) + click.echo("{}: {}".format(name, ctx.obj.grid_manager.storage_servers[name].public_key())) + if fp: + cert_count = 0 + while fp.child('{}.cert.{}'.format(name, cert_count)).exists(): + container = json.load(fp.child('{}.cert.{}'.format(name, cert_count)).open('r')) + cert_data = json.loads(container['certificate']) + expires = datetime.utcfromtimestamp(cert_data['expires']) + delta = datetime.utcnow() - expires + click.echo("{} cert {}: ".format(blank_name, cert_count), nl=False) + if delta.total_seconds() < 0: + click.echo("valid until {} ({})".format(expires, abbreviate_time(delta))) + else: + click.echo("expired {} ({})".format(expires, abbreviate_time(delta))) + cert_count += 1 + + +@grid_manager.command() +@click.argument("name") +@click.argument( + "expiry_days", + type=click.IntRange(1, 5*365), # XXX is 5 years a good maximum? +) +@click.pass_context +def sign(ctx, name, expiry_days): + """ + sign a new certificate + """ + fp = _config_path_from_option(ctx.parent.params["config"]) + expiry_seconds = int(expiry_days) * 86400 + + try: + certificate = ctx.obj.grid_manager.sign(name, expiry_seconds) + except KeyError: + raise click.ClickException( + "No storage-server called '{}' exists".format(name) + ) + + certificate_data = json.dumps(certificate, indent=4) + click.echo(certificate_data) + if fp is not None: + next_serial = 0 + while fp.child("{}.cert.{}".format(name, next_serial)).exists(): + next_serial += 1 + with fp.child('{}.cert.{}'.format(name, next_serial)).open('w') as f: + f.write(certificate_data) + + +def _config_path_from_option(config): + """ + :param string config: a path or - + :returns: a FilePath instance or None + """ + if config == "-": + return None + return FilePath(config) diff --git a/src/allmydata/grid_manager.py b/src/allmydata/grid_manager.py new file mode 100644 index 000000000..071e1233f --- /dev/null +++ b/src/allmydata/grid_manager.py @@ -0,0 +1,217 @@ +import sys +import json +from datetime import ( + datetime, + timedelta, +) + +from allmydata.crypto import ( + ed25519, +) +from allmydata.util import ( + fileutil, + base32, +) + + +class _GridManagerStorageServer(object): + """ + A Grid Manager's notion of a storage server + """ + + def __init__(self, name, public_key, certificates): + self.name = name + self._public_key = public_key + self._certificates = [] if certificates is None else certificates + + def add_certificate(self, certificate): + self._certificates.append(certificate) + + def public_key(self): + return ed25519.string_from_verifying_key(self._public_key) + + def marshal(self): + return { + u"public_key": self.public_key(), + } + + +def create_grid_manager(): + """ + Create a new Grid Manager with a fresh keypair + """ + private_key, public_key = ed25519.create_signing_keypair() + return _GridManager( + ed25519.string_from_signing_key(private_key), + {}, + ) + + +def load_grid_manager(config_path, config_location): + """ + Load a Grid Manager from existing configuration. + + :param FilePath config_path: the configuratino location (or None for + stdin) + + :param str config_location: a string describing the config's location + + :returns: a GridManager instance + """ + if config_path is None: + config_file = sys.stdin + else: + try: + config_file = config_path.child("config.json").open("r") + except IOError: + raise ValueError( + "'{}' is not a Grid Manager config-directory".format(config) + ) + with config_file: + config = json.load(config_file) + + if not config: + raise ValueError( + "Invalid Grid Manager config in '{}'".format(config_location) + ) + if 'private_key' not in config: + raise ValueError( + "Grid Manager config from '{}' requires a 'private_key'".format( + config_location, + ) + ) + + private_key_bytes = config['private_key'].encode('ascii') + try: + private_key, public_key = ed25519.signing_keypair_from_string(private_key_bytes) + except Exception as e: + raise ValueError( + "Invalid Grid Manager private_key: {}".format(e) + ) + + storage_servers = dict() + for name, srv_config in config.get(u'storage_servers', {}).items(): + if not 'public_key' in srv_config: + raise ValueError( + "No 'public_key' for storage server '{}'".format(name) + ) + storage_servers[name] = _GridManagerStorageServer( + name, + ed25519.verifying_key_from_string(srv_config['public_key'].encode('ascii')), + None, + ) + + gm_version = config.get(u'grid_manager_config_version', None) + if gm_version != 0: + raise ValueError( + "Missing or unknown version '{}' of Grid Manager config".format( + gm_version + ) + ) + return _GridManager(private_key_bytes, storage_servers) + + +class _GridManager(object): + """ + A Grid Manager's configuration. + """ + + def __init__(self, private_key_bytes, storage_servers): + self._storage_servers = dict() if storage_servers is None else storage_servers + self._private_key_bytes = private_key_bytes + self._private_key, self._public_key = ed25519.signing_keypair_from_string(self._private_key_bytes) + self._version = 0 + + @property + def storage_servers(self): + return self._storage_servers + + def public_identity(self): + return ed25519.string_from_verifying_key(self._public_key) + + def sign(self, name, expiry_seconds): + try: + srv = self._storage_servers[name] + except KeyError: + raise KeyError( + u"No storage server named '{}'".format(name) + ) + expiration = datetime.utcnow() + timedelta(seconds=expiry_seconds) + epoch_offset = (expiration - datetime(1970, 1, 1)).total_seconds() + cert_info = { + "expires": epoch_offset, + "public_key": srv.public_key(), + "version": 1, + } + cert_data = json.dumps(cert_info, separators=(',',':'), sort_keys=True).encode('utf8') + sig = ed25519.sign_data(self._private_key, cert_data) + certificate = { + u"certificate": cert_data, + u"signature": base32.b2a(sig), + } + + vk = ed25519.verifying_key_from_signing_key(self._private_key) + ed25519.verify_signature(vk, sig, cert_data) + + return certificate + + def add_storage_server(self, name, public_key): + """ + :param name: a user-meaningful name for the server + :param public_key: ed25519.VerifyingKey the public-key of the + storage provider (e.g. from the contents of node.pubkey + for the client) + """ + if name in self._storage_servers: + raise KeyError( + "Already have a storage server called '{}'".format(name) + ) + ss = _GridManagerStorageServer(name, public_key, None) + self._storage_servers[name] = ss + return ss + + def remove_storage_server(self, name): + """ + :param name: a user-meaningful name for the server + """ + try: + del self._storage_servers[name] + except KeyError: + raise KeyError( + "No storage server called '{}'".format(name) + ) + + def marshal(self): + data = { + u"grid_manager_config_version": self._version, + u"private_key": self._private_key_bytes.decode('ascii'), + } + if self._storage_servers: + data[u"storage_servers"] = { + name: srv.marshal() + for name, srv + in self._storage_servers.items() + } + return data + + +def save_grid_manager(file_path, grid_manager): + """ + Writes a Grid Manager configuration. + + :param file_path: a FilePath specifying where to write the config + (if None, stdout is used) + + :param grid_manager: a _GridManager instance + """ + data = json.dumps( + grid_manager.marshal(), + indent=4, + ) + + if file_path is None: + print("{}\n".format(data)) + else: + fileutil.make_dirs(file_path.path, mode=0o700) + with file_path.child("config.json").open("w") as f: + f.write("{}\n".format(data)) diff --git a/src/allmydata/scripts/tahoe_grid_manager.py b/src/allmydata/scripts/tahoe_grid_manager.py deleted file mode 100644 index 42ded3344..000000000 --- a/src/allmydata/scripts/tahoe_grid_manager.py +++ /dev/null @@ -1,497 +0,0 @@ -from __future__ import print_function - -import sys -import json -from datetime import datetime, timedelta - -from allmydata.scripts.common import BaseOptions -from allmydata.util.abbreviate import abbreviate_time -from twisted.python import usage -from twisted.python.filepath import FilePath -from allmydata.util import fileutil -from allmydata.util import base32 -from allmydata.crypto import ed25519 -from twisted.internet.defer import inlineCallbacks, returnValue - - -class CreateOptions(BaseOptions): - description = ( - "Create a new identity key and configuration of a Grid Manager" - ) - - -class ShowIdentityOptions(BaseOptions): - description = ( - "Show the public identity key of a Grid Manager\n" - "\n" - "This is what you give to clients to add to their configuration" - " so they use announcements from this Grid Manager" - ) - - -class AddOptions(BaseOptions): - description = ( - "Add a new storage-server's key to a Grid Manager configuration\n" - "using NAME and PUBIC_KEY (comes from a node.pubkey file)" - ) - - def getSynopsis(self): - return "{} add NAME PUBLIC_KEY".format(super(AddOptions, self).getSynopsis()) - - def parseArgs(self, *args, **kw): - BaseOptions.parseArgs(self, **kw) - if len(args) != 2: - raise usage.UsageError( - "Requires two arguments: name public_key" - ) - self['name'] = unicode(args[0]) - try: - # WTF?! why does it want 'str' and not six.text_type? - self['storage_public_key'] = ed25519.verifying_key_from_string(args[1]) - except Exception as e: - raise usage.UsageError( - "Invalid public_key argument: {}".format(e) - ) - - -class RemoveOptions(BaseOptions): - description = ( - "Remove a storage-server from a Grid Manager configuration" - ) - - def parseArgs(self, *args, **kw): - BaseOptions.parseArgs(self, **kw) - if len(args) != 1: - raise usage.UsageError( - "Requires one arguments: name" - ) - self['name'] = unicode(args[0]) - - -class ListOptions(BaseOptions): - description = ( - "List all storage servers in this Grid Manager" - ) - - -class SignOptions(BaseOptions): - description = ( - "Create and sign a new certificate for a storage-server" - ) - - def getSynopsis(self): - return "{} NAME EXPIRY_DAYS".format(super(SignOptions, self).getSynopsis()) - - def parseArgs(self, *args, **kw): - BaseOptions.parseArgs(self, **kw) - if len(args) != 2: - raise usage.UsageError( - "Requires two arguments: name expiry_days" - ) - self['name'] = unicode(args[0]) - self['expiry_days'] = int(args[1]) - if self['expiry_days'] < 1 or self['expiry_days'] > 20*365: - raise usage.UsageError( - "Certificate expires in an unreasonable number of days: {}".format( - self['expiry_days'], - ) - ) - - -class GridManagerOptions(BaseOptions): - subCommands = [ - ["create", None, CreateOptions, "Create a Grid Manager."], - ["public-identity", None, ShowIdentityOptions, "Get the public-key for this Grid Manager."], - ["add", None, AddOptions, "Add a storage server to this Grid Manager."], - ["remove", None, RemoveOptions, "Remove a storage server from this Grid Manager."], - ["list", None, ListOptions, "List all storage servers in this Grid Manager."], - ["sign", None, SignOptions, "Create and sign a new Storage Certificate."], - ] - - optParameters = [ - ("config", "c", None, "How to find the Grid Manager's configuration") - ] - - def postOptions(self): - if not hasattr(self, 'subOptions'): - raise usage.UsageError("must specify a subcommand") - if self['config'] is None: - raise usage.UsageError("Must supply configuration with --config") - - description = ( - 'A "grid-manager" consists of some data defining a keypair (along with ' - 'some other details) and Tahoe sub-commands to manipulate the data and ' - 'produce certificates to give to storage-servers. Certificates assert ' - 'the statement: "Grid Manager X suggests you use storage-server Y to ' - 'upload shares to" (X and Y are public-keys).' - '\n\n' - 'Clients can use Grid Managers to decide which storage servers to ' - 'upload shares to. They do this by adding one or more Grid Manager ' - 'public keys to their config.' - ) - - -def _create_gridmanager(): - """ - :return: an object providing the GridManager interface initialized - with a new random keypair - """ - private_key, public_key = ed25519.create_signing_keypair() - return _GridManager( - ed25519.string_from_signing_key(private_key), - {}, - ) - -def _create(gridoptions, options): - """ - Create a new Grid Manager - """ - gm_config = gridoptions['config'] - - # pre-conditions check - fp = None - if gm_config.strip() != '-': - fp = FilePath(gm_config.strip()) - if fp.exists(): - raise usage.UsageError( - "The directory '{}' already exists.".format(gm_config) - ) - - gm = _create_gridmanager() - _save_gridmanager_config(fp, gm) - - -class _GridManagerStorageServer(object): - """ - A Grid Manager's notion of a storage server - """ - - def __init__(self, name, public_key, certificates): - self.name = name - self._public_key = public_key - self._certificates = [] if certificates is None else certificates - - def add_certificate(self, certificate): - self._certificates.append(certificate) - - def public_key(self): - return ed25519.string_from_verifying_key(self._public_key) - - def marshal(self): - return { - u"public_key": self.public_key(), - } - -class _GridManager(object): - """ - A Grid Manager's configuration. - """ - - @staticmethod - def from_config(config, config_location): - if not config: - raise ValueError( - "Invalid Grid Manager config in '{}'".format(config_location) - ) - if 'private_key' not in config: - raise ValueError( - "Grid Manager config from '{}' requires a 'private_key'".format( - config_location, - ) - ) - - private_key_bytes = config['private_key'].encode('ascii') - try: - private_key, public_key = ed25519.signing_keypair_from_string(private_key_bytes) - except Exception as e: - raise ValueError( - "Invalid Grid Manager private_key: {}".format(e) - ) - - storage_servers = dict() - for name, srv_config in config.get(u'storage_servers', {}).items(): - if not 'public_key' in srv_config: - raise ValueError( - "No 'public_key' for storage server '{}'".format(name) - ) - storage_servers[name] = _GridManagerStorageServer( - name, - ed25519.verifying_key_from_string(srv_config['public_key'].encode('ascii')), - None, - ) - - gm_version = config.get(u'grid_manager_config_version', None) - if gm_version != 0: - raise ValueError( - "Missing or unknown version '{}' of Grid Manager config".format( - gm_version - ) - ) - return _GridManager(private_key_bytes, storage_servers) - - def __init__(self, private_key_bytes, storage_servers): - self._storage_servers = dict() if storage_servers is None else storage_servers - self._private_key_bytes = private_key_bytes - self._private_key, self._public_key = ed25519.signing_keypair_from_string(self._private_key_bytes) - self._version = 0 - - @property - def storage_servers(self): - return self._storage_servers - - def public_identity(self): - return ed25519.string_from_verifying_key(self._public_key) - - def sign(self, name, expiry_seconds): - try: - srv = self._storage_servers[name] - except KeyError: - raise KeyError( - u"No storage server named '{}'".format(name) - ) - expiration = datetime.utcnow() + timedelta(seconds=expiry_seconds) - epoch_offset = (expiration - datetime(1970, 1, 1)).total_seconds() - cert_info = { - "expires": epoch_offset, - "public_key": srv.public_key(), - "version": 1, - } - cert_data = json.dumps(cert_info, separators=(',',':'), sort_keys=True).encode('utf8') - sig = ed25519.sign_data(self._private_key, cert_data) - certificate = { - u"certificate": cert_data, - u"signature": base32.b2a(sig), - } - - vk = ed25519.verifying_key_from_signing_key(self._private_key) - ed25519.verify_signature(vk, sig, cert_data) - - return certificate - - def add_storage_server(self, name, public_key): - """ - :param name: a user-meaningful name for the server - :param public_key: ed25519.VerifyingKey the public-key of the - storage provider (e.g. from the contents of node.pubkey - for the client) - """ - if name in self._storage_servers: - raise KeyError( - "Already have a storage server called '{}'".format(name) - ) - ss = _GridManagerStorageServer(name, public_key, None) - self._storage_servers[name] = ss - return ss - - def remove_storage_server(self, name): - """ - :param name: a user-meaningful name for the server - """ - try: - del self._storage_servers[name] - except KeyError: - raise KeyError( - "No storage server called '{}'".format(name) - ) - - def marshal(self): - data = { - u"grid_manager_config_version": self._version, - u"private_key": self._private_key_bytes.decode('ascii'), - } - if self._storage_servers: - data[u"storage_servers"] = { - name: srv.marshal() - for name, srv - in self._storage_servers.items() - } - return data - - -def _save_gridmanager_config(file_path, grid_manager): - """ - Writes a Grid Manager configuration. - - :param file_path: a FilePath specifying where to write the config - (if None, stdout is used) - - :param grid_manager: a _GridManager instance - """ - data = json.dumps( - grid_manager.marshal(), - indent=4, - ) - - if file_path is None: - print("{}\n".format(data)) - else: - fileutil.make_dirs(file_path.path, mode=0o700) - with file_path.child("config.json").open("w") as f: - f.write("{}\n".format(data)) - - -def _load_gridmanager_config(fp): - """ - Loads a Grid Manager configuration and returns it (a dict) after - validating. Exceptions if the config can't be found, or has - problems. - - :param FilePath fp: None for stdin or a path to a Grid Manager - configuration directory - """ - if fp is None: - gm = json.load(sys.stdin) - else: - with fp.child("config.json").open("r") as f: - gm = json.load(f) - - try: - return _GridManager.from_config(gm, fp or "") - except ValueError as e: - raise usage.UsageError(str(e)) - - -def _show_identity(gridoptions, options): - """ - Output the public-key of a Grid Manager - """ - gm_config = gridoptions['config'].strip() - assert gm_config is not None - - gm = _load_gridmanager_config(gm_config) - print(gm.public_identity()) - - -def _add(gridoptions, options): - """ - Add a new storage-server by name to a Grid Manager - """ - gm_config = gridoptions['config'].strip() - fp = FilePath(gm_config) if gm_config.strip() != '-' else None - - gm = _load_gridmanager_config(gm_config) - try: - gm.add_storage_server( - options['name'], - options['storage_public_key'], - ) - except KeyError: - raise usage.UsageError( - "A storage-server called '{}' already exists".format(options['name']) - ) - - _save_gridmanager_config(fp, gm) - return 0 - - -def _remove(gridoptions, options): - """ - Remove an existing storage-server by name from a Grid Manager - """ - gm_config = gridoptions['config'].strip() - fp = FilePath(gm_config) if gm_config.strip() != '-' else None - gm = _load_gridmanager_config(gm_config) - - try: - gm.remove_storage_server(options['name']) - except KeyError: - raise usage.UsageError( - "No storage-server called '{}' exists".format(options['name']) - ) - cert_count = 0 - if fp is not None: - while fp.child('{}.cert.{}'.format(options['name'], cert_count)).exists(): - fp.child('{}.cert.{}'.format(options['name'], cert_count)).remove() - cert_count += 1 - - _save_gridmanager_config(fp, gm) - return 0 - - -def _list(gridoptions, options): - """ - List all storage-servers known to a Grid Manager - """ - gm_config = gridoptions['config'].strip() - fp = FilePath(gm_config) if gm_config.strip() != '-' else None - - gm = _load_gridmanager_config(gm_config) - for name in sorted(gm.storage_servers.keys()): - print("{}: {}".format(name, gm.storage_servers[name].public_key())) - if fp: - cert_count = 0 - while fp.child('{}.cert.{}'.format(name, cert_count)).exists(): - container = json.load(fp.child('{}.cert.{}'.format(name, cert_count)).open('r')) - cert_data = json.loads(container['certificate']) - expires = datetime.utcfromtimestamp(cert_data['expires']) - delta = datetime.utcnow() - expires - if delta.total_seconds() < 0: - print("{}: cert {}: valid until {} ({})".format(name, cert_count, expires, abbreviate_time(delta))) - else: - print("{}: cert {}: expired ({})".format(name, cert_count, abbreviate_time(delta))) - cert_count += 1 - - -def _sign(gridoptions, options): - """ - sign a new certificate - """ - gm_config = gridoptions['config'].strip() - fp = FilePath(gm_config) if gm_config.strip() != '-' else None - gm = _load_gridmanager_config(gm_config) - - expiry_seconds = int(options['expiry_days']) * 86400 - - try: - certificate = gm.sign(options['name'], expiry_seconds) - except KeyError: - raise usage.UsageError( - "No storage-server called '{}' exists".format(options['name']) - ) - - certificate_data = json.dumps(certificate, indent=4) - print(certificate_data) - if fp is not None: - next_serial = 0 - while fp.child("{}.cert.{}".format(options['name'], next_serial)).exists(): - next_serial += 1 - with fp.child('{}.cert.{}'.format(options['name'], next_serial)).open('w') as f: - f.write(certificate_data) - - -grid_manager_commands = { - CreateOptions: _create, - ShowIdentityOptions: _show_identity, - AddOptions: _add, - RemoveOptions: _remove, - ListOptions: _list, - SignOptions: _sign, -} - -@inlineCallbacks -def gridmanager(config): - """ - Runs the 'tahoe grid-manager' command. - """ - if config.subCommand is None: - print(config) - returnValue(1) - - try: - f = grid_manager_commands[config.subOptions.__class__] - except KeyError: - print(config.subOptions, grid_manager_commands.keys()) - print("Unknown command 'tahoe grid-manager {}': no such grid-manager subcommand".format(config.subCommand)) - returnValue(2) - - x = yield f(config, config.subOptions) - returnValue(x) - -subCommands = [ - ["grid-manager", None, GridManagerOptions, - "Grid Manager subcommands: use 'tahoe grid-manager' for a list."], -] - -dispatch = { - "grid-manager": gridmanager, -} From d09690823df0b0bb7171044c37b7677947a532da Mon Sep 17 00:00:00 2001 From: meejah Date: Sat, 7 Nov 2020 02:18:18 -0700 Subject: [PATCH 0133/2309] spelling --- src/allmydata/grid_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/grid_manager.py b/src/allmydata/grid_manager.py index 071e1233f..0982d0aa4 100644 --- a/src/allmydata/grid_manager.py +++ b/src/allmydata/grid_manager.py @@ -51,7 +51,7 @@ def load_grid_manager(config_path, config_location): """ Load a Grid Manager from existing configuration. - :param FilePath config_path: the configuratino location (or None for + :param FilePath config_path: the configuration location (or None for stdin) :param str config_location: a string describing the config's location From a8382a5356d6e8b43484ac794a95435efe1962f3 Mon Sep 17 00:00:00 2001 From: meejah Date: Sat, 7 Nov 2020 02:18:54 -0700 Subject: [PATCH 0134/2309] cleanup, more tests --- src/allmydata/grid_manager.py | 149 ++++++++++++++++-- src/allmydata/scripts/admin.py | 10 +- src/allmydata/scripts/runner.py | 5 +- src/allmydata/storage_client.py | 118 -------------- src/allmydata/test/test_grid_manager.py | 194 ++++++++++++++++++++++++ 5 files changed, 339 insertions(+), 137 deletions(-) diff --git a/src/allmydata/grid_manager.py b/src/allmydata/grid_manager.py index 0982d0aa4..dadf3b361 100644 --- a/src/allmydata/grid_manager.py +++ b/src/allmydata/grid_manager.py @@ -70,9 +70,12 @@ def load_grid_manager(config_path, config_location): with config_file: config = json.load(config_file) - if not config: + gm_version = config.get(u'grid_manager_config_version', None) + if gm_version != 0: raise ValueError( - "Invalid Grid Manager config in '{}'".format(config_location) + "Missing or unknown version '{}' of Grid Manager config".format( + gm_version + ) ) if 'private_key' not in config: raise ValueError( @@ -101,13 +104,6 @@ def load_grid_manager(config_path, config_location): None, ) - gm_version = config.get(u'grid_manager_config_version', None) - if gm_version != 0: - raise ValueError( - "Missing or unknown version '{}' of Grid Manager config".format( - gm_version - ) - ) return _GridManager(private_key_bytes, storage_servers) @@ -153,6 +149,7 @@ class _GridManager(object): vk = ed25519.verifying_key_from_signing_key(self._private_key) ed25519.verify_signature(vk, sig, cert_data) + srv.add_certificate(certificate) return certificate def add_storage_server(self, name, public_key): @@ -215,3 +212,137 @@ def save_grid_manager(file_path, grid_manager): fileutil.make_dirs(file_path.path, mode=0o700) with file_path.child("config.json").open("w") as f: f.write("{}\n".format(data)) + + +def parse_grid_manager_certificate(gm_data): + """ + :param gm_data: some data that might be JSON that might be a valid + Grid Manager Certificate + + :returns: json data of a valid Grid Manager certificate, or an + exception if the data is not valid. + """ + + required_keys = { + 'certificate', + 'signature', + } + + js = json.loads(gm_data) + + if not isinstance(js, dict): + raise ValueError( + "Grid Manager certificate must be a dict" + ) + if set(js.keys()) != required_keys: + raise ValueError( + "Grid Manager certificate must contain: {}".format( + ", ".join("'{}'".format(k) for k in js.keys()), + ) + ) + return js + + +def validate_grid_manager_certificate(gm_key, alleged_cert): + """ + :param gm_key: a VerifyingKey instance, a Grid Manager's public + key. + + :param alleged_cert: dict with "certificate" and "signature" keys, where + "certificate" contains a JSON-serialized certificate for a Storage + Server (comes from a Grid Manager). + + :return: a dict consisting of the deserialized certificate data or + None if the signature is invalid. Note we do NOT check the + expiry time in this function. + """ + try: + ed25519.verify_signature( + gm_key, + base32.a2b(alleged_cert['signature'].encode('ascii')), + alleged_cert['certificate'].encode('ascii'), + ) + except ed25519.BadSignature: + return None + # signature is valid; now we can load the actual data + cert = json.loads(alleged_cert['certificate']) + return cert + + +def create_grid_manager_verifier(keys, certs, now_fn=None, bad_cert=None): + """ + Creates a predicate for confirming some Grid Manager-issued + certificates against Grid Manager keys. A predicate is used + (instead of just returning True/False here) so that the + expiry-time can be tested on each call. + + :param list keys: 0 or more `VerifyingKey` instances + + :param list certs: 1 or more Grid Manager certificates each of + which is a `dict` containing 'signature' and 'certificate' keys. + + :param callable now_fn: a callable which returns the current UTC + timestamp (or datetime.utcnow if None). + + :param callable bad_cert: a two-argument callable which is invoked + when a certificate verification fails. The first argument is + the verifying key and the second is the certificate. If None + (the default) errors are print()-ed. Note that we may have + several certificates and only one must be valid, so this may + be called (multiple times) even if the function ultimately + returns successfully. + + :returns: a callable which will return True only-if there is at + least one valid certificate (that has not at this moment + expired) in `certs` signed by one of the keys in `keys`. + """ + + now_fn = datetime.utcnow if now_fn is None else now_fn + valid_certs = [] + + # if we have zero grid-manager keys then everything is valid + if not keys: + return lambda: True + + if bad_cert is None: + + def bad_cert(key, alleged_cert): + """ + We might want to let the user know about this failed-to-verify + certificate .. but also if you have multiple grid-managers + then a bunch of these messages would appear. Better would + be to bubble this up to some sort of status API (or maybe + on the Welcome page?) + + The only thing that might actually be interesting, though, + is whether this whole function returns false or not.. + """ + print( + "Grid Manager certificate signature failed. Certificate: " + "\"{cert}\" for key \"{key}\".".format( + cert=alleged_cert, + key=ed25519.string_from_verifying_key(key), + ) + ) + + # validate the signatures on any certificates we have (not yet the expiry dates) + for alleged_cert in certs: + for key in keys: + cert = _validate_grid_manager_certificate(key, alleged_cert) + if cert is not None: + valid_certs.append(cert) + else: + bad_cert(key, alleged_cert) + + def validate(): + now = now_fn() + # if *any* certificate is still valid then we consider the server valid + for cert in valid_certs: + expires = datetime.utcfromtimestamp(cert['expires']) + # cert_pubkey = keyutil.parse_pubkey(cert['public_key'].encode('ascii')) + if expires > now: + # not-expired + return True + return False + + return validate diff --git a/src/allmydata/scripts/admin.py b/src/allmydata/scripts/admin.py index e5b7dca2d..b13dfb07d 100644 --- a/src/allmydata/scripts/admin.py +++ b/src/allmydata/scripts/admin.py @@ -7,8 +7,8 @@ from os.path import exists, join from twisted.python import usage #from allmydata.node import read_config from allmydata.client import read_config -from allmydata.storage_client import ( - parse_grid_manager_data, +from allmydata.grid_manager import ( + parse_grid_manager_certificate, ) from allmydata.scripts.cli import _default_nodedir from allmydata.scripts.common import BaseOptions @@ -90,7 +90,7 @@ class AddGridManagerCertOptions(BaseOptions): data = f.read() try: - self.certificate_data = parse_grid_manager_data(data) + self.certificate_data = parse_grid_manager_certificate(data) except ValueError as e: raise usage.UsageError( "Error parsing certificate: {}".format(e) @@ -142,11 +142,9 @@ def add_grid_manager_cert(options): # write all the data out fileutil.write(cert_path, cert_bytes) - # print("created {}: {} bytes".format(cert_fname, len(cert_bytes))) with open(config_path, "w") as f: - # XXX probably want a _Config.write_tahoe_cfg() or something? or just set_config() does that automagically + # XXX probably want a _Config.write_tahoe_cfg() or something? config.config.write(f) - # print("wrote {}".format(config_fname)) cert_count = len(config.enumerate_section("grid_manager_certificates")) print("There are now {} certificates".format(cert_count), file=options.parent.parent.stderr) diff --git a/src/allmydata/scripts/runner.py b/src/allmydata/scripts/runner.py index 209a379bc..24d027cc3 100644 --- a/src/allmydata/scripts/runner.py +++ b/src/allmydata/scripts/runner.py @@ -11,7 +11,7 @@ from allmydata.version_checks import get_package_versions_string from allmydata.scripts.common import get_default_nodedir from allmydata.scripts import debug, create_node, cli, \ stats_gatherer, admin, tahoe_daemonize, tahoe_start, \ - tahoe_stop, tahoe_restart, tahoe_run, tahoe_invite, tahoe_grid_manager + tahoe_stop, tahoe_restart, tahoe_run, tahoe_invite from allmydata.util.encodingutil import quote_output, quote_local_unicode_path, get_io_encoding from allmydata.util.eliotutil import ( opt_eliot_destination, @@ -63,7 +63,6 @@ class Options(usage.Options): + debug.subCommands + cli.subCommands + tahoe_invite.subCommands - + tahoe_grid_manager.subCommands ) optFlags = [ @@ -155,8 +154,6 @@ def dispatch(config, # these are blocking, and must be run in a thread f0 = cli.dispatch[command] f = lambda so: threads.deferToThread(f0, so) - elif command in tahoe_grid_manager.dispatch: - f = tahoe_grid_manager.dispatch[command] elif command in tahoe_invite.dispatch: f = tahoe_invite.dispatch[command] else: diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 5ffb44092..49c20866b 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -431,124 +431,6 @@ class StubServer(object): return "?" -def parse_grid_manager_data(gm_data): - """ - :param gm_data: some data that might be JSON that might be a valid - Grid Manager Certificate - - :returns: json data of a valid Grid Manager certificate, or an - exception if the data is not valid. - """ - - required_keys = allowed_keys = [ - 'certificate', - 'signature', - ] - - js = json.loads(gm_data) - for k in js.keys(): - if k not in allowed_keys: - raise ValueError( - "Grid Manager certificate JSON may not contain '{}'".format( - k, - ) - ) - for k in required_keys: - if k not in js: - raise ValueError( - "Grid Manager certificate JSON must contain '{}'".format( - k, - ) - ) - return js - - -def _validate_grid_manager_certificate(gm_key, alleged_cert): - """ - :param gm_key: a VerifyingKey instance, a Grid Manager's public - key. - - :param alleged_cert: dict with "certificate" and "signature" keys, where - "certificate" contains a JSON-serialized certificate for a Storage - Server (comes from a Grid Manager). - - :return: a dict consisting of the deserialized certificate data or - None if the signature is invalid. Note we do NOT check the - expiry time in this function. - """ - try: - ed25519.verify_signature( - gm_key, - base32.a2b(alleged_cert['signature'].encode('ascii')), - alleged_cert['certificate'].encode('ascii'), - ) - except ed25519.BadSignature: - return None - # signature is valid; now we can load the actual data - cert = json.loads(alleged_cert['certificate']) - return cert - - -def create_grid_manager_verifier(keys, certs, now_fn=None): - """ - Creates a predicate for confirming some Grid Manager-issued - certificates against Grid Manager keys. A predicate is used - (instead of just returning True/False here) so that the - expiry-time can be tested on each call. - - :param list keys: 0 or more `VerifyingKey` instances - - :param list certs: 1 or more Grid Manager certificates each of - which is a `dict` containing 'signature' and 'certificate' keys. - - :param callable now_fn: a callable which returns the current UTC - timestamp (or datetime.utcnow if None). - - :returns: a callable which will return True only-if there is at - least one valid certificate in `certs` signed by one of the keys - in `keys`. - """ - - now_fn = datetime.utcnow if now_fn is None else now_fn - valid_certs = [] - - # if we have zero grid-manager keys then everything is valid - if not keys: - return lambda: True - - # validate the signatures on any certificates we have (not yet the expiry dates) - for alleged_cert in certs: - for key in keys: - cert = _validate_grid_manager_certificate(key, alleged_cert) - if cert is not None: - valid_certs.append(cert) - else: - # we might want to let the user know about this - # failed-to-verify certificate .. but also if you have - # multiple grid-managers then a bunch of these - # messages would appear - print( - "Grid Manager certificate signature failed. Certificate: " - "\"{cert}\" for key \"{key}\".".format( - key=key, - cert=alleged_cert, - ) - ) - - def validate(): - now = now_fn() - # if *any* certificate is still valid then we consider the server valid - for cert in valid_certs: - expires = datetime.utcfromtimestamp(cert['expires']) - # cert_pubkey = keyutil.parse_pubkey(cert['public_key'].encode('ascii')) - if expires > now: - # not-expired - return True - return False - - return validate - - class IFoolscapStorageServer(Interface): """ An internal interface that mediates between ``NativeStorageServer`` and diff --git a/src/allmydata/test/test_grid_manager.py b/src/allmydata/test/test_grid_manager.py index a60739634..893941f7d 100644 --- a/src/allmydata/test/test_grid_manager.py +++ b/src/allmydata/test/test_grid_manager.py @@ -1,6 +1,10 @@ import json +from twisted.python.filepath import ( + FilePath, +) + from allmydata.client import ( create_storage_farm_broker, ) @@ -10,6 +14,17 @@ from allmydata.node import ( from allmydata.client import ( _valid_config as client_valid_config, ) +from allmydata.crypto import ( + ed25519, +) +from allmydata.util import ( + base32, +) +from allmydata.grid_manager import ( + load_grid_manager, + save_grid_manager, + create_grid_manager, +) from .common import SyncTestCase @@ -78,3 +93,182 @@ class GridManagerUtilities(SyncTestCase): 1, len(config.enumerate_section("grid_managers")) ) + + +class GridManagerVerifier(SyncTestCase): + """ + Tests related to rejecting or accepting Grid Manager certificates. + """ + + def setUp(self): + self.gm = create_grid_manager() + return super(GridManagerVerifier, self).setUp() + + def test_sign_cert(self): + """ + Add a storage-server and sign a certificate for it + """ + priv, pub = ed25519.create_signing_keypair() + self.gm.add_storage_server("test", pub) + cert = self.gm.sign("test", 86400) + + self.assertEqual( + set(cert.keys()), + {"certificate", "signature"}, + ) + gm_key = ed25519.verifying_key_from_string(self.gm.public_identity()) + self.assertEqual( + ed25519.verify_signature( + gm_key, + base32.a2b(cert["signature"]), + cert["certificate"], + ), + None + ) + + def test_sign_cert_wrong_name(self): + """ + Try to sign a storage-server that doesn't exist + """ + with self.assertRaises(KeyError): + self.gm.sign("doesn't exist", 86400) + + def test_add_cert(self): + """ + Add a storage-server and serialize it + """ + priv, pub = ed25519.create_signing_keypair() + self.gm.add_storage_server("test", pub) + + data = self.gm.marshal() + self.assertEqual( + data["storage_servers"], + { + "test": { + "public_key": ed25519.string_from_verifying_key(pub), + } + } + ) + + def test_remove(self): + """ + Add then remove a storage-server + """ + priv, pub = ed25519.create_signing_keypair() + self.gm.add_storage_server("test", pub) + self.gm.remove_storage_server("test") + self.assertEqual(len(self.gm.storage_servers), 0) + + def test_serialize(self): + """ + Write and then read a Grid Manager config + """ + priv0, pub0 = ed25519.create_signing_keypair() + priv1, pub1 = ed25519.create_signing_keypair() + self.gm.add_storage_server("test0", pub0) + self.gm.add_storage_server("test1", pub1) + + tempdir = self.mktemp() + fp = FilePath(tempdir) + + save_grid_manager(fp, self.gm) + gm2 = load_grid_manager(fp, tempdir) + self.assertEqual( + self.gm.public_identity(), + gm2.public_identity(), + ) + self.assertEqual( + len(self.gm.storage_servers), + len(gm2.storage_servers), + ) + for name, ss0 in self.gm.storage_servers.items(): + ss1 = gm2.storage_servers[name] + self.assertEqual(ss0.name, ss1.name) + self.assertEqual(ss0.public_key(), ss1.public_key()) + self.assertEqual(self.gm.marshal(), gm2.marshal()) + + def test_invalid_no_version(self): + """ + Invalid Grid Manager config with no version + """ + tempdir = self.mktemp() + fp = FilePath(tempdir) + bad_config = { + "private_key": "at least we have one", + } + fp.makedirs() + with fp.child("config.json").open("w") as f: + json.dump(bad_config, f) + + with self.assertRaises(ValueError) as ctx: + load_grid_manager(fp, tempdir) + self.assertIn( + "unknown version", + str(ctx.exception), + ) + + def test_invalid_no_private_key(self): + """ + Invalid Grid Manager config with no private key + """ + tempdir = self.mktemp() + fp = FilePath(tempdir) + bad_config = { + "grid_manager_config_version": 0, + } + fp.makedirs() + with fp.child("config.json").open("w") as f: + json.dump(bad_config, f) + + with self.assertRaises(ValueError) as ctx: + load_grid_manager(fp, tempdir) + self.assertIn( + "requires a 'private_key'", + str(ctx.exception), + ) + + def test_invalid_bad_private_key(self): + """ + Invalid Grid Manager config with bad private-key + """ + tempdir = self.mktemp() + fp = FilePath(tempdir) + bad_config = { + "grid_manager_config_version": 0, + "private_key": "not actually encoded key", + } + fp.makedirs() + with fp.child("config.json").open("w") as f: + json.dump(bad_config, f) + + with self.assertRaises(ValueError) as ctx: + load_grid_manager(fp, tempdir) + self.assertIn( + "Invalid Grid Manager private_key", + str(ctx.exception), + ) + + def test_invalid_storage_server(self): + """ + Invalid Grid Manager config with missing public-key for + storage-server + """ + tempdir = self.mktemp() + fp = FilePath(tempdir) + bad_config = { + "grid_manager_config_version": 0, + "private_key": "priv-v0-ub7knkkmkptqbsax4tznymwzc4nk5lynskwjsiubmnhcpd7lvlqa", + "storage_servers": { + "bad": {} + } + } + fp.makedirs() + with fp.child("config.json").open("w") as f: + json.dump(bad_config, f) + + with self.assertRaises(ValueError) as ctx: + load_grid_manager(fp, tempdir) + self.assertIn( + "No 'public_key' for storage server", + str(ctx.exception), + ) From 41fa8238d55b79be6f8ec013a17f5cf06d0d3912 Mon Sep 17 00:00:00 2001 From: meejah Date: Sat, 7 Nov 2020 03:26:05 -0700 Subject: [PATCH 0135/2309] more unittests --- src/allmydata/grid_manager.py | 2 +- src/allmydata/storage_client.py | 3 ++ src/allmydata/test/test_grid_manager.py | 51 +++++++++++++++++++++++++ 3 files changed, 55 insertions(+), 1 deletion(-) diff --git a/src/allmydata/grid_manager.py b/src/allmydata/grid_manager.py index dadf3b361..6c5230901 100644 --- a/src/allmydata/grid_manager.py +++ b/src/allmydata/grid_manager.py @@ -328,7 +328,7 @@ def create_grid_manager_verifier(keys, certs, now_fn=None, bad_cert=None): # validate the signatures on any certificates we have (not yet the expiry dates) for alleged_cert in certs: for key in keys: - cert = _validate_grid_manager_certificate(key, alleged_cert) + cert = validate_grid_manager_certificate(key, alleged_cert) if cert is not None: valid_certs.append(cert) else: diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 49c20866b..c17fbe4f7 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -66,6 +66,9 @@ from allmydata.interfaces import ( IStorageServer, IFoolscapStoragePlugin, ) +from allmydata.grid_manager import ( + create_grid_manager_verifier, +) from allmydata.util import log, base32, connection_status from allmydata.util.assertutil import precondition from allmydata.util.observer import ObserverList diff --git a/src/allmydata/test/test_grid_manager.py b/src/allmydata/test/test_grid_manager.py index 893941f7d..a420385f0 100644 --- a/src/allmydata/test/test_grid_manager.py +++ b/src/allmydata/test/test_grid_manager.py @@ -24,6 +24,8 @@ from allmydata.grid_manager import ( load_grid_manager, save_grid_manager, create_grid_manager, + parse_grid_manager_certificate, + create_grid_manager_verifier, ) from .common import SyncTestCase @@ -272,3 +274,52 @@ class GridManagerVerifier(SyncTestCase): "No 'public_key' for storage server", str(ctx.exception), ) + + def test_parse_cert(self): + """ + Parse an ostensibly valid storage certificate + """ + js = parse_grid_manager_certificate('{"certificate": "", "signature": ""}') + self.assertEqual( + set(js.keys()), + {"certificate", "signature"} + ) + # the signature isn't *valid*, but that's checked in a + # different function + + def test_parse_cert_not_dict(self): + """ + Certificate data not even a dict + """ + with self.assertRaises(ValueError) as ctx: + parse_grid_manager_certificate("[]") + self.assertIn( + "must be a dict", + str(ctx.exception), + ) + + def test_parse_cert_missing_signature(self): + """ + Missing the signature + """ + with self.assertRaises(ValueError) as ctx: + parse_grid_manager_certificate('{"certificate": ""}') + self.assertIn( + "must contain", + str(ctx.exception), + ) + + def test_validate_cert(self): + """ + Validate a correctly-signed certificate + """ + priv0, pub0 = ed25519.create_signing_keypair() + self.gm.add_storage_server("test0", pub0) + cert0 = self.gm.sign("test0", 86400) + + verify = create_grid_manager_verifier( + [self.gm._public_key], + [cert0], + ) + + self.assertTrue(verify()) From f3d530fc0bb673d9837bb287cc6b7bb0fc47e989 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 13 Nov 2020 17:54:36 -0700 Subject: [PATCH 0136/2309] grid-manager CLI tests --- src/allmydata/test/cli/test_grid_manager.py | 56 +++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 src/allmydata/test/cli/test_grid_manager.py diff --git a/src/allmydata/test/cli/test_grid_manager.py b/src/allmydata/test/cli/test_grid_manager.py new file mode 100644 index 000000000..ebb963ab8 --- /dev/null +++ b/src/allmydata/test/cli/test_grid_manager.py @@ -0,0 +1,56 @@ + +import os +import json + +from ..common import SyncTestCase +from allmydata.cli.grid_manager import ( + grid_manager, +) + +import click.testing + + +class GridManagerCommandLine(SyncTestCase): + """ + Test the mechanics of the `grid-manager` command + """ + + def setUp(self): + self.runner = click.testing.CliRunner() + super(GridManagerCommandLine, self).setUp() + + def test_create(self): + """ + Create a new grid-manager + """ + with self.runner.isolated_filesystem(): + result = self.runner.invoke(grid_manager, ["--config", "foo", "create"]) + self.assertEqual(["foo"], os.listdir(".")) + self.assertEqual(["config.json"], os.listdir("./foo")) + + def test_create_stdout(self): + """ + Create a new grid-manager with no files + """ + with self.runner.isolated_filesystem(): + result = self.runner.invoke(grid_manager, ["--config", "-", "create"]) + self.assertEqual([], os.listdir(".")) + config = json.loads(result.output) + self.assertEqual( + {"private_key", "grid_manager_config_version"}, + set(config.keys()), + ) + + def test_add_and_sign(self): + """ + Add a new storage-server and sign a certificate for it + """ + pubkey = "pub-v0-cbq6hcf3pxcz6ouoafrbktmkixkeuywpcpbcomzd3lqbkq4nmfga" + with self.runner.isolated_filesystem(): + self.runner.invoke(grid_manager, ["--config", "foo", "create"]) + self.runner.invoke(grid_manager, ["--config", "foo", "add", "storage0", pubkey]) + result = self.runner.invoke(grid_manager, ["--config", "foo", "sign", "storage0", "10"]) + sigcert = json.loads(result.output) + self.assertEqual({"certificate", "signature"}, set(sigcert.keys())) + cert = json.loads(sigcert['certificate']) + self.assertEqual(cert["public_key"], pubkey) From 6e1bb1ecd54e6e47fb997e52b20e591be1ed7025 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 13 Nov 2020 18:01:32 -0700 Subject: [PATCH 0137/2309] grid-manager CLI tests --- src/allmydata/test/cli/test_grid_manager.py | 23 +++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/src/allmydata/test/cli/test_grid_manager.py b/src/allmydata/test/cli/test_grid_manager.py index ebb963ab8..8383bb9cd 100644 --- a/src/allmydata/test/cli/test_grid_manager.py +++ b/src/allmydata/test/cli/test_grid_manager.py @@ -27,6 +27,8 @@ class GridManagerCommandLine(SyncTestCase): result = self.runner.invoke(grid_manager, ["--config", "foo", "create"]) self.assertEqual(["foo"], os.listdir(".")) self.assertEqual(["config.json"], os.listdir("./foo")) + result = self.runner.invoke(grid_manager, ["--config", "foo", "public-identity"]) + self.assertTrue(result.output.startswith("pub-v0-")) def test_create_stdout(self): """ @@ -54,3 +56,24 @@ class GridManagerCommandLine(SyncTestCase): self.assertEqual({"certificate", "signature"}, set(sigcert.keys())) cert = json.loads(sigcert['certificate']) self.assertEqual(cert["public_key"], pubkey) + + def test_add_list_remove(self): + """ + Add a storage server, list it, remove it. + """ + pubkey = "pub-v0-cbq6hcf3pxcz6ouoafrbktmkixkeuywpcpbcomzd3lqbkq4nmfga" + with self.runner.isolated_filesystem(): + self.runner.invoke(grid_manager, ["--config", "foo", "create"]) + self.runner.invoke(grid_manager, ["--config", "foo", "add", "storage0", pubkey]) + + result = self.runner.invoke(grid_manager, ["--config", "foo", "list"]) + names = [ + line.split(':')[0] + for line in result.output.strip().split('\n') + ] + self.assertEqual(names, ["storage0"]) + + self.runner.invoke(grid_manager, ["--config", "foo", "remove", "storage0"]) + + result = self.runner.invoke(grid_manager, ["--config", "foo", "list"]) + self.assertEqual(result.output.strip(), "") From b6fbfeee480331606484c7ed8f455f11f47846d7 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 13 Nov 2020 18:23:21 -0700 Subject: [PATCH 0138/2309] more utests --- src/allmydata/client.py | 2 +- src/allmydata/test/test_grid_manager.py | 14 +++++++++----- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/src/allmydata/client.py b/src/allmydata/client.py index 019535a14..006b0eed3 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -565,7 +565,7 @@ def create_storage_farm_broker(config, default_connection_handlers, foolscap_con grid_manager_keys = [] for name, gm_key in config.enumerate_section('grid_managers').items(): grid_manager_keys.append( - ed25519.verifying_key_from_string(gm_key) + ed25519.verifying_key_from_string(gm_key.encode("ascii")) ) # we don't actually use this keypair for anything (yet) as far diff --git a/src/allmydata/test/test_grid_manager.py b/src/allmydata/test/test_grid_manager.py index a420385f0..6ff4fb065 100644 --- a/src/allmydata/test/test_grid_manager.py +++ b/src/allmydata/test/test_grid_manager.py @@ -7,6 +7,7 @@ from twisted.python.filepath import ( from allmydata.client import ( create_storage_farm_broker, + _load_grid_manager_certificates, ) from allmydata.node import ( config_from_string, @@ -70,7 +71,7 @@ class GridManagerUtilities(SyncTestCase): } } sfb.set_static_servers(static_servers) - nss = sfb._make_storage_server(u"server0", {"ann": announcement}) + nss = sfb._make_storage_server(b"server0", {"ann": announcement}) # we have some grid-manager keys defined so the server should # only upload if there's a valid certificate -- but the only @@ -79,11 +80,12 @@ class GridManagerUtilities(SyncTestCase): def test_load_certificates(self): cert_path = self.mktemp() + fake_cert = { + "certificate": "{\"expires\":1601687822,\"public_key\":\"pub-v0-cbq6hcf3pxcz6ouoafrbktmkixkeuywpcpbcomzd3lqbkq4nmfga\",\"version\":1}", + "signature": "fvjd3uvvupf2v6tnvkwjd473u3m3inyqkwiclhp7balmchkmn3px5pei3qyfjnhymq4cjcwvbpqmcwwnwswdtrfkpnlaxuih2zbdmda" + } with open(cert_path, "w") as f: - f.write(json.dumps({ - "certificate": "{\"expires\":1601687822,\"public_key\":\"pub-v0-cbq6hcf3pxcz6ouoafrbktmkixkeuywpcpbcomzd3lqbkq4nmfga\",\"version\":1}", - "signature": "fvjd3uvvupf2v6tnvkwjd473u3m3inyqkwiclhp7balmchkmn3px5pei3qyfjnhymq4cjcwvbpqmcwwnwswdtrfkpnlaxuih2zbdmda" - })) + f.write(json.dumps(fake_cert)) config_data = ( "[grid_managers]\n" "fluffy = pub-v0-vqimc4s5eflwajttsofisp5st566dbq36xnpp4siz57ufdavpvlq\n" @@ -95,6 +97,8 @@ class GridManagerUtilities(SyncTestCase): 1, len(config.enumerate_section("grid_managers")) ) + certs = _load_grid_manager_certificates(config) + self.assertEqual([fake_cert], certs) class GridManagerVerifier(SyncTestCase): From 32b19fa4d0868fd67bee9756d282e18be8cd3438 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 13 Nov 2020 18:40:44 -0700 Subject: [PATCH 0139/2309] flake8 --- src/allmydata/grid_manager.py | 2 +- src/allmydata/storage_client.py | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/src/allmydata/grid_manager.py b/src/allmydata/grid_manager.py index 6c5230901..464e90b94 100644 --- a/src/allmydata/grid_manager.py +++ b/src/allmydata/grid_manager.py @@ -65,7 +65,7 @@ def load_grid_manager(config_path, config_location): config_file = config_path.child("config.json").open("r") except IOError: raise ValueError( - "'{}' is not a Grid Manager config-directory".format(config) + "'{}' is not a Grid Manager config-directory".format(config_path) ) with config_file: config = json.load(config_file) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 7f24af410..bb5039f46 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -41,9 +41,7 @@ if PY2: import re import time -import json import hashlib -from datetime import datetime # On Python 2 this will be the backport. from configparser import NoSectionError From c7f4f36c8af02156505b4c15856da8f542ea7a5f Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 13 Nov 2020 21:30:23 -0700 Subject: [PATCH 0140/2309] merge config --- src/allmydata/client.py | 12 ------------ src/allmydata/storage_client.py | 21 ++++++++++++++++++--- 2 files changed, 18 insertions(+), 15 deletions(-) diff --git a/src/allmydata/client.py b/src/allmydata/client.py index 006b0eed3..b2b819916 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -92,9 +92,6 @@ _client_config = configutil.ValidConfiguration( ), "grid_managers": None, # means "any options valid" "grid_manager_certificates": None, - "drop_upload": ( # deprecated already? - "enabled", - ), "ftpd": ( "accounts.file", "accounts.url", @@ -560,14 +557,6 @@ def create_storage_farm_broker(config, default_connection_handlers, foolscap_con **kwargs ) - # grid manager setup - - grid_manager_keys = [] - for name, gm_key in config.enumerate_section('grid_managers').items(): - grid_manager_keys.append( - ed25519.verifying_key_from_string(gm_key.encode("ascii")) - ) - # we don't actually use this keypair for anything (yet) as far # as I can see. # my_pubkey = keyutil.parse_pubkey( @@ -581,7 +570,6 @@ def create_storage_farm_broker(config, default_connection_handlers, foolscap_con tub_maker=tub_creator, node_config=config, storage_client_config=storage_client_config, - grid_manager_keys=grid_manager_keys, # XXX maybe roll into above storage_client_config? ) for ic in introducer_clients: sb.use_introducer(ic) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index bb5039f46..aa8431e05 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -74,6 +74,9 @@ from allmydata.interfaces import ( from allmydata.grid_manager import ( create_grid_manager_verifier, ) +from allmydata.crypto import ( + ed25519, +) from allmydata.util import log, base32, connection_status from allmydata.util.assertutil import precondition from allmydata.util.observer import ObserverList @@ -111,9 +114,15 @@ class StorageClientConfig(object): :ivar dict[unicode, dict[unicode, unicode]] storage_plugins: A mapping from names of ``IFoolscapStoragePlugin`` configured in *tahoe.cfg* to the respective configuration. + + :ivar list[ed25519.VerifyKey] grid_manager_keys: with no keys in + this list, we'll upload to any storage server. Otherwise, we will + only upload to a storage-server that has a valid certificate + signed by at least one of these keys. """ preferred_peers = attr.ib(default=()) storage_plugins = attr.ib(default=attr.Factory(dict)) + grid_manager_keys = attr.ib(default=attr.Factory(list)) @classmethod def from_node_config(cls, config): @@ -145,9 +154,17 @@ class StorageClientConfig(object): plugin_config = [] storage_plugins[plugin_name] = dict(plugin_config) + grid_manager_keys = [] + for name, gm_key in config.enumerate_section('grid_managers').items(): + grid_manager_keys.append( + ed25519.verifying_key_from_string(gm_key.encode("ascii")) + ) + + return cls( preferred_peers, storage_plugins, + grid_manager_keys, ) @@ -173,13 +190,11 @@ class StorageFarmBroker(service.MultiService): tub_maker, node_config, storage_client_config=None, - grid_manager_keys=None, ): service.MultiService.__init__(self) assert permute_peers # False not implemented yet self.permute_peers = permute_peers self._tub_maker = tub_maker - self._grid_manager_keys = grid_manager_keys if grid_manager_keys else list() self.node_config = node_config @@ -268,7 +283,7 @@ class StorageFarmBroker(service.MultiService): assert isinstance(server_id, bytes) handler_overrides = server.get("connections", {}) gm_verifier = create_grid_manager_verifier( - self._grid_manager_keys, + self.storage_client_config.grid_manager_keys, server["ann"].get("grid-manager-certificates", []), ) From 45128f49c5b77997f834461aed295cc2037d9c05 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 13 Nov 2020 22:22:18 -0700 Subject: [PATCH 0141/2309] more works on config --- docs/proposed/grid-manager/managed-grid.rst | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/docs/proposed/grid-manager/managed-grid.rst b/docs/proposed/grid-manager/managed-grid.rst index 01181d42a..289ecd004 100644 --- a/docs/proposed/grid-manager/managed-grid.rst +++ b/docs/proposed/grid-manager/managed-grid.rst @@ -1,7 +1,3 @@ -(This document is "in-progress", with feedback and input from two -devchats with Brain Warner and exarkun as well as other input, -discussion and edits from exarkun. It is NOT done). Search for -"DECIDE" for open questions. Managed Grid @@ -65,7 +61,16 @@ use ``--config -`` (the last character is a dash) and write a valid JSON configuration to stdin. All commands require the ``--config`` option and they all behave -similarly for "data from stdin" versus "data from disk". +similarly for "data from stdin" versus "data from disk". A directory +(and not a file) is used on disk because in that mode, each +certificate issued is also stored alongside the configuration +document; in "stdin / stdout" mode, an issued certificate is only +ever available on stdout. + +The configuration is a JSON document. It is subject to change as Grid +Manager evolves. It contains a version number in the +`grid_manager_config_version` key which should increment whenever the +document schema changes. tahoe grid-manager create From 5b4f5e89e14c2f736380db7c451cfcdea01e8892 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 13 Nov 2020 22:22:27 -0700 Subject: [PATCH 0142/2309] explicit unicode --- src/allmydata/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/client.py b/src/allmydata/client.py index b2b819916..4570e660d 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -937,7 +937,7 @@ class _Client(node.Node, pollmixin.PollMixin): if self.config.get_config("storage", "grid_management", default=False, boolean=True): grid_manager_certificates = _load_grid_manager_certificates(self.config) - announcement["grid-manager-certificates"] = grid_manager_certificates + announcement[u"grid-manager-certificates"] = grid_manager_certificates # XXX we should probably verify that the certificates are # valid and not expired, as that could be confusing for the From c58dd36c8729c9ad0b4f22ff072b8e80ad89e9c4 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 13 Nov 2020 22:28:06 -0700 Subject: [PATCH 0143/2309] eliot changes --- src/allmydata/mutable/publish.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/src/allmydata/mutable/publish.py b/src/allmydata/mutable/publish.py index f8f486f51..a6e0d900f 100644 --- a/src/allmydata/mutable/publish.py +++ b/src/allmydata/mutable/publish.py @@ -35,8 +35,10 @@ from allmydata.mutable.layout import get_version_from_checkstring,\ MDMFSlotWriteProxy, \ SDMFSlotWriteProxy -import eliot - +from eliot import ( + Message, + start_action, +) KiB = 1024 DEFAULT_MAX_SEGMENT_SIZE = 128 * KiB @@ -959,7 +961,7 @@ class Publish(object): serverlist = [] - action = eliot.start_action( + action = start_action( action_type=u"mutable:upload:update_goal", homeless_shares=len(homeless_shares), ) @@ -967,17 +969,17 @@ class Publish(object): for i, server in enumerate(self.full_serverlist): serverid = server.get_serverid() if server in self.bad_servers: - action.log( + Message.log( + message_type=u"mutable:upload:bad-server", server_id=server.get_server_id(), - message="Server is bad", ) continue # if we have >= 1 grid-managers, this checks that we have # a valid certificate for this server if not server.upload_permitted(): - action.log( + Message.log( + message_type=u"mutable:upload:no-gm-certs", server_id=server.get_server_id(), - message="No valid grid-manager certificates", ) continue From bab77eded87505b0bb399a3076d7329159f78608 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 13 Nov 2020 22:57:05 -0700 Subject: [PATCH 0144/2309] function not method --- src/allmydata/node.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/src/allmydata/node.py b/src/allmydata/node.py index 6333f8424..498b70a5d 100644 --- a/src/allmydata/node.py +++ b/src/allmydata/node.py @@ -298,19 +298,20 @@ class _Config(object): self.config = configparser if write_new_tahoecfg is None: - write_new_tahoecfg = self._default_write_new_tahoecfg + + def write_new_tahoecfg(config): + """ + Write to the default place, /tahoe.cfg + """ + fn = os.path.join(self._basedir, "tahoe.cfg") + with open(fn, "w") as f: + config.write(f) + self._write_config = write_new_tahoecfg self.nickname = self.get_config("node", "nickname", u"") assert isinstance(self.nickname, str) - def _default_write_new_tahoecfg(self, config): - """ - Write to the default place, /tahoe.cfg - """ - fn = os.path.join(self._basedir, "tahoe.cfg") - with open(fn, "w") as f: - config.write(f) def validate(self, valid_config_sections): configutil.validate_config(self._config_fname, self.config, valid_config_sections) From 0af033beb7453ef02c854e9a820e685c75ffe258 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 13 Nov 2020 22:57:26 -0700 Subject: [PATCH 0145/2309] use filepath --- src/allmydata/scripts/admin.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/src/allmydata/scripts/admin.py b/src/allmydata/scripts/admin.py index b13dfb07d..2b896fcfb 100644 --- a/src/allmydata/scripts/admin.py +++ b/src/allmydata/scripts/admin.py @@ -2,10 +2,10 @@ from __future__ import print_function import sys import json -from os.path import exists, join from twisted.python import usage -#from allmydata.node import read_config +from twisted.python.filepath import FilePath + from allmydata.client import read_config from allmydata.grid_manager import ( parse_grid_manager_certificate, @@ -16,6 +16,7 @@ from allmydata.util.encodingutil import argv_to_abspath from allmydata.util import fileutil + class GenerateKeypairOptions(BaseOptions): def getUsage(self, width=None): @@ -80,7 +81,7 @@ class AddGridManagerCertOptions(BaseOptions): ) if self['filename'] == '-': print("reading certificate from stdin", file=self.parent.parent.stderr) - data = sys.stdin.read() + data = self.parent.parent.stdin.read() if len(data) == 0: raise usage.UsageError( "Reading certificate from stdin failed" @@ -124,11 +125,11 @@ def add_grid_manager_cert(options): config = read_config(nd, "portnum") config_path = join(nd, "tahoe.cfg") cert_fname = "{}.cert".format(options['name']) - cert_path = config.get_config_path(cert_fname) + cert_path = FilePath(config.get_config_path(cert_fname)) cert_bytes = json.dumps(options.certificate_data, indent=4) + '\n' cert_name = options['name'] - if exists(cert_path): + if cert_path.exists(): msg = "Already have certificate for '{}' (at {})".format( options['name'], cert_path, @@ -140,11 +141,8 @@ def add_grid_manager_cert(options): config.set_config("grid_manager_certificates", cert_name, cert_fname) # write all the data out - - fileutil.write(cert_path, cert_bytes) - with open(config_path, "w") as f: - # XXX probably want a _Config.write_tahoe_cfg() or something? - config.config.write(f) + with cert_path.open("wb") as f: + f.wrwite(cert_bytes) cert_count = len(config.enumerate_section("grid_manager_certificates")) print("There are now {} certificates".format(cert_count), file=options.parent.parent.stderr) From 15e33f8b35a59b368f1d5923fde8921d67bf153a Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 13 Nov 2020 22:57:44 -0700 Subject: [PATCH 0146/2309] undo a wording change --- src/allmydata/test/test_abbreviate.py | 2 +- src/allmydata/util/abbreviate.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/test_abbreviate.py b/src/allmydata/test/test_abbreviate.py index 2fb3d6b95..3ef1e96a6 100644 --- a/src/allmydata/test/test_abbreviate.py +++ b/src/allmydata/test/test_abbreviate.py @@ -33,7 +33,7 @@ class Abbreviate(unittest.TestCase): def test_abbrev_time_future_5_minutes(self): diff = timedelta(minutes=-5) s = abbreviate.abbreviate_time(diff) - self.assertEqual('5 minutes from now', s) + self.assertEqual('5 minutes in the future', s) def test_abbrev_time_hours(self): diff = timedelta(hours=4) diff --git a/src/allmydata/util/abbreviate.py b/src/allmydata/util/abbreviate.py index 5ca8aa3d1..f895c3727 100644 --- a/src/allmydata/util/abbreviate.py +++ b/src/allmydata/util/abbreviate.py @@ -40,7 +40,7 @@ def abbreviate_time(s): if s >= 0.0: postfix = ' ago' else: - postfix = ' from now' + postfix = ' in the future' s = -s def _plural(count, unit): count = int(count) From 1ccc074d04b42d7f5f509ca0c594046d1ce40fbf Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 13 Nov 2020 22:59:11 -0700 Subject: [PATCH 0147/2309] docstring --- src/allmydata/test/test_grid_manager.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/allmydata/test/test_grid_manager.py b/src/allmydata/test/test_grid_manager.py index 6ff4fb065..9188626c6 100644 --- a/src/allmydata/test/test_grid_manager.py +++ b/src/allmydata/test/test_grid_manager.py @@ -38,6 +38,10 @@ class GridManagerUtilities(SyncTestCase): """ def test_client_grid_manager(self): + """ + A client refuses to upload to a storage-server with an invalid + certificate when using Grid Manager. + """ config_data = ( "[grid_managers]\n" "fluffy = pub-v0-vqimc4s5eflwajttsofisp5st566dbq36xnpp4siz57ufdavpvlq\n" From 51b52174e19d292fc5f28152a31ffb1ae259884d Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 13 Nov 2020 22:59:43 -0700 Subject: [PATCH 0148/2309] docstring --- src/allmydata/test/test_grid_manager.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/allmydata/test/test_grid_manager.py b/src/allmydata/test/test_grid_manager.py index 9188626c6..84ac280e1 100644 --- a/src/allmydata/test/test_grid_manager.py +++ b/src/allmydata/test/test_grid_manager.py @@ -83,6 +83,9 @@ class GridManagerUtilities(SyncTestCase): self.assertFalse(nss.upload_permitted()) def test_load_certificates(self): + """ + Grid Manager certificates are deserialized from config properly + """ cert_path = self.mktemp() fake_cert = { "certificate": "{\"expires\":1601687822,\"public_key\":\"pub-v0-cbq6hcf3pxcz6ouoafrbktmkixkeuywpcpbcomzd3lqbkq4nmfga\",\"version\":1}", From 3a552f68caa1cc470708417248a35463831d6c71 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 13 Nov 2020 23:09:15 -0700 Subject: [PATCH 0149/2309] don't ned this test --- src/allmydata/test/test_grid_manager.py | 45 ------------------------- 1 file changed, 45 deletions(-) diff --git a/src/allmydata/test/test_grid_manager.py b/src/allmydata/test/test_grid_manager.py index 84ac280e1..946722540 100644 --- a/src/allmydata/test/test_grid_manager.py +++ b/src/allmydata/test/test_grid_manager.py @@ -37,51 +37,6 @@ class GridManagerUtilities(SyncTestCase): Confirm operation of utility functions used by GridManager """ - def test_client_grid_manager(self): - """ - A client refuses to upload to a storage-server with an invalid - certificate when using Grid Manager. - """ - config_data = ( - "[grid_managers]\n" - "fluffy = pub-v0-vqimc4s5eflwajttsofisp5st566dbq36xnpp4siz57ufdavpvlq\n" - ) - config = config_from_string("/foo", "portnum", config_data, client_valid_config()) - sfb = create_storage_farm_broker(config, {}, {}, {}, []) - # could introspect sfb._grid_manager_certificates, but that's - # "cheating"? even though _make_storage_sever is also - # "private"? - - # ...but, okay, a "real" client will call set_static_servers() - # with any configured/cached servers (thus causing - # _make_storage_server to be called). The other way - # _make_storage_server is called is when _got_announcement - # runs, which is when an introducer client gets an - # announcement... - - invalid_cert = { - "certificate": "foo", - "signature": "43564356435643564356435643564356", - } - announcement = { - "anonymous-storage-FURL": b"pb://abcde@nowhere/fake", - "grid-manager-certificates": [ - invalid_cert, - ] - } - static_servers = { - "v0-4uazse3xb6uu5qpkb7tel2bm6bpea4jhuigdhqcuvvse7hugtsia": { - "ann": announcement, - } - } - sfb.set_static_servers(static_servers) - nss = sfb._make_storage_server(b"server0", {"ann": announcement}) - - # we have some grid-manager keys defined so the server should - # only upload if there's a valid certificate -- but the only - # one we have is invalid - self.assertFalse(nss.upload_permitted()) - def test_load_certificates(self): """ Grid Manager certificates are deserialized from config properly From cb6ed4c47ffd8a11b411c157b0dd16883e2261d8 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 13 Nov 2020 23:14:27 -0700 Subject: [PATCH 0150/2309] flake8 --- src/allmydata/scripts/admin.py | 3 --- src/allmydata/test/test_grid_manager.py | 1 - 2 files changed, 4 deletions(-) diff --git a/src/allmydata/scripts/admin.py b/src/allmydata/scripts/admin.py index 2b896fcfb..803f6561c 100644 --- a/src/allmydata/scripts/admin.py +++ b/src/allmydata/scripts/admin.py @@ -1,6 +1,5 @@ from __future__ import print_function -import sys import json from twisted.python import usage @@ -13,7 +12,6 @@ from allmydata.grid_manager import ( from allmydata.scripts.cli import _default_nodedir from allmydata.scripts.common import BaseOptions from allmydata.util.encodingutil import argv_to_abspath -from allmydata.util import fileutil @@ -123,7 +121,6 @@ def add_grid_manager_cert(options): nd = _default_nodedir config = read_config(nd, "portnum") - config_path = join(nd, "tahoe.cfg") cert_fname = "{}.cert".format(options['name']) cert_path = FilePath(config.get_config_path(cert_fname)) cert_bytes = json.dumps(options.certificate_data, indent=4) + '\n' diff --git a/src/allmydata/test/test_grid_manager.py b/src/allmydata/test/test_grid_manager.py index 946722540..f7c4d4c7f 100644 --- a/src/allmydata/test/test_grid_manager.py +++ b/src/allmydata/test/test_grid_manager.py @@ -6,7 +6,6 @@ from twisted.python.filepath import ( ) from allmydata.client import ( - create_storage_farm_broker, _load_grid_manager_certificates, ) from allmydata.node import ( From 7fa0f64f1ccd4bbef8aff4acc3fe13d8788e75ed Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 13 Nov 2020 23:38:00 -0700 Subject: [PATCH 0151/2309] better assert --- src/allmydata/test/test_grid_manager.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/test_grid_manager.py b/src/allmydata/test/test_grid_manager.py index f7c4d4c7f..45d2061b2 100644 --- a/src/allmydata/test/test_grid_manager.py +++ b/src/allmydata/test/test_grid_manager.py @@ -55,8 +55,8 @@ class GridManagerUtilities(SyncTestCase): ) config = config_from_string("/foo", "portnum", config_data, client_valid_config()) self.assertEqual( - 1, - len(config.enumerate_section("grid_managers")) + {"fluffy": "pub-v0-vqimc4s5eflwajttsofisp5st566dbq36xnpp4siz57ufdavpvlq"}, + config.enumerate_section("grid_managers") ) certs = _load_grid_manager_certificates(config) self.assertEqual([fake_cert], certs) From f3b33412678bdd1466ae30150f4f1a2317f9dc77 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 13 Nov 2020 23:40:17 -0700 Subject: [PATCH 0152/2309] better-ize test --- src/allmydata/test/test_node.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/allmydata/test/test_node.py b/src/allmydata/test/test_node.py index 5491ffc0e..c032d20d7 100644 --- a/src/allmydata/test/test_node.py +++ b/src/allmydata/test/test_node.py @@ -432,10 +432,17 @@ class TestCase(testutil.SignalMixin, unittest.TestCase): self.failUnless(ns.called) def test_set_config_new_section(self): + """ + set_config() can create a new config section + """ basedir = "test_node/test_set_config_new_section" config = config_from_string(basedir, "", "") config.set_config("foo", "bar", "value1") config.set_config("foo", "bar", "value2") + self.assertEqual( + config.get_config("foo", "bar"), + "value2" + ) class TestMissingPorts(unittest.TestCase): From f6f53ad16514b379e99c02c7ad6eb4a0f5cf2896 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 13 Nov 2020 23:41:08 -0700 Subject: [PATCH 0153/2309] proper spot --- docs/{proposed/grid-manager => }/managed-grid.rst | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename docs/{proposed/grid-manager => }/managed-grid.rst (100%) diff --git a/docs/proposed/grid-manager/managed-grid.rst b/docs/managed-grid.rst similarity index 100% rename from docs/proposed/grid-manager/managed-grid.rst rename to docs/managed-grid.rst From 5a88dfd5753ba4243613566f02a878bc435d4e34 Mon Sep 17 00:00:00 2001 From: meejah Date: Sat, 14 Nov 2020 01:56:03 -0700 Subject: [PATCH 0154/2309] refactor integration tests and add some for grid-manager --- integration/conftest.py | 172 ++------ integration/grid.py | 507 +++++++++++++++++++++++ integration/test_servers_of_happiness.py | 7 +- integration/test_tor.py | 39 +- integration/test_web.py | 10 +- integration/util.py | 54 ++- 6 files changed, 614 insertions(+), 175 deletions(-) create mode 100644 integration/grid.py diff --git a/integration/conftest.py b/integration/conftest.py index ca18230cd..15450767b 100644 --- a/integration/conftest.py +++ b/integration/conftest.py @@ -8,10 +8,6 @@ from os.path import join, exists from tempfile import mkdtemp, mktemp from functools import partial -from foolscap.furl import ( - decode_furl, -) - from eliot import ( to_file, log_call, @@ -38,6 +34,11 @@ from util import ( await_client_ready, TahoeProcess, ) +from grid import ( + create_port_allocator, + create_flog_gatherer, + create_grid, +) # pytest customization hooks @@ -74,6 +75,12 @@ def reactor(): return _reactor +@pytest.fixture(scope='session') +@log_call(action_type=u"integration:port_allocator", include_result=False) +def port_allocator(reactor): + return create_port_allocator(start_port=45000) + + @pytest.fixture(scope='session') @log_call(action_type=u"integration:temp_dir", include_args=[]) def temp_dir(request): @@ -108,137 +115,30 @@ def flog_binary(): @pytest.fixture(scope='session') @log_call(action_type=u"integration:flog_gatherer", include_args=[]) def flog_gatherer(reactor, temp_dir, flog_binary, request): - out_protocol = _CollectOutputProtocol() - gather_dir = join(temp_dir, 'flog_gather') - reactor.spawnProcess( - out_protocol, - flog_binary, - ( - 'flogtool', 'create-gatherer', - '--location', 'tcp:localhost:3117', - '--port', '3117', - gather_dir, - ) + fg = pytest_twisted.blockon( + create_flog_gatherer(reactor, request, temp_dir, flog_binary) ) - pytest_twisted.blockon(out_protocol.done) - - twistd_protocol = _MagicTextProtocol("Gatherer waiting at") - twistd_process = reactor.spawnProcess( - twistd_protocol, - which('twistd')[0], - ( - 'twistd', '--nodaemon', '--python', - join(gather_dir, 'gatherer.tac'), - ), - path=gather_dir, - ) - pytest_twisted.blockon(twistd_protocol.magic_seen) - - def cleanup(): - _cleanup_tahoe_process(twistd_process, twistd_protocol.exited) - - flog_file = mktemp('.flog_dump') - flog_protocol = _DumpOutputProtocol(open(flog_file, 'w')) - flog_dir = join(temp_dir, 'flog_gather') - flogs = [x for x in listdir(flog_dir) if x.endswith('.flog')] - - print("Dumping {} flogtool logfiles to '{}'".format(len(flogs), flog_file)) - reactor.spawnProcess( - flog_protocol, - flog_binary, - ( - 'flogtool', 'dump', join(temp_dir, 'flog_gather', flogs[0]) - ), - ) - print("Waiting for flogtool to complete") - try: - pytest_twisted.blockon(flog_protocol.done) - except ProcessTerminated as e: - print("flogtool exited unexpectedly: {}".format(str(e))) - print("Flogtool completed") - - request.addfinalizer(cleanup) - - with open(join(gather_dir, 'log_gatherer.furl'), 'r') as f: - furl = f.read().strip() - return furl + return fg @pytest.fixture(scope='session') -@log_call( - action_type=u"integration:introducer", - include_args=["temp_dir", "flog_gatherer"], - include_result=False, -) -def introducer(reactor, temp_dir, flog_gatherer, request): - config = ''' -[node] -nickname = introducer0 -web.port = 4560 -log_gatherer.furl = {log_furl} -'''.format(log_furl=flog_gatherer) - - intro_dir = join(temp_dir, 'introducer') - print("making introducer", intro_dir) - - if not exists(intro_dir): - mkdir(intro_dir) - done_proto = _ProcessExitedProtocol() - _tahoe_runner_optional_coverage( - done_proto, - reactor, - request, - ( - 'create-introducer', - '--listen=tcp', - '--hostname=localhost', - intro_dir, - ), - ) - pytest_twisted.blockon(done_proto.done) - - # over-write the config file with our stuff - with open(join(intro_dir, 'tahoe.cfg'), 'w') as f: - f.write(config) - - # on windows, "tahoe start" means: run forever in the foreground, - # but on linux it means daemonize. "tahoe run" is consistent - # between platforms. - protocol = _MagicTextProtocol('introducer running') - transport = _tahoe_runner_optional_coverage( - protocol, - reactor, - request, - ( - 'run', - intro_dir, - ), +@log_call(action_type=u"integration:grid", include_args=[]) +def grid(reactor, request, temp_dir, flog_gatherer, port_allocator): + g = pytest_twisted.blockon( + create_grid(reactor, request, temp_dir, flog_gatherer, port_allocator) ) - request.addfinalizer(partial(_cleanup_tahoe_process, transport, protocol.exited)) + return g - pytest_twisted.blockon(protocol.magic_seen) - return TahoeProcess(transport, intro_dir) + +@pytest.fixture(scope='session') +def introducer(grid): + return grid.introducer @pytest.fixture(scope='session') @log_call(action_type=u"integration:introducer:furl", include_args=["temp_dir"]) def introducer_furl(introducer, temp_dir): - furl_fname = join(temp_dir, 'introducer', 'private', 'introducer.furl') - while not exists(furl_fname): - print("Don't see {} yet".format(furl_fname)) - sleep(.1) - furl = open(furl_fname, 'r').read() - tubID, location_hints, name = decode_furl(furl) - if not location_hints: - # If there are no location hints then nothing can ever possibly - # connect to it and the only thing that can happen next is something - # will hang or time out. So just give up right now. - raise ValueError( - "Introducer ({!r}) fURL has no location hints!".format( - introducer_furl, - ), - ) - return furl + return introducer.furl @pytest.fixture(scope='session') @@ -317,28 +217,20 @@ def tor_introducer_furl(tor_introducer, temp_dir): @pytest.fixture(scope='session') @log_call( action_type=u"integration:storage_nodes", - include_args=["temp_dir", "introducer_furl", "flog_gatherer"], + include_args=["grid"], include_result=False, ) -def storage_nodes(reactor, temp_dir, introducer, introducer_furl, flog_gatherer, request): +def storage_nodes(grid): nodes_d = [] # start all 5 nodes in parallel for x in range(5): - name = 'node{}'.format(x) - web_port= 9990 + x - nodes_d.append( - _create_node( - reactor, request, temp_dir, introducer_furl, flog_gatherer, name, - web_port="tcp:{}:interface=localhost".format(web_port), - storage=True, - ) - ) + #nodes_d.append(grid.add_storage_node()) + pytest_twisted.blockon(grid.add_storage_node()) + nodes_status = pytest_twisted.blockon(DeferredList(nodes_d)) - nodes = [] - for ok, process in nodes_status: - assert ok, "Storage node creation failed: {}".format(process) - nodes.append(process) - return nodes + for ok, value in nodes_status: + assert ok, "Storage node creation failed: {}".format(value) + return grid.storage_servers @pytest.fixture(scope='session') diff --git a/integration/grid.py b/integration/grid.py new file mode 100644 index 000000000..5c3086eea --- /dev/null +++ b/integration/grid.py @@ -0,0 +1,507 @@ +""" +Classes which directly represent various kinds of Tahoe processes +that co-operate to for "a Grid". + +These methods and objects are used by conftest.py fixtures but may +also be used as direct helpers for tests that don't want to (or can't) +rely on 'the' global grid as provided by fixtures like 'alice' or +'storage_servers'. +""" + +from os import mkdir, listdir, environ +from os.path import join, exists +from tempfile import mkdtemp, mktemp + +from eliot import ( + log_call, +) + +from foolscap.furl import ( + decode_furl, +) + +from twisted.python.procutils import which +from twisted.internet.defer import ( + inlineCallbacks, + returnValue, + maybeDeferred, +) +from twisted.internet.task import ( + deferLater, +) +from twisted.internet.interfaces import ( + IProcessTransport, + IProcessProtocol, + IProtocol, +) +from twisted.internet.endpoints import ( + TCP4ServerEndpoint, +) +from twisted.internet.protocol import ( + Factory, + Protocol, +) + +from util import ( + _CollectOutputProtocol, + _MagicTextProtocol, + _DumpOutputProtocol, + _ProcessExitedProtocol, + _create_node, + _run_node, + _cleanup_tahoe_process, + _tahoe_runner_optional_coverage, + TahoeProcess, + await_client_ready, +) + +import attr +import pytest_twisted + + +# further directions: +# - "Grid" is unused, basically -- tie into the rest? +# - could make a Grid instance mandatory for create_* calls +# - could instead make create_* calls methods of Grid +# - Bring more 'util' or 'conftest' code into here +# - stop()/start()/restart() methods on StorageServer etc +# - more-complex stuff like config changes (which imply a restart too)? + + +@attr.s +class FlogGatherer(object): + """ + Flog Gatherer process. + """ + + process = attr.ib( + validator=attr.validators.provides(IProcessTransport) + ) + protocol = attr.ib( + validator=attr.validators.provides(IProcessProtocol) + ) + furl = attr.ib() + + +@inlineCallbacks +def create_flog_gatherer(reactor, request, temp_dir, flog_binary): + out_protocol = _CollectOutputProtocol() + gather_dir = join(temp_dir, 'flog_gather') + reactor.spawnProcess( + out_protocol, + flog_binary, + ( + 'flogtool', 'create-gatherer', + '--location', 'tcp:localhost:3117', + '--port', '3117', + gather_dir, + ) + ) + yield out_protocol.done + + twistd_protocol = _MagicTextProtocol("Gatherer waiting at") + twistd_process = reactor.spawnProcess( + twistd_protocol, + which('twistd')[0], + ( + 'twistd', '--nodaemon', '--python', + join(gather_dir, 'gatherer.tac'), + ), + path=gather_dir, + ) + yield twistd_protocol.magic_seen + + def cleanup(): + _cleanup_tahoe_process(twistd_process, twistd_protocol.exited) + + flog_file = mktemp('.flog_dump') + flog_protocol = _DumpOutputProtocol(open(flog_file, 'w')) + flog_dir = join(temp_dir, 'flog_gather') + flogs = [x for x in listdir(flog_dir) if x.endswith('.flog')] + + print("Dumping {} flogtool logfiles to '{}'".format(len(flogs), flog_file)) + reactor.spawnProcess( + flog_protocol, + flog_binary, + ( + 'flogtool', 'dump', join(temp_dir, 'flog_gather', flogs[0]) + ), + ) + print("Waiting for flogtool to complete") + try: + pytest_twisted.blockon(flog_protocol.done) + except ProcessTerminated as e: + print("flogtool exited unexpectedly: {}".format(str(e))) + print("Flogtool completed") + + request.addfinalizer(cleanup) + + with open(join(gather_dir, 'log_gatherer.furl'), 'r') as f: + furl = f.read().strip() + returnValue( + FlogGatherer( + protocol=twistd_protocol, + process=twistd_process, + furl=furl, + ) + ) + + +@attr.s +class StorageServer(object): + """ + Represents a Tahoe Storage Server + """ + + process = attr.ib( + validator=attr.validators.instance_of(TahoeProcess) + ) + protocol = attr.ib( + validator=attr.validators.provides(IProcessProtocol) + ) + + @inlineCallbacks + def restart(self, reactor, request): + """ + re-start our underlying process by issuing a TERM, waiting and + then running again. await_client_ready() will be done as well + + Note that self.process and self.protocol will be new instances + after this. + """ + self.process.transport.signalProcess('TERM') + yield self.protocol.exited + self.process = yield _run_node( + reactor, self.process.node_dir, request, None, + ) + self.protocol = self.process.transport._protocol + + +@inlineCallbacks +def create_storage_server(reactor, request, temp_dir, introducer, flog_gatherer, name, web_port, + needed=2, happy=3, total=4): + """ + Create a new storage server + """ + from util import _create_node + node_process = yield _create_node( + reactor, request, temp_dir, introducer.furl, flog_gatherer, + name, web_port, storage=True, needed=needed, happy=happy, total=total, + ) + storage = StorageServer( + process=node_process, + protocol=node_process.transport._protocol, + ) + returnValue(storage) + + +@attr.s +class Client(object): + """ + Represents a Tahoe client + """ + + process = attr.ib( + validator=attr.validators.instance_of(TahoeProcess) + ) + protocol = attr.ib( + validator=attr.validators.provides(IProcessProtocol) + ) + + @inlineCallbacks + def restart(self, reactor, request, servers=1): + """ + re-start our underlying process by issuing a TERM, waiting and + then running again. + + :param int servers: number of server connections we will wait + for before being 'ready' + + Note that self.process and self.protocol will be new instances + after this. + """ + self.process.transport.signalProcess('TERM') + yield self.protocol.exited + process = yield _run_node( + reactor, self.process.node_dir, request, None, + ) + self.process = process + self.protocol = self.process.transport._protocol + + + # XXX add stop / start / restart + # ...maybe "reconfig" of some kind? + + +@inlineCallbacks +def create_client(reactor, request, temp_dir, introducer, flog_gatherer, name, web_port, + needed=2, happy=3, total=4): + """ + Create a new storage server + """ + from util import _create_node + node_process = yield _create_node( + reactor, request, temp_dir, introducer.furl, flog_gatherer, + name, web_port, storage=False, needed=needed, happy=happy, total=total, + ) + returnValue( + Client( + process=node_process, + protocol=node_process.transport._protocol, + ) + ) + + +@attr.s +class Introducer(object): + """ + Reprsents a running introducer + """ + + process = attr.ib( + validator=attr.validators.instance_of(TahoeProcess) + ) + protocol = attr.ib( + validator=attr.validators.provides(IProcessProtocol) + ) + furl = attr.ib() + + +def _validate_furl(furl_fname): + """ + Opens and validates a fURL, ensuring location hints. + :returns: the furl + :raises: ValueError if no location hints + """ + while not exists(furl_fname): + print("Don't see {} yet".format(furl_fname)) + sleep(.1) + furl = open(furl_fname, 'r').read() + tubID, location_hints, name = decode_furl(furl) + if not location_hints: + # If there are no location hints then nothing can ever possibly + # connect to it and the only thing that can happen next is something + # will hang or time out. So just give up right now. + raise ValueError( + "Introducer ({!r}) fURL has no location hints!".format( + introducer_furl, + ), + ) + return furl + + +@inlineCallbacks +@log_call( + action_type=u"integration:introducer", + include_args=["temp_dir", "flog_gatherer"], + include_result=False, +) +def create_introducer(reactor, request, temp_dir, flog_gatherer, port): + """ + Run a new Introducer and return an Introducer instance. + """ + config = ( + '[node]\n' + 'nickname = introducer{port}\n' + 'web.port = {port}\n' + 'log_gatherer.furl = {log_furl}\n' + ).format( + port=port, + log_furl=flog_gatherer.furl, + ) + + intro_dir = join(temp_dir, 'introducer{}'.format(port)) + + if not exists(intro_dir): + mkdir(intro_dir) + done_proto = _ProcessExitedProtocol() + _tahoe_runner_optional_coverage( + done_proto, + reactor, + request, + ( + 'create-introducer', + '--listen=tcp', + '--hostname=localhost', + intro_dir, + ), + ) + yield done_proto.done + + # over-write the config file with our stuff + with open(join(intro_dir, 'tahoe.cfg'), 'w') as f: + f.write(config) + + # on windows, "tahoe start" means: run forever in the foreground, + # but on linux it means daemonize. "tahoe run" is consistent + # between platforms. + protocol = _MagicTextProtocol('introducer running') + transport = _tahoe_runner_optional_coverage( + protocol, + reactor, + request, + ( + 'run', + intro_dir, + ), + ) + + def clean(): + return _cleanup_tahoe_process(transport, protocol.exited) + request.addfinalizer(clean) + + yield protocol.magic_seen + + furl_fname = join(intro_dir, 'private', 'introducer.furl') + while not exists(furl_fname): + print("Don't see {} yet".format(furl_fname)) + yield deferLater(reactor, .1, lambda: None) + furl = _validate_furl(furl_fname) + + returnValue( + Introducer( + process=TahoeProcess(transport, intro_dir), + protocol=protocol, + furl=furl, + ) + ) + + +@attr.s +class Grid(object): + """ + Represents an entire Tahoe Grid setup + + A Grid includes an Introducer, Flog Gatherer and some number of + Storage Servers. + """ + + _reactor = attr.ib() + _request = attr.ib() + _temp_dir = attr.ib() + _port_allocator = attr.ib() + introducer = attr.ib() + flog_gatherer = attr.ib() + storage_servers = attr.ib(factory=list) + clients = attr.ib(factory=dict) + + @storage_servers.validator + def check(self, attribute, value): + for server in value: + if not isinstance(server, StorageServer): + raise ValueError( + "storage_servers must be StorageServer" + ) + + @inlineCallbacks + def add_storage_node(self): + """ + Creates a new storage node, returns a StorageServer instance + (which will already be added to our .storage_servers list) + """ + port = yield self._port_allocator() + print("make {}".format(port)) + name = 'node{}'.format(port) + web_port = 'tcp:{}:interface=localhost'.format(port) + server = yield create_storage_server( + self._reactor, + self._request, + self._temp_dir, + self.introducer, + self.flog_gatherer, + name, + web_port, + ) + self.storage_servers.append(server) + returnValue(server) + + @inlineCallbacks + def add_client(self, name, needed=2, happy=3, total=4): + """ + Create a new client node + """ + port = yield self._port_allocator() + web_port = 'tcp:{}:interface=localhost'.format(port) + client = yield create_client( + self._reactor, + self._request, + self._temp_dir, + self.introducer, + self.flog_gatherer, + name, + web_port, + needed=needed, + happy=happy, + total=total, + ) + self.clients[name] = client + yield await_client_ready(client.process) + returnValue(client) + + + +# XXX THINK can we tie a whole *grid* to a single request? (I think +# that's all that makes sense) +@inlineCallbacks +def create_grid(reactor, request, temp_dir, flog_gatherer, port_allocator): + """ + """ + intro_port = yield port_allocator() + introducer = yield create_introducer(reactor, request, temp_dir, flog_gatherer, intro_port) + grid = Grid( + reactor, + request, + temp_dir, + port_allocator, + introducer, + flog_gatherer, + ) + returnValue(grid) + + +def create_port_allocator(start_port): + """ + Returns a new port-allocator .. which is a zero-argument function + that returns Deferreds that fire with new, sequential ports + starting at `start_port` skipping any that already appear to have + a listener. + + There can still be a race against other processes allocating ports + -- between the time when we check the status of the port and when + our subprocess starts up. This *could* be mitigated by instructing + the OS to not randomly-allocate ports in some range, and then + using that range here (explicitly, ourselves). + + NB once we're Python3-only this could be an async-generator + """ + port = [start_port - 1] + + # import stays here to not interfere with reactor selection -- but + # maybe this function should be arranged to be called once from a + # fixture (with the reactor)? + from twisted.internet import reactor + + class NothingProtocol(Protocol): + """ + I do nothing. + """ + + def port_generator(): + print("Checking port {}".format(port)) + port[0] += 1 + ep = TCP4ServerEndpoint(reactor, port[0], interface="localhost") + d = ep.listen(Factory.forProtocol(NothingProtocol)) + + def good(listening_port): + unlisten_d = maybeDeferred(listening_port.stopListening) + def return_port(_): + return port[0] + unlisten_d.addBoth(return_port) + return unlisten_d + + def try_again(fail): + return port_generator() + + d.addCallbacks(good, try_again) + return d + return port_generator diff --git a/integration/test_servers_of_happiness.py b/integration/test_servers_of_happiness.py index 97392bf00..fe3a466eb 100644 --- a/integration/test_servers_of_happiness.py +++ b/integration/test_servers_of_happiness.py @@ -38,8 +38,7 @@ def test_upload_immutable(reactor, temp_dir, introducer_furl, flog_gatherer, sto try: yield proto.done assert False, "should raise exception" - except Exception as e: - assert isinstance(e, ProcessTerminated) + except util.ProcessFailed as e: + assert "UploadUnhappinessError" in e.output - output = proto.output.getvalue() - assert "shares could be placed on only" in output + assert "shares could be placed on only" in proto.output.getvalue() diff --git a/integration/test_tor.py b/integration/test_tor.py index 3d169a88f..db38b13ea 100644 --- a/integration/test_tor.py +++ b/integration/test_tor.py @@ -69,25 +69,28 @@ def _create_anonymous_node(reactor, name, control_port, request, temp_dir, flog_ node_dir = join(temp_dir, name) web_port = "tcp:{}:interface=localhost".format(control_port + 2000) - if True: - print("creating", node_dir) - mkdir(node_dir) - proto = util._DumpOutputProtocol(None) - reactor.spawnProcess( - proto, - sys.executable, - ( - sys.executable, '-m', 'allmydata.scripts.runner', - 'create-node', - '--nickname', name, - '--introducer', introducer_furl, - '--hide-ip', - '--tor-control-port', 'tcp:localhost:{}'.format(control_port), - '--listen', 'tor', - node_dir, - ) + if exists(node_dir): + raise RuntimeError( + "A node already exists in '{}'".format(node_dir) ) - yield proto.done + print("creating", node_dir) + mkdir(node_dir) + proto = util._DumpOutputProtocol(None) + reactor.spawnProcess( + proto, + sys.executable, + ( + sys.executable, '-m', 'allmydata.scripts.runner', + 'create-node', + '--nickname', name, + '--introducer', introducer_furl, + '--hide-ip', + '--tor-control-port', 'tcp:localhost:{}'.format(control_port), + '--listen', 'tor', + node_dir, + ) + ) + yield proto.done with open(join(node_dir, 'tahoe.cfg'), 'w') as f: f.write(''' diff --git a/integration/test_web.py b/integration/test_web.py index fe2137ff3..6986e74c5 100644 --- a/integration/test_web.py +++ b/integration/test_web.py @@ -91,7 +91,7 @@ def test_helper_status(storage_nodes): successfully GET the /helper_status page """ - url = util.node_url(storage_nodes[0].node_dir, "helper_status") + url = util.node_url(storage_nodes[0].process.node_dir, "helper_status") resp = requests.get(url) assert resp.status_code >= 200 and resp.status_code < 300 dom = BeautifulSoup(resp.content, "html5lib") @@ -412,7 +412,7 @@ def test_storage_info(storage_nodes): storage0 = storage_nodes[0] requests.get( - util.node_url(storage0.node_dir, u"storage"), + util.node_url(storage0.process.node_dir, u"storage"), ) @@ -423,7 +423,7 @@ def test_storage_info_json(storage_nodes): storage0 = storage_nodes[0] resp = requests.get( - util.node_url(storage0.node_dir, u"storage"), + util.node_url(storage0.process.node_dir, u"storage"), params={u"t": u"json"}, ) data = json.loads(resp.content) @@ -435,12 +435,12 @@ def test_introducer_info(introducer): retrieve and confirm /introducer URI for the introducer """ resp = requests.get( - util.node_url(introducer.node_dir, u""), + util.node_url(introducer.process.node_dir, u""), ) assert "Introducer" in resp.content resp = requests.get( - util.node_url(introducer.node_dir, u""), + util.node_url(introducer.process.node_dir, u""), params={u"t": u"json"}, ) data = json.loads(resp.content) diff --git a/integration/util.py b/integration/util.py index a64bcbf8e..54898ec4a 100644 --- a/integration/util.py +++ b/integration/util.py @@ -5,6 +5,7 @@ from os import mkdir, environ from os.path import exists, join from six.moves import StringIO from functools import partial +from shutil import rmtree from twisted.internet.defer import Deferred, succeed from twisted.internet.protocol import ProcessProtocol @@ -35,15 +36,38 @@ class _ProcessExitedProtocol(ProcessProtocol): self.done.callback(None) +class ProcessFailed(Exception): + """ + A subprocess has failed. + + :ivar ProcessTerminated reason: the original reason from .processExited + + :ivar StringIO output: all stdout and stderr collected to this point. + """ + + def __init__(self, reason, output): + self.reason = reason + self.output = output + + def __str__(self): + return ":\n{}".format(self.reason, self.output) + + class _CollectOutputProtocol(ProcessProtocol): """ Internal helper. Collects all output (stdout + stderr) into self.output, and callback's on done with all of it after the process exits (for any reason). """ - def __init__(self): + def __init__(self, stdin=None): self.done = Deferred() self.output = StringIO() + self._stdin = stdin + + def connectionMade(self): + if self._stdin is not None: + self.transport.write(self._stdin) + self.transport.closeStdin() def processEnded(self, reason): if not self.done.called: @@ -51,7 +75,7 @@ class _CollectOutputProtocol(ProcessProtocol): def processExited(self, reason): if not isinstance(reason.value, ProcessDone): - self.done.errback(reason) + self.done.errback(ProcessFailed(reason, self.output.getvalue())) def outReceived(self, data): self.output.write(data) @@ -123,13 +147,27 @@ def _cleanup_tahoe_process(tahoe_transport, exited): try: print("signaling {} with TERM".format(tahoe_transport.pid)) tahoe_transport.signalProcess('TERM') - print("signaled, blocking on exit") + print("signaled, blocking on exit {}".format(exited)) pytest_twisted.blockon(exited) print("exited, goodbye") except ProcessExitedAlready: pass +def run_tahoe(reactor, request, *args, **kwargs): + """ + Helper to run tahoe with optional coverage. + + :returns: a Deferred that fires when the command is done (or a + ProcessFailed exception if it exits non-zero) + """ + stdin = kwargs.get("stdin", None) + protocol = _CollectOutputProtocol(stdin=stdin) + process = _tahoe_runner_optional_coverage(protocol, reactor, request, args) + process.exited = protocol.done + return protocol.done + + def _tahoe_runner_optional_coverage(proto, reactor, request, other_args): """ Internal helper. Calls spawnProcess with `-m @@ -232,7 +270,7 @@ def _create_node(reactor, request, temp_dir, introducer_furl, flog_gatherer, nam if exists(node_dir): created_d = succeed(None) else: - print("creating", node_dir) + print("creating: {}".format(node_dir)) mkdir(node_dir) done_proto = _ProcessExitedProtocol() args = [ @@ -257,7 +295,7 @@ def _create_node(reactor, request, temp_dir, introducer_furl, flog_gatherer, nam def created(_): config_path = join(node_dir, 'tahoe.cfg') config = get_config(config_path) - set_config(config, 'node', 'log_gatherer.furl', flog_gatherer) + set_config(config, 'node', 'log_gatherer.furl', flog_gatherer.furl) write_config(config_path, config) created_d.addCallback(created) @@ -444,7 +482,7 @@ def web_post(tahoe, uri_fragment, **kwargs): return resp.content -def await_client_ready(tahoe, timeout=10, liveness=60*2): +def await_client_ready(tahoe, timeout=10, liveness=60*2, servers=1): """ Uses the status API to wait for a client-type node (in `tahoe`, a `TahoeProcess` instance usually from a fixture e.g. `alice`) to be @@ -468,8 +506,8 @@ def await_client_ready(tahoe, timeout=10, liveness=60*2): time.sleep(1) continue - if len(js['servers']) == 0: - print("waiting because no servers at all") + if len(js['servers']) < servers: + print("waiting because fewer than {} server(s)".format(servers)) time.sleep(1) continue server_times = [ From 2e2128619335d7b6a87f7fac060544568582232d Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 16 Nov 2020 01:19:01 -0700 Subject: [PATCH 0155/2309] grid-manager tests --- integration/test_grid_manager.py | 274 +++++++++++++++++++++++++++++++ 1 file changed, 274 insertions(+) create mode 100644 integration/test_grid_manager.py diff --git a/integration/test_grid_manager.py b/integration/test_grid_manager.py new file mode 100644 index 000000000..ce426c6d7 --- /dev/null +++ b/integration/test_grid_manager.py @@ -0,0 +1,274 @@ +import sys +import time +import json +import shutil +from os import mkdir, unlink, listdir, utime +from os.path import join, exists, getmtime + +from cryptography.hazmat.primitives.serialization import ( + Encoding, + PublicFormat, +) + +from twisted.internet.utils import ( + getProcessOutputAndValue, +) +from twisted.internet.defer import ( + inlineCallbacks, + returnValue, +) + +from allmydata.crypto import ed25519 +from allmydata.util import base32 +from allmydata.util import configutil + +import util +from grid import ( + create_grid, +) + +import pytest_twisted + + +@inlineCallbacks +def _run_gm(reactor, *args, **kwargs): + """ + Run the grid-manager process, passing all arguments as extra CLI + args. + + :returns: all process output + """ + output, errput, exit_code = yield getProcessOutputAndValue( + sys.executable, + ("-m", "allmydata.cli.grid_manager") + args, + reactor=reactor, + **kwargs + ) + if exit_code != 0: + raise util.ProcessFailed( + RuntimeError("Exit code {}".format(exit_code)), + output + errput, + ) + returnValue(output) + + +@pytest_twisted.inlineCallbacks +def test_create_certificate(reactor, request): + """ + The Grid Manager produces a valid, correctly-signed certificate. + """ + gm_config = yield _run_gm(reactor, "--config", "-", "create") + privkey_bytes = json.loads(gm_config)['private_key'].encode('ascii') + privkey, pubkey = ed25519.signing_keypair_from_string(privkey_bytes) + + # Note that zara + her key here are arbitrary and don't match any + # "actual" clients in the test-grid; we're just checking that the + # Grid Manager signs this properly. + gm_config = yield _run_gm( + reactor, "--config", "-", "add", + "zara", "pub-v0-kzug3ut2m7ziihf3ndpqlquuxeie4foyl36wn54myqc4wmiwe4ga", + stdinBytes=gm_config, + ) + zara_cert_bytes = yield _run_gm( + reactor, "--config", "-", "sign", "zara", "1", + stdinBytes=gm_config, + ) + zara_cert = json.loads(zara_cert_bytes) + + # confirm that zara's certificate is made by the Grid Manager + # (.verify returns None on success, raises exception on error) + pubkey.verify( + base32.a2b(zara_cert['signature'].encode('ascii')), + zara_cert['certificate'].encode('ascii'), + ) + + +@pytest_twisted.inlineCallbacks +def test_remove_client(reactor, request): + """ + A Grid Manager can add and successfully remove a client + """ + gm_config = yield _run_gm( + reactor, "--config", "-", "create", + ) + + gm_config = yield _run_gm( + reactor, "--config", "-", "add", + "zara", "pub-v0-kzug3ut2m7ziihf3ndpqlquuxeie4foyl36wn54myqc4wmiwe4ga", + stdinBytes=gm_config, + ) + gm_config = yield _run_gm( + reactor, "--config", "-", "add", + "yakov", "pub-v0-kvxhb3nexybmipkrar2ztfrwp4uxxsmrjzkpzafit3ket4u5yldq", + stdinBytes=gm_config, + ) + assert "zara" in json.loads(gm_config)['storage_servers'] + assert "yakov" in json.loads(gm_config)['storage_servers'] + + gm_config = yield _run_gm( + reactor, "--config", "-", "remove", + "zara", + stdinBytes=gm_config, + ) + assert "zara" not in json.loads(gm_config)['storage_servers'] + assert "yakov" in json.loads(gm_config)['storage_servers'] + + +@pytest_twisted.inlineCallbacks +def test_remove_last_client(reactor, request): + """ + A Grid Manager can remove all clients + """ + gm_config = yield _run_gm( + reactor, "--config", "-", "create", + ) + + gm_config = yield _run_gm( + reactor, "--config", "-", "add", + "zara", "pub-v0-kzug3ut2m7ziihf3ndpqlquuxeie4foyl36wn54myqc4wmiwe4ga", + stdinBytes=gm_config, + ) + assert "zara" in json.loads(gm_config)['storage_servers'] + + gm_config = yield _run_gm( + reactor, "--config", "-", "remove", + "zara", + stdinBytes=gm_config, + ) + # there are no storage servers left at all now + assert "storage_servers" not in json.loads(gm_config) + + +@pytest_twisted.inlineCallbacks +def test_add_remove_client_file(reactor, request, temp_dir): + """ + A Grid Manager can add and successfully remove a client (when + keeping data on disk) + """ + gmconfig = join(temp_dir, "gmtest") + gmconfig_file = join(temp_dir, "gmtest", "config.json") + yield _run_gm( + reactor, "--config", gmconfig, "create", + ) + + yield _run_gm( + reactor, "--config", gmconfig, "add", + "zara", "pub-v0-kzug3ut2m7ziihf3ndpqlquuxeie4foyl36wn54myqc4wmiwe4ga", + ) + yield _run_gm( + reactor, "--config", gmconfig, "add", + "yakov", "pub-v0-kvxhb3nexybmipkrar2ztfrwp4uxxsmrjzkpzafit3ket4u5yldq", + ) + assert "zara" in json.load(open(gmconfig_file, "r"))['storage_servers'] + assert "yakov" in json.load(open(gmconfig_file, "r"))['storage_servers'] + + yield _run_gm( + reactor, "--config", gmconfig, "remove", + "zara", + ) + assert "zara" not in json.load(open(gmconfig_file, "r"))['storage_servers'] + assert "yakov" in json.load(open(gmconfig_file, "r"))['storage_servers'] + + +@pytest_twisted.inlineCallbacks +def test_reject_storage_server(reactor, request, temp_dir, flog_gatherer, port_allocator): + """ + A client with happines=2 fails to upload to a Grid when it is + using Grid Manager and there is only 1 storage server with a valid + certificate. + """ + grid = yield create_grid(reactor, request, temp_dir, flog_gatherer, port_allocator) + storage0 = yield grid.add_storage_node() + storage1 = yield grid.add_storage_node() + + gm_config = yield _run_gm( + reactor, "--config", "-", "create", + ) + gm_privkey_bytes = json.loads(gm_config)['private_key'].encode('ascii') + gm_privkey, gm_pubkey = ed25519.signing_keypair_from_string(gm_privkey_bytes) + + # create certificate for the first storage-server + pubkey_fname = join(storage0.process.node_dir, "node.pubkey") + with open(pubkey_fname, 'r') as f: + pubkey_str = f.read().strip() + + gm_config = yield _run_gm( + reactor, "--config", "-", "add", + "storage0", pubkey_str, + stdinBytes=gm_config, + ) + assert json.loads(gm_config)['storage_servers'].keys() == ['storage0'] + + print("inserting certificate") + cert = yield _run_gm( + reactor, "--config", "-", "sign", "storage0", "1", + stdinBytes=gm_config, + ) + print(cert) + + yield util.run_tahoe( + reactor, request, "--node-directory", storage0.process.node_dir, + "admin", "add-grid-manager-cert", + "--name", "default", + "--filename", "-", + stdin=cert, + ) + + # re-start this storage server + yield storage0.restart(reactor, request) + + # now only one storage-server has the certificate .. configure + # diana to have the grid-manager certificate + + diana = yield grid.add_client("diana", needed=2, happy=2, total=2) + + config = configutil.get_config(join(diana.process.node_dir, "tahoe.cfg")) + config.add_section("grid_managers") + config.set("grid_managers", "test", ed25519.string_from_verifying_key(gm_pubkey)) + with open(join(diana.process.node_dir, "tahoe.cfg"), "w") as f: + config.write(f) + + yield diana.restart(reactor, request, servers=2) + + # try to put something into the grid, which should fail (because + # diana has happy=2 but should only find storage0 to be acceptable + # to upload to) + + try: + yield util.run_tahoe( + reactor, request, "--node-directory", diana.process.node_dir, + "put", "-", + stdin="some content\n" * 200, + ) + assert False, "Should get a failure" + except util.ProcessFailed as e: + assert 'UploadUnhappinessError' in e.output + + +@pytest_twisted.inlineCallbacks +def test_identity(reactor, request, temp_dir): + """ + Dump public key to CLI + """ + gm_config = join(temp_dir, "test_identity") + yield _run_gm( + reactor, "--config", gm_config, "create", + ) + + # ask the CLI for the grid-manager pubkey + pubkey = yield _run_gm( + reactor, "--config", gm_config, "public-identity", + ) + alleged_pubkey = ed25519.verifying_key_from_string(pubkey.strip()) + + # load the grid-manager pubkey "ourselves" + with open(join(gm_config, "config.json"), "r") as f: + real_config = json.load(f) + real_privkey, real_pubkey = ed25519.signing_keypair_from_string( + real_config["private_key"].encode("ascii"), + ) + + # confirm the CLI told us the correct thing + alleged_bytes = alleged_pubkey.public_bytes(Encoding.Raw, PublicFormat.Raw) + real_bytes = real_pubkey.public_bytes(Encoding.Raw, PublicFormat.Raw) + assert alleged_bytes == real_bytes, "Keys don't match" From 885f72ff2bebdca815fbc3a695db17dbe888c01c Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 16 Nov 2020 01:23:37 -0700 Subject: [PATCH 0156/2309] decisions --- docs/managed-grid.rst | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/docs/managed-grid.rst b/docs/managed-grid.rst index 289ecd004..1f062db19 100644 --- a/docs/managed-grid.rst +++ b/docs/managed-grid.rst @@ -212,7 +212,7 @@ Enrolling a Client: CLI tahoe add-grid-manager (PROPOSED) ````````````````````````````````` -DECIDE: this command hasn't actually been written yet. +(Note: this command hasn't actually been written yet). This takes two arguments: ``name`` and ``public-identity``. @@ -336,15 +336,6 @@ Put the key printed out above into Alice's ``tahoe.cfg`` in section example_name = pub-v0-vqimc4s5eflwajttsofisp5st566dbq36xnpp4siz57ufdavpvlq -DECIDE: - - should the grid-manager be identified by a certificate? exarkun - points out: --name seems like the hint of the beginning of a - use-case for certificates rather than bare public keys?). - - (note the "--name" thing came from a former version of this - proposal that used CLI commands to add the public-keys -- but the - point remains, if there's to be metadata associated with "grid - managers" maybe they should be certificates..) - Now, re-start the "alice" client. Since we made Alice's parameters require 3 storage servers to be reachable (``--happy=3``), all their uploads should now fail (so ``tahoe put`` will fail) because they From 409e9bd65d2396cbf214d749a4816edf5b04ceca Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 16 Nov 2020 01:27:16 -0700 Subject: [PATCH 0157/2309] file tickets for unimplemented commands --- docs/managed-grid.rst | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/docs/managed-grid.rst b/docs/managed-grid.rst index 1f062db19..1fdafda7b 100644 --- a/docs/managed-grid.rst +++ b/docs/managed-grid.rst @@ -212,7 +212,8 @@ Enrolling a Client: CLI tahoe add-grid-manager (PROPOSED) ````````````````````````````````` -(Note: this command hasn't actually been written yet). +(Note: this command hasn't actually been written yet, see +https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3507). This takes two arguments: ``name`` and ``public-identity``. @@ -341,8 +342,9 @@ require 3 storage servers to be reachable (``--happy=3``), all their uploads should now fail (so ``tahoe put`` will fail) because they won't use storage2 and thus can't "achieve happiness". -You can check Alice's "Welcome" page (where the list of connected servers -is) at http://localhost:6301/ and should be able to see details about -the "work-grid" Grid Manager that you added. When any Grid Managers -are enabled, each storage-server line will show whether it has a valid +(PROPOSED https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3506) You can +check Alice's "Welcome" page (where the list of connected servers is) +at http://localhost:6301/ and should be able to see details about the +"work-grid" Grid Manager that you added. When any Grid Managers are +enabled, each storage-server line will show whether it has a valid certificate or not (and how much longer it's valid until). From 8400893976966cb698608811473a438511662428 Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 16 Nov 2020 01:30:38 -0700 Subject: [PATCH 0158/2309] news --- newsfragments/3508.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3508.minor diff --git a/newsfragments/3508.minor b/newsfragments/3508.minor new file mode 100644 index 000000000..e69de29bb From 019772a2c28fc8389100691906572ca0b30fa19a Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 16 Nov 2020 11:24:46 -0700 Subject: [PATCH 0159/2309] typo --- src/allmydata/scripts/admin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/scripts/admin.py b/src/allmydata/scripts/admin.py index 803f6561c..28ab0f2ed 100644 --- a/src/allmydata/scripts/admin.py +++ b/src/allmydata/scripts/admin.py @@ -139,7 +139,7 @@ def add_grid_manager_cert(options): # write all the data out with cert_path.open("wb") as f: - f.wrwite(cert_bytes) + f.write(cert_bytes) cert_count = len(config.enumerate_section("grid_manager_certificates")) print("There are now {} certificates".format(cert_count), file=options.parent.parent.stderr) From a34093ed0e9c29d7e838dc65aa6c104bb8cc7195 Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 16 Nov 2020 11:58:27 -0700 Subject: [PATCH 0160/2309] fix some test-ability problems and add tests for 'tahoe admin add-grid-manager-cert' --- src/allmydata/scripts/runner.py | 6 +- src/allmydata/test/cli/test_grid_manager.py | 90 ++++++++++++++++++++- src/allmydata/test/common_util.py | 13 +-- 3 files changed, 102 insertions(+), 7 deletions(-) diff --git a/src/allmydata/scripts/runner.py b/src/allmydata/scripts/runner.py index 24d027cc3..2184569b6 100644 --- a/src/allmydata/scripts/runner.py +++ b/src/allmydata/scripts/runner.py @@ -115,8 +115,11 @@ def parse_options(argv, config=None): config.parseOptions(argv) # may raise usage.error return config -def parse_or_exit_with_explanation(argv, stdout=sys.stdout): +def parse_or_exit_with_explanation(argv, stdout=sys.stdout, stderr=sys.stderr, stdin=sys.stdin): config = Options() + config.stdout = stdout + config.stdin = stdin + config.stderr = stderr try: parse_options(argv, config=config) except usage.error as e: @@ -141,6 +144,7 @@ def dispatch(config, so.stdout = stdout so.stderr = stderr so.stdin = stdin + config.stdin = stdin if command in create_dispatch: f = create_dispatch[command] diff --git a/src/allmydata/test/cli/test_grid_manager.py b/src/allmydata/test/cli/test_grid_manager.py index 8383bb9cd..6d71a14eb 100644 --- a/src/allmydata/test/cli/test_grid_manager.py +++ b/src/allmydata/test/cli/test_grid_manager.py @@ -2,13 +2,27 @@ import os import json -from ..common import SyncTestCase +from ..common import ( + SyncTestCase, + AsyncTestCase, +) from allmydata.cli.grid_manager import ( grid_manager, ) import click.testing +# these imports support the tests for `tahoe *` subcommands +from ..common_util import ( + run_cli, +) +from twisted.internet.defer import ( + inlineCallbacks, +) +from twisted.python.filepath import ( + FilePath, +) + class GridManagerCommandLine(SyncTestCase): """ @@ -77,3 +91,77 @@ class GridManagerCommandLine(SyncTestCase): result = self.runner.invoke(grid_manager, ["--config", "foo", "list"]) self.assertEqual(result.output.strip(), "") + + +# note: CLITestMixin can't function without also GridTestMixin ... :/ +class TahoeAddGridManagerCert(AsyncTestCase): + """ + Test `tahoe admin add-grid-manager-cert` subcommand + """ + + @inlineCallbacks + def test_help(self): + """ + some kind of help is printed + """ + code, out, err = yield run_cli("admin", "add-grid-manager-cert") + self.assertEqual(err, "") + self.assertNotEqual(0, code) + + @inlineCallbacks + def test_no_name(self): + """ + error to miss --name option + """ + code, out, err = yield run_cli( + "admin", "add-grid-manager-cert", "--filename", "-", + stdin="the cert", + ) + self.assertIn( + "Must provide --name", + out + ) + + @inlineCallbacks + def test_no_filename(self): + """ + error to miss --name option + """ + code, out, err = yield run_cli( + "admin", "add-grid-manager-cert", "--name", "foo", + stdin="the cert", + ) + self.assertIn( + "Must provide --filename", + out + ) + + @inlineCallbacks + def test_add_one(self): + """ + we can add a certificate + """ + nodedir = self.mktemp() + fake_cert = """{"certificate": "", "signature": ""}""" + + code, out, err = yield run_cli( + "--node-directory", nodedir, + "admin", "add-grid-manager-cert", "-f", "-", "--name", "foo", + stdin=fake_cert, + ignore_stderr=True, + ) + nodepath = FilePath(nodedir) + with nodepath.child("tahoe.cfg").open("r") as f: + config_data = f.read() + + self.assertIn("tahoe.cfg", nodepath.listdir()) + self.assertIn( + "foo = foo.cert", + config_data, + ) + self.assertIn("foo.cert", nodepath.listdir()) + with nodepath.child("foo.cert").open("r") as f: + self.assertEqual( + json.load(f), + json.loads(fake_cert) + ) diff --git a/src/allmydata/test/common_util.py b/src/allmydata/test/common_util.py index e3f5cf750..cbc00a135 100644 --- a/src/allmydata/test/common_util.py +++ b/src/allmydata/test/common_util.py @@ -40,14 +40,17 @@ def run_cli(verb, *args, **kwargs): "arguments to do_cli must be strs -- convert using unicode_to_argv", args=args) nodeargs = kwargs.get("nodeargs", []) argv = nodeargs + [verb] + list(args) - stdin = kwargs.get("stdin", "") + stdin = StringIO(kwargs.get("stdin", "")) stdout = StringIO() stderr = StringIO() d = defer.succeed(argv) - d.addCallback(runner.parse_or_exit_with_explanation, stdout=stdout) - d.addCallback(runner.dispatch, - stdin=StringIO(stdin), - stdout=stdout, stderr=stderr) + d.addCallback(runner.parse_or_exit_with_explanation, stdout=stdout, stderr=stderr, stdin=stdin) + d.addCallback( + runner.dispatch, + stdin=stdin, + stdout=stdout, + stderr=stderr, + ) def _done(rc): return 0, stdout.getvalue(), stderr.getvalue() def _err(f): From 6e77eba68837d3cc38f1576004e4218fa6922351 Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 16 Nov 2020 15:15:49 -0700 Subject: [PATCH 0161/2309] test another path --- src/allmydata/test/cli/test_grid_manager.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/allmydata/test/cli/test_grid_manager.py b/src/allmydata/test/cli/test_grid_manager.py index 6d71a14eb..85dc757fd 100644 --- a/src/allmydata/test/cli/test_grid_manager.py +++ b/src/allmydata/test/cli/test_grid_manager.py @@ -79,11 +79,13 @@ class GridManagerCommandLine(SyncTestCase): with self.runner.isolated_filesystem(): self.runner.invoke(grid_manager, ["--config", "foo", "create"]) self.runner.invoke(grid_manager, ["--config", "foo", "add", "storage0", pubkey]) + self.runner.invoke(grid_manager, ["--config", "foo", "sign", "storage0", "1"]) result = self.runner.invoke(grid_manager, ["--config", "foo", "list"]) names = [ line.split(':')[0] for line in result.output.strip().split('\n') + if not line.startswith(" ") # "cert" lines start with whitespace ] self.assertEqual(names, ["storage0"]) @@ -93,7 +95,6 @@ class GridManagerCommandLine(SyncTestCase): self.assertEqual(result.output.strip(), "") -# note: CLITestMixin can't function without also GridTestMixin ... :/ class TahoeAddGridManagerCert(AsyncTestCase): """ Test `tahoe admin add-grid-manager-cert` subcommand From 3b3b95838e6d7f0e0e3efdacfc1dc50e1d0597de Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 16 Nov 2020 16:55:06 -0700 Subject: [PATCH 0162/2309] tempdir should be native-string --- src/allmydata/node.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/allmydata/node.py b/src/allmydata/node.py index 498b70a5d..a72303e33 100644 --- a/src/allmydata/node.py +++ b/src/allmydata/node.py @@ -28,6 +28,7 @@ import configparser from twisted.python import log as twlog from twisted.application import service from twisted.python.failure import Failure +from twisted.python.compat import nativeString from foolscap.api import Tub, app_versions import foolscap.logging.log from allmydata.version_checks import get_package_versions, get_package_versions_string @@ -828,7 +829,7 @@ class Node(service.MultiService): tempdir = self.config.get_config_path(tempdir_config) if not os.path.exists(tempdir): fileutil.make_dirs(tempdir) - tempfile.tempdir = tempdir + tempfile.tempdir = nativeString(tempdir) # this should cause twisted.web.http (which uses # tempfile.TemporaryFile) to put large request bodies in the given # directory. Without this, the default temp dir is usually /tmp/, From d34c32cc503fc50438c1ead36be80d8a9a8342f1 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 17 Nov 2020 10:49:07 -0700 Subject: [PATCH 0163/2309] cleanup / review --- src/allmydata/grid_manager.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/grid_manager.py b/src/allmydata/grid_manager.py index 464e90b94..61b2c90c1 100644 --- a/src/allmydata/grid_manager.py +++ b/src/allmydata/grid_manager.py @@ -94,7 +94,7 @@ def load_grid_manager(config_path, config_location): storage_servers = dict() for name, srv_config in config.get(u'storage_servers', {}).items(): - if not 'public_key' in srv_config: + if 'public_key' not in srv_config: raise ValueError( "No 'public_key' for storage server '{}'".format(name) ) @@ -130,7 +130,7 @@ class _GridManager(object): srv = self._storage_servers[name] except KeyError: raise KeyError( - u"No storage server named '{}'".format(name) + "No storage server named '{}'".format(name) ) expiration = datetime.utcnow() + timedelta(seconds=expiry_seconds) epoch_offset = (expiration - datetime(1970, 1, 1)).total_seconds() From 800497a71977130272e1c26ea5d085a89510c7da Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 17 Nov 2020 10:57:30 -0700 Subject: [PATCH 0164/2309] refactor load_grid_manager() to better present errors (review) --- src/allmydata/cli/grid_manager.py | 7 ++++++- src/allmydata/grid_manager.py | 10 ++++------ src/allmydata/test/test_grid_manager.py | 12 ++++++------ 3 files changed, 16 insertions(+), 13 deletions(-) diff --git a/src/allmydata/cli/grid_manager.py b/src/allmydata/cli/grid_manager.py index ce0197345..c839684ca 100644 --- a/src/allmydata/cli/grid_manager.py +++ b/src/allmydata/cli/grid_manager.py @@ -54,7 +54,12 @@ def grid_manager(ctx, config): def grid_manager(self): if self._grid_manager is None: config_path = _config_path_from_option(config) - self._grid_manager = load_grid_manager(config_path, config) + try: + self._grid_manager = load_grid_manager(config_path) + except ValueError as e: + raise click.ClickException( + "Error loading Grid Manager from '{}': {}".format(config, e) + ) return self._grid_manager ctx.obj = Config() diff --git a/src/allmydata/grid_manager.py b/src/allmydata/grid_manager.py index 61b2c90c1..68465a6c1 100644 --- a/src/allmydata/grid_manager.py +++ b/src/allmydata/grid_manager.py @@ -47,16 +47,16 @@ def create_grid_manager(): ) -def load_grid_manager(config_path, config_location): +def load_grid_manager(config_path): """ Load a Grid Manager from existing configuration. :param FilePath config_path: the configuration location (or None for stdin) - :param str config_location: a string describing the config's location - :returns: a GridManager instance + + :raises: ValueError if the confguration is invalid """ if config_path is None: config_file = sys.stdin @@ -79,9 +79,7 @@ def load_grid_manager(config_path, config_location): ) if 'private_key' not in config: raise ValueError( - "Grid Manager config from '{}' requires a 'private_key'".format( - config_location, - ) + "'private_key' required in config" ) private_key_bytes = config['private_key'].encode('ascii') diff --git a/src/allmydata/test/test_grid_manager.py b/src/allmydata/test/test_grid_manager.py index 45d2061b2..36d15bc46 100644 --- a/src/allmydata/test/test_grid_manager.py +++ b/src/allmydata/test/test_grid_manager.py @@ -139,7 +139,7 @@ class GridManagerVerifier(SyncTestCase): fp = FilePath(tempdir) save_grid_manager(fp, self.gm) - gm2 = load_grid_manager(fp, tempdir) + gm2 = load_grid_manager(fp) self.assertEqual( self.gm.public_identity(), gm2.public_identity(), @@ -168,7 +168,7 @@ class GridManagerVerifier(SyncTestCase): json.dump(bad_config, f) with self.assertRaises(ValueError) as ctx: - load_grid_manager(fp, tempdir) + load_grid_manager(fp) self.assertIn( "unknown version", str(ctx.exception), @@ -188,9 +188,9 @@ class GridManagerVerifier(SyncTestCase): json.dump(bad_config, f) with self.assertRaises(ValueError) as ctx: - load_grid_manager(fp, tempdir) + load_grid_manager(fp) self.assertIn( - "requires a 'private_key'", + "'private_key' required", str(ctx.exception), ) @@ -209,7 +209,7 @@ class GridManagerVerifier(SyncTestCase): json.dump(bad_config, f) with self.assertRaises(ValueError) as ctx: - load_grid_manager(fp, tempdir) + load_grid_manager(fp) self.assertIn( "Invalid Grid Manager private_key", str(ctx.exception), @@ -234,7 +234,7 @@ class GridManagerVerifier(SyncTestCase): json.dump(bad_config, f) with self.assertRaises(ValueError) as ctx: - load_grid_manager(fp, tempdir) + load_grid_manager(fp) self.assertIn( "No 'public_key' for storage server", str(ctx.exception), From 32625bf655c7b8fc7214ebcfe338c913e1247f8c Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 17 Nov 2020 10:59:03 -0700 Subject: [PATCH 0165/2309] tahoe grid-manager -> grid-manager --- docs/managed-grid.rst | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/docs/managed-grid.rst b/docs/managed-grid.rst index 1fdafda7b..b045e9565 100644 --- a/docs/managed-grid.rst +++ b/docs/managed-grid.rst @@ -73,8 +73,8 @@ Manager evolves. It contains a version number in the document schema changes. -tahoe grid-manager create -````````````````````````` +grid-manager create +``````````````````` Create a new grid-manager. @@ -84,8 +84,8 @@ directory specified by the ``--config`` option. It is an error if the directory already exists. -tahoe grid-manager public-identity -`````````````````````````````````` +grid-manager public-identity +```````````````````````````` Print out a grid-manager's public key. This key is derived from the private-key of the grid-manager, so a valid grid-manager config must @@ -95,8 +95,8 @@ This public key is what is put in clients' configuration to actually validate and use grid-manager certificates. -tahoe grid-manager add -`````````````````````` +grid-manager add +```````````````` Takes two args: ``name pubkey``. The ``name`` is an arbitrary local identifier for the new storage node (also sometimes called "a petname" @@ -105,7 +105,7 @@ file in the storage-server's node directory (minus any whitespace). For example, if ``~/storage0`` contains a storage-node, you might do something like this: - tahoe grid-manager --config ./gm0 add storage0 $(cat ~/storage0/node.pubkey) + grid-manager --config ./gm0 add storage0 $(cat ~/storage0/node.pubkey) This adds a new storage-server to a Grid Manager's configuration. (Since it mutates the configuration, if you used @@ -114,15 +114,15 @@ usefulness of the ``name`` is solely for reference within this Grid Manager. -tahoe grid-manager list -``````````````````````` +grid-manager list +````````````````` Lists all storage-servers that have previously been added using -``tahoe grid-manager add``. +``grid-manager add``. -tahoe grid-manager sign -``````````````````````` +grid-manager sign +````````````````` Takes one arg: ``name``, the nickname used previously in a ``tahoe grid-manager add`` command. @@ -190,7 +190,7 @@ turn on grid-management with ``grid_management = true``. You then must also provide a ``[grid_management_certificates]`` section in the config-file which lists ``name = path/to/certificate`` pairs. -These certificate files are issued by the ``tahoe grid-manager sign`` +These certificate files are issued by the ``grid-manager sign`` command; these should be **securely transmitted** to the storage server. Relative paths are based from the node directory. Example:: @@ -223,7 +223,7 @@ client with zero Grid Managers will accept any announcement from an Introducer. The ``public-identity`` argument is the encoded public key of the Grid -Manager (that is, the output of ``tahoe grid-manager +Manager (that is, the output of ``grid-manager public-identity``). The client will have to be re-started once this change is made. @@ -262,7 +262,7 @@ own shell/terminal window or via something like ``systemd`` We'll store our Grid Manager configuration on disk, in ``./gm0``. To initialize this directory:: - tahoe grid-manager --config ./gm0 create + grid-manager --config ./gm0 create (If you already have a grid, you can :ref:`skip ahead `.) @@ -284,13 +284,13 @@ Next, we attach a couple of storage nodes:: We can now tell the Grid Manager about our new storage servers:: - tahoe grid-manager --config ./gm0 add storage0 $(cat storage0/node.pubkey) - tahoe grid-manager --config ./gm0 add storage1 $(cat storage1/node.pubkey) + grid-manager --config ./gm0 add storage0 $(cat storage0/node.pubkey) + grid-manager --config ./gm0 add storage1 $(cat storage1/node.pubkey) To produce a new certificate for each node, we do this:: - tahoe grid-manager --config ./gm0 sign storage0 > ./storage0/gridmanager.cert - tahoe grid-manager --config ./gm0 sign storage1 > ./storage1/gridmanager.cert + grid-manager --config ./gm0 sign storage0 > ./storage0/gridmanager.cert + grid-manager --config ./gm0 sign storage1 > ./storage1/gridmanager.cert Now, we want our storage servers to actually announce these certificates into the grid. We do this by adding some configuration @@ -328,7 +328,7 @@ grid-manager has given certificates to (``storage0`` and ``storage1``). We need the grid-manager's public key to put in Alice's configuration:: - tahoe grid-manager --config ./gm0 public-identity + grid-manager --config ./gm0 public-identity Put the key printed out above into Alice's ``tahoe.cfg`` in section ``client``:: From eb11809798f2d78ac6afa51a99879cdeebe0a700 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 17 Nov 2020 11:02:33 -0700 Subject: [PATCH 0166/2309] introduction (review) --- docs/managed-grid.rst | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/managed-grid.rst b/docs/managed-grid.rst index b045e9565..187651380 100644 --- a/docs/managed-grid.rst +++ b/docs/managed-grid.rst @@ -3,6 +3,15 @@ Managed Grid ============ +This document explains the "Grid Manager" concept and the +`grid-manager` command. Someone operating a grid may choose to use a +Grid Manager. Operators of storage-servers and clients will then be +given additional configuration in this case. + + +Overview and Motivation +----------------------- + In a grid using an Introducer, a client will use any storage-server the Introducer announces (and the Introducer will announce any storage-server that connects to it). This means that anyone with the From 61f348f7a5e853fe09ad6d520130a581f1df0809 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 17 Nov 2020 11:05:40 -0700 Subject: [PATCH 0167/2309] clarify --- docs/managed-grid.rst | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/docs/managed-grid.rst b/docs/managed-grid.rst index 187651380..eab72ca27 100644 --- a/docs/managed-grid.rst +++ b/docs/managed-grid.rst @@ -133,8 +133,8 @@ Lists all storage-servers that have previously been added using grid-manager sign ````````````````` -Takes one arg: ``name``, the nickname used previously in a ``tahoe -grid-manager add`` command. +Takes one arg: ``name``, the nickname used previously in a +``grid-manager add`` command. Note that this mutates the state of the grid-manager if it is on disk, by adding this certificate to our collection of issued @@ -200,8 +200,9 @@ also provide a ``[grid_management_certificates]`` section in the config-file which lists ``name = path/to/certificate`` pairs. These certificate files are issued by the ``grid-manager sign`` -command; these should be **securely transmitted** to the storage -server. Relative paths are based from the node directory. Example:: +command; these should be transmitted to the storage server operator +who includes them in the config for the storage sever. Relative paths +are based from the node directory. Example:: [storage] grid_management = true From 649cb9380e0e0fa771634c5dfbd6a0275c08556a Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 17 Nov 2020 11:11:55 -0700 Subject: [PATCH 0168/2309] move proposals out --- docs/managed-grid.rst | 34 +++++----------------------------- 1 file changed, 5 insertions(+), 29 deletions(-) diff --git a/docs/managed-grid.rst b/docs/managed-grid.rst index eab72ca27..11c5035e8 100644 --- a/docs/managed-grid.rst +++ b/docs/managed-grid.rst @@ -215,29 +215,6 @@ connect to (and subsequently, the Introducer will give the certificate out to clients). -Enrolling a Client: CLI ------------------------ - - -tahoe add-grid-manager (PROPOSED) -````````````````````````````````` - -(Note: this command hasn't actually been written yet, see -https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3507). - -This takes two arguments: ``name`` and ``public-identity``. - -The ``name`` argument is a nickname to call this Grid Manager. A -client may have any number of grid-managers, so each one has a name. A -client with zero Grid Managers will accept any announcement from an -Introducer. - -The ``public-identity`` argument is the encoded public key of the Grid -Manager (that is, the output of ``grid-manager -public-identity``). The client will have to be re-started once this -change is made. - - Enrolling a Client: Config -------------------------- @@ -254,6 +231,8 @@ Manager. Example:: [grid_managers] example_grid = pub-v0-vqimc4s5eflwajttsofisp5st566dbq36xnpp4siz57ufdavpvlq +See also https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3507 which +proposes a command to edit the config. Example Setup of a New Managed Grid @@ -352,9 +331,6 @@ require 3 storage servers to be reachable (``--happy=3``), all their uploads should now fail (so ``tahoe put`` will fail) because they won't use storage2 and thus can't "achieve happiness". -(PROPOSED https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3506) You can -check Alice's "Welcome" page (where the list of connected servers is) -at http://localhost:6301/ and should be able to see details about the -"work-grid" Grid Manager that you added. When any Grid Managers are -enabled, each storage-server line will show whether it has a valid -certificate or not (and how much longer it's valid until). +A proposal to expose more information about Grid Manager and +certifcate status in the Welcome page is discussed in +https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3506 From 938cc5616e82f1dba8213777efc0c9d744dc3eac Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 17 Nov 2020 17:46:26 -0700 Subject: [PATCH 0169/2309] refactor (review): move grid-manager certificate loading code out of cli --- src/allmydata/cli/grid_manager.py | 34 ++++----- src/allmydata/grid_manager.py | 98 ++++++++++++++++++++----- src/allmydata/test/test_grid_manager.py | 2 +- 3 files changed, 97 insertions(+), 37 deletions(-) diff --git a/src/allmydata/cli/grid_manager.py b/src/allmydata/cli/grid_manager.py index c839684ca..d937c0f30 100644 --- a/src/allmydata/cli/grid_manager.py +++ b/src/allmydata/cli/grid_manager.py @@ -152,23 +152,16 @@ def list(ctx): """ List all storage-servers known to a Grid Manager """ - fp = _config_path_from_option(ctx.parent.params["config"]) for name in sorted(ctx.obj.grid_manager.storage_servers.keys()): blank_name = " " * len(name) - click.echo("{}: {}".format(name, ctx.obj.grid_manager.storage_servers[name].public_key())) - if fp: - cert_count = 0 - while fp.child('{}.cert.{}'.format(name, cert_count)).exists(): - container = json.load(fp.child('{}.cert.{}'.format(name, cert_count)).open('r')) - cert_data = json.loads(container['certificate']) - expires = datetime.utcfromtimestamp(cert_data['expires']) - delta = datetime.utcnow() - expires - click.echo("{} cert {}: ".format(blank_name, cert_count), nl=False) - if delta.total_seconds() < 0: - click.echo("valid until {} ({})".format(expires, abbreviate_time(delta))) - else: - click.echo("expired {} ({})".format(expires, abbreviate_time(delta))) - cert_count += 1 + click.echo("{}: {}".format(name, ctx.obj.grid_manager.storage_servers[name].public_key_string())) + for cert in ctx.obj.grid_manager.storage_servers[name].certificates: + delta = datetime.utcnow() - cert.expires + click.echo("{} cert {}: ".format(blank_name, cert.index), nl=False) + if delta.total_seconds() < 0: + click.echo("valid until {} ({})".format(cert.expires, abbreviate_time(delta))) + else: + click.echo("expired {} ({})".format(cert.expires, abbreviate_time(delta))) @grid_manager.command() @@ -196,15 +189,20 @@ def sign(ctx, name, expiry_days): click.echo(certificate_data) if fp is not None: next_serial = 0 - while fp.child("{}.cert.{}".format(name, next_serial)).exists(): + f = None + while f is None: + try: + f = fp.child("{}.cert.{}".format(name, next_serial)).create() + except Exception: + f = None next_serial += 1 - with fp.child('{}.cert.{}'.format(name, next_serial)).open('w') as f: + with f: f.write(certificate_data) def _config_path_from_option(config): """ - :param string config: a path or - + :param str config: a path or - :returns: a FilePath instance or None """ if config == "-": diff --git a/src/allmydata/grid_manager.py b/src/allmydata/grid_manager.py index 68465a6c1..d753295ab 100644 --- a/src/allmydata/grid_manager.py +++ b/src/allmydata/grid_manager.py @@ -1,3 +1,4 @@ + import sys import json from datetime import ( @@ -13,29 +14,46 @@ from allmydata.util import ( base32, ) +import attr + +@attr.s class _GridManagerStorageServer(object): """ A Grid Manager's notion of a storage server """ - def __init__(self, name, public_key, certificates): - self.name = name - self._public_key = public_key - self._certificates = [] if certificates is None else certificates + name = attr.ib() + public_key = attr.ib(validator=attr.validators.instance_of(ed25519.Ed25519PublicKey)) + certificates = attr.ib( + default=attr.Factory(list), + validator=attr.validators.instance_of(list), + ) def add_certificate(self, certificate): - self._certificates.append(certificate) + self.certificates.append(certificate) - def public_key(self): - return ed25519.string_from_verifying_key(self._public_key) + def public_key_string(self): + return ed25519.string_from_verifying_key(self.public_key) def marshal(self): return { - u"public_key": self.public_key(), + u"public_key": self.public_key_string(), } +@attr.s +class _GridManagerCertificate(object): + """ + Represents a single certificate for a single storage-server + """ + + filename = attr.ib() + index = attr.ib(validator=attr.validators.instance_of(int)) + expires = attr.ib(validator=attr.validators.instance_of(datetime)) + public_key = attr.ib(validator=attr.validators.instance_of(ed25519.Ed25519PublicKey)) + + def create_grid_manager(): """ Create a new Grid Manager with a fresh keypair @@ -47,6 +65,52 @@ def create_grid_manager(): ) +def _load_certificates_for(config_path, name, gm_key=None): + """ + Load any existing certificates for the given storage-server. + + :param FilePath config_path: the configuration location (or None for + stdin) + + :param str name: the name of an existing storage-server + + :param ed25519.VerifyingKey gm_key: an optional Grid Manager + public key. If provided, certificates will be verified against it. + + :returns: list containing any known certificates (may be empty) + + :raises: ed25519.BadSignature if any certificate signature fails to verify + """ + if config_path is None: + return [] + cert_index = 0 + cert_path = config_path.child('{}.cert.{}'.format(name, cert_index)) + certificates = [] + while cert_path.exists(): + container = json.load(cert_path.open('r')) + if gm_key is not None: + validate_grid_manager_certificate(gm_key, container) + cert_data = json.loads(container['certificate']) + if cert_data['version'] != 1: + raise ValueError( + "Unknown certificate version '{}' in '{}'".format( + cert_data['version'], + cert_path.path, + ) + ) + certificates.append( + _GridManagerCertificate( + filename=cert_path.path, + index=cert_index, + expires=datetime.utcfromtimestamp(cert_data['expires']), + public_key=ed25519.verifying_key_from_string(cert_data['public_key'].encode('ascii')), + ) + ) + cert_index += 1 + cert_path = config_path.child('{}.cert.{}'.format(name, cert_index)) + return certificates + + def load_grid_manager(config_path): """ Load a Grid Manager from existing configuration. @@ -56,17 +120,15 @@ def load_grid_manager(config_path): :returns: a GridManager instance - :raises: ValueError if the confguration is invalid + :raises: ValueError if the confguration is invalid or IOError if + expected files can't be opened. """ if config_path is None: config_file = sys.stdin else: - try: - config_file = config_path.child("config.json").open("r") - except IOError: - raise ValueError( - "'{}' is not a Grid Manager config-directory".format(config_path) - ) + # this might raise IOError or similar but caller must handle it + config_file = config_path.child("config.json").open("r") + with config_file: config = json.load(config_file) @@ -99,7 +161,7 @@ def load_grid_manager(config_path): storage_servers[name] = _GridManagerStorageServer( name, ed25519.verifying_key_from_string(srv_config['public_key'].encode('ascii')), - None, + _load_certificates_for(config_path, name, public_key), ) return _GridManager(private_key_bytes, storage_servers) @@ -134,7 +196,7 @@ class _GridManager(object): epoch_offset = (expiration - datetime(1970, 1, 1)).total_seconds() cert_info = { "expires": epoch_offset, - "public_key": srv.public_key(), + "public_key": srv.public_key_string(), "version": 1, } cert_data = json.dumps(cert_info, separators=(',',':'), sort_keys=True).encode('utf8') @@ -161,7 +223,7 @@ class _GridManager(object): raise KeyError( "Already have a storage server called '{}'".format(name) ) - ss = _GridManagerStorageServer(name, public_key, None) + ss = _GridManagerStorageServer(name, public_key, []) self._storage_servers[name] = ss return ss diff --git a/src/allmydata/test/test_grid_manager.py b/src/allmydata/test/test_grid_manager.py index 36d15bc46..ec780918d 100644 --- a/src/allmydata/test/test_grid_manager.py +++ b/src/allmydata/test/test_grid_manager.py @@ -151,7 +151,7 @@ class GridManagerVerifier(SyncTestCase): for name, ss0 in self.gm.storage_servers.items(): ss1 = gm2.storage_servers[name] self.assertEqual(ss0.name, ss1.name) - self.assertEqual(ss0.public_key(), ss1.public_key()) + self.assertEqual(ss0.public_key_string(), ss1.public_key_string()) self.assertEqual(self.gm.marshal(), gm2.marshal()) def test_invalid_no_version(self): From 72f2e25f86022659632f873dd05b506a1b918bd9 Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 23 Nov 2020 17:20:10 -0700 Subject: [PATCH 0170/2309] ask forgiveness not permission --- src/allmydata/cli/grid_manager.py | 11 ++++++----- src/allmydata/grid_manager.py | 3 ++- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/allmydata/cli/grid_manager.py b/src/allmydata/cli/grid_manager.py index d937c0f30..d07e2d412 100644 --- a/src/allmydata/cli/grid_manager.py +++ b/src/allmydata/cli/grid_manager.py @@ -75,13 +75,14 @@ def create(ctx): fp = None if config_location != '-': fp = FilePath(config_location) - if fp.exists(): - raise click.ClickException( - "The directory '{}' already exists.".format(config_location) - ) gm = create_grid_manager() - save_grid_manager(fp, gm) + try: + save_grid_manager(fp, gm) + except OSError as e: + raise click.ClickException( + "Can't create '{}': {}".format(config_location, e) + ) @grid_manager.command() diff --git a/src/allmydata/grid_manager.py b/src/allmydata/grid_manager.py index d753295ab..df0a46e6d 100644 --- a/src/allmydata/grid_manager.py +++ b/src/allmydata/grid_manager.py @@ -269,7 +269,8 @@ def save_grid_manager(file_path, grid_manager): if file_path is None: print("{}\n".format(data)) else: - fileutil.make_dirs(file_path.path, mode=0o700) + file_path.makedirs() + file_path.chmod(0o700) with file_path.child("config.json").open("w") as f: f.write("{}\n".format(data)) From 1e1aad8cc8586fc2e2681eedcc247663f9811d18 Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 23 Nov 2020 17:36:39 -0700 Subject: [PATCH 0171/2309] save only fails sometimes --- src/allmydata/cli/grid_manager.py | 6 +++--- src/allmydata/grid_manager.py | 13 ++++++++++--- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/src/allmydata/cli/grid_manager.py b/src/allmydata/cli/grid_manager.py index d07e2d412..a5e0ee09a 100644 --- a/src/allmydata/cli/grid_manager.py +++ b/src/allmydata/cli/grid_manager.py @@ -144,7 +144,7 @@ def remove(ctx, name): fp.child('{}.cert.{}'.format(name, cert_count)).remove() cert_count += 1 - save_grid_manager(fp, ctx.obj.grid_manager) + save_grid_manager(fp, ctx.obj.grid_manager, create=False) @grid_manager.command() @@ -177,10 +177,10 @@ def sign(ctx, name, expiry_days): sign a new certificate """ fp = _config_path_from_option(ctx.parent.params["config"]) - expiry_seconds = int(expiry_days) * 86400 + expiry = timedelta(days=expiry_days) try: - certificate = ctx.obj.grid_manager.sign(name, expiry_seconds) + certificate = ctx.obj.grid_manager.sign(name, expiry) except KeyError: raise click.ClickException( "No storage-server called '{}' exists".format(name) diff --git a/src/allmydata/grid_manager.py b/src/allmydata/grid_manager.py index df0a46e6d..71395fd11 100644 --- a/src/allmydata/grid_manager.py +++ b/src/allmydata/grid_manager.py @@ -252,7 +252,7 @@ class _GridManager(object): return data -def save_grid_manager(file_path, grid_manager): +def save_grid_manager(file_path, grid_manager, create=True): """ Writes a Grid Manager configuration. @@ -260,6 +260,9 @@ def save_grid_manager(file_path, grid_manager): (if None, stdout is used) :param grid_manager: a _GridManager instance + + :param bool create: if True (the default) we are creating a new + grid-manager and will fail if the directory already exists. """ data = json.dumps( grid_manager.marshal(), @@ -269,8 +272,12 @@ def save_grid_manager(file_path, grid_manager): if file_path is None: print("{}\n".format(data)) else: - file_path.makedirs() - file_path.chmod(0o700) + try: + file_path.makedirs() + file_path.chmod(0o700) + except OSError: + if create: + raise with file_path.child("config.json").open("w") as f: f.write("{}\n".format(data)) From a98d784ce44140e4b93ef9c627364bbbb5553578 Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 23 Nov 2020 17:36:50 -0700 Subject: [PATCH 0172/2309] timedelta, not seconds --- src/allmydata/cli/grid_manager.py | 2 ++ src/allmydata/grid_manager.py | 15 +++++++++++++-- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/src/allmydata/cli/grid_manager.py b/src/allmydata/cli/grid_manager.py index a5e0ee09a..8e561d36a 100644 --- a/src/allmydata/cli/grid_manager.py +++ b/src/allmydata/cli/grid_manager.py @@ -1,5 +1,6 @@ from datetime import ( datetime, + timedelta, ) import json @@ -120,6 +121,7 @@ def add(ctx, name, public_key): save_grid_manager( _config_path_from_option(ctx.parent.params["config"]), ctx.obj.grid_manager, + create=False, ) return 0 diff --git a/src/allmydata/grid_manager.py b/src/allmydata/grid_manager.py index 71395fd11..d3256813d 100644 --- a/src/allmydata/grid_manager.py +++ b/src/allmydata/grid_manager.py @@ -185,14 +185,25 @@ class _GridManager(object): def public_identity(self): return ed25519.string_from_verifying_key(self._public_key) - def sign(self, name, expiry_seconds): + def sign(self, name, expiry): + """ + Create a new signed certificate for a particular server + + :param str name: the server to create a certificate for + + :param timedelta expiry: how far in the future the certificate + should expire. + + :returns: a dict defining the certificate (it has + "certificate" and "signature" keys). + """ try: srv = self._storage_servers[name] except KeyError: raise KeyError( "No storage server named '{}'".format(name) ) - expiration = datetime.utcnow() + timedelta(seconds=expiry_seconds) + expiration = datetime.utcnow() + expiry epoch_offset = (expiration - datetime(1970, 1, 1)).total_seconds() cert_info = { "expires": epoch_offset, From d1adbe0f64c15dd137c71f808c37e3e574d6db8d Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 23 Nov 2020 17:37:37 -0700 Subject: [PATCH 0173/2309] unused --- src/allmydata/grid_manager.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/allmydata/grid_manager.py b/src/allmydata/grid_manager.py index d3256813d..4c16b572a 100644 --- a/src/allmydata/grid_manager.py +++ b/src/allmydata/grid_manager.py @@ -3,14 +3,12 @@ import sys import json from datetime import ( datetime, - timedelta, ) from allmydata.crypto import ( ed25519, ) from allmydata.util import ( - fileutil, base32, ) From a21381997e8962dd798531d6a49d5b726e07f313 Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 23 Nov 2020 23:23:20 -0700 Subject: [PATCH 0174/2309] sign uses timedelta --- src/allmydata/test/test_grid_manager.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/allmydata/test/test_grid_manager.py b/src/allmydata/test/test_grid_manager.py index ec780918d..bcea741d7 100644 --- a/src/allmydata/test/test_grid_manager.py +++ b/src/allmydata/test/test_grid_manager.py @@ -1,4 +1,6 @@ - +from datetime import ( + timedelta, +) import json from twisted.python.filepath import ( @@ -77,7 +79,7 @@ class GridManagerVerifier(SyncTestCase): """ priv, pub = ed25519.create_signing_keypair() self.gm.add_storage_server("test", pub) - cert = self.gm.sign("test", 86400) + cert = self.gm.sign("test", timedelta(seconds=86400)) self.assertEqual( set(cert.keys()), @@ -98,7 +100,7 @@ class GridManagerVerifier(SyncTestCase): Try to sign a storage-server that doesn't exist """ with self.assertRaises(KeyError): - self.gm.sign("doesn't exist", 86400) + self.gm.sign("doesn't exist", timedelta(seconds=86400)) def test_add_cert(self): """ @@ -280,7 +282,7 @@ class GridManagerVerifier(SyncTestCase): """ priv0, pub0 = ed25519.create_signing_keypair() self.gm.add_storage_server("test0", pub0) - cert0 = self.gm.sign("test0", 86400) + cert0 = self.gm.sign("test0", timedelta(seconds=86400)) verify = create_grid_manager_verifier( [self.gm._public_key], From 1b531359d7df9038151a438db4379eca4d86e2b2 Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 23 Nov 2020 23:33:27 -0700 Subject: [PATCH 0175/2309] test create-twice --- src/allmydata/test/cli/test_grid_manager.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/allmydata/test/cli/test_grid_manager.py b/src/allmydata/test/cli/test_grid_manager.py index 85dc757fd..a8b0315e0 100644 --- a/src/allmydata/test/cli/test_grid_manager.py +++ b/src/allmydata/test/cli/test_grid_manager.py @@ -44,6 +44,20 @@ class GridManagerCommandLine(SyncTestCase): result = self.runner.invoke(grid_manager, ["--config", "foo", "public-identity"]) self.assertTrue(result.output.startswith("pub-v0-")) + def test_create_already(self): + """ + It's an error to create a new grid-manager in an existing + directory. + """ + with self.runner.isolated_filesystem(): + result = self.runner.invoke(grid_manager, ["--config", "foo", "create"]) + result = self.runner.invoke(grid_manager, ["--config", "foo", "create"]) + self.assertEqual(1, result.exit_code) + self.assertIn( + "Can't create", + result.stdout, + ) + def test_create_stdout(self): """ Create a new grid-manager with no files From 42b7d3974f5c70261132ba34fbf492b4aee8c16a Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 24 Nov 2020 01:19:10 -0700 Subject: [PATCH 0176/2309] check public key / server-id --- src/allmydata/grid_manager.py | 14 +++++++++----- src/allmydata/storage_client.py | 1 + src/allmydata/test/test_grid_manager.py | 1 + 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/src/allmydata/grid_manager.py b/src/allmydata/grid_manager.py index 4c16b572a..fde88824e 100644 --- a/src/allmydata/grid_manager.py +++ b/src/allmydata/grid_manager.py @@ -346,7 +346,7 @@ def validate_grid_manager_certificate(gm_key, alleged_cert): return cert -def create_grid_manager_verifier(keys, certs, now_fn=None, bad_cert=None): +def create_grid_manager_verifier(keys, certs, public_key, now_fn=None, bad_cert=None): """ Creates a predicate for confirming some Grid Manager-issued certificates against Grid Manager keys. A predicate is used @@ -358,6 +358,9 @@ def create_grid_manager_verifier(keys, certs, now_fn=None, bad_cert=None): :param list certs: 1 or more Grid Manager certificates each of which is a `dict` containing 'signature' and 'certificate' keys. + :param str public_key: the identifier of the server we expect + certificates for. + :param callable now_fn: a callable which returns the current UTC timestamp (or datetime.utcnow if None). @@ -416,10 +419,11 @@ def create_grid_manager_verifier(keys, certs, now_fn=None, bad_cert=None): # if *any* certificate is still valid then we consider the server valid for cert in valid_certs: expires = datetime.utcfromtimestamp(cert['expires']) - # cert_pubkey = keyutil.parse_pubkey(cert['public_key'].encode('ascii')) - if expires > now: - # not-expired - return True + cert_pubkey = ed25519.verifying_key_from_string(cert['public_key'].encode('ascii')) + if cert['public_key'] == public_key: + if expires > now: + # not-expired + return True return False return validate diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index aa8431e05..293683f66 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -285,6 +285,7 @@ class StorageFarmBroker(service.MultiService): gm_verifier = create_grid_manager_verifier( self.storage_client_config.grid_manager_keys, server["ann"].get("grid-manager-certificates", []), + "pub-{}".format(server_id), # server_id is v0- not pub-v0-key .. for reasons? ) s = NativeStorageServer( diff --git a/src/allmydata/test/test_grid_manager.py b/src/allmydata/test/test_grid_manager.py index bcea741d7..5323a6548 100644 --- a/src/allmydata/test/test_grid_manager.py +++ b/src/allmydata/test/test_grid_manager.py @@ -287,6 +287,7 @@ class GridManagerVerifier(SyncTestCase): verify = create_grid_manager_verifier( [self.gm._public_key], [cert0], + ed25519.string_from_verifying_key(pub0), ) self.assertTrue(verify()) From da0fe23082060ccfc1f151f0a3c705c57c85b116 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 24 Nov 2020 14:08:41 -0700 Subject: [PATCH 0177/2309] _load_grid_manager_certificates -> method --- src/allmydata/client.py | 30 +----------------- src/allmydata/node.py | 41 +++++++++++++++++++++++++ src/allmydata/test/test_grid_manager.py | 5 +-- 3 files changed, 43 insertions(+), 33 deletions(-) diff --git a/src/allmydata/client.py b/src/allmydata/client.py index 4570e660d..53047a81b 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -576,34 +576,6 @@ def create_storage_farm_broker(config, default_connection_handlers, foolscap_con return sb -def _load_grid_manager_certificates(config): - """ - Load all Grid Manager certificates in the config in a list. An - empty list is returned if there are none. - """ - grid_manager_certificates = [] - - cert_fnames = list(config.enumerate_section("grid_manager_certificates").values()) - for fname in cert_fnames: - fname = config.get_config_path(fname.decode('utf8')) - if not os.path.exists(fname): - raise ValueError( - "Grid Manager certificate file '{}' doesn't exist".format( - fname - ) - ) - with open(fname, 'r') as f: - cert = json.load(f) - if set(cert.keys()) != {"certificate", "signature"}: - raise ValueError( - "Unknown key in Grid Manager certificate '{}'".format( - fname - ) - ) - grid_manager_certificates.append(cert) - return grid_manager_certificates - - def _register_reference(key, config, tub, referenceable): """ Register a referenceable in a tub with a stable fURL. @@ -936,7 +908,7 @@ class _Client(node.Node, pollmixin.PollMixin): announcement.update(plugins_announcement) if self.config.get_config("storage", "grid_management", default=False, boolean=True): - grid_manager_certificates = _load_grid_manager_certificates(self.config) + grid_manager_certificates = self.config.get_grid_manager_certificates() announcement[u"grid-manager-certificates"] = grid_manager_certificates # XXX we should probably verify that the certificates are diff --git a/src/allmydata/node.py b/src/allmydata/node.py index a72303e33..b1c5b0eb6 100644 --- a/src/allmydata/node.py +++ b/src/allmydata/node.py @@ -464,6 +464,12 @@ class _Config(object): """ returns an absolute path inside the 'private' directory with any extra args join()-ed + + This exists for historical reasons. New code should ideally + not call this because it makes it harder for e.g. a SQL-based + _Config object to exist. Code that needs to call this method + should probably be a _Config method itself. See + e.g. get_grid_manager_certificates() """ return os.path.join(self._basedir, "private", *args) @@ -471,6 +477,12 @@ class _Config(object): """ returns an absolute path inside the config directory with any extra args join()-ed + + This exists for historical reasons. New code should ideally + not call this because it makes it harder for e.g. a SQL-based + _Config object to exist. Code that needs to call this method + should probably be a _Config method itself. See + e.g. get_grid_manager_certificates() """ # note: we re-expand here (_basedir already went through this # expanduser function) in case the path we're being asked for @@ -479,6 +491,35 @@ class _Config(object): os.path.join(self._basedir, *args) ) + def get_grid_manager_certificates(self): + """ + Load all Grid Manager certificates in the config. + + :returns: A list of all certificates. An empty list is + returned if there are none. + """ + grid_manager_certificates = [] + + cert_fnames = list(self.enumerate_section("grid_manager_certificates").values()) + for fname in cert_fnames: + fname = self.get_config_path(fname.decode('utf8')) + if not os.path.exists(fname): + raise ValueError( + "Grid Manager certificate file '{}' doesn't exist".format( + fname + ) + ) + with open(fname, 'r') as f: + cert = json.load(f) + if set(cert.keys()) != {"certificate", "signature"}: + raise ValueError( + "Unknown key in Grid Manager certificate '{}'".format( + fname + ) + ) + grid_manager_certificates.append(cert) + return grid_manager_certificates + def create_tub_options(config): """ diff --git a/src/allmydata/test/test_grid_manager.py b/src/allmydata/test/test_grid_manager.py index 5323a6548..8747afe08 100644 --- a/src/allmydata/test/test_grid_manager.py +++ b/src/allmydata/test/test_grid_manager.py @@ -7,9 +7,6 @@ from twisted.python.filepath import ( FilePath, ) -from allmydata.client import ( - _load_grid_manager_certificates, -) from allmydata.node import ( config_from_string, ) @@ -60,7 +57,7 @@ class GridManagerUtilities(SyncTestCase): {"fluffy": "pub-v0-vqimc4s5eflwajttsofisp5st566dbq36xnpp4siz57ufdavpvlq"}, config.enumerate_section("grid_managers") ) - certs = _load_grid_manager_certificates(config) + certs = config.get_grid_manager_certificates() self.assertEqual([fake_cert], certs) From 4450a7a4b23d84519b44292054e00b72b5cfd4ee Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 25 Nov 2020 19:33:32 -0700 Subject: [PATCH 0178/2309] better words --- src/allmydata/test/test_grid_manager.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/allmydata/test/test_grid_manager.py b/src/allmydata/test/test_grid_manager.py index 8747afe08..c01266264 100644 --- a/src/allmydata/test/test_grid_manager.py +++ b/src/allmydata/test/test_grid_manager.py @@ -72,7 +72,11 @@ class GridManagerVerifier(SyncTestCase): def test_sign_cert(self): """ - Add a storage-server and sign a certificate for it + For a storage server previously added to a grid manager, + _GridManager.sign returns a dict with "certificate" and + "signature" properties where the value of "signature" gives + the ed25519 signature (using the grid manager's private key of + the value) of "certificate". """ priv, pub = ed25519.create_signing_keypair() self.gm.add_storage_server("test", pub) From 884c26495ffcb050309d6b01e594296b89aa5a8b Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 25 Nov 2020 19:50:11 -0700 Subject: [PATCH 0179/2309] flake8 --- src/allmydata/client.py | 1 - src/allmydata/grid_manager.py | 1 - src/allmydata/node.py | 1 + 3 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/allmydata/client.py b/src/allmydata/client.py index e2c0d354a..c8e03c16d 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -2,7 +2,6 @@ import os import stat import time import weakref -import json from allmydata import node from base64 import urlsafe_b64encode from functools import partial diff --git a/src/allmydata/grid_manager.py b/src/allmydata/grid_manager.py index fde88824e..0a5753cc4 100644 --- a/src/allmydata/grid_manager.py +++ b/src/allmydata/grid_manager.py @@ -419,7 +419,6 @@ def create_grid_manager_verifier(keys, certs, public_key, now_fn=None, bad_cert= # if *any* certificate is still valid then we consider the server valid for cert in valid_certs: expires = datetime.utcfromtimestamp(cert['expires']) - cert_pubkey = ed25519.verifying_key_from_string(cert['public_key'].encode('ascii')) if cert['public_key'] == public_key: if expires > now: # not-expired diff --git a/src/allmydata/node.py b/src/allmydata/node.py index a8ee973e2..d9d0a55e5 100644 --- a/src/allmydata/node.py +++ b/src/allmydata/node.py @@ -14,6 +14,7 @@ if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from six import ensure_str, ensure_text +import json import datetime import os.path import re From 706d3085da42a17a1a5a796cb4614946ce6dfadb Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 14 Dec 2020 13:42:51 -0700 Subject: [PATCH 0180/2309] create directories on save --- src/allmydata/util/configutil.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/allmydata/util/configutil.py b/src/allmydata/util/configutil.py index dc7639991..4855241cb 100644 --- a/src/allmydata/util/configutil.py +++ b/src/allmydata/util/configutil.py @@ -66,13 +66,18 @@ def write_config(tahoe_cfg, config): """ Write a configuration to a file. - :param FilePath tahoe_cfg: The path to which to write the config. + :param FilePath tahoe_cfg: The path to which to write the + config. The directories are created if they do not already exist. :param ConfigParser config: The configuration to write. :return: ``None`` """ tmp = tahoe_cfg.temporarySibling() + try: + tahoe_cfg.parent().makedirs() + except OSError: + pass # FilePath.open can only open files in binary mode which does not work # with ConfigParser.write. with open(tmp.path, "wt") as fp: From 38968b40346f54a2f9edd1a92b7e3159cac73fdd Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 14 Dec 2020 13:53:56 -0700 Subject: [PATCH 0181/2309] correct merge resolution --- src/allmydata/test/common_util.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/allmydata/test/common_util.py b/src/allmydata/test/common_util.py index af484ef45..bf85e0580 100644 --- a/src/allmydata/test/common_util.py +++ b/src/allmydata/test/common_util.py @@ -82,7 +82,6 @@ def run_cli_bytes(verb, *args, **kwargs): nodeargs=nodeargs, ) argv = nodeargs + [verb] + list(args) - stdin = kwargs.get("stdin", "") if encoding is None: # The original behavior, the Python 2 behavior, is to accept either # bytes or unicode and try to automatically encode or decode as @@ -91,6 +90,7 @@ def run_cli_bytes(verb, *args, **kwargs): # away from this behavior. stdout = StringIO() stderr = StringIO() + stdin = StringIO(kwargs.get("stdin", "")) else: # The new behavior, the Python 3 behavior, is to accept unicode and # encode it using a specific encoding. For older versions of Python @@ -99,6 +99,7 @@ def run_cli_bytes(verb, *args, **kwargs): # encodings to exercise different behaviors. stdout = TextIOWrapper(BytesIO(), encoding) stderr = TextIOWrapper(BytesIO(), encoding) + stdin = TextIOWrapper(BytesIO(kwargs.get("stdin", b""), encoding) d = defer.succeed(argv) d.addCallback(runner.parse_or_exit_with_explanation, stdout=stdout, stderr=stderr, stdin=stdin) d.addCallback( From 4884f81235c16185bbb34c7abac185489bcb5d40 Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 14 Dec 2020 15:49:42 -0700 Subject: [PATCH 0182/2309] brackets are important --- src/allmydata/test/common_util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/common_util.py b/src/allmydata/test/common_util.py index bf85e0580..dcf81b267 100644 --- a/src/allmydata/test/common_util.py +++ b/src/allmydata/test/common_util.py @@ -99,7 +99,7 @@ def run_cli_bytes(verb, *args, **kwargs): # encodings to exercise different behaviors. stdout = TextIOWrapper(BytesIO(), encoding) stderr = TextIOWrapper(BytesIO(), encoding) - stdin = TextIOWrapper(BytesIO(kwargs.get("stdin", b""), encoding) + stdin = TextIOWrapper(BytesIO(kwargs.get("stdin", b""), encoding)) d = defer.succeed(argv) d.addCallback(runner.parse_or_exit_with_explanation, stdout=stdout, stderr=stderr, stdin=stdin) d.addCallback( From 4fe65a8efde8be1a807ca92f8efc2e33f40029a9 Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 14 Dec 2020 15:57:03 -0700 Subject: [PATCH 0183/2309] brackets are hard --- src/allmydata/test/common_util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/common_util.py b/src/allmydata/test/common_util.py index dcf81b267..23bf15d9b 100644 --- a/src/allmydata/test/common_util.py +++ b/src/allmydata/test/common_util.py @@ -99,7 +99,7 @@ def run_cli_bytes(verb, *args, **kwargs): # encodings to exercise different behaviors. stdout = TextIOWrapper(BytesIO(), encoding) stderr = TextIOWrapper(BytesIO(), encoding) - stdin = TextIOWrapper(BytesIO(kwargs.get("stdin", b""), encoding)) + stdin = TextIOWrapper(BytesIO(kwargs.get("stdin", b"")), encoding) d = defer.succeed(argv) d.addCallback(runner.parse_or_exit_with_explanation, stdout=stdout, stderr=stderr, stdin=stdin) d.addCallback( From bd46ff2f67c2a3df98cc15346c669beb459c1657 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 15 Dec 2020 02:03:10 -0700 Subject: [PATCH 0184/2309] windows-only checks --- src/allmydata/util/configutil.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/allmydata/util/configutil.py b/src/allmydata/util/configutil.py index 4855241cb..b84d96319 100644 --- a/src/allmydata/util/configutil.py +++ b/src/allmydata/util/configutil.py @@ -85,7 +85,10 @@ def write_config(tahoe_cfg, config): # Windows doesn't have atomic overwrite semantics for moveTo. Thus we end # up slightly less than atomic. if platform.isWindows(): - tahoe_cfg.remove() + try: + tahoe_cfg.remove() + except OSError: + pass tmp.moveTo(tahoe_cfg) def validate_config(fname, cfg, valid_config): From fbddff39ea039b46f4c878a90a7e39becf21fdea Mon Sep 17 00:00:00 2001 From: meejah Date: Sun, 20 Dec 2020 22:02:56 -0700 Subject: [PATCH 0185/2309] link grid-manager into the ToC --- docs/index.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/index.rst b/docs/index.rst index 3d0a41302..e778319a4 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -27,6 +27,7 @@ Contents: CODE_OF_CONDUCT servers + managed-grid helper convergence-secret garbage-collection From e38a1c3ee18fd3dd25431d82448fa05ccca7666c Mon Sep 17 00:00:00 2001 From: meejah Date: Sun, 20 Dec 2020 22:03:24 -0700 Subject: [PATCH 0186/2309] two colons --- docs/managed-grid.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/managed-grid.rst b/docs/managed-grid.rst index 11c5035e8..cc427d2c5 100644 --- a/docs/managed-grid.rst +++ b/docs/managed-grid.rst @@ -112,7 +112,7 @@ identifier for the new storage node (also sometimes called "a petname" or "nickname"). The pubkey is the tahoe-encoded key from a ``node.pubkey`` file in the storage-server's node directory (minus any whitespace). For example, if ``~/storage0`` contains a storage-node, -you might do something like this: +you might do something like this:: grid-manager --config ./gm0 add storage0 $(cat ~/storage0/node.pubkey) From f4164aa1c9aa384094d0dfc60fbc0ac7205c0b3d Mon Sep 17 00:00:00 2001 From: meejah Date: Sun, 20 Dec 2020 22:05:40 -0700 Subject: [PATCH 0187/2309] missed a new arg --- docs/managed-grid.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/managed-grid.rst b/docs/managed-grid.rst index cc427d2c5..1613260eb 100644 --- a/docs/managed-grid.rst +++ b/docs/managed-grid.rst @@ -133,8 +133,9 @@ Lists all storage-servers that have previously been added using grid-manager sign ````````````````` -Takes one arg: ``name``, the nickname used previously in a -``grid-manager add`` command. +Takes two args: ``name expiry_days``. The ``name`` is a nickname used +previously in a ``grid-manager add`` command and ``expiry_days`` is +the number of days in the future when the certificate should expire. Note that this mutates the state of the grid-manager if it is on disk, by adding this certificate to our collection of issued From 07180b295bb984822afec7098c88f13f8b3e3e93 Mon Sep 17 00:00:00 2001 From: meejah Date: Sun, 20 Dec 2020 22:12:11 -0700 Subject: [PATCH 0188/2309] spelling --- src/allmydata/mutable/publish.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/mutable/publish.py b/src/allmydata/mutable/publish.py index ef93be721..1f2c04018 100644 --- a/src/allmydata/mutable/publish.py +++ b/src/allmydata/mutable/publish.py @@ -971,7 +971,7 @@ class Publish(object): if server in self.bad_servers: Message.log( message_type=u"mutable:upload:bad-server", - server_id=server.get_server_id(), + server_id=serverid, ) continue # if we have >= 1 grid-managers, this checks that we have @@ -979,7 +979,7 @@ class Publish(object): if not server.upload_permitted(): Message.log( message_type=u"mutable:upload:no-gm-certs", - server_id=server.get_server_id(), + server_id=serverid, ) continue From d267bb3ee1f87627a35719eaf0d860c147f6e1e4 Mon Sep 17 00:00:00 2001 From: meejah Date: Sun, 20 Dec 2020 22:15:23 -0700 Subject: [PATCH 0189/2309] leftover comment --- src/allmydata/storage_client.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 6c6ea5eb4..04b87141d 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -641,10 +641,6 @@ class _FoolscapStorage(object): short_description = server_id[:8] nickname = ann.get("nickname", "") - ## XXX FIXME post-merge, need to parse out - ## "grid-manager-certificates" and .. do something with them - ## (like store them on this object) - return cls( nickname=nickname, permutation_seed=permutation_seed, From 24ac2c69942efffe3202d9fe83fd5b132a64ffc6 Mon Sep 17 00:00:00 2001 From: meejah Date: Sun, 20 Dec 2020 22:19:42 -0700 Subject: [PATCH 0190/2309] I guess only OSError we care about? --- src/allmydata/cli/grid_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/cli/grid_manager.py b/src/allmydata/cli/grid_manager.py index 8e561d36a..70f1a7bed 100644 --- a/src/allmydata/cli/grid_manager.py +++ b/src/allmydata/cli/grid_manager.py @@ -196,7 +196,7 @@ def sign(ctx, name, expiry_days): while f is None: try: f = fp.child("{}.cert.{}".format(name, next_serial)).create() - except Exception: + except OSError: f = None next_serial += 1 with f: From 361acc206d62b608509aadae14ab35011be298ce Mon Sep 17 00:00:00 2001 From: meejah Date: Sun, 20 Dec 2020 22:21:00 -0700 Subject: [PATCH 0191/2309] irrelevant comment --- src/allmydata/client.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/allmydata/client.py b/src/allmydata/client.py index b64feae46..22f5bd589 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -517,14 +517,7 @@ def create_storage_farm_broker(config, default_connection_handlers, foolscap_con **kwargs ) - # we don't actually use this keypair for anything (yet) as far - # as I can see. - # my_pubkey = keyutil.parse_pubkey( - # self.get_config_from_file("node.pubkey") - # ) - # create the actual storage-broker - sb = storage_client.StorageFarmBroker( permute_peers=True, tub_maker=tub_creator, From bfd45d3a017a3efd7ceec51aa0ea33ea2285092a Mon Sep 17 00:00:00 2001 From: meejah Date: Sun, 20 Dec 2020 22:21:31 -0700 Subject: [PATCH 0192/2309] typo --- docs/managed-grid.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/managed-grid.rst b/docs/managed-grid.rst index 1613260eb..3630b956b 100644 --- a/docs/managed-grid.rst +++ b/docs/managed-grid.rst @@ -202,7 +202,7 @@ config-file which lists ``name = path/to/certificate`` pairs. These certificate files are issued by the ``grid-manager sign`` command; these should be transmitted to the storage server operator -who includes them in the config for the storage sever. Relative paths +who includes them in the config for the storage server. Relative paths are based from the node directory. Example:: [storage] From 5d4253be3e91ade95d5773b4a51c19e076de489a Mon Sep 17 00:00:00 2001 From: meejah Date: Sun, 20 Dec 2020 23:03:06 -0700 Subject: [PATCH 0193/2309] reject a bunch of invalid signatures --- src/allmydata/test/strategies.py | 15 ++++++++++ src/allmydata/test/test_grid_manager.py | 37 +++++++++++++++++++++++++ 2 files changed, 52 insertions(+) diff --git a/src/allmydata/test/strategies.py b/src/allmydata/test/strategies.py index 553b2c226..d8b758dab 100644 --- a/src/allmydata/test/strategies.py +++ b/src/allmydata/test/strategies.py @@ -15,6 +15,11 @@ from ..uri import ( MDMFDirectoryURI, ) +from allmydata.util.base32 import ( + b2a, +) + + def write_capabilities(): """ Build ``IURI`` providers representing all kinds of write capabilities. @@ -109,3 +114,13 @@ def dir2_mdmf_capabilities(): MDMFDirectoryURI, mdmf_capabilities(), ) + + +def base32text(): + """ + Build text()s that are valid base32 + """ + return builds( + b2a, + binary(), + ) diff --git a/src/allmydata/test/test_grid_manager.py b/src/allmydata/test/test_grid_manager.py index c01266264..734b54e4c 100644 --- a/src/allmydata/test/test_grid_manager.py +++ b/src/allmydata/test/test_grid_manager.py @@ -7,6 +7,8 @@ from twisted.python.filepath import ( FilePath, ) +from hypothesis import given + from allmydata.node import ( config_from_string, ) @@ -26,6 +28,9 @@ from allmydata.grid_manager import ( parse_grid_manager_certificate, create_grid_manager_verifier, ) +from allmydata.test.strategies import ( + base32text, +) from .common import SyncTestCase @@ -292,3 +297,35 @@ class GridManagerVerifier(SyncTestCase): ) self.assertTrue(verify()) + + +class GridManagerVerifier(SyncTestCase): + """ + Invalid certificate rejection tests + """ + + def setUp(self): + self.gm = create_grid_manager() + self.priv0, self.pub0 = ed25519.create_signing_keypair() + self.gm.add_storage_server("test0", self.pub0) + self.cert0 = self.gm.sign("test0", timedelta(seconds=86400)) + return super(GridManagerVerifier, self).setUp() + + @given( + base32text(), + ) + def test_validate_cert_invalid(self, invalid_signature): + """ + An incorrect signature is rejected + """ + # make signature invalid + self.cert0["signature"] = invalid_signature + + verify = create_grid_manager_verifier( + [self.gm._public_key], + [self.cert0], + ed25519.string_from_verifying_key(self.pub0), + bad_cert = lambda key, cert: None, + ) + + self.assertFalse(verify()) From 5eff6f9ddd109bca00ec56b5b5cffcb56206855e Mon Sep 17 00:00:00 2001 From: meejah Date: Sun, 20 Dec 2020 23:15:14 -0700 Subject: [PATCH 0194/2309] add a few docstrings for accessors etc --- src/allmydata/grid_manager.py | 19 ++++++++++++++++++- src/allmydata/scripts/admin.py | 3 +++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/src/allmydata/grid_manager.py b/src/allmydata/grid_manager.py index 0a5753cc4..120701255 100644 --- a/src/allmydata/grid_manager.py +++ b/src/allmydata/grid_manager.py @@ -29,12 +29,21 @@ class _GridManagerStorageServer(object): ) def add_certificate(self, certificate): + """ + Add ``certificate`` + """ self.certificates.append(certificate) def public_key_string(self): + """ + :returns: the public key as a string + """ return ed25519.string_from_verifying_key(self.public_key) def marshal(self): + """ + :returns: a dict suitable for JSON representing this object + """ return { u"public_key": self.public_key_string(), } @@ -181,6 +190,9 @@ class _GridManager(object): return self._storage_servers def public_identity(self): + """ + :returns: public key as a string + """ return ed25519.string_from_verifying_key(self._public_key) def sign(self, name, expiry): @@ -248,6 +260,9 @@ class _GridManager(object): ) def marshal(self): + """ + :returns: a dict suitable for JSON representing this object + """ data = { u"grid_manager_config_version": self._version, u"private_key": self._private_key_bytes.decode('ascii'), @@ -415,8 +430,10 @@ def create_grid_manager_verifier(keys, certs, public_key, now_fn=None, bad_cert= bad_cert(key, alleged_cert) def validate(): + """ + :returns: True if if *any* certificate is still valid for a server + """ now = now_fn() - # if *any* certificate is still valid then we consider the server valid for cert in valid_certs: expires = datetime.utcfromtimestamp(cert['expires']) if cert['public_key'] == public_key: diff --git a/src/allmydata/scripts/admin.py b/src/allmydata/scripts/admin.py index 28ab0f2ed..8710641ec 100644 --- a/src/allmydata/scripts/admin.py +++ b/src/allmydata/scripts/admin.py @@ -59,6 +59,9 @@ def derive_pubkey(options): class AddGridManagerCertOptions(BaseOptions): + """ + Options for add-grid-manager-cert + """ optParameters = [ ['filename', 'f', None, "Filename of the certificate ('-', a dash, for stdin)"], From d31d8e19c5fde5a96eb44fa7f3053b79053b0997 Mon Sep 17 00:00:00 2001 From: meejah Date: Sun, 20 Dec 2020 23:17:21 -0700 Subject: [PATCH 0195/2309] module docstring --- src/allmydata/grid_manager.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/allmydata/grid_manager.py b/src/allmydata/grid_manager.py index 120701255..39f258fd1 100644 --- a/src/allmydata/grid_manager.py +++ b/src/allmydata/grid_manager.py @@ -1,3 +1,6 @@ +""" +Functions and classes relating to the Grid Manager internal state +""" import sys import json From 80c68a41fd39b3620da8cdf35280b58ae7a280da Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 21 Dec 2020 00:49:56 -0700 Subject: [PATCH 0196/2309] cover more error-cases --- src/allmydata/test/cli/test_grid_manager.py | 56 ++++++++++++++++++ src/allmydata/test/test_grid_manager.py | 65 ++++++++++++++++++++- 2 files changed, 119 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/cli/test_grid_manager.py b/src/allmydata/test/cli/test_grid_manager.py index a8b0315e0..a6f1f7ec2 100644 --- a/src/allmydata/test/cli/test_grid_manager.py +++ b/src/allmydata/test/cli/test_grid_manager.py @@ -44,6 +44,20 @@ class GridManagerCommandLine(SyncTestCase): result = self.runner.invoke(grid_manager, ["--config", "foo", "public-identity"]) self.assertTrue(result.output.startswith("pub-v0-")) + def test_load_invalid(self): + """ + An invalid config is reported to the user + """ + with self.runner.isolated_filesystem(): + with open("config.json", "w") as f: + json.dump({"not": "valid"}, f) + result = self.runner.invoke(grid_manager, ["--config", ".", "public-identity"]) + self.assertNotEqual(result.exit_code, 0) + self.assertIn( + "Error loading Grid Manager", + result.output, + ) + def test_create_already(self): """ It's an error to create a new grid-manager in an existing @@ -85,6 +99,22 @@ class GridManagerCommandLine(SyncTestCase): cert = json.loads(sigcert['certificate']) self.assertEqual(cert["public_key"], pubkey) + def test_add_twice(self): + """ + An error is reported trying to add an existing server + """ + pubkey0 = "pub-v0-cbq6hcf3pxcz6ouoafrbktmkixkeuywpcpbcomzd3lqbkq4nmfga" + pubkey1 = "pub-v0-5ysc55trfvfvg466v46j4zmfyltgus3y2gdejifctv7h4zkuyveq" + with self.runner.isolated_filesystem(): + self.runner.invoke(grid_manager, ["--config", "foo", "create"]) + self.runner.invoke(grid_manager, ["--config", "foo", "add", "storage0", pubkey0]) + result = self.runner.invoke(grid_manager, ["--config", "foo", "add", "storage0", pubkey1]) + self.assertNotEquals(result.exit_code, 0) + self.assertIn( + "A storage-server called 'storage0' already exists", + result.output, + ) + def test_add_list_remove(self): """ Add a storage server, list it, remove it. @@ -108,6 +138,32 @@ class GridManagerCommandLine(SyncTestCase): result = self.runner.invoke(grid_manager, ["--config", "foo", "list"]) self.assertEqual(result.output.strip(), "") + def test_remove_missing(self): + """ + Error reported when removing non-existant server + """ + with self.runner.isolated_filesystem(): + self.runner.invoke(grid_manager, ["--config", "foo", "create"]) + result = self.runner.invoke(grid_manager, ["--config", "foo", "remove", "storage0"]) + self.assertNotEquals(result.exit_code, 0) + self.assertIn( + "No storage-server called 'storage0' exists", + result.output, + ) + + def test_sign_missing(self): + """ + Error reported when signing non-existant server + """ + with self.runner.isolated_filesystem(): + self.runner.invoke(grid_manager, ["--config", "foo", "create"]) + result = self.runner.invoke(grid_manager, ["--config", "foo", "sign", "storage0", "42"]) + self.assertNotEquals(result.exit_code, 0) + self.assertIn( + "No storage-server called 'storage0' exists", + result.output, + ) + class TahoeAddGridManagerCert(AsyncTestCase): """ diff --git a/src/allmydata/test/test_grid_manager.py b/src/allmydata/test/test_grid_manager.py index 734b54e4c..4c899257d 100644 --- a/src/allmydata/test/test_grid_manager.py +++ b/src/allmydata/test/test_grid_manager.py @@ -65,6 +65,32 @@ class GridManagerUtilities(SyncTestCase): certs = config.get_grid_manager_certificates() self.assertEqual([fake_cert], certs) + def test_load_certificates_invalid_version(self): + """ + An error is reported loading invalid certificate version + """ + cert_path = self.mktemp() + fake_cert = { + "certificate": "{\"expires\":1601687822,\"public_key\":\"pub-v0-cbq6hcf3pxcz6ouoafrbktmkixkeuywpcpbcomzd3lqbkq4nmfga\",\"version\":22}", + "signature": "fvjd3uvvupf2v6tnvkwjd473u3m3inyqkwiclhp7balmchkmn3px5pei3qyfjnhymq4cjcwvbpqmcwwnwswdtrfkpnlaxuih2zbdmda" + } + with open(cert_path, "w") as f: + f.write(json.dumps(fake_cert)) + config_data = ( + "[grid_managers]\n" + "fluffy = pub-v0-vqimc4s5eflwajttsofisp5st566dbq36xnpp4siz57ufdavpvlq\n" + "[grid_manager_certificates]\n" + "ding = {}\n".format(cert_path) + ) + config = config_from_string("/foo", "portnum", config_data, client_valid_config()) + self.assertEqual( + {"fluffy": "pub-v0-vqimc4s5eflwajttsofisp5st566dbq36xnpp4siz57ufdavpvlq"}, + config.enumerate_section("grid_managers") + ) + certs = config.get_grid_manager_certificates() + self.assertEqual([fake_cert], certs) + print(certs) + class GridManagerVerifier(SyncTestCase): """ @@ -182,6 +208,41 @@ class GridManagerVerifier(SyncTestCase): str(ctx.exception), ) + def test_invalid_certificate_bad_version(self): + """ + Invalid Grid Manager config containing a certificate with an + illegal version + """ + tempdir = self.mktemp() + fp = FilePath(tempdir) + config = { + "grid_manager_config_version": 0, + "private_key": "priv-v0-ub7knkkmkptqbsax4tznymwzc4nk5lynskwjsiubmnhcpd7lvlqa", + "storage_servers": { + "alice": { + "public_key": "pub-v0-cbq6hcf3pxcz6ouoafrbktmkixkeuywpcpbcomzd3lqbkq4nmfga" + } + } + } + bad_cert = { + "certificate": "{\"expires\":1601687822,\"public_key\":\"pub-v0-cbq6hcf3pxcz6ouoafrbktmkixkeuywpcpbcomzd3lqbkq4nmfga\",\"version\":0}", + "signature": "fvjd3uvvupf2v6tnvkwjd473u3m3inyqkwiclhp7balmchkmn3px5pei3qyfjnhymq4cjcwvbpqmcwwnwswdtrfkpnlaxuih2zbdmda" + } + + fp.makedirs() + with fp.child("config.json").open("w") as f: + json.dump(config, f) + with fp.child("alice.cert.0").open("w") as f: + json.dump(bad_cert, f) + + with self.assertRaises(ValueError) as ctx: + load_grid_manager(fp) + + self.assertIn( + "Unknown certificate version", + str(ctx.exception), + ) + def test_invalid_no_private_key(self): """ Invalid Grid Manager config with no private key @@ -299,7 +360,7 @@ class GridManagerVerifier(SyncTestCase): self.assertTrue(verify()) -class GridManagerVerifier(SyncTestCase): +class GridManagerInvalidVerifier(SyncTestCase): """ Invalid certificate rejection tests """ @@ -309,7 +370,7 @@ class GridManagerVerifier(SyncTestCase): self.priv0, self.pub0 = ed25519.create_signing_keypair() self.gm.add_storage_server("test0", self.pub0) self.cert0 = self.gm.sign("test0", timedelta(seconds=86400)) - return super(GridManagerVerifier, self).setUp() + return super(GridManagerInvalidVerifier, self).setUp() @given( base32text(), From 30e3cd23da641a6f3a9120e80c80154840aff44d Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 4 Jan 2021 15:39:08 -0700 Subject: [PATCH 0197/2309] un-writable directory test --- src/allmydata/cli/grid_manager.py | 12 +++++++++--- src/allmydata/test/cli/test_grid_manager.py | 17 +++++++++++++++++ 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/src/allmydata/cli/grid_manager.py b/src/allmydata/cli/grid_manager.py index 70f1a7bed..acf233b29 100644 --- a/src/allmydata/cli/grid_manager.py +++ b/src/allmydata/cli/grid_manager.py @@ -194,10 +194,16 @@ def sign(ctx, name, expiry_days): next_serial = 0 f = None while f is None: + fname = "{}.cert.{}".format(name, next_serial) try: - f = fp.child("{}.cert.{}".format(name, next_serial)).create() - except OSError: - f = None + f = fp.child(fname).create() + except OSError as e: + if e.errno == 17: # file exists + f = None + else: + raise click.ClickException( + "{}: {}".format(fname, e) + ) next_serial += 1 with f: f.write(certificate_data) diff --git a/src/allmydata/test/cli/test_grid_manager.py b/src/allmydata/test/cli/test_grid_manager.py index a6f1f7ec2..8c8ab8140 100644 --- a/src/allmydata/test/cli/test_grid_manager.py +++ b/src/allmydata/test/cli/test_grid_manager.py @@ -164,6 +164,23 @@ class GridManagerCommandLine(SyncTestCase): result.output, ) + def test_sign_bad_perms(self): + """ + Error reported if we can't create certificate file + """ + pubkey = "pub-v0-cbq6hcf3pxcz6ouoafrbktmkixkeuywpcpbcomzd3lqbkq4nmfga" + with self.runner.isolated_filesystem(): + self.runner.invoke(grid_manager, ["--config", "foo", "create"]) + self.runner.invoke(grid_manager, ["--config", "foo", "add", "storage0", pubkey]) + # make the directory un-writable (so we can't create a new cert) + os.chmod("foo", 0o550) + result = self.runner.invoke(grid_manager, ["--config", "foo", "sign", "storage0", "42"]) + self.assertEquals(result.exit_code, 1) + self.assertIn( + "Permission denied", + result.output, + ) + class TahoeAddGridManagerCert(AsyncTestCase): """ From 66de141b8a8a7e0b667048cab672acf5b02c0187 Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 4 Jan 2021 16:10:20 -0700 Subject: [PATCH 0198/2309] test: list certs from stdin --- src/allmydata/test/cli/test_grid_manager.py | 26 +++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/src/allmydata/test/cli/test_grid_manager.py b/src/allmydata/test/cli/test_grid_manager.py index 8c8ab8140..4bb82a433 100644 --- a/src/allmydata/test/cli/test_grid_manager.py +++ b/src/allmydata/test/cli/test_grid_manager.py @@ -1,6 +1,9 @@ import os import json +from io import ( + BytesIO, +) from ..common import ( SyncTestCase, @@ -85,6 +88,29 @@ class GridManagerCommandLine(SyncTestCase): set(config.keys()), ) + def test_list_stdout(self): + """ + Load Grid Manager without files (using 'list' subcommand, but any will do) + """ + config = { + "storage_servers": { + "storage0": { + "public_key": "pub-v0-cbq6hcf3pxcz6ouoafrbktmkixkeuywpcpbcomzd3lqbkq4nmfga" + } + }, + "private_key": "priv-v0-6uinzyaxy3zvscwgsps5pxcfezhrkfb43kvnrbrhhfzyduyqnniq", + "grid_manager_config_version": 0 + } + result = self.runner.invoke( + grid_manager, ["--config", "-", "list"], + input=BytesIO(json.dumps(config)), + ) + self.assertEqual(result.exit_code, 0) + self.assertEqual( + "storage0: pub-v0-cbq6hcf3pxcz6ouoafrbktmkixkeuywpcpbcomzd3lqbkq4nmfga\n", + result.output, + ) + def test_add_and_sign(self): """ Add a new storage-server and sign a certificate for it From 6c9632b5ece73727a64fe74baa200c77ee1a3713 Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 4 Jan 2021 16:10:32 -0700 Subject: [PATCH 0199/2309] test: missing cert from config is reported --- src/allmydata/test/test_grid_manager.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/src/allmydata/test/test_grid_manager.py b/src/allmydata/test/test_grid_manager.py index 4c899257d..c3b85bbee 100644 --- a/src/allmydata/test/test_grid_manager.py +++ b/src/allmydata/test/test_grid_manager.py @@ -89,7 +89,25 @@ class GridManagerUtilities(SyncTestCase): ) certs = config.get_grid_manager_certificates() self.assertEqual([fake_cert], certs) - print(certs) + + def test_load_certificates_missing(self): + """ + An error is reported for missing certificates + """ + cert_path = self.mktemp() + config_data = ( + "[grid_managers]\n" + "fluffy = pub-v0-vqimc4s5eflwajttsofisp5st566dbq36xnpp4siz57ufdavpvlq\n" + "[grid_manager_certificates]\n" + "ding = {}\n".format(cert_path) + ) + config = config_from_string("/foo", "portnum", config_data, client_valid_config()) + with self.assertRaises(ValueError) as ctx: + certs = config.get_grid_manager_certificates() + self.assertIn( + "Grid Manager certificate file '{}' doesn't exist".format(cert_path), + str(ctx.exception) + ) class GridManagerVerifier(SyncTestCase): From dbf385bd796997ab0e649cbb100cd25a78f7f4ec Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 4 Jan 2021 19:40:23 -0700 Subject: [PATCH 0200/2309] test: missing config section --- src/allmydata/test/test_node.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/allmydata/test/test_node.py b/src/allmydata/test/test_node.py index 1e0f3020c..74ea6e5eb 100644 --- a/src/allmydata/test/test_node.py +++ b/src/allmydata/test/test_node.py @@ -245,6 +245,20 @@ class TestCase(testutil.SignalMixin, unittest.TestCase): with self.assertRaises(MissingConfigEntry): config.get_config("node", "log_gatherer.furl") + def test_missing_config_section(self): + """ + Enumerating a missing section returns empty dict + """ + basedir = self.mktemp() + fileutil.make_dirs(basedir) + with open(os.path.join(basedir, 'tahoe.cfg'), 'w'): + pass + config = read_config(basedir, "") + self.assertEquals( + config.enumerate_section("not-a-section"), + {} + ) + def test_config_required(self): """ Asking for missing (but required) configuration is an error From 22239022c7ad4e3f8537794971a2d5f468c8e79d Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 4 Jan 2021 21:49:15 -0700 Subject: [PATCH 0201/2309] test: invalid cert key --- src/allmydata/grid_manager.py | 2 +- src/allmydata/test/test_grid_manager.py | 25 +++++++++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/src/allmydata/grid_manager.py b/src/allmydata/grid_manager.py index 39f258fd1..c288a34bb 100644 --- a/src/allmydata/grid_manager.py +++ b/src/allmydata/grid_manager.py @@ -332,7 +332,7 @@ def parse_grid_manager_certificate(gm_data): if set(js.keys()) != required_keys: raise ValueError( "Grid Manager certificate must contain: {}".format( - ", ".join("'{}'".format(k) for k in js.keys()), + ", ".join("'{}'".format(k) for k in required_keys), ) ) return js diff --git a/src/allmydata/test/test_grid_manager.py b/src/allmydata/test/test_grid_manager.py index c3b85bbee..c28afd5fd 100644 --- a/src/allmydata/test/test_grid_manager.py +++ b/src/allmydata/test/test_grid_manager.py @@ -90,6 +90,31 @@ class GridManagerUtilities(SyncTestCase): certs = config.get_grid_manager_certificates() self.assertEqual([fake_cert], certs) + def test_load_certificates_unknown_key(self): + """ + An error is reported loading certificates with invalid keys in them + """ + cert_path = self.mktemp() + fake_cert = { + "certificate": "{\"expires\":1601687822,\"public_key\":\"pub-v0-cbq6hcf3pxcz6ouoafrbktmkixkeuywpcpbcomzd3lqbkq4nmfga\",\"version\":22}", + "signature": "fvjd3uvvupf2v6tnvkwjd473u3m3inyqkwiclhp7balmchkmn3px5pei3qyfjnhymq4cjcwvbpqmcwwnwswdtrfkpnlaxuih2zbdmda", + "something-else": "not valid in a v0 certificate" + } + with open(cert_path, "w") as f: + f.write(json.dumps(fake_cert)) + config_data = ( + "[grid_manager_certificates]\n" + "ding = {}\n".format(cert_path) + ) + config = config_from_string("/foo", "portnum", config_data, client_valid_config()) + with self.assertRaises(ValueError) as ctx: + certs = config.get_grid_manager_certificates() + + self.assertIn( + "Unknown key in Grid Manager certificate", + str(ctx.exception) + ) + def test_load_certificates_missing(self): """ An error is reported for missing certificates From 7783f31c1b0580211f7c05f8b8a08c40c4e4c976 Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 4 Jan 2021 21:49:40 -0700 Subject: [PATCH 0202/2309] tests for 'tahoe admin add-grid-manager-cert' --- src/allmydata/scripts/admin.py | 4 +- src/allmydata/test/cli/test_admin.py | 178 ++++++++++++++++++++++++ src/allmydata/test/test_grid_manager.py | 10 +- 3 files changed, 185 insertions(+), 7 deletions(-) create mode 100644 src/allmydata/test/cli/test_admin.py diff --git a/src/allmydata/scripts/admin.py b/src/allmydata/scripts/admin.py index 8710641ec..a0014a24e 100644 --- a/src/allmydata/scripts/admin.py +++ b/src/allmydata/scripts/admin.py @@ -115,8 +115,6 @@ def add_grid_manager_cert(options): """ Add a new Grid Manager certificate to our config """ - if options.certificate_data is None: - return 1 # XXX is there really not already a function for this? if options.parent.parent['node-directory']: nd = argv_to_abspath(options.parent.parent['node-directory']) @@ -132,7 +130,7 @@ def add_grid_manager_cert(options): if cert_path.exists(): msg = "Already have certificate for '{}' (at {})".format( options['name'], - cert_path, + cert_path.path, ) print(msg, file=options.parent.parent.stderr) return 1 diff --git a/src/allmydata/test/cli/test_admin.py b/src/allmydata/test/cli/test_admin.py new file mode 100644 index 000000000..0d825945c --- /dev/null +++ b/src/allmydata/test/cli/test_admin.py @@ -0,0 +1,178 @@ +import json +from io import ( + BytesIO, + StringIO, +) + +from twisted.python.usage import ( + UsageError, +) +from twisted.python.filepath import ( + FilePath, +) + +from allmydata.scripts.admin import ( + AdminCommand, + AddGridManagerCertOptions, + add_grid_manager_cert, +) +from allmydata.scripts.runner import ( + Options, +) +from ..common import ( + SyncTestCase, +) + + +fake_cert = { + "certificate": "{\"expires\":1601687822,\"public_key\":\"pub-v0-cbq6hcf3pxcz6ouoafrbktmkixkeuywpcpbcomzd3lqbkq4nmfga\",\"version\":1}", + "signature": "fvjd3uvvupf2v6tnvkwjd473u3m3inyqkwiclhp7balmchkmn3px5pei3qyfjnhymq4cjcwvbpqmcwwnwswdtrfkpnlaxuih2zbdmda" +} + + +class AddCertificateOptions(SyncTestCase): + """ + Tests for 'tahoe admin add-grid-manager-cert' option validation + """ + + def setUp(self): + self.tahoe = Options() + return super(AddCertificateOptions, self).setUp() + + def test_parse_no_data(self): + """ + When no data is passed to stdin an error is produced + """ + self.tahoe.stdin = BytesIO(b"") + self.tahoe.stderr = BytesIO() # suppress message + + with self.assertRaises(UsageError) as ctx: + self.tahoe.parseOptions( + [ + "admin", "add-grid-manager-cert", + "--name", "random-name", + "--filename", "-", + ] + ) + + self.assertIn( + "Reading certificate from stdin failed", + str(ctx.exception) + ) + + def test_read_cert_file(self): + """ + A certificate can be read from a file + """ + tmp = self.mktemp() + with open(tmp, "w") as f: + json.dump(fake_cert, f) + + # certificate should be loaded + o = self.tahoe.parseOptions( + [ + "admin", "add-grid-manager-cert", + "--name", "random-name", + "--filename", tmp, + ] + ) + opts = self.tahoe.subOptions.subOptions + self.assertEqual( + fake_cert, + opts.certificate_data + ) + + def test_bad_certificate(self): + """ + Unparseable data produces an error + """ + self.tahoe.stdin = BytesIO(b"{}") + self.tahoe.stderr = BytesIO() # suppress message + + with self.assertRaises(UsageError) as ctx: + self.tahoe.parseOptions( + [ + "admin", "add-grid-manager-cert", + "--name", "random-name", + "--filename", "-", + ] + ) + + self.assertIn( + "Grid Manager certificate must contain", + str(ctx.exception) + ) + + +class AddCertificateCommand(SyncTestCase): + """ + Tests for 'tahoe admin add-grid-manager-cert' operation + """ + + def setUp(self): + self.tahoe = Options() + self.node_path = FilePath(self.mktemp()) + self.node_path.makedirs() + with self.node_path.child("tahoe.cfg").open("w") as f: + f.write("# minimal test config\n") + return super(AddCertificateCommand, self).setUp() + + def test_add_one(self): + """ + Adding a certificate succeeds + """ + self.tahoe.stdin = BytesIO(json.dumps(fake_cert)) + self.tahoe.stderr = BytesIO() + self.tahoe.parseOptions( + [ + "--node-directory", self.node_path.path, + "admin", "add-grid-manager-cert", + "--name", "zero", + "--filename", "-", + ] + ) + rc = add_grid_manager_cert(self.tahoe.subOptions.subOptions) + + self.assertEqual(rc, 0) + self.assertEqual( + ["zero.cert", "tahoe.cfg"], + self.node_path.listdir() + ) + self.assertIn( + "There are now 1 certificates", + self.tahoe.stderr.getvalue() + ) + + def test_add_two(self): + """ + An error message is produced when adding a certificate with a + duplicate name. + """ + self.tahoe.stdin = BytesIO(json.dumps(fake_cert)) + self.tahoe.stderr = BytesIO() + self.tahoe.parseOptions( + [ + "--node-directory", self.node_path.path, + "admin", "add-grid-manager-cert", + "--name", "zero", + "--filename", "-", + ] + ) + rc = add_grid_manager_cert(self.tahoe.subOptions.subOptions) + self.assertEqual(rc, 0) + + self.tahoe.stdin = BytesIO(json.dumps(fake_cert)) + self.tahoe.parseOptions( + [ + "--node-directory", self.node_path.path, + "admin", "add-grid-manager-cert", + "--name", "zero", + "--filename", "-", + ] + ) + rc = add_grid_manager_cert(self.tahoe.subOptions.subOptions) + self.assertEqual(rc, 1) + self.assertIn( + "Already have certificate for 'zero'", + self.tahoe.stderr.getvalue() + ) diff --git a/src/allmydata/test/test_grid_manager.py b/src/allmydata/test/test_grid_manager.py index c28afd5fd..01cb50ccf 100644 --- a/src/allmydata/test/test_grid_manager.py +++ b/src/allmydata/test/test_grid_manager.py @@ -154,18 +154,20 @@ class GridManagerVerifier(SyncTestCase): """ priv, pub = ed25519.create_signing_keypair() self.gm.add_storage_server("test", pub) - cert = self.gm.sign("test", timedelta(seconds=86400)) + cert0 = self.gm.sign("test", timedelta(seconds=86400)) + cert1 = self.gm.sign("test", timedelta(seconds=86400)) + self.assertNotEqual(cert0, cert1) self.assertEqual( - set(cert.keys()), + set(cert0.keys()), {"certificate", "signature"}, ) gm_key = ed25519.verifying_key_from_string(self.gm.public_identity()) self.assertEqual( ed25519.verify_signature( gm_key, - base32.a2b(cert["signature"]), - cert["certificate"], + base32.a2b(cert0["signature"]), + cert0["certificate"], ), None ) From c0f0076a3ea40be1619ae8b57f59637b763bbffc Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 5 Jan 2021 00:00:13 -0700 Subject: [PATCH 0203/2309] undo --- src/allmydata/scripts/runner.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/allmydata/scripts/runner.py b/src/allmydata/scripts/runner.py index 30b29f68e..f57397f74 100644 --- a/src/allmydata/scripts/runner.py +++ b/src/allmydata/scripts/runner.py @@ -162,13 +162,7 @@ def dispatch(config, # 4: return a Deferred that does 1 or 2 or 3 def _raise_sys_exit(rc): sys.exit(rc) - - def _error(f): - f.trap(usage.UsageError) - print("Error: {}".format(f.value.message)) - sys.exit(1) d.addCallback(_raise_sys_exit) - d.addErrback(_error) return d def _maybe_enable_eliot_logging(options, reactor): From 7d30bd53c7445c1bbb4d3709649b2ffe9796f4c4 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 5 Jan 2021 01:14:29 -0700 Subject: [PATCH 0204/2309] test: announcements contain grid-manager certs --- src/allmydata/test/test_client.py | 43 ++++++++++++++++++++++++++++++- 1 file changed, 42 insertions(+), 1 deletion(-) diff --git a/src/allmydata/test/test_client.py b/src/allmydata/test/test_client.py index 342fe4af1..2c4ccca25 100644 --- a/src/allmydata/test/test_client.py +++ b/src/allmydata/test/test_client.py @@ -1,4 +1,6 @@ -import os, sys +import os +import sys +import json from functools import ( partial, ) @@ -35,6 +37,7 @@ from testtools.matchers import ( AfterPreprocessing, MatchesListwise, MatchesDict, + ContainsDict, Always, Is, raises, @@ -1543,3 +1546,41 @@ enabled = {storage_enabled} ), ), ) + + def test_announcement_includes_grid_manager(self): + """ + When Grid Manager is enabled certificates are included in the + announcement + """ + fake_cert = { + "certificate": "{\"expires\":1601687822,\"public_key\":\"pub-v0-cbq6hcf3pxcz6ouoafrbktmkixkeuywpcpbcomzd3lqbkq4nmfga\",\"version\":1}", + "signature": "fvjd3uvvupf2v6tnvkwjd473u3m3inyqkwiclhp7balmchkmn3px5pei3qyfjnhymq4cjcwvbpqmcwwnwswdtrfkpnlaxuih2zbdmda", + } + with self.basedir.child("zero.cert").open("w") as f: + json.dump(fake_cert, f) + + config = client.config_from_string( + self.basedir.path, + "tub.port", + self.get_config( + storage_enabled=True, + more_storage="grid_management = True", + more_sections=( + "[grid_manager_certificates]\n" + "foo = zero.cert\n" + ) + ), + ) + + self.assertThat( + client.create_client_from_config( + config, + _introducer_factory=MemoryIntroducerClient, + ), + succeeded(AfterPreprocessing( + lambda client: get_published_announcements(client)[0].ann, + ContainsDict({ + "grid-manager-certificates": Equals([fake_cert]), + }), + )), + ) From 2dc6c5f2e808b2208767f9f07e38ccf56912e3da Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 19 Jan 2021 10:11:36 -0700 Subject: [PATCH 0205/2309] add a gm cert in the client test --- src/allmydata/test/test_client.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/allmydata/test/test_client.py b/src/allmydata/test/test_client.py index 2c4ccca25..8ce63dc8a 100644 --- a/src/allmydata/test/test_client.py +++ b/src/allmydata/test/test_client.py @@ -1558,6 +1558,8 @@ enabled = {storage_enabled} } with self.basedir.child("zero.cert").open("w") as f: json.dump(fake_cert, f) + with self.basedir.child("gm0.cert").open("w") as f: + json.dump(fake_cert, f) config = client.config_from_string( self.basedir.path, @@ -1566,6 +1568,8 @@ enabled = {storage_enabled} storage_enabled=True, more_storage="grid_management = True", more_sections=( + "[grid_managers]\n" + "gm0 = pub-v0-ibpbsexcjfbv3ni7gwlclgn6mldaqnqd5mrtan2fnq2b27xnovca\n" "[grid_manager_certificates]\n" "foo = zero.cert\n" ) From 6560621e959ff269788e2fbcf6027dca1dc52e99 Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 21 Jan 2021 00:08:00 -0700 Subject: [PATCH 0206/2309] cover 'second certificate' codepath --- src/allmydata/test/cli/test_grid_manager.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/allmydata/test/cli/test_grid_manager.py b/src/allmydata/test/cli/test_grid_manager.py index 4bb82a433..6150c4710 100644 --- a/src/allmydata/test/cli/test_grid_manager.py +++ b/src/allmydata/test/cli/test_grid_manager.py @@ -125,6 +125,22 @@ class GridManagerCommandLine(SyncTestCase): cert = json.loads(sigcert['certificate']) self.assertEqual(cert["public_key"], pubkey) + def test_add_and_sign_second_cert(self): + """ + Add a new storage-server and sign two certificates. + """ + pubkey = "pub-v0-cbq6hcf3pxcz6ouoafrbktmkixkeuywpcpbcomzd3lqbkq4nmfga" + with self.runner.isolated_filesystem(): + self.runner.invoke(grid_manager, ["--config", "foo", "create"]) + self.runner.invoke(grid_manager, ["--config", "foo", "add", "storage0", pubkey]) + result = self.runner.invoke(grid_manager, ["--config", "foo", "sign", "storage0", "10"]) + result = self.runner.invoke(grid_manager, ["--config", "foo", "sign", "storage0", "10"]) + # we should now have two certificates stored + self.assertEqual( + set(FilePath("foo").listdir()), + {'storage0.cert.1', 'storage0.cert.0', 'config.json'}, + ) + def test_add_twice(self): """ An error is reported trying to add an existing server From a2f0f6581ed0515360f4a65b8fbcb6554dcd6cd3 Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 21 Jan 2021 18:36:18 -0700 Subject: [PATCH 0207/2309] order not important --- src/allmydata/test/cli/test_admin.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/cli/test_admin.py b/src/allmydata/test/cli/test_admin.py index 0d825945c..6521272fa 100644 --- a/src/allmydata/test/cli/test_admin.py +++ b/src/allmydata/test/cli/test_admin.py @@ -135,8 +135,8 @@ class AddCertificateCommand(SyncTestCase): self.assertEqual(rc, 0) self.assertEqual( - ["zero.cert", "tahoe.cfg"], - self.node_path.listdir() + {"zero.cert", "tahoe.cfg"}, + set(self.node_path.listdir()) ) self.assertIn( "There are now 1 certificates", From 66d4118e5468f5a52714f522377bc3dcfff32145 Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 21 Jan 2021 18:42:02 -0700 Subject: [PATCH 0208/2309] I know know how permissions work on linux --- src/allmydata/test/cli/test_grid_manager.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/allmydata/test/cli/test_grid_manager.py b/src/allmydata/test/cli/test_grid_manager.py index 6150c4710..80a39d540 100644 --- a/src/allmydata/test/cli/test_grid_manager.py +++ b/src/allmydata/test/cli/test_grid_manager.py @@ -4,6 +4,9 @@ import json from io import ( BytesIO, ) +from unittest import ( + skipIf, +) from ..common import ( SyncTestCase, @@ -25,6 +28,9 @@ from twisted.internet.defer import ( from twisted.python.filepath import ( FilePath, ) +from twisted.python.runtime import ( + platform, +) class GridManagerCommandLine(SyncTestCase): @@ -206,6 +212,7 @@ class GridManagerCommandLine(SyncTestCase): result.output, ) + @skipIf(not platform.isLinux(), "I only know how permissions work on linux") def test_sign_bad_perms(self): """ Error reported if we can't create certificate file From 75db6f3e622be1cc519c1a41a9093f60597c811a Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 22 Jan 2021 16:01:29 -0700 Subject: [PATCH 0209/2309] relax test --- src/allmydata/test/test_grid_manager.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/allmydata/test/test_grid_manager.py b/src/allmydata/test/test_grid_manager.py index 01cb50ccf..56541aa16 100644 --- a/src/allmydata/test/test_grid_manager.py +++ b/src/allmydata/test/test_grid_manager.py @@ -129,8 +129,15 @@ class GridManagerUtilities(SyncTestCase): config = config_from_string("/foo", "portnum", config_data, client_valid_config()) with self.assertRaises(ValueError) as ctx: certs = config.get_grid_manager_certificates() + # we don't reliably know how Windows or MacOS will represent + # the path in the exception, so we don't check for the *exact* + # message with full-path here.. self.assertIn( - "Grid Manager certificate file '{}' doesn't exist".format(cert_path), + "Grid Manager certificate file", + str(ctx.exception) + ) + self.assertIn( + " doesn't exist", str(ctx.exception) ) From 7aaf86ac946791ee8b4bd39d59a56671110bbe1e Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 25 Jan 2021 15:23:47 -0700 Subject: [PATCH 0210/2309] flake8 cleanup --- src/allmydata/test/cli/test_admin.py | 5 +---- src/allmydata/test/cli/test_grid_manager.py | 4 ++-- src/allmydata/test/test_grid_manager.py | 4 ++-- 3 files changed, 5 insertions(+), 8 deletions(-) diff --git a/src/allmydata/test/cli/test_admin.py b/src/allmydata/test/cli/test_admin.py index 6521272fa..37f733f07 100644 --- a/src/allmydata/test/cli/test_admin.py +++ b/src/allmydata/test/cli/test_admin.py @@ -1,7 +1,6 @@ import json from io import ( BytesIO, - StringIO, ) from twisted.python.usage import ( @@ -12,8 +11,6 @@ from twisted.python.filepath import ( ) from allmydata.scripts.admin import ( - AdminCommand, - AddGridManagerCertOptions, add_grid_manager_cert, ) from allmydata.scripts.runner import ( @@ -69,7 +66,7 @@ class AddCertificateOptions(SyncTestCase): json.dump(fake_cert, f) # certificate should be loaded - o = self.tahoe.parseOptions( + self.tahoe.parseOptions( [ "admin", "add-grid-manager-cert", "--name", "random-name", diff --git a/src/allmydata/test/cli/test_grid_manager.py b/src/allmydata/test/cli/test_grid_manager.py index 80a39d540..397824876 100644 --- a/src/allmydata/test/cli/test_grid_manager.py +++ b/src/allmydata/test/cli/test_grid_manager.py @@ -139,8 +139,8 @@ class GridManagerCommandLine(SyncTestCase): with self.runner.isolated_filesystem(): self.runner.invoke(grid_manager, ["--config", "foo", "create"]) self.runner.invoke(grid_manager, ["--config", "foo", "add", "storage0", pubkey]) - result = self.runner.invoke(grid_manager, ["--config", "foo", "sign", "storage0", "10"]) - result = self.runner.invoke(grid_manager, ["--config", "foo", "sign", "storage0", "10"]) + self.runner.invoke(grid_manager, ["--config", "foo", "sign", "storage0", "10"]) + self.runner.invoke(grid_manager, ["--config", "foo", "sign", "storage0", "10"]) # we should now have two certificates stored self.assertEqual( set(FilePath("foo").listdir()), diff --git a/src/allmydata/test/test_grid_manager.py b/src/allmydata/test/test_grid_manager.py index 56541aa16..222631111 100644 --- a/src/allmydata/test/test_grid_manager.py +++ b/src/allmydata/test/test_grid_manager.py @@ -108,7 +108,7 @@ class GridManagerUtilities(SyncTestCase): ) config = config_from_string("/foo", "portnum", config_data, client_valid_config()) with self.assertRaises(ValueError) as ctx: - certs = config.get_grid_manager_certificates() + config.get_grid_manager_certificates() self.assertIn( "Unknown key in Grid Manager certificate", @@ -128,7 +128,7 @@ class GridManagerUtilities(SyncTestCase): ) config = config_from_string("/foo", "portnum", config_data, client_valid_config()) with self.assertRaises(ValueError) as ctx: - certs = config.get_grid_manager_certificates() + config.get_grid_manager_certificates() # we don't reliably know how Windows or MacOS will represent # the path in the exception, so we don't check for the *exact* # message with full-path here.. From 87c78fd8f341b4f2b8c3beb85794850e11dbb572 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 26 Jan 2021 13:32:23 -0700 Subject: [PATCH 0211/2309] use plain TestCase (fix eliot tests, maybe) --- src/allmydata/test/cli/test_grid_manager.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/allmydata/test/cli/test_grid_manager.py b/src/allmydata/test/cli/test_grid_manager.py index 397824876..e63e6aaaa 100644 --- a/src/allmydata/test/cli/test_grid_manager.py +++ b/src/allmydata/test/cli/test_grid_manager.py @@ -8,9 +8,8 @@ from unittest import ( skipIf, ) -from ..common import ( - SyncTestCase, - AsyncTestCase, +from twisted.trial.unittest import ( + TestCase, ) from allmydata.cli.grid_manager import ( grid_manager, @@ -33,7 +32,7 @@ from twisted.python.runtime import ( ) -class GridManagerCommandLine(SyncTestCase): +class GridManagerCommandLine(TestCase): """ Test the mechanics of the `grid-manager` command """ @@ -231,7 +230,7 @@ class GridManagerCommandLine(SyncTestCase): ) -class TahoeAddGridManagerCert(AsyncTestCase): +class TahoeAddGridManagerCert(TestCase): """ Test `tahoe admin add-grid-manager-cert` subcommand """ From 8bb83e797eeac3659a133116ccc159dde0250ad6 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 26 Jan 2021 15:13:57 -0700 Subject: [PATCH 0212/2309] actually not-equal --- src/allmydata/test/test_grid_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/test_grid_manager.py b/src/allmydata/test/test_grid_manager.py index 222631111..e53984a05 100644 --- a/src/allmydata/test/test_grid_manager.py +++ b/src/allmydata/test/test_grid_manager.py @@ -162,7 +162,7 @@ class GridManagerVerifier(SyncTestCase): priv, pub = ed25519.create_signing_keypair() self.gm.add_storage_server("test", pub) cert0 = self.gm.sign("test", timedelta(seconds=86400)) - cert1 = self.gm.sign("test", timedelta(seconds=86400)) + cert1 = self.gm.sign("test", timedelta(seconds=3600)) self.assertNotEqual(cert0, cert1) self.assertEqual( From 492bcbbd128b5c3a3d4c4c9f9898e9ddac522713 Mon Sep 17 00:00:00 2001 From: fenn-cs Date: Fri, 13 Aug 2021 18:22:10 +0100 Subject: [PATCH 0213/2309] Refactored test_logs to be consistent with base testcases Signed-off-by: fenn-cs --- newsfragments/3758.other | 1 + src/allmydata/test/web/test_logs.py | 27 ++++++++++++++++----------- src/allmydata/util/eliotutil.py | 9 +++++++-- 3 files changed, 24 insertions(+), 13 deletions(-) create mode 100644 newsfragments/3758.other diff --git a/newsfragments/3758.other b/newsfragments/3758.other new file mode 100644 index 000000000..d0eb1d4c1 --- /dev/null +++ b/newsfragments/3758.other @@ -0,0 +1 @@ +Refactored test_logs, test_grid and test_root in web tests to use custom base test cases diff --git a/src/allmydata/test/web/test_logs.py b/src/allmydata/test/web/test_logs.py index 89ec7ba42..043541690 100644 --- a/src/allmydata/test/web/test_logs.py +++ b/src/allmydata/test/web/test_logs.py @@ -17,10 +17,8 @@ if PY2: import json -from twisted.trial import unittest from twisted.internet.defer import inlineCallbacks -from eliot import log_call from autobahn.twisted.testing import create_memory_agent, MemoryReactorClockResolver, create_pumper @@ -48,6 +46,7 @@ from .matchers import ( from ..common import ( SyncTestCase, + AsyncTestCase, ) from ...web.logs import ( @@ -55,6 +54,10 @@ from ...web.logs import ( TokenAuthenticatedWebSocketServerProtocol, ) +from ...util.eliotutil import ( + log_call_deferred +) + class StreamingEliotLogsTests(SyncTestCase): """ Tests for the log streaming resources created by ``create_log_resources``. @@ -75,18 +78,20 @@ class StreamingEliotLogsTests(SyncTestCase): ) -class TestStreamingLogs(unittest.TestCase): +class TestStreamingLogs(AsyncTestCase): """ Test websocket streaming of logs """ def setUp(self): + super(TestStreamingLogs, self).setUp() self.reactor = MemoryReactorClockResolver() self.pumper = create_pumper() self.agent = create_memory_agent(self.reactor, self.pumper, TokenAuthenticatedWebSocketServerProtocol) return self.pumper.start() def tearDown(self): + super(TestStreamingLogs, self).tearDown() return self.pumper.stop() @inlineCallbacks @@ -105,7 +110,7 @@ class TestStreamingLogs(unittest.TestCase): messages.append(json.loads(msg)) proto.on("message", got_message) - @log_call(action_type=u"test:cli:some-exciting-action") + @log_call_deferred(action_type=u"test:cli:some-exciting-action") def do_a_thing(arguments): pass @@ -114,10 +119,10 @@ class TestStreamingLogs(unittest.TestCase): proto.transport.loseConnection() yield proto.is_closed - self.assertEqual(len(messages), 2) - self.assertEqual(messages[0]["action_type"], "test:cli:some-exciting-action") - self.assertEqual(messages[0]["arguments"], - ["hello", "good-\\xff-day", 123, {"a": 35}, [None]]) - self.assertEqual(messages[1]["action_type"], "test:cli:some-exciting-action") - self.assertEqual("started", messages[0]["action_status"]) - self.assertEqual("succeeded", messages[1]["action_status"]) + self.assertThat(len(messages), Equals(3)) + self.assertThat(messages[0]["action_type"], Equals("test:cli:some-exciting-action")) + self.assertThat(messages[0]["arguments"], + Equals(["hello", "good-\\xff-day", 123, {"a": 35}, [None]])) + self.assertThat(messages[1]["action_type"], Equals("test:cli:some-exciting-action")) + self.assertThat("started", Equals(messages[0]["action_status"])) + self.assertThat("succeeded", Equals(messages[1]["action_status"])) diff --git a/src/allmydata/util/eliotutil.py b/src/allmydata/util/eliotutil.py index 4e48fbb9f..ec4c0bf97 100644 --- a/src/allmydata/util/eliotutil.py +++ b/src/allmydata/util/eliotutil.py @@ -87,7 +87,11 @@ from twisted.internet.defer import ( ) from twisted.application.service import Service -from .jsonbytes import AnyBytesJSONEncoder +from .jsonbytes import ( + AnyBytesJSONEncoder, + bytes_to_unicode +) + def validateInstanceOf(t): @@ -320,7 +324,8 @@ def log_call_deferred(action_type): def logged_f(*a, **kw): # Use the action's context method to avoid ending the action when # the `with` block ends. - with start_action(action_type=action_type).context(): + args = bytes_to_unicode(True, kw['arguments']) + with start_action(action_type=action_type, arguments=args).context(): # Use addActionFinish so that the action finishes when the # Deferred fires. d = maybeDeferred(f, *a, **kw) From 27c8e62cf648a1186a91e84aa0cd84e62774c5e9 Mon Sep 17 00:00:00 2001 From: fenn-cs Date: Sat, 14 Aug 2021 00:09:34 +0100 Subject: [PATCH 0214/2309] Replaced fixed arg with dynamic args in log_call_deferred Signed-off-by: fenn-cs --- src/allmydata/util/eliotutil.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/util/eliotutil.py b/src/allmydata/util/eliotutil.py index ec4c0bf97..d989c9e2a 100644 --- a/src/allmydata/util/eliotutil.py +++ b/src/allmydata/util/eliotutil.py @@ -324,8 +324,8 @@ def log_call_deferred(action_type): def logged_f(*a, **kw): # Use the action's context method to avoid ending the action when # the `with` block ends. - args = bytes_to_unicode(True, kw['arguments']) - with start_action(action_type=action_type, arguments=args).context(): + args = {k: bytes_to_unicode(True, kw[k]) for k in kw} + with start_action(action_type=action_type, **args).context(): # Use addActionFinish so that the action finishes when the # Deferred fires. d = maybeDeferred(f, *a, **kw) From f7f08c93f9088b187bbef8de225c0e8352a6cd36 Mon Sep 17 00:00:00 2001 From: fenn-cs Date: Mon, 16 Aug 2021 12:57:24 +0100 Subject: [PATCH 0215/2309] Refactored test_root to be consistent with base testcases Signed-off-by: fenn-cs --- src/allmydata/test/web/test_root.py | 39 ++++++++++++++++++----------- 1 file changed, 24 insertions(+), 15 deletions(-) diff --git a/src/allmydata/test/web/test_root.py b/src/allmydata/test/web/test_root.py index ca3cc695d..1d5e45ba4 100644 --- a/src/allmydata/test/web/test_root.py +++ b/src/allmydata/test/web/test_root.py @@ -20,10 +20,11 @@ from bs4 import ( BeautifulSoup, ) -from twisted.trial import unittest from twisted.web.template import Tag from twisted.web.test.requesthelper import DummyRequest from twisted.application import service +from testtools.twistedsupport import succeeded +from twisted.internet.defer import inlineCallbacks from ...storage_client import ( NativeStorageServer, @@ -44,7 +45,17 @@ from ..common import ( EMPTY_CLIENT_CONFIG, ) -class RenderSlashUri(unittest.TestCase): +from ..common import ( + SyncTestCase, +) + +from testtools.matchers import ( + Equals, + Contains, + AfterPreprocessing, +) + +class RenderSlashUri(SyncTestCase): """ Ensure that URIs starting with /uri?uri= only accept valid capabilities @@ -53,7 +64,9 @@ class RenderSlashUri(unittest.TestCase): def setUp(self): self.client = object() self.res = URIHandler(self.client) + super(RenderSlashUri, self).setUp() + @inlineCallbacks def test_valid_query_redirect(self): """ A syntactically valid capability given in the ``uri`` query argument @@ -64,9 +77,7 @@ class RenderSlashUri(unittest.TestCase): b"mukesarwdjxiyqsjinbfiiro6q7kgmmekocxfjcngh23oxwyxtzq:2:5:5874882" ) query_args = {b"uri": [cap]} - response_body = self.successResultOf( - render(self.res, query_args), - ) + response_body = yield render(self.res, query_args) soup = BeautifulSoup(response_body, 'html5lib') tag = assert_soup_has_tag_with_attributes( self, @@ -74,9 +85,9 @@ class RenderSlashUri(unittest.TestCase): u"meta", {u"http-equiv": "refresh"}, ) - self.assertIn( - quote(cap, safe=""), + self.assertThat( tag.attrs.get(u"content"), + Contains(quote(cap, safe="")), ) def test_invalid(self): @@ -84,16 +95,14 @@ class RenderSlashUri(unittest.TestCase): A syntactically invalid capbility results in an error. """ query_args = {b"uri": [b"not a capability"]} - response_body = self.successResultOf( - render(self.res, query_args), - ) - self.assertEqual( + response_body = render(self.res, query_args) + self.assertThat( response_body, - b"Invalid capability", + succeeded(AfterPreprocessing(bytes, Equals(b"Invalid capability"))), ) -class RenderServiceRow(unittest.TestCase): +class RenderServiceRow(SyncTestCase): def test_missing(self): """ minimally-defined static servers just need anonymous-storage-FURL @@ -127,5 +136,5 @@ class RenderServiceRow(unittest.TestCase): # Coerce `items` to list and pick the first item from it. item = list(items)[0] - self.assertEqual(item.slotData.get("version"), "") - self.assertEqual(item.slotData.get("nickname"), "") + self.assertThat(item.slotData.get("version"), Equals("")) + self.assertThat(item.slotData.get("nickname"), Equals("")) From bef2413e4b9bfbfe3553be3b56c9d3a57cf4f623 Mon Sep 17 00:00:00 2001 From: fenn-cs Date: Tue, 17 Aug 2021 13:11:54 +0100 Subject: [PATCH 0216/2309] Refactored test_grid to be consistent with base testcases Signed-off-by: fenn-cs --- src/allmydata/test/web/test_grid.py | 204 +++++++++++++++------------- 1 file changed, 107 insertions(+), 97 deletions(-) diff --git a/src/allmydata/test/web/test_grid.py b/src/allmydata/test/web/test_grid.py index edcf32268..54aa13941 100644 --- a/src/allmydata/test/web/test_grid.py +++ b/src/allmydata/test/web/test_grid.py @@ -18,7 +18,6 @@ from six.moves import StringIO from bs4 import BeautifulSoup from twisted.web import resource -from twisted.trial import unittest from allmydata import uri, dirnode from allmydata.util import base32 from allmydata.util.encodingutil import to_bytes @@ -43,6 +42,20 @@ from .common import ( unknown_rwcap, ) +from ..common import ( + AsyncTestCase, +) + +from testtools.matchers import ( + Equals, + Contains, + Is, + Not, +) + +from testtools.twistedsupport import flush_logged_errors + + DIR_HTML_TAG = '' class CompletelyUnhandledError(Exception): @@ -53,7 +66,7 @@ class ErrorBoom(resource.Resource, object): def render(self, req): raise CompletelyUnhandledError("whoops") -class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMixin, unittest.TestCase): +class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMixin, AsyncTestCase): def CHECK(self, ign, which, args, clientnum=0): fileurl = self.fileurls[which] @@ -117,37 +130,37 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi d.addCallback(self.CHECK, "good", "t=check") def _got_html_good(res): - self.failUnlessIn("Healthy", res) - self.failIfIn("Not Healthy", res) + self.assertThat(res, Contains("Healthy")) + self.assertThat(res, Not(Contains("Not Healthy", ))) soup = BeautifulSoup(res, 'html5lib') assert_soup_has_favicon(self, soup) d.addCallback(_got_html_good) d.addCallback(self.CHECK, "good", "t=check&return_to=somewhere") def _got_html_good_return_to(res): - self.failUnlessIn("Healthy", res) - self.failIfIn("Not Healthy", res) - self.failUnlessIn('Return to file', res) + self.assertThat(res, Contains("Healthy")) + self.assertThat(res, Not(Contains("Not Healthy"))) + self.assertThat(res, Contains('Return to file')) d.addCallback(_got_html_good_return_to) d.addCallback(self.CHECK, "good", "t=check&output=json") def _got_json_good(res): r = json.loads(res) self.failUnlessEqual(r["summary"], "Healthy") self.failUnless(r["results"]["healthy"]) - self.failIfIn("needs-rebalancing", r["results"]) + self.assertThat(r["results"], Not(Contains("needs-rebalancing",))) self.failUnless(r["results"]["recoverable"]) d.addCallback(_got_json_good) d.addCallback(self.CHECK, "small", "t=check") def _got_html_small(res): - self.failUnlessIn("Literal files are always healthy", res) - self.failIfIn("Not Healthy", res) + self.assertThat(res, Contains("Literal files are always healthy")) + self.assertThat(res, Not(Contains("Not Healthy"))) d.addCallback(_got_html_small) d.addCallback(self.CHECK, "small", "t=check&return_to=somewhere") def _got_html_small_return_to(res): - self.failUnlessIn("Literal files are always healthy", res) - self.failIfIn("Not Healthy", res) - self.failUnlessIn('Return to file', res) + self.assertThat(res, Contains("Literal files are always healthy")) + self.assertThat(res, Not(Contains("Not Healthy"))) + self.assertThat(res, Contains('Return to file')) d.addCallback(_got_html_small_return_to) d.addCallback(self.CHECK, "small", "t=check&output=json") def _got_json_small(res): @@ -158,8 +171,8 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi d.addCallback(self.CHECK, "smalldir", "t=check") def _got_html_smalldir(res): - self.failUnlessIn("Literal files are always healthy", res) - self.failIfIn("Not Healthy", res) + self.assertThat(res, Contains("Literal files are always healthy")) + self.assertThat(res, Not(Contains("Not Healthy"))) d.addCallback(_got_html_smalldir) d.addCallback(self.CHECK, "smalldir", "t=check&output=json") def _got_json_smalldir(res): @@ -170,43 +183,43 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi d.addCallback(self.CHECK, "sick", "t=check") def _got_html_sick(res): - self.failUnlessIn("Not Healthy", res) + self.assertThat(res, Contains("Not Healthy")) d.addCallback(_got_html_sick) d.addCallback(self.CHECK, "sick", "t=check&output=json") def _got_json_sick(res): r = json.loads(res) self.failUnlessEqual(r["summary"], "Not Healthy: 9 shares (enc 3-of-10)") - self.failIf(r["results"]["healthy"]) + self.assertThat(r["results"]["healthy"], Is(False)) self.failUnless(r["results"]["recoverable"]) - self.failIfIn("needs-rebalancing", r["results"]) + self.assertThat(r["results"], Not(Contains("needs-rebalancing"))) d.addCallback(_got_json_sick) d.addCallback(self.CHECK, "dead", "t=check") def _got_html_dead(res): - self.failUnlessIn("Not Healthy", res) + self.assertThat(res, Contains("Not Healthy")) d.addCallback(_got_html_dead) d.addCallback(self.CHECK, "dead", "t=check&output=json") def _got_json_dead(res): r = json.loads(res) self.failUnlessEqual(r["summary"], "Not Healthy: 1 shares (enc 3-of-10)") - self.failIf(r["results"]["healthy"]) - self.failIf(r["results"]["recoverable"]) - self.failIfIn("needs-rebalancing", r["results"]) + self.assertThat(r["results"]["healthy"], Is(False)) + self.assertThat(r["results"]["recoverable"], Is(False)) + self.assertThat(r["results"], Not(Contains("needs-rebalancing"))) d.addCallback(_got_json_dead) d.addCallback(self.CHECK, "corrupt", "t=check&verify=true") def _got_html_corrupt(res): - self.failUnlessIn("Not Healthy! : Unhealthy", res) + self.assertThat(res, Contains("Not Healthy! : Unhealthy")) d.addCallback(_got_html_corrupt) d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&output=json") def _got_json_corrupt(res): r = json.loads(res) - self.failUnlessIn("Unhealthy: 9 shares (enc 3-of-10)", r["summary"]) - self.failIf(r["results"]["healthy"]) + self.assertThat(r["summary"], Contains("Unhealthy: 9 shares (enc 3-of-10)")) + self.assertThat(r["results"]["healthy"], Is(False)) self.failUnless(r["results"]["recoverable"]) - self.failIfIn("needs-rebalancing", r["results"]) + self.assertThat(r["results"], Not(Contains("needs-rebalancing"))) self.failUnlessReallyEqual(r["results"]["count-happiness"], 9) self.failUnlessReallyEqual(r["results"]["count-shares-good"], 9) self.failUnlessReallyEqual(r["results"]["count-corrupt-shares"], 1) @@ -261,9 +274,9 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi d.addCallback(self.CHECK, "good", "t=check&repair=true") def _got_html_good(res): - self.failUnlessIn("Healthy", res) - self.failIfIn("Not Healthy", res) - self.failUnlessIn("No repair necessary", res) + self.assertThat(res, Contains("Healthy")) + self.assertThat(res, Not(Contains("Not Healthy"))) + self.assertThat(res, Contains("No repair necessary", )) soup = BeautifulSoup(res, 'html5lib') assert_soup_has_favicon(self, soup) @@ -271,9 +284,9 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi d.addCallback(self.CHECK, "sick", "t=check&repair=true") def _got_html_sick(res): - self.failUnlessIn("Healthy : healthy", res) - self.failIfIn("Not Healthy", res) - self.failUnlessIn("Repair successful", res) + self.assertThat(res, Contains("Healthy : healthy")) + self.assertThat(res, Not(Contains("Not Healthy"))) + self.assertThat(res, Contains("Repair successful")) d.addCallback(_got_html_sick) # repair of a dead file will fail, of course, but it isn't yet @@ -290,9 +303,9 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi d.addCallback(self.CHECK, "corrupt", "t=check&verify=true&repair=true") def _got_html_corrupt(res): - self.failUnlessIn("Healthy : Healthy", res) - self.failIfIn("Not Healthy", res) - self.failUnlessIn("Repair successful", res) + self.assertThat(res, Contains("Healthy : Healthy")) + self.assertThat(res, Not(Contains("Not Healthy"))) + self.assertThat(res, Contains("Repair successful")) d.addCallback(_got_html_corrupt) d.addErrback(self.explain_web_error) @@ -392,31 +405,31 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi if expect_rw_uri: self.failUnlessReallyEqual(to_bytes(f[1]["rw_uri"]), unknown_rwcap, data) else: - self.failIfIn("rw_uri", f[1]) + self.assertThat(f[1], Not(Contains("rw_uri"))) if immutable: self.failUnlessReallyEqual(to_bytes(f[1]["ro_uri"]), unknown_immcap, data) else: self.failUnlessReallyEqual(to_bytes(f[1]["ro_uri"]), unknown_rocap, data) - self.failUnlessIn("metadata", f[1]) + self.assertThat(f[1], Contains("metadata")) d.addCallback(_check_directory_json, expect_rw_uri=not immutable) def _check_info(res, expect_rw_uri, expect_ro_uri): if expect_rw_uri: - self.failUnlessIn(unknown_rwcap, res) + self.assertThat(res, Contains(unknown_rwcap)) if expect_ro_uri: if immutable: - self.failUnlessIn(unknown_immcap, res) + self.assertThat(res, Contains(unknown_immcap)) else: - self.failUnlessIn(unknown_rocap, res) + self.assertThat(res, Contains(unknown_rocap)) else: - self.failIfIn(unknown_rocap, res) + self.assertThat(res, Not(Contains(unknown_rocap))) res = str(res, "utf-8") - self.failUnlessIn("Object Type: unknown", res) - self.failIfIn("Raw data as", res) - self.failIfIn("Directory writecap", res) - self.failIfIn("Checker Operations", res) - self.failIfIn("Mutable File Operations", res) - self.failIfIn("Directory Operations", res) + self.assertThat(res, Contains("Object Type: unknown")) + self.assertThat(res, Not(Contains("Raw data as"))) + self.assertThat(res, Not(Contains("Directory writecap"))) + self.assertThat(res, Not(Contains("Checker Operations"))) + self.assertThat(res, Not(Contains("Mutable File Operations"))) + self.assertThat(res, Not(Contains("Directory Operations"))) # FIXME: these should have expect_rw_uri=not immutable; I don't know # why they fail. Possibly related to ticket #922. @@ -432,7 +445,7 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi if expect_rw_uri: self.failUnlessReallyEqual(to_bytes(data[1]["rw_uri"]), unknown_rwcap, data) else: - self.failIfIn("rw_uri", data[1]) + self.assertThat(data[1], Not(Contains("rw_uri"))) if immutable: self.failUnlessReallyEqual(to_bytes(data[1]["ro_uri"]), unknown_immcap, data) @@ -442,10 +455,10 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi self.failUnlessReallyEqual(data[1]["mutable"], True) else: self.failUnlessReallyEqual(to_bytes(data[1]["ro_uri"]), unknown_rocap, data) - self.failIfIn("mutable", data[1]) + self.assertThat(data[1], Not(Contains("mutable"))) # TODO: check metadata contents - self.failUnlessIn("metadata", data[1]) + self.assertThat(data[1], Contains("metadata")) d.addCallback(lambda ign: self.GET("%s/%s?t=json" % (self.rooturl, str(name)))) d.addCallback(_check_json, expect_rw_uri=not immutable) @@ -519,14 +532,14 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi def _created(dn): self.failUnless(isinstance(dn, dirnode.DirectoryNode)) - self.failIf(dn.is_mutable()) + self.assertThat(dn.is_mutable(), Is(False)) self.failUnless(dn.is_readonly()) # This checks that if we somehow ended up calling dn._decrypt_rwcapdata, it would fail. - self.failIf(hasattr(dn._node, 'get_writekey')) + self.assertThat(hasattr(dn._node, 'get_writekey'), Is(False)) rep = str(dn) - self.failUnlessIn("RO-IMM", rep) + self.assertThat(rep, Contains("RO-IMM")) cap = dn.get_cap() - self.failUnlessIn(b"CHK", cap.to_string()) + self.assertThat(cap.to_string(), Contains(b"CHK")) self.cap = cap self.rootnode = dn self.rooturl = "uri/" + url_quote(dn.get_uri()) @@ -546,7 +559,7 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi (name_utf8, ro_uri, rwcapdata, metadata_s), subpos = split_netstring(entry, 4) name = name_utf8.decode("utf-8") self.failUnlessEqual(rwcapdata, b"") - self.failUnlessIn(name, kids) + self.assertThat(kids, Contains(name)) (expected_child, ign) = kids[name] self.failUnlessReallyEqual(ro_uri, expected_child.get_readonly_uri()) numkids += 1 @@ -572,27 +585,27 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi d.addCallback(lambda ign: self.GET(self.rooturl)) def _check_html(res): soup = BeautifulSoup(res, 'html5lib') - self.failIfIn(b"URI:SSK", res) + self.assertThat(res, Not(Contains(b"URI:SSK"))) found = False for td in soup.find_all(u"td"): if td.text != u"FILE": continue a = td.findNextSibling()(u"a")[0] - self.assertIn(url_quote(lonely_uri), a[u"href"]) - self.assertEqual(u"lonely", a.text) - self.assertEqual(a[u"rel"], [u"noreferrer"]) - self.assertEqual(u"{}".format(len("one")), td.findNextSibling().findNextSibling().text) + self.assertThat(a[u"href"], Contains(url_quote(lonely_uri))) + self.assertThat(a.text, Equals(u"lonely")) + self.assertThat(a[u"rel"], Equals([u"noreferrer"])) + self.assertThat(td.findNextSibling().findNextSibling().text, Equals(u"{}".format(len("one")))) found = True break - self.assertTrue(found) + self.assertThat(found, Is(True)) infos = list( a[u"href"] for a in soup.find_all(u"a") if a.text == u"More Info" ) - self.assertEqual(1, len(infos)) - self.assertTrue(infos[0].endswith(url_quote(lonely_uri) + "?t=info")) + self.assertThat(len(infos), Equals(1)) + self.assertThat(infos[0].endswith(url_quote(lonely_uri) + "?t=info"), Is(True)) d.addCallback(_check_html) # ... and in JSON. @@ -604,7 +617,7 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi self.failUnlessReallyEqual(sorted(listed_children.keys()), [u"lonely"]) ll_type, ll_data = listed_children[u"lonely"] self.failUnlessEqual(ll_type, "filenode") - self.failIfIn("rw_uri", ll_data) + self.assertThat(ll_data, Not(Contains("rw_uri"))) self.failUnlessReallyEqual(to_bytes(ll_data["ro_uri"]), lonely_uri) d.addCallback(_check_json) return d @@ -744,8 +757,7 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi error_line = lines[first_error] error_msg = lines[first_error+1:] error_msg_s = "\n".join(error_msg) + "\n" - self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)", - error_line) + self.assertThat(error_line, Contains("ERROR: UnrecoverableFileError(no recoverable versions)")) self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback units = [json.loads(line) for line in lines[:first_error]] self.failUnlessReallyEqual(len(units), 6) # includes subdir @@ -765,8 +777,7 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi error_line = lines[first_error] error_msg = lines[first_error+1:] error_msg_s = "\n".join(error_msg) + "\n" - self.failUnlessIn("ERROR: UnrecoverableFileError(no recoverable versions)", - error_line) + self.assertThat(error_line, Contains("ERROR: UnrecoverableFileError(no recoverable versions)")) self.failUnless(len(error_msg) > 2, error_msg_s) # some traceback units = [json.loads(line) for line in lines[:first_error]] self.failUnlessReallyEqual(len(units), 6) # includes subdir @@ -936,8 +947,8 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi d.addCallback(self.CHECK, "one", "t=check") # no add-lease def _got_html_good(res): - self.failUnlessIn("Healthy", res) - self.failIfIn("Not Healthy", res) + self.assertThat(res, Contains("Healthy")) + self.assertThat(res, Not(Contains("Not Healthy"))) d.addCallback(_got_html_good) d.addCallback(self._count_leases, "one") @@ -1111,7 +1122,7 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi self.GET, self.fileurls["0shares"])) def _check_zero_shares(body): body = str(body, "utf-8") - self.failIfIn("", body) + self.assertThat(body, Not(Contains(""))) body = " ".join(body.strip().split()) exp = ("NoSharesError: no shares could be found. " "Zero shares usually indicates a corrupt URI, or that " @@ -1129,7 +1140,7 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi self.GET, self.fileurls["1share"])) def _check_one_share(body): body = str(body, "utf-8") - self.failIfIn("", body) + self.assertThat(body, Not(Contains(""))) body = " ".join(body.strip().split()) msgbase = ("NotEnoughSharesError: This indicates that some " "servers were unavailable, or that shares have been " @@ -1154,17 +1165,16 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi self.GET, self.fileurls["imaginary"])) def _missing_child(body): body = str(body, "utf-8") - self.failUnlessIn("No such child: imaginary", body) + self.assertThat(body, Contains("No such child: imaginary")) d.addCallback(_missing_child) d.addCallback(lambda ignored: self.GET_unicode(self.fileurls["dir-0share"])) def _check_0shares_dir_html(body): - self.failUnlessIn(DIR_HTML_TAG, body) + self.assertThat(body, Contains(DIR_HTML_TAG)) # we should see the regular page, but without the child table or # the dirops forms body = " ".join(body.strip().split()) - self.failUnlessIn('href="?t=info">More info on this directory', - body) + self.assertThat(body, Contains('href="?t=info">More info on this directory')) exp = ("UnrecoverableFileError: the directory (or mutable file) " "could not be retrieved, because there were insufficient " "good shares. This might indicate that no servers were " @@ -1172,8 +1182,8 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi "was corrupt, or that shares have been lost due to server " "departure, hard drive failure, or disk corruption. You " "should perform a filecheck on this object to learn more.") - self.failUnlessIn(exp, body) - self.failUnlessIn("No upload forms: directory is unreadable", body) + self.assertThat(body, Contains(exp)) + self.assertThat(body, Contains("No upload forms: directory is unreadable")) d.addCallback(_check_0shares_dir_html) d.addCallback(lambda ignored: self.GET_unicode(self.fileurls["dir-1share"])) @@ -1182,10 +1192,9 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi # and some-shares like we did for immutable files (since there # are different sorts of advice to offer in each case). For now, # they present the same way. - self.failUnlessIn(DIR_HTML_TAG, body) + self.assertThat(body, Contains(DIR_HTML_TAG)) body = " ".join(body.strip().split()) - self.failUnlessIn('href="?t=info">More info on this directory', - body) + self.assertThat(body, Contains('href="?t=info">More info on this directory')) exp = ("UnrecoverableFileError: the directory (or mutable file) " "could not be retrieved, because there were insufficient " "good shares. This might indicate that no servers were " @@ -1193,8 +1202,8 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi "was corrupt, or that shares have been lost due to server " "departure, hard drive failure, or disk corruption. You " "should perform a filecheck on this object to learn more.") - self.failUnlessIn(exp, body) - self.failUnlessIn("No upload forms: directory is unreadable", body) + self.assertThat(body, Contains(exp)) + self.assertThat(body, Contains("No upload forms: directory is unreadable")) d.addCallback(_check_1shares_dir_html) d.addCallback(lambda ignored: @@ -1204,7 +1213,7 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi self.fileurls["dir-0share-json"])) def _check_unrecoverable_file(body): body = str(body, "utf-8") - self.failIfIn("", body) + self.assertThat(body, Not(Contains(""))) body = " ".join(body.strip().split()) exp = ("UnrecoverableFileError: the directory (or mutable file) " "could not be retrieved, because there were insufficient " @@ -1213,7 +1222,7 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi "was corrupt, or that shares have been lost due to server " "departure, hard drive failure, or disk corruption. You " "should perform a filecheck on this object to learn more.") - self.failUnlessIn(exp, body) + self.assertThat(body, Contains(exp)) d.addCallback(_check_unrecoverable_file) d.addCallback(lambda ignored: @@ -1245,7 +1254,7 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi headers={"accept": "*/*"})) def _internal_error_html1(body): body = str(body, "utf-8") - self.failUnlessIn("", "expected HTML, not '%s'" % body) + self.assertThat("expected HTML, not '%s'" % body, Contains("")) d.addCallback(_internal_error_html1) d.addCallback(lambda ignored: @@ -1255,8 +1264,9 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi headers={"accept": "text/plain"})) def _internal_error_text2(body): body = str(body, "utf-8") - self.failIfIn("", body) + self.assertThat(body, Not(Contains(""))) self.failUnless(body.startswith("Traceback "), body) + d.addCallback(_internal_error_text2) CLI_accepts = "text/plain, application/octet-stream" @@ -1267,7 +1277,7 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi headers={"accept": CLI_accepts})) def _internal_error_text3(body): body = str(body, "utf-8") - self.failIfIn("", body) + self.assertThat(body, Not(Contains(""))) self.failUnless(body.startswith("Traceback "), body) d.addCallback(_internal_error_text3) @@ -1276,12 +1286,12 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi 500, "Internal Server Error", None, self.GET, "ERRORBOOM")) def _internal_error_html4(body): - self.failUnlessIn(b"", body) + self.assertThat(body, Contains(b"")) d.addCallback(_internal_error_html4) def _flush_errors(res): # Trial: please ignore the CompletelyUnhandledError in the logs - self.flushLoggedErrors(CompletelyUnhandledError) + flush_logged_errors(CompletelyUnhandledError) return res d.addBoth(_flush_errors) @@ -1312,8 +1322,8 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi d.addCallback(_stash_dir) d.addCallback(lambda ign: self.GET_unicode(self.dir_url, followRedirect=True)) def _check_dir_html(body): - self.failUnlessIn(DIR_HTML_TAG, body) - self.failUnlessIn("blacklisted.txt", body) + self.assertThat(body, Contains(DIR_HTML_TAG)) + self.assertThat(body, Contains("blacklisted.txt")) d.addCallback(_check_dir_html) d.addCallback(lambda ign: self.GET(self.url)) d.addCallback(lambda body: self.failUnlessEqual(DATA, body)) @@ -1336,8 +1346,8 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi # We should still be able to list the parent directory, in HTML... d.addCallback(lambda ign: self.GET_unicode(self.dir_url, followRedirect=True)) def _check_dir_html2(body): - self.failUnlessIn(DIR_HTML_TAG, body) - self.failUnlessIn("blacklisted.txt", body) + self.assertThat(body, Contains(DIR_HTML_TAG)) + self.assertThat(body, Contains("blacklisted.txt")) d.addCallback(_check_dir_html2) # ... and in JSON (used by CLI). @@ -1347,8 +1357,8 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi self.failUnless(isinstance(data, list), data) self.failUnlessEqual(data[0], "dirnode") self.failUnless(isinstance(data[1], dict), data) - self.failUnlessIn("children", data[1]) - self.failUnlessIn("blacklisted.txt", data[1]["children"]) + self.assertThat(data[1], Contains("children")) + self.assertThat(data[1]["children"], Contains("blacklisted.txt")) childdata = data[1]["children"]["blacklisted.txt"] self.failUnless(isinstance(childdata, list), data) self.failUnlessEqual(childdata[0], "filenode") @@ -1387,7 +1397,7 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi self.child_url = b"uri/"+dn.get_readonly_uri()+b"/child" d.addCallback(_get_dircap) d.addCallback(lambda ign: self.GET(self.dir_url_base, followRedirect=True)) - d.addCallback(lambda body: self.failUnlessIn(DIR_HTML_TAG, str(body, "utf-8"))) + d.addCallback(lambda body: self.assertThat(str(body, "utf-8"), Contains(DIR_HTML_TAG))) d.addCallback(lambda ign: self.GET(self.dir_url_json1)) d.addCallback(lambda res: json.loads(res)) # just check it decodes d.addCallback(lambda ign: self.GET(self.dir_url_json2)) From 7fa180176e5b656a789e83095875da5d481a26d2 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 26 Aug 2021 14:18:03 -0400 Subject: [PATCH 0217/2309] Pass tests on Python 2. --- src/allmydata/scripts/admin.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/allmydata/scripts/admin.py b/src/allmydata/scripts/admin.py index 86209637e..178701f84 100644 --- a/src/allmydata/scripts/admin.py +++ b/src/allmydata/scripts/admin.py @@ -10,7 +10,7 @@ from future.utils import PY2 if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 -from six import ensure_binary +from six import ensure_binary, ensure_str import json try: @@ -100,7 +100,7 @@ class AddGridManagerCertOptions(BaseOptions): "Must provide --filename option" ) if self['filename'] == '-': - print("reading certificate from stdin", file=self.parent.parent.stderr) + print(ensure_str("reading certificate from stdin"), file=self.parent.parent.stderr) data = self.parent.parent.stdin.read() if len(data) == 0: raise usage.UsageError( @@ -151,7 +151,7 @@ def add_grid_manager_cert(options): options['name'], cert_path.path, ) - print(msg, file=options.parent.parent.stderr) + print(ensure_str(msg), file=options.parent.parent.stderr) return 1 config.set_config("storage", "grid_management", "True") @@ -162,7 +162,8 @@ def add_grid_manager_cert(options): f.write(cert_bytes) cert_count = len(config.enumerate_section("grid_manager_certificates")) - print("There are now {} certificates".format(cert_count), file=options.parent.parent.stderr) + print(ensure_str("There are now {} certificates").format(cert_count), + file=options.parent.parent.stderr) return 0 From e23767db1b6921038ae8e46e60e954c0dcab23a0 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 26 Aug 2021 14:39:41 -0400 Subject: [PATCH 0218/2309] Tests pass on Python 2 and Python 3. --- src/allmydata/scripts/admin.py | 5 ++-- src/allmydata/test/cli/test_admin.py | 36 +++++++++++++++------------- 2 files changed, 22 insertions(+), 19 deletions(-) diff --git a/src/allmydata/scripts/admin.py b/src/allmydata/scripts/admin.py index 178701f84..7bce914cc 100644 --- a/src/allmydata/scripts/admin.py +++ b/src/allmydata/scripts/admin.py @@ -11,7 +11,6 @@ if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from six import ensure_binary, ensure_str -import json try: from allmydata.scripts.types_ import SubCommands @@ -28,7 +27,7 @@ from allmydata.grid_manager import ( from allmydata.scripts.cli import _default_nodedir from allmydata.scripts.common import BaseOptions from allmydata.util.encodingutil import argv_to_abspath - +from allmydata.util import jsonbytes class GenerateKeypairOptions(BaseOptions): @@ -143,7 +142,7 @@ def add_grid_manager_cert(options): config = read_config(nd, "portnum") cert_fname = "{}.cert".format(options['name']) cert_path = FilePath(config.get_config_path(cert_fname)) - cert_bytes = json.dumps(options.certificate_data, indent=4) + '\n' + cert_bytes = jsonbytes.dumps_bytes(options.certificate_data, indent=4) + b'\n' cert_name = options['name'] if cert_path.exists(): diff --git a/src/allmydata/test/cli/test_admin.py b/src/allmydata/test/cli/test_admin.py index 37f733f07..1ac047acb 100644 --- a/src/allmydata/test/cli/test_admin.py +++ b/src/allmydata/test/cli/test_admin.py @@ -1,7 +1,11 @@ -import json -from io import ( - BytesIO, -) +from future.utils import PY3 +from six import ensure_str + +# We're going to override stdin/stderr, so want to match their behavior on respective Python versions. +if PY3: + from io import StringIO +else: + from StringIO import StringIO from twisted.python.usage import ( UsageError, @@ -16,6 +20,7 @@ from allmydata.scripts.admin import ( from allmydata.scripts.runner import ( Options, ) +from allmydata.util import jsonbytes as json from ..common import ( SyncTestCase, ) @@ -31,7 +36,6 @@ class AddCertificateOptions(SyncTestCase): """ Tests for 'tahoe admin add-grid-manager-cert' option validation """ - def setUp(self): self.tahoe = Options() return super(AddCertificateOptions, self).setUp() @@ -40,8 +44,8 @@ class AddCertificateOptions(SyncTestCase): """ When no data is passed to stdin an error is produced """ - self.tahoe.stdin = BytesIO(b"") - self.tahoe.stderr = BytesIO() # suppress message + self.tahoe.stdin = StringIO(ensure_str("")) + self.tahoe.stderr = StringIO() # suppress message with self.assertRaises(UsageError) as ctx: self.tahoe.parseOptions( @@ -63,7 +67,7 @@ class AddCertificateOptions(SyncTestCase): """ tmp = self.mktemp() with open(tmp, "w") as f: - json.dump(fake_cert, f) + f.write(json.dumps(fake_cert)) # certificate should be loaded self.tahoe.parseOptions( @@ -83,8 +87,8 @@ class AddCertificateOptions(SyncTestCase): """ Unparseable data produces an error """ - self.tahoe.stdin = BytesIO(b"{}") - self.tahoe.stderr = BytesIO() # suppress message + self.tahoe.stdin = StringIO(ensure_str("{}")) + self.tahoe.stderr = StringIO() # suppress message with self.assertRaises(UsageError) as ctx: self.tahoe.parseOptions( @@ -111,15 +115,15 @@ class AddCertificateCommand(SyncTestCase): self.node_path = FilePath(self.mktemp()) self.node_path.makedirs() with self.node_path.child("tahoe.cfg").open("w") as f: - f.write("# minimal test config\n") + f.write(b"# minimal test config\n") return super(AddCertificateCommand, self).setUp() def test_add_one(self): """ Adding a certificate succeeds """ - self.tahoe.stdin = BytesIO(json.dumps(fake_cert)) - self.tahoe.stderr = BytesIO() + self.tahoe.stdin = StringIO(json.dumps(fake_cert)) + self.tahoe.stderr = StringIO() self.tahoe.parseOptions( [ "--node-directory", self.node_path.path, @@ -145,8 +149,8 @@ class AddCertificateCommand(SyncTestCase): An error message is produced when adding a certificate with a duplicate name. """ - self.tahoe.stdin = BytesIO(json.dumps(fake_cert)) - self.tahoe.stderr = BytesIO() + self.tahoe.stdin = StringIO(json.dumps(fake_cert)) + self.tahoe.stderr = StringIO() self.tahoe.parseOptions( [ "--node-directory", self.node_path.path, @@ -158,7 +162,7 @@ class AddCertificateCommand(SyncTestCase): rc = add_grid_manager_cert(self.tahoe.subOptions.subOptions) self.assertEqual(rc, 0) - self.tahoe.stdin = BytesIO(json.dumps(fake_cert)) + self.tahoe.stdin = StringIO(json.dumps(fake_cert)) self.tahoe.parseOptions( [ "--node-directory", self.node_path.path, From 969f3fa9b3f87bd789a0da46dae8c0a8fbe1d4a8 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 26 Aug 2021 14:46:28 -0400 Subject: [PATCH 0219/2309] Port to Python 3. --- src/allmydata/test/cli/test_admin.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/src/allmydata/test/cli/test_admin.py b/src/allmydata/test/cli/test_admin.py index 1ac047acb..fe7aa21a5 100644 --- a/src/allmydata/test/cli/test_admin.py +++ b/src/allmydata/test/cli/test_admin.py @@ -1,4 +1,15 @@ -from future.utils import PY3 +""" +Ported to Python 3. +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2, PY3 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from six import ensure_str # We're going to override stdin/stderr, so want to match their behavior on respective Python versions. @@ -66,8 +77,8 @@ class AddCertificateOptions(SyncTestCase): A certificate can be read from a file """ tmp = self.mktemp() - with open(tmp, "w") as f: - f.write(json.dumps(fake_cert)) + with open(tmp, "wb") as f: + f.write(json.dumps_bytes(fake_cert)) # certificate should be loaded self.tahoe.parseOptions( From 6df4fa315ecea6d8f06b991ba49ddf07a8721f42 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 26 Aug 2021 14:55:41 -0400 Subject: [PATCH 0220/2309] Add explicit dependency on Click. --- setup.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/setup.py b/setup.py index 991f79e93..f79f1feed 100644 --- a/setup.py +++ b/setup.py @@ -137,6 +137,9 @@ install_requires = [ # Backported configparser for Python 2: "configparser ; python_version < '3.0'", + + # Command-line parsing + "click >= 7.0", ] setup_requires = [ From c7f0a099e1533f4b41563bf05d22cea3277fa21c Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 26 Aug 2021 15:44:54 -0400 Subject: [PATCH 0221/2309] More compatibility with stdlib json module. --- src/allmydata/util/jsonbytes.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/allmydata/util/jsonbytes.py b/src/allmydata/util/jsonbytes.py index 08e0cb68e..8ddc4c453 100644 --- a/src/allmydata/util/jsonbytes.py +++ b/src/allmydata/util/jsonbytes.py @@ -116,6 +116,7 @@ def dumps_bytes(obj, *args, **kwargs): # To make this module drop-in compatible with json module: loads = json.loads +load = json.load __all__ = ["dumps", "loads"] From c3f6184960c88df8e54e0d5d9b45eac3c482600a Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 26 Aug 2021 16:55:03 -0400 Subject: [PATCH 0222/2309] Match documented behavior. --- src/allmydata/test/strategies.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/strategies.py b/src/allmydata/test/strategies.py index fea71799a..61d6d6e78 100644 --- a/src/allmydata/test/strategies.py +++ b/src/allmydata/test/strategies.py @@ -131,6 +131,6 @@ def base32text(): Build text()s that are valid base32 """ return builds( - b2a, + lambda b: str(b2a(b), "ascii"), binary(), ) From c88130d8a819d0600538182d9e1c6103d656d403 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 26 Aug 2021 16:55:12 -0400 Subject: [PATCH 0223/2309] Make signed certificates an object instead of a dict. --- src/allmydata/cli/grid_manager.py | 4 +- src/allmydata/grid_manager.py | 51 ++++++++++++++++--------- src/allmydata/test/test_grid_manager.py | 12 +++--- 3 files changed, 42 insertions(+), 25 deletions(-) diff --git a/src/allmydata/cli/grid_manager.py b/src/allmydata/cli/grid_manager.py index acf233b29..eb09e787a 100644 --- a/src/allmydata/cli/grid_manager.py +++ b/src/allmydata/cli/grid_manager.py @@ -1,3 +1,5 @@ +from __future__ import print_function + from datetime import ( datetime, timedelta, @@ -188,7 +190,7 @@ def sign(ctx, name, expiry_days): "No storage-server called '{}' exists".format(name) ) - certificate_data = json.dumps(certificate, indent=4) + certificate_data = json.dumps(certificate.asdict(), indent=4) click.echo(certificate_data) if fp is not None: next_serial = 0 diff --git a/src/allmydata/grid_manager.py b/src/allmydata/grid_manager.py index c288a34bb..a5a120a67 100644 --- a/src/allmydata/grid_manager.py +++ b/src/allmydata/grid_manager.py @@ -2,6 +2,9 @@ Functions and classes relating to the Grid Manager internal state """ +from future.utils import PY2, PY3 +from past.builtins import unicode + import sys import json from datetime import ( @@ -18,6 +21,24 @@ from allmydata.util import ( import attr +@attr.s +class SignedCertificate(object): + """ + A signed certificate. + """ + # A JSON-encoded certificate. + certificate = attr.ib(type=unicode) + # The signature in base32. + signature = attr.ib(type=unicode) + + @classmethod + def load(cls, file_like): + return cls(**json.load(file_like)) + + def asdict(self): + return attr.asdict(self) + + @attr.s class _GridManagerStorageServer(object): """ @@ -97,10 +118,10 @@ def _load_certificates_for(config_path, name, gm_key=None): cert_path = config_path.child('{}.cert.{}'.format(name, cert_index)) certificates = [] while cert_path.exists(): - container = json.load(cert_path.open('r')) + container = SignedCertificate.load(cert_path.open('r')) if gm_key is not None: validate_grid_manager_certificate(gm_key, container) - cert_data = json.loads(container['certificate']) + cert_data = json.loads(container.certificate) if cert_data['version'] != 1: raise ValueError( "Unknown certificate version '{}' in '{}'".format( @@ -207,8 +228,7 @@ class _GridManager(object): :param timedelta expiry: how far in the future the certificate should expire. - :returns: a dict defining the certificate (it has - "certificate" and "signature" keys). + :returns SignedCertificate: the signed certificate. """ try: srv = self._storage_servers[name] @@ -225,11 +245,10 @@ class _GridManager(object): } cert_data = json.dumps(cert_info, separators=(',',':'), sort_keys=True).encode('utf8') sig = ed25519.sign_data(self._private_key, cert_data) - certificate = { - u"certificate": cert_data, - u"signature": base32.b2a(sig), - } - + certificate = SignedCertificate( + certificate=cert_data, + signature=base32.b2a(sig), + ) vk = ed25519.verifying_key_from_signing_key(self._private_key) ed25519.verify_signature(vk, sig, cert_data) @@ -343,9 +362,7 @@ def validate_grid_manager_certificate(gm_key, alleged_cert): :param gm_key: a VerifyingKey instance, a Grid Manager's public key. - :param alleged_cert: dict with "certificate" and "signature" keys, where - "certificate" contains a JSON-serialized certificate for a Storage - Server (comes from a Grid Manager). + :param alleged_cert SignedCertificate: A signed certificate. :return: a dict consisting of the deserialized certificate data or None if the signature is invalid. Note we do NOT check the @@ -354,13 +371,13 @@ def validate_grid_manager_certificate(gm_key, alleged_cert): try: ed25519.verify_signature( gm_key, - base32.a2b(alleged_cert['signature'].encode('ascii')), - alleged_cert['certificate'].encode('ascii'), + base32.a2b(alleged_cert.signature.encode('ascii')), + alleged_cert.certificate.encode('ascii'), ) except ed25519.BadSignature: return None # signature is valid; now we can load the actual data - cert = json.loads(alleged_cert['certificate']) + cert = json.loads(alleged_cert.certificate) return cert @@ -371,10 +388,10 @@ def create_grid_manager_verifier(keys, certs, public_key, now_fn=None, bad_cert= (instead of just returning True/False here) so that the expiry-time can be tested on each call. - :param list keys: 0 or more `VerifyingKey` instances + :param list keys: 0 or more ``VerifyingKey`` instances :param list certs: 1 or more Grid Manager certificates each of - which is a `dict` containing 'signature' and 'certificate' keys. + which is a ``SignedCertificate``. :param str public_key: the identifier of the server we expect certificates for. diff --git a/src/allmydata/test/test_grid_manager.py b/src/allmydata/test/test_grid_manager.py index e53984a05..539ff5c2e 100644 --- a/src/allmydata/test/test_grid_manager.py +++ b/src/allmydata/test/test_grid_manager.py @@ -27,6 +27,7 @@ from allmydata.grid_manager import ( create_grid_manager, parse_grid_manager_certificate, create_grid_manager_verifier, + SignedCertificate, ) from allmydata.test.strategies import ( base32text, @@ -165,16 +166,13 @@ class GridManagerVerifier(SyncTestCase): cert1 = self.gm.sign("test", timedelta(seconds=3600)) self.assertNotEqual(cert0, cert1) - self.assertEqual( - set(cert0.keys()), - {"certificate", "signature"}, - ) + self.assertIsInstance(cert0, SignedCertificate) gm_key = ed25519.verifying_key_from_string(self.gm.public_identity()) self.assertEqual( ed25519.verify_signature( gm_key, - base32.a2b(cert0["signature"]), - cert0["certificate"], + base32.a2b(cert0.signature.encode("ascii")), + cert0.certificate.encode("ascii"), ), None ) @@ -432,7 +430,7 @@ class GridManagerInvalidVerifier(SyncTestCase): An incorrect signature is rejected """ # make signature invalid - self.cert0["signature"] = invalid_signature + self.cert0.signature = invalid_signature verify = create_grid_manager_verifier( [self.gm._public_key], From e3a844e68444ca661342937b10c16fd31ed23ae2 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 27 Aug 2021 09:25:49 -0400 Subject: [PATCH 0224/2309] Maybe fields are better off as bytes in SignedCertificate. --- src/allmydata/grid_manager.py | 25 +++++++++++++++++-------- src/allmydata/test/test_grid_manager.py | 4 +++- 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/src/allmydata/grid_manager.py b/src/allmydata/grid_manager.py index a5a120a67..e99dd1863 100644 --- a/src/allmydata/grid_manager.py +++ b/src/allmydata/grid_manager.py @@ -6,7 +6,6 @@ from future.utils import PY2, PY3 from past.builtins import unicode import sys -import json from datetime import ( datetime, ) @@ -16,6 +15,7 @@ from allmydata.crypto import ( ) from allmydata.util import ( base32, + jsonbytes as json, ) import attr @@ -26,14 +26,23 @@ class SignedCertificate(object): """ A signed certificate. """ - # A JSON-encoded certificate. - certificate = attr.ib(type=unicode) + # A JSON-encoded, UTF-8-encoded certificate. + certificate = attr.ib( + type=bytes, validator=attr.validators.instance_of(bytes) + ) # The signature in base32. - signature = attr.ib(type=unicode) + signature = attr.ib( + type=bytes, + validator=attr.validators.instance_of(bytes) + ) @classmethod def load(cls, file_like): - return cls(**json.load(file_like)) + data = json.load(file_like) + return cls( + certificate=data["certificate"].encode("ascii"), + signature=data["signature"].encode("ascii") + ) def asdict(self): return attr.asdict(self) @@ -243,7 +252,7 @@ class _GridManager(object): "public_key": srv.public_key_string(), "version": 1, } - cert_data = json.dumps(cert_info, separators=(',',':'), sort_keys=True).encode('utf8') + cert_data = json.dumps_bytes(cert_info, separators=(',',':'), sort_keys=True) sig = ed25519.sign_data(self._private_key, cert_data) certificate = SignedCertificate( certificate=cert_data, @@ -371,8 +380,8 @@ def validate_grid_manager_certificate(gm_key, alleged_cert): try: ed25519.verify_signature( gm_key, - base32.a2b(alleged_cert.signature.encode('ascii')), - alleged_cert.certificate.encode('ascii'), + base32.a2b(alleged_cert.signature), + alleged_cert.certificate, ) except ed25519.BadSignature: return None diff --git a/src/allmydata/test/test_grid_manager.py b/src/allmydata/test/test_grid_manager.py index 539ff5c2e..5276df4f0 100644 --- a/src/allmydata/test/test_grid_manager.py +++ b/src/allmydata/test/test_grid_manager.py @@ -1,3 +1,5 @@ +from past.builtins import unicode + from datetime import ( timedelta, ) @@ -430,7 +432,7 @@ class GridManagerInvalidVerifier(SyncTestCase): An incorrect signature is rejected """ # make signature invalid - self.cert0.signature = invalid_signature + self.cert0.signature = invalid_signature.encode("ascii") verify = create_grid_manager_verifier( [self.gm._public_key], From f99f9cf7d1f1928d8d67adb32339565ffaf82196 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 27 Aug 2021 09:47:42 -0400 Subject: [PATCH 0225/2309] Tests pass on Python 3. --- src/allmydata/grid_manager.py | 5 +++-- src/allmydata/node.py | 2 +- src/allmydata/test/test_grid_manager.py | 18 +++++++++--------- src/allmydata/test/test_util.py | 1 - 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/allmydata/grid_manager.py b/src/allmydata/grid_manager.py index e99dd1863..018fcd461 100644 --- a/src/allmydata/grid_manager.py +++ b/src/allmydata/grid_manager.py @@ -334,7 +334,8 @@ def save_grid_manager(file_path, grid_manager, create=True): if create: raise with file_path.child("config.json").open("w") as f: - f.write("{}\n".format(data)) + f.write(data.encode("utf-8")) + f.write(b"\n") def parse_grid_manager_certificate(gm_data): @@ -465,7 +466,7 @@ def create_grid_manager_verifier(keys, certs, public_key, now_fn=None, bad_cert= now = now_fn() for cert in valid_certs: expires = datetime.utcfromtimestamp(cert['expires']) - if cert['public_key'] == public_key: + if cert['public_key'].encode("ascii") == public_key: if expires > now: # not-expired return True diff --git a/src/allmydata/node.py b/src/allmydata/node.py index 0feaffb35..e6f3820d0 100644 --- a/src/allmydata/node.py +++ b/src/allmydata/node.py @@ -534,7 +534,7 @@ class _Config(object): cert_fnames = list(self.enumerate_section("grid_manager_certificates").values()) for fname in cert_fnames: - fname = self.get_config_path(fname.decode('utf8')) + fname = self.get_config_path(fname) if not os.path.exists(fname): raise ValueError( "Grid Manager certificate file '{}' doesn't exist".format( diff --git a/src/allmydata/test/test_grid_manager.py b/src/allmydata/test/test_grid_manager.py index 5276df4f0..3ee4f8a88 100644 --- a/src/allmydata/test/test_grid_manager.py +++ b/src/allmydata/test/test_grid_manager.py @@ -3,7 +3,6 @@ from past.builtins import unicode from datetime import ( timedelta, ) -import json from twisted.python.filepath import ( FilePath, @@ -22,6 +21,7 @@ from allmydata.crypto import ( ) from allmydata.util import ( base32, + jsonbytes as json, ) from allmydata.grid_manager import ( load_grid_manager, @@ -173,8 +173,8 @@ class GridManagerVerifier(SyncTestCase): self.assertEqual( ed25519.verify_signature( gm_key, - base32.a2b(cert0.signature.encode("ascii")), - cert0.certificate.encode("ascii"), + base32.a2b(cert0.signature), + cert0.certificate, ), None ) @@ -251,7 +251,7 @@ class GridManagerVerifier(SyncTestCase): } fp.makedirs() with fp.child("config.json").open("w") as f: - json.dump(bad_config, f) + f.write(json.dumps_bytes(bad_config)) with self.assertRaises(ValueError) as ctx: load_grid_manager(fp) @@ -283,9 +283,9 @@ class GridManagerVerifier(SyncTestCase): fp.makedirs() with fp.child("config.json").open("w") as f: - json.dump(config, f) + f.write(json.dumps_bytes(config)) with fp.child("alice.cert.0").open("w") as f: - json.dump(bad_cert, f) + f.write(json.dumps_bytes(bad_cert)) with self.assertRaises(ValueError) as ctx: load_grid_manager(fp) @@ -306,7 +306,7 @@ class GridManagerVerifier(SyncTestCase): } fp.makedirs() with fp.child("config.json").open("w") as f: - json.dump(bad_config, f) + f.write(json.dumps_bytes(bad_config)) with self.assertRaises(ValueError) as ctx: load_grid_manager(fp) @@ -327,7 +327,7 @@ class GridManagerVerifier(SyncTestCase): } fp.makedirs() with fp.child("config.json").open("w") as f: - json.dump(bad_config, f) + f.write(json.dumps_bytes(bad_config)) with self.assertRaises(ValueError) as ctx: load_grid_manager(fp) @@ -352,7 +352,7 @@ class GridManagerVerifier(SyncTestCase): } fp.makedirs() with fp.child("config.json").open("w") as f: - json.dump(bad_config, f) + f.write(json.dumps_bytes(bad_config)) with self.assertRaises(ValueError) as ctx: load_grid_manager(fp) diff --git a/src/allmydata/test/test_util.py b/src/allmydata/test/test_util.py index a03845ed6..251a60d70 100644 --- a/src/allmydata/test/test_util.py +++ b/src/allmydata/test/test_util.py @@ -559,7 +559,6 @@ class JSONBytes(unittest.TestCase): ) - class FakeGetVersion(object): """Emulate an object with a get_version.""" From bb48974fd8645906431182f8fda21db0269fd88f Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 27 Aug 2021 09:50:03 -0400 Subject: [PATCH 0226/2309] Ported to Python 3. --- src/allmydata/test/test_grid_manager.py | 26 +++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/src/allmydata/test/test_grid_manager.py b/src/allmydata/test/test_grid_manager.py index 3ee4f8a88..ded73e1e5 100644 --- a/src/allmydata/test/test_grid_manager.py +++ b/src/allmydata/test/test_grid_manager.py @@ -1,4 +1,14 @@ -from past.builtins import unicode +""" +Ported to Python 3. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from datetime import ( timedelta, @@ -52,8 +62,8 @@ class GridManagerUtilities(SyncTestCase): "certificate": "{\"expires\":1601687822,\"public_key\":\"pub-v0-cbq6hcf3pxcz6ouoafrbktmkixkeuywpcpbcomzd3lqbkq4nmfga\",\"version\":1}", "signature": "fvjd3uvvupf2v6tnvkwjd473u3m3inyqkwiclhp7balmchkmn3px5pei3qyfjnhymq4cjcwvbpqmcwwnwswdtrfkpnlaxuih2zbdmda" } - with open(cert_path, "w") as f: - f.write(json.dumps(fake_cert)) + with open(cert_path, "wb") as f: + f.write(json.dumps_bytes(fake_cert)) config_data = ( "[grid_managers]\n" "fluffy = pub-v0-vqimc4s5eflwajttsofisp5st566dbq36xnpp4siz57ufdavpvlq\n" @@ -77,8 +87,8 @@ class GridManagerUtilities(SyncTestCase): "certificate": "{\"expires\":1601687822,\"public_key\":\"pub-v0-cbq6hcf3pxcz6ouoafrbktmkixkeuywpcpbcomzd3lqbkq4nmfga\",\"version\":22}", "signature": "fvjd3uvvupf2v6tnvkwjd473u3m3inyqkwiclhp7balmchkmn3px5pei3qyfjnhymq4cjcwvbpqmcwwnwswdtrfkpnlaxuih2zbdmda" } - with open(cert_path, "w") as f: - f.write(json.dumps(fake_cert)) + with open(cert_path, "wb") as f: + f.write(json.dumps_bytes(fake_cert)) config_data = ( "[grid_managers]\n" "fluffy = pub-v0-vqimc4s5eflwajttsofisp5st566dbq36xnpp4siz57ufdavpvlq\n" @@ -103,8 +113,8 @@ class GridManagerUtilities(SyncTestCase): "signature": "fvjd3uvvupf2v6tnvkwjd473u3m3inyqkwiclhp7balmchkmn3px5pei3qyfjnhymq4cjcwvbpqmcwwnwswdtrfkpnlaxuih2zbdmda", "something-else": "not valid in a v0 certificate" } - with open(cert_path, "w") as f: - f.write(json.dumps(fake_cert)) + with open(cert_path, "wb") as f: + f.write(json.dumps_bytes(fake_cert)) config_data = ( "[grid_manager_certificates]\n" "ding = {}\n".format(cert_path) @@ -234,7 +244,7 @@ class GridManagerVerifier(SyncTestCase): len(self.gm.storage_servers), len(gm2.storage_servers), ) - for name, ss0 in self.gm.storage_servers.items(): + for name, ss0 in list(self.gm.storage_servers.items()): ss1 = gm2.storage_servers[name] self.assertEqual(ss0.name, ss1.name) self.assertEqual(ss0.public_key_string(), ss1.public_key_string()) From d5b48e65c7dd1f8d6943c85602aed1c8114f9a42 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 27 Aug 2021 10:14:39 -0400 Subject: [PATCH 0227/2309] Fix bug in jsonbytes. --- src/allmydata/test/test_util.py | 6 ++++++ src/allmydata/util/jsonbytes.py | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/allmydata/test/test_util.py b/src/allmydata/test/test_util.py index 251a60d70..50532274a 100644 --- a/src/allmydata/test/test_util.py +++ b/src/allmydata/test/test_util.py @@ -558,6 +558,12 @@ class JSONBytes(unittest.TestCase): expected ) + def test_dumps_bytes_unicode_separators(self): + """Unicode separators don't prevent the result from being bytes.""" + result = jsonbytes.dumps_bytes([1, 2], separators=(u',', u':')) + self.assertIsInstance(result, bytes) + self.assertEqual(result, b"[1,2]") + class FakeGetVersion(object): """Emulate an object with a get_version.""" diff --git a/src/allmydata/util/jsonbytes.py b/src/allmydata/util/jsonbytes.py index 8ddc4c453..159ae7990 100644 --- a/src/allmydata/util/jsonbytes.py +++ b/src/allmydata/util/jsonbytes.py @@ -109,7 +109,7 @@ def dumps_bytes(obj, *args, **kwargs): human consumption. """ result = dumps(obj, *args, **kwargs) - if PY3: + if PY3 or isinstance(result, str): result = result.encode("utf-8") return result From 4c750cf88dd73c4d90c0f5a503d24792a6aa9498 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 27 Aug 2021 10:15:26 -0400 Subject: [PATCH 0228/2309] Port to Python 3. --- src/allmydata/grid_manager.py | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/src/allmydata/grid_manager.py b/src/allmydata/grid_manager.py index 018fcd461..e6fe29326 100644 --- a/src/allmydata/grid_manager.py +++ b/src/allmydata/grid_manager.py @@ -1,9 +1,16 @@ """ Functions and classes relating to the Grid Manager internal state -""" -from future.utils import PY2, PY3 -from past.builtins import unicode +Ported to Python 3. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import sys from datetime import ( @@ -16,6 +23,7 @@ from allmydata.crypto import ( from allmydata.util import ( base32, jsonbytes as json, + dictutil, ) import attr @@ -193,7 +201,7 @@ def load_grid_manager(config_path): ) storage_servers = dict() - for name, srv_config in config.get(u'storage_servers', {}).items(): + for name, srv_config in list(config.get(u'storage_servers', {}).items()): if 'public_key' not in srv_config: raise ValueError( "No 'public_key' for storage server '{}'".format(name) @@ -213,7 +221,10 @@ class _GridManager(object): """ def __init__(self, private_key_bytes, storage_servers): - self._storage_servers = dict() if storage_servers is None else storage_servers + self._storage_servers = dictutil.UnicodeKeyDict( + {} if storage_servers is None else storage_servers + ) + assert isinstance(private_key_bytes, bytes) self._private_key_bytes = private_key_bytes self._private_key, self._public_key = ed25519.signing_keypair_from_string(self._private_key_bytes) self._version = 0 @@ -239,6 +250,7 @@ class _GridManager(object): :returns SignedCertificate: the signed certificate. """ + assert isinstance(name, str) # must be unicode try: srv = self._storage_servers[name] except KeyError: From c0d07e1894d91fd1e37d5ff5f6d477c2c1290184 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 27 Aug 2021 11:12:34 -0400 Subject: [PATCH 0229/2309] Bit more input validation. --- src/allmydata/grid_manager.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/allmydata/grid_manager.py b/src/allmydata/grid_manager.py index e6fe29326..8589e87c0 100644 --- a/src/allmydata/grid_manager.py +++ b/src/allmydata/grid_manager.py @@ -283,6 +283,7 @@ class _GridManager(object): storage provider (e.g. from the contents of node.pubkey for the client) """ + assert isinstance(name, str) # must be unicode if name in self._storage_servers: raise KeyError( "Already have a storage server called '{}'".format(name) @@ -295,6 +296,7 @@ class _GridManager(object): """ :param name: a user-meaningful name for the server """ + assert isinstance(name, str) # must be unicode try: del self._storage_servers[name] except KeyError: From 1acc80b56312f3533d098f76b923f18e27616ece Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 27 Aug 2021 11:13:03 -0400 Subject: [PATCH 0230/2309] Validate commands actually succeeded! --- src/allmydata/test/cli/test_grid_manager.py | 54 ++++++++++++--------- 1 file changed, 30 insertions(+), 24 deletions(-) diff --git a/src/allmydata/test/cli/test_grid_manager.py b/src/allmydata/test/cli/test_grid_manager.py index e63e6aaaa..8d2acc542 100644 --- a/src/allmydata/test/cli/test_grid_manager.py +++ b/src/allmydata/test/cli/test_grid_manager.py @@ -41,15 +41,21 @@ class GridManagerCommandLine(TestCase): self.runner = click.testing.CliRunner() super(GridManagerCommandLine, self).setUp() + def invoke_and_check(self, *args, **kwargs): + """Invoke a command with the runner and ensure it succeeded.""" + result = self.runner.invoke(*args, **kwargs) + self.assertEqual(result.exit_code, 0, result) + return result + def test_create(self): """ Create a new grid-manager """ with self.runner.isolated_filesystem(): - result = self.runner.invoke(grid_manager, ["--config", "foo", "create"]) + result = self.invoke_and_check(grid_manager, ["--config", "foo", "create"]) self.assertEqual(["foo"], os.listdir(".")) self.assertEqual(["config.json"], os.listdir("./foo")) - result = self.runner.invoke(grid_manager, ["--config", "foo", "public-identity"]) + result = self.invoke_and_check(grid_manager, ["--config", "foo", "public-identity"]) self.assertTrue(result.output.startswith("pub-v0-")) def test_load_invalid(self): @@ -72,7 +78,7 @@ class GridManagerCommandLine(TestCase): directory. """ with self.runner.isolated_filesystem(): - result = self.runner.invoke(grid_manager, ["--config", "foo", "create"]) + result = self.invoke_and_check(grid_manager, ["--config", "foo", "create"]) result = self.runner.invoke(grid_manager, ["--config", "foo", "create"]) self.assertEqual(1, result.exit_code) self.assertIn( @@ -85,7 +91,7 @@ class GridManagerCommandLine(TestCase): Create a new grid-manager with no files """ with self.runner.isolated_filesystem(): - result = self.runner.invoke(grid_manager, ["--config", "-", "create"]) + result = self.invoke_and_check(grid_manager, ["--config", "-", "create"]) self.assertEqual([], os.listdir(".")) config = json.loads(result.output) self.assertEqual( @@ -106,7 +112,7 @@ class GridManagerCommandLine(TestCase): "private_key": "priv-v0-6uinzyaxy3zvscwgsps5pxcfezhrkfb43kvnrbrhhfzyduyqnniq", "grid_manager_config_version": 0 } - result = self.runner.invoke( + result = self.invoke_and_check( grid_manager, ["--config", "-", "list"], input=BytesIO(json.dumps(config)), ) @@ -122,9 +128,9 @@ class GridManagerCommandLine(TestCase): """ pubkey = "pub-v0-cbq6hcf3pxcz6ouoafrbktmkixkeuywpcpbcomzd3lqbkq4nmfga" with self.runner.isolated_filesystem(): - self.runner.invoke(grid_manager, ["--config", "foo", "create"]) - self.runner.invoke(grid_manager, ["--config", "foo", "add", "storage0", pubkey]) - result = self.runner.invoke(grid_manager, ["--config", "foo", "sign", "storage0", "10"]) + self.invoke_and_check(grid_manager, ["--config", "foo", "create"]) + self.invoke_and_check(grid_manager, ["--config", "foo", "add", "storage0", pubkey]) + result = self.invoke_and_check(grid_manager, ["--config", "foo", "sign", "storage0", "10"]) sigcert = json.loads(result.output) self.assertEqual({"certificate", "signature"}, set(sigcert.keys())) cert = json.loads(sigcert['certificate']) @@ -136,10 +142,10 @@ class GridManagerCommandLine(TestCase): """ pubkey = "pub-v0-cbq6hcf3pxcz6ouoafrbktmkixkeuywpcpbcomzd3lqbkq4nmfga" with self.runner.isolated_filesystem(): - self.runner.invoke(grid_manager, ["--config", "foo", "create"]) - self.runner.invoke(grid_manager, ["--config", "foo", "add", "storage0", pubkey]) - self.runner.invoke(grid_manager, ["--config", "foo", "sign", "storage0", "10"]) - self.runner.invoke(grid_manager, ["--config", "foo", "sign", "storage0", "10"]) + self.invoke_and_check(grid_manager, ["--config", "foo", "create"]) + self.invoke_and_check(grid_manager, ["--config", "foo", "add", "storage0", pubkey]) + self.invoke_and_check(grid_manager, ["--config", "foo", "sign", "storage0", "10"]) + self.invoke_and_check(grid_manager, ["--config", "foo", "sign", "storage0", "10"]) # we should now have two certificates stored self.assertEqual( set(FilePath("foo").listdir()), @@ -153,8 +159,8 @@ class GridManagerCommandLine(TestCase): pubkey0 = "pub-v0-cbq6hcf3pxcz6ouoafrbktmkixkeuywpcpbcomzd3lqbkq4nmfga" pubkey1 = "pub-v0-5ysc55trfvfvg466v46j4zmfyltgus3y2gdejifctv7h4zkuyveq" with self.runner.isolated_filesystem(): - self.runner.invoke(grid_manager, ["--config", "foo", "create"]) - self.runner.invoke(grid_manager, ["--config", "foo", "add", "storage0", pubkey0]) + self.invoke_and_check(grid_manager, ["--config", "foo", "create"]) + self.invoke_and_check(grid_manager, ["--config", "foo", "add", "storage0", pubkey0]) result = self.runner.invoke(grid_manager, ["--config", "foo", "add", "storage0", pubkey1]) self.assertNotEquals(result.exit_code, 0) self.assertIn( @@ -168,11 +174,11 @@ class GridManagerCommandLine(TestCase): """ pubkey = "pub-v0-cbq6hcf3pxcz6ouoafrbktmkixkeuywpcpbcomzd3lqbkq4nmfga" with self.runner.isolated_filesystem(): - self.runner.invoke(grid_manager, ["--config", "foo", "create"]) - self.runner.invoke(grid_manager, ["--config", "foo", "add", "storage0", pubkey]) - self.runner.invoke(grid_manager, ["--config", "foo", "sign", "storage0", "1"]) + self.invoke_and_check(grid_manager, ["--config", "foo", "create"]) + self.invoke_and_check(grid_manager, ["--config", "foo", "add", "storage0", pubkey]) + self.invoke_and_check(grid_manager, ["--config", "foo", "sign", "storage0", "1"]) - result = self.runner.invoke(grid_manager, ["--config", "foo", "list"]) + result = self.invoke_and_check(grid_manager, ["--config", "foo", "list"]) names = [ line.split(':')[0] for line in result.output.strip().split('\n') @@ -180,9 +186,9 @@ class GridManagerCommandLine(TestCase): ] self.assertEqual(names, ["storage0"]) - self.runner.invoke(grid_manager, ["--config", "foo", "remove", "storage0"]) + self.invoke_and_check(grid_manager, ["--config", "foo", "remove", "storage0"]) - result = self.runner.invoke(grid_manager, ["--config", "foo", "list"]) + result = self.invoke_and_check(grid_manager, ["--config", "foo", "list"]) self.assertEqual(result.output.strip(), "") def test_remove_missing(self): @@ -190,7 +196,7 @@ class GridManagerCommandLine(TestCase): Error reported when removing non-existant server """ with self.runner.isolated_filesystem(): - self.runner.invoke(grid_manager, ["--config", "foo", "create"]) + self.invoke_and_check(grid_manager, ["--config", "foo", "create"]) result = self.runner.invoke(grid_manager, ["--config", "foo", "remove", "storage0"]) self.assertNotEquals(result.exit_code, 0) self.assertIn( @@ -203,7 +209,7 @@ class GridManagerCommandLine(TestCase): Error reported when signing non-existant server """ with self.runner.isolated_filesystem(): - self.runner.invoke(grid_manager, ["--config", "foo", "create"]) + self.invoke_and_check(grid_manager, ["--config", "foo", "create"]) result = self.runner.invoke(grid_manager, ["--config", "foo", "sign", "storage0", "42"]) self.assertNotEquals(result.exit_code, 0) self.assertIn( @@ -218,8 +224,8 @@ class GridManagerCommandLine(TestCase): """ pubkey = "pub-v0-cbq6hcf3pxcz6ouoafrbktmkixkeuywpcpbcomzd3lqbkq4nmfga" with self.runner.isolated_filesystem(): - self.runner.invoke(grid_manager, ["--config", "foo", "create"]) - self.runner.invoke(grid_manager, ["--config", "foo", "add", "storage0", pubkey]) + self.invoke_and_check(grid_manager, ["--config", "foo", "create"]) + self.invoke_and_check(grid_manager, ["--config", "foo", "add", "storage0", pubkey]) # make the directory un-writable (so we can't create a new cert) os.chmod("foo", 0o550) result = self.runner.invoke(grid_manager, ["--config", "foo", "sign", "storage0", "42"]) From aacdf9906daa47e072ac1fbcc18933442a4bc64f Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 27 Aug 2021 11:21:59 -0400 Subject: [PATCH 0231/2309] Make failures easier to debug. --- src/allmydata/cli/grid_manager.py | 1 + src/allmydata/test/cli/test_grid_manager.py | 3 +++ 2 files changed, 4 insertions(+) diff --git a/src/allmydata/cli/grid_manager.py b/src/allmydata/cli/grid_manager.py index eb09e787a..44a524db9 100644 --- a/src/allmydata/cli/grid_manager.py +++ b/src/allmydata/cli/grid_manager.py @@ -111,6 +111,7 @@ def add(ctx, name, public_key): PUBLIC_KEY is the contents of a node.pubkey file from a Tahoe node-directory. NAME is an arbitrary label. """ + public_key = public_key.encode("ascii") try: ctx.obj.grid_manager.add_storage_server( name, diff --git a/src/allmydata/test/cli/test_grid_manager.py b/src/allmydata/test/cli/test_grid_manager.py index 8d2acc542..f105dabeb 100644 --- a/src/allmydata/test/cli/test_grid_manager.py +++ b/src/allmydata/test/cli/test_grid_manager.py @@ -1,3 +1,4 @@ +from future.utils import raise_ import os import json @@ -44,6 +45,8 @@ class GridManagerCommandLine(TestCase): def invoke_and_check(self, *args, **kwargs): """Invoke a command with the runner and ensure it succeeded.""" result = self.runner.invoke(*args, **kwargs) + if result.exception is not None: + raise_(*result.exc_info) self.assertEqual(result.exit_code, 0, result) return result From 6bfcda2a3d82898d504db29f60bdd8aabc922029 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 27 Aug 2021 11:26:51 -0400 Subject: [PATCH 0232/2309] Tests pass on Python 3. --- src/allmydata/cli/grid_manager.py | 11 +++++++---- src/allmydata/test/cli/test_grid_manager.py | 16 ++++++++-------- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/src/allmydata/cli/grid_manager.py b/src/allmydata/cli/grid_manager.py index 44a524db9..78b037649 100644 --- a/src/allmydata/cli/grid_manager.py +++ b/src/allmydata/cli/grid_manager.py @@ -1,10 +1,10 @@ from __future__ import print_function +from past.builtins import unicode from datetime import ( datetime, timedelta, ) -import json import click @@ -23,6 +23,7 @@ from allmydata.grid_manager import ( save_grid_manager, load_grid_manager, ) +from allmydata.util import jsonbytes as json @click.group() @@ -102,7 +103,7 @@ def public_identity(config): @grid_manager.command() @click.argument("name") -@click.argument("public_key", type=click.UNPROCESSED) +@click.argument("public_key", type=click.STRING) @click.pass_context def add(ctx, name, public_key): """ @@ -160,7 +161,9 @@ def list(ctx): """ for name in sorted(ctx.obj.grid_manager.storage_servers.keys()): blank_name = " " * len(name) - click.echo("{}: {}".format(name, ctx.obj.grid_manager.storage_servers[name].public_key_string())) + click.echo("{}: {}".format( + name, + unicode(ctx.obj.grid_manager.storage_servers[name].public_key_string(), "utf-8"))) for cert in ctx.obj.grid_manager.storage_servers[name].certificates: delta = datetime.utcnow() - cert.expires click.echo("{} cert {}: ".format(blank_name, cert.index), nl=False) @@ -209,7 +212,7 @@ def sign(ctx, name, expiry_days): ) next_serial += 1 with f: - f.write(certificate_data) + f.write(certificate_data.encode("ascii")) def _config_path_from_option(config): diff --git a/src/allmydata/test/cli/test_grid_manager.py b/src/allmydata/test/cli/test_grid_manager.py index f105dabeb..7fbc6a563 100644 --- a/src/allmydata/test/cli/test_grid_manager.py +++ b/src/allmydata/test/cli/test_grid_manager.py @@ -1,7 +1,6 @@ from future.utils import raise_ import os -import json from io import ( BytesIO, ) @@ -31,6 +30,7 @@ from twisted.python.filepath import ( from twisted.python.runtime import ( platform, ) +from allmydata.util import jsonbytes as json class GridManagerCommandLine(TestCase): @@ -66,8 +66,8 @@ class GridManagerCommandLine(TestCase): An invalid config is reported to the user """ with self.runner.isolated_filesystem(): - with open("config.json", "w") as f: - json.dump({"not": "valid"}, f) + with open("config.json", "wb") as f: + f.write(json.dumps_bytes({"not": "valid"})) result = self.runner.invoke(grid_manager, ["--config", ".", "public-identity"]) self.assertNotEqual(result.exit_code, 0) self.assertIn( @@ -117,7 +117,7 @@ class GridManagerCommandLine(TestCase): } result = self.invoke_and_check( grid_manager, ["--config", "-", "list"], - input=BytesIO(json.dumps(config)), + input=BytesIO(json.dumps_bytes(config)), ) self.assertEqual(result.exit_code, 0) self.assertEqual( @@ -260,7 +260,7 @@ class TahoeAddGridManagerCert(TestCase): """ code, out, err = yield run_cli( "admin", "add-grid-manager-cert", "--filename", "-", - stdin="the cert", + stdin=b"the cert", ) self.assertIn( "Must provide --name", @@ -274,7 +274,7 @@ class TahoeAddGridManagerCert(TestCase): """ code, out, err = yield run_cli( "admin", "add-grid-manager-cert", "--name", "foo", - stdin="the cert", + stdin=b"the cert", ) self.assertIn( "Must provide --filename", @@ -287,7 +287,7 @@ class TahoeAddGridManagerCert(TestCase): we can add a certificate """ nodedir = self.mktemp() - fake_cert = """{"certificate": "", "signature": ""}""" + fake_cert = b"""{"certificate": "", "signature": ""}""" code, out, err = yield run_cli( "--node-directory", nodedir, @@ -301,7 +301,7 @@ class TahoeAddGridManagerCert(TestCase): self.assertIn("tahoe.cfg", nodepath.listdir()) self.assertIn( - "foo = foo.cert", + b"foo = foo.cert", config_data, ) self.assertIn("foo.cert", nodepath.listdir()) From e8e0ecd39e8f95659711305f381410d750ab9a7a Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 27 Aug 2021 11:38:15 -0400 Subject: [PATCH 0233/2309] Port to Python 3. --- src/allmydata/test/cli/test_grid_manager.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/allmydata/test/cli/test_grid_manager.py b/src/allmydata/test/cli/test_grid_manager.py index 7fbc6a563..03e50efdd 100644 --- a/src/allmydata/test/cli/test_grid_manager.py +++ b/src/allmydata/test/cli/test_grid_manager.py @@ -1,4 +1,14 @@ -from future.utils import raise_ +""" +Ported to Python 3. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2, raise_ +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import os from io import ( From 2d2e8051f6a52bac860ed74858687743c842d4db Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 27 Aug 2021 11:39:32 -0400 Subject: [PATCH 0234/2309] Port to Python 3. --- src/allmydata/cli/grid_manager.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/allmydata/cli/grid_manager.py b/src/allmydata/cli/grid_manager.py index 78b037649..24d12398c 100644 --- a/src/allmydata/cli/grid_manager.py +++ b/src/allmydata/cli/grid_manager.py @@ -1,5 +1,14 @@ +""" +Ported to Python 3. +""" +from __future__ import absolute_import +from __future__ import division from __future__ import print_function -from past.builtins import unicode +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from datetime import ( datetime, @@ -163,7 +172,7 @@ def list(ctx): blank_name = " " * len(name) click.echo("{}: {}".format( name, - unicode(ctx.obj.grid_manager.storage_servers[name].public_key_string(), "utf-8"))) + str(ctx.obj.grid_manager.storage_servers[name].public_key_string(), "utf-8"))) for cert in ctx.obj.grid_manager.storage_servers[name].certificates: delta = datetime.utcnow() - cert.expires click.echo("{} cert {}: ".format(blank_name, cert.index), nl=False) From 75ba5c71fce340dcf6724bdb9ee015b51f55b6e3 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 27 Aug 2021 12:04:42 -0400 Subject: [PATCH 0235/2309] More passing tests on Python 3. --- src/allmydata/test/common_util.py | 1 - src/allmydata/test/test_client.py | 6 +++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/allmydata/test/common_util.py b/src/allmydata/test/common_util.py index ec449c2af..b82a7f774 100644 --- a/src/allmydata/test/common_util.py +++ b/src/allmydata/test/common_util.py @@ -127,7 +127,6 @@ def run_cli_native(verb, *args, **kwargs): stdin = TextIOWrapper(BytesIO(stdin), encoding) stdout = TextIOWrapper(BytesIO(), encoding) stderr = TextIOWrapper(BytesIO(), encoding) - stdin = TextIOWrapper(BytesIO(kwargs.get("stdin", b"")), encoding) d = defer.succeed(argv) d.addCallback(runner.parse_or_exit_with_explanation, stdout=stdout, stderr=stderr, stdin=stdin) d.addCallback(runner.dispatch, diff --git a/src/allmydata/test/test_client.py b/src/allmydata/test/test_client.py index 5d91b5ea4..f870f9fa5 100644 --- a/src/allmydata/test/test_client.py +++ b/src/allmydata/test/test_client.py @@ -12,7 +12,6 @@ if PY2: import os import sys -import json from functools import ( partial, ) @@ -75,6 +74,7 @@ from allmydata.util import ( fileutil, encodingutil, configutil, + jsonbytes as json, ) from allmydata.util.eliotutil import capture_logging from allmydata.util.fileutil import abspath_expanduser_unicode @@ -1520,9 +1520,9 @@ enabled = {storage_enabled} "signature": "fvjd3uvvupf2v6tnvkwjd473u3m3inyqkwiclhp7balmchkmn3px5pei3qyfjnhymq4cjcwvbpqmcwwnwswdtrfkpnlaxuih2zbdmda", } with self.basedir.child("zero.cert").open("w") as f: - json.dump(fake_cert, f) + f.write(json.dumps_bytes(fake_cert)) with self.basedir.child("gm0.cert").open("w") as f: - json.dump(fake_cert, f) + f.write(json.dumps_bytes(fake_cert)) config = client.config_from_string( self.basedir.path, From b9a1cc3dde82662bdfe498c9c09ec96433c77235 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 30 Aug 2021 10:27:15 -0400 Subject: [PATCH 0236/2309] Pacify flake8. --- src/allmydata/cli/grid_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/cli/grid_manager.py b/src/allmydata/cli/grid_manager.py index 24d12398c..b12839eae 100644 --- a/src/allmydata/cli/grid_manager.py +++ b/src/allmydata/cli/grid_manager.py @@ -162,7 +162,7 @@ def remove(ctx, name): save_grid_manager(fp, ctx.obj.grid_manager, create=False) -@grid_manager.command() +@grid_manager.command() # noqa: F811 @click.pass_context def list(ctx): """ From 056f7748c5d38811caa8b14e1327d86e19f61098 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 30 Aug 2021 10:29:49 -0400 Subject: [PATCH 0237/2309] Fix errant str()-of-bytes bug. --- src/allmydata/storage_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index f6f2919b4..6bd8b3ce5 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -289,7 +289,7 @@ class StorageFarmBroker(service.MultiService): gm_verifier = create_grid_manager_verifier( self.storage_client_config.grid_manager_keys, server["ann"].get("grid-manager-certificates", []), - "pub-{}".format(server_id), # server_id is v0- not pub-v0-key .. for reasons? + "pub-{}".format(str(server_id, "ascii")), # server_id is v0- not pub-v0-key .. for reasons? ) s = NativeStorageServer( From b4cdf7f96915943be85ee2b48c6954a1b2c128e7 Mon Sep 17 00:00:00 2001 From: fenn-cs Date: Wed, 8 Sep 2021 00:08:37 +0100 Subject: [PATCH 0238/2309] changed fragment to minor, improved test_grid.py refactor Signed-off-by: fenn-cs --- newsfragments/3758.minor | 0 newsfragments/3758.other | 1 - src/allmydata/test/web/test_grid.py | 21 +++++++++++---------- 3 files changed, 11 insertions(+), 11 deletions(-) create mode 100644 newsfragments/3758.minor delete mode 100644 newsfragments/3758.other diff --git a/newsfragments/3758.minor b/newsfragments/3758.minor new file mode 100644 index 000000000..e69de29bb diff --git a/newsfragments/3758.other b/newsfragments/3758.other deleted file mode 100644 index d0eb1d4c1..000000000 --- a/newsfragments/3758.other +++ /dev/null @@ -1 +0,0 @@ -Refactored test_logs, test_grid and test_root in web tests to use custom base test cases diff --git a/src/allmydata/test/web/test_grid.py b/src/allmydata/test/web/test_grid.py index 54aa13941..1ebe3a90f 100644 --- a/src/allmydata/test/web/test_grid.py +++ b/src/allmydata/test/web/test_grid.py @@ -49,8 +49,9 @@ from ..common import ( from testtools.matchers import ( Equals, Contains, - Is, Not, + HasLength, + EndsWith, ) from testtools.twistedsupport import flush_logged_errors @@ -190,7 +191,7 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi r = json.loads(res) self.failUnlessEqual(r["summary"], "Not Healthy: 9 shares (enc 3-of-10)") - self.assertThat(r["results"]["healthy"], Is(False)) + self.assertThat(r["results"]["healthy"], Equals(False)) self.failUnless(r["results"]["recoverable"]) self.assertThat(r["results"], Not(Contains("needs-rebalancing"))) d.addCallback(_got_json_sick) @@ -204,8 +205,8 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi r = json.loads(res) self.failUnlessEqual(r["summary"], "Not Healthy: 1 shares (enc 3-of-10)") - self.assertThat(r["results"]["healthy"], Is(False)) - self.assertThat(r["results"]["recoverable"], Is(False)) + self.assertThat(r["results"]["healthy"], Equals(False)) + self.assertThat(r["results"]["recoverable"], Equals(False)) self.assertThat(r["results"], Not(Contains("needs-rebalancing"))) d.addCallback(_got_json_dead) @@ -217,7 +218,7 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi def _got_json_corrupt(res): r = json.loads(res) self.assertThat(r["summary"], Contains("Unhealthy: 9 shares (enc 3-of-10)")) - self.assertThat(r["results"]["healthy"], Is(False)) + self.assertThat(r["results"]["healthy"], Equals(False)) self.failUnless(r["results"]["recoverable"]) self.assertThat(r["results"], Not(Contains("needs-rebalancing"))) self.failUnlessReallyEqual(r["results"]["count-happiness"], 9) @@ -532,10 +533,10 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi def _created(dn): self.failUnless(isinstance(dn, dirnode.DirectoryNode)) - self.assertThat(dn.is_mutable(), Is(False)) + self.assertThat(dn.is_mutable(), Equals(False)) self.failUnless(dn.is_readonly()) # This checks that if we somehow ended up calling dn._decrypt_rwcapdata, it would fail. - self.assertThat(hasattr(dn._node, 'get_writekey'), Is(False)) + self.assertThat(hasattr(dn._node, 'get_writekey'), Equals(False)) rep = str(dn) self.assertThat(rep, Contains("RO-IMM")) cap = dn.get_cap() @@ -597,15 +598,15 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi self.assertThat(td.findNextSibling().findNextSibling().text, Equals(u"{}".format(len("one")))) found = True break - self.assertThat(found, Is(True)) + self.assertThat(found, Equals(True)) infos = list( a[u"href"] for a in soup.find_all(u"a") if a.text == u"More Info" ) - self.assertThat(len(infos), Equals(1)) - self.assertThat(infos[0].endswith(url_quote(lonely_uri) + "?t=info"), Is(True)) + self.assertThat(infos, HasLength(1)) + self.assertThat(infos[0], EndsWith(url_quote(lonely_uri) + "?t=info")) d.addCallback(_check_html) # ... and in JSON. From 55221d4532fe051e01e67c21e53507feda5f7feb Mon Sep 17 00:00:00 2001 From: fenn-cs Date: Thu, 9 Sep 2021 01:50:21 +0100 Subject: [PATCH 0239/2309] replaced testools.unittest.TestCase with common base case Signed-off-by: fenn-cs --- src/allmydata/test/mutable/test_checker.py | 5 +++-- src/allmydata/test/mutable/test_datahandle.py | 6 ++++-- src/allmydata/test/mutable/test_different_encoding.py | 5 +++-- src/allmydata/test/mutable/test_exceptions.py | 5 +++-- src/allmydata/test/mutable/test_filehandle.py | 6 ++++-- 5 files changed, 17 insertions(+), 10 deletions(-) diff --git a/src/allmydata/test/mutable/test_checker.py b/src/allmydata/test/mutable/test_checker.py index 11ba776fd..6d9145d68 100644 --- a/src/allmydata/test/mutable/test_checker.py +++ b/src/allmydata/test/mutable/test_checker.py @@ -10,14 +10,15 @@ from future.utils import PY2 if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 -from twisted.trial import unittest +from ..common import AsyncTestCase from foolscap.api import flushEventualQueue from allmydata.monitor import Monitor from allmydata.mutable.common import CorruptShareError from .util import PublishMixin, corrupt, CheckerMixin -class Checker(unittest.TestCase, CheckerMixin, PublishMixin): +class Checker(AsyncTestCase, CheckerMixin, PublishMixin): def setUp(self): + super(Checker, self).setUp() return self.publish_one() diff --git a/src/allmydata/test/mutable/test_datahandle.py b/src/allmydata/test/mutable/test_datahandle.py index 1819cba01..53e2983d1 100644 --- a/src/allmydata/test/mutable/test_datahandle.py +++ b/src/allmydata/test/mutable/test_datahandle.py @@ -10,11 +10,13 @@ from future.utils import PY2 if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 -from twisted.trial import unittest +from ..common import SyncTestCase from allmydata.mutable.publish import MutableData -class DataHandle(unittest.TestCase): + +class DataHandle(SyncTestCase): def setUp(self): + super(DataHandle, self).setUp() self.test_data = b"Test Data" * 50000 self.uploadable = MutableData(self.test_data) diff --git a/src/allmydata/test/mutable/test_different_encoding.py b/src/allmydata/test/mutable/test_different_encoding.py index a5165532c..f1796d373 100644 --- a/src/allmydata/test/mutable/test_different_encoding.py +++ b/src/allmydata/test/mutable/test_different_encoding.py @@ -10,11 +10,12 @@ from future.utils import PY2 if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 -from twisted.trial import unittest +from ..common import AsyncTestCase from .util import FakeStorage, make_nodemaker -class DifferentEncoding(unittest.TestCase): +class DifferentEncoding(AsyncTestCase): def setUp(self): + super(DifferentEncoding, self).setUp() self._storage = s = FakeStorage() self.nodemaker = make_nodemaker(s) diff --git a/src/allmydata/test/mutable/test_exceptions.py b/src/allmydata/test/mutable/test_exceptions.py index 6a9b2b575..aa2b56b86 100644 --- a/src/allmydata/test/mutable/test_exceptions.py +++ b/src/allmydata/test/mutable/test_exceptions.py @@ -11,10 +11,11 @@ from future.utils import PY2 if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 -from twisted.trial import unittest +from ..common import SyncTestCase from allmydata.mutable.common import NeedMoreDataError, UncoordinatedWriteError -class Exceptions(unittest.TestCase): + +class Exceptions(SyncTestCase): def test_repr(self): nmde = NeedMoreDataError(100, 50, 100) self.failUnless("NeedMoreDataError" in repr(nmde), repr(nmde)) diff --git a/src/allmydata/test/mutable/test_filehandle.py b/src/allmydata/test/mutable/test_filehandle.py index 8db02f3fd..795f60654 100644 --- a/src/allmydata/test/mutable/test_filehandle.py +++ b/src/allmydata/test/mutable/test_filehandle.py @@ -12,11 +12,13 @@ if PY2: import os from io import BytesIO -from twisted.trial import unittest +from ..common import SyncTestCase from allmydata.mutable.publish import MutableFileHandle -class FileHandle(unittest.TestCase): + +class FileHandle(SyncTestCase): def setUp(self): + super(FileHandle, self).setUp() self.test_data = b"Test Data" * 50000 self.sio = BytesIO(self.test_data) self.uploadable = MutableFileHandle(self.sio) From bbbc8592f09d8f7bac1f434dab6b305f6b58ea54 Mon Sep 17 00:00:00 2001 From: fenn-cs Date: Thu, 9 Sep 2021 14:41:06 +0100 Subject: [PATCH 0240/2309] removed deprecated methods, already refactored mutable files Signed-off-by: fenn-cs --- src/allmydata/test/mutable/test_datahandle.py | 11 +-- src/allmydata/test/mutable/test_exceptions.py | 5 +- .../test/mutable/test_interoperability.py | 9 +-- .../test/mutable/test_multiple_encodings.py | 8 ++- .../test/mutable/test_multiple_versions.py | 38 ++++++----- src/allmydata/test/mutable/test_problems.py | 19 +++--- src/allmydata/test/mutable/test_repair.py | 67 ++++++++++--------- 7 files changed, 83 insertions(+), 74 deletions(-) diff --git a/src/allmydata/test/mutable/test_datahandle.py b/src/allmydata/test/mutable/test_datahandle.py index 53e2983d1..7aabcd8e1 100644 --- a/src/allmydata/test/mutable/test_datahandle.py +++ b/src/allmydata/test/mutable/test_datahandle.py @@ -12,6 +12,7 @@ if PY2: from ..common import SyncTestCase from allmydata.mutable.publish import MutableData +from testtools.matchers import Equals, HasLength class DataHandle(SyncTestCase): @@ -28,13 +29,13 @@ class DataHandle(SyncTestCase): data = b"".join(data) start = i end = i + chunk_size - self.failUnlessEqual(data, self.test_data[start:end]) + self.assertThat(data, Equals(self.test_data[start:end])) def test_datahandle_get_size(self): actual_size = len(self.test_data) size = self.uploadable.get_size() - self.failUnlessEqual(size, actual_size) + self.assertThat(size, Equals(actual_size)) def test_datahandle_get_size_out_of_order(self): @@ -42,14 +43,14 @@ class DataHandle(SyncTestCase): # disturbing the location of the seek pointer. chunk_size = 100 data = self.uploadable.read(chunk_size) - self.failUnlessEqual(b"".join(data), self.test_data[:chunk_size]) + self.assertThat(b"".join(data), Equals(self.test_data[:chunk_size])) # Now get the size. size = self.uploadable.get_size() - self.failUnlessEqual(size, len(self.test_data)) + self.assertThat(self.test_data, HasLength(size)) # Now get more data. We should be right where we left off. more_data = self.uploadable.read(chunk_size) start = chunk_size end = chunk_size * 2 - self.failUnlessEqual(b"".join(more_data), self.test_data[start:end]) + self.assertThat(b"".join(more_data), Equals(self.test_data[start:end])) diff --git a/src/allmydata/test/mutable/test_exceptions.py b/src/allmydata/test/mutable/test_exceptions.py index aa2b56b86..23674d036 100644 --- a/src/allmydata/test/mutable/test_exceptions.py +++ b/src/allmydata/test/mutable/test_exceptions.py @@ -18,6 +18,7 @@ from allmydata.mutable.common import NeedMoreDataError, UncoordinatedWriteError class Exceptions(SyncTestCase): def test_repr(self): nmde = NeedMoreDataError(100, 50, 100) - self.failUnless("NeedMoreDataError" in repr(nmde), repr(nmde)) + self.assertTrue("NeedMoreDataError" in repr(nmde), msg=repr(nmde)) + self.assertTrue("NeedMoreDataError" in repr(nmde), msg=repr(nmde)) ucwe = UncoordinatedWriteError() - self.failUnless("UncoordinatedWriteError" in repr(ucwe), repr(ucwe)) + self.assertTrue("UncoordinatedWriteError" in repr(ucwe), msg=repr(ucwe)) diff --git a/src/allmydata/test/mutable/test_interoperability.py b/src/allmydata/test/mutable/test_interoperability.py index 5d7414907..496da1d2a 100644 --- a/src/allmydata/test/mutable/test_interoperability.py +++ b/src/allmydata/test/mutable/test_interoperability.py @@ -11,14 +11,15 @@ if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import os, base64 -from twisted.trial import unittest +from ..common import AsyncTestCase +from testtools.matchers import HasLength from allmydata import uri from allmydata.storage.common import storage_index_to_dir from allmydata.util import fileutil from .. import common_util as testutil from ..no_network import GridTestMixin -class Interoperability(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin): +class Interoperability(GridTestMixin, AsyncTestCase, testutil.ShouldFailMixin): sdmf_old_shares = {} sdmf_old_shares[0] = b"VGFob2UgbXV0YWJsZSBjb250YWluZXIgdjEKdQlEA47ESLbTdKdpLJXCpBxd5OH239tl5hvAiz1dvGdE5rIOpf8cbfxbPcwNF+Y5dM92uBVbmV6KAAAAAAAAB/wAAAAAAAAJ0AAAAAFOWSw7jSx7WXzaMpdleJYXwYsRCV82jNA5oex9m2YhXSnb2POh+vvC1LE1NAfRc9GOb2zQG84Xdsx1Jub2brEeKkyt0sRIttN0p2kslcKkHF3k4fbf22XmAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABamJprL6ecrsOoFKdrXUmWveLq8nzEGDOjFnyK9detI3noX3uyK2MwSnFdAfyN0tuAwoAAAAAAAAAFQAAAAAAAAAVAAABjwAAAo8AAAMXAAADNwAAAAAAAAM+AAAAAAAAB/wwggEgMA0GCSqGSIb3DQEBAQUAA4IBDQAwggEIAoIBAQC1IkainlJF12IBXBQdpRK1zXB7a26vuEYqRmQM09YjC6sQjCs0F2ICk8n9m/2Kw4l16eIEboB2Au9pODCE+u/dEAakEFh4qidTMn61rbGUbsLK8xzuWNW22ezzz9/nPia0HDrulXt51/FYtfnnAuD1RJGXJv/8tDllE9FL/18TzlH4WuB6Fp8FTgv7QdbZAfWJHDGFIpVCJr1XxOCsSZNFJIqGwZnD2lsChiWw5OJDbKd8otqN1hIbfHyMyfMOJ/BzRzvZXaUt4Dv5nf93EmQDWClxShRwpuX/NkZ5B2K9OFonFTbOCexm/MjMAdCBqebKKaiHFkiknUCn9eJQpZ5bAgERgV50VKj+AVTDfgTpqfO2vfo4wrufi6ZBb8QV7hllhUFBjYogQ9C96dnS7skv0s+cqFuUjwMILr5/rsbEmEMGvl0T0ytyAbtlXuowEFVj/YORNknM4yjY72YUtEPTlMpk0Cis7aIgTvu5qWMPER26PMApZuRqiwRsGIkaJIvOVOTHHjFYe3/YzdMkc7OZtqRMfQLtwVl2/zKQQV8b/a9vaT6q3mRLRd4P3esaAFe/+7sR/t+9tmB+a8kxtKM6kmaVQJMbXJZ4aoHGfeLX0m35Rcvu2Bmph7QfSDjk/eaE3q55zYSoGWShmlhlw4Kwg84sMuhmcVhLvo0LovR8bKmbdgACtTh7+7gs/l5w1lOkgbF6w7rkXLNslK7L2KYF4SPFLUcABOOLy8EETxh7h7/z9d62EiPu9CNpRrCOLxUhn+JUS+DuAAhgcAb/adrQFrhlrRNoRpvjDuxmFebA4F0qCyqWssm61AAQ/EX4eC/1+hGOQ/h4EiKUkqxdsfzdcPlDvd11SGWZ0VHsUclZChTzuBAU2zLTXm+cG8IFhO50ly6Ey/DB44NtMKVaVzO0nU8DE0Wua7Lx6Bnad5n91qmHAnwSEJE5YIhQM634omd6cq9Wk4seJCUIn+ucoknrpxp0IR9QMxpKSMRHRUg2K8ZegnY3YqFunRZKCfsq9ufQEKgjZN12AFqi551KPBdn4/3V5HK6xTv0P4robSsE/BvuIfByvRf/W7ZrDx+CFC4EEcsBOACOZCrkhhqd5TkYKbe9RA+vs56+9N5qZGurkxcoKviiyEncxvTuShD65DK/6x6kMDMgQv/EdZDI3x9GtHTnRBYXwDGnPJ19w+q2zC3e2XarbxTGYQIPEC5mYx0gAA0sbjf018NGfwBhl6SB54iGsa8uLvR3jHv6OSRJgwxL6j7P0Ts4Hv2EtO12P0Lv21pwi3JC1O/WviSrKCvrQD5lMHL9Uym3hwFi2zu0mqwZvxOAbGy7kfOPXkLYKOHTZLthzKj3PsdjeceWBfYIvPGKYcd6wDr36d1aXSYS4IWeApTS2AQ2lu0DUcgSefAvsA8NkgOklvJY1cjTMSg6j6cxQo48Bvl8RAWGLbr4h2S/8KwDGxwLsSv0Gop/gnFc3GzCsmL0EkEyHHWkCA8YRXCghfW80KLDV495ff7yF5oiwK56GniqowZ3RG9Jxp5MXoJQgsLV1VMQFMAmsY69yz8eoxRH3wl9L0dMyndLulhWWzNwPMQ2I0yAWdzA/pksVmwTJTFenB3MHCiWc5rEwJ3yofe6NZZnZQrYyL9r1TNnVwfTwRUiykPiLSk4x9Mi6DX7RamDAxc8u3gDVfjPsTOTagBOEGUWlGAL54KE/E6sgCQ5DEAt12chk8AxbjBFLPgV+/idrzS0lZHOL+IVBI9D0i3Bq1yZcSIqcjZB0M3IbxbPm4gLAYOWEiTUN2ecsEHHg9nt6rhgffVoqSbCCFPbpC0xf7WOC3+BQORIZECOCC7cUAciXq3xn+GuxpFE40RWRJeKAK7bBQ21X89ABIXlQFkFddZ9kRvlZ2Pnl0oeF+2pjnZu0Yc2czNfZEQF2P7BKIdLrgMgxG89snxAY8qAYTCKyQw6xTG87wkjDcpy1wzsZLP3WsOuO7cAm7b27xU0jRKq8Cw4d1hDoyRG+RdS53F8RFJzVMaNNYgxU2tfRwUvXpTRXiOheeRVvh25+YGVnjakUXjx/dSDnOw4ETHGHD+7styDkeSfc3BdSZxswzc6OehgMI+xsCxeeRym15QUm9hxvg8X7Bfz/0WulgFwgzrm11TVynZYOmvyHpiZKoqQyQyKahIrfhwuchCr7lMsZ4a+umIkNkKxCLZnI+T7jd+eGFMgKItjz3kTTxRl3IhaJG3LbPmwRUJynMxQKdMi4Uf0qy0U7+i8hIJ9m50QXc+3tw2bwDSbx22XYJ9Wf14gxx5G5SPTb1JVCbhe4fxNt91xIxCow2zk62tzbYfRe6dfmDmgYHkv2PIEtMJZK8iKLDjFfu2ZUxsKT2A5g1q17og6o9MeXeuFS3mzJXJYFQZd+3UzlFR9qwkFkby9mg5y4XSeMvRLOHPt/H/r5SpEqBE6a9MadZYt61FBV152CUEzd43ihXtrAa0XH9HdsiySBcWI1SpM3mv9rRP0DiLjMUzHw/K1D8TE2f07zW4t/9kvE11tFj/NpICixQAAAAA=" sdmf_old_shares[1] = b"VGFob2UgbXV0YWJsZSBjb250YWluZXIgdjEKdQlEA47ESLbTdKdpLJXCpBxd5OH239tl5hvAiz1dvGdE5rIOpf8cbfxbPcwNF+Y5dM92uBVbmV6KAAAAAAAAB/wAAAAAAAAJ0AAAAAFOWSw7jSx7WXzaMpdleJYXwYsRCV82jNA5oex9m2YhXSnb2POh+vvC1LE1NAfRc9GOb2zQG84Xdsx1Jub2brEeKkyt0sRIttN0p2kslcKkHF3k4fbf22XmAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABamJprL6ecrsOoFKdrXUmWveLq8nzEGDOjFnyK9detI3noX3uyK2MwSnFdAfyN0tuAwoAAAAAAAAAFQAAAAAAAAAVAAABjwAAAo8AAAMXAAADNwAAAAAAAAM+AAAAAAAAB/wwggEgMA0GCSqGSIb3DQEBAQUAA4IBDQAwggEIAoIBAQC1IkainlJF12IBXBQdpRK1zXB7a26vuEYqRmQM09YjC6sQjCs0F2ICk8n9m/2Kw4l16eIEboB2Au9pODCE+u/dEAakEFh4qidTMn61rbGUbsLK8xzuWNW22ezzz9/nPia0HDrulXt51/FYtfnnAuD1RJGXJv/8tDllE9FL/18TzlH4WuB6Fp8FTgv7QdbZAfWJHDGFIpVCJr1XxOCsSZNFJIqGwZnD2lsChiWw5OJDbKd8otqN1hIbfHyMyfMOJ/BzRzvZXaUt4Dv5nf93EmQDWClxShRwpuX/NkZ5B2K9OFonFTbOCexm/MjMAdCBqebKKaiHFkiknUCn9eJQpZ5bAgERgV50VKj+AVTDfgTpqfO2vfo4wrufi6ZBb8QV7hllhUFBjYogQ9C96dnS7skv0s+cqFuUjwMILr5/rsbEmEMGvl0T0ytyAbtlXuowEFVj/YORNknM4yjY72YUtEPTlMpk0Cis7aIgTvu5qWMPER26PMApZuRqiwRsGIkaJIvOVOTHHjFYe3/YzdMkc7OZtqRMfQLtwVl2/zKQQV8b/a9vaT6q3mRLRd4P3esaAFe/+7sR/t+9tmB+a8kxtKM6kmaVQJMbXJZ4aoHGfeLX0m35Rcvu2Bmph7QfSDjk/eaE3q55zYSoGWShmlhlw4Kwg84sMuhmcVhLvo0LovR8bKmbdgACtTh7+7gs/l5w1lOkgbF6w7rkXLNslK7L2KYF4SPFLUcABOOLy8EETxh7h7/z9d62EiPu9CNpRrCOLxUhn+JUS+DuAAhgcAb/adrQFrhlrRNoRpvjDuxmFebA4F0qCyqWssm61AAP7FHJWQoU87gQFNsy015vnBvCBYTudJcuhMvwweODbTD8Rfh4L/X6EY5D+HgSIpSSrF2x/N1w+UO93XVIZZnRUeePDXEwhqYDE0Wua7Lx6Bnad5n91qmHAnwSEJE5YIhQM634omd6cq9Wk4seJCUIn+ucoknrpxp0IR9QMxpKSMRHRUg2K8ZegnY3YqFunRZKCfsq9ufQEKgjZN12AFqi551KPBdn4/3V5HK6xTv0P4robSsE/BvuIfByvRf/W7ZrDx+CFC4EEcsBOACOZCrkhhqd5TkYKbe9RA+vs56+9N5qZGurkxcoKviiyEncxvTuShD65DK/6x6kMDMgQv/EdZDI3x9GtHTnRBYXwDGnPJ19w+q2zC3e2XarbxTGYQIPEC5mYx0gAA0sbjf018NGfwBhl6SB54iGsa8uLvR3jHv6OSRJgwxL6j7P0Ts4Hv2EtO12P0Lv21pwi3JC1O/WviSrKCvrQD5lMHL9Uym3hwFi2zu0mqwZvxOAbGy7kfOPXkLYKOHTZLthzKj3PsdjeceWBfYIvPGKYcd6wDr36d1aXSYS4IWeApTS2AQ2lu0DUcgSefAvsA8NkgOklvJY1cjTMSg6j6cxQo48Bvl8RAWGLbr4h2S/8KwDGxwLsSv0Gop/gnFc3GzCsmL0EkEyHHWkCA8YRXCghfW80KLDV495ff7yF5oiwK56GniqowZ3RG9Jxp5MXoJQgsLV1VMQFMAmsY69yz8eoxRH3wl9L0dMyndLulhWWzNwPMQ2I0yAWdzA/pksVmwTJTFenB3MHCiWc5rEwJ3yofe6NZZnZQrYyL9r1TNnVwfTwRUiykPiLSk4x9Mi6DX7RamDAxc8u3gDVfjPsTOTagBOEGUWlGAL54KE/E6sgCQ5DEAt12chk8AxbjBFLPgV+/idrzS0lZHOL+IVBI9D0i3Bq1yZcSIqcjZB0M3IbxbPm4gLAYOWEiTUN2ecsEHHg9nt6rhgffVoqSbCCFPbpC0xf7WOC3+BQORIZECOCC7cUAciXq3xn+GuxpFE40RWRJeKAK7bBQ21X89ABIXlQFkFddZ9kRvlZ2Pnl0oeF+2pjnZu0Yc2czNfZEQF2P7BKIdLrgMgxG89snxAY8qAYTCKyQw6xTG87wkjDcpy1wzsZLP3WsOuO7cAm7b27xU0jRKq8Cw4d1hDoyRG+RdS53F8RFJzVMaNNYgxU2tfRwUvXpTRXiOheeRVvh25+YGVnjakUXjx/dSDnOw4ETHGHD+7styDkeSfc3BdSZxswzc6OehgMI+xsCxeeRym15QUm9hxvg8X7Bfz/0WulgFwgzrm11TVynZYOmvyHpiZKoqQyQyKahIrfhwuchCr7lMsZ4a+umIkNkKxCLZnI+T7jd+eGFMgKItjz3kTTxRl3IhaJG3LbPmwRUJynMxQKdMi4Uf0qy0U7+i8hIJ9m50QXc+3tw2bwDSbx22XYJ9Wf14gxx5G5SPTb1JVCbhe4fxNt91xIxCow2zk62tzbYfRe6dfmDmgYHkv2PIEtMJZK8iKLDjFfu2ZUxsKT2A5g1q17og6o9MeXeuFS3mzJXJYFQZd+3UzlFR9qwkFkby9mg5y4XSeMvRLOHPt/H/r5SpEqBE6a9MadZYt61FBV152CUEzd43ihXtrAa0XH9HdsiySBcWI1SpM3mv9rRP0DiLjMUzHw/K1D8TE2f07zW4t/9kvE11tFj/NpICixQAAAAA=" @@ -53,7 +54,7 @@ class Interoperability(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixi sharedata) # ...and verify that the shares are there. shares = self.find_uri_shares(self.sdmf_old_cap) - assert len(shares) == 10 + self.assertThat(shares, HasLength(10)) def test_new_downloader_can_read_old_shares(self): self.basedir = "mutable/Interoperability/new_downloader_can_read_old_shares" @@ -62,5 +63,5 @@ class Interoperability(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixi nm = self.g.clients[0].nodemaker n = nm.create_from_cap(self.sdmf_old_cap) d = n.download_best_version() - d.addCallback(self.failUnlessEqual, self.sdmf_old_contents) + d.addCallback(self.assertEqual, self.sdmf_old_contents) return d diff --git a/src/allmydata/test/mutable/test_multiple_encodings.py b/src/allmydata/test/mutable/test_multiple_encodings.py index 12c5be051..2291b60d8 100644 --- a/src/allmydata/test/mutable/test_multiple_encodings.py +++ b/src/allmydata/test/mutable/test_multiple_encodings.py @@ -10,7 +10,8 @@ from future.utils import PY2 if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 -from twisted.trial import unittest +from ..common import AsyncTestCase +from testtools.matchers import Equals from allmydata.interfaces import SDMF_VERSION from allmydata.monitor import Monitor from foolscap.logging import log @@ -20,8 +21,9 @@ from allmydata.mutable.servermap import ServerMap, ServermapUpdater from ..common_util import DevNullDictionary from .util import FakeStorage, make_nodemaker -class MultipleEncodings(unittest.TestCase): +class MultipleEncodings(AsyncTestCase): def setUp(self): + super(MultipleEncodings, self).setUp() self.CONTENTS = b"New contents go here" self.uploadable = MutableData(self.CONTENTS) self._storage = FakeStorage() @@ -159,6 +161,6 @@ class MultipleEncodings(unittest.TestCase): d.addCallback(lambda res: fn3.download_best_version()) def _retrieved(new_contents): # the current specified behavior is "first version recoverable" - self.failUnlessEqual(new_contents, contents1) + self.assertThat(new_contents, Equals(contents1)) d.addCallback(_retrieved) return d diff --git a/src/allmydata/test/mutable/test_multiple_versions.py b/src/allmydata/test/mutable/test_multiple_versions.py index 460cde4b3..c9b7e71df 100644 --- a/src/allmydata/test/mutable/test_multiple_versions.py +++ b/src/allmydata/test/mutable/test_multiple_versions.py @@ -10,15 +10,17 @@ from future.utils import PY2 if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 -from twisted.trial import unittest +from ..common import AsyncTestCase +from testtools.matchers import Equals, HasLength from allmydata.monitor import Monitor from allmydata.mutable.common import MODE_CHECK, MODE_READ from .util import PublishMixin, CheckerMixin -class MultipleVersions(unittest.TestCase, PublishMixin, CheckerMixin): +class MultipleVersions(AsyncTestCase, PublishMixin, CheckerMixin): def setUp(self): + super(MultipleVersions, self).setUp() return self.publish_multiple() def test_multiple_versions(self): @@ -26,7 +28,7 @@ class MultipleVersions(unittest.TestCase, PublishMixin, CheckerMixin): # should get the latest one self._set_versions(dict([(i,2) for i in (0,2,4,6,8)])) d = self._fn.download_best_version() - d.addCallback(lambda res: self.failUnlessEqual(res, self.CONTENTS[4])) + d.addCallback(lambda res: self.assertThat(res, Equals(self.CONTENTS[4]))) # and the checker should report problems d.addCallback(lambda res: self._fn.check(Monitor())) d.addCallback(self.check_bad, "test_multiple_versions") @@ -35,23 +37,23 @@ class MultipleVersions(unittest.TestCase, PublishMixin, CheckerMixin): d.addCallback(lambda res: self._set_versions(dict([(i,2) for i in range(10)]))) d.addCallback(lambda res: self._fn.download_best_version()) - d.addCallback(lambda res: self.failUnlessEqual(res, self.CONTENTS[2])) + d.addCallback(lambda res: self.assertThat(res, Equals(self.CONTENTS[2]))) # if exactly one share is at version 3, we should still get v2 d.addCallback(lambda res: self._set_versions({0:3})) d.addCallback(lambda res: self._fn.download_best_version()) - d.addCallback(lambda res: self.failUnlessEqual(res, self.CONTENTS[2])) + d.addCallback(lambda res: self.assertThat(res, Equals(self.CONTENTS[2]))) # but the servermap should see the unrecoverable version. This # depends upon the single newer share being queried early. d.addCallback(lambda res: self._fn.get_servermap(MODE_READ)) def _check_smap(smap): - self.failUnlessEqual(len(smap.unrecoverable_versions()), 1) + self.assertThat(smap.unrecoverable_versions(), HasLength(1)) newer = smap.unrecoverable_newer_versions() - self.failUnlessEqual(len(newer), 1) + self.assertThat(newer, HasLength(1)) verinfo, health = list(newer.items())[0] - self.failUnlessEqual(verinfo[0], 4) - self.failUnlessEqual(health, (1,3)) - self.failIf(smap.needs_merge()) + self.assertThat(verinfo[0], Equals(4)) + self.assertThat(health, Equals((1,3))) + self.assertThat(smap.needs_merge(), Equals(False)) d.addCallback(_check_smap) # if we have a mix of two parallel versions (s4a and s4b), we could # recover either @@ -60,13 +62,13 @@ class MultipleVersions(unittest.TestCase, PublishMixin, CheckerMixin): 1:4,3:4,5:4,7:4,9:4})) d.addCallback(lambda res: self._fn.get_servermap(MODE_READ)) def _check_smap_mixed(smap): - self.failUnlessEqual(len(smap.unrecoverable_versions()), 0) + self.assertThat(smap.unrecoverable_versions(), HasLength(0)) newer = smap.unrecoverable_newer_versions() - self.failUnlessEqual(len(newer), 0) - self.failUnless(smap.needs_merge()) + self.assertThat(newer, HasLength(0)) + self.assertTrue(smap.needs_merge()) d.addCallback(_check_smap_mixed) d.addCallback(lambda res: self._fn.download_best_version()) - d.addCallback(lambda res: self.failUnless(res == self.CONTENTS[3] or + d.addCallback(lambda res: self.assertTrue(res == self.CONTENTS[3] or res == self.CONTENTS[4])) return d @@ -86,12 +88,12 @@ class MultipleVersions(unittest.TestCase, PublishMixin, CheckerMixin): d = self._fn.modify(_modify) d.addCallback(lambda res: self._fn.download_best_version()) expected = self.CONTENTS[2] + b" modified" - d.addCallback(lambda res: self.failUnlessEqual(res, expected)) + d.addCallback(lambda res: self.assertThat(res, Equals(expected))) # and the servermap should indicate that the outlier was replaced too d.addCallback(lambda res: self._fn.get_servermap(MODE_CHECK)) def _check_smap(smap): - self.failUnlessEqual(smap.highest_seqnum(), 5) - self.failUnlessEqual(len(smap.unrecoverable_versions()), 0) - self.failUnlessEqual(len(smap.recoverable_versions()), 1) + self.assertThat(smap.highest_seqnum(), Equals(5)) + self.assertThat(smap.unrecoverable_versions(), HasLength(0)) + self.assertThat(smap.recoverable_versions(), HasLength(1)) d.addCallback(_check_smap) return d diff --git a/src/allmydata/test/mutable/test_problems.py b/src/allmydata/test/mutable/test_problems.py index 86a367596..9abee560d 100644 --- a/src/allmydata/test/mutable/test_problems.py +++ b/src/allmydata/test/mutable/test_problems.py @@ -11,7 +11,8 @@ if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import os, base64 -from twisted.trial import unittest +from ..common import AsyncTestCase +from testtools.matchers import HasLength from twisted.internet import defer from foolscap.logging import log from allmydata import uri @@ -61,7 +62,7 @@ class FirstServerGetsDeleted(object): return (True, {}) return retval -class Problems(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin): +class Problems(GridTestMixin, AsyncTestCase, testutil.ShouldFailMixin): def do_publish_surprise(self, version): self.basedir = "mutable/Problems/test_publish_surprise_%s" % version self.set_up_grid() @@ -198,8 +199,8 @@ class Problems(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin): def _overwritten_again(smap): # Make sure that all shares were updated by making sure that # there aren't any other versions in the sharemap. - self.failUnlessEqual(len(smap.recoverable_versions()), 1) - self.failUnlessEqual(len(smap.unrecoverable_versions()), 0) + self.assertThat(smap.recoverable_versions(), HasLength(1)) + self.assertThat(smap.unrecoverable_versions(), HasLength(0)) d.addCallback(_overwritten_again) return d @@ -240,7 +241,7 @@ class Problems(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin): # that ought to work def _got_node(n): d = n.download_best_version() - d.addCallback(lambda res: self.failUnlessEqual(res, b"contents 1")) + d.addCallback(lambda res: self.assertTrue(res, b"contents 1")) # now break the second peer def _break_peer1(res): self.g.break_server(self.server1.get_serverid()) @@ -248,7 +249,7 @@ class Problems(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin): d.addCallback(lambda res: n.overwrite(MutableData(b"contents 2"))) # that ought to work too d.addCallback(lambda res: n.download_best_version()) - d.addCallback(lambda res: self.failUnlessEqual(res, b"contents 2")) + d.addCallback(lambda res: self.assertTrue(res, b"contents 2")) def _explain_error(f): print(f) if f.check(NotEnoughServersError): @@ -280,7 +281,7 @@ class Problems(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin): d = nm.create_mutable_file(MutableData(b"contents 1")) def _created(n): d = n.download_best_version() - d.addCallback(lambda res: self.failUnlessEqual(res, b"contents 1")) + d.addCallback(lambda res: self.assertTrue(res, b"contents 1")) # now break one of the remaining servers def _break_second_server(res): self.g.break_server(peerids[1]) @@ -288,7 +289,7 @@ class Problems(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin): d.addCallback(lambda res: n.overwrite(MutableData(b"contents 2"))) # that ought to work too d.addCallback(lambda res: n.download_best_version()) - d.addCallback(lambda res: self.failUnlessEqual(res, b"contents 2")) + d.addCallback(lambda res: self.assertTrue(res, b"contents 2")) return d d.addCallback(_created) return d @@ -419,7 +420,7 @@ class Problems(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin): return self._node.download_version(servermap, ver) d.addCallback(_then) d.addCallback(lambda data: - self.failUnlessEqual(data, CONTENTS)) + self.assertTrue(data, CONTENTS)) return d def test_1654(self): diff --git a/src/allmydata/test/mutable/test_repair.py b/src/allmydata/test/mutable/test_repair.py index fb1caa974..987b21cc3 100644 --- a/src/allmydata/test/mutable/test_repair.py +++ b/src/allmydata/test/mutable/test_repair.py @@ -10,7 +10,8 @@ from future.utils import PY2 if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 -from twisted.trial import unittest +from ..common import AsyncTestCase +from testtools.matchers import Equals, HasLength from allmydata.interfaces import IRepairResults, ICheckAndRepairResults from allmydata.monitor import Monitor from allmydata.mutable.common import MODE_CHECK @@ -19,7 +20,7 @@ from allmydata.mutable.repairer import MustForceRepairError from ..common import ShouldFailMixin from .util import PublishMixin -class Repair(unittest.TestCase, PublishMixin, ShouldFailMixin): +class Repair(AsyncTestCase, PublishMixin, ShouldFailMixin): def get_shares(self, s): all_shares = {} # maps (peerid, shnum) to share data @@ -40,8 +41,8 @@ class Repair(unittest.TestCase, PublishMixin, ShouldFailMixin): d.addCallback(lambda res: self._fn.check(Monitor())) d.addCallback(lambda check_results: self._fn.repair(check_results)) def _check_results(rres): - self.failUnless(IRepairResults.providedBy(rres)) - self.failUnless(rres.get_successful()) + self.assertThat(IRepairResults.providedBy(rres), Equals(True)) + self.assertThat(rres.get_successful(), Equals(True)) # TODO: examine results self.copy_shares() @@ -50,11 +51,11 @@ class Repair(unittest.TestCase, PublishMixin, ShouldFailMixin): new_shares = self.old_shares[1] # TODO: this really shouldn't change anything. When we implement # a "minimal-bandwidth" repairer", change this test to assert: - #self.failUnlessEqual(new_shares, initial_shares) + #self.assertThat(new_shares, Equals(initial_shares)) # all shares should be in the same place as before - self.failUnlessEqual(set(initial_shares.keys()), - set(new_shares.keys())) + self.assertThat(set(initial_shares.keys()), + Equals(set(new_shares.keys()))) # but they should all be at a newer seqnum. The IV will be # different, so the roothash will be too. for key in initial_shares: @@ -70,19 +71,19 @@ class Repair(unittest.TestCase, PublishMixin, ShouldFailMixin): IV1, k1, N1, segsize1, datalen1, o1) = unpack_header(new_shares[key]) - self.failUnlessEqual(version0, version1) - self.failUnlessEqual(seqnum0+1, seqnum1) - self.failUnlessEqual(k0, k1) - self.failUnlessEqual(N0, N1) - self.failUnlessEqual(segsize0, segsize1) - self.failUnlessEqual(datalen0, datalen1) + self.assertThat(version0, Equals(version1)) + self.assertThat(seqnum0+1, Equals(seqnum1)) + self.assertThat(k0, Equals(k1)) + self.assertThat(N0, Equals(N1)) + self.assertThat(segsize0, Equals(segsize1)) + self.assertThat(datalen0, Equals(datalen1)) d.addCallback(_check_results) return d def failIfSharesChanged(self, ignored=None): old_shares = self.old_shares[-2] current_shares = self.old_shares[-1] - self.failUnlessEqual(old_shares, current_shares) + self.assertThat(old_shares, Equals(current_shares)) def _test_whether_repairable(self, publisher, nshares, expected_result): @@ -96,12 +97,12 @@ class Repair(unittest.TestCase, PublishMixin, ShouldFailMixin): d.addCallback(_delete_some_shares) d.addCallback(lambda ign: self._fn.check(Monitor())) def _check(cr): - self.failIf(cr.is_healthy()) - self.failUnlessEqual(cr.is_recoverable(), expected_result) + self.assertThat(cr.is_healthy(), Equals(False)) + self.assertThat(cr.is_recoverable(), Equals(expected_result)) return cr d.addCallback(_check) d.addCallback(lambda check_results: self._fn.repair(check_results)) - d.addCallback(lambda crr: self.failUnlessEqual(crr.get_successful(), expected_result)) + d.addCallback(lambda crr: self.assertThat(crr.get_successful(), Equals(expected_result))) return d def test_unrepairable_0shares(self): @@ -136,7 +137,7 @@ class Repair(unittest.TestCase, PublishMixin, ShouldFailMixin): del shares[peerid][shnum] d.addCallback(_delete_some_shares) d.addCallback(lambda ign: self._fn.check_and_repair(Monitor())) - d.addCallback(lambda crr: self.failUnlessEqual(crr.get_repair_successful(), expected_result)) + d.addCallback(lambda crr: self.assertThat(crr.get_repair_successful(), Equals(expected_result))) return d def test_unrepairable_0shares_checkandrepair(self): @@ -181,13 +182,13 @@ class Repair(unittest.TestCase, PublishMixin, ShouldFailMixin): self._fn.repair(check_results, force=True)) # this should give us 10 shares of the highest roothash def _check_repair_results(rres): - self.failUnless(rres.get_successful()) + self.assertThat(rres.get_successful(), Equals(True)) pass # TODO d.addCallback(_check_repair_results) d.addCallback(lambda res: self._fn.get_servermap(MODE_CHECK)) def _check_smap(smap): - self.failUnlessEqual(len(smap.recoverable_versions()), 1) - self.failIf(smap.unrecoverable_versions()) + self.assertThat(smap.recoverable_versions(), HasLength(1)) + self.assertThat(smap.unrecoverable_versions(), HasLength(0)) # now, which should have won? roothash_s4a = self.get_roothash_for(3) roothash_s4b = self.get_roothash_for(4) @@ -196,9 +197,9 @@ class Repair(unittest.TestCase, PublishMixin, ShouldFailMixin): else: expected_contents = self.CONTENTS[3] new_versionid = smap.best_recoverable_version() - self.failUnlessEqual(new_versionid[0], 5) # seqnum 5 + self.assertThat(new_versionid[0], Equals(5)) # seqnum 5 d2 = self._fn.download_version(smap, new_versionid) - d2.addCallback(self.failUnlessEqual, expected_contents) + d2.addCallback(self.assertEqual, expected_contents) return d2 d.addCallback(_check_smap) return d @@ -216,19 +217,19 @@ class Repair(unittest.TestCase, PublishMixin, ShouldFailMixin): d.addCallback(lambda check_results: self._fn.repair(check_results)) # this should give us 10 shares of v3 def _check_repair_results(rres): - self.failUnless(rres.get_successful()) + self.assertThat(rres.get_successful(), Equals(True)) pass # TODO d.addCallback(_check_repair_results) d.addCallback(lambda res: self._fn.get_servermap(MODE_CHECK)) def _check_smap(smap): - self.failUnlessEqual(len(smap.recoverable_versions()), 1) - self.failIf(smap.unrecoverable_versions()) + self.assertThat(smap.recoverable_versions(), HasLength(1)) + self.assertThat(smap.unrecoverable_versions(), HasLength(0)) # now, which should have won? expected_contents = self.CONTENTS[3] new_versionid = smap.best_recoverable_version() - self.failUnlessEqual(new_versionid[0], 5) # seqnum 5 + self.assertThat(new_versionid[0], Equals(5)) # seqnum 5 d2 = self._fn.download_version(smap, new_versionid) - d2.addCallback(self.failUnlessEqual, expected_contents) + d2.addCallback(self.assertTrue, expected_contents) return d2 d.addCallback(_check_smap) return d @@ -256,12 +257,12 @@ class Repair(unittest.TestCase, PublishMixin, ShouldFailMixin): d.addCallback(_get_readcap) d.addCallback(lambda res: self._fn3.check_and_repair(Monitor())) def _check_results(crr): - self.failUnless(ICheckAndRepairResults.providedBy(crr)) + self.assertThat(ICheckAndRepairResults.providedBy(crr), Equals(True)) # we should detect the unhealthy, but skip over mutable-readcap # repairs until #625 is fixed - self.failIf(crr.get_pre_repair_results().is_healthy()) - self.failIf(crr.get_repair_attempted()) - self.failIf(crr.get_post_repair_results().is_healthy()) + self.assertThat(crr.get_pre_repair_results().is_healthy(), Equals(False)) + self.assertThat(crr.get_repair_attempted(), Equals(False)) + self.assertThat(crr.get_post_repair_results().is_healthy(), Equals(False)) d.addCallback(_check_results) return d @@ -281,6 +282,6 @@ class Repair(unittest.TestCase, PublishMixin, ShouldFailMixin): d.addCallback(lambda ign: self._fn2.check(Monitor())) d.addCallback(lambda check_results: self._fn2.repair(check_results)) def _check(crr): - self.failUnlessEqual(crr.get_successful(), True) + self.assertThat(crr.get_successful(), Equals(True)) d.addCallback(_check) return d From 61b9f15fd1f27f428c1492a14ff3e998e07ac79b Mon Sep 17 00:00:00 2001 From: fenn-cs Date: Fri, 10 Sep 2021 00:59:55 +0100 Subject: [PATCH 0241/2309] test.mutable : refactored roundtrip and servermap tests Signed-off-by: fenn-cs --- src/allmydata/test/mutable/test_roundtrip.py | 44 +++++++-------- src/allmydata/test/mutable/test_servermap.py | 57 ++++++++++---------- 2 files changed, 52 insertions(+), 49 deletions(-) diff --git a/src/allmydata/test/mutable/test_roundtrip.py b/src/allmydata/test/mutable/test_roundtrip.py index 79292b000..96ecdf640 100644 --- a/src/allmydata/test/mutable/test_roundtrip.py +++ b/src/allmydata/test/mutable/test_roundtrip.py @@ -11,7 +11,8 @@ if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from six.moves import cStringIO as StringIO -from twisted.trial import unittest +from ..common import AsyncTestCase +from testtools.matchers import Equals, HasLength, Contains from twisted.internet import defer from allmydata.util import base32, consumer @@ -23,8 +24,9 @@ from allmydata.mutable.retrieve import Retrieve from .util import PublishMixin, make_storagebroker, corrupt from .. import common_util as testutil -class Roundtrip(unittest.TestCase, testutil.ShouldFailMixin, PublishMixin): +class Roundtrip(AsyncTestCase, testutil.ShouldFailMixin, PublishMixin): def setUp(self): + super(Roundtrip, self).setUp() return self.publish_one() def make_servermap(self, mode=MODE_READ, oldmap=None, sb=None): @@ -73,11 +75,11 @@ class Roundtrip(unittest.TestCase, testutil.ShouldFailMixin, PublishMixin): def _do_retrieve(servermap): self._smap = servermap #self.dump_servermap(servermap) - self.failUnlessEqual(len(servermap.recoverable_versions()), 1) + self.assertThat(servermap.recoverable_versions(), HasLength(1)) return self.do_download(servermap) d.addCallback(_do_retrieve) def _retrieved(new_contents): - self.failUnlessEqual(new_contents, self.CONTENTS) + self.assertThat(new_contents, Equals(self.CONTENTS)) d.addCallback(_retrieved) # we should be able to re-use the same servermap, both with and # without updating it. @@ -132,10 +134,10 @@ class Roundtrip(unittest.TestCase, testutil.ShouldFailMixin, PublishMixin): # back empty d = self.make_servermap(sb=sb2) def _check_servermap(servermap): - self.failUnlessEqual(servermap.best_recoverable_version(), None) - self.failIf(servermap.recoverable_versions()) - self.failIf(servermap.unrecoverable_versions()) - self.failIf(servermap.all_servers()) + self.assertThat(servermap.best_recoverable_version(), Equals(None)) + self.assertFalse(servermap.recoverable_versions()) + self.assertFalse(servermap.unrecoverable_versions()) + self.assertFalse(servermap.all_servers()) d.addCallback(_check_servermap) return d @@ -154,7 +156,7 @@ class Roundtrip(unittest.TestCase, testutil.ShouldFailMixin, PublishMixin): self._fn._storage_broker = self._storage_broker return self._fn.download_best_version() def _retrieved(new_contents): - self.failUnlessEqual(new_contents, self.CONTENTS) + self.assertThat(new_contents, Equals(self.CONTENTS)) d.addCallback(_restore) d.addCallback(_retrieved) return d @@ -178,13 +180,13 @@ class Roundtrip(unittest.TestCase, testutil.ShouldFailMixin, PublishMixin): # should be noted in the servermap's list of problems. if substring: allproblems = [str(f) for f in servermap.get_problems()] - self.failUnlessIn(substring, "".join(allproblems)) + self.assertThat("".join(allproblems), Contains(substring)) return servermap if should_succeed: d1 = self._fn.download_version(servermap, ver, fetch_privkey) d1.addCallback(lambda new_contents: - self.failUnlessEqual(new_contents, self.CONTENTS)) + self.assertThat(new_contents, Equals(self.CONTENTS))) else: d1 = self.shouldFail(NotEnoughSharesError, "_corrupt_all(offset=%s)" % (offset,), @@ -207,7 +209,7 @@ class Roundtrip(unittest.TestCase, testutil.ShouldFailMixin, PublishMixin): # and the dump should mention the problems s = StringIO() dump = servermap.dump(s).getvalue() - self.failUnless("30 PROBLEMS" in dump, dump) + self.assertTrue("30 PROBLEMS" in dump, msg=dump) d.addCallback(_check_servermap) return d @@ -299,8 +301,8 @@ class Roundtrip(unittest.TestCase, testutil.ShouldFailMixin, PublishMixin): # in NotEnoughSharesError, since each share will look invalid def _check(res): f = res[0] - self.failUnless(f.check(NotEnoughSharesError)) - self.failUnless("uncoordinated write" in str(f)) + self.assertThat(f.check(NotEnoughSharesError), HasLength(1)) + self.assertThat("uncoordinated write" in str(f), Equals(True)) return self._test_corrupt_all(1, "ran out of servers", corrupt_early=False, failure_checker=_check) @@ -309,7 +311,7 @@ class Roundtrip(unittest.TestCase, testutil.ShouldFailMixin, PublishMixin): def test_corrupt_all_block_late(self): def _check(res): f = res[0] - self.failUnless(f.check(NotEnoughSharesError)) + self.assertTrue(f.check(NotEnoughSharesError)) return self._test_corrupt_all("share_data", "block hash tree failure", corrupt_early=False, failure_checker=_check) @@ -330,9 +332,9 @@ class Roundtrip(unittest.TestCase, testutil.ShouldFailMixin, PublishMixin): shnums_to_corrupt=list(range(0, N-k))) d.addCallback(lambda res: self.make_servermap()) def _do_retrieve(servermap): - self.failUnless(servermap.get_problems()) - self.failUnless("pubkey doesn't match fingerprint" - in str(servermap.get_problems()[0])) + self.assertTrue(servermap.get_problems()) + self.assertThat("pubkey doesn't match fingerprint" + in str(servermap.get_problems()[0]), Equals(True)) ver = servermap.best_recoverable_version() r = Retrieve(self._fn, self._storage_broker, servermap, ver) c = consumer.MemoryConsumer() @@ -340,7 +342,7 @@ class Roundtrip(unittest.TestCase, testutil.ShouldFailMixin, PublishMixin): d.addCallback(_do_retrieve) d.addCallback(lambda mc: b"".join(mc.chunks)) d.addCallback(lambda new_contents: - self.failUnlessEqual(new_contents, self.CONTENTS)) + self.assertThat(new_contents, Equals(self.CONTENTS))) return d @@ -355,11 +357,11 @@ class Roundtrip(unittest.TestCase, testutil.ShouldFailMixin, PublishMixin): self.make_servermap()) def _do_retrieve(servermap): ver = servermap.best_recoverable_version() - self.failUnless(ver) + self.assertTrue(ver) return self._fn.download_best_version() d.addCallback(_do_retrieve) d.addCallback(lambda new_contents: - self.failUnlessEqual(new_contents, self.CONTENTS)) + self.assertThat(new_contents, Equals(self.CONTENTS))) return d diff --git a/src/allmydata/test/mutable/test_servermap.py b/src/allmydata/test/mutable/test_servermap.py index e8f933977..505d31e73 100644 --- a/src/allmydata/test/mutable/test_servermap.py +++ b/src/allmydata/test/mutable/test_servermap.py @@ -11,7 +11,8 @@ from future.utils import PY2 if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 -from twisted.trial import unittest +from ..common import AsyncTestCase +from testtools.matchers import Equals, NotEquals, HasLength from twisted.internet import defer from allmydata.monitor import Monitor from allmydata.mutable.common import \ @@ -20,8 +21,9 @@ from allmydata.mutable.publish import MutableData from allmydata.mutable.servermap import ServerMap, ServermapUpdater from .util import PublishMixin -class Servermap(unittest.TestCase, PublishMixin): +class Servermap(AsyncTestCase, PublishMixin): def setUp(self): + super(Servermap, self).setUp() return self.publish_one() def make_servermap(self, mode=MODE_CHECK, fn=None, sb=None, @@ -42,17 +44,17 @@ class Servermap(unittest.TestCase, PublishMixin): return d def failUnlessOneRecoverable(self, sm, num_shares): - self.failUnlessEqual(len(sm.recoverable_versions()), 1) - self.failUnlessEqual(len(sm.unrecoverable_versions()), 0) + self.assertThat(sm.recoverable_versions(), HasLength(1)) + self.assertThat(sm.unrecoverable_versions(), HasLength(0)) best = sm.best_recoverable_version() - self.failIfEqual(best, None) - self.failUnlessEqual(sm.recoverable_versions(), set([best])) - self.failUnlessEqual(len(sm.shares_available()), 1) - self.failUnlessEqual(sm.shares_available()[best], (num_shares, 3, 10)) + self.assertThat(best, NotEquals(None)) + self.assertThat(sm.recoverable_versions(), Equals(set([best]))) + self.assertThat(sm.shares_available(), HasLength(1)) + self.assertThat(sm.shares_available()[best], Equals((num_shares, 3, 10))) shnum, servers = list(sm.make_sharemap().items())[0] server = list(servers)[0] - self.failUnlessEqual(sm.version_on_server(server, shnum), best) - self.failUnlessEqual(sm.version_on_server(server, 666), None) + self.assertThat(sm.version_on_server(server, shnum), Equals(best)) + self.assertThat(sm.version_on_server(server, 666), Equals(None)) return sm def test_basic(self): @@ -117,7 +119,7 @@ class Servermap(unittest.TestCase, PublishMixin): v = sm.best_recoverable_version() vm = sm.make_versionmap() shares = list(vm[v]) - self.failUnlessEqual(len(shares), 6) + self.assertThat(shares, HasLength(6)) self._corrupted = set() # mark the first 5 shares as corrupt, then update the servermap. # The map should not have the marked shares it in any more, and @@ -135,18 +137,17 @@ class Servermap(unittest.TestCase, PublishMixin): shares = list(vm[v]) for (server, shnum) in self._corrupted: server_shares = sm.debug_shares_on_server(server) - self.failIf(shnum in server_shares, - "%d was in %s" % (shnum, server_shares)) - self.failUnlessEqual(len(shares), 5) + self.assertFalse(shnum in server_shares, "%d was in %s" % (shnum, server_shares)) + self.assertThat(shares, HasLength(5)) d.addCallback(_check_map) return d def failUnlessNoneRecoverable(self, sm): - self.failUnlessEqual(len(sm.recoverable_versions()), 0) - self.failUnlessEqual(len(sm.unrecoverable_versions()), 0) + self.assertThat(sm.recoverable_versions(), HasLength(0)) + self.assertThat(sm.unrecoverable_versions(), HasLength(0)) best = sm.best_recoverable_version() - self.failUnlessEqual(best, None) - self.failUnlessEqual(len(sm.shares_available()), 0) + self.assertThat(best, Equals(None)) + self.assertThat(sm.shares_available(), HasLength(0)) def test_no_shares(self): self._storage._peers = {} # delete all shares @@ -168,12 +169,12 @@ class Servermap(unittest.TestCase, PublishMixin): return d def failUnlessNotQuiteEnough(self, sm): - self.failUnlessEqual(len(sm.recoverable_versions()), 0) - self.failUnlessEqual(len(sm.unrecoverable_versions()), 1) + self.assertThat(sm.recoverable_versions(), HasLength(0)) + self.assertThat(sm.unrecoverable_versions(), HasLength(1)) best = sm.best_recoverable_version() - self.failUnlessEqual(best, None) - self.failUnlessEqual(len(sm.shares_available()), 1) - self.failUnlessEqual(list(sm.shares_available().values())[0], (2,3,10) ) + self.assertThat(best, Equals(None)) + self.assertThat(sm.shares_available(), HasLength(1)) + self.assertThat(list(sm.shares_available().values())[0], Equals((2,3,10))) return sm def test_not_quite_enough_shares(self): @@ -193,7 +194,7 @@ class Servermap(unittest.TestCase, PublishMixin): d.addCallback(lambda res: ms(mode=MODE_CHECK)) d.addCallback(lambda sm: self.failUnlessNotQuiteEnough(sm)) d.addCallback(lambda sm: - self.failUnlessEqual(len(sm.make_sharemap()), 2)) + self.assertThat(sm.make_sharemap(), HasLength(2))) d.addCallback(lambda res: ms(mode=MODE_ANYTHING)) d.addCallback(lambda sm: self.failUnlessNotQuiteEnough(sm)) d.addCallback(lambda res: ms(mode=MODE_WRITE)) @@ -216,7 +217,7 @@ class Servermap(unittest.TestCase, PublishMixin): # Calling make_servermap also updates the servermap in the mode # that we specify, so we just need to see what it says. def _check_servermap(sm): - self.failUnlessEqual(len(sm.recoverable_versions()), 1) + self.assertThat(sm.recoverable_versions(), HasLength(1)) d.addCallback(_check_servermap) return d @@ -229,10 +230,10 @@ class Servermap(unittest.TestCase, PublishMixin): self.make_servermap(mode=MODE_WRITE, update_range=(1, 2))) def _check_servermap(sm): # 10 shares - self.failUnlessEqual(len(sm.update_data), 10) + self.assertThat(sm.update_data, HasLength(10)) # one version for data in sm.update_data.values(): - self.failUnlessEqual(len(data), 1) + self.assertThat(data, HasLength(1)) d.addCallback(_check_servermap) return d @@ -244,5 +245,5 @@ class Servermap(unittest.TestCase, PublishMixin): d.addCallback(lambda ignored: self.make_servermap(mode=MODE_CHECK)) d.addCallback(lambda servermap: - self.failUnlessEqual(len(servermap.recoverable_versions()), 1)) + self.assertThat(servermap.recoverable_versions(), HasLength(1))) return d From 3b80b8cbe97a14e9e14e46a0a329c9e6bdcc8c12 Mon Sep 17 00:00:00 2001 From: fenn-cs Date: Fri, 10 Sep 2021 14:24:20 +0100 Subject: [PATCH 0242/2309] test.mutable : refactored test_version.py Signed-off-by: fenn-cs --- newsfragments/3788.minor | 0 src/allmydata/test/mutable/test_version.py | 116 +++++++++++---------- 2 files changed, 61 insertions(+), 55 deletions(-) create mode 100644 newsfragments/3788.minor diff --git a/newsfragments/3788.minor b/newsfragments/3788.minor new file mode 100644 index 000000000..e69de29bb diff --git a/src/allmydata/test/mutable/test_version.py b/src/allmydata/test/mutable/test_version.py index 042305c24..d5c44f204 100644 --- a/src/allmydata/test/mutable/test_version.py +++ b/src/allmydata/test/mutable/test_version.py @@ -14,7 +14,13 @@ import os from six.moves import cStringIO as StringIO from twisted.internet import defer -from twisted.trial import unittest +from ..common import AsyncTestCase +from testtools.matchers import ( + Equals, + IsInstance, + HasLength, + Contains, +) from allmydata import uri from allmydata.interfaces import SDMF_VERSION, MDMF_VERSION @@ -29,7 +35,7 @@ from ..no_network import GridTestMixin from .util import PublishMixin from .. import common_util as testutil -class Version(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin, \ +class Version(GridTestMixin, AsyncTestCase, testutil.ShouldFailMixin, \ PublishMixin): def setUp(self): GridTestMixin.setUp(self) @@ -47,8 +53,8 @@ class Version(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin, \ d = self.nm.create_mutable_file(MutableData(data), version=MDMF_VERSION) def _then(n): - assert isinstance(n, MutableFileNode) - assert n._protocol_version == MDMF_VERSION + self.assertThat(n, IsInstance(MutableFileNode)) + self.assertThat(n._protocol_version, Equals(MDMF_VERSION)) self.mdmf_node = n return n d.addCallback(_then) @@ -59,8 +65,8 @@ class Version(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin, \ data = self.small_data d = self.nm.create_mutable_file(MutableData(data)) def _then(n): - assert isinstance(n, MutableFileNode) - assert n._protocol_version == SDMF_VERSION + self.assertThat(n, IsInstance(MutableFileNode)) + self.assertThat(n._protocol_version, Equals(SDMF_VERSION)) self.sdmf_node = n return n d.addCallback(_then) @@ -69,9 +75,9 @@ class Version(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin, \ def do_upload_empty_sdmf(self): d = self.nm.create_mutable_file(MutableData(b"")) def _then(n): - assert isinstance(n, MutableFileNode) + self.assertThat(n, IsInstance(MutableFileNode)) self.sdmf_zero_length_node = n - assert n._protocol_version == SDMF_VERSION + self.assertThat(n._protocol_version, Equals(SDMF_VERSION)) return n d.addCallback(_then) return d @@ -95,7 +101,7 @@ class Version(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin, \ debug.find_shares(fso) sharefiles = fso.stdout.getvalue().splitlines() expected = self.nm.default_encoding_parameters["n"] - self.failUnlessEqual(len(sharefiles), expected) + self.assertThat(sharefiles, HasLength(expected)) do = debug.DumpOptions() do["filename"] = sharefiles[0] @@ -103,17 +109,17 @@ class Version(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin, \ debug.dump_share(do) output = do.stdout.getvalue() lines = set(output.splitlines()) - self.failUnless("Mutable slot found:" in lines, output) - self.failUnless(" share_type: MDMF" in lines, output) - self.failUnless(" num_extra_leases: 0" in lines, output) - self.failUnless(" MDMF contents:" in lines, output) - self.failUnless(" seqnum: 1" in lines, output) - self.failUnless(" required_shares: 3" in lines, output) - self.failUnless(" total_shares: 10" in lines, output) - self.failUnless(" segsize: 131073" in lines, output) - self.failUnless(" datalen: %d" % len(self.data) in lines, output) + self.assertTrue("Mutable slot found:" in lines, output) + self.assertTrue(" share_type: MDMF" in lines, output) + self.assertTrue(" num_extra_leases: 0" in lines, output) + self.assertTrue(" MDMF contents:" in lines, output) + self.assertTrue(" seqnum: 1" in lines, output) + self.assertTrue(" required_shares: 3" in lines, output) + self.assertTrue(" total_shares: 10" in lines, output) + self.assertTrue(" segsize: 131073" in lines, output) + self.assertTrue(" datalen: %d" % len(self.data) in lines, output) vcap = str(n.get_verify_cap().to_string(), "utf-8") - self.failUnless(" verify-cap: %s" % vcap in lines, output) + self.assertTrue(" verify-cap: %s" % vcap in lines, output) cso = debug.CatalogSharesOptions() cso.nodedirs = fso.nodedirs cso.stdout = StringIO() @@ -122,13 +128,13 @@ class Version(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin, \ shares = cso.stdout.getvalue().splitlines() oneshare = shares[0] # all shares should be MDMF self.failIf(oneshare.startswith("UNKNOWN"), oneshare) - self.failUnless(oneshare.startswith("MDMF"), oneshare) + self.assertTrue(oneshare.startswith("MDMF"), oneshare) fields = oneshare.split() - self.failUnlessEqual(fields[0], "MDMF") - self.failUnlessEqual(fields[1].encode("ascii"), storage_index) - self.failUnlessEqual(fields[2], "3/10") - self.failUnlessEqual(fields[3], "%d" % len(self.data)) - self.failUnless(fields[4].startswith("#1:"), fields[3]) + self.assertThat(fields[0], Equals("MDMF")) + self.assertThat(fields[1].encode("ascii"), Equals(storage_index)) + self.assertThat(fields[2], Equals("3/10")) + self.assertThat(fields[3], Equals("%d" % len(self.data))) + self.assertTrue(fields[4].startswith("#1:"), fields[3]) # the rest of fields[4] is the roothash, which depends upon # encryption salts and is not constant. fields[5] is the # remaining time on the longest lease, which is timing dependent. @@ -140,11 +146,11 @@ class Version(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin, \ d = self.do_upload() d.addCallback(lambda ign: self.mdmf_node.get_best_readable_version()) d.addCallback(lambda bv: - self.failUnlessEqual(bv.get_sequence_number(), 1)) + self.assertThat(bv.get_sequence_number(), Equals(1))) d.addCallback(lambda ignored: self.sdmf_node.get_best_readable_version()) d.addCallback(lambda bv: - self.failUnlessEqual(bv.get_sequence_number(), 1)) + self.assertThat(bv.get_sequence_number(), Equals(1))) # Now update. The sequence number in both cases should be 1 in # both cases. def _do_update(ignored): @@ -158,11 +164,11 @@ class Version(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin, \ d.addCallback(lambda ignored: self.mdmf_node.get_best_readable_version()) d.addCallback(lambda bv: - self.failUnlessEqual(bv.get_sequence_number(), 2)) + self.assertThat(bv.get_sequence_number(), Equals(2))) d.addCallback(lambda ignored: self.sdmf_node.get_best_readable_version()) d.addCallback(lambda bv: - self.failUnlessEqual(bv.get_sequence_number(), 2)) + self.assertThat(bv.get_sequence_number(), Equals(2))) return d @@ -175,10 +181,10 @@ class Version(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin, \ def _then(ign): mdmf_uri = self.mdmf_node.get_uri() cap = uri.from_string(mdmf_uri) - self.failUnless(isinstance(cap, uri.WriteableMDMFFileURI)) + self.assertTrue(isinstance(cap, uri.WriteableMDMFFileURI)) readonly_mdmf_uri = self.mdmf_node.get_readonly_uri() cap = uri.from_string(readonly_mdmf_uri) - self.failUnless(isinstance(cap, uri.ReadonlyMDMFFileURI)) + self.assertTrue(isinstance(cap, uri.ReadonlyMDMFFileURI)) d.addCallback(_then) return d @@ -189,16 +195,16 @@ class Version(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin, \ d.addCallback(lambda ign: self.mdmf_node.get_best_mutable_version()) def _check_mdmf(bv): n = self.mdmf_node - self.failUnlessEqual(bv.get_writekey(), n.get_writekey()) - self.failUnlessEqual(bv.get_storage_index(), n.get_storage_index()) - self.failIf(bv.is_readonly()) + self.assertThat(bv.get_writekey(), Equals(n.get_writekey())) + self.assertThat(bv.get_storage_index(), Equals(n.get_storage_index())) + self.assertFalse(bv.is_readonly()) d.addCallback(_check_mdmf) d.addCallback(lambda ign: self.sdmf_node.get_best_mutable_version()) def _check_sdmf(bv): n = self.sdmf_node - self.failUnlessEqual(bv.get_writekey(), n.get_writekey()) - self.failUnlessEqual(bv.get_storage_index(), n.get_storage_index()) - self.failIf(bv.is_readonly()) + self.assertThat(bv.get_writekey(), Equals(n.get_writekey())) + self.assertThat(bv.get_storage_index(), Equals(n.get_storage_index())) + self.assertFalse(bv.is_readonly()) d.addCallback(_check_sdmf) return d @@ -206,21 +212,21 @@ class Version(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin, \ def test_get_readonly_version(self): d = self.do_upload() d.addCallback(lambda ign: self.mdmf_node.get_best_readable_version()) - d.addCallback(lambda bv: self.failUnless(bv.is_readonly())) + d.addCallback(lambda bv: self.assertTrue(bv.is_readonly())) # Attempting to get a mutable version of a mutable file from a # filenode initialized with a readcap should return a readonly # version of that same node. d.addCallback(lambda ign: self.mdmf_node.get_readonly()) d.addCallback(lambda ro: ro.get_best_mutable_version()) - d.addCallback(lambda v: self.failUnless(v.is_readonly())) + d.addCallback(lambda v: self.assertTrue(v.is_readonly())) d.addCallback(lambda ign: self.sdmf_node.get_best_readable_version()) - d.addCallback(lambda bv: self.failUnless(bv.is_readonly())) + d.addCallback(lambda bv: self.assertTrue(bv.is_readonly())) d.addCallback(lambda ign: self.sdmf_node.get_readonly()) d.addCallback(lambda ro: ro.get_best_mutable_version()) - d.addCallback(lambda v: self.failUnless(v.is_readonly())) + d.addCallback(lambda v: self.assertTrue(v.is_readonly())) return d @@ -232,13 +238,13 @@ class Version(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin, \ d.addCallback(lambda ignored: self.mdmf_node.download_best_version()) d.addCallback(lambda data: - self.failUnlessEqual(data, b"foo bar baz" * 100000)) + self.assertThat(data, Equals(b"foo bar baz" * 100000))) d.addCallback(lambda ignored: self.sdmf_node.overwrite(new_small_data)) d.addCallback(lambda ignored: self.sdmf_node.download_best_version()) d.addCallback(lambda data: - self.failUnlessEqual(data, b"foo bar baz" * 10)) + self.assertThat(data, Equals(b"foo bar baz" * 10))) return d @@ -250,13 +256,13 @@ class Version(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin, \ d.addCallback(lambda ignored: self.mdmf_node.download_best_version()) d.addCallback(lambda data: - self.failUnlessIn(b"modified", data)) + self.assertThat(data, Contains(b"modified"))) d.addCallback(lambda ignored: self.sdmf_node.modify(modifier)) d.addCallback(lambda ignored: self.sdmf_node.download_best_version()) d.addCallback(lambda data: - self.failUnlessIn(b"modified", data)) + self.assertThat(data, Contains(b"modified"))) return d @@ -271,13 +277,13 @@ class Version(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin, \ d.addCallback(lambda ignored: self.mdmf_node.download_best_version()) d.addCallback(lambda data: - self.failUnlessIn(b"modified", data)) + self.assertThat(data, Contains(b"modified"))) d.addCallback(lambda ignored: self.sdmf_node.modify(modifier)) d.addCallback(lambda ignored: self.sdmf_node.download_best_version()) d.addCallback(lambda data: - self.failUnlessIn(b"modified", data)) + self.assertThat(data, Contains(b"modified"))) return d @@ -308,13 +314,13 @@ class Version(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin, \ d.addCallback(lambda ignored: self._fn.download_version(self.servermap, self.version1)) d.addCallback(lambda results: - self.failUnlessEqual(self.CONTENTS[self.version1_index], - results)) + self.assertThat(self.CONTENTS[self.version1_index], + Equals(results))) d.addCallback(lambda ignored: self._fn.download_version(self.servermap, self.version2)) d.addCallback(lambda results: - self.failUnlessEqual(self.CONTENTS[self.version2_index], - results)) + self.assertThat(self.CONTENTS[self.version2_index], + Equals(results))) return d @@ -344,7 +350,7 @@ class Version(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin, \ for i in range(0, len(expected), step): d2.addCallback(lambda ignored, i=i: version.read(c, i, step)) d2.addCallback(lambda ignored: - self.failUnlessEqual(expected, b"".join(c.chunks))) + self.assertThat(expected, Equals(b"".join(c.chunks)))) return d2 d.addCallback(_read_data) return d @@ -447,16 +453,16 @@ class Version(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin, \ d2 = defer.succeed(None) d2.addCallback(lambda ignored: version.read(c)) d2.addCallback(lambda ignored: - self.failUnlessEqual(expected, b"".join(c.chunks))) + self.assertThat(expected, Equals(b"".join(c.chunks)))) d2.addCallback(lambda ignored: version.read(c2, offset=0, size=len(expected))) d2.addCallback(lambda ignored: - self.failUnlessEqual(expected, b"".join(c2.chunks))) + self.assertThat(expected, Equals(b"".join(c2.chunks)))) return d2 d.addCallback(_read_data) d.addCallback(lambda ignored: node.download_best_version()) - d.addCallback(lambda data: self.failUnlessEqual(expected, data)) + d.addCallback(lambda data: self.assertThat(expected, Equals(data))) return d def test_read_and_download_mdmf(self): From a3168b384450dbd4a9a576425442d0a92267e950 Mon Sep 17 00:00:00 2001 From: fenn-cs Date: Mon, 13 Sep 2021 23:45:16 +0100 Subject: [PATCH 0243/2309] test.mutable : refactored test_update.py Signed-off-by: fenn-cs --- src/allmydata/test/mutable/test_update.py | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/src/allmydata/test/mutable/test_update.py b/src/allmydata/test/mutable/test_update.py index da5d53e4c..c3ba1e9f7 100644 --- a/src/allmydata/test/mutable/test_update.py +++ b/src/allmydata/test/mutable/test_update.py @@ -11,7 +11,12 @@ if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import re -from twisted.trial import unittest +from ..common import AsyncTestCase +from testtools.matchers import ( + Equals, + IsInstance, + GreaterThan, +) from twisted.internet import defer from allmydata.interfaces import MDMF_VERSION from allmydata.mutable.filenode import MutableFileNode @@ -25,7 +30,7 @@ from .. import common_util as testutil # this up. SEGSIZE = 128*1024 -class Update(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin): +class Update(GridTestMixin, AsyncTestCase, testutil.ShouldFailMixin): def setUp(self): GridTestMixin.setUp(self) self.basedir = self.mktemp() @@ -35,14 +40,14 @@ class Update(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin): # self.data should be at least three segments long. td = b"testdata " self.data = td*(int(3*SEGSIZE//len(td))+10) # currently about 400kB - assert len(self.data) > 3*SEGSIZE + self.assertThat(len(self.data), GreaterThan(3*SEGSIZE)) self.small_data = b"test data" * 10 # 90 B; SDMF def do_upload_sdmf(self): d = self.nm.create_mutable_file(MutableData(self.small_data)) def _then(n): - assert isinstance(n, MutableFileNode) + self.assertThat(n, IsInstance(MutableFileNode)) self.sdmf_node = n d.addCallback(_then) return d @@ -51,7 +56,7 @@ class Update(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin): d = self.nm.create_mutable_file(MutableData(self.data), version=MDMF_VERSION) def _then(n): - assert isinstance(n, MutableFileNode) + self.assertThat(n, IsInstance(MutableFileNode)) self.mdmf_node = n d.addCallback(_then) return d @@ -185,7 +190,7 @@ class Update(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin): len(self.data))) d.addCallback(lambda ign: self.mdmf_node.download_best_version()) d.addCallback(lambda results: - self.failUnlessEqual(results, new_data)) + self.assertThat(results, Equals(new_data))) return d d0.addCallback(_run) return d0 @@ -201,7 +206,7 @@ class Update(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin): len(self.small_data))) d.addCallback(lambda ign: self.sdmf_node.download_best_version()) d.addCallback(lambda results: - self.failUnlessEqual(results, new_data)) + self.assertThat(results, Equals(new_data))) return d d0.addCallback(_run) return d0 @@ -221,7 +226,7 @@ class Update(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin): replace_offset)) d.addCallback(lambda ign: self.mdmf_node.download_best_version()) d.addCallback(lambda results: - self.failUnlessEqual(results, new_data)) + self.assertThat(results, Equals(new_data))) return d d0.addCallback(_run) return d0 @@ -242,7 +247,7 @@ class Update(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin): replace_offset)) d.addCallback(lambda ignored: self.mdmf_node.download_best_version()) d.addCallback(lambda results: - self.failUnlessEqual(results, new_data)) + self.assertThat(results, Equals(new_data))) return d d0.addCallback(_run) return d0 From 5db3540c206b8bc9b12b50d61800b669334e3555 Mon Sep 17 00:00:00 2001 From: fenn-cs Date: Fri, 17 Sep 2021 10:56:28 +0100 Subject: [PATCH 0244/2309] update NEWS.txt for release --- NEWS.rst | 98 ++++++++++++++++++++++++++++++++ newsfragments/1549.installation | 1 - newsfragments/2928.minor | 0 newsfragments/3037.other | 1 - newsfragments/3283.minor | 0 newsfragments/3314.minor | 0 newsfragments/3326.installation | 1 - newsfragments/3384.minor | 0 newsfragments/3385.minor | 0 newsfragments/3390.minor | 0 newsfragments/3399.feature | 1 - newsfragments/3404.minor | 0 newsfragments/3428.minor | 0 newsfragments/3432.minor | 0 newsfragments/3433.installation | 1 - newsfragments/3434.minor | 0 newsfragments/3435.minor | 0 newsfragments/3454.minor | 0 newsfragments/3459.minor | 0 newsfragments/3460.minor | 0 newsfragments/3465.minor | 0 newsfragments/3466.minor | 0 newsfragments/3467.minor | 0 newsfragments/3468.minor | 0 newsfragments/3470.minor | 0 newsfragments/3471.minor | 0 newsfragments/3472.minor | 0 newsfragments/3473.minor | 0 newsfragments/3474.minor | 0 newsfragments/3475.minor | 0 newsfragments/3477.minor | 0 newsfragments/3478.minor | 1 - newsfragments/3479.minor | 0 newsfragments/3481.minor | 0 newsfragments/3482.minor | 0 newsfragments/3483.minor | 0 newsfragments/3485.minor | 0 newsfragments/3486.installation | 1 - newsfragments/3488.minor | 0 newsfragments/3490.minor | 0 newsfragments/3491.minor | 0 newsfragments/3492.minor | 0 newsfragments/3493.minor | 0 newsfragments/3496.minor | 0 newsfragments/3497.installation | 1 - newsfragments/3499.minor | 0 newsfragments/3500.minor | 0 newsfragments/3501.minor | 0 newsfragments/3502.minor | 0 newsfragments/3503.other | 1 - newsfragments/3504.configuration | 1 - newsfragments/3509.bugfix | 1 - newsfragments/3510.bugfix | 1 - newsfragments/3511.minor | 0 newsfragments/3513.minor | 0 newsfragments/3514.minor | 0 newsfragments/3515.minor | 0 newsfragments/3517.minor | 0 newsfragments/3518.removed | 1 - newsfragments/3520.minor | 0 newsfragments/3521.minor | 0 newsfragments/3522.minor | 0 newsfragments/3523.minor | 0 newsfragments/3524.minor | 0 newsfragments/3528.minor | 0 newsfragments/3529.minor | 0 newsfragments/3532.minor | 0 newsfragments/3533.minor | 0 newsfragments/3534.minor | 0 newsfragments/3536.minor | 0 newsfragments/3537.minor | 0 newsfragments/3539.bugfix | 1 - newsfragments/3542.minor | 0 newsfragments/3544.minor | 0 newsfragments/3545.other | 1 - newsfragments/3546.minor | 0 newsfragments/3547.minor | 0 newsfragments/3549.removed | 1 - newsfragments/3550.removed | 1 - newsfragments/3551.minor | 0 newsfragments/3552.minor | 0 newsfragments/3553.minor | 0 newsfragments/3555.minor | 0 newsfragments/3557.minor | 0 newsfragments/3558.minor | 0 newsfragments/3560.minor | 0 newsfragments/3563.minor | 0 newsfragments/3564.minor | 0 newsfragments/3565.minor | 0 newsfragments/3566.minor | 0 newsfragments/3567.minor | 0 newsfragments/3568.minor | 0 newsfragments/3572.minor | 0 newsfragments/3574.minor | 0 newsfragments/3575.minor | 0 newsfragments/3576.minor | 0 newsfragments/3577.minor | 0 newsfragments/3578.minor | 0 newsfragments/3579.minor | 0 newsfragments/3580.minor | 0 newsfragments/3582.minor | 0 newsfragments/3583.removed | 1 - newsfragments/3584.bugfix | 1 - newsfragments/3587.minor | 1 - newsfragments/3588.incompat | 1 - newsfragments/3588.minor | 0 newsfragments/3589.minor | 0 newsfragments/3590.bugfix | 1 - newsfragments/3591.minor | 0 newsfragments/3592.minor | 0 newsfragments/3593.minor | 0 newsfragments/3594.minor | 0 newsfragments/3595.minor | 0 newsfragments/3596.minor | 0 newsfragments/3599.minor | 0 newsfragments/3600.minor | 0 newsfragments/3603.minor.rst | 0 newsfragments/3605.minor | 0 newsfragments/3606.minor | 0 newsfragments/3607.minor | 0 newsfragments/3608.minor | 0 newsfragments/3611.minor | 0 newsfragments/3612.minor | 0 newsfragments/3613.minor | 0 newsfragments/3615.minor | 0 newsfragments/3616.minor | 0 newsfragments/3617.minor | 0 newsfragments/3618.minor | 0 newsfragments/3619.minor | 0 newsfragments/3620.minor | 0 newsfragments/3621.minor | 0 newsfragments/3623.minor | 1 - newsfragments/3624.minor | 0 newsfragments/3625.minor | 0 newsfragments/3626.minor | 0 newsfragments/3628.minor | 0 newsfragments/3629.feature | 1 - newsfragments/3630.minor | 0 newsfragments/3631.minor | 0 newsfragments/3632.minor | 0 newsfragments/3633.installation | 1 - newsfragments/3634.minor | 0 newsfragments/3635.minor | 0 newsfragments/3637.minor | 0 newsfragments/3638.minor | 0 newsfragments/3640.minor | 0 newsfragments/3642.minor | 0 newsfragments/3644.other | 1 - newsfragments/3645.minor | 0 newsfragments/3646.minor | 0 newsfragments/3647.minor | 0 newsfragments/3648.minor | 0 newsfragments/3649.minor | 0 newsfragments/3650.bugfix | 1 - newsfragments/3651.minor | 1 - newsfragments/3652.removed | 1 - newsfragments/3653.minor | 0 newsfragments/3654.minor | 0 newsfragments/3655.minor | 0 newsfragments/3656.minor | 0 newsfragments/3657.minor | 0 newsfragments/3658.minor | 0 newsfragments/3659.documentation | 0 newsfragments/3662.minor | 0 newsfragments/3663.other | 1 - newsfragments/3664.documentation | 1 - newsfragments/3666.documentation | 1 - newsfragments/3667.minor | 0 newsfragments/3669.minor | 0 newsfragments/3670.minor | 0 newsfragments/3671.minor | 0 newsfragments/3672.minor | 0 newsfragments/3674.minor | 0 newsfragments/3675.minor | 0 newsfragments/3676.minor | 0 newsfragments/3677.documentation | 1 - newsfragments/3678.minor | 0 newsfragments/3679.minor | 0 newsfragments/3681.minor | 8 --- newsfragments/3682.documentation | 1 - newsfragments/3683.minor | 0 newsfragments/3686.minor | 0 newsfragments/3687.minor | 0 newsfragments/3691.minor | 0 newsfragments/3692.minor | 0 newsfragments/3699.minor | 0 newsfragments/3700.minor | 0 newsfragments/3701.minor | 0 newsfragments/3702.minor | 0 newsfragments/3703.minor | 0 newsfragments/3704.minor | 0 newsfragments/3705.minor | 0 newsfragments/3707.minor | 0 newsfragments/3708.minor | 0 newsfragments/3709.minor | 0 newsfragments/3711.minor | 0 newsfragments/3712.installation | 1 - newsfragments/3713.minor | 0 newsfragments/3714.minor | 0 newsfragments/3715.minor | 0 newsfragments/3716.incompat | 1 - newsfragments/3717.minor | 0 newsfragments/3718.minor | 0 newsfragments/3721.documentation | 1 - newsfragments/3722.minor | 0 newsfragments/3723.minor | 0 newsfragments/3726.documentation | 1 - newsfragments/3727.minor | 0 newsfragments/3728.minor | 0 newsfragments/3729.minor | 0 newsfragments/3730.minor | 0 newsfragments/3731.minor | 0 newsfragments/3732.minor | 0 newsfragments/3733.installation | 1 - newsfragments/3734.minor | 0 newsfragments/3735.minor | 0 newsfragments/3736.minor | 0 newsfragments/3738.bugfix | 1 - newsfragments/3739.bugfix | 1 - newsfragments/3741.minor | 0 newsfragments/3743.minor | 0 newsfragments/3744.minor | 0 newsfragments/3745.minor | 0 newsfragments/3746.minor | 0 newsfragments/3747.documentation | 1 - newsfragments/3749.documentation | 1 - newsfragments/3751.minor | 0 newsfragments/3757.other | 1 - newsfragments/3759.minor | 0 newsfragments/3760.minor | 0 newsfragments/3763.minor | 0 newsfragments/3764.documentation | 1 - newsfragments/3765.documentation | 1 - newsfragments/3769.documentation | 1 - newsfragments/3773.minor | 0 newsfragments/3774.documentation | 1 - newsfragments/3777.documentation | 1 - newsfragments/3779.bugfix | 1 - newsfragments/3781.minor | 0 newsfragments/3782.documentation | 1 - newsfragments/3785.documentation | 1 - 241 files changed, 98 insertions(+), 60 deletions(-) delete mode 100644 newsfragments/1549.installation delete mode 100644 newsfragments/2928.minor delete mode 100644 newsfragments/3037.other delete mode 100644 newsfragments/3283.minor delete mode 100644 newsfragments/3314.minor delete mode 100644 newsfragments/3326.installation delete mode 100644 newsfragments/3384.minor delete mode 100644 newsfragments/3385.minor delete mode 100644 newsfragments/3390.minor delete mode 100644 newsfragments/3399.feature delete mode 100644 newsfragments/3404.minor delete mode 100644 newsfragments/3428.minor delete mode 100644 newsfragments/3432.minor delete mode 100644 newsfragments/3433.installation delete mode 100644 newsfragments/3434.minor delete mode 100644 newsfragments/3435.minor delete mode 100644 newsfragments/3454.minor delete mode 100644 newsfragments/3459.minor delete mode 100644 newsfragments/3460.minor delete mode 100644 newsfragments/3465.minor delete mode 100644 newsfragments/3466.minor delete mode 100644 newsfragments/3467.minor delete mode 100644 newsfragments/3468.minor delete mode 100644 newsfragments/3470.minor delete mode 100644 newsfragments/3471.minor delete mode 100644 newsfragments/3472.minor delete mode 100644 newsfragments/3473.minor delete mode 100644 newsfragments/3474.minor delete mode 100644 newsfragments/3475.minor delete mode 100644 newsfragments/3477.minor delete mode 100644 newsfragments/3478.minor delete mode 100644 newsfragments/3479.minor delete mode 100644 newsfragments/3481.minor delete mode 100644 newsfragments/3482.minor delete mode 100644 newsfragments/3483.minor delete mode 100644 newsfragments/3485.minor delete mode 100644 newsfragments/3486.installation delete mode 100644 newsfragments/3488.minor delete mode 100644 newsfragments/3490.minor delete mode 100644 newsfragments/3491.minor delete mode 100644 newsfragments/3492.minor delete mode 100644 newsfragments/3493.minor delete mode 100644 newsfragments/3496.minor delete mode 100644 newsfragments/3497.installation delete mode 100644 newsfragments/3499.minor delete mode 100644 newsfragments/3500.minor delete mode 100644 newsfragments/3501.minor delete mode 100644 newsfragments/3502.minor delete mode 100644 newsfragments/3503.other delete mode 100644 newsfragments/3504.configuration delete mode 100644 newsfragments/3509.bugfix delete mode 100644 newsfragments/3510.bugfix delete mode 100644 newsfragments/3511.minor delete mode 100644 newsfragments/3513.minor delete mode 100644 newsfragments/3514.minor delete mode 100644 newsfragments/3515.minor delete mode 100644 newsfragments/3517.minor delete mode 100644 newsfragments/3518.removed delete mode 100644 newsfragments/3520.minor delete mode 100644 newsfragments/3521.minor delete mode 100644 newsfragments/3522.minor delete mode 100644 newsfragments/3523.minor delete mode 100644 newsfragments/3524.minor delete mode 100644 newsfragments/3528.minor delete mode 100644 newsfragments/3529.minor delete mode 100644 newsfragments/3532.minor delete mode 100644 newsfragments/3533.minor delete mode 100644 newsfragments/3534.minor delete mode 100644 newsfragments/3536.minor delete mode 100644 newsfragments/3537.minor delete mode 100644 newsfragments/3539.bugfix delete mode 100644 newsfragments/3542.minor delete mode 100644 newsfragments/3544.minor delete mode 100644 newsfragments/3545.other delete mode 100644 newsfragments/3546.minor delete mode 100644 newsfragments/3547.minor delete mode 100644 newsfragments/3549.removed delete mode 100644 newsfragments/3550.removed delete mode 100644 newsfragments/3551.minor delete mode 100644 newsfragments/3552.minor delete mode 100644 newsfragments/3553.minor delete mode 100644 newsfragments/3555.minor delete mode 100644 newsfragments/3557.minor delete mode 100644 newsfragments/3558.minor delete mode 100644 newsfragments/3560.minor delete mode 100644 newsfragments/3563.minor delete mode 100644 newsfragments/3564.minor delete mode 100644 newsfragments/3565.minor delete mode 100644 newsfragments/3566.minor delete mode 100644 newsfragments/3567.minor delete mode 100644 newsfragments/3568.minor delete mode 100644 newsfragments/3572.minor delete mode 100644 newsfragments/3574.minor delete mode 100644 newsfragments/3575.minor delete mode 100644 newsfragments/3576.minor delete mode 100644 newsfragments/3577.minor delete mode 100644 newsfragments/3578.minor delete mode 100644 newsfragments/3579.minor delete mode 100644 newsfragments/3580.minor delete mode 100644 newsfragments/3582.minor delete mode 100644 newsfragments/3583.removed delete mode 100644 newsfragments/3584.bugfix delete mode 100644 newsfragments/3587.minor delete mode 100644 newsfragments/3588.incompat delete mode 100644 newsfragments/3588.minor delete mode 100644 newsfragments/3589.minor delete mode 100644 newsfragments/3590.bugfix delete mode 100644 newsfragments/3591.minor delete mode 100644 newsfragments/3592.minor delete mode 100644 newsfragments/3593.minor delete mode 100644 newsfragments/3594.minor delete mode 100644 newsfragments/3595.minor delete mode 100644 newsfragments/3596.minor delete mode 100644 newsfragments/3599.minor delete mode 100644 newsfragments/3600.minor delete mode 100644 newsfragments/3603.minor.rst delete mode 100644 newsfragments/3605.minor delete mode 100644 newsfragments/3606.minor delete mode 100644 newsfragments/3607.minor delete mode 100644 newsfragments/3608.minor delete mode 100644 newsfragments/3611.minor delete mode 100644 newsfragments/3612.minor delete mode 100644 newsfragments/3613.minor delete mode 100644 newsfragments/3615.minor delete mode 100644 newsfragments/3616.minor delete mode 100644 newsfragments/3617.minor delete mode 100644 newsfragments/3618.minor delete mode 100644 newsfragments/3619.minor delete mode 100644 newsfragments/3620.minor delete mode 100644 newsfragments/3621.minor delete mode 100644 newsfragments/3623.minor delete mode 100644 newsfragments/3624.minor delete mode 100644 newsfragments/3625.minor delete mode 100644 newsfragments/3626.minor delete mode 100644 newsfragments/3628.minor delete mode 100644 newsfragments/3629.feature delete mode 100644 newsfragments/3630.minor delete mode 100644 newsfragments/3631.minor delete mode 100644 newsfragments/3632.minor delete mode 100644 newsfragments/3633.installation delete mode 100644 newsfragments/3634.minor delete mode 100644 newsfragments/3635.minor delete mode 100644 newsfragments/3637.minor delete mode 100644 newsfragments/3638.minor delete mode 100644 newsfragments/3640.minor delete mode 100644 newsfragments/3642.minor delete mode 100644 newsfragments/3644.other delete mode 100644 newsfragments/3645.minor delete mode 100644 newsfragments/3646.minor delete mode 100644 newsfragments/3647.minor delete mode 100644 newsfragments/3648.minor delete mode 100644 newsfragments/3649.minor delete mode 100644 newsfragments/3650.bugfix delete mode 100644 newsfragments/3651.minor delete mode 100644 newsfragments/3652.removed delete mode 100644 newsfragments/3653.minor delete mode 100644 newsfragments/3654.minor delete mode 100644 newsfragments/3655.minor delete mode 100644 newsfragments/3656.minor delete mode 100644 newsfragments/3657.minor delete mode 100644 newsfragments/3658.minor delete mode 100644 newsfragments/3659.documentation delete mode 100644 newsfragments/3662.minor delete mode 100644 newsfragments/3663.other delete mode 100644 newsfragments/3664.documentation delete mode 100644 newsfragments/3666.documentation delete mode 100644 newsfragments/3667.minor delete mode 100644 newsfragments/3669.minor delete mode 100644 newsfragments/3670.minor delete mode 100644 newsfragments/3671.minor delete mode 100644 newsfragments/3672.minor delete mode 100644 newsfragments/3674.minor delete mode 100644 newsfragments/3675.minor delete mode 100644 newsfragments/3676.minor delete mode 100644 newsfragments/3677.documentation delete mode 100644 newsfragments/3678.minor delete mode 100644 newsfragments/3679.minor delete mode 100644 newsfragments/3681.minor delete mode 100644 newsfragments/3682.documentation delete mode 100644 newsfragments/3683.minor delete mode 100644 newsfragments/3686.minor delete mode 100644 newsfragments/3687.minor delete mode 100644 newsfragments/3691.minor delete mode 100644 newsfragments/3692.minor delete mode 100644 newsfragments/3699.minor delete mode 100644 newsfragments/3700.minor delete mode 100644 newsfragments/3701.minor delete mode 100644 newsfragments/3702.minor delete mode 100644 newsfragments/3703.minor delete mode 100644 newsfragments/3704.minor delete mode 100644 newsfragments/3705.minor delete mode 100644 newsfragments/3707.minor delete mode 100644 newsfragments/3708.minor delete mode 100644 newsfragments/3709.minor delete mode 100644 newsfragments/3711.minor delete mode 100644 newsfragments/3712.installation delete mode 100644 newsfragments/3713.minor delete mode 100644 newsfragments/3714.minor delete mode 100644 newsfragments/3715.minor delete mode 100644 newsfragments/3716.incompat delete mode 100644 newsfragments/3717.minor delete mode 100644 newsfragments/3718.minor delete mode 100644 newsfragments/3721.documentation delete mode 100644 newsfragments/3722.minor delete mode 100644 newsfragments/3723.minor delete mode 100644 newsfragments/3726.documentation delete mode 100644 newsfragments/3727.minor delete mode 100644 newsfragments/3728.minor delete mode 100644 newsfragments/3729.minor delete mode 100644 newsfragments/3730.minor delete mode 100644 newsfragments/3731.minor delete mode 100644 newsfragments/3732.minor delete mode 100644 newsfragments/3733.installation delete mode 100644 newsfragments/3734.minor delete mode 100644 newsfragments/3735.minor delete mode 100644 newsfragments/3736.minor delete mode 100644 newsfragments/3738.bugfix delete mode 100644 newsfragments/3739.bugfix delete mode 100644 newsfragments/3741.minor delete mode 100644 newsfragments/3743.minor delete mode 100644 newsfragments/3744.minor delete mode 100644 newsfragments/3745.minor delete mode 100644 newsfragments/3746.minor delete mode 100644 newsfragments/3747.documentation delete mode 100644 newsfragments/3749.documentation delete mode 100644 newsfragments/3751.minor delete mode 100644 newsfragments/3757.other delete mode 100644 newsfragments/3759.minor delete mode 100644 newsfragments/3760.minor delete mode 100644 newsfragments/3763.minor delete mode 100644 newsfragments/3764.documentation delete mode 100644 newsfragments/3765.documentation delete mode 100644 newsfragments/3769.documentation delete mode 100644 newsfragments/3773.minor delete mode 100644 newsfragments/3774.documentation delete mode 100644 newsfragments/3777.documentation delete mode 100644 newsfragments/3779.bugfix delete mode 100644 newsfragments/3781.minor delete mode 100644 newsfragments/3782.documentation delete mode 100644 newsfragments/3785.documentation diff --git a/NEWS.rst b/NEWS.rst index 1cfc726ae..a7d814c70 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -5,6 +5,104 @@ User-Visible Changes in Tahoe-LAFS ================================== .. towncrier start line +Release 1.15.1.post2188.dev0 (2021-09-17)Release 1.15.1.post2188.dev0 (2021-09-17) +''''''''''''''''''''''''''''''''''''''''' + +Backwards Incompatible Changes +------------------------------ + +- The Tahoe command line now always uses UTF-8 to decode its arguments, regardless of locale. (`#3588 `_) +- tahoe backup's --exclude-from has been renamed to --exclude-from-utf-8, and correspondingly requires the file to be UTF-8 encoded. (`#3716 `_) + + +Features +-------- + +- Added 'typechecks' environment for tox running mypy and performing static typechecks. (`#3399 `_) +- The NixOS-packaged Tahoe-LAFS now knows its own version. (`#3629 `_) + + +Bug Fixes +--------- + +- Fix regression that broke flogtool results on Python 2. (`#3509 `_) +- Fix a logging regression on Python 2 involving unicode strings. (`#3510 `_) +- Certain implementation-internal weakref KeyErrors are now handled and should no longer cause user-initiated operations to fail. (`#3539 `_) +- SFTP public key auth likely works more consistently, and SFTP in general was previously broken. (`#3584 `_) +- Fixed issue where redirecting old-style URIs (/uri/?uri=...) didn't work. (`#3590 `_) +- ``tahoe invite`` will now read share encoding/placement configuration values from a Tahoe client node configuration file if they are not given on the command line, instead of raising an unhandled exception. (`#3650 `_) +- Fix regression where uploading files with non-ASCII names failed. (`#3738 `_) +- Fixed annoying UnicodeWarning message on Python 2 when running CLI tools. (`#3739 `_) +- Fixed bug where share corruption events were not logged on storage servers running on Windows. (`#3779 `_) + + +Dependency/Installation Changes +------------------------------- + +- Tahoe-LAFS now requires Twisted 19.10.0 or newer. As a result, it now has a transitive dependency on bcrypt. (`#1549 `_) +- Debian 8 support has been replaced with Debian 10 support. (`#3326 `_) +- Tahoe-LAFS no longer depends on Nevow. (`#3433 `_) +- Tahoe-LAFS now requires the `netifaces` Python package and no longer requires the external `ip`, `ifconfig`, or `route.exe` executables. (`#3486 `_) +- The Tahoe-LAFS project no longer commits to maintaining binary packages for all dependencies at . Please use PyPI instead. (`#3497 `_) +- Tahoe-LAFS now uses a forked version of txi2p (named txi2p-tahoe) with Python 3 support. (`#3633 `_) +- The Nix package now includes correct version information. (`#3712 `_) +- Use netifaces 0.11.0 wheel package from PyPI.org if you use 64-bit Python 2.7 on Windows. VCPython27 downloads are no longer available at Microsoft's website, which has made building Python 2.7 wheel packages of Python libraries with C extensions (such as netifaces) on Windows difficult. (`#3733 `_) + + +Configuration Changes +--------------------- + +- The ``[client]introducer.furl`` configuration item is now deprecated in favor of the ``private/introducers.yaml`` file. (`#3504 `_) + + +Documentation Changes +--------------------- + +- (`#3659 `_) +- Documentation now has its own towncrier category. (`#3664 `_) +- `tox -e docs` will treat warnings about docs as errors. (`#3666 `_) +- The visibility of the Tahoe-LAFS logo has been improved for "dark" themed viewing. (`#3677 `_) +- A cheatsheet-style document for contributors was created at CONTRIBUTORS.rst (`#3682 `_) +- Our IRC channel, #tahoe-lafs, has been moved to irc.libera.chat. (`#3721 `_) +- Tahoe-LAFS project is now registered with Libera.Chat IRC network. (`#3726 `_) +- Rewriting the installation guide for Tahoe-LAFS. (`#3747 `_) +- Documentation and installation links in the README have been fixed. (`#3749 `_) +- The Great Black Swamp proposed specification now includes sample interactions to demonstrate expected usage patterns. (`#3764 `_) +- The Great Black Swamp proposed specification now includes a glossary. (`#3765 `_) +- The Great Black Swamp specification now allows parallel upload of immutable share data. (`#3769 `_) +- There is now a specification for the scheme which Tahoe-LAFS storage clients use to derive their lease renewal secrets. (`#3774 `_) +- The Great Black Swamp proposed specification now has a simplified interface for reading data from immutable shares. (`#3777 `_) +- tahoe-dev mailing list is now at tahoe-dev@lists.tahoe-lafs.org. (`#3782 `_) +- The Great Black Swamp specification now describes the required authorization scheme. (`#3785 `_) + + +Removed Features +---------------- + +- Announcements delivered through the introducer system are no longer automatically annotated with copious information about the Tahoe-LAFS software version nor the versions of its dependencies. (`#3518 `_) +- The stats gatherer, broken since at least Tahoe-LAFS 1.13.0, has been removed. The ``[client]stats_gatherer.furl`` configuration item in ``tahoe.cfg`` is no longer allowed. The Tahoe-LAFS project recommends using a third-party metrics aggregation tool instead. (`#3549 `_) +- The deprecated ``tahoe`` start, restart, stop, and daemonize sub-commands have been removed. (`#3550 `_) +- FTP is no longer supported by Tahoe-LAFS. Please use the SFTP support instead. (`#3583 `_) +- Removed support for the Account Server frontend authentication type. (`#3652 `_) + + +Other Changes +------------- + +- The "Great Black Swamp" proposed specification has been expanded to include two lease management APIs. (`#3037 `_) +- The specification section of the Tahoe-LAFS documentation now includes explicit discussion of the security properties of Foolscap "fURLs" on which it depends. (`#3503 `_) +- The README, revised by Viktoriia with feedback from the team, is now more focused on the developer community and provides more information about Tahoe-LAFS, why it's important, and how someone can use it or start contributing to it. (`#3545 `_) +- The "Great Black Swamp" proposed specification has been changed use ``v=1`` as the URL version identifier. (`#3644 `_) +- You can run `make livehtml` in docs directory to invoke sphinx-autobuild. (`#3663 `_) +- Refactored test_introducer in web tests to use custom base test cases (`#3757 `_) + + +Misc/Other +---------- + +- `#2928 `_, `#3283 `_, `#3314 `_, `#3384 `_, `#3385 `_, `#3390 `_, `#3404 `_, `#3428 `_, `#3432 `_, `#3434 `_, `#3435 `_, `#3454 `_, `#3459 `_, `#3460 `_, `#3465 `_, `#3466 `_, `#3467 `_, `#3468 `_, `#3470 `_, `#3471 `_, `#3472 `_, `#3473 `_, `#3474 `_, `#3475 `_, `#3477 `_, `#3478 `_, `#3479 `_, `#3481 `_, `#3482 `_, `#3483 `_, `#3485 `_, `#3488 `_, `#3490 `_, `#3491 `_, `#3492 `_, `#3493 `_, `#3496 `_, `#3499 `_, `#3500 `_, `#3501 `_, `#3502 `_, `#3511 `_, `#3513 `_, `#3514 `_, `#3515 `_, `#3517 `_, `#3520 `_, `#3521 `_, `#3522 `_, `#3523 `_, `#3524 `_, `#3528 `_, `#3529 `_, `#3532 `_, `#3533 `_, `#3534 `_, `#3536 `_, `#3537 `_, `#3542 `_, `#3544 `_, `#3546 `_, `#3547 `_, `#3551 `_, `#3552 `_, `#3553 `_, `#3555 `_, `#3557 `_, `#3558 `_, `#3560 `_, `#3563 `_, `#3564 `_, `#3565 `_, `#3566 `_, `#3567 `_, `#3568 `_, `#3572 `_, `#3574 `_, `#3575 `_, `#3576 `_, `#3577 `_, `#3578 `_, `#3579 `_, `#3580 `_, `#3582 `_, `#3587 `_, `#3588 `_, `#3589 `_, `#3591 `_, `#3592 `_, `#3593 `_, `#3594 `_, `#3595 `_, `#3596 `_, `#3599 `_, `#3600 `_, `#3603 `_, `#3605 `_, `#3606 `_, `#3607 `_, `#3608 `_, `#3611 `_, `#3612 `_, `#3613 `_, `#3615 `_, `#3616 `_, `#3617 `_, `#3618 `_, `#3619 `_, `#3620 `_, `#3621 `_, `#3623 `_, `#3624 `_, `#3625 `_, `#3626 `_, `#3628 `_, `#3630 `_, `#3631 `_, `#3632 `_, `#3634 `_, `#3635 `_, `#3637 `_, `#3638 `_, `#3640 `_, `#3642 `_, `#3645 `_, `#3646 `_, `#3647 `_, `#3648 `_, `#3649 `_, `#3651 `_, `#3653 `_, `#3654 `_, `#3655 `_, `#3656 `_, `#3657 `_, `#3658 `_, `#3662 `_, `#3667 `_, `#3669 `_, `#3670 `_, `#3671 `_, `#3672 `_, `#3674 `_, `#3675 `_, `#3676 `_, `#3678 `_, `#3679 `_, `#3681 `_, `#3683 `_, `#3686 `_, `#3687 `_, `#3691 `_, `#3692 `_, `#3699 `_, `#3700 `_, `#3701 `_, `#3702 `_, `#3703 `_, `#3704 `_, `#3705 `_, `#3707 `_, `#3708 `_, `#3709 `_, `#3711 `_, `#3713 `_, `#3714 `_, `#3715 `_, `#3717 `_, `#3718 `_, `#3722 `_, `#3723 `_, `#3727 `_, `#3728 `_, `#3729 `_, `#3730 `_, `#3731 `_, `#3732 `_, `#3734 `_, `#3735 `_, `#3736 `_, `#3741 `_, `#3743 `_, `#3744 `_, `#3745 `_, `#3746 `_, `#3751 `_, `#3759 `_, `#3760 `_, `#3763 `_, `#3773 `_, `#3781 `_ + + Release 1.15.1 '''''''''''''' diff --git a/newsfragments/1549.installation b/newsfragments/1549.installation deleted file mode 100644 index cbb91cea5..000000000 --- a/newsfragments/1549.installation +++ /dev/null @@ -1 +0,0 @@ -Tahoe-LAFS now requires Twisted 19.10.0 or newer. As a result, it now has a transitive dependency on bcrypt. diff --git a/newsfragments/2928.minor b/newsfragments/2928.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3037.other b/newsfragments/3037.other deleted file mode 100644 index 947dc8f60..000000000 --- a/newsfragments/3037.other +++ /dev/null @@ -1 +0,0 @@ -The "Great Black Swamp" proposed specification has been expanded to include two lease management APIs. \ No newline at end of file diff --git a/newsfragments/3283.minor b/newsfragments/3283.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3314.minor b/newsfragments/3314.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3326.installation b/newsfragments/3326.installation deleted file mode 100644 index 2a3a64e32..000000000 --- a/newsfragments/3326.installation +++ /dev/null @@ -1 +0,0 @@ -Debian 8 support has been replaced with Debian 10 support. diff --git a/newsfragments/3384.minor b/newsfragments/3384.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3385.minor b/newsfragments/3385.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3390.minor b/newsfragments/3390.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3399.feature b/newsfragments/3399.feature deleted file mode 100644 index d30a91679..000000000 --- a/newsfragments/3399.feature +++ /dev/null @@ -1 +0,0 @@ -Added 'typechecks' environment for tox running mypy and performing static typechecks. diff --git a/newsfragments/3404.minor b/newsfragments/3404.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3428.minor b/newsfragments/3428.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3432.minor b/newsfragments/3432.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3433.installation b/newsfragments/3433.installation deleted file mode 100644 index 3c06e53d3..000000000 --- a/newsfragments/3433.installation +++ /dev/null @@ -1 +0,0 @@ -Tahoe-LAFS no longer depends on Nevow. \ No newline at end of file diff --git a/newsfragments/3434.minor b/newsfragments/3434.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3435.minor b/newsfragments/3435.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3454.minor b/newsfragments/3454.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3459.minor b/newsfragments/3459.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3460.minor b/newsfragments/3460.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3465.minor b/newsfragments/3465.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3466.minor b/newsfragments/3466.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3467.minor b/newsfragments/3467.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3468.minor b/newsfragments/3468.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3470.minor b/newsfragments/3470.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3471.minor b/newsfragments/3471.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3472.minor b/newsfragments/3472.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3473.minor b/newsfragments/3473.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3474.minor b/newsfragments/3474.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3475.minor b/newsfragments/3475.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3477.minor b/newsfragments/3477.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3478.minor b/newsfragments/3478.minor deleted file mode 100644 index 8b1378917..000000000 --- a/newsfragments/3478.minor +++ /dev/null @@ -1 +0,0 @@ - diff --git a/newsfragments/3479.minor b/newsfragments/3479.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3481.minor b/newsfragments/3481.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3482.minor b/newsfragments/3482.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3483.minor b/newsfragments/3483.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3485.minor b/newsfragments/3485.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3486.installation b/newsfragments/3486.installation deleted file mode 100644 index 7b24956b2..000000000 --- a/newsfragments/3486.installation +++ /dev/null @@ -1 +0,0 @@ -Tahoe-LAFS now requires the `netifaces` Python package and no longer requires the external `ip`, `ifconfig`, or `route.exe` executables. diff --git a/newsfragments/3488.minor b/newsfragments/3488.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3490.minor b/newsfragments/3490.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3491.minor b/newsfragments/3491.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3492.minor b/newsfragments/3492.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3493.minor b/newsfragments/3493.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3496.minor b/newsfragments/3496.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3497.installation b/newsfragments/3497.installation deleted file mode 100644 index 4a50be97e..000000000 --- a/newsfragments/3497.installation +++ /dev/null @@ -1 +0,0 @@ -The Tahoe-LAFS project no longer commits to maintaining binary packages for all dependencies at . Please use PyPI instead. diff --git a/newsfragments/3499.minor b/newsfragments/3499.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3500.minor b/newsfragments/3500.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3501.minor b/newsfragments/3501.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3502.minor b/newsfragments/3502.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3503.other b/newsfragments/3503.other deleted file mode 100644 index 5d0c681b6..000000000 --- a/newsfragments/3503.other +++ /dev/null @@ -1 +0,0 @@ -The specification section of the Tahoe-LAFS documentation now includes explicit discussion of the security properties of Foolscap "fURLs" on which it depends. diff --git a/newsfragments/3504.configuration b/newsfragments/3504.configuration deleted file mode 100644 index 9ff74482c..000000000 --- a/newsfragments/3504.configuration +++ /dev/null @@ -1 +0,0 @@ -The ``[client]introducer.furl`` configuration item is now deprecated in favor of the ``private/introducers.yaml`` file. \ No newline at end of file diff --git a/newsfragments/3509.bugfix b/newsfragments/3509.bugfix deleted file mode 100644 index 4d633feab..000000000 --- a/newsfragments/3509.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix regression that broke flogtool results on Python 2. \ No newline at end of file diff --git a/newsfragments/3510.bugfix b/newsfragments/3510.bugfix deleted file mode 100644 index d4a2bd5dc..000000000 --- a/newsfragments/3510.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a logging regression on Python 2 involving unicode strings. \ No newline at end of file diff --git a/newsfragments/3511.minor b/newsfragments/3511.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3513.minor b/newsfragments/3513.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3514.minor b/newsfragments/3514.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3515.minor b/newsfragments/3515.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3517.minor b/newsfragments/3517.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3518.removed b/newsfragments/3518.removed deleted file mode 100644 index 460af5142..000000000 --- a/newsfragments/3518.removed +++ /dev/null @@ -1 +0,0 @@ -Announcements delivered through the introducer system are no longer automatically annotated with copious information about the Tahoe-LAFS software version nor the versions of its dependencies. diff --git a/newsfragments/3520.minor b/newsfragments/3520.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3521.minor b/newsfragments/3521.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3522.minor b/newsfragments/3522.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3523.minor b/newsfragments/3523.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3524.minor b/newsfragments/3524.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3528.minor b/newsfragments/3528.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3529.minor b/newsfragments/3529.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3532.minor b/newsfragments/3532.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3533.minor b/newsfragments/3533.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3534.minor b/newsfragments/3534.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3536.minor b/newsfragments/3536.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3537.minor b/newsfragments/3537.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3539.bugfix b/newsfragments/3539.bugfix deleted file mode 100644 index ed4aeb9af..000000000 --- a/newsfragments/3539.bugfix +++ /dev/null @@ -1 +0,0 @@ -Certain implementation-internal weakref KeyErrors are now handled and should no longer cause user-initiated operations to fail. diff --git a/newsfragments/3542.minor b/newsfragments/3542.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3544.minor b/newsfragments/3544.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3545.other b/newsfragments/3545.other deleted file mode 100644 index fd8adc37b..000000000 --- a/newsfragments/3545.other +++ /dev/null @@ -1 +0,0 @@ -The README, revised by Viktoriia with feedback from the team, is now more focused on the developer community and provides more information about Tahoe-LAFS, why it's important, and how someone can use it or start contributing to it. \ No newline at end of file diff --git a/newsfragments/3546.minor b/newsfragments/3546.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3547.minor b/newsfragments/3547.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3549.removed b/newsfragments/3549.removed deleted file mode 100644 index 53c7a7de1..000000000 --- a/newsfragments/3549.removed +++ /dev/null @@ -1 +0,0 @@ -The stats gatherer, broken since at least Tahoe-LAFS 1.13.0, has been removed. The ``[client]stats_gatherer.furl`` configuration item in ``tahoe.cfg`` is no longer allowed. The Tahoe-LAFS project recommends using a third-party metrics aggregation tool instead. diff --git a/newsfragments/3550.removed b/newsfragments/3550.removed deleted file mode 100644 index 2074bf676..000000000 --- a/newsfragments/3550.removed +++ /dev/null @@ -1 +0,0 @@ -The deprecated ``tahoe`` start, restart, stop, and daemonize sub-commands have been removed. \ No newline at end of file diff --git a/newsfragments/3551.minor b/newsfragments/3551.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3552.minor b/newsfragments/3552.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3553.minor b/newsfragments/3553.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3555.minor b/newsfragments/3555.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3557.minor b/newsfragments/3557.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3558.minor b/newsfragments/3558.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3560.minor b/newsfragments/3560.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3563.minor b/newsfragments/3563.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3564.minor b/newsfragments/3564.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3565.minor b/newsfragments/3565.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3566.minor b/newsfragments/3566.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3567.minor b/newsfragments/3567.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3568.minor b/newsfragments/3568.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3572.minor b/newsfragments/3572.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3574.minor b/newsfragments/3574.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3575.minor b/newsfragments/3575.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3576.minor b/newsfragments/3576.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3577.minor b/newsfragments/3577.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3578.minor b/newsfragments/3578.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3579.minor b/newsfragments/3579.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3580.minor b/newsfragments/3580.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3582.minor b/newsfragments/3582.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3583.removed b/newsfragments/3583.removed deleted file mode 100644 index a3fce48be..000000000 --- a/newsfragments/3583.removed +++ /dev/null @@ -1 +0,0 @@ -FTP is no longer supported by Tahoe-LAFS. Please use the SFTP support instead. \ No newline at end of file diff --git a/newsfragments/3584.bugfix b/newsfragments/3584.bugfix deleted file mode 100644 index faf57713b..000000000 --- a/newsfragments/3584.bugfix +++ /dev/null @@ -1 +0,0 @@ -SFTP public key auth likely works more consistently, and SFTP in general was previously broken. \ No newline at end of file diff --git a/newsfragments/3587.minor b/newsfragments/3587.minor deleted file mode 100644 index 8b1378917..000000000 --- a/newsfragments/3587.minor +++ /dev/null @@ -1 +0,0 @@ - diff --git a/newsfragments/3588.incompat b/newsfragments/3588.incompat deleted file mode 100644 index 402ae8479..000000000 --- a/newsfragments/3588.incompat +++ /dev/null @@ -1 +0,0 @@ -The Tahoe command line now always uses UTF-8 to decode its arguments, regardless of locale. diff --git a/newsfragments/3588.minor b/newsfragments/3588.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3589.minor b/newsfragments/3589.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3590.bugfix b/newsfragments/3590.bugfix deleted file mode 100644 index aa504a5e3..000000000 --- a/newsfragments/3590.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fixed issue where redirecting old-style URIs (/uri/?uri=...) didn't work. \ No newline at end of file diff --git a/newsfragments/3591.minor b/newsfragments/3591.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3592.minor b/newsfragments/3592.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3593.minor b/newsfragments/3593.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3594.minor b/newsfragments/3594.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3595.minor b/newsfragments/3595.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3596.minor b/newsfragments/3596.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3599.minor b/newsfragments/3599.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3600.minor b/newsfragments/3600.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3603.minor.rst b/newsfragments/3603.minor.rst deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3605.minor b/newsfragments/3605.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3606.minor b/newsfragments/3606.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3607.minor b/newsfragments/3607.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3608.minor b/newsfragments/3608.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3611.minor b/newsfragments/3611.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3612.minor b/newsfragments/3612.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3613.minor b/newsfragments/3613.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3615.minor b/newsfragments/3615.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3616.minor b/newsfragments/3616.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3617.minor b/newsfragments/3617.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3618.minor b/newsfragments/3618.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3619.minor b/newsfragments/3619.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3620.minor b/newsfragments/3620.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3621.minor b/newsfragments/3621.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3623.minor b/newsfragments/3623.minor deleted file mode 100644 index 8b1378917..000000000 --- a/newsfragments/3623.minor +++ /dev/null @@ -1 +0,0 @@ - diff --git a/newsfragments/3624.minor b/newsfragments/3624.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3625.minor b/newsfragments/3625.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3626.minor b/newsfragments/3626.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3628.minor b/newsfragments/3628.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3629.feature b/newsfragments/3629.feature deleted file mode 100644 index cdca48a18..000000000 --- a/newsfragments/3629.feature +++ /dev/null @@ -1 +0,0 @@ -The NixOS-packaged Tahoe-LAFS now knows its own version. diff --git a/newsfragments/3630.minor b/newsfragments/3630.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3631.minor b/newsfragments/3631.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3632.minor b/newsfragments/3632.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3633.installation b/newsfragments/3633.installation deleted file mode 100644 index 8f6d7efdd..000000000 --- a/newsfragments/3633.installation +++ /dev/null @@ -1 +0,0 @@ -Tahoe-LAFS now uses a forked version of txi2p (named txi2p-tahoe) with Python 3 support. diff --git a/newsfragments/3634.minor b/newsfragments/3634.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3635.minor b/newsfragments/3635.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3637.minor b/newsfragments/3637.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3638.minor b/newsfragments/3638.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3640.minor b/newsfragments/3640.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3642.minor b/newsfragments/3642.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3644.other b/newsfragments/3644.other deleted file mode 100644 index 4b159e45d..000000000 --- a/newsfragments/3644.other +++ /dev/null @@ -1 +0,0 @@ -The "Great Black Swamp" proposed specification has been changed use ``v=1`` as the URL version identifier. \ No newline at end of file diff --git a/newsfragments/3645.minor b/newsfragments/3645.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3646.minor b/newsfragments/3646.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3647.minor b/newsfragments/3647.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3648.minor b/newsfragments/3648.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3649.minor b/newsfragments/3649.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3650.bugfix b/newsfragments/3650.bugfix deleted file mode 100644 index 09a810239..000000000 --- a/newsfragments/3650.bugfix +++ /dev/null @@ -1 +0,0 @@ -``tahoe invite`` will now read share encoding/placement configuration values from a Tahoe client node configuration file if they are not given on the command line, instead of raising an unhandled exception. diff --git a/newsfragments/3651.minor b/newsfragments/3651.minor deleted file mode 100644 index 9a2f5a0ed..000000000 --- a/newsfragments/3651.minor +++ /dev/null @@ -1 +0,0 @@ -We added documentation detailing the project's ticket triage process diff --git a/newsfragments/3652.removed b/newsfragments/3652.removed deleted file mode 100644 index a3e964702..000000000 --- a/newsfragments/3652.removed +++ /dev/null @@ -1 +0,0 @@ -Removed support for the Account Server frontend authentication type. diff --git a/newsfragments/3653.minor b/newsfragments/3653.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3654.minor b/newsfragments/3654.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3655.minor b/newsfragments/3655.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3656.minor b/newsfragments/3656.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3657.minor b/newsfragments/3657.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3658.minor b/newsfragments/3658.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3659.documentation b/newsfragments/3659.documentation deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3662.minor b/newsfragments/3662.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3663.other b/newsfragments/3663.other deleted file mode 100644 index 62abf2666..000000000 --- a/newsfragments/3663.other +++ /dev/null @@ -1 +0,0 @@ -You can run `make livehtml` in docs directory to invoke sphinx-autobuild. diff --git a/newsfragments/3664.documentation b/newsfragments/3664.documentation deleted file mode 100644 index ab5de8884..000000000 --- a/newsfragments/3664.documentation +++ /dev/null @@ -1 +0,0 @@ -Documentation now has its own towncrier category. diff --git a/newsfragments/3666.documentation b/newsfragments/3666.documentation deleted file mode 100644 index 3f9e34777..000000000 --- a/newsfragments/3666.documentation +++ /dev/null @@ -1 +0,0 @@ -`tox -e docs` will treat warnings about docs as errors. diff --git a/newsfragments/3667.minor b/newsfragments/3667.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3669.minor b/newsfragments/3669.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3670.minor b/newsfragments/3670.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3671.minor b/newsfragments/3671.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3672.minor b/newsfragments/3672.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3674.minor b/newsfragments/3674.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3675.minor b/newsfragments/3675.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3676.minor b/newsfragments/3676.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3677.documentation b/newsfragments/3677.documentation deleted file mode 100644 index 51730e765..000000000 --- a/newsfragments/3677.documentation +++ /dev/null @@ -1 +0,0 @@ -The visibility of the Tahoe-LAFS logo has been improved for "dark" themed viewing. diff --git a/newsfragments/3678.minor b/newsfragments/3678.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3679.minor b/newsfragments/3679.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3681.minor b/newsfragments/3681.minor deleted file mode 100644 index bc84b6b8f..000000000 --- a/newsfragments/3681.minor +++ /dev/null @@ -1,8 +0,0 @@ -(The below text is no longer valid: netifaces has released a 64-bit -Python 2.7 wheel for Windows. Ticket #3733 made the switch in CI. We -should be able to test and run Tahoe-LAFS without needing vcpython27 -now.) - -Tahoe-LAFS CI now runs tests only on 32-bit Windows. Microsoft has -removed vcpython27 compiler downloads from their site, and Tahoe-LAFS -needs vcpython27 to build and install netifaces on 64-bit Windows. diff --git a/newsfragments/3682.documentation b/newsfragments/3682.documentation deleted file mode 100644 index 5cf78bd90..000000000 --- a/newsfragments/3682.documentation +++ /dev/null @@ -1 +0,0 @@ -A cheatsheet-style document for contributors was created at CONTRIBUTORS.rst \ No newline at end of file diff --git a/newsfragments/3683.minor b/newsfragments/3683.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3686.minor b/newsfragments/3686.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3687.minor b/newsfragments/3687.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3691.minor b/newsfragments/3691.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3692.minor b/newsfragments/3692.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3699.minor b/newsfragments/3699.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3700.minor b/newsfragments/3700.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3701.minor b/newsfragments/3701.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3702.minor b/newsfragments/3702.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3703.minor b/newsfragments/3703.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3704.minor b/newsfragments/3704.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3705.minor b/newsfragments/3705.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3707.minor b/newsfragments/3707.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3708.minor b/newsfragments/3708.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3709.minor b/newsfragments/3709.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3711.minor b/newsfragments/3711.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3712.installation b/newsfragments/3712.installation deleted file mode 100644 index b80e1558b..000000000 --- a/newsfragments/3712.installation +++ /dev/null @@ -1 +0,0 @@ -The Nix package now includes correct version information. \ No newline at end of file diff --git a/newsfragments/3713.minor b/newsfragments/3713.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3714.minor b/newsfragments/3714.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3715.minor b/newsfragments/3715.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3716.incompat b/newsfragments/3716.incompat deleted file mode 100644 index aa03eea47..000000000 --- a/newsfragments/3716.incompat +++ /dev/null @@ -1 +0,0 @@ -tahoe backup's --exclude-from has been renamed to --exclude-from-utf-8, and correspondingly requires the file to be UTF-8 encoded. \ No newline at end of file diff --git a/newsfragments/3717.minor b/newsfragments/3717.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3718.minor b/newsfragments/3718.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3721.documentation b/newsfragments/3721.documentation deleted file mode 100644 index 36ae33236..000000000 --- a/newsfragments/3721.documentation +++ /dev/null @@ -1 +0,0 @@ -Our IRC channel, #tahoe-lafs, has been moved to irc.libera.chat. diff --git a/newsfragments/3722.minor b/newsfragments/3722.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3723.minor b/newsfragments/3723.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3726.documentation b/newsfragments/3726.documentation deleted file mode 100644 index fb94fff32..000000000 --- a/newsfragments/3726.documentation +++ /dev/null @@ -1 +0,0 @@ -Tahoe-LAFS project is now registered with Libera.Chat IRC network. diff --git a/newsfragments/3727.minor b/newsfragments/3727.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3728.minor b/newsfragments/3728.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3729.minor b/newsfragments/3729.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3730.minor b/newsfragments/3730.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3731.minor b/newsfragments/3731.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3732.minor b/newsfragments/3732.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3733.installation b/newsfragments/3733.installation deleted file mode 100644 index c1cac649b..000000000 --- a/newsfragments/3733.installation +++ /dev/null @@ -1 +0,0 @@ -Use netifaces 0.11.0 wheel package from PyPI.org if you use 64-bit Python 2.7 on Windows. VCPython27 downloads are no longer available at Microsoft's website, which has made building Python 2.7 wheel packages of Python libraries with C extensions (such as netifaces) on Windows difficult. diff --git a/newsfragments/3734.minor b/newsfragments/3734.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3735.minor b/newsfragments/3735.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3736.minor b/newsfragments/3736.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3738.bugfix b/newsfragments/3738.bugfix deleted file mode 100644 index 6a4bc1cd9..000000000 --- a/newsfragments/3738.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix regression where uploading files with non-ASCII names failed. \ No newline at end of file diff --git a/newsfragments/3739.bugfix b/newsfragments/3739.bugfix deleted file mode 100644 index 875941cf8..000000000 --- a/newsfragments/3739.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fixed annoying UnicodeWarning message on Python 2 when running CLI tools. \ No newline at end of file diff --git a/newsfragments/3741.minor b/newsfragments/3741.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3743.minor b/newsfragments/3743.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3744.minor b/newsfragments/3744.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3745.minor b/newsfragments/3745.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3746.minor b/newsfragments/3746.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3747.documentation b/newsfragments/3747.documentation deleted file mode 100644 index a2559a6a0..000000000 --- a/newsfragments/3747.documentation +++ /dev/null @@ -1 +0,0 @@ -Rewriting the installation guide for Tahoe-LAFS. diff --git a/newsfragments/3749.documentation b/newsfragments/3749.documentation deleted file mode 100644 index 554564a0b..000000000 --- a/newsfragments/3749.documentation +++ /dev/null @@ -1 +0,0 @@ -Documentation and installation links in the README have been fixed. diff --git a/newsfragments/3751.minor b/newsfragments/3751.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3757.other b/newsfragments/3757.other deleted file mode 100644 index 3d2d3f272..000000000 --- a/newsfragments/3757.other +++ /dev/null @@ -1 +0,0 @@ -Refactored test_introducer in web tests to use custom base test cases \ No newline at end of file diff --git a/newsfragments/3759.minor b/newsfragments/3759.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3760.minor b/newsfragments/3760.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3763.minor b/newsfragments/3763.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3764.documentation b/newsfragments/3764.documentation deleted file mode 100644 index d473cd27c..000000000 --- a/newsfragments/3764.documentation +++ /dev/null @@ -1 +0,0 @@ -The Great Black Swamp proposed specification now includes sample interactions to demonstrate expected usage patterns. \ No newline at end of file diff --git a/newsfragments/3765.documentation b/newsfragments/3765.documentation deleted file mode 100644 index a3b59c4d6..000000000 --- a/newsfragments/3765.documentation +++ /dev/null @@ -1 +0,0 @@ -The Great Black Swamp proposed specification now includes a glossary. \ No newline at end of file diff --git a/newsfragments/3769.documentation b/newsfragments/3769.documentation deleted file mode 100644 index 3d4ef7d4c..000000000 --- a/newsfragments/3769.documentation +++ /dev/null @@ -1 +0,0 @@ -The Great Black Swamp specification now allows parallel upload of immutable share data. diff --git a/newsfragments/3773.minor b/newsfragments/3773.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3774.documentation b/newsfragments/3774.documentation deleted file mode 100644 index d58105966..000000000 --- a/newsfragments/3774.documentation +++ /dev/null @@ -1 +0,0 @@ -There is now a specification for the scheme which Tahoe-LAFS storage clients use to derive their lease renewal secrets. diff --git a/newsfragments/3777.documentation b/newsfragments/3777.documentation deleted file mode 100644 index 7635cc1e6..000000000 --- a/newsfragments/3777.documentation +++ /dev/null @@ -1 +0,0 @@ -The Great Black Swamp proposed specification now has a simplified interface for reading data from immutable shares. diff --git a/newsfragments/3779.bugfix b/newsfragments/3779.bugfix deleted file mode 100644 index 073046474..000000000 --- a/newsfragments/3779.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fixed bug where share corruption events were not logged on storage servers running on Windows. \ No newline at end of file diff --git a/newsfragments/3781.minor b/newsfragments/3781.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3782.documentation b/newsfragments/3782.documentation deleted file mode 100644 index 5e5cecc13..000000000 --- a/newsfragments/3782.documentation +++ /dev/null @@ -1 +0,0 @@ -tahoe-dev mailing list is now at tahoe-dev@lists.tahoe-lafs.org. diff --git a/newsfragments/3785.documentation b/newsfragments/3785.documentation deleted file mode 100644 index 4eb268f79..000000000 --- a/newsfragments/3785.documentation +++ /dev/null @@ -1 +0,0 @@ -The Great Black Swamp specification now describes the required authorization scheme. From 0377c619cb358697e0379d664bc8eee487338f0b Mon Sep 17 00:00:00 2001 From: fenn-cs Date: Fri, 17 Sep 2021 10:58:07 +0100 Subject: [PATCH 0245/2309] release 1.16.0-rc1 Signed-off-by: fenn-cs --- newsfragments/3754.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3754.minor diff --git a/newsfragments/3754.minor b/newsfragments/3754.minor new file mode 100644 index 000000000..e69de29bb From ac603c5e17f32cf18fcde479ab289329dfc3570b Mon Sep 17 00:00:00 2001 From: fenn-cs Date: Fri, 17 Sep 2021 11:05:47 +0100 Subject: [PATCH 0246/2309] added proper release title Signed-off-by: fenn-cs --- NEWS.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/NEWS.rst b/NEWS.rst index a7d814c70..366e45907 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -5,8 +5,9 @@ User-Visible Changes in Tahoe-LAFS ================================== .. towncrier start line -Release 1.15.1.post2188.dev0 (2021-09-17)Release 1.15.1.post2188.dev0 (2021-09-17) -''''''''''''''''''''''''''''''''''''''''' + +Release 1.16.0 (2021-09-17) +''''''''''''''''''''''''''' Backwards Incompatible Changes ------------------------------ From 3d9644f42915fc3140bf88007020d6526b32f139 Mon Sep 17 00:00:00 2001 From: fenn-cs Date: Fri, 17 Sep 2021 11:06:19 +0100 Subject: [PATCH 0247/2309] release notes for 1.16.0 Signed-off-by: fenn-cs --- relnotes.txt | 37 +++++++++++++++++++++++-------------- 1 file changed, 23 insertions(+), 14 deletions(-) diff --git a/relnotes.txt b/relnotes.txt index 4afbd6cc5..c97b42664 100644 --- a/relnotes.txt +++ b/relnotes.txt @@ -1,6 +1,6 @@ -ANNOUNCING Tahoe, the Least-Authority File Store, v1.15.1 +ANNOUNCING Tahoe, the Least-Authority File Store, v1.16.0 -The Tahoe-LAFS team is pleased to announce version 1.15.1 of +The Tahoe-LAFS team is pleased to announce version 1.16.0 of Tahoe-LAFS, an extremely reliable decentralized storage system. Get it with "pip install tahoe-lafs", or download a tarball here: @@ -16,14 +16,23 @@ unique security and fault-tolerance properties: https://tahoe-lafs.readthedocs.org/en/latest/about.html The previous stable release of Tahoe-LAFS was v1.15.0, released on -January 19, 2021. +March 23rd, 2021. -In this release: PyPI does not accept uploads of packages that use -PEP-508 version specifiers. +The major change in this release is the completion of the Python 3 +port -- while maintaining support for Python 2. A future release will +remove Python 2 support. -Note that Python3 porting is underway but not yet complete in this -release. Developers may notice python3 as new targets for certain -tools. +The previously deprecated subcommands "start", "stop", "restart" and +"daemonize" have been removed. You must now use "tahoe run" (possibly +along with your favourite daemonization software). + +Several features are now removed: the Account Server, stats-gatherer +and FTP support. + +There are several dependency changes that will be interesting for +distribution maintainers. + +As well 196 bugs have been fixed since the last release. Please see ``NEWS.rst`` for a more complete list of changes. @@ -142,19 +151,19 @@ solely as a labor of love by volunteers. Thank you very much to the team of "hackers in the public interest" who make Tahoe-LAFS possible. -meejah +fenn-cs on behalf of the Tahoe-LAFS team -March 23, 2021 +September 16, 2021 Planet Earth -[1] https://github.com/tahoe-lafs/tahoe-lafs/blob/tahoe-lafs-1.15.1/NEWS.rst +[1] https://github.com/tahoe-lafs/tahoe-lafs/blob/tahoe-lafs-1.16.0/NEWS.rst [2] https://github.com/tahoe-lafs/tahoe-lafs/blob/master/docs/known_issues.rst [3] https://tahoe-lafs.org/trac/tahoe-lafs/wiki/RelatedProjects -[4] https://github.com/tahoe-lafs/tahoe-lafs/blob/tahoe-lafs-1.15.1/COPYING.GPL -[5] https://github.com/tahoe-lafs/tahoe-lafs/blob/tahoe-lafs-1.15.1/COPYING.TGPPL.rst -[6] https://tahoe-lafs.readthedocs.org/en/tahoe-lafs-1.15.1/INSTALL.html +[4] https://github.com/tahoe-lafs/tahoe-lafs/blob/tahoe-lafs-1.16.0/COPYING.GPL +[5] https://github.com/tahoe-lafs/tahoe-lafs/blob/tahoe-lafs-1.16.0/COPYING.TGPPL.rst +[6] https://tahoe-lafs.readthedocs.org/en/tahoe-lafs-1.16.0/INSTALL.html [7] https://lists.tahoe-lafs.org/mailman/listinfo/tahoe-dev [8] https://tahoe-lafs.org/trac/tahoe-lafs/roadmap [9] https://github.com/tahoe-lafs/tahoe-lafs/blob/master/CREDITS From 87ea676502cc5a231a2efabfc50f2cb7fd42d9bf Mon Sep 17 00:00:00 2001 From: fenn-cs Date: Fri, 17 Sep 2021 11:06:37 +0100 Subject: [PATCH 0248/2309] update nix version Signed-off-by: fenn-cs --- nix/tahoe-lafs.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nix/tahoe-lafs.nix b/nix/tahoe-lafs.nix index 35b29f1cc..2aff6af18 100644 --- a/nix/tahoe-lafs.nix +++ b/nix/tahoe-lafs.nix @@ -7,7 +7,7 @@ , html5lib, pyutil, distro, configparser }: python.pkgs.buildPythonPackage rec { - # Most of the time this is not exactly the release version (eg 1.15.1). + # Most of the time this is not exactly the release version (eg 1.16.0). # Give it a `post` component to make it look newer than the release version # and we'll bump this up at the time of each release. # @@ -20,7 +20,7 @@ python.pkgs.buildPythonPackage rec { # is not a reproducable artifact (in the sense of "reproducable builds") so # it is excluded from the source tree by default. When it is included, the # package tends to be frequently spuriously rebuilt. - version = "1.15.1.post1"; + version = "1.16.0.post1"; name = "tahoe-lafs-${version}"; src = lib.cleanSourceWith { src = ../.; From d26101c82528243af03d427e0c6f411d114572a1 Mon Sep 17 00:00:00 2001 From: fenn-cs Date: Fri, 17 Sep 2021 11:07:24 +0100 Subject: [PATCH 0249/2309] acknowledge new contributors Signed-off-by: fenn-cs --- CREDITS | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/CREDITS b/CREDITS index b0923fc35..8a6e876ec 100644 --- a/CREDITS +++ b/CREDITS @@ -240,3 +240,23 @@ N: Lukas Pirl E: tahoe@lukas-pirl.de W: http://lukas-pirl.de D: Buildslaves (Debian, Fedora, CentOS; 2016-2021) + +N: Anxhelo Lushka +E: anxhelo1995@gmail.com +D: Web site design and updates + +N: Fon E. Noel +E: fenn25.fn@gmail.com +D: bug-fixes and refactoring + +N: Jehad Baeth +E: jehad@leastauthority.com +D: Documentation improvement + +N: May-Lee Sia +E: mayleesia@gmail.com +D: Community-manager and documentation improvements + +N: Yash Nayani +E: yashaswi.nram@gmail.com +D: Installation Guide improvements From f6a96ae3976ee21ad0376f7b6a22fc3d12110dce Mon Sep 17 00:00:00 2001 From: fenn-cs Date: Fri, 17 Sep 2021 11:07:58 +0100 Subject: [PATCH 0250/2309] fix tarballs target for release Signed-off-by: fenn-cs --- tox.ini | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 9b0f71038..3aa6a6d43 100644 --- a/tox.ini +++ b/tox.ini @@ -258,7 +258,8 @@ commands= pyinstaller -y --clean pyinstaller.spec [testenv:tarballs] +basepython = python3 deps = commands = python setup.py update_version - python setup.py sdist --formats=bztar,gztar,zip bdist_wheel + python setup.py sdist --formats=bztar,gztar,zip bdist_wheel --universal From 5bd5ee580acd3d0a95b190074d2da1fc5d98975e Mon Sep 17 00:00:00 2001 From: fenn-cs Date: Sat, 18 Sep 2021 23:50:34 +0100 Subject: [PATCH 0251/2309] layout for tests that check if log_call_deffered decorates parametized functions correctly Signed-off-by: fenn-cs --- src/allmydata/test/test_eliotutil.py | 41 ++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/src/allmydata/test/test_eliotutil.py b/src/allmydata/test/test_eliotutil.py index 3f915ecd2..7db23ce9b 100644 --- a/src/allmydata/test/test_eliotutil.py +++ b/src/allmydata/test/test_eliotutil.py @@ -281,3 +281,44 @@ class LogCallDeferredTests(TestCase): ), ), ) + + @capture_logging( + lambda self, logger: + assertHasAction(self, logger, u"the-action", succeeded=True), + ) + def test_gets_positional_arguments(self, logger): + """ + Check that positional arguments are logged when using ``log_call_deferred`` + """ + @log_call_deferred(action_type=u"the-action") + def f(a): + return a ** 2 + self.assertThat( + f(4), succeeded(Equals(16))) + + @capture_logging( + lambda self, logger: + assertHasAction(self, logger, u"the-action", succeeded=True), + ) + def test_gets_keyword_arguments(self, logger): + """ + Check that keyword arguments are logged when using ``log_call_deferred`` + """ + @log_call_deferred(action_type=u"the-action") + def f(base, exp): + return base ** exp + self.assertThat(f(exp=2,base=10), succeeded(Equals(100))) + + + @capture_logging( + lambda self, logger: + assertHasAction(self, logger, u"the-action", succeeded=True), + ) + def test_gets_keyword_and_positional_arguments(self, logger): + """ + Check that both keyword and positional arguments are logged when using ``log_call_deferred`` + """ + @log_call_deferred(action_type=u"the-action") + def f(base, exp, message): + return base ** exp + self.assertThat(f(10, 2, message="an exponential function"), succeeded(Equals(100))) \ No newline at end of file From dd8aa8a66648a9582a8732afde704cebdd4b16fa Mon Sep 17 00:00:00 2001 From: fenn-cs Date: Wed, 22 Sep 2021 23:37:33 +0100 Subject: [PATCH 0252/2309] test if log_call_deffered decorates parametized functions correctly Signed-off-by: fenn-cs --- src/allmydata/test/test_eliotutil.py | 9 ++++++++- src/allmydata/util/eliotutil.py | 4 ++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/allmydata/test/test_eliotutil.py b/src/allmydata/test/test_eliotutil.py index 7db23ce9b..7edd4e780 100644 --- a/src/allmydata/test/test_eliotutil.py +++ b/src/allmydata/test/test_eliotutil.py @@ -56,6 +56,7 @@ from eliot.testing import ( capture_logging, assertHasAction, swap_logger, + assertContainsFields, ) from twisted.internet.defer import ( @@ -295,6 +296,8 @@ class LogCallDeferredTests(TestCase): return a ** 2 self.assertThat( f(4), succeeded(Equals(16))) + msg = logger.messages[0] + assertContainsFields(self, msg, {"args": (4,)}) @capture_logging( lambda self, logger: @@ -308,6 +311,8 @@ class LogCallDeferredTests(TestCase): def f(base, exp): return base ** exp self.assertThat(f(exp=2,base=10), succeeded(Equals(100))) + msg = logger.messages[0] + assertContainsFields(self, msg, {"base": 10, "exp": 2}) @capture_logging( @@ -321,4 +326,6 @@ class LogCallDeferredTests(TestCase): @log_call_deferred(action_type=u"the-action") def f(base, exp, message): return base ** exp - self.assertThat(f(10, 2, message="an exponential function"), succeeded(Equals(100))) \ No newline at end of file + self.assertThat(f(10, 2, message="an exponential function"), succeeded(Equals(100))) + msg = logger.messages[0] + assertContainsFields(self, msg, {"args": (10, 2), "message": "an exponential function"}) diff --git a/src/allmydata/util/eliotutil.py b/src/allmydata/util/eliotutil.py index d989c9e2a..ac2d3e4e0 100644 --- a/src/allmydata/util/eliotutil.py +++ b/src/allmydata/util/eliotutil.py @@ -324,8 +324,8 @@ def log_call_deferred(action_type): def logged_f(*a, **kw): # Use the action's context method to avoid ending the action when # the `with` block ends. - args = {k: bytes_to_unicode(True, kw[k]) for k in kw} - with start_action(action_type=action_type, **args).context(): + kwargs = {k: bytes_to_unicode(True, kw[k]) for k in kw} + with start_action(action_type=action_type, args=a, **kwargs).context(): # Use addActionFinish so that the action finishes when the # Deferred fires. d = maybeDeferred(f, *a, **kw) From 88cbb7b109946709fb93e982a73b4b4e84fda595 Mon Sep 17 00:00:00 2001 From: fenn-cs Date: Fri, 24 Sep 2021 23:04:01 +0100 Subject: [PATCH 0253/2309] remove methods that break test_filenode with AsyncBrokenTest Signed-off-by: fenn-cs --- src/allmydata/test/mutable/test_filenode.py | 95 +++++++++++---------- 1 file changed, 52 insertions(+), 43 deletions(-) diff --git a/src/allmydata/test/mutable/test_filenode.py b/src/allmydata/test/mutable/test_filenode.py index de03afc5a..748df1fde 100644 --- a/src/allmydata/test/mutable/test_filenode.py +++ b/src/allmydata/test/mutable/test_filenode.py @@ -13,6 +13,14 @@ if PY2: from six.moves import cStringIO as StringIO from twisted.internet import defer, reactor from twisted.trial import unittest +from ..common import AsyncTestCase, AsyncBrokenTestCase +from testtools.matchers import ( + Equals, + Contains, + HasLength, + Is, + IsInstance, +) from allmydata import uri, client from allmydata.util.consumer import MemoryConsumer from allmydata.interfaces import SDMF_VERSION, MDMF_VERSION, DownloadStopped @@ -29,12 +37,13 @@ from .util import ( make_peer, ) -class Filenode(unittest.TestCase, testutil.ShouldFailMixin): +class Filenode(AsyncBrokenTestCase, testutil.ShouldFailMixin): # this used to be in Publish, but we removed the limit. Some of # these tests test whether the new code correctly allows files # larger than the limit. OLD_MAX_SEGMENT_SIZE = 3500000 def setUp(self): + super(Filenode, self).setUp() self._storage = FakeStorage() self._peers = list( make_peer(self._storage, n) @@ -48,12 +57,12 @@ class Filenode(unittest.TestCase, testutil.ShouldFailMixin): def test_create(self): d = self.nodemaker.create_mutable_file() def _created(n): - self.failUnless(isinstance(n, MutableFileNode)) - self.failUnlessEqual(n.get_storage_index(), n._storage_index) + self.assertThat(n, IsInstance(MutableFileNode)) + self.assertThat(n.get_storage_index(), Equals(n._storage_index)) sb = self.nodemaker.storage_broker peer0 = sorted(sb.get_all_serverids())[0] shnums = self._storage._peers[peer0].keys() - self.failUnlessEqual(len(shnums), 1) + self.assertThat(shnums, HasLength(1)) d.addCallback(_created) return d @@ -61,12 +70,12 @@ class Filenode(unittest.TestCase, testutil.ShouldFailMixin): def test_create_mdmf(self): d = self.nodemaker.create_mutable_file(version=MDMF_VERSION) def _created(n): - self.failUnless(isinstance(n, MutableFileNode)) - self.failUnlessEqual(n.get_storage_index(), n._storage_index) + self.assertThat(n, IsInstance(MutableFileNode)) + self.assertThat(n.get_storage_index(), Equals(n._storage_index)) sb = self.nodemaker.storage_broker peer0 = sorted(sb.get_all_serverids())[0] shnums = self._storage._peers[peer0].keys() - self.failUnlessEqual(len(shnums), 1) + self.assertThat(shnums, HasLength(1)) d.addCallback(_created) return d @@ -80,7 +89,7 @@ class Filenode(unittest.TestCase, testutil.ShouldFailMixin): d.addCallback(lambda ignored, v=v: self.nodemaker.create_mutable_file(version=v)) def _created(n): - self.failUnless(isinstance(n, MutableFileNode)) + self.assertThat(n, IsInstance(MutableFileNode)) self._node = n return n d.addCallback(_created) @@ -89,19 +98,19 @@ class Filenode(unittest.TestCase, testutil.ShouldFailMixin): d.addCallback(lambda ignored: self._node.download_best_version()) d.addCallback(lambda contents: - self.failUnlessEqual(contents, b"Contents" * 50000)) + self.assertThat(contents, Equals(b"Contents" * 50000))) return d def test_max_shares(self): self.nodemaker.default_encoding_parameters['n'] = 255 d = self.nodemaker.create_mutable_file(version=SDMF_VERSION) def _created(n): - self.failUnless(isinstance(n, MutableFileNode)) - self.failUnlessEqual(n.get_storage_index(), n._storage_index) + self.assertThat(n, IsInstance(MutableFileNode)) + self.assertThat(n.get_storage_index(), Equals(n._storage_index)) sb = self.nodemaker.storage_broker num_shares = sum([len(self._storage._peers[x].keys()) for x \ in sb.get_all_serverids()]) - self.failUnlessEqual(num_shares, 255) + self.assertThat(num_shares, Equals(255)) self._node = n return n d.addCallback(_created) @@ -121,12 +130,12 @@ class Filenode(unittest.TestCase, testutil.ShouldFailMixin): self.nodemaker.default_encoding_parameters['n'] = 255 d = self.nodemaker.create_mutable_file(version=MDMF_VERSION) def _created(n): - self.failUnless(isinstance(n, MutableFileNode)) - self.failUnlessEqual(n.get_storage_index(), n._storage_index) + self.assertThat(n, IsInstance(MutableFileNode)) + self.assertThat(n.get_storage_index(), Equals(n._storage_index)) sb = self.nodemaker.storage_broker num_shares = sum([len(self._storage._peers[x].keys()) for x \ in sb.get_all_serverids()]) - self.failUnlessEqual(num_shares, 255) + self.assertThat(num_shares, Equals(255)) self._node = n return n d.addCallback(_created) @@ -135,20 +144,20 @@ class Filenode(unittest.TestCase, testutil.ShouldFailMixin): d.addCallback(lambda ignored: self._node.download_best_version()) d.addCallback(lambda contents: - self.failUnlessEqual(contents, b"contents" * 50000)) + self.assertThat(contents, Equals(b"contents" * 50000))) return d def test_mdmf_filenode_cap(self): # Test that an MDMF filenode, once created, returns an MDMF URI. d = self.nodemaker.create_mutable_file(version=MDMF_VERSION) def _created(n): - self.failUnless(isinstance(n, MutableFileNode)) + self.assertThat(n, IsInstance(MutableFileNode)) cap = n.get_cap() - self.failUnless(isinstance(cap, uri.WriteableMDMFFileURI)) + self.assertThat(cap, IsInstance(uri.WriteableMDMFFileURI)) rcap = n.get_readcap() - self.failUnless(isinstance(rcap, uri.ReadonlyMDMFFileURI)) + self.assertThat(rcap, IsInstance(uri.ReadonlyMDMFFileURI)) vcap = n.get_verify_cap() - self.failUnless(isinstance(vcap, uri.MDMFVerifierURI)) + self.assertThat(vcap, IsInstance(uri.MDMFVerifierURI)) d.addCallback(_created) return d @@ -158,13 +167,13 @@ class Filenode(unittest.TestCase, testutil.ShouldFailMixin): # filenode given an MDMF cap. d = self.nodemaker.create_mutable_file(version=MDMF_VERSION) def _created(n): - self.failUnless(isinstance(n, MutableFileNode)) + self.assertThat(n, IsInstance(MutableFileNode)) s = n.get_uri() self.failUnless(s.startswith(b"URI:MDMF")) n2 = self.nodemaker.create_from_cap(s) - self.failUnless(isinstance(n2, MutableFileNode)) - self.failUnlessEqual(n.get_storage_index(), n2.get_storage_index()) - self.failUnlessEqual(n.get_uri(), n2.get_uri()) + self.assertThat(n2, IsInstance(MutableFileNode)) + self.assertThat(n.get_storage_index(), Equals(n2.get_storage_index())) + self.assertThat(n.get_uri(), Equals(n2.get_uri())) d.addCallback(_created) return d @@ -172,10 +181,10 @@ class Filenode(unittest.TestCase, testutil.ShouldFailMixin): def test_create_from_mdmf_readcap(self): d = self.nodemaker.create_mutable_file(version=MDMF_VERSION) def _created(n): - self.failUnless(isinstance(n, MutableFileNode)) + self.assertThat(n, IsInstance(MutableFileNode)) s = n.get_readonly_uri() n2 = self.nodemaker.create_from_cap(s) - self.failUnless(isinstance(n2, MutableFileNode)) + self.assertThat(n2, IsInstance(MutableFileNode)) # Check that it's a readonly node self.failUnless(n2.is_readonly()) @@ -191,10 +200,10 @@ class Filenode(unittest.TestCase, testutil.ShouldFailMixin): d = self.nodemaker.create_mutable_file(version=MDMF_VERSION) def _created(n): self.uri = n.get_uri() - self.failUnlessEqual(n._protocol_version, MDMF_VERSION) + self.assertThat(n._protocol_version, Equals(MDMF_VERSION)) n2 = self.nodemaker.create_from_cap(self.uri) - self.failUnlessEqual(n2._protocol_version, MDMF_VERSION) + self.assertThat(n2._protocol_version, Equals(MDMF_VERSION)) d.addCallback(_created) return d @@ -203,14 +212,14 @@ class Filenode(unittest.TestCase, testutil.ShouldFailMixin): n = MutableFileNode(None, None, {"k": 3, "n": 10}, None) calls = [] def _callback(*args, **kwargs): - self.failUnlessEqual(args, (4,) ) - self.failUnlessEqual(kwargs, {"foo": 5}) + self.assertThat(args, Equals((4,))) + self.assertThat(kwargs, Equals({"foo": 5})) calls.append(1) return 6 d = n._do_serialized(_callback, 4, foo=5) def _check_callback(res): - self.failUnlessEqual(res, 6) - self.failUnlessEqual(calls, [1]) + self.assertThat(res, Equals(6)) + self.assertThat(calls, Equals([1])) d.addCallback(_check_callback) def _errback(): @@ -229,24 +238,24 @@ class Filenode(unittest.TestCase, testutil.ShouldFailMixin): d.addCallback(lambda sio: self.failUnless("3-of-10" in sio.getvalue())) d.addCallback(lambda res: n.overwrite(MutableData(b"contents 1"))) - d.addCallback(lambda res: self.failUnlessIdentical(res, None)) + d.addCallback(lambda res: self.assertThat(res, Is(None))) d.addCallback(lambda res: n.download_best_version()) - d.addCallback(lambda res: self.failUnlessEqual(res, b"contents 1")) + d.addCallback(lambda res: self.assertThat(res, Equals(b"contents 1"))) d.addCallback(lambda res: n.get_size_of_best_version()) d.addCallback(lambda size: - self.failUnlessEqual(size, len(b"contents 1"))) + self.assertThat(size, Equals(len(b"contents 1")))) d.addCallback(lambda res: n.overwrite(MutableData(b"contents 2"))) d.addCallback(lambda res: n.download_best_version()) - d.addCallback(lambda res: self.failUnlessEqual(res, b"contents 2")) + d.addCallback(lambda res: self.assertThat(res, Equals(b"contents 2"))) d.addCallback(lambda res: n.get_servermap(MODE_WRITE)) d.addCallback(lambda smap: n.upload(MutableData(b"contents 3"), smap)) d.addCallback(lambda res: n.download_best_version()) - d.addCallback(lambda res: self.failUnlessEqual(res, b"contents 3")) + d.addCallback(lambda res: self.assertThat(res, Equals(b"contents 3"))) d.addCallback(lambda res: n.get_servermap(MODE_ANYTHING)) d.addCallback(lambda smap: n.download_version(smap, smap.best_recoverable_version())) - d.addCallback(lambda res: self.failUnlessEqual(res, b"contents 3")) + d.addCallback(lambda res: self.assertThat(res, Equals(b"contents 3"))) # test a file that is large enough to overcome the # mapupdate-to-retrieve data caching (i.e. make the shares larger # than the default readsize, which is 2000 bytes). A 15kB file @@ -254,7 +263,7 @@ class Filenode(unittest.TestCase, testutil.ShouldFailMixin): d.addCallback(lambda res: n.overwrite(MutableData(b"large size file" * 1000))) d.addCallback(lambda res: n.download_best_version()) d.addCallback(lambda res: - self.failUnlessEqual(res, b"large size file" * 1000)) + self.assertThat(res, Equals(b"large size file" * 1000))) return d d.addCallback(_created) return d @@ -268,7 +277,7 @@ class Filenode(unittest.TestCase, testutil.ShouldFailMixin): n.get_servermap(MODE_READ)) def _then(servermap): dumped = servermap.dump(StringIO()) - self.failUnlessIn("3-of-10", dumped.getvalue()) + self.assertThat(dumped.getvalue(), Contains("3-of-10")) d.addCallback(_then) # Now overwrite the contents with some new contents. We want # to make them big enough to force the file to be uploaded @@ -431,7 +440,7 @@ class Filenode(unittest.TestCase, testutil.ShouldFailMixin): def test_create_with_initial_contents_function(self): data = b"initial contents" def _make_contents(n): - self.failUnless(isinstance(n, MutableFileNode)) + self.assertThat(n, IsInstance(MutableFileNode)) key = n.get_writekey() self.failUnless(isinstance(key, bytes), key) self.failUnlessEqual(len(key), 16) # AES key size @@ -447,7 +456,7 @@ class Filenode(unittest.TestCase, testutil.ShouldFailMixin): def test_create_mdmf_with_initial_contents_function(self): data = b"initial contents" * 100000 def _make_contents(n): - self.failUnless(isinstance(n, MutableFileNode)) + self.assertThat(n, IsInstance(MutableFileNode)) key = n.get_writekey() self.failUnless(isinstance(key, bytes), key) self.failUnlessEqual(len(key), 16) @@ -643,7 +652,7 @@ class Filenode(unittest.TestCase, testutil.ShouldFailMixin): d.addCallback(lambda sio: self.failUnless("3-of-10" in sio.getvalue())) d.addCallback(lambda res: n.overwrite(MutableData(b"contents 1"))) - d.addCallback(lambda res: self.failUnlessIdentical(res, None)) + d.addCallback(lambda res: self.assertThat(res, Is(None))) d.addCallback(lambda res: n.download_best_version()) d.addCallback(lambda res: self.failUnlessEqual(res, b"contents 1")) d.addCallback(lambda res: n.overwrite(MutableData(b"contents 2"))) From 49b6080097e12a9150db45534b81cfee48f39b9d Mon Sep 17 00:00:00 2001 From: fenn-cs Date: Sat, 25 Sep 2021 21:03:01 +0100 Subject: [PATCH 0254/2309] remove depracated assert methods Signed-off-by: fenn-cs --- src/allmydata/test/mutable/test_filenode.py | 95 ++++++++++----------- 1 file changed, 47 insertions(+), 48 deletions(-) diff --git a/src/allmydata/test/mutable/test_filenode.py b/src/allmydata/test/mutable/test_filenode.py index 748df1fde..579734433 100644 --- a/src/allmydata/test/mutable/test_filenode.py +++ b/src/allmydata/test/mutable/test_filenode.py @@ -12,8 +12,7 @@ if PY2: from six.moves import cStringIO as StringIO from twisted.internet import defer, reactor -from twisted.trial import unittest -from ..common import AsyncTestCase, AsyncBrokenTestCase +from ..common import AsyncBrokenTestCase from testtools.matchers import ( Equals, Contains, @@ -122,7 +121,7 @@ class Filenode(AsyncBrokenTestCase, testutil.ShouldFailMixin): self._node.download_best_version()) # ...and check to make sure everything went okay. d.addCallback(lambda contents: - self.failUnlessEqual(b"contents" * 50000, contents)) + self.assertThat(b"contents" * 50000, Equals(contents))) return d def test_max_shares_mdmf(self): @@ -169,7 +168,7 @@ class Filenode(AsyncBrokenTestCase, testutil.ShouldFailMixin): def _created(n): self.assertThat(n, IsInstance(MutableFileNode)) s = n.get_uri() - self.failUnless(s.startswith(b"URI:MDMF")) + self.assertTrue(s.startswith(b"URI:MDMF")) n2 = self.nodemaker.create_from_cap(s) self.assertThat(n2, IsInstance(MutableFileNode)) self.assertThat(n.get_storage_index(), Equals(n2.get_storage_index())) @@ -187,7 +186,7 @@ class Filenode(AsyncBrokenTestCase, testutil.ShouldFailMixin): self.assertThat(n2, IsInstance(MutableFileNode)) # Check that it's a readonly node - self.failUnless(n2.is_readonly()) + self.assertTrue(n2.is_readonly()) d.addCallback(_created) return d @@ -236,7 +235,7 @@ class Filenode(AsyncBrokenTestCase, testutil.ShouldFailMixin): d.addCallback(lambda res: n.get_servermap(MODE_READ)) d.addCallback(lambda smap: smap.dump(StringIO())) d.addCallback(lambda sio: - self.failUnless("3-of-10" in sio.getvalue())) + self.assertTrue("3-of-10" in sio.getvalue())) d.addCallback(lambda res: n.overwrite(MutableData(b"contents 1"))) d.addCallback(lambda res: self.assertThat(res, Is(None))) d.addCallback(lambda res: n.download_best_version()) @@ -289,7 +288,7 @@ class Filenode(AsyncBrokenTestCase, testutil.ShouldFailMixin): d.addCallback(lambda ignored: n.download_best_version()) d.addCallback(lambda data: - self.failUnlessEqual(data, big_contents)) + self.assertThat(data, Equals(big_contents))) # Overwrite the contents again with some new contents. As # before, they need to be big enough to force multiple # segments, so that we make the downloader deal with @@ -301,7 +300,7 @@ class Filenode(AsyncBrokenTestCase, testutil.ShouldFailMixin): d.addCallback(lambda ignored: n.download_best_version()) d.addCallback(lambda data: - self.failUnlessEqual(data, bigger_contents)) + self.assertThat(data, Equals(bigger_contents))) return d d.addCallback(_created) return d @@ -332,7 +331,7 @@ class Filenode(AsyncBrokenTestCase, testutil.ShouldFailMixin): # Now we'll retrieve it into a pausing consumer. c = PausingConsumer() d = version.read(c) - d.addCallback(lambda ign: self.failUnlessEqual(c.size, len(data))) + d.addCallback(lambda ign: self.assertThat(c.size, Equals(len(data)))) c2 = PausingAndStoppingConsumer() d.addCallback(lambda ign: @@ -369,14 +368,14 @@ class Filenode(AsyncBrokenTestCase, testutil.ShouldFailMixin): self.uri = node.get_uri() # also confirm that the cap has no extension fields pieces = self.uri.split(b":") - self.failUnlessEqual(len(pieces), 4) + self.assertThat(pieces, HasLength(4)) return node.overwrite(MutableData(b"contents1" * 100000)) def _then(ignored): node = self.nodemaker.create_from_cap(self.uri) return node.download_best_version() def _downloaded(data): - self.failUnlessEqual(data, b"contents1" * 100000) + self.assertThat(data, Equals(b"contents1" * 100000)) d.addCallback(_created) d.addCallback(_then) d.addCallback(_downloaded) @@ -406,11 +405,11 @@ class Filenode(AsyncBrokenTestCase, testutil.ShouldFailMixin): d = self.nodemaker.create_mutable_file(upload1) def _created(n): d = n.download_best_version() - d.addCallback(lambda res: self.failUnlessEqual(res, b"contents 1")) + d.addCallback(lambda res: self.assertThat(res, Equals(b"contents 1"))) upload2 = MutableData(b"contents 2") d.addCallback(lambda res: n.overwrite(upload2)) d.addCallback(lambda res: n.download_best_version()) - d.addCallback(lambda res: self.failUnlessEqual(res, b"contents 2")) + d.addCallback(lambda res: self.assertThat(res, Equals(b"contents 2"))) return d d.addCallback(_created) return d @@ -424,15 +423,15 @@ class Filenode(AsyncBrokenTestCase, testutil.ShouldFailMixin): def _created(n): d = n.download_best_version() d.addCallback(lambda data: - self.failUnlessEqual(data, initial_contents)) + self.assertThat(data, Equals(initial_contents))) uploadable2 = MutableData(initial_contents + b"foobarbaz") d.addCallback(lambda ignored: n.overwrite(uploadable2)) d.addCallback(lambda ignored: n.download_best_version()) d.addCallback(lambda data: - self.failUnlessEqual(data, initial_contents + - b"foobarbaz")) + self.assertThat(data, Equals(initial_contents + + b"foobarbaz"))) return d d.addCallback(_created) return d @@ -442,14 +441,14 @@ class Filenode(AsyncBrokenTestCase, testutil.ShouldFailMixin): def _make_contents(n): self.assertThat(n, IsInstance(MutableFileNode)) key = n.get_writekey() - self.failUnless(isinstance(key, bytes), key) - self.failUnlessEqual(len(key), 16) # AES key size + self.assertTrue(isinstance(key, bytes), key) + self.assertThat(key, HasLength(16)) # AES key size return MutableData(data) d = self.nodemaker.create_mutable_file(_make_contents) def _created(n): return n.download_best_version() d.addCallback(_created) - d.addCallback(lambda data2: self.failUnlessEqual(data2, data)) + d.addCallback(lambda data2: self.assertThat(data2, Equals(data))) return d @@ -458,15 +457,15 @@ class Filenode(AsyncBrokenTestCase, testutil.ShouldFailMixin): def _make_contents(n): self.assertThat(n, IsInstance(MutableFileNode)) key = n.get_writekey() - self.failUnless(isinstance(key, bytes), key) - self.failUnlessEqual(len(key), 16) + self.assertTrue(isinstance(key, bytes), key) + self.assertThat(key, HasLength(16)) return MutableData(data) d = self.nodemaker.create_mutable_file(_make_contents, version=MDMF_VERSION) d.addCallback(lambda n: n.download_best_version()) d.addCallback(lambda data2: - self.failUnlessEqual(data2, data)) + self.assertThat(data2, Equals(data))) return d @@ -485,7 +484,7 @@ class Filenode(AsyncBrokenTestCase, testutil.ShouldFailMixin): d = n.get_servermap(MODE_READ) d.addCallback(lambda servermap: servermap.best_recoverable_version()) d.addCallback(lambda verinfo: - self.failUnlessEqual(verinfo[0], expected_seqnum, which)) + self.assertThat(verinfo[0], Equals(expected_seqnum), which)) return d def test_modify(self): @@ -522,36 +521,36 @@ class Filenode(AsyncBrokenTestCase, testutil.ShouldFailMixin): def _created(n): d = n.modify(_modifier) d.addCallback(lambda res: n.download_best_version()) - d.addCallback(lambda res: self.failUnlessEqual(res, b"line1line2")) + d.addCallback(lambda res: self.assertThat(res, Equals(b"line1line2"))) d.addCallback(lambda res: self.failUnlessCurrentSeqnumIs(n, 2, "m")) d.addCallback(lambda res: n.modify(_non_modifier)) d.addCallback(lambda res: n.download_best_version()) - d.addCallback(lambda res: self.failUnlessEqual(res, b"line1line2")) + d.addCallback(lambda res: self.assertThat(res, Equals(b"line1line2"))) d.addCallback(lambda res: self.failUnlessCurrentSeqnumIs(n, 2, "non")) d.addCallback(lambda res: n.modify(_none_modifier)) d.addCallback(lambda res: n.download_best_version()) - d.addCallback(lambda res: self.failUnlessEqual(res, b"line1line2")) + d.addCallback(lambda res: self.assertThat(res, Equals(b"line1line2"))) d.addCallback(lambda res: self.failUnlessCurrentSeqnumIs(n, 2, "none")) d.addCallback(lambda res: self.shouldFail(ValueError, "error_modifier", None, n.modify, _error_modifier)) d.addCallback(lambda res: n.download_best_version()) - d.addCallback(lambda res: self.failUnlessEqual(res, b"line1line2")) + d.addCallback(lambda res: self.assertThat(res, Equals(b"line1line2"))) d.addCallback(lambda res: self.failUnlessCurrentSeqnumIs(n, 2, "err")) d.addCallback(lambda res: n.download_best_version()) - d.addCallback(lambda res: self.failUnlessEqual(res, b"line1line2")) + d.addCallback(lambda res: self.assertThat(res, Equals(b"line1line2"))) d.addCallback(lambda res: self.failUnlessCurrentSeqnumIs(n, 2, "big")) d.addCallback(lambda res: n.modify(_ucw_error_modifier)) - d.addCallback(lambda res: self.failUnlessEqual(len(calls), 2)) + d.addCallback(lambda res: self.assertThat(calls, HasLength(2))) d.addCallback(lambda res: n.download_best_version()) - d.addCallback(lambda res: self.failUnlessEqual(res, - b"line1line2line3")) + d.addCallback(lambda res: self.assertThat(res, + Equals(b"line1line2line3"))) d.addCallback(lambda res: self.failUnlessCurrentSeqnumIs(n, 3, "ucw")) def _reset_ucw_error_modifier(res): @@ -566,10 +565,10 @@ class Filenode(AsyncBrokenTestCase, testutil.ShouldFailMixin): # will only be one larger than the previous test, not two (i.e. 4 # instead of 5). d.addCallback(lambda res: n.modify(_ucw_error_non_modifier)) - d.addCallback(lambda res: self.failUnlessEqual(len(calls), 2)) + d.addCallback(lambda res: self.assertThat(calls, HasLength(2))) d.addCallback(lambda res: n.download_best_version()) - d.addCallback(lambda res: self.failUnlessEqual(res, - b"line1line2line3")) + d.addCallback(lambda res: self.assertThat(res, + Equals(b"line1line2line3"))) d.addCallback(lambda res: self.failUnlessCurrentSeqnumIs(n, 4, "ucw")) d.addCallback(lambda res: n.modify(_toobig_modifier)) return d @@ -605,7 +604,7 @@ class Filenode(AsyncBrokenTestCase, testutil.ShouldFailMixin): def _created(n): d = n.modify(_modifier) d.addCallback(lambda res: n.download_best_version()) - d.addCallback(lambda res: self.failUnlessEqual(res, b"line1line2")) + d.addCallback(lambda res: self.assertThat(res, Equals(b"line1line2"))) d.addCallback(lambda res: self.failUnlessCurrentSeqnumIs(n, 2, "m")) d.addCallback(lambda res: @@ -614,7 +613,7 @@ class Filenode(AsyncBrokenTestCase, testutil.ShouldFailMixin): n.modify, _ucw_error_modifier, _backoff_stopper)) d.addCallback(lambda res: n.download_best_version()) - d.addCallback(lambda res: self.failUnlessEqual(res, b"line1line2")) + d.addCallback(lambda res: self.assertThat(res, Equals(b"line1line2"))) d.addCallback(lambda res: self.failUnlessCurrentSeqnumIs(n, 2, "stop")) def _reset_ucw_error_modifier(res): @@ -624,8 +623,8 @@ class Filenode(AsyncBrokenTestCase, testutil.ShouldFailMixin): d.addCallback(lambda res: n.modify(_ucw_error_modifier, _backoff_pauser)) d.addCallback(lambda res: n.download_best_version()) - d.addCallback(lambda res: self.failUnlessEqual(res, - b"line1line2line3")) + d.addCallback(lambda res: self.assertThat(res, + Equals(b"line1line2line3"))) d.addCallback(lambda res: self.failUnlessCurrentSeqnumIs(n, 3, "pause")) d.addCallback(lambda res: @@ -634,8 +633,8 @@ class Filenode(AsyncBrokenTestCase, testutil.ShouldFailMixin): n.modify, _always_ucw_error_modifier, giveuper.delay)) d.addCallback(lambda res: n.download_best_version()) - d.addCallback(lambda res: self.failUnlessEqual(res, - b"line1line2line3")) + d.addCallback(lambda res: self.assertThat(res, + Equals(b"line1line2line3"))) d.addCallback(lambda res: self.failUnlessCurrentSeqnumIs(n, 3, "giveup")) return d @@ -650,23 +649,23 @@ class Filenode(AsyncBrokenTestCase, testutil.ShouldFailMixin): d.addCallback(lambda res: n.get_servermap(MODE_READ)) d.addCallback(lambda smap: smap.dump(StringIO())) d.addCallback(lambda sio: - self.failUnless("3-of-10" in sio.getvalue())) + self.assertTrue("3-of-10" in sio.getvalue())) d.addCallback(lambda res: n.overwrite(MutableData(b"contents 1"))) d.addCallback(lambda res: self.assertThat(res, Is(None))) d.addCallback(lambda res: n.download_best_version()) - d.addCallback(lambda res: self.failUnlessEqual(res, b"contents 1")) + d.addCallback(lambda res: self.assertThat(res, Equals(b"contents 1"))) d.addCallback(lambda res: n.overwrite(MutableData(b"contents 2"))) d.addCallback(lambda res: n.download_best_version()) - d.addCallback(lambda res: self.failUnlessEqual(res, b"contents 2")) + d.addCallback(lambda res: self.assertThat(res, Equals(b"contents 2"))) d.addCallback(lambda res: n.get_servermap(MODE_WRITE)) d.addCallback(lambda smap: n.upload(MutableData(b"contents 3"), smap)) d.addCallback(lambda res: n.download_best_version()) - d.addCallback(lambda res: self.failUnlessEqual(res, b"contents 3")) + d.addCallback(lambda res: self.assertThat(res, Equals(b"contents 3"))) d.addCallback(lambda res: n.get_servermap(MODE_ANYTHING)) d.addCallback(lambda smap: n.download_version(smap, smap.best_recoverable_version())) - d.addCallback(lambda res: self.failUnlessEqual(res, b"contents 3")) + d.addCallback(lambda res: self.assertThat(res, Equals(b"contents 3"))) return d d.addCallback(_created) return d @@ -682,14 +681,14 @@ class Filenode(AsyncBrokenTestCase, testutil.ShouldFailMixin): return n.get_servermap(MODE_READ) d.addCallback(_created) d.addCallback(lambda ignored: - self.failUnlessEqual(self.n.get_size(), 0)) + self.assertThat(self.n.get_size(), Equals(0))) d.addCallback(lambda ignored: self.n.overwrite(MutableData(b"foobarbaz"))) d.addCallback(lambda ignored: - self.failUnlessEqual(self.n.get_size(), 9)) + self.assertThat(self.n.get_size(), Equals(9))) d.addCallback(lambda ignored: self.nodemaker.create_mutable_file(MutableData(b"foobarbaz"))) d.addCallback(_created) d.addCallback(lambda ignored: - self.failUnlessEqual(self.n.get_size(), 9)) + self.assertThat(self.n.get_size(), Equals(9))) return d From 759d4c85a295cfd620aacbc55225ee1d8aa236b2 Mon Sep 17 00:00:00 2001 From: fenn-cs Date: Tue, 28 Sep 2021 09:56:14 +0100 Subject: [PATCH 0255/2309] avoid argument collision in call of start_action in eliotutil Signed-off-by: fenn-cs --- src/allmydata/test/test_eliotutil.py | 5 +++-- src/allmydata/test/web/test_logs.py | 2 +- src/allmydata/util/eliotutil.py | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/allmydata/test/test_eliotutil.py b/src/allmydata/test/test_eliotutil.py index 7edd4e780..1fbb9ec8d 100644 --- a/src/allmydata/test/test_eliotutil.py +++ b/src/allmydata/test/test_eliotutil.py @@ -312,7 +312,7 @@ class LogCallDeferredTests(TestCase): return base ** exp self.assertThat(f(exp=2,base=10), succeeded(Equals(100))) msg = logger.messages[0] - assertContainsFields(self, msg, {"base": 10, "exp": 2}) + assertContainsFields(self, msg, {"kwargs": {"base": 10, "exp": 2}}) @capture_logging( @@ -328,4 +328,5 @@ class LogCallDeferredTests(TestCase): return base ** exp self.assertThat(f(10, 2, message="an exponential function"), succeeded(Equals(100))) msg = logger.messages[0] - assertContainsFields(self, msg, {"args": (10, 2), "message": "an exponential function"}) + assertContainsFields(self, msg, {"args": (10, 2)}) + assertContainsFields(self, msg, {"kwargs": {"message": "an exponential function"}}) diff --git a/src/allmydata/test/web/test_logs.py b/src/allmydata/test/web/test_logs.py index 043541690..fe0a0445d 100644 --- a/src/allmydata/test/web/test_logs.py +++ b/src/allmydata/test/web/test_logs.py @@ -121,7 +121,7 @@ class TestStreamingLogs(AsyncTestCase): self.assertThat(len(messages), Equals(3)) self.assertThat(messages[0]["action_type"], Equals("test:cli:some-exciting-action")) - self.assertThat(messages[0]["arguments"], + self.assertThat(messages[0]["kwargs"]["arguments"], Equals(["hello", "good-\\xff-day", 123, {"a": 35}, [None]])) self.assertThat(messages[1]["action_type"], Equals("test:cli:some-exciting-action")) self.assertThat("started", Equals(messages[0]["action_status"])) diff --git a/src/allmydata/util/eliotutil.py b/src/allmydata/util/eliotutil.py index ac2d3e4e0..e0c2fd8ae 100644 --- a/src/allmydata/util/eliotutil.py +++ b/src/allmydata/util/eliotutil.py @@ -325,7 +325,7 @@ def log_call_deferred(action_type): # Use the action's context method to avoid ending the action when # the `with` block ends. kwargs = {k: bytes_to_unicode(True, kw[k]) for k in kw} - with start_action(action_type=action_type, args=a, **kwargs).context(): + with start_action(action_type=action_type, args=a, kwargs=kwargs).context(): # Use addActionFinish so that the action finishes when the # Deferred fires. d = maybeDeferred(f, *a, **kw) From 5803d9999d3763e1cc7ac746273cf2873cede646 Mon Sep 17 00:00:00 2001 From: fenn-cs Date: Mon, 11 Oct 2021 13:49:29 +0100 Subject: [PATCH 0256/2309] remove unseriable args in log_call_deferred passed to start_action Signed-off-by: fenn-cs --- src/allmydata/test/test_eliotutil.py | 4 ++-- src/allmydata/util/eliotutil.py | 21 +++++++++++++++++++-- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/src/allmydata/test/test_eliotutil.py b/src/allmydata/test/test_eliotutil.py index 1fbb9ec8d..3d7b2bd42 100644 --- a/src/allmydata/test/test_eliotutil.py +++ b/src/allmydata/test/test_eliotutil.py @@ -297,7 +297,7 @@ class LogCallDeferredTests(TestCase): self.assertThat( f(4), succeeded(Equals(16))) msg = logger.messages[0] - assertContainsFields(self, msg, {"args": (4,)}) + assertContainsFields(self, msg, {"args": {'arg_0': 4}}) @capture_logging( lambda self, logger: @@ -328,5 +328,5 @@ class LogCallDeferredTests(TestCase): return base ** exp self.assertThat(f(10, 2, message="an exponential function"), succeeded(Equals(100))) msg = logger.messages[0] - assertContainsFields(self, msg, {"args": (10, 2)}) + assertContainsFields(self, msg, {"args": {'arg_0': 10, 'arg_1': 2}}) assertContainsFields(self, msg, {"kwargs": {"message": "an exponential function"}}) diff --git a/src/allmydata/util/eliotutil.py b/src/allmydata/util/eliotutil.py index e0c2fd8ae..fb18ed332 100644 --- a/src/allmydata/util/eliotutil.py +++ b/src/allmydata/util/eliotutil.py @@ -91,7 +91,7 @@ from .jsonbytes import ( AnyBytesJSONEncoder, bytes_to_unicode ) - +import json def validateInstanceOf(t): @@ -315,6 +315,14 @@ class _DestinationParser(object): _parse_destination_description = _DestinationParser().parse +def is_json_serializable(object): + try: + json.dumps(object) + return True + except (TypeError, OverflowError): + return False + + def log_call_deferred(action_type): """ Like ``eliot.log_call`` but for functions which return ``Deferred``. @@ -325,7 +333,14 @@ def log_call_deferred(action_type): # Use the action's context method to avoid ending the action when # the `with` block ends. kwargs = {k: bytes_to_unicode(True, kw[k]) for k in kw} - with start_action(action_type=action_type, args=a, kwargs=kwargs).context(): + # Remove complex (unserializable) objects from positional args to + # prevent eliot from throwing errors when it attempts serialization + args = { + "arg_" + str(pos): bytes_to_unicode(True, a[pos]) + for pos in range(len(a)) + if is_json_serializable(a[pos]) + } + with start_action(action_type=action_type, args=args, kwargs=kwargs).context(): # Use addActionFinish so that the action finishes when the # Deferred fires. d = maybeDeferred(f, *a, **kw) @@ -339,3 +354,5 @@ if PY2: capture_logging = eliot_capture_logging else: capture_logging = partial(eliot_capture_logging, encoder_=AnyBytesJSONEncoder) + + From 57a0f76e1f6ed5116c1defa221ff93241f60291d Mon Sep 17 00:00:00 2001 From: fenn-cs Date: Wed, 13 Oct 2021 23:41:42 +0100 Subject: [PATCH 0257/2309] maintain list of positional arguments as tuple Signed-off-by: fenn-cs --- src/allmydata/test/test_eliotutil.py | 4 ++-- src/allmydata/util/eliotutil.py | 6 +----- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/src/allmydata/test/test_eliotutil.py b/src/allmydata/test/test_eliotutil.py index 3d7b2bd42..1fbb9ec8d 100644 --- a/src/allmydata/test/test_eliotutil.py +++ b/src/allmydata/test/test_eliotutil.py @@ -297,7 +297,7 @@ class LogCallDeferredTests(TestCase): self.assertThat( f(4), succeeded(Equals(16))) msg = logger.messages[0] - assertContainsFields(self, msg, {"args": {'arg_0': 4}}) + assertContainsFields(self, msg, {"args": (4,)}) @capture_logging( lambda self, logger: @@ -328,5 +328,5 @@ class LogCallDeferredTests(TestCase): return base ** exp self.assertThat(f(10, 2, message="an exponential function"), succeeded(Equals(100))) msg = logger.messages[0] - assertContainsFields(self, msg, {"args": {'arg_0': 10, 'arg_1': 2}}) + assertContainsFields(self, msg, {"args": (10, 2)}) assertContainsFields(self, msg, {"kwargs": {"message": "an exponential function"}}) diff --git a/src/allmydata/util/eliotutil.py b/src/allmydata/util/eliotutil.py index fb18ed332..997dadb8d 100644 --- a/src/allmydata/util/eliotutil.py +++ b/src/allmydata/util/eliotutil.py @@ -335,11 +335,7 @@ def log_call_deferred(action_type): kwargs = {k: bytes_to_unicode(True, kw[k]) for k in kw} # Remove complex (unserializable) objects from positional args to # prevent eliot from throwing errors when it attempts serialization - args = { - "arg_" + str(pos): bytes_to_unicode(True, a[pos]) - for pos in range(len(a)) - if is_json_serializable(a[pos]) - } + args = tuple([a[pos] for pos in range(len(a)) if is_json_serializable(a[pos])]) with start_action(action_type=action_type, args=args, kwargs=kwargs).context(): # Use addActionFinish so that the action finishes when the # Deferred fires. From 1a12a8acdffea5491e223e82cafc557b1d8dbda6 Mon Sep 17 00:00:00 2001 From: fenn-cs Date: Fri, 15 Oct 2021 00:50:11 +0100 Subject: [PATCH 0258/2309] don't throw away unserializable parameter Signed-off-by: fenn-cs --- src/allmydata/util/eliotutil.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/allmydata/util/eliotutil.py b/src/allmydata/util/eliotutil.py index 997dadb8d..f2272a731 100644 --- a/src/allmydata/util/eliotutil.py +++ b/src/allmydata/util/eliotutil.py @@ -335,7 +335,12 @@ def log_call_deferred(action_type): kwargs = {k: bytes_to_unicode(True, kw[k]) for k in kw} # Remove complex (unserializable) objects from positional args to # prevent eliot from throwing errors when it attempts serialization - args = tuple([a[pos] for pos in range(len(a)) if is_json_serializable(a[pos])]) + args = tuple( + a[pos] + if is_json_serializable(a[pos]) + else str(a[pos]) + for pos in range(len(a)) + ) with start_action(action_type=action_type, args=args, kwargs=kwargs).context(): # Use addActionFinish so that the action finishes when the # Deferred fires. From 347377aaab20b0a806af2661bca8093f5644fea8 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 15 Oct 2021 11:43:34 -0400 Subject: [PATCH 0259/2309] Get rid of `check_memory` which depends on the control port This was some kind of memory usage analysis tool. It depends on the control port so it cannot work after I delete the control port. The code itself is messy, undocumented, and has no automated tests. I don't know if it works at all anymore. Even if it does, no one ever runs it. Measuring Tahoe-LAFS' memory usage over the course of maintenance and development is a lovely idea but the project has not managed to adopt (or maintain?) that practice based on this tool. Given sufficient interest we can resurrect this idea using a more streamlined process and less invasive tools in the future. --- .gitignore | 1 - misc/checkers/check_memory.py | 522 ---------------------------------- tox.ini | 11 - 3 files changed, 534 deletions(-) delete mode 100644 misc/checkers/check_memory.py diff --git a/.gitignore b/.gitignore index d6a58b88b..50a1352a2 100644 --- a/.gitignore +++ b/.gitignore @@ -30,7 +30,6 @@ zope.interface-*.egg /src/allmydata/test/plugins/dropin.cache /_trial_temp* -/_test_memory/ /tmp* /*.patch /dist/ diff --git a/misc/checkers/check_memory.py b/misc/checkers/check_memory.py deleted file mode 100644 index 268d77451..000000000 --- a/misc/checkers/check_memory.py +++ /dev/null @@ -1,522 +0,0 @@ -from __future__ import print_function - -import os, shutil, sys, urllib, time, stat, urlparse - -# Python 2 compatibility -from future.utils import PY2 -if PY2: - from future.builtins import str # noqa: F401 -from six.moves import cStringIO as StringIO - -from twisted.python.filepath import ( - FilePath, -) -from twisted.internet import defer, reactor, protocol, error -from twisted.application import service, internet -from twisted.web import client as tw_client -from twisted.python import log, procutils -from foolscap.api import Tub, fireEventually, flushEventualQueue - -from allmydata import client, introducer -from allmydata.immutable import upload -from allmydata.scripts import create_node -from allmydata.util import fileutil, pollmixin -from allmydata.util.fileutil import abspath_expanduser_unicode -from allmydata.util.encodingutil import get_filesystem_encoding - -from allmydata.scripts.common import ( - write_introducer, -) - -class StallableHTTPGetterDiscarder(tw_client.HTTPPageGetter, object): - full_speed_ahead = False - _bytes_so_far = 0 - stalled = None - def handleResponsePart(self, data): - self._bytes_so_far += len(data) - if not self.factory.do_stall: - return - if self.full_speed_ahead: - return - if self._bytes_so_far > 1e6+100: - if not self.stalled: - print("STALLING") - self.transport.pauseProducing() - self.stalled = reactor.callLater(10.0, self._resume_speed) - def _resume_speed(self): - print("RESUME SPEED") - self.stalled = None - self.full_speed_ahead = True - self.transport.resumeProducing() - def handleResponseEnd(self): - if self.stalled: - print("CANCEL") - self.stalled.cancel() - self.stalled = None - return tw_client.HTTPPageGetter.handleResponseEnd(self) - -class StallableDiscardingHTTPClientFactory(tw_client.HTTPClientFactory, object): - protocol = StallableHTTPGetterDiscarder - -def discardPage(url, stall=False, *args, **kwargs): - """Start fetching the URL, but stall our pipe after the first 1MB. - Wait 10 seconds, then resume downloading (and discarding) everything. - """ - # adapted from twisted.web.client.getPage . We can't just wrap or - # subclass because it provides no way to override the HTTPClientFactory - # that it creates. - scheme, netloc, path, params, query, fragment = urlparse.urlparse(url) - assert scheme == 'http' - host, port = netloc, 80 - if ":" in host: - host, port = host.split(":") - port = int(port) - factory = StallableDiscardingHTTPClientFactory(url, *args, **kwargs) - factory.do_stall = stall - reactor.connectTCP(host, port, factory) - return factory.deferred - -class ChildDidNotStartError(Exception): - pass - -class SystemFramework(pollmixin.PollMixin): - numnodes = 7 - - def __init__(self, basedir, mode): - self.basedir = basedir = abspath_expanduser_unicode(str(basedir)) - if not (basedir + os.path.sep).startswith(abspath_expanduser_unicode(u".") + os.path.sep): - raise AssertionError("safety issue: basedir must be a subdir") - self.testdir = testdir = os.path.join(basedir, "test") - if os.path.exists(testdir): - shutil.rmtree(testdir) - fileutil.make_dirs(testdir) - self.sparent = service.MultiService() - self.sparent.startService() - self.proc = None - self.tub = Tub() - self.tub.setOption("expose-remote-exception-types", False) - self.tub.setServiceParent(self.sparent) - self.mode = mode - self.failed = False - self.keepalive_file = None - - def run(self): - framelog = os.path.join(self.basedir, "driver.log") - log.startLogging(open(framelog, "a"), setStdout=False) - log.msg("CHECK_MEMORY(mode=%s) STARTING" % self.mode) - #logfile = open(os.path.join(self.testdir, "log"), "w") - #flo = log.FileLogObserver(logfile) - #log.startLoggingWithObserver(flo.emit, setStdout=False) - d = fireEventually() - d.addCallback(lambda res: self.setUp()) - d.addCallback(lambda res: self.record_initial_memusage()) - d.addCallback(lambda res: self.make_nodes()) - d.addCallback(lambda res: self.wait_for_client_connected()) - d.addCallback(lambda res: self.do_test()) - d.addBoth(self.tearDown) - def _err(err): - self.failed = err - log.err(err) - print(err) - d.addErrback(_err) - def _done(res): - reactor.stop() - return res - d.addBoth(_done) - reactor.run() - if self.failed: - # raiseException doesn't work for CopiedFailures - self.failed.raiseException() - - def setUp(self): - #print("STARTING") - self.stats = {} - self.statsfile = open(os.path.join(self.basedir, "stats.out"), "a") - self.make_introducer() - d = self.start_client() - def _record_control_furl(control_furl): - self.control_furl = control_furl - #print("OBTAINING '%s'" % (control_furl,)) - return self.tub.getReference(self.control_furl) - d.addCallback(_record_control_furl) - def _record_control(control_rref): - self.control_rref = control_rref - d.addCallback(_record_control) - def _ready(res): - #print("CLIENT READY") - pass - d.addCallback(_ready) - return d - - def record_initial_memusage(self): - print() - print("Client started (no connections yet)") - d = self._print_usage() - d.addCallback(self.stash_stats, "init") - return d - - def wait_for_client_connected(self): - print() - print("Client connecting to other nodes..") - return self.control_rref.callRemote("wait_for_client_connections", - self.numnodes+1) - - def tearDown(self, passthrough): - # the client node will shut down in a few seconds - #os.remove(os.path.join(self.clientdir, client.Client.EXIT_TRIGGER_FILE)) - log.msg("shutting down SystemTest services") - if self.keepalive_file and os.path.exists(self.keepalive_file): - age = time.time() - os.stat(self.keepalive_file)[stat.ST_MTIME] - log.msg("keepalive file at shutdown was %ds old" % age) - d = defer.succeed(None) - if self.proc: - d.addCallback(lambda res: self.kill_client()) - d.addCallback(lambda res: self.sparent.stopService()) - d.addCallback(lambda res: flushEventualQueue()) - def _close_statsfile(res): - self.statsfile.close() - d.addCallback(_close_statsfile) - d.addCallback(lambda res: passthrough) - return d - - def make_introducer(self): - iv_basedir = os.path.join(self.testdir, "introducer") - os.mkdir(iv_basedir) - self.introducer = introducer.IntroducerNode(basedir=iv_basedir) - self.introducer.setServiceParent(self) - self.introducer_furl = self.introducer.introducer_url - - def make_nodes(self): - root = FilePath(self.testdir) - self.nodes = [] - for i in range(self.numnodes): - nodedir = root.child("node%d" % (i,)) - private = nodedir.child("private") - private.makedirs() - write_introducer(nodedir, "default", self.introducer_url) - config = ( - "[client]\n" - "shares.happy = 1\n" - "[storage]\n" - ) - # the only tests for which we want the internal nodes to actually - # retain shares are the ones where somebody's going to download - # them. - if self.mode in ("download", "download-GET", "download-GET-slow"): - # retain shares - pass - else: - # for these tests, we tell the storage servers to pretend to - # accept shares, but really just throw them out, since we're - # only testing upload and not download. - config += "debug_discard = true\n" - if self.mode in ("receive",): - # for this mode, the client-under-test gets all the shares, - # so our internal nodes can refuse requests - config += "readonly = true\n" - nodedir.child("tahoe.cfg").setContent(config) - c = client.Client(basedir=nodedir.path) - c.setServiceParent(self) - self.nodes.append(c) - # the peers will start running, eventually they will connect to each - # other and the introducer - - def touch_keepalive(self): - if os.path.exists(self.keepalive_file): - age = time.time() - os.stat(self.keepalive_file)[stat.ST_MTIME] - log.msg("touching keepalive file, was %ds old" % age) - f = open(self.keepalive_file, "w") - f.write("""\ -If the node notices this file at startup, it will poll every 5 seconds and -terminate if the file is more than 10 seconds old, or if it has been deleted. -If the test harness has an internal failure and neglects to kill off the node -itself, this helps to avoid leaving processes lying around. The contents of -this file are ignored. - """) - f.close() - - def start_client(self): - # this returns a Deferred that fires with the client's control.furl - log.msg("MAKING CLIENT") - # self.testdir is an absolute Unicode path - clientdir = self.clientdir = os.path.join(self.testdir, u"client") - clientdir_str = clientdir.encode(get_filesystem_encoding()) - quiet = StringIO() - create_node.create_node({'basedir': clientdir}, out=quiet) - log.msg("DONE MAKING CLIENT") - write_introducer(clientdir, "default", self.introducer_furl) - # now replace tahoe.cfg - # set webport=0 and then ask the node what port it picked. - f = open(os.path.join(clientdir, "tahoe.cfg"), "w") - f.write("[node]\n" - "web.port = tcp:0:interface=127.0.0.1\n" - "[client]\n" - "shares.happy = 1\n" - "[storage]\n" - ) - - if self.mode in ("upload-self", "receive"): - # accept and store shares, to trigger the memory consumption bugs - pass - else: - # don't accept any shares - f.write("readonly = true\n") - ## also, if we do receive any shares, throw them away - #f.write("debug_discard = true") - if self.mode == "upload-self": - pass - f.close() - self.keepalive_file = os.path.join(clientdir, - client.Client.EXIT_TRIGGER_FILE) - # now start updating the mtime. - self.touch_keepalive() - ts = internet.TimerService(1.0, self.touch_keepalive) - ts.setServiceParent(self.sparent) - - pp = ClientWatcher() - self.proc_done = pp.d = defer.Deferred() - logfile = os.path.join(self.basedir, "client.log") - tahoes = procutils.which("tahoe") - if not tahoes: - raise RuntimeError("unable to find a 'tahoe' executable") - cmd = [tahoes[0], "run", ".", "-l", logfile] - env = os.environ.copy() - self.proc = reactor.spawnProcess(pp, cmd[0], cmd, env, path=clientdir_str) - log.msg("CLIENT STARTED") - - # now we wait for the client to get started. we're looking for the - # control.furl file to appear. - furl_file = os.path.join(clientdir, "private", "control.furl") - url_file = os.path.join(clientdir, "node.url") - def _check(): - if pp.ended and pp.ended.value.status != 0: - # the twistd process ends normally (with rc=0) if the child - # is successfully launched. It ends abnormally (with rc!=0) - # if the child cannot be launched. - raise ChildDidNotStartError("process ended while waiting for startup") - return os.path.exists(furl_file) - d = self.poll(_check, 0.1) - # once it exists, wait a moment before we read from it, just in case - # it hasn't finished writing the whole thing. Ideally control.furl - # would be created in some atomic fashion, or made non-readable until - # it's ready, but I can't think of an easy way to do that, and I - # think the chances that we'll observe a half-write are pretty low. - def _stall(res): - d2 = defer.Deferred() - reactor.callLater(0.1, d2.callback, None) - return d2 - d.addCallback(_stall) - def _read(res): - # read the node's URL - self.webish_url = open(url_file, "r").read().strip() - if self.webish_url[-1] == "/": - # trim trailing slash, since the rest of the code wants it gone - self.webish_url = self.webish_url[:-1] - f = open(furl_file, "r") - furl = f.read() - return furl.strip() - d.addCallback(_read) - return d - - - def kill_client(self): - # returns a Deferred that fires when the process exits. This may only - # be called once. - try: - self.proc.signalProcess("INT") - except error.ProcessExitedAlready: - pass - return self.proc_done - - - def create_data(self, name, size): - filename = os.path.join(self.testdir, name + ".data") - f = open(filename, "wb") - block = "a" * 8192 - while size > 0: - l = min(size, 8192) - f.write(block[:l]) - size -= l - return filename - - def stash_stats(self, stats, name): - self.statsfile.write("%s %s: %d\n" % (self.mode, name, stats['VmPeak'])) - self.statsfile.flush() - self.stats[name] = stats['VmPeak'] - - def POST(self, urlpath, **fields): - url = self.webish_url + urlpath - sepbase = "boogabooga" - sep = "--" + sepbase - form = [] - form.append(sep) - form.append('Content-Disposition: form-data; name="_charset"') - form.append('') - form.append('UTF-8') - form.append(sep) - for name, value in fields.iteritems(): - if isinstance(value, tuple): - filename, value = value - form.append('Content-Disposition: form-data; name="%s"; ' - 'filename="%s"' % (name, filename)) - else: - form.append('Content-Disposition: form-data; name="%s"' % name) - form.append('') - form.append(value) - form.append(sep) - form[-1] += "--" - body = "\r\n".join(form) + "\r\n" - headers = {"content-type": "multipart/form-data; boundary=%s" % sepbase, - } - return tw_client.getPage(url, method="POST", postdata=body, - headers=headers, followRedirect=False) - - def GET_discard(self, urlpath, stall): - url = self.webish_url + urlpath + "?filename=dummy-get.out" - return discardPage(url, stall) - - def _print_usage(self, res=None): - d = self.control_rref.callRemote("get_memory_usage") - def _print(stats): - print("VmSize: %9d VmPeak: %9d" % (stats["VmSize"], - stats["VmPeak"])) - return stats - d.addCallback(_print) - return d - - def _do_upload(self, res, size, files, uris): - name = '%d' % size - print() - print("uploading %s" % name) - if self.mode in ("upload", "upload-self"): - d = self.control_rref.callRemote("upload_random_data_from_file", - size, - convergence="check-memory") - elif self.mode == "upload-POST": - data = "a" * size - url = "/uri" - d = self.POST(url, t="upload", file=("%d.data" % size, data)) - elif self.mode in ("receive", - "download", "download-GET", "download-GET-slow"): - # mode=receive: upload the data from a local peer, so that the - # client-under-test receives and stores the shares - # - # mode=download*: upload the data from a local peer, then have - # the client-under-test download it. - # - # we need to wait until the uploading node has connected to all - # peers, since the wait_for_client_connections() above doesn't - # pay attention to our self.nodes[] and their connections. - files[name] = self.create_data(name, size) - u = self.nodes[0].getServiceNamed("uploader") - d = self.nodes[0].debug_wait_for_client_connections(self.numnodes+1) - d.addCallback(lambda res: - u.upload(upload.FileName(files[name], - convergence="check-memory"))) - d.addCallback(lambda results: results.get_uri()) - else: - raise ValueError("unknown mode=%s" % self.mode) - def _complete(uri): - uris[name] = uri - print("uploaded %s" % name) - d.addCallback(_complete) - return d - - def _do_download(self, res, size, uris): - if self.mode not in ("download", "download-GET", "download-GET-slow"): - return - name = '%d' % size - print("downloading %s" % name) - uri = uris[name] - - if self.mode == "download": - d = self.control_rref.callRemote("download_to_tempfile_and_delete", - uri) - elif self.mode == "download-GET": - url = "/uri/%s" % uri - d = self.GET_discard(urllib.quote(url), stall=False) - elif self.mode == "download-GET-slow": - url = "/uri/%s" % uri - d = self.GET_discard(urllib.quote(url), stall=True) - - def _complete(res): - print("downloaded %s" % name) - return res - d.addCallback(_complete) - return d - - def do_test(self): - #print("CLIENT STARTED") - #print("FURL", self.control_furl) - #print("RREF", self.control_rref) - #print() - kB = 1000; MB = 1000*1000 - files = {} - uris = {} - - d = self._print_usage() - d.addCallback(self.stash_stats, "0B") - - for i in range(10): - d.addCallback(self._do_upload, 10*kB+i, files, uris) - d.addCallback(self._do_download, 10*kB+i, uris) - d.addCallback(self._print_usage) - d.addCallback(self.stash_stats, "10kB") - - for i in range(3): - d.addCallback(self._do_upload, 10*MB+i, files, uris) - d.addCallback(self._do_download, 10*MB+i, uris) - d.addCallback(self._print_usage) - d.addCallback(self.stash_stats, "10MB") - - for i in range(1): - d.addCallback(self._do_upload, 50*MB+i, files, uris) - d.addCallback(self._do_download, 50*MB+i, uris) - d.addCallback(self._print_usage) - d.addCallback(self.stash_stats, "50MB") - - #for i in range(1): - # d.addCallback(self._do_upload, 100*MB+i, files, uris) - # d.addCallback(self._do_download, 100*MB+i, uris) - # d.addCallback(self._print_usage) - #d.addCallback(self.stash_stats, "100MB") - - #d.addCallback(self.stall) - def _done(res): - print("FINISHING") - d.addCallback(_done) - return d - - def stall(self, res): - d = defer.Deferred() - reactor.callLater(5, d.callback, None) - return d - - -class ClientWatcher(protocol.ProcessProtocol, object): - ended = False - def outReceived(self, data): - print("OUT:", data) - def errReceived(self, data): - print("ERR:", data) - def processEnded(self, reason): - self.ended = reason - self.d.callback(None) - - -if __name__ == '__main__': - mode = "upload" - if len(sys.argv) > 1: - mode = sys.argv[1] - if sys.maxsize == 2147483647: - bits = "32" - elif sys.maxsize == 9223372036854775807: - bits = "64" - else: - bits = "?" - print("%s-bit system (sys.maxsize=%d)" % (bits, sys.maxsize)) - # put the logfile and stats.out in _test_memory/ . These stick around. - # put the nodes and other files in _test_memory/test/ . These are - # removed each time we run. - sf = SystemFramework("_test_memory", mode) - sf.run() diff --git a/tox.ini b/tox.ini index 610570be5..1b1e8e5e3 100644 --- a/tox.ini +++ b/tox.ini @@ -206,17 +206,6 @@ commands = flogtool --version python misc/build_helpers/run-deprecations.py --package allmydata --warnings={env:TAHOE_LAFS_WARNINGS_LOG:_trial_temp/deprecation-warnings.log} trial {env:TAHOE_LAFS_TRIAL_ARGS:--rterrors} {posargs:allmydata} -[testenv:checkmemory] -commands = - rm -rf _test_memory - python src/allmydata/test/check_memory.py upload - python src/allmydata/test/check_memory.py upload-self - python src/allmydata/test/check_memory.py upload-POST - python src/allmydata/test/check_memory.py download - python src/allmydata/test/check_memory.py download-GET - python src/allmydata/test/check_memory.py download-GET-slow - python src/allmydata/test/check_memory.py receive - # Use 'tox -e docs' to check formatting and cross-references in docs .rst # files. The published docs are built by code run over at readthedocs.org, # which does not use this target (but does something similar). From 1b8e013991ffb71620fcd6589945624e8cac0e97 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 15 Oct 2021 11:46:34 -0400 Subject: [PATCH 0260/2309] Get rid of `check_speed` The motivation and justification here are roughly the same as for `check_memory`. --- Makefile | 28 +---- misc/checkers/check_speed.py | 234 ----------------------------------- 2 files changed, 1 insertion(+), 261 deletions(-) delete mode 100644 misc/checkers/check_speed.py diff --git a/Makefile b/Makefile index f7a357588..5d8bf18ba 100644 --- a/Makefile +++ b/Makefile @@ -142,31 +142,6 @@ count-lines: # src/allmydata/test/bench_dirnode.py -# The check-speed and check-grid targets are disabled, since they depend upon -# the pre-located $(TAHOE) executable that was removed when we switched to -# tox. They will eventually be resurrected as dedicated tox environments. - -# The check-speed target uses a pre-established client node to run a canned -# set of performance tests against a test network that is also -# pre-established (probably on a remote machine). Provide it with the path to -# a local directory where this client node has been created (and populated -# with the necessary FURLs of the test network). This target will start that -# client with the current code and then run the tests. Afterwards it will -# stop the client. -# -# The 'sleep 5' is in there to give the new client a chance to connect to its -# storageservers, since check_speed.py has no good way of doing that itself. - -##.PHONY: check-speed -##check-speed: .built -## if [ -z '$(TESTCLIENTDIR)' ]; then exit 1; fi -## @echo "stopping any leftover client code" -## -$(TAHOE) stop $(TESTCLIENTDIR) -## $(TAHOE) start $(TESTCLIENTDIR) -## sleep 5 -## $(TAHOE) @src/allmydata/test/check_speed.py $(TESTCLIENTDIR) -## $(TAHOE) stop $(TESTCLIENTDIR) - # The check-grid target also uses a pre-established client node, along with a # long-term directory that contains some well-known files. See the docstring # in src/allmydata/test/check_grid.py to see how to set this up. @@ -195,12 +170,11 @@ test-clean: # Use 'make distclean' instead to delete all generated files. .PHONY: clean clean: - rm -rf build _trial_temp _test_memory .built + rm -rf build _trial_temp .built rm -f `find src *.egg -name '*.so' -or -name '*.pyc'` rm -rf support dist rm -rf `ls -d *.egg | grep -vEe"setuptools-|setuptools_darcs-|darcsver-"` rm -rf *.pyc - rm -f bin/tahoe bin/tahoe.pyscript rm -f *.pkg .PHONY: distclean diff --git a/misc/checkers/check_speed.py b/misc/checkers/check_speed.py deleted file mode 100644 index 2fce53387..000000000 --- a/misc/checkers/check_speed.py +++ /dev/null @@ -1,234 +0,0 @@ -from __future__ import print_function - -import os, sys -from twisted.internet import reactor, defer -from twisted.python import log -from twisted.application import service -from foolscap.api import Tub, fireEventually - -MB = 1000000 - -class SpeedTest(object): - DO_IMMUTABLE = True - DO_MUTABLE_CREATE = True - DO_MUTABLE = True - - def __init__(self, test_client_dir): - #self.real_stderr = sys.stderr - log.startLogging(open("st.log", "a"), setStdout=False) - f = open(os.path.join(test_client_dir, "private", "control.furl"), "r") - self.control_furl = f.read().strip() - f.close() - self.base_service = service.MultiService() - self.failed = None - self.upload_times = {} - self.download_times = {} - - def run(self): - print("STARTING") - d = fireEventually() - d.addCallback(lambda res: self.setUp()) - d.addCallback(lambda res: self.do_test()) - d.addBoth(self.tearDown) - def _err(err): - self.failed = err - log.err(err) - print(err) - d.addErrback(_err) - def _done(res): - reactor.stop() - return res - d.addBoth(_done) - reactor.run() - if self.failed: - print("EXCEPTION") - print(self.failed) - sys.exit(1) - - def setUp(self): - self.base_service.startService() - self.tub = Tub() - self.tub.setOption("expose-remote-exception-types", False) - self.tub.setServiceParent(self.base_service) - d = self.tub.getReference(self.control_furl) - def _gotref(rref): - self.client_rref = rref - print("Got Client Control reference") - return self.stall(5) - d.addCallback(_gotref) - return d - - def stall(self, delay, result=None): - d = defer.Deferred() - reactor.callLater(delay, d.callback, result) - return d - - def record_times(self, times, key): - print("TIME (%s): %s up, %s down" % (key, times[0], times[1])) - self.upload_times[key], self.download_times[key] = times - - def one_test(self, res, name, count, size, mutable): - # values for 'mutable': - # False (upload a different CHK file for each 'count') - # "create" (upload different contents into a new SSK file) - # "upload" (upload different contents into the same SSK file. The - # time consumed does not include the creation of the file) - d = self.client_rref.callRemote("speed_test", count, size, mutable) - d.addCallback(self.record_times, name) - return d - - def measure_rtt(self, res): - # use RIClient.get_nodeid() to measure the foolscap-level RTT - d = self.client_rref.callRemote("measure_peer_response_time") - def _got(res): - assert len(res) # need at least one peer - times = res.values() - self.total_rtt = sum(times) - self.average_rtt = sum(times) / len(times) - self.max_rtt = max(times) - print("num-peers: %d" % len(times)) - print("total-RTT: %f" % self.total_rtt) - print("average-RTT: %f" % self.average_rtt) - print("max-RTT: %f" % self.max_rtt) - d.addCallback(_got) - return d - - def do_test(self): - print("doing test") - d = defer.succeed(None) - d.addCallback(self.one_test, "startup", 1, 1000, False) #ignore this one - d.addCallback(self.measure_rtt) - - if self.DO_IMMUTABLE: - # immutable files - d.addCallback(self.one_test, "1x 200B", 1, 200, False) - d.addCallback(self.one_test, "10x 200B", 10, 200, False) - def _maybe_do_100x_200B(res): - if self.upload_times["10x 200B"] < 5: - print("10x 200B test went too fast, doing 100x 200B test") - return self.one_test(None, "100x 200B", 100, 200, False) - return - d.addCallback(_maybe_do_100x_200B) - d.addCallback(self.one_test, "1MB", 1, 1*MB, False) - d.addCallback(self.one_test, "10MB", 1, 10*MB, False) - def _maybe_do_100MB(res): - if self.upload_times["10MB"] > 30: - print("10MB test took too long, skipping 100MB test") - return - return self.one_test(None, "100MB", 1, 100*MB, False) - d.addCallback(_maybe_do_100MB) - - if self.DO_MUTABLE_CREATE: - # mutable file creation - d.addCallback(self.one_test, "10x 200B SSK creation", 10, 200, - "create") - - if self.DO_MUTABLE: - # mutable file upload/download - d.addCallback(self.one_test, "10x 200B SSK", 10, 200, "upload") - def _maybe_do_100x_200B_SSK(res): - if self.upload_times["10x 200B SSK"] < 5: - print("10x 200B SSK test went too fast, doing 100x 200B SSK") - return self.one_test(None, "100x 200B SSK", 100, 200, - "upload") - return - d.addCallback(_maybe_do_100x_200B_SSK) - d.addCallback(self.one_test, "1MB SSK", 1, 1*MB, "upload") - - d.addCallback(self.calculate_speeds) - return d - - def calculate_speeds(self, res): - # time = A*size+B - # we assume that A*200bytes is negligible - - if self.DO_IMMUTABLE: - # upload - if "100x 200B" in self.upload_times: - B = self.upload_times["100x 200B"] / 100 - else: - B = self.upload_times["10x 200B"] / 10 - print("upload per-file time: %.3fs" % B) - print("upload per-file times-avg-RTT: %f" % (B / self.average_rtt)) - print("upload per-file times-total-RTT: %f" % (B / self.total_rtt)) - A1 = 1*MB / (self.upload_times["1MB"] - B) # in bytes per second - print("upload speed (1MB):", self.number(A1, "Bps")) - A2 = 10*MB / (self.upload_times["10MB"] - B) - print("upload speed (10MB):", self.number(A2, "Bps")) - if "100MB" in self.upload_times: - A3 = 100*MB / (self.upload_times["100MB"] - B) - print("upload speed (100MB):", self.number(A3, "Bps")) - - # download - if "100x 200B" in self.download_times: - B = self.download_times["100x 200B"] / 100 - else: - B = self.download_times["10x 200B"] / 10 - print("download per-file time: %.3fs" % B) - print("download per-file times-avg-RTT: %f" % (B / self.average_rtt)) - print("download per-file times-total-RTT: %f" % (B / self.total_rtt)) - A1 = 1*MB / (self.download_times["1MB"] - B) # in bytes per second - print("download speed (1MB):", self.number(A1, "Bps")) - A2 = 10*MB / (self.download_times["10MB"] - B) - print("download speed (10MB):", self.number(A2, "Bps")) - if "100MB" in self.download_times: - A3 = 100*MB / (self.download_times["100MB"] - B) - print("download speed (100MB):", self.number(A3, "Bps")) - - if self.DO_MUTABLE_CREATE: - # SSK creation - B = self.upload_times["10x 200B SSK creation"] / 10 - print("create per-file time SSK: %.3fs" % B) - - if self.DO_MUTABLE: - # upload SSK - if "100x 200B SSK" in self.upload_times: - B = self.upload_times["100x 200B SSK"] / 100 - else: - B = self.upload_times["10x 200B SSK"] / 10 - print("upload per-file time SSK: %.3fs" % B) - A1 = 1*MB / (self.upload_times["1MB SSK"] - B) # in bytes per second - print("upload speed SSK (1MB):", self.number(A1, "Bps")) - - # download SSK - if "100x 200B SSK" in self.download_times: - B = self.download_times["100x 200B SSK"] / 100 - else: - B = self.download_times["10x 200B SSK"] / 10 - print("download per-file time SSK: %.3fs" % B) - A1 = 1*MB / (self.download_times["1MB SSK"] - B) # in bytes per - # second - print("download speed SSK (1MB):", self.number(A1, "Bps")) - - def number(self, value, suffix=""): - scaling = 1 - if value < 1: - fmt = "%1.2g%s" - elif value < 100: - fmt = "%.1f%s" - elif value < 1000: - fmt = "%d%s" - elif value < 1e6: - fmt = "%.2fk%s"; scaling = 1e3 - elif value < 1e9: - fmt = "%.2fM%s"; scaling = 1e6 - elif value < 1e12: - fmt = "%.2fG%s"; scaling = 1e9 - elif value < 1e15: - fmt = "%.2fT%s"; scaling = 1e12 - elif value < 1e18: - fmt = "%.2fP%s"; scaling = 1e15 - else: - fmt = "huge! %g%s" - return fmt % (value / scaling, suffix) - - def tearDown(self, res): - d = self.base_service.stopService() - d.addCallback(lambda ignored: res) - return d - - -if __name__ == '__main__': - test_client_dir = sys.argv[1] - st = SpeedTest(test_client_dir) - st.run() From 1aae92b18e0180b31f1bb91eca2cfda2d8fb5058 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 15 Oct 2021 11:47:05 -0400 Subject: [PATCH 0261/2309] Get rid of `getmem.py` helper Platforms provide an interface for retrieving this information. Just use those interfaces instead. --- misc/operations_helpers/getmem.py | 20 -------------------- 1 file changed, 20 deletions(-) delete mode 100644 misc/operations_helpers/getmem.py diff --git a/misc/operations_helpers/getmem.py b/misc/operations_helpers/getmem.py deleted file mode 100644 index b3c6285fe..000000000 --- a/misc/operations_helpers/getmem.py +++ /dev/null @@ -1,20 +0,0 @@ -#! /usr/bin/env python - -from __future__ import print_function - -from foolscap import Tub -from foolscap.eventual import eventually -import sys -from twisted.internet import reactor - -def go(): - t = Tub() - d = t.getReference(sys.argv[1]) - d.addCallback(lambda rref: rref.callRemote("get_memory_usage")) - def _got(res): - print(res) - reactor.stop() - d.addCallback(_got) - -eventually(go) -reactor.run() From 95b765e3092b8f076fe8e72053e1986a4e642086 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 15 Oct 2021 11:54:18 -0400 Subject: [PATCH 0262/2309] stop creating a control tub for the introducer --- src/allmydata/introducer/server.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/allmydata/introducer/server.py b/src/allmydata/introducer/server.py index 1e28f511b..950602f98 100644 --- a/src/allmydata/introducer/server.py +++ b/src/allmydata/introducer/server.py @@ -39,7 +39,6 @@ from allmydata.introducer.common import unsign_from_foolscap, \ from allmydata.node import read_config from allmydata.node import create_node_dir from allmydata.node import create_connection_handlers -from allmydata.node import create_control_tub from allmydata.node import create_tub_options from allmydata.node import create_main_tub @@ -88,7 +87,7 @@ def create_introducer(basedir=u"."): config, tub_options, default_connection_handlers, foolscap_connection_handlers, i2p_provider, tor_provider, ) - control_tub = create_control_tub() + control_tub = None node = _IntroducerNode( config, From e0312eae57fd35e7f68a62dd4ceb4957082bf6fa Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 15 Oct 2021 12:02:24 -0400 Subject: [PATCH 0263/2309] stop creating a control tub for client nodes --- src/allmydata/client.py | 10 +--------- src/allmydata/test/test_system.py | 20 -------------------- 2 files changed, 1 insertion(+), 29 deletions(-) diff --git a/src/allmydata/client.py b/src/allmydata/client.py index aabae9065..8a953937a 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -40,7 +40,6 @@ from allmydata.storage.server import StorageServer from allmydata import storage_client from allmydata.immutable.upload import Uploader from allmydata.immutable.offloaded import Helper -from allmydata.control import ControlServer from allmydata.introducer.client import IntroducerClient from allmydata.util import ( hashutil, base32, pollmixin, log, idlib, @@ -283,7 +282,7 @@ def create_client_from_config(config, _client_factory=None, _introducer_factory= config, tub_options, default_connection_handlers, foolscap_connection_handlers, i2p_provider, tor_provider, ) - control_tub = node.create_control_tub() + control_tub = None introducer_clients = create_introducer_clients(config, main_tub, _introducer_factory) storage_broker = create_storage_farm_broker( @@ -648,7 +647,6 @@ class _Client(node.Node, pollmixin.PollMixin): self.init_stats_provider() self.init_secrets() self.init_node_key() - self.init_control() self._key_generator = KeyGenerator() key_gen_furl = config.get_config("client", "key_generator.furl", None) if key_gen_furl: @@ -985,12 +983,6 @@ class _Client(node.Node, pollmixin.PollMixin): def get_history(self): return self.history - def init_control(self): - c = ControlServer() - c.setServiceParent(self) - control_url = self.control_tub.registerReference(c) - self.config.write_private_config("control.furl", control_url + "\n") - def init_helper(self): self.helper = Helper(self.config.get_config_path("helper"), self.storage_broker, self._secret_holder, diff --git a/src/allmydata/test/test_system.py b/src/allmydata/test/test_system.py index 3e1bdcdd4..c01dd0afc 100644 --- a/src/allmydata/test/test_system.py +++ b/src/allmydata/test/test_system.py @@ -780,7 +780,6 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase): d.addCallback(self._check_publish_private) d.addCallback(self.log, "did _check_publish_private") d.addCallback(self._test_web) - d.addCallback(self._test_control) d.addCallback(self._test_cli) # P now has four top-level children: # P/personal/sekrit data @@ -1343,25 +1342,6 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase): if line.startswith("CHK %s " % storage_index_s)] self.failUnlessEqual(len(matching), 10) - def _test_control(self, res): - # exercise the remote-control-the-client foolscap interfaces in - # allmydata.control (mostly used for performance tests) - c0 = self.clients[0] - control_furl_file = c0.config.get_private_path("control.furl") - control_furl = ensure_str(open(control_furl_file, "r").read().strip()) - # it doesn't really matter which Tub we use to connect to the client, - # so let's just use our IntroducerNode's - d = self.introducer.tub.getReference(control_furl) - d.addCallback(self._test_control2, control_furl_file) - return d - def _test_control2(self, rref, filename): - d = defer.succeed(None) - d.addCallback(lambda res: rref.callRemote("speed_test", 1, 200, False)) - if sys.platform in ("linux2", "linux3"): - d.addCallback(lambda res: rref.callRemote("get_memory_usage")) - d.addCallback(lambda res: rref.callRemote("measure_peer_response_time")) - return d - def _test_cli(self, res): # run various CLI commands (in a thread, since they use blocking # network calls) From ddf5f461bf69224c47b0b8d41c84d2abdc63c5c4 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 15 Oct 2021 12:11:53 -0400 Subject: [PATCH 0264/2309] Stop half-pretending to have a control port --- src/allmydata/test/no_network.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/allmydata/test/no_network.py b/src/allmydata/test/no_network.py index 2f75f9274..3b88a1cc6 100644 --- a/src/allmydata/test/no_network.py +++ b/src/allmydata/test/no_network.py @@ -17,8 +17,7 @@ from __future__ import unicode_literals # This should be useful for tests which want to examine and/or manipulate the # uploaded shares, checker/verifier/repairer tests, etc. The clients have no -# Tubs, so it is not useful for tests that involve a Helper or the -# control.furl . +# Tubs, so it is not useful for tests that involve a Helper. from future.utils import PY2 if PY2: @@ -274,8 +273,6 @@ class _NoNetworkClient(_Client): # type: ignore # tahoe-lafs/ticket/3573 pass def init_introducer_client(self): pass - def create_control_tub(self): - pass def create_log_tub(self): pass def setup_logging(self): @@ -284,8 +281,6 @@ class _NoNetworkClient(_Client): # type: ignore # tahoe-lafs/ticket/3573 service.MultiService.startService(self) def stopService(self): return service.MultiService.stopService(self) - def init_control(self): - pass def init_helper(self): pass def init_key_gen(self): From 1de480dc37b24c75441724a99bc0b265347afb16 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 15 Oct 2021 12:12:03 -0400 Subject: [PATCH 0265/2309] Stop offering an API to create a control tub or handling the control tub --- src/allmydata/node.py | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/src/allmydata/node.py b/src/allmydata/node.py index 5a6f8c66f..08271fc5f 100644 --- a/src/allmydata/node.py +++ b/src/allmydata/node.py @@ -919,18 +919,6 @@ def create_main_tub(config, tub_options, return tub -def create_control_tub(): - """ - Creates a Foolscap Tub for use by the control port. This is a - localhost-only ephemeral Tub, with no control over the listening - port or location - """ - control_tub = Tub() - portnum = iputil.listenOnUnused(control_tub) - log.msg("Control Tub location set to 127.0.0.1:%s" % (portnum,)) - return control_tub - - class Node(service.MultiService): """ This class implements common functionality of both Client nodes and Introducer nodes. @@ -967,10 +955,6 @@ class Node(service.MultiService): else: self.nodeid = self.short_nodeid = None - self.control_tub = control_tub - if self.control_tub is not None: - self.control_tub.setServiceParent(self) - self.log("Node constructed. " + __full_version__) iputil.increase_rlimits() From fe2e2cc1d697f562284cfffcfe3b250dea4ed36c Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 15 Oct 2021 12:12:19 -0400 Subject: [PATCH 0266/2309] Get rid of the control service --- src/allmydata/control.py | 273 --------------------------------------- 1 file changed, 273 deletions(-) delete mode 100644 src/allmydata/control.py diff --git a/src/allmydata/control.py b/src/allmydata/control.py deleted file mode 100644 index 7efa174ab..000000000 --- a/src/allmydata/control.py +++ /dev/null @@ -1,273 +0,0 @@ -"""Ported to Python 3. -""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals - -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - -import os, time, tempfile -from zope.interface import implementer -from twisted.application import service -from twisted.internet import defer -from twisted.internet.interfaces import IConsumer -from foolscap.api import Referenceable -from allmydata.interfaces import RIControlClient, IFileNode -from allmydata.util import fileutil, mathutil -from allmydata.immutable import upload -from allmydata.mutable.publish import MutableData -from twisted.python import log - -def get_memory_usage(): - # this is obviously linux-specific - stat_names = (b"VmPeak", - b"VmSize", - #b"VmHWM", - b"VmData") - stats = {} - try: - with open("/proc/self/status", "rb") as f: - for line in f: - name, right = line.split(b":",2) - if name in stat_names: - assert right.endswith(b" kB\n") - right = right[:-4] - stats[name] = int(right) * 1024 - except: - # Probably not on (a compatible version of) Linux - stats['VmSize'] = 0 - stats['VmPeak'] = 0 - return stats - -def log_memory_usage(where=""): - stats = get_memory_usage() - log.msg("VmSize: %9d VmPeak: %9d %s" % (stats[b"VmSize"], - stats[b"VmPeak"], - where)) - -@implementer(IConsumer) -class FileWritingConsumer(object): - def __init__(self, filename): - self.done = False - self.f = open(filename, "wb") - def registerProducer(self, p, streaming): - if streaming: - p.resumeProducing() - else: - while not self.done: - p.resumeProducing() - def write(self, data): - self.f.write(data) - def unregisterProducer(self): - self.done = True - self.f.close() - -@implementer(RIControlClient) -class ControlServer(Referenceable, service.Service): - - def remote_wait_for_client_connections(self, num_clients): - return self.parent.debug_wait_for_client_connections(num_clients) - - def remote_upload_random_data_from_file(self, size, convergence): - tempdir = tempfile.mkdtemp() - filename = os.path.join(tempdir, "data") - f = open(filename, "wb") - block = b"a" * 8192 - while size > 0: - l = min(size, 8192) - f.write(block[:l]) - size -= l - f.close() - uploader = self.parent.getServiceNamed("uploader") - u = upload.FileName(filename, convergence=convergence) - # XXX should pass reactor arg - d = uploader.upload(u) - d.addCallback(lambda results: results.get_uri()) - def _done(uri): - os.remove(filename) - os.rmdir(tempdir) - return uri - d.addCallback(_done) - return d - - def remote_download_to_tempfile_and_delete(self, uri): - tempdir = tempfile.mkdtemp() - filename = os.path.join(tempdir, "data") - filenode = self.parent.create_node_from_uri(uri, name=filename) - if not IFileNode.providedBy(filenode): - raise AssertionError("The URI does not reference a file.") - c = FileWritingConsumer(filename) - d = filenode.read(c) - def _done(res): - os.remove(filename) - os.rmdir(tempdir) - return None - d.addCallback(_done) - return d - - def remote_speed_test(self, count, size, mutable): - assert size > 8 - log.msg("speed_test: count=%d, size=%d, mutable=%s" % (count, size, - mutable)) - st = SpeedTest(self.parent, count, size, mutable) - return st.run() - - def remote_get_memory_usage(self): - return get_memory_usage() - - def remote_measure_peer_response_time(self): - # I'd like to average together several pings, but I don't want this - # phase to take more than 10 seconds. Expect worst-case latency to be - # 300ms. - results = {} - sb = self.parent.get_storage_broker() - everyone = sb.get_connected_servers() - num_pings = int(mathutil.div_ceil(10, (len(everyone) * 0.3))) - everyone = list(everyone) * num_pings - d = self._do_one_ping(None, everyone, results) - return d - def _do_one_ping(self, res, everyone_left, results): - if not everyone_left: - return results - server = everyone_left.pop(0) - server_name = server.get_longname() - storage_server = server.get_storage_server() - start = time.time() - d = storage_server.get_buckets(b"\x00" * 16) - def _done(ignored): - stop = time.time() - elapsed = stop - start - if server_name in results: - results[server_name].append(elapsed) - else: - results[server_name] = [elapsed] - d.addCallback(_done) - d.addCallback(self._do_one_ping, everyone_left, results) - def _average(res): - averaged = {} - for server_name,times in results.items(): - averaged[server_name] = sum(times) / len(times) - return averaged - d.addCallback(_average) - return d - -class SpeedTest(object): - def __init__(self, parent, count, size, mutable): - self.parent = parent - self.count = count - self.size = size - self.mutable_mode = mutable - self.uris = {} - self.basedir = self.parent.config.get_config_path("_speed_test_data") - - def run(self): - self.create_data() - d = self.do_upload() - d.addCallback(lambda res: self.do_download()) - d.addBoth(self.do_cleanup) - d.addCallback(lambda res: (self.upload_time, self.download_time)) - return d - - def create_data(self): - fileutil.make_dirs(self.basedir) - for i in range(self.count): - s = self.size - fn = os.path.join(self.basedir, str(i)) - if os.path.exists(fn): - os.unlink(fn) - f = open(fn, "wb") - f.write(os.urandom(8)) - s -= 8 - while s > 0: - chunk = min(s, 4096) - f.write(b"\x00" * chunk) - s -= chunk - f.close() - - def do_upload(self): - d = defer.succeed(None) - def _create_slot(res): - d1 = self.parent.create_mutable_file(b"") - def _created(n): - self._n = n - d1.addCallback(_created) - return d1 - if self.mutable_mode == "upload": - d.addCallback(_create_slot) - def _start(res): - self._start = time.time() - d.addCallback(_start) - - def _record_uri(uri, i): - self.uris[i] = uri - def _upload_one_file(ignored, i): - if i >= self.count: - return - fn = os.path.join(self.basedir, str(i)) - if self.mutable_mode == "create": - data = open(fn,"rb").read() - d1 = self.parent.create_mutable_file(data) - d1.addCallback(lambda n: n.get_uri()) - elif self.mutable_mode == "upload": - data = open(fn,"rb").read() - d1 = self._n.overwrite(MutableData(data)) - d1.addCallback(lambda res: self._n.get_uri()) - else: - up = upload.FileName(fn, convergence=None) - d1 = self.parent.upload(up) - d1.addCallback(lambda results: results.get_uri()) - d1.addCallback(_record_uri, i) - d1.addCallback(_upload_one_file, i+1) - return d1 - d.addCallback(_upload_one_file, 0) - def _upload_done(ignored): - stop = time.time() - self.upload_time = stop - self._start - d.addCallback(_upload_done) - return d - - def do_download(self): - start = time.time() - d = defer.succeed(None) - def _download_one_file(ignored, i): - if i >= self.count: - return - n = self.parent.create_node_from_uri(self.uris[i]) - if not IFileNode.providedBy(n): - raise AssertionError("The URI does not reference a file.") - if n.is_mutable(): - d1 = n.download_best_version() - else: - d1 = n.read(DiscardingConsumer()) - d1.addCallback(_download_one_file, i+1) - return d1 - d.addCallback(_download_one_file, 0) - def _download_done(ignored): - stop = time.time() - self.download_time = stop - start - d.addCallback(_download_done) - return d - - def do_cleanup(self, res): - for i in range(self.count): - fn = os.path.join(self.basedir, str(i)) - os.unlink(fn) - return res - -@implementer(IConsumer) -class DiscardingConsumer(object): - def __init__(self): - self.done = False - def registerProducer(self, p, streaming): - if streaming: - p.resumeProducing() - else: - while not self.done: - p.resumeProducing() - def write(self, data): - pass - def unregisterProducer(self): - self.done = True From 0611af6b0b7de30ef9720ab08b796f615107f5bf Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 15 Oct 2021 13:10:18 -0400 Subject: [PATCH 0267/2309] Stop passing even a dummy value for control tub into Nodes --- src/allmydata/client.py | 6 ++---- src/allmydata/introducer/server.py | 6 ++---- src/allmydata/node.py | 2 +- src/allmydata/test/no_network.py | 1 - src/allmydata/test/web/test_introducer.py | 2 +- 5 files changed, 6 insertions(+), 11 deletions(-) diff --git a/src/allmydata/client.py b/src/allmydata/client.py index 8a953937a..a2f88ebd6 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -282,7 +282,6 @@ def create_client_from_config(config, _client_factory=None, _introducer_factory= config, tub_options, default_connection_handlers, foolscap_connection_handlers, i2p_provider, tor_provider, ) - control_tub = None introducer_clients = create_introducer_clients(config, main_tub, _introducer_factory) storage_broker = create_storage_farm_broker( @@ -293,7 +292,6 @@ def create_client_from_config(config, _client_factory=None, _introducer_factory= client = _client_factory( config, main_tub, - control_tub, i2p_provider, tor_provider, introducer_clients, @@ -630,12 +628,12 @@ class _Client(node.Node, pollmixin.PollMixin): "max_segment_size": DEFAULT_MAX_SEGMENT_SIZE, } - def __init__(self, config, main_tub, control_tub, i2p_provider, tor_provider, introducer_clients, + def __init__(self, config, main_tub, i2p_provider, tor_provider, introducer_clients, storage_farm_broker): """ Use :func:`allmydata.client.create_client` to instantiate one of these. """ - node.Node.__init__(self, config, main_tub, control_tub, i2p_provider, tor_provider) + node.Node.__init__(self, config, main_tub, i2p_provider, tor_provider) self.started_timestamp = time.time() self.logSource = "Client" diff --git a/src/allmydata/introducer/server.py b/src/allmydata/introducer/server.py index 950602f98..8678ad5bf 100644 --- a/src/allmydata/introducer/server.py +++ b/src/allmydata/introducer/server.py @@ -87,12 +87,10 @@ def create_introducer(basedir=u"."): config, tub_options, default_connection_handlers, foolscap_connection_handlers, i2p_provider, tor_provider, ) - control_tub = None node = _IntroducerNode( config, main_tub, - control_tub, i2p_provider, tor_provider, ) @@ -104,8 +102,8 @@ def create_introducer(basedir=u"."): class _IntroducerNode(node.Node): NODETYPE = "introducer" - def __init__(self, config, main_tub, control_tub, i2p_provider, tor_provider): - node.Node.__init__(self, config, main_tub, control_tub, i2p_provider, tor_provider) + def __init__(self, config, main_tub, i2p_provider, tor_provider): + node.Node.__init__(self, config, main_tub, i2p_provider, tor_provider) self.init_introducer() webport = self.get_config("node", "web.port", None) if webport: diff --git a/src/allmydata/node.py b/src/allmydata/node.py index 08271fc5f..3ac4c507b 100644 --- a/src/allmydata/node.py +++ b/src/allmydata/node.py @@ -926,7 +926,7 @@ class Node(service.MultiService): NODETYPE = "unknown NODETYPE" CERTFILE = "node.pem" - def __init__(self, config, main_tub, control_tub, i2p_provider, tor_provider): + def __init__(self, config, main_tub, i2p_provider, tor_provider): """ Initialize the node with the given configuration. Its base directory is the current directory by default. diff --git a/src/allmydata/test/no_network.py b/src/allmydata/test/no_network.py index 3b88a1cc6..7a84580bf 100644 --- a/src/allmydata/test/no_network.py +++ b/src/allmydata/test/no_network.py @@ -250,7 +250,6 @@ def create_no_network_client(basedir): client = _NoNetworkClient( config, main_tub=None, - control_tub=None, i2p_provider=None, tor_provider=None, introducer_clients=[], diff --git a/src/allmydata/test/web/test_introducer.py b/src/allmydata/test/web/test_introducer.py index ba0a5beb9..4b5850cbc 100644 --- a/src/allmydata/test/web/test_introducer.py +++ b/src/allmydata/test/web/test_introducer.py @@ -211,7 +211,7 @@ class IntroducerRootTests(SyncTestCase): main_tub = Tub() main_tub.listenOn(b"tcp:0") main_tub.setLocation(b"tcp:127.0.0.1:1") - introducer_node = _IntroducerNode(config, main_tub, None, None, None) + introducer_node = _IntroducerNode(config, main_tub, None, None) introducer_service = introducer_node.getServiceNamed("introducer") for n in range(2): From 9e59e6922383639014f48d8c496bd91da8806a0f Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 15 Oct 2021 13:13:08 -0400 Subject: [PATCH 0268/2309] news fragment --- newsfragments/3814.removed | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/3814.removed diff --git a/newsfragments/3814.removed b/newsfragments/3814.removed new file mode 100644 index 000000000..939d20ffc --- /dev/null +++ b/newsfragments/3814.removed @@ -0,0 +1 @@ +The little-used "control port" has been removed from all node types. From ad216e0f237fc7e32f296eaf0a38fac69e4ba70f Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 15 Oct 2021 13:13:37 -0400 Subject: [PATCH 0269/2309] remove unused import --- src/allmydata/test/test_system.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/test_system.py b/src/allmydata/test/test_system.py index c01dd0afc..087a1c634 100644 --- a/src/allmydata/test/test_system.py +++ b/src/allmydata/test/test_system.py @@ -12,7 +12,7 @@ if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, dict, list, object, range, max, min, str # noqa: F401 from past.builtins import chr as byteschr, long -from six import ensure_text, ensure_str +from six import ensure_text import os, re, sys, time, json From 1c347c593130f606cfdc7d0e2f52a0ec1db5b20a Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 15 Oct 2021 15:05:21 -0400 Subject: [PATCH 0270/2309] replace sensitive introducer fURL with path where it can be found --- src/allmydata/introducer/server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/introducer/server.py b/src/allmydata/introducer/server.py index 1e28f511b..aa0ae8336 100644 --- a/src/allmydata/introducer/server.py +++ b/src/allmydata/introducer/server.py @@ -136,7 +136,7 @@ class _IntroducerNode(node.Node): os.rename(old_public_fn, private_fn) furl = self.tub.registerReference(introducerservice, furlFile=private_fn) - self.log(" introducer is at %s" % furl, umid="qF2L9A") + self.log(" introducer can be found in {!r}".format(private_fn), umid="qF2L9A") self.introducer_url = furl # for tests def init_web(self, webport): From f2ef72e935126f55be103ac856836ebf4b2c140e Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 18 Oct 2021 08:14:42 -0400 Subject: [PATCH 0271/2309] newsfragment in temporary location --- newsfragments/LFS-01-001.security | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/LFS-01-001.security diff --git a/newsfragments/LFS-01-001.security b/newsfragments/LFS-01-001.security new file mode 100644 index 000000000..975fd0035 --- /dev/null +++ b/newsfragments/LFS-01-001.security @@ -0,0 +1 @@ +The introducer server no longer writes the sensitive introducer fURL value to its log at startup time. Instead it writes the well-known path of the file from which this value can be read. From 58112ba75b8a53e6f361600cc9a0b602e5aebc7c Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 18 Oct 2021 12:50:29 -0400 Subject: [PATCH 0272/2309] Plan of implementation for lease tests. --- newsfragments/3800.minor | 0 src/allmydata/test/test_istorageserver.py | 10 ++++++++++ 2 files changed, 10 insertions(+) create mode 100644 newsfragments/3800.minor diff --git a/newsfragments/3800.minor b/newsfragments/3800.minor new file mode 100644 index 000000000..e69de29bb diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index bd056ae13..1c5496ea4 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -444,6 +444,10 @@ class IStorageServerImmutableAPIsTestsMixin(object): b"immutable", storage_index, 0, b"ono" ) + # TODO allocate_buckets creates lease + # TODO add_lease renews lease if existing storage index and secret + # TODO add_lease creates new lease if new secret + class IStorageServerMutableAPIsTestsMixin(object): """ @@ -820,6 +824,12 @@ class IStorageServerMutableAPIsTestsMixin(object): b"mutable", storage_index, 0, b"ono" ) + # TODO STARAW creates lease for new data + # TODO STARAW renews lease if same secret is used on existing data + # TODO STARAW creates new lease for existing data if new secret is given + # TODO add_lease renews lease if existing storage index and secret + # TODO add_lease creates new lease if new secret + class _FoolscapMixin(SystemTestMixin): """Run tests on Foolscap version of ``IStorageServer.""" From 2b40610a27c93ae7cf639063dafe518580193906 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 18 Oct 2021 12:55:30 -0400 Subject: [PATCH 0273/2309] "Server" is extremely ambiguous, so let's just call this a client, which it is. --- src/allmydata/test/test_istorageserver.py | 66 +++++++++++------------ 1 file changed, 33 insertions(+), 33 deletions(-) diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index 1c5496ea4..b34964463 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -56,7 +56,7 @@ class IStorageServerSharedAPIsTestsMixin(object): """ Tests for ``IStorageServer``'s shared APIs. - ``self.storage_server`` is expected to provide ``IStorageServer``. + ``self.storage_client`` is expected to provide ``IStorageServer``. """ @inlineCallbacks @@ -65,7 +65,7 @@ class IStorageServerSharedAPIsTestsMixin(object): ``IStorageServer`` returns a dictionary where the key is an expected protocol version. """ - result = yield self.storage_server.get_version() + result = yield self.storage_client.get_version() self.assertIsInstance(result, dict) self.assertIn(b"http://allmydata.org/tahoe/protocols/storage/v1", result) @@ -74,10 +74,10 @@ class IStorageServerImmutableAPIsTestsMixin(object): """ Tests for ``IStorageServer``'s immutable APIs. - ``self.storage_server`` is expected to provide ``IStorageServer``. + ``self.storage_client`` is expected to provide ``IStorageServer``. ``self.disconnect()`` should disconnect and then reconnect, creating a new - ``self.storage_server``. Some implementations may wish to skip tests using + ``self.storage_client``. Some implementations may wish to skip tests using this; HTTP has no notion of disconnection. """ @@ -87,7 +87,7 @@ class IStorageServerImmutableAPIsTestsMixin(object): allocate_buckets() with a new storage index returns the matching shares. """ - (already_got, allocated) = yield self.storage_server.allocate_buckets( + (already_got, allocated) = yield self.storage_client.allocate_buckets( new_storage_index(), renew_secret=new_secret(), cancel_secret=new_secret(), @@ -110,7 +110,7 @@ class IStorageServerImmutableAPIsTestsMixin(object): new_secret(), new_secret(), ) - (already_got, allocated) = yield self.storage_server.allocate_buckets( + (already_got, allocated) = yield self.storage_client.allocate_buckets( storage_index, renew_secret, cancel_secret, @@ -118,7 +118,7 @@ class IStorageServerImmutableAPIsTestsMixin(object): allocated_size=1024, canary=Referenceable(), ) - (already_got2, allocated2) = yield self.storage_server.allocate_buckets( + (already_got2, allocated2) = yield self.storage_client.allocate_buckets( storage_index, renew_secret, cancel_secret, @@ -146,7 +146,7 @@ class IStorageServerImmutableAPIsTestsMixin(object): new_secret(), new_secret(), ) - (_, allocated) = yield self.storage_server.allocate_buckets( + (_, allocated) = yield self.storage_client.allocate_buckets( storage_index, renew_secret, cancel_secret, @@ -162,7 +162,7 @@ class IStorageServerImmutableAPIsTestsMixin(object): yield abort_or_disconnect(allocated[0]) # Write different data with no complaint: - (_, allocated) = yield self.storage_server.allocate_buckets( + (_, allocated) = yield self.storage_client.allocate_buckets( storage_index, renew_secret, cancel_secret, @@ -198,7 +198,7 @@ class IStorageServerImmutableAPIsTestsMixin(object): new_secret(), new_secret(), ) - (_, allocated) = yield self.storage_server.allocate_buckets( + (_, allocated) = yield self.storage_client.allocate_buckets( storage_index, renew_secret, cancel_secret, @@ -219,7 +219,7 @@ class IStorageServerImmutableAPIsTestsMixin(object): # Bucket 0 has partial write. yield allocated[0].callRemote("write", 0, b"1" * 512) - (already_got, _) = yield self.storage_server.allocate_buckets( + (already_got, _) = yield self.storage_client.allocate_buckets( storage_index, renew_secret, cancel_secret, @@ -242,7 +242,7 @@ class IStorageServerImmutableAPIsTestsMixin(object): new_secret(), new_secret(), ) - (_, allocated) = yield self.storage_server.allocate_buckets( + (_, allocated) = yield self.storage_client.allocate_buckets( storage_index, renew_secret, cancel_secret, @@ -261,7 +261,7 @@ class IStorageServerImmutableAPIsTestsMixin(object): yield allocated[2].callRemote("write", 0, b"3" * 512) yield allocated[2].callRemote("close") - buckets = yield self.storage_server.get_buckets(storage_index) + buckets = yield self.storage_client.get_buckets(storage_index) self.assertEqual(set(buckets.keys()), {1, 2}) self.assertEqual( @@ -282,7 +282,7 @@ class IStorageServerImmutableAPIsTestsMixin(object): new_secret(), new_secret(), ) - (_, allocated) = yield self.storage_server.allocate_buckets( + (_, allocated) = yield self.storage_client.allocate_buckets( storage_index, renew_secret, cancel_secret, @@ -307,7 +307,7 @@ class IStorageServerImmutableAPIsTestsMixin(object): new_secret(), new_secret(), ) - (_, allocated) = yield self.storage_server.allocate_buckets( + (_, allocated) = yield self.storage_client.allocate_buckets( storage_index, renew_secret, cancel_secret, @@ -321,7 +321,7 @@ class IStorageServerImmutableAPIsTestsMixin(object): yield allocated[0].callRemote("write", 5, b"1" * 20) yield allocated[0].callRemote("close") - buckets = yield self.storage_server.get_buckets(storage_index) + buckets = yield self.storage_client.get_buckets(storage_index) self.assertEqual(set(buckets.keys()), {0}) self.assertEqual((yield buckets[0].callRemote("read", 0, 25)), b"1" * 25) @@ -346,7 +346,7 @@ class IStorageServerImmutableAPIsTestsMixin(object): ``IStorageServer.get_buckets()`` implementations. """ storage_index = new_storage_index() - (_, allocated) = yield self.storage_server.allocate_buckets( + (_, allocated) = yield self.storage_client.allocate_buckets( storage_index, renew_secret=new_secret(), cancel_secret=new_secret(), @@ -362,7 +362,7 @@ class IStorageServerImmutableAPIsTestsMixin(object): # Bucket 2 is partially written yield allocated[2].callRemote("write", 0, b"1" * 5) - buckets = yield self.storage_server.get_buckets(storage_index) + buckets = yield self.storage_client.get_buckets(storage_index) self.assertEqual(set(buckets.keys()), {1}) @inlineCallbacks @@ -375,7 +375,7 @@ class IStorageServerImmutableAPIsTestsMixin(object): length = 256 * 17 storage_index = new_storage_index() - (_, allocated) = yield self.storage_server.allocate_buckets( + (_, allocated) = yield self.storage_client.allocate_buckets( storage_index, renew_secret=new_secret(), cancel_secret=new_secret(), @@ -388,7 +388,7 @@ class IStorageServerImmutableAPIsTestsMixin(object): yield allocated[0].callRemote("write", 0, total_data) yield allocated[0].callRemote("close") - buckets = yield self.storage_server.get_buckets(storage_index) + buckets = yield self.storage_client.get_buckets(storage_index) bucket = buckets[0] for start, to_read in [ (0, 250), # fraction @@ -408,7 +408,7 @@ class IStorageServerImmutableAPIsTestsMixin(object): def create_share(self): """Create a share, return the storage index.""" storage_index = new_storage_index() - (_, allocated) = yield self.storage_server.allocate_buckets( + (_, allocated) = yield self.storage_client.allocate_buckets( storage_index, renew_secret=new_secret(), cancel_secret=new_secret(), @@ -429,7 +429,7 @@ class IStorageServerImmutableAPIsTestsMixin(object): behavior is opaque at this level of abstraction). """ storage_index = yield self.create_share() - buckets = yield self.storage_server.get_buckets(storage_index) + buckets = yield self.storage_client.get_buckets(storage_index) yield buckets[0].callRemote("advise_corrupt_share", b"OH NO") @inlineCallbacks @@ -440,7 +440,7 @@ class IStorageServerImmutableAPIsTestsMixin(object): abstraction). """ storage_index = yield self.create_share() - yield self.storage_server.advise_corrupt_share( + yield self.storage_client.advise_corrupt_share( b"immutable", storage_index, 0, b"ono" ) @@ -453,7 +453,7 @@ class IStorageServerMutableAPIsTestsMixin(object): """ Tests for ``IStorageServer``'s mutable APIs. - ``self.storage_server`` is expected to provide ``IStorageServer``. + ``self.storage_client`` is expected to provide ``IStorageServer``. ``STARAW`` is short for ``slot_testv_and_readv_and_writev``. """ @@ -464,7 +464,7 @@ class IStorageServerMutableAPIsTestsMixin(object): def staraw(self, *args, **kwargs): """Like ``slot_testv_and_readv_and_writev``, but less typing.""" - return self.storage_server.slot_testv_and_readv_and_writev(*args, **kwargs) + return self.storage_client.slot_testv_and_readv_and_writev(*args, **kwargs) @inlineCallbacks def test_STARAW_reads_after_write(self): @@ -760,7 +760,7 @@ class IStorageServerMutableAPIsTestsMixin(object): ) self.assertEqual(written, True) - reads = yield self.storage_server.slot_readv( + reads = yield self.storage_client.slot_readv( storage_index, shares=[0, 1], # Whole thing, partial, going beyond the edge, completely outside @@ -791,7 +791,7 @@ class IStorageServerMutableAPIsTestsMixin(object): ) self.assertEqual(written, True) - reads = yield self.storage_server.slot_readv( + reads = yield self.storage_client.slot_readv( storage_index, shares=[], readv=[(0, 7)], @@ -820,7 +820,7 @@ class IStorageServerMutableAPIsTestsMixin(object): ) self.assertEqual(written, True) - yield self.storage_server.advise_corrupt_share( + yield self.storage_client.advise_corrupt_share( b"mutable", storage_index, 0, b"ono" ) @@ -843,8 +843,8 @@ class _FoolscapMixin(SystemTestMixin): self.basedir = "test_istorageserver/" + self.id() yield SystemTestMixin.setUp(self) yield self.set_up_nodes(1) - self.storage_server = self._get_native_server().get_storage_server() - self.assertTrue(IStorageServer.providedBy(self.storage_server)) + self.storage_client = self._get_native_server().get_storage_server() + self.assertTrue(IStorageServer.providedBy(self.storage_client)) @inlineCallbacks def tearDown(self): @@ -856,10 +856,10 @@ class _FoolscapMixin(SystemTestMixin): """ Disconnect and then reconnect with a new ``IStorageServer``. """ - current = self.storage_server + current = self.storage_client yield self.bounce_client(0) - self.storage_server = self._get_native_server().get_storage_server() - assert self.storage_server is not current + self.storage_client = self._get_native_server().get_storage_server() + assert self.storage_client is not current class FoolscapSharedAPIsTests( From b7be91e3d0d02fb2d83957acfbc2b9f414e25c60 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 18 Oct 2021 13:17:07 -0400 Subject: [PATCH 0274/2309] First test for leases. --- src/allmydata/test/test_istorageserver.py | 29 ++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index b34964463..3aa48c35b 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -19,15 +19,16 @@ if PY2: # fmt: on from random import Random +import time from twisted.internet.defer import inlineCallbacks, returnValue from foolscap.api import Referenceable, RemoteException -from allmydata.interfaces import IStorageServer +from allmydata.interfaces import IStorageServer # really, IStorageClient from .common_system import SystemTestMixin from .common import AsyncTestCase - +from allmydata.storage.server import StorageServer # not a IStorageServer!! # Use random generator with known seed, so results are reproducible if tests # are run in the same order. @@ -79,6 +80,9 @@ class IStorageServerImmutableAPIsTestsMixin(object): ``self.disconnect()`` should disconnect and then reconnect, creating a new ``self.storage_client``. Some implementations may wish to skip tests using this; HTTP has no notion of disconnection. + + ``self.server`` is expected to be the corresponding + ``allmydata.storage.server.StorageServer`` instance. """ @inlineCallbacks @@ -444,7 +448,17 @@ class IStorageServerImmutableAPIsTestsMixin(object): b"immutable", storage_index, 0, b"ono" ) - # TODO allocate_buckets creates lease + @inlineCallbacks + def test_allocate_buckets_creates_lease(self): + """ + When buckets are created using ``allocate_buckets()``, a lease is + created once writing is done. + """ + storage_index = yield self.create_share() + [lease] = self.server.get_leases(storage_index) + # Lease expires in 31 days. + assert lease.get_expiration_time() - time.time() > (31 * 24 * 60 * 60 - 10) + # TODO add_lease renews lease if existing storage index and secret # TODO add_lease creates new lease if new secret @@ -455,6 +469,9 @@ class IStorageServerMutableAPIsTestsMixin(object): ``self.storage_client`` is expected to provide ``IStorageServer``. + ``self.server`` is expected to be the corresponding + ``allmydata.storage.server.StorageServer`` instance. + ``STARAW`` is short for ``slot_testv_and_readv_and_writev``. """ @@ -845,6 +862,12 @@ class _FoolscapMixin(SystemTestMixin): yield self.set_up_nodes(1) self.storage_client = self._get_native_server().get_storage_server() self.assertTrue(IStorageServer.providedBy(self.storage_client)) + self.server = None + for s in self.clients[0].services: + if isinstance(s, StorageServer): + self.server = s + break + assert self.server is not None, "Couldn't find StorageServer" @inlineCallbacks def tearDown(self): From 7d04e6ab8613010e98f807fa95826451d79b2d1d Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 18 Oct 2021 10:45:10 -0400 Subject: [PATCH 0275/2309] news fragment --- newsfragments/LFS-01-007.security | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 newsfragments/LFS-01-007.security diff --git a/newsfragments/LFS-01-007.security b/newsfragments/LFS-01-007.security new file mode 100644 index 000000000..75d9904a2 --- /dev/null +++ b/newsfragments/LFS-01-007.security @@ -0,0 +1,2 @@ +The storage protocol operation ``add_lease`` now safely rejects an attempt to add a 4,294,967,296th lease to an immutable share. +Previously this failed with an error after recording the new lease in the share file, resulting in the share file losing track of a one previous lease. From f60bbbd3e201b1b49598b7b2b6a05ad8db8a3dfd Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 18 Oct 2021 10:45:58 -0400 Subject: [PATCH 0276/2309] make it possible to test this behavior of `add_lease` --- src/allmydata/storage/immutable.py | 66 ++++++++++++++++++++++++++++-- src/allmydata/test/test_storage.py | 59 +++++++++++++++++++++++++- 2 files changed, 120 insertions(+), 5 deletions(-) diff --git a/src/allmydata/storage/immutable.py b/src/allmydata/storage/immutable.py index b8b18f140..acd09854f 100644 --- a/src/allmydata/storage/immutable.py +++ b/src/allmydata/storage/immutable.py @@ -53,13 +53,64 @@ from allmydata.storage.common import UnknownImmutableContainerVersionError # then the value stored in this field will be the actual share data length # modulo 2**32. +def _fix_lease_count_format(lease_count_format): + """ + Turn a single character struct format string into a format string suitable + for use in encoding and decoding the lease count value inside a share + file, if possible. + + :param str lease_count_format: A single character format string like + ``"B"`` or ``"L"``. + + :raise ValueError: If the given format string is not suitable for use + encoding and decoding a lease count. + + :return str: A complete format string which can safely be used to encode + and decode lease counts in a share file. + """ + if len(lease_count_format) != 1: + raise ValueError( + "Cannot construct ShareFile with lease_count_format={!r}; " + "format must accept a single value".format( + lease_count_format, + ), + ) + # Make it big-endian with standard size so all platforms agree on the + # result. + fixed = ">" + lease_count_format + if struct.calcsize(fixed) > 4: + # There is only room for at most 4 bytes in the share file format so + # we can't allow any larger formats. + raise ValueError( + "Cannot construct ShareFile with lease_count_format={!r}; " + "size must be smaller than size of '>L'".format( + lease_count_format, + ), + ) + return fixed + + class ShareFile(object): + """ + Support interaction with persistent storage of a share. + + :ivar str _lease_count_format: The format string which is used to encode + and decode the lease count inside the share file. As stated in the + comment in this module there is room for at most 4 bytes in this part + of the file. A format string that works on fewer bytes is allowed to + restrict the number of leases allowed in the share file to a smaller + number than could be supported by using the full 4 bytes. This is + mostly of interest for testing. + """ LEASE_SIZE = struct.calcsize(">L32s32sL") sharetype = "immutable" - def __init__(self, filename, max_size=None, create=False): + def __init__(self, filename, max_size=None, create=False, lease_count_format="L"): """ If max_size is not None then I won't allow more than max_size to be written to me. If create=True and max_size must not be None. """ precondition((max_size is not None) or (not create), max_size, create) + + self._lease_count_format = _fix_lease_count_format(lease_count_format) + self._lease_count_size = struct.calcsize(self._lease_count_format) self.home = filename self._max_size = max_size if create: @@ -126,12 +177,21 @@ class ShareFile(object): def _read_num_leases(self, f): f.seek(0x08) - (num_leases,) = struct.unpack(">L", f.read(4)) + (num_leases,) = struct.unpack( + self._lease_count_format, + f.read(self._lease_count_size), + ) return num_leases def _write_num_leases(self, f, num_leases): + self._write_encoded_num_leases( + f, + struct.pack(self._lease_count_format, num_leases), + ) + + def _write_encoded_num_leases(self, f, encoded_num_leases): f.seek(0x08) - f.write(struct.pack(">L", num_leases)) + f.write(encoded_num_leases) def _truncate_leases(self, f, num_leases): f.truncate(self._lease_offset + num_leases * self.LEASE_SIZE) diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index d18960a1e..0a37dffc2 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -19,6 +19,7 @@ import platform import stat import struct import shutil +from functools import partial from uuid import uuid4 from twisted.trial import unittest @@ -3009,8 +3010,8 @@ class Stats(unittest.TestCase): class ShareFileTests(unittest.TestCase): """Tests for allmydata.storage.immutable.ShareFile.""" - def get_sharefile(self): - sf = ShareFile(self.mktemp(), max_size=1000, create=True) + def get_sharefile(self, **kwargs): + sf = ShareFile(self.mktemp(), max_size=1000, create=True, **kwargs) sf.write_share_data(0, b"abc") sf.write_share_data(2, b"DEF") # Should be b'abDEF' now. @@ -3039,3 +3040,57 @@ class ShareFileTests(unittest.TestCase): sf = self.get_sharefile() with self.assertRaises(IndexError): sf.cancel_lease(b"garbage") + + def test_long_lease_count_format(self): + """ + ``ShareFile.__init__`` raises ``ValueError`` if the lease count format + given is longer than one character. + """ + with self.assertRaises(ValueError): + self.get_sharefile(lease_count_format="BB") + + def test_large_lease_count_format(self): + """ + ``ShareFile.__init__`` raises ``ValueError`` if the lease count format + encodes to a size larger than 8 bytes. + """ + with self.assertRaises(ValueError): + self.get_sharefile(lease_count_format="Q") + + def test_avoid_lease_overflow(self): + """ + If the share file already has the maximum number of leases supported then + ``ShareFile.add_lease`` raises ``struct.error`` and makes no changes + to the share file contents. + """ + make_lease = partial( + LeaseInfo, + renew_secret=b"r" * 32, + cancel_secret=b"c" * 32, + expiration_time=2 ** 31, + ) + # Make it a little easier to reach the condition by limiting the + # number of leases to only 255. + sf = self.get_sharefile(lease_count_format="B") + + # Add the leases. + for i in range(2 ** 8 - 1): + lease = make_lease(owner_num=i) + sf.add_lease(lease) + + # Capture the state of the share file at this point so we can + # determine whether the next operation modifies it or not. + with open(sf.home, "rb") as f: + before_data = f.read() + + # It is not possible to add a 256th lease. + lease = make_lease(owner_num=256) + with self.assertRaises(struct.error): + sf.add_lease(lease) + + # Compare the share file state to what we captured earlier. Any + # change is a bug. + with open(sf.home, "rb") as f: + after_data = f.read() + + self.assertEqual(before_data, after_data) From df64bbb1e443cbbad272067e7716b4d9a3f3408d Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 18 Oct 2021 10:50:28 -0400 Subject: [PATCH 0277/2309] fail to encode the lease count *before* changing anything This preserves the failure behavior - `struct.error` is raised - but leaves the actual share file contents untouched if the new lease count cannot be encoded. There are still two separate write operations so it is conceivable that some other problem could cause `write_lease_record` to happen but `write_encoded_num_leases` not to happen. As far as I can tell we have severely limited options for addressing that problem in general as long as share files are backed by a POSIX filesystem. However, by removing the failure mode that depends on user input, it may be that this is all that is needed to close the *security* hole. --- src/allmydata/storage/immutable.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/allmydata/storage/immutable.py b/src/allmydata/storage/immutable.py index acd09854f..887ccc931 100644 --- a/src/allmydata/storage/immutable.py +++ b/src/allmydata/storage/immutable.py @@ -209,8 +209,11 @@ class ShareFile(object): def add_lease(self, lease_info): with open(self.home, 'rb+') as f: num_leases = self._read_num_leases(f) + # Before we write the new lease record, make sure we can encode + # the new lease count. + new_lease_count = struct.pack(self._lease_count_format, num_leases + 1) self._write_lease_record(f, num_leases, lease_info) - self._write_num_leases(f, num_leases+1) + self._write_encoded_num_leases(f, new_lease_count) def renew_lease(self, renew_secret, new_expire_time): for i,lease in enumerate(self.get_leases()): From 4a5e4be0069ed41eb25deeb09828de76e1db041d Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 18 Oct 2021 14:35:11 -0400 Subject: [PATCH 0278/2309] news fragment --- newsfragments/LFS-01-008.security | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 newsfragments/LFS-01-008.security diff --git a/newsfragments/LFS-01-008.security b/newsfragments/LFS-01-008.security new file mode 100644 index 000000000..5d6c07ab5 --- /dev/null +++ b/newsfragments/LFS-01-008.security @@ -0,0 +1,2 @@ +The storage protocol operation ``readv`` now safely rejects attempts to read negative lengths. +Previously these read requests were satisfied with the complete contents of the share file (including trailing metadata) starting from the specified offset. From 5e58b62979b7ad2c813f95e1f50c550da1f69f36 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 18 Oct 2021 14:36:24 -0400 Subject: [PATCH 0279/2309] Add a test for negative offset or length to MutableShareFile.readv --- src/allmydata/test/strategies.py | 15 ++++ src/allmydata/test/test_storage.py | 117 +++++++++++++++++++++++++++++ 2 files changed, 132 insertions(+) diff --git a/src/allmydata/test/strategies.py b/src/allmydata/test/strategies.py index c0f558ef6..2bb23a373 100644 --- a/src/allmydata/test/strategies.py +++ b/src/allmydata/test/strategies.py @@ -16,6 +16,7 @@ from hypothesis.strategies import ( one_of, builds, binary, + integers, ) from ..uri import ( @@ -119,3 +120,17 @@ def dir2_mdmf_capabilities(): MDMFDirectoryURI, mdmf_capabilities(), ) + +def offsets(min_value=0, max_value=2 ** 16): + """ + Build ``int`` values that could be used as valid offsets into a sequence + (such as share data in a share file). + """ + return integers(min_value, max_value) + +def lengths(min_value=1, max_value=2 ** 16): + """ + Build ``int`` values that could be used as valid lengths of data (such as + share data in a share file). + """ + return integers(min_value, max_value) diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index 0a37dffc2..f19073f3e 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -13,6 +13,9 @@ if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from six import ensure_str +from io import ( + BytesIO, +) import time import os.path import platform @@ -59,6 +62,10 @@ from allmydata.storage_client import ( ) from .common import LoggingServiceParent, ShouldFailMixin from .common_util import FakeCanary +from .strategies import ( + offsets, + lengths, +) class UtilTests(unittest.TestCase): @@ -3094,3 +3101,113 @@ class ShareFileTests(unittest.TestCase): after_data = f.read() self.assertEqual(before_data, after_data) + + +class MutableShareFileTests(unittest.TestCase): + """ + Tests for allmydata.storage.mutable.MutableShareFile. + """ + def get_sharefile(self): + return MutableShareFile(self.mktemp()) + + @given( + nodeid=strategies.just(b"x" * 20), + write_enabler=strategies.just(b"y" * 32), + datav=strategies.lists( + # Limit the max size of these so we don't write *crazy* amounts of + # data to disk. + strategies.tuples(offsets(), strategies.binary(max_size=2 ** 8)), + max_size=2 ** 8, + ), + new_length=offsets(), + ) + def test_readv_reads_share_data(self, nodeid, write_enabler, datav, new_length): + """ + ``MutableShareFile.readv`` returns bytes from the share data portion + of the share file. + """ + sf = self.get_sharefile() + sf.create(my_nodeid=nodeid, write_enabler=write_enabler) + sf.writev(datav=datav, new_length=new_length) + + # Apply all of the writes to a simple in-memory buffer so we can + # resolve the final state of the share data. In particular, this + # helps deal with overlapping writes which otherwise make it tricky to + # figure out what data to expect to be able to read back. + buf = BytesIO() + for (offset, data) in datav: + buf.seek(offset) + buf.write(data) + buf.truncate(new_length) + + # Using that buffer, determine the expected result of a readv for all + # of the data just written. + def read_from_buf(offset, length): + buf.seek(offset) + return buf.read(length) + expected_data = list( + read_from_buf(offset, len(data)) + for (offset, data) + in datav + ) + + # Perform a read that gives back all of the data written to the share + # file. + read_vectors = list((offset, len(data)) for (offset, data) in datav) + read_data = sf.readv(read_vectors) + + # Make sure the read reproduces the value we computed using our local + # buffer. + self.assertEqual(expected_data, read_data) + + @given( + nodeid=strategies.just(b"x" * 20), + write_enabler=strategies.just(b"y" * 32), + readv=strategies.lists(strategies.tuples(offsets(), lengths()), min_size=1), + random=strategies.randoms(), + ) + def test_readv_rejects_negative_length(self, nodeid, write_enabler, readv, random): + """ + If a negative length is given to ``MutableShareFile.readv`` in a read + vector then ``AssertionError`` is raised. + """ + # Pick a read vector to break with a negative value + readv_index = random.randrange(len(readv)) + # Decide on whether we're breaking offset or length + offset_or_length = random.randrange(2) + + # A helper function that will take a valid offset and length and break + # one of them. + def corrupt(break_length, offset, length): + if break_length: + # length must not be 0 or flipping the sign does nothing + # length must not be negative or flipping the sign *fixes* it + assert length > 0 + return (offset, -length) + else: + if offset > 0: + # We can break offset just by flipping the sign. + return (-offset, length) + else: + # Otherwise it has to be zero. If it was negative, what's + # going on? + assert offset == 0 + # Since we can't just flip the sign on 0 to break things, + # replace a 0 offset with a simple negative value. All + # other negative values will be tested by the `offset > 0` + # case above. + return (-1, length) + + # Break the read vector very slightly! + broken_readv = readv[:] + broken_readv[readv_index] = corrupt( + offset_or_length, + *broken_readv[readv_index] + ) + + sf = self.get_sharefile() + sf.create(my_nodeid=nodeid, write_enabler=write_enabler) + + # A read with a broken read vector is an error. + with self.assertRaises(AssertionError): + sf.readv(broken_readv) From 3cd9a02c810f6aa1dba9dbd664980b49bec39048 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 18 Oct 2021 20:13:24 -0400 Subject: [PATCH 0280/2309] Reject negative lengths in MutableShareFile._read_share_data and readv --- src/allmydata/storage/mutable.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/storage/mutable.py b/src/allmydata/storage/mutable.py index 2ef0c3215..cdb4faeaf 100644 --- a/src/allmydata/storage/mutable.py +++ b/src/allmydata/storage/mutable.py @@ -120,6 +120,7 @@ class MutableShareFile(object): def _read_share_data(self, f, offset, length): precondition(offset >= 0) + precondition(length >= 0) data_length = self._read_data_length(f) if offset+length > data_length: # reads beyond the end of the data are truncated. Reads that @@ -454,4 +455,3 @@ def create_mutable_sharefile(filename, my_nodeid, write_enabler, parent): ms.create(my_nodeid, write_enabler) del ms return MutableShareFile(filename, parent) - From 4b8b6052f33c9513986170e5b6ad3066747b7560 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 19 Oct 2021 09:05:48 -0400 Subject: [PATCH 0281/2309] Finish testing leases on immutables. --- src/allmydata/test/test_istorageserver.py | 71 +++++++++++++++++++---- 1 file changed, 60 insertions(+), 11 deletions(-) diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index 3aa48c35b..b6d3e4b9a 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -19,7 +19,6 @@ if PY2: # fmt: on from random import Random -import time from twisted.internet.defer import inlineCallbacks, returnValue @@ -82,7 +81,9 @@ class IStorageServerImmutableAPIsTestsMixin(object): this; HTTP has no notion of disconnection. ``self.server`` is expected to be the corresponding - ``allmydata.storage.server.StorageServer`` instance. + ``allmydata.storage.server.StorageServer`` instance. Time should be + instrumented, such that ``self.fake_time()`` and ``self.fake_sleep()`` + return and advance the server time, respectively. """ @inlineCallbacks @@ -412,10 +413,12 @@ class IStorageServerImmutableAPIsTestsMixin(object): def create_share(self): """Create a share, return the storage index.""" storage_index = new_storage_index() + renew_secret = new_secret() + cancel_secret = new_secret() (_, allocated) = yield self.storage_client.allocate_buckets( storage_index, - renew_secret=new_secret(), - cancel_secret=new_secret(), + renew_secret=renew_secret, + cancel_secret=cancel_secret, sharenums=set(range(1)), allocated_size=10, canary=Referenceable(), @@ -423,7 +426,7 @@ class IStorageServerImmutableAPIsTestsMixin(object): yield allocated[0].callRemote("write", 0, b"0123456789") yield allocated[0].callRemote("close") - returnValue(storage_index) + returnValue((storage_index, renew_secret, cancel_secret)) @inlineCallbacks def test_bucket_advise_corrupt_share(self): @@ -432,7 +435,7 @@ class IStorageServerImmutableAPIsTestsMixin(object): ``IStorageServer.get_buckets()`` does not result in error (other behavior is opaque at this level of abstraction). """ - storage_index = yield self.create_share() + storage_index, _, _ = yield self.create_share() buckets = yield self.storage_client.get_buckets(storage_index) yield buckets[0].callRemote("advise_corrupt_share", b"OH NO") @@ -443,7 +446,7 @@ class IStorageServerImmutableAPIsTestsMixin(object): result in error (other behavior is opaque at this level of abstraction). """ - storage_index = yield self.create_share() + storage_index, _, _ = yield self.create_share() yield self.storage_client.advise_corrupt_share( b"immutable", storage_index, 0, b"ono" ) @@ -454,13 +457,49 @@ class IStorageServerImmutableAPIsTestsMixin(object): When buckets are created using ``allocate_buckets()``, a lease is created once writing is done. """ - storage_index = yield self.create_share() + storage_index, _, _ = yield self.create_share() [lease] = self.server.get_leases(storage_index) # Lease expires in 31 days. - assert lease.get_expiration_time() - time.time() > (31 * 24 * 60 * 60 - 10) + assert lease.get_expiration_time() - self.fake_time() > (31 * 24 * 60 * 60 - 10) - # TODO add_lease renews lease if existing storage index and secret - # TODO add_lease creates new lease if new secret + @inlineCallbacks + def test_add_lease_renewal(self): + """ + If the lease secret is reused, ``add_lease()`` extends the existing + lease. + """ + storage_index, renew_secret, cancel_secret = yield self.create_share() + [lease] = self.server.get_leases(storage_index) + initial_expiration_time = lease.get_expiration_time() + + # Time passes: + self.fake_sleep(178) + + # We renew the lease: + yield self.storage_client.add_lease(storage_index, renew_secret, cancel_secret) + [lease] = self.server.get_leases(storage_index) + new_expiration_time = lease.get_expiration_time() + self.assertEqual(new_expiration_time - initial_expiration_time, 178) + + @inlineCallbacks + def test_add_new_lease(self): + """ + If a new lease secret is used, ``add_lease()`` creates a new lease. + """ + storage_index, _, _ = yield self.create_share() + [lease] = self.server.get_leases(storage_index) + initial_expiration_time = lease.get_expiration_time() + + # Time passes: + self.fake_sleep(167) + + # We create a new lease: + renew_secret = new_secret() + cancel_secret = new_secret() + yield self.storage_client.add_lease(storage_index, renew_secret, cancel_secret) + [lease1, lease2] = self.server.get_leases(storage_index) + self.assertEqual(lease1.get_expiration_time(), initial_expiration_time) + self.assertEqual(lease2.get_expiration_time() - initial_expiration_time, 167) class IStorageServerMutableAPIsTestsMixin(object): @@ -868,6 +907,16 @@ class _FoolscapMixin(SystemTestMixin): self.server = s break assert self.server is not None, "Couldn't find StorageServer" + self._current_time = 123456 + self.server._get_current_time = self.fake_time + + def fake_time(self): + """Return the current fake, test-controlled, time.""" + return self._current_time + + def fake_sleep(self, seconds): + """Advance the fake time by the given number of seconds.""" + self._current_time += seconds @inlineCallbacks def tearDown(self): From 2a5dbcb05edc2fdc325028b8ca58be0d2fe5ac21 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 19 Oct 2021 09:28:11 -0400 Subject: [PATCH 0282/2309] Tests for mutable leases. --- src/allmydata/test/test_istorageserver.py | 134 ++++++++++++++++++++-- 1 file changed, 122 insertions(+), 12 deletions(-) diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index b6d3e4b9a..fe494a9d4 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -460,7 +460,9 @@ class IStorageServerImmutableAPIsTestsMixin(object): storage_index, _, _ = yield self.create_share() [lease] = self.server.get_leases(storage_index) # Lease expires in 31 days. - assert lease.get_expiration_time() - self.fake_time() > (31 * 24 * 60 * 60 - 10) + self.assertTrue( + lease.get_expiration_time() - self.fake_time() > (31 * 24 * 60 * 60 - 10) + ) @inlineCallbacks def test_add_lease_renewal(self): @@ -858,12 +860,8 @@ class IStorageServerMutableAPIsTestsMixin(object): ) @inlineCallbacks - def test_advise_corrupt_share(self): - """ - Calling ``advise_corrupt_share()`` on a mutable share does not - result in error (other behavior is opaque at this level of - abstraction). - """ + def create_slot(self): + """Create a slot with sharenum 0.""" secrets = self.new_secrets() storage_index = new_storage_index() (written, _) = yield self.staraw( @@ -875,16 +873,128 @@ class IStorageServerMutableAPIsTestsMixin(object): r_vector=[], ) self.assertEqual(written, True) + returnValue((secrets, storage_index)) + + @inlineCallbacks + def test_advise_corrupt_share(self): + """ + Calling ``advise_corrupt_share()`` on a mutable share does not + result in error (other behavior is opaque at this level of + abstraction). + """ + secrets, storage_index = yield self.create_slot() yield self.storage_client.advise_corrupt_share( b"mutable", storage_index, 0, b"ono" ) - # TODO STARAW creates lease for new data - # TODO STARAW renews lease if same secret is used on existing data - # TODO STARAW creates new lease for existing data if new secret is given - # TODO add_lease renews lease if existing storage index and secret - # TODO add_lease creates new lease if new secret + @inlineCallbacks + def test_STARAW_create_lease(self): + """ + When STARAW creates a new slot, it also creates a lease. + """ + _, storage_index = yield self.create_slot() + [lease] = self.server.get_slot_leases(storage_index) + # Lease expires in 31 days. + self.assertTrue( + lease.get_expiration_time() - self.fake_time() > (31 * 24 * 60 * 60 - 10) + ) + + @inlineCallbacks + def test_STARAW_renews_lease(self): + """ + When STARAW is run on an existing slot with same renewal secret, it + renews the lease. + """ + secrets, storage_index = yield self.create_slot() + [lease] = self.server.get_slot_leases(storage_index) + initial_expire = lease.get_expiration_time() + + # Time passes... + self.fake_sleep(17) + + # We do another write: + (written, _) = yield self.staraw( + storage_index, + secrets, + tw_vectors={ + 0: ([], [(0, b"1234567")], 7), + }, + r_vector=[], + ) + self.assertEqual(written, True) + + # The lease has been renewed: + [lease] = self.server.get_slot_leases(storage_index) + self.assertEqual(lease.get_expiration_time() - initial_expire, 17) + + @inlineCallbacks + def test_STARAW_new_lease(self): + """ + When STARAW is run with a new renewal secret on an existing slot, it + adds a new lease. + """ + secrets, storage_index = yield self.create_slot() + [lease] = self.server.get_slot_leases(storage_index) + initial_expire = lease.get_expiration_time() + + # Time passes... + self.fake_sleep(19) + + # We do another write: + (written, _) = yield self.staraw( + storage_index, + (secrets[0], new_secret(), new_secret()), + tw_vectors={ + 0: ([], [(0, b"1234567")], 7), + }, + r_vector=[], + ) + self.assertEqual(written, True) + + # A new lease was added: + [lease1, lease2] = self.server.get_slot_leases(storage_index) + self.assertEqual(lease1.get_expiration_time(), initial_expire) + self.assertEqual(lease2.get_expiration_time() - initial_expire, 19) + + @inlineCallbacks + def test_add_lease_renewal(self): + """ + If the lease secret is reused, ``add_lease()`` extends the existing + lease. + """ + secrets, storage_index = yield self.create_slot() + [lease] = self.server.get_slot_leases(storage_index) + initial_expiration_time = lease.get_expiration_time() + + # Time passes: + self.fake_sleep(178) + + # We renew the lease: + yield self.storage_client.add_lease(storage_index, secrets[1], secrets[2]) + [lease] = self.server.get_slot_leases(storage_index) + new_expiration_time = lease.get_expiration_time() + self.assertEqual(new_expiration_time - initial_expiration_time, 178) + + @inlineCallbacks + def test_add_new_lease(self): + """ + If a new lease secret is used, ``add_lease()`` creates a new lease. + """ + secrets, storage_index = yield self.create_slot() + [lease] = self.server.get_slot_leases(storage_index) + initial_expiration_time = lease.get_expiration_time() + + # Time passes: + self.fake_sleep(167) + + # We create a new lease: + renew_secret = new_secret() + cancel_secret = new_secret() + yield self.storage_client.add_lease(storage_index, renew_secret, cancel_secret) + [lease1, lease2] = self.server.get_slot_leases(storage_index) + self.assertEqual(lease1.get_expiration_time(), initial_expiration_time) + self.assertEqual(lease2.get_expiration_time() - initial_expiration_time, 167) class _FoolscapMixin(SystemTestMixin): From e1dfee1d7b35d494b55178568612f6f648cf1205 Mon Sep 17 00:00:00 2001 From: fenn-cs Date: Tue, 19 Oct 2021 23:20:38 +0100 Subject: [PATCH 0283/2309] put notes under correct categories Signed-off-by: fenn-cs --- NEWS.rst | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/NEWS.rst b/NEWS.rst index 366e45907..e4fef833a 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -59,7 +59,6 @@ Configuration Changes Documentation Changes --------------------- -- (`#3659 `_) - Documentation now has its own towncrier category. (`#3664 `_) - `tox -e docs` will treat warnings about docs as errors. (`#3666 `_) - The visibility of the Tahoe-LAFS logo has been improved for "dark" themed viewing. (`#3677 `_) @@ -75,6 +74,11 @@ Documentation Changes - The Great Black Swamp proposed specification now has a simplified interface for reading data from immutable shares. (`#3777 `_) - tahoe-dev mailing list is now at tahoe-dev@lists.tahoe-lafs.org. (`#3782 `_) - The Great Black Swamp specification now describes the required authorization scheme. (`#3785 `_) +- The "Great Black Swamp" proposed specification has been expanded to include two lease management APIs. (`#3037 `_) +- The specification section of the Tahoe-LAFS documentation now includes explicit discussion of the security properties of Foolscap "fURLs" on which it depends. (`#3503 `_) +- The README, revised by Viktoriia with feedback from the team, is now more focused on the developer community and provides more information about Tahoe-LAFS, why it's important, and how someone can use it or start contributing to it. (`#3545 `_) +- The "Great Black Swamp" proposed specification has been changed use ``v=1`` as the URL version identifier. (`#3644 `_) +- You can run `make livehtml` in docs directory to invoke sphinx-autobuild. (`#3663 `_) Removed Features @@ -90,11 +94,6 @@ Removed Features Other Changes ------------- -- The "Great Black Swamp" proposed specification has been expanded to include two lease management APIs. (`#3037 `_) -- The specification section of the Tahoe-LAFS documentation now includes explicit discussion of the security properties of Foolscap "fURLs" on which it depends. (`#3503 `_) -- The README, revised by Viktoriia with feedback from the team, is now more focused on the developer community and provides more information about Tahoe-LAFS, why it's important, and how someone can use it or start contributing to it. (`#3545 `_) -- The "Great Black Swamp" proposed specification has been changed use ``v=1`` as the URL version identifier. (`#3644 `_) -- You can run `make livehtml` in docs directory to invoke sphinx-autobuild. (`#3663 `_) - Refactored test_introducer in web tests to use custom base test cases (`#3757 `_) From 20ad6cd9e79cc62b23c6de4c4dba8ff9300f7a2c Mon Sep 17 00:00:00 2001 From: fenn-cs Date: Tue, 19 Oct 2021 23:57:52 +0100 Subject: [PATCH 0284/2309] iterate over args directly without indexing Signed-off-by: fenn-cs --- src/allmydata/util/eliotutil.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/allmydata/util/eliotutil.py b/src/allmydata/util/eliotutil.py index f2272a731..fe431568f 100644 --- a/src/allmydata/util/eliotutil.py +++ b/src/allmydata/util/eliotutil.py @@ -335,12 +335,7 @@ def log_call_deferred(action_type): kwargs = {k: bytes_to_unicode(True, kw[k]) for k in kw} # Remove complex (unserializable) objects from positional args to # prevent eliot from throwing errors when it attempts serialization - args = tuple( - a[pos] - if is_json_serializable(a[pos]) - else str(a[pos]) - for pos in range(len(a)) - ) + args = tuple(arg if is_json_serializable(arg) else str(arg) for arg in a) with start_action(action_type=action_type, args=args, kwargs=kwargs).context(): # Use addActionFinish so that the action finishes when the # Deferred fires. From 1e6265b87cdb5c0c04a79b69a82027edf07072a1 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 19 Oct 2021 17:24:29 -0600 Subject: [PATCH 0285/2309] update relnotes --- relnotes.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/relnotes.txt b/relnotes.txt index c97b42664..fc18f4e96 100644 --- a/relnotes.txt +++ b/relnotes.txt @@ -32,7 +32,7 @@ and FTP support. There are several dependency changes that will be interesting for distribution maintainers. -As well 196 bugs have been fixed since the last release. +In all, 240 issues have been fixed since the last release. Please see ``NEWS.rst`` for a more complete list of changes. From 4bfb9d21700b8084d5fb2c697ceeb7088dd97c37 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 19 Oct 2021 17:25:34 -0600 Subject: [PATCH 0286/2309] correct previous-release version --- relnotes.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/relnotes.txt b/relnotes.txt index fc18f4e96..e5976a97b 100644 --- a/relnotes.txt +++ b/relnotes.txt @@ -15,7 +15,7 @@ unique security and fault-tolerance properties: https://tahoe-lafs.readthedocs.org/en/latest/about.html -The previous stable release of Tahoe-LAFS was v1.15.0, released on +The previous stable release of Tahoe-LAFS was v1.15.1, released on March 23rd, 2021. The major change in this release is the completion of the Python 3 From a7ce84f4d5a884e165232a4e009e345c976cabff Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 19 Oct 2021 18:02:29 -0600 Subject: [PATCH 0287/2309] correct names, dates --- relnotes.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/relnotes.txt b/relnotes.txt index e5976a97b..2748bc4fa 100644 --- a/relnotes.txt +++ b/relnotes.txt @@ -151,10 +151,10 @@ solely as a labor of love by volunteers. Thank you very much to the team of "hackers in the public interest" who make Tahoe-LAFS possible. -fenn-cs +fenn-cs + meejah on behalf of the Tahoe-LAFS team -September 16, 2021 +October 19, 2021 Planet Earth From 027df0982894642a5b7a8c645278106d5e8b118f Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 20 Oct 2021 16:10:23 -0600 Subject: [PATCH 0288/2309] release two things: wheels, and a .tar.gz source dist --- docs/release-checklist.rst | 4 +--- tox.ini | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/docs/release-checklist.rst b/docs/release-checklist.rst index da1bbe16f..f943abb5d 100644 --- a/docs/release-checklist.rst +++ b/docs/release-checklist.rst @@ -120,10 +120,8 @@ they will need to evaluate which contributors' signatures they trust. - when satisfied, sign the tarballs: - - gpg --pinentry=loopback --armor --detach-sign dist/tahoe_lafs-1.15.0rc0-py2-none-any.whl - - gpg --pinentry=loopback --armor --detach-sign dist/tahoe_lafs-1.15.0rc0.tar.bz2 + - gpg --pinentry=loopback --armor --detach-sign dist/tahoe_lafs-1.15.0rc0-py2.py3-none-any.whl - gpg --pinentry=loopback --armor --detach-sign dist/tahoe_lafs-1.15.0rc0.tar.gz - - gpg --pinentry=loopback --armor --detach-sign dist/tahoe_lafs-1.15.0rc0.zip Privileged Contributor diff --git a/tox.ini b/tox.ini index af79ea4c7..94c9835ad 100644 --- a/tox.ini +++ b/tox.ini @@ -264,4 +264,4 @@ basepython = python3 deps = commands = python setup.py update_version - python setup.py sdist --formats=bztar,gztar,zip bdist_wheel --universal + python setup.py sdist --formats=gztar bdist_wheel --universal From b8ff0e7fa913c4e6c73991c2ffd1a3278d688ceb Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 20 Oct 2021 16:11:03 -0600 Subject: [PATCH 0289/2309] news --- newsfragments/3735.feature | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/3735.feature diff --git a/newsfragments/3735.feature b/newsfragments/3735.feature new file mode 100644 index 000000000..5a86d5547 --- /dev/null +++ b/newsfragments/3735.feature @@ -0,0 +1 @@ +Tahoe-LAFS releases now have just a .tar.gz source release and a (universal) wheel From a8d3555ebb6eae1d65cf4cfc928357de8d9a2268 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 21 Oct 2021 15:24:53 -0400 Subject: [PATCH 0290/2309] reference the eventually-public ticket number --- newsfragments/{LFS-01-001.security => 3819.security} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename newsfragments/{LFS-01-001.security => 3819.security} (100%) diff --git a/newsfragments/LFS-01-001.security b/newsfragments/3819.security similarity index 100% rename from newsfragments/LFS-01-001.security rename to newsfragments/3819.security From 61a20e245029ecfa626aa5f522af2eb08b7e19d3 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 22 Oct 2021 10:10:53 -0400 Subject: [PATCH 0291/2309] Add concept of upload secret to immutable uploads. --- docs/proposed/http-storage-node-protocol.rst | 33 +++++++++++++++++--- newsfragments/3820.minor | 0 2 files changed, 29 insertions(+), 4 deletions(-) create mode 100644 newsfragments/3820.minor diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index 521bf476d..16db0fed9 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -451,6 +451,7 @@ Details of the buckets to create are encoded in the request body. For example:: {"renew-secret": "efgh", "cancel-secret": "ijkl", + "upload-secret": "xyzf", "share-numbers": [1, 7, ...], "allocated-size": 12345} The response body includes encoded information about the created buckets. @@ -458,6 +459,8 @@ For example:: {"already-have": [1, ...], "allocated": [7, ...]} +The session secret is an opaque _byte_ string. + Discussion `````````` @@ -482,6 +485,13 @@ The response includes ``already-have`` and ``allocated`` for two reasons: This might be because a server has become unavailable and a remaining server needs to store more shares for the upload. It could also just be that the client's preferred servers have changed. +Regarding upload secrets, +the goal is for uploading and aborting (see next sections) to be authenticated by more than just the storage index. +In the future, we will want to generate them in a way that allows resuming/canceling when the client has issues. +In the short term, they can just be a random byte string. +The key security constraint is that each upload to each server has its own, unique upload key, +tied to uploading that particular storage index to this particular server. + ``PATCH /v1/immutable/:storage_index/:share_number`` !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! @@ -498,6 +508,12 @@ If any one of these requests fails then at most 128KiB of upload work needs to b The server must recognize when all of the data has been received and mark the share as complete (which it can do because it was informed of the size when the storage index was initialized). +The request body looks this, with data and upload secret being bytes:: + + { "upload-secret": "xyzf", "data": "thedata" } + +Responses: + * When a chunk that does not complete the share is successfully uploaded the response is ``OK``. The response body indicates the range of share data that has yet to be uploaded. That is:: @@ -522,6 +538,10 @@ The server must recognize when all of the data has been received and mark the sh This cancels an *in-progress* upload. +The request body looks this:: + + { "upload-secret": "xyzf" } + The response code: * When the upload is still in progress and therefore the abort has succeeded, @@ -695,6 +715,7 @@ Immutable Data POST /v1/immutable/AAAAAAAAAAAAAAAA {"renew-secret": "efgh", "cancel-secret": "ijkl", + "upload-secret": "xyzf", "share-numbers": [1, 7], "allocated-size": 48} 200 OK @@ -704,25 +725,29 @@ Immutable Data PATCH /v1/immutable/AAAAAAAAAAAAAAAA/7 Content-Range: bytes 0-15/48 - + + {"upload-secret": b"xyzf", "data": "first 16 bytes!!" 200 OK PATCH /v1/immutable/AAAAAAAAAAAAAAAA/7 Content-Range: bytes 16-31/48 - + + {"upload-secret": "xyzf", "data": "second 16 bytes!" 200 OK PATCH /v1/immutable/AAAAAAAAAAAAAAAA/7 Content-Range: bytes 32-47/48 - + + {"upload-secret": "xyzf", "data": "final 16 bytes!!" 201 CREATED #. Download the content of the previously uploaded immutable share ``7``:: - GET /v1/immutable/AAAAAAAAAAAAAAAA?share=7&offset=0&size=48 + GET /v1/immutable/AAAAAAAAAAAAAAAA?share=7 + Range: bytes=0-47 200 OK diff --git a/newsfragments/3820.minor b/newsfragments/3820.minor new file mode 100644 index 000000000..e69de29bb From e0c8bab5d7a97d539a5364c962cd5861430432a0 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 22 Oct 2021 10:32:44 -0400 Subject: [PATCH 0292/2309] Add proposal on how to generate upload secret. --- docs/proposed/http-storage-node-protocol.rst | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index 16db0fed9..d5b6653be 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -459,7 +459,13 @@ For example:: {"already-have": [1, ...], "allocated": [7, ...]} -The session secret is an opaque _byte_ string. +The uplaod secret is an opaque _byte_ string. +It will be generated by hashing a combination of:b + +1. A tag. +2. The storage index, so it's unique across different source files. +3. The server ID, so it's unique across different servers. +4. The convergence secret, so that servers can't guess the upload secret for other servers. Discussion `````````` @@ -492,6 +498,13 @@ In the short term, they can just be a random byte string. The key security constraint is that each upload to each server has its own, unique upload key, tied to uploading that particular storage index to this particular server. +Rejected designs for upload secrets: + +* Upload secret per share number. + In order to make the secret unguessable by attackers, which includes other servers, + it must contain randomness. + Randomness means there is no need to have a secret per share, since adding share-specific content to randomness doesn't actually make the secret any better. + ``PATCH /v1/immutable/:storage_index/:share_number`` !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! From 6c0ca0b88592bffd8954cf06142cd962c1a3c654 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 22 Oct 2021 08:41:09 -0400 Subject: [PATCH 0293/2309] try making windows let us use longer paths --- src/allmydata/test/test_download.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/test_download.py b/src/allmydata/test/test_download.py index d61942839..8e7aa9d27 100644 --- a/src/allmydata/test/test_download.py +++ b/src/allmydata/test/test_download.py @@ -493,7 +493,7 @@ class DownloadTest(_Base, unittest.TestCase): d.addCallback(_done) return d - def test_simultaneous_onefails_onecancelled(self): + def test_simul_1fail_1cancel(self): # This exercises an mplayer behavior in ticket #1154. I believe that # mplayer made two simultaneous webapi GET requests: first one for an # index region at the end of the (mp3/video) file, then one for the From d8c466e9a7ba5f121cb6d9f891569db7e01e87b6 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 22 Oct 2021 12:35:11 -0400 Subject: [PATCH 0294/2309] try to explain `lease_count_format` more clearly --- src/allmydata/storage/immutable.py | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/src/allmydata/storage/immutable.py b/src/allmydata/storage/immutable.py index 887ccc931..e23abb080 100644 --- a/src/allmydata/storage/immutable.py +++ b/src/allmydata/storage/immutable.py @@ -106,7 +106,30 @@ class ShareFile(object): sharetype = "immutable" def __init__(self, filename, max_size=None, create=False, lease_count_format="L"): - """ If max_size is not None then I won't allow more than max_size to be written to me. If create=True and max_size must not be None. """ + """ + Initialize a ``ShareFile``. + + :param Optional[int] max_size: If given, the maximum number of bytes + that this ``ShareFile`` will accept to be stored. ``write`` will + accept in total. + + :param bool create: If ``True``, create the file (and fail if it + exists already). ``max_size`` must not be ``None`` in this case. + If ``False``, open an existing file for reading. + + :param str lease_count_format: A format character to use to encode and + decode the number of leases in the share file. There are only 4 + bytes available in the file so the format must be 4 bytes or + smaller. If different formats are used at different times with + the same share file, the result will likely be nonsense. + + This parameter is intended for the test suite to use to be able to + exercise values near the maximum encodeable value without having + to create billions of leases. + + :raise ValueError: If the encoding of ``lease_count_format`` is too + large or if it is not a single format character. + """ precondition((max_size is not None) or (not create), max_size, create) self._lease_count_format = _fix_lease_count_format(lease_count_format) From bcdfb8155c28c94e75b8e7acc7344dc1f01aa798 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 22 Oct 2021 12:53:17 -0400 Subject: [PATCH 0295/2309] give the news fragment its proper name --- newsfragments/{LFS-01-007.security => 3821.security} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename newsfragments/{LFS-01-007.security => 3821.security} (100%) diff --git a/newsfragments/LFS-01-007.security b/newsfragments/3821.security similarity index 100% rename from newsfragments/LFS-01-007.security rename to newsfragments/3821.security From 7f3d9316d2dc8d2fe99b211a006bc45749f184c3 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 22 Oct 2021 12:59:26 -0400 Subject: [PATCH 0296/2309] Give the news fragment its real name --- newsfragments/{LFS-01-008.security => 3822.security} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename newsfragments/{LFS-01-008.security => 3822.security} (100%) diff --git a/newsfragments/LFS-01-008.security b/newsfragments/3822.security similarity index 100% rename from newsfragments/LFS-01-008.security rename to newsfragments/3822.security From ce30f9dd0663ba22a985571f8029ad35026bb91e Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 22 Oct 2021 15:04:45 -0400 Subject: [PATCH 0297/2309] clean up copyediting errors --- src/allmydata/storage/immutable.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/allmydata/storage/immutable.py b/src/allmydata/storage/immutable.py index e23abb080..55bcdda64 100644 --- a/src/allmydata/storage/immutable.py +++ b/src/allmydata/storage/immutable.py @@ -110,8 +110,7 @@ class ShareFile(object): Initialize a ``ShareFile``. :param Optional[int] max_size: If given, the maximum number of bytes - that this ``ShareFile`` will accept to be stored. ``write`` will - accept in total. + that this ``ShareFile`` will accept to be stored. :param bool create: If ``True``, create the file (and fail if it exists already). ``max_size`` must not be ``None`` in this case. From bb5b26638de4729254b6febb2549a08cd82471e7 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 20 Oct 2021 14:20:53 -0400 Subject: [PATCH 0298/2309] news fragment --- newsfragments/LFS-01-005.security | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 newsfragments/LFS-01-005.security diff --git a/newsfragments/LFS-01-005.security b/newsfragments/LFS-01-005.security new file mode 100644 index 000000000..135b2487c --- /dev/null +++ b/newsfragments/LFS-01-005.security @@ -0,0 +1,3 @@ +The storage server implementation now respects the ``reserved_space`` configuration value when writing lease information. +Previously, new leases could be created and written to disk even when the storage server had less remaining space than the configured reserve space value. +Now this operation will fail with an exception and the lease will not be created. From c77425693769ecd1ce73fcf064b62ef0eaf29ec6 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 20 Oct 2021 14:25:01 -0400 Subject: [PATCH 0299/2309] Add a test for ``remote_add_lease`` with respect to reserved space --- src/allmydata/interfaces.py | 2 ++ src/allmydata/test/common.py | 37 ++++++++++++++++++++++++++++ src/allmydata/test/common_storage.py | 33 +++++++++++++++++++++++++ src/allmydata/test/test_storage.py | 35 +++++++++++++++++++++++++- 4 files changed, 106 insertions(+), 1 deletion(-) create mode 100644 src/allmydata/test/common_storage.py diff --git a/src/allmydata/interfaces.py b/src/allmydata/interfaces.py index 5522663ee..f055a01e2 100644 --- a/src/allmydata/interfaces.py +++ b/src/allmydata/interfaces.py @@ -52,6 +52,8 @@ WriteEnablerSecret = Hash # used to protect mutable share modifications LeaseRenewSecret = Hash # used to protect lease renewal requests LeaseCancelSecret = Hash # was used to protect lease cancellation requests +class NoSpace(Exception): + """Storage space was not available for a space-allocating operation.""" class DataTooLargeError(Exception): """The write went past the expected size of the bucket.""" diff --git a/src/allmydata/test/common.py b/src/allmydata/test/common.py index 0f2dc7c62..38282297a 100644 --- a/src/allmydata/test/common.py +++ b/src/allmydata/test/common.py @@ -87,6 +87,7 @@ from allmydata.interfaces import ( SDMF_VERSION, MDMF_VERSION, IAddressFamily, + NoSpace, ) from allmydata.check_results import CheckResults, CheckAndRepairResults, \ DeepCheckResults, DeepCheckAndRepairResults @@ -139,6 +140,42 @@ EMPTY_CLIENT_CONFIG = config_from_string( "" ) +@attr.s +class FakeDisk(object): + """ + Just enough of a disk to be able to report free / used information. + """ + total = attr.ib() + used = attr.ib() + + def use(self, num_bytes): + """ + Mark some amount of available bytes as used (and no longer available). + + :param int num_bytes: The number of bytes to use. + + :raise NoSpace: If there are fewer bytes available than ``num_bytes``. + + :return: ``None`` + """ + if num_bytes > self.total - self.used: + raise NoSpace() + self.used += num_bytes + + @property + def available(self): + return self.total - self.used + + def get_disk_stats(self, whichdir, reserved_space): + avail = self.available + return { + 'total': self.total, + 'free_for_root': avail, + 'free_for_nonroot': avail, + 'used': self.used, + 'avail': avail - reserved_space, + } + @attr.s class MemoryIntroducerClient(object): diff --git a/src/allmydata/test/common_storage.py b/src/allmydata/test/common_storage.py new file mode 100644 index 000000000..f020a8146 --- /dev/null +++ b/src/allmydata/test/common_storage.py @@ -0,0 +1,33 @@ + +from .common_util import ( + FakeCanary, +) + +def upload_immutable(storage_server, storage_index, renew_secret, cancel_secret, shares): + """ + Synchronously upload some immutable shares to a ``StorageServer``. + + :param allmydata.storage.server.StorageServer storage_server: The storage + server object to use to perform the upload. + + :param bytes storage_index: The storage index for the immutable shares. + + :param bytes renew_secret: The renew secret for the implicitly created lease. + :param bytes cancel_secret: The cancel secret for the implicitly created lease. + + :param dict[int, bytes] shares: A mapping from share numbers to share data + to upload. The data for all shares must be of the same length. + + :return: ``None`` + """ + already, writers = storage_server.remote_allocate_buckets( + storage_index, + renew_secret, + cancel_secret, + shares.keys(), + len(next(iter(shares.values()))), + canary=FakeCanary(), + ) + for shnum, writer in writers.items(): + writer.remote_write(0, shares[shnum]) + writer.remote_close() diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index f19073f3e..67d690047 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -60,8 +60,15 @@ from allmydata.test.no_network import NoNetworkServer from allmydata.storage_client import ( _StorageServer, ) -from .common import LoggingServiceParent, ShouldFailMixin +from .common import ( + LoggingServiceParent, + ShouldFailMixin, + FakeDisk, +) from .common_util import FakeCanary +from .common_storage import ( + upload_immutable, +) from .strategies import ( offsets, lengths, @@ -651,6 +658,32 @@ class Server(unittest.TestCase): self.failUnlessEqual(already, set()) self.failUnlessEqual(set(writers.keys()), set([0,1,2])) + def test_reserved_space_immutable_lease(self): + """ + If there is not enough available space to store an additional lease then + ``remote_add_lease`` fails with ``NoSpace`` when an attempt is made to + use it to create a new lease. + """ + disk = FakeDisk(total=1024, used=0) + self.patch(fileutil, "get_disk_stats", disk.get_disk_stats) + + ss = self.create("test_reserved_space_immutable_lease") + + storage_index = b"x" * 16 + renew_secret = b"r" * 32 + cancel_secret = b"c" * 32 + shares = {0: b"y" * 500} + upload_immutable(ss, storage_index, renew_secret, cancel_secret, shares) + + # use up all the available space + disk.use(disk.available) + + # Different secrets to produce a different lease, not a renewal. + renew_secret = b"R" * 32 + cancel_secret = b"C" * 32 + with self.assertRaises(interfaces.NoSpace): + ss.remote_add_lease(storage_index, renew_secret, cancel_secret) + def test_reserved_space(self): reserved = 10000 allocated = 0 From b3aa1e224f226fb09fdc38312c189369a7aa8847 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 20 Oct 2021 14:27:27 -0400 Subject: [PATCH 0300/2309] Add a helper to LeaseInfo for computing size This lets some code LBYL and avoid writing if the lease won't fit in the immutable share in the space available. --- src/allmydata/storage/lease.py | 14 ++++++++++++-- src/allmydata/test/test_storage.py | 23 +++++++++++++++++++++++ 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/src/allmydata/storage/lease.py b/src/allmydata/storage/lease.py index 187f32406..d3b3eef88 100644 --- a/src/allmydata/storage/lease.py +++ b/src/allmydata/storage/lease.py @@ -13,6 +13,9 @@ if PY2: import struct, time +# struct format for representation of a lease in an immutable share +IMMUTABLE_FORMAT = ">L32s32sL" + class LeaseInfo(object): def __init__(self, owner_num=None, renew_secret=None, cancel_secret=None, expiration_time=None, nodeid=None): @@ -39,12 +42,19 @@ class LeaseInfo(object): (self.owner_num, self.renew_secret, self.cancel_secret, - self.expiration_time) = struct.unpack(">L32s32sL", data) + self.expiration_time) = struct.unpack(IMMUTABLE_FORMAT, data) self.nodeid = None return self + def immutable_size(self): + """ + :return int: The size, in bytes, of the representation of this lease in an + immutable share file. + """ + return struct.calcsize(IMMUTABLE_FORMAT) + def to_immutable_data(self): - return struct.pack(">L32s32sL", + return struct.pack(IMMUTABLE_FORMAT, self.owner_num, self.renew_secret, self.cancel_secret, int(self.expiration_time)) diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index 67d690047..329953e99 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -117,6 +117,29 @@ class FakeStatsProvider(object): def register_producer(self, producer): pass + +class LeaseInfoTests(unittest.TestCase): + """ + Tests for ``LeaseInfo``. + """ + @given( + strategies.tuples( + strategies.integers(min_value=0, max_value=2 ** 31 - 1), + strategies.binary(min_size=32, max_size=32), + strategies.binary(min_size=32, max_size=32), + strategies.integers(min_value=0, max_value=2 ** 31 - 1), + strategies.binary(min_size=20, max_size=20), + ), + ) + def test_immutable_size(self, initializer_args): + """ + ``LeaseInfo.immutable_size`` returns the length of the result of + ``LeaseInfo.to_immutable_data``. + """ + info = LeaseInfo(*initializer_args) + self.assertEqual(len(info.to_immutable_data()), info.immutable_size()) + + class Bucket(unittest.TestCase): def make_workdir(self, name): basedir = os.path.join("storage", "Bucket", name) From 1264c3be1e225e0573aa1e5b30ffa52f5af2d3be Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 20 Oct 2021 14:35:13 -0400 Subject: [PATCH 0301/2309] Use `_add_or_renew_leases` helper consistently in StorageServer This will make it easier to add a new argument to the underlying `add_or_renew_lease` call. --- src/allmydata/storage/server.py | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/src/allmydata/storage/server.py b/src/allmydata/storage/server.py index 041783a4e..21c612a59 100644 --- a/src/allmydata/storage/server.py +++ b/src/allmydata/storage/server.py @@ -286,7 +286,7 @@ class StorageServer(service.MultiService, Referenceable): # to a particular owner. start = self._get_current_time() self.count("allocate") - alreadygot = set() + alreadygot = {} bucketwriters = {} # k: shnum, v: BucketWriter si_dir = storage_index_to_dir(storage_index) si_s = si_b2a(storage_index) @@ -318,9 +318,8 @@ class StorageServer(service.MultiService, Referenceable): # leases for all of them: if they want us to hold shares for this # file, they'll want us to hold leases for this file. for (shnum, fn) in self._get_bucket_shares(storage_index): - alreadygot.add(shnum) - sf = ShareFile(fn) - sf.add_or_renew_lease(lease_info) + alreadygot[shnum] = ShareFile(fn) + self._add_or_renew_leases(alreadygot.values(), lease_info) for shnum in sharenums: incominghome = os.path.join(self.incomingdir, si_dir, "%d" % shnum) @@ -352,7 +351,7 @@ class StorageServer(service.MultiService, Referenceable): fileutil.make_dirs(os.path.join(self.sharedir, si_dir)) self.add_latency("allocate", self._get_current_time() - start) - return alreadygot, bucketwriters + return set(alreadygot), bucketwriters def remote_allocate_buckets(self, storage_index, renew_secret, cancel_secret, @@ -392,8 +391,10 @@ class StorageServer(service.MultiService, Referenceable): lease_info = LeaseInfo(owner_num, renew_secret, cancel_secret, new_expire_time, self.my_nodeid) - for sf in self._iter_share_files(storage_index): - sf.add_or_renew_lease(lease_info) + self._add_or_renew_leases( + self._iter_share_files(storage_index), + lease_info, + ) self.add_latency("add-lease", self._get_current_time() - start) return None @@ -611,12 +612,12 @@ class StorageServer(service.MultiService, Referenceable): """ Put the given lease onto the given shares. - :param dict[int, MutableShareFile] shares: The shares to put the lease - onto. + :param Iterable[Union[MutableShareFile, ShareFile]] shares: The shares + to put the lease onto. :param LeaseInfo lease_info: The lease to put on the shares. """ - for share in six.viewvalues(shares): + for share in shares: share.add_or_renew_lease(lease_info) def slot_testv_and_readv_and_writev( # type: ignore # warner/foolscap#78 @@ -675,7 +676,7 @@ class StorageServer(service.MultiService, Referenceable): ) if renew_leases: lease_info = self._make_lease_info(renew_secret, cancel_secret) - self._add_or_renew_leases(remaining_shares, lease_info) + self._add_or_renew_leases(remaining_shares.values(), lease_info) # all done self.add_latency("writev", self._get_current_time() - start) From 4defc641a2da2b20898f15eb1c9234dcc1cbeb38 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 20 Oct 2021 14:36:05 -0400 Subject: [PATCH 0302/2309] Have ShareFile only write a new lease if there is room for it StorageServer passes available space down so it can make the decision. ShareFile has to do it because `add_or_renew_lease` only *sometimes* adds a lease and only ShareFile knows when that is. --- src/allmydata/storage/immutable.py | 20 ++++++++++++++++++-- src/allmydata/storage/server.py | 2 +- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/src/allmydata/storage/immutable.py b/src/allmydata/storage/immutable.py index 55bcdda64..ad2d19f5f 100644 --- a/src/allmydata/storage/immutable.py +++ b/src/allmydata/storage/immutable.py @@ -21,6 +21,7 @@ from zope.interface import implementer from allmydata.interfaces import ( RIBucketWriter, RIBucketReader, ConflictingWriteError, DataTooLargeError, + NoSpace, ) from allmydata.util import base32, fileutil, log from allmydata.util.assertutil import precondition @@ -249,14 +250,29 @@ class ShareFile(object): return raise IndexError("unable to renew non-existent lease") - def add_or_renew_lease(self, lease_info): + def add_or_renew_lease(self, available_space, lease_info): + """ + Renew an existing lease if possible, otherwise allocate a new one. + + :param int available_space: The maximum number of bytes of storage to + commit in this operation. If more than this number of bytes is + required, raise ``NoSpace`` instead. + + :param LeaseInfo lease_info: The details of the lease to renew or add. + + :raise NoSpace: If more than ``available_space`` bytes is required to + complete the operation. In this case, no lease is added. + + :return: ``None`` + """ try: self.renew_lease(lease_info.renew_secret, lease_info.expiration_time) except IndexError: + if lease_info.immutable_size() > available_space: + raise NoSpace() self.add_lease(lease_info) - def cancel_lease(self, cancel_secret): """Remove a lease with the given cancel_secret. If the last lease is cancelled, the file will be removed. Return the number of bytes that diff --git a/src/allmydata/storage/server.py b/src/allmydata/storage/server.py index 21c612a59..66d9df998 100644 --- a/src/allmydata/storage/server.py +++ b/src/allmydata/storage/server.py @@ -618,7 +618,7 @@ class StorageServer(service.MultiService, Referenceable): :param LeaseInfo lease_info: The lease to put on the shares. """ for share in shares: - share.add_or_renew_lease(lease_info) + share.add_or_renew_lease(self.get_available_space(), lease_info) def slot_testv_and_readv_and_writev( # type: ignore # warner/foolscap#78 self, From e0ed04c1033f995aa5cf90f829b63d127cd290af Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 21 Oct 2021 14:27:20 -0400 Subject: [PATCH 0303/2309] use SyncTestCase to get `expectThat` --- src/allmydata/test/test_storage.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index 329953e99..5b5cfa89d 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -25,6 +25,10 @@ import shutil from functools import partial from uuid import uuid4 +from testtools.matchers import ( + HasLength, +) + from twisted.trial import unittest from twisted.internet import defer @@ -64,6 +68,7 @@ from .common import ( LoggingServiceParent, ShouldFailMixin, FakeDisk, + SyncTestCase, ) from .common_util import FakeCanary from .common_storage import ( @@ -118,7 +123,7 @@ class FakeStatsProvider(object): pass -class LeaseInfoTests(unittest.TestCase): +class LeaseInfoTests(SyncTestCase): """ Tests for ``LeaseInfo``. """ @@ -137,7 +142,10 @@ class LeaseInfoTests(unittest.TestCase): ``LeaseInfo.to_immutable_data``. """ info = LeaseInfo(*initializer_args) - self.assertEqual(len(info.to_immutable_data()), info.immutable_size()) + self.expectThat( + info.to_immutable_data(), + HasLength(info.immutable_size()), + ) class Bucket(unittest.TestCase): From dd1ab2afe8299f8b96651112e4117ffb267ad054 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 21 Oct 2021 14:27:45 -0400 Subject: [PATCH 0304/2309] Add a helper to compute the size of a lease in a mutable share --- src/allmydata/storage/lease.py | 14 ++++++++++++-- src/allmydata/test/test_storage.py | 7 +++++++ 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/src/allmydata/storage/lease.py b/src/allmydata/storage/lease.py index d3b3eef88..3453c1ecc 100644 --- a/src/allmydata/storage/lease.py +++ b/src/allmydata/storage/lease.py @@ -16,6 +16,9 @@ import struct, time # struct format for representation of a lease in an immutable share IMMUTABLE_FORMAT = ">L32s32sL" +# struct format for representation of a lease in a mutable share +MUTABLE_FORMAT = ">LL32s32s20s" + class LeaseInfo(object): def __init__(self, owner_num=None, renew_secret=None, cancel_secret=None, expiration_time=None, nodeid=None): @@ -53,6 +56,13 @@ class LeaseInfo(object): """ return struct.calcsize(IMMUTABLE_FORMAT) + def mutable_size(self): + """ + :return int: The size, in bytes, of the representation of this lease in a + mutable share file. + """ + return struct.calcsize(MUTABLE_FORMAT) + def to_immutable_data(self): return struct.pack(IMMUTABLE_FORMAT, self.owner_num, @@ -60,7 +70,7 @@ class LeaseInfo(object): int(self.expiration_time)) def to_mutable_data(self): - return struct.pack(">LL32s32s20s", + return struct.pack(MUTABLE_FORMAT, self.owner_num, int(self.expiration_time), self.renew_secret, self.cancel_secret, @@ -70,5 +80,5 @@ class LeaseInfo(object): (self.owner_num, self.expiration_time, self.renew_secret, self.cancel_secret, - self.nodeid) = struct.unpack(">LL32s32s20s", data) + self.nodeid) = struct.unpack(MUTABLE_FORMAT, data) return self diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index 5b5cfa89d..9ce6482ea 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -140,12 +140,19 @@ class LeaseInfoTests(SyncTestCase): """ ``LeaseInfo.immutable_size`` returns the length of the result of ``LeaseInfo.to_immutable_data``. + + ``LeaseInfo.mutable_size`` returns the length of the result of + ``LeaseInfo.to_mutable_data``. """ info = LeaseInfo(*initializer_args) self.expectThat( info.to_immutable_data(), HasLength(info.immutable_size()), ) + self.expectThat( + info.to_mutable_data(), + HasLength(info.mutable_size()), + ) class Bucket(unittest.TestCase): From f789339a79c995d617e09010563e6f418e815067 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 21 Oct 2021 15:16:56 -0400 Subject: [PATCH 0305/2309] Have MutableShare file only write a new lease if there is room for it This is analagous to the earlier ShareFile change. --- src/allmydata/storage/mutable.py | 25 ++++++++++++--- src/allmydata/test/common_storage.py | 32 +++++++++++++++++++ src/allmydata/test/test_storage.py | 47 ++++++++++++++++++++++++++-- 3 files changed, 97 insertions(+), 7 deletions(-) diff --git a/src/allmydata/storage/mutable.py b/src/allmydata/storage/mutable.py index cdb4faeaf..74c0d1051 100644 --- a/src/allmydata/storage/mutable.py +++ b/src/allmydata/storage/mutable.py @@ -13,7 +13,10 @@ if PY2: import os, stat, struct -from allmydata.interfaces import BadWriteEnablerError +from allmydata.interfaces import ( + BadWriteEnablerError, + NoSpace, +) from allmydata.util import idlib, log from allmydata.util.assertutil import precondition from allmydata.util.hashutil import timing_safe_compare @@ -289,7 +292,19 @@ class MutableShareFile(object): except IndexError: return - def add_lease(self, lease_info): + def add_lease(self, available_space, lease_info): + """ + Add a new lease to this share. + + :param int available_space: The maximum number of bytes of storage to + commit in this operation. If more than this number of bytes is + required, raise ``NoSpace`` instead. + + :raise NoSpace: If more than ``available_space`` bytes is required to + complete the operation. In this case, no lease is added. + + :return: ``None`` + """ precondition(lease_info.owner_num != 0) # 0 means "no lease here" with open(self.home, 'rb+') as f: num_lease_slots = self._get_num_lease_slots(f) @@ -297,6 +312,8 @@ class MutableShareFile(object): if empty_slot is not None: self._write_lease_record(f, empty_slot, lease_info) else: + if lease_info.mutable_size() > available_space: + raise NoSpace() self._write_lease_record(f, num_lease_slots, lease_info) def renew_lease(self, renew_secret, new_expire_time): @@ -321,13 +338,13 @@ class MutableShareFile(object): msg += " ." raise IndexError(msg) - def add_or_renew_lease(self, lease_info): + def add_or_renew_lease(self, available_space, lease_info): precondition(lease_info.owner_num != 0) # 0 means "no lease here" try: self.renew_lease(lease_info.renew_secret, lease_info.expiration_time) except IndexError: - self.add_lease(lease_info) + self.add_lease(available_space, lease_info) def cancel_lease(self, cancel_secret): """Remove any leases with the given cancel_secret. If the last lease diff --git a/src/allmydata/test/common_storage.py b/src/allmydata/test/common_storage.py index f020a8146..529ebe586 100644 --- a/src/allmydata/test/common_storage.py +++ b/src/allmydata/test/common_storage.py @@ -31,3 +31,35 @@ def upload_immutable(storage_server, storage_index, renew_secret, cancel_secret, for shnum, writer in writers.items(): writer.remote_write(0, shares[shnum]) writer.remote_close() + + +def upload_mutable(storage_server, storage_index, secrets, shares): + """ + Synchronously upload some mutable shares to a ``StorageServer``. + + :param allmydata.storage.server.StorageServer storage_server: The storage + server object to use to perform the upload. + + :param bytes storage_index: The storage index for the immutable shares. + + :param secrets: A three-tuple of a write enabler, renew secret, and cancel + secret. + + :param dict[int, bytes] shares: A mapping from share numbers to share data + to upload. + + :return: ``None`` + """ + test_and_write_vectors = { + sharenum: ([], [(0, data)], None) + for sharenum, data + in shares.items() + } + read_vector = [] + + storage_server.remote_slot_testv_and_readv_and_writev( + storage_index, + secrets, + test_and_write_vectors, + read_vector, + ) diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index 9ce6482ea..e03c07203 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -73,6 +73,7 @@ from .common import ( from .common_util import FakeCanary from .common_storage import ( upload_immutable, + upload_mutable, ) from .strategies import ( offsets, @@ -698,9 +699,9 @@ class Server(unittest.TestCase): def test_reserved_space_immutable_lease(self): """ - If there is not enough available space to store an additional lease then - ``remote_add_lease`` fails with ``NoSpace`` when an attempt is made to - use it to create a new lease. + If there is not enough available space to store an additional lease on an + immutable share then ``remote_add_lease`` fails with ``NoSpace`` when + an attempt is made to use it to create a new lease. """ disk = FakeDisk(total=1024, used=0) self.patch(fileutil, "get_disk_stats", disk.get_disk_stats) @@ -722,6 +723,46 @@ class Server(unittest.TestCase): with self.assertRaises(interfaces.NoSpace): ss.remote_add_lease(storage_index, renew_secret, cancel_secret) + def test_reserved_space_mutable_lease(self): + """ + If there is not enough available space to store an additional lease on a + mutable share then ``remote_add_lease`` fails with ``NoSpace`` when an + attempt is made to use it to create a new lease. + """ + disk = FakeDisk(total=1024, used=0) + self.patch(fileutil, "get_disk_stats", disk.get_disk_stats) + + ss = self.create("test_reserved_space_mutable_lease") + + renew_secrets = iter( + "{}{}".format("r" * 31, i).encode("ascii") + for i + in range(5) + ) + + storage_index = b"x" * 16 + write_enabler = b"w" * 32 + cancel_secret = b"c" * 32 + secrets = (write_enabler, next(renew_secrets), cancel_secret) + shares = {0: b"y" * 500} + upload_mutable(ss, storage_index, secrets, shares) + + # use up all the available space + disk.use(disk.available) + + # The upload created one lease. There is room for three more leases + # in the share header. Even if we're out of disk space, on a boring + # enough filesystem we can write these. + for i in range(3): + ss.remote_add_lease(storage_index, next(renew_secrets), cancel_secret) + + # Having used all of the space for leases in the header, we would have + # to allocate storage for the next lease. Since there is no space + # available, this must fail instead. + with self.assertRaises(interfaces.NoSpace): + ss.remote_add_lease(storage_index, next(renew_secrets), cancel_secret) + + def test_reserved_space(self): reserved = 10000 allocated = 0 From 6449ad03de20db407dc96ba2a6651b9d80ff797a Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 22 Oct 2021 13:38:37 -0400 Subject: [PATCH 0306/2309] Do not record corruption advisories if there is no available space --- src/allmydata/storage/server.py | 86 ++++++++++++++++++++++++------ src/allmydata/test/test_storage.py | 21 ++++++++ 2 files changed, 92 insertions(+), 15 deletions(-) diff --git a/src/allmydata/storage/server.py b/src/allmydata/storage/server.py index 66d9df998..3ee494786 100644 --- a/src/allmydata/storage/server.py +++ b/src/allmydata/storage/server.py @@ -737,24 +737,80 @@ class StorageServer(service.MultiService, Referenceable): # protocol backwards compatibility reasons. assert isinstance(share_type, bytes) assert isinstance(reason, bytes), "%r is not bytes" % (reason,) - fileutil.make_dirs(self.corruption_advisory_dir) - now = time_format.iso_utc(sep="T") + si_s = si_b2a(storage_index) - # windows can't handle colons in the filename - fn = os.path.join( - self.corruption_advisory_dir, - ("%s--%s-%d" % (now, str(si_s, "utf-8"), shnum)).replace(":","") - ) - with open(fn, "w") as f: - f.write("report: Share Corruption\n") - f.write("type: %s\n" % bytes_to_native_str(share_type)) - f.write("storage_index: %s\n" % bytes_to_native_str(si_s)) - f.write("share_number: %d\n" % shnum) - f.write("\n") - f.write(bytes_to_native_str(reason)) - f.write("\n") + log.msg(format=("client claims corruption in (%(share_type)s) " + "%(si)s-%(shnum)d: %(reason)s"), share_type=share_type, si=si_s, shnum=shnum, reason=reason, level=log.SCARY, umid="SGx2fA") + + fileutil.make_dirs(self.corruption_advisory_dir) + now = time_format.iso_utc(sep="T") + + report = render_corruption_report(share_type, si_s, shnum, reason) + if len(report) > self.get_available_space(): + return None + + report_path = get_corruption_report_path( + self.corruption_advisory_dir, + now, + si_s, + shnum, + ) + with open(report_path, "w") as f: + f.write(report) + return None + +CORRUPTION_REPORT_FORMAT = """\ +report: Share Corruption +type: {type} +storage_index: {storage_index} +share_number: {share_number} + +{reason} + +""" + +def render_corruption_report(share_type, si_s, shnum, reason): + """ + Create a string that explains a corruption report using freeform text. + + :param bytes share_type: The type of the share which the report is about. + + :param bytes si_s: The encoded representation of the storage index which + the report is about. + + :param int shnum: The share number which the report is about. + + :param bytes reason: The reason given by the client for the corruption + report. + """ + return CORRUPTION_REPORT_FORMAT.format( + type=bytes_to_native_str(share_type), + storage_index=bytes_to_native_str(si_s), + share_number=shnum, + reason=bytes_to_native_str(reason), + ) + +def get_corruption_report_path(base_dir, now, si_s, shnum): + """ + Determine the path to which a certain corruption report should be written. + + :param str base_dir: The directory beneath which to construct the path. + + :param str now: The time of the report. + + :param str si_s: The encoded representation of the storage index which the + report is about. + + :param int shnum: The share number which the report is about. + + :return str: A path to which the report can be written. + """ + # windows can't handle colons in the filename + return os.path.join( + base_dir, + ("%s--%s-%d" % (now, str(si_s, "utf-8"), shnum)).replace(":","") + ) diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index e03c07203..314069ce2 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -1006,6 +1006,27 @@ class Server(unittest.TestCase): self.failUnlessEqual(set(b.keys()), set([0,1,2])) self.failUnlessEqual(b[0].remote_read(0, 25), b"\x00" * 25) + def test_reserved_space_advise_corruption(self): + """ + If there is no available space then ``remote_advise_corrupt_share`` does + not write a corruption report. + """ + disk = FakeDisk(total=1024, used=1024) + self.patch(fileutil, "get_disk_stats", disk.get_disk_stats) + + workdir = self.workdir("test_reserved_space_advise_corruption") + ss = StorageServer(workdir, b"\x00" * 20, discard_storage=True) + ss.setServiceParent(self.sparent) + + si0_s = base32.b2a(b"si0") + ss.remote_advise_corrupt_share(b"immutable", b"si0", 0, + b"This share smells funny.\n") + + self.assertEqual( + [], + os.listdir(ss.corruption_advisory_dir), + ) + def test_advise_corruption(self): workdir = self.workdir("test_advise_corruption") ss = StorageServer(workdir, b"\x00" * 20, discard_storage=True) From 5837841c090d110e1ec772f0aed137642a7d6aaa Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 22 Oct 2021 14:15:47 -0400 Subject: [PATCH 0307/2309] mention corruption advisories in the news fragment too --- newsfragments/LFS-01-005.security | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/newsfragments/LFS-01-005.security b/newsfragments/LFS-01-005.security index 135b2487c..ba2bbd741 100644 --- a/newsfragments/LFS-01-005.security +++ b/newsfragments/LFS-01-005.security @@ -1,3 +1,4 @@ -The storage server implementation now respects the ``reserved_space`` configuration value when writing lease information. +The storage server implementation now respects the ``reserved_space`` configuration value when writing lease information and recording corruption advisories. Previously, new leases could be created and written to disk even when the storage server had less remaining space than the configured reserve space value. Now this operation will fail with an exception and the lease will not be created. +Similarly, if there is no space available, corruption advisories will be logged but not written to disk. From 8d15d61ff2600b3a4f560e3b55f14a13bc3138e5 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 22 Oct 2021 15:58:48 -0400 Subject: [PATCH 0308/2309] put the news fragment in the right place --- newsfragments/{LFS-01-005.security => 3823.security} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename newsfragments/{LFS-01-005.security => 3823.security} (100%) diff --git a/newsfragments/LFS-01-005.security b/newsfragments/3823.security similarity index 100% rename from newsfragments/LFS-01-005.security rename to newsfragments/3823.security From 194499aafe42399185cbc4185fa078f09adfb608 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 22 Oct 2021 16:09:54 -0400 Subject: [PATCH 0309/2309] remove unused import --- src/allmydata/storage/server.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/allmydata/storage/server.py b/src/allmydata/storage/server.py index 3ee494786..30fa5adc2 100644 --- a/src/allmydata/storage/server.py +++ b/src/allmydata/storage/server.py @@ -15,7 +15,6 @@ else: from typing import Dict import os, re, struct, time -import six from foolscap.api import Referenceable from foolscap.ipb import IRemoteReference From cb675df48d08f4f0a42061d9261a3f5d47ac1673 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 22 Oct 2021 16:10:24 -0400 Subject: [PATCH 0310/2309] remove unused encoding of storage index --- src/allmydata/test/test_storage.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index 314069ce2..70cad7db2 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -1018,7 +1018,6 @@ class Server(unittest.TestCase): ss = StorageServer(workdir, b"\x00" * 20, discard_storage=True) ss.setServiceParent(self.sparent) - si0_s = base32.b2a(b"si0") ss.remote_advise_corrupt_share(b"immutable", b"si0", 0, b"This share smells funny.\n") From ea202ba61b90545ab78127f3340a8bb4bac18612 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 22 Oct 2021 14:51:37 -0400 Subject: [PATCH 0311/2309] news fragment --- newsfragments/3824.security | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/3824.security diff --git a/newsfragments/3824.security b/newsfragments/3824.security new file mode 100644 index 000000000..b29b2acc8 --- /dev/null +++ b/newsfragments/3824.security @@ -0,0 +1 @@ +The storage server implementation no longer records corruption advisories about storage indexes for which it holds no shares. From 470657b337ca199418aee6777866281769d8f38c Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 22 Oct 2021 14:56:09 -0400 Subject: [PATCH 0312/2309] Drop corruption advisories if we don't have a matching share --- src/allmydata/storage/server.py | 32 ++++++++++++++++++++++++++---- src/allmydata/test/test_storage.py | 23 +++++++++++++++++++++ 2 files changed, 51 insertions(+), 4 deletions(-) diff --git a/src/allmydata/storage/server.py b/src/allmydata/storage/server.py index 30fa5adc2..c81d88bfc 100644 --- a/src/allmydata/storage/server.py +++ b/src/allmydata/storage/server.py @@ -77,9 +77,9 @@ class StorageServer(service.MultiService, Referenceable): sharedir = os.path.join(storedir, "shares") fileutil.make_dirs(sharedir) self.sharedir = sharedir - # we don't actually create the corruption-advisory dir until necessary self.corruption_advisory_dir = os.path.join(storedir, "corruption-advisories") + fileutil.make_dirs(self.corruption_advisory_dir) self.reserved_space = int(reserved_space) self.no_storage = discard_storage self.readonly_storage = readonly_storage @@ -730,6 +730,21 @@ class StorageServer(service.MultiService, Referenceable): self.add_latency("readv", self._get_current_time() - start) return datavs + def _share_exists(self, storage_index, shnum): + """ + Check local share storage to see if a matching share exists. + + :param bytes storage_index: The storage index to inspect. + :param int shnum: The share number to check for. + + :return bool: ``True`` if a share with the given number exists at the + given storage index, ``False`` otherwise. + """ + for existing_sharenum, ignored in self._get_bucket_shares(storage_index): + if existing_sharenum == shnum: + return True + return False + def remote_advise_corrupt_share(self, share_type, storage_index, shnum, reason): # This is a remote API, I believe, so this has to be bytes for legacy @@ -739,18 +754,27 @@ class StorageServer(service.MultiService, Referenceable): si_s = si_b2a(storage_index) + if not self._share_exists(storage_index, shnum): + log.msg( + format=( + "discarding client corruption claim for %(si)s/%(shnum)d " + "which I do not have" + ), + si=si_s, + shnum=shnum, + ) + return + log.msg(format=("client claims corruption in (%(share_type)s) " + "%(si)s-%(shnum)d: %(reason)s"), share_type=share_type, si=si_s, shnum=shnum, reason=reason, level=log.SCARY, umid="SGx2fA") - fileutil.make_dirs(self.corruption_advisory_dir) - now = time_format.iso_utc(sep="T") - report = render_corruption_report(share_type, si_s, shnum, reason) if len(report) > self.get_available_space(): return None + now = time_format.iso_utc(sep="T") report_path = get_corruption_report_path( self.corruption_advisory_dir, now, diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index 70cad7db2..9889a001a 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -1018,6 +1018,7 @@ class Server(unittest.TestCase): ss = StorageServer(workdir, b"\x00" * 20, discard_storage=True) ss.setServiceParent(self.sparent) + upload_immutable(ss, "si0", b"r" * 32, b"c" * 32, {0: b""}) ss.remote_advise_corrupt_share(b"immutable", b"si0", 0, b"This share smells funny.\n") @@ -1032,6 +1033,7 @@ class Server(unittest.TestCase): ss.setServiceParent(self.sparent) si0_s = base32.b2a(b"si0") + upload_immutable(ss, "si0", b"r" * 32, b"c" * 32, {0: b""}) ss.remote_advise_corrupt_share(b"immutable", b"si0", 0, b"This share smells funny.\n") reportdir = os.path.join(workdir, "corruption-advisories") @@ -1070,6 +1072,27 @@ class Server(unittest.TestCase): self.failUnlessIn(b"share_number: 1", report) self.failUnlessIn(b"This share tastes like dust.", report) + def test_advise_corruption_missing(self): + """ + If a corruption advisory is received for a share that is not present on + this server then it is not persisted. + """ + workdir = self.workdir("test_advise_corruption_missing") + ss = StorageServer(workdir, b"\x00" * 20, discard_storage=True) + ss.setServiceParent(self.sparent) + + # Upload one share for this storage index + upload_immutable(ss, "si0", b"r" * 32, b"c" * 32, {0: b""}) + + # And try to submit a corruption advisory about a different share + si0_s = base32.b2a(b"si0") + ss.remote_advise_corrupt_share(b"immutable", b"si0", 1, + b"This share smells funny.\n") + + self.assertEqual( + [], + os.listdir(ss.corruption_advisory_dir), + ) class MutableServer(unittest.TestCase): From 0ada9d93f794efed68d185b8f68bcb31f6394ee0 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Sat, 23 Oct 2021 07:43:22 -0400 Subject: [PATCH 0313/2309] remove unused local --- src/allmydata/test/test_storage.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index 9889a001a..06b8d7957 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -1085,7 +1085,6 @@ class Server(unittest.TestCase): upload_immutable(ss, "si0", b"r" * 32, b"c" * 32, {0: b""}) # And try to submit a corruption advisory about a different share - si0_s = base32.b2a(b"si0") ss.remote_advise_corrupt_share(b"immutable", b"si0", 1, b"This share smells funny.\n") From b51f0ac8ff60a39b34c51d5f8b4c4b7aad232c37 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Sat, 23 Oct 2021 08:04:19 -0400 Subject: [PATCH 0314/2309] storage_index is a byte string and Python 3 cares --- src/allmydata/test/test_storage.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index 06b8d7957..738e218eb 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -1018,7 +1018,7 @@ class Server(unittest.TestCase): ss = StorageServer(workdir, b"\x00" * 20, discard_storage=True) ss.setServiceParent(self.sparent) - upload_immutable(ss, "si0", b"r" * 32, b"c" * 32, {0: b""}) + upload_immutable(ss, b"si0", b"r" * 32, b"c" * 32, {0: b""}) ss.remote_advise_corrupt_share(b"immutable", b"si0", 0, b"This share smells funny.\n") @@ -1033,7 +1033,7 @@ class Server(unittest.TestCase): ss.setServiceParent(self.sparent) si0_s = base32.b2a(b"si0") - upload_immutable(ss, "si0", b"r" * 32, b"c" * 32, {0: b""}) + upload_immutable(ss, b"si0", b"r" * 32, b"c" * 32, {0: b""}) ss.remote_advise_corrupt_share(b"immutable", b"si0", 0, b"This share smells funny.\n") reportdir = os.path.join(workdir, "corruption-advisories") @@ -1082,7 +1082,7 @@ class Server(unittest.TestCase): ss.setServiceParent(self.sparent) # Upload one share for this storage index - upload_immutable(ss, "si0", b"r" * 32, b"c" * 32, {0: b""}) + upload_immutable(ss, b"si0", b"r" * 32, b"c" * 32, {0: b""}) # And try to submit a corruption advisory about a different share ss.remote_advise_corrupt_share(b"immutable", b"si0", 1, From 0b4e6754a34ee9ba8d7d71f6f16e7e29f4fd8ec8 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 25 Oct 2021 20:47:35 -0400 Subject: [PATCH 0315/2309] news fragment --- newsfragments/3827.security | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 newsfragments/3827.security diff --git a/newsfragments/3827.security b/newsfragments/3827.security new file mode 100644 index 000000000..4fee19c76 --- /dev/null +++ b/newsfragments/3827.security @@ -0,0 +1,4 @@ +The SFTP server no longer accepts password-based credentials for authentication. +Public/private key-based credentials are now the only supported authentication type. +This removes plaintext password storage from the SFTP credentials file. +It also removes a possible timing side-channel vulnerability which might have allowed attackers to discover an account's plaintext password. From 5878a64890ba0a395f61432a9b5dd534daa9a64a Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 25 Oct 2021 20:50:19 -0400 Subject: [PATCH 0316/2309] Remove password-based authentication from the SFTP frontend --- docs/frontends/FTP-and-SFTP.rst | 14 +- src/allmydata/frontends/auth.py | 122 ++++++++++------ src/allmydata/test/test_auth.py | 244 +++++++++++++++++++++++--------- 3 files changed, 255 insertions(+), 125 deletions(-) diff --git a/docs/frontends/FTP-and-SFTP.rst b/docs/frontends/FTP-and-SFTP.rst index 9d4f1dcec..ede719e26 100644 --- a/docs/frontends/FTP-and-SFTP.rst +++ b/docs/frontends/FTP-and-SFTP.rst @@ -47,8 +47,8 @@ servers must be configured with a way to first authenticate a user (confirm that a prospective client has a legitimate claim to whatever authorities we might grant a particular user), and second to decide what directory cap should be used as the root directory for a log-in by the authenticated user. -A username and password can be used; as of Tahoe-LAFS v1.11, RSA or DSA -public key authentication is also supported. +As of Tahoe-LAFS v1.17, +RSA/DSA public key authentication is the only supported mechanism. Tahoe-LAFS provides two mechanisms to perform this user-to-cap mapping. The first (recommended) is a simple flat file with one account per line. @@ -59,20 +59,14 @@ Creating an Account File To use the first form, create a file (for example ``BASEDIR/private/accounts``) in which each non-comment/non-blank line is a space-separated line of -(USERNAME, PASSWORD, ROOTCAP), like so:: +(USERNAME, KEY-TYPE, PUBLIC-KEY, ROOTCAP), like so:: % cat BASEDIR/private/accounts - # This is a password line: username password cap - alice password URI:DIR2:ioej8xmzrwilg772gzj4fhdg7a:wtiizszzz2rgmczv4wl6bqvbv33ag4kvbr6prz3u6w3geixa6m6a - bob sekrit URI:DIR2:6bdmeitystckbl9yqlw7g56f4e:serp5ioqxnh34mlbmzwvkp3odehsyrr7eytt5f64we3k9hhcrcja - # This is a public key line: username keytype pubkey cap # (Tahoe-LAFS v1.11 or later) carol ssh-rsa AAAA... URI:DIR2:ovjy4yhylqlfoqg2vcze36dhde:4d4f47qko2xm5g7osgo2yyidi5m4muyo2vjjy53q4vjju2u55mfa -For public key authentication, the keytype may be either "ssh-rsa" or "ssh-dsa". -To avoid ambiguity between passwords and public key types, a password cannot -start with "ssh-". +The key type may be either "ssh-rsa" or "ssh-dsa". Now add an ``accounts.file`` directive to your ``tahoe.cfg`` file, as described in the next sections. diff --git a/src/allmydata/frontends/auth.py b/src/allmydata/frontends/auth.py index b61062334..312a9da1a 100644 --- a/src/allmydata/frontends/auth.py +++ b/src/allmydata/frontends/auth.py @@ -32,65 +32,93 @@ class FTPAvatarID(object): @implementer(checkers.ICredentialsChecker) class AccountFileChecker(object): - credentialInterfaces = (credentials.IUsernamePassword, - credentials.IUsernameHashedPassword, - credentials.ISSHPrivateKey) + credentialInterfaces = (credentials.ISSHPrivateKey,) + def __init__(self, client, accountfile): self.client = client - self.passwords = BytesKeyDict() - pubkeys = BytesKeyDict() - self.rootcaps = BytesKeyDict() - with open(abspath_expanduser_unicode(accountfile), "rb") as f: - for line in f: - line = line.strip() - if line.startswith(b"#") or not line: - continue - name, passwd, rest = line.split(None, 2) - if passwd.startswith(b"ssh-"): - bits = rest.split() - keystring = b" ".join([passwd] + bits[:-1]) - key = keys.Key.fromString(keystring) - rootcap = bits[-1] - pubkeys[name] = [key] - else: - self.passwords[name] = passwd - rootcap = rest - self.rootcaps[name] = rootcap + path = abspath_expanduser_unicode(accountfile) + with open_account_file(path) as f: + self.rootcaps, pubkeys = load_account_file(f) self._pubkeychecker = SSHPublicKeyChecker(InMemorySSHKeyDB(pubkeys)) def _avatarId(self, username): return FTPAvatarID(username, self.rootcaps[username]) - def _cbPasswordMatch(self, matched, username): - if matched: - return self._avatarId(username) - raise error.UnauthorizedLogin - def requestAvatarId(self, creds): if credentials.ISSHPrivateKey.providedBy(creds): d = defer.maybeDeferred(self._pubkeychecker.requestAvatarId, creds) d.addCallback(self._avatarId) return d - elif credentials.IUsernameHashedPassword.providedBy(creds): - return self._checkPassword(creds) - elif credentials.IUsernamePassword.providedBy(creds): - return self._checkPassword(creds) - else: - raise NotImplementedError() + raise NotImplementedError() - def _checkPassword(self, creds): - """ - Determine whether the password in the given credentials matches the - password in the account file. +def open_account_file(path): + """ + Open and return the accounts file at the given path. + """ + return open(path, "rt", encoding="utf-8") - Returns a Deferred that fires with the username if the password matches - or with an UnauthorizedLogin failure otherwise. - """ - try: - correct = self.passwords[creds.username] - except KeyError: - return defer.fail(error.UnauthorizedLogin()) +def load_account_file(lines): + """ + Load credentials from an account file. - d = defer.maybeDeferred(creds.checkPassword, correct) - d.addCallback(self._cbPasswordMatch, creds.username) - return d + :param lines: An iterable of account lines to load. + + :return: See ``create_account_maps``. + """ + return create_account_maps( + parse_accounts( + content_lines( + lines, + ), + ), + ) + +def content_lines(lines): + """ + Drop empty and commented-out lines (``#``-prefixed) from an iterator of + lines. + + :param lines: An iterator of lines to process. + + :return: An iterator of lines including only those from ``lines`` that + include content intended to be loaded. + """ + for line in lines: + line = line.strip() + if line and not line.startswith("#"): + yield line + +def parse_accounts(lines): + """ + Parse account lines into their components (name, key, rootcap). + """ + for line in lines: + name, passwd, rest = line.split(None, 2) + if not passwd.startswith("ssh-"): + raise ValueError( + "Password-based authentication is not supported; " + "configure key-based authentication instead." + ) + + bits = rest.split() + keystring = " ".join([passwd] + bits[:-1]) + key = keys.Key.fromString(keystring) + rootcap = bits[-1] + yield (name, key, rootcap) + +def create_account_maps(accounts): + """ + Build mappings from account names to keys and rootcaps. + + :param accounts: An iterator if (name, key, rootcap) tuples. + + :return: A tuple of two dicts. The first maps account names to rootcaps. + The second maps account names to public keys. + """ + rootcaps = BytesKeyDict() + pubkeys = BytesKeyDict() + for (name, key, rootcap) in accounts: + name_bytes = name.encode("utf-8") + rootcaps[name_bytes] = rootcap.encode("utf-8") + pubkeys[name_bytes] = [key] + return rootcaps, pubkeys diff --git a/src/allmydata/test/test_auth.py b/src/allmydata/test/test_auth.py index d5198d326..19c2f7c01 100644 --- a/src/allmydata/test/test_auth.py +++ b/src/allmydata/test/test_auth.py @@ -8,7 +8,17 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: - from future.builtins import str # noqa: F401 + from future.builtins import str, open # noqa: F401 + +from hypothesis import ( + given, +) +from hypothesis.strategies import ( + text, + characters, + tuples, + lists, +) from twisted.trial import unittest from twisted.python import filepath @@ -38,25 +48,184 @@ dBSD8940XU3YW+oeq8e+p3yQ2GinHfeJ3BYQyNQLuMAJ -----END RSA PRIVATE KEY----- """) -DUMMY_ACCOUNTS = u"""\ -alice herpassword URI:DIR2:aaaaaaaaaaaaaaaaaaaaaaaaaa:1111111111111111111111111111111111111111111111111111 -bob sekrit URI:DIR2:bbbbbbbbbbbbbbbbbbbbbbbbbb:2222222222222222222222222222222222222222222222222222 +DUMMY_KEY_DSA = keys.Key.fromString("""\ +-----BEGIN OPENSSH PRIVATE KEY----- +b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAABsQAAAAdzc2gtZH +NzAAAAgQDKMh/ELaiP21LYRBuPbUy7dUhv/XZwV7aS1LzxSP+KaJvtDOei8X76XEAfkqX+ +aGh9eup+BLkezrV6LlpO9uPzhY8ChlKpkvw5PZKv/2agSrVxZyG7yEzHNtSBQXE6qNMwIk +N/ycXLGCqyAhQSzRhLz9ETNaslRDLo7YyVWkiuAQAAABUA5nTatFKux5EqZS4EarMWFRBU +i1UAAACAFpkkK+JsPixSTPyn0DNMoGKA0Klqy8h61Ds6pws+4+aJQptUBshpwNw1ypo7MO ++goDZy3wwdWtURTPGMgesNdEfxp8L2/kqE4vpMK0myoczCqOiWMeNB/x1AStbSkBI8WmHW +2htgsC01xbaix/FrA3edK8WEyv+oIxlbV1FkrPkAAACANb0EpCc8uoR4/32rO2JLsbcLBw +H5wc2khe7AKkIa9kUknRIRvoCZUtXF5XuXXdRmnpVEm2KcsLdtZjip43asQcqgt0Kz3nuF +kAf7bI98G1waFUimcCSPsal4kCmW2HC11sg/BWOt5qczX/0/3xVxpo6juUeBq9ncnFTvPX +5fOlEAAAHoJkFqHiZBah4AAAAHc3NoLWRzcwAAAIEAyjIfxC2oj9tS2EQbj21Mu3VIb/12 +cFe2ktS88Uj/imib7QznovF++lxAH5Kl/mhofXrqfgS5Hs61ei5aTvbj84WPAoZSqZL8OT +2Sr/9moEq1cWchu8hMxzbUgUFxOqjTMCJDf8nFyxgqsgIUEs0YS8/REzWrJUQy6O2MlVpI +rgEAAAAVAOZ02rRSrseRKmUuBGqzFhUQVItVAAAAgBaZJCvibD4sUkz8p9AzTKBigNCpas +vIetQ7OqcLPuPmiUKbVAbIacDcNcqaOzDvoKA2ct8MHVrVEUzxjIHrDXRH8afC9v5KhOL6 +TCtJsqHMwqjoljHjQf8dQErW0pASPFph1tobYLAtNcW2osfxawN3nSvFhMr/qCMZW1dRZK +z5AAAAgDW9BKQnPLqEeP99qztiS7G3CwcB+cHNpIXuwCpCGvZFJJ0SEb6AmVLVxeV7l13U +Zp6VRJtinLC3bWY4qeN2rEHKoLdCs957hZAH+2yPfBtcGhVIpnAkj7GpeJAplthwtdbIPw +VjreanM1/9P98VcaaOo7lHgavZ3JxU7z1+XzpRAAAAFQC7360pZLbv7PFt4BPFJ8zAHxAe +QwAAAA5leGFya3VuQGJhcnlvbgECAwQ= +-----END OPENSSH PRIVATE KEY----- +""") -# dennis password URI:DIR2:aaaaaaaaaaaaaaaaaaaaaaaaaa:1111111111111111111111111111111111111111111111111111 +ACCOUNTS = u"""\ +# dennis {key} URI:DIR2:aaaaaaaaaaaaaaaaaaaaaaaaaa:1111111111111111111111111111111111111111111111111111 carol {key} URI:DIR2:cccccccccccccccccccccccccc:3333333333333333333333333333333333333333333333333333 """.format(key=str(DUMMY_KEY.public().toString("openssh"), "ascii")).encode("ascii") +# Python str.splitlines considers NEXT LINE, LINE SEPARATOR, and PARAGRAPH +# separator to be line separators, too. However, file.readlines() does not... +LINE_SEPARATORS = ( + '\x0a', # line feed + '\x0b', # vertical tab + '\x0c', # form feed + '\x0d', # carriage return +) + +class AccountFileParserTests(unittest.TestCase): + """ + Tests for ``load_account_file`` and its helper functions. + """ + @given(lists( + text(alphabet=characters( + blacklist_categories=( + # Surrogates are an encoding trick to help out UTF-16. + # They're not necessary to represent any non-surrogate code + # point in unicode. They're also not legal individually but + # only in pairs. + 'Cs', + ), + # Exclude all our line separators too. + blacklist_characters=("\n", "\r"), + )), + )) + def test_ignore_comments(self, lines): + """ + ``auth.content_lines`` filters out lines beginning with `#` and empty + lines. + """ + expected = set() + + # It's not clear that real files and StringIO behave sufficiently + # similarly to use the latter instead of the former here. In + # particular, they seem to have distinct and incompatible + # line-splitting rules. + bufpath = self.mktemp() + with open(bufpath, "wt", encoding="utf-8") as buf: + for line in lines: + stripped = line.strip() + is_content = stripped and not stripped.startswith("#") + if is_content: + expected.add(stripped) + buf.write(line + "\n") + + with auth.open_account_file(bufpath) as buf: + actual = set(auth.content_lines(buf)) + + self.assertEqual(expected, actual) + + def test_parse_accounts(self): + """ + ``auth.parse_accounts`` accepts an iterator of account lines and returns + an iterator of structured account data. + """ + alice_key = DUMMY_KEY.public().toString("openssh").decode("utf-8") + alice_cap = "URI:DIR2:aaaa:1111" + + bob_key = DUMMY_KEY_DSA.public().toString("openssh").decode("utf-8") + bob_cap = "URI:DIR2:aaaa:2222" + self.assertEqual( + list(auth.parse_accounts([ + "alice {} {}".format(alice_key, alice_cap), + "bob {} {}".format(bob_key, bob_cap), + ])), + [ + ("alice", DUMMY_KEY.public(), alice_cap), + ("bob", DUMMY_KEY_DSA.public(), bob_cap), + ], + ) + + def test_parse_accounts_rejects_passwords(self): + """ + The iterator returned by ``auth.parse_accounts`` raises ``ValueError`` + when processing reaches a line that has what looks like a password + instead of an ssh key. + """ + with self.assertRaises(ValueError): + list(auth.parse_accounts(["alice apassword URI:DIR2:aaaa:1111"])) + + def test_create_account_maps(self): + """ + ``auth.create_account_maps`` accepts an iterator of structured account + data and returns two mappings: one from account name to rootcap, the + other from account name to public keys. + """ + alice_cap = "URI:DIR2:aaaa:1111" + alice_key = DUMMY_KEY.public() + bob_cap = "URI:DIR2:aaaa:2222" + bob_key = DUMMY_KEY_DSA.public() + accounts = [ + ("alice", alice_key, alice_cap), + ("bob", bob_key, bob_cap), + ] + self.assertEqual( + auth.create_account_maps(accounts), + ({ + b"alice": alice_cap.encode("utf-8"), + b"bob": bob_cap.encode("utf-8"), + }, + { + b"alice": [alice_key], + b"bob": [bob_key], + }), + ) + + def test_load_account_file(self): + """ + ``auth.load_account_file`` accepts an iterator of serialized account lines + and returns two mappings: one from account name to rootcap, the other + from account name to public keys. + """ + alice_key = DUMMY_KEY.public().toString("openssh").decode("utf-8") + alice_cap = "URI:DIR2:aaaa:1111" + + bob_key = DUMMY_KEY_DSA.public().toString("openssh").decode("utf-8") + bob_cap = "URI:DIR2:aaaa:2222" + + accounts = [ + "alice {} {}".format(alice_key, alice_cap), + "bob {} {}".format(bob_key, bob_cap), + "# carol {} {}".format(alice_key, alice_cap), + ] + + self.assertEqual( + auth.load_account_file(accounts), + ({ + b"alice": alice_cap.encode("utf-8"), + b"bob": bob_cap.encode("utf-8"), + }, + { + b"alice": [DUMMY_KEY.public()], + b"bob": [DUMMY_KEY_DSA.public()], + }), + ) + + class AccountFileCheckerKeyTests(unittest.TestCase): """ Tests for key handling done by allmydata.frontends.auth.AccountFileChecker. """ def setUp(self): self.account_file = filepath.FilePath(self.mktemp()) - self.account_file.setContent(DUMMY_ACCOUNTS) + self.account_file.setContent(ACCOUNTS) abspath = abspath_expanduser_unicode(str(self.account_file.path)) self.checker = auth.AccountFileChecker(None, abspath) - def test_unknown_user_ssh(self): + def test_unknown_user(self): """ AccountFileChecker.requestAvatarId returns a Deferred that fires with UnauthorizedLogin if called with an SSHPrivateKey object with a @@ -67,67 +236,6 @@ class AccountFileCheckerKeyTests(unittest.TestCase): avatarId = self.checker.requestAvatarId(key_credentials) return self.assertFailure(avatarId, error.UnauthorizedLogin) - def test_unknown_user_password(self): - """ - AccountFileChecker.requestAvatarId returns a Deferred that fires with - UnauthorizedLogin if called with an SSHPrivateKey object with a - username not present in the account file. - - We use a commented out user, so we're also checking that comments are - skipped. - """ - key_credentials = credentials.UsernamePassword(b"dennis", b"password") - d = self.checker.requestAvatarId(key_credentials) - return self.assertFailure(d, error.UnauthorizedLogin) - - def test_password_auth_user_with_ssh_key(self): - """ - AccountFileChecker.requestAvatarId returns a Deferred that fires with - UnauthorizedLogin if called with an SSHPrivateKey object for a username - only associated with a password in the account file. - """ - key_credentials = credentials.SSHPrivateKey( - b"alice", b"md5", None, None, None) - avatarId = self.checker.requestAvatarId(key_credentials) - return self.assertFailure(avatarId, error.UnauthorizedLogin) - - def test_password_auth_user_with_correct_password(self): - """ - AccountFileChecker.requestAvatarId returns a Deferred that fires with - the user if the correct password is given. - """ - key_credentials = credentials.UsernamePassword(b"alice", b"herpassword") - d = self.checker.requestAvatarId(key_credentials) - def authenticated(avatarId): - self.assertEqual( - (b"alice", - b"URI:DIR2:aaaaaaaaaaaaaaaaaaaaaaaaaa:1111111111111111111111111111111111111111111111111111"), - (avatarId.username, avatarId.rootcap)) - return d - - def test_password_auth_user_with_correct_hashed_password(self): - """ - AccountFileChecker.requestAvatarId returns a Deferred that fires with - the user if the correct password is given in hashed form. - """ - key_credentials = credentials.UsernameHashedPassword(b"alice", b"herpassword") - d = self.checker.requestAvatarId(key_credentials) - def authenticated(avatarId): - self.assertEqual( - (b"alice", - b"URI:DIR2:aaaaaaaaaaaaaaaaaaaaaaaaaa:1111111111111111111111111111111111111111111111111111"), - (avatarId.username, avatarId.rootcap)) - return d - - def test_password_auth_user_with_wrong_password(self): - """ - AccountFileChecker.requestAvatarId returns a Deferred that fires with - UnauthorizedLogin if the wrong password is given. - """ - key_credentials = credentials.UsernamePassword(b"alice", b"WRONG") - avatarId = self.checker.requestAvatarId(key_credentials) - return self.assertFailure(avatarId, error.UnauthorizedLogin) - def test_unrecognized_key(self): """ AccountFileChecker.requestAvatarId returns a Deferred that fires with From 3de481ab6bbabde6943648c80722cdacabb1d3e1 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 25 Oct 2021 20:52:35 -0400 Subject: [PATCH 0317/2309] remove unused imports --- src/allmydata/frontends/auth.py | 2 +- src/allmydata/test/test_auth.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/allmydata/frontends/auth.py b/src/allmydata/frontends/auth.py index 312a9da1a..b6f9c2b7e 100644 --- a/src/allmydata/frontends/auth.py +++ b/src/allmydata/frontends/auth.py @@ -12,7 +12,7 @@ if PY2: from zope.interface import implementer from twisted.internet import defer -from twisted.cred import error, checkers, credentials +from twisted.cred import checkers, credentials from twisted.conch.ssh import keys from twisted.conch.checkers import SSHPublicKeyChecker, InMemorySSHKeyDB diff --git a/src/allmydata/test/test_auth.py b/src/allmydata/test/test_auth.py index 19c2f7c01..bfe717f79 100644 --- a/src/allmydata/test/test_auth.py +++ b/src/allmydata/test/test_auth.py @@ -16,7 +16,6 @@ from hypothesis import ( from hypothesis.strategies import ( text, characters, - tuples, lists, ) From 9764ac740ada46b8ee23b3060e951e0cd5dab9a9 Mon Sep 17 00:00:00 2001 From: fenn-cs Date: Tue, 26 Oct 2021 11:22:32 +0100 Subject: [PATCH 0318/2309] test kwargs overlap with params in start_action Signed-off-by: fenn-cs --- src/allmydata/test/test_eliotutil.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/src/allmydata/test/test_eliotutil.py b/src/allmydata/test/test_eliotutil.py index 1fbb9ec8d..bab37243c 100644 --- a/src/allmydata/test/test_eliotutil.py +++ b/src/allmydata/test/test_eliotutil.py @@ -330,3 +330,27 @@ class LogCallDeferredTests(TestCase): msg = logger.messages[0] assertContainsFields(self, msg, {"args": (10, 2)}) assertContainsFields(self, msg, {"kwargs": {"message": "an exponential function"}}) + + + @capture_logging( + lambda self, logger: + assertHasAction(self, logger, u"the-action", succeeded=True), + ) + def test_keyword_args_dont_overlap_with_start_action(self, logger): + """ + Check that both keyword and positional arguments are logged when using ``log_call_deferred`` + """ + @log_call_deferred(action_type=u"the-action") + def f(base, exp, kwargs, args): + return base ** exp + self.assertThat( + f(10, 2, kwargs={"kwarg_1": "value_1", "kwarg_2": 2}, args=(1, 2, 3)), + succeeded(Equals(100)), + ) + msg = logger.messages[0] + assertContainsFields(self, msg, {"args": (10, 2)}) + assertContainsFields( + self, + msg, + {"kwargs": {"args": [1, 2, 3], "kwargs": {"kwarg_1": "value_1", "kwarg_2": 2}}}, + ) From 5b9997f388ccca089081d8f5939f0c84edea3542 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 26 Oct 2021 07:16:24 -0400 Subject: [PATCH 0319/2309] update the integration tests to reflect removal of sftp password auth --- integration/conftest.py | 29 ++++++++++++++++---------- integration/test_sftp.py | 45 +++++++++++++++++++++++++++++----------- 2 files changed, 51 insertions(+), 23 deletions(-) diff --git a/integration/conftest.py b/integration/conftest.py index 39ff3b42b..ef5c518a8 100644 --- a/integration/conftest.py +++ b/integration/conftest.py @@ -353,10 +353,23 @@ def storage_nodes(reactor, temp_dir, introducer, introducer_furl, flog_gatherer, nodes.append(process) return nodes +@pytest.fixture(scope="session") +def alice_sftp_client_key_path(temp_dir): + # The client SSH key path is typically going to be somewhere else (~/.ssh, + # typically), but for convenience sake for testing we'll put it inside node. + return join(temp_dir, "alice", "private", "ssh_client_rsa_key") @pytest.fixture(scope='session') @log_call(action_type=u"integration:alice", include_args=[], include_result=False) -def alice(reactor, temp_dir, introducer_furl, flog_gatherer, storage_nodes, request): +def alice( + reactor, + temp_dir, + introducer_furl, + flog_gatherer, + storage_nodes, + alice_sftp_client_key_path, + request, +): process = pytest_twisted.blockon( _create_node( reactor, request, temp_dir, introducer_furl, flog_gatherer, "alice", @@ -387,19 +400,13 @@ accounts.file = {accounts_path} """.format(ssh_key_path=host_ssh_key_path, accounts_path=accounts_path)) generate_ssh_key(host_ssh_key_path) - # 3. Add a SFTP access file with username/password and SSH key auth. - - # The client SSH key path is typically going to be somewhere else (~/.ssh, - # typically), but for convenience sake for testing we'll put it inside node. - client_ssh_key_path = join(process.node_dir, "private", "ssh_client_rsa_key") - generate_ssh_key(client_ssh_key_path) + # 3. Add a SFTP access file with an SSH key for auth. + generate_ssh_key(alice_sftp_client_key_path) # Pub key format is "ssh-rsa ". We want the key. - ssh_public_key = open(client_ssh_key_path + ".pub").read().strip().split()[1] + ssh_public_key = open(alice_sftp_client_key_path + ".pub").read().strip().split()[1] with open(accounts_path, "w") as f: f.write("""\ -alice password {rwcap} - -alice2 ssh-rsa {ssh_public_key} {rwcap} +alice-key ssh-rsa {ssh_public_key} {rwcap} """.format(rwcap=rwcap, ssh_public_key=ssh_public_key)) # 4. Restart the node with new SFTP config. diff --git a/integration/test_sftp.py b/integration/test_sftp.py index 6171c7413..3fdbb56d7 100644 --- a/integration/test_sftp.py +++ b/integration/test_sftp.py @@ -19,6 +19,7 @@ from future.utils import PY2 if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 +import os.path from posixpath import join from stat import S_ISDIR @@ -33,7 +34,7 @@ import pytest from .util import generate_ssh_key, run_in_thread -def connect_sftp(connect_args={"username": "alice", "password": "password"}): +def connect_sftp(connect_args): """Create an SFTP client.""" client = SSHClient() client.set_missing_host_key_policy(AutoAddPolicy) @@ -60,24 +61,24 @@ def connect_sftp(connect_args={"username": "alice", "password": "password"}): @run_in_thread def test_bad_account_password_ssh_key(alice, tmpdir): """ - Can't login with unknown username, wrong password, or wrong SSH pub key. + Can't login with unknown username, any password, or wrong SSH pub key. """ - # Wrong password, wrong username: - for u, p in [("alice", "wrong"), ("someuser", "password")]: + # Any password, wrong username: + for u, p in [("alice-key", "wrong"), ("someuser", "password")]: with pytest.raises(AuthenticationException): connect_sftp(connect_args={ "username": u, "password": p, }) - another_key = join(str(tmpdir), "ssh_key") + another_key = os.path.join(str(tmpdir), "ssh_key") generate_ssh_key(another_key) - good_key = RSAKey(filename=join(alice.node_dir, "private", "ssh_client_rsa_key")) + good_key = RSAKey(filename=os.path.join(alice.node_dir, "private", "ssh_client_rsa_key")) bad_key = RSAKey(filename=another_key) # Wrong key: with pytest.raises(AuthenticationException): connect_sftp(connect_args={ - "username": "alice2", "pkey": bad_key, + "username": "alice-key", "pkey": bad_key, }) # Wrong username: @@ -86,13 +87,24 @@ def test_bad_account_password_ssh_key(alice, tmpdir): "username": "someoneelse", "pkey": good_key, }) +def sftp_client_key(node): + return RSAKey( + filename=os.path.join(node.node_dir, "private", "ssh_client_rsa_key"), + ) + +def test_sftp_client_key_exists(alice, alice_sftp_client_key_path): + """ + Weakly validate the sftp client key fixture by asserting that *something* + exists at the supposed key path. + """ + assert os.path.exists(alice_sftp_client_key_path) @run_in_thread def test_ssh_key_auth(alice): """It's possible to login authenticating with SSH public key.""" - key = RSAKey(filename=join(alice.node_dir, "private", "ssh_client_rsa_key")) + key = sftp_client_key(alice) sftp = connect_sftp(connect_args={ - "username": "alice2", "pkey": key + "username": "alice-key", "pkey": key }) assert sftp.listdir() == [] @@ -100,7 +112,10 @@ def test_ssh_key_auth(alice): @run_in_thread def test_read_write_files(alice): """It's possible to upload and download files.""" - sftp = connect_sftp() + sftp = connect_sftp(connect_args={ + "username": "alice-key", + "pkey": sftp_client_key(alice), + }) with sftp.file("myfile", "wb") as f: f.write(b"abc") f.write(b"def") @@ -117,7 +132,10 @@ def test_directories(alice): It's possible to create, list directories, and create and remove files in them. """ - sftp = connect_sftp() + sftp = connect_sftp(connect_args={ + "username": "alice-key", + "pkey": sftp_client_key(alice), + }) assert sftp.listdir() == [] sftp.mkdir("childdir") @@ -148,7 +166,10 @@ def test_directories(alice): @run_in_thread def test_rename(alice): """Directories and files can be renamed.""" - sftp = connect_sftp() + sftp = connect_sftp(connect_args={ + "username": "alice-key", + "pkey": sftp_client_key(alice), + }) sftp.mkdir("dir") filepath = join("dir", "file") From 69d335c1e1503544850e4e3014ca1a6d1d89180b Mon Sep 17 00:00:00 2001 From: fenn-cs Date: Tue, 26 Oct 2021 13:14:26 +0100 Subject: [PATCH 0320/2309] update test overlap function docstring Signed-off-by: fenn-cs --- src/allmydata/test/test_eliotutil.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/test_eliotutil.py b/src/allmydata/test/test_eliotutil.py index bab37243c..61e0a6958 100644 --- a/src/allmydata/test/test_eliotutil.py +++ b/src/allmydata/test/test_eliotutil.py @@ -338,7 +338,7 @@ class LogCallDeferredTests(TestCase): ) def test_keyword_args_dont_overlap_with_start_action(self, logger): """ - Check that both keyword and positional arguments are logged when using ``log_call_deferred`` + Check that kwargs passed to decorated functions don't overlap with params in ``start_action`` """ @log_call_deferred(action_type=u"the-action") def f(base, exp, kwargs, args): From 28cc3cad66e0367da10ee97326d100c686f78d10 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 26 Oct 2021 14:10:29 -0400 Subject: [PATCH 0321/2309] news fragment --- newsfragments/3829.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3829.minor diff --git a/newsfragments/3829.minor b/newsfragments/3829.minor new file mode 100644 index 000000000..e69de29bb From 7ec7cd45dd41b0f828a581865ab3b7bb15a655be Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 26 Oct 2021 14:10:41 -0400 Subject: [PATCH 0322/2309] Use "concurrency groups" to auto-cancel redundant builds --- .github/workflows/ci.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 45b2986a3..8209108bf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,6 +6,23 @@ on: - "master" pull_request: +# Control to what degree jobs in this workflow will run concurrently with +# other instances of themselves. +# +# https://docs.github.com/en/actions/learn-github-actions/workflow-syntax-for-github-actions#concurrency +concurrency: + # We want every revision on master to run the workflow completely. + # "head_ref" is not set for the "push" event but it is set for the + # "pull_request" event. If it is set then it is the name of the branch and + # we can use it to make sure each branch has only one active workflow at a + # time. If it is not set then we can compute a unique string that gives + # every master/push workflow its own group. + group: "${{ github.head_ref || format('{0}-{1}', github.run_number, github.run_attempt) }}" + + # Then, we say that if a new workflow wants to start in the same group as a + # running workflow, the running workflow should be cancelled. + cancel-in-progress: true + env: # Tell Hypothesis which configuration we want it to use. TAHOE_LAFS_HYPOTHESIS_PROFILE: "ci" From eddfd244a761e006c70b80326edc987b16ef2c6a Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 26 Oct 2021 13:37:26 -0600 Subject: [PATCH 0323/2309] code and tests to check RSA key sizes --- src/allmydata/crypto/rsa.py | 12 ++++++ .../test/data/pycryptopp-rsa-1024-priv.txt | 1 + .../test/data/pycryptopp-rsa-32768-priv.txt | 1 + src/allmydata/test/test_crypto.py | 38 +++++++++++++++++++ 4 files changed, 52 insertions(+) create mode 100644 src/allmydata/test/data/pycryptopp-rsa-1024-priv.txt create mode 100644 src/allmydata/test/data/pycryptopp-rsa-32768-priv.txt diff --git a/src/allmydata/crypto/rsa.py b/src/allmydata/crypto/rsa.py index b5d15ad4a..d290388da 100644 --- a/src/allmydata/crypto/rsa.py +++ b/src/allmydata/crypto/rsa.py @@ -77,6 +77,18 @@ def create_signing_keypair_from_string(private_key_der): password=None, backend=default_backend(), ) + if not isinstance(priv_key, rsa.RSAPrivateKey): + raise ValueError( + "Private Key did not decode to an RSA key" + ) + if priv_key.key_size < 2048: + raise ValueError( + "Private Key is smaller than 2048 bits" + ) + if priv_key.key_size > (2048 * 8): + raise ValueError( + "Private Key is unreasonably large" + ) return priv_key, priv_key.public_key() diff --git a/src/allmydata/test/data/pycryptopp-rsa-1024-priv.txt b/src/allmydata/test/data/pycryptopp-rsa-1024-priv.txt new file mode 100644 index 000000000..6f5e67950 --- /dev/null +++ b/src/allmydata/test/data/pycryptopp-rsa-1024-priv.txt @@ -0,0 +1 @@ +MIICdQIBADANBgkqhkiG9w0BAQEFAASCAl8wggJbAgEAAoGBAJLEAfZueLuT4vUQ1+c8ZM9dJ/LA29CYgA5toaMklQjbVQ2Skywvw1wEkRjhMpjQAx5+lpLTE2xCtqtfkHooMRNnquOxoh0o1Xya60jUHze7VB5QMV7BMKeUTff1hQqpIgw/GLvJRtar53cVY+SYf4SXx2/slDbVr8BI3DPwdeNtAgERAoGABzHD3GTJrteQJRxu+cQ3I0NPwx2IQ/Nlplq1GZDaIQ/FbJY+bhZrdXOswnl4cOcPNjNhu+c1qHGznv0ntayjCGgJ9dDySGqknDau+ezZcBO1JrIpPOABS7MVMst79mn47vB2+t8w5krrBYahAVp/L5kY8k+Pr9AU+L9mbevFW9MCQQDA+bAeMRNBfGc4gvoVV8ecovE1KRksFDlkaDVEOc76zNW6JZazHhQF/zIoMkV81rrg5UBntw3WR3R8A3l9osgDAkEAwrLQICJ3zjsJBt0xEkCBv9tK6IvSIc7MUQIc4J2Y1hiSjqsnTRACRy3UMsODfx/Lg7ITlDbABCLfv3v4D39jzwJBAKpFuYQNLxuqALlkgk8RN6hTiYlCYYE/BXa2TR4U4848RBy3wTSiEarwO1Ck0+afWZlCwFuDZo/kshMSH+dTZS8CQQC3PuIAIHDCGXHoV7W200zwzmSeoba2aEfTxcDTZyZvJi+VVcqi4eQGwbioP4rR/86aEQNeUaWpijv/g7xK0j/RAkBbt2U9bFFcja10KIpgw2bBxDU/c67h4+38lkrBUnM9XVBZxjbtQbnkkeAfOgQDiq3oBDBrHF3/Q8XM0CzZJBWS \ No newline at end of file diff --git a/src/allmydata/test/data/pycryptopp-rsa-32768-priv.txt b/src/allmydata/test/data/pycryptopp-rsa-32768-priv.txt new file mode 100644 index 000000000..d949f3f60 --- /dev/null +++ b/src/allmydata/test/data/pycryptopp-rsa-32768-priv.txt @@ -0,0 +1 @@ +MIJIQQIBADANBgkqhkiG9w0BAQEFAASCSCswgkgnAgEAAoIQAQC3x9r2dfYoTp7oIMsPdOhyNK5CB3TOtiaxhf3EkGAIaLWTXUVbxvOkiSu3Tca9VqFVnN7EkbT790uDjh4rviGeZF8oplVN+FDxKfcg5tXWv4ec9LnOUUAVRUnrUQA2azkOT+ozXQwZnJwUYr210VoV8D0MkrvOzNgGpb8aErDhW8SwrJcoYkObIE7n3C3zEMaEIyA1OFWSJDiXNGnBDvO54t1/y+/o4IuzLWWG7TPx8hnV+jcHRoxJTX2MZusJ7kugvxhgB0+avwXFTQr6ogvPNcUXak0+aLInLRtkYJ+0DYqo1hLAh8EBY/cLrhZM5LGGC4BAwGgUwsx3KKeOeduNnob3s/1rZpvZGwbGtfiWYQwDB8q68j3Ypf2Qvn7hPwicdOr0Dwe4TXJQ4yRHPeQaToOBUjtTJnrHsKDZET6i+jQ9e07Ct+yYrUwZjiaSXJYU/gCyPCui7L37NasXBJ00f1Ogm3gt4uxl3abO8mO1nKSWM+HbFBEyyO0apT+sSwYj6IL7cyCSJtWYMD4APdW5rXSArhyiaHV+xNbVUXAdBrZSNuwet925hTOf4IQD9uqfzeV3HIoiUCxn5GKYPZy01Kft+DExuDbJjMmES2GhfPWRIFB5MN0UdjlagDHLzFraQUcLTDKlxL0iZ+uV4Itv5dQyaf93Szu2LD1jnkvZOV5GN1RxTmZCH1FIPYCNwS6mIRG/4aPWA0HCZX8HzSMOBshAS6wECaoLWxv8D3K4Tm1rp/EgP7NZRxTj2ToOostJtjzTrVb3f3+zaT5svxD1Exw8tA1fZNRThIDKZXVSSLDYaiRDAUg7xEMD2eDCvNQasjAwX5Tnw7R4M/CZoZhgYVwIE+vHQTh8H+M/J8CNLxPT4N3fuXCqT8YoJVUOmKHe0kE5Rtd87X2BQY5SSx6LFMRRSVdBBpWB6cwLo8egehYAScEDQh0ht/ssaraWZ2LGt5hZL0I5V58iS/6C4IOu+1ry75g6mecWoHD0fBQELB3Q3Qi6c6Hik/jgTLQHb5UMqKj/MDSdTWuxwH2dYU5H4EGAkbfufBoxw9hIpdeS7/aDulvRKtFVPfi/pxmrd1lxQCBA4ionRe4IOY0E9i419TOgMtGgZxNlEXtp445MbeIlurxIDIX8N+RGHWljGR/9K6sjbgtGKyKLUxg51DZeuDKQGdyKXtIIkZ+Od9HN+3Mv0Ch5B9htIRV9hE6oLWLT+grqJCFAOD3olGgrRXByDsd8YouahYfjqb4KNCOyFPS3j5MdUpq+fiLrG3O98/L/xtmXxw+ekl95EGAnlwiCwULsjzVjHJDzSc68cldMnzNqLwhwWXpc0iswCWCQVFce/d1KlWqrtwq2ThH2pX3BJ5Pnu+KMISNNC/tagLe9vjmrh6ZhEks7hefn0srytJdivGDFqMs/ISmcld0U/0ZqE05b7BpErpfVrG9kb5QxWBTpaEb2O0pRsaYRcllFuNF6Nl/jPDBnn4BMYnOFnn9OKGPEDUeV/6CYP9x+Wi96M5Ni6vtv+zw9Xg8drslS5DJazXQFbJ0aqW3EgalUJVV0NgykB6Hr4pxTzrwo0+R/ro32DEj5OfjjU7TB4fYie0eax8tpdvzcWJRZ/c5b/Dg1yK+hbiMg9aTctHAsYJkOvMpxvull20IuV2sErWZ7KZhId19AFOnEQ6ILlHRwUf35AyEVmUL5BqLl137EeEVShEmage4+E/N6PdKzJdJGl1AQGyb7NTD86m0Jj2+8qu6zsBgyUfiJqZ17fixKV6l9HGJKSmY9If2XrX/IhNZ5dvqSmODJ1ZRGC5gjJcxcdHp2Q1179SlNmXiR/7DMcprL/+iVhRyxzM2GEJ78q9jS6j/Z+0vLzdNOPo1KxD191ogYjl5ck9gnHAkbaiANaK4rrfMytDkNm0JRua4p0mVyVHWZWwatoMhJxVl3+9x37OkF24ICTJZ4LSKDLJxi9WCQbhgACIA1mjcW0P+4AszpbuSXOQkPtT+MQ0IxHMzX261yHAIPbGsbSzoTy+PWJywFdMDy5afXDTNpmMfpzWkw2fhBQasNoGHl2CwFftJdr4WWxuN6mSwhNVHJTw1xe4A5fa6bjip5kmrLQK85YF4Ron0OIOofjcCzvjKCkNkGVKBhRiqBoqV6Pzz1XauVHFhFgZZNWXI+le+Fg9SJojeDtFQp5w6dZKBJMxV2uNPqV0U4VOtvAas2+Ul4zIJDB/FJyDX8POrsR+VkW7via64xM1hQlOZ5ispEOUvmO/NWkAsJM0n3S7qgud6NaFqOofQZcbh5r1z2uIrXwUIb85m2t/sPJBI1J/Dql4dmzgfn/q6Siqi8FeDoma/lQBZWyEeGz+/ckHdw/BGPx5FZlc8xLegNrQj4sVkUZXVAjNoUguA5HT9GcAmE5FeOHdHtD0bdTaNFkQbKdi3yUlGA1GZeyPThwfBaizgX3i6oOtGguX3HQMQtExip5xR2vsiYJsbWXuzlKEws8GwXoiJo8xEh+TPavxxtZ7dDdnJY1mUhKTVGLBCqCrJ+uhWdWuHKvC9x++V5NO6WQrUiG/o8oOwkpWyH7GC/VtulpxkoJlxAej3JxlHn91cN4PstDo4goOhQBi9k2A5rsmvjGG75BOKlqvhaQ6BPOa+9F5D5H0RhT0hw43TZmJri+0Ba2WT3FigcHHYGtx4UJfyqfg7d+WXvpIynC7i3SIN3N7atg3EsWwPuzDKE6ycjWTD6ToKmYLMnDgl4PzOEBFstG12OdcuQwhk2Dy5uEdxqGfViy3fV+Muev0yAkE/pRwutgQjQdw0OPXyGoqchYx33/cHq1fDWmkXZab8wuVThcx3He30UI4rr3MMff0gxdnJt3e6YcHHF0R8fGwkVC03zWXI2hfqHq+rNQkBnIbbRnepKvJylmcHn8KVJ13Nm2iHRTw7B8r6fE6LsmUJndh/M2Poa1AtxfGBniMIfqtV0RuT7UR1nDI0C8Lnx7E2KTw1MXCLh4xzGr5wZ+4T5FTeUnzd6yc7EEduLxktqh7RpmnBBPRNIufI9ztPTmRPXgF7r9PxI8MI09Sr2HQq2ZmEs6G0w8l8WMiABvlG/YQd+UHGn29acrzSYp6AfggjuUV7PrCC4flKk5IGBNdUtUqFxBRUuvn0ln7HayAAYLJuVMNv9daBwqMpp3Faor/0K+jC0FhIan3R6wBpKSuJo/6jZJoSlSCLGCkFqM9ks3sgD5cDvxahV7HNOv7AisDws2LsVATHbF0HFeoEA7lp6NzjK5dgqd+9rA95U0c7w31E1E9GbmzLADC/0eSDKEkdKGIJ4mP1erpBOc+cdJ2tVP5e6cZ7KNhzjYf19tORINCTrPAp9/aLXnoHgtLp3ozkFS/dGowLZ6Q5XInPBchgiI4TVHDDxGpwMAZp3G3yM1QDptd3pxRSv4m97QIOa7ma9l3TCK8RA/bs/akYoZnxM92GvG/3FQdws1y3Lz2NjoikVSaX0TS1t16TupL3PQioaeRJLnTZu0WGR20WLL6kEBz6cHJC3ZN9Zilnoje8lEm/7/WYOCt490+w4KS24aJcgDPzV7Z1npXy19p3ywEY0AJND8uurWeTEHIBJNxMPU2OMGd0bGa2S0yr/dfbIz3FmD06noX7/XKMjQ+gW8EBXAA7s8TA2RE0HbD8IGKlg3CCIaYsS4BbvK0B71qHhe/yM8qnUo5+vv1UpbioYVBI77UfiqqUDUAIIg+apIKJjU352GqXiEovXGR6Jeag+ufzPkPq9BqvyIfW0+3r2/wp4nIu7Z9XM6iU1Lj1j/wM1goktBnDfY6hbjHA0acQFCgUrzeGqyzYSe9kufDTSw7ePbx2rLG+fXa9qwqVwY0iBjJ8Hu6xIFmvesHwq0ySH0IqyI/Y53ee2hhju0xWAz8GishuMv4/apVLWQ4MbmG788ybGRxePWqYx/KI8M1fUvZGRXmtwAqEIaakewUVpL3QhawB4eR074Yhl5gY/ElwlcxNboUVayqJwgh4BO+/2tAutTDCtkzdLMjH4JoDpMNsf4GiLVvlSahU76B+oOlttcIm69oRB5BklrgbPCwqbQldsvvP3nHuFxBAlunefMMGZFbTd59JbO5UAkAHQ7XRw3MWDq8B3V1uCF59r4uXc+kvYFS/y8DTpQGKtO0RQx5yIonoNCbJjYWtx+zMACXoXWkrH03IQJMKmPM3IMbtMDMxIdqjD1hdaQ4dAnVcCq7ZvcbIThtCHX0+Vqo9eHoqA2kBtZLRq5rq4GG8Jm7o9mrpuVTLvym0goJuK2KQbF39CxlTG8eIIRKFQNhKC1XtuTGiIQzd14UsHWHhqhWo8uXHGhAvkl3ga8+5bDuJRhJ3ndsNE/tnq/VlJf329ATseDCLmVEDRiqe7CJeeyvMLgN0oE0lGZkmf2iYfRpB0zdkj6EpVdVZs2f/vRTp7S0ldwvV0pTDj5dzboY+nhd2hzR1+EnLPuUbVGqotTz8BWkxo9DpoGkA//5ZMeCkqFtKh3f7/UAWC5EyBZpjoPN3JGtEOdBRLX9pKrvY6tqpwaiGAHA85LywmB3UoudiGyifKe3ydIlMltsSpgc8IESwQaku2+ZlvZklm8N8KVl+ctF+n58bYS0ex63FfYoJEbUzJMcyC8Gse7zfC5MFX7nVQPWRrJ6waRu+r33KKllmKp1pqtTH1SO0N3WTP8W/npELnG6A9RnnsbtXO1WhN1HuyT5yv9KRaVPq+2EkoweAEq/Q1SGtJBX0hxWaK2UDRb4VRMHC1uDF/CVMCcfvTOQ8/ihWgrZtroDQ8J8TU0ICZVCdz3duvw5/C0eCLB5szT1EsMY2x1hKpnfS21Y7SCpG3SYv2Ii47kCex1A35Et/7MMwilelxgrwDCsXyObkepVwdrBwV6YF2qd+jMj+H4mCfhempxwCSlhXgwhS0svSPmPPAJOU4gSmcVktfs/CyqCKLzpGxHXjdcA41/gWVCeYDdjOEirh9rUIy8KlIspI+3y+XNdWrRfH9UkYQsjH7mwvixOQfc3NUvMLOSnCe4bLZ1gR4mIiaGwR15YT+Tl3AkfHu3Ic062iPlWON5Sn6ZOBE1FnGi25YOiBCdDkF1vGdzPb2SLBnucVnEqKfBB3/0KcMrT6bDApKrPxfVQfx7YJnKO6T8nddFdPne2sr2Joz+QJ4DR7nnSBvu0VEZTXLAr+K7OOSJwlE76WYT/oHDHM4LivUit0ChnsUegNFwD7zO6nz3OWYzDaB+XzVr0c5wtpZP1IYRCs20L5jOc2P1dzV7WHErHJ8/VhDZ76d//2SCCdjv5kTfwXXHsfWRK8jMV+TZSmKlKgq+pDd9Um8Ao5ShvGqMz6TThFihNrXUL2xCEXJ1ki7xL3fTTCgK/SlMt7NYeOv5xqIdQdc7tSjYt9y76UbY6bVe+i1H3ppaYh2+oBaSDyzbInglXpHEWS4yJfh7kJxXV5P2u+LeOIzmz3xpZJJCiRjdW/Bl6jbAgERAoIQABPRyc9I9OY6rL6uNAQtPDR5Idnxvsr/kLjKr3IPkeLKCYrfZFezkr7rp9oK5b8V5DjrRTNQ9+j6CqdJDUr96ocK0wvpx/HR/rCYmqave3QFmKoGUEXvqgxVRrd+sjgQlTY/1X4CgU4OYSVV8VJaV4TgLr2XWoc+P3Qq+QBNT0+E4IF8BkMZp+sVDYdvloYib8L0urBn9SZZPVGPsQ1KZZQL6rXwWJ4iQUMCYsrJRFjWWB6a++UtQVMzBgKXpeV2j69z+xlqM0Bf5QO1fCoWfsOFzHh8Z7PoJ0p/2EmR8xryZsvu7fGgNXEXVF4fUrf6i52DwAb7ptUP/PPAnp5sg5lP11byyIGLEM6hCEKbJ1uC77oNY6q/xWowBMHOROYYXcqZKGWdOo7bLPSlC3EYPj8SgaIGW7spy/xv6TCB3BaYeRWwb2VQEfxjAK1sMVYPASBhqr3jWgoKeOFdoYJ7el2BLqprHod1Vbqr+2ahq2Fjt2WIGt3mjmdb8WnGht3f7xfzbX+CYGATPzEKOOHojQJ0lpptITSm336cwdW//4qo4XdMMo/cnO5cKzbjgbAdI1eCIEaSIvmpRgs0PNQuzSKPZ3GBqvPLFPeePeOZsq+IdNXs5YqPTw7BdJ3Wm/VZzZACBSbdjP3Mbr/yG+qEIx2i0x6I690twqy+fxdKy/HHcRGcjiBMODROq+cpxRROjxHqd9/8udNQqjqcg6j/iMzOiQv0FQ9+iEyEzk/jjF8rmFlp9FtSKe4FJ+ZgNfKFAdhDVt+cu5MpW5NZJ1wKkOM2xEzSKZlYrXx1MQbEqsUb6uopkHWoS435jsGrkzgjbDUTN2SW21o/xaiSJn7/27oUiezK7sKqK70Sf2ixdqXQXwBC6sBItE6aK/VFR+r8YcU0ysxzj7WhJB+CDNatv4d4M0oFZkXB9wZ7GIPD282KqAUM+TUOqMnpLKftZAEpRGC5ck/keBU+J7/vGO//HUKOjtPsqYPPV6qY1Pc6jrUn5RkIxzc+qo5lSoae3DL/e/7a+SCKN97Elac/bOtTRy/of4jYf8HgNQVd56NxQeoy+fUboH11jwuz3BSrHmBLnbljxz42gglBRFY4Zw0Vh35KISziV9yXqj+a+72dj1iOXCc0w/27E3gQERaex5m+8eGTxKb1R32HKV9Ww94UYDdkLZwW3g7sG6uXO9+tjJY2uZk8GHFxyYlCUB8a0URVNVMYdKDHqTuhrFLOv/CWjCBg92VB19bwSGFWEfwUroQlZa9nU6FHp0a9SgpLvq2VSeReOppoSngAuft8vxNUDXeDRfZfwf4jtUdp14zLE3QvSU83RKy+Wv/4jC/Y2ro7SqZ6wAWIlYr9Js1ixbOyeXu7e99D8sjWZbB3QMD5zYpsW416jOxZ0OXKrRZ9om+B6CtGgugjxZri8us9VpZXw9Q5TDcW88Ym6Dersajy71qnndzvo0K2FJBW7EMi64J/2lr70yAJADNU9z90B3BK0X5junIBbp88MfJNKVjrm7VV4DVVk5YdmpMqxWUVW/xj51ARIxmu2boXSpUxHs9ZXAoF1C/OoIVcM/7/tOtOERzUFFRClGsw6yeTEPvPlYY6eKnKQJputuCMD/+qbhj6kpxjclAnfEJMr+Wa/QnOLp+0/Lvz9gh5hyMdgYCBIaPe1rJ7TglrqsdcoIjHObvMm2OjeYdZUAHB+Hgozu0H82XC+OD57wax1n4fw+YktMtgobt2YRENRAcyYReehwfMKM0ahR6GVIdRCXQ4RggEbyQUoTArKSS13JpliMLNEhwocFsahqxazDm//tadLKCPEjnuKrWGXEwiHpJBOLas/J2HhQEQ3XKMDCAGz+QIfkjxGvbhYARpBTgf2AWNoj1BzWwPWn1vUQk8v7osEoP0s2kaSencOFlPfRzkVowKJAnR5IZ/xv6lau7bjqsOnMutoKjJ3lWUzvjhuvAHUh7AG/t/Uubn0ZdZalVIvDR4xcjcRdQSsyxcVKg5cw9V7e8fOFocHlb/JKYUqWaG7edondhueTNK9n4YAwjgykPhcj7+aJOWJAP6tTlqIt10lC09mHIkgfGdEU7gGmODgXMj6C5bW51TGKi38mtAs4YwCiUJ/m1x+yGFP3LBsB0jswMxSIL1/5B9djzeqbYRoZAUoBuS/qPzDtSNqOO7ZLmCb2YL6vV1x9nCEUkmIvEyDNB83MxZeMMv3cIp8VXPx8X5U78sLfqTHlq8dZnhvGs9zwVOUk729bfGLuk9ZQxHuFwoodFOUMLTdgJGPaXWjEaY/rdzKnuN5GDhtJ7MDqipVFd4O7PUNCjeqQo9hJAbPRaCXh7cweIWcBkVl/0df+Y4vGtmvQEyt4wvQyYYCCVE3J5m1UK60Uf/DB3OtM08Xcr/DiRG6zdIUVcdpQzRBRIJLUoP5vDp/jj4qpoh+bsR4uIQpvU1ityWixGiAAMVZuuvnJ+G/A7mc5naLN+hH6wELoqRxDbUqNerfxulkEKIpPwiZ3l5AI5O8yLiG2Pu9tPj0QoTz5neBDDNyx2EyAlQh6Be7hSZyWqOuS5YWbs+h+XVmsNdQaY0CKDsX5NjgmtYeh1KF+RPYTs44982RosMVUnijKP5LrtM945zk38/RZ5qR/Wn66Qm2ToKEiTnw5wQFFx86/lZPeFDQKpsxx+qi9rf7pxVALvl+p7vehLrNajnFDAh5DvsNlWkID/jgipuNSFIN6TsLuMvRAbqWWJBpOOVaE9Mj174Lv+/C75EJPVMUAkzvBpr2scTNl9sSixXgdFsc1TZ3zXs+vV4AKuYjw3Gq6dmnAj6Qu0XaYfgnGZqz4lzYJIff2mP1AAPHN7rCfnlza03cAppazc1WvTqIC22Gx1Sn906cdcG8LUobdx08sXTVxi6wgyqfQUuU+JbCpH4eoHFpUMifXmGHRHciQCytE/UIOKTPX1JNFnRKmEM5DYhfD8/wi5nHgNS/L6zHqpsrWfu5UyvumZJ7XA/djiZ37x7JdpTVj/8EgIn146AYRoVlS+V1xWDOz6c1BG9BUN8ZWdpY/Y4W65owEN19CNg9eKWizEQD8TH7X5rz874WVlrsEuBOTN9feYylhT0uyJCAPWX/ARhwX2iTSVsIemAGwI8tvoqq9u8vXU/j0+EtiFYjBm+GTo/E/GqLjSsEIc+B7RnARWTjfMNqNu49DoGVLUtvQWAoZlYqGLGpvis7PlO1tNIRbhaXcSXasBbO6DpASLBZwGTfZzpm3D2OC60v52f22uwJx/2tHRUILWXgbmc7/kWnkb1FZbpUSfrkxiLcX6cK+3RLT//Pnbk9wva+noJ/aVFb9ldBkkAk4iX5XYHSTWf2IdPe5Lz1bBB2Y3WtFo0MR1LKf46yQncL+FbzWTLRSHPY3UeRhVg3FHkH6MnXYpov8hHwZ4FrJaT7LMmdj13DL3HF5lwwYzvkclyUJ2taQCwnXPlgXvWRgmYfNblc98/yn3m3wWzx5rS4gGFHqBkJYwTqW2cGuRDVZ0V3t3+UfzqIJmK8nXpm0GKjZT50PfMjsS6+uVgTHaQ38HDFvpBM/1z2Sh2fcGfbkxVBWt8Wwl0Xntt6tYYamFGfqR+8W6VRVQJitb6uZZiA+wcbO+kfZOw55VGHld/USRiRv8QuxGe95TZV47f1CcCJzZhWqiaNH65DLsLAja7DeNwxd6CHaDAik6S6rD0FyZ9PQPaICPPI4/xAo/0ZVnd/yEc8OI+3yM4Ks+YgQ02Gnrl1z9lv2Y9zytEPBDFy8iWYtiyXZ8i4U7AXOGd5i4h3jKPlW7h0OkRKiSSh4TgO7dD+5Sxk5kAMUo9nxumcCmTBWL6i6yRnsKmS0nkIyZI4wuEihk4Icof6JsPqrvXxc9VgQ6QWQ0FgAeubKbqIFgV58l2JK4Qfv3JKYrKMS/n/BCjRVZh3DfkTcZzQg+m9Ytcze7bv52bN0S2xrDITaw4q0IKPgmXI5Nwb4HA2t4p0iBHgoqtMbU2tkoVyh16EVnCwnS/IhHi4HTlcKSNDCWp52NXf0cWGjgxDV2ds37QYD6JoLz6Jf+NIUElPQ/CySdVnfcTHK6h1xjG3K5OoeIboMqJ0WxKdRm+Eu/2OpC2T/x4i0YxM6pthPXUQ+tYnjYd4csTbjE9aAVexoM+ARW6WJj/utUp0VvRQOiFRTLDVNJfzG1YUDXq3u0cAWkezq9q8bny97HBHP5vnjzymajF89NHP+bjZrvPNigJOXSPybJPPFLhTPZGjryD+78fT0VrvMHkXutC/Yqa2OEXe+jYXOhx5phxknCngScLmIudX2c/fXXxxoLeJHD9Hjv2ASlDszSEuBFDawPEMuQaNf6sjTi3PLgOaVZDID+NAh9sw3RqcnQjMcyR6ojGxkDpzxj5VBNHxbPXNuAUXPNkl8KfkAgwbP1qBWbyHAzUBg0+rBcRBjnD+WHkhiJRqKW7RMyyGMgpk7E2p75ZsdtjDX1uzxJ99QT+q3qEoM8qfAMniuUoxeVX4WWaL+eS3aDhE9hJtz2qVJjx/oYu+X6tSjSoY/3OHlum80NLM5h/tVBXi8kSFmtV9NkiGPXT3OVpEodhhCXBZOblOTOkolbawoROX1tJNXpNAJCxz5d7jkjPM/VUoBrvtXcfMBJOGyAgrfCu/qZ787tsi49ZwMKPjW7SAWzgzsVVynVS3SyPfUs69um4QESoW5rMqbnh0jTRCiCGAjK/2jDjhqpA3r395j0TDlQh9goCzwzYfEyFEAPspF73GcEcR2eb64S0bRjT/SUrPrRFUSV0MhFefwXwd+mv2VcF7Zr8GzlR9fOpngy3xrC7GkyeSz2jNSwIkpssLpvXPbG4mzXs4WBFDcDb0hZmFHvU+fLI1+Do9lQ3KbSyCXxA3VoveSEv7spX+9EGJpHjesN8cPcjChjVozfOzGWDXw9xRAFVbE/eLLrik+ftGqzmqm1zNSbXInJqfFmgeJAH95eS7j6r/kqO6b38rKtMIRMWj/2xtArTtpqmEbF7JgQNM56dIsKgf+Iea3XeV2A5wa/d1EMj7omPTUezw5beqBExgShFc5xkibXHuSTLD/ibQTya42F514GH+1CpmXJ2MtoQMBv5mxJ5l+HynS6i11kfku33m6CMPzv9H7vsO+0OMgK9zf7qOIPIN6tpOkHXJPy6ytHkPNJoQ1SStUawwwddGGOVu0u/IfaCp47sLMqIoUAF1kZSt3laLGeW0Y3/Mbdb5j5NwK+36XuWUvJs+eHIKRvc7KqcW8Ww+ReglXFdc9HGmUOHV6t7hQ6YT059ThcDZQf0JasLJwFPAo9BfHL2sgBUdF4rRt0jLBVNaXbcwO+tg374KIf7dHcKKkPQ9HT0fzkBu0+SlsEJfpqMklksImd6Ls1clJSORvKAnzcPvSbxA2vcGg++Lu2vdqSzQXD+2BegqE95A7h0Dd7VH6AvuqosfLpuarI5Hs+FX4H6vpxMa9lb8RTIi2lAI70CgggBALr8nb9910Az4BdF02PCn0uM5oa1W94D2wQN9sW88ivd2pXMRlht4y0546P96ud8Daxtv1acT2henrCw1S3I9CpR/0HDoKywEzPgN3JQsJhDfsvEhRCrKnU9miwvjCe38nlkMG9PVZmVTjlvt5UWihzbTnjv9nBSnQ6fhz4QqqRBAi8Lcmc6IKuz7CuROsY4lNCHW1xLcVoKJOTOMV1DUKCXn36K4bkiYE0lhWCtAZQBVHkJWupZpogjd5mr9qy8IfXF91iIPKw02XLgNiclPX6q4r3m98aMD0c/slvsIH0r5fphjLdoQHYPt4Mp+Vum1cGk+ogmpcwSJnBJ1qbrFvlBmcGb5LoMd9z4qhvWwWVOKw565kyWkaB5WO4v1KFx67KVdPszzAUF8u2Ac5RIPY+4Db8hvTCovDH2y3q3mBynYJX2FjHS+3Q02E66thuzHfbxHIKHSazq5gJWzr+hYfal+5kZxOfydFMIC+jdRmFajNmoKFM2LOUlZMVAHPVTK40DshixVjakvEMUCJyDHURyydgDbs9W0ElSYq9mVMXF/2m11KY0Eptzvuh1LkFHIfDOdUCjKOrsd7JeUqF860WPgxHUnAas5HKBTM2xNXEyAsQXtQk1jU/CxKgLr3WDLF4eQ76a/BO3SeGhytpasDKUMQiqXyN7v1gJeBQoyiFitC1oHUVVTg7EgJfN0B0dFWKL8iyYItWB7xKtXHPsedU9EWRfghBAxoAqf8GLW0905DMHdnIQKg/43iaKWNqmNqCVRMKQnShA6GN6tOxtvaVV4WRNtwtEuOP2U42cNA702e0qFtmWDBjARuee1qhJCuklkYdDFKrzn0MXT/5xxNCtGVLeZCFPWw0uDUQu+HjD8Izc42fnVGS8fLwGLjj0Ajnn/MtVusCHvUFJSPLG8qsCXBuhsywmtpZGKKe2EP+KKphBFfExQQJWXR9tbBGIcygK9c6wj3Tnrwii8D3oIGvEgnNYWUL0pRVSs6tpRwzXwK1el1wAoU7rUQ16UoJQx01tWEvxN7wTsbo/V3IHp8F/UAMNnK1GQDZqn/NDR1Ln70yT56kqXsNf88WI38eox55vtOCePiFmpHddvRuMZrmSu9FFQtd2rK4eDMrDuGxFJh63+n53iLFlCBbNcc1XV5CP99B3STPSzYHPS9n0aCoiDL5kJ96LelFEkFqr9gOhG/3JpW7rGw30Mv1rFN4dFKn58dSyfi2tHbz2geuIVG5BEhujxvhYg53CC8v1agYd2zlSPQnCKU2efI47iXbGw66l1ACwLWsI21pR/HVt4YyjKwy8IWJoNPPN0AjcDq1Czis6kUXfmLRDks7DciEdhOqT49zQyn4hNebkFg+VCs3Y1JfMilRYdCH5aJJn6g6w9wqE/qCx6wQuq/7Y5ImEpKEYme40uqJMjO2oekz1FhsZ8PWSku+d+Srus0pQkB8MMjHoFrAtXi0QWY1y0wo6Ci1kM6T9wbVLmF8hXkfqhEdB+RcyNqQeGquNxM6rU2JKvy/HLwO+zTD53CQC1ToYV2+5MCRr9+N2/CbcifMUN4VIEn1Eej0zwHF/yN2Dc+UYWiyEQtlG14z2hlkDP0CPGq4tt8VdftJ+HvCw8DXvTWTnLnn1Zp8JOcQmEeP99YAYcjKhKnol+34BK6OqlAPxBhpdin+TRG05T1CoGS4qDFCdS/mIdCVFv9g2/QS1SdUQIS52zaRHnQQCSCWEa+ZSTfRHd58wlVwt58M3tCbGyNiM6wA90GWFA+zPn5OSuWleAC/cHp8uaJ5p1tC2CPYxbU19N/pQmg+fwNTBO24wUN+3zJXC++eGtiFofpQjnDWXLH27+oIG+YuutaWh1jf4Jsf3HybnAmDBUf4D39zprOur24+buf+h5uDddADdFHnQ8GHo7txQ0pEU1Q5L6tUw7JY4zVLZ7PF04Bl/XLIRHwb9hGoAEGsblcahUXa6SWq6oQmyoNO5l91ZDZk2ovSdq0kMrEB543Y6Uo8UvPIDgOwvcVhjrx2BDy7H0YG8rMIerCI+4mXi+xrU5Akhyom5b8TFqsEmZN5lvrsdcNtYc4/d7qnkbVYBZlx2MyeDC+ch5f1yVBY1cLnpjFFHFUXZFmpzUrhXPc20vgeXnQQgqQtV5fbQDYUGz5KIe8d1wVGIVMut1rmRa9/dspSJMmE24mNe/K11eSymPBI+oSmwmo2KobIOb4otMXXGiNmwVSN8Yv22FoF3u2zgpx6esCfGLLScnsXOpCf0f7aP4aqwqN5yeypzAlhF3+yakuuv0m/dHUEhxuOqStrxEG8ShJv5tkHsM3V1WLRkpBAadXPy6gSysA265grR8BX4LbUZnFqvoDDNrvRSweNv2HddvI2fcgltJ/fIcEu8Qk/WNLUUWJXdMRbaUwO9IPvhQULFEUCLdqvK5bB5oDnUQQ3FTq7Lspp/naoolLMn7k6K5gx2IxQnpq9+iTCzU/vrKL+O7Mi86AHJxPCr9tk/MPEqzaH1SjA8zrPqdZdtyngTEMn5ZPiHV2zMUuWPJ2xXT2zrpyx7mVXJdl0SE2gbnOTs2/5wFPy9aTKynFtKxZB1y1iWEAlBWsTnoS8FE+6CBZH01xww9GRjoMi9xDee+wXV/olDo/dROj4RPYSvIeB3tIxorxRR17YjyzZPssKDGTvfzKM8kqYYNE/BqEKBKLCz0bhPCCWxu3JaVJomTVJTrFy9JzmBMy2O3sgLRDl6X7vkqOm1AoIIAQD7nE/lfkcKEttlB0HLSKT+yDGo8kJAR4zKmi5fZpVgWYK30Aib5HFTA9BHVZElnhTeNyvYMdSO1FdtNsa7tQ1/0rD985d/GLXe/f25PAbEsmgnFMmc9zSmpLIZ5vTxIC7Bk73mqwwgZZxSNvpqurbUO+787vMn2wKC74fJHC6NF5FMFrCypu4B5RLs6C9fGjRKab1vW2mi2967gCZrB1celCcgkBzN6XA7tvjDozDz7JU+x7ugmBx+6MKpsLc/FPrRgEhwWdPIsV6R+vOqRugeTBtr+NyvFhAa639l/e9EQwpEbVJgbNg5okOZliYDF4UM7YADgv0aKJtir+4xN5Cka7Jb8vyYIAcchy4cjz8IDNK3SuvhRmPTbEOs/xwZpoN3YqUiARI0RvYznaByKpOJSpxzqqP1W/026K6n0KagIjyQht6p5ElpsXlIgcH0fwpXseNYl2pQAzj0jAGFaJNYSBdgyQdZkoiUDprKUm9dZfDL8m9FFpoDV+BuJmxDe2XUpLfDhTnF5n/F9wYjmd4Vhfui0HA6kh0dLvOS0EZEvz4mT6zD7Sxx+T4uyZJE9nq1KOEpQTW27mzJad4jXJkiYe5C33DSEdOpwVAu8pIYFxmcj9uuNHoK2hpYcst/wYuNzgAHB9LuJaJRFLZSXN+IVyBWU2S8iejIVYzAKhm7Pj72hIE25Z7oQE/MniMQeUgmoIlqxbSpnWho+K4koZGNIyiGv3N9XFTjN9YCdWSC4AVuyfyKa8c8Wl1cWggnOwhj1CkFeMCK+f02a64kupllLUL5I2bzC2drmjpdEGB8m7KaCWl+W86pWKHKltns7u6Z0TlEPCk2Y2+ypD7GEicZSbMwAPt5jpTfxoMk2h9ICzgDbFPaJTtAsYNMiAYz9Sa+w0ELdSYoGD1OqN/ZkPE/sGRcXfAk4efEkfRDbCU0hiH2HMbKFLhH63/RfGSbgeYSGDHTs66JOJ3htSh1arYOmkwBB5v33cnVCmRiUGgE4QijTnMmYLKH42txfzD6fU1TJKUr2woazXiPvpS53tgSbO/zmBUE6fiFIaOGpT0iHXhx38sDX21VPVY4zwkYvmFNKliwgnZTZiThCNF8e1r4W5SlOyoCm+cc6UnPB1XOYx/Nd1W7Njm46rL4rsfZ2w18vATLl4ofn+6M1dgN39FO6ueKvZzxHUH1Gp2J3Z1cphfke3+O8NKi0BmIe+TjfuTzCt6l/rkr0UjKqXqYF1OedZe0kwkIRDmY6cY+gQlIdIFOaefF/3bBu95mAozWMTtZZGAPrf1QM52AJ/0fZKjoBvvZTVbeP6TnuulOcahtZVGDs3Q2Io9d5Y/c/adXwEyizH19Z8dV/ImY9JdmXDB80wDodoo0/uL8Ig/2NslKCu4KtxjzLwgKHhsz2wWgjagn3AGkD6nlVdElCPwRMdHW0v1Ld5RzZG+oXD88tXe91cLH7YY6k44pB86gD2EauwqDPSk1Q0TPy+Fj8sLEWwg/prsVZWMvwLvGCRRCCUWiDJhuWT1dzOxHTcbLJSAqSTaRDccvIrFR9YdqqmZtinnSwzByzOG0xY4uO3j4EhK3GVpi6L8zgoEqP4F1vU1EwPn/W7VfsLggBBRhG06yk+R4zOBtUNOHi3Ra/P/D7smXKmgR5hnz8tfObTgCO6FdIZAnP7DbS4bw1eykk55rG9x/k76Kd9iB6PtlnTl2gqaCcx/JX09lhWNbXL0NL9J1T+aEyJiHZyViVcHBKjXaUSlf8yYbuFMSV82iT/LgYLSmEb+tsS3bm6Sa1r4uoOrET40Dky88Oru7hoZ49f1HJrGLhoRlDO4rCnXV7QABqwAE5qJCDZ0Kx1Vvs0WrK1yypHAjbmK9O4+98Ih+65HhdXoR5Ds2Yj1ovv+d9NWBMEQpLEpOdtEoZ6xqAr1DDgdPVg5wSPtEavKOEfQWfPERqCQC/oqcO9rMbwEZGx3wcJyIZZ6jbupWGcHmSu3bvb0sJjdX69wQGL9Gl5WzR3xrqMYDX/ObNKml0QM0//SX0+j3FhMzMzwzqDc79a0FnXjjMBloIRVWsFdGqt5ZF8fXSEkHejycJDbyXZ2amxtPN9LgOZ6GvboFEnoEpslW4shx2+zO3Q/u0YYbaLGZu5zKumObpau92s8clYwC37htg/IT/JYLUVvSx6HaWj3GaVfvFlQ2/oH+Pk3MOVAyx1GXpZoOtjcs44/U1fKVIIAn0jX4g//wcsdt9jdbdU1PD6UpH5VlH8xJ3fNWxr37R8nIw8HzBnbrgm6PWH1wiWzbZSR5dAn5WUv8MS8JxMKC+QyNjZ6/kgfO1Yt0PV1EPJ4ji6A0F+akKWlYVXdbgGVQyISsje66u4fncZOMHgVwlF3X2sNe+ybRMUTysPsTAmRm2YUvIX6b0IGL+CcSWMKM7PeCyX+utfIn2IWZ0Wa5mjN56TRFBx0b9Xdnq9gLbx+HaUHSLERJloYg8jfeshmUIha6qfb7ywtLBixXcJTQUYtlXkQJ5pzXyYNWqv5gKShjAxsOMxvg/AvXw1g2TKjq/vZs7X+lIbghfEilIu8UUn1r2Lkwak0AI4si1prjsqNCaxduiZGGjeKiOlDA9c+72AmrGj8hbgCyzOq8mAYTlvadCUH2GRmQQnGVvw2pxoHpFFFBx1ZPWmmU44lnBjlWxPfQ2Ic9u1yLYHEnUVYTxDKHK5bT8940F86YFfjozWK67PFKWju0iuriL7cvbi8yxyeiTwKCCABX/mhaHRoAGGl0XRgu8izYQk5dgoWVp3YgBpI+74EFlZQKQgL8b/JvosV6WV97/iSNYNKDHGGahuFEFvroXpEE20rxxXjJvEVFlrCuRBbePeFQ1PNTI19GOxtgFmASsOqTenElUoKioJ1INJKggxPRWCTtnhmeRP6deD+kvIyJiAEHFHISdbUFgdiM+QyZhAnLiv3RFHGTyInVFbzgmCXxOEsOX3lIEC1RexGW6AC+Hr5XE3YT7fQD1HSEjSjJwfHdEd3PTyucVRsI4ftdtyv/X3nCxwswQekSeFPvBbTvnC/9WxULA+IZcM7UT/zf1go9AlfHmbdvF5meQN17ueyxiEhbHC9mnHSkOMiFkjzkYQUz/ZmNdAhLhGYVvCfTgOdjGSf9vgWoAsysADZj5cKd/EK0TBzLmrLqVgVm7PxJuC1zvxmA28GgGN5DKrANCP8Ky9EuXchRX3tMZRX/03llAtDAhJjln0XMuH4TOvPxlAYMEuXMzjM+qC9r4e+CgX3oAb04y+xV8ytq3EBJpxzU6rlWmDQlVgeqCKbpIRjViloToNyKctuUcrQxKBXEXbWef0Y8iQQyUSlE4RfThhRc+D2uCbLV9wIXxGBgy9zp+Wq2ob6a7AZDpvMh52GgtjL/HU0OZw02dF8AxJuyDI8m3FNPXzvUdngpbd4nmrl5H2PZIe+oKCS7p8QLM6064IKIulPYwBBkeWFyM3bNI/0ZDa3U4aaePJmluaWIQZRhoGtjTs5Ty18WkztdbkfubFXxNy9qnmgS8V5M7nNCFYZr7C3U2UcUXJM+GZC7HFS7voSr15JIRpxH4gM/0kblyAUibAg/pxjI6x3FOCWk6j6AUXVULGta+CrZBpzUys9H47x+hhCpXc1clO9ninAazS45Xhyb7Bul5YY81zFjMHIyW3ajl2NgEjfOPyIwziYd5qqiAILL2vFqgv6lYKtTi4F8QWSdgEOCTuj1AWH/A9MFiabM3kgfgi+RkFSM5j+NkrUGSqGUtQCdm+noOZA9UzCc6CmNJjhYgb0MWgsIfBK1aRaYBmfZEgAZm5aQmCGQbSVRNosibkq2S0WKIkswx+V3vBjiLFl5IT5WSjrfyZnAvYWPqB90dBUGpLq5xYP2tyD/ZaMOVl5xmPS/b70VVkdTFpK8dF6u+coe+COx3G1BAPbwLyHSI4Ta8xbBQd0u4meGfQKOjMFv+nJZI1UdOtyMOWK+ch1Cq9HCVeLJMRisWttYTRJWwD3v4thf+wS3lZRXNcJe8fVRs/5hDPVlEj331ZDxQ9kjT3ZInw1kb/GrmBRCOmoQMQncJJ4iSXBRiNl9wTVODt5y8p9wW/l4/tUjGGs6vJuGpjd7tqD4RiMzsVT8JATcZdxMOSImx300FwrXxh14zDJcUjLSR/MTibbiZe4VvnXyBef3XlervqD9sdrN6p9/0d6qyq65j1LhbyauEt2AFVl+nkhCkGNQG1AVXFSJ4NOgnAt4D7Plm4mK8d6hgQqnlbIynRFSMoGXqrRSuBYf4VGAdZTFpvruKZKO7bxNX/wuzpTG/l8I+nR69L1oIDmGNnit4cfvxWO3GoTJp6b81gsVKLexavCW2e5wFYOoK/9yHTu8j4AZYY3VIX9Ic3uWInWJe1O2laC0wDW9eQTuL/3g8X3yqqAB2tWyDebSn6e67cr4x5NhBLqASgWimpECey0adDrVCgSggA+dZRV6fA2niJpsAhSonzO+P7/ScTc6b/SYGjao1gQpq7nh/vioPphvcMOQvYRt0eH4Z5Xwjk9ZmzfpvxNGrdkVaBpXrXWs/+JGAJRwFrylg6uRxSs/xuxL9PBFtmegv5x3Z4Tx5SojnYKoTCiSzyFPCuF7uAEeeReGmGlY5m999oVwwcDwxKjiShh44IIbNSXTuOjgJgi8voJhFKq+rZyC7Y3MosnbCdLe0oX5cXgDSiAx4emb0L70D63dhNdBSMRAzIfrKilZGtk5CqcJs5vmJBTTDC7OOZBDVQ2fELUj2hc4p2F3S8ro1oC1hfbx8FEBDoioatCFGOPID+bXZlK285umMC93t2jQhlM6C4GtHSUEp7r7S/PvRq6pLpwwiGw7CKAKc4BXfPa81igg3qEjCRfeRywkkUpd7P6Yh9cUZKh0JawCXY7bi4WLCjzbEvq6M7BXU34O/uqgQJAtv3mYLLMkc4RRPytT4TzIUxuN5uKuJOkx9yZViprAy3Nb/kyzoPuIOFgzPIrhO54w1bqvWMMv6MW4cw7sf/G5vIuz+aNfRS5HqGlgSL2cFoEllTrxeU6JQRQqy8t/kD5nhJIA55++zc9j8yAHk/sJY0DzJulv33tQYstVofdkSEmUFmmAYMrNVB8BnguDd2fKLOpeyfSw1stu7y5DsBjNrzi+/q2wZr2naA+Fly3FEXGHySjJUGwWz9LuCYgGevfZyUT9aTsi5eufmlIG1/PJoQFy5Xud4TGAVGPB2BMs9/b1DZpbMcYW54M5Dq2eqrsfCgTLZ+jNIwopJJuDzLybSC+EA3RvbzYRrMdCCvgzQbgU7t6+9WTggPoS39Fcq7LSFqB277kIzIXQm6hD+zKECmHmPN9ruEvZ5EWdalz5ZCj+NSe2xXjW7+Pd8HYg3Sx81IllU2azy+C26QDiGjbYqbvNU7DOLvY3rQjUAXVJWkIusxfVsQmO8biXxE9iNoDPNEARvQzqhNyExrr5kMmVDbgbD5+c9/BeI2tmV7SUp9cEkQKCCAEA3gJknVFNvZgq/soq/qmChnRoDYp2sTAS0OJlJwApcyHNsT8Wp6tzDQNdbB5S5PTlPIsIkZVhMrtcMzBU//oa+FB+DUBYfzPrxMH9/cuNgGEuuRJXin/FC4JCy4+M1MILI0YgB8QZwjuJ7jCCmmiDM7xpdcPHfYUCN0vSKeuwmpxTBubYJSnhELsQsur8nzU9MpmJB+c/Fzp5PAepbX7yhGSa/p1Gl5G9Yd2uUkSyuRwLN2Tw2P6vu0XY8BRlc+VVx+mpVBMGKY1xj91tlj6QkzQYMhfRx6oONd7Z0nal8O/b4gYbgkHr9p47paKaArpmVrNw9AoqnpxM3ps7lNaszU/3uosbHND3N0oZoLqhBxpfkquE1dSyb0Fo4/An2mW/SzjsDvHi4tUzlvR+gtpF8ZwvsVpUbxTue74/wT+iFNLqJSu1aLpe5MnFXhgjm38nPlGqe1hs3TAFFAMQZqeREakFkaJRx4FLVXZMWCqef5Yu0hIl8aqH5NURUiHnDl3SUjb8f1dvNiW8CQcjiNMPQCrtFzBjBoDsgyltgYqYWsbcfCgvBzquvur6ocDqeRW3kMm3nN8vZSy6V11pprsdtOz/aC6QuVsGDkEooeUXfqr4exWFmbXVGKJTezgc+EFdBKa0uujJLHuPOHuv7lHyaT3RPRxn8abcdIe4bVJS8II3jjiuP39P+hqgw5qXaON75djxuJBUHTCJTZAhL2FiT1tB4E6TFEJpBLjL5A06kZh9Q6MqH8iCnqoWJE9wmxX4WBWNm2qLxeujMASotv7/0b6GY1t49JGXfQ+c6LQY5mtDPJ7knKtb/tW77v2THFpaD0AjeHFRile86OtGcoh82hPaV4hla0GSaxiR1TjubL6a1dgNwHs0SCQojtJf0331AqxIc4V8BUKQcpUBv/hcZV9nnMtba9ZjFtsi0hQg0/3huwpVDKje1gwHXnzRPesWTDN3QlM/pkEDxydf7yHr7sRhLhXF2rSjB0Vnogq2Imw0zFRHfDc0HYxt3J1nc5u8ssX7JrI2F6Y9M4oKwh764xTTuNF79UbqV1nqo/s18OzTr8V25Nu60r2mblxTUhFk6bvz5wmzsv/GL/i41z+qnudlCkNDL3qAoQoT8uhaxSpJPNK1DplB/YPLF6lG7WbtyGmp4NEBZzLDbTUoDD3060e9Pi7VxbBnX8wwptKZ6FZRUSGsyWsUNU40pZp+qp0kXfqIOBz9vUAxK0o+/qsrqe9Jn1SPf8O6Wb82c2LL9KMIrpmuY2jwUJa1LNUS2xxhixxUwop2GZb0YgUqozqzJxU4ko+I4jgoF8MKGAnu9x0pzo9IbABgYeisHVhIXHx/2vCq9i5klyofDn12h36FIthMGGiYEKSqKcOzuyFIMkXhGINwXhpwgWXbxFfXyeZnMjqYCTr/UeJPIK2THjsEckGyUaW/OKPqDQYZrgmHxZ5+sGgrJKBQQlIuyXb7U9I2c8yNxZW1L9IDG/RRgBQWVkfSQA4qV0+0vcvGlJ7E+GV3cGzbyzxYAq4Jwk3vNF63rSpGVsRCGyPv9LR4fsV6jMpX7NLlRSbIv2Gm+QDjVkOL/Ot7h82BByj5wj2eh/WRSrpUvdgp/iG3oPn6JRkU4w8gYHR+aIobX1e0f7STwZ3jWxZTIor6pxUTTUOsf1nZjAsFdjOVLtrf3IJAfKAc6QnkXA9krtyhleUUlb6S65LBsa5zO3WyBVHT/JOblK/phDiGlZc/GofnMfgRZkec+k8Dgd7f4wIt6ZHWTYKBRzzWTfav/gHNeZBNdG/eNL6pmb4ano4tLP46arruihMVIMH8WSmG2q7gcXbDxTyHi9qPKzkwNq/h+SW18WJ+9/qBEDQ5AVKsAfJaUd7qIUmJ040lL/xUTV075bnpkBuHb5+M29JAFJe2P2vULBtv3Jc56pq/lri35sSME9eniAzUexzUp/iT4Y8fFib8TJ+ZLQ5ezHDs4o8yngXg7xDUF+V8IGazHUMDICtl+IpeuViut68EH4jR7KldLsO5syRkJU+2lpaeh+7HUwXzBPRbm0iO42h8PW5rIDxp1iQKruVtnS+e5B/0P0OLD/JFReX2TWAEWMWGBHm29Quil/VHc4XQ8sMODvUb+hEVLUsv/iv9iVXx48ERGTiotz3e9zgv84SEZFbYjM5DhG2+CWwCS24OEmgYWM2P8G7OSuwa0RmmDPshoBQVf4+ZzuBxFBPVRLC0pvvdJMow2DpTRcKCq9CS4MG0QS1AH2QRCuT9VsrYTueWGxi75+Sq6tOcSR0CEM/MkLgz/KPeNcu6r8ywuSKbIYDZtoAvwOrZ0swTEE4F05yeVJ0CmTxaQa2GkpLPaxPSpMOWCHNF9Bp9RTeeGNAVzEcEIf5L5TK/ayA6eN4MGob3PjByTlNtxOTn5cIHkYSd1mROIyh14hMeZ4gPTXNqWwZG3G8tHz1GKDTflZkb9a6Wm0iUd2xPaiStKFpQSlm7zxyRfm7b1K6hbIQvs8ulciXVr1dz4w9Y62+cGyDbox4JikfONtmKEcsroixC2JVSgqVIHYvHoMR4mXX2Mft2Occ04gE+iCbE5wcIheYFncStlNeLvFGSjCQvw9y9PJ6wLI482gAaaivJIFgGxsvu6DRDu4XrwF1ISoH6KALeSRlMJ+ZdKQUAxFDJLnGPXew7GFoGXNygE6IexiWV/swbq/VLl2BM4IvboDzAhtERI3zLRPMLfEg4OOjAO9zmvGMCgggBAI0U6B4zfbor2UG5zkmlHcBbOc+a3/N4PNZLwcWfMcS6hzsU8v7fgM1sOz03K4EEPr1ULSI/Tq71XsIcaGPt124quX2O6wzplYsDYy40MBeeKry7xsaLnGo5UCqvCprelYx2zGUY/fuz2UJxbeMyM9m9uTBZ8h3rOuioGQgmRDhI+ACcti4kMKg1W0nqd2pZ69tgCEGt3H2puq9SmukNm41xYE3YkMvo5e7yjlWVcdQ93K3x3dPP8mtr6ckkKMhOoxDB3tsd69LTxXc3ebhD1u/pGhqyAvpXcPaN0TqjhNMKdnn+G+g7BfOjmO0FsF4ElRO5d/O7KrUs/E6vfvE4m46KeWlE1plG8C6Ukx/Af6UwCHtWTMQihLfskuIMz67o/YDOnJ7miGb146yd3E1nOjydRwUoSeVPYzLCL4R7aO8DCdKbmVnQyh/xUBSM1m+MWH/UyqFQMx+vFMseDoPjx/+G2ZvKa/GXNRoThXonVpAFFXUzEU2DzIzxa75FWUNU4Nhc9h3HLsYCG4hYYb2ab45cQD3uOjHIS1VB6tXKLbwBfIQFH9bi3wnUdmGBnRHAU3NEvflxmNFCejBZoLsbqp/niVr1BIzvmmZHOR0di07sVkKdoRGBFuLuS53UPOBndoQJre+SESEyVNwdN8jnDFsCQ3k4KZbS9d85MgoCagtNA9XaZ0kvQtwP7zBqVAwEeCn/cJG2yKbVMXOstGGW4TexTHiGlSCT0Q9lSAYPLJqT96x8vL5JoUeGaIL1h7b1hdwR14LZgp0nmROKzCKovASkuMaPvSDv8kA3TLG9mJD0flp8cB2y3+njj3j8O8aY/RHx3qNJwIR8djGnmcpw5hjzFA6rbx0zj0UCc69ogNTbdeh9Ia5Z9RMdsEUkBLj1+AABk5AV90xv8wAUjxzpflhR+fz51wsvAL9CIPwvJIvbzSHZEPOgKiW1zwOkO8NOrG1GdyPYgD2JseLxfZQ3pivqfcOekLJ+X8ZT3VN+wqojz99lXFUDA4IU5WSRYVROmWypZm5LfhulX13+REGgDs0sGNmjqCNcsQ6UW6NFFIK6dh6OmVnuKW1+lSG097xlhv64IDabYM/wf9kH34QLyyZvI0OVVoUnKuiebZMExAZJ8NzxTyM6ol/J8wIHRuSHXwu812AVgUdIDGdswNF2PjapNrb/6TRXZeP6BtizlHWoMlpJp9QaNOqhNPj3uONB7P8EJrS0u23SXunTz+GIKzGP4x/a1hDburtYmoKUrls+rF2eufbTypANSJf5u7niVnXaQn2Mpy07FeeeptyYi2hWgXOrWjtsUy9OLRgR4TKyzKtj3rJX+jRJ/SgNv59VQta83JN0Xw+4qIJPWhYHvgSAdp1EugnSK70PvoLN7T3OX3Ox3HjtKcZR/ClR9w2hpoWbGEMmqeiUug64aYFQ6UyBeKWEUpPT3rd7Cusu57WiSoj7OsXX5vWlUz23Dmz/UqJq91qo7UorjlueIkyMgPpaKfEeF6FM5i/lkBBPlB0rD8l5RaJ7c6EgD/6ahcyM7bteQIpL/7P8G9VcWD/45D1HqOhc0DXenSJuZBnA50IMLaT77bomcEtEigMCiBjKpTCSjNJ/CL9aJe+1EOQpYL8kEH7ZHrUlQtO8tnOCM3tQ+0d72g0zo35pPTbwgEOdH7BAqs4z/EEdjmEaE15VdmVeDXYUUnl2XSX4TO45G2l5O7wzLwEvYWRx+cez6ro+Hv4f5MkPeQvyqLzKwuwocOG6GD4TzsUl3w/h3Tw+kEkbzPW2UijeygLSYad44jmfwTQCwee/DzPbiGXv8a3Zo1KrT0+RLgKQ4K5/RLqLFfcHZtgKqSIFKbPaNPoBs9YWNkR82Be75bYtyAWuaNj/taw9h06hHBlN9JAlXE46wUjZ8ScG9Lw+pI9SxW+k5sWzrOjCv0rH6wGF2XwEjU7zTXe9njj4zPj+Jgsc1Q01ThYUNfAXG0M1cm9SteVEovgRXT14nv3yqgyOMW2Q/REGqNuyRvrbxjfwfk7ZbvVF6mDR5ayB0qdnH5YlEbDfE/MmbEQ1UQvEkbMZsyrCzNjUoG/DxThsuCARZt1P9OpDSYmcG1LL4TgFfSZIF40QfHeJJjZhwotCwrBSWCkThF/TAHO6MFYaUvX0iofIMzIjdojuf7eTLU2dVaxDLoYWorvKl18T1zo9ESws0Ro453sXTzvQbyGJaDhYQbAhkYwvzX3D1tq4r4iBDqTlJKGsX59z2G1m5K48dIAqpjysknMyDeCz2MyfpKbj1ja0GzuNbtv0X48PMR+6PTMc25zatNU93aDR30fE1BEtjRgUrUuZzMSC0FkAkuqWTuN0mK7kYZZ8Uv3fSa0pOGh/uEyuIZ2+slOobCeqiG9hgmOnjvPAY/DXRTu+sSsRyeICSfLaawja3ZGhpz/fTFKygSY8O8Iolyg1MeyPrIz3eNndkd7RlBifbN+RZD8pNHJhljHnBRvO579Kn5eBey9cih0/DCXrqiJrxz2/rulNezKuLsY3m+l//IqzA38kpR5sbHEDoO+0HZcNTpU7hsc+3yj806eZ0SvJdDLxjiOoebLBLo6JebfOmaBAjplam8GLLuoJfH0DlwJkAEUEQvcx4Y0AbUAL3CmQUHWHiGrlCrWml7nlIyEhLj7Uj32z9lRXxBBrH5obgwl8RWpmCAti7K4ryFSveRMo0A67wR3APYYvF1DoSbIRABn2ikQVvPrcjiXDNwkx \ No newline at end of file diff --git a/src/allmydata/test/test_crypto.py b/src/allmydata/test/test_crypto.py index 0aefa757f..052ddfbd7 100644 --- a/src/allmydata/test/test_crypto.py +++ b/src/allmydata/test/test_crypto.py @@ -60,6 +60,28 @@ class TestRegression(unittest.TestCase): # The public key corresponding to `RSA_2048_PRIV_KEY`. RSA_2048_PUB_KEY = b64decode(f.read().strip()) + with RESOURCE_DIR.child('pycryptopp-rsa-1024-priv.txt').open('r') as f: + # Created using `pycryptopp`: + # + # from base64 import b64encode + # from pycryptopp.publickey import rsa + # priv = rsa.generate(1024) + # priv_str = b64encode(priv.serialize()) + # pub_str = b64encode(priv.get_verifying_key().serialize()) + RSA_TINY_PRIV_KEY = b64decode(f.read().strip()) + assert isinstance(RSA_TINY_PRIV_KEY, native_bytes) + + with RESOURCE_DIR.child('pycryptopp-rsa-32768-priv.txt').open('r') as f: + # Created using `pycryptopp`: + # + # from base64 import b64encode + # from pycryptopp.publickey import rsa + # priv = rsa.generate(32768) + # priv_str = b64encode(priv.serialize()) + # pub_str = b64encode(priv.get_verifying_key().serialize()) + RSA_HUGE_PRIV_KEY = b64decode(f.read().strip()) + assert isinstance(RSA_HUGE_PRIV_KEY, native_bytes) + def test_old_start_up_test(self): """ This was the old startup test run at import time in `pycryptopp.cipher.aes`. @@ -232,6 +254,22 @@ class TestRegression(unittest.TestCase): priv_key, pub_key = rsa.create_signing_keypair_from_string(self.RSA_2048_PRIV_KEY) rsa.verify_signature(pub_key, self.RSA_2048_SIG, b'test') + def test_decode_tiny_rsa_keypair(self): + ''' + An unreasonably small RSA key is rejected ("unreasonably small" + means less that 2048 bits) + ''' + with self.assertRaises(ValueError): + rsa.create_signing_keypair_from_string(self.RSA_TINY_PRIV_KEY) + + def test_decode_huge_rsa_keypair(self): + ''' + An unreasonably _large_ RSA key is rejected ("unreasonably large" + means 32768 or more bits) + ''' + with self.assertRaises(ValueError): + rsa.create_signing_keypair_from_string(self.RSA_HUGE_PRIV_KEY) + def test_encrypt_data_not_bytes(self): ''' only bytes can be encrypted From 2336cae78c02adec4fe9516f75f506c5f20ce075 Mon Sep 17 00:00:00 2001 From: fenn-cs Date: Thu, 28 Oct 2021 08:26:13 +0100 Subject: [PATCH 0324/2309] remove step, release checklist Signed-off-by: fenn-cs --- .github/CONTRIBUTING.rst | 6 ++++++ README.rst | 6 ++++++ docs/release-checklist.rst | 43 +++++++++++++++++++++++++------------- newsfragments/3816.minor | 0 4 files changed, 40 insertions(+), 15 deletions(-) create mode 100644 newsfragments/3816.minor diff --git a/.github/CONTRIBUTING.rst b/.github/CONTRIBUTING.rst index b59385aa4..fa3d66ffe 100644 --- a/.github/CONTRIBUTING.rst +++ b/.github/CONTRIBUTING.rst @@ -18,3 +18,9 @@ Examples of contributions include: Before authoring or reviewing a patch, please familiarize yourself with the `Coding Standards `_ and the `Contributor Code of Conduct <../docs/CODE_OF_CONDUCT.md>`_. + + +🥳 First Contribution? +====================== + +If you are committing to Tahoe for the very first time, consider adding your name to our contributor list in `CREDITS <../CREDITS>`__ diff --git a/README.rst b/README.rst index 705ed11bb..20748a8db 100644 --- a/README.rst +++ b/README.rst @@ -97,6 +97,12 @@ As a community-driven open source project, Tahoe-LAFS welcomes contributions of Before authoring or reviewing a patch, please familiarize yourself with the `Coding Standard `__ and the `Contributor Code of Conduct `__. +🥳 First Contribution? +---------------------- + +If you are committing to Tahoe for the very first time, consider adding your name to our contributor list in `CREDITS `__ + + 🤝 Supporters -------------- diff --git a/docs/release-checklist.rst b/docs/release-checklist.rst index f943abb5d..dc060cd8d 100644 --- a/docs/release-checklist.rst +++ b/docs/release-checklist.rst @@ -3,9 +3,8 @@ Release Checklist ================= -These instructions were produced while making the 1.15.0 release. They -are based on the original instructions (in old revisions in the file -`docs/how_to_make_a_tahoe-lafs_release.org`). +This release checklist specifies a series of checks that anyone engaged in +releasing a version of Tahoe should follow. Any contributor can do the first part of the release preparation. Only certain contributors can perform other parts. These are the two main @@ -13,9 +12,12 @@ sections of this checklist (and could be done by different people). A final section describes how to announce the release. +This checklist is based on the original instructions (in old revisions in the file +`docs/how_to_make_a_tahoe-lafs_release.org`). + Any Contributor ---------------- +``````````````` Anyone who can create normal PRs should be able to complete this portion of the release process. @@ -33,12 +35,29 @@ Tuesday if you want to get anything in"). - Create a ticket for the release in Trac - Ticket number needed in next section +Get a clean checkout +```````````````````` + +The release proccess involves compressing source files and putting them in formats +suitable for distribution such as ``.tar.gz`` and ``zip``. That said, it's neccesary to +the release process begins with a clean checkout to avoid making a release with +previously generated files. + +- Inside the tahoe root dir run ``git clone . ../tahoe-release-x.x.x`` where (x.x.x is the release number such as 1.16.0). + +*The above command would create a new directory at the same level as your original clone named +``tahoe-release-x.x.x``. You could name the folder however you want but it would be a good +practice to give it the release name. You MAY also discard this directory once the release +process is complete.* + +- ``cd into the release directory and install dependencies by running ``python -m venv venv && source venv/bin/activate && pip install --editable .[test]```` + Create Branch and Apply Updates ``````````````````````````````` -- Create a branch for release-candidates (e.g. `XXXX.release-1.15.0.rc0`) -- run `tox -e news` to produce a new NEWS.txt file (this does a commit) +- Create a branch for the release (e.g. `XXXX.release-1.16.0`) +- run ``tox -e news`` to produce a new NEWS.txt file (this does a commit) - create the news for the release - newsfragments/.minor @@ -46,7 +65,7 @@ Create Branch and Apply Updates - manually fix NEWS.txt - - proper title for latest release ("Release 1.15.0" instead of "Release ...post1432") + - proper title for latest release ("Release 1.16.0" instead of "Release ...post1432") - double-check date (maybe release will be in the future) - spot-check the release notes (these come from the newsfragments files though so don't do heavy editing) @@ -54,7 +73,7 @@ Create Branch and Apply Updates - update "relnotes.txt" - - update all mentions of 1.14.0 -> 1.15.0 + - update all mentions of 1.16.0 -> 1.16.x - update "previous release" statement and date - summarize major changes - commit it @@ -63,12 +82,6 @@ Create Branch and Apply Updates - change the value given for `version` from `OLD.post1` to `NEW.post1` -- update "CREDITS" - - - are there any new contributors in this release? - - one way: git log release-1.14.0.. | grep Author | sort | uniq - - commit it - - update "docs/known_issues.rst" if appropriate - update "docs/Installation/install-tahoe.rst" references to the new release - Push the branch to github @@ -125,7 +138,7 @@ they will need to evaluate which contributors' signatures they trust. Privileged Contributor ------------------------ +`````````````````````` Steps in this portion require special access to keys or infrastructure. For example, **access to tahoe-lafs.org** to upload diff --git a/newsfragments/3816.minor b/newsfragments/3816.minor new file mode 100644 index 000000000..e69de29bb From 972790cdebe6056ae93c59452975c7c1b67c7c9c Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 28 Oct 2021 09:47:47 -0400 Subject: [PATCH 0325/2309] news fragment --- newsfragments/3830.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3830.minor diff --git a/newsfragments/3830.minor b/newsfragments/3830.minor new file mode 100644 index 000000000..e69de29bb From 70fb5d563abfcd809ce627b5ed35c0b09d55d684 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 28 Oct 2021 09:48:26 -0400 Subject: [PATCH 0326/2309] Get rid of the public expiration_time attribute LeaseInfo now has a getter and a setter for this attribute. LeaseInfo is now also immutable by way of `attrs`. LeaseInfo is now also comparable by way of `attrs`. --- src/allmydata/scripts/debug.py | 8 +-- src/allmydata/storage/immutable.py | 6 +-- src/allmydata/storage/lease.py | 72 ++++++++++++++++++++------ src/allmydata/storage/mutable.py | 7 ++- src/allmydata/test/test_storage.py | 23 +++----- src/allmydata/test/test_storage_web.py | 2 +- 6 files changed, 73 insertions(+), 45 deletions(-) diff --git a/src/allmydata/scripts/debug.py b/src/allmydata/scripts/debug.py index 2d6ba4602..4d3f4cb21 100644 --- a/src/allmydata/scripts/debug.py +++ b/src/allmydata/scripts/debug.py @@ -170,7 +170,7 @@ def dump_immutable_lease_info(f, out): leases = list(f.get_leases()) if leases: for i,lease in enumerate(leases): - when = format_expiration_time(lease.expiration_time) + when = format_expiration_time(lease.get_expiration_time()) print(" Lease #%d: owner=%d, expire in %s" \ % (i, lease.owner_num, when), file=out) else: @@ -223,7 +223,7 @@ def dump_mutable_share(options): print(file=out) print(" Lease #%d:" % leasenum, file=out) print(" ownerid: %d" % lease.owner_num, file=out) - when = format_expiration_time(lease.expiration_time) + when = format_expiration_time(lease.get_expiration_time()) print(" expires in %s" % when, file=out) print(" renew_secret: %s" % str(base32.b2a(lease.renew_secret), "utf-8"), file=out) print(" cancel_secret: %s" % str(base32.b2a(lease.cancel_secret), "utf-8"), file=out) @@ -730,7 +730,7 @@ def describe_share(abs_sharefile, si_s, shnum_s, now, out): m = MutableShareFile(abs_sharefile) WE, nodeid = m._read_write_enabler_and_nodeid(f) data_length = m._read_data_length(f) - expiration_time = min( [lease.expiration_time + expiration_time = min( [lease.get_expiration_time() for (i,lease) in m._enumerate_leases(f)] ) expiration = max(0, expiration_time - now) @@ -811,7 +811,7 @@ def describe_share(abs_sharefile, si_s, shnum_s, now, out): sf = ShareFile(abs_sharefile) bp = ImmediateReadBucketProxy(sf) - expiration_time = min( [lease.expiration_time + expiration_time = min( [lease.get_expiration_time() for lease in sf.get_leases()] ) expiration = max(0, expiration_time - now) diff --git a/src/allmydata/storage/immutable.py b/src/allmydata/storage/immutable.py index b8b18f140..81470eed8 100644 --- a/src/allmydata/storage/immutable.py +++ b/src/allmydata/storage/immutable.py @@ -156,9 +156,9 @@ class ShareFile(object): for i,lease in enumerate(self.get_leases()): if timing_safe_compare(lease.renew_secret, renew_secret): # yup. See if we need to update the owner time. - if new_expire_time > lease.expiration_time: + if new_expire_time > lease.get_expiration_time(): # yes - lease.expiration_time = new_expire_time + lease = lease.renew(new_expire_time) with open(self.home, 'rb+') as f: self._write_lease_record(f, i, lease) return @@ -167,7 +167,7 @@ class ShareFile(object): def add_or_renew_lease(self, lease_info): try: self.renew_lease(lease_info.renew_secret, - lease_info.expiration_time) + lease_info.get_expiration_time()) except IndexError: self.add_lease(lease_info) diff --git a/src/allmydata/storage/lease.py b/src/allmydata/storage/lease.py index 187f32406..594d61cf5 100644 --- a/src/allmydata/storage/lease.py +++ b/src/allmydata/storage/lease.py @@ -13,24 +13,64 @@ if PY2: import struct, time +import attr + +@attr.s(frozen=True) class LeaseInfo(object): - def __init__(self, owner_num=None, renew_secret=None, cancel_secret=None, - expiration_time=None, nodeid=None): - self.owner_num = owner_num - self.renew_secret = renew_secret - self.cancel_secret = cancel_secret - self.expiration_time = expiration_time - if nodeid is not None: - assert isinstance(nodeid, bytes) - assert len(nodeid) == 20 - self.nodeid = nodeid + """ + Represent the details of one lease, a marker which is intended to inform + the storage server how long to store a particular share. + """ + owner_num = attr.ib(default=None) + + # Don't put secrets into the default string representation. This makes it + # slightly less likely the secrets will accidentally be leaked to + # someplace they're not meant to be. + renew_secret = attr.ib(default=None, repr=False) + cancel_secret = attr.ib(default=None, repr=False) + + _expiration_time = attr.ib(default=None) + + nodeid = attr.ib(default=None) + + @nodeid.validator + def _validate_nodeid(self, attribute, value): + if value is not None: + if not isinstance(value, bytes): + raise ValueError( + "nodeid value must be bytes, not {!r}".format(value), + ) + if len(value) != 20: + raise ValueError( + "nodeid value must be 20 bytes long, not {!r}".format(value), + ) + return None def get_expiration_time(self): - return self.expiration_time + # type: () -> float + """ + Retrieve a POSIX timestamp representing the time at which this lease is + set to expire. + """ + return self._expiration_time + + def renew(self, new_expire_time): + # type: (float) -> LeaseInfo + """ + Create a new lease the same as this one but with a new expiration time. + + :param new_expire_time: The new expiration time. + + :return: The new lease info. + """ + return attr.assoc( + self, + _expiration_time=new_expire_time, + ) def get_grant_renew_time_time(self): # hack, based upon fixed 31day expiration period - return self.expiration_time - 31*24*60*60 + return self._expiration_time - 31*24*60*60 def get_age(self): return time.time() - self.get_grant_renew_time_time() @@ -39,7 +79,7 @@ class LeaseInfo(object): (self.owner_num, self.renew_secret, self.cancel_secret, - self.expiration_time) = struct.unpack(">L32s32sL", data) + self._expiration_time) = struct.unpack(">L32s32sL", data) self.nodeid = None return self @@ -47,18 +87,18 @@ class LeaseInfo(object): return struct.pack(">L32s32sL", self.owner_num, self.renew_secret, self.cancel_secret, - int(self.expiration_time)) + int(self._expiration_time)) def to_mutable_data(self): return struct.pack(">LL32s32s20s", self.owner_num, - int(self.expiration_time), + int(self._expiration_time), self.renew_secret, self.cancel_secret, self.nodeid) def from_mutable_data(self, data): (self.owner_num, - self.expiration_time, + self._expiration_time, self.renew_secret, self.cancel_secret, self.nodeid) = struct.unpack(">LL32s32s20s", data) return self diff --git a/src/allmydata/storage/mutable.py b/src/allmydata/storage/mutable.py index 2ef0c3215..53a38fae9 100644 --- a/src/allmydata/storage/mutable.py +++ b/src/allmydata/storage/mutable.py @@ -304,9 +304,9 @@ class MutableShareFile(object): for (leasenum,lease) in self._enumerate_leases(f): if timing_safe_compare(lease.renew_secret, renew_secret): # yup. See if we need to update the owner time. - if new_expire_time > lease.expiration_time: + if new_expire_time > lease.get_expiration_time(): # yes - lease.expiration_time = new_expire_time + lease = lease.renew(new_expire_time) self._write_lease_record(f, leasenum, lease) return accepting_nodeids.add(lease.nodeid) @@ -324,7 +324,7 @@ class MutableShareFile(object): precondition(lease_info.owner_num != 0) # 0 means "no lease here" try: self.renew_lease(lease_info.renew_secret, - lease_info.expiration_time) + lease_info.get_expiration_time()) except IndexError: self.add_lease(lease_info) @@ -454,4 +454,3 @@ def create_mutable_sharefile(filename, my_nodeid, write_enabler, parent): ms.create(my_nodeid, write_enabler) del ms return MutableShareFile(filename, parent) - diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index d18960a1e..8123be2c5 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -835,7 +835,7 @@ class Server(unittest.TestCase): # Start out with single lease created with bucket: renewal_secret, cancel_secret = self.create_bucket_5_shares(ss, b"si0") [lease] = ss.get_leases(b"si0") - self.assertEqual(lease.expiration_time, 123 + DEFAULT_RENEWAL_TIME) + self.assertEqual(lease.get_expiration_time(), 123 + DEFAULT_RENEWAL_TIME) # Time passes: clock.advance(123456) @@ -843,7 +843,7 @@ class Server(unittest.TestCase): # Adding a lease with matching renewal secret just renews it: ss.remote_add_lease(b"si0", renewal_secret, cancel_secret) [lease] = ss.get_leases(b"si0") - self.assertEqual(lease.expiration_time, 123 + 123456 + DEFAULT_RENEWAL_TIME) + self.assertEqual(lease.get_expiration_time(), 123 + 123456 + DEFAULT_RENEWAL_TIME) def test_have_shares(self): """By default the StorageServer has no shares.""" @@ -1230,17 +1230,6 @@ class MutableServer(unittest.TestCase): self.failUnlessEqual(a.cancel_secret, b.cancel_secret) self.failUnlessEqual(a.nodeid, b.nodeid) - def compare_leases(self, leases_a, leases_b): - self.failUnlessEqual(len(leases_a), len(leases_b)) - for i in range(len(leases_a)): - a = leases_a[i] - b = leases_b[i] - self.failUnlessEqual(a.owner_num, b.owner_num) - self.failUnlessEqual(a.renew_secret, b.renew_secret) - self.failUnlessEqual(a.cancel_secret, b.cancel_secret) - self.failUnlessEqual(a.nodeid, b.nodeid) - self.failUnlessEqual(a.expiration_time, b.expiration_time) - def test_leases(self): ss = self.create("test_leases") def secrets(n): @@ -1321,11 +1310,11 @@ class MutableServer(unittest.TestCase): self.failUnlessIn("I have leases accepted by nodeids:", e_s) self.failUnlessIn("nodeids: 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' .", e_s) - self.compare_leases(all_leases, list(s0.get_leases())) + self.assertEqual(all_leases, list(s0.get_leases())) # reading shares should not modify the timestamp read(b"si1", [], [(0,200)]) - self.compare_leases(all_leases, list(s0.get_leases())) + self.assertEqual(all_leases, list(s0.get_leases())) write(b"si1", secrets(0), {0: ([], [(200, b"make me bigger")], None)}, []) @@ -1359,7 +1348,7 @@ class MutableServer(unittest.TestCase): "shares", storage_index_to_dir(b"si1")) s0 = MutableShareFile(os.path.join(bucket_dir, "0")) [lease] = s0.get_leases() - self.assertEqual(lease.expiration_time, 235 + DEFAULT_RENEWAL_TIME) + self.assertEqual(lease.get_expiration_time(), 235 + DEFAULT_RENEWAL_TIME) # Time passes... clock.advance(835) @@ -1367,7 +1356,7 @@ class MutableServer(unittest.TestCase): # Adding a lease renews it: ss.remote_add_lease(b"si1", renew_secret, cancel_secret) [lease] = s0.get_leases() - self.assertEqual(lease.expiration_time, + self.assertEqual(lease.get_expiration_time(), 235 + 835 + DEFAULT_RENEWAL_TIME) def test_remove(self): diff --git a/src/allmydata/test/test_storage_web.py b/src/allmydata/test/test_storage_web.py index b3f5fac98..e905b240d 100644 --- a/src/allmydata/test/test_storage_web.py +++ b/src/allmydata/test/test_storage_web.py @@ -490,7 +490,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin): # current lease has), so we have to reach inside it. for i,lease in enumerate(sf.get_leases()): if lease.renew_secret == renew_secret: - lease.expiration_time = new_expire_time + lease = lease.renew(new_expire_time) f = open(sf.home, 'rb+') sf._write_lease_record(f, i, lease) f.close() From 76caf4634710344f839b8b8e58bfc424a124fdcc Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 28 Oct 2021 10:23:58 -0400 Subject: [PATCH 0327/2309] make the alternate LeaseInfo constructors into class methods --- src/allmydata/storage/immutable.py | 2 +- src/allmydata/storage/lease.py | 46 +++++++++++++++++++++--------- src/allmydata/storage/mutable.py | 2 +- 3 files changed, 35 insertions(+), 15 deletions(-) diff --git a/src/allmydata/storage/immutable.py b/src/allmydata/storage/immutable.py index 81470eed8..0042673f5 100644 --- a/src/allmydata/storage/immutable.py +++ b/src/allmydata/storage/immutable.py @@ -144,7 +144,7 @@ class ShareFile(object): for i in range(num_leases): data = f.read(self.LEASE_SIZE) if data: - yield LeaseInfo().from_immutable_data(data) + yield LeaseInfo.from_immutable_data(data) def add_lease(self, lease_info): with open(self.home, 'rb+') as f: diff --git a/src/allmydata/storage/lease.py b/src/allmydata/storage/lease.py index 594d61cf5..191edbe1a 100644 --- a/src/allmydata/storage/lease.py +++ b/src/allmydata/storage/lease.py @@ -75,13 +75,22 @@ class LeaseInfo(object): def get_age(self): return time.time() - self.get_grant_renew_time_time() - def from_immutable_data(self, data): - (self.owner_num, - self.renew_secret, - self.cancel_secret, - self._expiration_time) = struct.unpack(">L32s32sL", data) - self.nodeid = None - return self + @classmethod + def from_immutable_data(cls, data): + # type: (bytes) -> cls + """ + Create a new instance from the encoded data given. + + :param data: A lease serialized using the immutable-share-file format. + """ + names = [ + "owner_num", + "renew_secret", + "cancel_secret", + "expiration_time", + ] + values = struct.unpack(">L32s32sL", data) + return cls(nodeid=None, **dict(zip(names, values))) def to_immutable_data(self): return struct.pack(">L32s32sL", @@ -96,9 +105,20 @@ class LeaseInfo(object): self.renew_secret, self.cancel_secret, self.nodeid) - def from_mutable_data(self, data): - (self.owner_num, - self._expiration_time, - self.renew_secret, self.cancel_secret, - self.nodeid) = struct.unpack(">LL32s32s20s", data) - return self + @classmethod + def from_mutable_data(cls, data): + # (bytes) -> cls + """ + Create a new instance from the encoded data given. + + :param data: A lease serialized using the mutable-share-file format. + """ + names = [ + "owner_num", + "expiration_time", + "renew_secret", + "cancel_secret", + "nodeid", + ] + values = struct.unpack(">LL32s32s20s", data) + return cls(**dict(zip(names, values))) diff --git a/src/allmydata/storage/mutable.py b/src/allmydata/storage/mutable.py index 53a38fae9..e6f24679b 100644 --- a/src/allmydata/storage/mutable.py +++ b/src/allmydata/storage/mutable.py @@ -253,7 +253,7 @@ class MutableShareFile(object): f.seek(offset) assert f.tell() == offset data = f.read(self.LEASE_SIZE) - lease_info = LeaseInfo().from_mutable_data(data) + lease_info = LeaseInfo.from_mutable_data(data) if lease_info.owner_num == 0: return None return lease_info From 3514995068b7b132d3f7c590b4ecb8347f36655e Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 28 Oct 2021 10:26:30 -0400 Subject: [PATCH 0328/2309] some versions of mypy don't like this so nevermind --- src/allmydata/storage/lease.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/allmydata/storage/lease.py b/src/allmydata/storage/lease.py index 191edbe1a..17683a888 100644 --- a/src/allmydata/storage/lease.py +++ b/src/allmydata/storage/lease.py @@ -77,7 +77,6 @@ class LeaseInfo(object): @classmethod def from_immutable_data(cls, data): - # type: (bytes) -> cls """ Create a new instance from the encoded data given. @@ -107,7 +106,6 @@ class LeaseInfo(object): @classmethod def from_mutable_data(cls, data): - # (bytes) -> cls """ Create a new instance from the encoded data given. From 125c937d466db13e27ab06cc40e5d68aa5d93d28 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 28 Oct 2021 10:49:08 -0400 Subject: [PATCH 0329/2309] Switch to HTTP header scheme. --- docs/proposed/http-storage-node-protocol.rst | 38 ++++++++++++-------- 1 file changed, 23 insertions(+), 15 deletions(-) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index d5b6653be..fd1db5c4c 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -450,16 +450,22 @@ A lease is also created for the shares. Details of the buckets to create are encoded in the request body. For example:: - {"renew-secret": "efgh", "cancel-secret": "ijkl", - "upload-secret": "xyzf", - "share-numbers": [1, 7, ...], "allocated-size": 12345} + {"share-numbers": [1, 7, ...], "allocated-size": 12345} + +The request must include ``WWW-Authenticate`` HTTP headers that set the various secrets—upload, lease renewal, lease cancellation—that will be later used to authorize various operations. +Typically this is a header sent by the server, but in Tahoe-LAFS keys are set by the client, so may as well reuse it. +For example:: + + WWW-Authenticate: x-tahoe-renew-secret + WWW-Authenticate: x-tahoe-cancel-secret + WWW-Authenticate: x-tahoe-upload-secret The response body includes encoded information about the created buckets. For example:: {"already-have": [1, ...], "allocated": [7, ...]} -The uplaod secret is an opaque _byte_ string. +The upload secret is an opaque _byte_ string. It will be generated by hashing a combination of:b 1. A tag. @@ -521,9 +527,9 @@ If any one of these requests fails then at most 128KiB of upload work needs to b The server must recognize when all of the data has been received and mark the share as complete (which it can do because it was informed of the size when the storage index was initialized). -The request body looks this, with data and upload secret being bytes:: +The request must include a ``Authorization`` header that includes the upload secret:: - { "upload-secret": "xyzf", "data": "thedata" } + Authorization: x-tahoe-upload-secret Responses: @@ -727,9 +733,11 @@ Immutable Data 1. Create a bucket for storage index ``AAAAAAAAAAAAAAAA`` to hold two immutable shares, discovering that share ``1`` was already uploaded:: POST /v1/immutable/AAAAAAAAAAAAAAAA - {"renew-secret": "efgh", "cancel-secret": "ijkl", - "upload-secret": "xyzf", - "share-numbers": [1, 7], "allocated-size": 48} + WWW-Authenticate: x-tahoe-renew-secret efgh + WWW-Authenticate: x-tahoe-cancel-secret jjkl + WWW-Authenticate: x-tahoe-upload-secret xyzf + + {"share-numbers": [1, 7], "allocated-size": 48} 200 OK {"already-have": [1], "allocated": [7]} @@ -738,22 +746,22 @@ Immutable Data PATCH /v1/immutable/AAAAAAAAAAAAAAAA/7 Content-Range: bytes 0-15/48 - - {"upload-secret": b"xyzf", "data": "first 16 bytes!!" + Authorization: x-tahoe-upload-secret xyzf + 200 OK PATCH /v1/immutable/AAAAAAAAAAAAAAAA/7 Content-Range: bytes 16-31/48 - - {"upload-secret": "xyzf", "data": "second 16 bytes!" + Authorization: x-tahoe-upload-secret xyzf + 200 OK PATCH /v1/immutable/AAAAAAAAAAAAAAAA/7 Content-Range: bytes 32-47/48 - - {"upload-secret": "xyzf", "data": "final 16 bytes!!" + Authorization: x-tahoe-upload-secret xyzf + 201 CREATED From f635aec5bebda81ad1c073efa598e3546861ebf9 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 28 Oct 2021 10:53:29 -0400 Subject: [PATCH 0330/2309] news fragment --- newsfragments/3832.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3832.minor diff --git a/newsfragments/3832.minor b/newsfragments/3832.minor new file mode 100644 index 000000000..e69de29bb From 65d3ab614256a21430dc77b2982137de1cccfd8b Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 28 Oct 2021 10:53:52 -0400 Subject: [PATCH 0331/2309] move backdating logic into mutable/immutable share files --- src/allmydata/storage/immutable.py | 15 +++++++++++++-- src/allmydata/storage/mutable.py | 15 +++++++++++++-- src/allmydata/test/test_storage_web.py | 12 +----------- 3 files changed, 27 insertions(+), 15 deletions(-) diff --git a/src/allmydata/storage/immutable.py b/src/allmydata/storage/immutable.py index 0042673f5..7712e568a 100644 --- a/src/allmydata/storage/immutable.py +++ b/src/allmydata/storage/immutable.py @@ -152,11 +152,22 @@ class ShareFile(object): self._write_lease_record(f, num_leases, lease_info) self._write_num_leases(f, num_leases+1) - def renew_lease(self, renew_secret, new_expire_time): + def renew_lease(self, renew_secret, new_expire_time, allow_backdate=False): + # type: (bytes, int, bool) -> None + """ + Update the expiration time on an existing lease. + + :param allow_backdate: If ``True`` then allow the new expiration time + to be before the current expiration time. Otherwise, make no + change when this is the case. + + :raise IndexError: If there is no lease matching the given renew + secret. + """ for i,lease in enumerate(self.get_leases()): if timing_safe_compare(lease.renew_secret, renew_secret): # yup. See if we need to update the owner time. - if new_expire_time > lease.get_expiration_time(): + if allow_backdate or new_expire_time > lease.get_expiration_time(): # yes lease = lease.renew(new_expire_time) with open(self.home, 'rb+') as f: diff --git a/src/allmydata/storage/mutable.py b/src/allmydata/storage/mutable.py index e6f24679b..de840b89a 100644 --- a/src/allmydata/storage/mutable.py +++ b/src/allmydata/storage/mutable.py @@ -298,13 +298,24 @@ class MutableShareFile(object): else: self._write_lease_record(f, num_lease_slots, lease_info) - def renew_lease(self, renew_secret, new_expire_time): + def renew_lease(self, renew_secret, new_expire_time, allow_backdate=False): + # type: (bytes, int, bool) -> None + """ + Update the expiration time on an existing lease. + + :param allow_backdate: If ``True`` then allow the new expiration time + to be before the current expiration time. Otherwise, make no + change when this is the case. + + :raise IndexError: If there is no lease matching the given renew + secret. + """ accepting_nodeids = set() with open(self.home, 'rb+') as f: for (leasenum,lease) in self._enumerate_leases(f): if timing_safe_compare(lease.renew_secret, renew_secret): # yup. See if we need to update the owner time. - if new_expire_time > lease.get_expiration_time(): + if allow_backdate or new_expire_time > lease.get_expiration_time(): # yes lease = lease.renew(new_expire_time) self._write_lease_record(f, leasenum, lease) diff --git a/src/allmydata/test/test_storage_web.py b/src/allmydata/test/test_storage_web.py index e905b240d..38e380223 100644 --- a/src/allmydata/test/test_storage_web.py +++ b/src/allmydata/test/test_storage_web.py @@ -485,17 +485,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin): return d def backdate_lease(self, sf, renew_secret, new_expire_time): - # ShareFile.renew_lease ignores attempts to back-date a lease (i.e. - # "renew" a lease with a new_expire_time that is older than what the - # current lease has), so we have to reach inside it. - for i,lease in enumerate(sf.get_leases()): - if lease.renew_secret == renew_secret: - lease = lease.renew(new_expire_time) - f = open(sf.home, 'rb+') - sf._write_lease_record(f, i, lease) - f.close() - return - raise IndexError("unable to renew non-existent lease") + sf.renew_lease(renew_secret, new_expire_time, allow_backdate=True) def test_expire_age(self): basedir = "storage/LeaseCrawler/expire_age" From 54bf271fbebdb7d63642cc4d86dcf9507ae839df Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 28 Oct 2021 11:12:08 -0400 Subject: [PATCH 0332/2309] news fragment --- newsfragments/3833.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3833.minor diff --git a/newsfragments/3833.minor b/newsfragments/3833.minor new file mode 100644 index 000000000..e69de29bb From 34d2f74ede88107d6e30c927fdce704be06e2c3d Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 28 Oct 2021 11:12:17 -0400 Subject: [PATCH 0333/2309] Tell RTD how to install Sphinx. --- .readthedocs.yaml | 5 +++++ docs/requirements.txt | 4 ++++ newsfragments/3831.minor | 0 tox.ini | 7 +------ 4 files changed, 10 insertions(+), 6 deletions(-) create mode 100644 .readthedocs.yaml create mode 100644 docs/requirements.txt create mode 100644 newsfragments/3831.minor diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 000000000..65b390f26 --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,5 @@ +version: 2 + +python: + install: + - requirements: docs/requirements.txt diff --git a/docs/requirements.txt b/docs/requirements.txt new file mode 100644 index 000000000..39c4c20f0 --- /dev/null +++ b/docs/requirements.txt @@ -0,0 +1,4 @@ +sphinx +docutils<0.18 # https://github.com/sphinx-doc/sphinx/issues/9788 +recommonmark +sphinx_rtd_theme diff --git a/newsfragments/3831.minor b/newsfragments/3831.minor new file mode 100644 index 000000000..e69de29bb diff --git a/tox.ini b/tox.ini index 61a811b71..38cee1f9f 100644 --- a/tox.ini +++ b/tox.ini @@ -217,13 +217,8 @@ commands = # your web browser. [testenv:docs] -# we pin docutils because of https://sourceforge.net/p/docutils/bugs/301/ -# which asserts when it reads links to .svg files (e.g. about.rst) deps = - sphinx - docutils==0.12 - recommonmark - sphinx_rtd_theme + -r docs/requirements.txt # normal install is not needed for docs, and slows things down skip_install = True commands = From 66845c9a1786778e145aab65e30fb0068e2f8245 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 28 Oct 2021 11:12:20 -0400 Subject: [PATCH 0334/2309] Add ShareFile.is_valid_header and use it instead of manual header inspection --- src/allmydata/scripts/debug.py | 2 +- src/allmydata/storage/immutable.py | 15 +++++++++++++++ src/allmydata/storage/server.py | 2 +- src/allmydata/test/test_system.py | 8 ++++---- 4 files changed, 21 insertions(+), 6 deletions(-) diff --git a/src/allmydata/scripts/debug.py b/src/allmydata/scripts/debug.py index 4d3f4cb21..71e1ccb41 100644 --- a/src/allmydata/scripts/debug.py +++ b/src/allmydata/scripts/debug.py @@ -795,7 +795,7 @@ def describe_share(abs_sharefile, si_s, shnum_s, now, out): else: print("UNKNOWN mutable %s" % quote_output(abs_sharefile), file=out) - elif struct.unpack(">L", prefix[:4]) == (1,): + elif ShareFile.is_valid_header(prefix): # immutable class ImmediateReadBucketProxy(ReadBucketProxy): diff --git a/src/allmydata/storage/immutable.py b/src/allmydata/storage/immutable.py index 7712e568a..407116038 100644 --- a/src/allmydata/storage/immutable.py +++ b/src/allmydata/storage/immutable.py @@ -57,6 +57,21 @@ class ShareFile(object): LEASE_SIZE = struct.calcsize(">L32s32sL") sharetype = "immutable" + @classmethod + def is_valid_header(cls, header): + # (bytes) -> bool + """ + Determine if the given bytes constitute a valid header for this type of + container. + + :param header: Some bytes from the beginning of a container. + + :return: ``True`` if the bytes could belong to this container, + ``False`` otherwise. + """ + (version,) = struct.unpack(">L", header[:4]) + return version == 1 + def __init__(self, filename, max_size=None, create=False): """ If max_size is not None then I won't allow more than max_size to be written to me. If create=True and max_size must not be None. """ precondition((max_size is not None) or (not create), max_size, create) diff --git a/src/allmydata/storage/server.py b/src/allmydata/storage/server.py index 041783a4e..f339c579b 100644 --- a/src/allmydata/storage/server.py +++ b/src/allmydata/storage/server.py @@ -378,7 +378,7 @@ class StorageServer(service.MultiService, Referenceable): # note: if the share has been migrated, the renew_lease() # call will throw an exception, with information to help the # client update the lease. - elif header[:4] == struct.pack(">L", 1): + elif ShareFile.is_valid_header(header): sf = ShareFile(filename) else: continue # non-sharefile diff --git a/src/allmydata/test/test_system.py b/src/allmydata/test/test_system.py index 087a1c634..72ce4b6ec 100644 --- a/src/allmydata/test/test_system.py +++ b/src/allmydata/test/test_system.py @@ -22,7 +22,7 @@ from twisted.trial import unittest from twisted.internet import defer from allmydata import uri -from allmydata.storage.mutable import MutableShareFile +from allmydata.storage.mutable import ShareFile, MutableShareFile from allmydata.storage.server import si_a2b from allmydata.immutable import offloaded, upload from allmydata.immutable.literal import LiteralFileNode @@ -1290,9 +1290,9 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase): # are sharefiles here filename = os.path.join(dirpath, filenames[0]) # peek at the magic to see if it is a chk share - magic = open(filename, "rb").read(4) - if magic == b'\x00\x00\x00\x01': - break + with open(filename, "rb") as f: + if ShareFile.is_valid_header(f.read(32)): + break else: self.fail("unable to find any uri_extension files in %r" % self.basedir) From 1b46ac7a241e719cc0d7ddc4b66fa9fcdca5992d Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 28 Oct 2021 11:38:18 -0400 Subject: [PATCH 0335/2309] add MutableShareFile.is_valid_header and use it --- src/allmydata/scripts/debug.py | 287 ++++++++++++++--------------- src/allmydata/storage/immutable.py | 2 +- src/allmydata/storage/mutable.py | 18 +- src/allmydata/storage/server.py | 2 +- src/allmydata/storage/shares.py | 3 +- src/allmydata/test/common.py | 2 +- src/allmydata/test/test_system.py | 3 +- 7 files changed, 163 insertions(+), 154 deletions(-) diff --git a/src/allmydata/scripts/debug.py b/src/allmydata/scripts/debug.py index 71e1ccb41..ab48b0fd0 100644 --- a/src/allmydata/scripts/debug.py +++ b/src/allmydata/scripts/debug.py @@ -15,15 +15,22 @@ try: except ImportError: pass - -# do not import any allmydata modules at this level. Do that from inside -# individual functions instead. import struct, time, os, sys + from twisted.python import usage, failure from twisted.internet import defer from foolscap.logging import cli as foolscap_cli -from allmydata.scripts.common import BaseOptions +from allmydata.scripts.common import BaseOptions +from allmydata import uri +from allmydata.storage.mutable import MutableShareFile +from allmydata.storage.immutable import ShareFile +from allmydata.mutable.layout import unpack_share +from allmydata.mutable.layout import MDMFSlotReadProxy +from allmydata.mutable.common import NeedMoreDataError +from allmydata.immutable.layout import ReadBucketProxy +from allmydata.util import base32 +from allmydata.util.encodingutil import quote_output class DumpOptions(BaseOptions): def getSynopsis(self): @@ -56,13 +63,11 @@ def dump_share(options): # check the version, to see if we have a mutable or immutable share print("share filename: %s" % quote_output(options['filename']), file=out) - f = open(options['filename'], "rb") - prefix = f.read(32) - f.close() - if prefix == MutableShareFile.MAGIC: - return dump_mutable_share(options) - # otherwise assume it's immutable - return dump_immutable_share(options) + with open(options['filename'], "rb") as f: + if MutableShareFile.is_valid_header(f.read(32)): + return dump_mutable_share(options) + # otherwise assume it's immutable + return dump_immutable_share(options) def dump_immutable_share(options): from allmydata.storage.immutable import ShareFile @@ -712,125 +717,115 @@ def call(c, *args, **kwargs): return results[0] def describe_share(abs_sharefile, si_s, shnum_s, now, out): - from allmydata import uri - from allmydata.storage.mutable import MutableShareFile - from allmydata.storage.immutable import ShareFile - from allmydata.mutable.layout import unpack_share - from allmydata.mutable.common import NeedMoreDataError - from allmydata.immutable.layout import ReadBucketProxy - from allmydata.util import base32 - from allmydata.util.encodingutil import quote_output - import struct - - f = open(abs_sharefile, "rb") - prefix = f.read(32) - - if prefix == MutableShareFile.MAGIC: - # mutable share - m = MutableShareFile(abs_sharefile) - WE, nodeid = m._read_write_enabler_and_nodeid(f) - data_length = m._read_data_length(f) - expiration_time = min( [lease.get_expiration_time() - for (i,lease) in m._enumerate_leases(f)] ) - expiration = max(0, expiration_time - now) - - share_type = "unknown" - f.seek(m.DATA_OFFSET) - version = f.read(1) - if version == b"\x00": - # this slot contains an SMDF share - share_type = "SDMF" - elif version == b"\x01": - share_type = "MDMF" - - if share_type == "SDMF": - f.seek(m.DATA_OFFSET) - data = f.read(min(data_length, 2000)) - - try: - pieces = unpack_share(data) - except NeedMoreDataError as e: - # retry once with the larger size - size = e.needed_bytes - f.seek(m.DATA_OFFSET) - data = f.read(min(data_length, size)) - pieces = unpack_share(data) - (seqnum, root_hash, IV, k, N, segsize, datalen, - pubkey, signature, share_hash_chain, block_hash_tree, - share_data, enc_privkey) = pieces - - print("SDMF %s %d/%d %d #%d:%s %d %s" % \ - (si_s, k, N, datalen, - seqnum, str(base32.b2a(root_hash), "utf-8"), - expiration, quote_output(abs_sharefile)), file=out) - elif share_type == "MDMF": - from allmydata.mutable.layout import MDMFSlotReadProxy - fake_shnum = 0 - # TODO: factor this out with dump_MDMF_share() - class ShareDumper(MDMFSlotReadProxy): - def _read(self, readvs, force_remote=False, queue=False): - data = [] - for (where,length) in readvs: - f.seek(m.DATA_OFFSET+where) - data.append(f.read(length)) - return defer.succeed({fake_shnum: data}) - - p = ShareDumper(None, "fake-si", fake_shnum) - def extract(func): - stash = [] - # these methods return Deferreds, but we happen to know that - # they run synchronously when not actually talking to a - # remote server - d = func() - d.addCallback(stash.append) - return stash[0] - - verinfo = extract(p.get_verinfo) - (seqnum, root_hash, salt_to_use, segsize, datalen, k, N, prefix, - offsets) = verinfo - print("MDMF %s %d/%d %d #%d:%s %d %s" % \ - (si_s, k, N, datalen, - seqnum, str(base32.b2a(root_hash), "utf-8"), - expiration, quote_output(abs_sharefile)), file=out) + with open(abs_sharefile, "rb") as f: + prefix = f.read(32) + if MutableShareFile.is_valid_header(prefix): + _describe_mutable_share(abs_sharefile, f, now, si_s, out) + elif ShareFile.is_valid_header(prefix): + _describe_immutable_share(abs_sharefile, now, si_s, out) else: - print("UNKNOWN mutable %s" % quote_output(abs_sharefile), file=out) + print("UNKNOWN really-unknown %s" % quote_output(abs_sharefile), file=out) - elif ShareFile.is_valid_header(prefix): - # immutable +def _describe_mutable_share(abs_sharefile, f, now, si_s, out): + # mutable share + m = MutableShareFile(abs_sharefile) + WE, nodeid = m._read_write_enabler_and_nodeid(f) + data_length = m._read_data_length(f) + expiration_time = min( [lease.get_expiration_time() + for (i,lease) in m._enumerate_leases(f)] ) + expiration = max(0, expiration_time - now) - class ImmediateReadBucketProxy(ReadBucketProxy): - def __init__(self, sf): - self.sf = sf - ReadBucketProxy.__init__(self, None, None, "") - def __repr__(self): - return "" - def _read(self, offset, size): - return defer.succeed(sf.read_share_data(offset, size)) + share_type = "unknown" + f.seek(m.DATA_OFFSET) + version = f.read(1) + if version == b"\x00": + # this slot contains an SMDF share + share_type = "SDMF" + elif version == b"\x01": + share_type = "MDMF" - # use a ReadBucketProxy to parse the bucket and find the uri extension - sf = ShareFile(abs_sharefile) - bp = ImmediateReadBucketProxy(sf) + if share_type == "SDMF": + f.seek(m.DATA_OFFSET) + data = f.read(min(data_length, 2000)) - expiration_time = min( [lease.get_expiration_time() - for lease in sf.get_leases()] ) - expiration = max(0, expiration_time - now) + try: + pieces = unpack_share(data) + except NeedMoreDataError as e: + # retry once with the larger size + size = e.needed_bytes + f.seek(m.DATA_OFFSET) + data = f.read(min(data_length, size)) + pieces = unpack_share(data) + (seqnum, root_hash, IV, k, N, segsize, datalen, + pubkey, signature, share_hash_chain, block_hash_tree, + share_data, enc_privkey) = pieces - UEB_data = call(bp.get_uri_extension) - unpacked = uri.unpack_extension_readable(UEB_data) + print("SDMF %s %d/%d %d #%d:%s %d %s" % \ + (si_s, k, N, datalen, + seqnum, str(base32.b2a(root_hash), "utf-8"), + expiration, quote_output(abs_sharefile)), file=out) + elif share_type == "MDMF": + fake_shnum = 0 + # TODO: factor this out with dump_MDMF_share() + class ShareDumper(MDMFSlotReadProxy): + def _read(self, readvs, force_remote=False, queue=False): + data = [] + for (where,length) in readvs: + f.seek(m.DATA_OFFSET+where) + data.append(f.read(length)) + return defer.succeed({fake_shnum: data}) - k = unpacked["needed_shares"] - N = unpacked["total_shares"] - filesize = unpacked["size"] - ueb_hash = unpacked["UEB_hash"] - - print("CHK %s %d/%d %d %s %d %s" % (si_s, k, N, filesize, - str(ueb_hash, "utf-8"), expiration, - quote_output(abs_sharefile)), file=out) + p = ShareDumper(None, "fake-si", fake_shnum) + def extract(func): + stash = [] + # these methods return Deferreds, but we happen to know that + # they run synchronously when not actually talking to a + # remote server + d = func() + d.addCallback(stash.append) + return stash[0] + verinfo = extract(p.get_verinfo) + (seqnum, root_hash, salt_to_use, segsize, datalen, k, N, prefix, + offsets) = verinfo + print("MDMF %s %d/%d %d #%d:%s %d %s" % \ + (si_s, k, N, datalen, + seqnum, str(base32.b2a(root_hash), "utf-8"), + expiration, quote_output(abs_sharefile)), file=out) else: - print("UNKNOWN really-unknown %s" % quote_output(abs_sharefile), file=out) + print("UNKNOWN mutable %s" % quote_output(abs_sharefile), file=out) + + +def _describe_immutable_share(abs_sharefile, now, si_s, out): + class ImmediateReadBucketProxy(ReadBucketProxy): + def __init__(self, sf): + self.sf = sf + ReadBucketProxy.__init__(self, None, None, "") + def __repr__(self): + return "" + def _read(self, offset, size): + return defer.succeed(sf.read_share_data(offset, size)) + + # use a ReadBucketProxy to parse the bucket and find the uri extension + sf = ShareFile(abs_sharefile) + bp = ImmediateReadBucketProxy(sf) + + expiration_time = min( [lease.get_expiration_time() + for lease in sf.get_leases()] ) + expiration = max(0, expiration_time - now) + + UEB_data = call(bp.get_uri_extension) + unpacked = uri.unpack_extension_readable(UEB_data) + + k = unpacked["needed_shares"] + N = unpacked["total_shares"] + filesize = unpacked["size"] + ueb_hash = unpacked["UEB_hash"] + + print("CHK %s %d/%d %d %s %d %s" % (si_s, k, N, filesize, + str(ueb_hash, "utf-8"), expiration, + quote_output(abs_sharefile)), file=out) - f.close() def catalog_shares(options): from allmydata.util.encodingutil import listdir_unicode, quote_output @@ -933,34 +928,34 @@ def corrupt_share(options): f.write(d) f.close() - f = open(fn, "rb") - prefix = f.read(32) - f.close() - if prefix == MutableShareFile.MAGIC: - # mutable - m = MutableShareFile(fn) - f = open(fn, "rb") - f.seek(m.DATA_OFFSET) - data = f.read(2000) - # make sure this slot contains an SMDF share - assert data[0:1] == b"\x00", "non-SDMF mutable shares not supported" - f.close() + with open(fn, "rb") as f: + prefix = f.read(32) - (version, ig_seqnum, ig_roothash, ig_IV, ig_k, ig_N, ig_segsize, - ig_datalen, offsets) = unpack_header(data) + if MutableShareFile.is_valid_header(prefix): + # mutable + m = MutableShareFile(fn) + f = open(fn, "rb") + f.seek(m.DATA_OFFSET) + data = f.read(2000) + # make sure this slot contains an SMDF share + assert data[0:1] == b"\x00", "non-SDMF mutable shares not supported" + f.close() - assert version == 0, "we only handle v0 SDMF files" - start = m.DATA_OFFSET + offsets["share_data"] - end = m.DATA_OFFSET + offsets["enc_privkey"] - flip_bit(start, end) - else: - # otherwise assume it's immutable - f = ShareFile(fn) - bp = ReadBucketProxy(None, None, '') - offsets = bp._parse_offsets(f.read_share_data(0, 0x24)) - start = f._data_offset + offsets["data"] - end = f._data_offset + offsets["plaintext_hash_tree"] - flip_bit(start, end) + (version, ig_seqnum, ig_roothash, ig_IV, ig_k, ig_N, ig_segsize, + ig_datalen, offsets) = unpack_header(data) + + assert version == 0, "we only handle v0 SDMF files" + start = m.DATA_OFFSET + offsets["share_data"] + end = m.DATA_OFFSET + offsets["enc_privkey"] + flip_bit(start, end) + else: + # otherwise assume it's immutable + f = ShareFile(fn) + bp = ReadBucketProxy(None, None, '') + offsets = bp._parse_offsets(f.read_share_data(0, 0x24)) + start = f._data_offset + offsets["data"] + end = f._data_offset + offsets["plaintext_hash_tree"] + flip_bit(start, end) diff --git a/src/allmydata/storage/immutable.py b/src/allmydata/storage/immutable.py index 407116038..24465c1ed 100644 --- a/src/allmydata/storage/immutable.py +++ b/src/allmydata/storage/immutable.py @@ -59,7 +59,7 @@ class ShareFile(object): @classmethod def is_valid_header(cls, header): - # (bytes) -> bool + # type: (bytes) -> bool """ Determine if the given bytes constitute a valid header for this type of container. diff --git a/src/allmydata/storage/mutable.py b/src/allmydata/storage/mutable.py index de840b89a..1b29b4a65 100644 --- a/src/allmydata/storage/mutable.py +++ b/src/allmydata/storage/mutable.py @@ -67,6 +67,20 @@ class MutableShareFile(object): MAX_SIZE = MAX_MUTABLE_SHARE_SIZE # TODO: decide upon a policy for max share size + @classmethod + def is_valid_header(cls, header): + # type: (bytes) -> bool + """ + Determine if the given bytes constitute a valid header for this type of + container. + + :param header: Some bytes from the beginning of a container. + + :return: ``True`` if the bytes could belong to this container, + ``False`` otherwise. + """ + return header.startswith(cls.MAGIC) + def __init__(self, filename, parent=None): self.home = filename if os.path.exists(self.home): @@ -77,7 +91,7 @@ class MutableShareFile(object): write_enabler_nodeid, write_enabler, data_length, extra_least_offset) = \ struct.unpack(">32s20s32sQQ", data) - if magic != self.MAGIC: + if not self.is_valid_header(data): msg = "sharefile %s had magic '%r' but we wanted '%r'" % \ (filename, magic, self.MAGIC) raise UnknownMutableContainerVersionError(msg) @@ -388,7 +402,7 @@ class MutableShareFile(object): write_enabler_nodeid, write_enabler, data_length, extra_least_offset) = \ struct.unpack(">32s20s32sQQ", data) - assert magic == self.MAGIC + assert self.is_valid_header(data) return (write_enabler, write_enabler_nodeid) def readv(self, readv): diff --git a/src/allmydata/storage/server.py b/src/allmydata/storage/server.py index f339c579b..0f30dad6a 100644 --- a/src/allmydata/storage/server.py +++ b/src/allmydata/storage/server.py @@ -373,7 +373,7 @@ class StorageServer(service.MultiService, Referenceable): for shnum, filename in self._get_bucket_shares(storage_index): with open(filename, 'rb') as f: header = f.read(32) - if header[:32] == MutableShareFile.MAGIC: + if MutableShareFile.is_valid_header(header): sf = MutableShareFile(filename, self) # note: if the share has been migrated, the renew_lease() # call will throw an exception, with information to help the diff --git a/src/allmydata/storage/shares.py b/src/allmydata/storage/shares.py index ec6c0a501..59e7b1539 100644 --- a/src/allmydata/storage/shares.py +++ b/src/allmydata/storage/shares.py @@ -17,8 +17,7 @@ from allmydata.storage.immutable import ShareFile def get_share_file(filename): with open(filename, "rb") as f: prefix = f.read(32) - if prefix == MutableShareFile.MAGIC: + if MutableShareFile.is_valid_header(prefix): return MutableShareFile(filename) # otherwise assume it's immutable return ShareFile(filename) - diff --git a/src/allmydata/test/common.py b/src/allmydata/test/common.py index 0f2dc7c62..97368ee92 100644 --- a/src/allmydata/test/common.py +++ b/src/allmydata/test/common.py @@ -1068,7 +1068,7 @@ def _corrupt_offset_of_uri_extension_to_force_short_read(data, debug=False): def _corrupt_mutable_share_data(data, debug=False): prefix = data[:32] - assert prefix == MutableShareFile.MAGIC, "This function is designed to corrupt mutable shares of v1, and the magic number doesn't look right: %r vs %r" % (prefix, MutableShareFile.MAGIC) + assert MutableShareFile.is_valid_header(prefix), "This function is designed to corrupt mutable shares of v1, and the magic number doesn't look right: %r vs %r" % (prefix, MutableShareFile.MAGIC) data_offset = MutableShareFile.DATA_OFFSET sharetype = data[data_offset:data_offset+1] assert sharetype == b"\x00", "non-SDMF mutable shares not supported" diff --git a/src/allmydata/test/test_system.py b/src/allmydata/test/test_system.py index 72ce4b6ec..d859a0e00 100644 --- a/src/allmydata/test/test_system.py +++ b/src/allmydata/test/test_system.py @@ -22,7 +22,8 @@ from twisted.trial import unittest from twisted.internet import defer from allmydata import uri -from allmydata.storage.mutable import ShareFile, MutableShareFile +from allmydata.storage.mutable import MutableShareFile +from allmydata.storage.immutable import ShareFile from allmydata.storage.server import si_a2b from allmydata.immutable import offloaded, upload from allmydata.immutable.literal import LiteralFileNode From 8d202a4018bc5121800ea5551fd925d8432b2996 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 28 Oct 2021 12:37:37 -0400 Subject: [PATCH 0336/2309] news fragment --- newsfragments/3835.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3835.minor diff --git a/newsfragments/3835.minor b/newsfragments/3835.minor new file mode 100644 index 000000000..e69de29bb From d0ee17d99efff05738f0eb3140c3a5c947c20b5a Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 28 Oct 2021 12:39:01 -0400 Subject: [PATCH 0337/2309] some docstrings --- src/allmydata/test/no_network.py | 26 +++++++++++++++ src/allmydata/test/test_download.py | 52 ++++++++++++++++++++++++++++- 2 files changed, 77 insertions(+), 1 deletion(-) diff --git a/src/allmydata/test/no_network.py b/src/allmydata/test/no_network.py index 7a84580bf..aa41ab6bc 100644 --- a/src/allmydata/test/no_network.py +++ b/src/allmydata/test/no_network.py @@ -479,6 +479,18 @@ class GridTestMixin(object): def set_up_grid(self, num_clients=1, num_servers=10, client_config_hooks={}, oneshare=False): + """ + Create a Tahoe-LAFS storage grid. + + :param num_clients: See ``NoNetworkGrid`` + :param num_servers: See `NoNetworkGrid`` + :param client_config_hooks: See ``NoNetworkGrid`` + + :param bool oneshare: If ``True`` then the first client node is + configured with ``n == k == happy == 1``. + + :return: ``None`` + """ # self.basedir must be set port_assigner = SameProcessStreamEndpointAssigner() port_assigner.setUp() @@ -557,6 +569,15 @@ class GridTestMixin(object): return sorted(shares) def copy_shares(self, uri): + # type: (bytes) -> Dict[bytes, bytes] + """ + Read all of the share files for the given capability from the storage area + of the storage servers created by ``set_up_grid``. + + :param bytes uri: A Tahoe-LAFS data capability. + + :return: A ``dict`` mapping share file names to share file contents. + """ shares = {} for (shnum, serverid, sharefile) in self.find_uri_shares(uri): with open(sharefile, "rb") as f: @@ -601,6 +622,11 @@ class GridTestMixin(object): f.write(corruptdata) def corrupt_all_shares(self, uri, corruptor, debug=False): + # type: (bytes, Callable[[bytes, bool], bytes] -> bytes), bool) -> None + """ + Apply ``corruptor`` to the contents of all share files associated with a + given capability and replace the share file contents with its result. + """ for (i_shnum, i_serverid, i_sharefile) in self.find_uri_shares(uri): with open(i_sharefile, "rb") as f: sharedata = f.read() diff --git a/src/allmydata/test/test_download.py b/src/allmydata/test/test_download.py index d61942839..6b8dc6a31 100644 --- a/src/allmydata/test/test_download.py +++ b/src/allmydata/test/test_download.py @@ -951,12 +951,52 @@ class Corruption(_Base, unittest.TestCase): self.corrupt_shares_numbered(imm_uri, [2], _corruptor) def _corrupt_set(self, ign, imm_uri, which, newvalue): + # type: (Any, bytes, int, int) -> None + """ + Replace a single byte share file number 2 for the given capability with a + new byte. + + :param imm_uri: Corrupt share number 2 belonging to this capability. + :param which: The byte position to replace. + :param newvalue: The new byte value to set in the share. + """ log.msg("corrupt %d" % which) def _corruptor(s, debug=False): return s[:which] + bchr(newvalue) + s[which+1:] self.corrupt_shares_numbered(imm_uri, [2], _corruptor) def test_each_byte(self): + """ + Test share selection behavior of the downloader in the face of certain + kinds of data corruption. + + 1. upload a small share to the no-network grid + 2. read all of the resulting share files out of the no-network storage servers + 3. for each of + + a. each byte of the share file version field + b. each byte of the immutable share version field + c. each byte of the immutable share data offset field + d. the most significant byte of the block_shares offset field + e. one of the bytes of one of the merkle trees + f. one of the bytes of the share hashes list + + i. flip the least significant bit in all of the the share files + ii. perform the download/check/restore process + + 4. add 2 ** 24 to the share file version number + 5. perform the download/check/restore process + + 6. add 2 ** 24 to the share version number + 7. perform the download/check/restore process + + The download/check/restore process is: + + 1. attempt to download the data + 2. assert that the recovered plaintext is correct + 3. assert that only the "correct" share numbers were used to reconstruct the plaintext + 4. restore all of the share files to their pristine condition + """ # Setting catalog_detection=True performs an exhaustive test of the # Downloader's response to corruption in the lsb of each byte of the # 2070-byte share, with two goals: make sure we tolerate all forms of @@ -1145,8 +1185,18 @@ class Corruption(_Base, unittest.TestCase): return d def _corrupt_flip_all(self, ign, imm_uri, which): + # type: (Any, bytes, int) -> None + """ + Flip the least significant bit at a given byte position in all share files + for the given capability. + """ def _corruptor(s, debug=False): - return s[:which] + bchr(ord(s[which:which+1])^0x01) + s[which+1:] + # type: (bytes, bool) -> bytes + before_corruption = s[:which] + after_corruption = s[which+1:] + original_byte = s[which:which+1] + corrupt_byte = bchr(ord(original_byte) ^ 0x01) + return b"".join([before_corruption, corrupt_byte, after_corruption]) self.corrupt_all_shares(imm_uri, _corruptor) class DownloadV2(_Base, unittest.TestCase): From 8cb1f4f57cc6591e46573bd214ef0a7c43ad2c04 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 28 Oct 2021 14:25:24 -0400 Subject: [PATCH 0338/2309] news fragment --- newsfragments/3527.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3527.minor diff --git a/newsfragments/3527.minor b/newsfragments/3527.minor new file mode 100644 index 000000000..e69de29bb From 54d80222c9cdf6662510f67d15de0cf7494a723e Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 28 Oct 2021 14:34:47 -0400 Subject: [PATCH 0339/2309] switch to monkey-patching from other sources This is not much of an improvement to the tests themselves, unfortunately. However, it does get us one step closer to dropping `mock` as a dependency. --- src/allmydata/test/cli/test_create.py | 144 ++++++++++++++++---------- src/allmydata/test/common.py | 25 +++++ 2 files changed, 115 insertions(+), 54 deletions(-) diff --git a/src/allmydata/test/cli/test_create.py b/src/allmydata/test/cli/test_create.py index 282f26163..609888fb3 100644 --- a/src/allmydata/test/cli/test_create.py +++ b/src/allmydata/test/cli/test_create.py @@ -11,16 +11,24 @@ if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import os -import mock + +try: + from typing import Any, List, Tuple +except ImportError: + pass + from twisted.trial import unittest from twisted.internet import defer, reactor from twisted.python import usage from allmydata.util import configutil +from allmydata.util import tor_provider, i2p_provider from ..common_util import run_cli, parse_cli +from ..common import ( + disable_modules, +) from ...scripts import create_node from ... import client - def read_config(basedir): tahoe_cfg = os.path.join(basedir, "tahoe.cfg") config = configutil.get_config(tahoe_cfg) @@ -105,11 +113,12 @@ class Config(unittest.TestCase): @defer.inlineCallbacks def test_client_hide_ip_no_i2p_txtorcon(self): - # hmm, I must be doing something weird, these don't work as - # @mock.patch decorators for some reason - txi2p = mock.patch('allmydata.util.i2p_provider._import_txi2p', return_value=None) - txtorcon = mock.patch('allmydata.util.tor_provider._import_txtorcon', return_value=None) - with txi2p, txtorcon: + """ + The ``create-client`` sub-command tells the user to install the necessary + dependencies if they have neither tor nor i2p support installed and + they request network location privacy with the ``--hide-ip`` flag. + """ + with disable_modules("txi2p", "txtorcon"): basedir = self.mktemp() rc, out, err = yield run_cli("create-client", "--hide-ip", basedir) self.assertTrue(rc != 0, out) @@ -118,8 +127,7 @@ class Config(unittest.TestCase): @defer.inlineCallbacks def test_client_i2p_option_no_txi2p(self): - txi2p = mock.patch('allmydata.util.i2p_provider._import_txi2p', return_value=None) - with txi2p: + with disable_modules("txi2p"): basedir = self.mktemp() rc, out, err = yield run_cli("create-node", "--listen=i2p", "--i2p-launch", basedir) self.assertTrue(rc != 0) @@ -127,8 +135,7 @@ class Config(unittest.TestCase): @defer.inlineCallbacks def test_client_tor_option_no_txtorcon(self): - txtorcon = mock.patch('allmydata.util.tor_provider._import_txtorcon', return_value=None) - with txtorcon: + with disable_modules("txtorcon"): basedir = self.mktemp() rc, out, err = yield run_cli("create-node", "--listen=tor", "--tor-launch", basedir) self.assertTrue(rc != 0) @@ -145,9 +152,7 @@ class Config(unittest.TestCase): @defer.inlineCallbacks def test_client_hide_ip_no_txtorcon(self): - txtorcon = mock.patch('allmydata.util.tor_provider._import_txtorcon', - return_value=None) - with txtorcon: + with disable_modules("txtorcon"): basedir = self.mktemp() rc, out, err = yield run_cli("create-client", "--hide-ip", basedir) self.assertEqual(0, rc) @@ -295,11 +300,10 @@ class Config(unittest.TestCase): def test_node_slow_tor(self): basedir = self.mktemp() d = defer.Deferred() - with mock.patch("allmydata.util.tor_provider.create_config", - return_value=d): - d2 = run_cli("create-node", "--listen=tor", basedir) - d.callback(({}, "port", "location")) - rc, out, err = yield d2 + self.patch(tor_provider, "create_config", lambda *a, **kw: d) + d2 = run_cli("create-node", "--listen=tor", basedir) + d.callback(({}, "port", "location")) + rc, out, err = yield d2 self.assertEqual(rc, 0) self.assertIn("Node created", out) self.assertEqual(err, "") @@ -308,11 +312,10 @@ class Config(unittest.TestCase): def test_node_slow_i2p(self): basedir = self.mktemp() d = defer.Deferred() - with mock.patch("allmydata.util.i2p_provider.create_config", - return_value=d): - d2 = run_cli("create-node", "--listen=i2p", basedir) - d.callback(({}, "port", "location")) - rc, out, err = yield d2 + self.patch(i2p_provider, "create_config", lambda *a, **kw: d) + d2 = run_cli("create-node", "--listen=i2p", basedir) + d.callback(({}, "port", "location")) + rc, out, err = yield d2 self.assertEqual(rc, 0) self.assertIn("Node created", out) self.assertEqual(err, "") @@ -353,6 +356,27 @@ class Config(unittest.TestCase): self.assertIn("is not empty", err) self.assertIn("To avoid clobbering anything, I am going to quit now", err) +def fake_config(testcase, module, result): + # type: (unittest.TestCase, Any, Any) -> List[Tuple] + """ + Monkey-patch a fake configuration function into the given module. + + :param testcase: The test case to use to do the monkey-patching. + + :param module: The module into which to patch the fake function. + + :param result: The return value for the fake function. + + :return: A list of tuples of the arguments the fake function was called + with. + """ + calls = [] + def fake_config(reactor, cli_config): + calls.append((reactor, cli_config)) + return result + testcase.patch(module, "create_config", fake_config) + return calls + class Tor(unittest.TestCase): def test_default(self): basedir = self.mktemp() @@ -360,12 +384,14 @@ class Tor(unittest.TestCase): tor_port = "ghi" tor_location = "jkl" config_d = defer.succeed( (tor_config, tor_port, tor_location) ) - with mock.patch("allmydata.util.tor_provider.create_config", - return_value=config_d) as co: - rc, out, err = self.successResultOf( - run_cli("create-node", "--listen=tor", basedir)) - self.assertEqual(len(co.mock_calls), 1) - args = co.mock_calls[0][1] + + calls = fake_config(self, tor_provider, config_d) + rc, out, err = self.successResultOf( + run_cli("create-node", "--listen=tor", basedir), + ) + + self.assertEqual(len(calls), 1) + args = calls[0] self.assertIdentical(args[0], reactor) self.assertIsInstance(args[1], create_node.CreateNodeOptions) self.assertEqual(args[1]["listen"], "tor") @@ -380,12 +406,15 @@ class Tor(unittest.TestCase): tor_port = "ghi" tor_location = "jkl" config_d = defer.succeed( (tor_config, tor_port, tor_location) ) - with mock.patch("allmydata.util.tor_provider.create_config", - return_value=config_d) as co: - rc, out, err = self.successResultOf( - run_cli("create-node", "--listen=tor", "--tor-launch", - basedir)) - args = co.mock_calls[0][1] + + calls = fake_config(self, tor_provider, config_d) + rc, out, err = self.successResultOf( + run_cli( + "create-node", "--listen=tor", "--tor-launch", + basedir, + ), + ) + args = calls[0] self.assertEqual(args[1]["listen"], "tor") self.assertEqual(args[1]["tor-launch"], True) self.assertEqual(args[1]["tor-control-port"], None) @@ -396,12 +425,15 @@ class Tor(unittest.TestCase): tor_port = "ghi" tor_location = "jkl" config_d = defer.succeed( (tor_config, tor_port, tor_location) ) - with mock.patch("allmydata.util.tor_provider.create_config", - return_value=config_d) as co: - rc, out, err = self.successResultOf( - run_cli("create-node", "--listen=tor", "--tor-control-port=mno", - basedir)) - args = co.mock_calls[0][1] + + calls = fake_config(self, tor_provider, config_d) + rc, out, err = self.successResultOf( + run_cli( + "create-node", "--listen=tor", "--tor-control-port=mno", + basedir, + ), + ) + args = calls[0] self.assertEqual(args[1]["listen"], "tor") self.assertEqual(args[1]["tor-launch"], False) self.assertEqual(args[1]["tor-control-port"], "mno") @@ -434,12 +466,13 @@ class I2P(unittest.TestCase): i2p_port = "ghi" i2p_location = "jkl" dest_d = defer.succeed( (i2p_config, i2p_port, i2p_location) ) - with mock.patch("allmydata.util.i2p_provider.create_config", - return_value=dest_d) as co: - rc, out, err = self.successResultOf( - run_cli("create-node", "--listen=i2p", basedir)) - self.assertEqual(len(co.mock_calls), 1) - args = co.mock_calls[0][1] + + calls = fake_config(self, i2p_provider, dest_d) + rc, out, err = self.successResultOf( + run_cli("create-node", "--listen=i2p", basedir), + ) + self.assertEqual(len(calls), 1) + args = calls[0] self.assertIdentical(args[0], reactor) self.assertIsInstance(args[1], create_node.CreateNodeOptions) self.assertEqual(args[1]["listen"], "i2p") @@ -461,12 +494,15 @@ class I2P(unittest.TestCase): i2p_port = "ghi" i2p_location = "jkl" dest_d = defer.succeed( (i2p_config, i2p_port, i2p_location) ) - with mock.patch("allmydata.util.i2p_provider.create_config", - return_value=dest_d) as co: - rc, out, err = self.successResultOf( - run_cli("create-node", "--listen=i2p", "--i2p-sam-port=mno", - basedir)) - args = co.mock_calls[0][1] + + calls = fake_config(self, i2p_provider, dest_d) + rc, out, err = self.successResultOf( + run_cli( + "create-node", "--listen=i2p", "--i2p-sam-port=mno", + basedir, + ), + ) + args = calls[0] self.assertEqual(args[1]["listen"], "i2p") self.assertEqual(args[1]["i2p-launch"], False) self.assertEqual(args[1]["i2p-sam-port"], "mno") diff --git a/src/allmydata/test/common.py b/src/allmydata/test/common.py index 0f2dc7c62..2e6da9801 100644 --- a/src/allmydata/test/common.py +++ b/src/allmydata/test/common.py @@ -26,8 +26,14 @@ __all__ = [ "PIPE", ] +try: + from typing import Tuple, ContextManager +except ImportError: + pass + import sys import os, random, struct +from contextlib import contextmanager import six import tempfile from tempfile import mktemp @@ -1213,6 +1219,25 @@ class ConstantAddresses(object): raise Exception("{!r} has no client endpoint.") return self._handler +@contextmanager +def disable_modules(*names): + # type: (Tuple[str]) -> ContextManager + """ + A context manager which makes modules appear to be missing while it is + active. + + :param *names: The names of the modules to disappear. + """ + missing = object() + modules = list(sys.modules.get(n, missing) for n in names) + for n in names: + sys.modules[n] = None + yield + for n, original in zip(names, modules): + if original is missing: + del sys.modules[n] + else: + sys.modules[n] = original class _TestCaseMixin(object): """ From 8d5727977b9a1a7865954db30f9d4771518b97c0 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 28 Oct 2021 14:47:42 -0400 Subject: [PATCH 0340/2309] it doesn't typecheck, nevermind --- src/allmydata/test/common.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/allmydata/test/common.py b/src/allmydata/test/common.py index 2e6da9801..8e97fa598 100644 --- a/src/allmydata/test/common.py +++ b/src/allmydata/test/common.py @@ -26,11 +26,6 @@ __all__ = [ "PIPE", ] -try: - from typing import Tuple, ContextManager -except ImportError: - pass - import sys import os, random, struct from contextlib import contextmanager @@ -1221,7 +1216,6 @@ class ConstantAddresses(object): @contextmanager def disable_modules(*names): - # type: (Tuple[str]) -> ContextManager """ A context manager which makes modules appear to be missing while it is active. From f8655f149bb0754013adc985a6041738f18327f2 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 28 Oct 2021 15:04:19 -0400 Subject: [PATCH 0341/2309] fix the type annotations and such --- src/allmydata/test/no_network.py | 9 +++++++-- src/allmydata/test/test_download.py | 5 +++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/no_network.py b/src/allmydata/test/no_network.py index aa41ab6bc..b9fa99005 100644 --- a/src/allmydata/test/no_network.py +++ b/src/allmydata/test/no_network.py @@ -25,6 +25,11 @@ if PY2: from past.builtins import unicode from six import ensure_text +try: + from typing import Dict, Callable +except ImportError: + pass + import os from base64 import b32encode from functools import ( @@ -622,7 +627,7 @@ class GridTestMixin(object): f.write(corruptdata) def corrupt_all_shares(self, uri, corruptor, debug=False): - # type: (bytes, Callable[[bytes, bool], bytes] -> bytes), bool) -> None + # type: (bytes, Callable[[bytes, bool], bytes], bool) -> None """ Apply ``corruptor`` to the contents of all share files associated with a given capability and replace the share file contents with its result. @@ -630,7 +635,7 @@ class GridTestMixin(object): for (i_shnum, i_serverid, i_sharefile) in self.find_uri_shares(uri): with open(i_sharefile, "rb") as f: sharedata = f.read() - corruptdata = corruptor(sharedata, debug=debug) + corruptdata = corruptor(sharedata, debug) with open(i_sharefile, "wb") as f: f.write(corruptdata) diff --git a/src/allmydata/test/test_download.py b/src/allmydata/test/test_download.py index 6b8dc6a31..aeea9642e 100644 --- a/src/allmydata/test/test_download.py +++ b/src/allmydata/test/test_download.py @@ -14,6 +14,11 @@ if PY2: # a previous run. This asserts that the current code is capable of decoding # shares from a previous version. +try: + from typing import Any +except ImportError: + pass + import six import os from twisted.trial import unittest From 78dbe7699403dbe38f94d574367f5d5e95916f4a Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 28 Oct 2021 15:20:44 -0400 Subject: [PATCH 0342/2309] remove unused import --- src/allmydata/storage/server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/storage/server.py b/src/allmydata/storage/server.py index 0f30dad6a..3e2d3b5c6 100644 --- a/src/allmydata/storage/server.py +++ b/src/allmydata/storage/server.py @@ -14,7 +14,7 @@ if PY2: else: from typing import Dict -import os, re, struct, time +import os, re, time import six from foolscap.api import Referenceable From 8b976b441e793f45b50e5d5ebcb4314beba889ee Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 28 Oct 2021 12:05:34 -0400 Subject: [PATCH 0343/2309] add LeaseInfo.is_renew_secret and use it --- src/allmydata/storage/immutable.py | 2 +- src/allmydata/storage/lease.py | 12 +++++++++ src/allmydata/storage/mutable.py | 2 +- src/allmydata/test/test_storage.py | 39 ++++++++++++++++++++++-------- 4 files changed, 43 insertions(+), 12 deletions(-) diff --git a/src/allmydata/storage/immutable.py b/src/allmydata/storage/immutable.py index 24465c1ed..c9b8995b5 100644 --- a/src/allmydata/storage/immutable.py +++ b/src/allmydata/storage/immutable.py @@ -180,7 +180,7 @@ class ShareFile(object): secret. """ for i,lease in enumerate(self.get_leases()): - if timing_safe_compare(lease.renew_secret, renew_secret): + if lease.is_renew_secret(renew_secret): # yup. See if we need to update the owner time. if allow_backdate or new_expire_time > lease.get_expiration_time(): # yes diff --git a/src/allmydata/storage/lease.py b/src/allmydata/storage/lease.py index 17683a888..2132048ce 100644 --- a/src/allmydata/storage/lease.py +++ b/src/allmydata/storage/lease.py @@ -15,6 +15,8 @@ import struct, time import attr +from allmydata.util.hashutil import timing_safe_compare + @attr.s(frozen=True) class LeaseInfo(object): """ @@ -68,6 +70,16 @@ class LeaseInfo(object): _expiration_time=new_expire_time, ) + def is_renew_secret(self, candidate_secret): + # type: (bytes) -> bool + """ + Check a string to see if it is the correct renew secret. + + :return: ``True`` if it is the correct renew secret, ``False`` + otherwise. + """ + return timing_safe_compare(self.renew_secret, candidate_secret) + def get_grant_renew_time_time(self): # hack, based upon fixed 31day expiration period return self._expiration_time - 31*24*60*60 diff --git a/src/allmydata/storage/mutable.py b/src/allmydata/storage/mutable.py index 1b29b4a65..017f2dbb7 100644 --- a/src/allmydata/storage/mutable.py +++ b/src/allmydata/storage/mutable.py @@ -327,7 +327,7 @@ class MutableShareFile(object): accepting_nodeids = set() with open(self.home, 'rb+') as f: for (leasenum,lease) in self._enumerate_leases(f): - if timing_safe_compare(lease.renew_secret, renew_secret): + if lease.is_renew_secret(renew_secret): # yup. See if we need to update the owner time. if allow_backdate or new_expire_time > lease.get_expiration_time(): # yes diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index 8123be2c5..005309f87 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -755,28 +755,28 @@ class Server(unittest.TestCase): # Create a bucket: rs0, cs0 = self.create_bucket_5_shares(ss, b"si0") - leases = list(ss.get_leases(b"si0")) - self.failUnlessEqual(len(leases), 1) - self.failUnlessEqual(set([l.renew_secret for l in leases]), set([rs0])) + (lease,) = ss.get_leases(b"si0") + self.assertTrue(lease.is_renew_secret(rs0)) rs1, cs1 = self.create_bucket_5_shares(ss, b"si1") # take out a second lease on si1 rs2, cs2 = self.create_bucket_5_shares(ss, b"si1", 5, 0) - leases = list(ss.get_leases(b"si1")) - self.failUnlessEqual(len(leases), 2) - self.failUnlessEqual(set([l.renew_secret for l in leases]), set([rs1, rs2])) + (lease1, lease2) = ss.get_leases(b"si1") + self.assertTrue(lease1.is_renew_secret(rs1)) + self.assertTrue(lease2.is_renew_secret(rs2)) # and a third lease, using add-lease rs2a,cs2a = (hashutil.my_renewal_secret_hash(b"%d" % next(self._lease_secret)), hashutil.my_cancel_secret_hash(b"%d" % next(self._lease_secret))) ss.remote_add_lease(b"si1", rs2a, cs2a) - leases = list(ss.get_leases(b"si1")) - self.failUnlessEqual(len(leases), 3) - self.failUnlessEqual(set([l.renew_secret for l in leases]), set([rs1, rs2, rs2a])) + (lease1, lease2, lease3) = ss.get_leases(b"si1") + self.assertTrue(lease1.is_renew_secret(rs1)) + self.assertTrue(lease2.is_renew_secret(rs2)) + self.assertTrue(lease3.is_renew_secret(rs2a)) # add-lease on a missing storage index is silently ignored - self.failUnlessEqual(ss.remote_add_lease(b"si18", b"", b""), None) + self.assertIsNone(ss.remote_add_lease(b"si18", b"", b"")) # check that si0 is readable readers = ss.remote_get_buckets(b"si0") @@ -3028,3 +3028,22 @@ class ShareFileTests(unittest.TestCase): sf = self.get_sharefile() with self.assertRaises(IndexError): sf.cancel_lease(b"garbage") + + def test_renew_secret(self): + """ + A lease loaded from a share file can have its renew secret verified. + """ + renew_secret = b"r" * 32 + cancel_secret = b"c" * 32 + expiration_time = 2 ** 31 + + sf = self.get_sharefile() + lease = LeaseInfo( + owner_num=0, + renew_secret=renew_secret, + cancel_secret=cancel_secret, + expiration_time=expiration_time, + ) + sf.add_lease(lease) + (loaded_lease,) = sf.get_leases() + self.assertTrue(loaded_lease.is_renew_secret(renew_secret)) From b5f882ffa60574f193a18e70e3c310077a2f097e Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 28 Oct 2021 12:21:22 -0400 Subject: [PATCH 0344/2309] introduce and use LeaseInfo.is_cancel_secret --- src/allmydata/storage/immutable.py | 2 +- src/allmydata/storage/lease.py | 10 ++++++++++ src/allmydata/storage/mutable.py | 2 +- src/allmydata/test/test_storage.py | 19 +++++++++++++++++++ 4 files changed, 31 insertions(+), 2 deletions(-) diff --git a/src/allmydata/storage/immutable.py b/src/allmydata/storage/immutable.py index c9b8995b5..4f6a1c9c7 100644 --- a/src/allmydata/storage/immutable.py +++ b/src/allmydata/storage/immutable.py @@ -209,7 +209,7 @@ class ShareFile(object): leases = list(self.get_leases()) num_leases_removed = 0 for i,lease in enumerate(leases): - if timing_safe_compare(lease.cancel_secret, cancel_secret): + if lease.is_cancel_secret(cancel_secret): leases[i] = None num_leases_removed += 1 if not num_leases_removed: diff --git a/src/allmydata/storage/lease.py b/src/allmydata/storage/lease.py index 2132048ce..ff96ebaf4 100644 --- a/src/allmydata/storage/lease.py +++ b/src/allmydata/storage/lease.py @@ -80,6 +80,16 @@ class LeaseInfo(object): """ return timing_safe_compare(self.renew_secret, candidate_secret) + def is_cancel_secret(self, candidate_secret): + # type: (bytes) -> bool + """ + Check a string to see if it is the correct cancel secret. + + :return: ``True`` if it is the correct cancel secret, ``False`` + otherwise. + """ + return timing_safe_compare(self.cancel_secret, candidate_secret) + def get_grant_renew_time_time(self): # hack, based upon fixed 31day expiration period return self._expiration_time - 31*24*60*60 diff --git a/src/allmydata/storage/mutable.py b/src/allmydata/storage/mutable.py index 017f2dbb7..9480a3c03 100644 --- a/src/allmydata/storage/mutable.py +++ b/src/allmydata/storage/mutable.py @@ -371,7 +371,7 @@ class MutableShareFile(object): with open(self.home, 'rb+') as f: for (leasenum,lease) in self._enumerate_leases(f): accepting_nodeids.add(lease.nodeid) - if timing_safe_compare(lease.cancel_secret, cancel_secret): + if lease.is_cancel_secret(cancel_secret): self._write_lease_record(f, leasenum, blank_lease) modified += 1 else: diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index 005309f87..aac40362c 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -3047,3 +3047,22 @@ class ShareFileTests(unittest.TestCase): sf.add_lease(lease) (loaded_lease,) = sf.get_leases() self.assertTrue(loaded_lease.is_renew_secret(renew_secret)) + + def test_cancel_secret(self): + """ + A lease loaded from a share file can have its cancel secret verified. + """ + renew_secret = b"r" * 32 + cancel_secret = b"c" * 32 + expiration_time = 2 ** 31 + + sf = self.get_sharefile() + lease = LeaseInfo( + owner_num=0, + renew_secret=renew_secret, + cancel_secret=cancel_secret, + expiration_time=expiration_time, + ) + sf.add_lease(lease) + (loaded_lease,) = sf.get_leases() + self.assertTrue(loaded_lease.is_cancel_secret(cancel_secret)) From 696a260ddfc02be35a63ed1446eda0b5434cc86f Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 29 Oct 2021 09:00:38 -0400 Subject: [PATCH 0345/2309] news fragment --- newsfragments/3836.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3836.minor diff --git a/newsfragments/3836.minor b/newsfragments/3836.minor new file mode 100644 index 000000000..e69de29bb From 892b4683654cd1281be8feeaa65a2ef946ed4f5a Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 29 Oct 2021 09:03:37 -0400 Subject: [PATCH 0346/2309] use the port assigner to assign a port for the main tub --- src/allmydata/test/common_system.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/common_system.py b/src/allmydata/test/common_system.py index 9d14c8642..874c7f6ba 100644 --- a/src/allmydata/test/common_system.py +++ b/src/allmydata/test/common_system.py @@ -672,11 +672,14 @@ class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin): """ iv_dir = self.getdir("introducer") if not os.path.isdir(iv_dir): - _, port_endpoint = self.port_assigner.assign(reactor) + _, web_port_endpoint = self.port_assigner.assign(reactor) + main_location_hint, main_port_endpoint = self.port_assigner.assign(reactor) introducer_config = ( u"[node]\n" u"nickname = introducer \N{BLACK SMILING FACE}\n" + - u"web.port = {}\n".format(port_endpoint) + u"web.port = {}\n".format(web_port_endpoint) + + u"tub.port = {}\n".format(main_port_endpoint) + + u"tub.location = {}\n".format(main_location_hint) ).encode("utf-8") fileutil.make_dirs(iv_dir) From ffe23452a4e83b6f912d2b6c94584f10235ed457 Mon Sep 17 00:00:00 2001 From: fenn-cs Date: Sat, 30 Oct 2021 13:32:42 +0100 Subject: [PATCH 0347/2309] gpg setup Signed-off-by: fenn-cs --- docs/release-checklist.rst | 36 +++++++++++++++++++++++++++++------- 1 file changed, 29 insertions(+), 7 deletions(-) diff --git a/docs/release-checklist.rst b/docs/release-checklist.rst index dc060cd8d..75a9e2f4a 100644 --- a/docs/release-checklist.rst +++ b/docs/release-checklist.rst @@ -56,7 +56,7 @@ process is complete.* Create Branch and Apply Updates ``````````````````````````````` -- Create a branch for the release (e.g. `XXXX.release-1.16.0`) +- Create a branch for the release/candidate (e.g. `XXXX.release-1.16.0`) - run ``tox -e news`` to produce a new NEWS.txt file (this does a commit) - create the news for the release @@ -92,6 +92,27 @@ Create Branch and Apply Updates - Confirm CI runs successfully on all platforms +Preparing to Authenticate Release (Setting up GPG) +`````````````````````````````````````````````````` +*Skip the section if you already have GPG setup.* + +In other to keep releases authentic it's required that releases are signed before being +published. This ensure's that users of Tahoe are able to verify that the version of Tahoe +they are using is coming from a trusted or at the very least known source. + +The authentication is done using the ``GPG`` implementation of ``OpenGPG`` to be able to complete +the release steps you would have to download the ``GPG`` software and setup a key(identity). + +- `Download `__ and install GPG for your operating system. +- Generate a key pair using ``gpg --gen-key``. *Some questions would be asked to personalize your key configuration.* + +You might take additional steps including: + +- Setting up a revocation certificate (Incase you lose your secret key) +- Backing up your key pair +- Upload your fingerprint to a keyserver such as `openpgp.org `__ + + Create Release Candidate ```````````````````````` @@ -108,8 +129,10 @@ they will need to evaluate which contributors' signatures they trust. - (all steps above are completed) - sign the release - - git tag -s -u 0xE34E62D06D0E69CFCA4179FFBDE0D31D68666A7A -m "release Tahoe-LAFS-1.15.0rc0" tahoe-lafs-1.15.0rc0 - - (replace the key-id above with your own) + - ``git tag -s -u 0xE34E62D06D0E69CFCA4179FFBDE0D31D68666A7A -m "release Tahoe-LAFS-1.16.0rc0" tahoe-lafs-1.16.0rc0`` + +*Replace the key-id above with your own, which can simply be your email if's attached your fingerprint.* +*Don't forget to put the correct tag message and name in this example the tag message is "release Tahoe-LAFS-1.16.0rc0" and the tag name is `tahoe-lafs-1.16.0rc0`* - build all code locally - these should all pass: @@ -123,8 +146,7 @@ they will need to evaluate which contributors' signatures they trust. - build tarballs - tox -e tarballs - - confirm it at least exists: - - ls dist/ | grep 1.15.0rc0 + - Confirm that release tarballs exist by runnig: ``ls dist/ | grep 1.16.0rc0`` - inspect and test the tarballs @@ -133,8 +155,8 @@ they will need to evaluate which contributors' signatures they trust. - when satisfied, sign the tarballs: - - gpg --pinentry=loopback --armor --detach-sign dist/tahoe_lafs-1.15.0rc0-py2.py3-none-any.whl - - gpg --pinentry=loopback --armor --detach-sign dist/tahoe_lafs-1.15.0rc0.tar.gz + - ``gpg --pinentry=loopback --armor --detach-sign dist/tahoe_lafs-1.16.0rc0-py2.py3-none-any.whl`` + - ``gpg --pinentry=loopback --armor --detach-sign dist/tahoe_lafs-1.16.0rc0.tar.gz`` Privileged Contributor From 882f1973062df9b3d1aec49d468621e46f72dfb6 Mon Sep 17 00:00:00 2001 From: fenn-cs Date: Sat, 30 Oct 2021 13:37:58 +0100 Subject: [PATCH 0348/2309] format updates Signed-off-by: fenn-cs --- docs/release-checklist.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/release-checklist.rst b/docs/release-checklist.rst index 75a9e2f4a..403a6f933 100644 --- a/docs/release-checklist.rst +++ b/docs/release-checklist.rst @@ -50,13 +50,13 @@ previously generated files. practice to give it the release name. You MAY also discard this directory once the release process is complete.* -- ``cd into the release directory and install dependencies by running ``python -m venv venv && source venv/bin/activate && pip install --editable .[test]```` +- ``cd`` into the release directory and install dependencies by running ``python -m venv venv && source venv/bin/activate && pip install --editable .[test]`` Create Branch and Apply Updates ``````````````````````````````` -- Create a branch for the release/candidate (e.g. `XXXX.release-1.16.0`) +- Create a branch for the release/candidate (e.g. ``XXXX.release-1.16.0``) - run ``tox -e news`` to produce a new NEWS.txt file (this does a commit) - create the news for the release @@ -73,7 +73,7 @@ Create Branch and Apply Updates - update "relnotes.txt" - - update all mentions of 1.16.0 -> 1.16.x + - update all mentions of ``1.16.0`` to new and higher release version for example ``1.16.1`` - update "previous release" statement and date - summarize major changes - commit it From 5ba636c7b10fd146c39eff9a60c34f9eb5943a9a Mon Sep 17 00:00:00 2001 From: fenn-cs Date: Tue, 2 Nov 2021 10:36:32 +0100 Subject: [PATCH 0349/2309] removed deferred logger from basic function in test_logs Signed-off-by: fenn-cs --- src/allmydata/test/web/test_logs.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/allmydata/test/web/test_logs.py b/src/allmydata/test/web/test_logs.py index fe0a0445d..81ec357c0 100644 --- a/src/allmydata/test/web/test_logs.py +++ b/src/allmydata/test/web/test_logs.py @@ -54,9 +54,7 @@ from ...web.logs import ( TokenAuthenticatedWebSocketServerProtocol, ) -from ...util.eliotutil import ( - log_call_deferred -) +from eliot import log_call class StreamingEliotLogsTests(SyncTestCase): """ @@ -110,7 +108,7 @@ class TestStreamingLogs(AsyncTestCase): messages.append(json.loads(msg)) proto.on("message", got_message) - @log_call_deferred(action_type=u"test:cli:some-exciting-action") + @log_call(action_type=u"test:cli:some-exciting-action") def do_a_thing(arguments): pass @@ -121,7 +119,7 @@ class TestStreamingLogs(AsyncTestCase): self.assertThat(len(messages), Equals(3)) self.assertThat(messages[0]["action_type"], Equals("test:cli:some-exciting-action")) - self.assertThat(messages[0]["kwargs"]["arguments"], + self.assertThat(messages[0]["arguments"], Equals(["hello", "good-\\xff-day", 123, {"a": 35}, [None]])) self.assertThat(messages[1]["action_type"], Equals("test:cli:some-exciting-action")) self.assertThat("started", Equals(messages[0]["action_status"])) From fcfc89e3ae4d2a73ba110b2b23eaf24001e78dd9 Mon Sep 17 00:00:00 2001 From: fenn-cs Date: Tue, 2 Nov 2021 14:32:20 +0100 Subject: [PATCH 0350/2309] moved new tests/update for eliotutils Signed-off-by: fenn-cs --- src/allmydata/test/test_eliotutil.py | 73 ---------------------------- src/allmydata/util/eliotutil.py | 22 +-------- 2 files changed, 2 insertions(+), 93 deletions(-) diff --git a/src/allmydata/test/test_eliotutil.py b/src/allmydata/test/test_eliotutil.py index 61e0a6958..3f915ecd2 100644 --- a/src/allmydata/test/test_eliotutil.py +++ b/src/allmydata/test/test_eliotutil.py @@ -56,7 +56,6 @@ from eliot.testing import ( capture_logging, assertHasAction, swap_logger, - assertContainsFields, ) from twisted.internet.defer import ( @@ -282,75 +281,3 @@ class LogCallDeferredTests(TestCase): ), ), ) - - @capture_logging( - lambda self, logger: - assertHasAction(self, logger, u"the-action", succeeded=True), - ) - def test_gets_positional_arguments(self, logger): - """ - Check that positional arguments are logged when using ``log_call_deferred`` - """ - @log_call_deferred(action_type=u"the-action") - def f(a): - return a ** 2 - self.assertThat( - f(4), succeeded(Equals(16))) - msg = logger.messages[0] - assertContainsFields(self, msg, {"args": (4,)}) - - @capture_logging( - lambda self, logger: - assertHasAction(self, logger, u"the-action", succeeded=True), - ) - def test_gets_keyword_arguments(self, logger): - """ - Check that keyword arguments are logged when using ``log_call_deferred`` - """ - @log_call_deferred(action_type=u"the-action") - def f(base, exp): - return base ** exp - self.assertThat(f(exp=2,base=10), succeeded(Equals(100))) - msg = logger.messages[0] - assertContainsFields(self, msg, {"kwargs": {"base": 10, "exp": 2}}) - - - @capture_logging( - lambda self, logger: - assertHasAction(self, logger, u"the-action", succeeded=True), - ) - def test_gets_keyword_and_positional_arguments(self, logger): - """ - Check that both keyword and positional arguments are logged when using ``log_call_deferred`` - """ - @log_call_deferred(action_type=u"the-action") - def f(base, exp, message): - return base ** exp - self.assertThat(f(10, 2, message="an exponential function"), succeeded(Equals(100))) - msg = logger.messages[0] - assertContainsFields(self, msg, {"args": (10, 2)}) - assertContainsFields(self, msg, {"kwargs": {"message": "an exponential function"}}) - - - @capture_logging( - lambda self, logger: - assertHasAction(self, logger, u"the-action", succeeded=True), - ) - def test_keyword_args_dont_overlap_with_start_action(self, logger): - """ - Check that kwargs passed to decorated functions don't overlap with params in ``start_action`` - """ - @log_call_deferred(action_type=u"the-action") - def f(base, exp, kwargs, args): - return base ** exp - self.assertThat( - f(10, 2, kwargs={"kwarg_1": "value_1", "kwarg_2": 2}, args=(1, 2, 3)), - succeeded(Equals(100)), - ) - msg = logger.messages[0] - assertContainsFields(self, msg, {"args": (10, 2)}) - assertContainsFields( - self, - msg, - {"kwargs": {"args": [1, 2, 3], "kwargs": {"kwarg_1": "value_1", "kwarg_2": 2}}}, - ) diff --git a/src/allmydata/util/eliotutil.py b/src/allmydata/util/eliotutil.py index fe431568f..4e48fbb9f 100644 --- a/src/allmydata/util/eliotutil.py +++ b/src/allmydata/util/eliotutil.py @@ -87,11 +87,7 @@ from twisted.internet.defer import ( ) from twisted.application.service import Service -from .jsonbytes import ( - AnyBytesJSONEncoder, - bytes_to_unicode -) -import json +from .jsonbytes import AnyBytesJSONEncoder def validateInstanceOf(t): @@ -315,14 +311,6 @@ class _DestinationParser(object): _parse_destination_description = _DestinationParser().parse -def is_json_serializable(object): - try: - json.dumps(object) - return True - except (TypeError, OverflowError): - return False - - def log_call_deferred(action_type): """ Like ``eliot.log_call`` but for functions which return ``Deferred``. @@ -332,11 +320,7 @@ def log_call_deferred(action_type): def logged_f(*a, **kw): # Use the action's context method to avoid ending the action when # the `with` block ends. - kwargs = {k: bytes_to_unicode(True, kw[k]) for k in kw} - # Remove complex (unserializable) objects from positional args to - # prevent eliot from throwing errors when it attempts serialization - args = tuple(arg if is_json_serializable(arg) else str(arg) for arg in a) - with start_action(action_type=action_type, args=args, kwargs=kwargs).context(): + with start_action(action_type=action_type).context(): # Use addActionFinish so that the action finishes when the # Deferred fires. d = maybeDeferred(f, *a, **kw) @@ -350,5 +334,3 @@ if PY2: capture_logging = eliot_capture_logging else: capture_logging = partial(eliot_capture_logging, encoder_=AnyBytesJSONEncoder) - - From 39c4a2c4eb1963b2035644d97c1760b649c21278 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 2 Nov 2021 15:10:54 -0400 Subject: [PATCH 0351/2309] tidy up some corners --- src/allmydata/scripts/debug.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/src/allmydata/scripts/debug.py b/src/allmydata/scripts/debug.py index ab48b0fd0..260cca55b 100644 --- a/src/allmydata/scripts/debug.py +++ b/src/allmydata/scripts/debug.py @@ -746,6 +746,13 @@ def _describe_mutable_share(abs_sharefile, f, now, si_s, out): if share_type == "SDMF": f.seek(m.DATA_OFFSET) + + # Read at least the mutable header length, if possible. If there's + # less data than that in the share, don't try to read more (we won't + # be able to unpack the header in this case but we surely don't want + # to try to unpack bytes *following* the data section as if they were + # header data). Rather than 2000 we could use HEADER_LENGTH from + # allmydata/mutable/layout.py, probably. data = f.read(min(data_length, 2000)) try: @@ -810,8 +817,8 @@ def _describe_immutable_share(abs_sharefile, now, si_s, out): sf = ShareFile(abs_sharefile) bp = ImmediateReadBucketProxy(sf) - expiration_time = min( [lease.get_expiration_time() - for lease in sf.get_leases()] ) + expiration_time = min(lease.get_expiration_time() + for lease in sf.get_leases()) expiration = max(0, expiration_time - now) UEB_data = call(bp.get_uri_extension) @@ -934,9 +941,10 @@ def corrupt_share(options): if MutableShareFile.is_valid_header(prefix): # mutable m = MutableShareFile(fn) - f = open(fn, "rb") - f.seek(m.DATA_OFFSET) - data = f.read(2000) + with open(fn, "rb") as f: + f.seek(m.DATA_OFFSET) + # Read enough data to get a mutable header to unpack. + data = f.read(2000) # make sure this slot contains an SMDF share assert data[0:1] == b"\x00", "non-SDMF mutable shares not supported" f.close() From 08cf881e28f84e74865284697234e9760776d9f5 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 2 Nov 2021 22:16:14 -0600 Subject: [PATCH 0352/2309] test with real-size keys --- src/allmydata/test/common.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/allmydata/test/common.py b/src/allmydata/test/common.py index 38282297a..fc268311e 100644 --- a/src/allmydata/test/common.py +++ b/src/allmydata/test/common.py @@ -133,6 +133,7 @@ from subprocess import ( ) TEST_RSA_KEY_SIZE = 522 +TEST_RSA_KEY_SIZE = 2048 EMPTY_CLIENT_CONFIG = config_from_string( "/dev/null", From b3d1acd14a1f602df5bba424214070a4643a8bab Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 3 Nov 2021 09:55:16 -0400 Subject: [PATCH 0353/2309] try skipping Tor integration tests on Python 2 --- integration/test_tor.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/integration/test_tor.py b/integration/test_tor.py index 15d888e36..b0419f0d2 100644 --- a/integration/test_tor.py +++ b/integration/test_tor.py @@ -35,6 +35,9 @@ from allmydata.test.common import ( if sys.platform.startswith('win'): pytest.skip('Skipping Tor tests on Windows', allow_module_level=True) +if PY2: + pytest.skip('Skipping Tor tests on Python 2 because dependencies are hard to come by', allow_module_level=True) + @pytest_twisted.inlineCallbacks def test_onion_service_storage(reactor, request, temp_dir, flog_gatherer, tor_network, tor_introducer_furl): yield _create_anonymous_node(reactor, 'carol', 8008, request, temp_dir, flog_gatherer, tor_network, tor_introducer_furl) From 4606c3c9dde91de11d769bf9d8c6fd6f2fd1f877 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 3 Nov 2021 09:59:19 -0400 Subject: [PATCH 0354/2309] news fragment --- newsfragments/3837.other | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/3837.other diff --git a/newsfragments/3837.other b/newsfragments/3837.other new file mode 100644 index 000000000..a9e4e6986 --- /dev/null +++ b/newsfragments/3837.other @@ -0,0 +1 @@ +Tahoe-LAFS no longer runs its Tor integration test suite on Python 2 due to the increased complexity of obtaining compatible versions of necessary dependencies. From 8e150cce6a27b6616db54cfd4c2ac08fbdd13794 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 3 Nov 2021 13:14:55 -0400 Subject: [PATCH 0355/2309] add explicit direct tests for the new methods --- src/allmydata/test/test_storage.py | 61 ++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index aac40362c..460653bd0 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -3066,3 +3066,64 @@ class ShareFileTests(unittest.TestCase): sf.add_lease(lease) (loaded_lease,) = sf.get_leases() self.assertTrue(loaded_lease.is_cancel_secret(cancel_secret)) + + +class LeaseInfoTests(unittest.TestCase): + """ + Tests for ``allmydata.storage.lease.LeaseInfo``. + """ + def test_is_renew_secret(self): + """ + ``LeaseInfo.is_renew_secret`` returns ``True`` if the value given is the + renew secret. + """ + renew_secret = b"r" * 32 + cancel_secret = b"c" * 32 + lease = LeaseInfo( + owner_num=1, + renew_secret=renew_secret, + cancel_secret=cancel_secret, + ) + self.assertTrue(lease.is_renew_secret(renew_secret)) + + def test_is_not_renew_secret(self): + """ + ``LeaseInfo.is_renew_secret`` returns ``False`` if the value given is not + the renew secret. + """ + renew_secret = b"r" * 32 + cancel_secret = b"c" * 32 + lease = LeaseInfo( + owner_num=1, + renew_secret=renew_secret, + cancel_secret=cancel_secret, + ) + self.assertFalse(lease.is_renew_secret(cancel_secret)) + + def test_is_cancel_secret(self): + """ + ``LeaseInfo.is_cancel_secret`` returns ``True`` if the value given is the + cancel secret. + """ + renew_secret = b"r" * 32 + cancel_secret = b"c" * 32 + lease = LeaseInfo( + owner_num=1, + renew_secret=renew_secret, + cancel_secret=cancel_secret, + ) + self.assertTrue(lease.is_cancel_secret(cancel_secret)) + + def test_is_not_cancel_secret(self): + """ + ``LeaseInfo.is_cancel_secret`` returns ``False`` if the value given is not + the cancel secret. + """ + renew_secret = b"r" * 32 + cancel_secret = b"c" * 32 + lease = LeaseInfo( + owner_num=1, + renew_secret=renew_secret, + cancel_secret=cancel_secret, + ) + self.assertFalse(lease.is_cancel_secret(renew_secret)) From 7335b2a5977752c0805a7fd9c7759cafa8ac31b1 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 3 Nov 2021 13:16:15 -0400 Subject: [PATCH 0356/2309] remove unused import --- src/allmydata/storage/immutable.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/allmydata/storage/immutable.py b/src/allmydata/storage/immutable.py index 4f6a1c9c7..8a7a5a966 100644 --- a/src/allmydata/storage/immutable.py +++ b/src/allmydata/storage/immutable.py @@ -24,7 +24,6 @@ from allmydata.interfaces import ( ) from allmydata.util import base32, fileutil, log from allmydata.util.assertutil import precondition -from allmydata.util.hashutil import timing_safe_compare from allmydata.storage.lease import LeaseInfo from allmydata.storage.common import UnknownImmutableContainerVersionError From 86ca463c3198746e31b61569f79e860f4a6e7d6d Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 3 Nov 2021 13:24:04 -0400 Subject: [PATCH 0357/2309] news fragment --- newsfragments/3834.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3834.minor diff --git a/newsfragments/3834.minor b/newsfragments/3834.minor new file mode 100644 index 000000000..e69de29bb From 797e0994596dd916a978b5fc8a757d15322b3100 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 3 Nov 2021 16:05:28 -0400 Subject: [PATCH 0358/2309] make create_introducer_webish assign a main tub port --- src/allmydata/test/web/test_introducer.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/allmydata/test/web/test_introducer.py b/src/allmydata/test/web/test_introducer.py index 4b5850cbc..69309d35b 100644 --- a/src/allmydata/test/web/test_introducer.py +++ b/src/allmydata/test/web/test_introducer.py @@ -83,12 +83,18 @@ def create_introducer_webish(reactor, port_assigner, basedir): with the node and its webish service. """ node.create_node_dir(basedir, "testing") - _, port_endpoint = port_assigner.assign(reactor) + main_tub_location, main_tub_endpoint = port_assigner.assign(reactor) + _, web_port_endpoint = port_assigner.assign(reactor) with open(join(basedir, "tahoe.cfg"), "w") as f: f.write( "[node]\n" - "tub.location = 127.0.0.1:1\n" + - "web.port = {}\n".format(port_endpoint) + "tub.port = {main_tub_endpoint}\n" + "tub.location = {main_tub_location}\n" + "web.port = {web_port_endpoint}\n".format( + main_tub_endpoint=main_tub_endpoint, + main_tub_location=main_tub_location, + web_port_endpoint=web_port_endpoint, + ) ) intro_node = yield create_introducer(basedir) From 31649890ef47c2169a0aedee2d7488b8f6da6959 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 3 Nov 2021 16:08:08 -0400 Subject: [PATCH 0359/2309] Teach UseNode to use a port assigner for tub.port Then use it to assign ports for tub.port unless the caller supplied their own value. --- src/allmydata/test/common.py | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/common.py b/src/allmydata/test/common.py index 97368ee92..e0472edce 100644 --- a/src/allmydata/test/common.py +++ b/src/allmydata/test/common.py @@ -267,8 +267,12 @@ class UseNode(object): node_config = attr.ib(default=attr.Factory(dict)) config = attr.ib(default=None) + reactor = attr.ib(default=None) def setUp(self): + self.assigner = SameProcessStreamEndpointAssigner() + self.assigner.setUp() + def format_config_items(config): return "\n".join( " = ".join((key, value)) @@ -292,6 +296,23 @@ class UseNode(object): "default", self.introducer_furl, ) + + node_config = self.node_config.copy() + if "tub.port" not in node_config: + if "tub.location" in node_config: + raise ValueError( + "UseNode fixture does not support specifying tub.location " + "without tub.port" + ) + + # Don't use the normal port auto-assignment logic. It produces + # collisions and makes tests fail spuriously. + tub_location, tub_endpoint = self.assigner.assign(self.reactor) + node_config.update({ + "tub.port": tub_endpoint, + "tub.location": tub_location, + }) + self.config = config_from_string( self.basedir.asTextMode().path, "tub.port", @@ -304,7 +325,7 @@ storage.plugins = {storage_plugin} {plugin_config_section} """.format( storage_plugin=self.storage_plugin, - node_config=format_config_items(self.node_config), + node_config=format_config_items(node_config), plugin_config_section=plugin_config_section, ) ) @@ -316,7 +337,7 @@ storage.plugins = {storage_plugin} ) def cleanUp(self): - pass + self.assigner.tearDown() def getDetails(self): From 5a71774bf875a71c8ddbfb8b4fcfcb2dda7a4f9d Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 3 Nov 2021 16:10:32 -0400 Subject: [PATCH 0360/2309] use port assigner and UseNode more in test_node.py --- src/allmydata/test/test_node.py | 36 +++++++++++++++++++++++++-------- 1 file changed, 28 insertions(+), 8 deletions(-) diff --git a/src/allmydata/test/test_node.py b/src/allmydata/test/test_node.py index cf5fa27f3..c6cff1bab 100644 --- a/src/allmydata/test/test_node.py +++ b/src/allmydata/test/test_node.py @@ -69,6 +69,8 @@ import allmydata.test.common_util as testutil from .common import ( ConstantAddresses, + SameProcessStreamEndpointAssigner, + UseNode, ) def port_numbers(): @@ -80,11 +82,10 @@ class LoggingMultiService(service.MultiService): # see https://tahoe-lafs.org/trac/tahoe-lafs/ticket/2946 -def testing_tub(config_data=''): +def testing_tub(reactor, config_data=''): """ Creates a 'main' Tub for testing purposes, from config data """ - from twisted.internet import reactor basedir = 'dummy_basedir' config = config_from_string(basedir, 'DEFAULT_PORTNUMFILE_BLANK', config_data) fileutil.make_dirs(os.path.join(basedir, 'private')) @@ -112,6 +113,9 @@ class TestCase(testutil.SignalMixin, unittest.TestCase): # try to bind the port. We'll use a low-numbered one that's likely to # conflict with another service to prove it. self._available_port = 22 + self.port_assigner = SameProcessStreamEndpointAssigner() + self.port_assigner.setUp() + self.addCleanup(self.port_assigner.tearDown) def _test_location( self, @@ -137,11 +141,23 @@ class TestCase(testutil.SignalMixin, unittest.TestCase): :param local_addresses: If not ``None`` then a list of addresses to supply to the system under test as local addresses. """ + from twisted.internet import reactor + basedir = self.mktemp() create_node_dir(basedir, "testing") + if tub_port is None: + # Always configure a usable tub.port address instead of relying on + # the automatic port assignment. The automatic port assignment is + # prone to collisions and spurious test failures. + _, tub_port = self.port_assigner.assign(reactor) + config_data = "[node]\n" - if tub_port: - config_data += "tub.port = {}\n".format(tub_port) + config_data += "tub.port = {}\n".format(tub_port) + + # If they wanted a certain location, go for it. This probably won't + # agree with the tub.port value we set but that only matters if + # anything tries to use this to establish a connection ... which + # nothing in this test suite will. if tub_location is not None: config_data += "tub.location = {}\n".format(tub_location) @@ -149,7 +165,7 @@ class TestCase(testutil.SignalMixin, unittest.TestCase): self.patch(iputil, 'get_local_addresses_sync', lambda: local_addresses) - tub = testing_tub(config_data) + tub = testing_tub(reactor, config_data) class Foo(object): pass @@ -431,7 +447,12 @@ class TestCase(testutil.SignalMixin, unittest.TestCase): @defer.inlineCallbacks def test_logdir_is_str(self): - basedir = "test_node/test_logdir_is_str" + from twisted.internet import reactor + + basedir = FilePath(self.mktemp()) + fixture = UseNode(None, None, basedir, "pb://introducer/furl", {}, reactor=reactor) + fixture.setUp() + self.addCleanup(fixture.cleanUp) ns = Namespace() ns.called = False @@ -440,8 +461,7 @@ class TestCase(testutil.SignalMixin, unittest.TestCase): self.failUnless(isinstance(logdir, str), logdir) self.patch(foolscap.logging.log, 'setLogDir', call_setLogDir) - create_node_dir(basedir, "nothing to see here") - yield client.create_client(basedir) + yield fixture.create_node() self.failUnless(ns.called) def test_set_config_unescaped_furl_hash(self): From 5caa80fe383630aab8afa8a9a1667fb3d4cd8f60 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 3 Nov 2021 16:11:08 -0400 Subject: [PATCH 0361/2309] use UseNode more in test_client.py Also make write_introducer more lenient about filesystem state --- src/allmydata/scripts/common.py | 4 +++- src/allmydata/test/test_client.py | 16 +++++++++------- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/src/allmydata/scripts/common.py b/src/allmydata/scripts/common.py index 0a9ab8714..c9fc8e031 100644 --- a/src/allmydata/scripts/common.py +++ b/src/allmydata/scripts/common.py @@ -141,7 +141,9 @@ def write_introducer(basedir, petname, furl): """ if isinstance(furl, bytes): furl = furl.decode("utf-8") - basedir.child(b"private").child(b"introducers.yaml").setContent( + private = basedir.child(b"private") + private.makedirs(ignoreExistingDirectory=True) + private.child(b"introducers.yaml").setContent( safe_dump({ "introducers": { petname: { diff --git a/src/allmydata/test/test_client.py b/src/allmydata/test/test_client.py index fd2837f1d..a2572e735 100644 --- a/src/allmydata/test/test_client.py +++ b/src/allmydata/test/test_client.py @@ -89,6 +89,7 @@ from .common import ( UseTestPlugins, MemoryIntroducerClient, get_published_announcements, + UseNode, ) from .matchers import ( MatchesSameElements, @@ -953,13 +954,14 @@ class Run(unittest.TestCase, testutil.StallMixin): @defer.inlineCallbacks def test_reloadable(self): - basedir = FilePath("test_client.Run.test_reloadable") - private = basedir.child("private") - private.makedirs() + from twisted.internet import reactor + dummy = "pb://wl74cyahejagspqgy4x5ukrvfnevlknt@127.0.0.1:58889/bogus" - write_introducer(basedir, "someintroducer", dummy) - basedir.child("tahoe.cfg").setContent(BASECONFIG. encode("ascii")) - c1 = yield client.create_client(basedir.path) + fixture = UseNode(None, None, FilePath(self.mktemp()), dummy, reactor=reactor) + fixture.setUp() + self.addCleanup(fixture.cleanUp) + + c1 = yield fixture.create_node() c1.setServiceParent(self.sparent) # delay to let the service start up completely. I'm not entirely sure @@ -981,7 +983,7 @@ class Run(unittest.TestCase, testutil.StallMixin): # also change _check_exit_trigger to use it instead of a raw # reactor.stop, also instrument the shutdown event in an # attribute that we can check.) - c2 = yield client.create_client(basedir.path) + c2 = yield fixture.create_node() c2.setServiceParent(self.sparent) yield c2.disownServiceParent() From 780be2691b9ca4a7b1b2d08c6d0bb44b11d8b9a1 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 3 Nov 2021 16:11:28 -0400 Subject: [PATCH 0362/2309] assign a tub.port to all system test nodes --- src/allmydata/test/common_system.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/src/allmydata/test/common_system.py b/src/allmydata/test/common_system.py index 874c7f6ba..0c424136a 100644 --- a/src/allmydata/test/common_system.py +++ b/src/allmydata/test/common_system.py @@ -767,13 +767,15 @@ class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin): def _generate_config(self, which, basedir): config = {} - except1 = set(range(self.numclients)) - {1} + allclients = set(range(self.numclients)) + except1 = allclients - {1} feature_matrix = { ("client", "nickname"): except1, - # client 1 has to auto-assign an address. - ("node", "tub.port"): except1, - ("node", "tub.location"): except1, + # Auto-assigning addresses is extremely failure prone and not + # amenable to automated testing in _this_ manner. + ("node", "tub.port"): allclients, + ("node", "tub.location"): allclients, # client 0 runs a webserver and a helper # client 3 runs a webserver but no helper @@ -855,7 +857,13 @@ class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin): # connection-lost code basedir = FilePath(self.getdir("client%d" % client_num)) basedir.makedirs() - config = "[client]\n" + config = ( + "[node]\n" + "tub.location = {}\n" + "tub.port = {}\n" + "[client]\n" + ).format(*self.port_assigner.assign(reactor)) + if helper_furl: config += "helper.furl = %s\n" % helper_furl basedir.child("tahoe.cfg").setContent(config.encode("utf-8")) From b4bc95cb5a36b7507ce3745cfc3a273b5eedecb6 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 3 Nov 2021 16:15:38 -0400 Subject: [PATCH 0363/2309] news fragment --- newsfragments/3838.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3838.minor diff --git a/newsfragments/3838.minor b/newsfragments/3838.minor new file mode 100644 index 000000000..e69de29bb From 0459b712b02b8ba686687d696325bcdb650f770c Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 4 Nov 2021 08:54:55 -0400 Subject: [PATCH 0364/2309] news fragment --- newsfragments/3839.security | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/3839.security diff --git a/newsfragments/3839.security b/newsfragments/3839.security new file mode 100644 index 000000000..1ae054542 --- /dev/null +++ b/newsfragments/3839.security @@ -0,0 +1 @@ +The storage server now keeps hashes of lease renew and cancel secrets for immutable share files instead of keeping the original secrets. From 274dc6e837dd7181fb1f6ba9116570dc4b255d66 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 4 Nov 2021 08:55:37 -0400 Subject: [PATCH 0365/2309] Introduce `UnknownContainerVersionError` base w/ structured args --- src/allmydata/storage/common.py | 11 ++++++++--- src/allmydata/storage/immutable.py | 4 +--- src/allmydata/storage/mutable.py | 4 +--- src/allmydata/test/test_storage.py | 7 ++++--- 4 files changed, 14 insertions(+), 12 deletions(-) diff --git a/src/allmydata/storage/common.py b/src/allmydata/storage/common.py index e5563647f..48fc77840 100644 --- a/src/allmydata/storage/common.py +++ b/src/allmydata/storage/common.py @@ -16,11 +16,16 @@ from allmydata.util import base32 # Backwards compatibility. from allmydata.interfaces import DataTooLargeError # noqa: F401 -class UnknownMutableContainerVersionError(Exception): - pass -class UnknownImmutableContainerVersionError(Exception): +class UnknownContainerVersionError(Exception): + def __init__(self, filename, version): + self.filename = filename + self.version = version + +class UnknownMutableContainerVersionError(UnknownContainerVersionError): pass +class UnknownImmutableContainerVersionError(UnknownContainerVersionError): + pass def si_b2a(storageindex): return base32.b2a(storageindex) diff --git a/src/allmydata/storage/immutable.py b/src/allmydata/storage/immutable.py index a43860138..fcc60509c 100644 --- a/src/allmydata/storage/immutable.py +++ b/src/allmydata/storage/immutable.py @@ -174,9 +174,7 @@ class ShareFile(object): filesize = os.path.getsize(self.home) (version, unused, num_leases) = struct.unpack(">LLL", f.read(0xc)) if version != 1: - msg = "sharefile %s had version %d but we wanted 1" % \ - (filename, version) - raise UnknownImmutableContainerVersionError(msg) + raise UnknownImmutableContainerVersionError(filename, version) self._num_leases = num_leases self._lease_offset = filesize - (num_leases * self.LEASE_SIZE) self._data_offset = 0xc diff --git a/src/allmydata/storage/mutable.py b/src/allmydata/storage/mutable.py index 4abf22064..ce9cc5ff4 100644 --- a/src/allmydata/storage/mutable.py +++ b/src/allmydata/storage/mutable.py @@ -95,9 +95,7 @@ class MutableShareFile(object): data_length, extra_least_offset) = \ struct.unpack(">32s20s32sQQ", data) if not self.is_valid_header(data): - msg = "sharefile %s had magic '%r' but we wanted '%r'" % \ - (filename, magic, self.MAGIC) - raise UnknownMutableContainerVersionError(msg) + raise UnknownMutableContainerVersionError(filename, magic) self.parent = parent # for logging def log(self, *args, **kwargs): diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index 2c8d84b9e..bf9eff37a 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -646,7 +646,8 @@ class Server(unittest.TestCase): e = self.failUnlessRaises(UnknownImmutableContainerVersionError, ss.remote_get_buckets, b"si1") - self.failUnlessIn(" had version 0 but we wanted 1", str(e)) + self.assertEqual(e.filename, fn) + self.assertEqual(e.version, 0) def test_disconnect(self): # simulate a disconnection @@ -1127,8 +1128,8 @@ class MutableServer(unittest.TestCase): read = ss.remote_slot_readv e = self.failUnlessRaises(UnknownMutableContainerVersionError, read, b"si1", [0], [(0,10)]) - self.failUnlessIn(" had magic ", str(e)) - self.failUnlessIn(" but we wanted ", str(e)) + self.assertEqual(e.filename, fn) + self.assertTrue(e.version.startswith(b"BAD MAGIC")) def test_container_size(self): ss = self.create("test_container_size") From 10724a91f9ca2fe929f4e29adb03b876b21f9fe5 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 4 Nov 2021 10:17:36 -0400 Subject: [PATCH 0366/2309] introduce an explicit representation of the v1 immutable container schema This is only a partial representation, sufficient to express the changes that are coming in v2. --- src/allmydata/storage/immutable.py | 37 ++++++----- src/allmydata/storage/immutable_schema.py | 81 +++++++++++++++++++++++ 2 files changed, 102 insertions(+), 16 deletions(-) create mode 100644 src/allmydata/storage/immutable_schema.py diff --git a/src/allmydata/storage/immutable.py b/src/allmydata/storage/immutable.py index fcc60509c..ae5a710af 100644 --- a/src/allmydata/storage/immutable.py +++ b/src/allmydata/storage/immutable.py @@ -25,9 +25,14 @@ from allmydata.interfaces import ( ) from allmydata.util import base32, fileutil, log from allmydata.util.assertutil import precondition -from allmydata.storage.lease import LeaseInfo from allmydata.storage.common import UnknownImmutableContainerVersionError +from .immutable_schema import ( + NEWEST_SCHEMA_VERSION, + schema_from_version, +) + + # each share file (in storage/shares/$SI/$SHNUM) contains lease information # and share data. The share data is accessed by RIBucketWriter.write and # RIBucketReader.read . The lease information is not accessible through these @@ -118,9 +123,16 @@ class ShareFile(object): ``False`` otherwise. """ (version,) = struct.unpack(">L", header[:4]) - return version == 1 + return schema_from_version(version) is not None - def __init__(self, filename, max_size=None, create=False, lease_count_format="L"): + def __init__( + self, + filename, + max_size=None, + create=False, + lease_count_format="L", + schema=NEWEST_SCHEMA_VERSION, + ): """ Initialize a ``ShareFile``. @@ -156,24 +168,17 @@ class ShareFile(object): # it. Also construct the metadata. assert not os.path.exists(self.home) fileutil.make_dirs(os.path.dirname(self.home)) - # The second field -- the four-byte share data length -- is no - # longer used as of Tahoe v1.3.0, but we continue to write it in - # there in case someone downgrades a storage server from >= - # Tahoe-1.3.0 to < Tahoe-1.3.0, or moves a share file from one - # server to another, etc. We do saturation -- a share data length - # larger than 2**32-1 (what can fit into the field) is marked as - # the largest length that can fit into the field. That way, even - # if this does happen, the old < v1.3.0 server will still allow - # clients to read the first part of the share. + self._schema = schema with open(self.home, 'wb') as f: - f.write(struct.pack(">LLL", 1, min(2**32-1, max_size), 0)) + f.write(self._schema.header(max_size)) self._lease_offset = max_size + 0x0c self._num_leases = 0 else: with open(self.home, 'rb') as f: filesize = os.path.getsize(self.home) (version, unused, num_leases) = struct.unpack(">LLL", f.read(0xc)) - if version != 1: + self._schema = schema_from_version(version) + if self._schema is None: raise UnknownImmutableContainerVersionError(filename, version) self._num_leases = num_leases self._lease_offset = filesize - (num_leases * self.LEASE_SIZE) @@ -209,7 +214,7 @@ class ShareFile(object): offset = self._lease_offset + lease_number * self.LEASE_SIZE f.seek(offset) assert f.tell() == offset - f.write(lease_info.to_immutable_data()) + f.write(self._schema.serialize_lease(lease_info)) def _read_num_leases(self, f): f.seek(0x08) @@ -240,7 +245,7 @@ class ShareFile(object): for i in range(num_leases): data = f.read(self.LEASE_SIZE) if data: - yield LeaseInfo.from_immutable_data(data) + yield self._schema.unserialize_lease(data) def add_lease(self, lease_info): with open(self.home, 'rb+') as f: diff --git a/src/allmydata/storage/immutable_schema.py b/src/allmydata/storage/immutable_schema.py new file mode 100644 index 000000000..759752bed --- /dev/null +++ b/src/allmydata/storage/immutable_schema.py @@ -0,0 +1,81 @@ +""" +Ported to Python 3. +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + +import struct + +from .lease import ( + LeaseInfo, +) + +def _header(version, max_size): + # (int, int) -> bytes + """ + Construct the header for an immutable container. + + :param version: The container version to include the in header. + :param max_size: The maximum data size the container will hold. + + :return: Some bytes to write at the beginning of the container. + """ + # The second field -- the four-byte share data length -- is no longer + # used as of Tahoe v1.3.0, but we continue to write it in there in + # case someone downgrades a storage server from >= Tahoe-1.3.0 to < + # Tahoe-1.3.0, or moves a share file from one server to another, + # etc. We do saturation -- a share data length larger than 2**32-1 + # (what can fit into the field) is marked as the largest length that + # can fit into the field. That way, even if this does happen, the old + # < v1.3.0 server will still allow clients to read the first part of + # the share. + return struct.pack(">LLL", version, min(2**32 - 1, max_size), 0) + +class _V1(object): + """ + Implement encoding and decoding for v1 of the immutable container. + """ + version = 1 + + @classmethod + def header(cls, max_size): + return _header(cls.version, max_size) + + @classmethod + def serialize_lease(cls, lease): + if isinstance(lease, LeaseInfo): + return lease.to_immutable_data() + raise ValueError( + "ShareFile v1 schema only supports LeaseInfo, not {!r}".format( + lease, + ), + ) + + @classmethod + def unserialize_lease(cls, data): + # In v1 of the immutable schema lease secrets are stored plaintext. + # So load the data into a plain LeaseInfo which works on plaintext + # secrets. + return LeaseInfo.from_immutable_data(data) + + +ALL_SCHEMAS = {_V1} +ALL_SCHEMA_VERSIONS = {schema.version for schema in ALL_SCHEMAS} +NEWEST_SCHEMA_VERSION = max(ALL_SCHEMAS, key=lambda schema: schema.version) + +def schema_from_version(version): + # (int) -> Optional[type] + """ + Find the schema object that corresponds to a certain version number. + """ + for schema in ALL_SCHEMAS: + if schema.version == version: + return schema + return None From 3b4141952387452894ed5c0ed58113e272ad3e4f Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 4 Nov 2021 10:32:59 -0400 Subject: [PATCH 0367/2309] apply the ShareFile tests to all schema versions using hypothesis --- src/allmydata/test/test_storage.py | 60 +++++++++++++++++++----------- 1 file changed, 38 insertions(+), 22 deletions(-) diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index bf9eff37a..655395042 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -43,6 +43,9 @@ from allmydata.storage.server import StorageServer, DEFAULT_RENEWAL_TIME from allmydata.storage.shares import get_share_file from allmydata.storage.mutable import MutableShareFile from allmydata.storage.immutable import BucketWriter, BucketReader, ShareFile +from allmydata.storage.immutable_schema import ( + ALL_SCHEMAS, +) from allmydata.storage.common import storage_index_to_dir, \ UnknownMutableContainerVersionError, UnknownImmutableContainerVersionError, \ si_b2a, si_a2b @@ -844,6 +847,9 @@ class Server(unittest.TestCase): # Create a bucket: rs0, cs0 = self.create_bucket_5_shares(ss, b"si0") + + # Upload of an immutable implies creation of a single lease with the + # supplied secrets. (lease,) = ss.get_leases(b"si0") self.assertTrue(lease.is_renew_secret(rs0)) @@ -3125,6 +3131,7 @@ class Stats(unittest.TestCase): self.failUnless(output["get"]["99_0_percentile"] is None, output) self.failUnless(output["get"]["99_9_percentile"] is None, output) +immutable_schemas = strategies.sampled_from(list(ALL_SCHEMAS)) class ShareFileTests(unittest.TestCase): """Tests for allmydata.storage.immutable.ShareFile.""" @@ -3136,47 +3143,54 @@ class ShareFileTests(unittest.TestCase): # Should be b'abDEF' now. return sf - def test_read_write(self): + @given(immutable_schemas) + def test_read_write(self, schema): """Basic writes can be read.""" - sf = self.get_sharefile() + sf = self.get_sharefile(schema=schema) self.assertEqual(sf.read_share_data(0, 3), b"abD") self.assertEqual(sf.read_share_data(1, 4), b"bDEF") - def test_reads_beyond_file_end(self): + @given(immutable_schemas) + def test_reads_beyond_file_end(self, schema): """Reads beyond the file size are truncated.""" - sf = self.get_sharefile() + sf = self.get_sharefile(schema=schema) self.assertEqual(sf.read_share_data(0, 10), b"abDEF") self.assertEqual(sf.read_share_data(5, 10), b"") - def test_too_large_write(self): + @given(immutable_schemas) + def test_too_large_write(self, schema): """Can't do write larger than file size.""" - sf = self.get_sharefile() + sf = self.get_sharefile(schema=schema) with self.assertRaises(DataTooLargeError): sf.write_share_data(0, b"x" * 3000) - def test_no_leases_cancelled(self): + @given(immutable_schemas) + def test_no_leases_cancelled(self, schema): """If no leases were cancelled, IndexError is raised.""" - sf = self.get_sharefile() + sf = self.get_sharefile(schema=schema) with self.assertRaises(IndexError): sf.cancel_lease(b"garbage") - def test_long_lease_count_format(self): + @given(immutable_schemas) + def test_long_lease_count_format(self, schema): """ ``ShareFile.__init__`` raises ``ValueError`` if the lease count format given is longer than one character. """ with self.assertRaises(ValueError): - self.get_sharefile(lease_count_format="BB") + self.get_sharefile(schema=schema, lease_count_format="BB") - def test_large_lease_count_format(self): + @given(immutable_schemas) + def test_large_lease_count_format(self, schema): """ ``ShareFile.__init__`` raises ``ValueError`` if the lease count format encodes to a size larger than 8 bytes. """ with self.assertRaises(ValueError): - self.get_sharefile(lease_count_format="Q") + self.get_sharefile(schema=schema, lease_count_format="Q") - def test_avoid_lease_overflow(self): + @given(immutable_schemas) + def test_avoid_lease_overflow(self, schema): """ If the share file already has the maximum number of leases supported then ``ShareFile.add_lease`` raises ``struct.error`` and makes no changes @@ -3190,7 +3204,7 @@ class ShareFileTests(unittest.TestCase): ) # Make it a little easier to reach the condition by limiting the # number of leases to only 255. - sf = self.get_sharefile(lease_count_format="B") + sf = self.get_sharefile(schema=schema, lease_count_format="B") # Add the leases. for i in range(2 ** 8 - 1): @@ -3214,16 +3228,17 @@ class ShareFileTests(unittest.TestCase): self.assertEqual(before_data, after_data) - def test_renew_secret(self): + @given(immutable_schemas) + def test_renew_secret(self, schema): """ - A lease loaded from an immutable share file can have its renew secret - verified. + A lease loaded from an immutable share file at any schema version can have + its renew secret verified. """ renew_secret = b"r" * 32 cancel_secret = b"c" * 32 expiration_time = 2 ** 31 - sf = self.get_sharefile() + sf = self.get_sharefile(schema=schema) lease = LeaseInfo( owner_num=0, renew_secret=renew_secret, @@ -3234,16 +3249,17 @@ class ShareFileTests(unittest.TestCase): (loaded_lease,) = sf.get_leases() self.assertTrue(loaded_lease.is_renew_secret(renew_secret)) - def test_cancel_secret(self): + @given(immutable_schemas) + def test_cancel_secret(self, schema): """ - A lease loaded from an immutable share file can have its cancel secret - verified. + A lease loaded from an immutable share file at any schema version can have + its cancel secret verified. """ renew_secret = b"r" * 32 cancel_secret = b"c" * 32 expiration_time = 2 ** 31 - sf = self.get_sharefile() + sf = self.get_sharefile(schema=schema) lease = LeaseInfo( owner_num=0, renew_secret=renew_secret, From 234b8dcde2febc2b3eee96e1ede4d123f634dcb1 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 4 Nov 2021 11:56:49 -0400 Subject: [PATCH 0368/2309] Formalize LeaseInfo interface in preparation for another implementation --- src/allmydata/storage/lease.py | 83 ++++++++++++++++++++++++++++++++++ 1 file changed, 83 insertions(+) diff --git a/src/allmydata/storage/lease.py b/src/allmydata/storage/lease.py index 6d21bb2b2..23071707a 100644 --- a/src/allmydata/storage/lease.py +++ b/src/allmydata/storage/lease.py @@ -15,6 +15,11 @@ import struct, time import attr +from zope.interface import ( + Interface, + implementer, +) + from allmydata.util.hashutil import timing_safe_compare # struct format for representation of a lease in an immutable share @@ -23,6 +28,84 @@ IMMUTABLE_FORMAT = ">L32s32sL" # struct format for representation of a lease in a mutable share MUTABLE_FORMAT = ">LL32s32s20s" + +class ILeaseInfo(Interface): + """ + Represent a marker attached to a share that indicates that share should be + retained for some amount of time. + + Typically clients will create and renew leases on their shares as a way to + inform storage servers that there is still interest in those shares. A + share may have more than one lease. If all leases on a share have + expiration times in the past then the storage server may take this as a + strong hint that no one is interested in the share anymore and therefore + the share may be deleted to reclaim the space. + """ + def renew(new_expire_time): + """ + Create a new ``ILeaseInfo`` with the given expiration time. + + :param Union[int, float] new_expire_time: The expiration time the new + ``ILeaseInfo`` will have. + + :return: The new ``ILeaseInfo`` provider with the new expiration time. + """ + + def get_expiration_time(): + """ + :return Union[int, float]: this lease's expiration time + """ + + def get_grant_renew_time_time(): + """ + :return Union[int, float]: a guess about the last time this lease was + renewed + """ + + def get_age(): + """ + :return Union[int, float]: a guess about how long it has been since this + lease was renewed + """ + + def to_immutable_data(): + """ + :return bytes: a serialized representation of this lease suitable for + inclusion in an immutable container + """ + + def to_mutable_data(): + """ + :return bytes: a serialized representation of this lease suitable for + inclusion in a mutable container + """ + + def immutable_size(): + """ + :return int: the size of the serialized representation of this lease in an + immutable container + """ + + def mutable_size(): + """ + :return int: the size of the serialized representation of this lease in a + mutable container + """ + + def is_renew_secret(candidate_secret): + """ + :return bool: ``True`` if the given byte string is this lease's renew + secret, ``False`` otherwise + """ + + def is_cancel_secret(candidate_secret): + """ + :return bool: ``True`` if the given byte string is this lease's cancel + secret, ``False`` otherwise + """ + + +@implementer(ILeaseInfo) @attr.s(frozen=True) class LeaseInfo(object): """ From b69e8d013bfc32f8b7ca5948ad36a5c60b3db73a Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 4 Nov 2021 14:07:49 -0400 Subject: [PATCH 0369/2309] introduce immutable container schema version 2 This version used on-disk hashed secrets to reduce the chance of secrets leaking to unintended parties. --- src/allmydata/storage/immutable.py | 23 ++++- src/allmydata/storage/immutable_schema.py | 105 +++++++++++++++++++++- src/allmydata/storage/lease.py | 82 +++++++++++++++++ src/allmydata/test/test_download.py | 14 ++- 4 files changed, 214 insertions(+), 10 deletions(-) diff --git a/src/allmydata/storage/immutable.py b/src/allmydata/storage/immutable.py index ae5a710af..216262a81 100644 --- a/src/allmydata/storage/immutable.py +++ b/src/allmydata/storage/immutable.py @@ -39,14 +39,14 @@ from .immutable_schema import ( # interfaces. # The share file has the following layout: -# 0x00: share file version number, four bytes, current version is 1 +# 0x00: share file version number, four bytes, current version is 2 # 0x04: share data length, four bytes big-endian = A # See Footnote 1 below. # 0x08: number of leases, four bytes big-endian # 0x0c: beginning of share data (see immutable.layout.WriteBucketProxy) # A+0x0c = B: first lease. Lease format is: # B+0x00: owner number, 4 bytes big-endian, 0 is reserved for no-owner -# B+0x04: renew secret, 32 bytes (SHA256) -# B+0x24: cancel secret, 32 bytes (SHA256) +# B+0x04: renew secret, 32 bytes (SHA256 + blake2b) # See Footnote 2 below. +# B+0x24: cancel secret, 32 bytes (SHA256 + blake2b) # B+0x44: expiration time, 4 bytes big-endian seconds-since-epoch # B+0x48: next lease, or end of record @@ -58,6 +58,23 @@ from .immutable_schema import ( # then the value stored in this field will be the actual share data length # modulo 2**32. +# Footnote 2: The change between share file version number 1 and 2 is that +# storage of lease secrets is changed from plaintext to hashed. This change +# protects the secrets from compromises of local storage on the server: if a +# plaintext cancel secret is somehow exfiltrated from the storage server, an +# attacker could use it to cancel that lease and potentially cause user data +# to be discarded before intended by the real owner. As of this comment, +# lease cancellation is disabled because there have been at least two bugs +# which leak the persisted value of the cancellation secret. If lease secrets +# were stored hashed instead of plaintext then neither of these bugs would +# have allowed an attacker to learn a usable cancel secret. +# +# Clients are free to construct these secrets however they like. The +# Tahoe-LAFS client uses a SHA256-based construction. The server then uses +# blake2b to hash these values for storage so that it retains no persistent +# copy of the original secret. +# + def _fix_lease_count_format(lease_count_format): """ Turn a single character struct format string into a format string suitable diff --git a/src/allmydata/storage/immutable_schema.py b/src/allmydata/storage/immutable_schema.py index 759752bed..fc823507a 100644 --- a/src/allmydata/storage/immutable_schema.py +++ b/src/allmydata/storage/immutable_schema.py @@ -13,8 +13,14 @@ if PY2: import struct +import attr + +from nacl.hash import blake2b +from nacl.encoding import RawEncoder + from .lease import ( LeaseInfo, + HashedLeaseInfo, ) def _header(version, max_size): @@ -22,10 +28,10 @@ def _header(version, max_size): """ Construct the header for an immutable container. - :param version: The container version to include the in header. - :param max_size: The maximum data size the container will hold. + :param version: the container version to include the in header + :param max_size: the maximum data size the container will hold - :return: Some bytes to write at the beginning of the container. + :return: some bytes to write at the beginning of the container """ # The second field -- the four-byte share data length -- is no longer # used as of Tahoe v1.3.0, but we continue to write it in there in @@ -38,6 +44,97 @@ def _header(version, max_size): # the share. return struct.pack(">LLL", version, min(2**32 - 1, max_size), 0) + +class _V2(object): + """ + Implement encoding and decoding for v2 of the immutable container. + """ + version = 2 + + @classmethod + def _hash_secret(cls, secret): + # type: (bytes) -> bytes + """ + Hash a lease secret for storage. + """ + return blake2b(secret, digest_size=32, encoder=RawEncoder()) + + @classmethod + def _hash_lease_info(cls, lease_info): + # type: (LeaseInfo) -> HashedLeaseInfo + """ + Hash the cleartext lease info secrets into a ``HashedLeaseInfo``. + """ + if not isinstance(lease_info, LeaseInfo): + # Provide a little safety against misuse, especially an attempt to + # re-hash an already-hashed lease info which is represented as a + # different type. + raise TypeError( + "Can only hash LeaseInfo, not {!r}".format(lease_info), + ) + + # Hash the cleartext secrets in the lease info and wrap the result in + # a new type. + return HashedLeaseInfo( + attr.assoc( + lease_info, + renew_secret=cls._hash_secret(lease_info.renew_secret), + cancel_secret=cls._hash_secret(lease_info.cancel_secret), + ), + cls._hash_secret, + ) + + @classmethod + def header(cls, max_size): + # type: (int) -> bytes + """ + Construct a container header. + + :param max_size: the maximum size the container can hold + + :return: the header bytes + """ + return _header(cls.version, max_size) + + @classmethod + def serialize_lease(cls, lease): + # type: (Union[LeaseInfo, HashedLeaseInfo]) -> bytes + """ + Serialize a lease to be written to a v2 container. + + :param lease: the lease to serialize + + :return: the serialized bytes + """ + if isinstance(lease, LeaseInfo): + # v2 of the immutable schema stores lease secrets hashed. If + # we're given a LeaseInfo then it holds plaintext secrets. Hash + # them before trying to serialize. + lease = cls._hash_lease_info(lease) + if isinstance(lease, HashedLeaseInfo): + return lease.to_immutable_data() + raise ValueError( + "ShareFile v2 schema cannot represent lease {!r}".format( + lease, + ), + ) + + @classmethod + def unserialize_lease(cls, data): + # type: (bytes) -> HashedLeaseInfo + """ + Unserialize some bytes from a v2 container. + + :param data: the bytes from the container + + :return: the ``HashedLeaseInfo`` the bytes represent + """ + # In v2 of the immutable schema lease secrets are stored hashed. Wrap + # a LeaseInfo in a HashedLeaseInfo so it can supply the correct + # interpretation for those values. + return HashedLeaseInfo(LeaseInfo.from_immutable_data(data), cls._hash_secret) + + class _V1(object): """ Implement encoding and decoding for v1 of the immutable container. @@ -66,7 +163,7 @@ class _V1(object): return LeaseInfo.from_immutable_data(data) -ALL_SCHEMAS = {_V1} +ALL_SCHEMAS = {_V2, _V1} ALL_SCHEMA_VERSIONS = {schema.version for schema in ALL_SCHEMAS} NEWEST_SCHEMA_VERSION = max(ALL_SCHEMAS, key=lambda schema: schema.version) diff --git a/src/allmydata/storage/lease.py b/src/allmydata/storage/lease.py index 23071707a..895a0970c 100644 --- a/src/allmydata/storage/lease.py +++ b/src/allmydata/storage/lease.py @@ -20,6 +20,10 @@ from zope.interface import ( implementer, ) +from twisted.python.components import ( + proxyForInterface, +) + from allmydata.util.hashutil import timing_safe_compare # struct format for representation of a lease in an immutable share @@ -245,3 +249,81 @@ class LeaseInfo(object): ] values = struct.unpack(">LL32s32s20s", data) return cls(**dict(zip(names, values))) + + +@attr.s(frozen=True) +class HashedLeaseInfo(proxyForInterface(ILeaseInfo, "_lease_info")): + """ + A ``HashedLeaseInfo`` wraps lease information in which the secrets have + been hashed. + """ + _lease_info = attr.ib() + _hash = attr.ib() + + def is_renew_secret(self, candidate_secret): + """ + Hash the candidate secret and compare the result to the stored hashed + secret. + """ + return super(HashedLeaseInfo, self).is_renew_secret(self._hash(candidate_secret)) + + def is_cancel_secret(self, candidate_secret): + """ + Hash the candidate secret and compare the result to the stored hashed + secret. + """ + if isinstance(candidate_secret, _HashedCancelSecret): + # Someone read it off of this object in this project - probably + # the lease crawler - and is just trying to use it to identify + # which lease it wants to operate on. Avoid re-hashing the value. + # + # It is important that this codepath is only availably internally + # for this process to talk to itself. If it were to be exposed to + # clients over the network, they could just provide the hashed + # value to avoid having to ever learn the original value. + hashed_candidate = candidate_secret.hashed_value + else: + # It is not yet hashed so hash it. + hashed_candidate = self._hash(candidate_secret) + + return super(HashedLeaseInfo, self).is_cancel_secret(hashed_candidate) + + @property + def owner_num(self): + return self._lease_info.owner_num + + @property + def cancel_secret(self): + """ + Give back an opaque wrapper around the hashed cancel secret which can + later be presented for a succesful equality comparison. + """ + # We don't *have* the cancel secret. We hashed it and threw away the + # original. That's good. It does mean that some code that runs + # in-process with the storage service (LeaseCheckingCrawler) runs into + # some difficulty. That code wants to cancel leases and does so using + # the same interface that faces storage clients (or would face them, + # if lease cancellation were exposed). + # + # Since it can't use the hashed secret to cancel a lease (that's the + # point of the hashing) and we don't have the unhashed secret to give + # it, instead we give it a marker that `cancel_lease` will recognize. + # On recognizing it, if the hashed value given matches the hashed + # value stored it is considered a match and the lease can be + # cancelled. + # + # This isn't great. Maybe the internal and external consumers of + # cancellation should use different interfaces. + return _HashedCancelSecret(self._lease_info.cancel_secret) + + +@attr.s(frozen=True) +class _HashedCancelSecret(object): + """ + ``_HashedCancelSecret`` is a marker type for an already-hashed lease + cancel secret that lets internal lease cancellers bypass the hash-based + protection that's imposed on external lease cancellers. + + :ivar bytes hashed_value: The already-hashed secret. + """ + hashed_value = attr.ib() diff --git a/src/allmydata/test/test_download.py b/src/allmydata/test/test_download.py index ca5b5650b..85d89cde6 100644 --- a/src/allmydata/test/test_download.py +++ b/src/allmydata/test/test_download.py @@ -1113,9 +1113,17 @@ class Corruption(_Base, unittest.TestCase): d.addCallback(_download, imm_uri, i, expected) d.addCallback(lambda ign: self.restore_all_shares(self.shares)) d.addCallback(fireEventually) - corrupt_values = [(3, 2, "no-sh2"), - (15, 2, "need-4th"), # share looks v2 - ] + corrupt_values = [ + # Make the container version for share number 2 look + # unsupported. If you add support for immutable share file + # version number much past 16 million then you will have to + # update this test. Also maybe you have other problems. + (1, 255, "no-sh2"), + # Make the immutable share number 2 (not the container, the + # thing inside the container) look unsupported. Ditto the + # above about version numbers in the ballpark of 16 million. + (13, 255, "need-4th"), + ] for i,newvalue,expected in corrupt_values: d.addCallback(self._corrupt_set, imm_uri, i, newvalue) d.addCallback(_download, imm_uri, i, expected) From 7a59aa83bb9e429d0b44f47fff6365dbfa24f42f Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 4 Nov 2021 14:12:54 -0400 Subject: [PATCH 0370/2309] add missing import --- src/allmydata/storage/immutable_schema.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/allmydata/storage/immutable_schema.py b/src/allmydata/storage/immutable_schema.py index fc823507a..6ac49f6f1 100644 --- a/src/allmydata/storage/immutable_schema.py +++ b/src/allmydata/storage/immutable_schema.py @@ -13,6 +13,11 @@ if PY2: import struct +try: + from typing import Union +except ImportError: + pass + import attr from nacl.hash import blake2b From 6889ab2a76a1665dc7adb11dfa2205760641f303 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 4 Nov 2021 14:16:55 -0400 Subject: [PATCH 0371/2309] fix syntax of type hint --- src/allmydata/storage/immutable_schema.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/storage/immutable_schema.py b/src/allmydata/storage/immutable_schema.py index 6ac49f6f1..7ffec418a 100644 --- a/src/allmydata/storage/immutable_schema.py +++ b/src/allmydata/storage/immutable_schema.py @@ -29,7 +29,7 @@ from .lease import ( ) def _header(version, max_size): - # (int, int) -> bytes + # type: (int, int) -> bytes """ Construct the header for an immutable container. From 2186bfcc372d01ab79f6899e8d0a54157ee83444 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 4 Nov 2021 14:40:43 -0400 Subject: [PATCH 0372/2309] silence some mypy errors :/ I don't know the "right" way to make mypy happy with these things --- src/allmydata/storage/immutable_schema.py | 4 ++-- src/allmydata/storage/lease.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/allmydata/storage/immutable_schema.py b/src/allmydata/storage/immutable_schema.py index 7ffec418a..440755b01 100644 --- a/src/allmydata/storage/immutable_schema.py +++ b/src/allmydata/storage/immutable_schema.py @@ -169,8 +169,8 @@ class _V1(object): ALL_SCHEMAS = {_V2, _V1} -ALL_SCHEMA_VERSIONS = {schema.version for schema in ALL_SCHEMAS} -NEWEST_SCHEMA_VERSION = max(ALL_SCHEMAS, key=lambda schema: schema.version) +ALL_SCHEMA_VERSIONS = {schema.version for schema in ALL_SCHEMAS} # type: ignore +NEWEST_SCHEMA_VERSION = max(ALL_SCHEMAS, key=lambda schema: schema.version) # type: ignore def schema_from_version(version): # (int) -> Optional[type] diff --git a/src/allmydata/storage/lease.py b/src/allmydata/storage/lease.py index 895a0970c..63dba15e8 100644 --- a/src/allmydata/storage/lease.py +++ b/src/allmydata/storage/lease.py @@ -252,7 +252,7 @@ class LeaseInfo(object): @attr.s(frozen=True) -class HashedLeaseInfo(proxyForInterface(ILeaseInfo, "_lease_info")): +class HashedLeaseInfo(proxyForInterface(ILeaseInfo, "_lease_info")): # type: ignore # unsupported dynamic base class """ A ``HashedLeaseInfo`` wraps lease information in which the secrets have been hashed. From 931ddf85a532178ab83584820eda8605a495d5ab Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 4 Nov 2021 15:26:58 -0400 Subject: [PATCH 0373/2309] introduce an explicit representation of the v1 mutable container schema This is only a partial representation, sufficient to express the changes that are coming in v2. --- src/allmydata/storage/mutable.py | 46 +++------ src/allmydata/storage/mutable_schema.py | 119 ++++++++++++++++++++++++ 2 files changed, 134 insertions(+), 31 deletions(-) create mode 100644 src/allmydata/storage/mutable_schema.py diff --git a/src/allmydata/storage/mutable.py b/src/allmydata/storage/mutable.py index ce9cc5ff4..346edd53a 100644 --- a/src/allmydata/storage/mutable.py +++ b/src/allmydata/storage/mutable.py @@ -24,7 +24,10 @@ from allmydata.storage.lease import LeaseInfo from allmydata.storage.common import UnknownMutableContainerVersionError, \ DataTooLargeError from allmydata.mutable.layout import MAX_MUTABLE_SHARE_SIZE - +from .mutable_schema import ( + NEWEST_SCHEMA_VERSION, + schema_from_header, +) # the MutableShareFile is like the ShareFile, but used for mutable data. It # has a different layout. See docs/mutable.txt for more details. @@ -64,9 +67,6 @@ class MutableShareFile(object): # our sharefiles share with a recognizable string, plus some random # binary data to reduce the chance that a regular text file will look # like a sharefile. - MAGIC = b"Tahoe mutable container v1\n" + b"\x75\x09\x44\x03\x8e" - assert len(MAGIC) == 32 - assert isinstance(MAGIC, bytes) MAX_SIZE = MAX_MUTABLE_SHARE_SIZE # TODO: decide upon a policy for max share size @@ -82,20 +82,19 @@ class MutableShareFile(object): :return: ``True`` if the bytes could belong to this container, ``False`` otherwise. """ - return header.startswith(cls.MAGIC) + return schema_from_header(header) is not None - def __init__(self, filename, parent=None): + def __init__(self, filename, parent=None, schema=NEWEST_SCHEMA_VERSION): self.home = filename if os.path.exists(self.home): # we don't cache anything, just check the magic with open(self.home, 'rb') as f: - data = f.read(self.HEADER_SIZE) - (magic, - write_enabler_nodeid, write_enabler, - data_length, extra_least_offset) = \ - struct.unpack(">32s20s32sQQ", data) - if not self.is_valid_header(data): - raise UnknownMutableContainerVersionError(filename, magic) + header = f.read(self.HEADER_SIZE) + self._schema = schema_from_header(header) + if self._schema is None: + raise UnknownMutableContainerVersionError(filename, header) + else: + self._schema = schema self.parent = parent # for logging def log(self, *args, **kwargs): @@ -103,23 +102,8 @@ class MutableShareFile(object): def create(self, my_nodeid, write_enabler): assert not os.path.exists(self.home) - data_length = 0 - extra_lease_offset = (self.HEADER_SIZE - + 4 * self.LEASE_SIZE - + data_length) - assert extra_lease_offset == self.DATA_OFFSET # true at creation - num_extra_leases = 0 with open(self.home, 'wb') as f: - header = struct.pack( - ">32s20s32sQQ", - self.MAGIC, my_nodeid, write_enabler, - data_length, extra_lease_offset, - ) - leases = (b"\x00" * self.LEASE_SIZE) * 4 - f.write(header + leases) - # data goes here, empty after creation - f.write(struct.pack(">L", num_extra_leases)) - # extra leases go here, none at creation + f.write(self._schema.header(my_nodeid, write_enabler)) def unlink(self): os.unlink(self.home) @@ -252,7 +236,7 @@ class MutableShareFile(object): + (lease_number-4)*self.LEASE_SIZE) f.seek(offset) assert f.tell() == offset - f.write(lease_info.to_mutable_data()) + f.write(self._schema.serialize_lease(lease_info)) def _read_lease_record(self, f, lease_number): # returns a LeaseInfo instance, or None @@ -269,7 +253,7 @@ class MutableShareFile(object): f.seek(offset) assert f.tell() == offset data = f.read(self.LEASE_SIZE) - lease_info = LeaseInfo.from_mutable_data(data) + lease_info = self._schema.unserialize_lease(data) if lease_info.owner_num == 0: return None return lease_info diff --git a/src/allmydata/storage/mutable_schema.py b/src/allmydata/storage/mutable_schema.py new file mode 100644 index 000000000..25f24ea1f --- /dev/null +++ b/src/allmydata/storage/mutable_schema.py @@ -0,0 +1,119 @@ +""" +Ported to Python 3. +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + +import struct + +from .lease import ( + LeaseInfo, +) + +class _V1(object): + """ + Implement encoding and decoding for v1 of the mutable container. + """ + version = 1 + + _MAGIC = ( + # Make it easy for people to recognize + b"Tahoe mutable container v1\n" + # But also keep the chance of accidental collision low + b"\x75\x09\x44\x03\x8e" + ) + assert len(_MAGIC) == 32 + + _HEADER_FORMAT = ">32s20s32sQQ" + + # This size excludes leases + _HEADER_SIZE = struct.calcsize(_HEADER_FORMAT) + + _EXTRA_LEASE_OFFSET = _HEADER_SIZE + 4 * LeaseInfo().mutable_size() + + @classmethod + def magic_matches(cls, candidate_magic): + # type: (bytes) -> bool + """ + Return ``True`` if a candidate string matches the expected magic string + from a mutable container header, ``False`` otherwise. + """ + return candidate_magic[:len(cls._MAGIC)] == cls._MAGIC + + @classmethod + def header(cls, nodeid, write_enabler): + # type: (bytes, bytes) -> bytes + """ + Construct a container header. + + :param nodeid: A unique identifier for the node holding this + container. + + :param write_enabler: A secret shared with the client used to + authorize changes to the contents of this container. + """ + fixed_header = struct.pack( + ">32s20s32sQQ", + cls._MAGIC, + nodeid, + write_enabler, + # data length, initially the container is empty + 0, + cls._EXTRA_LEASE_OFFSET, + ) + blank_leases = b"\x00" * LeaseInfo().mutable_size() * 4 + extra_lease_count = struct.pack(">L", 0) + + return b"".join([ + fixed_header, + # share data will go in between the next two items eventually but + # for now there is none. + blank_leases, + extra_lease_count, + ]) + + @classmethod + def serialize_lease(cls, lease_info): + # type: (LeaseInfo) -> bytes + """ + Serialize a lease to be written to a v1 container. + + :param lease: the lease to serialize + + :return: the serialized bytes + """ + return lease_info.to_mutable_data() + + @classmethod + def unserialize_lease(cls, data): + # type: (bytes) -> LeaseInfo + """ + Unserialize some bytes from a v1 container. + + :param data: the bytes from the container + + :return: the ``LeaseInfo`` the bytes represent + """ + return LeaseInfo.from_mutable_data(data) + + +ALL_SCHEMAS = {_V1} +ALL_SCHEMA_VERSIONS = {schema.version for schema in ALL_SCHEMAS} # type: ignore +NEWEST_SCHEMA_VERSION = max(ALL_SCHEMAS, key=lambda schema: schema.version) # type: ignore + +def schema_from_header(header): + # (int) -> Optional[type] + """ + Find the schema object that corresponds to a certain version number. + """ + for schema in ALL_SCHEMAS: + if schema.magic_matches(header): + return schema + return None From 728638fe230dfdf0149c5835b0a8077230dbf021 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 4 Nov 2021 15:37:29 -0400 Subject: [PATCH 0374/2309] apply the MutableShareFile tests to all known schemas --- src/allmydata/test/test_storage.py | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index 655395042..fbd005050 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -42,9 +42,12 @@ from allmydata.util import fileutil, hashutil, base32 from allmydata.storage.server import StorageServer, DEFAULT_RENEWAL_TIME from allmydata.storage.shares import get_share_file from allmydata.storage.mutable import MutableShareFile +from allmydata.storage.mutable_schema import ( + ALL_SCHEMAS as ALL_MUTABLE_SCHEMAS, +) from allmydata.storage.immutable import BucketWriter, BucketReader, ShareFile from allmydata.storage.immutable_schema import ( - ALL_SCHEMAS, + ALL_SCHEMAS as ALL_IMMUTABLE_SCHEMAS, ) from allmydata.storage.common import storage_index_to_dir, \ UnknownMutableContainerVersionError, UnknownImmutableContainerVersionError, \ @@ -3131,7 +3134,7 @@ class Stats(unittest.TestCase): self.failUnless(output["get"]["99_0_percentile"] is None, output) self.failUnless(output["get"]["99_9_percentile"] is None, output) -immutable_schemas = strategies.sampled_from(list(ALL_SCHEMAS)) +immutable_schemas = strategies.sampled_from(list(ALL_IMMUTABLE_SCHEMAS)) class ShareFileTests(unittest.TestCase): """Tests for allmydata.storage.immutable.ShareFile.""" @@ -3270,15 +3273,17 @@ class ShareFileTests(unittest.TestCase): (loaded_lease,) = sf.get_leases() self.assertTrue(loaded_lease.is_cancel_secret(cancel_secret)) +mutable_schemas = strategies.sampled_from(list(ALL_MUTABLE_SCHEMAS)) class MutableShareFileTests(unittest.TestCase): """ Tests for allmydata.storage.mutable.MutableShareFile. """ - def get_sharefile(self): - return MutableShareFile(self.mktemp()) + def get_sharefile(self, **kwargs): + return MutableShareFile(self.mktemp(), **kwargs) @given( + schema=mutable_schemas, nodeid=strategies.just(b"x" * 20), write_enabler=strategies.just(b"y" * 32), datav=strategies.lists( @@ -3289,12 +3294,12 @@ class MutableShareFileTests(unittest.TestCase): ), new_length=offsets(), ) - def test_readv_reads_share_data(self, nodeid, write_enabler, datav, new_length): + def test_readv_reads_share_data(self, schema, nodeid, write_enabler, datav, new_length): """ ``MutableShareFile.readv`` returns bytes from the share data portion of the share file. """ - sf = self.get_sharefile() + sf = self.get_sharefile(schema=schema) sf.create(my_nodeid=nodeid, write_enabler=write_enabler) sf.writev(datav=datav, new_length=new_length) @@ -3329,12 +3334,13 @@ class MutableShareFileTests(unittest.TestCase): self.assertEqual(expected_data, read_data) @given( + schema=mutable_schemas, nodeid=strategies.just(b"x" * 20), write_enabler=strategies.just(b"y" * 32), readv=strategies.lists(strategies.tuples(offsets(), lengths()), min_size=1), random=strategies.randoms(), ) - def test_readv_rejects_negative_length(self, nodeid, write_enabler, readv, random): + def test_readv_rejects_negative_length(self, schema, nodeid, write_enabler, readv, random): """ If a negative length is given to ``MutableShareFile.readv`` in a read vector then ``AssertionError`` is raised. @@ -3373,7 +3379,7 @@ class MutableShareFileTests(unittest.TestCase): *broken_readv[readv_index] ) - sf = self.get_sharefile() + sf = self.get_sharefile(schema=schema) sf.create(my_nodeid=nodeid, write_enabler=write_enabler) # A read with a broken read vector is an error. From 8adff050a7f179c6c4796f4d3b04fab60924cbad Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 5 Nov 2021 13:51:46 -0400 Subject: [PATCH 0375/2309] compare without breaking out all of the fields HashedLeaseInfo doesn't have all of these attributes --- src/allmydata/test/test_storage.py | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index fbd005050..92176ce52 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -1361,14 +1361,21 @@ class MutableServer(unittest.TestCase): 2: [b"2"*10]}) def compare_leases_without_timestamps(self, leases_a, leases_b): - self.failUnlessEqual(len(leases_a), len(leases_b)) - for i in range(len(leases_a)): - a = leases_a[i] - b = leases_b[i] - self.failUnlessEqual(a.owner_num, b.owner_num) - self.failUnlessEqual(a.renew_secret, b.renew_secret) - self.failUnlessEqual(a.cancel_secret, b.cancel_secret) - self.failUnlessEqual(a.nodeid, b.nodeid) + for a, b in zip(leases_a, leases_b): + # The leases aren't always of the same type (though of course + # corresponding elements in the two lists should be of the same + # type as each other) so it's inconvenient to just reach in and + # normalize the expiration timestamp. We don't want to call + # `renew` on both objects to normalize the expiration timestamp in + # case `renew` is broken and gives us back equal outputs from + # non-equal inputs (expiration timestamp aside). It seems + # reasonably safe to use `renew` to make _one_ of the timestamps + # equal to the other though. + self.assertEqual( + a.renew(b.get_expiration_time()), + b, + ) + self.assertEqual(len(leases_a), len(leases_b)) def test_leases(self): ss = self.create("test_leases") From 0cd96ed713ba6429b76e2520752acb7e8e166e40 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 5 Nov 2021 14:09:46 -0400 Subject: [PATCH 0376/2309] fix the debug tool for the hashed lease secret case --- src/allmydata/scripts/debug.py | 4 ++-- src/allmydata/storage/lease.py | 43 ++++++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 2 deletions(-) diff --git a/src/allmydata/scripts/debug.py b/src/allmydata/scripts/debug.py index 260cca55b..6201ce28f 100644 --- a/src/allmydata/scripts/debug.py +++ b/src/allmydata/scripts/debug.py @@ -230,8 +230,8 @@ def dump_mutable_share(options): print(" ownerid: %d" % lease.owner_num, file=out) when = format_expiration_time(lease.get_expiration_time()) print(" expires in %s" % when, file=out) - print(" renew_secret: %s" % str(base32.b2a(lease.renew_secret), "utf-8"), file=out) - print(" cancel_secret: %s" % str(base32.b2a(lease.cancel_secret), "utf-8"), file=out) + print(" renew_secret: %s" % lease.present_renew_secret(), file=out) + print(" cancel_secret: %s" % lease.present_cancel_secret(), file=out) print(" secrets are for nodeid: %s" % idlib.nodeid_b2a(lease.nodeid), file=out) else: print("No leases.", file=out) diff --git a/src/allmydata/storage/lease.py b/src/allmydata/storage/lease.py index 63dba15e8..3ec760dbe 100644 --- a/src/allmydata/storage/lease.py +++ b/src/allmydata/storage/lease.py @@ -25,6 +25,7 @@ from twisted.python.components import ( ) from allmydata.util.hashutil import timing_safe_compare +from allmydata.util import base32 # struct format for representation of a lease in an immutable share IMMUTABLE_FORMAT = ">L32s32sL" @@ -102,12 +103,24 @@ class ILeaseInfo(Interface): secret, ``False`` otherwise """ + def present_renew_secret(): + """ + :return str: Text which could reasonably be shown to a person representing + this lease's renew secret. + """ + def is_cancel_secret(candidate_secret): """ :return bool: ``True`` if the given byte string is this lease's cancel secret, ``False`` otherwise """ + def present_cancel_secret(): + """ + :return str: Text which could reasonably be shown to a person representing + this lease's cancel secret. + """ + @implementer(ILeaseInfo) @attr.s(frozen=True) @@ -173,6 +186,13 @@ class LeaseInfo(object): """ return timing_safe_compare(self.renew_secret, candidate_secret) + def present_renew_secret(self): + # type: () -> bytes + """ + Return the renew secret, base32-encoded. + """ + return str(base32.b2a(self.renew_secret), "utf-8") + def is_cancel_secret(self, candidate_secret): # type: (bytes) -> bool """ @@ -183,6 +203,13 @@ class LeaseInfo(object): """ return timing_safe_compare(self.cancel_secret, candidate_secret) + def present_cancel_secret(self): + # type: () -> bytes + """ + Return the cancel secret, base32-encoded. + """ + return str(base32.b2a(self.cancel_secret), "utf-8") + def get_grant_renew_time_time(self): # hack, based upon fixed 31day expiration period return self._expiration_time - 31*24*60*60 @@ -267,6 +294,12 @@ class HashedLeaseInfo(proxyForInterface(ILeaseInfo, "_lease_info")): # type: ign """ return super(HashedLeaseInfo, self).is_renew_secret(self._hash(candidate_secret)) + def present_renew_secret(self): + """ + Present the hash of the secret with a marker indicating it is a hash. + """ + return u"hash:" + super(HashedLeaseInfo, self).present_renew_secret() + def is_cancel_secret(self, candidate_secret): """ Hash the candidate secret and compare the result to the stored hashed @@ -288,10 +321,20 @@ class HashedLeaseInfo(proxyForInterface(ILeaseInfo, "_lease_info")): # type: ign return super(HashedLeaseInfo, self).is_cancel_secret(hashed_candidate) + def present_cancel_secret(self): + """ + Present the hash of the secret with a marker indicating it is a hash. + """ + return u"hash:" + super(HashedLeaseInfo, self).present_cancel_secret() + @property def owner_num(self): return self._lease_info.owner_num + @property + def nodeid(self): + return self._lease_info.nodeid + @property def cancel_secret(self): """ From 5d703d989339587cfd5706fea1728ecb59e17808 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 5 Nov 2021 14:10:27 -0400 Subject: [PATCH 0377/2309] some type annotations --- src/allmydata/storage/lease.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/allmydata/storage/lease.py b/src/allmydata/storage/lease.py index 3ec760dbe..9ddbc9c68 100644 --- a/src/allmydata/storage/lease.py +++ b/src/allmydata/storage/lease.py @@ -187,7 +187,7 @@ class LeaseInfo(object): return timing_safe_compare(self.renew_secret, candidate_secret) def present_renew_secret(self): - # type: () -> bytes + # type: () -> str """ Return the renew secret, base32-encoded. """ @@ -204,7 +204,7 @@ class LeaseInfo(object): return timing_safe_compare(self.cancel_secret, candidate_secret) def present_cancel_secret(self): - # type: () -> bytes + # type: () -> str """ Return the cancel secret, base32-encoded. """ @@ -288,6 +288,7 @@ class HashedLeaseInfo(proxyForInterface(ILeaseInfo, "_lease_info")): # type: ign _hash = attr.ib() def is_renew_secret(self, candidate_secret): + # type: (bytes) -> bool """ Hash the candidate secret and compare the result to the stored hashed secret. @@ -295,12 +296,14 @@ class HashedLeaseInfo(proxyForInterface(ILeaseInfo, "_lease_info")): # type: ign return super(HashedLeaseInfo, self).is_renew_secret(self._hash(candidate_secret)) def present_renew_secret(self): + # type: () -> str """ Present the hash of the secret with a marker indicating it is a hash. """ return u"hash:" + super(HashedLeaseInfo, self).present_renew_secret() def is_cancel_secret(self, candidate_secret): + # type: (bytes) -> bool """ Hash the candidate secret and compare the result to the stored hashed secret. @@ -322,6 +325,7 @@ class HashedLeaseInfo(proxyForInterface(ILeaseInfo, "_lease_info")): # type: ign return super(HashedLeaseInfo, self).is_cancel_secret(hashed_candidate) def present_cancel_secret(self): + # type: () -> str """ Present the hash of the secret with a marker indicating it is a hash. """ From 3de9c73b0b066e5e15978a15c2903d10e398ed0a Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 5 Nov 2021 14:11:05 -0400 Subject: [PATCH 0378/2309] preserve the type when renewing HashedLeaseInfo does this mean immutable lease renewal is untested? maybe --- src/allmydata/storage/lease.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/allmydata/storage/lease.py b/src/allmydata/storage/lease.py index 9ddbc9c68..1a5416d6a 100644 --- a/src/allmydata/storage/lease.py +++ b/src/allmydata/storage/lease.py @@ -287,6 +287,13 @@ class HashedLeaseInfo(proxyForInterface(ILeaseInfo, "_lease_info")): # type: ign _lease_info = attr.ib() _hash = attr.ib() + def renew(self, new_expire_time): + # Preserve the HashedLeaseInfo wrapper around the renewed LeaseInfo. + return attr.assoc( + self, + _lease_info=super(HashedLeaseInfo, self).renew(new_expire_time), + ) + def is_renew_secret(self, candidate_secret): # type: (bytes) -> bool """ From 456df65a07a0c48f3a056519282cc96b5e4e2f25 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 5 Nov 2021 14:16:43 -0400 Subject: [PATCH 0379/2309] Add v2 of the mutable container schema It uses hashed lease secrets, like v2 of the immutable container schema. --- src/allmydata/storage/mutable_schema.py | 225 ++++++++++++++++++++---- 1 file changed, 187 insertions(+), 38 deletions(-) diff --git a/src/allmydata/storage/mutable_schema.py b/src/allmydata/storage/mutable_schema.py index 25f24ea1f..9496fe571 100644 --- a/src/allmydata/storage/mutable_schema.py +++ b/src/allmydata/storage/mutable_schema.py @@ -13,23 +13,193 @@ if PY2: import struct +try: + from typing import Union +except ImportError: + pass + +import attr + +from nacl.hash import blake2b +from nacl.encoding import RawEncoder + +from ..util.hashutil import ( + tagged_hash, +) from .lease import ( LeaseInfo, + HashedLeaseInfo, ) +def _magic(version): + # type: (int) -> bytes + """ + Compute a "magic" header string for a container of the given version. + + :param version: The version number of the container. + """ + # Make it easy for people to recognize + human_readable = u"Tahoe mutable container v{:d}\n".format(version).encode("ascii") + # But also keep the chance of accidental collision low + if version == 1: + # It's unclear where this byte sequence came from. It may have just + # been random. In any case, preserve it since it is the magic marker + # in all v1 share files. + random_bytes = b"\x75\x09\x44\x03\x8e" + else: + # For future versions, use a reproducable scheme. + random_bytes = tagged_hash( + b"allmydata_mutable_container_header", + human_readable, + truncate_to=5, + ) + magic = human_readable + random_bytes + assert len(magic) == 32 + if version > 1: + # The chance of collision is pretty low but let's just be sure about + # it. + assert magic != _magic(version - 1) + + return magic + +def _header(magic, extra_lease_offset, nodeid, write_enabler): + # type: (bytes, int, bytes, bytes) -> bytes + """ + Construct a container header. + + :param nodeid: A unique identifier for the node holding this + container. + + :param write_enabler: A secret shared with the client used to + authorize changes to the contents of this container. + """ + fixed_header = struct.pack( + ">32s20s32sQQ", + magic, + nodeid, + write_enabler, + # data length, initially the container is empty + 0, + extra_lease_offset, + ) + blank_leases = b"\x00" * LeaseInfo().mutable_size() * 4 + extra_lease_count = struct.pack(">L", 0) + + return b"".join([ + fixed_header, + # share data will go in between the next two items eventually but + # for now there is none. + blank_leases, + extra_lease_count, + ]) + + +class _V2(object): + """ + Implement encoding and decoding for v2 of the mutable container. + """ + version = 2 + _MAGIC = _magic(version) + + _HEADER_FORMAT = ">32s20s32sQQ" + + # This size excludes leases + _HEADER_SIZE = struct.calcsize(_HEADER_FORMAT) + + _EXTRA_LEASE_OFFSET = _HEADER_SIZE + 4 * LeaseInfo().mutable_size() + + @classmethod + def _hash_secret(cls, secret): + # type: (bytes) -> bytes + """ + Hash a lease secret for storage. + """ + return blake2b(secret, digest_size=32, encoder=RawEncoder()) + + @classmethod + def _hash_lease_info(cls, lease_info): + # type: (LeaseInfo) -> HashedLeaseInfo + """ + Hash the cleartext lease info secrets into a ``HashedLeaseInfo``. + """ + if not isinstance(lease_info, LeaseInfo): + # Provide a little safety against misuse, especially an attempt to + # re-hash an already-hashed lease info which is represented as a + # different type. + raise TypeError( + "Can only hash LeaseInfo, not {!r}".format(lease_info), + ) + + # Hash the cleartext secrets in the lease info and wrap the result in + # a new type. + return HashedLeaseInfo( + attr.assoc( + lease_info, + renew_secret=cls._hash_secret(lease_info.renew_secret), + cancel_secret=cls._hash_secret(lease_info.cancel_secret), + ), + cls._hash_secret, + ) + + @classmethod + def magic_matches(cls, candidate_magic): + # type: (bytes) -> bool + """ + Return ``True`` if a candidate string matches the expected magic string + from a mutable container header, ``False`` otherwise. + """ + return candidate_magic[:len(cls._MAGIC)] == cls._MAGIC + + @classmethod + def header(cls, nodeid, write_enabler): + return _header(cls._MAGIC, cls._EXTRA_LEASE_OFFSET, nodeid, write_enabler) + + @classmethod + def serialize_lease(cls, lease): + # type: (Union[LeaseInfo, HashedLeaseInfo]) -> bytes + """ + Serialize a lease to be written to a v2 container. + + :param lease: the lease to serialize + + :return: the serialized bytes + """ + if isinstance(lease, LeaseInfo): + # v2 of the mutable schema stores lease secrets hashed. If we're + # given a LeaseInfo then it holds plaintext secrets. Hash them + # before trying to serialize. + lease = cls._hash_lease_info(lease) + if isinstance(lease, HashedLeaseInfo): + return lease.to_mutable_data() + raise ValueError( + "MutableShareFile v2 schema cannot represent lease {!r}".format( + lease, + ), + ) + + @classmethod + def unserialize_lease(cls, data): + # type: (bytes) -> HashedLeaseInfo + """ + Unserialize some bytes from a v2 container. + + :param data: the bytes from the container + + :return: the ``HashedLeaseInfo`` the bytes represent + """ + # In v2 of the immutable schema lease secrets are stored hashed. Wrap + # a LeaseInfo in a HashedLeaseInfo so it can supply the correct + # interpretation for those values. + lease = LeaseInfo.from_mutable_data(data) + return HashedLeaseInfo(lease, cls._hash_secret) + + class _V1(object): """ Implement encoding and decoding for v1 of the mutable container. """ version = 1 - - _MAGIC = ( - # Make it easy for people to recognize - b"Tahoe mutable container v1\n" - # But also keep the chance of accidental collision low - b"\x75\x09\x44\x03\x8e" - ) - assert len(_MAGIC) == 32 + _MAGIC = _magic(version) _HEADER_FORMAT = ">32s20s32sQQ" @@ -49,35 +219,8 @@ class _V1(object): @classmethod def header(cls, nodeid, write_enabler): - # type: (bytes, bytes) -> bytes - """ - Construct a container header. + return _header(cls._MAGIC, cls._EXTRA_LEASE_OFFSET, nodeid, write_enabler) - :param nodeid: A unique identifier for the node holding this - container. - - :param write_enabler: A secret shared with the client used to - authorize changes to the contents of this container. - """ - fixed_header = struct.pack( - ">32s20s32sQQ", - cls._MAGIC, - nodeid, - write_enabler, - # data length, initially the container is empty - 0, - cls._EXTRA_LEASE_OFFSET, - ) - blank_leases = b"\x00" * LeaseInfo().mutable_size() * 4 - extra_lease_count = struct.pack(">L", 0) - - return b"".join([ - fixed_header, - # share data will go in between the next two items eventually but - # for now there is none. - blank_leases, - extra_lease_count, - ]) @classmethod def serialize_lease(cls, lease_info): @@ -89,7 +232,13 @@ class _V1(object): :return: the serialized bytes """ - return lease_info.to_mutable_data() + if isinstance(lease, LeaseInfo): + return lease_info.to_mutable_data() + raise ValueError( + "MutableShareFile v1 schema only supports LeaseInfo, not {!r}".format( + lease, + ), + ) @classmethod def unserialize_lease(cls, data): @@ -104,7 +253,7 @@ class _V1(object): return LeaseInfo.from_mutable_data(data) -ALL_SCHEMAS = {_V1} +ALL_SCHEMAS = {_V2, _V1} ALL_SCHEMA_VERSIONS = {schema.version for schema in ALL_SCHEMAS} # type: ignore NEWEST_SCHEMA_VERSION = max(ALL_SCHEMAS, key=lambda schema: schema.version) # type: ignore From 617a1eac9d848661b1fce2fe18976796ce02ac2a Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 5 Nov 2021 15:30:49 -0400 Subject: [PATCH 0380/2309] refactor lease hashing logic to avoid mutable/immutable duplication --- src/allmydata/storage/immutable.py | 4 +- src/allmydata/storage/immutable_schema.py | 169 ++++--------------- src/allmydata/storage/lease.py | 4 +- src/allmydata/storage/lease_schema.py | 129 ++++++++++++++ src/allmydata/storage/mutable.py | 4 +- src/allmydata/storage/mutable_schema.py | 194 ++++------------------ 6 files changed, 199 insertions(+), 305 deletions(-) create mode 100644 src/allmydata/storage/lease_schema.py diff --git a/src/allmydata/storage/immutable.py b/src/allmydata/storage/immutable.py index 216262a81..e9992d96e 100644 --- a/src/allmydata/storage/immutable.py +++ b/src/allmydata/storage/immutable.py @@ -231,7 +231,7 @@ class ShareFile(object): offset = self._lease_offset + lease_number * self.LEASE_SIZE f.seek(offset) assert f.tell() == offset - f.write(self._schema.serialize_lease(lease_info)) + f.write(self._schema.lease_serializer.serialize(lease_info)) def _read_num_leases(self, f): f.seek(0x08) @@ -262,7 +262,7 @@ class ShareFile(object): for i in range(num_leases): data = f.read(self.LEASE_SIZE) if data: - yield self._schema.unserialize_lease(data) + yield self._schema.lease_serializer.unserialize(data) def add_lease(self, lease_info): with open(self.home, 'rb+') as f: diff --git a/src/allmydata/storage/immutable_schema.py b/src/allmydata/storage/immutable_schema.py index 440755b01..40663b935 100644 --- a/src/allmydata/storage/immutable_schema.py +++ b/src/allmydata/storage/immutable_schema.py @@ -13,84 +13,28 @@ if PY2: import struct -try: - from typing import Union -except ImportError: - pass - import attr -from nacl.hash import blake2b -from nacl.encoding import RawEncoder - -from .lease import ( - LeaseInfo, - HashedLeaseInfo, +from .lease_schema import ( + v1_immutable, + v2_immutable, ) -def _header(version, max_size): - # type: (int, int) -> bytes +@attr.s(frozen=True) +class _Schema(object): """ - Construct the header for an immutable container. + Implement encoding and decoding for multiple versions of the immutable + container schema. - :param version: the container version to include the in header - :param max_size: the maximum data size the container will hold + :ivar int version: the version number of the schema this object supports - :return: some bytes to write at the beginning of the container + :ivar lease_serializer: an object that is responsible for lease + serialization and unserialization """ - # The second field -- the four-byte share data length -- is no longer - # used as of Tahoe v1.3.0, but we continue to write it in there in - # case someone downgrades a storage server from >= Tahoe-1.3.0 to < - # Tahoe-1.3.0, or moves a share file from one server to another, - # etc. We do saturation -- a share data length larger than 2**32-1 - # (what can fit into the field) is marked as the largest length that - # can fit into the field. That way, even if this does happen, the old - # < v1.3.0 server will still allow clients to read the first part of - # the share. - return struct.pack(">LLL", version, min(2**32 - 1, max_size), 0) + version = attr.ib() + lease_serializer = attr.ib() - -class _V2(object): - """ - Implement encoding and decoding for v2 of the immutable container. - """ - version = 2 - - @classmethod - def _hash_secret(cls, secret): - # type: (bytes) -> bytes - """ - Hash a lease secret for storage. - """ - return blake2b(secret, digest_size=32, encoder=RawEncoder()) - - @classmethod - def _hash_lease_info(cls, lease_info): - # type: (LeaseInfo) -> HashedLeaseInfo - """ - Hash the cleartext lease info secrets into a ``HashedLeaseInfo``. - """ - if not isinstance(lease_info, LeaseInfo): - # Provide a little safety against misuse, especially an attempt to - # re-hash an already-hashed lease info which is represented as a - # different type. - raise TypeError( - "Can only hash LeaseInfo, not {!r}".format(lease_info), - ) - - # Hash the cleartext secrets in the lease info and wrap the result in - # a new type. - return HashedLeaseInfo( - attr.assoc( - lease_info, - renew_secret=cls._hash_secret(lease_info.renew_secret), - cancel_secret=cls._hash_secret(lease_info.cancel_secret), - ), - cls._hash_secret, - ) - - @classmethod - def header(cls, max_size): + def header(self, max_size): # type: (int) -> bytes """ Construct a container header. @@ -99,78 +43,23 @@ class _V2(object): :return: the header bytes """ - return _header(cls.version, max_size) + # The second field -- the four-byte share data length -- is no longer + # used as of Tahoe v1.3.0, but we continue to write it in there in + # case someone downgrades a storage server from >= Tahoe-1.3.0 to < + # Tahoe-1.3.0, or moves a share file from one server to another, + # etc. We do saturation -- a share data length larger than 2**32-1 + # (what can fit into the field) is marked as the largest length that + # can fit into the field. That way, even if this does happen, the old + # < v1.3.0 server will still allow clients to read the first part of + # the share. + return struct.pack(">LLL", self.version, min(2**32 - 1, max_size), 0) - @classmethod - def serialize_lease(cls, lease): - # type: (Union[LeaseInfo, HashedLeaseInfo]) -> bytes - """ - Serialize a lease to be written to a v2 container. - - :param lease: the lease to serialize - - :return: the serialized bytes - """ - if isinstance(lease, LeaseInfo): - # v2 of the immutable schema stores lease secrets hashed. If - # we're given a LeaseInfo then it holds plaintext secrets. Hash - # them before trying to serialize. - lease = cls._hash_lease_info(lease) - if isinstance(lease, HashedLeaseInfo): - return lease.to_immutable_data() - raise ValueError( - "ShareFile v2 schema cannot represent lease {!r}".format( - lease, - ), - ) - - @classmethod - def unserialize_lease(cls, data): - # type: (bytes) -> HashedLeaseInfo - """ - Unserialize some bytes from a v2 container. - - :param data: the bytes from the container - - :return: the ``HashedLeaseInfo`` the bytes represent - """ - # In v2 of the immutable schema lease secrets are stored hashed. Wrap - # a LeaseInfo in a HashedLeaseInfo so it can supply the correct - # interpretation for those values. - return HashedLeaseInfo(LeaseInfo.from_immutable_data(data), cls._hash_secret) - - -class _V1(object): - """ - Implement encoding and decoding for v1 of the immutable container. - """ - version = 1 - - @classmethod - def header(cls, max_size): - return _header(cls.version, max_size) - - @classmethod - def serialize_lease(cls, lease): - if isinstance(lease, LeaseInfo): - return lease.to_immutable_data() - raise ValueError( - "ShareFile v1 schema only supports LeaseInfo, not {!r}".format( - lease, - ), - ) - - @classmethod - def unserialize_lease(cls, data): - # In v1 of the immutable schema lease secrets are stored plaintext. - # So load the data into a plain LeaseInfo which works on plaintext - # secrets. - return LeaseInfo.from_immutable_data(data) - - -ALL_SCHEMAS = {_V2, _V1} -ALL_SCHEMA_VERSIONS = {schema.version for schema in ALL_SCHEMAS} # type: ignore -NEWEST_SCHEMA_VERSION = max(ALL_SCHEMAS, key=lambda schema: schema.version) # type: ignore +ALL_SCHEMAS = { + _Schema(version=2, lease_serializer=v2_immutable), + _Schema(version=1, lease_serializer=v1_immutable), +} +ALL_SCHEMA_VERSIONS = {schema.version for schema in ALL_SCHEMAS} +NEWEST_SCHEMA_VERSION = max(ALL_SCHEMAS, key=lambda schema: schema.version) def schema_from_version(version): # (int) -> Optional[type] diff --git a/src/allmydata/storage/lease.py b/src/allmydata/storage/lease.py index 1a5416d6a..8be44bafd 100644 --- a/src/allmydata/storage/lease.py +++ b/src/allmydata/storage/lease.py @@ -230,7 +230,7 @@ class LeaseInfo(object): "cancel_secret", "expiration_time", ] - values = struct.unpack(">L32s32sL", data) + values = struct.unpack(IMMUTABLE_FORMAT, data) return cls(nodeid=None, **dict(zip(names, values))) def immutable_size(self): @@ -274,7 +274,7 @@ class LeaseInfo(object): "cancel_secret", "nodeid", ] - values = struct.unpack(">LL32s32s20s", data) + values = struct.unpack(MUTABLE_FORMAT, data) return cls(**dict(zip(names, values))) diff --git a/src/allmydata/storage/lease_schema.py b/src/allmydata/storage/lease_schema.py new file mode 100644 index 000000000..697ac9e34 --- /dev/null +++ b/src/allmydata/storage/lease_schema.py @@ -0,0 +1,129 @@ +""" +Ported to Python 3. +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + +try: + from typing import Union +except ImportError: + pass + +import attr + +from nacl.hash import blake2b +from nacl.encoding import RawEncoder + +from .lease import ( + LeaseInfo, + HashedLeaseInfo, +) + +@attr.s(frozen=True) +class CleartextLeaseSerializer(object): + _to_data = attr.ib() + _from_data = attr.ib() + + def serialize(self, lease): + # type: (LeaseInfo) -> bytes + if isinstance(lease, LeaseInfo): + return self._to_data(lease) + raise ValueError( + "ShareFile v1 schema only supports LeaseInfo, not {!r}".format( + lease, + ), + ) + + def unserialize(self, data): + # type: (bytes) -> LeaseInfo + # In v1 of the immutable schema lease secrets are stored plaintext. + # So load the data into a plain LeaseInfo which works on plaintext + # secrets. + return self._from_data(data) + +@attr.s(frozen=True) +class HashedLeaseSerializer(object): + _to_data = attr.ib() + _from_data = attr.ib() + + @classmethod + def _hash_secret(cls, secret): + # type: (bytes) -> bytes + """ + Hash a lease secret for storage. + """ + return blake2b(secret, digest_size=32, encoder=RawEncoder()) + + @classmethod + def _hash_lease_info(cls, lease_info): + # type: (LeaseInfo) -> HashedLeaseInfo + """ + Hash the cleartext lease info secrets into a ``HashedLeaseInfo``. + """ + if not isinstance(lease_info, LeaseInfo): + # Provide a little safety against misuse, especially an attempt to + # re-hash an already-hashed lease info which is represented as a + # different type. + raise TypeError( + "Can only hash LeaseInfo, not {!r}".format(lease_info), + ) + + # Hash the cleartext secrets in the lease info and wrap the result in + # a new type. + return HashedLeaseInfo( + attr.assoc( + lease_info, + renew_secret=cls._hash_secret(lease_info.renew_secret), + cancel_secret=cls._hash_secret(lease_info.cancel_secret), + ), + cls._hash_secret, + ) + + def serialize(self, lease): + # type: (Union[LeaseInfo, HashedLeaseInfo]) -> bytes + if isinstance(lease, LeaseInfo): + # v2 of the immutable schema stores lease secrets hashed. If + # we're given a LeaseInfo then it holds plaintext secrets. Hash + # them before trying to serialize. + lease = self._hash_lease_info(lease) + if isinstance(lease, HashedLeaseInfo): + return self._to_data(lease) + raise ValueError( + "ShareFile v2 schema cannot represent lease {!r}".format( + lease, + ), + ) + + def unserialize(self, data): + # type: (bytes) -> HashedLeaseInfo + # In v2 of the immutable schema lease secrets are stored hashed. Wrap + # a LeaseInfo in a HashedLeaseInfo so it can supply the correct + # interpretation for those values. + return HashedLeaseInfo(self._from_data(data), self._hash_secret) + +v1_immutable = CleartextLeaseSerializer( + LeaseInfo.to_immutable_data, + LeaseInfo.from_immutable_data, +) + +v2_immutable = HashedLeaseSerializer( + HashedLeaseInfo.to_immutable_data, + LeaseInfo.from_immutable_data, +) + +v1_mutable = CleartextLeaseSerializer( + LeaseInfo.to_mutable_data, + LeaseInfo.from_mutable_data, +) + +v2_mutable = HashedLeaseSerializer( + HashedLeaseInfo.to_mutable_data, + LeaseInfo.from_mutable_data, +) diff --git a/src/allmydata/storage/mutable.py b/src/allmydata/storage/mutable.py index 346edd53a..bd59d96b8 100644 --- a/src/allmydata/storage/mutable.py +++ b/src/allmydata/storage/mutable.py @@ -236,7 +236,7 @@ class MutableShareFile(object): + (lease_number-4)*self.LEASE_SIZE) f.seek(offset) assert f.tell() == offset - f.write(self._schema.serialize_lease(lease_info)) + f.write(self._schema.lease_serializer.serialize(lease_info)) def _read_lease_record(self, f, lease_number): # returns a LeaseInfo instance, or None @@ -253,7 +253,7 @@ class MutableShareFile(object): f.seek(offset) assert f.tell() == offset data = f.read(self.LEASE_SIZE) - lease_info = self._schema.unserialize_lease(data) + lease_info = self._schema.lease_serializer.unserialize(data) if lease_info.owner_num == 0: return None return lease_info diff --git a/src/allmydata/storage/mutable_schema.py b/src/allmydata/storage/mutable_schema.py index 9496fe571..4be0d2137 100644 --- a/src/allmydata/storage/mutable_schema.py +++ b/src/allmydata/storage/mutable_schema.py @@ -13,22 +13,17 @@ if PY2: import struct -try: - from typing import Union -except ImportError: - pass - import attr -from nacl.hash import blake2b -from nacl.encoding import RawEncoder - from ..util.hashutil import ( tagged_hash, ) from .lease import ( LeaseInfo, - HashedLeaseInfo, +) +from .lease_schema import ( + v1_mutable, + v2_mutable, ) def _magic(version): @@ -94,168 +89,49 @@ def _header(magic, extra_lease_offset, nodeid, write_enabler): ]) -class _V2(object): +_HEADER_FORMAT = ">32s20s32sQQ" + +# This size excludes leases +_HEADER_SIZE = struct.calcsize(_HEADER_FORMAT) + +_EXTRA_LEASE_OFFSET = _HEADER_SIZE + 4 * LeaseInfo().mutable_size() + + +@attr.s(frozen=True) +class _Schema(object): """ - Implement encoding and decoding for v2 of the mutable container. + Implement encoding and decoding for the mutable container. + + :ivar int version: the version number of the schema this object supports + + :ivar lease_serializer: an object that is responsible for lease + serialization and unserialization """ - version = 2 - _MAGIC = _magic(version) - - _HEADER_FORMAT = ">32s20s32sQQ" - - # This size excludes leases - _HEADER_SIZE = struct.calcsize(_HEADER_FORMAT) - - _EXTRA_LEASE_OFFSET = _HEADER_SIZE + 4 * LeaseInfo().mutable_size() + version = attr.ib() + lease_serializer = attr.ib() + _magic = attr.ib() @classmethod - def _hash_secret(cls, secret): - # type: (bytes) -> bytes - """ - Hash a lease secret for storage. - """ - return blake2b(secret, digest_size=32, encoder=RawEncoder()) + def for_version(cls, version, lease_serializer): + return cls(version, lease_serializer, magic=_magic(version)) - @classmethod - def _hash_lease_info(cls, lease_info): - # type: (LeaseInfo) -> HashedLeaseInfo - """ - Hash the cleartext lease info secrets into a ``HashedLeaseInfo``. - """ - if not isinstance(lease_info, LeaseInfo): - # Provide a little safety against misuse, especially an attempt to - # re-hash an already-hashed lease info which is represented as a - # different type. - raise TypeError( - "Can only hash LeaseInfo, not {!r}".format(lease_info), - ) - - # Hash the cleartext secrets in the lease info and wrap the result in - # a new type. - return HashedLeaseInfo( - attr.assoc( - lease_info, - renew_secret=cls._hash_secret(lease_info.renew_secret), - cancel_secret=cls._hash_secret(lease_info.cancel_secret), - ), - cls._hash_secret, - ) - - @classmethod - def magic_matches(cls, candidate_magic): + def magic_matches(self, candidate_magic): # type: (bytes) -> bool """ Return ``True`` if a candidate string matches the expected magic string from a mutable container header, ``False`` otherwise. """ - return candidate_magic[:len(cls._MAGIC)] == cls._MAGIC + return candidate_magic[:len(self._magic)] == self._magic - @classmethod - def header(cls, nodeid, write_enabler): - return _header(cls._MAGIC, cls._EXTRA_LEASE_OFFSET, nodeid, write_enabler) + def header(self, nodeid, write_enabler): + return _header(self._magic, _EXTRA_LEASE_OFFSET, nodeid, write_enabler) - @classmethod - def serialize_lease(cls, lease): - # type: (Union[LeaseInfo, HashedLeaseInfo]) -> bytes - """ - Serialize a lease to be written to a v2 container. - - :param lease: the lease to serialize - - :return: the serialized bytes - """ - if isinstance(lease, LeaseInfo): - # v2 of the mutable schema stores lease secrets hashed. If we're - # given a LeaseInfo then it holds plaintext secrets. Hash them - # before trying to serialize. - lease = cls._hash_lease_info(lease) - if isinstance(lease, HashedLeaseInfo): - return lease.to_mutable_data() - raise ValueError( - "MutableShareFile v2 schema cannot represent lease {!r}".format( - lease, - ), - ) - - @classmethod - def unserialize_lease(cls, data): - # type: (bytes) -> HashedLeaseInfo - """ - Unserialize some bytes from a v2 container. - - :param data: the bytes from the container - - :return: the ``HashedLeaseInfo`` the bytes represent - """ - # In v2 of the immutable schema lease secrets are stored hashed. Wrap - # a LeaseInfo in a HashedLeaseInfo so it can supply the correct - # interpretation for those values. - lease = LeaseInfo.from_mutable_data(data) - return HashedLeaseInfo(lease, cls._hash_secret) - - -class _V1(object): - """ - Implement encoding and decoding for v1 of the mutable container. - """ - version = 1 - _MAGIC = _magic(version) - - _HEADER_FORMAT = ">32s20s32sQQ" - - # This size excludes leases - _HEADER_SIZE = struct.calcsize(_HEADER_FORMAT) - - _EXTRA_LEASE_OFFSET = _HEADER_SIZE + 4 * LeaseInfo().mutable_size() - - @classmethod - def magic_matches(cls, candidate_magic): - # type: (bytes) -> bool - """ - Return ``True`` if a candidate string matches the expected magic string - from a mutable container header, ``False`` otherwise. - """ - return candidate_magic[:len(cls._MAGIC)] == cls._MAGIC - - @classmethod - def header(cls, nodeid, write_enabler): - return _header(cls._MAGIC, cls._EXTRA_LEASE_OFFSET, nodeid, write_enabler) - - - @classmethod - def serialize_lease(cls, lease_info): - # type: (LeaseInfo) -> bytes - """ - Serialize a lease to be written to a v1 container. - - :param lease: the lease to serialize - - :return: the serialized bytes - """ - if isinstance(lease, LeaseInfo): - return lease_info.to_mutable_data() - raise ValueError( - "MutableShareFile v1 schema only supports LeaseInfo, not {!r}".format( - lease, - ), - ) - - @classmethod - def unserialize_lease(cls, data): - # type: (bytes) -> LeaseInfo - """ - Unserialize some bytes from a v1 container. - - :param data: the bytes from the container - - :return: the ``LeaseInfo`` the bytes represent - """ - return LeaseInfo.from_mutable_data(data) - - -ALL_SCHEMAS = {_V2, _V1} -ALL_SCHEMA_VERSIONS = {schema.version for schema in ALL_SCHEMAS} # type: ignore -NEWEST_SCHEMA_VERSION = max(ALL_SCHEMAS, key=lambda schema: schema.version) # type: ignore +ALL_SCHEMAS = { + _Schema.for_version(version=2, lease_serializer=v2_mutable), + _Schema.for_version(version=1, lease_serializer=v1_mutable), +} +ALL_SCHEMA_VERSIONS = {schema.version for schema in ALL_SCHEMAS} +NEWEST_SCHEMA_VERSION = max(ALL_SCHEMAS, key=lambda schema: schema.version) def schema_from_header(header): # (int) -> Optional[type] From 66644791cbce41f31a08a7a9ba56449ccf02e33e Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 5 Nov 2021 15:36:26 -0400 Subject: [PATCH 0381/2309] news fragment --- newsfragments/3841.security | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/3841.security diff --git a/newsfragments/3841.security b/newsfragments/3841.security new file mode 100644 index 000000000..867322e0a --- /dev/null +++ b/newsfragments/3841.security @@ -0,0 +1 @@ +The storage server now keeps hashes of lease renew and cancel secrets for mutable share files instead of keeping the original secrets. \ No newline at end of file From 8dd4aaebb6e579b4874951bc4b6c6218ed667b79 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 10 Nov 2021 14:42:22 -0500 Subject: [PATCH 0382/2309] More consistent header system. --- docs/proposed/http-storage-node-protocol.rst | 106 +++++++++++-------- 1 file changed, 63 insertions(+), 43 deletions(-) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index fd1db5c4c..2a392fb20 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -363,11 +363,11 @@ one branch contains all of the share data; another branch contains all of the lease data; etc. -Authorization is required for all endpoints. +An ``Authorization`` header in requests is required for all endpoints. The standard HTTP authorization protocol is used. The authentication *type* used is ``Tahoe-LAFS``. The swissnum from the NURL used to locate the storage service is used as the *credentials*. -If credentials are not presented or the swissnum is not associated with a storage service then no storage processing is performed and the request receives an ``UNAUTHORIZED`` response. +If credentials are not presented or the swissnum is not associated with a storage service then no storage processing is performed and the request receives an ``401 UNAUTHORIZED`` response. General ~~~~~~~ @@ -396,17 +396,26 @@ For example:: !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! Either renew or create a new lease on the bucket addressed by ``storage_index``. -The details of the lease are encoded in the request body. + +For a renewal, the renew secret and cancellation secret should be included as ``X-Tahoe-Authorization`` headers. For example:: - {"renew-secret": "abcd", "cancel-secret": "efgh"} + X-Tahoe-Authorization: lease-renew-secret + X-Tahoe-Authorization: lease-cancel-secret -If the ``renew-secret`` value matches an existing lease -then the expiration time of that lease will be changed to 31 days after the time of this operation. -If it does not match an existing lease -then a new lease will be created with this ``renew-secret`` which expires 31 days after the time of this operation. +For a new lease, ``X-Tahoe-Set-Authorization`` headers should be used instead. +For example:: -``renew-secret`` and ``cancel-secret`` values must be 32 bytes long. + X-Tahoe-Set-Authorization: lease-renew-secret + X-Tahoe-Set-Authorization: lease-cancel-secret + +For renewal, the expiration time of that lease will be changed to 31 days after the time of this operation. +If the renewal secret does not match, a new lease will be created, but clients should still not rely on this behavior if possible, and instead use the appropriate new lease headers. + +For the creation path, +then a new lease will be created with this ``lease-renew-secret`` which expires 31 days after the time of this operation. + +``lease-renew-secret`` and ``lease-cancel-secret`` values must be 32 bytes long. The server treats them as opaque values. :ref:`Share Leases` gives details about how the Tahoe-LAFS storage client constructs these values. @@ -423,7 +432,7 @@ In these cases the server takes no action and returns ``NOT FOUND``. Discussion `````````` -We considered an alternative where ``renew-secret`` and ``cancel-secret`` are placed in query arguments on the request path. +We considered an alternative where ``lease-renew-secret`` and ``lease-cancel-secret`` are placed in query arguments on the request path. We chose to put these values into the request body to make the URL simpler. Several behaviors here are blindly copied from the Foolscap-based storage server protocol. @@ -452,13 +461,13 @@ For example:: {"share-numbers": [1, 7, ...], "allocated-size": 12345} -The request must include ``WWW-Authenticate`` HTTP headers that set the various secrets—upload, lease renewal, lease cancellation—that will be later used to authorize various operations. +The request must include ``X-Tahoe-Set-Authorization`` HTTP headers that set the various secrets—upload, lease renewal, lease cancellation—that will be later used to authorize various operations. Typically this is a header sent by the server, but in Tahoe-LAFS keys are set by the client, so may as well reuse it. For example:: - WWW-Authenticate: x-tahoe-renew-secret - WWW-Authenticate: x-tahoe-cancel-secret - WWW-Authenticate: x-tahoe-upload-secret + X-Tahoe-Set-Authorization: lease-renew-secret + X-Tahoe-Set-Authorization: lease-cancel-secret + X-Tahoe-Set-Authorization: upload-secret The response body includes encoded information about the created buckets. For example:: @@ -527,9 +536,9 @@ If any one of these requests fails then at most 128KiB of upload work needs to b The server must recognize when all of the data has been received and mark the share as complete (which it can do because it was informed of the size when the storage index was initialized). -The request must include a ``Authorization`` header that includes the upload secret:: +The request must include a ``X-Tahoe-Authorization`` header that includes the upload secret:: - Authorization: x-tahoe-upload-secret + X-Tahoe-Authorization: upload-secret Responses: @@ -557,9 +566,9 @@ Responses: This cancels an *in-progress* upload. -The request body looks this:: +The request must include a ``Authorization`` header that includes the upload secret:: - { "upload-secret": "xyzf" } + X-Tahoe-Authorization: upload-secret The response code: @@ -658,16 +667,16 @@ The first write operation on a mutable storage index creates it (that is, there is no separate "create this storage index" operation as there is for the immutable storage index type). -The request body includes the secrets necessary to rewrite to the shares -along with test, read, and write vectors for the operation. +The request must include ``X-Tahoe-Authorization`` headers with write enabler and lease secrets:: + + X-Tahoe-Authorization: write-enabler + X-Tahoe-Authorization: lease-lease-cancel-secret + X-Tahoe-Authorization: lease-renew-secret + +The request body includes test, read, and write vectors for the operation. For example:: { - "secrets": { - "write-enabler": "abcd", - "lease-renew": "efgh", - "lease-cancel": "ijkl" - }, "test-write-vectors": { 0: { "test": [{ @@ -733,9 +742,10 @@ Immutable Data 1. Create a bucket for storage index ``AAAAAAAAAAAAAAAA`` to hold two immutable shares, discovering that share ``1`` was already uploaded:: POST /v1/immutable/AAAAAAAAAAAAAAAA - WWW-Authenticate: x-tahoe-renew-secret efgh - WWW-Authenticate: x-tahoe-cancel-secret jjkl - WWW-Authenticate: x-tahoe-upload-secret xyzf + Authorization: Tahoe-LAFS nurl-swissnum + X-Tahoe-Set-Authorization: lease-renew-secret efgh + X-Tahoe-Set-Authorization: lease-cancel-secret jjkl + X-Tahoe-Set-Authorization: upload-secret xyzf {"share-numbers": [1, 7], "allocated-size": 48} @@ -745,22 +755,25 @@ Immutable Data #. Upload the content for immutable share ``7``:: PATCH /v1/immutable/AAAAAAAAAAAAAAAA/7 + Authorization: Tahoe-LAFS nurl-swissnum Content-Range: bytes 0-15/48 - Authorization: x-tahoe-upload-secret xyzf + X-Tahoe-Authorization: upload-secret xyzf 200 OK PATCH /v1/immutable/AAAAAAAAAAAAAAAA/7 + Authorization: Tahoe-LAFS nurl-swissnum Content-Range: bytes 16-31/48 - Authorization: x-tahoe-upload-secret xyzf + X-Tahoe-Authorization: upload-secret xyzf 200 OK PATCH /v1/immutable/AAAAAAAAAAAAAAAA/7 + Authorization: Tahoe-LAFS nurl-swissnum Content-Range: bytes 32-47/48 - Authorization: x-tahoe-upload-secret xyzf + X-Tahoe-Authorization: upload-secret xyzf 201 CREATED @@ -768,6 +781,7 @@ Immutable Data #. Download the content of the previously uploaded immutable share ``7``:: GET /v1/immutable/AAAAAAAAAAAAAAAA?share=7 + Authorization: Tahoe-LAFS nurl-swissnum Range: bytes=0-47 200 OK @@ -776,7 +790,9 @@ Immutable Data #. Renew the lease on all immutable shares in bucket ``AAAAAAAAAAAAAAAA``:: PUT /v1/lease/AAAAAAAAAAAAAAAA - {"renew-secret": "efgh", "cancel-secret": "ijkl"} + Authorization: Tahoe-LAFS nurl-swissnum + X-Tahoe-Authorization: lease-cancel-secret jjkl + X-Tahoe-Authorization: upload-secret xyzf 204 NO CONTENT @@ -789,12 +805,12 @@ if there is no existing share, otherwise it will read a byte which won't match `b""`:: POST /v1/mutable/BBBBBBBBBBBBBBBB/read-test-write + Authorization: Tahoe-LAFS nurl-swissnum + X-Tahoe-Authorization: write-enabler abcd + X-Tahoe-Authorization: lease-cancel-secret efgh + X-Tahoe-Authorization: lease-renew-secret ijkl + { - "secrets": { - "write-enabler": "abcd", - "lease-renew": "efgh", - "lease-cancel": "ijkl" - }, "test-write-vectors": { 3: { "test": [{ @@ -821,12 +837,12 @@ otherwise it will read a byte which won't match `b""`:: #. Safely rewrite the contents of a known version of mutable share number ``3`` (or fail):: POST /v1/mutable/BBBBBBBBBBBBBBBB/read-test-write + Authorization: Tahoe-LAFS nurl-swissnum + X-Tahoe-Authorization: write-enabler abcd + X-Tahoe-Authorization: lease-cancel-secret efgh + X-Tahoe-Authorization: lease-renew-secret ijkl + { - "secrets": { - "write-enabler": "abcd", - "lease-renew": "efgh", - "lease-cancel": "ijkl" - }, "test-write-vectors": { 3: { "test": [{ @@ -853,12 +869,16 @@ otherwise it will read a byte which won't match `b""`:: #. Download the contents of share number ``3``:: GET /v1/mutable/BBBBBBBBBBBBBBBB?share=3&offset=0&size=10 + Authorization: Tahoe-LAFS nurl-swissnum + #. Renew the lease on previously uploaded mutable share in slot ``BBBBBBBBBBBBBBBB``:: PUT /v1/lease/BBBBBBBBBBBBBBBB - {"renew-secret": "efgh", "cancel-secret": "ijkl"} + Authorization: Tahoe-LAFS nurl-swissnum + X-Tahoe-Authorization: lease-cancel-secret efgh + X-Tahoe-Authorization: lease-renew-secret ijkl 204 NO CONTENT From 7faec6e5a0bb53f5d58a4253795047144a58d62d Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 10 Nov 2021 15:48:58 -0500 Subject: [PATCH 0383/2309] news fragment --- newsfragments/3842.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3842.minor diff --git a/newsfragments/3842.minor b/newsfragments/3842.minor new file mode 100644 index 000000000..e69de29bb From 9af81d21c5af232c8e02e874b09ac33202cb5158 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 10 Nov 2021 16:08:40 -0500 Subject: [PATCH 0384/2309] add a way to turn off implicit bucket lease renewal too --- src/allmydata/storage/server.py | 50 ++++++++++++++++++++----- src/allmydata/test/test_storage.py | 59 +++++++++++++++++++++++++++++- 2 files changed, 98 insertions(+), 11 deletions(-) diff --git a/src/allmydata/storage/server.py b/src/allmydata/storage/server.py index 3e2d3b5c6..d142646a8 100644 --- a/src/allmydata/storage/server.py +++ b/src/allmydata/storage/server.py @@ -57,9 +57,23 @@ DEFAULT_RENEWAL_TIME = 31 * 24 * 60 * 60 @implementer(RIStorageServer, IStatsProducer) class StorageServer(service.MultiService, Referenceable): + """ + A filesystem-based implementation of ``RIStorageServer``. + + :ivar bool _implicit_bucket_lease_renewal: If and only if this is ``True`` + then ``allocate_buckets`` will renew leases on existing shares + associated with the storage index it operates on. + + :ivar bool _implicit_slot_lease_renewal: If and only if this is ``True`` + then ``slot_testv_and_readv_and_writev`` will renew leases on shares + associated with the slot it operates on. + """ name = 'storage' LeaseCheckerClass = LeaseCheckingCrawler + _implicit_bucket_lease_renewal = True + _implicit_slot_lease_renewal = True + def __init__(self, storedir, nodeid, reserved_space=0, discard_storage=False, readonly_storage=False, stats_provider=None, @@ -135,6 +149,29 @@ class StorageServer(service.MultiService, Referenceable): def __repr__(self): return "" % (idlib.shortnodeid_b2a(self.my_nodeid),) + def set_implicit_bucket_lease_renewal(self, enabled): + # type: (bool) -> None + """ + Control the behavior of implicit lease renewal by *allocate_buckets*. + + :param enabled: If and only if ``True`` then future *allocate_buckets* + calls will renew leases on shares that already exist in the bucket. + """ + self._implicit_bucket_lease_renewal = enabled + + def set_implicit_slot_lease_renewal(self, enabled): + # type: (bool) -> None + """ + Control the behavior of implicit lease renewal by + *slot_testv_and_readv_and_writev*. + + :param enabled: If and only if ``True`` then future + *slot_testv_and_readv_and_writev* calls will renew leases on + shares that still exist in the slot after the writev is applied + and which were touched by the writev. + """ + self._implicit_slot_lease_renewal = enabled + def have_shares(self): # quick test to decide if we need to commit to an implicit # permutation-seed or if we should use a new one @@ -319,8 +356,9 @@ class StorageServer(service.MultiService, Referenceable): # file, they'll want us to hold leases for this file. for (shnum, fn) in self._get_bucket_shares(storage_index): alreadygot.add(shnum) - sf = ShareFile(fn) - sf.add_or_renew_lease(lease_info) + if self._implicit_bucket_lease_renewal: + sf = ShareFile(fn) + sf.add_or_renew_lease(lease_info) for shnum in sharenums: incominghome = os.path.join(self.incomingdir, si_dir, "%d" % shnum) @@ -625,15 +663,10 @@ class StorageServer(service.MultiService, Referenceable): secrets, test_and_write_vectors, read_vector, - renew_leases, ): """ Read data from shares and conditionally write some data to them. - :param bool renew_leases: If and only if this is ``True`` and the test - vectors pass then shares in this slot will also have an updated - lease applied to them. - See ``allmydata.interfaces.RIStorageServer`` for details about other parameters and return value. """ @@ -673,7 +706,7 @@ class StorageServer(service.MultiService, Referenceable): test_and_write_vectors, shares, ) - if renew_leases: + if self._implicit_slot_lease_renewal: lease_info = self._make_lease_info(renew_secret, cancel_secret) self._add_or_renew_leases(remaining_shares, lease_info) @@ -690,7 +723,6 @@ class StorageServer(service.MultiService, Referenceable): secrets, test_and_write_vectors, read_vector, - renew_leases=True, ) def _allocate_slot_share(self, bucketdir, secrets, sharenum, diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index 460653bd0..efa889f8d 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -608,6 +608,61 @@ class Server(unittest.TestCase): for i,wb in writers.items(): wb.remote_abort() + def test_allocate_without_lease_renewal(self): + """ + ``remote_allocate_buckets`` does not renew leases on existing shares if + ``set_implicit_bucket_lease_renewal(False)`` is called first. + """ + first_lease = 456 + second_lease = 543 + storage_index = b"allocate" + + clock = Clock() + clock.advance(first_lease) + ss = self.create( + "test_allocate_without_lease_renewal", + get_current_time=clock.seconds, + ) + ss.set_implicit_bucket_lease_renewal(False) + + # Put a share on there + already, writers = self.allocate(ss, storage_index, [0], 1) + (writer,) = writers.values() + writer.remote_write(0, b"x") + writer.remote_close() + + # It should have a lease granted at the current time. + shares = dict(ss._get_bucket_shares(storage_index)) + self.assertEqual( + [first_lease], + list( + lease.get_grant_renew_time_time() + for lease + in ShareFile(shares[0]).get_leases() + ), + ) + + # Let some time pass so we can tell if the lease on share 0 is + # renewed. + clock.advance(second_lease) + + # Put another share on there. + already, writers = self.allocate(ss, storage_index, [1], 1) + (writer,) = writers.values() + writer.remote_write(0, b"x") + writer.remote_close() + + # The first share's lease expiration time is unchanged. + shares = dict(ss._get_bucket_shares(storage_index)) + self.assertEqual( + [first_lease], + list( + lease.get_grant_renew_time_time() + for lease + in ShareFile(shares[0]).get_leases() + ), + ) + def test_bad_container_version(self): ss = self.create("test_bad_container_version") a,w = self.allocate(ss, b"si1", [0], 10) @@ -1408,9 +1463,10 @@ class MutableServer(unittest.TestCase): def test_writev_without_renew_lease(self): """ The helper method ``slot_testv_and_readv_and_writev`` does not renew - leases if ``False`` is passed for the ``renew_leases`` parameter. + leases if ``set_implicit_bucket_lease_renewal(False)`` is called first. """ ss = self.create("test_writev_without_renew_lease") + ss.set_implicit_slot_lease_renewal(False) storage_index = b"si2" secrets = ( @@ -1429,7 +1485,6 @@ class MutableServer(unittest.TestCase): sharenum: ([], datav, None), }, read_vector=[], - renew_leases=False, ) leases = list(ss.get_slot_leases(storage_index)) self.assertEqual([], leases) From 2742de6f7c1fa6cf77e35ecc5854bcf7db3e5963 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 10 Nov 2021 16:08:53 -0500 Subject: [PATCH 0385/2309] drop some ancient cruft allocated_size not used anywhere, so why have it --- src/allmydata/storage/server.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/allmydata/storage/server.py b/src/allmydata/storage/server.py index d142646a8..36cf06d0e 100644 --- a/src/allmydata/storage/server.py +++ b/src/allmydata/storage/server.py @@ -617,10 +617,8 @@ class StorageServer(service.MultiService, Referenceable): else: if sharenum not in shares: # allocate a new share - allocated_size = 2000 # arbitrary, really share = self._allocate_slot_share(bucketdir, secrets, sharenum, - allocated_size, owner_num=0) shares[sharenum] = share shares[sharenum].writev(datav, new_length) @@ -726,7 +724,7 @@ class StorageServer(service.MultiService, Referenceable): ) def _allocate_slot_share(self, bucketdir, secrets, sharenum, - allocated_size, owner_num=0): + owner_num=0): (write_enabler, renew_secret, cancel_secret) = secrets my_nodeid = self.my_nodeid fileutil.make_dirs(bucketdir) From c270a346c6c7c247db08bf107bef93c4cccc7ced Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 11 Nov 2021 11:02:51 -0500 Subject: [PATCH 0386/2309] Remove typo. --- docs/proposed/http-storage-node-protocol.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index 2a392fb20..19a64f5ca 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -670,7 +670,7 @@ there is no separate "create this storage index" operation as there is for the i The request must include ``X-Tahoe-Authorization`` headers with write enabler and lease secrets:: X-Tahoe-Authorization: write-enabler - X-Tahoe-Authorization: lease-lease-cancel-secret + X-Tahoe-Authorization: lease-cancel-secret X-Tahoe-Authorization: lease-renew-secret The request body includes test, read, and write vectors for the operation. From 24646c56d0aae56bd18d2d2ffa2acf1616cc2a62 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 11 Nov 2021 11:29:05 -0500 Subject: [PATCH 0387/2309] Updates based on review. --- docs/proposed/http-storage-node-protocol.rst | 43 ++++++++------------ 1 file changed, 16 insertions(+), 27 deletions(-) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index 19a64f5ca..44bda1205 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -397,22 +397,15 @@ For example:: Either renew or create a new lease on the bucket addressed by ``storage_index``. -For a renewal, the renew secret and cancellation secret should be included as ``X-Tahoe-Authorization`` headers. +The renew secret and cancellation secret should be included as ``X-Tahoe-Authorization`` headers. For example:: X-Tahoe-Authorization: lease-renew-secret X-Tahoe-Authorization: lease-cancel-secret -For a new lease, ``X-Tahoe-Set-Authorization`` headers should be used instead. -For example:: - - X-Tahoe-Set-Authorization: lease-renew-secret - X-Tahoe-Set-Authorization: lease-cancel-secret - -For renewal, the expiration time of that lease will be changed to 31 days after the time of this operation. -If the renewal secret does not match, a new lease will be created, but clients should still not rely on this behavior if possible, and instead use the appropriate new lease headers. - -For the creation path, +If the ``lease-renew-secret`` value matches an existing lease +then the expiration time of that lease will be changed to 31 days after the time of this operation. +If it does not match an existing lease then a new lease will be created with this ``lease-renew-secret`` which expires 31 days after the time of this operation. ``lease-renew-secret`` and ``lease-cancel-secret`` values must be 32 bytes long. @@ -433,7 +426,9 @@ Discussion `````````` We considered an alternative where ``lease-renew-secret`` and ``lease-cancel-secret`` are placed in query arguments on the request path. -We chose to put these values into the request body to make the URL simpler. +This increases chances of leaking secrets in logs. +Putting the secrets in the body reduces the chances of leaking secrets, +but eventually we chose headers as the least likely information to be logged. Several behaviors here are blindly copied from the Foolscap-based storage server protocol. @@ -461,13 +456,13 @@ For example:: {"share-numbers": [1, 7, ...], "allocated-size": 12345} -The request must include ``X-Tahoe-Set-Authorization`` HTTP headers that set the various secrets—upload, lease renewal, lease cancellation—that will be later used to authorize various operations. +The request must include ``X-Tahoe-Authorization`` HTTP headers that set the various secrets—upload, lease renewal, lease cancellation—that will be later used to authorize various operations. Typically this is a header sent by the server, but in Tahoe-LAFS keys are set by the client, so may as well reuse it. For example:: - X-Tahoe-Set-Authorization: lease-renew-secret - X-Tahoe-Set-Authorization: lease-cancel-secret - X-Tahoe-Set-Authorization: upload-secret + X-Tahoe-Authorization: lease-renew-secret + X-Tahoe-Authorization: lease-cancel-secret + X-Tahoe-Authorization: upload-secret The response body includes encoded information about the created buckets. For example:: @@ -475,12 +470,6 @@ For example:: {"already-have": [1, ...], "allocated": [7, ...]} The upload secret is an opaque _byte_ string. -It will be generated by hashing a combination of:b - -1. A tag. -2. The storage index, so it's unique across different source files. -3. The server ID, so it's unique across different servers. -4. The convergence secret, so that servers can't guess the upload secret for other servers. Discussion `````````` @@ -508,7 +497,7 @@ The response includes ``already-have`` and ``allocated`` for two reasons: Regarding upload secrets, the goal is for uploading and aborting (see next sections) to be authenticated by more than just the storage index. -In the future, we will want to generate them in a way that allows resuming/canceling when the client has issues. +In the future, we may want to generate them in a way that allows resuming/canceling when the client has issues. In the short term, they can just be a random byte string. The key security constraint is that each upload to each server has its own, unique upload key, tied to uploading that particular storage index to this particular server. @@ -566,7 +555,7 @@ Responses: This cancels an *in-progress* upload. -The request must include a ``Authorization`` header that includes the upload secret:: +The request must include a ``X-Tahoe-Authorization`` header that includes the upload secret:: X-Tahoe-Authorization: upload-secret @@ -743,9 +732,9 @@ Immutable Data POST /v1/immutable/AAAAAAAAAAAAAAAA Authorization: Tahoe-LAFS nurl-swissnum - X-Tahoe-Set-Authorization: lease-renew-secret efgh - X-Tahoe-Set-Authorization: lease-cancel-secret jjkl - X-Tahoe-Set-Authorization: upload-secret xyzf + X-Tahoe-Authorization: lease-renew-secret efgh + X-Tahoe-Authorization: lease-cancel-secret jjkl + X-Tahoe-Authorization: upload-secret xyzf {"share-numbers": [1, 7], "allocated-size": 48} From bea4cf18a0d7d91dece9fb4a45bb39c5b41b8e9d Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 12 Nov 2021 11:19:29 -0500 Subject: [PATCH 0388/2309] News file. --- newsfragments/3843.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3843.minor diff --git a/newsfragments/3843.minor b/newsfragments/3843.minor new file mode 100644 index 000000000..e69de29bb From e7a5d14c0e8c0077880e2a9ffbd1e3db3738dd93 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 12 Nov 2021 11:25:10 -0500 Subject: [PATCH 0389/2309] New requirements. --- nix/tahoe-lafs.nix | 2 +- setup.py | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/nix/tahoe-lafs.nix b/nix/tahoe-lafs.nix index e42afc57f..f691677f6 100644 --- a/nix/tahoe-lafs.nix +++ b/nix/tahoe-lafs.nix @@ -4,7 +4,7 @@ , setuptools, setuptoolsTrial, pyasn1, zope_interface , service-identity, pyyaml, magic-wormhole, treq, appdirs , beautifulsoup4, eliot, autobahn, cryptography, netifaces -, html5lib, pyutil, distro, configparser +, html5lib, pyutil, distro, configparser, klein, treq }: python.pkgs.buildPythonPackage rec { # Most of the time this is not exactly the release version (eg 1.16.0). diff --git a/setup.py b/setup.py index 8c6396937..3d9f5a509 100644 --- a/setup.py +++ b/setup.py @@ -140,6 +140,10 @@ install_requires = [ # For the RangeMap datastructure. "collections-extended", + + # HTTP server and client + "klein", + "treq", ] setup_requires = [ @@ -397,7 +401,6 @@ setup(name="tahoe-lafs", # also set in __init__.py # Python 2.7. "decorator < 5", "hypothesis >= 3.6.1", - "treq", "towncrier", "testtools", "fixtures", From 777d630f481e3010c399d0cc2e872bacd572e700 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 12 Nov 2021 12:00:07 -0500 Subject: [PATCH 0390/2309] Another dependency. --- nix/tahoe-lafs.nix | 2 +- setup.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/nix/tahoe-lafs.nix b/nix/tahoe-lafs.nix index f691677f6..a6a8a69ec 100644 --- a/nix/tahoe-lafs.nix +++ b/nix/tahoe-lafs.nix @@ -4,7 +4,7 @@ , setuptools, setuptoolsTrial, pyasn1, zope_interface , service-identity, pyyaml, magic-wormhole, treq, appdirs , beautifulsoup4, eliot, autobahn, cryptography, netifaces -, html5lib, pyutil, distro, configparser, klein, treq +, html5lib, pyutil, distro, configparser, klein, treq, cbor2 }: python.pkgs.buildPythonPackage rec { # Most of the time this is not exactly the release version (eg 1.16.0). diff --git a/setup.py b/setup.py index 3d9f5a509..7e7a955c6 100644 --- a/setup.py +++ b/setup.py @@ -144,6 +144,7 @@ install_requires = [ # HTTP server and client "klein", "treq", + "cbor2" ] setup_requires = [ From a32c6be978f0c857ee0465cf123b56058178a21e Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 12 Nov 2021 12:02:58 -0500 Subject: [PATCH 0391/2309] A sketch of what the HTTP server will look like. --- src/allmydata/storage/http_server.py | 66 ++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 src/allmydata/storage/http_server.py diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py new file mode 100644 index 000000000..87edda999 --- /dev/null +++ b/src/allmydata/storage/http_server.py @@ -0,0 +1,66 @@ +""" +HTTP server for storage. +""" + +from functools import wraps + +from klein import Klein +from twisted.web import http + +# Make sure to use pure Python versions: +from cbor2.encoder import dumps +from cbor2.decoder import loads + +from .server import StorageServer + + +def _authorization_decorator(f): + """ + Check the ``Authorization`` header, and (TODO: in later revision of code) + extract ``X-Tahoe-Authorization`` headers and pass them in. + """ + + @wraps(f) + def route(self, request, *args, **kwargs): + if request.headers["Authorization"] != self._swissnum: + request.setResponseCode(http.NOT_ALLOWED) + return b"" + # authorization = request.headers.getRawHeaders("X-Tahoe-Authorization", []) + # For now, just a placeholder: + authorization = None + return f(self, request, authorization, *args, **kwargs) + + +def _route(app, *route_args, **route_kwargs): + """ + Like Klein's @route, but with additional support for checking the + ``Authorization`` header as well as ``X-Tahoe-Authorization`` headers. The + latter will (TODO: in later revision of code) get passed in as second + argument to wrapped functions. + """ + + def decorator(f): + @app.route(*route_args, **route_kwargs) + @_authorization_decorator + def handle_route(*args, **kwargs): + return f(*args, **kwargs) + + return handle_route + + return decorator + + +class HTTPServer(object): + """ + A HTTP interface to the storage server. + """ + + _app = Klein() + + def __init__(self, storage_server: StorageServer, swissnum): + self._storage_server = storage_server + self._swissnum = swissnum + + @_route(_app, "/v1/version", methods=["GET"]) + def version(self, request, authorization): + return dumps(self._storage_server.remote_get_version()) From ddd2780bd243436d3630fdcee8b0340480736e27 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 12 Nov 2021 12:51:52 -0500 Subject: [PATCH 0392/2309] First sketch of HTTP client. --- src/allmydata/storage/http_client.py | 36 ++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 src/allmydata/storage/http_client.py diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py new file mode 100644 index 000000000..ca80b704e --- /dev/null +++ b/src/allmydata/storage/http_client.py @@ -0,0 +1,36 @@ +""" +HTTP client that talks to the HTTP storage server. +""" + +# Make sure to import Python version: +from cbor2.encoder import loads +from cbor2.decoder import loads + +from twisted.internet.defer import inlineCallbacks, returnValue +from hyperlink import DecodedURL +import treq + + +def _decode_cbor(response): + """Given HTTP response, return decoded CBOR body.""" + return treq.content(response).addCallback(loads) + + +class StorageClient(object): + """ + HTTP client that talks to the HTTP storage server. + """ + + def __init__(self, url: DecodedURL, swissnum, treq=treq): + self._base_url = url + self._swissnum = swissnum + self._treq = treq + + @inlineCallbacks + def get_version(self): + """ + Return the version metadata for the server. + """ + url = self._base_url.child("v1", "version") + response = _decode_cbor((yield self._treq.get(url))) + returnValue(response) From 12cbf8a90109548aaba570d977863bacc2e8fdad Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 12 Nov 2021 13:03:53 -0500 Subject: [PATCH 0393/2309] First sketch of HTTP testing infrastructure. --- src/allmydata/test/test_storage_http.py | 38 +++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 src/allmydata/test/test_storage_http.py diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py new file mode 100644 index 000000000..589cfdddf --- /dev/null +++ b/src/allmydata/test/test_storage_http.py @@ -0,0 +1,38 @@ +""" +Tests for HTTP storage client + server. +""" + +from twisted.trial.unittest import TestCase +from twisted.internet.defer import inlineCallbacks + +from treq.testing import StubTreq +from hyperlink import DecodedURL + +from ..storage.server import StorageServer +from ..storage.http_server import HTTPServer +from ..storage.http_client import StorageClient + + +class HTTPTests(TestCase): + """ + Tests of HTTP client talking to the HTTP server. + """ + + def setUp(self): + self.storage_server = StorageServer(self.mktemp(), b"\x00" * 20) + # TODO what should the swissnum _actually_ be? + self._http_server = HTTPServer(self._storage_server, b"abcd") + self.client = StorageClient( + DecodedURL.from_text("http://example.com"), + b"abcd", + treq=StubTreq(self._http_server.get_resource()), + ) + + @inlineCallbacks + def test_version(self): + """ + The client can return the version. + """ + version = yield self.client.get_version() + expected_version = self.storage_server.remote_get_version() + self.assertEqual(version, expected_version) From c101dd4dc9e33190da63daedd1963a1fb0e9f7cf Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 12 Nov 2021 13:13:19 -0500 Subject: [PATCH 0394/2309] Closer to first passing test. --- src/allmydata/storage/http_client.py | 20 +++++++++++++------- src/allmydata/storage/http_server.py | 20 +++++++++++++++----- src/allmydata/test/test_storage_http.py | 2 +- 3 files changed, 29 insertions(+), 13 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index ca80b704e..e593fd379 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -2,18 +2,23 @@ HTTP client that talks to the HTTP storage server. """ -# Make sure to import Python version: -from cbor2.encoder import loads -from cbor2.decoder import loads +# TODO Make sure to import Python version? +from cbor2 import loads, dumps -from twisted.internet.defer import inlineCallbacks, returnValue +from twisted.internet.defer import inlineCallbacks, returnValue, fail from hyperlink import DecodedURL import treq +class ClientException(Exception): + """An unexpected error.""" + + def _decode_cbor(response): """Given HTTP response, return decoded CBOR body.""" - return treq.content(response).addCallback(loads) + if response.code > 199 and response.code < 300: + return treq.content(response).addCallback(loads) + return fail(ClientException(response.code, response.phrase)) class StorageClient(object): @@ -32,5 +37,6 @@ class StorageClient(object): Return the version metadata for the server. """ url = self._base_url.child("v1", "version") - response = _decode_cbor((yield self._treq.get(url))) - returnValue(response) + response = yield self._treq.get(url) + decoded_response = yield _decode_cbor(response) + returnValue(decoded_response) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 87edda999..b862fe7b1 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -7,9 +7,8 @@ from functools import wraps from klein import Klein from twisted.web import http -# Make sure to use pure Python versions: -from cbor2.encoder import dumps -from cbor2.decoder import loads +# TODO Make sure to use pure Python versions? +from cbor2 import loads, dumps from .server import StorageServer @@ -22,14 +21,19 @@ def _authorization_decorator(f): @wraps(f) def route(self, request, *args, **kwargs): - if request.headers["Authorization"] != self._swissnum: + if ( + request.requestHeaders.getRawHeaders("Authorization", [None])[0] + != self._swissnum + ): request.setResponseCode(http.NOT_ALLOWED) return b"" - # authorization = request.headers.getRawHeaders("X-Tahoe-Authorization", []) + # authorization = request.requestHeaders.getRawHeaders("X-Tahoe-Authorization", []) # For now, just a placeholder: authorization = None return f(self, request, authorization, *args, **kwargs) + return route + def _route(app, *route_args, **route_kwargs): """ @@ -53,6 +57,8 @@ def _route(app, *route_args, **route_kwargs): class HTTPServer(object): """ A HTTP interface to the storage server. + + TODO returning CBOR should set CBOR content-type """ _app = Klein() @@ -61,6 +67,10 @@ class HTTPServer(object): self._storage_server = storage_server self._swissnum = swissnum + def get_resource(self): + """Return twisted.web Resource for this object.""" + return self._app.resource() + @_route(_app, "/v1/version", methods=["GET"]) def version(self, request, authorization): return dumps(self._storage_server.remote_get_version()) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 589cfdddf..663675f40 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -21,7 +21,7 @@ class HTTPTests(TestCase): def setUp(self): self.storage_server = StorageServer(self.mktemp(), b"\x00" * 20) # TODO what should the swissnum _actually_ be? - self._http_server = HTTPServer(self._storage_server, b"abcd") + self._http_server = HTTPServer(self.storage_server, b"abcd") self.client = StorageClient( DecodedURL.from_text("http://example.com"), b"abcd", From c3cb0ebaeaa196c24272ac1fd834ed3c30baa377 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 12 Nov 2021 16:20:27 -0500 Subject: [PATCH 0395/2309] Switch to per-call parameter for controlling lease renewal behavior This is closer to an implementation where you could have two frontends, say a Foolscap frontend and an HTTP frontend or even just two different HTTP frontends, which had different opinions about what the behaviour should be. --- src/allmydata/storage/server.py | 51 ++++++------------------ src/allmydata/test/test_storage.py | 63 +++++++++++++++++++++++------- 2 files changed, 61 insertions(+), 53 deletions(-) diff --git a/src/allmydata/storage/server.py b/src/allmydata/storage/server.py index 36cf06d0e..70d71f841 100644 --- a/src/allmydata/storage/server.py +++ b/src/allmydata/storage/server.py @@ -59,21 +59,10 @@ DEFAULT_RENEWAL_TIME = 31 * 24 * 60 * 60 class StorageServer(service.MultiService, Referenceable): """ A filesystem-based implementation of ``RIStorageServer``. - - :ivar bool _implicit_bucket_lease_renewal: If and only if this is ``True`` - then ``allocate_buckets`` will renew leases on existing shares - associated with the storage index it operates on. - - :ivar bool _implicit_slot_lease_renewal: If and only if this is ``True`` - then ``slot_testv_and_readv_and_writev`` will renew leases on shares - associated with the slot it operates on. """ name = 'storage' LeaseCheckerClass = LeaseCheckingCrawler - _implicit_bucket_lease_renewal = True - _implicit_slot_lease_renewal = True - def __init__(self, storedir, nodeid, reserved_space=0, discard_storage=False, readonly_storage=False, stats_provider=None, @@ -149,29 +138,6 @@ class StorageServer(service.MultiService, Referenceable): def __repr__(self): return "" % (idlib.shortnodeid_b2a(self.my_nodeid),) - def set_implicit_bucket_lease_renewal(self, enabled): - # type: (bool) -> None - """ - Control the behavior of implicit lease renewal by *allocate_buckets*. - - :param enabled: If and only if ``True`` then future *allocate_buckets* - calls will renew leases on shares that already exist in the bucket. - """ - self._implicit_bucket_lease_renewal = enabled - - def set_implicit_slot_lease_renewal(self, enabled): - # type: (bool) -> None - """ - Control the behavior of implicit lease renewal by - *slot_testv_and_readv_and_writev*. - - :param enabled: If and only if ``True`` then future - *slot_testv_and_readv_and_writev* calls will renew leases on - shares that still exist in the slot after the writev is applied - and which were touched by the writev. - """ - self._implicit_slot_lease_renewal = enabled - def have_shares(self): # quick test to decide if we need to commit to an implicit # permutation-seed or if we should use a new one @@ -314,9 +280,12 @@ class StorageServer(service.MultiService, Referenceable): def _allocate_buckets(self, storage_index, renew_secret, cancel_secret, sharenums, allocated_size, - owner_num=0): + owner_num=0, renew_leases=True): """ Generic bucket allocation API. + + :param bool renew_leases: If and only if this is ``True`` then + renew leases on existing shares in this bucket. """ # owner_num is not for clients to set, but rather it should be # curried into the PersonalStorageServer instance that is dedicated @@ -356,7 +325,7 @@ class StorageServer(service.MultiService, Referenceable): # file, they'll want us to hold leases for this file. for (shnum, fn) in self._get_bucket_shares(storage_index): alreadygot.add(shnum) - if self._implicit_bucket_lease_renewal: + if renew_leases: sf = ShareFile(fn) sf.add_or_renew_lease(lease_info) @@ -399,7 +368,7 @@ class StorageServer(service.MultiService, Referenceable): """Foolscap-specific ``allocate_buckets()`` API.""" alreadygot, bucketwriters = self._allocate_buckets( storage_index, renew_secret, cancel_secret, sharenums, allocated_size, - owner_num=owner_num, + owner_num=owner_num, renew_leases=True, ) # Abort BucketWriters if disconnection happens. for bw in bucketwriters.values(): @@ -661,12 +630,17 @@ class StorageServer(service.MultiService, Referenceable): secrets, test_and_write_vectors, read_vector, + renew_leases, ): """ Read data from shares and conditionally write some data to them. See ``allmydata.interfaces.RIStorageServer`` for details about other parameters and return value. + + :param bool renew_leases: If and only if this is ``True`` then renew + leases on all shares mentioned in ``test_and_write_vectors` that + still exist after the changes are made. """ start = self._get_current_time() self.count("writev") @@ -704,7 +678,7 @@ class StorageServer(service.MultiService, Referenceable): test_and_write_vectors, shares, ) - if self._implicit_slot_lease_renewal: + if renew_leases: lease_info = self._make_lease_info(renew_secret, cancel_secret) self._add_or_renew_leases(remaining_shares, lease_info) @@ -721,6 +695,7 @@ class StorageServer(service.MultiService, Referenceable): secrets, test_and_write_vectors, read_vector, + renew_leases=True, ) def _allocate_slot_share(self, bucketdir, secrets, sharenum, diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index efa889f8d..a6c1ac2c2 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -468,14 +468,19 @@ class Server(unittest.TestCase): sv1 = ver[b'http://allmydata.org/tahoe/protocols/storage/v1'] self.failUnlessIn(b'available-space', sv1) - def allocate(self, ss, storage_index, sharenums, size, canary=None): + def allocate(self, ss, storage_index, sharenums, size, renew_leases=True): + """ + Call directly into the storage server's allocate_buckets implementation, + skipping the Foolscap layer. + """ renew_secret = hashutil.my_renewal_secret_hash(b"%d" % next(self._lease_secret)) cancel_secret = hashutil.my_cancel_secret_hash(b"%d" % next(self._lease_secret)) - if not canary: - canary = FakeCanary() - return ss.remote_allocate_buckets(storage_index, - renew_secret, cancel_secret, - sharenums, size, canary) + return ss._allocate_buckets( + storage_index, + renew_secret, cancel_secret, + sharenums, size, + renew_leases=renew_leases, + ) def test_large_share(self): syslow = platform.system().lower() @@ -611,7 +616,7 @@ class Server(unittest.TestCase): def test_allocate_without_lease_renewal(self): """ ``remote_allocate_buckets`` does not renew leases on existing shares if - ``set_implicit_bucket_lease_renewal(False)`` is called first. + ``renew_leases`` is ``False``. """ first_lease = 456 second_lease = 543 @@ -623,10 +628,11 @@ class Server(unittest.TestCase): "test_allocate_without_lease_renewal", get_current_time=clock.seconds, ) - ss.set_implicit_bucket_lease_renewal(False) # Put a share on there - already, writers = self.allocate(ss, storage_index, [0], 1) + already, writers = self.allocate( + ss, storage_index, [0], 1, renew_leases=False, + ) (writer,) = writers.values() writer.remote_write(0, b"x") writer.remote_close() @@ -647,7 +653,9 @@ class Server(unittest.TestCase): clock.advance(second_lease) # Put another share on there. - already, writers = self.allocate(ss, storage_index, [1], 1) + already, writers = self.allocate( + ss, storage_index, [1], 1, renew_leases=False, + ) (writer,) = writers.values() writer.remote_write(0, b"x") writer.remote_close() @@ -684,8 +692,17 @@ class Server(unittest.TestCase): def test_disconnect(self): # simulate a disconnection ss = self.create("test_disconnect") + renew_secret = b"r" * 32 + cancel_secret = b"c" * 32 canary = FakeCanary() - already,writers = self.allocate(ss, b"disconnect", [0,1,2], 75, canary) + already,writers = ss.remote_allocate_buckets( + b"disconnect", + renew_secret, + cancel_secret, + sharenums=[0,1,2], + allocated_size=75, + canary=canary, + ) self.failUnlessEqual(already, set()) self.failUnlessEqual(set(writers.keys()), set([0,1,2])) for (f,args,kwargs) in list(canary.disconnectors.values()): @@ -717,8 +734,17 @@ class Server(unittest.TestCase): # the size we request. OVERHEAD = 3*4 LEASE_SIZE = 4+32+32+4 + renew_secret = b"r" * 32 + cancel_secret = b"c" * 32 canary = FakeCanary() - already, writers = self.allocate(ss, b"vid1", [0,1,2], 1000, canary) + already, writers = ss.remote_allocate_buckets( + b"vid1", + renew_secret, + cancel_secret, + sharenums=[0,1,2], + allocated_size=1000, + canary=canary, + ) self.failUnlessEqual(len(writers), 3) # now the StorageServer should have 3000 bytes provisionally # allocated, allowing only 2000 more to be claimed @@ -751,7 +777,14 @@ class Server(unittest.TestCase): # now there should be ALLOCATED=1001+12+72=1085 bytes allocated, and # 5000-1085=3915 free, therefore we can fit 39 100byte shares canary3 = FakeCanary() - already3, writers3 = self.allocate(ss, b"vid3", list(range(100)), 100, canary3) + already3, writers3 = ss.remote_allocate_buckets( + b"vid3", + renew_secret, + cancel_secret, + sharenums=list(range(100)), + allocated_size=100, + canary=canary3, + ) self.failUnlessEqual(len(writers3), 39) self.failUnlessEqual(len(ss._bucket_writers), 39) @@ -1463,10 +1496,9 @@ class MutableServer(unittest.TestCase): def test_writev_without_renew_lease(self): """ The helper method ``slot_testv_and_readv_and_writev`` does not renew - leases if ``set_implicit_bucket_lease_renewal(False)`` is called first. + leases if ``renew_leases```` is ``False``. """ ss = self.create("test_writev_without_renew_lease") - ss.set_implicit_slot_lease_renewal(False) storage_index = b"si2" secrets = ( @@ -1485,6 +1517,7 @@ class MutableServer(unittest.TestCase): sharenum: ([], datav, None), }, read_vector=[], + renew_leases=False, ) leases = list(ss.get_slot_leases(storage_index)) self.assertEqual([], leases) From 85977e48a7dde8ea29e196a6d466ae8685c2f6fc Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 12 Nov 2021 16:23:15 -0500 Subject: [PATCH 0396/2309] put this comment back and merge info from the two versions --- src/allmydata/storage/server.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/allmydata/storage/server.py b/src/allmydata/storage/server.py index 70d71f841..9b73963ae 100644 --- a/src/allmydata/storage/server.py +++ b/src/allmydata/storage/server.py @@ -635,12 +635,13 @@ class StorageServer(service.MultiService, Referenceable): """ Read data from shares and conditionally write some data to them. + :param bool renew_leases: If and only if this is ``True`` and the test + vectors pass then shares mentioned in ``test_and_write_vectors`` + that still exist after the changes are made will also have an + updated lease applied to them. + See ``allmydata.interfaces.RIStorageServer`` for details about other parameters and return value. - - :param bool renew_leases: If and only if this is ``True`` then renew - leases on all shares mentioned in ``test_and_write_vectors` that - still exist after the changes are made. """ start = self._get_current_time() self.count("writev") From dece67ee3ac8d2bd06b42e07a01492e3c4497ae6 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 12 Nov 2021 16:24:29 -0500 Subject: [PATCH 0397/2309] it is not the remote interface that varies anymore --- src/allmydata/test/test_storage.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index a6c1ac2c2..076e9f3d1 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -615,8 +615,8 @@ class Server(unittest.TestCase): def test_allocate_without_lease_renewal(self): """ - ``remote_allocate_buckets`` does not renew leases on existing shares if - ``renew_leases`` is ``False``. + ``StorageServer._allocate_buckets`` does not renew leases on existing + shares if ``renew_leases`` is ``False``. """ first_lease = 456 second_lease = 543 From 6c2e85e99145652625ff7a4d6791a410ce13c742 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 12 Nov 2021 16:25:36 -0500 Subject: [PATCH 0398/2309] put the comment back --- src/allmydata/test/test_storage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index 076e9f3d1..4e40a76a5 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -1496,7 +1496,7 @@ class MutableServer(unittest.TestCase): def test_writev_without_renew_lease(self): """ The helper method ``slot_testv_and_readv_and_writev`` does not renew - leases if ``renew_leases```` is ``False``. + leases if ``False`` is passed for the ``renew_leases`` parameter. """ ss = self.create("test_writev_without_renew_lease") From ad6017e63df94dbac7916f4673332b33deb8d5be Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 15 Nov 2021 08:08:14 -0500 Subject: [PATCH 0399/2309] clarify renew_leases docs on allocate_buckets --- src/allmydata/storage/server.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/allmydata/storage/server.py b/src/allmydata/storage/server.py index 9b73963ae..bfbc10b59 100644 --- a/src/allmydata/storage/server.py +++ b/src/allmydata/storage/server.py @@ -284,8 +284,10 @@ class StorageServer(service.MultiService, Referenceable): """ Generic bucket allocation API. - :param bool renew_leases: If and only if this is ``True`` then - renew leases on existing shares in this bucket. + :param bool renew_leases: If and only if this is ``True`` then renew a + secret-matching lease on (or, if none match, add a new lease to) + existing shares in this bucket. Any *new* shares are given a new + lease regardless. """ # owner_num is not for clients to set, but rather it should be # curried into the PersonalStorageServer instance that is dedicated From 84c19f5468b04279e1826ed77dc1e7d4b4ae00e8 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 15 Nov 2021 08:12:07 -0500 Subject: [PATCH 0400/2309] clarify renew_leases docs on slot_testv_and_readv_and_writev --- src/allmydata/storage/server.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/allmydata/storage/server.py b/src/allmydata/storage/server.py index bfbc10b59..ee2ea1c61 100644 --- a/src/allmydata/storage/server.py +++ b/src/allmydata/storage/server.py @@ -639,8 +639,9 @@ class StorageServer(service.MultiService, Referenceable): :param bool renew_leases: If and only if this is ``True`` and the test vectors pass then shares mentioned in ``test_and_write_vectors`` - that still exist after the changes are made will also have an - updated lease applied to them. + that still exist after the changes are made will also have a + secret-matching lease renewed (or, if none match, a new lease + added). See ``allmydata.interfaces.RIStorageServer`` for details about other parameters and return value. From fcd634fc43c42c838ac415767bd6eeb05172c82b Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 15 Nov 2021 13:34:46 -0500 Subject: [PATCH 0401/2309] some direct tests for the new utility function --- src/allmydata/test/common.py | 7 ++- src/allmydata/test/test_common_util.py | 78 +++++++++++++++++++++++++- 2 files changed, 81 insertions(+), 4 deletions(-) diff --git a/src/allmydata/test/common.py b/src/allmydata/test/common.py index 8e97fa598..76127fb57 100644 --- a/src/allmydata/test/common.py +++ b/src/allmydata/test/common.py @@ -1220,8 +1220,13 @@ def disable_modules(*names): A context manager which makes modules appear to be missing while it is active. - :param *names: The names of the modules to disappear. + :param *names: The names of the modules to disappear. Only top-level + modules are supported (that is, "." is not allowed in any names). + This is an implementation shortcoming which could be lifted if + desired. """ + if any("." in name for name in names): + raise ValueError("Names containing '.' are not supported.") missing = object() modules = list(sys.modules.get(n, missing) for n in names) for n in names: diff --git a/src/allmydata/test/test_common_util.py b/src/allmydata/test/test_common_util.py index 55986d123..c141adc8d 100644 --- a/src/allmydata/test/test_common_util.py +++ b/src/allmydata/test/test_common_util.py @@ -10,16 +10,30 @@ from future.utils import PY2 if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 +import sys import random -import unittest +from hypothesis import given +from hypothesis.strategies import lists, sampled_from +from testtools.matchers import Equals +from twisted.python.reflect import ( + ModuleNotFound, + namedAny, +) + +from .common import ( + SyncTestCase, + disable_modules, +) from allmydata.test.common_util import flip_one_bit -class TestFlipOneBit(unittest.TestCase): +class TestFlipOneBit(SyncTestCase): def setUp(self): - random.seed(42) # I tried using version=1 on PY3 to avoid the if below, to no avail. + super(TestFlipOneBit, self).setUp() + # I tried using version=1 on PY3 to avoid the if below, to no avail. + random.seed(42) def test_accepts_byte_string(self): actual = flip_one_bit(b'foo') @@ -27,3 +41,61 @@ class TestFlipOneBit(unittest.TestCase): def test_rejects_unicode_string(self): self.assertRaises(AssertionError, flip_one_bit, u'foo') + + + +def some_existing_modules(): + """ + Build the names of modules (as native strings) that exist and can be + imported. + """ + candidates = sorted( + name + for name + in sys.modules + if "." not in name + and sys.modules[name] is not None + ) + return sampled_from(candidates) + +class DisableModulesTests(SyncTestCase): + """ + Tests for ``disable_modules``. + """ + def setup_example(self): + return sys.modules.copy() + + def teardown_example(self, safe_modules): + sys.modules.update(safe_modules) + + @given(lists(some_existing_modules(), unique=True)) + def test_importerror(self, module_names): + """ + While the ``disable_modules`` context manager is active any import of the + modules identified by the names passed to it result in ``ImportError`` + being raised. + """ + def get_modules(): + return list( + namedAny(name) + for name + in module_names + ) + before_modules = get_modules() + + with disable_modules(*module_names): + for name in module_names: + with self.assertRaises(ModuleNotFound): + namedAny(name) + + after_modules = get_modules() + self.assertThat(before_modules, Equals(after_modules)) + + def test_dotted_names_rejected(self): + """ + If names with "." in them are passed to ``disable_modules`` then + ``ValueError`` is raised. + """ + with self.assertRaises(ValueError): + with disable_modules("foo.bar"): + pass From 304b0269e3afe6499eaa1a92abd4856c970da60b Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 16 Nov 2021 10:14:04 -0500 Subject: [PATCH 0402/2309] Apply suggestions from code review Co-authored-by: Jean-Paul Calderone --- docs/proposed/http-storage-node-protocol.rst | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index 44bda1205..bc109ac7e 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -400,8 +400,8 @@ Either renew or create a new lease on the bucket addressed by ``storage_index``. The renew secret and cancellation secret should be included as ``X-Tahoe-Authorization`` headers. For example:: - X-Tahoe-Authorization: lease-renew-secret - X-Tahoe-Authorization: lease-cancel-secret + X-Tahoe-Authorization: lease-renew-secret + X-Tahoe-Authorization: lease-cancel-secret If the ``lease-renew-secret`` value matches an existing lease then the expiration time of that lease will be changed to 31 days after the time of this operation. @@ -457,7 +457,6 @@ For example:: {"share-numbers": [1, 7, ...], "allocated-size": 12345} The request must include ``X-Tahoe-Authorization`` HTTP headers that set the various secrets—upload, lease renewal, lease cancellation—that will be later used to authorize various operations. -Typically this is a header sent by the server, but in Tahoe-LAFS keys are set by the client, so may as well reuse it. For example:: X-Tahoe-Authorization: lease-renew-secret @@ -499,7 +498,7 @@ Regarding upload secrets, the goal is for uploading and aborting (see next sections) to be authenticated by more than just the storage index. In the future, we may want to generate them in a way that allows resuming/canceling when the client has issues. In the short term, they can just be a random byte string. -The key security constraint is that each upload to each server has its own, unique upload key, +The primary security constraint is that each upload to each server has its own unique upload key, tied to uploading that particular storage index to this particular server. Rejected designs for upload secrets: @@ -527,7 +526,7 @@ The server must recognize when all of the data has been received and mark the sh The request must include a ``X-Tahoe-Authorization`` header that includes the upload secret:: - X-Tahoe-Authorization: upload-secret + X-Tahoe-Authorization: upload-secret Responses: @@ -557,7 +556,7 @@ This cancels an *in-progress* upload. The request must include a ``X-Tahoe-Authorization`` header that includes the upload secret:: - X-Tahoe-Authorization: upload-secret + X-Tahoe-Authorization: upload-secret The response code: @@ -658,7 +657,7 @@ there is no separate "create this storage index" operation as there is for the i The request must include ``X-Tahoe-Authorization`` headers with write enabler and lease secrets:: - X-Tahoe-Authorization: write-enabler + X-Tahoe-Authorization: write-enabler X-Tahoe-Authorization: lease-cancel-secret X-Tahoe-Authorization: lease-renew-secret From 7caffce8d509e1293248cb83d89e81e030b88e16 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 16 Nov 2021 10:14:19 -0500 Subject: [PATCH 0403/2309] Another review suggestion Co-authored-by: Jean-Paul Calderone --- docs/proposed/http-storage-node-protocol.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index bc109ac7e..490d3f3ca 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -780,7 +780,7 @@ Immutable Data PUT /v1/lease/AAAAAAAAAAAAAAAA Authorization: Tahoe-LAFS nurl-swissnum X-Tahoe-Authorization: lease-cancel-secret jjkl - X-Tahoe-Authorization: upload-secret xyzf + X-Tahoe-Authorization: lease-renew-secret efgh 204 NO CONTENT From 41ec63f7586124eaaf9ca65bb4d6c4884e16b48f Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 16 Nov 2021 10:56:21 -0500 Subject: [PATCH 0404/2309] Passing first tests. --- src/allmydata/storage/http_client.py | 22 ++++++++++++++++++++-- src/allmydata/storage/http_server.py | 8 ++++---- src/allmydata/test/test_storage_http.py | 19 +++++++++++++++++-- 3 files changed, 41 insertions(+), 8 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index e593fd379..412bf9cec 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -2,9 +2,13 @@ HTTP client that talks to the HTTP storage server. """ +import base64 + # TODO Make sure to import Python version? from cbor2 import loads, dumps + +from twisted.web.http_headers import Headers from twisted.internet.defer import inlineCallbacks, returnValue, fail from hyperlink import DecodedURL import treq @@ -21,6 +25,11 @@ def _decode_cbor(response): return fail(ClientException(response.code, response.phrase)) +def swissnum_auth_header(swissnum): + """Return value for ``Authentication`` header.""" + return b"Tahoe-LAFS " + base64.encodestring(swissnum).strip() + + class StorageClient(object): """ HTTP client that talks to the HTTP storage server. @@ -31,12 +40,21 @@ class StorageClient(object): self._swissnum = swissnum self._treq = treq + def _get_headers(self): + """Return the basic headers to be used by default.""" + headers = Headers() + headers.addRawHeader( + "Authorization", + swissnum_auth_header(self._swissnum), + ) + return headers + @inlineCallbacks def get_version(self): """ Return the version metadata for the server. """ - url = self._base_url.child("v1", "version") - response = yield self._treq.get(url) + url = self._base_url.click("/v1/version") + response = yield self._treq.get(url, headers=self._get_headers()) decoded_response = yield _decode_cbor(response) returnValue(decoded_response) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index b862fe7b1..2d6308baf 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -11,6 +11,7 @@ from twisted.web import http from cbor2 import loads, dumps from .server import StorageServer +from .http_client import swissnum_auth_header def _authorization_decorator(f): @@ -21,11 +22,10 @@ def _authorization_decorator(f): @wraps(f) def route(self, request, *args, **kwargs): - if ( - request.requestHeaders.getRawHeaders("Authorization", [None])[0] - != self._swissnum + if request.requestHeaders.getRawHeaders("Authorization", [None])[0] != str( + swissnum_auth_header(self._swissnum), "ascii" ): - request.setResponseCode(http.NOT_ALLOWED) + request.setResponseCode(http.UNAUTHORIZED) return b"" # authorization = request.requestHeaders.getRawHeaders("X-Tahoe-Authorization", []) # For now, just a placeholder: diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 663675f40..b659a6ace 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -10,7 +10,7 @@ from hyperlink import DecodedURL from ..storage.server import StorageServer from ..storage.http_server import HTTPServer -from ..storage.http_client import StorageClient +from ..storage.http_client import StorageClient, ClientException class HTTPTests(TestCase): @@ -23,11 +23,26 @@ class HTTPTests(TestCase): # TODO what should the swissnum _actually_ be? self._http_server = HTTPServer(self.storage_server, b"abcd") self.client = StorageClient( - DecodedURL.from_text("http://example.com"), + DecodedURL.from_text("http://127.0.0.1"), b"abcd", treq=StubTreq(self._http_server.get_resource()), ) + @inlineCallbacks + def test_bad_authentication(self): + """ + If the wrong swissnum is used, an ``Unauthorized`` response code is + returned. + """ + client = StorageClient( + DecodedURL.from_text("http://127.0.0.1"), + b"something wrong", + treq=StubTreq(self._http_server.get_resource()), + ) + with self.assertRaises(ClientException) as e: + yield client.get_version() + self.assertEqual(e.exception.args[0], 401) + @inlineCallbacks def test_version(self): """ From 671b670154f62cb6c7876c707f254a6c7b3a2f4f Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 16 Nov 2021 11:09:08 -0500 Subject: [PATCH 0405/2309] Some type annotations. --- src/allmydata/storage/http_client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 412bf9cec..8e14d1137 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -25,7 +25,7 @@ def _decode_cbor(response): return fail(ClientException(response.code, response.phrase)) -def swissnum_auth_header(swissnum): +def swissnum_auth_header(swissnum): # type: (bytes) -> bytes """Return value for ``Authentication`` header.""" return b"Tahoe-LAFS " + base64.encodestring(swissnum).strip() @@ -40,7 +40,7 @@ class StorageClient(object): self._swissnum = swissnum self._treq = treq - def _get_headers(self): + def _get_headers(self): # type: () -> Headers """Return the basic headers to be used by default.""" headers = Headers() headers.addRawHeader( From 171d1053ec803f2d2de57f0970fbad049d49f2da Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 16 Nov 2021 11:09:17 -0500 Subject: [PATCH 0406/2309] CBOR content-type on responses. --- src/allmydata/storage/http_server.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 2d6308baf..91387c58f 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -57,8 +57,6 @@ def _route(app, *route_args, **route_kwargs): class HTTPServer(object): """ A HTTP interface to the storage server. - - TODO returning CBOR should set CBOR content-type """ _app = Klein() @@ -71,6 +69,12 @@ class HTTPServer(object): """Return twisted.web Resource for this object.""" return self._app.resource() + def _cbor(self, request, data): + """Return CBOR-encoded data.""" + request.setHeader("Content-Type", "application/cbor") + # TODO if data is big, maybe want to use a temporary file eventually... + return dumps(data) + @_route(_app, "/v1/version", methods=["GET"]) def version(self, request, authorization): - return dumps(self._storage_server.remote_get_version()) + return self._cbor(request, self._storage_server.remote_get_version()) From c195f895db7bd3ec7a8618956a71e67152e32df7 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 16 Nov 2021 11:16:26 -0500 Subject: [PATCH 0407/2309] Python 2 support. --- src/allmydata/storage/http_client.py | 19 ++++++++++++++++++- src/allmydata/storage/http_server.py | 18 ++++++++++++++++-- src/allmydata/test/test_storage_http.py | 12 ++++++++++++ 3 files changed, 46 insertions(+), 3 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 8e14d1137..4a143a60b 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -2,6 +2,21 @@ HTTP client that talks to the HTTP storage server. """ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 + +if PY2: + # fmt: off + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + # fmt: on +else: + from typing import Union + from treq.testing import StubTreq + import base64 # TODO Make sure to import Python version? @@ -35,7 +50,9 @@ class StorageClient(object): HTTP client that talks to the HTTP storage server. """ - def __init__(self, url: DecodedURL, swissnum, treq=treq): + def __init__( + self, url, swissnum, treq=treq + ): # type: (DecodedURL, bytes, Union[treq,StubTreq]) -> None self._base_url = url self._swissnum = swissnum self._treq = treq diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 91387c58f..373d31e2e 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -2,6 +2,18 @@ HTTP server for storage. """ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 + +if PY2: + # fmt: off + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + # fmt: on + from functools import wraps from klein import Klein @@ -61,12 +73,14 @@ class HTTPServer(object): _app = Klein() - def __init__(self, storage_server: StorageServer, swissnum): + def __init__( + self, storage_server, swissnum + ): # type: (StorageServer, bytes) -> None self._storage_server = storage_server self._swissnum = swissnum def get_resource(self): - """Return twisted.web Resource for this object.""" + """Return twisted.web ``Resource`` for this object.""" return self._app.resource() def _cbor(self, request, data): diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index b659a6ace..9ba8adf21 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -2,6 +2,18 @@ Tests for HTTP storage client + server. """ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 + +if PY2: + # fmt: off + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + # fmt: on + from twisted.trial.unittest import TestCase from twisted.internet.defer import inlineCallbacks From a64778ddb0fb774ea43fa8a3c59be67b84e957ff Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 16 Nov 2021 11:28:13 -0500 Subject: [PATCH 0408/2309] Flakes. --- src/allmydata/storage/http_client.py | 2 +- src/allmydata/storage/http_server.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 4a143a60b..d5ca6caec 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -20,7 +20,7 @@ else: import base64 # TODO Make sure to import Python version? -from cbor2 import loads, dumps +from cbor2 import loads from twisted.web.http_headers import Headers diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 373d31e2e..3baa336fa 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -20,7 +20,7 @@ from klein import Klein from twisted.web import http # TODO Make sure to use pure Python versions? -from cbor2 import loads, dumps +from cbor2 import dumps from .server import StorageServer from .http_client import swissnum_auth_header From e5b5b50602268314e035c89e56b740c745b85c84 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 16 Nov 2021 11:28:19 -0500 Subject: [PATCH 0409/2309] Duplicate package. --- nix/tahoe-lafs.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nix/tahoe-lafs.nix b/nix/tahoe-lafs.nix index a6a8a69ec..8092dfaa7 100644 --- a/nix/tahoe-lafs.nix +++ b/nix/tahoe-lafs.nix @@ -95,7 +95,7 @@ EOF propagatedBuildInputs = with python.pkgs; [ twisted foolscap zfec appdirs setuptoolsTrial pyasn1 zope_interface - service-identity pyyaml magic-wormhole treq + service-identity pyyaml magic-wormhole eliot autobahn cryptography netifaces setuptools future pyutil distro configparser collections-extended ]; From a1424e90e18ae1dfbed245277120fdf3f0aaedc8 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 16 Nov 2021 11:34:44 -0500 Subject: [PATCH 0410/2309] Another duplicate. --- nix/tahoe-lafs.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nix/tahoe-lafs.nix b/nix/tahoe-lafs.nix index 8092dfaa7..df12f21d4 100644 --- a/nix/tahoe-lafs.nix +++ b/nix/tahoe-lafs.nix @@ -4,7 +4,7 @@ , setuptools, setuptoolsTrial, pyasn1, zope_interface , service-identity, pyyaml, magic-wormhole, treq, appdirs , beautifulsoup4, eliot, autobahn, cryptography, netifaces -, html5lib, pyutil, distro, configparser, klein, treq, cbor2 +, html5lib, pyutil, distro, configparser, klein, cbor2 }: python.pkgs.buildPythonPackage rec { # Most of the time this is not exactly the release version (eg 1.16.0). From f549488bb508a8377d968d16addb07a98559d8fd Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 16 Nov 2021 11:47:09 -0500 Subject: [PATCH 0411/2309] Don't use a deprecated API. --- src/allmydata/storage/http_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index d5ca6caec..e1743343d 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -42,7 +42,7 @@ def _decode_cbor(response): def swissnum_auth_header(swissnum): # type: (bytes) -> bytes """Return value for ``Authentication`` header.""" - return b"Tahoe-LAFS " + base64.encodestring(swissnum).strip() + return b"Tahoe-LAFS " + base64.b64encode(swissnum).strip() class StorageClient(object): From 3b69df36b0604a0981c92a4d4c0da0611bc04535 Mon Sep 17 00:00:00 2001 From: meejah Date: Sat, 23 Oct 2021 15:38:51 -0600 Subject: [PATCH 0412/2309] crawler: pickle -> json --- src/allmydata/storage/crawler.py | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/src/allmydata/storage/crawler.py b/src/allmydata/storage/crawler.py index bd4f4f432..d7dee78dc 100644 --- a/src/allmydata/storage/crawler.py +++ b/src/allmydata/storage/crawler.py @@ -11,15 +11,12 @@ from __future__ import print_function from future.utils import PY2, PY3 if PY2: - # We don't import bytes, object, dict, and list just in case they're used, - # so as not to create brittle pickles with random magic objects. - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min -import os, time, struct -try: - import cPickle as pickle -except ImportError: - import pickle # type: ignore +import os +import time +import json +import struct from twisted.internet import reactor from twisted.application import service from allmydata.storage.common import si_b2a @@ -214,7 +211,7 @@ class ShareCrawler(service.MultiService): # None if we are sleeping between cycles try: with open(self.statefile, "rb") as f: - state = pickle.load(f) + state = json.load(f) except Exception: state = {"version": 1, "last-cycle-finished": None, @@ -252,9 +249,7 @@ class ShareCrawler(service.MultiService): self.state["last-complete-prefix"] = last_complete_prefix tmpfile = self.statefile + ".tmp" with open(tmpfile, "wb") as f: - # Newer protocols won't work in Python 2; when it is dropped, - # protocol v4 can be used (added in Python 3.4). - pickle.dump(self.state, f, protocol=2) + json.dump(self.state, f) fileutil.move_into_place(tmpfile, self.statefile) def startService(self): From 758dcea2d4a73d472050d5f21bc84217a71802b8 Mon Sep 17 00:00:00 2001 From: meejah Date: Sat, 23 Oct 2021 16:19:27 -0600 Subject: [PATCH 0413/2309] news --- newsfragments/3825.security | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 newsfragments/3825.security diff --git a/newsfragments/3825.security b/newsfragments/3825.security new file mode 100644 index 000000000..b16418d2b --- /dev/null +++ b/newsfragments/3825.security @@ -0,0 +1,5 @@ +The lease-checker now uses JSON instead of pickle to serialize its state. + +Once you have run this version the lease state files will be stored in JSON +and an older version of the software won't load them (it simply won't notice +them so it will appear to have never run). From f7b385f9544f48ebfc7da69b314d3d1dda30ee2c Mon Sep 17 00:00:00 2001 From: meejah Date: Sun, 24 Oct 2021 22:27:59 -0600 Subject: [PATCH 0414/2309] play nice with subclasses --- src/allmydata/storage/crawler.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/allmydata/storage/crawler.py b/src/allmydata/storage/crawler.py index d7dee78dc..b931f1ab5 100644 --- a/src/allmydata/storage/crawler.py +++ b/src/allmydata/storage/crawler.py @@ -248,8 +248,12 @@ class ShareCrawler(service.MultiService): last_complete_prefix = self.prefixes[lcpi] self.state["last-complete-prefix"] = last_complete_prefix tmpfile = self.statefile + ".tmp" + + # Note: we use self.get_state() here because e.g + # LeaseCheckingCrawler stores non-JSON-able state in + # self.state() but converts it in self.get_state() with open(tmpfile, "wb") as f: - json.dump(self.state, f) + json.dump(self.get_state(), f) fileutil.move_into_place(tmpfile, self.statefile) def startService(self): From bb70e00065ab42f0e9f8faabff50d587532f49f7 Mon Sep 17 00:00:00 2001 From: meejah Date: Sun, 24 Oct 2021 23:47:24 -0600 Subject: [PATCH 0415/2309] Make internal state JSON-able for lease-crawler --- src/allmydata/storage/expirer.py | 53 +++++++++++++------------- src/allmydata/test/test_storage_web.py | 32 ++++++++-------- src/allmydata/web/storage.py | 11 +++--- 3 files changed, 49 insertions(+), 47 deletions(-) diff --git a/src/allmydata/storage/expirer.py b/src/allmydata/storage/expirer.py index 7c6cd8218..4513dadb2 100644 --- a/src/allmydata/storage/expirer.py +++ b/src/allmydata/storage/expirer.py @@ -5,10 +5,11 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: - # We omit anything that might end up in pickle, just in case. - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, range, str, max, min # noqa: F401 - -import time, os, pickle, struct + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 +import json +import time +import os +import struct from allmydata.storage.crawler import ShareCrawler from allmydata.storage.shares import get_share_file from allmydata.storage.common import UnknownMutableContainerVersionError, \ @@ -95,9 +96,7 @@ class LeaseCheckingCrawler(ShareCrawler): if not os.path.exists(self.historyfile): history = {} # cyclenum -> dict with open(self.historyfile, "wb") as f: - # Newer protocols won't work in Python 2; when it is dropped, - # protocol v4 can be used (added in Python 3.4). - pickle.dump(history, f, protocol=2) + json.dump(history, f) def create_empty_cycle_dict(self): recovered = self.create_empty_recovered_dict() @@ -142,7 +141,7 @@ class LeaseCheckingCrawler(ShareCrawler): struct.error): twlog.msg("lease-checker error processing %s" % sharefile) twlog.err() - which = (storage_index_b32, shnum) + which = [storage_index_b32, shnum] self.state["cycle-to-date"]["corrupt-shares"].append(which) wks = (1, 1, 1, "unknown") would_keep_shares.append(wks) @@ -212,7 +211,7 @@ class LeaseCheckingCrawler(ShareCrawler): num_valid_leases_configured += 1 so_far = self.state["cycle-to-date"] - self.increment(so_far["leases-per-share-histogram"], num_leases, 1) + self.increment(so_far["leases-per-share-histogram"], str(num_leases), 1) self.increment_space("examined", s, sharetype) would_keep_share = [1, 1, 1, sharetype] @@ -291,12 +290,14 @@ class LeaseCheckingCrawler(ShareCrawler): start = self.state["current-cycle-start-time"] now = time.time() - h["cycle-start-finish-times"] = (start, now) + h["cycle-start-finish-times"] = [start, now] h["expiration-enabled"] = self.expiration_enabled - h["configured-expiration-mode"] = (self.mode, - self.override_lease_duration, - self.cutoff_date, - self.sharetypes_to_expire) + h["configured-expiration-mode"] = [ + self.mode, + self.override_lease_duration, + self.cutoff_date, + self.sharetypes_to_expire, + ] s = self.state["cycle-to-date"] @@ -315,15 +316,13 @@ class LeaseCheckingCrawler(ShareCrawler): h["space-recovered"] = s["space-recovered"].copy() with open(self.historyfile, "rb") as f: - history = pickle.load(f) - history[cycle] = h + history = json.load(f) + history[str(cycle)] = h while len(history) > 10: - oldcycles = sorted(history.keys()) - del history[oldcycles[0]] + oldcycles = sorted(int(k) for k in history.keys()) + del history[str(oldcycles[0])] with open(self.historyfile, "wb") as f: - # Newer protocols won't work in Python 2; when it is dropped, - # protocol v4 can be used (added in Python 3.4). - pickle.dump(history, f, protocol=2) + json.dump(history, f) def get_state(self): """In addition to the crawler state described in @@ -393,7 +392,7 @@ class LeaseCheckingCrawler(ShareCrawler): state = ShareCrawler.get_state(self) # does a shallow copy with open(self.historyfile, "rb") as f: - history = pickle.load(f) + history = json.load(f) state["history"] = history if not progress["cycle-in-progress"]: @@ -406,10 +405,12 @@ class LeaseCheckingCrawler(ShareCrawler): lah = so_far["lease-age-histogram"] so_far["lease-age-histogram"] = self.convert_lease_age_histogram(lah) so_far["expiration-enabled"] = self.expiration_enabled - so_far["configured-expiration-mode"] = (self.mode, - self.override_lease_duration, - self.cutoff_date, - self.sharetypes_to_expire) + so_far["configured-expiration-mode"] = [ + self.mode, + self.override_lease_duration, + self.cutoff_date, + self.sharetypes_to_expire, + ] so_far_sr = so_far["space-recovered"] remaining_sr = {} diff --git a/src/allmydata/test/test_storage_web.py b/src/allmydata/test/test_storage_web.py index 38e380223..b9fa548d3 100644 --- a/src/allmydata/test/test_storage_web.py +++ b/src/allmydata/test/test_storage_web.py @@ -376,7 +376,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin): self.failUnlessEqual(type(lah), list) self.failUnlessEqual(len(lah), 1) self.failUnlessEqual(lah, [ (0.0, DAY, 1) ] ) - self.failUnlessEqual(so_far["leases-per-share-histogram"], {1: 1}) + self.failUnlessEqual(so_far["leases-per-share-histogram"], {"1": 1}) self.failUnlessEqual(so_far["corrupt-shares"], []) sr1 = so_far["space-recovered"] self.failUnlessEqual(sr1["examined-buckets"], 1) @@ -427,9 +427,9 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin): self.failIf("cycle-to-date" in s) self.failIf("estimated-remaining-cycle" in s) self.failIf("estimated-current-cycle" in s) - last = s["history"][0] + last = s["history"]["0"] self.failUnlessIn("cycle-start-finish-times", last) - self.failUnlessEqual(type(last["cycle-start-finish-times"]), tuple) + self.failUnlessEqual(type(last["cycle-start-finish-times"]), list) self.failUnlessEqual(last["expiration-enabled"], False) self.failUnlessIn("configured-expiration-mode", last) @@ -437,9 +437,9 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin): lah = last["lease-age-histogram"] self.failUnlessEqual(type(lah), list) self.failUnlessEqual(len(lah), 1) - self.failUnlessEqual(lah, [ (0.0, DAY, 6) ] ) + self.failUnlessEqual(lah, [ [0.0, DAY, 6] ] ) - self.failUnlessEqual(last["leases-per-share-histogram"], {1: 2, 2: 2}) + self.failUnlessEqual(last["leases-per-share-histogram"], {"1": 2, "2": 2}) self.failUnlessEqual(last["corrupt-shares"], []) rec = last["space-recovered"] @@ -587,12 +587,12 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin): self.failUnlessEqual(count_leases(mutable_si_3), 1) s = lc.get_state() - last = s["history"][0] + last = s["history"]["0"] self.failUnlessEqual(last["expiration-enabled"], True) self.failUnlessEqual(last["configured-expiration-mode"], - ("age", 2000, None, ("mutable", "immutable"))) - self.failUnlessEqual(last["leases-per-share-histogram"], {1: 2, 2: 2}) + ["age", 2000, None, ["mutable", "immutable"]]) + self.failUnlessEqual(last["leases-per-share-histogram"], {"1": 2, "2": 2}) rec = last["space-recovered"] self.failUnlessEqual(rec["examined-buckets"], 4) @@ -731,14 +731,14 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin): self.failUnlessEqual(count_leases(mutable_si_3), 1) s = lc.get_state() - last = s["history"][0] + last = s["history"]["0"] self.failUnlessEqual(last["expiration-enabled"], True) self.failUnlessEqual(last["configured-expiration-mode"], - ("cutoff-date", None, then, - ("mutable", "immutable"))) + ["cutoff-date", None, then, + ["mutable", "immutable"]]) self.failUnlessEqual(last["leases-per-share-histogram"], - {1: 2, 2: 2}) + {"1": 2, "2": 2}) rec = last["space-recovered"] self.failUnlessEqual(rec["examined-buckets"], 4) @@ -924,8 +924,8 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin): s = lc.get_state() h = s["history"] self.failUnlessEqual(len(h), 10) - self.failUnlessEqual(max(h.keys()), 15) - self.failUnlessEqual(min(h.keys()), 6) + self.failUnlessEqual(max(int(k) for k in h.keys()), 15) + self.failUnlessEqual(min(int(k) for k in h.keys()), 6) d.addCallback(_check) return d @@ -1014,7 +1014,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin): def _check(ignored): s = lc.get_state() - last = s["history"][0] + last = s["history"]["0"] rec = last["space-recovered"] self.failUnlessEqual(rec["configured-buckets"], 4) self.failUnlessEqual(rec["configured-shares"], 4) @@ -1110,7 +1110,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin): def _after_first_cycle(ignored): s = lc.get_state() - last = s["history"][0] + last = s["history"]["0"] rec = last["space-recovered"] self.failUnlessEqual(rec["examined-buckets"], 5) self.failUnlessEqual(rec["examined-shares"], 3) diff --git a/src/allmydata/web/storage.py b/src/allmydata/web/storage.py index f2f021a15..e568d5ed5 100644 --- a/src/allmydata/web/storage.py +++ b/src/allmydata/web/storage.py @@ -256,8 +256,8 @@ class StorageStatusElement(Element): if so_far["corrupt-shares"]: add("Corrupt shares:", - T.ul( (T.li( ["SI %s shnum %d" % corrupt_share - for corrupt_share in so_far["corrupt-shares"] ] + T.ul( (T.li( ["SI %s shnum %d" % (si, shnum) + for si, shnum in so_far["corrupt-shares"] ] )))) return tag("Current cycle:", p) @@ -267,7 +267,8 @@ class StorageStatusElement(Element): h = lc.get_state()["history"] if not h: return "" - last = h[max(h.keys())] + biggest = str(max(int(k) for k in h.keys())) + last = h[biggest] start, end = last["cycle-start-finish-times"] tag("Last complete cycle (which took %s and finished %s ago)" @@ -290,8 +291,8 @@ class StorageStatusElement(Element): if last["corrupt-shares"]: add("Corrupt shares:", - T.ul( (T.li( ["SI %s shnum %d" % corrupt_share - for corrupt_share in last["corrupt-shares"] ] + T.ul( (T.li( ["SI %s shnum %d" % (si, shnum) + for si, shnum in last["corrupt-shares"] ] )))) return tag(p) From fa6950f08dc054ec770af1c402e8eb5d3c581b3e Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 25 Oct 2021 12:18:28 -0600 Subject: [PATCH 0416/2309] an old pickle-format lease-checker state file --- src/allmydata/test/data/lease_checker.state | 545 ++++++++++++++++++++ 1 file changed, 545 insertions(+) create mode 100644 src/allmydata/test/data/lease_checker.state diff --git a/src/allmydata/test/data/lease_checker.state b/src/allmydata/test/data/lease_checker.state new file mode 100644 index 000000000..b32554434 --- /dev/null +++ b/src/allmydata/test/data/lease_checker.state @@ -0,0 +1,545 @@ +(dp1 +S'last-complete-prefix' +p2 +NsS'version' +p3 +I1 +sS'current-cycle-start-time' +p4 +F1635003106.611748 +sS'last-cycle-finished' +p5 +I312 +sS'cycle-to-date' +p6 +(dp7 +Vleases-per-share-histogram +p8 +(dp9 +I1 +I36793 +sI2 +I1 +ssVspace-recovered +p10 +(dp11 +Vexamined-buckets-immutable +p12 +I17183 +sVconfigured-buckets-mutable +p13 +I0 +sVexamined-shares-mutable +p14 +I1796 +sVoriginal-shares-mutable +p15 +I1563 +sVconfigured-buckets-immutable +p16 +I0 +sVoriginal-shares-immutable +p17 +I27926 +sVoriginal-diskbytes-immutable +p18 +I431149056 +sVexamined-shares-immutable +p19 +I34998 +sVoriginal-buckets +p20 +I14661 +sVactual-shares-immutable +p21 +I0 +sVconfigured-shares +p22 +I0 +sVoriginal-buckets-immutable +p23 +I13761 +sVactual-diskbytes +p24 +I4096 +sVactual-shares-mutable +p25 +I0 +sVconfigured-buckets +p26 +I1 +sVexamined-buckets-unknown +p27 +I14 +sVactual-sharebytes +p28 +I0 +sVoriginal-shares +p29 +I29489 +sVoriginal-sharebytes +p30 +I312664812 +sVexamined-sharebytes-immutable +p31 +I383801602 +sVactual-shares +p32 +I0 +sVactual-sharebytes-immutable +p33 +I0 +sVoriginal-diskbytes +p34 +I441643008 +sVconfigured-diskbytes-mutable +p35 +I0 +sVconfigured-sharebytes-immutable +p36 +I0 +sVconfigured-shares-mutable +p37 +I0 +sVactual-diskbytes-immutable +p38 +I0 +sVconfigured-diskbytes-immutable +p39 +I0 +sVoriginal-diskbytes-mutable +p40 +I10489856 +sVactual-sharebytes-mutable +p41 +I0 +sVconfigured-sharebytes +p42 +I0 +sVexamined-shares +p43 +I36794 +sVactual-diskbytes-mutable +p44 +I0 +sVactual-buckets +p45 +I1 +sVoriginal-buckets-mutable +p46 +I899 +sVconfigured-sharebytes-mutable +p47 +I0 +sVexamined-sharebytes +p48 +I390369660 +sVoriginal-sharebytes-immutable +p49 +I308125753 +sVoriginal-sharebytes-mutable +p50 +I4539059 +sVactual-buckets-mutable +p51 +I0 +sVexamined-diskbytes-mutable +p52 +I9154560 +sVexamined-buckets-mutable +p53 +I1043 +sVconfigured-shares-immutable +p54 +I0 +sVexamined-diskbytes +p55 +I476598272 +sVactual-buckets-immutable +p56 +I0 +sVexamined-sharebytes-mutable +p57 +I6568058 +sVexamined-buckets +p58 +I18241 +sVconfigured-diskbytes +p59 +I4096 +sVexamined-diskbytes-immutable +p60 +I467443712 +ssVcorrupt-shares +p61 +(lp62 +(V2dn6xnlnsqwtnapwxfdivpm3s4 +p63 +I4 +tp64 +a(g63 +I1 +tp65 +a(V2rrzthwsrrxolevmwdvbdy3rqi +p66 +I4 +tp67 +a(g66 +I1 +tp68 +a(V2skfngcto6h7eqmn4uo7ntk3ne +p69 +I4 +tp70 +a(g69 +I1 +tp71 +a(V32d5swqpqx2mwix7xmqzvhdwje +p72 +I4 +tp73 +a(g72 +I1 +tp74 +a(V5mmayp66yflmpon3o6unsnbaca +p75 +I4 +tp76 +a(g75 +I1 +tp77 +a(V6ixhpvbtre7fnrl6pehlrlflc4 +p78 +I4 +tp79 +a(g78 +I1 +tp80 +a(Vewzhvswjsz4vp2bqkb6mi3bz2u +p81 +I4 +tp82 +a(g81 +I1 +tp83 +a(Vfu7pazf6ogavkqj6z4q5qqex3u +p84 +I4 +tp85 +a(g84 +I1 +tp86 +a(Vhbyjtqvpcimwxiyqbcbbdn2i4a +p87 +I4 +tp88 +a(g87 +I1 +tp89 +a(Vpmcjbdkbjdl26k3e6yja77femq +p90 +I4 +tp91 +a(g90 +I1 +tp92 +a(Vr6swof4v2uttbiiqwj5pi32cm4 +p93 +I4 +tp94 +a(g93 +I1 +tp95 +a(Vt45v5akoktf53evc2fi6gwnv6y +p96 +I4 +tp97 +a(g96 +I1 +tp98 +a(Vy6zb4faar3rdvn3e6pfg4wlotm +p99 +I4 +tp100 +a(g99 +I1 +tp101 +a(Vz3yghutvqoqbchjao4lndnrh3a +p102 +I4 +tp103 +a(g102 +I1 +tp104 +asVlease-age-histogram +p105 +(dp106 +(I45619200 +I45705600 +tp107 +I4 +s(I12441600 +I12528000 +tp108 +I78 +s(I11923200 +I12009600 +tp109 +I89 +s(I33436800 +I33523200 +tp110 +I7 +s(I37411200 +I37497600 +tp111 +I4 +s(I38361600 +I38448000 +tp112 +I5 +s(I4665600 +I4752000 +tp113 +I256 +s(I11491200 +I11577600 +tp114 +I20 +s(I10713600 +I10800000 +tp115 +I183 +s(I42076800 +I42163200 +tp116 +I4 +s(I47865600 +I47952000 +tp117 +I7 +s(I3110400 +I3196800 +tp118 +I328 +s(I5788800 +I5875200 +tp119 +I954 +s(I9331200 +I9417600 +tp120 +I12 +s(I7430400 +I7516800 +tp121 +I7228 +s(I1555200 +I1641600 +tp122 +I492 +s(I37929600 +I38016000 +tp123 +I3 +s(I38880000 +I38966400 +tp124 +I3 +s(I12528000 +I12614400 +tp125 +I193 +s(I10454400 +I10540800 +tp126 +I1239 +s(I11750400 +I11836800 +tp127 +I7 +s(I950400 +I1036800 +tp128 +I4435 +s(I44409600 +I44496000 +tp129 +I13 +s(I12787200 +I12873600 +tp130 +I218 +s(I10368000 +I10454400 +tp131 +I117 +s(I3283200 +I3369600 +tp132 +I86 +s(I7516800 +I7603200 +tp133 +I993 +s(I42336000 +I42422400 +tp134 +I33 +s(I46310400 +I46396800 +tp135 +I1 +s(I39052800 +I39139200 +tp136 +I51 +s(I7603200 +I7689600 +tp137 +I2004 +s(I10540800 +I10627200 +tp138 +I16 +s(I36374400 +I36460800 +tp139 +I3 +s(I3369600 +I3456000 +tp140 +I79 +s(I12700800 +I12787200 +tp141 +I25 +s(I4838400 +I4924800 +tp142 +I386 +s(I10972800 +I11059200 +tp143 +I122 +s(I8812800 +I8899200 +tp144 +I57 +s(I38966400 +I39052800 +tp145 +I61 +s(I3196800 +I3283200 +tp146 +I628 +s(I9244800 +I9331200 +tp147 +I73 +s(I30499200 +I30585600 +tp148 +I5 +s(I12009600 +I12096000 +tp149 +I329 +s(I12960000 +I13046400 +tp150 +I8 +s(I12614400 +I12700800 +tp151 +I210 +s(I3801600 +I3888000 +tp152 +I32 +s(I10627200 +I10713600 +tp153 +I43 +s(I44928000 +I45014400 +tp154 +I2 +s(I8208000 +I8294400 +tp155 +I38 +s(I8640000 +I8726400 +tp156 +I32 +s(I7344000 +I7430400 +tp157 +I12689 +s(I49075200 +I49161600 +tp158 +I19 +s(I2764800 +I2851200 +tp159 +I76 +s(I2592000 +I2678400 +tp160 +I40 +s(I2073600 +I2160000 +tp161 +I388 +s(I37497600 +I37584000 +tp162 +I11 +s(I1641600 +I1728000 +tp163 +I78 +s(I12873600 +I12960000 +tp164 +I5 +s(I1814400 +I1900800 +tp165 +I1860 +s(I40176000 +I40262400 +tp166 +I1 +s(I3715200 +I3801600 +tp167 +I104 +s(I2332800 +I2419200 +tp168 +I12 +s(I2678400 +I2764800 +tp169 +I278 +s(I12268800 +I12355200 +tp170 +I2 +s(I28771200 +I28857600 +tp171 +I6 +s(I41990400 +I42076800 +tp172 +I10 +sssS'last-complete-bucket' +p173 +NsS'current-cycle' +p174 +Ns. \ No newline at end of file From f81e4e2d25e4d12362f05824075915d52b3878cc Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 25 Oct 2021 13:15:38 -0600 Subject: [PATCH 0417/2309] refactor to use serializers / pickle->json upgraders --- src/allmydata/storage/crawler.py | 143 +++++++++++++++++++++++-- src/allmydata/storage/expirer.py | 82 +++++++++++--- src/allmydata/storage/server.py | 1 + src/allmydata/test/test_storage_web.py | 10 +- 4 files changed, 212 insertions(+), 24 deletions(-) diff --git a/src/allmydata/storage/crawler.py b/src/allmydata/storage/crawler.py index b931f1ab5..48b03ec8b 100644 --- a/src/allmydata/storage/crawler.py +++ b/src/allmydata/storage/crawler.py @@ -19,12 +19,145 @@ import json import struct from twisted.internet import reactor from twisted.application import service +from twisted.python.filepath import FilePath from allmydata.storage.common import si_b2a from allmydata.util import fileutil class TimeSliceExceeded(Exception): pass + +def _convert_pickle_state_to_json(state): + """ + :param dict state: the pickled state + + :return dict: the state in the JSON form + """ + # ["cycle-to-date"]["corrupt-shares"] from 2-tuple to list + # ["leases-per-share-histogram"] gets str keys instead of int + # ["cycle-start-finish-times"] from 2-tuple to list + # ["configured-expiration-mode"] from 4-tuple to list + # ["history"] keys are strings + if state["version"] != 1: + raise ValueError( + "Unknown version {version} in pickle state".format(**state) + ) + + def convert_lpsh(value): + return { + str(k): v + for k, v in value.items() + } + + def convert_cem(value): + # original is a 4-tuple, with the last element being a 2-tuple + # .. convert both to lists + return [ + value[0], + value[1], + value[2], + list(value[3]), + ] + + def convert_history(value): + print("convert history") + print(value) + return { + str(k): v + for k, v in value + } + + converters = { + "cycle-to-date": list, + "leases-per-share-histogram": convert_lpsh, + "cycle-starte-finish-times": list, + "configured-expiration-mode": convert_cem, + "history": convert_history, + } + + def convert_value(key, value): + converter = converters.get(key, None) + if converter is None: + return value + return converter(value) + + new_state = { + k: convert_value(k, v) + for k, v in state.items() + } + return new_state + + +def _maybe_upgrade_pickle_to_json(state_path, convert_pickle): + """ + :param FilePath state_path: the filepath to ensure is json + + :param Callable[dict] convert_pickle: function to change + pickle-style state into JSON-style state + + :returns unicode: the local path where the state is stored + + If this state path is JSON, simply return it. + + If this state is pickle, convert to the JSON format and return the + JSON path. + """ + if state_path.path.endswith(".json"): + return state_path.path + + json_state_path = state_path.siblingExtension(".json") + + # if there's no file there at all, we're done because there's + # nothing to upgrade + if not state_path.exists(): + return json_state_path.path + + # upgrade the pickle data to JSON + import pickle + with state_path.open("r") as f: + state = pickle.load(f) + state = convert_pickle(state) + json_state_path = state_path.siblingExtension(".json") + with json_state_path.open("w") as f: + json.dump(state, f) + # we've written the JSON, delete the pickle + state_path.remove() + return json_state_path.path + + +class _LeaseStateSerializer(object): + """ + Read and write state for LeaseCheckingCrawler. This understands + how to read the legacy pickle format files and upgrade them to the + new JSON format (which will occur automatically). + """ + + def __init__(self, state_path): + self._path = FilePath( + _maybe_upgrade_pickle_to_json( + FilePath(state_path), + _convert_pickle_state_to_json, + ) + ) + # XXX want this to .. load and save the state + # - if the state is pickle-only: + # - load it and convert to json format + # - save json + # - delete pickle + # - if the state is json, load it + + def load(self): + with self._path.open("r") as f: + return json.load(f) + + def save(self, data): + tmpfile = self._path.siblingExtension(".tmp") + with tmpfile.open("wb") as f: + json.dump(data, f) + fileutil.move_into_place(tmpfile.path, self._path.path) + return None + + class ShareCrawler(service.MultiService): """A ShareCrawler subclass is attached to a StorageServer, and periodically walks all of its shares, processing each one in some @@ -87,7 +220,7 @@ class ShareCrawler(service.MultiService): self.allowed_cpu_percentage = allowed_cpu_percentage self.server = server self.sharedir = server.sharedir - self.statefile = statefile + self._state_serializer = _LeaseStateSerializer(statefile) self.prefixes = [si_b2a(struct.pack(">H", i << (16-10)))[:2] for i in range(2**10)] if PY3: @@ -210,8 +343,7 @@ class ShareCrawler(service.MultiService): # of the last bucket to be processed, or # None if we are sleeping between cycles try: - with open(self.statefile, "rb") as f: - state = json.load(f) + state = self._state_serializer.load() except Exception: state = {"version": 1, "last-cycle-finished": None, @@ -247,14 +379,11 @@ class ShareCrawler(service.MultiService): else: last_complete_prefix = self.prefixes[lcpi] self.state["last-complete-prefix"] = last_complete_prefix - tmpfile = self.statefile + ".tmp" # Note: we use self.get_state() here because e.g # LeaseCheckingCrawler stores non-JSON-able state in # self.state() but converts it in self.get_state() - with open(tmpfile, "wb") as f: - json.dump(self.get_state(), f) - fileutil.move_into_place(tmpfile, self.statefile) + self._state_serializer.save(self.get_state()) def startService(self): # arrange things to look like we were just sleeping, so diff --git a/src/allmydata/storage/expirer.py b/src/allmydata/storage/expirer.py index 4513dadb2..d2f48004a 100644 --- a/src/allmydata/storage/expirer.py +++ b/src/allmydata/storage/expirer.py @@ -10,11 +10,72 @@ import json import time import os import struct -from allmydata.storage.crawler import ShareCrawler +from allmydata.storage.crawler import ( + ShareCrawler, + _maybe_upgrade_pickle_to_json, +) from allmydata.storage.shares import get_share_file from allmydata.storage.common import UnknownMutableContainerVersionError, \ UnknownImmutableContainerVersionError from twisted.python import log as twlog +from twisted.python.filepath import FilePath + + +def _convert_pickle_state_to_json(state): + """ + :param dict state: the pickled state + + :return dict: the state in the JSON form + """ + print("CONVERT", state) + for k, v in state.items(): + print(k, v) + if state["version"] != 1: + raise ValueError( + "Unknown version {version} in pickle state".format(**state) + ) + + return state + + +class _HistorySerializer(object): + """ + Serialize the 'history' file of the lease-crawler state. This is + "storage/history.state" for the pickle or + "storage/history.state.json" for the new JSON format. + """ + + def __init__(self, history_path): + self._path = FilePath( + _maybe_upgrade_pickle_to_json( + FilePath(history_path), + _convert_pickle_state_to_json, + ) + ) + if not self._path.exists(): + with self._path.open("wb") as f: + json.dump({}, f) + + def read(self): + """ + Deserialize the existing data. + + :return dict: the existing history state + """ + assert self._path is not None, "Not initialized" + with self._path.open("rb") as f: + history = json.load(f) + return history + + def write(self, new_history): + """ + Serialize the existing data as JSON. + """ + assert self._path is not None, "Not initialized" + with self._path.open("wb") as f: + json.dump(new_history, f) + return None + class LeaseCheckingCrawler(ShareCrawler): """I examine the leases on all shares, determining which are still valid @@ -64,7 +125,8 @@ class LeaseCheckingCrawler(ShareCrawler): override_lease_duration, # used if expiration_mode=="age" cutoff_date, # used if expiration_mode=="cutoff-date" sharetypes): - self.historyfile = historyfile + self._history_serializer = _HistorySerializer(historyfile) + ##self.historyfile = historyfile self.expiration_enabled = expiration_enabled self.mode = mode self.override_lease_duration = None @@ -92,12 +154,6 @@ class LeaseCheckingCrawler(ShareCrawler): for k in so_far: self.state["cycle-to-date"].setdefault(k, so_far[k]) - # initialize history - if not os.path.exists(self.historyfile): - history = {} # cyclenum -> dict - with open(self.historyfile, "wb") as f: - json.dump(history, f) - def create_empty_cycle_dict(self): recovered = self.create_empty_recovered_dict() so_far = {"corrupt-shares": [], @@ -315,14 +371,12 @@ class LeaseCheckingCrawler(ShareCrawler): # copy() needs to become a deepcopy h["space-recovered"] = s["space-recovered"].copy() - with open(self.historyfile, "rb") as f: - history = json.load(f) + history = self._history_serializer.read() history[str(cycle)] = h while len(history) > 10: oldcycles = sorted(int(k) for k in history.keys()) del history[str(oldcycles[0])] - with open(self.historyfile, "wb") as f: - json.dump(history, f) + self._history_serializer.write(history) def get_state(self): """In addition to the crawler state described in @@ -391,9 +445,7 @@ class LeaseCheckingCrawler(ShareCrawler): progress = self.get_progress() state = ShareCrawler.get_state(self) # does a shallow copy - with open(self.historyfile, "rb") as f: - history = json.load(f) - state["history"] = history + state["history"] = self._history_serializer.read() if not progress["cycle-in-progress"]: del state["cycle-to-date"] diff --git a/src/allmydata/storage/server.py b/src/allmydata/storage/server.py index 49cb7fa82..9211535b7 100644 --- a/src/allmydata/storage/server.py +++ b/src/allmydata/storage/server.py @@ -57,6 +57,7 @@ DEFAULT_RENEWAL_TIME = 31 * 24 * 60 * 60 @implementer(RIStorageServer, IStatsProducer) class StorageServer(service.MultiService, Referenceable): name = 'storage' + # only the tests change this to anything else LeaseCheckerClass = LeaseCheckingCrawler def __init__(self, storedir, nodeid, reserved_space=0, diff --git a/src/allmydata/test/test_storage_web.py b/src/allmydata/test/test_storage_web.py index b9fa548d3..d91242449 100644 --- a/src/allmydata/test/test_storage_web.py +++ b/src/allmydata/test/test_storage_web.py @@ -25,14 +25,20 @@ from twisted.trial import unittest from twisted.internet import defer from twisted.application import service from twisted.web.template import flattenString +from twisted.python.filepath import FilePath from foolscap.api import fireEventually from allmydata.util import fileutil, hashutil, base32, pollmixin from allmydata.storage.common import storage_index_to_dir, \ UnknownMutableContainerVersionError, UnknownImmutableContainerVersionError from allmydata.storage.server import StorageServer -from allmydata.storage.crawler import BucketCountingCrawler -from allmydata.storage.expirer import LeaseCheckingCrawler +from allmydata.storage.crawler import ( + BucketCountingCrawler, + _LeaseStateSerializer, +) +from allmydata.storage.expirer import ( + LeaseCheckingCrawler, +) from allmydata.web.storage import ( StorageStatus, StorageStatusElement, From bf5e682d71e086f351126129c2e586a80442c2bb Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 25 Oct 2021 13:17:46 -0600 Subject: [PATCH 0418/2309] test upgrade of main state works --- src/allmydata/test/test_storage_web.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/allmydata/test/test_storage_web.py b/src/allmydata/test/test_storage_web.py index d91242449..0b287d667 100644 --- a/src/allmydata/test/test_storage_web.py +++ b/src/allmydata/test/test_storage_web.py @@ -1145,6 +1145,22 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin): d.addBoth(_cleanup) return d + def test_deserialize_pickle(self): + """ + The crawler can read existing state from the old pickle format + """ + original_pickle = FilePath(__file__).parent().child("data").child("lease_checker.state") + test_pickle = FilePath("lease_checker.state") + with test_pickle.open("w") as local, original_pickle.open("r") as remote: + local.write(remote.read()) + + serial = _LeaseStateSerializer(test_pickle.path) + + # the (existing) state file should have been upgraded to JSON + self.assertNot(test_pickle.exists()) + self.assertTrue(test_pickle.siblingExtension(".json").exists()) + + class WebStatus(unittest.TestCase, pollmixin.PollMixin): From 89c2aacadca5cc7ba13f4afda2c4d3817ea9b5c0 Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 25 Oct 2021 15:52:01 -0600 Subject: [PATCH 0419/2309] working test of 'in the wild' data, working converters --- src/allmydata/storage/crawler.py | 36 +++--- src/allmydata/test/test_storage_web.py | 165 +++++++++++++++++++++++++ 2 files changed, 184 insertions(+), 17 deletions(-) diff --git a/src/allmydata/storage/crawler.py b/src/allmydata/storage/crawler.py index 48b03ec8b..548864e06 100644 --- a/src/allmydata/storage/crawler.py +++ b/src/allmydata/storage/crawler.py @@ -34,7 +34,7 @@ def _convert_pickle_state_to_json(state): :return dict: the state in the JSON form """ # ["cycle-to-date"]["corrupt-shares"] from 2-tuple to list - # ["leases-per-share-histogram"] gets str keys instead of int + # ["cycle-to-date"]["leases-per-share-histogram"] gets str keys instead of int # ["cycle-start-finish-times"] from 2-tuple to list # ["configured-expiration-mode"] from 4-tuple to list # ["history"] keys are strings @@ -43,12 +43,6 @@ def _convert_pickle_state_to_json(state): "Unknown version {version} in pickle state".format(**state) ) - def convert_lpsh(value): - return { - str(k): v - for k, v in value.items() - } - def convert_cem(value): # original is a 4-tuple, with the last element being a 2-tuple # .. convert both to lists @@ -59,20 +53,28 @@ def _convert_pickle_state_to_json(state): list(value[3]), ] - def convert_history(value): - print("convert history") - print(value) + def convert_ctd(value): + ctd_converter = { + "lease-age-histogram": lambda value: { + "{},{}".format(k[0], k[1]): v + for k, v in value.items() + }, + "corrupt-shares": lambda value: [ + list(x) + for x in value + ], + } return { - str(k): v - for k, v in value + k: ctd_converter.get(k, lambda z: z)(v) + for k, v in value.items() } + # we don't convert "history" here because that's in a separate + # file; see expirer.py converters = { - "cycle-to-date": list, - "leases-per-share-histogram": convert_lpsh, + "cycle-to-date": convert_ctd, "cycle-starte-finish-times": list, "configured-expiration-mode": convert_cem, - "history": convert_history, } def convert_value(key, value): @@ -116,10 +118,10 @@ def _maybe_upgrade_pickle_to_json(state_path, convert_pickle): import pickle with state_path.open("r") as f: state = pickle.load(f) - state = convert_pickle(state) + new_state = convert_pickle(state) json_state_path = state_path.siblingExtension(".json") with json_state_path.open("w") as f: - json.dump(state, f) + json.dump(new_state, f) # we've written the JSON, delete the pickle state_path.remove() return json_state_path.path diff --git a/src/allmydata/test/test_storage_web.py b/src/allmydata/test/test_storage_web.py index 0b287d667..5cdf02a25 100644 --- a/src/allmydata/test/test_storage_web.py +++ b/src/allmydata/test/test_storage_web.py @@ -1160,6 +1160,171 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin): self.assertNot(test_pickle.exists()) self.assertTrue(test_pickle.siblingExtension(".json").exists()) + self.assertEqual( + serial.load(), + { + u'last-complete-prefix': None, + u'version': 1, + u'current-cycle-start-time': 1635003106.611748, + u'last-cycle-finished': 312, + u'cycle-to-date': { + u'leases-per-share-histogram': { + u'1': 36793, + u'2': 1, + }, + u'space-recovered': { + u'examined-buckets-immutable': 17183, + u'configured-buckets-mutable': 0, + u'examined-shares-mutable': 1796, + u'original-shares-mutable': 1563, + u'configured-buckets-immutable': 0, + u'original-shares-immutable': 27926, + u'original-diskbytes-immutable': 431149056, + u'examined-shares-immutable': 34998, + u'original-buckets': 14661, + u'actual-shares-immutable': 0, + u'configured-shares': 0, + u'original-buckets-mutable': 899, + u'actual-diskbytes': 4096, + u'actual-shares-mutable': 0, + u'configured-buckets': 1, + u'examined-buckets-unknown': 14, + u'actual-sharebytes': 0, + u'original-shares': 29489, + u'actual-buckets-immutable': 0, + u'original-sharebytes': 312664812, + u'examined-sharebytes-immutable': 383801602, + u'actual-shares': 0, + u'actual-sharebytes-immutable': 0, + u'original-diskbytes': 441643008, + u'configured-diskbytes-mutable': 0, + u'configured-sharebytes-immutable': 0, + u'configured-shares-mutable': 0, + u'actual-diskbytes-immutable': 0, + u'configured-diskbytes-immutable': 0, + u'original-diskbytes-mutable': 10489856, + u'actual-sharebytes-mutable': 0, + u'configured-sharebytes': 0, + u'examined-shares': 36794, + u'actual-diskbytes-mutable': 0, + u'actual-buckets': 1, + u'original-buckets-immutable': 13761, + u'configured-sharebytes-mutable': 0, + u'examined-sharebytes': 390369660, + u'original-sharebytes-immutable': 308125753, + u'original-sharebytes-mutable': 4539059, + u'actual-buckets-mutable': 0, + u'examined-buckets-mutable': 1043, + u'configured-shares-immutable': 0, + u'examined-diskbytes': 476598272, + u'examined-diskbytes-mutable': 9154560, + u'examined-sharebytes-mutable': 6568058, + u'examined-buckets': 18241, + u'configured-diskbytes': 4096, + u'examined-diskbytes-immutable': 467443712}, + u'corrupt-shares': [ + [u'2dn6xnlnsqwtnapwxfdivpm3s4', 4], + [u'2dn6xnlnsqwtnapwxfdivpm3s4', 1], + [u'2rrzthwsrrxolevmwdvbdy3rqi', 4], + [u'2rrzthwsrrxolevmwdvbdy3rqi', 1], + [u'2skfngcto6h7eqmn4uo7ntk3ne', 4], + [u'2skfngcto6h7eqmn4uo7ntk3ne', 1], + [u'32d5swqpqx2mwix7xmqzvhdwje', 4], + [u'32d5swqpqx2mwix7xmqzvhdwje', 1], + [u'5mmayp66yflmpon3o6unsnbaca', 4], + [u'5mmayp66yflmpon3o6unsnbaca', 1], + [u'6ixhpvbtre7fnrl6pehlrlflc4', 4], + [u'6ixhpvbtre7fnrl6pehlrlflc4', 1], + [u'ewzhvswjsz4vp2bqkb6mi3bz2u', 4], + [u'ewzhvswjsz4vp2bqkb6mi3bz2u', 1], + [u'fu7pazf6ogavkqj6z4q5qqex3u', 4], + [u'fu7pazf6ogavkqj6z4q5qqex3u', 1], + [u'hbyjtqvpcimwxiyqbcbbdn2i4a', 4], + [u'hbyjtqvpcimwxiyqbcbbdn2i4a', 1], + [u'pmcjbdkbjdl26k3e6yja77femq', 4], + [u'pmcjbdkbjdl26k3e6yja77femq', 1], + [u'r6swof4v2uttbiiqwj5pi32cm4', 4], + [u'r6swof4v2uttbiiqwj5pi32cm4', 1], + [u't45v5akoktf53evc2fi6gwnv6y', 4], + [u't45v5akoktf53evc2fi6gwnv6y', 1], + [u'y6zb4faar3rdvn3e6pfg4wlotm', 4], + [u'y6zb4faar3rdvn3e6pfg4wlotm', 1], + [u'z3yghutvqoqbchjao4lndnrh3a', 4], + [u'z3yghutvqoqbchjao4lndnrh3a', 1], + ], + u'lease-age-histogram': { + "1641600,1728000": 78, + "12441600,12528000": 78, + "8640000,8726400": 32, + "1814400,1900800": 1860, + "2764800,2851200": 76, + "11491200,11577600": 20, + "10713600,10800000": 183, + "47865600,47952000": 7, + "3110400,3196800": 328, + "10627200,10713600": 43, + "45619200,45705600": 4, + "12873600,12960000": 5, + "7430400,7516800": 7228, + "1555200,1641600": 492, + "38880000,38966400": 3, + "12528000,12614400": 193, + "7344000,7430400": 12689, + "2678400,2764800": 278, + "2332800,2419200": 12, + "9244800,9331200": 73, + "12787200,12873600": 218, + "49075200,49161600": 19, + "10368000,10454400": 117, + "4665600,4752000": 256, + "7516800,7603200": 993, + "42336000,42422400": 33, + "10972800,11059200": 122, + "39052800,39139200": 51, + "12614400,12700800": 210, + "7603200,7689600": 2004, + "10540800,10627200": 16, + "950400,1036800": 4435, + "42076800,42163200": 4, + "8812800,8899200": 57, + "5788800,5875200": 954, + "36374400,36460800": 3, + "9331200,9417600": 12, + "30499200,30585600": 5, + "12700800,12787200": 25, + "2073600,2160000": 388, + "12960000,13046400": 8, + "11923200,12009600": 89, + "3369600,3456000": 79, + "3196800,3283200": 628, + "37497600,37584000": 11, + "33436800,33523200": 7, + "44928000,45014400": 2, + "37929600,38016000": 3, + "38966400,39052800": 61, + "3283200,3369600": 86, + "11750400,11836800": 7, + "3801600,3888000": 32, + "46310400,46396800": 1, + "4838400,4924800": 386, + "8208000,8294400": 38, + "37411200,37497600": 4, + "12009600,12096000": 329, + "10454400,10540800": 1239, + "40176000,40262400": 1, + "3715200,3801600": 104, + "44409600,44496000": 13, + "38361600,38448000": 5, + "12268800,12355200": 2, + "28771200,28857600": 6, + "41990400,42076800": 10, + "2592000,2678400": 40, + }, + }, + 'current-cycle': None, + 'last-complete-bucket': None, + } + ) class WebStatus(unittest.TestCase, pollmixin.PollMixin): From d4fc14f9ada372e94d68667da64643b52de9923c Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 25 Oct 2021 19:42:08 -0600 Subject: [PATCH 0420/2309] docstring --- src/allmydata/storage/expirer.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/allmydata/storage/expirer.py b/src/allmydata/storage/expirer.py index d2f48004a..ce126b6a4 100644 --- a/src/allmydata/storage/expirer.py +++ b/src/allmydata/storage/expirer.py @@ -23,6 +23,9 @@ from twisted.python.filepath import FilePath def _convert_pickle_state_to_json(state): """ + Convert a pickle-serialized crawler-history state to the new JSON + format. + :param dict state: the pickled state :return dict: the state in the JSON form From 75410e51f04f90007efce9fa2cba6504c54fe9ac Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 25 Oct 2021 21:10:43 -0600 Subject: [PATCH 0421/2309] refactor --- src/allmydata/storage/crawler.py | 86 ++++++++++++++++++-------------- src/allmydata/storage/expirer.py | 14 ++---- 2 files changed, 53 insertions(+), 47 deletions(-) diff --git a/src/allmydata/storage/crawler.py b/src/allmydata/storage/crawler.py index 548864e06..2e9bafd13 100644 --- a/src/allmydata/storage/crawler.py +++ b/src/allmydata/storage/crawler.py @@ -27,23 +27,14 @@ class TimeSliceExceeded(Exception): pass -def _convert_pickle_state_to_json(state): +def _convert_cycle_data(state): """ - :param dict state: the pickled state + :param dict state: cycle-to-date or history-item state :return dict: the state in the JSON form """ - # ["cycle-to-date"]["corrupt-shares"] from 2-tuple to list - # ["cycle-to-date"]["leases-per-share-histogram"] gets str keys instead of int - # ["cycle-start-finish-times"] from 2-tuple to list - # ["configured-expiration-mode"] from 4-tuple to list - # ["history"] keys are strings - if state["version"] != 1: - raise ValueError( - "Unknown version {version} in pickle state".format(**state) - ) - def convert_cem(value): + def _convert_expiration_mode(value): # original is a 4-tuple, with the last element being a 2-tuple # .. convert both to lists return [ @@ -53,41 +44,60 @@ def _convert_pickle_state_to_json(state): list(value[3]), ] - def convert_ctd(value): - ctd_converter = { - "lease-age-histogram": lambda value: { + def _convert_lease_age(value): + # if we're in cycle-to-date, this is a dict + if isinstance(value, dict): + return { "{},{}".format(k[0], k[1]): v for k, v in value.items() - }, - "corrupt-shares": lambda value: [ - list(x) - for x in value - ], - } - return { - k: ctd_converter.get(k, lambda z: z)(v) - for k, v in value.items() - } + } + # otherwise, it's a history-item and they're 3-tuples + return [ + list(v) + for v in value + ] - # we don't convert "history" here because that's in a separate - # file; see expirer.py converters = { - "cycle-to-date": convert_ctd, - "cycle-starte-finish-times": list, - "configured-expiration-mode": convert_cem, + "configured-expiration-mode": _convert_expiration_mode, + "cycle-start-finish-times": list, + "lease-age-histogram": _convert_lease_age, + "corrupt-shares": lambda value: [ + list(x) + for x in value + ], + "leases-per-share-histogram": lambda value: { + str(k): v + for k, v in value.items() + }, + } + return { + k: converters.get(k, lambda z: z)(v) + for k, v in state.items() } - def convert_value(key, value): - converter = converters.get(key, None) - if converter is None: - return value - return converter(value) - new_state = { - k: convert_value(k, v) +def _convert_pickle_state_to_json(state): + """ + :param dict state: the pickled state + + :return dict: the state in the JSON form + """ + # ["cycle-to-date"]["corrupt-shares"] from 2-tuple to list + # ["cycle-to-date"]["leases-per-share-histogram"] gets str keys instead of int + # ["cycle-start-finish-times"] from 2-tuple to list + # ["history"] keys are strings + if state["version"] != 1: + raise ValueError( + "Unknown version {version} in pickle state".format(**state) + ) + + converters = { + "cycle-to-date": _convert_cycle_data, + } + return { + k: converters.get(k, lambda x: x)(v) for k, v in state.items() } - return new_state def _maybe_upgrade_pickle_to_json(state_path, convert_pickle): diff --git a/src/allmydata/storage/expirer.py b/src/allmydata/storage/expirer.py index ce126b6a4..946498eaf 100644 --- a/src/allmydata/storage/expirer.py +++ b/src/allmydata/storage/expirer.py @@ -13,6 +13,7 @@ import struct from allmydata.storage.crawler import ( ShareCrawler, _maybe_upgrade_pickle_to_json, + _convert_cycle_data, ) from allmydata.storage.shares import get_share_file from allmydata.storage.common import UnknownMutableContainerVersionError, \ @@ -30,15 +31,10 @@ def _convert_pickle_state_to_json(state): :return dict: the state in the JSON form """ - print("CONVERT", state) - for k, v in state.items(): - print(k, v) - if state["version"] != 1: - raise ValueError( - "Unknown version {version} in pickle state".format(**state) - ) - - return state + return { + str(k): _convert_cycle_data(v) + for k, v in state.items() + } class _HistorySerializer(object): From a867294e00addbf4a0f4426820beb07895b81441 Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 25 Oct 2021 21:12:17 -0600 Subject: [PATCH 0422/2309] dead --- src/allmydata/storage/expirer.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/allmydata/storage/expirer.py b/src/allmydata/storage/expirer.py index 946498eaf..254264e38 100644 --- a/src/allmydata/storage/expirer.py +++ b/src/allmydata/storage/expirer.py @@ -61,7 +61,6 @@ class _HistorySerializer(object): :return dict: the existing history state """ - assert self._path is not None, "Not initialized" with self._path.open("rb") as f: history = json.load(f) return history @@ -70,7 +69,6 @@ class _HistorySerializer(object): """ Serialize the existing data as JSON. """ - assert self._path is not None, "Not initialized" with self._path.open("wb") as f: json.dump(new_history, f) return None From 94670461f1d93bef6766652a03a2b9bd85916224 Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 25 Oct 2021 21:37:51 -0600 Subject: [PATCH 0423/2309] tests --- src/allmydata/storage/expirer.py | 10 +- src/allmydata/test/test_storage_web.py | 168 +++++++++++++++++++++++++ 2 files changed, 173 insertions(+), 5 deletions(-) diff --git a/src/allmydata/storage/expirer.py b/src/allmydata/storage/expirer.py index 254264e38..9ba71539c 100644 --- a/src/allmydata/storage/expirer.py +++ b/src/allmydata/storage/expirer.py @@ -55,7 +55,7 @@ class _HistorySerializer(object): with self._path.open("wb") as f: json.dump({}, f) - def read(self): + def load(self): """ Deserialize the existing data. @@ -65,7 +65,7 @@ class _HistorySerializer(object): history = json.load(f) return history - def write(self, new_history): + def save(self, new_history): """ Serialize the existing data as JSON. """ @@ -368,12 +368,12 @@ class LeaseCheckingCrawler(ShareCrawler): # copy() needs to become a deepcopy h["space-recovered"] = s["space-recovered"].copy() - history = self._history_serializer.read() + history = self._history_serializer.load() history[str(cycle)] = h while len(history) > 10: oldcycles = sorted(int(k) for k in history.keys()) del history[str(oldcycles[0])] - self._history_serializer.write(history) + self._history_serializer.save(history) def get_state(self): """In addition to the crawler state described in @@ -442,7 +442,7 @@ class LeaseCheckingCrawler(ShareCrawler): progress = self.get_progress() state = ShareCrawler.get_state(self) # does a shallow copy - state["history"] = self._history_serializer.read() + state["history"] = self._history_serializer.load() if not progress["cycle-in-progress"]: del state["cycle-to-date"] diff --git a/src/allmydata/test/test_storage_web.py b/src/allmydata/test/test_storage_web.py index 5cdf02a25..033462d46 100644 --- a/src/allmydata/test/test_storage_web.py +++ b/src/allmydata/test/test_storage_web.py @@ -38,6 +38,7 @@ from allmydata.storage.crawler import ( ) from allmydata.storage.expirer import ( LeaseCheckingCrawler, + _HistorySerializer, ) from allmydata.web.storage import ( StorageStatus, @@ -1149,6 +1150,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin): """ The crawler can read existing state from the old pickle format """ + # this file came from an "in the wild" tahoe version 1.16.0 original_pickle = FilePath(__file__).parent().child("data").child("lease_checker.state") test_pickle = FilePath("lease_checker.state") with test_pickle.open("w") as local, original_pickle.open("r") as remote: @@ -1326,6 +1328,172 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin): } ) + def test_deserialize_history_pickle(self): + """ + The crawler can read existing history state from the old pickle + format + """ + # this file came from an "in the wild" tahoe version 1.16.0 + original_pickle = FilePath(__file__).parent().child("data").child("lease_checker.history") + test_pickle = FilePath("lease_checker.history") + with test_pickle.open("w") as local, original_pickle.open("r") as remote: + local.write(remote.read()) + + serial = _HistorySerializer(test_pickle.path) + + self.maxDiff = None + self.assertEqual( + serial.load(), + { + "363": { + 'configured-expiration-mode': ['age', None, None, ['immutable', 'mutable']], + 'expiration-enabled': False, + 'leases-per-share-histogram': { + '1': 39774, + }, + 'lease-age-histogram': [ + [0, 86400, 3125], + [345600, 432000, 4175], + [950400, 1036800, 141], + [1036800, 1123200, 345], + [1123200, 1209600, 81], + [1296000, 1382400, 1832], + [1555200, 1641600, 390], + [1728000, 1814400, 12], + [2073600, 2160000, 84], + [2160000, 2246400, 228], + [2246400, 2332800, 75], + [2592000, 2678400, 644], + [2678400, 2764800, 273], + [2764800, 2851200, 94], + [2851200, 2937600, 97], + [3196800, 3283200, 143], + [3283200, 3369600, 48], + [4147200, 4233600, 374], + [4320000, 4406400, 534], + [5270400, 5356800, 1005], + [6739200, 6825600, 8704], + [6825600, 6912000, 3986], + [6912000, 6998400, 7592], + [6998400, 7084800, 2607], + [7689600, 7776000, 35], + [8035200, 8121600, 33], + [8294400, 8380800, 54], + [8640000, 8726400, 45], + [8726400, 8812800, 27], + [8812800, 8899200, 12], + [9763200, 9849600, 77], + [9849600, 9936000, 91], + [9936000, 10022400, 1210], + [10022400, 10108800, 45], + [10108800, 10195200, 186], + [10368000, 10454400, 113], + [10972800, 11059200, 21], + [11232000, 11318400, 5], + [11318400, 11404800, 19], + [11404800, 11491200, 238], + [11491200, 11577600, 159], + [11750400, 11836800, 1], + [11836800, 11923200, 32], + [11923200, 12009600, 192], + [12009600, 12096000, 222], + [12096000, 12182400, 18], + [12182400, 12268800, 224], + [12268800, 12355200, 9], + [12355200, 12441600, 9], + [12441600, 12528000, 10], + [12528000, 12614400, 6], + [12614400, 12700800, 6], + [12700800, 12787200, 18], + [12787200, 12873600, 6], + [12873600, 12960000, 62], + ], + 'cycle-start-finish-times': [1634446505.241972, 1634446666.055401], + 'space-recovered': { + 'examined-buckets-immutable': 17896, + 'configured-buckets-mutable': 0, + 'examined-shares-mutable': 2473, + 'original-shares-mutable': 1185, + 'configured-buckets-immutable': 0, + 'original-shares-immutable': 27457, + 'original-diskbytes-immutable': 2810982400, + 'examined-shares-immutable': 37301, + 'original-buckets': 14047, + 'actual-shares-immutable': 0, + 'configured-shares': 0, + 'original-buckets-mutable': 691, + 'actual-diskbytes': 4096, + 'actual-shares-mutable': 0, + 'configured-buckets': 1, + 'examined-buckets-unknown': 14, + 'actual-sharebytes': 0, + 'original-shares': 28642, + 'actual-buckets-immutable': 0, + 'original-sharebytes': 2695552941, + 'examined-sharebytes-immutable': 2754798505, + 'actual-shares': 0, + 'actual-sharebytes-immutable': 0, + 'original-diskbytes': 2818981888, + 'configured-diskbytes-mutable': 0, + 'configured-sharebytes-immutable': 0, + 'configured-shares-mutable': 0, + 'actual-diskbytes-immutable': 0, + 'configured-diskbytes-immutable': 0, + 'original-diskbytes-mutable': 7995392, + 'actual-sharebytes-mutable': 0, + 'configured-sharebytes': 0, + 'examined-shares': 39774, + 'actual-diskbytes-mutable': 0, + 'actual-buckets': 1, + 'original-buckets-immutable': 13355, + 'configured-sharebytes-mutable': 0, + 'examined-sharebytes': 2763646972, + 'original-sharebytes-immutable': 2692076909, + 'original-sharebytes-mutable': 3476032, + 'actual-buckets-mutable': 0, + 'examined-buckets-mutable': 1286, + 'configured-shares-immutable': 0, + 'examined-diskbytes': 2854801408, + 'examined-diskbytes-mutable': 12161024, + 'examined-sharebytes-mutable': 8848467, + 'examined-buckets': 19197, + 'configured-diskbytes': 4096, + 'examined-diskbytes-immutable': 2842640384 + }, + 'corrupt-shares': [ + ['2dn6xnlnsqwtnapwxfdivpm3s4', 3], + ['2dn6xnlnsqwtnapwxfdivpm3s4', 0], + ['2rrzthwsrrxolevmwdvbdy3rqi', 3], + ['2rrzthwsrrxolevmwdvbdy3rqi', 0], + ['2skfngcto6h7eqmn4uo7ntk3ne', 3], + ['2skfngcto6h7eqmn4uo7ntk3ne', 0], + ['32d5swqpqx2mwix7xmqzvhdwje', 3], + ['32d5swqpqx2mwix7xmqzvhdwje', 0], + ['5mmayp66yflmpon3o6unsnbaca', 3], + ['5mmayp66yflmpon3o6unsnbaca', 0], + ['6ixhpvbtre7fnrl6pehlrlflc4', 3], + ['6ixhpvbtre7fnrl6pehlrlflc4', 0], + ['ewzhvswjsz4vp2bqkb6mi3bz2u', 3], + ['ewzhvswjsz4vp2bqkb6mi3bz2u', 0], + ['fu7pazf6ogavkqj6z4q5qqex3u', 3], + ['fu7pazf6ogavkqj6z4q5qqex3u', 0], + ['hbyjtqvpcimwxiyqbcbbdn2i4a', 3], + ['hbyjtqvpcimwxiyqbcbbdn2i4a', 0], + ['pmcjbdkbjdl26k3e6yja77femq', 3], + ['pmcjbdkbjdl26k3e6yja77femq', 0], + ['r6swof4v2uttbiiqwj5pi32cm4', 3], + ['r6swof4v2uttbiiqwj5pi32cm4', 0], + ['t45v5akoktf53evc2fi6gwnv6y', 3], + ['t45v5akoktf53evc2fi6gwnv6y', 0], + ['y6zb4faar3rdvn3e6pfg4wlotm', 3], + ['y6zb4faar3rdvn3e6pfg4wlotm', 0], + ['z3yghutvqoqbchjao4lndnrh3a', 3], + ['z3yghutvqoqbchjao4lndnrh3a', 0], + ] + } + } + ) + class WebStatus(unittest.TestCase, pollmixin.PollMixin): From 069c332a6815c6c67b77a7af328e7bb2993d175d Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 25 Oct 2021 21:49:25 -0600 Subject: [PATCH 0424/2309] straight assert --- src/allmydata/storage/crawler.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/allmydata/storage/crawler.py b/src/allmydata/storage/crawler.py index 2e9bafd13..d1366765e 100644 --- a/src/allmydata/storage/crawler.py +++ b/src/allmydata/storage/crawler.py @@ -86,10 +86,7 @@ def _convert_pickle_state_to_json(state): # ["cycle-to-date"]["leases-per-share-histogram"] gets str keys instead of int # ["cycle-start-finish-times"] from 2-tuple to list # ["history"] keys are strings - if state["version"] != 1: - raise ValueError( - "Unknown version {version} in pickle state".format(**state) - ) + assert state["version"] == 1, "Only known version is 1" converters = { "cycle-to-date": _convert_cycle_data, From 9b3c55e4aa856e48b6d6fb5ee0252d69aeb64110 Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 25 Oct 2021 21:59:29 -0600 Subject: [PATCH 0425/2309] test a second deserialzation --- src/allmydata/test/test_storage_web.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/allmydata/test/test_storage_web.py b/src/allmydata/test/test_storage_web.py index 033462d46..70866cba9 100644 --- a/src/allmydata/test/test_storage_web.py +++ b/src/allmydata/test/test_storage_web.py @@ -1327,6 +1327,11 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin): 'last-complete-bucket': None, } ) + second_serial = _LeaseStateSerializer(serial._path.path) + self.assertEqual( + serial.load(), + second_serial.load(), + ) def test_deserialize_history_pickle(self): """ From 4f64bbaa0086af61745f7f55fd04fb716f018960 Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 25 Oct 2021 22:15:49 -0600 Subject: [PATCH 0426/2309] data --- src/allmydata/test/data/lease_checker.history | 501 ++++++++++++++++++ 1 file changed, 501 insertions(+) create mode 100644 src/allmydata/test/data/lease_checker.history diff --git a/src/allmydata/test/data/lease_checker.history b/src/allmydata/test/data/lease_checker.history new file mode 100644 index 000000000..0c27a5ad0 --- /dev/null +++ b/src/allmydata/test/data/lease_checker.history @@ -0,0 +1,501 @@ +(dp0 +I363 +(dp1 +Vconfigured-expiration-mode +p2 +(S'age' +p3 +NN(S'immutable' +p4 +S'mutable' +p5 +tp6 +tp7 +sVexpiration-enabled +p8 +I00 +sVleases-per-share-histogram +p9 +(dp10 +I1 +I39774 +ssVlease-age-histogram +p11 +(lp12 +(I0 +I86400 +I3125 +tp13 +a(I345600 +I432000 +I4175 +tp14 +a(I950400 +I1036800 +I141 +tp15 +a(I1036800 +I1123200 +I345 +tp16 +a(I1123200 +I1209600 +I81 +tp17 +a(I1296000 +I1382400 +I1832 +tp18 +a(I1555200 +I1641600 +I390 +tp19 +a(I1728000 +I1814400 +I12 +tp20 +a(I2073600 +I2160000 +I84 +tp21 +a(I2160000 +I2246400 +I228 +tp22 +a(I2246400 +I2332800 +I75 +tp23 +a(I2592000 +I2678400 +I644 +tp24 +a(I2678400 +I2764800 +I273 +tp25 +a(I2764800 +I2851200 +I94 +tp26 +a(I2851200 +I2937600 +I97 +tp27 +a(I3196800 +I3283200 +I143 +tp28 +a(I3283200 +I3369600 +I48 +tp29 +a(I4147200 +I4233600 +I374 +tp30 +a(I4320000 +I4406400 +I534 +tp31 +a(I5270400 +I5356800 +I1005 +tp32 +a(I6739200 +I6825600 +I8704 +tp33 +a(I6825600 +I6912000 +I3986 +tp34 +a(I6912000 +I6998400 +I7592 +tp35 +a(I6998400 +I7084800 +I2607 +tp36 +a(I7689600 +I7776000 +I35 +tp37 +a(I8035200 +I8121600 +I33 +tp38 +a(I8294400 +I8380800 +I54 +tp39 +a(I8640000 +I8726400 +I45 +tp40 +a(I8726400 +I8812800 +I27 +tp41 +a(I8812800 +I8899200 +I12 +tp42 +a(I9763200 +I9849600 +I77 +tp43 +a(I9849600 +I9936000 +I91 +tp44 +a(I9936000 +I10022400 +I1210 +tp45 +a(I10022400 +I10108800 +I45 +tp46 +a(I10108800 +I10195200 +I186 +tp47 +a(I10368000 +I10454400 +I113 +tp48 +a(I10972800 +I11059200 +I21 +tp49 +a(I11232000 +I11318400 +I5 +tp50 +a(I11318400 +I11404800 +I19 +tp51 +a(I11404800 +I11491200 +I238 +tp52 +a(I11491200 +I11577600 +I159 +tp53 +a(I11750400 +I11836800 +I1 +tp54 +a(I11836800 +I11923200 +I32 +tp55 +a(I11923200 +I12009600 +I192 +tp56 +a(I12009600 +I12096000 +I222 +tp57 +a(I12096000 +I12182400 +I18 +tp58 +a(I12182400 +I12268800 +I224 +tp59 +a(I12268800 +I12355200 +I9 +tp60 +a(I12355200 +I12441600 +I9 +tp61 +a(I12441600 +I12528000 +I10 +tp62 +a(I12528000 +I12614400 +I6 +tp63 +a(I12614400 +I12700800 +I6 +tp64 +a(I12700800 +I12787200 +I18 +tp65 +a(I12787200 +I12873600 +I6 +tp66 +a(I12873600 +I12960000 +I62 +tp67 +asVcycle-start-finish-times +p68 +(F1634446505.241972 +F1634446666.055401 +tp69 +sVspace-recovered +p70 +(dp71 +Vexamined-buckets-immutable +p72 +I17896 +sVconfigured-buckets-mutable +p73 +I0 +sVexamined-shares-mutable +p74 +I2473 +sVoriginal-shares-mutable +p75 +I1185 +sVconfigured-buckets-immutable +p76 +I0 +sVoriginal-shares-immutable +p77 +I27457 +sVoriginal-diskbytes-immutable +p78 +I2810982400 +sVexamined-shares-immutable +p79 +I37301 +sVoriginal-buckets +p80 +I14047 +sVactual-shares-immutable +p81 +I0 +sVconfigured-shares +p82 +I0 +sVoriginal-buckets-mutable +p83 +I691 +sVactual-diskbytes +p84 +I4096 +sVactual-shares-mutable +p85 +I0 +sVconfigured-buckets +p86 +I1 +sVexamined-buckets-unknown +p87 +I14 +sVactual-sharebytes +p88 +I0 +sVoriginal-shares +p89 +I28642 +sVactual-buckets-immutable +p90 +I0 +sVoriginal-sharebytes +p91 +I2695552941 +sVexamined-sharebytes-immutable +p92 +I2754798505 +sVactual-shares +p93 +I0 +sVactual-sharebytes-immutable +p94 +I0 +sVoriginal-diskbytes +p95 +I2818981888 +sVconfigured-diskbytes-mutable +p96 +I0 +sVconfigured-sharebytes-immutable +p97 +I0 +sVconfigured-shares-mutable +p98 +I0 +sVactual-diskbytes-immutable +p99 +I0 +sVconfigured-diskbytes-immutable +p100 +I0 +sVoriginal-diskbytes-mutable +p101 +I7995392 +sVactual-sharebytes-mutable +p102 +I0 +sVconfigured-sharebytes +p103 +I0 +sVexamined-shares +p104 +I39774 +sVactual-diskbytes-mutable +p105 +I0 +sVactual-buckets +p106 +I1 +sVoriginal-buckets-immutable +p107 +I13355 +sVconfigured-sharebytes-mutable +p108 +I0 +sVexamined-sharebytes +p109 +I2763646972 +sVoriginal-sharebytes-immutable +p110 +I2692076909 +sVoriginal-sharebytes-mutable +p111 +I3476032 +sVactual-buckets-mutable +p112 +I0 +sVexamined-buckets-mutable +p113 +I1286 +sVconfigured-shares-immutable +p114 +I0 +sVexamined-diskbytes +p115 +I2854801408 +sVexamined-diskbytes-mutable +p116 +I12161024 +sVexamined-sharebytes-mutable +p117 +I8848467 +sVexamined-buckets +p118 +I19197 +sVconfigured-diskbytes +p119 +I4096 +sVexamined-diskbytes-immutable +p120 +I2842640384 +ssVcorrupt-shares +p121 +(lp122 +(V2dn6xnlnsqwtnapwxfdivpm3s4 +p123 +I3 +tp124 +a(g123 +I0 +tp125 +a(V2rrzthwsrrxolevmwdvbdy3rqi +p126 +I3 +tp127 +a(g126 +I0 +tp128 +a(V2skfngcto6h7eqmn4uo7ntk3ne +p129 +I3 +tp130 +a(g129 +I0 +tp131 +a(V32d5swqpqx2mwix7xmqzvhdwje +p132 +I3 +tp133 +a(g132 +I0 +tp134 +a(V5mmayp66yflmpon3o6unsnbaca +p135 +I3 +tp136 +a(g135 +I0 +tp137 +a(V6ixhpvbtre7fnrl6pehlrlflc4 +p138 +I3 +tp139 +a(g138 +I0 +tp140 +a(Vewzhvswjsz4vp2bqkb6mi3bz2u +p141 +I3 +tp142 +a(g141 +I0 +tp143 +a(Vfu7pazf6ogavkqj6z4q5qqex3u +p144 +I3 +tp145 +a(g144 +I0 +tp146 +a(Vhbyjtqvpcimwxiyqbcbbdn2i4a +p147 +I3 +tp148 +a(g147 +I0 +tp149 +a(Vpmcjbdkbjdl26k3e6yja77femq +p150 +I3 +tp151 +a(g150 +I0 +tp152 +a(Vr6swof4v2uttbiiqwj5pi32cm4 +p153 +I3 +tp154 +a(g153 +I0 +tp155 +a(Vt45v5akoktf53evc2fi6gwnv6y +p156 +I3 +tp157 +a(g156 +I0 +tp158 +a(Vy6zb4faar3rdvn3e6pfg4wlotm +p159 +I3 +tp160 +a(g159 +I0 +tp161 +a(Vz3yghutvqoqbchjao4lndnrh3a +p162 +I3 +tp163 +a(g162 +I0 +tp164 +ass. \ No newline at end of file From 1c93175583365966f0b476ecc95720efcbf2f827 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 2 Nov 2021 22:42:33 -0600 Subject: [PATCH 0427/2309] cleanup --- src/allmydata/storage/crawler.py | 22 ++++------------------ src/allmydata/storage/expirer.py | 1 - 2 files changed, 4 insertions(+), 19 deletions(-) diff --git a/src/allmydata/storage/crawler.py b/src/allmydata/storage/crawler.py index d1366765e..a06806d17 100644 --- a/src/allmydata/storage/crawler.py +++ b/src/allmydata/storage/crawler.py @@ -82,10 +82,6 @@ def _convert_pickle_state_to_json(state): :return dict: the state in the JSON form """ - # ["cycle-to-date"]["corrupt-shares"] from 2-tuple to list - # ["cycle-to-date"]["leases-per-share-histogram"] gets str keys instead of int - # ["cycle-start-finish-times"] from 2-tuple to list - # ["history"] keys are strings assert state["version"] == 1, "Only known version is 1" converters = { @@ -123,12 +119,12 @@ def _maybe_upgrade_pickle_to_json(state_path, convert_pickle): # upgrade the pickle data to JSON import pickle - with state_path.open("r") as f: + with state_path.open("rb") as f: state = pickle.load(f) new_state = convert_pickle(state) - json_state_path = state_path.siblingExtension(".json") - with json_state_path.open("w") as f: + with json_state_path.open("wb") as f: json.dump(new_state, f) + # we've written the JSON, delete the pickle state_path.remove() return json_state_path.path @@ -148,15 +144,9 @@ class _LeaseStateSerializer(object): _convert_pickle_state_to_json, ) ) - # XXX want this to .. load and save the state - # - if the state is pickle-only: - # - load it and convert to json format - # - save json - # - delete pickle - # - if the state is json, load it def load(self): - with self._path.open("r") as f: + with self._path.open("rb") as f: return json.load(f) def save(self, data): @@ -388,10 +378,6 @@ class ShareCrawler(service.MultiService): else: last_complete_prefix = self.prefixes[lcpi] self.state["last-complete-prefix"] = last_complete_prefix - - # Note: we use self.get_state() here because e.g - # LeaseCheckingCrawler stores non-JSON-able state in - # self.state() but converts it in self.get_state() self._state_serializer.save(self.get_state()) def startService(self): diff --git a/src/allmydata/storage/expirer.py b/src/allmydata/storage/expirer.py index 9ba71539c..ad1343ef5 100644 --- a/src/allmydata/storage/expirer.py +++ b/src/allmydata/storage/expirer.py @@ -123,7 +123,6 @@ class LeaseCheckingCrawler(ShareCrawler): cutoff_date, # used if expiration_mode=="cutoff-date" sharetypes): self._history_serializer = _HistorySerializer(historyfile) - ##self.historyfile = historyfile self.expiration_enabled = expiration_enabled self.mode = mode self.override_lease_duration = None From 23ff1b2430334d376abd426421039562f94bcfc8 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 2 Nov 2021 22:45:08 -0600 Subject: [PATCH 0428/2309] noqa --- src/allmydata/storage/crawler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/storage/crawler.py b/src/allmydata/storage/crawler.py index a06806d17..129659d27 100644 --- a/src/allmydata/storage/crawler.py +++ b/src/allmydata/storage/crawler.py @@ -11,7 +11,7 @@ from __future__ import print_function from future.utils import PY2, PY3 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import os import time From 2fe686135bf9513cbdbbe700e5faa63ee1743e83 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 2 Nov 2021 23:33:54 -0600 Subject: [PATCH 0429/2309] rename data to appease distutils --- .../data/{lease_checker.history => lease_checker.history.txt} | 0 .../data/{lease_checker.state => lease_checker.state.txt} | 0 src/allmydata/test/test_storage_web.py | 4 ++-- 3 files changed, 2 insertions(+), 2 deletions(-) rename src/allmydata/test/data/{lease_checker.history => lease_checker.history.txt} (100%) rename src/allmydata/test/data/{lease_checker.state => lease_checker.state.txt} (100%) diff --git a/src/allmydata/test/data/lease_checker.history b/src/allmydata/test/data/lease_checker.history.txt similarity index 100% rename from src/allmydata/test/data/lease_checker.history rename to src/allmydata/test/data/lease_checker.history.txt diff --git a/src/allmydata/test/data/lease_checker.state b/src/allmydata/test/data/lease_checker.state.txt similarity index 100% rename from src/allmydata/test/data/lease_checker.state rename to src/allmydata/test/data/lease_checker.state.txt diff --git a/src/allmydata/test/test_storage_web.py b/src/allmydata/test/test_storage_web.py index 70866cba9..269af2203 100644 --- a/src/allmydata/test/test_storage_web.py +++ b/src/allmydata/test/test_storage_web.py @@ -1151,7 +1151,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin): The crawler can read existing state from the old pickle format """ # this file came from an "in the wild" tahoe version 1.16.0 - original_pickle = FilePath(__file__).parent().child("data").child("lease_checker.state") + original_pickle = FilePath(__file__).parent().child("data").child("lease_checker.state.txt") test_pickle = FilePath("lease_checker.state") with test_pickle.open("w") as local, original_pickle.open("r") as remote: local.write(remote.read()) @@ -1339,7 +1339,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin): format """ # this file came from an "in the wild" tahoe version 1.16.0 - original_pickle = FilePath(__file__).parent().child("data").child("lease_checker.history") + original_pickle = FilePath(__file__).parent().child("data").child("lease_checker.history.txt") test_pickle = FilePath("lease_checker.history") with test_pickle.open("w") as local, original_pickle.open("r") as remote: local.write(remote.read()) From a208502e18c4f4faf85d500e86e3b2093d219ecf Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 16 Nov 2021 18:29:01 -0500 Subject: [PATCH 0430/2309] whitespace --- src/allmydata/storage/lease.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/storage/lease.py b/src/allmydata/storage/lease.py index 63dba15e8..bc94ca6d5 100644 --- a/src/allmydata/storage/lease.py +++ b/src/allmydata/storage/lease.py @@ -272,7 +272,7 @@ class HashedLeaseInfo(proxyForInterface(ILeaseInfo, "_lease_info")): # type: ign Hash the candidate secret and compare the result to the stored hashed secret. """ - if isinstance(candidate_secret, _HashedCancelSecret): + if isinstance(candidate_secret, _HashedCancelSecret): # Someone read it off of this object in this project - probably # the lease crawler - and is just trying to use it to identify # which lease it wants to operate on. Avoid re-hashing the value. From 3a8432713fb0885f3795d4501c77e80a21caea5a Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 16 Nov 2021 18:29:05 -0500 Subject: [PATCH 0431/2309] a note about what's happening with proxyForInterface --- src/allmydata/storage/lease.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/allmydata/storage/lease.py b/src/allmydata/storage/lease.py index bc94ca6d5..0c3b219f6 100644 --- a/src/allmydata/storage/lease.py +++ b/src/allmydata/storage/lease.py @@ -260,6 +260,10 @@ class HashedLeaseInfo(proxyForInterface(ILeaseInfo, "_lease_info")): # type: ign _lease_info = attr.ib() _hash = attr.ib() + # proxyForInterface will take care of forwarding all methods on ILeaseInfo + # to `_lease_info`. Here we override a few of those methods to adjust + # their behavior to make them suitable for use with hashed secrets. + def is_renew_secret(self, candidate_secret): """ Hash the candidate secret and compare the result to the stored hashed From e8adca40abdfa9f4c8616194bbe0bf1fe8817f1f Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 16 Nov 2021 18:32:35 -0500 Subject: [PATCH 0432/2309] give the ContainerVersionError exceptions a nice str --- src/allmydata/storage/common.py | 6 ++++++ src/allmydata/test/test_storage.py | 3 +++ 2 files changed, 9 insertions(+) diff --git a/src/allmydata/storage/common.py b/src/allmydata/storage/common.py index 48fc77840..17a3f41b7 100644 --- a/src/allmydata/storage/common.py +++ b/src/allmydata/storage/common.py @@ -21,6 +21,12 @@ class UnknownContainerVersionError(Exception): self.filename = filename self.version = version + def __str__(self): + return "sharefile {!r} had unexpected version {!r}".format( + self.filename, + self.version, + ) + class UnknownMutableContainerVersionError(UnknownContainerVersionError): pass diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index 655395042..ba3d3598f 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -651,6 +651,7 @@ class Server(unittest.TestCase): ss.remote_get_buckets, b"si1") self.assertEqual(e.filename, fn) self.assertEqual(e.version, 0) + self.assertIn("had unexpected version 0", str(e)) def test_disconnect(self): # simulate a disconnection @@ -1136,6 +1137,8 @@ class MutableServer(unittest.TestCase): read, b"si1", [0], [(0,10)]) self.assertEqual(e.filename, fn) self.assertTrue(e.version.startswith(b"BAD MAGIC")) + self.assertIn("had unexpected version", str(e)) + self.assertIn("BAD MAGIC", str(e)) def test_container_size(self): ss = self.create("test_container_size") From 6a78703675e9ce9e42a8401ac38410d78357a86e Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 17 Nov 2021 10:53:51 -0500 Subject: [PATCH 0433/2309] News file. --- newsfragments/3807.feature | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/3807.feature diff --git a/newsfragments/3807.feature b/newsfragments/3807.feature new file mode 100644 index 000000000..f82363ffd --- /dev/null +++ b/newsfragments/3807.feature @@ -0,0 +1 @@ +If uploading an immutable hasn't had a write for 30 minutes, the storage server will abort the upload. \ No newline at end of file From 92c36a67d8c98436e2cc2616d89ff0135307858c Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 17 Nov 2021 11:01:04 -0500 Subject: [PATCH 0434/2309] Use IReactorTime instead of ad-hoc solutions. --- src/allmydata/storage/server.py | 39 ++++++++++++----------- src/allmydata/test/test_istorageserver.py | 10 +++--- src/allmydata/test/test_storage.py | 16 +++++----- 3 files changed, 34 insertions(+), 31 deletions(-) diff --git a/src/allmydata/storage/server.py b/src/allmydata/storage/server.py index ee2ea1c61..499d47276 100644 --- a/src/allmydata/storage/server.py +++ b/src/allmydata/storage/server.py @@ -20,6 +20,7 @@ import six from foolscap.api import Referenceable from foolscap.ipb import IRemoteReference from twisted.application import service +from twisted.internet import reactor from zope.interface import implementer from allmydata.interfaces import RIStorageServer, IStatsProducer @@ -71,7 +72,7 @@ class StorageServer(service.MultiService, Referenceable): expiration_override_lease_duration=None, expiration_cutoff_date=None, expiration_sharetypes=("mutable", "immutable"), - get_current_time=time.time): + clock=reactor): service.MultiService.__init__(self) assert isinstance(nodeid, bytes) assert len(nodeid) == 20 @@ -122,7 +123,7 @@ class StorageServer(service.MultiService, Referenceable): expiration_cutoff_date, expiration_sharetypes) self.lease_checker.setServiceParent(self) - self._get_current_time = get_current_time + self._clock = clock # Currently being-written Bucketwriters. For Foolscap, lifetime is tied # to connection: when disconnection happens, the BucketWriters are @@ -292,7 +293,7 @@ class StorageServer(service.MultiService, Referenceable): # owner_num is not for clients to set, but rather it should be # curried into the PersonalStorageServer instance that is dedicated # to a particular owner. - start = self._get_current_time() + start = self._clock.seconds() self.count("allocate") alreadygot = set() bucketwriters = {} # k: shnum, v: BucketWriter @@ -305,7 +306,7 @@ class StorageServer(service.MultiService, Referenceable): # goes into the share files themselves. It could also be put into a # separate database. Note that the lease should not be added until # the BucketWriter has been closed. - expire_time = self._get_current_time() + DEFAULT_RENEWAL_TIME + expire_time = self._clock.seconds() + DEFAULT_RENEWAL_TIME lease_info = LeaseInfo(owner_num, renew_secret, cancel_secret, expire_time, self.my_nodeid) @@ -360,7 +361,7 @@ class StorageServer(service.MultiService, Referenceable): if bucketwriters: fileutil.make_dirs(os.path.join(self.sharedir, si_dir)) - self.add_latency("allocate", self._get_current_time() - start) + self.add_latency("allocate", self._clock.seconds() - start) return alreadygot, bucketwriters def remote_allocate_buckets(self, storage_index, @@ -395,26 +396,26 @@ class StorageServer(service.MultiService, Referenceable): def remote_add_lease(self, storage_index, renew_secret, cancel_secret, owner_num=1): - start = self._get_current_time() + start = self._clock.seconds() self.count("add-lease") - new_expire_time = self._get_current_time() + DEFAULT_RENEWAL_TIME + new_expire_time = self._clock.seconds() + DEFAULT_RENEWAL_TIME lease_info = LeaseInfo(owner_num, renew_secret, cancel_secret, new_expire_time, self.my_nodeid) for sf in self._iter_share_files(storage_index): sf.add_or_renew_lease(lease_info) - self.add_latency("add-lease", self._get_current_time() - start) + self.add_latency("add-lease", self._clock.seconds() - start) return None def remote_renew_lease(self, storage_index, renew_secret): - start = self._get_current_time() + start = self._clock.seconds() self.count("renew") - new_expire_time = self._get_current_time() + DEFAULT_RENEWAL_TIME + new_expire_time = self._clock.seconds() + DEFAULT_RENEWAL_TIME found_buckets = False for sf in self._iter_share_files(storage_index): found_buckets = True sf.renew_lease(renew_secret, new_expire_time) - self.add_latency("renew", self._get_current_time() - start) + self.add_latency("renew", self._clock.seconds() - start) if not found_buckets: raise IndexError("no such lease to renew") @@ -441,7 +442,7 @@ class StorageServer(service.MultiService, Referenceable): pass def remote_get_buckets(self, storage_index): - start = self._get_current_time() + start = self._clock.seconds() self.count("get") si_s = si_b2a(storage_index) log.msg("storage: get_buckets %r" % si_s) @@ -449,7 +450,7 @@ class StorageServer(service.MultiService, Referenceable): for shnum, filename in self._get_bucket_shares(storage_index): bucketreaders[shnum] = BucketReader(self, filename, storage_index, shnum) - self.add_latency("get", self._get_current_time() - start) + self.add_latency("get", self._clock.seconds() - start) return bucketreaders def get_leases(self, storage_index): @@ -608,7 +609,7 @@ class StorageServer(service.MultiService, Referenceable): :return LeaseInfo: Information for a new lease for a share. """ ownerid = 1 # TODO - expire_time = self._get_current_time() + DEFAULT_RENEWAL_TIME + expire_time = self._clock.seconds() + DEFAULT_RENEWAL_TIME lease_info = LeaseInfo(ownerid, renew_secret, cancel_secret, expire_time, self.my_nodeid) @@ -646,7 +647,7 @@ class StorageServer(service.MultiService, Referenceable): See ``allmydata.interfaces.RIStorageServer`` for details about other parameters and return value. """ - start = self._get_current_time() + start = self._clock.seconds() self.count("writev") si_s = si_b2a(storage_index) log.msg("storage: slot_writev %r" % si_s) @@ -687,7 +688,7 @@ class StorageServer(service.MultiService, Referenceable): self._add_or_renew_leases(remaining_shares, lease_info) # all done - self.add_latency("writev", self._get_current_time() - start) + self.add_latency("writev", self._clock.seconds() - start) return (testv_is_good, read_data) def remote_slot_testv_and_readv_and_writev(self, storage_index, @@ -713,7 +714,7 @@ class StorageServer(service.MultiService, Referenceable): return share def remote_slot_readv(self, storage_index, shares, readv): - start = self._get_current_time() + start = self._clock.seconds() self.count("readv") si_s = si_b2a(storage_index) lp = log.msg("storage: slot_readv %r %r" % (si_s, shares), @@ -722,7 +723,7 @@ class StorageServer(service.MultiService, Referenceable): # shares exist if there is a file for them bucketdir = os.path.join(self.sharedir, si_dir) if not os.path.isdir(bucketdir): - self.add_latency("readv", self._get_current_time() - start) + self.add_latency("readv", self._clock.seconds() - start) return {} datavs = {} for sharenum_s in os.listdir(bucketdir): @@ -736,7 +737,7 @@ class StorageServer(service.MultiService, Referenceable): datavs[sharenum] = msf.readv(readv) log.msg("returning shares %s" % (list(datavs.keys()),), facility="tahoe.storage", level=log.NOISY, parent=lp) - self.add_latency("readv", self._get_current_time() - start) + self.add_latency("readv", self._clock.seconds() - start) return datavs def remote_advise_corrupt_share(self, share_type, storage_index, shnum, diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index fe494a9d4..a17264713 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -21,6 +21,7 @@ if PY2: from random import Random from twisted.internet.defer import inlineCallbacks, returnValue +from twisted.internet.task import Clock from foolscap.api import Referenceable, RemoteException @@ -1017,16 +1018,17 @@ class _FoolscapMixin(SystemTestMixin): self.server = s break assert self.server is not None, "Couldn't find StorageServer" - self._current_time = 123456 - self.server._get_current_time = self.fake_time + self._clock = Clock() + self._clock.advance(123456) + self.server._clock = self._clock def fake_time(self): """Return the current fake, test-controlled, time.""" - return self._current_time + return self._clock.seconds() def fake_sleep(self, seconds): """Advance the fake time by the given number of seconds.""" - self._current_time += seconds + self._clock.advance(seconds) @inlineCallbacks def tearDown(self): diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index 4e40a76a5..e143bec63 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -23,7 +23,7 @@ from uuid import uuid4 from twisted.trial import unittest -from twisted.internet import defer +from twisted.internet import defer, reactor from twisted.internet.task import Clock from hypothesis import given, strategies @@ -438,11 +438,11 @@ class Server(unittest.TestCase): basedir = os.path.join("storage", "Server", name) return basedir - def create(self, name, reserved_space=0, klass=StorageServer, get_current_time=time.time): + def create(self, name, reserved_space=0, klass=StorageServer, clock=reactor): workdir = self.workdir(name) ss = klass(workdir, b"\x00" * 20, reserved_space=reserved_space, stats_provider=FakeStatsProvider(), - get_current_time=get_current_time) + clock=clock) ss.setServiceParent(self.sparent) return ss @@ -626,7 +626,7 @@ class Server(unittest.TestCase): clock.advance(first_lease) ss = self.create( "test_allocate_without_lease_renewal", - get_current_time=clock.seconds, + clock=clock, ) # Put a share on there @@ -918,7 +918,7 @@ class Server(unittest.TestCase): """ clock = Clock() clock.advance(123) - ss = self.create("test_immutable_add_lease_renews", get_current_time=clock.seconds) + ss = self.create("test_immutable_add_lease_renews", clock=clock) # Start out with single lease created with bucket: renewal_secret, cancel_secret = self.create_bucket_5_shares(ss, b"si0") @@ -1032,10 +1032,10 @@ class MutableServer(unittest.TestCase): basedir = os.path.join("storage", "MutableServer", name) return basedir - def create(self, name, get_current_time=time.time): + def create(self, name, clock=reactor): workdir = self.workdir(name) ss = StorageServer(workdir, b"\x00" * 20, - get_current_time=get_current_time) + clock=clock) ss.setServiceParent(self.sparent) return ss @@ -1420,7 +1420,7 @@ class MutableServer(unittest.TestCase): clock = Clock() clock.advance(235) ss = self.create("test_mutable_add_lease_renews", - get_current_time=clock.seconds) + clock=clock) def secrets(n): return ( self.write_enabler(b"we1"), self.renew_secret(b"we1-%d" % n), From bf7d03310fc1bd730ba12449112144f3e6935020 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 17 Nov 2021 11:09:45 -0500 Subject: [PATCH 0435/2309] Hide all _trial_temp. --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 50a1352a2..7c7fa2afd 100644 --- a/.gitignore +++ b/.gitignore @@ -29,7 +29,7 @@ zope.interface-*.egg .pc /src/allmydata/test/plugins/dropin.cache -/_trial_temp* +**/_trial_temp* /tmp* /*.patch /dist/ From 45c00e93c9709e216fe87410783b0a6125e159e7 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 17 Nov 2021 11:12:40 -0500 Subject: [PATCH 0436/2309] Use clock in BucketWriter. --- src/allmydata/storage/immutable.py | 11 ++++++----- src/allmydata/storage/server.py | 3 ++- src/allmydata/test/test_storage.py | 12 ++++++------ 3 files changed, 14 insertions(+), 12 deletions(-) diff --git a/src/allmydata/storage/immutable.py b/src/allmydata/storage/immutable.py index 8a7a5a966..7cfb7a1bf 100644 --- a/src/allmydata/storage/immutable.py +++ b/src/allmydata/storage/immutable.py @@ -233,7 +233,7 @@ class ShareFile(object): @implementer(RIBucketWriter) class BucketWriter(Referenceable): # type: ignore # warner/foolscap#78 - def __init__(self, ss, incominghome, finalhome, max_size, lease_info): + def __init__(self, ss, incominghome, finalhome, max_size, lease_info, clock): self.ss = ss self.incominghome = incominghome self.finalhome = finalhome @@ -245,12 +245,13 @@ class BucketWriter(Referenceable): # type: ignore # warner/foolscap#78 # added by simultaneous uploaders self._sharefile.add_lease(lease_info) self._already_written = RangeMap() + self._clock = clock def allocated_size(self): return self._max_size def remote_write(self, offset, data): - start = time.time() + start = self._clock.seconds() precondition(not self.closed) if self.throw_out_all_data: return @@ -268,12 +269,12 @@ class BucketWriter(Referenceable): # type: ignore # warner/foolscap#78 self._sharefile.write_share_data(offset, data) self._already_written.set(True, offset, end) - self.ss.add_latency("write", time.time() - start) + self.ss.add_latency("write", self._clock.seconds() - start) self.ss.count("write") def remote_close(self): precondition(not self.closed) - start = time.time() + start = self._clock.seconds() fileutil.make_dirs(os.path.dirname(self.finalhome)) fileutil.rename(self.incominghome, self.finalhome) @@ -306,7 +307,7 @@ class BucketWriter(Referenceable): # type: ignore # warner/foolscap#78 filelen = os.stat(self.finalhome)[stat.ST_SIZE] self.ss.bucket_writer_closed(self, filelen) - self.ss.add_latency("close", time.time() - start) + self.ss.add_latency("close", self._clock.seconds() - start) self.ss.count("close") def disconnected(self): diff --git a/src/allmydata/storage/server.py b/src/allmydata/storage/server.py index 499d47276..080c1aea1 100644 --- a/src/allmydata/storage/server.py +++ b/src/allmydata/storage/server.py @@ -347,7 +347,8 @@ class StorageServer(service.MultiService, Referenceable): elif (not limited) or (remaining_space >= max_space_per_bucket): # ok! we need to create the new share file. bw = BucketWriter(self, incominghome, finalhome, - max_space_per_bucket, lease_info) + max_space_per_bucket, lease_info, + clock=self._clock) if self.no_storage: bw.throw_out_all_data = True bucketwriters[shnum] = bw diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index e143bec63..36c776fba 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -128,7 +128,7 @@ class Bucket(unittest.TestCase): def test_create(self): incoming, final = self.make_workdir("test_create") - bw = BucketWriter(self, incoming, final, 200, self.make_lease()) + bw = BucketWriter(self, incoming, final, 200, self.make_lease(), Clock()) bw.remote_write(0, b"a"*25) bw.remote_write(25, b"b"*25) bw.remote_write(50, b"c"*25) @@ -137,7 +137,7 @@ class Bucket(unittest.TestCase): def test_readwrite(self): incoming, final = self.make_workdir("test_readwrite") - bw = BucketWriter(self, incoming, final, 200, self.make_lease()) + bw = BucketWriter(self, incoming, final, 200, self.make_lease(), Clock()) bw.remote_write(0, b"a"*25) bw.remote_write(25, b"b"*25) bw.remote_write(50, b"c"*7) # last block may be short @@ -155,7 +155,7 @@ class Bucket(unittest.TestCase): incoming, final = self.make_workdir( "test_write_past_size_errors-{}".format(i) ) - bw = BucketWriter(self, incoming, final, 200, self.make_lease()) + bw = BucketWriter(self, incoming, final, 200, self.make_lease(), Clock()) with self.assertRaises(DataTooLargeError): bw.remote_write(offset, b"a" * length) @@ -174,7 +174,7 @@ class Bucket(unittest.TestCase): expected_data = b"".join(bchr(i) for i in range(100)) incoming, final = self.make_workdir("overlapping_writes_{}".format(uuid4())) bw = BucketWriter( - self, incoming, final, length, self.make_lease(), + self, incoming, final, length, self.make_lease(), Clock() ) # Three writes: 10-19, 30-39, 50-59. This allows for a bunch of holes. bw.remote_write(10, expected_data[10:20]) @@ -212,7 +212,7 @@ class Bucket(unittest.TestCase): length = 100 incoming, final = self.make_workdir("overlapping_writes_{}".format(uuid4())) bw = BucketWriter( - self, incoming, final, length, self.make_lease(), + self, incoming, final, length, self.make_lease(), Clock() ) # Three writes: 10-19, 30-39, 50-59. This allows for a bunch of holes. bw.remote_write(10, b"1" * 10) @@ -312,7 +312,7 @@ class BucketProxy(unittest.TestCase): final = os.path.join(basedir, "bucket") fileutil.make_dirs(basedir) fileutil.make_dirs(os.path.join(basedir, "tmp")) - bw = BucketWriter(self, incoming, final, size, self.make_lease()) + bw = BucketWriter(self, incoming, final, size, self.make_lease(), Clock()) rb = RemoteBucket(bw) return bw, rb, final From 5e341ad43a85444ffc3c12c685463171a53838ff Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 17 Nov 2021 11:29:34 -0500 Subject: [PATCH 0437/2309] New tests to write. --- src/allmydata/test/test_storage.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index 36c776fba..93779bb29 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -285,6 +285,22 @@ class Bucket(unittest.TestCase): result_of_read = br.remote_read(0, len(share_data)+1) self.failUnlessEqual(result_of_read, share_data) + def test_bucket_expires_if_no_writes_for_30_minutes(self): + pass + + def test_bucket_writes_delay_timeout(self): + pass + + def test_bucket_finishing_writiing_cancels_timeout(self): + pass + + def test_bucket_closing_cancels_timeout(self): + pass + + def test_bucket_aborting_cancels_timeout(self): + pass + + class RemoteBucket(object): def __init__(self, target): @@ -559,7 +575,6 @@ class Server(unittest.TestCase): writer.remote_abort() self.failUnlessEqual(ss.allocated_size(), 0) - def test_allocate(self): ss = self.create("test_allocate") From 92e8d78a3db530f76c11188a9dc8a543d03b6fbb Mon Sep 17 00:00:00 2001 From: fenn-cs Date: Thu, 18 Nov 2021 11:46:01 +0100 Subject: [PATCH 0438/2309] stronger language for adding contributors & make commands stand out Signed-off-by: fenn-cs --- README.rst | 5 +++-- docs/release-checklist.rst | 18 ++++++++++++------ 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/README.rst b/README.rst index 20748a8db..0b73b520e 100644 --- a/README.rst +++ b/README.rst @@ -95,12 +95,13 @@ As a community-driven open source project, Tahoe-LAFS welcomes contributions of - `Patch reviews `__ -Before authoring or reviewing a patch, please familiarize yourself with the `Coding Standard `__ and the `Contributor Code of Conduct `__. +Before authoring or reviewing a patch, please familiarize yourself with the `Coding Standard `__ and the `Contributor Code of Conduct `__. + 🥳 First Contribution? ---------------------- -If you are committing to Tahoe for the very first time, consider adding your name to our contributor list in `CREDITS `__ +If you are committing to Tahoe for the very first time, it's required that you add your name to our contributor list in `CREDITS `__. Please ensure that this addition has it's own commit within your first contribution. 🤝 Supporters diff --git a/docs/release-checklist.rst b/docs/release-checklist.rst index 403a6f933..3f075dd34 100644 --- a/docs/release-checklist.rst +++ b/docs/release-checklist.rst @@ -17,7 +17,7 @@ This checklist is based on the original instructions (in old revisions in the fi Any Contributor -``````````````` +--------------- Anyone who can create normal PRs should be able to complete this portion of the release process. @@ -50,7 +50,11 @@ previously generated files. practice to give it the release name. You MAY also discard this directory once the release process is complete.* -- ``cd`` into the release directory and install dependencies by running ``python -m venv venv && source venv/bin/activate && pip install --editable .[test]`` +Get into the release directory and install dependencies by running + +- ``cd ../tahoe-release-x.x.x`` (assuming you are still in your original clone) +- ``python -m venv venv`` +- ``./venv/bin/pip install --editable .[test]`` Create Branch and Apply Updates @@ -137,16 +141,18 @@ they will need to evaluate which contributors' signatures they trust. - build all code locally - these should all pass: - - tox -e py27,codechecks,docs,integration + - ``tox -e py27,codechecks,docs,integration`` - these can fail (ideally they should not of course): - - tox -e deprecations,upcoming-deprecations + - ``tox -e deprecations,upcoming-deprecations`` - build tarballs - tox -e tarballs - - Confirm that release tarballs exist by runnig: ``ls dist/ | grep 1.16.0rc0`` + - Confirm that release tarballs exist by runnig: + + - ``ls dist/ | grep 1.16.0rc0`` - inspect and test the tarballs @@ -160,7 +166,7 @@ they will need to evaluate which contributors' signatures they trust. Privileged Contributor -`````````````````````` +----------------------- Steps in this portion require special access to keys or infrastructure. For example, **access to tahoe-lafs.org** to upload From 767948759d2a1a88acf61f0f5a843c7183c46778 Mon Sep 17 00:00:00 2001 From: fenn-cs Date: Thu, 18 Nov 2021 11:48:32 +0100 Subject: [PATCH 0439/2309] correct indent Signed-off-by: fenn-cs --- docs/release-checklist.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/release-checklist.rst b/docs/release-checklist.rst index 3f075dd34..0ba94df3a 100644 --- a/docs/release-checklist.rst +++ b/docs/release-checklist.rst @@ -139,6 +139,7 @@ they will need to evaluate which contributors' signatures they trust. *Don't forget to put the correct tag message and name in this example the tag message is "release Tahoe-LAFS-1.16.0rc0" and the tag name is `tahoe-lafs-1.16.0rc0`* - build all code locally + - these should all pass: - ``tox -e py27,codechecks,docs,integration`` From e9ae3aa885227f4ba8fa0f33ede779d1c698ae13 Mon Sep 17 00:00:00 2001 From: fenn-cs Date: Thu, 18 Nov 2021 12:04:56 +0100 Subject: [PATCH 0440/2309] move gpg signing instructions to seperate file Signed-off-by: fenn-cs --- docs/gpg-setup.rst | 18 ++++++++++++++++++ docs/release-checklist.rst | 22 +--------------------- 2 files changed, 19 insertions(+), 21 deletions(-) create mode 100644 docs/gpg-setup.rst diff --git a/docs/gpg-setup.rst b/docs/gpg-setup.rst new file mode 100644 index 000000000..cb8cbfd20 --- /dev/null +++ b/docs/gpg-setup.rst @@ -0,0 +1,18 @@ +Preparing to Authenticate Release (Setting up GPG) +-------------------------------------------------- + +In other to keep releases authentic it's required that releases are signed before being +published. This ensure's that users of Tahoe are able to verify that the version of Tahoe +they are using is coming from a trusted or at the very least known source. + +The authentication is done using the ``GPG`` implementation of ``OpenGPG`` to be able to complete +the release steps you would have to download the ``GPG`` software and setup a key(identity). + +- `Download `__ and install GPG for your operating system. +- Generate a key pair using ``gpg --gen-key``. *Some questions would be asked to personalize your key configuration.* + +You might take additional steps including: + +- Setting up a revocation certificate (Incase you lose your secret key) +- Backing up your key pair +- Upload your fingerprint to a keyserver such as `openpgp.org `__ diff --git a/docs/release-checklist.rst b/docs/release-checklist.rst index 0ba94df3a..3b313da61 100644 --- a/docs/release-checklist.rst +++ b/docs/release-checklist.rst @@ -34,6 +34,7 @@ Tuesday if you want to get anything in"). - Create a ticket for the release in Trac - Ticket number needed in next section +- Making first release? See `GPG Setup Instructions `__ to make sure you can sign releases. [One time setup] Get a clean checkout ```````````````````` @@ -96,27 +97,6 @@ Create Branch and Apply Updates - Confirm CI runs successfully on all platforms -Preparing to Authenticate Release (Setting up GPG) -`````````````````````````````````````````````````` -*Skip the section if you already have GPG setup.* - -In other to keep releases authentic it's required that releases are signed before being -published. This ensure's that users of Tahoe are able to verify that the version of Tahoe -they are using is coming from a trusted or at the very least known source. - -The authentication is done using the ``GPG`` implementation of ``OpenGPG`` to be able to complete -the release steps you would have to download the ``GPG`` software and setup a key(identity). - -- `Download `__ and install GPG for your operating system. -- Generate a key pair using ``gpg --gen-key``. *Some questions would be asked to personalize your key configuration.* - -You might take additional steps including: - -- Setting up a revocation certificate (Incase you lose your secret key) -- Backing up your key pair -- Upload your fingerprint to a keyserver such as `openpgp.org `__ - - Create Release Candidate ```````````````````````` From 8c8e377466bcf2659029f7d59636d5039e12abf7 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 18 Nov 2021 14:35:04 -0500 Subject: [PATCH 0441/2309] Implement timeout and corresponding tests. --- src/allmydata/storage/immutable.py | 28 +++++++++++++-- src/allmydata/test/test_storage.py | 58 ++++++++++++++++++++++++++---- 2 files changed, 76 insertions(+), 10 deletions(-) diff --git a/src/allmydata/storage/immutable.py b/src/allmydata/storage/immutable.py index 7cfb7a1bf..8a7519b7b 100644 --- a/src/allmydata/storage/immutable.py +++ b/src/allmydata/storage/immutable.py @@ -246,11 +246,17 @@ class BucketWriter(Referenceable): # type: ignore # warner/foolscap#78 self._sharefile.add_lease(lease_info) self._already_written = RangeMap() self._clock = clock + self._timeout = clock.callLater(30 * 60, self._abort_due_to_timeout) def allocated_size(self): return self._max_size def remote_write(self, offset, data): + self.write(offset, data) + + def write(self, offset, data): + # Delay the timeout, since we received data: + self._timeout.reset(30 * 60) start = self._clock.seconds() precondition(not self.closed) if self.throw_out_all_data: @@ -273,7 +279,11 @@ class BucketWriter(Referenceable): # type: ignore # warner/foolscap#78 self.ss.count("write") def remote_close(self): + self.close() + + def close(self): precondition(not self.closed) + self._timeout.cancel() start = self._clock.seconds() fileutil.make_dirs(os.path.dirname(self.finalhome)) @@ -312,15 +322,23 @@ class BucketWriter(Referenceable): # type: ignore # warner/foolscap#78 def disconnected(self): if not self.closed: - self._abort() + self.abort() + + def _abort_due_to_timeout(self): + """ + Called if we run out of time. + """ + log.msg("storage: aborting sharefile %s due to timeout" % self.incominghome, + facility="tahoe.storage", level=log.UNUSUAL) + self.abort() def remote_abort(self): log.msg("storage: aborting sharefile %s" % self.incominghome, facility="tahoe.storage", level=log.UNUSUAL) - self._abort() + self.abort() self.ss.count("abort") - def _abort(self): + def abort(self): if self.closed: return @@ -338,6 +356,10 @@ class BucketWriter(Referenceable): # type: ignore # warner/foolscap#78 self.closed = True self.ss.bucket_writer_closed(self, 0) + # Cancel timeout if it wasn't already cancelled. + if self._timeout.active(): + self._timeout.cancel() + @implementer(RIBucketReader) class BucketReader(Referenceable): # type: ignore # warner/foolscap#78 diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index 93779bb29..18dca9856 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -285,20 +285,64 @@ class Bucket(unittest.TestCase): result_of_read = br.remote_read(0, len(share_data)+1) self.failUnlessEqual(result_of_read, share_data) + def _assert_timeout_only_after_30_minutes(self, clock, bw): + """ + The ``BucketWriter`` times out and is closed after 30 minutes, but not + sooner. + """ + self.assertFalse(bw.closed) + # 29 minutes pass. Everything is fine. + for i in range(29): + clock.advance(60) + self.assertFalse(bw.closed, "Bucket closed after only %d minutes" % (i + 1,)) + # After the 30th minute, the bucket is closed due to lack of writes. + clock.advance(60) + self.assertTrue(bw.closed) + def test_bucket_expires_if_no_writes_for_30_minutes(self): - pass + """ + If a ``BucketWriter`` receives no writes for 30 minutes, it is removed. + """ + incoming, final = self.make_workdir("test_bucket_expires") + clock = Clock() + bw = BucketWriter(self, incoming, final, 200, self.make_lease(), clock) + self._assert_timeout_only_after_30_minutes(clock, bw) def test_bucket_writes_delay_timeout(self): - pass - - def test_bucket_finishing_writiing_cancels_timeout(self): - pass + """ + So long as the ``BucketWriter`` receives writes, the the removal + timeout is put off. + """ + incoming, final = self.make_workdir("test_bucket_writes_delay_timeout") + clock = Clock() + bw = BucketWriter(self, incoming, final, 200, self.make_lease(), clock) + # 20 minutes pass, getting close to the timeout... + clock.advance(29 * 60) + # .. but we receive a write! So that should delay the timeout. + bw.write(0, b"hello") + self._assert_timeout_only_after_30_minutes(clock, bw) def test_bucket_closing_cancels_timeout(self): - pass + """ + Closing cancels the ``BucketWriter`` timeout. + """ + incoming, final = self.make_workdir("test_bucket_close_timeout") + clock = Clock() + bw = BucketWriter(self, incoming, final, 10, self.make_lease(), clock) + self.assertTrue(clock.getDelayedCalls()) + bw.close() + self.assertFalse(clock.getDelayedCalls()) def test_bucket_aborting_cancels_timeout(self): - pass + """ + Closing cancels the ``BucketWriter`` timeout. + """ + incoming, final = self.make_workdir("test_bucket_abort_timeout") + clock = Clock() + bw = BucketWriter(self, incoming, final, 10, self.make_lease(), clock) + self.assertTrue(clock.getDelayedCalls()) + bw.abort() + self.assertFalse(clock.getDelayedCalls()) class RemoteBucket(object): From 1827faf36b60751811302d1d20ba87348b7e32c4 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 18 Nov 2021 14:45:44 -0500 Subject: [PATCH 0442/2309] Fix issue with leaked-past-end-of-test DelayedCalls. --- src/allmydata/test/test_storage.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index 18dca9856..977ed768f 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -498,7 +498,9 @@ class Server(unittest.TestCase): basedir = os.path.join("storage", "Server", name) return basedir - def create(self, name, reserved_space=0, klass=StorageServer, clock=reactor): + def create(self, name, reserved_space=0, klass=StorageServer, clock=None): + if clock is None: + clock = Clock() workdir = self.workdir(name) ss = klass(workdir, b"\x00" * 20, reserved_space=reserved_space, stats_provider=FakeStatsProvider(), @@ -1091,8 +1093,10 @@ class MutableServer(unittest.TestCase): basedir = os.path.join("storage", "MutableServer", name) return basedir - def create(self, name, clock=reactor): + def create(self, name, clock=None): workdir = self.workdir(name) + if clock is None: + clock = Clock() ss = StorageServer(workdir, b"\x00" * 20, clock=clock) ss.setServiceParent(self.sparent) From 5d915afe1c00d5832fec59d9a1599482d66a9e85 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 18 Nov 2021 15:42:54 -0500 Subject: [PATCH 0443/2309] Clean up BucketWriters on shutdown (also preventing DelayedCalls leaks in tests). --- src/allmydata/storage/server.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/allmydata/storage/server.py b/src/allmydata/storage/server.py index 080c1aea1..6e3d6f683 100644 --- a/src/allmydata/storage/server.py +++ b/src/allmydata/storage/server.py @@ -136,6 +136,12 @@ class StorageServer(service.MultiService, Referenceable): # Canaries and disconnect markers for BucketWriters created via Foolscap: self._bucket_writer_disconnect_markers = {} # type: Dict[BucketWriter,(IRemoteReference, object)] + def stopService(self): + # Cancel any in-progress uploads: + for bw in list(self._bucket_writers.values()): + bw.disconnected() + return service.MultiService.stopService(self) + def __repr__(self): return "" % (idlib.shortnodeid_b2a(self.my_nodeid),) From bd645edd9e68efef5c21b5864957b6e2858acc12 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 18 Nov 2021 15:44:51 -0500 Subject: [PATCH 0444/2309] Fix flake. --- src/allmydata/test/test_storage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index 977ed768f..92de63f0d 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -23,7 +23,7 @@ from uuid import uuid4 from twisted.trial import unittest -from twisted.internet import defer, reactor +from twisted.internet import defer from twisted.internet.task import Clock from hypothesis import given, strategies From e2636466b584fff59dcd513866e254443417e771 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 18 Nov 2021 15:47:25 -0500 Subject: [PATCH 0445/2309] Fix a flake. --- src/allmydata/storage/server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/storage/server.py b/src/allmydata/storage/server.py index 6e3d6f683..7dc277e39 100644 --- a/src/allmydata/storage/server.py +++ b/src/allmydata/storage/server.py @@ -14,7 +14,7 @@ if PY2: else: from typing import Dict -import os, re, time +import os, re import six from foolscap.api import Referenceable From 4c111773876bb97192dc2604e6a32e14f7898c16 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 18 Nov 2021 15:58:55 -0500 Subject: [PATCH 0446/2309] Fix a problem with typechecking. Using remote_write() isn't quite right given move to HTTP, but can fight that battle another day. --- src/allmydata/storage/immutable.py | 3 --- src/allmydata/test/test_storage.py | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/src/allmydata/storage/immutable.py b/src/allmydata/storage/immutable.py index 8a7519b7b..08b83cd87 100644 --- a/src/allmydata/storage/immutable.py +++ b/src/allmydata/storage/immutable.py @@ -252,9 +252,6 @@ class BucketWriter(Referenceable): # type: ignore # warner/foolscap#78 return self._max_size def remote_write(self, offset, data): - self.write(offset, data) - - def write(self, offset, data): # Delay the timeout, since we received data: self._timeout.reset(30 * 60) start = self._clock.seconds() diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index 92de63f0d..7fbe8f87b 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -319,7 +319,7 @@ class Bucket(unittest.TestCase): # 20 minutes pass, getting close to the timeout... clock.advance(29 * 60) # .. but we receive a write! So that should delay the timeout. - bw.write(0, b"hello") + bw.remote_write(0, b"hello") self._assert_timeout_only_after_30_minutes(clock, bw) def test_bucket_closing_cancels_timeout(self): From eeb1d90e7a83cb85e685c73bfd7a960140a16ff4 Mon Sep 17 00:00:00 2001 From: fenn-cs Date: Sat, 20 Nov 2021 18:26:02 +0100 Subject: [PATCH 0447/2309] Leveled headings and rst semantics for sidenotes Signed-off-by: fenn-cs --- docs/release-checklist.rst | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/docs/release-checklist.rst b/docs/release-checklist.rst index 3b313da61..796be75ba 100644 --- a/docs/release-checklist.rst +++ b/docs/release-checklist.rst @@ -17,7 +17,7 @@ This checklist is based on the original instructions (in old revisions in the fi Any Contributor ---------------- +=============== Anyone who can create normal PRs should be able to complete this portion of the release process. @@ -46,10 +46,10 @@ previously generated files. - Inside the tahoe root dir run ``git clone . ../tahoe-release-x.x.x`` where (x.x.x is the release number such as 1.16.0). -*The above command would create a new directory at the same level as your original clone named -``tahoe-release-x.x.x``. You could name the folder however you want but it would be a good -practice to give it the release name. You MAY also discard this directory once the release -process is complete.* +.. note:: + The above command would create a new directory at the same level as your original clone named ``tahoe-release-x.x.x``. You can name this folder however you want but it would be a good + practice to give it the release name. You MAY also discard this directory once the release + process is complete. Get into the release directory and install dependencies by running @@ -115,8 +115,9 @@ they will need to evaluate which contributors' signatures they trust. - ``git tag -s -u 0xE34E62D06D0E69CFCA4179FFBDE0D31D68666A7A -m "release Tahoe-LAFS-1.16.0rc0" tahoe-lafs-1.16.0rc0`` -*Replace the key-id above with your own, which can simply be your email if's attached your fingerprint.* -*Don't forget to put the correct tag message and name in this example the tag message is "release Tahoe-LAFS-1.16.0rc0" and the tag name is `tahoe-lafs-1.16.0rc0`* +.. note:: + - Replace the key-id above with your own, which can simply be your email if's attached your fingerprint. + - Don't forget to put the correct tag message and name in this example the tag message is "release Tahoe-LAFS-1.16.0rc0" and the tag name is `tahoe-lafs-1.16.0rc0` - build all code locally @@ -147,7 +148,7 @@ they will need to evaluate which contributors' signatures they trust. Privileged Contributor ------------------------ +====================== Steps in this portion require special access to keys or infrastructure. For example, **access to tahoe-lafs.org** to upload From 04e45f065ab496acf8aadb9f4cca0f60f55c41a2 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 22 Nov 2021 07:56:51 -0500 Subject: [PATCH 0448/2309] document `compare_leases_without_timestamps` --- src/allmydata/test/test_storage.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index 92176ce52..91c7adb7f 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -1361,6 +1361,10 @@ class MutableServer(unittest.TestCase): 2: [b"2"*10]}) def compare_leases_without_timestamps(self, leases_a, leases_b): + """ + Assert that, except for expiration times, ``leases_a`` contains the same + lease information as ``leases_b``. + """ for a, b in zip(leases_a, leases_b): # The leases aren't always of the same type (though of course # corresponding elements in the two lists should be of the same From b92343c664c15605df4a4244208d614f2b3390b2 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 22 Nov 2021 08:36:12 -0500 Subject: [PATCH 0449/2309] some more docstrings --- src/allmydata/storage/lease_schema.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/allmydata/storage/lease_schema.py b/src/allmydata/storage/lease_schema.py index 697ac9e34..c09a9279b 100644 --- a/src/allmydata/storage/lease_schema.py +++ b/src/allmydata/storage/lease_schema.py @@ -28,11 +28,17 @@ from .lease import ( @attr.s(frozen=True) class CleartextLeaseSerializer(object): + """ + Serialize and unserialize leases with cleartext secrets. + """ _to_data = attr.ib() _from_data = attr.ib() def serialize(self, lease): # type: (LeaseInfo) -> bytes + """ + Represent the given lease as bytes with cleartext secrets. + """ if isinstance(lease, LeaseInfo): return self._to_data(lease) raise ValueError( @@ -42,6 +48,9 @@ class CleartextLeaseSerializer(object): ) def unserialize(self, data): + """ + Load a lease with cleartext secrets from the given bytes representation. + """ # type: (bytes) -> LeaseInfo # In v1 of the immutable schema lease secrets are stored plaintext. # So load the data into a plain LeaseInfo which works on plaintext From d1839187f148f0e8f265123149c5fc2bc3c9d143 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 22 Nov 2021 08:45:10 -0500 Subject: [PATCH 0450/2309] "misplaced type annotation" --- src/allmydata/storage/lease_schema.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/storage/lease_schema.py b/src/allmydata/storage/lease_schema.py index c09a9279b..7e604388e 100644 --- a/src/allmydata/storage/lease_schema.py +++ b/src/allmydata/storage/lease_schema.py @@ -48,10 +48,10 @@ class CleartextLeaseSerializer(object): ) def unserialize(self, data): + # type: (bytes) -> LeaseInfo """ Load a lease with cleartext secrets from the given bytes representation. """ - # type: (bytes) -> LeaseInfo # In v1 of the immutable schema lease secrets are stored plaintext. # So load the data into a plain LeaseInfo which works on plaintext # secrets. From c341a86abdd4aa2b4244d0adeff53d5893be9a03 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 23 Nov 2021 10:01:03 -0500 Subject: [PATCH 0451/2309] Correct the comment. --- src/allmydata/test/test_storage.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index 7fbe8f87b..bc87e168d 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -316,9 +316,10 @@ class Bucket(unittest.TestCase): incoming, final = self.make_workdir("test_bucket_writes_delay_timeout") clock = Clock() bw = BucketWriter(self, incoming, final, 200, self.make_lease(), clock) - # 20 minutes pass, getting close to the timeout... + # 29 minutes pass, getting close to the timeout... clock.advance(29 * 60) - # .. but we receive a write! So that should delay the timeout. + # .. but we receive a write! So that should delay the timeout again to + # another 30 minutes. bw.remote_write(0, b"hello") self._assert_timeout_only_after_30_minutes(clock, bw) From 6c514dfda57bfc2ede45719db24acecfbfee3ed1 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 23 Nov 2021 10:33:45 -0500 Subject: [PATCH 0452/2309] Add klein. --- nix/klein.nix | 18 ++++++++++++++++++ nix/overlays.nix | 3 +++ 2 files changed, 21 insertions(+) create mode 100644 nix/klein.nix diff --git a/nix/klein.nix b/nix/klein.nix new file mode 100644 index 000000000..aa109e3d1 --- /dev/null +++ b/nix/klein.nix @@ -0,0 +1,18 @@ +{ lib, buildPythonPackage, fetchPypi }: +buildPythonPackage rec { + pname = "klein"; + version = "21.8.0"; + + src = fetchPypi { + sha256 = "09i1x5ppan3kqsgclbz8xdnlvzvp3amijbmdzv0kik8p5l5zswxa"; + inherit pname version; + }; + + doCheck = false; + + meta = with lib; { + homepage = https://github.com/twisted/klein; + description = "Nicer web server for Twisted"; + license = licenses.mit; + }; +} diff --git a/nix/overlays.nix b/nix/overlays.nix index fbd0ce3bb..011d8dd6b 100644 --- a/nix/overlays.nix +++ b/nix/overlays.nix @@ -28,6 +28,9 @@ self: super: { packageOverrides = python-self: python-super: { # collections-extended is not part of nixpkgs at this time. collections-extended = python-super.pythonPackages.callPackage ./collections-extended.nix { }; + + # klein is not in nixpkgs 21.05, at least: + klein = python-super.pythonPackages.callPackage ./klein.nix { }; }; }; } From c921b153f4990e98a32dde1286d3a9c11d5fd2e4 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 23 Nov 2021 10:39:15 -0500 Subject: [PATCH 0453/2309] A better name for the API. --- src/allmydata/storage/http_server.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 3baa336fa..327892ecd 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -47,7 +47,7 @@ def _authorization_decorator(f): return route -def _route(app, *route_args, **route_kwargs): +def _authorized_route(app, *route_args, **route_kwargs): """ Like Klein's @route, but with additional support for checking the ``Authorization`` header as well as ``X-Tahoe-Authorization`` headers. The @@ -89,6 +89,6 @@ class HTTPServer(object): # TODO if data is big, maybe want to use a temporary file eventually... return dumps(data) - @_route(_app, "/v1/version", methods=["GET"]) + @_authorized_route(_app, "/v1/version", methods=["GET"]) def version(self, request, authorization): return self._cbor(request, self._storage_server.remote_get_version()) From a593095dc935b6719e266e0bf3a996b39047d9c0 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 23 Nov 2021 10:39:53 -0500 Subject: [PATCH 0454/2309] Explain why it's a conditional import. --- src/allmydata/storage/http_client.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index e1743343d..f8a7590aa 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -14,6 +14,8 @@ if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 # fmt: on else: + # typing module not available in Python 2, and we only do type checking in + # Python 3 anyway. from typing import Union from treq.testing import StubTreq From 8abc1ad8f4e43a244d0bcead201a133f3cf8b0c1 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 23 Nov 2021 10:44:45 -0500 Subject: [PATCH 0455/2309] cbor2 for Python 2 on Nix. --- nix/cbor2.nix | 18 ++++++++++++++++++ nix/overlays.nix | 3 +++ 2 files changed, 21 insertions(+) create mode 100644 nix/cbor2.nix diff --git a/nix/cbor2.nix b/nix/cbor2.nix new file mode 100644 index 000000000..02c810e1e --- /dev/null +++ b/nix/cbor2.nix @@ -0,0 +1,18 @@ +{ lib, buildPythonPackage, fetchPypi }: +buildPythonPackage rec { + pname = "cbor2"; + version = "5.2.0"; + + src = fetchPypi { + sha256 = "1mmmncfbsx7cbdalcrsagp9hx7wqfawaz9361gjkmsk3lp6chd5w"; + inherit pname version; + }; + + doCheck = false; + + meta = with lib; { + homepage = https://github.com/agronholm/cbor2; + description = "CBOR encoder/decoder"; + license = licenses.mit; + }; +} diff --git a/nix/overlays.nix b/nix/overlays.nix index 011d8dd6b..5cfab200c 100644 --- a/nix/overlays.nix +++ b/nix/overlays.nix @@ -21,6 +21,9 @@ self: super: { # collections-extended is not part of nixpkgs at this time. collections-extended = python-super.pythonPackages.callPackage ./collections-extended.nix { }; + + # cbor2 is not part of nixpkgs at this time. + cbor2 = python-super.pythonPackages.callPackage ./cbor2.nix { }; }; }; From 30511ea8502dc04848f9ed3715b5517c51444c96 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 23 Nov 2021 11:39:51 -0500 Subject: [PATCH 0456/2309] Add more build inputs. --- nix/tahoe-lafs.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/nix/tahoe-lafs.nix b/nix/tahoe-lafs.nix index df12f21d4..59864d36d 100644 --- a/nix/tahoe-lafs.nix +++ b/nix/tahoe-lafs.nix @@ -98,6 +98,7 @@ EOF service-identity pyyaml magic-wormhole eliot autobahn cryptography netifaces setuptools future pyutil distro configparser collections-extended + klein cbor2 treq ]; checkInputs = with python.pkgs; [ From 5855a30e34e1f18d44cbd898dee1b128be2cd976 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 23 Nov 2021 14:01:43 -0700 Subject: [PATCH 0457/2309] add docstrings --- src/allmydata/storage/crawler.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/allmydata/storage/crawler.py b/src/allmydata/storage/crawler.py index 129659d27..dcbea909a 100644 --- a/src/allmydata/storage/crawler.py +++ b/src/allmydata/storage/crawler.py @@ -146,10 +146,17 @@ class _LeaseStateSerializer(object): ) def load(self): + """ + :returns: deserialized JSON state + """ with self._path.open("rb") as f: return json.load(f) def save(self, data): + """ + Serialize the given data as JSON into the state-path + :returns: None + """ tmpfile = self._path.siblingExtension(".tmp") with tmpfile.open("wb") as f: json.dump(data, f) From 54c032d0d7f467ff3861e5dcefa4c0024b415b34 Mon Sep 17 00:00:00 2001 From: fenn-cs Date: Sat, 27 Nov 2021 00:59:13 +0100 Subject: [PATCH 0458/2309] change assertTrue -> assertEquals for non bools Signed-off-by: fenn-cs --- src/allmydata/test/mutable/test_problems.py | 8 ++++---- src/allmydata/test/mutable/test_repair.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/allmydata/test/mutable/test_problems.py b/src/allmydata/test/mutable/test_problems.py index 9abee560d..40105142a 100644 --- a/src/allmydata/test/mutable/test_problems.py +++ b/src/allmydata/test/mutable/test_problems.py @@ -241,7 +241,7 @@ class Problems(GridTestMixin, AsyncTestCase, testutil.ShouldFailMixin): # that ought to work def _got_node(n): d = n.download_best_version() - d.addCallback(lambda res: self.assertTrue(res, b"contents 1")) + d.addCallback(lambda res: self.assertEquals(res, b"contents 1")) # now break the second peer def _break_peer1(res): self.g.break_server(self.server1.get_serverid()) @@ -249,7 +249,7 @@ class Problems(GridTestMixin, AsyncTestCase, testutil.ShouldFailMixin): d.addCallback(lambda res: n.overwrite(MutableData(b"contents 2"))) # that ought to work too d.addCallback(lambda res: n.download_best_version()) - d.addCallback(lambda res: self.assertTrue(res, b"contents 2")) + d.addCallback(lambda res: self.assertEquals(res, b"contents 2")) def _explain_error(f): print(f) if f.check(NotEnoughServersError): @@ -281,7 +281,7 @@ class Problems(GridTestMixin, AsyncTestCase, testutil.ShouldFailMixin): d = nm.create_mutable_file(MutableData(b"contents 1")) def _created(n): d = n.download_best_version() - d.addCallback(lambda res: self.assertTrue(res, b"contents 1")) + d.addCallback(lambda res: self.assertEquals(res, b"contents 1")) # now break one of the remaining servers def _break_second_server(res): self.g.break_server(peerids[1]) @@ -289,7 +289,7 @@ class Problems(GridTestMixin, AsyncTestCase, testutil.ShouldFailMixin): d.addCallback(lambda res: n.overwrite(MutableData(b"contents 2"))) # that ought to work too d.addCallback(lambda res: n.download_best_version()) - d.addCallback(lambda res: self.assertTrue(res, b"contents 2")) + d.addCallback(lambda res: self.assertEquals(res, b"contents 2")) return d d.addCallback(_created) return d diff --git a/src/allmydata/test/mutable/test_repair.py b/src/allmydata/test/mutable/test_repair.py index 987b21cc3..deddb8d92 100644 --- a/src/allmydata/test/mutable/test_repair.py +++ b/src/allmydata/test/mutable/test_repair.py @@ -229,7 +229,7 @@ class Repair(AsyncTestCase, PublishMixin, ShouldFailMixin): new_versionid = smap.best_recoverable_version() self.assertThat(new_versionid[0], Equals(5)) # seqnum 5 d2 = self._fn.download_version(smap, new_versionid) - d2.addCallback(self.assertTrue, expected_contents) + d2.addCallback(self.assertEquals, expected_contents) return d2 d.addCallback(_check_smap) return d From c02d8cab3ab7ebee5cc57a07dde5d4b52393eb03 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 29 Nov 2021 08:56:05 -0500 Subject: [PATCH 0459/2309] change one more assertTrue to assertEquals --- src/allmydata/test/mutable/test_problems.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/mutable/test_problems.py b/src/allmydata/test/mutable/test_problems.py index 40105142a..4bcb8161b 100644 --- a/src/allmydata/test/mutable/test_problems.py +++ b/src/allmydata/test/mutable/test_problems.py @@ -420,7 +420,7 @@ class Problems(GridTestMixin, AsyncTestCase, testutil.ShouldFailMixin): return self._node.download_version(servermap, ver) d.addCallback(_then) d.addCallback(lambda data: - self.assertTrue(data, CONTENTS)) + self.assertEquals(data, CONTENTS)) return d def test_1654(self): From 5fef83078d863843ef1f5f8a35990bfc3fcdb338 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 29 Nov 2021 13:08:11 -0500 Subject: [PATCH 0460/2309] news fragment --- newsfragments/3847.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3847.minor diff --git a/newsfragments/3847.minor b/newsfragments/3847.minor new file mode 100644 index 000000000..e69de29bb From 66a0c6f3f43ecd018cdbb571ebd6eab740b6cca7 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 29 Nov 2021 13:43:06 -0500 Subject: [PATCH 0461/2309] add a direct test for the non-utf-8 bytestring behavior --- src/allmydata/test/test_eliotutil.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/allmydata/test/test_eliotutil.py b/src/allmydata/test/test_eliotutil.py index 3f915ecd2..00110530c 100644 --- a/src/allmydata/test/test_eliotutil.py +++ b/src/allmydata/test/test_eliotutil.py @@ -78,6 +78,9 @@ from .common import ( class EliotLoggedTestTests(AsyncTestCase): + """ + Tests for the automatic log-related provided by ``EliotLoggedRunTest``. + """ def test_returns_none(self): Message.log(hello="world") @@ -95,6 +98,12 @@ class EliotLoggedTestTests(AsyncTestCase): # We didn't start an action. We're not finishing an action. return d.result + def test_logs_non_utf_8_byte(self): + """ + If an Eliot message is emitted that contains a non-UTF-8 byte string then + the test nevertheless passes. + """ + Message.log(hello=b"\xFF") class ParseDestinationDescriptionTests(SyncTestCase): From f40da7dc27d089e3bdfdca48e51aec25aea282c0 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 29 Nov 2021 13:23:59 -0500 Subject: [PATCH 0462/2309] Put the choice of JSON encoder for Eliot into its own module and use it in a few places --- src/allmydata/test/__init__.py | 4 ++-- src/allmydata/test/test_eliotutil.py | 4 ++-- src/allmydata/util/_eliot_updates.py | 28 ++++++++++++++++++++++++++++ src/allmydata/util/eliotutil.py | 7 ++++--- 4 files changed, 36 insertions(+), 7 deletions(-) create mode 100644 src/allmydata/util/_eliot_updates.py diff --git a/src/allmydata/test/__init__.py b/src/allmydata/test/__init__.py index 893aa15ce..ad245ca77 100644 --- a/src/allmydata/test/__init__.py +++ b/src/allmydata/test/__init__.py @@ -125,5 +125,5 @@ if sys.platform == "win32": initialize() from eliot import to_file -from allmydata.util.jsonbytes import AnyBytesJSONEncoder -to_file(open("eliot.log", "wb"), encoder=AnyBytesJSONEncoder) +from allmydata.util.eliotutil import eliot_json_encoder +to_file(open("eliot.log", "wb"), encoder=eliot_json_encoder) diff --git a/src/allmydata/test/test_eliotutil.py b/src/allmydata/test/test_eliotutil.py index 00110530c..0be02b277 100644 --- a/src/allmydata/test/test_eliotutil.py +++ b/src/allmydata/test/test_eliotutil.py @@ -65,11 +65,11 @@ from twisted.internet.task import deferLater from twisted.internet import reactor from ..util.eliotutil import ( + eliot_json_encoder, log_call_deferred, _parse_destination_description, _EliotLogging, ) -from ..util.jsonbytes import AnyBytesJSONEncoder from .common import ( SyncTestCase, @@ -118,7 +118,7 @@ class ParseDestinationDescriptionTests(SyncTestCase): reactor = object() self.assertThat( _parse_destination_description("file:-")(reactor), - Equals(FileDestination(stdout, encoder=AnyBytesJSONEncoder)), + Equals(FileDestination(stdout, encoder=eliot_json_encoder)), ) diff --git a/src/allmydata/util/_eliot_updates.py b/src/allmydata/util/_eliot_updates.py new file mode 100644 index 000000000..4300f2be8 --- /dev/null +++ b/src/allmydata/util/_eliot_updates.py @@ -0,0 +1,28 @@ +""" +Bring in some Eliot updates from newer versions of Eliot than we can +depend on in Python 2. + +Every API in this module (except ``eliot_json_encoder``) should be obsolete as +soon as we depend on Eliot 1.14 or newer. + +Ported to Python 3. +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + +from .jsonbytes import AnyBytesJSONEncoder + +# There are currently a number of log messages that include non-UTF-8 bytes. +# Allow these, at least for now. Later when the whole test suite has been +# converted to our SyncTestCase or AsyncTestCase it will be easier to turn +# this off and then attribute log failures to specific codepaths so they can +# be fixed (and then not regressed later) because those instances will result +# in test failures instead of only garbage being written to the eliot log. +eliot_json_encoder = AnyBytesJSONEncoder diff --git a/src/allmydata/util/eliotutil.py b/src/allmydata/util/eliotutil.py index 4e48fbb9f..ff858531d 100644 --- a/src/allmydata/util/eliotutil.py +++ b/src/allmydata/util/eliotutil.py @@ -87,8 +87,9 @@ from twisted.internet.defer import ( ) from twisted.application.service import Service -from .jsonbytes import AnyBytesJSONEncoder - +from ._eliot_updates import ( + eliot_json_encoder, +) def validateInstanceOf(t): """ @@ -306,7 +307,7 @@ class _DestinationParser(object): rotateLength=rotate_length, maxRotatedFiles=max_rotated_files, ) - return lambda reactor: FileDestination(get_file(), AnyBytesJSONEncoder) + return lambda reactor: FileDestination(get_file(), eliot_json_encoder) _parse_destination_description = _DestinationParser().parse From 3eb1a5e7cb6bc227feda6f254b43e35b1807446d Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 29 Nov 2021 13:25:03 -0500 Subject: [PATCH 0463/2309] Add a MemoryLogger that prefers our encoder and use it instead of Eliot's --- src/allmydata/test/eliotutil.py | 16 ++----- src/allmydata/util/_eliot_updates.py | 62 +++++++++++++++++++++++++++- src/allmydata/util/eliotutil.py | 2 + 3 files changed, 67 insertions(+), 13 deletions(-) diff --git a/src/allmydata/test/eliotutil.py b/src/allmydata/test/eliotutil.py index 1685744fd..dd21f1e9d 100644 --- a/src/allmydata/test/eliotutil.py +++ b/src/allmydata/test/eliotutil.py @@ -42,7 +42,6 @@ from zope.interface import ( from eliot import ( ActionType, Field, - MemoryLogger, ILogger, ) from eliot.testing import ( @@ -54,8 +53,9 @@ from twisted.python.monkey import ( MonkeyPatcher, ) -from ..util.jsonbytes import AnyBytesJSONEncoder - +from ..util.eliotutil import ( + MemoryLogger, +) _NAME = Field.for_types( u"name", @@ -71,14 +71,6 @@ RUN_TEST = ActionType( ) -# On Python 3, we want to use our custom JSON encoder when validating messages -# can be encoded to JSON: -if PY2: - _memory_logger = MemoryLogger -else: - _memory_logger = lambda: MemoryLogger(encoder=AnyBytesJSONEncoder) - - @attr.s class EliotLoggedRunTest(object): """ @@ -170,7 +162,7 @@ def with_logging( """ @wraps(test_method) def run_with_logging(*args, **kwargs): - validating_logger = _memory_logger() + validating_logger = MemoryLogger() original = swap_logger(None) try: swap_logger(_TwoLoggers(original, validating_logger)) diff --git a/src/allmydata/util/_eliot_updates.py b/src/allmydata/util/_eliot_updates.py index 4300f2be8..4ff0caf4d 100644 --- a/src/allmydata/util/_eliot_updates.py +++ b/src/allmydata/util/_eliot_updates.py @@ -1,6 +1,7 @@ """ Bring in some Eliot updates from newer versions of Eliot than we can -depend on in Python 2. +depend on in Python 2. The implementations are copied from Eliot 1.14 and +only changed enough to add Python 2 compatibility. Every API in this module (except ``eliot_json_encoder``) should be obsolete as soon as we depend on Eliot 1.14 or newer. @@ -17,6 +18,13 @@ from future.utils import PY2 if PY2: from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 +import json as pyjson +from functools import partial + +from eliot import ( + MemoryLogger as _MemoryLogger, +) + from .jsonbytes import AnyBytesJSONEncoder # There are currently a number of log messages that include non-UTF-8 bytes. @@ -26,3 +34,55 @@ from .jsonbytes import AnyBytesJSONEncoder # be fixed (and then not regressed later) because those instances will result # in test failures instead of only garbage being written to the eliot log. eliot_json_encoder = AnyBytesJSONEncoder + +class _CustomEncoderMemoryLogger(_MemoryLogger): + """ + Override message validation from the Eliot-supplied ``MemoryLogger`` to + use our chosen JSON encoder. + + This is only necessary on Python 2 where we use an old version of Eliot + that does not parameterize the encoder. + """ + def __init__(self, encoder=eliot_json_encoder): + """ + @param encoder: A JSONEncoder subclass to use when encoding JSON. + """ + self._encoder = encoder + super(_CustomEncoderMemoryLogger, self).__init__() + + def _validate_message(self, dictionary, serializer): + """Validate an individual message. + + As a side-effect, the message is replaced with its serialized contents. + + @param dictionary: A message C{dict} to be validated. Might be mutated + by the serializer! + + @param serializer: C{None} or a serializer. + + @raises TypeError: If a field name is not unicode, or the dictionary + fails to serialize to JSON. + + @raises eliot.ValidationError: If serializer was given and validation + failed. + """ + if serializer is not None: + serializer.validate(dictionary) + for key in dictionary: + if not isinstance(key, str): + if isinstance(key, bytes): + key.decode("utf-8") + else: + raise TypeError(dictionary, "%r is not unicode" % (key,)) + if serializer is not None: + serializer.serialize(dictionary) + + try: + pyjson.dumps(dictionary, cls=self._encoder) + except Exception as e: + raise TypeError("Message %s doesn't encode to JSON: %s" % (dictionary, e)) + +if PY2: + MemoryLogger = partial(_CustomEncoderMemoryLogger, encoder=eliot_json_encoder) +else: + MemoryLogger = partial(_MemoryLogger, encoder=eliot_json_encoder) diff --git a/src/allmydata/util/eliotutil.py b/src/allmydata/util/eliotutil.py index ff858531d..5067876c5 100644 --- a/src/allmydata/util/eliotutil.py +++ b/src/allmydata/util/eliotutil.py @@ -16,6 +16,7 @@ from __future__ import ( ) __all__ = [ + "MemoryLogger", "inline_callbacks", "eliot_logging_service", "opt_eliot_destination", @@ -88,6 +89,7 @@ from twisted.internet.defer import ( from twisted.application.service import Service from ._eliot_updates import ( + MemoryLogger, eliot_json_encoder, ) From 20e0626e424276c83cd1f2eb42fdddb7cf56072e Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 29 Nov 2021 13:27:17 -0500 Subject: [PATCH 0464/2309] add capture_logging that parameterizes JSON encoder --- src/allmydata/util/_eliot_updates.py | 100 ++++++++++++++++++++++++++- src/allmydata/util/eliotutil.py | 13 +--- 2 files changed, 102 insertions(+), 11 deletions(-) diff --git a/src/allmydata/util/_eliot_updates.py b/src/allmydata/util/_eliot_updates.py index 4ff0caf4d..8e3beca45 100644 --- a/src/allmydata/util/_eliot_updates.py +++ b/src/allmydata/util/_eliot_updates.py @@ -19,12 +19,17 @@ if PY2: from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import json as pyjson -from functools import partial +from functools import wraps, partial from eliot import ( MemoryLogger as _MemoryLogger, ) +from eliot.testing import ( + check_for_errors, + swap_logger, +) + from .jsonbytes import AnyBytesJSONEncoder # There are currently a number of log messages that include non-UTF-8 bytes. @@ -86,3 +91,96 @@ if PY2: MemoryLogger = partial(_CustomEncoderMemoryLogger, encoder=eliot_json_encoder) else: MemoryLogger = partial(_MemoryLogger, encoder=eliot_json_encoder) + +def validateLogging( + assertion, *assertionArgs, **assertionKwargs +): + """ + Decorator factory for L{unittest.TestCase} methods to add logging + validation. + + 1. The decorated test method gets a C{logger} keyword argument, a + L{MemoryLogger}. + 2. All messages logged to this logger will be validated at the end of + the test. + 3. Any unflushed logged tracebacks will cause the test to fail. + + For example: + + from unittest import TestCase + from eliot.testing import assertContainsFields, validateLogging + + class MyTests(TestCase): + def assertFooLogging(self, logger): + assertContainsFields(self, logger.messages[0], {"key": 123}) + + + @param assertion: A callable that will be called with the + L{unittest.TestCase} instance, the logger and C{assertionArgs} and + C{assertionKwargs} once the actual test has run, allowing for extra + logging-related assertions on the effects of the test. Use L{None} if you + want the cleanup assertions registered but no custom assertions. + + @param assertionArgs: Additional positional arguments to pass to + C{assertion}. + + @param assertionKwargs: Additional keyword arguments to pass to + C{assertion}. + + @param encoder_: C{json.JSONEncoder} subclass to use when validating JSON. + """ + encoder_ = assertionKwargs.pop("encoder_", eliot_json_encoder) + def decorator(function): + @wraps(function) + def wrapper(self, *args, **kwargs): + skipped = False + + kwargs["logger"] = logger = MemoryLogger(encoder=encoder_) + self.addCleanup(check_for_errors, logger) + # TestCase runs cleanups in reverse order, and we want this to + # run *before* tracebacks are checked: + if assertion is not None: + self.addCleanup( + lambda: skipped + or assertion(self, logger, *assertionArgs, **assertionKwargs) + ) + try: + return function(self, *args, **kwargs) + except self.skipException: + skipped = True + raise + + return wrapper + + return decorator + +# PEP 8 variant: +validate_logging = validateLogging + +def capture_logging( + assertion, *assertionArgs, **assertionKwargs +): + """ + Capture and validate all logging that doesn't specify a L{Logger}. + + See L{validate_logging} for details on the rest of its behavior. + """ + encoder_ = assertionKwargs.pop("encoder_", eliot_json_encoder) + def decorator(function): + @validate_logging( + assertion, *assertionArgs, encoder_=encoder_, **assertionKwargs + ) + @wraps(function) + def wrapper(self, *args, **kwargs): + logger = kwargs["logger"] + previous_logger = swap_logger(logger) + + def cleanup(): + swap_logger(previous_logger) + + self.addCleanup(cleanup) + return function(self, *args, **kwargs) + + return wrapper + + return decorator diff --git a/src/allmydata/util/eliotutil.py b/src/allmydata/util/eliotutil.py index 5067876c5..789ef38ff 100644 --- a/src/allmydata/util/eliotutil.py +++ b/src/allmydata/util/eliotutil.py @@ -23,6 +23,7 @@ __all__ = [ "opt_help_eliot_destinations", "validateInstanceOf", "validateSetMembership", + "capture_logging", ] from future.utils import PY2 @@ -33,7 +34,7 @@ from six import ensure_text from sys import ( stdout, ) -from functools import wraps, partial +from functools import wraps from logging import ( INFO, Handler, @@ -67,8 +68,6 @@ from eliot.twisted import ( DeferredContext, inline_callbacks, ) -from eliot.testing import capture_logging as eliot_capture_logging - from twisted.python.usage import ( UsageError, ) @@ -91,6 +90,7 @@ from twisted.application.service import Service from ._eliot_updates import ( MemoryLogger, eliot_json_encoder, + capture_logging, ) def validateInstanceOf(t): @@ -330,10 +330,3 @@ def log_call_deferred(action_type): return DeferredContext(d).addActionFinish() return logged_f return decorate_log_call_deferred - -# On Python 3, encoding bytes to JSON doesn't work, so we have a custom JSON -# encoder we want to use when validating messages. -if PY2: - capture_logging = eliot_capture_logging -else: - capture_logging = partial(eliot_capture_logging, encoder_=AnyBytesJSONEncoder) From 7626a02bdb84013e4f45bad81c8a3f5ba4586401 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 29 Nov 2021 13:27:28 -0500 Subject: [PATCH 0465/2309] remove redundant assertion --- src/allmydata/test/test_util.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/allmydata/test/test_util.py b/src/allmydata/test/test_util.py index a03845ed6..9a0af1e06 100644 --- a/src/allmydata/test/test_util.py +++ b/src/allmydata/test/test_util.py @@ -553,11 +553,6 @@ class JSONBytes(unittest.TestCase): o, cls=jsonbytes.AnyBytesJSONEncoder)), expected, ) - self.assertEqual( - json.loads(jsonbytes.dumps(o, any_bytes=True)), - expected - ) - class FakeGetVersion(object): From b01478659ea9868164aa9f7b7368f295f2d47921 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 30 Nov 2021 13:18:18 -0500 Subject: [PATCH 0466/2309] Apparently I generated wrong hashes. --- nix/cbor2.nix | 2 +- nix/klein.nix | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/nix/cbor2.nix b/nix/cbor2.nix index 02c810e1e..1bd9920e6 100644 --- a/nix/cbor2.nix +++ b/nix/cbor2.nix @@ -4,7 +4,7 @@ buildPythonPackage rec { version = "5.2.0"; src = fetchPypi { - sha256 = "1mmmncfbsx7cbdalcrsagp9hx7wqfawaz9361gjkmsk3lp6chd5w"; + sha256 = "1gwlgjl70vlv35cgkcw3cg7b5qsmws36hs4mmh0l9msgagjs4fm3"; inherit pname version; }; diff --git a/nix/klein.nix b/nix/klein.nix index aa109e3d1..0bb025cf8 100644 --- a/nix/klein.nix +++ b/nix/klein.nix @@ -4,7 +4,7 @@ buildPythonPackage rec { version = "21.8.0"; src = fetchPypi { - sha256 = "09i1x5ppan3kqsgclbz8xdnlvzvp3amijbmdzv0kik8p5l5zswxa"; + sha256 = "1mpydmz90d0n9dwa7mr6pgj5v0kczfs05ykssrasdq368dssw7ch"; inherit pname version; }; From 1fc77504aeec738acac315d65b68b4a7e01db095 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 30 Nov 2021 13:39:42 -0500 Subject: [PATCH 0467/2309] List dependencies. --- nix/klein.nix | 2 ++ 1 file changed, 2 insertions(+) diff --git a/nix/klein.nix b/nix/klein.nix index 0bb025cf8..196f95e88 100644 --- a/nix/klein.nix +++ b/nix/klein.nix @@ -10,6 +10,8 @@ buildPythonPackage rec { doCheck = false; + propagatedBuildInputs = [ attrs hyperlink incremental Tubes Twisted typing_extensions Werkzeug zope.interface ]; + meta = with lib; { homepage = https://github.com/twisted/klein; description = "Nicer web server for Twisted"; From c65a13e63228fada255a94b99ca4e61a1e9e58dc Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 30 Nov 2021 13:47:28 -0500 Subject: [PATCH 0468/2309] Rip out klein, maybe not necessary. --- nix/klein.nix | 20 -------------------- nix/overlays.nix | 3 --- 2 files changed, 23 deletions(-) delete mode 100644 nix/klein.nix diff --git a/nix/klein.nix b/nix/klein.nix deleted file mode 100644 index 196f95e88..000000000 --- a/nix/klein.nix +++ /dev/null @@ -1,20 +0,0 @@ -{ lib, buildPythonPackage, fetchPypi }: -buildPythonPackage rec { - pname = "klein"; - version = "21.8.0"; - - src = fetchPypi { - sha256 = "1mpydmz90d0n9dwa7mr6pgj5v0kczfs05ykssrasdq368dssw7ch"; - inherit pname version; - }; - - doCheck = false; - - propagatedBuildInputs = [ attrs hyperlink incremental Tubes Twisted typing_extensions Werkzeug zope.interface ]; - - meta = with lib; { - homepage = https://github.com/twisted/klein; - description = "Nicer web server for Twisted"; - license = licenses.mit; - }; -} diff --git a/nix/overlays.nix b/nix/overlays.nix index 5cfab200c..92f36e93e 100644 --- a/nix/overlays.nix +++ b/nix/overlays.nix @@ -31,9 +31,6 @@ self: super: { packageOverrides = python-self: python-super: { # collections-extended is not part of nixpkgs at this time. collections-extended = python-super.pythonPackages.callPackage ./collections-extended.nix { }; - - # klein is not in nixpkgs 21.05, at least: - klein = python-super.pythonPackages.callPackage ./klein.nix { }; }; }; } From 2f4d1079aa3b0621e4ad5991f810a5baf32c23db Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 30 Nov 2021 13:51:36 -0500 Subject: [PATCH 0469/2309] Needs setuptools_scm --- nix/cbor2.nix | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/nix/cbor2.nix b/nix/cbor2.nix index 1bd9920e6..4d9734a8b 100644 --- a/nix/cbor2.nix +++ b/nix/cbor2.nix @@ -1,4 +1,4 @@ -{ lib, buildPythonPackage, fetchPypi }: +{ lib, buildPythonPackage, fetchPypi , setuptools_scm }: buildPythonPackage rec { pname = "cbor2"; version = "5.2.0"; @@ -10,6 +10,8 @@ buildPythonPackage rec { doCheck = false; + nativeBuildInputs = [ setuptools_scm ]; + meta = with lib; { homepage = https://github.com/agronholm/cbor2; description = "CBOR encoder/decoder"; From 136bf95bdfcf0819285f7c4ed937f4de64a99125 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 30 Nov 2021 13:58:02 -0500 Subject: [PATCH 0470/2309] Simpler way. --- nix/cbor2.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nix/cbor2.nix b/nix/cbor2.nix index 4d9734a8b..ace5e13c6 100644 --- a/nix/cbor2.nix +++ b/nix/cbor2.nix @@ -1,4 +1,4 @@ -{ lib, buildPythonPackage, fetchPypi , setuptools_scm }: +{ lib, buildPythonPackage, fetchPypi }: buildPythonPackage rec { pname = "cbor2"; version = "5.2.0"; @@ -10,7 +10,7 @@ buildPythonPackage rec { doCheck = false; - nativeBuildInputs = [ setuptools_scm ]; + buildInputs = [ setuptools_scm ]; meta = with lib; { homepage = https://github.com/agronholm/cbor2; From f2b52f368d63059ebe559109b4dbe8c4720bdd2f Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 30 Nov 2021 13:58:22 -0500 Subject: [PATCH 0471/2309] Another way. --- nix/cbor2.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nix/cbor2.nix b/nix/cbor2.nix index ace5e13c6..0544b1eb1 100644 --- a/nix/cbor2.nix +++ b/nix/cbor2.nix @@ -10,7 +10,7 @@ buildPythonPackage rec { doCheck = false; - buildInputs = [ setuptools_scm ]; + propagatedBuildInputs = [ setuptools_scm ]; meta = with lib; { homepage = https://github.com/agronholm/cbor2; From 49f24893219482a53a882c8139e3c46ffedcd48e Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 30 Nov 2021 15:59:27 -0700 Subject: [PATCH 0472/2309] explicit 'migrate pickle files' command --- src/allmydata/scripts/admin.py | 55 +++++++++++++++++++++++++++++++++- 1 file changed, 54 insertions(+), 1 deletion(-) diff --git a/src/allmydata/scripts/admin.py b/src/allmydata/scripts/admin.py index a9feed0dd..c125bc9e6 100644 --- a/src/allmydata/scripts/admin.py +++ b/src/allmydata/scripts/admin.py @@ -18,7 +18,17 @@ except ImportError: pass from twisted.python import usage -from allmydata.scripts.common import BaseOptions +from twisted.python.filepath import ( + FilePath, +) +from allmydata.scripts.common import ( + BaseOptions, + BasedirOptions, +) +from allmydata.storage import ( + crawler, + expirer, +) class GenerateKeypairOptions(BaseOptions): @@ -65,12 +75,54 @@ def derive_pubkey(options): print("public:", str(ed25519.string_from_verifying_key(public_key), "ascii"), file=out) return 0 +class MigrateCrawlerOptions(BasedirOptions): + + def getSynopsis(self): + return "Usage: tahoe [global-options] admin migrate-crawler" + + def getUsage(self, width=None): + t = BasedirOptions.getUsage(self, width) + t += ( + "The crawler data is now stored as JSON to avoid" + " potential security issues with pickle files.\n\nIf" + " you are confident the state files in the 'storage/'" + " subdirectory of your node are trustworthy, run this" + " command to upgrade them to JSON.\n\nThe files are:" + " lease_checker.history, lease_checker.state, and" + " bucket_counter.state" + ) + return t + +def migrate_crawler(options): + out = options.stdout + storage = FilePath(options['basedir']).child("storage") + + conversions = [ + (storage.child("lease_checker.state"), crawler._convert_pickle_state_to_json), + (storage.child("bucket_counter.state"), crawler._convert_pickle_state_to_json), + (storage.child("lease_checker.history"), expirer._convert_pickle_state_to_json), + ] + + for fp, converter in conversions: + existed = fp.exists() + newfp = crawler._maybe_upgrade_pickle_to_json(fp, converter) + if existed: + print("Converted '{}' to '{}'".format(fp.path, newfp.path)) + else: + if newfp.exists(): + print("Already converted: '{}'".format(newfp.path)) + else: + print("Not found: '{}'".format(fp.path)) + + class AdminCommand(BaseOptions): subCommands = [ ("generate-keypair", None, GenerateKeypairOptions, "Generate a public/private keypair, write to stdout."), ("derive-pubkey", None, DerivePubkeyOptions, "Derive a public key from a private key."), + ("migrate-crawler", None, MigrateCrawlerOptions, + "Write the crawler-history data as JSON."), ] def postOptions(self): if not hasattr(self, 'subOptions'): @@ -88,6 +140,7 @@ each subcommand. subDispatch = { "generate-keypair": print_keypair, "derive-pubkey": derive_pubkey, + "migrate-crawler": migrate_crawler, } def do_admin(options): From ce25795e4e86b778ad6cc739fdc83d42d101101e Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 30 Nov 2021 16:00:19 -0700 Subject: [PATCH 0473/2309] new news --- newsfragments/3825.security | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/newsfragments/3825.security b/newsfragments/3825.security index b16418d2b..df83821de 100644 --- a/newsfragments/3825.security +++ b/newsfragments/3825.security @@ -1,5 +1,6 @@ The lease-checker now uses JSON instead of pickle to serialize its state. -Once you have run this version the lease state files will be stored in JSON -and an older version of the software won't load them (it simply won't notice -them so it will appear to have never run). +tahoe will now refuse to run until you either delete all pickle files or +migrate them using the new command: + + tahoe admin migrate-crawler From 3fd1ca8acbc3d046c97e3c520fdc6fab5d67541d Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 30 Nov 2021 16:00:35 -0700 Subject: [PATCH 0474/2309] it's an error to have pickle-format files --- src/allmydata/scripts/tahoe_run.py | 16 +++++++++++++++- src/allmydata/storage/crawler.py | 8 ++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/src/allmydata/scripts/tahoe_run.py b/src/allmydata/scripts/tahoe_run.py index 01f1a354c..51be32ee3 100644 --- a/src/allmydata/scripts/tahoe_run.py +++ b/src/allmydata/scripts/tahoe_run.py @@ -27,7 +27,9 @@ from allmydata.scripts.default_nodedir import _default_nodedir from allmydata.util.encodingutil import listdir_unicode, quote_local_unicode_path from allmydata.util.configutil import UnknownConfigError from allmydata.util.deferredutil import HookMixin - +from allmydata.storage.crawler import ( + MigratePickleFileError, +) from allmydata.node import ( PortAssignmentRequired, PrivacyError, @@ -164,6 +166,18 @@ class DaemonizeTheRealService(Service, HookMixin): self.stderr.write("\ntub.port cannot be 0: you must choose.\n\n") elif reason.check(PrivacyError): self.stderr.write("\n{}\n\n".format(reason.value)) + elif reason.check(MigratePickleFileError): + self.stderr.write( + "Error\nAt least one 'pickle' format file exists.\n" + "The file is {}\n" + "You must either delete the pickle-format files" + " or migrate them using the command:\n" + " tahoe admin migrate-crawler --basedir {}\n\n" + .format( + reason.value.args[0].path, + self.basedir, + ) + ) else: self.stderr.write("\nUnknown error\n") reason.printTraceback(self.stderr) diff --git a/src/allmydata/storage/crawler.py b/src/allmydata/storage/crawler.py index dcbea909a..2b8cde230 100644 --- a/src/allmydata/storage/crawler.py +++ b/src/allmydata/storage/crawler.py @@ -27,6 +27,14 @@ class TimeSliceExceeded(Exception): pass +class MigratePickleFileError(Exception): + """ + A pickle-format file exists (the FilePath to the file will be the + single arg). + """ + pass + + def _convert_cycle_data(state): """ :param dict state: cycle-to-date or history-item state From 1b8ae8039e79bf288916edb9f7949f76f943aef4 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 30 Nov 2021 16:01:15 -0700 Subject: [PATCH 0475/2309] no auto-migrate; produce error if pickle-files exist --- src/allmydata/storage/crawler.py | 31 +++++++++++++++++++++---------- src/allmydata/storage/expirer.py | 14 ++++++-------- 2 files changed, 27 insertions(+), 18 deletions(-) diff --git a/src/allmydata/storage/crawler.py b/src/allmydata/storage/crawler.py index 2b8cde230..f63754e10 100644 --- a/src/allmydata/storage/crawler.py +++ b/src/allmydata/storage/crawler.py @@ -108,7 +108,7 @@ def _maybe_upgrade_pickle_to_json(state_path, convert_pickle): :param Callable[dict] convert_pickle: function to change pickle-style state into JSON-style state - :returns unicode: the local path where the state is stored + :returns FilePath: the local path where the state is stored If this state path is JSON, simply return it. @@ -116,14 +116,14 @@ def _maybe_upgrade_pickle_to_json(state_path, convert_pickle): JSON path. """ if state_path.path.endswith(".json"): - return state_path.path + return state_path json_state_path = state_path.siblingExtension(".json") # if there's no file there at all, we're done because there's # nothing to upgrade if not state_path.exists(): - return json_state_path.path + return json_state_path # upgrade the pickle data to JSON import pickle @@ -135,7 +135,23 @@ def _maybe_upgrade_pickle_to_json(state_path, convert_pickle): # we've written the JSON, delete the pickle state_path.remove() - return json_state_path.path + return json_state_path + + +def _confirm_json_format(fp): + """ + :param FilePath fp: the original (pickle) name of a state file + + This confirms that we do _not_ have the pickle-version of a + state-file and _do_ either have nothing, or the JSON version. If + the pickle-version exists, an exception is raised. + + :returns FilePath: the JSON name of a state file + """ + jsonfp = fp.siblingExtension(".json") + if fp.exists(): + raise MigratePickleFileError(fp) + return jsonfp class _LeaseStateSerializer(object): @@ -146,12 +162,7 @@ class _LeaseStateSerializer(object): """ def __init__(self, state_path): - self._path = FilePath( - _maybe_upgrade_pickle_to_json( - FilePath(state_path), - _convert_pickle_state_to_json, - ) - ) + self._path = _confirm_json_format(FilePath(state_path)) def load(self): """ diff --git a/src/allmydata/storage/expirer.py b/src/allmydata/storage/expirer.py index ad1343ef5..cd0a9369a 100644 --- a/src/allmydata/storage/expirer.py +++ b/src/allmydata/storage/expirer.py @@ -12,6 +12,8 @@ import os import struct from allmydata.storage.crawler import ( ShareCrawler, + MigratePickleFileError, + _confirm_json_format, _maybe_upgrade_pickle_to_json, _convert_cycle_data, ) @@ -40,17 +42,13 @@ def _convert_pickle_state_to_json(state): class _HistorySerializer(object): """ Serialize the 'history' file of the lease-crawler state. This is - "storage/history.state" for the pickle or - "storage/history.state.json" for the new JSON format. + "storage/lease_checker.history" for the pickle or + "storage/lease_checker.history.json" for the new JSON format. """ def __init__(self, history_path): - self._path = FilePath( - _maybe_upgrade_pickle_to_json( - FilePath(history_path), - _convert_pickle_state_to_json, - ) - ) + self._path = _confirm_json_format(FilePath(history_path)) + if not self._path.exists(): with self._path.open("wb") as f: json.dump({}, f) From 0a4bc385c5ac7c5e9410e83703250b425608be57 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 30 Nov 2021 18:00:58 -0700 Subject: [PATCH 0476/2309] fix tests to use migrate command --- src/allmydata/scripts/admin.py | 7 ++-- src/allmydata/storage/crawler.py | 2 ++ src/allmydata/test/test_storage_web.py | 45 +++++++++++++++++++++++--- 3 files changed, 47 insertions(+), 7 deletions(-) diff --git a/src/allmydata/scripts/admin.py b/src/allmydata/scripts/admin.py index c125bc9e6..a6e826174 100644 --- a/src/allmydata/scripts/admin.py +++ b/src/allmydata/scripts/admin.py @@ -93,6 +93,7 @@ class MigrateCrawlerOptions(BasedirOptions): ) return t + def migrate_crawler(options): out = options.stdout storage = FilePath(options['basedir']).child("storage") @@ -107,12 +108,12 @@ def migrate_crawler(options): existed = fp.exists() newfp = crawler._maybe_upgrade_pickle_to_json(fp, converter) if existed: - print("Converted '{}' to '{}'".format(fp.path, newfp.path)) + print("Converted '{}' to '{}'".format(fp.path, newfp.path), file=out) else: if newfp.exists(): - print("Already converted: '{}'".format(newfp.path)) + print("Already converted: '{}'".format(newfp.path), file=out) else: - print("Not found: '{}'".format(fp.path)) + print("Not found: '{}'".format(fp.path), file=out) class AdminCommand(BaseOptions): diff --git a/src/allmydata/storage/crawler.py b/src/allmydata/storage/crawler.py index f63754e10..a1f70f4e5 100644 --- a/src/allmydata/storage/crawler.py +++ b/src/allmydata/storage/crawler.py @@ -148,6 +148,8 @@ def _confirm_json_format(fp): :returns FilePath: the JSON name of a state file """ + if fp.path.endswith(".json"): + return fp jsonfp = fp.siblingExtension(".json") if fp.exists(): raise MigratePickleFileError(fp) diff --git a/src/allmydata/test/test_storage_web.py b/src/allmydata/test/test_storage_web.py index 269af2203..86c2382f0 100644 --- a/src/allmydata/test/test_storage_web.py +++ b/src/allmydata/test/test_storage_web.py @@ -19,6 +19,7 @@ import time import os.path import re import json +from six.moves import StringIO from twisted.trial import unittest @@ -45,6 +46,13 @@ from allmydata.web.storage import ( StorageStatusElement, remove_prefix ) +from allmydata.scripts.admin import ( + MigrateCrawlerOptions, + migrate_crawler, +) +from allmydata.scripts.runner import ( + Options, +) from .common_util import FakeCanary from .common_web import ( @@ -1152,15 +1160,29 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin): """ # this file came from an "in the wild" tahoe version 1.16.0 original_pickle = FilePath(__file__).parent().child("data").child("lease_checker.state.txt") - test_pickle = FilePath("lease_checker.state") + root = FilePath(self.mktemp()) + storage = root.child("storage") + storage.makedirs() + test_pickle = storage.child("lease_checker.state") with test_pickle.open("w") as local, original_pickle.open("r") as remote: local.write(remote.read()) - serial = _LeaseStateSerializer(test_pickle.path) + # convert from pickle format to JSON + top = Options() + top.parseOptions([ + "admin", "migrate-crawler", + "--basedir", storage.parent().path, + ]) + options = top.subOptions + while hasattr(options, "subOptions"): + options = options.subOptions + options.stdout = StringIO() + migrate_crawler(options) # the (existing) state file should have been upgraded to JSON - self.assertNot(test_pickle.exists()) + self.assertFalse(test_pickle.exists()) self.assertTrue(test_pickle.siblingExtension(".json").exists()) + serial = _LeaseStateSerializer(test_pickle.path) self.assertEqual( serial.load(), @@ -1340,10 +1362,25 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin): """ # this file came from an "in the wild" tahoe version 1.16.0 original_pickle = FilePath(__file__).parent().child("data").child("lease_checker.history.txt") - test_pickle = FilePath("lease_checker.history") + root = FilePath(self.mktemp()) + storage = root.child("storage") + storage.makedirs() + test_pickle = storage.child("lease_checker.history") with test_pickle.open("w") as local, original_pickle.open("r") as remote: local.write(remote.read()) + # convert from pickle format to JSON + top = Options() + top.parseOptions([ + "admin", "migrate-crawler", + "--basedir", storage.parent().path, + ]) + options = top.subOptions + while hasattr(options, "subOptions"): + options = options.subOptions + options.stdout = StringIO() + migrate_crawler(options) + serial = _HistorySerializer(test_pickle.path) self.maxDiff = None From fc9671a8122bd085fa1d4ea74e2d4850abdf529f Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 30 Nov 2021 18:25:32 -0700 Subject: [PATCH 0477/2309] simplify, flake9 --- src/allmydata/scripts/admin.py | 2 +- src/allmydata/storage/crawler.py | 7 +------ src/allmydata/storage/expirer.py | 2 -- src/allmydata/test/test_storage_web.py | 1 - 4 files changed, 2 insertions(+), 10 deletions(-) diff --git a/src/allmydata/scripts/admin.py b/src/allmydata/scripts/admin.py index a6e826174..e0dcc8821 100644 --- a/src/allmydata/scripts/admin.py +++ b/src/allmydata/scripts/admin.py @@ -106,7 +106,7 @@ def migrate_crawler(options): for fp, converter in conversions: existed = fp.exists() - newfp = crawler._maybe_upgrade_pickle_to_json(fp, converter) + newfp = crawler._upgrade_pickle_to_json(fp, converter) if existed: print("Converted '{}' to '{}'".format(fp.path, newfp.path), file=out) else: diff --git a/src/allmydata/storage/crawler.py b/src/allmydata/storage/crawler.py index a1f70f4e5..dbf4b1300 100644 --- a/src/allmydata/storage/crawler.py +++ b/src/allmydata/storage/crawler.py @@ -101,7 +101,7 @@ def _convert_pickle_state_to_json(state): } -def _maybe_upgrade_pickle_to_json(state_path, convert_pickle): +def _upgrade_pickle_to_json(state_path, convert_pickle): """ :param FilePath state_path: the filepath to ensure is json @@ -110,14 +110,9 @@ def _maybe_upgrade_pickle_to_json(state_path, convert_pickle): :returns FilePath: the local path where the state is stored - If this state path is JSON, simply return it. - If this state is pickle, convert to the JSON format and return the JSON path. """ - if state_path.path.endswith(".json"): - return state_path - json_state_path = state_path.siblingExtension(".json") # if there's no file there at all, we're done because there's diff --git a/src/allmydata/storage/expirer.py b/src/allmydata/storage/expirer.py index cd0a9369a..abe3c37b6 100644 --- a/src/allmydata/storage/expirer.py +++ b/src/allmydata/storage/expirer.py @@ -12,9 +12,7 @@ import os import struct from allmydata.storage.crawler import ( ShareCrawler, - MigratePickleFileError, _confirm_json_format, - _maybe_upgrade_pickle_to_json, _convert_cycle_data, ) from allmydata.storage.shares import get_share_file diff --git a/src/allmydata/test/test_storage_web.py b/src/allmydata/test/test_storage_web.py index 86c2382f0..490a3f775 100644 --- a/src/allmydata/test/test_storage_web.py +++ b/src/allmydata/test/test_storage_web.py @@ -47,7 +47,6 @@ from allmydata.web.storage import ( remove_prefix ) from allmydata.scripts.admin import ( - MigrateCrawlerOptions, migrate_crawler, ) from allmydata.scripts.runner import ( From 679c46451764aae1213239bb90dd25b36bba324e Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 30 Nov 2021 18:43:06 -0700 Subject: [PATCH 0478/2309] tests --- src/allmydata/test/cli/test_admin.py | 86 ++++++++++++++++++++++++++++ 1 file changed, 86 insertions(+) create mode 100644 src/allmydata/test/cli/test_admin.py diff --git a/src/allmydata/test/cli/test_admin.py b/src/allmydata/test/cli/test_admin.py new file mode 100644 index 000000000..bdfc0a46f --- /dev/null +++ b/src/allmydata/test/cli/test_admin.py @@ -0,0 +1,86 @@ +""" +Ported to Python 3. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + +from six.moves import StringIO + +from testtools.matchers import ( + Contains, +) + +from twisted.trial import unittest +from twisted.python.filepath import FilePath + +from allmydata.scripts.admin import ( + migrate_crawler, +) +from allmydata.scripts.runner import ( + Options, +) +from ..common import ( + SyncTestCase, +) + +class AdminMigrateCrawler(SyncTestCase): + """ + Tests related to 'tahoe admin migrate-crawler' + """ + + def test_already(self): + """ + We've already migrated; don't do it again. + """ + + root = FilePath(self.mktemp()) + storage = root.child("storage") + storage.makedirs() + with storage.child("lease_checker.state.json").open("w") as f: + f.write(b"{}\n") + + top = Options() + top.parseOptions([ + "admin", "migrate-crawler", + "--basedir", storage.parent().path, + ]) + options = top.subOptions + while hasattr(options, "subOptions"): + options = options.subOptions + options.stdout = StringIO() + migrate_crawler(options) + + self.assertThat( + options.stdout.getvalue(), + Contains("Already converted:"), + ) + + def test_usage(self): + """ + We've already migrated; don't do it again. + """ + + root = FilePath(self.mktemp()) + storage = root.child("storage") + storage.makedirs() + with storage.child("lease_checker.state.json").open("w") as f: + f.write(b"{}\n") + + top = Options() + top.parseOptions([ + "admin", "migrate-crawler", + "--basedir", storage.parent().path, + ]) + options = top.subOptions + while hasattr(options, "subOptions"): + options = options.subOptions + self.assertThat( + str(options), + Contains("security issues with pickle") + ) From b47381401c589c056afe89744df5b3f01f2ae5ae Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 30 Nov 2021 19:01:09 -0700 Subject: [PATCH 0479/2309] flake8 --- src/allmydata/test/cli/test_admin.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/cli/test_admin.py b/src/allmydata/test/cli/test_admin.py index bdfc0a46f..082904652 100644 --- a/src/allmydata/test/cli/test_admin.py +++ b/src/allmydata/test/cli/test_admin.py @@ -16,8 +16,9 @@ from testtools.matchers import ( Contains, ) -from twisted.trial import unittest -from twisted.python.filepath import FilePath +from twisted.python.filepath import ( + FilePath, +) from allmydata.scripts.admin import ( migrate_crawler, From 85fa8fe32e05c68da46f15fe68b69781d05384a7 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 30 Nov 2021 23:00:59 -0700 Subject: [PATCH 0480/2309] py2/py3 glue code for json dumping --- src/allmydata/storage/crawler.py | 18 ++++++++++++++---- src/allmydata/storage/expirer.py | 7 +++---- src/allmydata/test/test_storage_web.py | 4 ++-- 3 files changed, 19 insertions(+), 10 deletions(-) diff --git a/src/allmydata/storage/crawler.py b/src/allmydata/storage/crawler.py index dbf4b1300..7516bc4e9 100644 --- a/src/allmydata/storage/crawler.py +++ b/src/allmydata/storage/crawler.py @@ -125,8 +125,7 @@ def _upgrade_pickle_to_json(state_path, convert_pickle): with state_path.open("rb") as f: state = pickle.load(f) new_state = convert_pickle(state) - with json_state_path.open("wb") as f: - json.dump(new_state, f) + _dump_json_to_file(new_state, json_state_path) # we've written the JSON, delete the pickle state_path.remove() @@ -151,6 +150,18 @@ def _confirm_json_format(fp): return jsonfp +def _dump_json_to_file(js, afile): + """ + Dump the JSON object `js` to the FilePath `afile` + """ + with afile.open("wb") as f: + data = json.dumps(js) + if PY2: + f.write(data) + else: + f.write(data.encode("utf8")) + + class _LeaseStateSerializer(object): """ Read and write state for LeaseCheckingCrawler. This understands @@ -174,8 +185,7 @@ class _LeaseStateSerializer(object): :returns: None """ tmpfile = self._path.siblingExtension(".tmp") - with tmpfile.open("wb") as f: - json.dump(data, f) + _dump_json_to_file(data, tmpfile) fileutil.move_into_place(tmpfile.path, self._path.path) return None diff --git a/src/allmydata/storage/expirer.py b/src/allmydata/storage/expirer.py index abe3c37b6..55ab51843 100644 --- a/src/allmydata/storage/expirer.py +++ b/src/allmydata/storage/expirer.py @@ -14,6 +14,7 @@ from allmydata.storage.crawler import ( ShareCrawler, _confirm_json_format, _convert_cycle_data, + _dump_json_to_file, ) from allmydata.storage.shares import get_share_file from allmydata.storage.common import UnknownMutableContainerVersionError, \ @@ -48,8 +49,7 @@ class _HistorySerializer(object): self._path = _confirm_json_format(FilePath(history_path)) if not self._path.exists(): - with self._path.open("wb") as f: - json.dump({}, f) + _dump_json_to_file({}, self._path) def load(self): """ @@ -65,8 +65,7 @@ class _HistorySerializer(object): """ Serialize the existing data as JSON. """ - with self._path.open("wb") as f: - json.dump(new_history, f) + _dump_json_to_file(new_history, self._path) return None diff --git a/src/allmydata/test/test_storage_web.py b/src/allmydata/test/test_storage_web.py index 490a3f775..dff3b36f5 100644 --- a/src/allmydata/test/test_storage_web.py +++ b/src/allmydata/test/test_storage_web.py @@ -1163,7 +1163,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin): storage = root.child("storage") storage.makedirs() test_pickle = storage.child("lease_checker.state") - with test_pickle.open("w") as local, original_pickle.open("r") as remote: + with test_pickle.open("wb") as local, original_pickle.open("rb") as remote: local.write(remote.read()) # convert from pickle format to JSON @@ -1365,7 +1365,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin): storage = root.child("storage") storage.makedirs() test_pickle = storage.child("lease_checker.history") - with test_pickle.open("w") as local, original_pickle.open("r") as remote: + with test_pickle.open("wb") as local, original_pickle.open("rb") as remote: local.write(remote.read()) # convert from pickle format to JSON From d985d1062295e2f816214bcbedb6746838d7a67d Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 1 Dec 2021 09:24:03 -0500 Subject: [PATCH 0481/2309] Update nix/cbor2.nix Co-authored-by: Jean-Paul Calderone --- nix/cbor2.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nix/cbor2.nix b/nix/cbor2.nix index 0544b1eb1..16ca8ff63 100644 --- a/nix/cbor2.nix +++ b/nix/cbor2.nix @@ -1,4 +1,4 @@ -{ lib, buildPythonPackage, fetchPypi }: +{ lib, buildPythonPackage, fetchPypi, setuptools_scm }: buildPythonPackage rec { pname = "cbor2"; version = "5.2.0"; From 18a5966f1d27791d3129690926791a623957472c Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 1 Dec 2021 09:38:56 -0500 Subject: [PATCH 0482/2309] Don't bother running HTTP server tests on Python 2, since it's going away any day now. --- src/allmydata/test/test_storage_http.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 9ba8adf21..442e154a0 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -14,6 +14,8 @@ if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 # fmt: on +from unittest import SkipTest + from twisted.trial.unittest import TestCase from twisted.internet.defer import inlineCallbacks @@ -31,6 +33,8 @@ class HTTPTests(TestCase): """ def setUp(self): + if PY2: + raise SkipTest("Not going to bother supporting Python 2") self.storage_server = StorageServer(self.mktemp(), b"\x00" * 20) # TODO what should the swissnum _actually_ be? self._http_server = HTTPServer(self.storage_server, b"abcd") From 50e21a90347a8a4bcc88487245c93f0379811dde Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 1 Dec 2021 09:55:44 -0500 Subject: [PATCH 0483/2309] Split StorageServer into generic part and Foolscap part. --- src/allmydata/storage/server.py | 127 ++++++++++++++++++++------------ 1 file changed, 78 insertions(+), 49 deletions(-) diff --git a/src/allmydata/storage/server.py b/src/allmydata/storage/server.py index 7dc277e39..9d3ac4012 100644 --- a/src/allmydata/storage/server.py +++ b/src/allmydata/storage/server.py @@ -12,7 +12,7 @@ if PY2: # strings. Omit bytes so we don't leak future's custom bytes. from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, pow, round, super, dict, list, object, range, str, max, min # noqa: F401 else: - from typing import Dict + from typing import Dict, Tuple import os, re import six @@ -56,12 +56,11 @@ NUM_RE=re.compile("^[0-9]+$") DEFAULT_RENEWAL_TIME = 31 * 24 * 60 * 60 -@implementer(RIStorageServer, IStatsProducer) -class StorageServer(service.MultiService, Referenceable): +@implementer(IStatsProducer) +class StorageServer(service.MultiService): """ - A filesystem-based implementation of ``RIStorageServer``. + Implement the business logic for the storage server. """ - name = 'storage' LeaseCheckerClass = LeaseCheckingCrawler def __init__(self, storedir, nodeid, reserved_space=0, @@ -125,16 +124,8 @@ class StorageServer(service.MultiService, Referenceable): self.lease_checker.setServiceParent(self) self._clock = clock - # Currently being-written Bucketwriters. For Foolscap, lifetime is tied - # to connection: when disconnection happens, the BucketWriters are - # removed. For HTTP, this makes no sense, so there will be - # timeout-based cleanup; see - # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3807. - # Map in-progress filesystem path -> BucketWriter: self._bucket_writers = {} # type: Dict[str,BucketWriter] - # Canaries and disconnect markers for BucketWriters created via Foolscap: - self._bucket_writer_disconnect_markers = {} # type: Dict[BucketWriter,(IRemoteReference, object)] def stopService(self): # Cancel any in-progress uploads: @@ -263,7 +254,7 @@ class StorageServer(service.MultiService, Referenceable): space += bw.allocated_size() return space - def remote_get_version(self): + def get_version(self): remaining_space = self.get_available_space() if remaining_space is None: # We're on a platform that has no API to get disk stats. @@ -284,7 +275,7 @@ class StorageServer(service.MultiService, Referenceable): } return version - def _allocate_buckets(self, storage_index, + def allocate_buckets(self, storage_index, renew_secret, cancel_secret, sharenums, allocated_size, owner_num=0, renew_leases=True): @@ -371,21 +362,6 @@ class StorageServer(service.MultiService, Referenceable): self.add_latency("allocate", self._clock.seconds() - start) return alreadygot, bucketwriters - def remote_allocate_buckets(self, storage_index, - renew_secret, cancel_secret, - sharenums, allocated_size, - canary, owner_num=0): - """Foolscap-specific ``allocate_buckets()`` API.""" - alreadygot, bucketwriters = self._allocate_buckets( - storage_index, renew_secret, cancel_secret, sharenums, allocated_size, - owner_num=owner_num, renew_leases=True, - ) - # Abort BucketWriters if disconnection happens. - for bw in bucketwriters.values(): - disconnect_marker = canary.notifyOnDisconnect(bw.disconnected) - self._bucket_writer_disconnect_markers[bw] = (canary, disconnect_marker) - return alreadygot, bucketwriters - def _iter_share_files(self, storage_index): for shnum, filename in self._get_bucket_shares(storage_index): with open(filename, 'rb') as f: @@ -401,8 +377,7 @@ class StorageServer(service.MultiService, Referenceable): continue # non-sharefile yield sf - def remote_add_lease(self, storage_index, renew_secret, cancel_secret, - owner_num=1): + def add_lease(self, storage_index, renew_secret, cancel_secret, owner_num=1): start = self._clock.seconds() self.count("add-lease") new_expire_time = self._clock.seconds() + DEFAULT_RENEWAL_TIME @@ -414,7 +389,7 @@ class StorageServer(service.MultiService, Referenceable): self.add_latency("add-lease", self._clock.seconds() - start) return None - def remote_renew_lease(self, storage_index, renew_secret): + def renew_lease(self, storage_index, renew_secret): start = self._clock.seconds() self.count("renew") new_expire_time = self._clock.seconds() + DEFAULT_RENEWAL_TIME @@ -448,7 +423,7 @@ class StorageServer(service.MultiService, Referenceable): # Commonly caused by there being no buckets at all. pass - def remote_get_buckets(self, storage_index): + def get_buckets(self, storage_index): start = self._clock.seconds() self.count("get") si_s = si_b2a(storage_index) @@ -698,18 +673,6 @@ class StorageServer(service.MultiService, Referenceable): self.add_latency("writev", self._clock.seconds() - start) return (testv_is_good, read_data) - def remote_slot_testv_and_readv_and_writev(self, storage_index, - secrets, - test_and_write_vectors, - read_vector): - return self.slot_testv_and_readv_and_writev( - storage_index, - secrets, - test_and_write_vectors, - read_vector, - renew_leases=True, - ) - def _allocate_slot_share(self, bucketdir, secrets, sharenum, owner_num=0): (write_enabler, renew_secret, cancel_secret) = secrets @@ -720,7 +683,7 @@ class StorageServer(service.MultiService, Referenceable): self) return share - def remote_slot_readv(self, storage_index, shares, readv): + def slot_readv(self, storage_index, shares, readv): start = self._clock.seconds() self.count("readv") si_s = si_b2a(storage_index) @@ -747,8 +710,8 @@ class StorageServer(service.MultiService, Referenceable): self.add_latency("readv", self._clock.seconds() - start) return datavs - def remote_advise_corrupt_share(self, share_type, storage_index, shnum, - reason): + def advise_corrupt_share(self, share_type, storage_index, shnum, + reason): # This is a remote API, I believe, so this has to be bytes for legacy # protocol backwards compatibility reasons. assert isinstance(share_type, bytes) @@ -774,3 +737,69 @@ class StorageServer(service.MultiService, Referenceable): share_type=share_type, si=si_s, shnum=shnum, reason=reason, level=log.SCARY, umid="SGx2fA") return None + + +@implementer(RIStorageServer) +class FoolscapStorageServer(Referenceable): # type: ignore # warner/foolscap#78 + """ + A filesystem-based implementation of ``RIStorageServer``. + + For Foolscap, BucketWriter lifetime is tied to connection: when + disconnection happens, the BucketWriters are removed. + """ + name = 'storage' + + def __init__(self, storage_server): # type: (StorageServer) -> None + self._server = storage_server + + # Canaries and disconnect markers for BucketWriters created via Foolscap: + self._bucket_writer_disconnect_markers = {} # type: Dict[BucketWriter,Tuple[IRemoteReference, object]] + + + def remote_get_version(self): + return self.get_version() + + def remote_allocate_buckets(self, storage_index, + renew_secret, cancel_secret, + sharenums, allocated_size, + canary, owner_num=0): + """Foolscap-specific ``allocate_buckets()`` API.""" + alreadygot, bucketwriters = self._server.allocate_buckets( + storage_index, renew_secret, cancel_secret, sharenums, allocated_size, + owner_num=owner_num, renew_leases=True, + ) + # Abort BucketWriters if disconnection happens. + for bw in bucketwriters.values(): + disconnect_marker = canary.notifyOnDisconnect(bw.disconnected) + self._bucket_writer_disconnect_markers[bw] = (canary, disconnect_marker) + return alreadygot, bucketwriters + + def remote_add_lease(self, storage_index, renew_secret, cancel_secret, + owner_num=1): + return self._server.add_lease(storage_index, renew_secret, cancel_secret) + + def remote_renew_lease(self, storage_index, renew_secret): + return self._server.renew_lease(storage_index, renew_secret) + + def remote_get_buckets(self, storage_index): + return self._server.get_buckets(storage_index) + + def remote_slot_testv_and_readv_and_writev(self, storage_index, + secrets, + test_and_write_vectors, + read_vector): + return self._server.slot_testv_and_readv_and_writev( + storage_index, + secrets, + test_and_write_vectors, + read_vector, + renew_leases=True, + ) + + def remote_slot_readv(self, storage_index, shares, readv): + return self._server.slot_readv(self, storage_index, shares, readv) + + def remote_advise_corrupt_share(self, share_type, storage_index, shnum, + reason): + return self._server.advise_corrupt_share(share_type, storage_index, shnum, + reason) From 25ca767095019f3f0d6288b2a870ef3b98eef112 Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 1 Dec 2021 11:49:52 -0700 Subject: [PATCH 0484/2309] an offering to the windows godesses --- src/allmydata/test/test_storage_web.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/allmydata/test/test_storage_web.py b/src/allmydata/test/test_storage_web.py index dff3b36f5..282fb67e1 100644 --- a/src/allmydata/test/test_storage_web.py +++ b/src/allmydata/test/test_storage_web.py @@ -22,11 +22,11 @@ import json from six.moves import StringIO from twisted.trial import unittest - from twisted.internet import defer from twisted.application import service from twisted.web.template import flattenString from twisted.python.filepath import FilePath +from twisted.python.runtime import platform from foolscap.api import fireEventually from allmydata.util import fileutil, hashutil, base32, pollmixin @@ -1163,8 +1163,12 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin): storage = root.child("storage") storage.makedirs() test_pickle = storage.child("lease_checker.state") - with test_pickle.open("wb") as local, original_pickle.open("rb") as remote: - local.write(remote.read()) + with test_pickle.open("w") as local, original_pickle.open("r") as remote: + for line in remote.readlines(): + if platform.isWindows(): + local.write(line.replace("\n", "\r\n")) + else: + local.write(line.replace("\n", "\r\n")) # convert from pickle format to JSON top = Options() @@ -1366,7 +1370,11 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin): storage.makedirs() test_pickle = storage.child("lease_checker.history") with test_pickle.open("wb") as local, original_pickle.open("rb") as remote: - local.write(remote.read()) + for line in remote.readlines(): + if platform.isWindows(): + local.write(line.replace("\n", "\r\n")) + else: + local.write(line) # convert from pickle format to JSON top = Options() From 7080ee6fc7747e1a9ca10ddb47fe81fd5e96a37b Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 1 Dec 2021 12:02:06 -0700 Subject: [PATCH 0485/2309] oops --- src/allmydata/test/test_storage_web.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/test_storage_web.py b/src/allmydata/test/test_storage_web.py index 282fb67e1..1cf96d660 100644 --- a/src/allmydata/test/test_storage_web.py +++ b/src/allmydata/test/test_storage_web.py @@ -1168,7 +1168,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin): if platform.isWindows(): local.write(line.replace("\n", "\r\n")) else: - local.write(line.replace("\n", "\r\n")) + local.write(line) # convert from pickle format to JSON top = Options() From 940c6343cf32318b9ba72f1330fdd5371346ce1f Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 1 Dec 2021 12:02:42 -0700 Subject: [PATCH 0486/2309] consistency --- src/allmydata/test/test_storage_web.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/test_storage_web.py b/src/allmydata/test/test_storage_web.py index 1cf96d660..961bbef98 100644 --- a/src/allmydata/test/test_storage_web.py +++ b/src/allmydata/test/test_storage_web.py @@ -1369,7 +1369,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin): storage = root.child("storage") storage.makedirs() test_pickle = storage.child("lease_checker.history") - with test_pickle.open("wb") as local, original_pickle.open("rb") as remote: + with test_pickle.open("w") as local, original_pickle.open("r") as remote: for line in remote.readlines(): if platform.isWindows(): local.write(line.replace("\n", "\r\n")) From 90d1e90a14b0a3455e2e5ac86cde814a8a81b378 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 1 Dec 2021 15:05:29 -0500 Subject: [PATCH 0487/2309] rewrite the Eliot interaction tests to make expected behavior clearer and to have explicit assertions about that behavior --- src/allmydata/test/test_eliotutil.py | 113 ++++++++++++++++++++++----- 1 file changed, 92 insertions(+), 21 deletions(-) diff --git a/src/allmydata/test/test_eliotutil.py b/src/allmydata/test/test_eliotutil.py index 0be02b277..cabe599b3 100644 --- a/src/allmydata/test/test_eliotutil.py +++ b/src/allmydata/test/test_eliotutil.py @@ -27,13 +27,12 @@ from fixtures import ( ) from testtools import ( TestCase, -) -from testtools import ( TestResult, ) from testtools.matchers import ( Is, IsInstance, + Not, MatchesStructure, Equals, HasLength, @@ -77,33 +76,105 @@ from .common import ( ) -class EliotLoggedTestTests(AsyncTestCase): +def passes(): """ - Tests for the automatic log-related provided by ``EliotLoggedRunTest``. + Create a matcher that matches a ``TestCase`` that runs without failures or + errors. """ - def test_returns_none(self): - Message.log(hello="world") + def run(case): + result = TestResult() + case.run(result) + return result.wasSuccessful() + return AfterPreprocessing(run, Equals(True)) - def test_returns_fired_deferred(self): - Message.log(hello="world") - return succeed(None) - def test_returns_unfired_deferred(self): - Message.log(hello="world") - # @eliot_logged_test automatically gives us an action context but it's - # still our responsibility to maintain it across stack-busting - # operations. - d = DeferredContext(deferLater(reactor, 0.0, lambda: None)) - d.addCallback(lambda ignored: Message.log(goodbye="world")) - # We didn't start an action. We're not finishing an action. - return d.result +class EliotLoggedTestTests(TestCase): + """ + Tests for the automatic log-related provided by ``AsyncTestCase``. + + This class uses ``testtools.TestCase`` because it is inconvenient to nest + ``AsyncTestCase`` inside ``AsyncTestCase`` (in particular, Eliot messages + emitted by the inner test case get observed by the outer test case and if + an inner case emits invalid messages they cause the outer test case to + fail). + """ + def test_fails(self): + """ + A test method of an ``AsyncTestCase`` subclass can fail. + """ + class UnderTest(AsyncTestCase): + def test_it(self): + self.fail("make sure it can fail") + + self.assertThat(UnderTest("test_it"), Not(passes())) + + def test_unserializable_fails(self): + """ + A test method of an ``AsyncTestCase`` subclass that logs an unserializable + value with Eliot fails. + """ + class world(object): + """ + an unserializable object + """ + + class UnderTest(AsyncTestCase): + def test_it(self): + Message.log(hello=world) + + self.assertThat(UnderTest("test_it"), Not(passes())) def test_logs_non_utf_8_byte(self): """ - If an Eliot message is emitted that contains a non-UTF-8 byte string then - the test nevertheless passes. + A test method of an ``AsyncTestCase`` subclass can log a message that + contains a non-UTF-8 byte string and return ``None`` and pass. """ - Message.log(hello=b"\xFF") + class UnderTest(AsyncTestCase): + def test_it(self): + Message.log(hello=b"\xFF") + + self.assertThat(UnderTest("test_it"), passes()) + + def test_returns_none(self): + """ + A test method of an ``AsyncTestCase`` subclass can log a message and + return ``None`` and pass. + """ + class UnderTest(AsyncTestCase): + def test_it(self): + Message.log(hello="world") + + self.assertThat(UnderTest("test_it"), passes()) + + def test_returns_fired_deferred(self): + """ + A test method of an ``AsyncTestCase`` subclass can log a message and + return an already-fired ``Deferred`` and pass. + """ + class UnderTest(AsyncTestCase): + def test_it(self): + Message.log(hello="world") + return succeed(None) + + self.assertThat(UnderTest("test_it"), passes()) + + def test_returns_unfired_deferred(self): + """ + A test method of an ``AsyncTestCase`` subclass can log a message and + return an unfired ``Deferred`` and pass when the ``Deferred`` fires. + """ + class UnderTest(AsyncTestCase): + def test_it(self): + Message.log(hello="world") + # @eliot_logged_test automatically gives us an action context + # but it's still our responsibility to maintain it across + # stack-busting operations. + d = DeferredContext(deferLater(reactor, 0.0, lambda: None)) + d.addCallback(lambda ignored: Message.log(goodbye="world")) + # We didn't start an action. We're not finishing an action. + return d.result + + self.assertThat(UnderTest("test_it"), passes()) class ParseDestinationDescriptionTests(SyncTestCase): From eee1f0975d5bd32acbb5d1c481623235558ae47c Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 1 Dec 2021 15:16:16 -0500 Subject: [PATCH 0488/2309] note about how to clean this up later --- src/allmydata/util/_eliot_updates.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/allmydata/util/_eliot_updates.py b/src/allmydata/util/_eliot_updates.py index 8e3beca45..81db566a4 100644 --- a/src/allmydata/util/_eliot_updates.py +++ b/src/allmydata/util/_eliot_updates.py @@ -6,6 +6,15 @@ only changed enough to add Python 2 compatibility. Every API in this module (except ``eliot_json_encoder``) should be obsolete as soon as we depend on Eliot 1.14 or newer. +When that happens: + +* replace ``capture_logging`` + with ``partial(eliot.testing.capture_logging, encoder_=eliot_json_encoder)`` +* replace ``validateLogging`` + with ``partial(eliot.testing.validateLogging, encoder_=eliot_json_encoder)`` +* replace ``MemoryLogger`` + with ``partial(eliot.MemoryLogger, encoder=eliot_json_encoder)`` + Ported to Python 3. """ From e0092ededaa64a800058658de2d9ab8472acb3bf Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 1 Dec 2021 20:52:22 -0700 Subject: [PATCH 0489/2309] fine, just skip tests on windows --- src/allmydata/test/test_storage_web.py | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/src/allmydata/test/test_storage_web.py b/src/allmydata/test/test_storage_web.py index 961bbef98..a49b71325 100644 --- a/src/allmydata/test/test_storage_web.py +++ b/src/allmydata/test/test_storage_web.py @@ -19,6 +19,7 @@ import time import os.path import re import json +from unittest import skipIf from six.moves import StringIO from twisted.trial import unittest @@ -1153,6 +1154,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin): d.addBoth(_cleanup) return d + @skipIf(platform.isWindows()) def test_deserialize_pickle(self): """ The crawler can read existing state from the old pickle format @@ -1163,12 +1165,8 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin): storage = root.child("storage") storage.makedirs() test_pickle = storage.child("lease_checker.state") - with test_pickle.open("w") as local, original_pickle.open("r") as remote: - for line in remote.readlines(): - if platform.isWindows(): - local.write(line.replace("\n", "\r\n")) - else: - local.write(line) + with test_pickle.open("wb") as local, original_pickle.open("rb") as remote: + test_pickle.write(original_pickle.read()) # convert from pickle format to JSON top = Options() @@ -1358,6 +1356,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin): second_serial.load(), ) + @skipIf(platform.isWindows()) def test_deserialize_history_pickle(self): """ The crawler can read existing history state from the old pickle @@ -1369,12 +1368,8 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin): storage = root.child("storage") storage.makedirs() test_pickle = storage.child("lease_checker.history") - with test_pickle.open("w") as local, original_pickle.open("r") as remote: - for line in remote.readlines(): - if platform.isWindows(): - local.write(line.replace("\n", "\r\n")) - else: - local.write(line) + with test_pickle.open("wb") as local, original_pickle.open("rb") as remote: + test_pickle.write(original_pickle.read()) # convert from pickle format to JSON top = Options() From 40e7be6d8d7581f4c9fa71c0817207e11ac1a7e6 Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 1 Dec 2021 23:46:10 -0700 Subject: [PATCH 0490/2309] needs reason --- src/allmydata/test/test_storage_web.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/test_storage_web.py b/src/allmydata/test/test_storage_web.py index a49b71325..9292c0b20 100644 --- a/src/allmydata/test/test_storage_web.py +++ b/src/allmydata/test/test_storage_web.py @@ -1154,7 +1154,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin): d.addBoth(_cleanup) return d - @skipIf(platform.isWindows()) + @skipIf(platform.isWindows(), "pickle test-data can't be loaded on windows") def test_deserialize_pickle(self): """ The crawler can read existing state from the old pickle format @@ -1356,7 +1356,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin): second_serial.load(), ) - @skipIf(platform.isWindows()) + @skipIf(platform.isWindows(), "pickle test-data can't be loaded on windows") def test_deserialize_history_pickle(self): """ The crawler can read existing history state from the old pickle From 4bc0df7cc14f53901470a3e0d0f78a6d975c4781 Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 2 Dec 2021 00:05:21 -0700 Subject: [PATCH 0491/2309] file, not path --- src/allmydata/test/test_storage_web.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/test_storage_web.py b/src/allmydata/test/test_storage_web.py index 9292c0b20..18ea0220c 100644 --- a/src/allmydata/test/test_storage_web.py +++ b/src/allmydata/test/test_storage_web.py @@ -1166,7 +1166,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin): storage.makedirs() test_pickle = storage.child("lease_checker.state") with test_pickle.open("wb") as local, original_pickle.open("rb") as remote: - test_pickle.write(original_pickle.read()) + local.write(remote.read()) # convert from pickle format to JSON top = Options() @@ -1369,7 +1369,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin): storage.makedirs() test_pickle = storage.child("lease_checker.history") with test_pickle.open("wb") as local, original_pickle.open("rb") as remote: - test_pickle.write(original_pickle.read()) + local.write(remote.read()) # convert from pickle format to JSON top = Options() From 6b8a42b0439bd81bbb8359c256538daf53622733 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 2 Dec 2021 09:34:29 -0500 Subject: [PATCH 0492/2309] Make the test more robust. --- newsfragments/3850.minor | 0 src/allmydata/test/test_storage_http.py | 8 ++++++++ 2 files changed, 8 insertions(+) create mode 100644 newsfragments/3850.minor diff --git a/newsfragments/3850.minor b/newsfragments/3850.minor new file mode 100644 index 000000000..e69de29bb diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 442e154a0..e30eb24c7 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -63,7 +63,15 @@ class HTTPTests(TestCase): def test_version(self): """ The client can return the version. + + We ignore available disk space since that might change across calls. """ version = yield self.client.get_version() + version[b"http://allmydata.org/tahoe/protocols/storage/v1"].pop( + b"available-space" + ) expected_version = self.storage_server.remote_get_version() + expected_version[b"http://allmydata.org/tahoe/protocols/storage/v1"].pop( + b"available-space" + ) self.assertEqual(version, expected_version) From 541b28f4693c900f83fc767c5e012918e98d3b9a Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 2 Dec 2021 09:36:56 -0500 Subject: [PATCH 0493/2309] News file. --- newsfragments/3849.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3849.minor diff --git a/newsfragments/3849.minor b/newsfragments/3849.minor new file mode 100644 index 000000000..e69de29bb From f7cb4d5c92e33b2b25286e4adb8c3ba4d32755bd Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 2 Dec 2021 10:02:46 -0500 Subject: [PATCH 0494/2309] Hook up the new FoolscapStorageServer, and fix enough bugs, such that almost all end-to-end and integration tests pass. --- src/allmydata/client.py | 4 ++-- src/allmydata/storage/immutable.py | 8 ++++---- src/allmydata/storage/server.py | 24 +++++++++++++++++++----- 3 files changed, 25 insertions(+), 11 deletions(-) diff --git a/src/allmydata/client.py b/src/allmydata/client.py index a2f88ebd6..645e157b6 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -36,7 +36,7 @@ from twisted.python.filepath import FilePath import allmydata from allmydata.crypto import rsa, ed25519 from allmydata.crypto.util import remove_prefix -from allmydata.storage.server import StorageServer +from allmydata.storage.server import StorageServer, FoolscapStorageServer from allmydata import storage_client from allmydata.immutable.upload import Uploader from allmydata.immutable.offloaded import Helper @@ -834,7 +834,7 @@ class _Client(node.Node, pollmixin.PollMixin): if anonymous_storage_enabled(self.config): furl_file = self.config.get_private_path("storage.furl").encode(get_filesystem_encoding()) - furl = self.tub.registerReference(ss, furlFile=furl_file) + furl = self.tub.registerReference(FoolscapStorageServer(ss), furlFile=furl_file) announcement["anonymous-storage-FURL"] = furl enabled_storage_servers = self._enable_storage_servers( diff --git a/src/allmydata/storage/immutable.py b/src/allmydata/storage/immutable.py index 08b83cd87..173a43e8e 100644 --- a/src/allmydata/storage/immutable.py +++ b/src/allmydata/storage/immutable.py @@ -382,7 +382,7 @@ class BucketReader(Referenceable): # type: ignore # warner/foolscap#78 return data def remote_advise_corrupt_share(self, reason): - return self.ss.remote_advise_corrupt_share(b"immutable", - self.storage_index, - self.shnum, - reason) + return self.ss.advise_corrupt_share(b"immutable", + self.storage_index, + self.shnum, + reason) diff --git a/src/allmydata/storage/server.py b/src/allmydata/storage/server.py index 9d3ac4012..5dd8cd0bc 100644 --- a/src/allmydata/storage/server.py +++ b/src/allmydata/storage/server.py @@ -127,6 +127,9 @@ class StorageServer(service.MultiService): # Map in-progress filesystem path -> BucketWriter: self._bucket_writers = {} # type: Dict[str,BucketWriter] + # These callables will be called with BucketWriters that closed: + self._call_on_bucket_writer_close = [] + def stopService(self): # Cancel any in-progress uploads: for bw in list(self._bucket_writers.values()): @@ -405,9 +408,14 @@ class StorageServer(service.MultiService): if self.stats_provider: self.stats_provider.count('storage_server.bytes_added', consumed_size) del self._bucket_writers[bw.incominghome] - if bw in self._bucket_writer_disconnect_markers: - canary, disconnect_marker = self._bucket_writer_disconnect_markers.pop(bw) - canary.dontNotifyOnDisconnect(disconnect_marker) + for handler in self._call_on_bucket_writer_close: + handler(bw) + + def register_bucket_writer_close_handler(self, handler): + """ + The handler will be called with any ``BucketWriter`` that closes. + """ + self._call_on_bucket_writer_close.append(handler) def _get_bucket_shares(self, storage_index): """Return a list of (shnum, pathname) tuples for files that hold @@ -755,9 +763,15 @@ class FoolscapStorageServer(Referenceable): # type: ignore # warner/foolscap#78 # Canaries and disconnect markers for BucketWriters created via Foolscap: self._bucket_writer_disconnect_markers = {} # type: Dict[BucketWriter,Tuple[IRemoteReference, object]] + self._server.register_bucket_writer_close_handler(self._bucket_writer_closed) + + def _bucket_writer_closed(self, bw): + if bw in self._bucket_writer_disconnect_markers: + canary, disconnect_marker = self._bucket_writer_disconnect_markers.pop(bw) + canary.dontNotifyOnDisconnect(disconnect_marker) def remote_get_version(self): - return self.get_version() + return self._server.get_version() def remote_allocate_buckets(self, storage_index, renew_secret, cancel_secret, @@ -797,7 +811,7 @@ class FoolscapStorageServer(Referenceable): # type: ignore # warner/foolscap#78 ) def remote_slot_readv(self, storage_index, shares, readv): - return self._server.slot_readv(self, storage_index, shares, readv) + return self._server.slot_readv(storage_index, shares, readv) def remote_advise_corrupt_share(self, share_type, storage_index, shnum, reason): From 476c41e49ec973db18b13d8f3b4e96a70e7c0933 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 2 Dec 2021 10:29:52 -0500 Subject: [PATCH 0495/2309] Split out Foolscap code from BucketReader/Writer. --- src/allmydata/storage/immutable.py | 60 +++++++++++++++++++++++------- src/allmydata/storage/server.py | 18 ++++++++- 2 files changed, 62 insertions(+), 16 deletions(-) diff --git a/src/allmydata/storage/immutable.py b/src/allmydata/storage/immutable.py index 173a43e8e..5fea7e2b6 100644 --- a/src/allmydata/storage/immutable.py +++ b/src/allmydata/storage/immutable.py @@ -230,8 +230,10 @@ class ShareFile(object): return space_freed -@implementer(RIBucketWriter) -class BucketWriter(Referenceable): # type: ignore # warner/foolscap#78 +class BucketWriter(object): + """ + Keep track of the process of writing to a ShareFile. + """ def __init__(self, ss, incominghome, finalhome, max_size, lease_info, clock): self.ss = ss @@ -251,7 +253,7 @@ class BucketWriter(Referenceable): # type: ignore # warner/foolscap#78 def allocated_size(self): return self._max_size - def remote_write(self, offset, data): + def write(self, offset, data): # Delay the timeout, since we received data: self._timeout.reset(30 * 60) start = self._clock.seconds() @@ -275,9 +277,6 @@ class BucketWriter(Referenceable): # type: ignore # warner/foolscap#78 self.ss.add_latency("write", self._clock.seconds() - start) self.ss.count("write") - def remote_close(self): - self.close() - def close(self): precondition(not self.closed) self._timeout.cancel() @@ -329,13 +328,10 @@ class BucketWriter(Referenceable): # type: ignore # warner/foolscap#78 facility="tahoe.storage", level=log.UNUSUAL) self.abort() - def remote_abort(self): + def abort(self): log.msg("storage: aborting sharefile %s" % self.incominghome, facility="tahoe.storage", level=log.UNUSUAL) - self.abort() self.ss.count("abort") - - def abort(self): if self.closed: return @@ -358,8 +354,28 @@ class BucketWriter(Referenceable): # type: ignore # warner/foolscap#78 self._timeout.cancel() -@implementer(RIBucketReader) -class BucketReader(Referenceable): # type: ignore # warner/foolscap#78 +@implementer(RIBucketWriter) +class FoolscapBucketWriter(Referenceable): # type: ignore # warner/foolscap#78 + """ + Foolscap-specific BucketWriter. + """ + def __init__(self, bucket_writer): + self._bucket_writer = bucket_writer + + def remote_write(self, offset, data): + return self._bucket_writer.write(offset, data) + + def remote_close(self): + return self._bucket_writer.close() + + def remote_abort(self): + return self._bucket_writer.abort() + + +class BucketReader(object): + """ + Manage the process for reading from a ``ShareFile``. + """ def __init__(self, ss, sharefname, storage_index=None, shnum=None): self.ss = ss @@ -374,15 +390,31 @@ class BucketReader(Referenceable): # type: ignore # warner/foolscap#78 ), self.shnum) - def remote_read(self, offset, length): + def read(self, offset, length): start = time.time() data = self._share_file.read_share_data(offset, length) self.ss.add_latency("read", time.time() - start) self.ss.count("read") return data - def remote_advise_corrupt_share(self, reason): + def advise_corrupt_share(self, reason): return self.ss.advise_corrupt_share(b"immutable", self.storage_index, self.shnum, reason) + + +@implementer(RIBucketReader) +class FoolscapBucketReader(Referenceable): # type: ignore # warner/foolscap#78 + """ + Foolscap wrapper for ``BucketReader`` + """ + + def __init__(self, bucket_reader): + self._bucket_reader = bucket_reader + + def remote_read(self, offset, length): + return self._bucket_reader.read(offset, length) + + def remote_advise_corrupt_share(self, reason): + return self._bucket_reader.advise_corrupt_share(reason) diff --git a/src/allmydata/storage/server.py b/src/allmydata/storage/server.py index 5dd8cd0bc..8c790b66f 100644 --- a/src/allmydata/storage/server.py +++ b/src/allmydata/storage/server.py @@ -33,7 +33,10 @@ from allmydata.storage.lease import LeaseInfo from allmydata.storage.mutable import MutableShareFile, EmptyShare, \ create_mutable_sharefile from allmydata.mutable.layout import MAX_MUTABLE_SHARE_SIZE -from allmydata.storage.immutable import ShareFile, BucketWriter, BucketReader +from allmydata.storage.immutable import ( + ShareFile, BucketWriter, BucketReader, FoolscapBucketWriter, + FoolscapBucketReader, +) from allmydata.storage.crawler import BucketCountingCrawler from allmydata.storage.expirer import LeaseCheckingCrawler @@ -782,10 +785,18 @@ class FoolscapStorageServer(Referenceable): # type: ignore # warner/foolscap#78 storage_index, renew_secret, cancel_secret, sharenums, allocated_size, owner_num=owner_num, renew_leases=True, ) + # Abort BucketWriters if disconnection happens. for bw in bucketwriters.values(): disconnect_marker = canary.notifyOnDisconnect(bw.disconnected) self._bucket_writer_disconnect_markers[bw] = (canary, disconnect_marker) + + # Wrap BucketWriters with Foolscap adapter: + bucketwriters = { + k: FoolscapBucketWriter(bw) + for (k, bw) in bucketwriters.items() + } + return alreadygot, bucketwriters def remote_add_lease(self, storage_index, renew_secret, cancel_secret, @@ -796,7 +807,10 @@ class FoolscapStorageServer(Referenceable): # type: ignore # warner/foolscap#78 return self._server.renew_lease(storage_index, renew_secret) def remote_get_buckets(self, storage_index): - return self._server.get_buckets(storage_index) + return { + k: FoolscapBucketReader(bucket) + for (k, bucket) in self._server.get_buckets(storage_index).items() + } def remote_slot_testv_and_readv_and_writev(self, storage_index, secrets, From 8c3d61a94e6da2e590831afc86afa85e0b0d6e36 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 2 Dec 2021 10:49:23 -0500 Subject: [PATCH 0496/2309] Bit more backwards compatible. --- src/allmydata/storage/server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/storage/server.py b/src/allmydata/storage/server.py index 8c790b66f..bb116ce8e 100644 --- a/src/allmydata/storage/server.py +++ b/src/allmydata/storage/server.py @@ -626,7 +626,7 @@ class StorageServer(service.MultiService): secrets, test_and_write_vectors, read_vector, - renew_leases, + renew_leases=True, ): """ Read data from shares and conditionally write some data to them. From 439e5f2998ac178f7773190842dba0a9855da893 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 2 Dec 2021 10:49:30 -0500 Subject: [PATCH 0497/2309] Insofar as possible, switch to testing without the Foolscap API. --- src/allmydata/test/test_storage.py | 303 +++++++++++++------------ src/allmydata/test/test_storage_web.py | 24 +- 2 files changed, 165 insertions(+), 162 deletions(-) diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index bc87e168d..bfc1a7cd4 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -31,10 +31,15 @@ from hypothesis import given, strategies import itertools from allmydata import interfaces from allmydata.util import fileutil, hashutil, base32 -from allmydata.storage.server import StorageServer, DEFAULT_RENEWAL_TIME +from allmydata.storage.server import ( + StorageServer, DEFAULT_RENEWAL_TIME, FoolscapStorageServer, +) from allmydata.storage.shares import get_share_file from allmydata.storage.mutable import MutableShareFile -from allmydata.storage.immutable import BucketWriter, BucketReader, ShareFile +from allmydata.storage.immutable import ( + BucketWriter, BucketReader, ShareFile, FoolscapBucketWriter, + FoolscapBucketReader, +) from allmydata.storage.common import storage_index_to_dir, \ UnknownMutableContainerVersionError, UnknownImmutableContainerVersionError, \ si_b2a, si_a2b @@ -129,25 +134,25 @@ class Bucket(unittest.TestCase): def test_create(self): incoming, final = self.make_workdir("test_create") bw = BucketWriter(self, incoming, final, 200, self.make_lease(), Clock()) - bw.remote_write(0, b"a"*25) - bw.remote_write(25, b"b"*25) - bw.remote_write(50, b"c"*25) - bw.remote_write(75, b"d"*7) - bw.remote_close() + bw.write(0, b"a"*25) + bw.write(25, b"b"*25) + bw.write(50, b"c"*25) + bw.write(75, b"d"*7) + bw.close() def test_readwrite(self): incoming, final = self.make_workdir("test_readwrite") bw = BucketWriter(self, incoming, final, 200, self.make_lease(), Clock()) - bw.remote_write(0, b"a"*25) - bw.remote_write(25, b"b"*25) - bw.remote_write(50, b"c"*7) # last block may be short - bw.remote_close() + bw.write(0, b"a"*25) + bw.write(25, b"b"*25) + bw.write(50, b"c"*7) # last block may be short + bw.close() # now read from it br = BucketReader(self, bw.finalhome) - self.failUnlessEqual(br.remote_read(0, 25), b"a"*25) - self.failUnlessEqual(br.remote_read(25, 25), b"b"*25) - self.failUnlessEqual(br.remote_read(50, 7), b"c"*7) + self.failUnlessEqual(br.read(0, 25), b"a"*25) + self.failUnlessEqual(br.read(25, 25), b"b"*25) + self.failUnlessEqual(br.read(50, 7), b"c"*7) def test_write_past_size_errors(self): """Writing beyond the size of the bucket throws an exception.""" @@ -157,7 +162,7 @@ class Bucket(unittest.TestCase): ) bw = BucketWriter(self, incoming, final, 200, self.make_lease(), Clock()) with self.assertRaises(DataTooLargeError): - bw.remote_write(offset, b"a" * length) + bw.write(offset, b"a" * length) @given( maybe_overlapping_offset=strategies.integers(min_value=0, max_value=98), @@ -177,25 +182,25 @@ class Bucket(unittest.TestCase): self, incoming, final, length, self.make_lease(), Clock() ) # Three writes: 10-19, 30-39, 50-59. This allows for a bunch of holes. - bw.remote_write(10, expected_data[10:20]) - bw.remote_write(30, expected_data[30:40]) - bw.remote_write(50, expected_data[50:60]) + bw.write(10, expected_data[10:20]) + bw.write(30, expected_data[30:40]) + bw.write(50, expected_data[50:60]) # Then, an overlapping write but with matching data: - bw.remote_write( + bw.write( maybe_overlapping_offset, expected_data[ maybe_overlapping_offset:maybe_overlapping_offset + maybe_overlapping_length ] ) # Now fill in the holes: - bw.remote_write(0, expected_data[0:10]) - bw.remote_write(20, expected_data[20:30]) - bw.remote_write(40, expected_data[40:50]) - bw.remote_write(60, expected_data[60:]) - bw.remote_close() + bw.write(0, expected_data[0:10]) + bw.write(20, expected_data[20:30]) + bw.write(40, expected_data[40:50]) + bw.write(60, expected_data[60:]) + bw.close() br = BucketReader(self, bw.finalhome) - self.assertEqual(br.remote_read(0, length), expected_data) + self.assertEqual(br.read(0, length), expected_data) @given( @@ -215,21 +220,21 @@ class Bucket(unittest.TestCase): self, incoming, final, length, self.make_lease(), Clock() ) # Three writes: 10-19, 30-39, 50-59. This allows for a bunch of holes. - bw.remote_write(10, b"1" * 10) - bw.remote_write(30, b"1" * 10) - bw.remote_write(50, b"1" * 10) + bw.write(10, b"1" * 10) + bw.write(30, b"1" * 10) + bw.write(50, b"1" * 10) # Then, write something that might overlap with some of them, but # conflicts. Then fill in holes left by first three writes. Conflict is # inevitable. with self.assertRaises(ConflictingWriteError): - bw.remote_write( + bw.write( maybe_overlapping_offset, b'X' * min(maybe_overlapping_length, length - maybe_overlapping_offset), ) - bw.remote_write(0, b"1" * 10) - bw.remote_write(20, b"1" * 10) - bw.remote_write(40, b"1" * 10) - bw.remote_write(60, b"1" * 40) + bw.write(0, b"1" * 10) + bw.write(20, b"1" * 10) + bw.write(40, b"1" * 10) + bw.write(60, b"1" * 40) def test_read_past_end_of_share_data(self): # test vector for immutable files (hard-coded contents of an immutable share @@ -274,15 +279,15 @@ class Bucket(unittest.TestCase): # Now read from it. br = BucketReader(mockstorageserver, final) - self.failUnlessEqual(br.remote_read(0, len(share_data)), share_data) + self.failUnlessEqual(br.read(0, len(share_data)), share_data) # Read past the end of share data to get the cancel secret. read_length = len(share_data) + len(ownernumber) + len(renewsecret) + len(cancelsecret) - result_of_read = br.remote_read(0, read_length) + result_of_read = br.read(0, read_length) self.failUnlessEqual(result_of_read, share_data) - result_of_read = br.remote_read(0, len(share_data)+1) + result_of_read = br.read(0, len(share_data)+1) self.failUnlessEqual(result_of_read, share_data) def _assert_timeout_only_after_30_minutes(self, clock, bw): @@ -320,7 +325,7 @@ class Bucket(unittest.TestCase): clock.advance(29 * 60) # .. but we receive a write! So that should delay the timeout again to # another 30 minutes. - bw.remote_write(0, b"hello") + bw.write(0, b"hello") self._assert_timeout_only_after_30_minutes(clock, bw) def test_bucket_closing_cancels_timeout(self): @@ -374,7 +379,7 @@ class BucketProxy(unittest.TestCase): fileutil.make_dirs(basedir) fileutil.make_dirs(os.path.join(basedir, "tmp")) bw = BucketWriter(self, incoming, final, size, self.make_lease(), Clock()) - rb = RemoteBucket(bw) + rb = RemoteBucket(FoolscapBucketWriter(bw)) return bw, rb, final def make_lease(self): @@ -446,7 +451,7 @@ class BucketProxy(unittest.TestCase): # now read everything back def _start_reading(res): br = BucketReader(self, sharefname) - rb = RemoteBucket(br) + rb = RemoteBucket(FoolscapBucketReader(br)) server = NoNetworkServer(b"abc", None) rbp = rbp_class(rb, server, storage_index=b"") self.failUnlessIn("to peer", repr(rbp)) @@ -514,20 +519,20 @@ class Server(unittest.TestCase): def test_declares_fixed_1528(self): ss = self.create("test_declares_fixed_1528") - ver = ss.remote_get_version() + ver = ss.get_version() sv1 = ver[b'http://allmydata.org/tahoe/protocols/storage/v1'] self.failUnless(sv1.get(b'prevents-read-past-end-of-share-data'), sv1) def test_declares_maximum_share_sizes(self): ss = self.create("test_declares_maximum_share_sizes") - ver = ss.remote_get_version() + ver = ss.get_version() sv1 = ver[b'http://allmydata.org/tahoe/protocols/storage/v1'] self.failUnlessIn(b'maximum-immutable-share-size', sv1) self.failUnlessIn(b'maximum-mutable-share-size', sv1) def test_declares_available_space(self): ss = self.create("test_declares_available_space") - ver = ss.remote_get_version() + ver = ss.get_version() sv1 = ver[b'http://allmydata.org/tahoe/protocols/storage/v1'] self.failUnlessIn(b'available-space', sv1) @@ -538,7 +543,9 @@ class Server(unittest.TestCase): """ renew_secret = hashutil.my_renewal_secret_hash(b"%d" % next(self._lease_secret)) cancel_secret = hashutil.my_cancel_secret_hash(b"%d" % next(self._lease_secret)) - return ss._allocate_buckets( + if isinstance(ss, FoolscapStorageServer): + ss = ss._server + return ss.allocate_buckets( storage_index, renew_secret, cancel_secret, sharenums, size, @@ -562,12 +569,12 @@ class Server(unittest.TestCase): shnum, bucket = list(writers.items())[0] # This test is going to hammer your filesystem if it doesn't make a sparse file for this. :-( - bucket.remote_write(2**32, b"ab") - bucket.remote_close() + bucket.write(2**32, b"ab") + bucket.close() - readers = ss.remote_get_buckets(b"allocate") + readers = ss.get_buckets(b"allocate") reader = readers[shnum] - self.failUnlessEqual(reader.remote_read(2**32, 2), b"ab") + self.failUnlessEqual(reader.read(2**32, 2), b"ab") def test_dont_overfill_dirs(self): """ @@ -578,8 +585,8 @@ class Server(unittest.TestCase): ss = self.create("test_dont_overfill_dirs") already, writers = self.allocate(ss, b"storageindex", [0], 10) for i, wb in writers.items(): - wb.remote_write(0, b"%10d" % i) - wb.remote_close() + wb.write(0, b"%10d" % i) + wb.close() storedir = os.path.join(self.workdir("test_dont_overfill_dirs"), "shares") children_of_storedir = set(os.listdir(storedir)) @@ -588,8 +595,8 @@ class Server(unittest.TestCase): # chars the same as the first storageindex. already, writers = self.allocate(ss, b"storageindey", [0], 10) for i, wb in writers.items(): - wb.remote_write(0, b"%10d" % i) - wb.remote_close() + wb.write(0, b"%10d" % i) + wb.close() storedir = os.path.join(self.workdir("test_dont_overfill_dirs"), "shares") new_children_of_storedir = set(os.listdir(storedir)) @@ -599,8 +606,8 @@ class Server(unittest.TestCase): ss = self.create("test_remove_incoming") already, writers = self.allocate(ss, b"vid", list(range(3)), 10) for i,wb in writers.items(): - wb.remote_write(0, b"%10d" % i) - wb.remote_close() + wb.write(0, b"%10d" % i) + wb.close() incoming_share_dir = wb.incominghome incoming_bucket_dir = os.path.dirname(incoming_share_dir) incoming_prefix_dir = os.path.dirname(incoming_bucket_dir) @@ -619,32 +626,32 @@ class Server(unittest.TestCase): # Now abort the writers. for writer in writers.values(): - writer.remote_abort() + writer.abort() self.failUnlessEqual(ss.allocated_size(), 0) def test_allocate(self): ss = self.create("test_allocate") - self.failUnlessEqual(ss.remote_get_buckets(b"allocate"), {}) + self.failUnlessEqual(ss.get_buckets(b"allocate"), {}) already,writers = self.allocate(ss, b"allocate", [0,1,2], 75) self.failUnlessEqual(already, set()) self.failUnlessEqual(set(writers.keys()), set([0,1,2])) # while the buckets are open, they should not count as readable - self.failUnlessEqual(ss.remote_get_buckets(b"allocate"), {}) + self.failUnlessEqual(ss.get_buckets(b"allocate"), {}) # close the buckets for i,wb in writers.items(): - wb.remote_write(0, b"%25d" % i) - wb.remote_close() + wb.write(0, b"%25d" % i) + wb.close() # aborting a bucket that was already closed is a no-op - wb.remote_abort() + wb.abort() # now they should be readable - b = ss.remote_get_buckets(b"allocate") + b = ss.get_buckets(b"allocate") self.failUnlessEqual(set(b.keys()), set([0,1,2])) - self.failUnlessEqual(b[0].remote_read(0, 25), b"%25d" % 0) + self.failUnlessEqual(b[0].read(0, 25), b"%25d" % 0) b_str = str(b[0]) self.failUnlessIn("BucketReader", b_str) self.failUnlessIn("mfwgy33dmf2g 0", b_str) @@ -665,15 +672,15 @@ class Server(unittest.TestCase): # aborting the writes should remove the tempfiles for i,wb in writers2.items(): - wb.remote_abort() + wb.abort() already2,writers2 = self.allocate(ss, b"allocate", [2,3,4,5], 75) self.failUnlessEqual(already2, set([0,1,2])) self.failUnlessEqual(set(writers2.keys()), set([5])) for i,wb in writers2.items(): - wb.remote_abort() + wb.abort() for i,wb in writers.items(): - wb.remote_abort() + wb.abort() def test_allocate_without_lease_renewal(self): """ @@ -696,8 +703,8 @@ class Server(unittest.TestCase): ss, storage_index, [0], 1, renew_leases=False, ) (writer,) = writers.values() - writer.remote_write(0, b"x") - writer.remote_close() + writer.write(0, b"x") + writer.close() # It should have a lease granted at the current time. shares = dict(ss._get_bucket_shares(storage_index)) @@ -719,8 +726,8 @@ class Server(unittest.TestCase): ss, storage_index, [1], 1, renew_leases=False, ) (writer,) = writers.values() - writer.remote_write(0, b"x") - writer.remote_close() + writer.write(0, b"x") + writer.close() # The first share's lease expiration time is unchanged. shares = dict(ss._get_bucket_shares(storage_index)) @@ -736,8 +743,8 @@ class Server(unittest.TestCase): def test_bad_container_version(self): ss = self.create("test_bad_container_version") a,w = self.allocate(ss, b"si1", [0], 10) - w[0].remote_write(0, b"\xff"*10) - w[0].remote_close() + w[0].write(0, b"\xff"*10) + w[0].close() fn = os.path.join(ss.sharedir, storage_index_to_dir(b"si1"), "0") f = open(fn, "rb+") @@ -745,15 +752,15 @@ class Server(unittest.TestCase): f.write(struct.pack(">L", 0)) # this is invalid: minimum used is v1 f.close() - ss.remote_get_buckets(b"allocate") + ss.get_buckets(b"allocate") e = self.failUnlessRaises(UnknownImmutableContainerVersionError, - ss.remote_get_buckets, b"si1") + ss.get_buckets, b"si1") self.failUnlessIn(" had version 0 but we wanted 1", str(e)) def test_disconnect(self): # simulate a disconnection - ss = self.create("test_disconnect") + ss = FoolscapStorageServer(self.create("test_disconnect")) renew_secret = b"r" * 32 cancel_secret = b"c" * 32 canary = FakeCanary() @@ -789,7 +796,7 @@ class Server(unittest.TestCase): } self.patch(fileutil, 'get_disk_stats', call_get_disk_stats) - ss = self.create("test_reserved_space", reserved_space=reserved) + ss = FoolscapStorageServer(self.create("test_reserved_space", reserved_space=reserved)) # 15k available, 10k reserved, leaves 5k for shares # a newly created and filled share incurs this much overhead, beyond @@ -810,28 +817,28 @@ class Server(unittest.TestCase): self.failUnlessEqual(len(writers), 3) # now the StorageServer should have 3000 bytes provisionally # allocated, allowing only 2000 more to be claimed - self.failUnlessEqual(len(ss._bucket_writers), 3) + self.failUnlessEqual(len(ss._server._bucket_writers), 3) # allocating 1001-byte shares only leaves room for one canary2 = FakeCanary() already2, writers2 = self.allocate(ss, b"vid2", [0,1,2], 1001, canary2) self.failUnlessEqual(len(writers2), 1) - self.failUnlessEqual(len(ss._bucket_writers), 4) + self.failUnlessEqual(len(ss._server._bucket_writers), 4) # we abandon the first set, so their provisional allocation should be # returned canary.disconnected() - self.failUnlessEqual(len(ss._bucket_writers), 1) + self.failUnlessEqual(len(ss._server._bucket_writers), 1) # now we have a provisional allocation of 1001 bytes # and we close the second set, so their provisional allocation should # become real, long-term allocation, and grows to include the # overhead. for bw in writers2.values(): - bw.remote_write(0, b"a"*25) - bw.remote_close() - self.failUnlessEqual(len(ss._bucket_writers), 0) + bw.write(0, b"a"*25) + bw.close() + self.failUnlessEqual(len(ss._server._bucket_writers), 0) # this also changes the amount reported as available by call_get_disk_stats allocated = 1001 + OVERHEAD + LEASE_SIZE @@ -848,12 +855,12 @@ class Server(unittest.TestCase): canary=canary3, ) self.failUnlessEqual(len(writers3), 39) - self.failUnlessEqual(len(ss._bucket_writers), 39) + self.failUnlessEqual(len(ss._server._bucket_writers), 39) canary3.disconnected() - self.failUnlessEqual(len(ss._bucket_writers), 0) - ss.disownServiceParent() + self.failUnlessEqual(len(ss._server._bucket_writers), 0) + ss._server.disownServiceParent() del ss def test_seek(self): @@ -882,24 +889,22 @@ class Server(unittest.TestCase): Given a StorageServer, create a bucket with 5 shares and return renewal and cancellation secrets. """ - canary = FakeCanary() sharenums = list(range(5)) size = 100 # Creating a bucket also creates a lease: rs, cs = (hashutil.my_renewal_secret_hash(b"%d" % next(self._lease_secret)), hashutil.my_cancel_secret_hash(b"%d" % next(self._lease_secret))) - already, writers = ss.remote_allocate_buckets(storage_index, rs, cs, - sharenums, size, canary) + already, writers = ss.allocate_buckets(storage_index, rs, cs, + sharenums, size) self.failUnlessEqual(len(already), expected_already) self.failUnlessEqual(len(writers), expected_writers) for wb in writers.values(): - wb.remote_close() + wb.close() return rs, cs def test_leases(self): ss = self.create("test_leases") - canary = FakeCanary() sharenums = list(range(5)) size = 100 @@ -919,54 +924,54 @@ class Server(unittest.TestCase): # and a third lease, using add-lease rs2a,cs2a = (hashutil.my_renewal_secret_hash(b"%d" % next(self._lease_secret)), hashutil.my_cancel_secret_hash(b"%d" % next(self._lease_secret))) - ss.remote_add_lease(b"si1", rs2a, cs2a) + ss.add_lease(b"si1", rs2a, cs2a) (lease1, lease2, lease3) = ss.get_leases(b"si1") self.assertTrue(lease1.is_renew_secret(rs1)) self.assertTrue(lease2.is_renew_secret(rs2)) self.assertTrue(lease3.is_renew_secret(rs2a)) # add-lease on a missing storage index is silently ignored - self.assertIsNone(ss.remote_add_lease(b"si18", b"", b"")) + self.assertIsNone(ss.add_lease(b"si18", b"", b"")) # check that si0 is readable - readers = ss.remote_get_buckets(b"si0") + readers = ss.get_buckets(b"si0") self.failUnlessEqual(len(readers), 5) # renew the first lease. Only the proper renew_secret should work - ss.remote_renew_lease(b"si0", rs0) - self.failUnlessRaises(IndexError, ss.remote_renew_lease, b"si0", cs0) - self.failUnlessRaises(IndexError, ss.remote_renew_lease, b"si0", rs1) + ss.renew_lease(b"si0", rs0) + self.failUnlessRaises(IndexError, ss.renew_lease, b"si0", cs0) + self.failUnlessRaises(IndexError, ss.renew_lease, b"si0", rs1) # check that si0 is still readable - readers = ss.remote_get_buckets(b"si0") + readers = ss.get_buckets(b"si0") self.failUnlessEqual(len(readers), 5) # There is no such method as remote_cancel_lease for now -- see # ticket #1528. - self.failIf(hasattr(ss, 'remote_cancel_lease'), \ - "ss should not have a 'remote_cancel_lease' method/attribute") + self.failIf(hasattr(FoolscapStorageServer(ss), 'remote_cancel_lease'), \ + "ss should not have a 'remote_cancel_lease' method/attribute") # test overlapping uploads rs3,cs3 = (hashutil.my_renewal_secret_hash(b"%d" % next(self._lease_secret)), hashutil.my_cancel_secret_hash(b"%d" % next(self._lease_secret))) rs4,cs4 = (hashutil.my_renewal_secret_hash(b"%d" % next(self._lease_secret)), hashutil.my_cancel_secret_hash(b"%d" % next(self._lease_secret))) - already,writers = ss.remote_allocate_buckets(b"si3", rs3, cs3, - sharenums, size, canary) + already,writers = ss.allocate_buckets(b"si3", rs3, cs3, + sharenums, size) self.failUnlessEqual(len(already), 0) self.failUnlessEqual(len(writers), 5) - already2,writers2 = ss.remote_allocate_buckets(b"si3", rs4, cs4, - sharenums, size, canary) + already2,writers2 = ss.allocate_buckets(b"si3", rs4, cs4, + sharenums, size) self.failUnlessEqual(len(already2), 0) self.failUnlessEqual(len(writers2), 0) for wb in writers.values(): - wb.remote_close() + wb.close() leases = list(ss.get_leases(b"si3")) self.failUnlessEqual(len(leases), 1) - already3,writers3 = ss.remote_allocate_buckets(b"si3", rs4, cs4, - sharenums, size, canary) + already3,writers3 = ss.allocate_buckets(b"si3", rs4, cs4, + sharenums, size) self.failUnlessEqual(len(already3), 5) self.failUnlessEqual(len(writers3), 0) @@ -991,7 +996,7 @@ class Server(unittest.TestCase): clock.advance(123456) # Adding a lease with matching renewal secret just renews it: - ss.remote_add_lease(b"si0", renewal_secret, cancel_secret) + ss.add_lease(b"si0", renewal_secret, cancel_secret) [lease] = ss.get_leases(b"si0") self.assertEqual(lease.get_expiration_time(), 123 + 123456 + DEFAULT_RENEWAL_TIME) @@ -1027,14 +1032,14 @@ class Server(unittest.TestCase): self.failUnlessEqual(already, set()) self.failUnlessEqual(set(writers.keys()), set([0,1,2])) for i,wb in writers.items(): - wb.remote_write(0, b"%25d" % i) - wb.remote_close() + wb.write(0, b"%25d" % i) + wb.close() # since we discard the data, the shares should be present but sparse. # Since we write with some seeks, the data we read back will be all # zeros. - b = ss.remote_get_buckets(b"vid") + b = ss.get_buckets(b"vid") self.failUnlessEqual(set(b.keys()), set([0,1,2])) - self.failUnlessEqual(b[0].remote_read(0, 25), b"\x00" * 25) + self.failUnlessEqual(b[0].read(0, 25), b"\x00" * 25) def test_advise_corruption(self): workdir = self.workdir("test_advise_corruption") @@ -1042,8 +1047,8 @@ class Server(unittest.TestCase): ss.setServiceParent(self.sparent) si0_s = base32.b2a(b"si0") - ss.remote_advise_corrupt_share(b"immutable", b"si0", 0, - b"This share smells funny.\n") + ss.advise_corrupt_share(b"immutable", b"si0", 0, + b"This share smells funny.\n") reportdir = os.path.join(workdir, "corruption-advisories") reports = os.listdir(reportdir) self.failUnlessEqual(len(reports), 1) @@ -1062,12 +1067,12 @@ class Server(unittest.TestCase): already,writers = self.allocate(ss, b"si1", [1], 75) self.failUnlessEqual(already, set()) self.failUnlessEqual(set(writers.keys()), set([1])) - writers[1].remote_write(0, b"data") - writers[1].remote_close() + writers[1].write(0, b"data") + writers[1].close() - b = ss.remote_get_buckets(b"si1") + b = ss.get_buckets(b"si1") self.failUnlessEqual(set(b.keys()), set([1])) - b[1].remote_advise_corrupt_share(b"This share tastes like dust.\n") + b[1].advise_corrupt_share(b"This share tastes like dust.\n") reports = os.listdir(reportdir) self.failUnlessEqual(len(reports), 2) @@ -1125,7 +1130,7 @@ class MutableServer(unittest.TestCase): write_enabler = self.write_enabler(we_tag) renew_secret = self.renew_secret(lease_tag) cancel_secret = self.cancel_secret(lease_tag) - rstaraw = ss.remote_slot_testv_and_readv_and_writev + rstaraw = ss.slot_testv_and_readv_and_writev testandwritev = dict( [ (shnum, ([], [], None) ) for shnum in sharenums ] ) readv = [] @@ -1146,7 +1151,7 @@ class MutableServer(unittest.TestCase): f.seek(0) f.write(b"BAD MAGIC") f.close() - read = ss.remote_slot_readv + read = ss.slot_readv e = self.failUnlessRaises(UnknownMutableContainerVersionError, read, b"si1", [0], [(0,10)]) self.failUnlessIn(" had magic ", str(e)) @@ -1156,8 +1161,8 @@ class MutableServer(unittest.TestCase): ss = self.create("test_container_size") self.allocate(ss, b"si1", b"we1", next(self._lease_secret), set([0,1,2]), 100) - read = ss.remote_slot_readv - rstaraw = ss.remote_slot_testv_and_readv_and_writev + read = ss.slot_readv + rstaraw = ss.slot_testv_and_readv_and_writev secrets = ( self.write_enabler(b"we1"), self.renew_secret(b"we1"), self.cancel_secret(b"we1") ) @@ -1237,7 +1242,7 @@ class MutableServer(unittest.TestCase): # Also see if the server explicitly declares that it supports this # feature. - ver = ss.remote_get_version() + ver = ss.get_version() storage_v1_ver = ver[b"http://allmydata.org/tahoe/protocols/storage/v1"] self.failUnless(storage_v1_ver.get(b"fills-holes-with-zero-bytes")) @@ -1255,7 +1260,7 @@ class MutableServer(unittest.TestCase): self.allocate(ss, b"si1", b"we1", next(self._lease_secret), set([0,1,2]), 100) - read = ss.remote_slot_readv + read = ss.slot_readv self.failUnlessEqual(read(b"si1", [0], [(0, 10)]), {0: [b""]}) self.failUnlessEqual(read(b"si1", [], [(0, 10)]), @@ -1268,7 +1273,7 @@ class MutableServer(unittest.TestCase): self.renew_secret(b"we1"), self.cancel_secret(b"we1") ) data = b"".join([ (b"%d" % i) * 10 for i in range(10) ]) - write = ss.remote_slot_testv_and_readv_and_writev + write = ss.slot_testv_and_readv_and_writev answer = write(b"si1", secrets, {0: ([], [(0,data)], None)}, []) @@ -1278,7 +1283,7 @@ class MutableServer(unittest.TestCase): {0: [b"00000000001111111111"]}) self.failUnlessEqual(read(b"si1", [0], [(95,10)]), {0: [b"99999"]}) - #self.failUnlessEqual(s0.remote_get_length(), 100) + #self.failUnlessEqual(s0.get_length(), 100) bad_secrets = (b"bad write enabler", secrets[1], secrets[2]) f = self.failUnlessRaises(BadWriteEnablerError, @@ -1312,8 +1317,8 @@ class MutableServer(unittest.TestCase): self.renew_secret(b"we1"), self.cancel_secret(b"we1") ) data = b"".join([ (b"%d" % i) * 10 for i in range(10) ]) - write = ss.remote_slot_testv_and_readv_and_writev - read = ss.remote_slot_readv + write = ss.slot_testv_and_readv_and_writev + read = ss.slot_readv def reset(): write(b"si1", secrets, @@ -1357,8 +1362,8 @@ class MutableServer(unittest.TestCase): self.renew_secret(b"we1"), self.cancel_secret(b"we1") ) data = b"".join([ (b"%d" % i) * 10 for i in range(10) ]) - write = ss.remote_slot_testv_and_readv_and_writev - read = ss.remote_slot_readv + write = ss.slot_testv_and_readv_and_writev + read = ss.slot_readv data = [(b"%d" % i) * 100 for i in range(3)] rc = write(b"si1", secrets, {0: ([], [(0,data[0])], None), @@ -1389,8 +1394,8 @@ class MutableServer(unittest.TestCase): self.renew_secret(b"we1-%d" % n), self.cancel_secret(b"we1-%d" % n) ) data = b"".join([ (b"%d" % i) * 10 for i in range(10) ]) - write = ss.remote_slot_testv_and_readv_and_writev - read = ss.remote_slot_readv + write = ss.slot_testv_and_readv_and_writev + read = ss.slot_readv rc = write(b"si1", secrets(0), {0: ([], [(0,data)], None)}, []) self.failUnlessEqual(rc, (True, {})) @@ -1406,7 +1411,7 @@ class MutableServer(unittest.TestCase): self.failUnlessEqual(len(list(s0.get_leases())), 1) # add-lease on a missing storage index is silently ignored - self.failUnlessEqual(ss.remote_add_lease(b"si18", b"", b""), None) + self.failUnlessEqual(ss.add_lease(b"si18", b"", b""), None) # re-allocate the slots and use the same secrets, that should update # the lease @@ -1414,7 +1419,7 @@ class MutableServer(unittest.TestCase): self.failUnlessEqual(len(list(s0.get_leases())), 1) # renew it directly - ss.remote_renew_lease(b"si1", secrets(0)[1]) + ss.renew_lease(b"si1", secrets(0)[1]) self.failUnlessEqual(len(list(s0.get_leases())), 1) # now allocate them with a bunch of different secrets, to trigger the @@ -1422,7 +1427,7 @@ class MutableServer(unittest.TestCase): write(b"si1", secrets(1), {0: ([], [(0,data)], None)}, []) self.failUnlessEqual(len(list(s0.get_leases())), 2) secrets2 = secrets(2) - ss.remote_add_lease(b"si1", secrets2[1], secrets2[2]) + ss.add_lease(b"si1", secrets2[1], secrets2[2]) self.failUnlessEqual(len(list(s0.get_leases())), 3) write(b"si1", secrets(3), {0: ([], [(0,data)], None)}, []) write(b"si1", secrets(4), {0: ([], [(0,data)], None)}, []) @@ -1440,11 +1445,11 @@ class MutableServer(unittest.TestCase): # read back the leases, make sure they're still intact. self.compare_leases_without_timestamps(all_leases, list(s0.get_leases())) - ss.remote_renew_lease(b"si1", secrets(0)[1]) - ss.remote_renew_lease(b"si1", secrets(1)[1]) - ss.remote_renew_lease(b"si1", secrets(2)[1]) - ss.remote_renew_lease(b"si1", secrets(3)[1]) - ss.remote_renew_lease(b"si1", secrets(4)[1]) + ss.renew_lease(b"si1", secrets(0)[1]) + ss.renew_lease(b"si1", secrets(1)[1]) + ss.renew_lease(b"si1", secrets(2)[1]) + ss.renew_lease(b"si1", secrets(3)[1]) + ss.renew_lease(b"si1", secrets(4)[1]) self.compare_leases_without_timestamps(all_leases, list(s0.get_leases())) # get a new copy of the leases, with the current timestamps. Reading # data and failing to renew/cancel leases should leave the timestamps @@ -1455,7 +1460,7 @@ class MutableServer(unittest.TestCase): # examine the exception thus raised, make sure the old nodeid is # present, to provide for share migration e = self.failUnlessRaises(IndexError, - ss.remote_renew_lease, b"si1", + ss.renew_lease, b"si1", secrets(20)[1]) e_s = str(e) self.failUnlessIn("Unable to renew non-existent lease", e_s) @@ -1490,7 +1495,7 @@ class MutableServer(unittest.TestCase): self.renew_secret(b"we1-%d" % n), self.cancel_secret(b"we1-%d" % n) ) data = b"".join([ (b"%d" % i) * 10 for i in range(10) ]) - write = ss.remote_slot_testv_and_readv_and_writev + write = ss.slot_testv_and_readv_and_writev write_enabler, renew_secret, cancel_secret = secrets(0) rc = write(b"si1", (write_enabler, renew_secret, cancel_secret), {0: ([], [(0,data)], None)}, []) @@ -1506,7 +1511,7 @@ class MutableServer(unittest.TestCase): clock.advance(835) # Adding a lease renews it: - ss.remote_add_lease(b"si1", renew_secret, cancel_secret) + ss.add_lease(b"si1", renew_secret, cancel_secret) [lease] = s0.get_leases() self.assertEqual(lease.get_expiration_time(), 235 + 835 + DEFAULT_RENEWAL_TIME) @@ -1515,8 +1520,8 @@ class MutableServer(unittest.TestCase): ss = self.create("test_remove") self.allocate(ss, b"si1", b"we1", next(self._lease_secret), set([0,1,2]), 100) - readv = ss.remote_slot_readv - writev = ss.remote_slot_testv_and_readv_and_writev + readv = ss.slot_readv + writev = ss.slot_testv_and_readv_and_writev secrets = ( self.write_enabler(b"we1"), self.renew_secret(b"we1"), self.cancel_secret(b"we1") ) @@ -1620,7 +1625,7 @@ class MutableServer(unittest.TestCase): # We don't even need to create any shares to exercise this # functionality. Just go straight to sending a truncate-to-zero # write. - testv_is_good, read_data = ss.remote_slot_testv_and_readv_and_writev( + testv_is_good, read_data = ss.slot_testv_and_readv_and_writev( storage_index=storage_index, secrets=secrets, test_and_write_vectors={ @@ -1638,7 +1643,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin): self.sparent = LoggingServiceParent() self._lease_secret = itertools.count() self.ss = self.create("MDMFProxies storage test server") - self.rref = RemoteBucket(self.ss) + self.rref = RemoteBucket(FoolscapStorageServer(self.ss)) self.storage_server = _StorageServer(lambda: self.rref) self.secrets = (self.write_enabler(b"we_secret"), self.renew_secret(b"renew_secret"), @@ -1805,7 +1810,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin): If tail_segment=True, then I will write a share that has a smaller tail segment than other segments. """ - write = self.ss.remote_slot_testv_and_readv_and_writev + write = self.ss.slot_testv_and_readv_and_writev data = self.build_test_mdmf_share(tail_segment, empty) # Finally, we write the whole thing to the storage server in one # pass. @@ -1873,7 +1878,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin): empty=False): # Some tests need SDMF shares to verify that we can still # read them. This method writes one, which resembles but is not - write = self.ss.remote_slot_testv_and_readv_and_writev + write = self.ss.slot_testv_and_readv_and_writev share = self.build_test_sdmf_share(empty) testvs = [(0, 1, b"eq", b"")] tws = {} @@ -2205,7 +2210,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin): # blocks. mw = self._make_new_mw(b"si1", 0) # Test writing some blocks. - read = self.ss.remote_slot_readv + read = self.ss.slot_readv expected_private_key_offset = struct.calcsize(MDMFHEADER) expected_sharedata_offset = struct.calcsize(MDMFHEADER) + \ PRIVATE_KEY_SIZE + \ @@ -2996,7 +3001,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin): d = sdmfr.finish_publishing() def _then(ignored): self.failUnlessEqual(self.rref.write_count, 1) - read = self.ss.remote_slot_readv + read = self.ss.slot_readv self.failUnlessEqual(read(b"si1", [0], [(0, len(data))]), {0: [data]}) d.addCallback(_then) @@ -3053,7 +3058,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin): sdmfw.finish_publishing()) def _then_again(results): self.failUnless(results[0]) - read = self.ss.remote_slot_readv + read = self.ss.slot_readv self.failUnlessEqual(read(b"si1", [0], [(1, 8)]), {0: [struct.pack(">Q", 1)]}) self.failUnlessEqual(read(b"si1", [0], [(9, len(data) - 9)]), diff --git a/src/allmydata/test/test_storage_web.py b/src/allmydata/test/test_storage_web.py index 38e380223..5d72fbd82 100644 --- a/src/allmydata/test/test_storage_web.py +++ b/src/allmydata/test/test_storage_web.py @@ -38,7 +38,6 @@ from allmydata.web.storage import ( StorageStatusElement, remove_prefix ) -from .common_util import FakeCanary from .common_web import ( render, @@ -289,28 +288,27 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin): mutable_si_3, rs3, cs3, we3 = make_mutable(b"\x03" * 16) rs3a, cs3a = make_extra_lease(mutable_si_3, 1) sharenums = [0] - canary = FakeCanary() # note: 'tahoe debug dump-share' will not handle this file, since the # inner contents are not a valid CHK share data = b"\xff" * 1000 - a,w = ss.remote_allocate_buckets(immutable_si_0, rs0, cs0, sharenums, - 1000, canary) - w[0].remote_write(0, data) - w[0].remote_close() + a,w = ss.allocate_buckets(immutable_si_0, rs0, cs0, sharenums, + 1000) + w[0].write(0, data) + w[0].close() - a,w = ss.remote_allocate_buckets(immutable_si_1, rs1, cs1, sharenums, - 1000, canary) - w[0].remote_write(0, data) - w[0].remote_close() - ss.remote_add_lease(immutable_si_1, rs1a, cs1a) + a,w = ss.allocate_buckets(immutable_si_1, rs1, cs1, sharenums, + 1000) + w[0].write(0, data) + w[0].close() + ss.add_lease(immutable_si_1, rs1a, cs1a) - writev = ss.remote_slot_testv_and_readv_and_writev + writev = ss.slot_testv_and_readv_and_writev writev(mutable_si_2, (we2, rs2, cs2), {0: ([], [(0,data)], len(data))}, []) writev(mutable_si_3, (we3, rs3, cs3), {0: ([], [(0,data)], len(data))}, []) - ss.remote_add_lease(mutable_si_3, rs3a, cs3a) + ss.add_lease(mutable_si_3, rs3a, cs3a) self.sis = [immutable_si_0, immutable_si_1, mutable_si_2, mutable_si_3] self.renew_secrets = [rs0, rs1, rs1a, rs2, rs3, rs3a] From 53ff16f1a43717dd86e9582c379beb4d92ea17e9 Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 2 Dec 2021 12:56:52 -0700 Subject: [PATCH 0498/2309] rst for news --- newsfragments/3825.security | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/newsfragments/3825.security b/newsfragments/3825.security index df83821de..3d112dd49 100644 --- a/newsfragments/3825.security +++ b/newsfragments/3825.security @@ -1,6 +1,8 @@ The lease-checker now uses JSON instead of pickle to serialize its state. tahoe will now refuse to run until you either delete all pickle files or -migrate them using the new command: +migrate them using the new command:: tahoe admin migrate-crawler + +This will migrate all crawler-related pickle files. From 314b20291442bd485b9919081611efb9c145c277 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 3 Dec 2021 12:58:12 -0500 Subject: [PATCH 0499/2309] Ignore another field which can change. --- src/allmydata/test/test_storage_http.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index e30eb24c7..23a3e3ea6 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -64,14 +64,21 @@ class HTTPTests(TestCase): """ The client can return the version. - We ignore available disk space since that might change across calls. + We ignore available disk space and max immutable share size, since that + might change across calls. """ version = yield self.client.get_version() version[b"http://allmydata.org/tahoe/protocols/storage/v1"].pop( b"available-space" ) + version[b"http://allmydata.org/tahoe/protocols/storage/v1"].pop( + b"maximum-immutable-share-size" + ) expected_version = self.storage_server.remote_get_version() expected_version[b"http://allmydata.org/tahoe/protocols/storage/v1"].pop( b"available-space" ) + expected_version[b"http://allmydata.org/tahoe/protocols/storage/v1"].pop( + b"maximum-immutable-share-size" + ) self.assertEqual(version, expected_version) From 90f8480cf0c2fb97fd5e420c6e9ae029dad203b7 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 3 Dec 2021 13:09:27 -0500 Subject: [PATCH 0500/2309] Make more of the unittests pass again with the StorageServer factoring. --- src/allmydata/storage/http_server.py | 2 +- src/allmydata/storage/server.py | 1 + src/allmydata/test/no_network.py | 6 ++++-- src/allmydata/test/test_checker.py | 6 +++--- src/allmydata/test/test_client.py | 2 +- src/allmydata/test/test_crawler.py | 14 +++++++------- src/allmydata/test/test_helper.py | 7 ++++--- src/allmydata/test/test_hung_server.py | 4 ++-- src/allmydata/test/test_storage_http.py | 2 +- 9 files changed, 24 insertions(+), 20 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 327892ecd..6297ef484 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -91,4 +91,4 @@ class HTTPServer(object): @_authorized_route(_app, "/v1/version", methods=["GET"]) def version(self, request, authorization): - return self._cbor(request, self._storage_server.remote_get_version()) + return self._cbor(request, self._storage_server.get_version()) diff --git a/src/allmydata/storage/server.py b/src/allmydata/storage/server.py index bb116ce8e..0df9f23d8 100644 --- a/src/allmydata/storage/server.py +++ b/src/allmydata/storage/server.py @@ -64,6 +64,7 @@ class StorageServer(service.MultiService): """ Implement the business logic for the storage server. """ + name = "storage" LeaseCheckerClass = LeaseCheckingCrawler def __init__(self, storedir, nodeid, reserved_space=0, diff --git a/src/allmydata/test/no_network.py b/src/allmydata/test/no_network.py index b9fa99005..97cb371e6 100644 --- a/src/allmydata/test/no_network.py +++ b/src/allmydata/test/no_network.py @@ -50,7 +50,9 @@ from allmydata.util.assertutil import _assert from allmydata import uri as tahoe_uri from allmydata.client import _Client -from allmydata.storage.server import StorageServer, storage_index_to_dir +from allmydata.storage.server import ( + StorageServer, storage_index_to_dir, FoolscapStorageServer, +) from allmydata.util import fileutil, idlib, hashutil from allmydata.util.hashutil import permute_server_hash from allmydata.util.fileutil import abspath_expanduser_unicode @@ -417,7 +419,7 @@ class NoNetworkGrid(service.MultiService): ss.setServiceParent(middleman) serverid = ss.my_nodeid self.servers_by_number[i] = ss - wrapper = wrap_storage_server(ss) + wrapper = wrap_storage_server(FoolscapStorageServer(ss)) self.wrappers_by_id[serverid] = wrapper self.proxies_by_id[serverid] = NoNetworkServer(serverid, wrapper) self.rebuild_serverlist() diff --git a/src/allmydata/test/test_checker.py b/src/allmydata/test/test_checker.py index f56ecd089..3d64d4976 100644 --- a/src/allmydata/test/test_checker.py +++ b/src/allmydata/test/test_checker.py @@ -773,13 +773,13 @@ class AddLease(GridTestMixin, unittest.TestCase): d.addCallback(_check_cr, "mutable-normal") really_did_break = [] - # now break the server's remote_add_lease call + # now break the server's add_lease call def _break_add_lease(ign): def broken_add_lease(*args, **kwargs): really_did_break.append(1) raise KeyError("intentional failure, should be ignored") - assert self.g.servers_by_number[0].remote_add_lease - self.g.servers_by_number[0].remote_add_lease = broken_add_lease + assert self.g.servers_by_number[0].add_lease + self.g.servers_by_number[0].add_lease = broken_add_lease d.addCallback(_break_add_lease) # and confirm that the files still look healthy diff --git a/src/allmydata/test/test_client.py b/src/allmydata/test/test_client.py index a2572e735..c65a2fa2c 100644 --- a/src/allmydata/test/test_client.py +++ b/src/allmydata/test/test_client.py @@ -601,7 +601,7 @@ class Basic(testutil.ReallyEqualMixin, unittest.TestCase): "enabled = true\n") c = yield client.create_client(basedir) ss = c.getServiceNamed("storage") - verdict = ss.remote_get_version() + verdict = ss.get_version() self.failUnlessReallyEqual(verdict[b"application-version"], allmydata.__full_version__.encode("ascii")) self.failIfEqual(str(allmydata.__version__), "unknown") diff --git a/src/allmydata/test/test_crawler.py b/src/allmydata/test/test_crawler.py index a9be90c43..80d732986 100644 --- a/src/allmydata/test/test_crawler.py +++ b/src/allmydata/test/test_crawler.py @@ -27,7 +27,7 @@ from allmydata.util import fileutil, hashutil, pollmixin from allmydata.storage.server import StorageServer, si_b2a from allmydata.storage.crawler import ShareCrawler, TimeSliceExceeded -from allmydata.test.common_util import StallMixin, FakeCanary +from allmydata.test.common_util import StallMixin class BucketEnumeratingCrawler(ShareCrawler): cpu_slice = 500 # make sure it can complete in a single slice @@ -124,12 +124,12 @@ class Basic(unittest.TestCase, StallMixin, pollmixin.PollMixin): def write(self, i, ss, serverid, tail=0): si = self.si(i) si = si[:-1] + bytes(bytearray((tail,))) - had,made = ss.remote_allocate_buckets(si, - self.rs(i, serverid), - self.cs(i, serverid), - set([0]), 99, FakeCanary()) - made[0].remote_write(0, b"data") - made[0].remote_close() + had,made = ss.allocate_buckets(si, + self.rs(i, serverid), + self.cs(i, serverid), + set([0]), 99) + made[0].write(0, b"data") + made[0].close() return si_b2a(si) def test_immediate(self): diff --git a/src/allmydata/test/test_helper.py b/src/allmydata/test/test_helper.py index 3faffbe0d..933a2b591 100644 --- a/src/allmydata/test/test_helper.py +++ b/src/allmydata/test/test_helper.py @@ -39,6 +39,7 @@ from allmydata.crypto import aes from allmydata.storage.server import ( si_b2a, StorageServer, + FoolscapStorageServer, ) from allmydata.storage_client import StorageFarmBroker from allmydata.immutable.layout import ( @@ -427,7 +428,7 @@ class CHKCheckerAndUEBFetcherTests(SyncTestCase): """ storage_index = b"a" * 16 serverid = b"b" * 20 - storage = StorageServer(self.mktemp(), serverid) + storage = FoolscapStorageServer(StorageServer(self.mktemp(), serverid)) rref_without_ueb = LocalWrapper(storage, fireNow) yield write_bad_share(rref_without_ueb, storage_index) server_without_ueb = NoNetworkServer(serverid, rref_without_ueb) @@ -451,7 +452,7 @@ class CHKCheckerAndUEBFetcherTests(SyncTestCase): """ storage_index = b"a" * 16 serverid = b"b" * 20 - storage = StorageServer(self.mktemp(), serverid) + storage = FoolscapStorageServer(StorageServer(self.mktemp(), serverid)) rref_with_ueb = LocalWrapper(storage, fireNow) ueb = { "needed_shares": 2, @@ -487,7 +488,7 @@ class CHKCheckerAndUEBFetcherTests(SyncTestCase): in [b"b", b"c"] ) storages = list( - StorageServer(self.mktemp(), serverid) + FoolscapStorageServer(StorageServer(self.mktemp(), serverid)) for serverid in serverids ) diff --git a/src/allmydata/test/test_hung_server.py b/src/allmydata/test/test_hung_server.py index 490315500..162b1d79c 100644 --- a/src/allmydata/test/test_hung_server.py +++ b/src/allmydata/test/test_hung_server.py @@ -73,7 +73,7 @@ class HungServerDownloadTest(GridTestMixin, ShouldFailMixin, PollMixin, def _copy_share(self, share, to_server): (sharenum, sharefile) = share (id, ss) = to_server - shares_dir = os.path.join(ss.original.storedir, "shares") + shares_dir = os.path.join(ss.original._server.storedir, "shares") si = uri.from_string(self.uri).get_storage_index() si_dir = os.path.join(shares_dir, storage_index_to_dir(si)) if not os.path.exists(si_dir): @@ -82,7 +82,7 @@ class HungServerDownloadTest(GridTestMixin, ShouldFailMixin, PollMixin, shutil.copy(sharefile, new_sharefile) self.shares = self.find_uri_shares(self.uri) # Make sure that the storage server has the share. - self.failUnless((sharenum, ss.original.my_nodeid, new_sharefile) + self.failUnless((sharenum, ss.original._server.my_nodeid, new_sharefile) in self.shares) def _corrupt_share(self, share, corruptor_func): diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 442e154a0..6504ac97f 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -65,5 +65,5 @@ class HTTPTests(TestCase): The client can return the version. """ version = yield self.client.get_version() - expected_version = self.storage_server.remote_get_version() + expected_version = self.storage_server.get_version() self.assertEqual(version, expected_version) From 5bb6fbc51f4d1d1d871410aba2cc91a09a2bb3ab Mon Sep 17 00:00:00 2001 From: meejah Date: Sat, 4 Dec 2021 10:14:31 -0700 Subject: [PATCH 0501/2309] merge errors --- src/allmydata/storage/server.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/storage/server.py b/src/allmydata/storage/server.py index acca83d6a..ac8c41c07 100644 --- a/src/allmydata/storage/server.py +++ b/src/allmydata/storage/server.py @@ -337,7 +337,7 @@ class StorageServer(service.MultiService, Referenceable): alreadygot[shnum] = ShareFile(fn) if renew_leases: sf = ShareFile(fn) - sf.add_or_renew_lease(lease_info) + sf.add_or_renew_lease(remaining_space, lease_info) for shnum in sharenums: incominghome = os.path.join(self.incomingdir, si_dir, "%d" % shnum) @@ -411,7 +411,7 @@ class StorageServer(service.MultiService, Referenceable): renew_secret, cancel_secret, new_expire_time, self.my_nodeid) for sf in self._iter_share_files(storage_index): - sf.add_or_renew_lease(lease_info) + sf.add_or_renew_lease(self.get_available_space(), lease_info) self.add_latency("add-lease", self._clock.seconds() - start) return None From 50cdd9bd9659ad9886de0ca021b34ef3028f411d Mon Sep 17 00:00:00 2001 From: meejah Date: Sat, 4 Dec 2021 17:20:10 -0700 Subject: [PATCH 0502/2309] unused --- src/allmydata/storage/server.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/allmydata/storage/server.py b/src/allmydata/storage/server.py index ac8c41c07..9a9b3e624 100644 --- a/src/allmydata/storage/server.py +++ b/src/allmydata/storage/server.py @@ -15,7 +15,6 @@ else: from typing import Dict import os, re -import six from foolscap.api import Referenceable from foolscap.ipb import IRemoteReference From 402d11ecd61cd821b0d6afe8f492253106747759 Mon Sep 17 00:00:00 2001 From: meejah Date: Sun, 5 Dec 2021 00:39:31 -0700 Subject: [PATCH 0503/2309] update NEWS.txt for release --- NEWS.rst | 75 ++++++++++++++++++++++++++++++++ newsfragments/3525.minor | 0 newsfragments/3527.minor | 0 newsfragments/3735.feature | 1 - newsfragments/3754.minor | 0 newsfragments/3758.minor | 0 newsfragments/3784.minor | 0 newsfragments/3786.feature | 1 - newsfragments/3792.minor | 0 newsfragments/3793.minor | 0 newsfragments/3795.minor | 0 newsfragments/3797.minor | 0 newsfragments/3798.minor | 0 newsfragments/3799.minor | 0 newsfragments/3800.minor | 0 newsfragments/3801.bugfix | 1 - newsfragments/3805.minor | 0 newsfragments/3806.minor | 0 newsfragments/3807.feature | 1 - newsfragments/3808.installation | 1 - newsfragments/3810.minor | 0 newsfragments/3812.minor | 0 newsfragments/3814.removed | 1 - newsfragments/3815.documentation | 1 - newsfragments/3819.security | 1 - newsfragments/3820.minor | 0 newsfragments/3821.security | 2 - newsfragments/3822.security | 2 - newsfragments/3823.security | 4 -- newsfragments/3824.security | 1 - newsfragments/3825.security | 8 ---- newsfragments/3827.security | 4 -- newsfragments/3829.minor | 0 newsfragments/3830.minor | 0 newsfragments/3831.minor | 0 newsfragments/3832.minor | 0 newsfragments/3833.minor | 0 newsfragments/3834.minor | 0 newsfragments/3835.minor | 0 newsfragments/3836.minor | 0 newsfragments/3837.other | 1 - newsfragments/3838.minor | 0 newsfragments/3839.security | 1 - newsfragments/3841.security | 1 - newsfragments/3842.minor | 0 newsfragments/3843.minor | 0 newsfragments/3847.minor | 0 47 files changed, 75 insertions(+), 32 deletions(-) delete mode 100644 newsfragments/3525.minor delete mode 100644 newsfragments/3527.minor delete mode 100644 newsfragments/3735.feature delete mode 100644 newsfragments/3754.minor delete mode 100644 newsfragments/3758.minor delete mode 100644 newsfragments/3784.minor delete mode 100644 newsfragments/3786.feature delete mode 100644 newsfragments/3792.minor delete mode 100644 newsfragments/3793.minor delete mode 100644 newsfragments/3795.minor delete mode 100644 newsfragments/3797.minor delete mode 100644 newsfragments/3798.minor delete mode 100644 newsfragments/3799.minor delete mode 100644 newsfragments/3800.minor delete mode 100644 newsfragments/3801.bugfix delete mode 100644 newsfragments/3805.minor delete mode 100644 newsfragments/3806.minor delete mode 100644 newsfragments/3807.feature delete mode 100644 newsfragments/3808.installation delete mode 100644 newsfragments/3810.minor delete mode 100644 newsfragments/3812.minor delete mode 100644 newsfragments/3814.removed delete mode 100644 newsfragments/3815.documentation delete mode 100644 newsfragments/3819.security delete mode 100644 newsfragments/3820.minor delete mode 100644 newsfragments/3821.security delete mode 100644 newsfragments/3822.security delete mode 100644 newsfragments/3823.security delete mode 100644 newsfragments/3824.security delete mode 100644 newsfragments/3825.security delete mode 100644 newsfragments/3827.security delete mode 100644 newsfragments/3829.minor delete mode 100644 newsfragments/3830.minor delete mode 100644 newsfragments/3831.minor delete mode 100644 newsfragments/3832.minor delete mode 100644 newsfragments/3833.minor delete mode 100644 newsfragments/3834.minor delete mode 100644 newsfragments/3835.minor delete mode 100644 newsfragments/3836.minor delete mode 100644 newsfragments/3837.other delete mode 100644 newsfragments/3838.minor delete mode 100644 newsfragments/3839.security delete mode 100644 newsfragments/3841.security delete mode 100644 newsfragments/3842.minor delete mode 100644 newsfragments/3843.minor delete mode 100644 newsfragments/3847.minor diff --git a/NEWS.rst b/NEWS.rst index e4fef833a..697c44c30 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -5,6 +5,81 @@ User-Visible Changes in Tahoe-LAFS ================================== .. towncrier start line +Release 1.16.0.post463 (2021-12-05)Release 1.16.0.post463 (2021-12-05) +''''''''''''''''''''''''''''''''''' + +Security-related Changes +------------------------ + +- The introducer server no longer writes the sensitive introducer fURL value to its log at startup time. Instead it writes the well-known path of the file from which this value can be read. (`#3819 `_) +- The storage protocol operation ``add_lease`` now safely rejects an attempt to add a 4,294,967,296th lease to an immutable share. + Previously this failed with an error after recording the new lease in the share file, resulting in the share file losing track of a one previous lease. (`#3821 `_) +- The storage protocol operation ``readv`` now safely rejects attempts to read negative lengths. + Previously these read requests were satisfied with the complete contents of the share file (including trailing metadata) starting from the specified offset. (`#3822 `_) +- The storage server implementation now respects the ``reserved_space`` configuration value when writing lease information and recording corruption advisories. + Previously, new leases could be created and written to disk even when the storage server had less remaining space than the configured reserve space value. + Now this operation will fail with an exception and the lease will not be created. + Similarly, if there is no space available, corruption advisories will be logged but not written to disk. (`#3823 `_) +- The storage server implementation no longer records corruption advisories about storage indexes for which it holds no shares. (`#3824 `_) +- The lease-checker now uses JSON instead of pickle to serialize its state. + + tahoe will now refuse to run until you either delete all pickle files or + migrate them using the new command:: + + tahoe admin migrate-crawler + + This will migrate all crawler-related pickle files. (`#3825 `_) +- The SFTP server no longer accepts password-based credentials for authentication. + Public/private key-based credentials are now the only supported authentication type. + This removes plaintext password storage from the SFTP credentials file. + It also removes a possible timing side-channel vulnerability which might have allowed attackers to discover an account's plaintext password. (`#3827 `_) +- The storage server now keeps hashes of lease renew and cancel secrets for immutable share files instead of keeping the original secrets. (`#3839 `_) +- The storage server now keeps hashes of lease renew and cancel secrets for mutable share files instead of keeping the original secrets. (`#3841 `_) + + +Features +-------- + +- Tahoe-LAFS releases now have just a .tar.gz source release and a (universal) wheel (`#3735 `_) +- tahoe-lafs now provides its statistics also in OpenMetrics format (for Prometheus et. al.) at `/statistics?t=openmetrics`. (`#3786 `_) +- If uploading an immutable hasn't had a write for 30 minutes, the storage server will abort the upload. (`#3807 `_) + + +Bug Fixes +--------- + +- When uploading an immutable, overlapping writes that include conflicting data are rejected. In practice, this likely didn't happen in real-world usage. (`#3801 `_) + + +Dependency/Installation Changes +------------------------------- + +- Tahoe-LAFS now supports running on NixOS 21.05 with Python 3. (`#3808 `_) + + +Documentation Changes +--------------------- + +- The news file for future releases will include a section for changes with a security impact. (`#3815 `_) + + +Removed Features +---------------- + +- The little-used "control port" has been removed from all node types. (`#3814 `_) + + +Other Changes +------------- + +- Tahoe-LAFS no longer runs its Tor integration test suite on Python 2 due to the increased complexity of obtaining compatible versions of necessary dependencies. (`#3837 `_) + + +Misc/Other +---------- + +- `#3525 `_, `#3527 `_, `#3754 `_, `#3758 `_, `#3784 `_, `#3792 `_, `#3793 `_, `#3795 `_, `#3797 `_, `#3798 `_, `#3799 `_, `#3800 `_, `#3805 `_, `#3806 `_, `#3810 `_, `#3812 `_, `#3820 `_, `#3829 `_, `#3830 `_, `#3831 `_, `#3832 `_, `#3833 `_, `#3834 `_, `#3835 `_, `#3836 `_, `#3838 `_, `#3842 `_, `#3843 `_, `#3847 `_ + Release 1.16.0 (2021-09-17) ''''''''''''''''''''''''''' diff --git a/newsfragments/3525.minor b/newsfragments/3525.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3527.minor b/newsfragments/3527.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3735.feature b/newsfragments/3735.feature deleted file mode 100644 index 5a86d5547..000000000 --- a/newsfragments/3735.feature +++ /dev/null @@ -1 +0,0 @@ -Tahoe-LAFS releases now have just a .tar.gz source release and a (universal) wheel diff --git a/newsfragments/3754.minor b/newsfragments/3754.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3758.minor b/newsfragments/3758.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3784.minor b/newsfragments/3784.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3786.feature b/newsfragments/3786.feature deleted file mode 100644 index ecbfc0372..000000000 --- a/newsfragments/3786.feature +++ /dev/null @@ -1 +0,0 @@ -tahoe-lafs now provides its statistics also in OpenMetrics format (for Prometheus et. al.) at `/statistics?t=openmetrics`. diff --git a/newsfragments/3792.minor b/newsfragments/3792.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3793.minor b/newsfragments/3793.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3795.minor b/newsfragments/3795.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3797.minor b/newsfragments/3797.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3798.minor b/newsfragments/3798.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3799.minor b/newsfragments/3799.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3800.minor b/newsfragments/3800.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3801.bugfix b/newsfragments/3801.bugfix deleted file mode 100644 index 504b3999d..000000000 --- a/newsfragments/3801.bugfix +++ /dev/null @@ -1 +0,0 @@ -When uploading an immutable, overlapping writes that include conflicting data are rejected. In practice, this likely didn't happen in real-world usage. \ No newline at end of file diff --git a/newsfragments/3805.minor b/newsfragments/3805.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3806.minor b/newsfragments/3806.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3807.feature b/newsfragments/3807.feature deleted file mode 100644 index f82363ffd..000000000 --- a/newsfragments/3807.feature +++ /dev/null @@ -1 +0,0 @@ -If uploading an immutable hasn't had a write for 30 minutes, the storage server will abort the upload. \ No newline at end of file diff --git a/newsfragments/3808.installation b/newsfragments/3808.installation deleted file mode 100644 index 157f08a0c..000000000 --- a/newsfragments/3808.installation +++ /dev/null @@ -1 +0,0 @@ -Tahoe-LAFS now supports running on NixOS 21.05 with Python 3. diff --git a/newsfragments/3810.minor b/newsfragments/3810.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3812.minor b/newsfragments/3812.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3814.removed b/newsfragments/3814.removed deleted file mode 100644 index 939d20ffc..000000000 --- a/newsfragments/3814.removed +++ /dev/null @@ -1 +0,0 @@ -The little-used "control port" has been removed from all node types. diff --git a/newsfragments/3815.documentation b/newsfragments/3815.documentation deleted file mode 100644 index 7abc70bd1..000000000 --- a/newsfragments/3815.documentation +++ /dev/null @@ -1 +0,0 @@ -The news file for future releases will include a section for changes with a security impact. \ No newline at end of file diff --git a/newsfragments/3819.security b/newsfragments/3819.security deleted file mode 100644 index 975fd0035..000000000 --- a/newsfragments/3819.security +++ /dev/null @@ -1 +0,0 @@ -The introducer server no longer writes the sensitive introducer fURL value to its log at startup time. Instead it writes the well-known path of the file from which this value can be read. diff --git a/newsfragments/3820.minor b/newsfragments/3820.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3821.security b/newsfragments/3821.security deleted file mode 100644 index 75d9904a2..000000000 --- a/newsfragments/3821.security +++ /dev/null @@ -1,2 +0,0 @@ -The storage protocol operation ``add_lease`` now safely rejects an attempt to add a 4,294,967,296th lease to an immutable share. -Previously this failed with an error after recording the new lease in the share file, resulting in the share file losing track of a one previous lease. diff --git a/newsfragments/3822.security b/newsfragments/3822.security deleted file mode 100644 index 5d6c07ab5..000000000 --- a/newsfragments/3822.security +++ /dev/null @@ -1,2 +0,0 @@ -The storage protocol operation ``readv`` now safely rejects attempts to read negative lengths. -Previously these read requests were satisfied with the complete contents of the share file (including trailing metadata) starting from the specified offset. diff --git a/newsfragments/3823.security b/newsfragments/3823.security deleted file mode 100644 index ba2bbd741..000000000 --- a/newsfragments/3823.security +++ /dev/null @@ -1,4 +0,0 @@ -The storage server implementation now respects the ``reserved_space`` configuration value when writing lease information and recording corruption advisories. -Previously, new leases could be created and written to disk even when the storage server had less remaining space than the configured reserve space value. -Now this operation will fail with an exception and the lease will not be created. -Similarly, if there is no space available, corruption advisories will be logged but not written to disk. diff --git a/newsfragments/3824.security b/newsfragments/3824.security deleted file mode 100644 index b29b2acc8..000000000 --- a/newsfragments/3824.security +++ /dev/null @@ -1 +0,0 @@ -The storage server implementation no longer records corruption advisories about storage indexes for which it holds no shares. diff --git a/newsfragments/3825.security b/newsfragments/3825.security deleted file mode 100644 index 3d112dd49..000000000 --- a/newsfragments/3825.security +++ /dev/null @@ -1,8 +0,0 @@ -The lease-checker now uses JSON instead of pickle to serialize its state. - -tahoe will now refuse to run until you either delete all pickle files or -migrate them using the new command:: - - tahoe admin migrate-crawler - -This will migrate all crawler-related pickle files. diff --git a/newsfragments/3827.security b/newsfragments/3827.security deleted file mode 100644 index 4fee19c76..000000000 --- a/newsfragments/3827.security +++ /dev/null @@ -1,4 +0,0 @@ -The SFTP server no longer accepts password-based credentials for authentication. -Public/private key-based credentials are now the only supported authentication type. -This removes plaintext password storage from the SFTP credentials file. -It also removes a possible timing side-channel vulnerability which might have allowed attackers to discover an account's plaintext password. diff --git a/newsfragments/3829.minor b/newsfragments/3829.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3830.minor b/newsfragments/3830.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3831.minor b/newsfragments/3831.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3832.minor b/newsfragments/3832.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3833.minor b/newsfragments/3833.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3834.minor b/newsfragments/3834.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3835.minor b/newsfragments/3835.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3836.minor b/newsfragments/3836.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3837.other b/newsfragments/3837.other deleted file mode 100644 index a9e4e6986..000000000 --- a/newsfragments/3837.other +++ /dev/null @@ -1 +0,0 @@ -Tahoe-LAFS no longer runs its Tor integration test suite on Python 2 due to the increased complexity of obtaining compatible versions of necessary dependencies. diff --git a/newsfragments/3838.minor b/newsfragments/3838.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3839.security b/newsfragments/3839.security deleted file mode 100644 index 1ae054542..000000000 --- a/newsfragments/3839.security +++ /dev/null @@ -1 +0,0 @@ -The storage server now keeps hashes of lease renew and cancel secrets for immutable share files instead of keeping the original secrets. diff --git a/newsfragments/3841.security b/newsfragments/3841.security deleted file mode 100644 index 867322e0a..000000000 --- a/newsfragments/3841.security +++ /dev/null @@ -1 +0,0 @@ -The storage server now keeps hashes of lease renew and cancel secrets for mutable share files instead of keeping the original secrets. \ No newline at end of file diff --git a/newsfragments/3842.minor b/newsfragments/3842.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3843.minor b/newsfragments/3843.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3847.minor b/newsfragments/3847.minor deleted file mode 100644 index e69de29bb..000000000 From b8d00ab04a1ae6309d3dd5cf93b937d759f3c9d6 Mon Sep 17 00:00:00 2001 From: meejah Date: Sun, 5 Dec 2021 00:50:22 -0700 Subject: [PATCH 0504/2309] update release notes --- NEWS.rst | 6 ++++-- relnotes.txt | 39 ++++++++++++++++----------------------- 2 files changed, 20 insertions(+), 25 deletions(-) diff --git a/NEWS.rst b/NEWS.rst index 697c44c30..15cb9459d 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -5,8 +5,10 @@ User-Visible Changes in Tahoe-LAFS ================================== .. towncrier start line -Release 1.16.0.post463 (2021-12-05)Release 1.16.0.post463 (2021-12-05) -''''''''''''''''''''''''''''''''''' + + +Release 1.17.0 (2021-12-06) +''''''''''''''''''''''''''' Security-related Changes ------------------------ diff --git a/relnotes.txt b/relnotes.txt index 2748bc4fa..dff4f192e 100644 --- a/relnotes.txt +++ b/relnotes.txt @@ -1,6 +1,6 @@ -ANNOUNCING Tahoe, the Least-Authority File Store, v1.16.0 +ANNOUNCING Tahoe, the Least-Authority File Store, v1.17.0 -The Tahoe-LAFS team is pleased to announce version 1.16.0 of +The Tahoe-LAFS team is pleased to announce version 1.17.0 of Tahoe-LAFS, an extremely reliable decentralized storage system. Get it with "pip install tahoe-lafs", or download a tarball here: @@ -15,24 +15,17 @@ unique security and fault-tolerance properties: https://tahoe-lafs.readthedocs.org/en/latest/about.html -The previous stable release of Tahoe-LAFS was v1.15.1, released on -March 23rd, 2021. +The previous stable release of Tahoe-LAFS was v1.16.0, released on +October 19, 2021. -The major change in this release is the completion of the Python 3 -port -- while maintaining support for Python 2. A future release will -remove Python 2 support. +This release fixes several security issues raised as part of an audit +by Cure53. We developed fixes for these issues in a private +repository. Shortly after this release, public tickets will be updated +with further information (along with, of course, all the code). -The previously deprecated subcommands "start", "stop", "restart" and -"daemonize" have been removed. You must now use "tahoe run" (possibly -along with your favourite daemonization software). +There is also OpenMetrics support now and several bug fixes. -Several features are now removed: the Account Server, stats-gatherer -and FTP support. - -There are several dependency changes that will be interesting for -distribution maintainers. - -In all, 240 issues have been fixed since the last release. +In all, 46 issues have been fixed since the last release. Please see ``NEWS.rst`` for a more complete list of changes. @@ -151,19 +144,19 @@ solely as a labor of love by volunteers. Thank you very much to the team of "hackers in the public interest" who make Tahoe-LAFS possible. -fenn-cs + meejah +meejah on behalf of the Tahoe-LAFS team -October 19, 2021 +December 6, 2021 Planet Earth -[1] https://github.com/tahoe-lafs/tahoe-lafs/blob/tahoe-lafs-1.16.0/NEWS.rst +[1] https://github.com/tahoe-lafs/tahoe-lafs/blob/tahoe-lafs-1.17.0/NEWS.rst [2] https://github.com/tahoe-lafs/tahoe-lafs/blob/master/docs/known_issues.rst [3] https://tahoe-lafs.org/trac/tahoe-lafs/wiki/RelatedProjects -[4] https://github.com/tahoe-lafs/tahoe-lafs/blob/tahoe-lafs-1.16.0/COPYING.GPL -[5] https://github.com/tahoe-lafs/tahoe-lafs/blob/tahoe-lafs-1.16.0/COPYING.TGPPL.rst -[6] https://tahoe-lafs.readthedocs.org/en/tahoe-lafs-1.16.0/INSTALL.html +[4] https://github.com/tahoe-lafs/tahoe-lafs/blob/tahoe-lafs-1.17.0/COPYING.GPL +[5] https://github.com/tahoe-lafs/tahoe-lafs/blob/tahoe-lafs-1.17.0/COPYING.TGPPL.rst +[6] https://tahoe-lafs.readthedocs.org/en/tahoe-lafs-1.17.0/INSTALL.html [7] https://lists.tahoe-lafs.org/mailman/listinfo/tahoe-dev [8] https://tahoe-lafs.org/trac/tahoe-lafs/roadmap [9] https://github.com/tahoe-lafs/tahoe-lafs/blob/master/CREDITS From 95fdaf286e2f195672d4f2cd3371ed41ee49aae1 Mon Sep 17 00:00:00 2001 From: meejah Date: Sun, 5 Dec 2021 00:51:13 -0700 Subject: [PATCH 0505/2309] update nix version --- nix/tahoe-lafs.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nix/tahoe-lafs.nix b/nix/tahoe-lafs.nix index 59864d36d..04d6c4163 100644 --- a/nix/tahoe-lafs.nix +++ b/nix/tahoe-lafs.nix @@ -7,7 +7,7 @@ , html5lib, pyutil, distro, configparser, klein, cbor2 }: python.pkgs.buildPythonPackage rec { - # Most of the time this is not exactly the release version (eg 1.16.0). + # Most of the time this is not exactly the release version (eg 1.17.0). # Give it a `post` component to make it look newer than the release version # and we'll bump this up at the time of each release. # @@ -20,7 +20,7 @@ python.pkgs.buildPythonPackage rec { # is not a reproducable artifact (in the sense of "reproducable builds") so # it is excluded from the source tree by default. When it is included, the # package tends to be frequently spuriously rebuilt. - version = "1.16.0.post1"; + version = "1.17.0.post1"; name = "tahoe-lafs-${version}"; src = lib.cleanSourceWith { src = ../.; From a8bdb8dcbb66bfa75389816817e25b6f9ec5d74b Mon Sep 17 00:00:00 2001 From: meejah Date: Sun, 5 Dec 2021 00:53:50 -0700 Subject: [PATCH 0506/2309] add Florian --- CREDITS | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CREDITS b/CREDITS index 8a6e876ec..89e1468aa 100644 --- a/CREDITS +++ b/CREDITS @@ -260,3 +260,7 @@ D: Community-manager and documentation improvements N: Yash Nayani E: yashaswi.nram@gmail.com D: Installation Guide improvements + +N: Florian Sesser +E: florian@private.storage +D: OpenMetrics support \ No newline at end of file From 5f6579d44622b0774aec8ba06b67f2d4f2ee7af7 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 6 Dec 2021 12:47:37 -0500 Subject: [PATCH 0507/2309] hew closer to security/master version of these lines --- src/allmydata/storage/server.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/allmydata/storage/server.py b/src/allmydata/storage/server.py index 9a9b3e624..80b337d36 100644 --- a/src/allmydata/storage/server.py +++ b/src/allmydata/storage/server.py @@ -334,9 +334,8 @@ class StorageServer(service.MultiService, Referenceable): # file, they'll want us to hold leases for this file. for (shnum, fn) in self._get_bucket_shares(storage_index): alreadygot[shnum] = ShareFile(fn) - if renew_leases: - sf = ShareFile(fn) - sf.add_or_renew_lease(remaining_space, lease_info) + if renew_leases: + self._add_or_renew_leases(alreadygot.values(), lease_info) for shnum in sharenums: incominghome = os.path.join(self.incomingdir, si_dir, "%d" % shnum) @@ -409,8 +408,10 @@ class StorageServer(service.MultiService, Referenceable): lease_info = LeaseInfo(owner_num, renew_secret, cancel_secret, new_expire_time, self.my_nodeid) - for sf in self._iter_share_files(storage_index): - sf.add_or_renew_lease(self.get_available_space(), lease_info) + self._add_or_renew_leases( + self._iter_share_files(storage_index), + lease_info, + ) self.add_latency("add-lease", self._clock.seconds() - start) return None From 91dd70ad2926d1759402f9ba9214ba21063337e2 Mon Sep 17 00:00:00 2001 From: fenn-cs Date: Mon, 6 Dec 2021 22:51:44 +0100 Subject: [PATCH 0508/2309] fixed typo, made version name inline literal Signed-off-by: fenn-cs --- docs/release-checklist.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/release-checklist.rst b/docs/release-checklist.rst index 796be75ba..8bba63e79 100644 --- a/docs/release-checklist.rst +++ b/docs/release-checklist.rst @@ -116,8 +116,8 @@ they will need to evaluate which contributors' signatures they trust. - ``git tag -s -u 0xE34E62D06D0E69CFCA4179FFBDE0D31D68666A7A -m "release Tahoe-LAFS-1.16.0rc0" tahoe-lafs-1.16.0rc0`` .. note:: - - Replace the key-id above with your own, which can simply be your email if's attached your fingerprint. - - Don't forget to put the correct tag message and name in this example the tag message is "release Tahoe-LAFS-1.16.0rc0" and the tag name is `tahoe-lafs-1.16.0rc0` + - Replace the key-id above with your own, which can simply be your email if it's attached to your fingerprint. + - Don't forget to put the correct tag message and name. In this example, the tag message is "release Tahoe-LAFS-1.16.0rc0" and the tag name is ``tahoe-lafs-1.16.0rc0`` - build all code locally From 911eb6cf9a4a5eccb39a40bc128364f4778c8e96 Mon Sep 17 00:00:00 2001 From: fenn-cs Date: Tue, 7 Dec 2021 11:10:51 +0100 Subject: [PATCH 0509/2309] add gpg-setup doc to toctree Signed-off-by: fenn-cs --- docs/index.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/index.rst b/docs/index.rst index 16067597a..3da03341a 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -29,6 +29,7 @@ Contents: contributing CODE_OF_CONDUCT release-checklist + gpg-setup servers helper From b32374c8bcee4120dd89e37fe2472aabf48abce2 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 16 Dec 2021 10:39:58 -0500 Subject: [PATCH 0510/2309] Secret header parsing. --- src/allmydata/storage/http_server.py | 38 +++++++++++++ src/allmydata/test/test_storage_http.py | 73 ++++++++++++++++++++++++- 2 files changed, 110 insertions(+), 1 deletion(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 6297ef484..47722180d 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -13,8 +13,12 @@ if PY2: # fmt: off from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 # fmt: on +else: + from typing import Dict, List, Set from functools import wraps +from enum import Enum +from base64 import b64decode from klein import Klein from twisted.web import http @@ -26,6 +30,40 @@ from .server import StorageServer from .http_client import swissnum_auth_header +class Secrets(Enum): + """Different kinds of secrets the client may send.""" + LEASE_RENEW = "lease-renew-secret" + LEASE_CANCEL = "lease-cancel-secret" + UPLOAD = "upload-secret" + + +class ClientSecretsException(Exception): + """The client did not send the appropriate secrets.""" + + +def _extract_secrets(header_values, required_secrets): # type: (List[str], Set[Secrets]) -> Dict[Secrets, bytes] + """ + Given list of values of ``X-Tahoe-Authorization`` headers, and required + secrets, return dictionary mapping secrets to decoded values. + + If too few secrets were given, or too many, a ``ClientSecretsException`` is + raised. + """ + key_to_enum = {e.value: e for e in Secrets} + result = {} + try: + for header_value in header_values: + key, value = header_value.strip().split(" ", 1) + result[key_to_enum[key]] = b64decode(value) + except (ValueError, KeyError) as e: + raise ClientSecretsException("Bad header value(s): {}".format(header_values)) + if result.keys() != required_secrets: + raise ClientSecretsException( + "Expected {} secrets, got {}".format(required_secrets, result.keys()) + ) + return result + + def _authorization_decorator(f): """ Check the ``Authorization`` header, and (TODO: in later revision of code) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index e413a0624..2a84d477f 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -15,6 +15,7 @@ if PY2: # fmt: on from unittest import SkipTest +from base64 import b64encode from twisted.trial.unittest import TestCase from twisted.internet.defer import inlineCallbacks @@ -23,10 +24,80 @@ from treq.testing import StubTreq from hyperlink import DecodedURL from ..storage.server import StorageServer -from ..storage.http_server import HTTPServer +from ..storage.http_server import HTTPServer, _extract_secrets, Secrets, ClientSecretsException from ..storage.http_client import StorageClient, ClientException +class ExtractSecretsTests(TestCase): + """ + Tests for ``_extract_secrets``. + """ + def test_extract_secrets(self): + """ + ``_extract_secrets()`` returns a dictionary with the extracted secrets + if the input secrets match the required secrets. + """ + secret1 = b"\xFF\x11ZEBRa" + secret2 = b"\x34\xF2lalalalalala" + lease_secret = "lease-renew-secret " + str(b64encode(secret1), "ascii").strip() + upload_secret = "upload-secret " + str(b64encode(secret2), "ascii").strip() + + # No secrets needed, none given: + self.assertEqual(_extract_secrets([], set()), {}) + + # One secret: + self.assertEqual( + _extract_secrets([lease_secret], + {Secrets.LEASE_RENEW}), + {Secrets.LEASE_RENEW: secret1} + ) + + # Two secrets: + self.assertEqual( + _extract_secrets([upload_secret, lease_secret], + {Secrets.LEASE_RENEW, Secrets.UPLOAD}), + {Secrets.LEASE_RENEW: secret1, Secrets.UPLOAD: secret2} + ) + + def test_wrong_number_of_secrets(self): + """ + If the wrong number of secrets are passed to ``_extract_secrets``, a + ``ClientSecretsException`` is raised. + """ + secret1 = b"\xFF\x11ZEBRa" + lease_secret = "lease-renew-secret " + str(b64encode(secret1), "ascii").strip() + + # Missing secret: + with self.assertRaises(ClientSecretsException): + _extract_secrets([], {Secrets.LEASE_RENEW}) + + # Wrong secret: + with self.assertRaises(ClientSecretsException): + _extract_secrets([lease_secret], {Secrets.UPLOAD}) + + # Extra secret: + with self.assertRaises(ClientSecretsException): + _extract_secrets([lease_secret], {}) + + def test_bad_secrets(self): + """ + Bad inputs to ``_extract_secrets`` result in + ``ClientSecretsException``. + """ + + # Missing value. + with self.assertRaises(ClientSecretsException): + _extract_secrets(["lease-renew-secret"], {Secrets.LEASE_RENEW}) + + # Garbage prefix + with self.assertRaises(ClientSecretsException): + _extract_secrets(["FOO eA=="], {}) + + # Not base64. + with self.assertRaises(ClientSecretsException): + _extract_secrets(["lease-renew-secret x"], {Secrets.LEASE_RENEW}) + + class HTTPTests(TestCase): """ Tests of HTTP client talking to the HTTP server. From 1737340df69ff443c1e548d9126066c05e00bf30 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 16 Dec 2021 10:52:02 -0500 Subject: [PATCH 0511/2309] Document response codes some more. --- docs/proposed/http-storage-node-protocol.rst | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index 490d3f3ca..0d8cee466 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -369,6 +369,19 @@ The authentication *type* used is ``Tahoe-LAFS``. The swissnum from the NURL used to locate the storage service is used as the *credentials*. If credentials are not presented or the swissnum is not associated with a storage service then no storage processing is performed and the request receives an ``401 UNAUTHORIZED`` response. +There are also, for some endpoints, secrets sent via ``X-Tahoe-Authorization`` headers. +If these are: + +1. Missing. +2. The wrong length. +3. Not the expected kind of secret. +4. They are otherwise unparseable before they are actually semantically used. + +the server will respond with ``400 BAD REQUEST``. +401 is not used because this isn't an authorization problem, this is a "you sent garbage and should know better" bug. + +If authorization using the secret fails, then a ``401 UNAUTHORIZED`` response should be sent. + General ~~~~~~~ From 87fa9ac2a8e507b385c4b0845c50d34cf6d30da6 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 16 Dec 2021 11:17:11 -0500 Subject: [PATCH 0512/2309] Infrastructure for sending secrets. --- src/allmydata/storage/http_client.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index f8a7590aa..774ecbde1 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -19,7 +19,7 @@ else: from typing import Union from treq.testing import StubTreq -import base64 +from base64 import b64encode # TODO Make sure to import Python version? from cbor2 import loads @@ -44,7 +44,7 @@ def _decode_cbor(response): def swissnum_auth_header(swissnum): # type: (bytes) -> bytes """Return value for ``Authentication`` header.""" - return b"Tahoe-LAFS " + base64.b64encode(swissnum).strip() + return b"Tahoe-LAFS " + b64encode(swissnum).strip() class StorageClient(object): @@ -68,12 +68,25 @@ class StorageClient(object): ) return headers + def _request(self, method, url, secrets, **kwargs): + """ + Like ``treq.request()``, but additional argument of secrets mapping + ``http_server.Secret`` to the bytes value of the secret. + """ + headers = self._get_headers() + for key, value in secrets.items(): + headers.addRawHeader( + "X-Tahoe-Authorization", + b"{} {}".format(key.value.encode("ascii"), b64encode(value).strip()) + ) + return self._treq.request(method, url, headers=headers, **kwargs) + @inlineCallbacks def get_version(self): """ Return the version metadata for the server. """ url = self._base_url.click("/v1/version") - response = yield self._treq.get(url, headers=self._get_headers()) + response = yield self._request("GET", url, {}) decoded_response = yield _decode_cbor(response) returnValue(decoded_response) From da52a9aedeebe39c908f1616bac6ce887d340066 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 16 Dec 2021 11:17:32 -0500 Subject: [PATCH 0513/2309] Test for server-side secret handling. --- src/allmydata/test/test_storage_http.py | 88 +++++++++++++++++++++---- 1 file changed, 75 insertions(+), 13 deletions(-) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 2a84d477f..648530852 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -21,10 +21,14 @@ from twisted.trial.unittest import TestCase from twisted.internet.defer import inlineCallbacks from treq.testing import StubTreq +from klein import Klein from hyperlink import DecodedURL from ..storage.server import StorageServer -from ..storage.http_server import HTTPServer, _extract_secrets, Secrets, ClientSecretsException +from ..storage.http_server import ( + HTTPServer, _extract_secrets, Secrets, ClientSecretsException, + _authorized_route, +) from ..storage.http_client import StorageClient, ClientException @@ -98,22 +102,80 @@ class ExtractSecretsTests(TestCase): _extract_secrets(["lease-renew-secret x"], {Secrets.LEASE_RENEW}) -class HTTPTests(TestCase): +SWISSNUM_FOR_TEST = b"abcd" + + +class TestApp(object): + """HTTP API for testing purposes.""" + + _app = Klein() + _swissnum = SWISSNUM_FOR_TEST # Match what the test client is using + + @_authorized_route(_app, {Secrets.UPLOAD}, "/upload_secret", methods=["GET"]) + def validate_upload_secret(self, request, authorization): + if authorization == {Secrets.UPLOAD: b"abc"}: + return "OK" + else: + return "BAD: {}".format(authorization) + + +class RoutingTests(TestCase): """ - Tests of HTTP client talking to the HTTP server. + Tests for the HTTP routing infrastructure. + """ + def setUp(self): + self._http_server = TestApp() + self.client = StorageClient( + DecodedURL.from_text("http://127.0.0.1"), + SWISSNUM_FOR_TEST, + treq=StubTreq(self._http_server._app.resource()), + ) + + @inlineCallbacks + def test_authorization_enforcement(self): + """ + The requirement for secrets is enforced; if they are not given, a 400 + response code is returned. + """ + secret = b"abc" + + # Without secret, get a 400 error. + response = yield self.client._request( + "GET", "http://127.0.0.1/upload_secret", {} + ) + self.assertEqual(response.code, 400) + + # With secret, we're good. + response = yield self.client._request( + "GET", "http://127.0.0.1/upload_secret", {Secrets.UPLOAD, b"abc"} + ) + self.assertEqual(response.code, 200) + + +def setup_http_test(self): + """ + Setup HTTP tests; call from ``setUp``. + """ + if PY2: + raise SkipTest("Not going to bother supporting Python 2") + self.storage_server = StorageServer(self.mktemp(), b"\x00" * 20) + # TODO what should the swissnum _actually_ be? + self._http_server = HTTPServer(self.storage_server, SWISSNUM_FOR_TEST) + self.client = StorageClient( + DecodedURL.from_text("http://127.0.0.1"), + SWISSNUM_FOR_TEST, + treq=StubTreq(self._http_server.get_resource()), + ) + + +class GenericHTTPAPITests(TestCase): + """ + Tests of HTTP client talking to the HTTP server, for generic HTTP API + endpoints and concerns. """ def setUp(self): - if PY2: - raise SkipTest("Not going to bother supporting Python 2") - self.storage_server = StorageServer(self.mktemp(), b"\x00" * 20) - # TODO what should the swissnum _actually_ be? - self._http_server = HTTPServer(self.storage_server, b"abcd") - self.client = StorageClient( - DecodedURL.from_text("http://127.0.0.1"), - b"abcd", - treq=StubTreq(self._http_server.get_resource()), - ) + setup_http_test(self) @inlineCallbacks def test_bad_authentication(self): From 816dc0c73ff3ed7522c63198c6659c17e39f7837 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 16 Dec 2021 11:42:06 -0500 Subject: [PATCH 0514/2309] X-Tahoe-Authorization can be validated and are passed to server methods. --- src/allmydata/storage/http_client.py | 2 +- src/allmydata/storage/http_server.py | 51 ++++++++++++++++--------- src/allmydata/test/test_storage_http.py | 2 +- 3 files changed, 34 insertions(+), 21 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 774ecbde1..72e1af080 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -77,7 +77,7 @@ class StorageClient(object): for key, value in secrets.items(): headers.addRawHeader( "X-Tahoe-Authorization", - b"{} {}".format(key.value.encode("ascii"), b64encode(value).strip()) + b"%s %s" % (key.value.encode("ascii"), b64encode(value).strip()) ) return self._treq.request(method, url, headers=headers, **kwargs) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 47722180d..a26a2c266 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -54,8 +54,10 @@ def _extract_secrets(header_values, required_secrets): # type: (List[str], Set[ try: for header_value in header_values: key, value = header_value.strip().split(" ", 1) + # TODO enforce secret is 32 bytes long for lease secrets. dunno + # about upload secret. result[key_to_enum[key]] = b64decode(value) - except (ValueError, KeyError) as e: + except (ValueError, KeyError): raise ClientSecretsException("Bad header value(s): {}".format(header_values)) if result.keys() != required_secrets: raise ClientSecretsException( @@ -64,38 +66,45 @@ def _extract_secrets(header_values, required_secrets): # type: (List[str], Set[ return result -def _authorization_decorator(f): +def _authorization_decorator(required_secrets): """ Check the ``Authorization`` header, and (TODO: in later revision of code) extract ``X-Tahoe-Authorization`` headers and pass them in. """ + def decorator(f): + @wraps(f) + def route(self, request, *args, **kwargs): + if request.requestHeaders.getRawHeaders("Authorization", [None])[0] != str( + swissnum_auth_header(self._swissnum), "ascii" + ): + request.setResponseCode(http.UNAUTHORIZED) + return b"" + authorization = request.requestHeaders.getRawHeaders("X-Tahoe-Authorization", []) + try: + secrets = _extract_secrets(authorization, required_secrets) + except ClientSecretsException: + request.setResponseCode(400) + return b"" + return f(self, request, secrets, *args, **kwargs) - @wraps(f) - def route(self, request, *args, **kwargs): - if request.requestHeaders.getRawHeaders("Authorization", [None])[0] != str( - swissnum_auth_header(self._swissnum), "ascii" - ): - request.setResponseCode(http.UNAUTHORIZED) - return b"" - # authorization = request.requestHeaders.getRawHeaders("X-Tahoe-Authorization", []) - # For now, just a placeholder: - authorization = None - return f(self, request, authorization, *args, **kwargs) + return route - return route + return decorator -def _authorized_route(app, *route_args, **route_kwargs): +def _authorized_route(app, required_secrets, *route_args, **route_kwargs): """ Like Klein's @route, but with additional support for checking the ``Authorization`` header as well as ``X-Tahoe-Authorization`` headers. The - latter will (TODO: in later revision of code) get passed in as second - argument to wrapped functions. + latter will get passed in as second argument to wrapped functions, a + dictionary mapping a ``Secret`` value to the uploaded secret. + + :param required_secrets: Set of required ``Secret`` types. """ def decorator(f): @app.route(*route_args, **route_kwargs) - @_authorization_decorator + @_authorization_decorator(required_secrets) def handle_route(*args, **kwargs): return f(*args, **kwargs) @@ -127,6 +136,10 @@ class HTTPServer(object): # TODO if data is big, maybe want to use a temporary file eventually... return dumps(data) - @_authorized_route(_app, "/v1/version", methods=["GET"]) + + ##### Generic APIs ##### + + @_authorized_route(_app, set(), "/v1/version", methods=["GET"]) def version(self, request, authorization): + """Return version information.""" return self._cbor(request, self._storage_server.get_version()) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 648530852..b28f4aafb 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -147,7 +147,7 @@ class RoutingTests(TestCase): # With secret, we're good. response = yield self.client._request( - "GET", "http://127.0.0.1/upload_secret", {Secrets.UPLOAD, b"abc"} + "GET", "http://127.0.0.1/upload_secret", {Secrets.UPLOAD: b"abc"} ) self.assertEqual(response.code, 200) From d2ce80dab88df8431a2188ae53bd40f2debe5ee4 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 16 Dec 2021 11:42:44 -0500 Subject: [PATCH 0515/2309] News file. --- newsfragments/3848.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3848.minor diff --git a/newsfragments/3848.minor b/newsfragments/3848.minor new file mode 100644 index 000000000..e69de29bb From fb0be6b8944dcf9b1254e5d3991ddd8d2ff6ad3c Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 16 Dec 2021 11:46:35 -0500 Subject: [PATCH 0516/2309] Enforce length of lease secrets. --- src/allmydata/storage/http_server.py | 12 +++++++----- src/allmydata/test/test_storage_http.py | 10 ++++++++-- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index a26a2c266..1143acce9 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -49,14 +49,16 @@ def _extract_secrets(header_values, required_secrets): # type: (List[str], Set[ If too few secrets were given, or too many, a ``ClientSecretsException`` is raised. """ - key_to_enum = {e.value: e for e in Secrets} + string_key_to_enum = {e.value: e for e in Secrets} result = {} try: for header_value in header_values: - key, value = header_value.strip().split(" ", 1) - # TODO enforce secret is 32 bytes long for lease secrets. dunno - # about upload secret. - result[key_to_enum[key]] = b64decode(value) + string_key, string_value = header_value.strip().split(" ", 1) + key = string_key_to_enum[string_key] + value = b64decode(string_value) + if key in (Secrets.LEASE_CANCEL, Secrets.LEASE_RENEW) and len(value) != 32: + raise ClientSecretsException("Lease secrets must be 32 bytes long") + result[key] = value except (ValueError, KeyError): raise ClientSecretsException("Bad header value(s): {}".format(header_values)) if result.keys() != required_secrets: diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index b28f4aafb..9d4ef3738 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -41,8 +41,8 @@ class ExtractSecretsTests(TestCase): ``_extract_secrets()`` returns a dictionary with the extracted secrets if the input secrets match the required secrets. """ - secret1 = b"\xFF\x11ZEBRa" - secret2 = b"\x34\xF2lalalalalala" + secret1 = b"\xFF" * 32 + secret2 = b"\x34" * 32 lease_secret = "lease-renew-secret " + str(b64encode(secret1), "ascii").strip() upload_secret = "upload-secret " + str(b64encode(secret2), "ascii").strip() @@ -101,6 +101,12 @@ class ExtractSecretsTests(TestCase): with self.assertRaises(ClientSecretsException): _extract_secrets(["lease-renew-secret x"], {Secrets.LEASE_RENEW}) + # Wrong length lease secrets (must be 32 bytes long). + with self.assertRaises(ClientSecretsException): + _extract_secrets(["lease-renew-secret eA=="], {Secrets.LEASE_RENEW}) + with self.assertRaises(ClientSecretsException): + _extract_secrets(["lease-upload-secret eA=="], {Secrets.LEASE_RENEW}) + SWISSNUM_FOR_TEST = b"abcd" From 428a9d0573628fdc91c5efd7fc3a2f94bbdd19bf Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 16 Dec 2021 11:47:40 -0500 Subject: [PATCH 0517/2309] Lint fix. --- src/allmydata/test/test_storage_http.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 9d4ef3738..b9aa59f3e 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -143,8 +143,6 @@ class RoutingTests(TestCase): The requirement for secrets is enforced; if they are not given, a 400 response code is returned. """ - secret = b"abc" - # Without secret, get a 400 error. response = yield self.client._request( "GET", "http://127.0.0.1/upload_secret", {} From 81b95f3335c7178fb64cb131d4423780cf04cd29 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 16 Dec 2021 11:53:31 -0500 Subject: [PATCH 0518/2309] Ensure secret was validated. --- src/allmydata/test/test_storage_http.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index b9aa59f3e..16420b266 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -119,8 +119,8 @@ class TestApp(object): @_authorized_route(_app, {Secrets.UPLOAD}, "/upload_secret", methods=["GET"]) def validate_upload_secret(self, request, authorization): - if authorization == {Secrets.UPLOAD: b"abc"}: - return "OK" + if authorization == {Secrets.UPLOAD: b"MAGIC"}: + return "GOOD SECRET" else: return "BAD: {}".format(authorization) @@ -151,9 +151,10 @@ class RoutingTests(TestCase): # With secret, we're good. response = yield self.client._request( - "GET", "http://127.0.0.1/upload_secret", {Secrets.UPLOAD: b"abc"} + "GET", "http://127.0.0.1/upload_secret", {Secrets.UPLOAD: b"MAGIC"} ) self.assertEqual(response.code, 200) + self.assertEqual((yield response.content()), b"GOOD SECRET") def setup_http_test(self): From a529ba7d5ea84c2b9f4cef48f6e5ef778cb5fbbc Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 17 Dec 2021 09:14:09 -0500 Subject: [PATCH 0519/2309] More skipping on Python 2. --- src/allmydata/test/test_storage_http.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 16420b266..aba33fad3 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -36,6 +36,10 @@ class ExtractSecretsTests(TestCase): """ Tests for ``_extract_secrets``. """ + def setUp(self): + if PY2: + raise SkipTest("Not going to bother supporting Python 2") + def test_extract_secrets(self): """ ``_extract_secrets()`` returns a dictionary with the extracted secrets @@ -130,6 +134,8 @@ class RoutingTests(TestCase): Tests for the HTTP routing infrastructure. """ def setUp(self): + if PY2: + raise SkipTest("Not going to bother supporting Python 2") self._http_server = TestApp() self.client = StorageClient( DecodedURL.from_text("http://127.0.0.1"), From 291b4e1896f52a07fbe92df7b67f90372ca0f052 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 20 Dec 2021 11:17:27 -0500 Subject: [PATCH 0520/2309] Use more secure comparison to prevent timing-based side-channel attacks. --- src/allmydata/storage/http_server.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 1143acce9..386368364 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -28,10 +28,12 @@ from cbor2 import dumps from .server import StorageServer from .http_client import swissnum_auth_header +from ..util.hashutil import timing_safe_compare class Secrets(Enum): """Different kinds of secrets the client may send.""" + LEASE_RENEW = "lease-renew-secret" LEASE_CANCEL = "lease-cancel-secret" UPLOAD = "upload-secret" @@ -41,7 +43,9 @@ class ClientSecretsException(Exception): """The client did not send the appropriate secrets.""" -def _extract_secrets(header_values, required_secrets): # type: (List[str], Set[Secrets]) -> Dict[Secrets, bytes] +def _extract_secrets( + header_values, required_secrets +): # type: (List[str], Set[Secrets]) -> Dict[Secrets, bytes] """ Given list of values of ``X-Tahoe-Authorization`` headers, and required secrets, return dictionary mapping secrets to decoded values. @@ -73,15 +77,21 @@ def _authorization_decorator(required_secrets): Check the ``Authorization`` header, and (TODO: in later revision of code) extract ``X-Tahoe-Authorization`` headers and pass them in. """ + def decorator(f): @wraps(f) def route(self, request, *args, **kwargs): - if request.requestHeaders.getRawHeaders("Authorization", [None])[0] != str( - swissnum_auth_header(self._swissnum), "ascii" + if not timing_safe_compare( + request.requestHeaders.getRawHeaders("Authorization", [None])[0].encode( + "utf-8" + ), + swissnum_auth_header(self._swissnum), ): request.setResponseCode(http.UNAUTHORIZED) return b"" - authorization = request.requestHeaders.getRawHeaders("X-Tahoe-Authorization", []) + authorization = request.requestHeaders.getRawHeaders( + "X-Tahoe-Authorization", [] + ) try: secrets = _extract_secrets(authorization, required_secrets) except ClientSecretsException: @@ -138,7 +148,6 @@ class HTTPServer(object): # TODO if data is big, maybe want to use a temporary file eventually... return dumps(data) - ##### Generic APIs ##### @_authorized_route(_app, set(), "/v1/version", methods=["GET"]) From 1721865b20d0d160891f91365dc477fae26ffcb0 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 20 Dec 2021 13:46:19 -0500 Subject: [PATCH 0521/2309] No longer TODO. --- src/allmydata/storage/http_server.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 386368364..83bbbe49d 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -74,8 +74,8 @@ def _extract_secrets( def _authorization_decorator(required_secrets): """ - Check the ``Authorization`` header, and (TODO: in later revision of code) - extract ``X-Tahoe-Authorization`` headers and pass them in. + Check the ``Authorization`` header, and extract ``X-Tahoe-Authorization`` + headers and pass them in. """ def decorator(f): From 2bda2a01278d1aba5f3a13c772c9c7b887119bdd Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 21 Dec 2021 11:10:53 -0500 Subject: [PATCH 0522/2309] Switch to using a fixture. --- src/allmydata/test/test_storage_http.py | 77 +++++++++++++++---------- 1 file changed, 46 insertions(+), 31 deletions(-) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index aba33fad3..1cf650875 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -17,28 +17,34 @@ if PY2: from unittest import SkipTest from base64 import b64encode -from twisted.trial.unittest import TestCase from twisted.internet.defer import inlineCallbacks +from fixtures import Fixture, TempDir from treq.testing import StubTreq from klein import Klein from hyperlink import DecodedURL +from .common import AsyncTestCase, SyncTestCase from ..storage.server import StorageServer from ..storage.http_server import ( - HTTPServer, _extract_secrets, Secrets, ClientSecretsException, + HTTPServer, + _extract_secrets, + Secrets, + ClientSecretsException, _authorized_route, ) from ..storage.http_client import StorageClient, ClientException -class ExtractSecretsTests(TestCase): +class ExtractSecretsTests(SyncTestCase): """ Tests for ``_extract_secrets``. """ + def setUp(self): if PY2: raise SkipTest("Not going to bother supporting Python 2") + super(ExtractSecretsTests, self).setUp() def test_extract_secrets(self): """ @@ -55,16 +61,16 @@ class ExtractSecretsTests(TestCase): # One secret: self.assertEqual( - _extract_secrets([lease_secret], - {Secrets.LEASE_RENEW}), - {Secrets.LEASE_RENEW: secret1} + _extract_secrets([lease_secret], {Secrets.LEASE_RENEW}), + {Secrets.LEASE_RENEW: secret1}, ) # Two secrets: self.assertEqual( - _extract_secrets([upload_secret, lease_secret], - {Secrets.LEASE_RENEW, Secrets.UPLOAD}), - {Secrets.LEASE_RENEW: secret1, Secrets.UPLOAD: secret2} + _extract_secrets( + [upload_secret, lease_secret], {Secrets.LEASE_RENEW, Secrets.UPLOAD} + ), + {Secrets.LEASE_RENEW: secret1, Secrets.UPLOAD: secret2}, ) def test_wrong_number_of_secrets(self): @@ -129,19 +135,23 @@ class TestApp(object): return "BAD: {}".format(authorization) -class RoutingTests(TestCase): +class RoutingTests(AsyncTestCase): """ Tests for the HTTP routing infrastructure. """ + def setUp(self): if PY2: raise SkipTest("Not going to bother supporting Python 2") + super(RoutingTests, self).setUp() + # Could be a fixture, but will only be used in this test class so not + # going to bother: self._http_server = TestApp() self.client = StorageClient( - DecodedURL.from_text("http://127.0.0.1"), - SWISSNUM_FOR_TEST, - treq=StubTreq(self._http_server._app.resource()), - ) + DecodedURL.from_text("http://127.0.0.1"), + SWISSNUM_FOR_TEST, + treq=StubTreq(self._http_server._app.resource()), + ) @inlineCallbacks def test_authorization_enforcement(self): @@ -163,30 +173,35 @@ class RoutingTests(TestCase): self.assertEqual((yield response.content()), b"GOOD SECRET") -def setup_http_test(self): +class HttpTestFixture(Fixture): """ - Setup HTTP tests; call from ``setUp``. + Setup HTTP tests' infrastructure, the storage server and corresponding + client. """ - if PY2: - raise SkipTest("Not going to bother supporting Python 2") - self.storage_server = StorageServer(self.mktemp(), b"\x00" * 20) - # TODO what should the swissnum _actually_ be? - self._http_server = HTTPServer(self.storage_server, SWISSNUM_FOR_TEST) - self.client = StorageClient( - DecodedURL.from_text("http://127.0.0.1"), - SWISSNUM_FOR_TEST, - treq=StubTreq(self._http_server.get_resource()), - ) + + def _setUp(self): + self.tempdir = self.useFixture(TempDir()) + self.storage_server = StorageServer(self.tempdir.path, b"\x00" * 20) + # TODO what should the swissnum _actually_ be? + self.http_server = HTTPServer(self.storage_server, SWISSNUM_FOR_TEST) + self.client = StorageClient( + DecodedURL.from_text("http://127.0.0.1"), + SWISSNUM_FOR_TEST, + treq=StubTreq(self.http_server.get_resource()), + ) -class GenericHTTPAPITests(TestCase): +class GenericHTTPAPITests(AsyncTestCase): """ Tests of HTTP client talking to the HTTP server, for generic HTTP API endpoints and concerns. """ def setUp(self): - setup_http_test(self) + if PY2: + raise SkipTest("Not going to bother supporting Python 2") + super(GenericHTTPAPITests, self).setUp() + self.http = self.useFixture(HttpTestFixture()) @inlineCallbacks def test_bad_authentication(self): @@ -197,7 +212,7 @@ class GenericHTTPAPITests(TestCase): client = StorageClient( DecodedURL.from_text("http://127.0.0.1"), b"something wrong", - treq=StubTreq(self._http_server.get_resource()), + treq=StubTreq(self.http.http_server.get_resource()), ) with self.assertRaises(ClientException) as e: yield client.get_version() @@ -211,14 +226,14 @@ class GenericHTTPAPITests(TestCase): We ignore available disk space and max immutable share size, since that might change across calls. """ - version = yield self.client.get_version() + version = yield self.http.client.get_version() version[b"http://allmydata.org/tahoe/protocols/storage/v1"].pop( b"available-space" ) version[b"http://allmydata.org/tahoe/protocols/storage/v1"].pop( b"maximum-immutable-share-size" ) - expected_version = self.storage_server.get_version() + expected_version = self.http.storage_server.get_version() expected_version[b"http://allmydata.org/tahoe/protocols/storage/v1"].pop( b"available-space" ) From b1f4e82adfc1d2ff8482b1edb28e0139e7ef0000 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 21 Dec 2021 11:55:16 -0500 Subject: [PATCH 0523/2309] Switch to using hypothesis. --- src/allmydata/test/test_storage_http.py | 31 +++++++++---------------- 1 file changed, 11 insertions(+), 20 deletions(-) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 1cf650875..2a3a5bc90 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -19,6 +19,7 @@ from base64 import b64encode from twisted.internet.defer import inlineCallbacks +from hypothesis import given, strategies as st from fixtures import Fixture, TempDir from treq.testing import StubTreq from klein import Klein @@ -46,32 +47,22 @@ class ExtractSecretsTests(SyncTestCase): raise SkipTest("Not going to bother supporting Python 2") super(ExtractSecretsTests, self).setUp() - def test_extract_secrets(self): + @given(secret_types=st.sets(st.sampled_from(Secrets))) + def test_extract_secrets(self, secret_types): """ ``_extract_secrets()`` returns a dictionary with the extracted secrets if the input secrets match the required secrets. """ - secret1 = b"\xFF" * 32 - secret2 = b"\x34" * 32 - lease_secret = "lease-renew-secret " + str(b64encode(secret1), "ascii").strip() - upload_secret = "upload-secret " + str(b64encode(secret2), "ascii").strip() + secrets = {s: bytes([i] * 32) for (i, s) in enumerate(secret_types)} + headers = [ + "{} {}".format( + secret_type.value, str(b64encode(secrets[secret_type]), "ascii").strip() + ) + for secret_type in secret_types + ] # No secrets needed, none given: - self.assertEqual(_extract_secrets([], set()), {}) - - # One secret: - self.assertEqual( - _extract_secrets([lease_secret], {Secrets.LEASE_RENEW}), - {Secrets.LEASE_RENEW: secret1}, - ) - - # Two secrets: - self.assertEqual( - _extract_secrets( - [upload_secret, lease_secret], {Secrets.LEASE_RENEW, Secrets.UPLOAD} - ), - {Secrets.LEASE_RENEW: secret1, Secrets.UPLOAD: secret2}, - ) + self.assertEqual(_extract_secrets(headers, secret_types), secrets) def test_wrong_number_of_secrets(self): """ From 776f19cbb2f96607f583dd53ce3c90cbc1353ea8 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 21 Dec 2021 12:34:02 -0500 Subject: [PATCH 0524/2309] Even more hypothesis, this time for secrets' contents. --- src/allmydata/test/test_storage_http.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 2a3a5bc90..3dc6bac96 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -47,13 +47,25 @@ class ExtractSecretsTests(SyncTestCase): raise SkipTest("Not going to bother supporting Python 2") super(ExtractSecretsTests, self).setUp() - @given(secret_types=st.sets(st.sampled_from(Secrets))) - def test_extract_secrets(self, secret_types): + @given( + params=st.sets(st.sampled_from(Secrets)).flatmap( + lambda secret_types: st.tuples( + st.just(secret_types), + st.lists( + st.binary(min_size=32, max_size=32), + min_size=len(secret_types), + max_size=len(secret_types), + ), + ) + ) + ) + def test_extract_secrets(self, params): """ ``_extract_secrets()`` returns a dictionary with the extracted secrets if the input secrets match the required secrets. """ - secrets = {s: bytes([i] * 32) for (i, s) in enumerate(secret_types)} + secret_types, secrets = params + secrets = {t: s for (t, s) in zip(secret_types, secrets)} headers = [ "{} {}".format( secret_type.value, str(b64encode(secrets[secret_type]), "ascii").strip() From 8b4d166a54ebc13c43743cd4263612234f6c68d4 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 22 Dec 2021 11:44:45 -0500 Subject: [PATCH 0525/2309] Use hypothesis for another test. --- src/allmydata/test/test_storage_http.py | 78 +++++++++++++------------ 1 file changed, 42 insertions(+), 36 deletions(-) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 3dc6bac96..80bd2661b 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -19,7 +19,7 @@ from base64 import b64encode from twisted.internet.defer import inlineCallbacks -from hypothesis import given, strategies as st +from hypothesis import assume, given, strategies as st from fixtures import Fixture, TempDir from treq.testing import StubTreq from klein import Klein @@ -37,6 +37,35 @@ from ..storage.http_server import ( from ..storage.http_client import StorageClient, ClientException +def _post_process(params): + secret_types, secrets = params + secrets = {t: s for (t, s) in zip(secret_types, secrets)} + headers = [ + "{} {}".format( + secret_type.value, str(b64encode(secrets[secret_type]), "ascii").strip() + ) + for secret_type in secret_types + ] + return secrets, headers + + +# Creates a tuple of ({Secret enum value: secret_bytes}, [http headers with secrets]). +SECRETS_STRATEGY = ( + st.sets(st.sampled_from(Secrets)) + .flatmap( + lambda secret_types: st.tuples( + st.just(secret_types), + st.lists( + st.binary(min_size=32, max_size=32), + min_size=len(secret_types), + max_size=len(secret_types), + ), + ) + ) + .map(_post_process) +) + + class ExtractSecretsTests(SyncTestCase): """ Tests for ``_extract_secrets``. @@ -47,54 +76,31 @@ class ExtractSecretsTests(SyncTestCase): raise SkipTest("Not going to bother supporting Python 2") super(ExtractSecretsTests, self).setUp() - @given( - params=st.sets(st.sampled_from(Secrets)).flatmap( - lambda secret_types: st.tuples( - st.just(secret_types), - st.lists( - st.binary(min_size=32, max_size=32), - min_size=len(secret_types), - max_size=len(secret_types), - ), - ) - ) - ) - def test_extract_secrets(self, params): + @given(secrets_to_send=SECRETS_STRATEGY) + def test_extract_secrets(self, secrets_to_send): """ ``_extract_secrets()`` returns a dictionary with the extracted secrets if the input secrets match the required secrets. """ - secret_types, secrets = params - secrets = {t: s for (t, s) in zip(secret_types, secrets)} - headers = [ - "{} {}".format( - secret_type.value, str(b64encode(secrets[secret_type]), "ascii").strip() - ) - for secret_type in secret_types - ] + secrets, headers = secrets_to_send # No secrets needed, none given: - self.assertEqual(_extract_secrets(headers, secret_types), secrets) + self.assertEqual(_extract_secrets(headers, secrets.keys()), secrets) - def test_wrong_number_of_secrets(self): + @given( + secrets_to_send=SECRETS_STRATEGY, + secrets_to_require=st.sets(st.sampled_from(Secrets)), + ) + def test_wrong_number_of_secrets(self, secrets_to_send, secrets_to_require): """ If the wrong number of secrets are passed to ``_extract_secrets``, a ``ClientSecretsException`` is raised. """ - secret1 = b"\xFF\x11ZEBRa" - lease_secret = "lease-renew-secret " + str(b64encode(secret1), "ascii").strip() + secrets_to_send, headers = secrets_to_send + assume(secrets_to_send.keys() != secrets_to_require) - # Missing secret: with self.assertRaises(ClientSecretsException): - _extract_secrets([], {Secrets.LEASE_RENEW}) - - # Wrong secret: - with self.assertRaises(ClientSecretsException): - _extract_secrets([lease_secret], {Secrets.UPLOAD}) - - # Extra secret: - with self.assertRaises(ClientSecretsException): - _extract_secrets([lease_secret], {}) + _extract_secrets(headers, secrets_to_require) def test_bad_secrets(self): """ From 7a0c83e71be89f9a1efc631f7cb75df8b063fad8 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 22 Dec 2021 11:52:13 -0500 Subject: [PATCH 0526/2309] Split up test. --- src/allmydata/test/test_storage_http.py | 30 ++++++++++++++++++------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 80bd2661b..160cf8479 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -102,29 +102,43 @@ class ExtractSecretsTests(SyncTestCase): with self.assertRaises(ClientSecretsException): _extract_secrets(headers, secrets_to_require) - def test_bad_secrets(self): + def test_bad_secret_missing_value(self): """ - Bad inputs to ``_extract_secrets`` result in + Missing value in ``_extract_secrets`` result in ``ClientSecretsException``. """ - - # Missing value. with self.assertRaises(ClientSecretsException): _extract_secrets(["lease-renew-secret"], {Secrets.LEASE_RENEW}) - # Garbage prefix + def test_bad_secret_unknown_prefix(self): + """ + Missing value in ``_extract_secrets`` result in + ``ClientSecretsException``. + """ with self.assertRaises(ClientSecretsException): _extract_secrets(["FOO eA=="], {}) - # Not base64. + def test_bad_secret_not_base64(self): + """ + A non-base64 value in ``_extract_secrets`` result in + ``ClientSecretsException``. + """ with self.assertRaises(ClientSecretsException): _extract_secrets(["lease-renew-secret x"], {Secrets.LEASE_RENEW}) - # Wrong length lease secrets (must be 32 bytes long). + def test_bad_secret_wrong_length_lease_renew(self): + """ + Lease renewal secrets must be 32-bytes long. + """ with self.assertRaises(ClientSecretsException): _extract_secrets(["lease-renew-secret eA=="], {Secrets.LEASE_RENEW}) + + def test_bad_secret_wrong_length_lease_cancel(self): + """ + Lease cancel secrets must be 32-bytes long. + """ with self.assertRaises(ClientSecretsException): - _extract_secrets(["lease-upload-secret eA=="], {Secrets.LEASE_RENEW}) + _extract_secrets(["lease-cancel-secret eA=="], {Secrets.LEASE_RENEW}) SWISSNUM_FOR_TEST = b"abcd" From 58a71517c1fdc74b735876541d2dfa0ddfb2e5c4 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 22 Dec 2021 13:16:43 -0500 Subject: [PATCH 0527/2309] Correct way to skip with testtools. --- src/allmydata/test/test_storage_http.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 160cf8479..181b6d347 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -14,7 +14,6 @@ if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 # fmt: on -from unittest import SkipTest from base64 import b64encode from twisted.internet.defer import inlineCallbacks @@ -73,7 +72,7 @@ class ExtractSecretsTests(SyncTestCase): def setUp(self): if PY2: - raise SkipTest("Not going to bother supporting Python 2") + self.skipTest("Not going to bother supporting Python 2") super(ExtractSecretsTests, self).setUp() @given(secrets_to_send=SECRETS_STRATEGY) @@ -165,7 +164,7 @@ class RoutingTests(AsyncTestCase): def setUp(self): if PY2: - raise SkipTest("Not going to bother supporting Python 2") + self.skipTest("Not going to bother supporting Python 2") super(RoutingTests, self).setUp() # Could be a fixture, but will only be used in this test class so not # going to bother: @@ -222,7 +221,7 @@ class GenericHTTPAPITests(AsyncTestCase): def setUp(self): if PY2: - raise SkipTest("Not going to bother supporting Python 2") + self.skipTest("Not going to bother supporting Python 2") super(GenericHTTPAPITests, self).setUp() self.http = self.useFixture(HttpTestFixture()) From e9aaaaccc4c9783a9d1eb74ca241d72797ceceec Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 22 Dec 2021 15:31:09 -0700 Subject: [PATCH 0528/2309] test for json welcome page --- src/allmydata/test/web/test_root.py | 88 ++++++++++++++++++++++++++++- 1 file changed, 86 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/web/test_root.py b/src/allmydata/test/web/test_root.py index 1d5e45ba4..b0789b1d2 100644 --- a/src/allmydata/test/web/test_root.py +++ b/src/allmydata/test/web/test_root.py @@ -11,6 +11,7 @@ if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import time +import json from urllib.parse import ( quote, @@ -24,14 +25,23 @@ from twisted.web.template import Tag from twisted.web.test.requesthelper import DummyRequest from twisted.application import service from testtools.twistedsupport import succeeded -from twisted.internet.defer import inlineCallbacks +from twisted.internet.defer import ( + inlineCallbacks, + succeed, +) from ...storage_client import ( NativeStorageServer, StorageFarmBroker, ) -from ...web.root import RootElement +from ...web.root import ( + RootElement, + Root, +) from ...util.connection_status import ConnectionStatus +from ...crypto.ed25519 import ( + create_signing_keypair, +) from allmydata.web.root import URIHandler from allmydata.client import _Client @@ -47,6 +57,7 @@ from ..common import ( from ..common import ( SyncTestCase, + AsyncTestCase, ) from testtools.matchers import ( @@ -138,3 +149,76 @@ class RenderServiceRow(SyncTestCase): self.assertThat(item.slotData.get("version"), Equals("")) self.assertThat(item.slotData.get("nickname"), Equals("")) + + +class RenderRoot(AsyncTestCase): + + @inlineCallbacks + def test_root_json(self): + """ + """ + ann = { + "anonymous-storage-FURL": "pb://w2hqnbaa25yw4qgcvghl5psa3srpfgw3@tcp:127.0.0.1:51309/vucto2z4fxment3vfxbqecblbf6zyp6x", + "permutation-seed-base32": "w2hqnbaa25yw4qgcvghl5psa3srpfgw3", + } + srv = NativeStorageServer(b"server_id", ann, None, {}, EMPTY_CLIENT_CONFIG) + srv.get_connection_status = lambda: ConnectionStatus(False, "summary", {}, 0, 0) + + class FakeClient(_Client): + history = [] + stats_provider = object() + nickname = "" + nodeid = b"asdf" + _node_public_key = create_signing_keypair()[1] + introducer_clients = [] + helper = None + + def __init__(self): + service.MultiService.__init__(self) + self.storage_broker = StorageFarmBroker( + permute_peers=True, + tub_maker=None, + node_config=EMPTY_CLIENT_CONFIG, + ) + self.storage_broker.test_add_server(b"test-srv", srv) + + root = Root(FakeClient(), now_fn=time.time) + + lines = [] + + req = DummyRequest(b"") + req.fields = {} + req.args = { + "t": ["json"], + } + + # for some reason, DummyRequest is already finished when we + # try to add a notifyFinish handler, so override that + # behavior. + + def nop(): + return succeed(None) + req.notifyFinish = nop + req.write = lines.append + + yield root.render(req) + + raw_js = b"".join(lines).decode("utf8") + self.assertThat( + json.loads(raw_js), + Equals({ + "introducers": { + "statuses": [] + }, + "servers": [ + { + "connection_status": "summary", + "nodeid": "server_id", + "last_received_data": 0, + "version": None, + "available_space": None, + "nickname": "" + } + ] + }) + ) From 94b540215f6c32db026cbcaf588f22a9ebdfa866 Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 22 Dec 2021 15:32:30 -0700 Subject: [PATCH 0529/2309] args are bytes --- src/allmydata/test/web/test_root.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/web/test_root.py b/src/allmydata/test/web/test_root.py index b0789b1d2..44b91fa48 100644 --- a/src/allmydata/test/web/test_root.py +++ b/src/allmydata/test/web/test_root.py @@ -189,7 +189,7 @@ class RenderRoot(AsyncTestCase): req = DummyRequest(b"") req.fields = {} req.args = { - "t": ["json"], + b"t": [b"json"], } # for some reason, DummyRequest is already finished when we From 5be5714bb378a9ad7180f7878ce75b96120afc5c Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 22 Dec 2021 15:32:40 -0700 Subject: [PATCH 0530/2309] fix; get rid of sorting --- src/allmydata/web/root.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/allmydata/web/root.py b/src/allmydata/web/root.py index 1debc1d10..f1a8569d6 100644 --- a/src/allmydata/web/root.py +++ b/src/allmydata/web/root.py @@ -297,14 +297,12 @@ class Root(MultiFormatResource): } return json.dumps(result, indent=1) + "\n" - def _describe_known_servers(self, broker): - return sorted(list( + return list( self._describe_server(server) for server in broker.get_known_servers() - ), key=lambda o: sorted(o.items())) - + ) def _describe_server(self, server): status = server.get_connection_status() From 872ce021c85b48321fe389200661cf3f087e959f Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 22 Dec 2021 15:32:59 -0700 Subject: [PATCH 0531/2309] news --- newsfragments/3852.minor | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/3852.minor diff --git a/newsfragments/3852.minor b/newsfragments/3852.minor new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/newsfragments/3852.minor @@ -0,0 +1 @@ + From 4c92f9c8cfd1d77d5549de3de50c552dbb442461 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 4 Jan 2022 13:10:23 -0500 Subject: [PATCH 0532/2309] Document additional semantics. --- docs/proposed/http-storage-node-protocol.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index 0d8cee466..a8555cd26 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -483,6 +483,13 @@ For example:: The upload secret is an opaque _byte_ string. +Handling repeat calls: + +* If the same API call is repeated with the same upload secret, the response is the same and no change is made to server state. + This is necessary to ensure retries work in the face of lost responses from the server. +* If the API calls is with a different upload secret, this implies a new client, perhaps because the old client died. + In this case, all relevant in-progress uploads are canceled, and then the command is handled as usual. + Discussion `````````` From cac291eb91129e39336ebc508727912224fda606 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 4 Jan 2022 13:10:38 -0500 Subject: [PATCH 0533/2309] News file. --- newsfragments/3855.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3855.minor diff --git a/newsfragments/3855.minor b/newsfragments/3855.minor new file mode 100644 index 000000000..e69de29bb From 5f4db487f787b7c88c01974005f19c1f854e8fa4 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 4 Jan 2022 13:43:19 -0500 Subject: [PATCH 0534/2309] Sketch of required business logic. --- src/allmydata/storage/http_server.py | 49 +++++++++++++++++++++++++++- 1 file changed, 48 insertions(+), 1 deletion(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 83bbbe49d..2dfb49b65 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -22,12 +22,14 @@ from base64 import b64decode from klein import Klein from twisted.web import http +import attr # TODO Make sure to use pure Python versions? -from cbor2 import dumps +from cbor2 import dumps, loads from .server import StorageServer from .http_client import swissnum_auth_header +from .immutable import BucketWriter from ..util.hashutil import timing_safe_compare @@ -125,6 +127,19 @@ def _authorized_route(app, required_secrets, *route_args, **route_kwargs): return decorator +@attr.s +class StorageIndexUploads(object): + """ + In-progress upload to storage index. + """ + + # Map share number to BucketWriter + shares = attr.ib() # type: Dict[int,BucketWriter] + + # The upload key. + upload_key = attr.ib() # type: bytes + + class HTTPServer(object): """ A HTTP interface to the storage server. @@ -137,6 +152,8 @@ class HTTPServer(object): ): # type: (StorageServer, bytes) -> None self._storage_server = storage_server self._swissnum = swissnum + # Maps storage index to StorageIndexUploads: + self._uploads = {} # type: Dict[bytes,StorageIndexUploads] def get_resource(self): """Return twisted.web ``Resource`` for this object.""" @@ -154,3 +171,33 @@ class HTTPServer(object): def version(self, request, authorization): """Return version information.""" return self._cbor(request, self._storage_server.get_version()) + + ##### Immutable APIs ##### + + @_authorized_route( + _app, + {Secrets.LEASE_RENEW, Secrets.LEASE_CANCEL, Secrets.UPLOAD}, + "/v1/immutable/", + methods=["POST"], + ) + def allocate_buckets(self, request, authorization, storage_index): + """Allocate buckets.""" + info = loads(request.content.read()) + upload_key = authorization[Secrets.UPLOAD] + + if storage_index in self._uploads: + # Pre-existing upload. + in_progress = self._uploads[storage_index] + if in_progress.upload_key == upload_key: + # Same session. + # TODO add BucketWriters only for new shares + pass + else: + # New session. + # TODO cancel all existing BucketWriters, then do + # self._storage_server.allocate_buckets() with given inputs. + pass + else: + # New upload. + # TODO self._storage_server.allocate_buckets() with given inputs. + # TODO add results to self._uploads. From 9c20ac8e7b1aaae6705b7980620410b792a4fee5 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 5 Jan 2022 16:06:29 -0500 Subject: [PATCH 0535/2309] Client API sketch for basic immutable interactions. --- src/allmydata/storage/http_client.py | 82 +++++++++++++++++++++++++++- 1 file changed, 79 insertions(+), 3 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 72e1af080..a13ab1ce6 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -16,17 +16,19 @@ if PY2: else: # typing module not available in Python 2, and we only do type checking in # Python 3 anyway. - from typing import Union + from typing import Union, Set, List from treq.testing import StubTreq from base64 import b64encode +import attr + # TODO Make sure to import Python version? from cbor2 import loads from twisted.web.http_headers import Headers -from twisted.internet.defer import inlineCallbacks, returnValue, fail +from twisted.internet.defer import inlineCallbacks, returnValue, fail, Deferred from hyperlink import DecodedURL import treq @@ -47,6 +49,80 @@ def swissnum_auth_header(swissnum): # type: (bytes) -> bytes return b"Tahoe-LAFS " + b64encode(swissnum).strip() +@attr.s +class ImmutableCreateResult(object): + """Result of creating a storage index for an immutable.""" + + already_have = attr.ib(type=Set[int]) + allocated = attr.ib(type=Set[int]) + + +class StorageClientImmutables(object): + """ + APIs for interacting with immutables. + """ + + def __init__(self, client): # type: (StorageClient) -> None + self._client = client + + @inlineCallbacks + def create( + self, + storage_index, + share_numbers, + allocated_size, + upload_secret, + lease_renew_secret, + lease_cancel_secret, + ): # type: (bytes, List[int], int, bytes, bytes, bytes) -> Deferred[ImmutableCreateResult] + """ + Create a new storage index for an immutable. + + TODO retry internally on failure, to ensure the operation fully + succeeded. If sufficient number of failures occurred, the result may + fire with an error, but there's no expectation that user code needs to + have a recovery codepath; it will most likely just report an error to + the user. + + Result fires when creating the storage index succeeded, if creating the + storage index failed the result will fire with an exception. + """ + + @inlineCallbacks + def write_share_chunk( + self, storage_index, share_number, upload_secret, offset, data + ): # type: (bytes, int, bytes, int, bytes) -> Deferred[bool] + """ + Upload a chunk of data for a specific share. + + TODO The implementation should retry failed uploads transparently a number + of times, so that if a failure percolates up, the caller can assume the + failure isn't a short-term blip. + + Result fires when the upload succeeded, with a boolean indicating + whether the _complete_ share (i.e. all chunks, not just this one) has + been uploaded. + """ + + @inlineCallbacks + def read_share_chunk( + self, storage_index, share_number, offset, length + ): # type: (bytes, int, int, int) -> Deferred[bytes] + """ + Download a chunk of data from a share. + + TODO Failed downloads should be transparently retried and redownloaded + by the implementation a few times so that if a failure percolates up, + the caller can assume the failure isn't a short-term blip. + + NOTE: the underlying HTTP protocol is much more flexible than this API, + so a future refactor may expand this in order to simplify the calling + code and perhaps download data more efficiently. But then again maybe + the HTTP protocol will be simplified, see + https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3777 + """ + + class StorageClient(object): """ HTTP client that talks to the HTTP storage server. @@ -77,7 +153,7 @@ class StorageClient(object): for key, value in secrets.items(): headers.addRawHeader( "X-Tahoe-Authorization", - b"%s %s" % (key.value.encode("ascii"), b64encode(value).strip()) + b"%s %s" % (key.value.encode("ascii"), b64encode(value).strip()), ) return self._treq.request(method, url, headers=headers, **kwargs) From 90a25d010953b1ddf702d27d21d354c13a703d2c Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 6 Jan 2022 12:36:46 -0500 Subject: [PATCH 0536/2309] Reorganize into shared file. --- src/allmydata/storage/http_client.py | 124 ++++++++++++++++----------- src/allmydata/storage/http_common.py | 25 ++++++ src/allmydata/storage/http_server.py | 12 +-- 3 files changed, 99 insertions(+), 62 deletions(-) create mode 100644 src/allmydata/storage/http_common.py diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index a13ab1ce6..cdcb94a94 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -24,7 +24,7 @@ from base64 import b64encode import attr # TODO Make sure to import Python version? -from cbor2 import loads +from cbor2 import loads, dumps from twisted.web.http_headers import Headers @@ -32,6 +32,8 @@ from twisted.internet.defer import inlineCallbacks, returnValue, fail, Deferred from hyperlink import DecodedURL import treq +from .http_common import swissnum_auth_header, Secrets + class ClientException(Exception): """An unexpected error.""" @@ -44,11 +46,6 @@ def _decode_cbor(response): return fail(ClientException(response.code, response.phrase)) -def swissnum_auth_header(swissnum): # type: (bytes) -> bytes - """Return value for ``Authentication`` header.""" - return b"Tahoe-LAFS " + b64encode(swissnum).strip() - - @attr.s class ImmutableCreateResult(object): """Result of creating a storage index for an immutable.""" @@ -57,12 +54,75 @@ class ImmutableCreateResult(object): allocated = attr.ib(type=Set[int]) +class StorageClient(object): + """ + HTTP client that talks to the HTTP storage server. + """ + + def __init__( + self, url, swissnum, treq=treq + ): # type: (DecodedURL, bytes, Union[treq,StubTreq]) -> None + self._base_url = url + self._swissnum = swissnum + self._treq = treq + + def _url(self, path): + """Get a URL relative to the base URL.""" + return self._base_url.click(path) + + def _get_headers(self): # type: () -> Headers + """Return the basic headers to be used by default.""" + headers = Headers() + headers.addRawHeader( + "Authorization", + swissnum_auth_header(self._swissnum), + ) + return headers + + def _request( + self, + method, + url, + lease_renewal_secret=None, + lease_cancel_secret=None, + upload_secret=None, + **kwargs + ): + """ + Like ``treq.request()``, but with optional secrets that get translated + into corresponding HTTP headers. + """ + headers = self._get_headers() + for secret, value in [ + (Secrets.LEASE_RENEW, lease_renewal_secret), + (Secrets.LEASE_CANCEL, lease_cancel_secret), + (Secrets.UPLOAD, upload_secret), + ]: + if value is None: + continue + headers.addRawHeader( + "X-Tahoe-Authorization", + b"%s %s" % (secret.value.encode("ascii"), b64encode(value).strip()), + ) + return self._treq.request(method, url, headers=headers, **kwargs) + + @inlineCallbacks + def get_version(self): + """ + Return the version metadata for the server. + """ + url = self._url("/v1/version") + response = yield self._request("GET", url, {}) + decoded_response = yield _decode_cbor(response) + returnValue(decoded_response) + + class StorageClientImmutables(object): """ APIs for interacting with immutables. """ - def __init__(self, client): # type: (StorageClient) -> None + def __init__(self, client: StorageClient):# # type: (StorageClient) -> None self._client = client @inlineCallbacks @@ -87,6 +147,11 @@ class StorageClientImmutables(object): Result fires when creating the storage index succeeded, if creating the storage index failed the result will fire with an exception. """ + url = self._client._url("/v1/immutable/" + str(storage_index, "ascii")) + message = dumps( + {"share-numbers": share_numbers, "allocated-size": allocated_size} + ) + self._client._request("POST", ) @inlineCallbacks def write_share_chunk( @@ -121,48 +186,3 @@ class StorageClientImmutables(object): the HTTP protocol will be simplified, see https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3777 """ - - -class StorageClient(object): - """ - HTTP client that talks to the HTTP storage server. - """ - - def __init__( - self, url, swissnum, treq=treq - ): # type: (DecodedURL, bytes, Union[treq,StubTreq]) -> None - self._base_url = url - self._swissnum = swissnum - self._treq = treq - - def _get_headers(self): # type: () -> Headers - """Return the basic headers to be used by default.""" - headers = Headers() - headers.addRawHeader( - "Authorization", - swissnum_auth_header(self._swissnum), - ) - return headers - - def _request(self, method, url, secrets, **kwargs): - """ - Like ``treq.request()``, but additional argument of secrets mapping - ``http_server.Secret`` to the bytes value of the secret. - """ - headers = self._get_headers() - for key, value in secrets.items(): - headers.addRawHeader( - "X-Tahoe-Authorization", - b"%s %s" % (key.value.encode("ascii"), b64encode(value).strip()), - ) - return self._treq.request(method, url, headers=headers, **kwargs) - - @inlineCallbacks - def get_version(self): - """ - Return the version metadata for the server. - """ - url = self._base_url.click("/v1/version") - response = yield self._request("GET", url, {}) - decoded_response = yield _decode_cbor(response) - returnValue(decoded_response) diff --git a/src/allmydata/storage/http_common.py b/src/allmydata/storage/http_common.py new file mode 100644 index 000000000..af4224bd0 --- /dev/null +++ b/src/allmydata/storage/http_common.py @@ -0,0 +1,25 @@ +""" +Common HTTP infrastructure for the storge server. +""" +from future.utils import PY2 + +if PY2: + # fmt: off + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + # fmt: on + +from enum import Enum +from base64 import b64encode + + +def swissnum_auth_header(swissnum): # type: (bytes) -> bytes + """Return value for ``Authentication`` header.""" + return b"Tahoe-LAFS " + b64encode(swissnum).strip() + + +class Secrets(Enum): + """Different kinds of secrets the client may send.""" + + LEASE_RENEW = "lease-renew-secret" + LEASE_CANCEL = "lease-cancel-secret" + UPLOAD = "upload-secret" diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 2dfb49b65..78752e9c5 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -17,7 +17,6 @@ else: from typing import Dict, List, Set from functools import wraps -from enum import Enum from base64 import b64decode from klein import Klein @@ -28,19 +27,11 @@ import attr from cbor2 import dumps, loads from .server import StorageServer -from .http_client import swissnum_auth_header +from .http_common import swissnum_auth_header, Secrets from .immutable import BucketWriter from ..util.hashutil import timing_safe_compare -class Secrets(Enum): - """Different kinds of secrets the client may send.""" - - LEASE_RENEW = "lease-renew-secret" - LEASE_CANCEL = "lease-cancel-secret" - UPLOAD = "upload-secret" - - class ClientSecretsException(Exception): """The client did not send the appropriate secrets.""" @@ -201,3 +192,4 @@ class HTTPServer(object): # New upload. # TODO self._storage_server.allocate_buckets() with given inputs. # TODO add results to self._uploads. + pass From 2f94fdf372116f18fde2d764b0331edb995303fb Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 6 Jan 2022 12:47:44 -0500 Subject: [PATCH 0537/2309] Extra testing coverage, including reproducer for #3854. --- src/allmydata/test/web/test_webish.py | 48 +++++++++++++++++++++++++-- 1 file changed, 46 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/web/test_webish.py b/src/allmydata/test/web/test_webish.py index 12a04a6eb..4a77d21ae 100644 --- a/src/allmydata/test/web/test_webish.py +++ b/src/allmydata/test/web/test_webish.py @@ -90,10 +90,11 @@ class TahoeLAFSRequestTests(SyncTestCase): """ self._fields_test(b"GET", {}, b"", Equals(None)) - def test_form_fields(self): + def test_form_fields_if_filename_set(self): """ When a ``POST`` request is received, form fields are parsed into - ``TahoeLAFSRequest.fields``. + ``TahoeLAFSRequest.fields`` and the body is bytes (presuming ``filename`` + is set). """ form_data, boundary = multipart_formdata([ [param(u"name", u"foo"), @@ -121,6 +122,49 @@ class TahoeLAFSRequestTests(SyncTestCase): ), ) + def test_form_fields_if_name_is_file(self): + """ + When a ``POST`` request is received, form fields are parsed into + ``TahoeLAFSRequest.fields`` and the body is bytes when ``name`` + is set to ``"file"``. + """ + form_data, boundary = multipart_formdata([ + [param(u"name", u"foo"), + body(u"bar"), + ], + [param(u"name", u"file"), + body(u"some file contents"), + ], + ]) + self._fields_test( + b"POST", + {b"content-type": b"multipart/form-data; boundary=" + bytes(boundary, 'ascii')}, + form_data.encode("ascii"), + AfterPreprocessing( + lambda fs: { + k: fs.getvalue(k) + for k + in fs.keys() + }, + Equals({ + "foo": "bar", + "file": b"some file contents", + }), + ), + ) + + def test_form_fields_require_correct_mime_type(self): + """ + The body of a ``POST`` is not parsed into fields if its mime type is + not ``multipart/form-data``. + + Reproducer for https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3854 + """ + data = u'{"lalala": "lolo"}' + data = data.encode("utf-8") + self._fields_test(b"POST", {"content-type": "application/json"}, + data, Equals(None)) + class TahoeLAFSSiteTests(SyncTestCase): """ From 9f5d7c6d22d40183aaab480990d83c122049495d Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 6 Jan 2022 13:09:25 -0500 Subject: [PATCH 0538/2309] Fix a bug where we did unnecessary parsing. --- src/allmydata/webish.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/allmydata/webish.py b/src/allmydata/webish.py index 9b63a220c..559b475cb 100644 --- a/src/allmydata/webish.py +++ b/src/allmydata/webish.py @@ -114,7 +114,8 @@ class TahoeLAFSRequest(Request, object): self.path, argstring = x self.args = parse_qs(argstring, 1) - if self.method == b'POST': + content_type = (self.requestHeaders.getRawHeaders("content-type") or [""])[0] + if self.method == b'POST' and content_type.split(";")[0] == "multipart/form-data": # We use FieldStorage here because it performs better than # cgi.parse_multipart(self.content, pdict) which is what # twisted.web.http.Request uses. From 310b77aef0765bf26a28ac4ed8d03f10c05dbb49 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 6 Jan 2022 13:10:13 -0500 Subject: [PATCH 0539/2309] News file. --- newsfragments/3854.bugfix | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/3854.bugfix diff --git a/newsfragments/3854.bugfix b/newsfragments/3854.bugfix new file mode 100644 index 000000000..d12e174f9 --- /dev/null +++ b/newsfragments/3854.bugfix @@ -0,0 +1 @@ +Fixed regression on Python 3 where JSON HTTP POSTs failed to be processed. \ No newline at end of file From 2864ff872d4ddb1f4a16f1669e597e6e7ab3565a Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 6 Jan 2022 13:34:56 -0500 Subject: [PATCH 0540/2309] Another MIME type that needs to be handled by FieldStorage. --- src/allmydata/webish.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/webish.py b/src/allmydata/webish.py index 559b475cb..519b3e1f0 100644 --- a/src/allmydata/webish.py +++ b/src/allmydata/webish.py @@ -115,7 +115,7 @@ class TahoeLAFSRequest(Request, object): self.args = parse_qs(argstring, 1) content_type = (self.requestHeaders.getRawHeaders("content-type") or [""])[0] - if self.method == b'POST' and content_type.split(";")[0] == "multipart/form-data": + if self.method == b'POST' and content_type.split(";")[0] in ("multipart/form-data", "application/x-www-form-urlencoded"): # We use FieldStorage here because it performs better than # cgi.parse_multipart(self.content, pdict) which is what # twisted.web.http.Request uses. From 983f90116b7b120d30a01b336f59bf1c0a62b9f2 Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 6 Jan 2022 13:15:31 -0700 Subject: [PATCH 0541/2309] check differently, don't depend on order --- src/allmydata/test/web/test_web.py | 50 +++++++++++++++++------------- 1 file changed, 29 insertions(+), 21 deletions(-) diff --git a/src/allmydata/test/web/test_web.py b/src/allmydata/test/web/test_web.py index 1c9d6b65c..03cd6e560 100644 --- a/src/allmydata/test/web/test_web.py +++ b/src/allmydata/test/web/test_web.py @@ -820,29 +820,37 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi """ d = self.GET("/?t=json") def _check(res): + """ + Check that the results are correct. + We can't depend on the order of servers in the output + """ decoded = json.loads(res) - expected = { - u'introducers': { - u'statuses': [], + self.assertEqual(decoded['introducers'], {u'statuses': []}) + actual_servers = decoded[u"servers"] + self.assertEquals(len(actual_servers), 2) + self.assertIn( + { + u"nodeid": u'other_nodeid', + u'available_space': 123456, + u'connection_status': u'summary', + u'last_received_data': 30, + u'nickname': u'other_nickname \u263b', + u'version': u'1.0', }, - u'servers': sorted([ - {u"nodeid": u'other_nodeid', - u'available_space': 123456, - u'connection_status': u'summary', - u'last_received_data': 30, - u'nickname': u'other_nickname \u263b', - u'version': u'1.0', - }, - {u"nodeid": u'disconnected_nodeid', - u'available_space': 123456, - u'connection_status': u'summary', - u'last_received_data': 35, - u'nickname': u'disconnected_nickname \u263b', - u'version': u'1.0', - }, - ], key=lambda o: sorted(o.items())), - } - self.assertEqual(expected, decoded) + actual_servers + ) + self.assertIn( + { + u"nodeid": u'disconnected_nodeid', + u'available_space': 123456, + u'connection_status': u'summary', + u'last_received_data': 35, + u'nickname': u'disconnected_nickname \u263b', + u'version': u'1.0', + }, + actual_servers + ) + d.addCallback(_check) return d From 0bf713c38ab36651e841ca8c84e23ecf104aea55 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 7 Jan 2022 10:12:21 -0500 Subject: [PATCH 0542/2309] News fragment. --- newsfragments/3856.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3856.minor diff --git a/newsfragments/3856.minor b/newsfragments/3856.minor new file mode 100644 index 000000000..e69de29bb From 7e3cb44ede60de3bed90470bf7f7803abac607b9 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 7 Jan 2022 10:13:29 -0500 Subject: [PATCH 0543/2309] Pin non-broken version of Paramiko. --- setup.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 7e7a955c6..53057b808 100644 --- a/setup.py +++ b/setup.py @@ -409,7 +409,9 @@ setup(name="tahoe-lafs", # also set in __init__.py "html5lib", "junitxml", "tenacity", - "paramiko", + # Pin old version until + # https://github.com/paramiko/paramiko/issues/1961 is fixed. + "paramiko < 2.9", "pytest-timeout", # Does our OpenMetrics endpoint adhere to the spec: "prometheus-client == 0.11.0", From 11f2097591e8161416237ecb4676d1843478eb5d Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 7 Jan 2022 10:58:58 -0700 Subject: [PATCH 0544/2309] docstring --- src/allmydata/test/web/test_root.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/allmydata/test/web/test_root.py b/src/allmydata/test/web/test_root.py index 44b91fa48..199c8e545 100644 --- a/src/allmydata/test/web/test_root.py +++ b/src/allmydata/test/web/test_root.py @@ -156,6 +156,11 @@ class RenderRoot(AsyncTestCase): @inlineCallbacks def test_root_json(self): """ + The 'welcome' / root page renders properly with ?t=json when some + servers show None for available_space while others show a + valid int + + See also https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3852 """ ann = { "anonymous-storage-FURL": "pb://w2hqnbaa25yw4qgcvghl5psa3srpfgw3@tcp:127.0.0.1:51309/vucto2z4fxment3vfxbqecblbf6zyp6x", From a49baf44b68eac81bb1538000c042ed537e57ef0 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 7 Jan 2022 10:59:13 -0700 Subject: [PATCH 0545/2309] actually-reproduce 3852 --- src/allmydata/test/web/test_root.py | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/src/allmydata/test/web/test_root.py b/src/allmydata/test/web/test_root.py index 199c8e545..8c46b809a 100644 --- a/src/allmydata/test/web/test_root.py +++ b/src/allmydata/test/web/test_root.py @@ -166,8 +166,13 @@ class RenderRoot(AsyncTestCase): "anonymous-storage-FURL": "pb://w2hqnbaa25yw4qgcvghl5psa3srpfgw3@tcp:127.0.0.1:51309/vucto2z4fxment3vfxbqecblbf6zyp6x", "permutation-seed-base32": "w2hqnbaa25yw4qgcvghl5psa3srpfgw3", } - srv = NativeStorageServer(b"server_id", ann, None, {}, EMPTY_CLIENT_CONFIG) - srv.get_connection_status = lambda: ConnectionStatus(False, "summary", {}, 0, 0) + srv0 = NativeStorageServer(b"server_id0", ann, None, {}, EMPTY_CLIENT_CONFIG) + srv0.get_connection_status = lambda: ConnectionStatus(False, "summary0", {}, 0, 0) + + srv1 = NativeStorageServer(b"server_id1", ann, None, {}, EMPTY_CLIENT_CONFIG) + srv1.get_connection_status = lambda: ConnectionStatus(False, "summary1", {}, 0, 0) + # arrange for this server to have some valid available space + srv1.get_available_space = lambda: 12345 class FakeClient(_Client): history = [] @@ -185,7 +190,8 @@ class RenderRoot(AsyncTestCase): tub_maker=None, node_config=EMPTY_CLIENT_CONFIG, ) - self.storage_broker.test_add_server(b"test-srv", srv) + self.storage_broker.test_add_server(b"test-srv0", srv0) + self.storage_broker.test_add_server(b"test-srv1", srv1) root = Root(FakeClient(), now_fn=time.time) @@ -217,12 +223,20 @@ class RenderRoot(AsyncTestCase): }, "servers": [ { - "connection_status": "summary", - "nodeid": "server_id", + "connection_status": "summary0", + "nodeid": "server_id0", "last_received_data": 0, "version": None, "available_space": None, "nickname": "" + }, + { + "connection_status": "summary1", + "nodeid": "server_id1", + "last_received_data": 0, + "version": None, + "available_space": 12345, + "nickname": "" } ] }) From e8f5023ae2e8b6404f3d7ad37db34fb28a3c4333 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 7 Jan 2022 10:59:34 -0700 Subject: [PATCH 0546/2309] its a bugfix --- newsfragments/3852.minor => 3852.bugfix | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename newsfragments/3852.minor => 3852.bugfix (100%) diff --git a/newsfragments/3852.minor b/3852.bugfix similarity index 100% rename from newsfragments/3852.minor rename to 3852.bugfix From 9d823aef67d7328c9b8ee2d2ae75703b5cd3b26a Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 7 Jan 2022 11:05:35 -0700 Subject: [PATCH 0547/2309] newsfragment to correct spot --- 3852.bugfix => newsfragments/3852.bugfix | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename 3852.bugfix => newsfragments/3852.bugfix (100%) diff --git a/3852.bugfix b/newsfragments/3852.bugfix similarity index 100% rename from 3852.bugfix rename to newsfragments/3852.bugfix From 9644532916de535b738f1e85f5ce060c5e77c604 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 7 Jan 2022 11:28:55 -0700 Subject: [PATCH 0548/2309] don't depend on order --- src/allmydata/test/web/test_root.py | 49 ++++++++++++++--------------- 1 file changed, 24 insertions(+), 25 deletions(-) diff --git a/src/allmydata/test/web/test_root.py b/src/allmydata/test/web/test_root.py index 8c46b809a..228b8e449 100644 --- a/src/allmydata/test/web/test_root.py +++ b/src/allmydata/test/web/test_root.py @@ -215,29 +215,28 @@ class RenderRoot(AsyncTestCase): yield root.render(req) raw_js = b"".join(lines).decode("utf8") - self.assertThat( - json.loads(raw_js), - Equals({ - "introducers": { - "statuses": [] - }, - "servers": [ - { - "connection_status": "summary0", - "nodeid": "server_id0", - "last_received_data": 0, - "version": None, - "available_space": None, - "nickname": "" - }, - { - "connection_status": "summary1", - "nodeid": "server_id1", - "last_received_data": 0, - "version": None, - "available_space": 12345, - "nickname": "" - } - ] - }) + js = json.loads(raw_js) + servers = js["servers"] + self.assertEquals(len(servers), 2) + self.assertIn( + { + "connection_status": "summary0", + "nodeid": "server_id0", + "last_received_data": 0, + "version": None, + "available_space": None, + "nickname": "" + }, + servers + ) + self.assertIn( + { + "connection_status": "summary1", + "nodeid": "server_id1", + "last_received_data": 0, + "version": None, + "available_space": 12345, + "nickname": "" + }, + servers ) From b91835a2007764fc924d05f484563b303100f8b5 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 7 Jan 2022 13:06:26 -0700 Subject: [PATCH 0549/2309] update NEWS.txt for release --- NEWS.rst | 14 ++++++++++++++ newsfragments/3848.minor | 0 newsfragments/3849.minor | 0 newsfragments/3850.minor | 0 newsfragments/3852.bugfix | 1 - newsfragments/3854.bugfix | 1 - newsfragments/3856.minor | 0 7 files changed, 14 insertions(+), 2 deletions(-) delete mode 100644 newsfragments/3848.minor delete mode 100644 newsfragments/3849.minor delete mode 100644 newsfragments/3850.minor delete mode 100644 newsfragments/3852.bugfix delete mode 100644 newsfragments/3854.bugfix delete mode 100644 newsfragments/3856.minor diff --git a/NEWS.rst b/NEWS.rst index 15cb9459d..62d1587dd 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -5,6 +5,20 @@ User-Visible Changes in Tahoe-LAFS ================================== .. towncrier start line +Release 1.17.0.post55 (2022-01-07)Release 1.17.0.post55 (2022-01-07) +'''''''''''''''''''''''''''''''''' + +Bug Fixes +--------- + +- (`#3852 `_) +- Fixed regression on Python 3 where JSON HTTP POSTs failed to be processed. (`#3854 `_) + + +Misc/Other +---------- + +- `#3848 `_, `#3849 `_, `#3850 `_, `#3856 `_ Release 1.17.0 (2021-12-06) diff --git a/newsfragments/3848.minor b/newsfragments/3848.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3849.minor b/newsfragments/3849.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3850.minor b/newsfragments/3850.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3852.bugfix b/newsfragments/3852.bugfix deleted file mode 100644 index 8b1378917..000000000 --- a/newsfragments/3852.bugfix +++ /dev/null @@ -1 +0,0 @@ - diff --git a/newsfragments/3854.bugfix b/newsfragments/3854.bugfix deleted file mode 100644 index d12e174f9..000000000 --- a/newsfragments/3854.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fixed regression on Python 3 where JSON HTTP POSTs failed to be processed. \ No newline at end of file diff --git a/newsfragments/3856.minor b/newsfragments/3856.minor deleted file mode 100644 index e69de29bb..000000000 From 22734dccba2cc95752de1c360830b728d4ac83b2 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 7 Jan 2022 13:13:44 -0700 Subject: [PATCH 0550/2309] fix text for 3852 --- NEWS.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NEWS.rst b/NEWS.rst index 62d1587dd..c2d405f8f 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -11,7 +11,7 @@ Release 1.17.0.post55 (2022-01-07)Release 1.17.0.post55 (2022-01-07) Bug Fixes --------- -- (`#3852 `_) +- Fixed regression on Python 3 causing the JSON version of the Welcome page to sometimes produce a 500 error (`#3852 `_) - Fixed regression on Python 3 where JSON HTTP POSTs failed to be processed. (`#3854 `_) From e9ece061f4a1a6373f39fe37291e1775df3a0391 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 7 Jan 2022 13:18:03 -0700 Subject: [PATCH 0551/2309] news --- newsfragments/3858.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3858.minor diff --git a/newsfragments/3858.minor b/newsfragments/3858.minor new file mode 100644 index 000000000..e69de29bb From f9ddd3b3bedf692ffdf598d9def96b3c79097602 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 7 Jan 2022 13:21:44 -0700 Subject: [PATCH 0552/2309] fix NEWS title --- NEWS.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/NEWS.rst b/NEWS.rst index c2d405f8f..0f9194cc4 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -5,8 +5,8 @@ User-Visible Changes in Tahoe-LAFS ================================== .. towncrier start line -Release 1.17.0.post55 (2022-01-07)Release 1.17.0.post55 (2022-01-07) -'''''''''''''''''''''''''''''''''' +Release 1.17.1 (2022-01-07) +''''''''''''''''''''''''''' Bug Fixes --------- From b5251eb0a12eaa07411473583facd5c21cee729f Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 7 Jan 2022 13:27:53 -0700 Subject: [PATCH 0553/2309] update relnotes --- relnotes.txt | 45 +++++++++++++++++++-------------------------- 1 file changed, 19 insertions(+), 26 deletions(-) diff --git a/relnotes.txt b/relnotes.txt index dff4f192e..e9b298771 100644 --- a/relnotes.txt +++ b/relnotes.txt @@ -1,6 +1,6 @@ -ANNOUNCING Tahoe, the Least-Authority File Store, v1.17.0 +ANNOUNCING Tahoe, the Least-Authority File Store, v1.17.1 -The Tahoe-LAFS team is pleased to announce version 1.17.0 of +The Tahoe-LAFS team is pleased to announce version 1.17.1 of Tahoe-LAFS, an extremely reliable decentralized storage system. Get it with "pip install tahoe-lafs", or download a tarball here: @@ -15,19 +15,12 @@ unique security and fault-tolerance properties: https://tahoe-lafs.readthedocs.org/en/latest/about.html -The previous stable release of Tahoe-LAFS was v1.16.0, released on -October 19, 2021. +The previous stable release of Tahoe-LAFS was v1.17.0, released on +December 6, 2021. -This release fixes several security issues raised as part of an audit -by Cure53. We developed fixes for these issues in a private -repository. Shortly after this release, public tickets will be updated -with further information (along with, of course, all the code). +This release fixes two Python3-releated regressions and 4 minor bugs. -There is also OpenMetrics support now and several bug fixes. - -In all, 46 issues have been fixed since the last release. - -Please see ``NEWS.rst`` for a more complete list of changes. +Please see ``NEWS.rst`` [1] for a complete list of changes. WHAT IS IT GOOD FOR? @@ -66,12 +59,12 @@ to v1.0 (which was released March 25, 2008). Clients from this release can read files and directories produced by clients of all versions since v1.0. -Network connections are limited by the Introducer protocol in -use. If the Introducer is running v1.10 or v1.11, then servers -from this release (v1.12) can serve clients of all versions -back to v1.0 . If it is running v1.12, then they can only -serve clients back to v1.10. Clients from this release can use -servers back to v1.10, but not older servers. +Network connections are limited by the Introducer protocol in use. If +the Introducer is running v1.10 or v1.11, then servers from this +release can serve clients of all versions back to v1.0 . If it is +running v1.12 or higher, then they can only serve clients back to +v1.10. Clients from this release can use servers back to v1.10, but +not older servers. Except for the new optional MDMF format, we have not made any intentional compatibility changes. However we do not yet have @@ -79,7 +72,7 @@ the test infrastructure to continuously verify that all new versions are interoperable with previous versions. We intend to build such an infrastructure in the future. -This is the twenty-first release in the version 1 series. This +This is the twenty-second release in the version 1 series. This series of Tahoe-LAFS will be actively supported and maintained for the foreseeable future, and future versions of Tahoe-LAFS will retain the ability to read and write files compatible @@ -139,7 +132,7 @@ Of Fame" [13]. ACKNOWLEDGEMENTS -This is the eighteenth release of Tahoe-LAFS to be created +This is the nineteenth release of Tahoe-LAFS to be created solely as a labor of love by volunteers. Thank you very much to the team of "hackers in the public interest" who make Tahoe-LAFS possible. @@ -147,16 +140,16 @@ Tahoe-LAFS possible. meejah on behalf of the Tahoe-LAFS team -December 6, 2021 +January 7, 2022 Planet Earth -[1] https://github.com/tahoe-lafs/tahoe-lafs/blob/tahoe-lafs-1.17.0/NEWS.rst +[1] https://github.com/tahoe-lafs/tahoe-lafs/blob/tahoe-lafs-1.17.1/NEWS.rst [2] https://github.com/tahoe-lafs/tahoe-lafs/blob/master/docs/known_issues.rst [3] https://tahoe-lafs.org/trac/tahoe-lafs/wiki/RelatedProjects -[4] https://github.com/tahoe-lafs/tahoe-lafs/blob/tahoe-lafs-1.17.0/COPYING.GPL -[5] https://github.com/tahoe-lafs/tahoe-lafs/blob/tahoe-lafs-1.17.0/COPYING.TGPPL.rst -[6] https://tahoe-lafs.readthedocs.org/en/tahoe-lafs-1.17.0/INSTALL.html +[4] https://github.com/tahoe-lafs/tahoe-lafs/blob/tahoe-lafs-1.17.1/COPYING.GPL +[5] https://github.com/tahoe-lafs/tahoe-lafs/blob/tahoe-lafs-1.17.1/COPYING.TGPPL.rst +[6] https://tahoe-lafs.readthedocs.org/en/tahoe-lafs-1.17.1/INSTALL.html [7] https://lists.tahoe-lafs.org/mailman/listinfo/tahoe-dev [8] https://tahoe-lafs.org/trac/tahoe-lafs/roadmap [9] https://github.com/tahoe-lafs/tahoe-lafs/blob/master/CREDITS From c7664762365e1e14dd2c52374e3206ecd9b077a5 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 7 Jan 2022 13:28:27 -0700 Subject: [PATCH 0554/2309] nix --- nix/tahoe-lafs.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nix/tahoe-lafs.nix b/nix/tahoe-lafs.nix index 04d6c4163..2b41e676e 100644 --- a/nix/tahoe-lafs.nix +++ b/nix/tahoe-lafs.nix @@ -7,7 +7,7 @@ , html5lib, pyutil, distro, configparser, klein, cbor2 }: python.pkgs.buildPythonPackage rec { - # Most of the time this is not exactly the release version (eg 1.17.0). + # Most of the time this is not exactly the release version (eg 1.17.1). # Give it a `post` component to make it look newer than the release version # and we'll bump this up at the time of each release. # @@ -20,7 +20,7 @@ python.pkgs.buildPythonPackage rec { # is not a reproducable artifact (in the sense of "reproducable builds") so # it is excluded from the source tree by default. When it is included, the # package tends to be frequently spuriously rebuilt. - version = "1.17.0.post1"; + version = "1.17.1.post1"; name = "tahoe-lafs-${version}"; src = lib.cleanSourceWith { src = ../.; From aa81bfc937b1ee7bbe2b43e814cc7683eed1d29e Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 7 Jan 2022 13:29:45 -0700 Subject: [PATCH 0555/2309] cleanup whitespace --- docs/Installation/install-tahoe.rst | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/Installation/install-tahoe.rst b/docs/Installation/install-tahoe.rst index 2fe47f4a8..8ceca2e01 100644 --- a/docs/Installation/install-tahoe.rst +++ b/docs/Installation/install-tahoe.rst @@ -28,15 +28,15 @@ To install Tahoe-LAFS on Windows: 3. Open the installer by double-clicking it. Select the **Add Python to PATH** check-box, then click **Install Now**. 4. Start PowerShell and enter the following command to verify python installation:: - + python --version 5. Enter the following command to install Tahoe-LAFS:: - + pip install tahoe-lafs 6. Verify installation by checking for the version:: - + tahoe --version If you want to hack on Tahoe's source code, you can install Tahoe in a ``virtualenv`` on your Windows Machine. To learn more, see :doc:`install-on-windows`. @@ -56,13 +56,13 @@ If you are working on MacOS or a Linux distribution which does not have Tahoe-LA * **pip**: Most python installations already include `pip`. However, if your installation does not, see `pip installation `_. 2. Install Tahoe-LAFS using pip:: - + pip install tahoe-lafs 3. Verify installation by checking for the version:: - + tahoe --version -If you are looking to hack on the source code or run pre-release code, we recommend you install Tahoe-LAFS on a `virtualenv` instance. To learn more, see :doc:`install-on-linux`. +If you are looking to hack on the source code or run pre-release code, we recommend you install Tahoe-LAFS on a `virtualenv` instance. To learn more, see :doc:`install-on-linux`. You can always write to the `tahoe-dev mailing list `_ or chat on the `Libera.chat IRC `_ if you are not able to get Tahoe-LAFS up and running on your deployment. From f7477043c5025642ef0fbeb042310decb774bd01 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 7 Jan 2022 13:29:52 -0700 Subject: [PATCH 0556/2309] unnecessary step --- docs/release-checklist.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/release-checklist.rst b/docs/release-checklist.rst index f943abb5d..2b954449e 100644 --- a/docs/release-checklist.rst +++ b/docs/release-checklist.rst @@ -70,7 +70,6 @@ Create Branch and Apply Updates - commit it - update "docs/known_issues.rst" if appropriate -- update "docs/Installation/install-tahoe.rst" references to the new release - Push the branch to github - Create a (draft) PR; this should trigger CI (note that github doesn't let you create a PR without some changes on the branch so From 852ebe90e5bd5b04b5a75d1850df87673a78955f Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 10 Jan 2022 11:48:55 -0700 Subject: [PATCH 0557/2309] clean clone --- docs/release-checklist.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/release-checklist.rst b/docs/release-checklist.rst index 2b954449e..edfe9e20f 100644 --- a/docs/release-checklist.rst +++ b/docs/release-checklist.rst @@ -106,6 +106,11 @@ they will need to evaluate which contributors' signatures they trust. - tox -e deprecations,upcoming-deprecations +- clone to a clean, local checkout (to avoid extra files being included in the release) + + - cd /tmp + - git clone /home/meejah/src/tahoe-lafs + - build tarballs - tox -e tarballs From d2ff2a7376f99f08fba22ee3c3b28cba535a0117 Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 10 Jan 2022 11:49:02 -0700 Subject: [PATCH 0558/2309] whitespace --- docs/release-checklist.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/release-checklist.rst b/docs/release-checklist.rst index edfe9e20f..3c984d122 100644 --- a/docs/release-checklist.rst +++ b/docs/release-checklist.rst @@ -158,7 +158,8 @@ need to be uploaded to https://tahoe-lafs.org in `~source/downloads` - secure-copy all release artifacts to the download area on the tahoe-lafs.org host machine. `~source/downloads` on there maps to https://tahoe-lafs.org/downloads/ on the Web. -- scp dist/*1.15.0* username@tahoe-lafs.org:/home/source/downloads + - scp dist/*1.15.0* username@tahoe-lafs.org:/home/source/downloads + - the following developers have access to do this: - exarkun From 1446c9c4adebda255276659dfef883f17770ca7f Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 10 Jan 2022 11:49:15 -0700 Subject: [PATCH 0559/2309] add 'push the tags' step --- docs/release-checklist.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/release-checklist.rst b/docs/release-checklist.rst index 3c984d122..7acca6bb3 100644 --- a/docs/release-checklist.rst +++ b/docs/release-checklist.rst @@ -166,6 +166,10 @@ need to be uploaded to https://tahoe-lafs.org in `~source/downloads` - meejah - warner +Push the signed tag to the main repository: + +- git push origin_push tahoe-lafs-1.17.1 + For the actual release, the tarball and signature files need to be uploaded to PyPI as well. From 8cd4e7a4b5069c3fb30c195934974755e8f0c53c Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 10 Jan 2022 11:49:31 -0700 Subject: [PATCH 0560/2309] news --- newsfragments/3859.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3859.minor diff --git a/newsfragments/3859.minor b/newsfragments/3859.minor new file mode 100644 index 000000000..e69de29bb From ea83b16d1171b789c2041ed1e67e2ffa6dec3ff4 Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 10 Jan 2022 14:17:50 -0700 Subject: [PATCH 0561/2309] most people say 'origin' --- docs/release-checklist.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/release-checklist.rst b/docs/release-checklist.rst index 7acca6bb3..165aa8826 100644 --- a/docs/release-checklist.rst +++ b/docs/release-checklist.rst @@ -168,7 +168,7 @@ need to be uploaded to https://tahoe-lafs.org in `~source/downloads` Push the signed tag to the main repository: -- git push origin_push tahoe-lafs-1.17.1 +- git push origin tahoe-lafs-1.17.1 For the actual release, the tarball and signature files need to be uploaded to PyPI as well. From a753a71105a8865cb27c9a59258fe349d55ba06a Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 10 Jan 2022 14:22:57 -0700 Subject: [PATCH 0562/2309] please the Sphinx --- docs/release-checklist.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/release-checklist.rst b/docs/release-checklist.rst index 165aa8826..d2f1b3eb8 100644 --- a/docs/release-checklist.rst +++ b/docs/release-checklist.rst @@ -157,7 +157,8 @@ need to be uploaded to https://tahoe-lafs.org in `~source/downloads` - secure-copy all release artifacts to the download area on the tahoe-lafs.org host machine. `~source/downloads` on there maps to - https://tahoe-lafs.org/downloads/ on the Web. + https://tahoe-lafs.org/downloads/ on the Web: + - scp dist/*1.15.0* username@tahoe-lafs.org:/home/source/downloads - the following developers have access to do this: From 57405ea722d9ea97b74d71c2dca9a7eacd1b4ad5 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 7 Jan 2022 14:15:16 -0500 Subject: [PATCH 0563/2309] Finish sketch of minimal immutable HTTP client code. --- src/allmydata/storage/http_client.py | 102 ++++++++++++++++++++++----- 1 file changed, 86 insertions(+), 16 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index cdcb94a94..3accb3c62 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -16,7 +16,7 @@ if PY2: else: # typing module not available in Python 2, and we only do type checking in # Python 3 anyway. - from typing import Union, Set, List + from typing import Union, Set, List, Optional from treq.testing import StubTreq from base64 import b64encode @@ -28,6 +28,7 @@ from cbor2 import loads, dumps from twisted.web.http_headers import Headers +from twisted.web import http from twisted.internet.defer import inlineCallbacks, returnValue, fail, Deferred from hyperlink import DecodedURL import treq @@ -70,9 +71,10 @@ class StorageClient(object): """Get a URL relative to the base URL.""" return self._base_url.click(path) - def _get_headers(self): # type: () -> Headers + def _get_headers(self, headers): # type: (Optional[Headers]) -> Headers """Return the basic headers to be used by default.""" - headers = Headers() + if headers is None: + headers = Headers() headers.addRawHeader( "Authorization", swissnum_auth_header(self._swissnum), @@ -86,13 +88,14 @@ class StorageClient(object): lease_renewal_secret=None, lease_cancel_secret=None, upload_secret=None, + headers=None, **kwargs ): """ Like ``treq.request()``, but with optional secrets that get translated into corresponding HTTP headers. """ - headers = self._get_headers() + headers = self._get_headers(headers) for secret, value in [ (Secrets.LEASE_RENEW, lease_renewal_secret), (Secrets.LEASE_CANCEL, lease_cancel_secret), @@ -122,7 +125,7 @@ class StorageClientImmutables(object): APIs for interacting with immutables. """ - def __init__(self, client: StorageClient):# # type: (StorageClient) -> None + def __init__(self, client: StorageClient): # # type: (StorageClient) -> None self._client = client @inlineCallbacks @@ -138,11 +141,12 @@ class StorageClientImmutables(object): """ Create a new storage index for an immutable. - TODO retry internally on failure, to ensure the operation fully - succeeded. If sufficient number of failures occurred, the result may - fire with an error, but there's no expectation that user code needs to - have a recovery codepath; it will most likely just report an error to - the user. + TODO https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3857 retry + internally on failure, to ensure the operation fully succeeded. If + sufficient number of failures occurred, the result may fire with an + error, but there's no expectation that user code needs to have a + recovery codepath; it will most likely just report an error to the + user. Result fires when creating the storage index succeeded, if creating the storage index failed the result will fire with an exception. @@ -151,7 +155,22 @@ class StorageClientImmutables(object): message = dumps( {"share-numbers": share_numbers, "allocated-size": allocated_size} ) - self._client._request("POST", ) + response = yield self._client._request( + "POST", + url, + lease_renew_secret=lease_renew_secret, + lease_cancel_secret=lease_cancel_secret, + upload_secret=upload_secret, + data=message, + headers=Headers({"content-type": "application/cbor"}), + ) + decoded_response = yield _decode_cbor(response) + returnValue( + ImmutableCreateResult( + already_have=decoded_response["already-have"], + allocated=decoded_response["allocated"], + ) + ) @inlineCallbacks def write_share_chunk( @@ -160,14 +179,45 @@ class StorageClientImmutables(object): """ Upload a chunk of data for a specific share. - TODO The implementation should retry failed uploads transparently a number - of times, so that if a failure percolates up, the caller can assume the + TODO https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3857 The + implementation should retry failed uploads transparently a number of + times, so that if a failure percolates up, the caller can assume the failure isn't a short-term blip. Result fires when the upload succeeded, with a boolean indicating whether the _complete_ share (i.e. all chunks, not just this one) has been uploaded. """ + url = self._client._url( + "/v1/immutable/{}/{}".format(str(storage_index, "ascii"), share_number) + ) + response = yield self._client._request( + "POST", + url, + upload_secret=upload_secret, + data=data, + headers=Headers( + { + # The range is inclusive, thus the '- 1'. '*' means "length + # unknown", which isn't technically true but adding it just + # makes things slightly harder for calling API. + "content-range": "bytes {}-{}/*".format( + offset, offset + len(data) - 1 + ) + } + ), + ) + + if response.code == http.OK: + # Upload is still unfinished. + returnValue(False) + elif response.code == http.CREATED: + # Upload is done! + returnValue(True) + else: + raise ClientException( + response.code, + ) @inlineCallbacks def read_share_chunk( @@ -176,9 +226,10 @@ class StorageClientImmutables(object): """ Download a chunk of data from a share. - TODO Failed downloads should be transparently retried and redownloaded - by the implementation a few times so that if a failure percolates up, - the caller can assume the failure isn't a short-term blip. + TODO https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3857 Failed + downloads should be transparently retried and redownloaded by the + implementation a few times so that if a failure percolates up, the + caller can assume the failure isn't a short-term blip. NOTE: the underlying HTTP protocol is much more flexible than this API, so a future refactor may expand this in order to simplify the calling @@ -186,3 +237,22 @@ class StorageClientImmutables(object): the HTTP protocol will be simplified, see https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3777 """ + url = self._client._url( + "/v1/immutable/{}/{}".format(str(storage_index, "ascii"), share_number) + ) + response = yield self._client._request( + "GET", + url, + headers=Headers( + { + # The range is inclusive, thus the -1. + "range": "bytes={}-{}".format(offset, offset + length - 1) + } + ), + ) + if response.code == 200: + returnValue(response.content.read()) + else: + raise ClientException( + response.code, + ) From db68defe8897c48fb4a0ad31b9959b89664882b8 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 11 Jan 2022 14:50:29 -0500 Subject: [PATCH 0564/2309] Sketch of basic immutable server-side logic. --- src/allmydata/storage/http_server.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 78752e9c5..d2c9f6b7a 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -193,3 +193,31 @@ class HTTPServer(object): # TODO self._storage_server.allocate_buckets() with given inputs. # TODO add results to self._uploads. pass + + @_authorized_route( + _app, + {Secrets.UPLOAD}, "/v1/immutable//", methods=["PATCH"] + ) + def write_share_data(self, request, authorization, storage_index, share_number): + """Write data to an in-progress immutable upload.""" + # TODO parse the content-range header to get offset for writing + # TODO basic checks on validity of offset + # TODO basic check that body isn't infinite. require content-length? if so, needs t be in protocol spec. + data = request.content.read() + # TODO write to bucket at that offset. + + # TODO check if it conflicts with existing data (probably underlying code already handles that) if so, CONFLICT. + + # TODO if it finished writing altogether, 201 CREATED. Otherwise 200 OK. + + @_authorized_route( + _app, set(), "/v1/immutable//", methods=["GET"] + ) + def read_share_chunk(self, request, authorization, storage_index, share_number): + """Read a chunk for an already uploaded immutable.""" + # TODO read offset and length from Range header + # TODO basic checks on validity + # TODO lookup the share + # TODO if not found, 404 + # TODO otherwise, return data from that offset + From 040569b47acac5a00eddcbd03ef880c2b4f6727a Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 11 Jan 2022 15:11:16 -0500 Subject: [PATCH 0565/2309] Sketch of tests to write for basic HTTP immutable APIs. --- src/allmydata/test/test_storage_http.py | 69 +++++++++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 181b6d347..6cf7a521e 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -263,3 +263,72 @@ class GenericHTTPAPITests(AsyncTestCase): b"maximum-immutable-share-size" ) self.assertEqual(version, expected_version) + + +class ImmutableHTTPAPITests(AsyncTestCase): + """ + Tests for immutable upload/download APIs. + """ + + def test_upload_can_be_downloaded(self): + """ + A single share can be uploaded in (possibly overlapping) chunks, and + then a random chunk can be downloaded, and it will match the original + file. + """ + + def test_multiple_shares_uploaded_to_different_place(self): + """ + If a storage index has multiple shares, uploads to different shares are + stored separately and can be downloaded separately. + """ + + def test_bucket_allocated_with_new_shares(self): + """ + If some shares already exist, allocating shares indicates only the new + ones were created. + """ + + def test_bucket_allocation_new_upload_key(self): + """ + If a bucket was allocated with one upload key, and a different upload + key is used to allocate the bucket again, the previous download is + cancelled. + """ + + def test_upload_with_wrong_upload_key_fails(self): + """ + Uploading with a key that doesn't match the one used to allocate the + bucket will fail. + """ + + def test_upload_offset_cannot_be_negative(self): + """ + A negative upload offset will be rejected. + """ + + def test_mismatching_upload_fails(self): + """ + If an uploaded chunk conflicts with an already uploaded chunk, a + CONFLICT error is returned. + """ + + def test_read_of_wrong_storage_index_fails(self): + """ + Reading from unknown storage index results in 404. + """ + + def test_read_of_wrong_share_number_fails(self): + """ + Reading from unknown storage index results in 404. + """ + + def test_read_with_negative_offset_fails(self): + """ + The offset for reads cannot be negative. + """ + + def test_read_with_negative_length_fails(self): + """ + The length for reads cannot be negative. + """ From 2369de6873f20791f36c884760085010cca64941 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 11 Jan 2022 15:45:15 -0500 Subject: [PATCH 0566/2309] Simple upload/download test for immutables. --- src/allmydata/test/test_storage_http.py | 69 ++++++++++++++++++++++++- 1 file changed, 68 insertions(+), 1 deletion(-) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 6cf7a521e..95708d211 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -15,6 +15,7 @@ if PY2: # fmt: on from base64 import b64encode +from os import urandom from twisted.internet.defer import inlineCallbacks @@ -33,7 +34,12 @@ from ..storage.http_server import ( ClientSecretsException, _authorized_route, ) -from ..storage.http_client import StorageClient, ClientException +from ..storage.http_client import ( + StorageClient, + ClientException, + StorageClientImmutables, + ImmutableCreateResult, +) def _post_process(params): @@ -270,12 +276,73 @@ class ImmutableHTTPAPITests(AsyncTestCase): Tests for immutable upload/download APIs. """ + def setUp(self): + if PY2: + self.skipTest("Not going to bother supporting Python 2") + super(ImmutableHTTPAPITests, self).setUp() + self.http = self.useFixture(HttpTestFixture()) + + @inlineCallbacks def test_upload_can_be_downloaded(self): """ A single share can be uploaded in (possibly overlapping) chunks, and then a random chunk can be downloaded, and it will match the original file. + + We don't exercise the full variation of overlapping chunks because + that's already done in test_storage.py. """ + length = 100 + expected_data = b"".join(bytes([i]) for i in range(100)) + + im_client = StorageClientImmutables(self.http.client) + + # Create a upload: + upload_secret = urandom(32) + lease_secret = urandom(32) + storage_index = b"".join(bytes([i]) for i in range(16)) + created = yield im_client.create( + storage_index, [1], 100, upload_secret, lease_secret, lease_secret + ) + self.assertEqual( + created, ImmutableCreateResult(already_have=set(), allocated={1}) + ) + + # Three writes: 10-19, 30-39, 50-59. This allows for a bunch of holes. + def write(offset, length): + return im_client.write_share_chunk( + storage_index, + 1, + upload_secret, + offset, + expected_data[offset : offset + length], + ) + + finished = yield write(10, 10) + self.assertFalse(finished) + finished = yield write(30, 10) + self.assertFalse(finished) + finished = yield write(50, 10) + self.assertFalse(finished) + + # Then, an overlapping write with matching data (15-35): + finished = yield write(15, 20) + self.assertFalse(finished) + + # Now fill in the holes: + finished = yield write(0, 10) + self.assertFalse(finished) + finished = yield write(40, 10) + self.assertFalse(finished) + finished = yield write(60, 40) + self.assertTrue(finished) + + # We can now read: + for offset, length in [(0, 100), (10, 19), (99, 0), (49, 200)]: + downloaded = yield im_client.read_share_chunk( + storage_index, 1, upload_secret, offset, length + ) + self.assertEqual(downloaded, expected_data[offset : offset + length]) def test_multiple_shares_uploaded_to_different_place(self): """ From 004e5fbc9d2266585508dc376900e33ba1557a49 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 11 Jan 2022 15:47:32 -0500 Subject: [PATCH 0567/2309] Get to point where we get failing HTTP response. --- src/allmydata/storage/http_client.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 3accb3c62..002ffc928 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -85,7 +85,7 @@ class StorageClient(object): self, method, url, - lease_renewal_secret=None, + lease_renew_secret=None, lease_cancel_secret=None, upload_secret=None, headers=None, @@ -97,7 +97,7 @@ class StorageClient(object): """ headers = self._get_headers(headers) for secret, value in [ - (Secrets.LEASE_RENEW, lease_renewal_secret), + (Secrets.LEASE_RENEW, lease_renew_secret), (Secrets.LEASE_CANCEL, lease_cancel_secret), (Secrets.UPLOAD, upload_secret), ]: @@ -162,7 +162,7 @@ class StorageClientImmutables(object): lease_cancel_secret=lease_cancel_secret, upload_secret=upload_secret, data=message, - headers=Headers({"content-type": "application/cbor"}), + headers=Headers({"content-type": ["application/cbor"]}), ) decoded_response = yield _decode_cbor(response) returnValue( @@ -201,9 +201,9 @@ class StorageClientImmutables(object): # The range is inclusive, thus the '- 1'. '*' means "length # unknown", which isn't technically true but adding it just # makes things slightly harder for calling API. - "content-range": "bytes {}-{}/*".format( - offset, offset + len(data) - 1 - ) + "content-range": [ + "bytes {}-{}/*".format(offset, offset + len(data) - 1) + ] } ), ) @@ -245,8 +245,8 @@ class StorageClientImmutables(object): url, headers=Headers( { - # The range is inclusive, thus the -1. - "range": "bytes={}-{}".format(offset, offset + length - 1) + # The range is inclusive. + "range": ["bytes={}-{}".format(offset, offset + length)] } ), ) From 6e2aaa8391e46bf02ee37a794dcc51c8ced84a25 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 12 Jan 2022 09:14:58 -0500 Subject: [PATCH 0568/2309] Refactor more integration-y tests out. --- integration/test_storage_http.py | 283 ++++++++++++++++++++++++ src/allmydata/test/test_storage_http.py | 274 +---------------------- 2 files changed, 284 insertions(+), 273 deletions(-) create mode 100644 integration/test_storage_http.py diff --git a/integration/test_storage_http.py b/integration/test_storage_http.py new file mode 100644 index 000000000..714562cc4 --- /dev/null +++ b/integration/test_storage_http.py @@ -0,0 +1,283 @@ +""" +Connect the HTTP storage client to the HTTP storage server and make sure they +can talk to each other. +""" + +from future.utils import PY2 + +from os import urandom + +from twisted.internet.defer import inlineCallbacks +from fixtures import Fixture, TempDir +from treq.testing import StubTreq +from hyperlink import DecodedURL +from klein import Klein + +from allmydata.storage.server import StorageServer +from allmydata.storage.http_server import ( + HTTPServer, + _authorized_route, +) +from allmydata.storage.http_client import ( + StorageClient, + ClientException, + StorageClientImmutables, + ImmutableCreateResult, +) +from allmydata.storage.http_common import Secrets +from allmydata.test.common import AsyncTestCase + + +# TODO should be actual swissnum +SWISSNUM_FOR_TEST = b"abcd" + + +class TestApp(object): + """HTTP API for testing purposes.""" + + _app = Klein() + _swissnum = SWISSNUM_FOR_TEST # Match what the test client is using + + @_authorized_route(_app, {Secrets.UPLOAD}, "/upload_secret", methods=["GET"]) + def validate_upload_secret(self, request, authorization): + if authorization == {Secrets.UPLOAD: b"MAGIC"}: + return "GOOD SECRET" + else: + return "BAD: {}".format(authorization) + + +class RoutingTests(AsyncTestCase): + """ + Tests for the HTTP routing infrastructure. + """ + + def setUp(self): + if PY2: + self.skipTest("Not going to bother supporting Python 2") + super(RoutingTests, self).setUp() + # Could be a fixture, but will only be used in this test class so not + # going to bother: + self._http_server = TestApp() + self.client = StorageClient( + DecodedURL.from_text("http://127.0.0.1"), + SWISSNUM_FOR_TEST, + treq=StubTreq(self._http_server._app.resource()), + ) + + @inlineCallbacks + def test_authorization_enforcement(self): + """ + The requirement for secrets is enforced; if they are not given, a 400 + response code is returned. + """ + # Without secret, get a 400 error. + response = yield self.client._request( + "GET", "http://127.0.0.1/upload_secret", {} + ) + self.assertEqual(response.code, 400) + + # With secret, we're good. + response = yield self.client._request( + "GET", "http://127.0.0.1/upload_secret", {Secrets.UPLOAD: b"MAGIC"} + ) + self.assertEqual(response.code, 200) + self.assertEqual((yield response.content()), b"GOOD SECRET") + + +class HttpTestFixture(Fixture): + """ + Setup HTTP tests' infrastructure, the storage server and corresponding + client. + """ + + def _setUp(self): + self.tempdir = self.useFixture(TempDir()) + self.storage_server = StorageServer(self.tempdir.path, b"\x00" * 20) + self.http_server = HTTPServer(self.storage_server, SWISSNUM_FOR_TEST) + self.client = StorageClient( + DecodedURL.from_text("http://127.0.0.1"), + SWISSNUM_FOR_TEST, + treq=StubTreq(self.http_server.get_resource()), + ) + + +class GenericHTTPAPITests(AsyncTestCase): + """ + Tests of HTTP client talking to the HTTP server, for generic HTTP API + endpoints and concerns. + """ + + def setUp(self): + if PY2: + self.skipTest("Not going to bother supporting Python 2") + super(GenericHTTPAPITests, self).setUp() + self.http = self.useFixture(HttpTestFixture()) + + @inlineCallbacks + def test_bad_authentication(self): + """ + If the wrong swissnum is used, an ``Unauthorized`` response code is + returned. + """ + client = StorageClient( + DecodedURL.from_text("http://127.0.0.1"), + b"something wrong", + treq=StubTreq(self.http.http_server.get_resource()), + ) + with self.assertRaises(ClientException) as e: + yield client.get_version() + self.assertEqual(e.exception.args[0], 401) + + @inlineCallbacks + def test_version(self): + """ + The client can return the version. + + We ignore available disk space and max immutable share size, since that + might change across calls. + """ + version = yield self.http.client.get_version() + version[b"http://allmydata.org/tahoe/protocols/storage/v1"].pop( + b"available-space" + ) + version[b"http://allmydata.org/tahoe/protocols/storage/v1"].pop( + b"maximum-immutable-share-size" + ) + expected_version = self.http.storage_server.get_version() + expected_version[b"http://allmydata.org/tahoe/protocols/storage/v1"].pop( + b"available-space" + ) + expected_version[b"http://allmydata.org/tahoe/protocols/storage/v1"].pop( + b"maximum-immutable-share-size" + ) + self.assertEqual(version, expected_version) + + +class ImmutableHTTPAPITests(AsyncTestCase): + """ + Tests for immutable upload/download APIs. + """ + + def setUp(self): + if PY2: + self.skipTest("Not going to bother supporting Python 2") + super(ImmutableHTTPAPITests, self).setUp() + self.http = self.useFixture(HttpTestFixture()) + + @inlineCallbacks + def test_upload_can_be_downloaded(self): + """ + A single share can be uploaded in (possibly overlapping) chunks, and + then a random chunk can be downloaded, and it will match the original + file. + + We don't exercise the full variation of overlapping chunks because + that's already done in test_storage.py. + """ + length = 100 + expected_data = b"".join(bytes([i]) for i in range(100)) + + im_client = StorageClientImmutables(self.http.client) + + # Create a upload: + upload_secret = urandom(32) + lease_secret = urandom(32) + storage_index = b"".join(bytes([i]) for i in range(16)) + created = yield im_client.create( + storage_index, [1], 100, upload_secret, lease_secret, lease_secret + ) + self.assertEqual( + created, ImmutableCreateResult(already_have=set(), allocated={1}) + ) + + # Three writes: 10-19, 30-39, 50-59. This allows for a bunch of holes. + def write(offset, length): + return im_client.write_share_chunk( + storage_index, + 1, + upload_secret, + offset, + expected_data[offset : offset + length], + ) + + finished = yield write(10, 10) + self.assertFalse(finished) + finished = yield write(30, 10) + self.assertFalse(finished) + finished = yield write(50, 10) + self.assertFalse(finished) + + # Then, an overlapping write with matching data (15-35): + finished = yield write(15, 20) + self.assertFalse(finished) + + # Now fill in the holes: + finished = yield write(0, 10) + self.assertFalse(finished) + finished = yield write(40, 10) + self.assertFalse(finished) + finished = yield write(60, 40) + self.assertTrue(finished) + + # We can now read: + for offset, length in [(0, 100), (10, 19), (99, 0), (49, 200)]: + downloaded = yield im_client.read_share_chunk( + storage_index, 1, upload_secret, offset, length + ) + self.assertEqual(downloaded, expected_data[offset : offset + length]) + + def test_multiple_shares_uploaded_to_different_place(self): + """ + If a storage index has multiple shares, uploads to different shares are + stored separately and can be downloaded separately. + """ + + def test_bucket_allocated_with_new_shares(self): + """ + If some shares already exist, allocating shares indicates only the new + ones were created. + """ + + def test_bucket_allocation_new_upload_key(self): + """ + If a bucket was allocated with one upload key, and a different upload + key is used to allocate the bucket again, the previous download is + cancelled. + """ + + def test_upload_with_wrong_upload_key_fails(self): + """ + Uploading with a key that doesn't match the one used to allocate the + bucket will fail. + """ + + def test_upload_offset_cannot_be_negative(self): + """ + A negative upload offset will be rejected. + """ + + def test_mismatching_upload_fails(self): + """ + If an uploaded chunk conflicts with an already uploaded chunk, a + CONFLICT error is returned. + """ + + def test_read_of_wrong_storage_index_fails(self): + """ + Reading from unknown storage index results in 404. + """ + + def test_read_of_wrong_share_number_fails(self): + """ + Reading from unknown storage index results in 404. + """ + + def test_read_with_negative_offset_fails(self): + """ + The offset for reads cannot be negative. + """ + + def test_read_with_negative_length_fails(self): + """ + The length for reads cannot be negative. + """ diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 95708d211..aaa455a03 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -15,30 +15,13 @@ if PY2: # fmt: on from base64 import b64encode -from os import urandom - -from twisted.internet.defer import inlineCallbacks from hypothesis import assume, given, strategies as st -from fixtures import Fixture, TempDir -from treq.testing import StubTreq -from klein import Klein -from hyperlink import DecodedURL - -from .common import AsyncTestCase, SyncTestCase -from ..storage.server import StorageServer +from .common import SyncTestCase from ..storage.http_server import ( - HTTPServer, _extract_secrets, Secrets, ClientSecretsException, - _authorized_route, -) -from ..storage.http_client import ( - StorageClient, - ClientException, - StorageClientImmutables, - ImmutableCreateResult, ) @@ -144,258 +127,3 @@ class ExtractSecretsTests(SyncTestCase): """ with self.assertRaises(ClientSecretsException): _extract_secrets(["lease-cancel-secret eA=="], {Secrets.LEASE_RENEW}) - - -SWISSNUM_FOR_TEST = b"abcd" - - -class TestApp(object): - """HTTP API for testing purposes.""" - - _app = Klein() - _swissnum = SWISSNUM_FOR_TEST # Match what the test client is using - - @_authorized_route(_app, {Secrets.UPLOAD}, "/upload_secret", methods=["GET"]) - def validate_upload_secret(self, request, authorization): - if authorization == {Secrets.UPLOAD: b"MAGIC"}: - return "GOOD SECRET" - else: - return "BAD: {}".format(authorization) - - -class RoutingTests(AsyncTestCase): - """ - Tests for the HTTP routing infrastructure. - """ - - def setUp(self): - if PY2: - self.skipTest("Not going to bother supporting Python 2") - super(RoutingTests, self).setUp() - # Could be a fixture, but will only be used in this test class so not - # going to bother: - self._http_server = TestApp() - self.client = StorageClient( - DecodedURL.from_text("http://127.0.0.1"), - SWISSNUM_FOR_TEST, - treq=StubTreq(self._http_server._app.resource()), - ) - - @inlineCallbacks - def test_authorization_enforcement(self): - """ - The requirement for secrets is enforced; if they are not given, a 400 - response code is returned. - """ - # Without secret, get a 400 error. - response = yield self.client._request( - "GET", "http://127.0.0.1/upload_secret", {} - ) - self.assertEqual(response.code, 400) - - # With secret, we're good. - response = yield self.client._request( - "GET", "http://127.0.0.1/upload_secret", {Secrets.UPLOAD: b"MAGIC"} - ) - self.assertEqual(response.code, 200) - self.assertEqual((yield response.content()), b"GOOD SECRET") - - -class HttpTestFixture(Fixture): - """ - Setup HTTP tests' infrastructure, the storage server and corresponding - client. - """ - - def _setUp(self): - self.tempdir = self.useFixture(TempDir()) - self.storage_server = StorageServer(self.tempdir.path, b"\x00" * 20) - # TODO what should the swissnum _actually_ be? - self.http_server = HTTPServer(self.storage_server, SWISSNUM_FOR_TEST) - self.client = StorageClient( - DecodedURL.from_text("http://127.0.0.1"), - SWISSNUM_FOR_TEST, - treq=StubTreq(self.http_server.get_resource()), - ) - - -class GenericHTTPAPITests(AsyncTestCase): - """ - Tests of HTTP client talking to the HTTP server, for generic HTTP API - endpoints and concerns. - """ - - def setUp(self): - if PY2: - self.skipTest("Not going to bother supporting Python 2") - super(GenericHTTPAPITests, self).setUp() - self.http = self.useFixture(HttpTestFixture()) - - @inlineCallbacks - def test_bad_authentication(self): - """ - If the wrong swissnum is used, an ``Unauthorized`` response code is - returned. - """ - client = StorageClient( - DecodedURL.from_text("http://127.0.0.1"), - b"something wrong", - treq=StubTreq(self.http.http_server.get_resource()), - ) - with self.assertRaises(ClientException) as e: - yield client.get_version() - self.assertEqual(e.exception.args[0], 401) - - @inlineCallbacks - def test_version(self): - """ - The client can return the version. - - We ignore available disk space and max immutable share size, since that - might change across calls. - """ - version = yield self.http.client.get_version() - version[b"http://allmydata.org/tahoe/protocols/storage/v1"].pop( - b"available-space" - ) - version[b"http://allmydata.org/tahoe/protocols/storage/v1"].pop( - b"maximum-immutable-share-size" - ) - expected_version = self.http.storage_server.get_version() - expected_version[b"http://allmydata.org/tahoe/protocols/storage/v1"].pop( - b"available-space" - ) - expected_version[b"http://allmydata.org/tahoe/protocols/storage/v1"].pop( - b"maximum-immutable-share-size" - ) - self.assertEqual(version, expected_version) - - -class ImmutableHTTPAPITests(AsyncTestCase): - """ - Tests for immutable upload/download APIs. - """ - - def setUp(self): - if PY2: - self.skipTest("Not going to bother supporting Python 2") - super(ImmutableHTTPAPITests, self).setUp() - self.http = self.useFixture(HttpTestFixture()) - - @inlineCallbacks - def test_upload_can_be_downloaded(self): - """ - A single share can be uploaded in (possibly overlapping) chunks, and - then a random chunk can be downloaded, and it will match the original - file. - - We don't exercise the full variation of overlapping chunks because - that's already done in test_storage.py. - """ - length = 100 - expected_data = b"".join(bytes([i]) for i in range(100)) - - im_client = StorageClientImmutables(self.http.client) - - # Create a upload: - upload_secret = urandom(32) - lease_secret = urandom(32) - storage_index = b"".join(bytes([i]) for i in range(16)) - created = yield im_client.create( - storage_index, [1], 100, upload_secret, lease_secret, lease_secret - ) - self.assertEqual( - created, ImmutableCreateResult(already_have=set(), allocated={1}) - ) - - # Three writes: 10-19, 30-39, 50-59. This allows for a bunch of holes. - def write(offset, length): - return im_client.write_share_chunk( - storage_index, - 1, - upload_secret, - offset, - expected_data[offset : offset + length], - ) - - finished = yield write(10, 10) - self.assertFalse(finished) - finished = yield write(30, 10) - self.assertFalse(finished) - finished = yield write(50, 10) - self.assertFalse(finished) - - # Then, an overlapping write with matching data (15-35): - finished = yield write(15, 20) - self.assertFalse(finished) - - # Now fill in the holes: - finished = yield write(0, 10) - self.assertFalse(finished) - finished = yield write(40, 10) - self.assertFalse(finished) - finished = yield write(60, 40) - self.assertTrue(finished) - - # We can now read: - for offset, length in [(0, 100), (10, 19), (99, 0), (49, 200)]: - downloaded = yield im_client.read_share_chunk( - storage_index, 1, upload_secret, offset, length - ) - self.assertEqual(downloaded, expected_data[offset : offset + length]) - - def test_multiple_shares_uploaded_to_different_place(self): - """ - If a storage index has multiple shares, uploads to different shares are - stored separately and can be downloaded separately. - """ - - def test_bucket_allocated_with_new_shares(self): - """ - If some shares already exist, allocating shares indicates only the new - ones were created. - """ - - def test_bucket_allocation_new_upload_key(self): - """ - If a bucket was allocated with one upload key, and a different upload - key is used to allocate the bucket again, the previous download is - cancelled. - """ - - def test_upload_with_wrong_upload_key_fails(self): - """ - Uploading with a key that doesn't match the one used to allocate the - bucket will fail. - """ - - def test_upload_offset_cannot_be_negative(self): - """ - A negative upload offset will be rejected. - """ - - def test_mismatching_upload_fails(self): - """ - If an uploaded chunk conflicts with an already uploaded chunk, a - CONFLICT error is returned. - """ - - def test_read_of_wrong_storage_index_fails(self): - """ - Reading from unknown storage index results in 404. - """ - - def test_read_of_wrong_share_number_fails(self): - """ - Reading from unknown storage index results in 404. - """ - - def test_read_with_negative_offset_fails(self): - """ - The offset for reads cannot be negative. - """ - - def test_read_with_negative_length_fails(self): - """ - The length for reads cannot be negative. - """ From 2bccb01be4bbd33b0b25642049f6ab2ae2697e17 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 12 Jan 2022 11:16:21 -0500 Subject: [PATCH 0569/2309] Fix bug wrapping endpoints. --- src/allmydata/storage/http_server.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index d2c9f6b7a..b371fc395 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -110,6 +110,7 @@ def _authorized_route(app, required_secrets, *route_args, **route_kwargs): def decorator(f): @app.route(*route_args, **route_kwargs) @_authorization_decorator(required_secrets) + @wraps(f) def handle_route(*args, **kwargs): return f(*args, **kwargs) From 018f53105e9ad6c29dd82e324f2405ef9c75eb54 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 12 Jan 2022 11:16:39 -0500 Subject: [PATCH 0570/2309] Pass correct arguments. --- src/allmydata/storage/http_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 002ffc928..e38525583 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -115,7 +115,7 @@ class StorageClient(object): Return the version metadata for the server. """ url = self._url("/v1/version") - response = yield self._request("GET", url, {}) + response = yield self._request("GET", url) decoded_response = yield _decode_cbor(response) returnValue(decoded_response) From c4bb3c21d13a757007c058c467ee1dbc602be521 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 12 Jan 2022 11:18:34 -0500 Subject: [PATCH 0571/2309] Update test to match current API. --- integration/test_storage_http.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/integration/test_storage_http.py b/integration/test_storage_http.py index 714562cc4..66a8b2af1 100644 --- a/integration/test_storage_http.py +++ b/integration/test_storage_http.py @@ -72,13 +72,14 @@ class RoutingTests(AsyncTestCase): """ # Without secret, get a 400 error. response = yield self.client._request( - "GET", "http://127.0.0.1/upload_secret", {} + "GET", + "http://127.0.0.1/upload_secret", ) self.assertEqual(response.code, 400) # With secret, we're good. response = yield self.client._request( - "GET", "http://127.0.0.1/upload_secret", {Secrets.UPLOAD: b"MAGIC"} + "GET", "http://127.0.0.1/upload_secret", upload_secret=b"MAGIC" ) self.assertEqual(response.code, 200) self.assertEqual((yield response.content()), b"GOOD SECRET") From f5437d9be73b42b891f892336551c95074d84b4d Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 12 Jan 2022 11:51:56 -0500 Subject: [PATCH 0572/2309] Some progress towards bucket allocation endpoint, and defining the protocol better. --- docs/proposed/http-storage-node-protocol.rst | 5 +++ src/allmydata/storage/http_client.py | 12 ++++++-- src/allmydata/storage/http_server.py | 32 ++++++++++++++++---- 3 files changed, 40 insertions(+), 9 deletions(-) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index a8555cd26..26f1a2bb7 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -382,6 +382,11 @@ the server will respond with ``400 BAD REQUEST``. If authorization using the secret fails, then a ``401 UNAUTHORIZED`` response should be sent. +Encoding +~~~~~~~~ + +* ``storage_index`` should be base32 encoded (RFC3548) in URLs. + General ~~~~~~~ diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index e38525583..5e964bfbe 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -34,6 +34,12 @@ from hyperlink import DecodedURL import treq from .http_common import swissnum_auth_header, Secrets +from .common import si_b2a + + +def _encode_si(si): # type: (bytes) -> str + """Encode the storage index into Unicode string.""" + return str(si_b2a(si), "ascii") class ClientException(Exception): @@ -151,7 +157,7 @@ class StorageClientImmutables(object): Result fires when creating the storage index succeeded, if creating the storage index failed the result will fire with an exception. """ - url = self._client._url("/v1/immutable/" + str(storage_index, "ascii")) + url = self._client._url("/v1/immutable/" + _encode_si(storage_index)) message = dumps( {"share-numbers": share_numbers, "allocated-size": allocated_size} ) @@ -189,7 +195,7 @@ class StorageClientImmutables(object): been uploaded. """ url = self._client._url( - "/v1/immutable/{}/{}".format(str(storage_index, "ascii"), share_number) + "/v1/immutable/{}/{}".format(_encode_si(storage_index), share_number) ) response = yield self._client._request( "POST", @@ -238,7 +244,7 @@ class StorageClientImmutables(object): https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3777 """ url = self._client._url( - "/v1/immutable/{}/{}".format(str(storage_index, "ascii"), share_number) + "/v1/immutable/{}/{}".format(_encode_si(storage_index), share_number) ) response = yield self._client._request( "GET", diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index b371fc395..23f0d2f1c 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -28,6 +28,7 @@ from cbor2 import dumps, loads from .server import StorageServer from .http_common import swissnum_auth_header, Secrets +from .common import si_a2b from .immutable import BucketWriter from ..util.hashutil import timing_safe_compare @@ -174,6 +175,7 @@ class HTTPServer(object): ) def allocate_buckets(self, request, authorization, storage_index): """Allocate buckets.""" + storage_index = si_a2b(storage_index.encode("ascii")) info = loads(request.content.read()) upload_key = authorization[Secrets.UPLOAD] @@ -191,13 +193,29 @@ class HTTPServer(object): pass else: # New upload. - # TODO self._storage_server.allocate_buckets() with given inputs. - # TODO add results to self._uploads. - pass + already_got, sharenum_to_bucket = self._storage_server.allocate_buckets( + storage_index, + renew_secret=authorization[Secrets.LEASE_RENEW], + cancel_secret=authorization[Secrets.LEASE_CANCEL], + sharenums=info["share-numbers"], + allocated_size=info["allocated-size"], + ) + self._uploads[storage_index] = StorageIndexUploads( + shares=sharenum_to_bucket, upload_key=authorization[Secrets.UPLOAD] + ) + return self._cbor( + request, + { + "already-have": set(already_got), + "allocated": set(sharenum_to_bucket), + }, + ) @_authorized_route( _app, - {Secrets.UPLOAD}, "/v1/immutable//", methods=["PATCH"] + {Secrets.UPLOAD}, + "/v1/immutable//", + methods=["PATCH"], ) def write_share_data(self, request, authorization, storage_index, share_number): """Write data to an in-progress immutable upload.""" @@ -212,7 +230,10 @@ class HTTPServer(object): # TODO if it finished writing altogether, 201 CREATED. Otherwise 200 OK. @_authorized_route( - _app, set(), "/v1/immutable//", methods=["GET"] + _app, + set(), + "/v1/immutable//", + methods=["GET"], ) def read_share_chunk(self, request, authorization, storage_index, share_number): """Read a chunk for an already uploaded immutable.""" @@ -221,4 +242,3 @@ class HTTPServer(object): # TODO lookup the share # TODO if not found, 404 # TODO otherwise, return data from that offset - From 3bed0678285deb5ac2064bbc4c045c0b75b4df2b Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 14 Jan 2022 08:34:17 -0500 Subject: [PATCH 0573/2309] Implement more of the writing logic. --- src/allmydata/storage/http_client.py | 2 +- src/allmydata/storage/http_server.py | 32 ++++++++++++++++++++++------ src/allmydata/storage/immutable.py | 11 ++++++++-- 3 files changed, 36 insertions(+), 9 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 5e964bfbe..697af91b5 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -198,7 +198,7 @@ class StorageClientImmutables(object): "/v1/immutable/{}/{}".format(_encode_si(storage_index), share_number) ) response = yield self._client._request( - "POST", + "PATCH", url, upload_secret=upload_secret, data=data, diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 23f0d2f1c..28367752d 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -219,15 +219,35 @@ class HTTPServer(object): ) def write_share_data(self, request, authorization, storage_index, share_number): """Write data to an in-progress immutable upload.""" - # TODO parse the content-range header to get offset for writing - # TODO basic checks on validity of offset - # TODO basic check that body isn't infinite. require content-length? if so, needs t be in protocol spec. + storage_index = si_a2b(storage_index.encode("ascii")) + content_range = request.getHeader("content-range") + if content_range is None: + offset = 0 + else: + offset = int(content_range.split()[1].split("-")[0]) + + # TODO basic checks on validity of start, offset, and content-range in general. also of share_number. + # TODO basic check that body isn't infinite. require content-length? or maybe we should require content-range (it's optional now)? if so, needs to be rflected in protocol spec. + data = request.content.read() - # TODO write to bucket at that offset. + try: + bucket = self._uploads[storage_index].shares[share_number] + except (KeyError, IndexError): + # TODO return 404 + raise - # TODO check if it conflicts with existing data (probably underlying code already handles that) if so, CONFLICT. + finished = bucket.write(offset, data) - # TODO if it finished writing altogether, 201 CREATED. Otherwise 200 OK. + # TODO if raises ConflictingWriteError, return HTTP CONFLICT code. + + if finished: + request.setResponseCode(http.CREATED) + else: + request.setResponseCode(http.OK) + + # TODO spec says we should return missing ranges. but client doesn't + # actually use them? So is it actually useful? + return b"" @_authorized_route( _app, diff --git a/src/allmydata/storage/immutable.py b/src/allmydata/storage/immutable.py index da9aa473f..5878e254a 100644 --- a/src/allmydata/storage/immutable.py +++ b/src/allmydata/storage/immutable.py @@ -13,7 +13,7 @@ if PY2: import os, stat, struct, time -from collections_extended import RangeMap +from collections_extended import RangeMap, MappedRange from foolscap.api import Referenceable @@ -375,7 +375,10 @@ class BucketWriter(object): def allocated_size(self): return self._max_size - def write(self, offset, data): + def write(self, offset, data): # type: (int, bytes) -> bool + """ + Write data at given offset, return whether the upload is complete. + """ # Delay the timeout, since we received data: self._timeout.reset(30 * 60) start = self._clock.seconds() @@ -399,6 +402,10 @@ class BucketWriter(object): self.ss.add_latency("write", self._clock.seconds() - start) self.ss.count("write") + # Return whether the whole thing has been written. + # TODO needs property test + return self._already_written.ranges() == [MappedRange(0, self._max_size, True)] + def close(self): precondition(not self.closed) self._timeout.cancel() From 4ea6bf2381f5a43f1265be988b2b9c50ad017384 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Sat, 15 Jan 2022 12:59:23 -0500 Subject: [PATCH 0574/2309] A test and some progress to making it pass. --- src/allmydata/storage/immutable.py | 12 +++++++---- src/allmydata/test/test_storage.py | 33 ++++++++++++++++++++++++++++-- 2 files changed, 39 insertions(+), 6 deletions(-) diff --git a/src/allmydata/storage/immutable.py b/src/allmydata/storage/immutable.py index 5878e254a..0bcef8246 100644 --- a/src/allmydata/storage/immutable.py +++ b/src/allmydata/storage/immutable.py @@ -13,7 +13,7 @@ if PY2: import os, stat, struct, time -from collections_extended import RangeMap, MappedRange +from collections_extended import RangeMap from foolscap.api import Referenceable @@ -402,9 +402,13 @@ class BucketWriter(object): self.ss.add_latency("write", self._clock.seconds() - start) self.ss.count("write") - # Return whether the whole thing has been written. - # TODO needs property test - return self._already_written.ranges() == [MappedRange(0, self._max_size, True)] + # Return whether the whole thing has been written. See + # https://github.com/mlenzen/collections-extended/issues/169 for why + # it's done this way. + print([tuple(mr) for mr in self._already_written.ranges()]) + return [tuple(mr) for mr in self._already_written.ranges()] == [ + (0, self._max_size, True) + ] def close(self): precondition(not self.closed) diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index bd74a1052..e4f01f6f1 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -13,6 +13,7 @@ if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from six import ensure_str +from array import array from io import ( BytesIO, ) @@ -34,7 +35,7 @@ from twisted.trial import unittest from twisted.internet import defer from twisted.internet.task import Clock -from hypothesis import given, strategies +from hypothesis import given, strategies, example import itertools from allmydata import interfaces @@ -230,7 +231,6 @@ class Bucket(unittest.TestCase): br = BucketReader(self, bw.finalhome) self.assertEqual(br.read(0, length), expected_data) - @given( maybe_overlapping_offset=strategies.integers(min_value=0, max_value=98), maybe_overlapping_length=strategies.integers(min_value=1, max_value=100), @@ -264,6 +264,35 @@ class Bucket(unittest.TestCase): bw.write(40, b"1" * 10) bw.write(60, b"1" * 40) + @given( + offsets=strategies.lists( + strategies.integers(min_value=0, max_value=99), + min_size=20, + max_size=20 + ), + ) + @example(offsets=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 40, 70]) + def test_writes_return_when_finished( + self, offsets + ): + """ + The ``BucketWriter.write()`` return true if and only if the maximum + size has been reached via potentially overlapping writes. + """ + length = 100 + incoming, final = self.make_workdir("overlapping_writes_{}".format(uuid4())) + bw = BucketWriter( + self, incoming, final, length, self.make_lease(), Clock() + ) + local_written = [0] * 100 + for offset in offsets: + length = min(30, 100 - offset) + data = b"1" * length + for i in range(offset, offset+length): + local_written[i] = 1 + finished = bw.write(offset, data) + self.assertEqual(finished, sum(local_written) == 100) + def test_read_past_end_of_share_data(self): # test vector for immutable files (hard-coded contents of an immutable share # file): From 25e2100219ddc067f5090072fb318930a1eb0660 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 17 Jan 2022 14:06:21 -0500 Subject: [PATCH 0575/2309] Immutable writing now knows when it's finished. --- src/allmydata/storage/immutable.py | 8 +++----- src/allmydata/test/test_storage.py | 3 +-- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/src/allmydata/storage/immutable.py b/src/allmydata/storage/immutable.py index 0bcef8246..d17f69c07 100644 --- a/src/allmydata/storage/immutable.py +++ b/src/allmydata/storage/immutable.py @@ -403,12 +403,10 @@ class BucketWriter(object): self.ss.count("write") # Return whether the whole thing has been written. See - # https://github.com/mlenzen/collections-extended/issues/169 for why + # https://github.com/mlenzen/collections-extended/issues/169 and + # https://github.com/mlenzen/collections-extended/issues/172 for why # it's done this way. - print([tuple(mr) for mr in self._already_written.ranges()]) - return [tuple(mr) for mr in self._already_written.ranges()] == [ - (0, self._max_size, True) - ] + return sum([mr.stop - mr.start for mr in self._already_written.ranges()]) == self._max_size def close(self): precondition(not self.closed) diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index e4f01f6f1..881bfb6fd 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -279,10 +279,9 @@ class Bucket(unittest.TestCase): The ``BucketWriter.write()`` return true if and only if the maximum size has been reached via potentially overlapping writes. """ - length = 100 incoming, final = self.make_workdir("overlapping_writes_{}".format(uuid4())) bw = BucketWriter( - self, incoming, final, length, self.make_lease(), Clock() + self, incoming, final, 100, self.make_lease(), Clock() ) local_written = [0] * 100 for offset in offsets: From d4ae7c89aa0ba41b45720bebfe90054bc9e53df7 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 17 Jan 2022 14:20:40 -0500 Subject: [PATCH 0576/2309] First end-to-end immutable upload then download test passes. --- integration/test_storage_http.py | 2 +- src/allmydata/storage/http_client.py | 5 +++-- src/allmydata/storage/http_server.py | 21 ++++++++++++++++++--- 3 files changed, 22 insertions(+), 6 deletions(-) diff --git a/integration/test_storage_http.py b/integration/test_storage_http.py index 66a8b2af1..0e2cf89a6 100644 --- a/integration/test_storage_http.py +++ b/integration/test_storage_http.py @@ -223,7 +223,7 @@ class ImmutableHTTPAPITests(AsyncTestCase): # We can now read: for offset, length in [(0, 100), (10, 19), (99, 0), (49, 200)]: downloaded = yield im_client.read_share_chunk( - storage_index, 1, upload_secret, offset, length + storage_index, 1, offset, length ) self.assertEqual(downloaded, expected_data[offset : offset + length]) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 697af91b5..b091b3ca7 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -252,12 +252,13 @@ class StorageClientImmutables(object): headers=Headers( { # The range is inclusive. - "range": ["bytes={}-{}".format(offset, offset + length)] + "range": ["bytes={}-{}".format(offset, offset + length - 1)] } ), ) if response.code == 200: - returnValue(response.content.read()) + body = yield response.content() + returnValue(body) else: raise ClientException( response.code, diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 28367752d..50d955127 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -241,6 +241,7 @@ class HTTPServer(object): # TODO if raises ConflictingWriteError, return HTTP CONFLICT code. if finished: + bucket.close() request.setResponseCode(http.CREATED) else: request.setResponseCode(http.OK) @@ -257,8 +258,22 @@ class HTTPServer(object): ) def read_share_chunk(self, request, authorization, storage_index, share_number): """Read a chunk for an already uploaded immutable.""" - # TODO read offset and length from Range header # TODO basic checks on validity - # TODO lookup the share + storage_index = si_a2b(storage_index.encode("ascii")) + range_header = request.getHeader("range") + if range_header is None: + offset = 0 + inclusive_end = None + else: + parts = range_header.split("=")[1].split("-") + offset = int(parts[0]) # TODO make sure valid + if len(parts) > 0: + inclusive_end = int(parts[1]) # TODO make sure valid + else: + inclusive_end = None + + assert inclusive_end != None # TODO support this case + # TODO if not found, 404 - # TODO otherwise, return data from that offset + bucket = self._storage_server.get_buckets(storage_index)[share_number] + return bucket.read(offset, inclusive_end - offset + 1) From 79cd9a3d6d236749943228ed5fcc2287c6187e48 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 17 Jan 2022 14:22:15 -0500 Subject: [PATCH 0577/2309] Fix lint. --- src/allmydata/test/test_storage.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index 881bfb6fd..27309a82a 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -13,7 +13,6 @@ if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from six import ensure_str -from array import array from io import ( BytesIO, ) From 7aed7dbd8a7219e257dd0dc638eecd57e26fb66a Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 17 Jan 2022 14:24:28 -0500 Subject: [PATCH 0578/2309] Make module import on Python 2 (so tests can pass). --- src/allmydata/storage/http_client.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index b091b3ca7..8fea86396 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -13,6 +13,11 @@ if PY2: # fmt: off from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 # fmt: on + from collections import defaultdict + + Optional = Set = defaultdict( + lambda: None + ) # some garbage to just make this module import else: # typing module not available in Python 2, and we only do type checking in # Python 3 anyway. @@ -131,7 +136,7 @@ class StorageClientImmutables(object): APIs for interacting with immutables. """ - def __init__(self, client: StorageClient): # # type: (StorageClient) -> None + def __init__(self, client): # type: (StorageClient) -> None self._client = client @inlineCallbacks From 28dbdbe019f53dc16e0b6ea4af0822eba61b0842 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 17 Jan 2022 14:31:29 -0500 Subject: [PATCH 0579/2309] Make sure return type is consistent. --- src/allmydata/storage/immutable.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/storage/immutable.py b/src/allmydata/storage/immutable.py index d17f69c07..0949929a9 100644 --- a/src/allmydata/storage/immutable.py +++ b/src/allmydata/storage/immutable.py @@ -384,7 +384,7 @@ class BucketWriter(object): start = self._clock.seconds() precondition(not self.closed) if self.throw_out_all_data: - return + return False # Make sure we're not conflicting with existing data: end = offset + len(data) From 406a06a5080c691c9216849bba43100061b8ac3c Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 17 Jan 2022 14:38:06 -0500 Subject: [PATCH 0580/2309] Make sure we don't violate the Foolscap interface definition for this method. --- src/allmydata/storage/immutable.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/storage/immutable.py b/src/allmydata/storage/immutable.py index 0949929a9..e35ae9782 100644 --- a/src/allmydata/storage/immutable.py +++ b/src/allmydata/storage/immutable.py @@ -494,7 +494,7 @@ class FoolscapBucketWriter(Referenceable): # type: ignore # warner/foolscap#78 self._bucket_writer = bucket_writer def remote_write(self, offset, data): - return self._bucket_writer.write(offset, data) + self._bucket_writer.write(offset, data) def remote_close(self): return self._bucket_writer.close() From 23368fc9d95811c2088d430cc0dced995d212527 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 20 Jan 2022 10:34:09 -0500 Subject: [PATCH 0581/2309] Move tests back into unittest module. --- integration/test_storage_http.py | 284 ------------------------ src/allmydata/test/test_storage_http.py | 275 ++++++++++++++++++++++- 2 files changed, 274 insertions(+), 285 deletions(-) delete mode 100644 integration/test_storage_http.py diff --git a/integration/test_storage_http.py b/integration/test_storage_http.py deleted file mode 100644 index 0e2cf89a6..000000000 --- a/integration/test_storage_http.py +++ /dev/null @@ -1,284 +0,0 @@ -""" -Connect the HTTP storage client to the HTTP storage server and make sure they -can talk to each other. -""" - -from future.utils import PY2 - -from os import urandom - -from twisted.internet.defer import inlineCallbacks -from fixtures import Fixture, TempDir -from treq.testing import StubTreq -from hyperlink import DecodedURL -from klein import Klein - -from allmydata.storage.server import StorageServer -from allmydata.storage.http_server import ( - HTTPServer, - _authorized_route, -) -from allmydata.storage.http_client import ( - StorageClient, - ClientException, - StorageClientImmutables, - ImmutableCreateResult, -) -from allmydata.storage.http_common import Secrets -from allmydata.test.common import AsyncTestCase - - -# TODO should be actual swissnum -SWISSNUM_FOR_TEST = b"abcd" - - -class TestApp(object): - """HTTP API for testing purposes.""" - - _app = Klein() - _swissnum = SWISSNUM_FOR_TEST # Match what the test client is using - - @_authorized_route(_app, {Secrets.UPLOAD}, "/upload_secret", methods=["GET"]) - def validate_upload_secret(self, request, authorization): - if authorization == {Secrets.UPLOAD: b"MAGIC"}: - return "GOOD SECRET" - else: - return "BAD: {}".format(authorization) - - -class RoutingTests(AsyncTestCase): - """ - Tests for the HTTP routing infrastructure. - """ - - def setUp(self): - if PY2: - self.skipTest("Not going to bother supporting Python 2") - super(RoutingTests, self).setUp() - # Could be a fixture, but will only be used in this test class so not - # going to bother: - self._http_server = TestApp() - self.client = StorageClient( - DecodedURL.from_text("http://127.0.0.1"), - SWISSNUM_FOR_TEST, - treq=StubTreq(self._http_server._app.resource()), - ) - - @inlineCallbacks - def test_authorization_enforcement(self): - """ - The requirement for secrets is enforced; if they are not given, a 400 - response code is returned. - """ - # Without secret, get a 400 error. - response = yield self.client._request( - "GET", - "http://127.0.0.1/upload_secret", - ) - self.assertEqual(response.code, 400) - - # With secret, we're good. - response = yield self.client._request( - "GET", "http://127.0.0.1/upload_secret", upload_secret=b"MAGIC" - ) - self.assertEqual(response.code, 200) - self.assertEqual((yield response.content()), b"GOOD SECRET") - - -class HttpTestFixture(Fixture): - """ - Setup HTTP tests' infrastructure, the storage server and corresponding - client. - """ - - def _setUp(self): - self.tempdir = self.useFixture(TempDir()) - self.storage_server = StorageServer(self.tempdir.path, b"\x00" * 20) - self.http_server = HTTPServer(self.storage_server, SWISSNUM_FOR_TEST) - self.client = StorageClient( - DecodedURL.from_text("http://127.0.0.1"), - SWISSNUM_FOR_TEST, - treq=StubTreq(self.http_server.get_resource()), - ) - - -class GenericHTTPAPITests(AsyncTestCase): - """ - Tests of HTTP client talking to the HTTP server, for generic HTTP API - endpoints and concerns. - """ - - def setUp(self): - if PY2: - self.skipTest("Not going to bother supporting Python 2") - super(GenericHTTPAPITests, self).setUp() - self.http = self.useFixture(HttpTestFixture()) - - @inlineCallbacks - def test_bad_authentication(self): - """ - If the wrong swissnum is used, an ``Unauthorized`` response code is - returned. - """ - client = StorageClient( - DecodedURL.from_text("http://127.0.0.1"), - b"something wrong", - treq=StubTreq(self.http.http_server.get_resource()), - ) - with self.assertRaises(ClientException) as e: - yield client.get_version() - self.assertEqual(e.exception.args[0], 401) - - @inlineCallbacks - def test_version(self): - """ - The client can return the version. - - We ignore available disk space and max immutable share size, since that - might change across calls. - """ - version = yield self.http.client.get_version() - version[b"http://allmydata.org/tahoe/protocols/storage/v1"].pop( - b"available-space" - ) - version[b"http://allmydata.org/tahoe/protocols/storage/v1"].pop( - b"maximum-immutable-share-size" - ) - expected_version = self.http.storage_server.get_version() - expected_version[b"http://allmydata.org/tahoe/protocols/storage/v1"].pop( - b"available-space" - ) - expected_version[b"http://allmydata.org/tahoe/protocols/storage/v1"].pop( - b"maximum-immutable-share-size" - ) - self.assertEqual(version, expected_version) - - -class ImmutableHTTPAPITests(AsyncTestCase): - """ - Tests for immutable upload/download APIs. - """ - - def setUp(self): - if PY2: - self.skipTest("Not going to bother supporting Python 2") - super(ImmutableHTTPAPITests, self).setUp() - self.http = self.useFixture(HttpTestFixture()) - - @inlineCallbacks - def test_upload_can_be_downloaded(self): - """ - A single share can be uploaded in (possibly overlapping) chunks, and - then a random chunk can be downloaded, and it will match the original - file. - - We don't exercise the full variation of overlapping chunks because - that's already done in test_storage.py. - """ - length = 100 - expected_data = b"".join(bytes([i]) for i in range(100)) - - im_client = StorageClientImmutables(self.http.client) - - # Create a upload: - upload_secret = urandom(32) - lease_secret = urandom(32) - storage_index = b"".join(bytes([i]) for i in range(16)) - created = yield im_client.create( - storage_index, [1], 100, upload_secret, lease_secret, lease_secret - ) - self.assertEqual( - created, ImmutableCreateResult(already_have=set(), allocated={1}) - ) - - # Three writes: 10-19, 30-39, 50-59. This allows for a bunch of holes. - def write(offset, length): - return im_client.write_share_chunk( - storage_index, - 1, - upload_secret, - offset, - expected_data[offset : offset + length], - ) - - finished = yield write(10, 10) - self.assertFalse(finished) - finished = yield write(30, 10) - self.assertFalse(finished) - finished = yield write(50, 10) - self.assertFalse(finished) - - # Then, an overlapping write with matching data (15-35): - finished = yield write(15, 20) - self.assertFalse(finished) - - # Now fill in the holes: - finished = yield write(0, 10) - self.assertFalse(finished) - finished = yield write(40, 10) - self.assertFalse(finished) - finished = yield write(60, 40) - self.assertTrue(finished) - - # We can now read: - for offset, length in [(0, 100), (10, 19), (99, 0), (49, 200)]: - downloaded = yield im_client.read_share_chunk( - storage_index, 1, offset, length - ) - self.assertEqual(downloaded, expected_data[offset : offset + length]) - - def test_multiple_shares_uploaded_to_different_place(self): - """ - If a storage index has multiple shares, uploads to different shares are - stored separately and can be downloaded separately. - """ - - def test_bucket_allocated_with_new_shares(self): - """ - If some shares already exist, allocating shares indicates only the new - ones were created. - """ - - def test_bucket_allocation_new_upload_key(self): - """ - If a bucket was allocated with one upload key, and a different upload - key is used to allocate the bucket again, the previous download is - cancelled. - """ - - def test_upload_with_wrong_upload_key_fails(self): - """ - Uploading with a key that doesn't match the one used to allocate the - bucket will fail. - """ - - def test_upload_offset_cannot_be_negative(self): - """ - A negative upload offset will be rejected. - """ - - def test_mismatching_upload_fails(self): - """ - If an uploaded chunk conflicts with an already uploaded chunk, a - CONFLICT error is returned. - """ - - def test_read_of_wrong_storage_index_fails(self): - """ - Reading from unknown storage index results in 404. - """ - - def test_read_of_wrong_share_number_fails(self): - """ - Reading from unknown storage index results in 404. - """ - - def test_read_with_negative_offset_fails(self): - """ - The offset for reads cannot be negative. - """ - - def test_read_with_negative_length_fails(self): - """ - The length for reads cannot be negative. - """ diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index aaa455a03..af53efbde 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -15,13 +15,30 @@ if PY2: # fmt: on from base64 import b64encode +from os import urandom + +from twisted.internet.defer import inlineCallbacks from hypothesis import assume, given, strategies as st -from .common import SyncTestCase +from fixtures import Fixture, TempDir +from treq.testing import StubTreq +from klein import Klein +from hyperlink import DecodedURL + +from .common import AsyncTestCase, SyncTestCase +from ..storage.server import StorageServer from ..storage.http_server import ( + HTTPServer, _extract_secrets, Secrets, ClientSecretsException, + _authorized_route, +) +from ..storage.http_client import ( + StorageClient, + ClientException, + StorageClientImmutables, + ImmutableCreateResult, ) @@ -127,3 +144,259 @@ class ExtractSecretsTests(SyncTestCase): """ with self.assertRaises(ClientSecretsException): _extract_secrets(["lease-cancel-secret eA=="], {Secrets.LEASE_RENEW}) + + +# TODO should be actual swissnum +SWISSNUM_FOR_TEST = b"abcd" + + +class TestApp(object): + """HTTP API for testing purposes.""" + + _app = Klein() + _swissnum = SWISSNUM_FOR_TEST # Match what the test client is using + + @_authorized_route(_app, {Secrets.UPLOAD}, "/upload_secret", methods=["GET"]) + def validate_upload_secret(self, request, authorization): + if authorization == {Secrets.UPLOAD: b"MAGIC"}: + return "GOOD SECRET" + else: + return "BAD: {}".format(authorization) + + +class RoutingTests(AsyncTestCase): + """ + Tests for the HTTP routing infrastructure. + """ + + def setUp(self): + if PY2: + self.skipTest("Not going to bother supporting Python 2") + super(RoutingTests, self).setUp() + # Could be a fixture, but will only be used in this test class so not + # going to bother: + self._http_server = TestApp() + self.client = StorageClient( + DecodedURL.from_text("http://127.0.0.1"), + SWISSNUM_FOR_TEST, + treq=StubTreq(self._http_server._app.resource()), + ) + + @inlineCallbacks + def test_authorization_enforcement(self): + """ + The requirement for secrets is enforced; if they are not given, a 400 + response code is returned. + """ + # Without secret, get a 400 error. + response = yield self.client._request( + "GET", + "http://127.0.0.1/upload_secret", + ) + self.assertEqual(response.code, 400) + + # With secret, we're good. + response = yield self.client._request( + "GET", "http://127.0.0.1/upload_secret", upload_secret=b"MAGIC" + ) + self.assertEqual(response.code, 200) + self.assertEqual((yield response.content()), b"GOOD SECRET") + + +class HttpTestFixture(Fixture): + """ + Setup HTTP tests' infrastructure, the storage server and corresponding + client. + """ + + def _setUp(self): + self.tempdir = self.useFixture(TempDir()) + self.storage_server = StorageServer(self.tempdir.path, b"\x00" * 20) + self.http_server = HTTPServer(self.storage_server, SWISSNUM_FOR_TEST) + self.client = StorageClient( + DecodedURL.from_text("http://127.0.0.1"), + SWISSNUM_FOR_TEST, + treq=StubTreq(self.http_server.get_resource()), + ) + + +class GenericHTTPAPITests(AsyncTestCase): + """ + Tests of HTTP client talking to the HTTP server, for generic HTTP API + endpoints and concerns. + """ + + def setUp(self): + if PY2: + self.skipTest("Not going to bother supporting Python 2") + super(GenericHTTPAPITests, self).setUp() + self.http = self.useFixture(HttpTestFixture()) + + @inlineCallbacks + def test_bad_authentication(self): + """ + If the wrong swissnum is used, an ``Unauthorized`` response code is + returned. + """ + client = StorageClient( + DecodedURL.from_text("http://127.0.0.1"), + b"something wrong", + treq=StubTreq(self.http.http_server.get_resource()), + ) + with self.assertRaises(ClientException) as e: + yield client.get_version() + self.assertEqual(e.exception.args[0], 401) + + @inlineCallbacks + def test_version(self): + """ + The client can return the version. + + We ignore available disk space and max immutable share size, since that + might change across calls. + """ + version = yield self.http.client.get_version() + version[b"http://allmydata.org/tahoe/protocols/storage/v1"].pop( + b"available-space" + ) + version[b"http://allmydata.org/tahoe/protocols/storage/v1"].pop( + b"maximum-immutable-share-size" + ) + expected_version = self.http.storage_server.get_version() + expected_version[b"http://allmydata.org/tahoe/protocols/storage/v1"].pop( + b"available-space" + ) + expected_version[b"http://allmydata.org/tahoe/protocols/storage/v1"].pop( + b"maximum-immutable-share-size" + ) + self.assertEqual(version, expected_version) + + +class ImmutableHTTPAPITests(AsyncTestCase): + """ + Tests for immutable upload/download APIs. + """ + + def setUp(self): + if PY2: + self.skipTest("Not going to bother supporting Python 2") + super(ImmutableHTTPAPITests, self).setUp() + self.http = self.useFixture(HttpTestFixture()) + + @inlineCallbacks + def test_upload_can_be_downloaded(self): + """ + A single share can be uploaded in (possibly overlapping) chunks, and + then a random chunk can be downloaded, and it will match the original + file. + + We don't exercise the full variation of overlapping chunks because + that's already done in test_storage.py. + """ + length = 100 + expected_data = b"".join(bytes([i]) for i in range(100)) + + im_client = StorageClientImmutables(self.http.client) + + # Create a upload: + upload_secret = urandom(32) + lease_secret = urandom(32) + storage_index = b"".join(bytes([i]) for i in range(16)) + created = yield im_client.create( + storage_index, [1], 100, upload_secret, lease_secret, lease_secret + ) + self.assertEqual( + created, ImmutableCreateResult(already_have=set(), allocated={1}) + ) + + # Three writes: 10-19, 30-39, 50-59. This allows for a bunch of holes. + def write(offset, length): + return im_client.write_share_chunk( + storage_index, + 1, + upload_secret, + offset, + expected_data[offset : offset + length], + ) + + finished = yield write(10, 10) + self.assertFalse(finished) + finished = yield write(30, 10) + self.assertFalse(finished) + finished = yield write(50, 10) + self.assertFalse(finished) + + # Then, an overlapping write with matching data (15-35): + finished = yield write(15, 20) + self.assertFalse(finished) + + # Now fill in the holes: + finished = yield write(0, 10) + self.assertFalse(finished) + finished = yield write(40, 10) + self.assertFalse(finished) + finished = yield write(60, 40) + self.assertTrue(finished) + + # We can now read: + for offset, length in [(0, 100), (10, 19), (99, 0), (49, 200)]: + downloaded = yield im_client.read_share_chunk( + storage_index, 1, offset, length + ) + self.assertEqual(downloaded, expected_data[offset : offset + length]) + + def test_multiple_shares_uploaded_to_different_place(self): + """ + If a storage index has multiple shares, uploads to different shares are + stored separately and can be downloaded separately. + """ + + def test_bucket_allocated_with_new_shares(self): + """ + If some shares already exist, allocating shares indicates only the new + ones were created. + """ + + def test_bucket_allocation_new_upload_key(self): + """ + If a bucket was allocated with one upload key, and a different upload + key is used to allocate the bucket again, the previous download is + cancelled. + """ + + def test_upload_with_wrong_upload_key_fails(self): + """ + Uploading with a key that doesn't match the one used to allocate the + bucket will fail. + """ + + def test_upload_offset_cannot_be_negative(self): + """ + A negative upload offset will be rejected. + """ + + def test_mismatching_upload_fails(self): + """ + If an uploaded chunk conflicts with an already uploaded chunk, a + CONFLICT error is returned. + """ + + def test_read_of_wrong_storage_index_fails(self): + """ + Reading from unknown storage index results in 404. + """ + + def test_read_of_wrong_share_number_fails(self): + """ + Reading from unknown storage index results in 404. + """ + + def test_read_with_negative_offset_fails(self): + """ + The offset for reads cannot be negative. + """ + + def test_read_with_negative_length_fails(self): + """ + The length for reads cannot be negative. + """ From 1bf2b2ee5f4bec2df84f2a8bfac8aaa40ca08c95 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 20 Jan 2022 10:52:44 -0500 Subject: [PATCH 0582/2309] Note follow-up issue. --- src/allmydata/test/test_storage_http.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index af53efbde..2689b429f 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -349,12 +349,16 @@ class ImmutableHTTPAPITests(AsyncTestCase): """ If a storage index has multiple shares, uploads to different shares are stored separately and can be downloaded separately. + + TBD in https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3860 """ def test_bucket_allocated_with_new_shares(self): """ If some shares already exist, allocating shares indicates only the new ones were created. + + TBD in https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3860 """ def test_bucket_allocation_new_upload_key(self): @@ -362,41 +366,57 @@ class ImmutableHTTPAPITests(AsyncTestCase): If a bucket was allocated with one upload key, and a different upload key is used to allocate the bucket again, the previous download is cancelled. + + TBD in https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3860 """ def test_upload_with_wrong_upload_key_fails(self): """ Uploading with a key that doesn't match the one used to allocate the bucket will fail. + + TBD in https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3860 """ def test_upload_offset_cannot_be_negative(self): """ A negative upload offset will be rejected. + + TBD in https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3860 """ def test_mismatching_upload_fails(self): """ If an uploaded chunk conflicts with an already uploaded chunk, a CONFLICT error is returned. + + TBD in https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3860 """ def test_read_of_wrong_storage_index_fails(self): """ Reading from unknown storage index results in 404. + + TBD in https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3860 """ def test_read_of_wrong_share_number_fails(self): """ Reading from unknown storage index results in 404. + + TBD in https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3860 """ def test_read_with_negative_offset_fails(self): """ The offset for reads cannot be negative. + + TBD in https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3860 """ def test_read_with_negative_length_fails(self): """ The length for reads cannot be negative. + + TBD in https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3860 """ From d5bac8e186859f16c7de5637b601b215a87782c0 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 20 Jan 2022 10:56:08 -0500 Subject: [PATCH 0583/2309] Make sure upload secret semantics are still supporting the security goals. --- docs/proposed/http-storage-node-protocol.rst | 3 ++- src/allmydata/storage/http_server.py | 4 +--- src/allmydata/test/test_storage_http.py | 9 ++++----- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index 26f1a2bb7..bb1db750c 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -493,7 +493,8 @@ Handling repeat calls: * If the same API call is repeated with the same upload secret, the response is the same and no change is made to server state. This is necessary to ensure retries work in the face of lost responses from the server. * If the API calls is with a different upload secret, this implies a new client, perhaps because the old client died. - In this case, all relevant in-progress uploads are canceled, and then the command is handled as usual. + In order to prevent storage servers from being able to mess with each other, this API call will fail, because the secret doesn't match. + The use case of restarting upload from scratch if the client dies can be implemented by having the client persist the upload secret. Discussion `````````` diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 50d955127..71c34124a 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -187,9 +187,7 @@ class HTTPServer(object): # TODO add BucketWriters only for new shares pass else: - # New session. - # TODO cancel all existing BucketWriters, then do - # self._storage_server.allocate_buckets() with given inputs. + # TODO Fail, since the secret doesnt match. pass else: # New upload. diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 2689b429f..a7aad608e 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -361,16 +361,15 @@ class ImmutableHTTPAPITests(AsyncTestCase): TBD in https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3860 """ - def test_bucket_allocation_new_upload_key(self): + def test_bucket_allocation_new_upload_secret(self): """ - If a bucket was allocated with one upload key, and a different upload - key is used to allocate the bucket again, the previous download is - cancelled. + If a bucket was allocated with one upload secret, and a different upload + key is used to allocate the bucket again, the second allocation fails. TBD in https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3860 """ - def test_upload_with_wrong_upload_key_fails(self): + def test_upload_with_wrong_upload_secret_fails(self): """ Uploading with a key that doesn't match the one used to allocate the bucket will fail. From f09aa8c7969d8455a958c278d3fdb889aae15d71 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 20 Jan 2022 11:16:06 -0500 Subject: [PATCH 0584/2309] Use pre-existing parser for Range and Content-Range headers. --- docs/proposed/http-storage-node-protocol.rst | 2 +- nix/tahoe-lafs.nix | 4 +- setup.py | 1 + src/allmydata/storage/http_client.py | 2 +- src/allmydata/storage/http_server.py | 41 ++++++++++---------- src/allmydata/test/test_storage_http.py | 2 +- 6 files changed, 27 insertions(+), 25 deletions(-) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index bb1db750c..560220d00 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -640,7 +640,7 @@ For example:: Read a contiguous sequence of bytes from one share in one bucket. The response body is the raw share data (i.e., ``application/octet-stream``). -The ``Range`` header may be used to request exactly one ``bytes`` range. +The ``Range`` header may be used to request exactly one ``bytes`` range, in which case the response code will be 206 (partial content). Interpretation and response behavior is as specified in RFC 7233 § 4.1. Multiple ranges in a single request are *not* supported. diff --git a/nix/tahoe-lafs.nix b/nix/tahoe-lafs.nix index 04d6c4163..1885dd9ca 100644 --- a/nix/tahoe-lafs.nix +++ b/nix/tahoe-lafs.nix @@ -4,7 +4,7 @@ , setuptools, setuptoolsTrial, pyasn1, zope_interface , service-identity, pyyaml, magic-wormhole, treq, appdirs , beautifulsoup4, eliot, autobahn, cryptography, netifaces -, html5lib, pyutil, distro, configparser, klein, cbor2 +, html5lib, pyutil, distro, configparser, klein, werkzeug, cbor2 }: python.pkgs.buildPythonPackage rec { # Most of the time this is not exactly the release version (eg 1.17.0). @@ -98,7 +98,7 @@ EOF service-identity pyyaml magic-wormhole eliot autobahn cryptography netifaces setuptools future pyutil distro configparser collections-extended - klein cbor2 treq + klein werkzeug cbor2 treq ]; checkInputs = with python.pkgs; [ diff --git a/setup.py b/setup.py index 7e7a955c6..36e82a2b2 100644 --- a/setup.py +++ b/setup.py @@ -143,6 +143,7 @@ install_requires = [ # HTTP server and client "klein", + "werkzeug", "treq", "cbor2" ] diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 8fea86396..cf453fcfc 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -261,7 +261,7 @@ class StorageClientImmutables(object): } ), ) - if response.code == 200: + if response.code == http.PARTIAL_CONTENT: body = yield response.content() returnValue(body) else: diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 71c34124a..bbb42dbe1 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -22,6 +22,7 @@ from base64 import b64decode from klein import Klein from twisted.web import http import attr +from werkzeug.http import parse_range_header, parse_content_range_header # TODO Make sure to use pure Python versions? from cbor2 import dumps, loads @@ -218,11 +219,12 @@ class HTTPServer(object): def write_share_data(self, request, authorization, storage_index, share_number): """Write data to an in-progress immutable upload.""" storage_index = si_a2b(storage_index.encode("ascii")) - content_range = request.getHeader("content-range") - if content_range is None: - offset = 0 - else: - offset = int(content_range.split()[1].split("-")[0]) + content_range = parse_content_range_header(request.getHeader("content-range")) + # TODO in https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3860 + # 1. Malformed header should result in error + # 2. Non-bytes unit should result in error + # 3. Missing header means full upload in one request + offset = content_range.start # TODO basic checks on validity of start, offset, and content-range in general. also of share_number. # TODO basic check that body isn't infinite. require content-length? or maybe we should require content-range (it's optional now)? if so, needs to be rflected in protocol spec. @@ -256,22 +258,21 @@ class HTTPServer(object): ) def read_share_chunk(self, request, authorization, storage_index, share_number): """Read a chunk for an already uploaded immutable.""" - # TODO basic checks on validity + # TODO in https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3860 + # 1. basic checks on validity on storage index, share number + # 2. missing range header should have response code 200 and return whole thing + # 3. malformed range header should result in error? or return everything? + # 4. non-bytes range results in error + # 5. ranges make sense semantically (positive, etc.) + # 6. multiple ranges fails with error + # 7. missing end of range means "to the end of share" storage_index = si_a2b(storage_index.encode("ascii")) - range_header = request.getHeader("range") - if range_header is None: - offset = 0 - inclusive_end = None - else: - parts = range_header.split("=")[1].split("-") - offset = int(parts[0]) # TODO make sure valid - if len(parts) > 0: - inclusive_end = int(parts[1]) # TODO make sure valid - else: - inclusive_end = None - - assert inclusive_end != None # TODO support this case + range_header = parse_range_header(request.getHeader("range")) + offset, end = range_header.ranges[0] + assert end != None # TODO support this case # TODO if not found, 404 bucket = self._storage_server.get_buckets(storage_index)[share_number] - return bucket.read(offset, inclusive_end - offset + 1) + data = bucket.read(offset, end - offset) + request.setResponseCode(http.PARTIAL_CONTENT) + return data diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index a7aad608e..540e40c16 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -339,7 +339,7 @@ class ImmutableHTTPAPITests(AsyncTestCase): self.assertTrue(finished) # We can now read: - for offset, length in [(0, 100), (10, 19), (99, 0), (49, 200)]: + for offset, length in [(0, 100), (10, 19), (99, 1), (49, 200)]: downloaded = yield im_client.read_share_chunk( storage_index, 1, offset, length ) From 5fa8c78f97c14a378ea51f457093647c88c4b597 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 20 Jan 2022 12:04:20 -0500 Subject: [PATCH 0585/2309] Don't use reactor, since it's not necessary. --- src/allmydata/test/test_storage_http.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 540e40c16..8b71666b0 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -25,7 +25,7 @@ from treq.testing import StubTreq from klein import Klein from hyperlink import DecodedURL -from .common import AsyncTestCase, SyncTestCase +from .common import SyncTestCase from ..storage.server import StorageServer from ..storage.http_server import ( HTTPServer, @@ -164,7 +164,7 @@ class TestApp(object): return "BAD: {}".format(authorization) -class RoutingTests(AsyncTestCase): +class RoutingTests(SyncTestCase): """ Tests for the HTTP routing infrastructure. """ @@ -220,7 +220,7 @@ class HttpTestFixture(Fixture): ) -class GenericHTTPAPITests(AsyncTestCase): +class GenericHTTPAPITests(SyncTestCase): """ Tests of HTTP client talking to the HTTP server, for generic HTTP API endpoints and concerns. @@ -272,7 +272,7 @@ class GenericHTTPAPITests(AsyncTestCase): self.assertEqual(version, expected_version) -class ImmutableHTTPAPITests(AsyncTestCase): +class ImmutableHTTPAPITests(SyncTestCase): """ Tests for immutable upload/download APIs. """ From 9a0a19c15a70287d82dbc49e3a102c5a7b62b1d4 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 20 Jan 2022 12:07:58 -0500 Subject: [PATCH 0586/2309] Reminder we might want to support JSON too. --- src/allmydata/storage/http_server.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index bbb42dbe1..236204d66 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -155,6 +155,8 @@ class HTTPServer(object): def _cbor(self, request, data): """Return CBOR-encoded data.""" + # TODO Might want to optionally send JSON someday, based on Accept + # headers, see https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3861 request.setHeader("Content-Type", "application/cbor") # TODO if data is big, maybe want to use a temporary file eventually... return dumps(data) From 587a510b06a406d47e4d61417830d1f69455fcce Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 20 Jan 2022 12:38:01 -0500 Subject: [PATCH 0587/2309] Note a better way to implement this. --- src/allmydata/storage/server.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/allmydata/storage/server.py b/src/allmydata/storage/server.py index 2daf081e4..0add9806b 100644 --- a/src/allmydata/storage/server.py +++ b/src/allmydata/storage/server.py @@ -353,6 +353,9 @@ class StorageServer(service.MultiService): max_space_per_bucket, lease_info, clock=self._clock) if self.no_storage: + # Really this should be done by having a separate class for + # this situation; see + # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3862 bw.throw_out_all_data = True bucketwriters[shnum] = bw self._bucket_writers[incominghome] = bw From 2a2ab1ead722f60cc7771819908dd1b2fb35f58c Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 20 Jan 2022 12:39:25 -0500 Subject: [PATCH 0588/2309] Use a set, not a list, for share numbers. --- src/allmydata/storage/http_client.py | 4 ++-- src/allmydata/test/test_storage_http.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index cf453fcfc..36e745395 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -21,7 +21,7 @@ if PY2: else: # typing module not available in Python 2, and we only do type checking in # Python 3 anyway. - from typing import Union, Set, List, Optional + from typing import Union, Set, Optional from treq.testing import StubTreq from base64 import b64encode @@ -148,7 +148,7 @@ class StorageClientImmutables(object): upload_secret, lease_renew_secret, lease_cancel_secret, - ): # type: (bytes, List[int], int, bytes, bytes, bytes) -> Deferred[ImmutableCreateResult] + ): # type: (bytes, Set[int], int, bytes, bytes, bytes) -> Deferred[ImmutableCreateResult] """ Create a new storage index for an immutable. diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 8b71666b0..a3e7d1640 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -303,7 +303,7 @@ class ImmutableHTTPAPITests(SyncTestCase): lease_secret = urandom(32) storage_index = b"".join(bytes([i]) for i in range(16)) created = yield im_client.create( - storage_index, [1], 100, upload_secret, lease_secret, lease_secret + storage_index, {1}, 100, upload_secret, lease_secret, lease_secret ) self.assertEqual( created, ImmutableCreateResult(already_have=set(), allocated={1}) From b952e738dd60e5b5b6f85cbe232f82a18c828846 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 20 Jan 2022 12:42:26 -0500 Subject: [PATCH 0589/2309] Try to clarify. --- src/allmydata/storage/http_client.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 36e745395..dca8b761c 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -210,8 +210,10 @@ class StorageClientImmutables(object): headers=Headers( { # The range is inclusive, thus the '- 1'. '*' means "length - # unknown", which isn't technically true but adding it just - # makes things slightly harder for calling API. + # unknown", which isn't technically true but it's not clear + # there's any value in passing it in. The server has to + # handle this case anyway, and requiring share length means + # a bit more work for the calling API with no benefit. "content-range": [ "bytes {}-{}/*".format(offset, offset + len(data) - 1) ] From 4b5c71ffbc21a0ff961c15fb27da521adf294bf8 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 20 Jan 2022 12:50:36 -0500 Subject: [PATCH 0590/2309] Bit more info. --- src/allmydata/storage/http_server.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 236204d66..84c1a2e69 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -223,9 +223,10 @@ class HTTPServer(object): storage_index = si_a2b(storage_index.encode("ascii")) content_range = parse_content_range_header(request.getHeader("content-range")) # TODO in https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3860 - # 1. Malformed header should result in error - # 2. Non-bytes unit should result in error + # 1. Malformed header should result in error 416 + # 2. Non-bytes unit should result in error 416 # 3. Missing header means full upload in one request + # 4. Impossible range should resul tin error 416 offset = content_range.start # TODO basic checks on validity of start, offset, and content-range in general. also of share_number. From 65787e5603215fb2b9df3893662e9cefa8112da3 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 20 Jan 2022 12:57:52 -0500 Subject: [PATCH 0591/2309] Get rid of inlineCallbacks. --- src/allmydata/test/test_storage_http.py | 67 ++++++++++++++++--------- 1 file changed, 42 insertions(+), 25 deletions(-) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index a3e7d1640..948b6c718 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -17,8 +17,6 @@ if PY2: from base64 import b64encode from os import urandom -from twisted.internet.defer import inlineCallbacks - from hypothesis import assume, given, strategies as st from fixtures import Fixture, TempDir from treq.testing import StubTreq @@ -164,6 +162,23 @@ class TestApp(object): return "BAD: {}".format(authorization) +def result_of(d): + """ + Synchronously extract the result of a Deferred. + """ + result = [] + error = [] + d.addCallbacks(result.append, error.append) + if result: + return result[0] + if error: + error[0].raiseException() + raise RuntimeError( + "We expected given Deferred to have result already, but it wasn't. " + + "This is probably a test design issue." + ) + + class RoutingTests(SyncTestCase): """ Tests for the HTTP routing infrastructure. @@ -182,25 +197,28 @@ class RoutingTests(SyncTestCase): treq=StubTreq(self._http_server._app.resource()), ) - @inlineCallbacks def test_authorization_enforcement(self): """ The requirement for secrets is enforced; if they are not given, a 400 response code is returned. """ # Without secret, get a 400 error. - response = yield self.client._request( - "GET", - "http://127.0.0.1/upload_secret", + response = result_of( + self.client._request( + "GET", + "http://127.0.0.1/upload_secret", + ) ) self.assertEqual(response.code, 400) # With secret, we're good. - response = yield self.client._request( - "GET", "http://127.0.0.1/upload_secret", upload_secret=b"MAGIC" + response = result_of( + self.client._request( + "GET", "http://127.0.0.1/upload_secret", upload_secret=b"MAGIC" + ) ) self.assertEqual(response.code, 200) - self.assertEqual((yield response.content()), b"GOOD SECRET") + self.assertEqual(result_of(response.content()), b"GOOD SECRET") class HttpTestFixture(Fixture): @@ -232,7 +250,6 @@ class GenericHTTPAPITests(SyncTestCase): super(GenericHTTPAPITests, self).setUp() self.http = self.useFixture(HttpTestFixture()) - @inlineCallbacks def test_bad_authentication(self): """ If the wrong swissnum is used, an ``Unauthorized`` response code is @@ -244,10 +261,9 @@ class GenericHTTPAPITests(SyncTestCase): treq=StubTreq(self.http.http_server.get_resource()), ) with self.assertRaises(ClientException) as e: - yield client.get_version() + result_of(client.get_version()) self.assertEqual(e.exception.args[0], 401) - @inlineCallbacks def test_version(self): """ The client can return the version. @@ -255,7 +271,7 @@ class GenericHTTPAPITests(SyncTestCase): We ignore available disk space and max immutable share size, since that might change across calls. """ - version = yield self.http.client.get_version() + version = result_of(self.http.client.get_version()) version[b"http://allmydata.org/tahoe/protocols/storage/v1"].pop( b"available-space" ) @@ -283,7 +299,6 @@ class ImmutableHTTPAPITests(SyncTestCase): super(ImmutableHTTPAPITests, self).setUp() self.http = self.useFixture(HttpTestFixture()) - @inlineCallbacks def test_upload_can_be_downloaded(self): """ A single share can be uploaded in (possibly overlapping) chunks, and @@ -302,8 +317,10 @@ class ImmutableHTTPAPITests(SyncTestCase): upload_secret = urandom(32) lease_secret = urandom(32) storage_index = b"".join(bytes([i]) for i in range(16)) - created = yield im_client.create( - storage_index, {1}, 100, upload_secret, lease_secret, lease_secret + created = result_of( + im_client.create( + storage_index, {1}, 100, upload_secret, lease_secret, lease_secret + ) ) self.assertEqual( created, ImmutableCreateResult(already_have=set(), allocated={1}) @@ -319,29 +336,29 @@ class ImmutableHTTPAPITests(SyncTestCase): expected_data[offset : offset + length], ) - finished = yield write(10, 10) + finished = result_of(write(10, 10)) self.assertFalse(finished) - finished = yield write(30, 10) + finished = result_of(write(30, 10)) self.assertFalse(finished) - finished = yield write(50, 10) + finished = result_of(write(50, 10)) self.assertFalse(finished) # Then, an overlapping write with matching data (15-35): - finished = yield write(15, 20) + finished = result_of(write(15, 20)) self.assertFalse(finished) # Now fill in the holes: - finished = yield write(0, 10) + finished = result_of(write(0, 10)) self.assertFalse(finished) - finished = yield write(40, 10) + finished = result_of(write(40, 10)) self.assertFalse(finished) - finished = yield write(60, 40) + finished = result_of(write(60, 40)) self.assertTrue(finished) # We can now read: for offset, length in [(0, 100), (10, 19), (99, 1), (49, 200)]: - downloaded = yield im_client.read_share_chunk( - storage_index, 1, offset, length + downloaded = result_of( + im_client.read_share_chunk(storage_index, 1, offset, length) ) self.assertEqual(downloaded, expected_data[offset : offset + length]) From c4d71a4636503df1afa5a7884c473c0674f427f0 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 20 Jan 2022 13:10:42 -0500 Subject: [PATCH 0592/2309] Use abstractions for generating headers on client, note another place we should generate headers. --- src/allmydata/storage/http_client.py | 14 +++++--------- src/allmydata/storage/http_server.py | 6 ++++++ 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index dca8b761c..4436f2fd2 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -31,7 +31,7 @@ import attr # TODO Make sure to import Python version? from cbor2 import loads, dumps - +from werkzeug.datastructures import Range, ContentRange from twisted.web.http_headers import Headers from twisted.web import http from twisted.internet.defer import inlineCallbacks, returnValue, fail, Deferred @@ -209,13 +209,8 @@ class StorageClientImmutables(object): data=data, headers=Headers( { - # The range is inclusive, thus the '- 1'. '*' means "length - # unknown", which isn't technically true but it's not clear - # there's any value in passing it in. The server has to - # handle this case anyway, and requiring share length means - # a bit more work for the calling API with no benefit. "content-range": [ - "bytes {}-{}/*".format(offset, offset + len(data) - 1) + ContentRange("bytes", offset, offset+len(data)).to_header() ] } ), @@ -258,8 +253,9 @@ class StorageClientImmutables(object): url, headers=Headers( { - # The range is inclusive. - "range": ["bytes={}-{}".format(offset, offset + length - 1)] + "range": [ + Range("bytes", [(offset, offset + length)]).to_header() + ] } ), ) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 84c1a2e69..6b792cf06 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -278,4 +278,10 @@ class HTTPServer(object): bucket = self._storage_server.get_buckets(storage_index)[share_number] data = bucket.read(offset, end - offset) request.setResponseCode(http.PARTIAL_CONTENT) + # TODO set content-range on response. We we need to expand the + # BucketReader interface to return share's length. + # + # request.setHeader( + # "content-range", range_header.make_content_range(share_length).to_header() + # ) return data From e8e3a3e663458b4df5cab8553890a75db02c6942 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 21 Jan 2022 11:37:46 -0500 Subject: [PATCH 0593/2309] Expand. --- src/allmydata/storage/http_server.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 6b792cf06..73ef8e09e 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -187,7 +187,8 @@ class HTTPServer(object): in_progress = self._uploads[storage_index] if in_progress.upload_key == upload_key: # Same session. - # TODO add BucketWriters only for new shares + # TODO add BucketWriters only for new shares that don't already have buckets; see the HTTP spec for details. + # The backend code may already implement this logic. pass else: # TODO Fail, since the secret doesnt match. From a4cb4837e6ea122a2273bd1560def2550b438664 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 21 Jan 2022 11:43:36 -0500 Subject: [PATCH 0594/2309] It's a secret, compare it securely. --- src/allmydata/storage/http_server.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 73ef8e09e..a19faf1fa 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -131,7 +131,7 @@ class StorageIndexUploads(object): shares = attr.ib() # type: Dict[int,BucketWriter] # The upload key. - upload_key = attr.ib() # type: bytes + upload_secret = attr.ib() # type: bytes class HTTPServer(object): @@ -180,12 +180,12 @@ class HTTPServer(object): """Allocate buckets.""" storage_index = si_a2b(storage_index.encode("ascii")) info = loads(request.content.read()) - upload_key = authorization[Secrets.UPLOAD] + upload_secret = authorization[Secrets.UPLOAD] if storage_index in self._uploads: # Pre-existing upload. in_progress = self._uploads[storage_index] - if in_progress.upload_key == upload_key: + if timing_safe_compare(in_progress.upload_secret, upload_secret): # Same session. # TODO add BucketWriters only for new shares that don't already have buckets; see the HTTP spec for details. # The backend code may already implement this logic. @@ -203,7 +203,7 @@ class HTTPServer(object): allocated_size=info["allocated-size"], ) self._uploads[storage_index] = StorageIndexUploads( - shares=sharenum_to_bucket, upload_key=authorization[Secrets.UPLOAD] + shares=sharenum_to_bucket, upload_secret=authorization[Secrets.UPLOAD] ) return self._cbor( request, From d2e3b74098c2a94c104f395d8c293ee40f0862be Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 21 Jan 2022 12:36:58 -0500 Subject: [PATCH 0595/2309] Some progress towards upload progress result from the server. --- src/allmydata/storage/http_client.py | 24 ++++++++++--- src/allmydata/storage/http_server.py | 8 +++-- src/allmydata/storage/immutable.py | 10 ++++++ src/allmydata/test/test_storage.py | 6 +++- src/allmydata/test/test_storage_http.py | 48 +++++++++++++++++-------- 5 files changed, 74 insertions(+), 22 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 4436f2fd2..d4837d4ab 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -30,7 +30,7 @@ import attr # TODO Make sure to import Python version? from cbor2 import loads, dumps - +from collections_extended import RangeMap from werkzeug.datastructures import Range, ContentRange from twisted.web.http_headers import Headers from twisted.web import http @@ -131,6 +131,17 @@ class StorageClient(object): returnValue(decoded_response) +@attr.s +class UploadProgress(object): + """ + Progress of immutable upload, per the server. + """ + # True when upload has finished. + finished = attr.ib(type=bool) + # Remaining ranges to upload. + required = attr.ib(type=RangeMap) + + class StorageClientImmutables(object): """ APIs for interacting with immutables. @@ -186,7 +197,7 @@ class StorageClientImmutables(object): @inlineCallbacks def write_share_chunk( self, storage_index, share_number, upload_secret, offset, data - ): # type: (bytes, int, bytes, int, bytes) -> Deferred[bool] + ): # type: (bytes, int, bytes, int, bytes) -> Deferred[UploadProgress] """ Upload a chunk of data for a specific share. @@ -218,14 +229,19 @@ class StorageClientImmutables(object): if response.code == http.OK: # Upload is still unfinished. - returnValue(False) + finished = False elif response.code == http.CREATED: # Upload is done! - returnValue(True) + finished = True else: raise ClientException( response.code, ) + body = loads((yield response.content())) + remaining = RangeMap() + for chunk in body["required"]: + remaining.set(True, chunk["begin"], chunk["end"]) + returnValue(UploadProgress(finished=finished, required=remaining)) @inlineCallbacks def read_share_chunk( diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index a19faf1fa..1d1a9466c 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -23,6 +23,7 @@ from klein import Klein from twisted.web import http import attr from werkzeug.http import parse_range_header, parse_content_range_header +from collections_extended import RangeMap # TODO Make sure to use pure Python versions? from cbor2 import dumps, loads @@ -250,9 +251,10 @@ class HTTPServer(object): else: request.setResponseCode(http.OK) - # TODO spec says we should return missing ranges. but client doesn't - # actually use them? So is it actually useful? - return b"" + required = [] + for start, end, _ in bucket.required_ranges().ranges(): + required.append({"begin": start, "end": end}) + return self._cbor(request, {"required": required}) @_authorized_route( _app, diff --git a/src/allmydata/storage/immutable.py b/src/allmydata/storage/immutable.py index e35ae9782..920bd3c5e 100644 --- a/src/allmydata/storage/immutable.py +++ b/src/allmydata/storage/immutable.py @@ -372,6 +372,16 @@ class BucketWriter(object): self._clock = clock self._timeout = clock.callLater(30 * 60, self._abort_due_to_timeout) + def required_ranges(self): # type: () -> RangeMap + """ + Return which ranges still need to be written. + """ + result = RangeMap() + result.set(True, 0, self._max_size) + for start, end, _ in self._already_written.ranges(): + result.delete(start, end) + return result + def allocated_size(self): return self._max_size diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index 27309a82a..b37f74c24 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -276,7 +276,8 @@ class Bucket(unittest.TestCase): ): """ The ``BucketWriter.write()`` return true if and only if the maximum - size has been reached via potentially overlapping writes. + size has been reached via potentially overlapping writes. The + remaining ranges can be checked via ``BucketWriter.required_ranges()``. """ incoming, final = self.make_workdir("overlapping_writes_{}".format(uuid4())) bw = BucketWriter( @@ -290,6 +291,9 @@ class Bucket(unittest.TestCase): local_written[i] = 1 finished = bw.write(offset, data) self.assertEqual(finished, sum(local_written) == 100) + required_ranges = bw.required_ranges() + for i in range(0, 100): + self.assertEqual(local_written[i] == 1, required_ranges.get(i) is None) def test_read_past_end_of_share_data(self): # test vector for immutable files (hard-coded contents of an immutable share diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 948b6c718..b1eeca4e7 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -22,6 +22,7 @@ from fixtures import Fixture, TempDir from treq.testing import StubTreq from klein import Klein from hyperlink import DecodedURL +from collections_extended import RangeMap from .common import SyncTestCase from ..storage.server import StorageServer @@ -37,6 +38,7 @@ from ..storage.http_client import ( ClientException, StorageClientImmutables, ImmutableCreateResult, + UploadProgress, ) @@ -326,8 +328,12 @@ class ImmutableHTTPAPITests(SyncTestCase): created, ImmutableCreateResult(already_have=set(), allocated={1}) ) + remaining = RangeMap() + remaining.set(True, 0, 100) + # Three writes: 10-19, 30-39, 50-59. This allows for a bunch of holes. def write(offset, length): + remaining.empty(offset, offset + length) return im_client.write_share_chunk( storage_index, 1, @@ -336,24 +342,38 @@ class ImmutableHTTPAPITests(SyncTestCase): expected_data[offset : offset + length], ) - finished = result_of(write(10, 10)) - self.assertFalse(finished) - finished = result_of(write(30, 10)) - self.assertFalse(finished) - finished = result_of(write(50, 10)) - self.assertFalse(finished) + upload_progress = result_of(write(10, 10)) + self.assertEqual( + upload_progress, UploadProgress(finished=False, required=remaining) + ) + upload_progress = result_of(write(30, 10)) + self.assertEqual( + upload_progress, UploadProgress(finished=False, required=remaining) + ) + upload_progress = result_of(write(50, 10)) + self.assertEqual( + upload_progress, UploadProgress(finished=False, required=remaining) + ) # Then, an overlapping write with matching data (15-35): - finished = result_of(write(15, 20)) - self.assertFalse(finished) + upload_progress = result_of(write(15, 20)) + self.assertEqual( + upload_progress, UploadProgress(finished=False, required=remaining) + ) # Now fill in the holes: - finished = result_of(write(0, 10)) - self.assertFalse(finished) - finished = result_of(write(40, 10)) - self.assertFalse(finished) - finished = result_of(write(60, 40)) - self.assertTrue(finished) + upload_progress = result_of(write(0, 10)) + self.assertEqual( + upload_progress, UploadProgress(finished=False, required=remaining) + ) + upload_progress = result_of(write(40, 10)) + self.assertEqual( + upload_progress, UploadProgress(finished=False, required=remaining) + ) + upload_progress = result_of(write(60, 40)) + self.assertEqual( + upload_progress, UploadProgress(finished=False, required=RangeMap()) + ) # We can now read: for offset, length in [(0, 100), (10, 19), (99, 1), (49, 200)]: From 764e493c98798f2b4248189e1c0faa3a9df27ffb Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 24 Jan 2022 10:32:27 -0500 Subject: [PATCH 0596/2309] News. --- newsfragments/3865.incompat | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/3865.incompat diff --git a/newsfragments/3865.incompat b/newsfragments/3865.incompat new file mode 100644 index 000000000..59381b269 --- /dev/null +++ b/newsfragments/3865.incompat @@ -0,0 +1 @@ +Python 3.6 is no longer supported, as it has reached end-of-life and is no longer receiving security updates. \ No newline at end of file From 8eb6ab47653f7e8be52c310332f8f6d9686cd2ae Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 24 Jan 2022 10:40:26 -0500 Subject: [PATCH 0597/2309] Switch to Python 3.7 as minimal version. --- .circleci/config.yml | 12 ++++++------ .github/workflows/ci.yml | 7 +++---- Makefile | 4 ++-- misc/python3/Makefile | 4 ++-- setup.py | 5 ++--- tox.ini | 5 ++--- 6 files changed, 17 insertions(+), 20 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 2fc8e88e7..d55b80469 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -49,8 +49,8 @@ workflows: - "pypy27-buster": {} - # Just one Python 3.6 configuration while the port is in-progress. - - "python36": + # Test against Python 3: + - "python37": {} # Other assorted tasks and configurations @@ -118,7 +118,7 @@ workflows: <<: *DOCKERHUB_CONTEXT - "build-image-pypy27-buster": <<: *DOCKERHUB_CONTEXT - - "build-image-python36-ubuntu": + - "build-image-python37-ubuntu": <<: *DOCKERHUB_CONTEXT @@ -379,7 +379,7 @@ jobs: user: "nobody" - python36: + python37: <<: *UBUNTU_18_04 docker: - <<: *DOCKERHUB_AUTH @@ -392,7 +392,7 @@ jobs: # this reporter on Python 3. So drop that and just specify the # reporter. TAHOE_LAFS_TRIAL_ARGS: "--reporter=subunitv2-file" - TAHOE_LAFS_TOX_ENVIRONMENT: "py36" + TAHOE_LAFS_TOX_ENVIRONMENT: "py37" ubuntu-20-04: @@ -577,7 +577,7 @@ jobs: PYTHON_VERSION: "2.7" - build-image-python36-ubuntu: + build-image-python37-ubuntu: <<: *BUILD_IMAGE environment: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8209108bf..5ae70a3bb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -39,12 +39,11 @@ jobs: - ubuntu-latest python-version: - 2.7 - - 3.6 - 3.7 - 3.8 - 3.9 include: - # On macOS don't bother with 3.6-3.8, just to get faster builds. + # On macOS don't bother with 3.7-3.8, just to get faster builds. - os: macos-10.15 python-version: 2.7 - os: macos-latest @@ -181,10 +180,10 @@ jobs: - ubuntu-latest python-version: - 2.7 - - 3.6 + - 3.7 - 3.9 include: - # On macOS don't bother with 3.6, just to get faster builds. + # On macOS don't bother with 3.7, just to get faster builds. - os: macos-10.15 python-version: 2.7 - os: macos-latest diff --git a/Makefile b/Makefile index 5d8bf18ba..33a40df02 100644 --- a/Makefile +++ b/Makefile @@ -35,7 +35,7 @@ test: .tox/create-venvs.log # Run codechecks first since it takes the least time to report issues early. tox --develop -e codechecks # Run all the test environments in parallel to reduce run-time - tox --develop -p auto -e 'py27,py36,pypy27' + tox --develop -p auto -e 'py27,py37,pypy27' .PHONY: test-venv-coverage ## Run all tests with coverage collection and reporting. test-venv-coverage: @@ -51,7 +51,7 @@ test-venv-coverage: .PHONY: test-py3-all ## Run all tests under Python 3 test-py3-all: .tox/create-venvs.log - tox --develop -e py36 allmydata + tox --develop -e py37 allmydata # This is necessary only if you want to automatically produce a new # _version.py file from the current git history (without doing a build). diff --git a/misc/python3/Makefile b/misc/python3/Makefile index f0ef8b12a..43cb3e3ce 100644 --- a/misc/python3/Makefile +++ b/misc/python3/Makefile @@ -37,8 +37,8 @@ test-py3-all-diff: ../../.tox/make-test-py3-all.diff # `$ make .tox/make-test-py3-all.diff` $(foreach side,old new,../../.tox/make-test-py3-all-$(side).log): cd "../../" - tox --develop --notest -e py36-coverage - (make VIRTUAL_ENV=./.tox/py36-coverage TEST_SUITE=allmydata \ + tox --develop --notest -e py37-coverage + (make VIRTUAL_ENV=./.tox/py37-coverage TEST_SUITE=allmydata \ test-venv-coverage || true) | \ sed -E 's/\([0-9]+\.[0-9]{3} secs\)/(#.### secs)/' | \ tee "./misc/python3/$(@)" diff --git a/setup.py b/setup.py index 53057b808..9a1c76bd8 100644 --- a/setup.py +++ b/setup.py @@ -376,9 +376,8 @@ setup(name="tahoe-lafs", # also set in __init__.py package_dir = {'':'src'}, packages=find_packages('src') + ['allmydata.test.plugins'], classifiers=trove_classifiers, - # We support Python 2.7, and we're working on support for 3.6 (the - # highest version that PyPy currently supports). - python_requires=">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*", + # We support Python 2.7, and Python 3.7 or later. + python_requires=">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*, !=3.6.*", install_requires=install_requires, extras_require={ # Duplicate the Twisted pywin32 dependency here. See diff --git a/tox.ini b/tox.ini index 38cee1f9f..34d555aa7 100644 --- a/tox.ini +++ b/tox.ini @@ -8,7 +8,6 @@ [gh-actions] python = 2.7: py27-coverage,codechecks - 3.6: py36-coverage 3.7: py37-coverage,typechecks,codechecks3 3.8: py38-coverage 3.9: py39-coverage @@ -18,7 +17,7 @@ python = twisted = 1 [tox] -envlist = typechecks,codechecks,codechecks3,py{27,36,37,38,39}-{coverage},pypy27,pypy3,integration,integration3 +envlist = typechecks,codechecks,codechecks3,py{27,37,38,39}-{coverage},pypy27,pypy3,integration,integration3 minversion = 2.4 [testenv] @@ -51,7 +50,7 @@ deps = # suffering we're trying to avoid with the above pins. certifi # VCS hooks support - py36,!coverage: pre-commit + py37,!coverage: pre-commit # We add usedevelop=False because testing against a true installation gives # more useful results. From fa2b4a11c76a1b55100ea51fa25f9d0dcda9ff6d Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 24 Jan 2022 10:50:40 -0500 Subject: [PATCH 0598/2309] Welcome to the WORLD OF TOMORROW --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index d55b80469..5ef9c81a8 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -511,7 +511,7 @@ jobs: # https://circleci.com/blog/how-to-build-a-docker-image-on-circleci-2-0/ docker: - <<: *DOCKERHUB_AUTH - image: "docker:17.05.0-ce-git" + image: "docker:20.10" environment: DISTRO: "tahoelafsci/:foo-py2" From f04e121a7d31d1b18bdd5c5d40618f98ce3bb2ce Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 24 Jan 2022 10:51:55 -0500 Subject: [PATCH 0599/2309] Try to use correct Docker image. --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 5ef9c81a8..05686fcf8 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -383,7 +383,7 @@ jobs: <<: *UBUNTU_18_04 docker: - <<: *DOCKERHUB_AUTH - image: "tahoelafsci/ubuntu:18.04-py3" + image: "tahoelafsci/ubuntu:18.04-py3.7" user: "nobody" environment: @@ -583,7 +583,7 @@ jobs: environment: DISTRO: "ubuntu" TAG: "18.04" - PYTHON_VERSION: "3" + PYTHON_VERSION: "3.7" build-image-ubuntu-20-04: From 02740f075bb69f74ae3ed198a426a0fb3750a5e1 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 24 Jan 2022 10:56:11 -0500 Subject: [PATCH 0600/2309] Temporarily enable image builds on every push. --- .circleci/config.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 05686fcf8..6578f1d1a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -84,12 +84,12 @@ workflows: # faster and takes various spurious failures out of the critical path. triggers: # Build once a day - - schedule: - cron: "0 0 * * *" - filters: - branches: - only: - - "master" + # - schedule: + # cron: "0 0 * * *" + # filters: + # branches: + # only: + # - "master" jobs: # Every job that pushes a Docker image from Docker Hub needs to provide From 31e4556bd1910c19a74e30b793d5098ca5f27b8c Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 24 Jan 2022 11:01:47 -0500 Subject: [PATCH 0601/2309] Need image with Docker _and_ git+ssh. --- .circleci/config.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 6578f1d1a..d28196097 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -511,7 +511,9 @@ jobs: # https://circleci.com/blog/how-to-build-a-docker-image-on-circleci-2-0/ docker: - <<: *DOCKERHUB_AUTH - image: "docker:20.10" + # CircleCI build images; https://github.com/CircleCI-Public/cimg-base + # for details. + image: "cimg/base:2022.01" environment: DISTRO: "tahoelafsci/:foo-py2" From 04cf206e0d78a174c0f80a80180340838f332b67 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 24 Jan 2022 11:06:58 -0500 Subject: [PATCH 0602/2309] Switch back to running image building on schedule. --- .circleci/config.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index d28196097..a650313ed 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -84,12 +84,12 @@ workflows: # faster and takes various spurious failures out of the critical path. triggers: # Build once a day - # - schedule: - # cron: "0 0 * * *" - # filters: - # branches: - # only: - # - "master" + - schedule: + cron: "0 0 * * *" + filters: + branches: + only: + - "master" jobs: # Every job that pushes a Docker image from Docker Hub needs to provide From b64e6552a44a5ae0e4149dbcb363fc51fc469fe4 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 24 Jan 2022 11:30:41 -0500 Subject: [PATCH 0603/2309] Fix assertion. --- src/allmydata/test/test_storage_http.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index b1eeca4e7..dcefc9950 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -372,7 +372,7 @@ class ImmutableHTTPAPITests(SyncTestCase): ) upload_progress = result_of(write(60, 40)) self.assertEqual( - upload_progress, UploadProgress(finished=False, required=RangeMap()) + upload_progress, UploadProgress(finished=True, required=RangeMap()) ) # We can now read: From e9d6eb8d0ec862616b839f3aba0a2b98e5931e3e Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 24 Jan 2022 11:30:49 -0500 Subject: [PATCH 0604/2309] Need some fixes in this version. --- setup.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 36e82a2b2..75a034ff5 100644 --- a/setup.py +++ b/setup.py @@ -138,8 +138,11 @@ install_requires = [ # Backported configparser for Python 2: "configparser ; python_version < '3.0'", - # For the RangeMap datastructure. - "collections-extended", + # For the RangeMap datastructure. Need 2.0.2 at least for bugfixes. Python + # 2 doesn't actually need this, since HTTP storage protocol isn't supported + # there, so we just pick whatever version so that code imports. + "collections-extended >= 2.0.2 ; python_version > '3.0'", + "collections-extended ; python_version < '3.0'", # HTTP server and client "klein", From 0346dfea60d92c9864d0e8cd9ac3ec5cb20e719b Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 25 Jan 2022 09:56:54 -0500 Subject: [PATCH 0605/2309] Note we can do this now. --- src/allmydata/util/encodingutil.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/allmydata/util/encodingutil.py b/src/allmydata/util/encodingutil.py index f32710688..5e28f59fe 100644 --- a/src/allmydata/util/encodingutil.py +++ b/src/allmydata/util/encodingutil.py @@ -320,6 +320,9 @@ def quote_output(s, quotemarks=True, quote_newlines=None, encoding=None): # Although the problem is that doesn't work in Python 3.6, only 3.7 or # later... For now not thinking about it, just returning unicode since # that is the right thing to do on Python 3. + # + # Now that Python 3.7 is the minimum, this can in theory be done: + # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3866 result = result.decode(encoding) return result From 0ad31e33eca75467c3bb5120c7ba05b1b3795323 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 25 Jan 2022 09:57:03 -0500 Subject: [PATCH 0606/2309] Not used. --- misc/python3/Makefile | 53 ------------------------------------------- 1 file changed, 53 deletions(-) delete mode 100644 misc/python3/Makefile diff --git a/misc/python3/Makefile b/misc/python3/Makefile deleted file mode 100644 index f0ef8b12a..000000000 --- a/misc/python3/Makefile +++ /dev/null @@ -1,53 +0,0 @@ -# Python 3 porting targets -# -# NOTE: this Makefile requires GNU make - -### Defensive settings for make: -# https://tech.davis-hansson.com/p/make/ -SHELL := bash -.ONESHELL: -.SHELLFLAGS := -xeu -o pipefail -c -.SILENT: -.DELETE_ON_ERROR: -MAKEFLAGS += --warn-undefined-variables -MAKEFLAGS += --no-builtin-rules - - -# Top-level, phony targets - -.PHONY: default -default: - @echo "no default target" - -.PHONY: test-py3-all-before -## Log the output of running all tests under Python 3 before changes -test-py3-all-before: ../../.tox/make-test-py3-all-old.log -.PHONY: test-py3-all-diff -## Compare the output of running all tests under Python 3 after changes -test-py3-all-diff: ../../.tox/make-test-py3-all.diff - - -# Real targets - -# Gauge the impact of changes on Python 3 compatibility -# Compare the output from running all tests under Python 3 before and after changes. -# Before changes: -# `$ rm -f .tox/make-test-py3-all-*.log && make .tox/make-test-py3-all-old.log` -# After changes: -# `$ make .tox/make-test-py3-all.diff` -$(foreach side,old new,../../.tox/make-test-py3-all-$(side).log): - cd "../../" - tox --develop --notest -e py36-coverage - (make VIRTUAL_ENV=./.tox/py36-coverage TEST_SUITE=allmydata \ - test-venv-coverage || true) | \ - sed -E 's/\([0-9]+\.[0-9]{3} secs\)/(#.### secs)/' | \ - tee "./misc/python3/$(@)" -../../.tox/make-test-py3-all.diff: ../../.tox/make-test-py3-all-new.log - (diff -u "$(<:%-new.log=%-old.log)" "$(<)" || true) | tee "$(@)" - -# Locate modules that are candidates for naively converting `unicode` -> `str`. -# List all Python source files that reference `unicode` but don't reference `str` -../../.tox/py3-unicode-no-str.ls: - cd "../../" - find src -type f -iname '*.py' -exec grep -l -E '\Wunicode\W' '{}' ';' | \ - xargs grep -L '\Wstr\W' | xargs ls -ld | tee "./misc/python3/$(@)" From 54996185dec11b7b2c78a1f90e6a17bee9ff3ed6 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 25 Jan 2022 10:06:05 -0500 Subject: [PATCH 0607/2309] No longer used. --- misc/python3/Makefile | 53 ------------------------------------------- 1 file changed, 53 deletions(-) delete mode 100644 misc/python3/Makefile diff --git a/misc/python3/Makefile b/misc/python3/Makefile deleted file mode 100644 index 43cb3e3ce..000000000 --- a/misc/python3/Makefile +++ /dev/null @@ -1,53 +0,0 @@ -# Python 3 porting targets -# -# NOTE: this Makefile requires GNU make - -### Defensive settings for make: -# https://tech.davis-hansson.com/p/make/ -SHELL := bash -.ONESHELL: -.SHELLFLAGS := -xeu -o pipefail -c -.SILENT: -.DELETE_ON_ERROR: -MAKEFLAGS += --warn-undefined-variables -MAKEFLAGS += --no-builtin-rules - - -# Top-level, phony targets - -.PHONY: default -default: - @echo "no default target" - -.PHONY: test-py3-all-before -## Log the output of running all tests under Python 3 before changes -test-py3-all-before: ../../.tox/make-test-py3-all-old.log -.PHONY: test-py3-all-diff -## Compare the output of running all tests under Python 3 after changes -test-py3-all-diff: ../../.tox/make-test-py3-all.diff - - -# Real targets - -# Gauge the impact of changes on Python 3 compatibility -# Compare the output from running all tests under Python 3 before and after changes. -# Before changes: -# `$ rm -f .tox/make-test-py3-all-*.log && make .tox/make-test-py3-all-old.log` -# After changes: -# `$ make .tox/make-test-py3-all.diff` -$(foreach side,old new,../../.tox/make-test-py3-all-$(side).log): - cd "../../" - tox --develop --notest -e py37-coverage - (make VIRTUAL_ENV=./.tox/py37-coverage TEST_SUITE=allmydata \ - test-venv-coverage || true) | \ - sed -E 's/\([0-9]+\.[0-9]{3} secs\)/(#.### secs)/' | \ - tee "./misc/python3/$(@)" -../../.tox/make-test-py3-all.diff: ../../.tox/make-test-py3-all-new.log - (diff -u "$(<:%-new.log=%-old.log)" "$(<)" || true) | tee "$(@)" - -# Locate modules that are candidates for naively converting `unicode` -> `str`. -# List all Python source files that reference `unicode` but don't reference `str` -../../.tox/py3-unicode-no-str.ls: - cd "../../" - find src -type f -iname '*.py' -exec grep -l -E '\Wunicode\W' '{}' ';' | \ - xargs grep -L '\Wstr\W' | xargs ls -ld | tee "./misc/python3/$(@)" From e1f9f7de94c68d8c4584ea650e6d9584614b3eb7 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 25 Jan 2022 10:06:18 -0500 Subject: [PATCH 0608/2309] Note for future improvement. --- src/allmydata/util/encodingutil.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/allmydata/util/encodingutil.py b/src/allmydata/util/encodingutil.py index f32710688..5e28f59fe 100644 --- a/src/allmydata/util/encodingutil.py +++ b/src/allmydata/util/encodingutil.py @@ -320,6 +320,9 @@ def quote_output(s, quotemarks=True, quote_newlines=None, encoding=None): # Although the problem is that doesn't work in Python 3.6, only 3.7 or # later... For now not thinking about it, just returning unicode since # that is the right thing to do on Python 3. + # + # Now that Python 3.7 is the minimum, this can in theory be done: + # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3866 result = result.decode(encoding) return result From 2583236ad8ab70e3b44bfd1bccbaf4e02aba8400 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 25 Jan 2022 10:56:45 -0500 Subject: [PATCH 0609/2309] Fix unused import. --- src/allmydata/storage/http_server.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 1d1a9466c..d79e9a38b 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -23,7 +23,6 @@ from klein import Klein from twisted.web import http import attr from werkzeug.http import parse_range_header, parse_content_range_header -from collections_extended import RangeMap # TODO Make sure to use pure Python versions? from cbor2 import dumps, loads From 08911a5bddf6f159a7614894ddd04a07425e9a63 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 27 Jan 2022 12:18:23 -0500 Subject: [PATCH 0610/2309] news fragment --- newsfragments/3867.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3867.minor diff --git a/newsfragments/3867.minor b/newsfragments/3867.minor new file mode 100644 index 000000000..e69de29bb From e482745a0bd4b0cfeb24cd782c99a4e3cfa42f57 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 27 Jan 2022 12:19:24 -0500 Subject: [PATCH 0611/2309] drop all of the hand-rolled nix packaging expressions --- nix/autobahn.nix | 34 ---------- nix/cbor2.nix | 20 ------ nix/collections-extended.nix | 19 ------ nix/default.nix | 7 -- nix/eliot.nix | 31 --------- nix/future.nix | 35 ---------- nix/overlays.nix | 36 ---------- nix/py3.nix | 7 -- nix/pyutil.nix | 48 ------------- nix/tahoe-lafs.nix | 126 ----------------------------------- nix/twisted.nix | 63 ------------------ 11 files changed, 426 deletions(-) delete mode 100644 nix/autobahn.nix delete mode 100644 nix/cbor2.nix delete mode 100644 nix/collections-extended.nix delete mode 100644 nix/default.nix delete mode 100644 nix/eliot.nix delete mode 100644 nix/future.nix delete mode 100644 nix/overlays.nix delete mode 100644 nix/py3.nix delete mode 100644 nix/pyutil.nix delete mode 100644 nix/tahoe-lafs.nix delete mode 100644 nix/twisted.nix diff --git a/nix/autobahn.nix b/nix/autobahn.nix deleted file mode 100644 index 83148c4f8..000000000 --- a/nix/autobahn.nix +++ /dev/null @@ -1,34 +0,0 @@ -{ lib, buildPythonPackage, fetchPypi, isPy3k, - six, txaio, twisted, zope_interface, cffi, futures, - mock, pytest, cryptography, pynacl -}: -buildPythonPackage rec { - pname = "autobahn"; - version = "19.8.1"; - - src = fetchPypi { - inherit pname version; - sha256 = "294e7381dd54e73834354832604ae85567caf391c39363fed0ea2bfa86aa4304"; - }; - - propagatedBuildInputs = [ six txaio twisted zope_interface cffi cryptography pynacl ] ++ - (lib.optionals (!isPy3k) [ futures ]); - - checkInputs = [ mock pytest ]; - checkPhase = '' - runHook preCheck - USE_TWISTED=true py.test $out - runHook postCheck - ''; - - # Tests do no seem to be compatible yet with pytest 5.1 - # https://github.com/crossbario/autobahn-python/issues/1235 - doCheck = false; - - meta = with lib; { - description = "WebSocket and WAMP in Python for Twisted and asyncio."; - homepage = "https://crossbar.io/autobahn"; - license = licenses.mit; - maintainers = with maintainers; [ nand0p ]; - }; -} diff --git a/nix/cbor2.nix b/nix/cbor2.nix deleted file mode 100644 index 16ca8ff63..000000000 --- a/nix/cbor2.nix +++ /dev/null @@ -1,20 +0,0 @@ -{ lib, buildPythonPackage, fetchPypi, setuptools_scm }: -buildPythonPackage rec { - pname = "cbor2"; - version = "5.2.0"; - - src = fetchPypi { - sha256 = "1gwlgjl70vlv35cgkcw3cg7b5qsmws36hs4mmh0l9msgagjs4fm3"; - inherit pname version; - }; - - doCheck = false; - - propagatedBuildInputs = [ setuptools_scm ]; - - meta = with lib; { - homepage = https://github.com/agronholm/cbor2; - description = "CBOR encoder/decoder"; - license = licenses.mit; - }; -} diff --git a/nix/collections-extended.nix b/nix/collections-extended.nix deleted file mode 100644 index 3f1ad165a..000000000 --- a/nix/collections-extended.nix +++ /dev/null @@ -1,19 +0,0 @@ -{ lib, buildPythonPackage, fetchPypi }: -buildPythonPackage rec { - pname = "collections-extended"; - version = "1.0.3"; - - src = fetchPypi { - inherit pname version; - sha256 = "0lb69x23asd68n0dgw6lzxfclavrp2764xsnh45jm97njdplznkw"; - }; - - # Tests aren't in tarball, for 1.0.3 at least. - doCheck = false; - - meta = with lib; { - homepage = https://github.com/mlenzen/collections-extended; - description = "Extra Python Collections - bags (multisets), setlists (unique list / indexed set), RangeMap and IndexedDict"; - license = licenses.asl20; - }; -} diff --git a/nix/default.nix b/nix/default.nix deleted file mode 100644 index bd7460c2f..000000000 --- a/nix/default.nix +++ /dev/null @@ -1,7 +0,0 @@ -# This is the main entrypoint for the Tahoe-LAFS derivation. -{ pkgs ? import { } }: -# Add our Python packages to nixpkgs to simplify the expression for the -# Tahoe-LAFS derivation. -let pkgs' = pkgs.extend (import ./overlays.nix); -# Evaluate the expression for our Tahoe-LAFS derivation. -in pkgs'.python2.pkgs.callPackage ./tahoe-lafs.nix { } diff --git a/nix/eliot.nix b/nix/eliot.nix deleted file mode 100644 index c5975e990..000000000 --- a/nix/eliot.nix +++ /dev/null @@ -1,31 +0,0 @@ -{ lib, buildPythonPackage, fetchPypi, zope_interface, pyrsistent, boltons -, hypothesis, testtools, pytest }: -buildPythonPackage rec { - pname = "eliot"; - version = "1.7.0"; - - src = fetchPypi { - inherit pname version; - sha256 = "0ylyycf717s5qsrx8b9n6m38vyj2k8328lfhn8y6r31824991wv8"; - }; - - postPatch = '' - substituteInPlace setup.py \ - --replace "boltons >= 19.0.1" boltons - ''; - - # A seemingly random subset of the test suite fails intermittently. After - # Tahoe-LAFS is ported to Python 3 we can update to a newer Eliot and, if - # the test suite continues to fail, maybe it will be more likely that we can - # have upstream fix it for us. - doCheck = false; - - checkInputs = [ testtools pytest hypothesis ]; - propagatedBuildInputs = [ zope_interface pyrsistent boltons ]; - - meta = with lib; { - homepage = https://github.com/itamarst/eliot/; - description = "Logging library that tells you why it happened"; - license = licenses.asl20; - }; -} diff --git a/nix/future.nix b/nix/future.nix deleted file mode 100644 index 814b7c1b5..000000000 --- a/nix/future.nix +++ /dev/null @@ -1,35 +0,0 @@ -{ lib -, buildPythonPackage -, fetchPypi -}: - -buildPythonPackage rec { - pname = "future"; - version = "0.18.2"; - - src = fetchPypi { - inherit pname version; - sha256 = "sha256:0zakvfj87gy6mn1nba06sdha63rn4njm7bhh0wzyrxhcny8avgmi"; - }; - - doCheck = false; - - meta = { - description = "Clean single-source support for Python 3 and 2"; - longDescription = '' - python-future is the missing compatibility layer between Python 2 and - Python 3. It allows you to use a single, clean Python 3.x-compatible - codebase to support both Python 2 and Python 3 with minimal overhead. - - It provides future and past packages with backports and forward ports - of features from Python 3 and 2. It also comes with futurize and - pasteurize, customized 2to3-based scripts that helps you to convert - either Py2 or Py3 code easily to support both Python 2 and 3 in a - single clean Py3-style codebase, module by module. - ''; - homepage = https://python-future.org; - downloadPage = https://github.com/PythonCharmers/python-future/releases; - license = with lib.licenses; [ mit ]; - maintainers = with lib.maintainers; [ prikhi ]; - }; -} diff --git a/nix/overlays.nix b/nix/overlays.nix deleted file mode 100644 index 92f36e93e..000000000 --- a/nix/overlays.nix +++ /dev/null @@ -1,36 +0,0 @@ -self: super: { - python27 = super.python27.override { - packageOverrides = python-self: python-super: { - # eliot is not part of nixpkgs at all at this time. - eliot = python-self.pythonPackages.callPackage ./eliot.nix { }; - - # NixOS autobahn package has trollius as a dependency, although - # it is optional. Trollius is unmaintained and fails on CI. - autobahn = python-super.pythonPackages.callPackage ./autobahn.nix { }; - - # Porting to Python 3 is greatly aided by the future package. A - # slightly newer version than appears in nixos 19.09 is helpful. - future = python-super.pythonPackages.callPackage ./future.nix { }; - - # Need version of pyutil that supports Python 3. The version in 19.09 - # is too old. - pyutil = python-super.pythonPackages.callPackage ./pyutil.nix { }; - - # Need a newer version of Twisted, too. - twisted = python-super.pythonPackages.callPackage ./twisted.nix { }; - - # collections-extended is not part of nixpkgs at this time. - collections-extended = python-super.pythonPackages.callPackage ./collections-extended.nix { }; - - # cbor2 is not part of nixpkgs at this time. - cbor2 = python-super.pythonPackages.callPackage ./cbor2.nix { }; - }; - }; - - python39 = super.python39.override { - packageOverrides = python-self: python-super: { - # collections-extended is not part of nixpkgs at this time. - collections-extended = python-super.pythonPackages.callPackage ./collections-extended.nix { }; - }; - }; -} diff --git a/nix/py3.nix b/nix/py3.nix deleted file mode 100644 index 34ede49dd..000000000 --- a/nix/py3.nix +++ /dev/null @@ -1,7 +0,0 @@ -# This is the main entrypoint for the Tahoe-LAFS derivation. -{ pkgs ? import { } }: -# Add our Python packages to nixpkgs to simplify the expression for the -# Tahoe-LAFS derivation. -let pkgs' = pkgs.extend (import ./overlays.nix); -# Evaluate the expression for our Tahoe-LAFS derivation. -in pkgs'.python39.pkgs.callPackage ./tahoe-lafs.nix { } diff --git a/nix/pyutil.nix b/nix/pyutil.nix deleted file mode 100644 index 6852c2acc..000000000 --- a/nix/pyutil.nix +++ /dev/null @@ -1,48 +0,0 @@ -{ stdenv -, buildPythonPackage -, fetchPypi -, setuptoolsDarcs -, setuptoolsTrial -, simplejson -, twisted -, isPyPy -}: - -buildPythonPackage rec { - pname = "pyutil"; - version = "3.3.0"; - - src = fetchPypi { - inherit pname version; - sha256 = "8c4d4bf668c559186389bb9bce99e4b1b871c09ba252a756ccaacd2b8f401848"; - }; - - buildInputs = [ setuptoolsDarcs setuptoolsTrial ] ++ (if doCheck then [ simplejson ] else []); - propagatedBuildInputs = [ twisted ]; - - # Tests fail because they try to write new code into the twisted - # package, apparently some kind of plugin. - doCheck = false; - - prePatch = stdenv.lib.optionalString isPyPy '' - grep -rl 'utf-8-with-signature-unix' ./ | xargs sed -i -e "s|utf-8-with-signature-unix|utf-8|g" - ''; - - meta = with stdenv.lib; { - description = "Pyutil, a collection of mature utilities for Python programmers"; - - longDescription = '' - These are a few data structures, classes and functions which - we've needed over many years of Python programming and which - seem to be of general use to other Python programmers. Many of - the modules that have existed in pyutil over the years have - subsequently been obsoleted by new features added to the - Python language or its standard library, thus showing that - we're not alone in wanting tools like these. - ''; - - homepage = "http://allmydata.org/trac/pyutil"; - license = licenses.gpl2Plus; - }; - -} \ No newline at end of file diff --git a/nix/tahoe-lafs.nix b/nix/tahoe-lafs.nix deleted file mode 100644 index 2b41e676e..000000000 --- a/nix/tahoe-lafs.nix +++ /dev/null @@ -1,126 +0,0 @@ -{ fetchFromGitHub, lib -, git, python -, twisted, foolscap, zfec -, setuptools, setuptoolsTrial, pyasn1, zope_interface -, service-identity, pyyaml, magic-wormhole, treq, appdirs -, beautifulsoup4, eliot, autobahn, cryptography, netifaces -, html5lib, pyutil, distro, configparser, klein, cbor2 -}: -python.pkgs.buildPythonPackage rec { - # Most of the time this is not exactly the release version (eg 1.17.1). - # Give it a `post` component to make it look newer than the release version - # and we'll bump this up at the time of each release. - # - # It's difficult to read the version from Git the way the Python code does - # for two reasons. First, doing so involves populating the Nix expression - # with values from the source. Nix calls this "import from derivation" or - # "IFD" (). This is - # discouraged in most cases - including this one, I think. Second, the - # Python code reads the contents of `.git` to determine its version. `.git` - # is not a reproducable artifact (in the sense of "reproducable builds") so - # it is excluded from the source tree by default. When it is included, the - # package tends to be frequently spuriously rebuilt. - version = "1.17.1.post1"; - name = "tahoe-lafs-${version}"; - src = lib.cleanSourceWith { - src = ../.; - filter = name: type: - let - basename = baseNameOf name; - - split = lib.splitString "."; - join = builtins.concatStringsSep "."; - ext = join (builtins.tail (split basename)); - - # Build up a bunch of knowledge about what kind of file this is. - isTox = type == "directory" && basename == ".tox"; - isTrialTemp = type == "directory" && basename == "_trial_temp"; - isVersion = basename == "_version.py"; - isBytecode = ext == "pyc" || ext == "pyo"; - isBackup = lib.hasSuffix "~" basename; - isTemporary = lib.hasPrefix "#" basename && lib.hasSuffix "#" basename; - isSymlink = type == "symlink"; - isGit = type == "directory" && basename == ".git"; - in - # Exclude all these things - ! (isTox - || isTrialTemp - || isVersion - || isBytecode - || isBackup - || isTemporary - || isSymlink - || isGit - ); - }; - - postPatch = '' - # Chroots don't have /etc/hosts and /etc/resolv.conf, so work around - # that. - for i in $(find src/allmydata/test -type f) - do - sed -i "$i" -e"s/localhost/127.0.0.1/g" - done - - # Some tests are flaky or fail to skip when dependencies are missing. - # This list is over-zealous because it's more work to disable individual - # tests with in a module. - - # Many of these tests don't properly skip when i2p or tor dependencies are - # not supplied (and we are not supplying them). - rm src/allmydata/test/test_i2p_provider.py - rm src/allmydata/test/test_connections.py - rm src/allmydata/test/cli/test_create.py - - # Generate _version.py ourselves since we can't rely on the Python code - # extracting the information from the .git directory we excluded. - cat > src/allmydata/_version.py < /dev/null - ''; - - checkPhase = '' - ${python.interpreter} -m unittest discover -s twisted/test - ''; - # Tests require network - doCheck = false; - - meta = with stdenv.lib; { - homepage = https://twistedmatrix.com/; - description = "Twisted, an event-driven networking engine written in Python"; - longDescription = '' - Twisted is an event-driven networking engine written in Python - and licensed under the MIT license. - ''; - license = licenses.mit; - maintainers = [ ]; - }; -} From 8a1d4617c23b8addf2955931ef1a3e234de916b1 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 27 Jan 2022 13:00:36 -0500 Subject: [PATCH 0612/2309] Add new Nix packaging using mach-nix.buildPythonPackage --- default.nix | 52 ++++++++++++++ nix/sources.json | 62 +++++++++++++++++ nix/sources.nix | 174 +++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 288 insertions(+) create mode 100644 default.nix create mode 100644 nix/sources.json create mode 100644 nix/sources.nix diff --git a/default.nix b/default.nix new file mode 100644 index 000000000..4c9ed1cc9 --- /dev/null +++ b/default.nix @@ -0,0 +1,52 @@ +let + sources = import nix/sources.nix; +in +{ pkgsVersion ? "nixpkgs-21.11" +, pkgs ? import sources.${pkgsVersion} { } +, pypiData ? sources.pypi-deps-db +, pythonVersion ? "python37" +, mach-nix ? import sources.mach-nix { + inherit pkgs pypiData; + python = pythonVersion; + } +}: +# The project name, version, and most other metadata are automatically +# extracted from the source. Some requirements are not properly extracted +# and those cases are handled below. The version can only be extracted if +# `setup.py update_version` has been run (this is not at all ideal but it +# seems difficult to fix) - so for now just be sure to run that first. +mach-nix.buildPythonPackage { + # Define the location of the Tahoe-LAFS source to be packaged. Clean up all + # as many of the non-source files (eg the `.git` directory, `~` backup + # files, nix's own `result` symlink, etc) as possible to avoid needing to + # re-build when files that make no difference to the package have changed. + src = pkgs.lib.cleanSource ./.; + + # Define some extra requirements that mach-nix does not automatically detect + # from inspection of the source. We typically don't need to put version + # constraints on any of these requirements. The pypi-deps-db we're + # operating with makes dependency resolution deterministic so as long as it + # works once it will always work. It could be that in the future we update + # pypi-deps-db and an incompatibility arises - in which case it would make + # sense to apply some version constraints here. + requirementsExtra = '' + # mach-nix does not yet support pyproject.toml which means it misses any + # build-time requirements of our dependencies which are declared in such a + # file. Tell it about them here. + setuptools_rust + + # mach-nix does not yet parse environment markers correctly. It misses + # all of our requirements which have an environment marker. Duplicate them + # here. + foolscap + eliot + pyrsistent + ''; + + providers = { + # Through zfec 1.5.5 the wheel has an incorrect runtime dependency + # declared on argparse, not available for recent versions of Python 3. + # Force mach-nix to use the sdist instead, side-stepping this issue. + zfec = "sdist"; + }; +} diff --git a/nix/sources.json b/nix/sources.json new file mode 100644 index 000000000..1169911e2 --- /dev/null +++ b/nix/sources.json @@ -0,0 +1,62 @@ +{ + "mach-nix": { + "branch": "master", + "description": "Create highly reproducible python environments", + "homepage": "", + "owner": "davhau", + "repo": "mach-nix", + "rev": "bdc97ba6b2ecd045a467b008cff4ae337b6a7a6b", + "sha256": "12b3jc0g0ak6s93g3ifvdpwxbyqx276k1kl66bpwz8a67qjbcbwf", + "type": "tarball", + "url": "https://github.com/davhau/mach-nix/archive/bdc97ba6b2ecd045a467b008cff4ae337b6a7a6b.tar.gz", + "url_template": "https://github.com///archive/.tar.gz" + }, + "niv": { + "branch": "master", + "description": "Easy dependency management for Nix projects", + "homepage": "https://github.com/nmattia/niv", + "owner": "nmattia", + "repo": "niv", + "rev": "5830a4dd348d77e39a0f3c4c762ff2663b602d4c", + "sha256": "1d3lsrqvci4qz2hwjrcnd8h5vfkg8aypq3sjd4g3izbc8frwz5sm", + "type": "tarball", + "url": "https://github.com/nmattia/niv/archive/5830a4dd348d77e39a0f3c4c762ff2663b602d4c.tar.gz", + "url_template": "https://github.com///archive/.tar.gz" + }, + "nixpkgs": { + "branch": "release-20.03", + "description": "Nix Packages collection", + "homepage": "", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "eb73405ecceb1dc505b7cbbd234f8f94165e2696", + "sha256": "06k21wbyhhvq2f1xczszh3c2934p0m02by3l2ixvd6nkwrqklax7", + "type": "tarball", + "url": "https://github.com/NixOS/nixpkgs/archive/eb73405ecceb1dc505b7cbbd234f8f94165e2696.tar.gz", + "url_template": "https://github.com///archive/.tar.gz" + }, + "nixpkgs-21.11": { + "branch": "nixos-21.11", + "description": "Nix Packages collection", + "homepage": "", + "owner": "nixos", + "repo": "nixpkgs", + "rev": "6c4b9f1a2fd761e2d384ef86cff0d208ca27fdca", + "sha256": "1yl5gj0mzczhl1j8sl8iqpwa1jzsgr12fdszw9rq13cdig2a2r5f", + "type": "tarball", + "url": "https://github.com/nixos/nixpkgs/archive/6c4b9f1a2fd761e2d384ef86cff0d208ca27fdca.tar.gz", + "url_template": "https://github.com///archive/.tar.gz" + }, + "pypi-deps-db": { + "branch": "master", + "description": "Probably the most complete python dependency database", + "homepage": "", + "owner": "DavHau", + "repo": "pypi-deps-db", + "rev": "0f6de8bf1f186c275af862ec9667abb95aae8542", + "sha256": "1ygw9pywyl4p25hx761d1sbwl3qjhm630fa36gdf6b649im4mx8y", + "type": "tarball", + "url": "https://github.com/DavHau/pypi-deps-db/archive/0f6de8bf1f186c275af862ec9667abb95aae8542.tar.gz", + "url_template": "https://github.com///archive/.tar.gz" + } +} diff --git a/nix/sources.nix b/nix/sources.nix new file mode 100644 index 000000000..1938409dd --- /dev/null +++ b/nix/sources.nix @@ -0,0 +1,174 @@ +# This file has been generated by Niv. + +let + + # + # The fetchers. fetch_ fetches specs of type . + # + + fetch_file = pkgs: name: spec: + let + name' = sanitizeName name + "-src"; + in + if spec.builtin or true then + builtins_fetchurl { inherit (spec) url sha256; name = name'; } + else + pkgs.fetchurl { inherit (spec) url sha256; name = name'; }; + + fetch_tarball = pkgs: name: spec: + let + name' = sanitizeName name + "-src"; + in + if spec.builtin or true then + builtins_fetchTarball { name = name'; inherit (spec) url sha256; } + else + pkgs.fetchzip { name = name'; inherit (spec) url sha256; }; + + fetch_git = name: spec: + let + ref = + if spec ? ref then spec.ref else + if spec ? branch then "refs/heads/${spec.branch}" else + if spec ? tag then "refs/tags/${spec.tag}" else + abort "In git source '${name}': Please specify `ref`, `tag` or `branch`!"; + in + builtins.fetchGit { url = spec.repo; inherit (spec) rev; inherit ref; }; + + fetch_local = spec: spec.path; + + fetch_builtin-tarball = name: throw + ''[${name}] The niv type "builtin-tarball" is deprecated. You should instead use `builtin = true`. + $ niv modify ${name} -a type=tarball -a builtin=true''; + + fetch_builtin-url = name: throw + ''[${name}] The niv type "builtin-url" will soon be deprecated. You should instead use `builtin = true`. + $ niv modify ${name} -a type=file -a builtin=true''; + + # + # Various helpers + # + + # https://github.com/NixOS/nixpkgs/pull/83241/files#diff-c6f540a4f3bfa4b0e8b6bafd4cd54e8bR695 + sanitizeName = name: + ( + concatMapStrings (s: if builtins.isList s then "-" else s) + ( + builtins.split "[^[:alnum:]+._?=-]+" + ((x: builtins.elemAt (builtins.match "\\.*(.*)" x) 0) name) + ) + ); + + # The set of packages used when specs are fetched using non-builtins. + mkPkgs = sources: system: + let + sourcesNixpkgs = + import (builtins_fetchTarball { inherit (sources.nixpkgs) url sha256; }) { inherit system; }; + hasNixpkgsPath = builtins.any (x: x.prefix == "nixpkgs") builtins.nixPath; + hasThisAsNixpkgsPath = == ./.; + in + if builtins.hasAttr "nixpkgs" sources + then sourcesNixpkgs + else if hasNixpkgsPath && ! hasThisAsNixpkgsPath then + import {} + else + abort + '' + Please specify either (through -I or NIX_PATH=nixpkgs=...) or + add a package called "nixpkgs" to your sources.json. + ''; + + # The actual fetching function. + fetch = pkgs: name: spec: + + if ! builtins.hasAttr "type" spec then + abort "ERROR: niv spec ${name} does not have a 'type' attribute" + else if spec.type == "file" then fetch_file pkgs name spec + else if spec.type == "tarball" then fetch_tarball pkgs name spec + else if spec.type == "git" then fetch_git name spec + else if spec.type == "local" then fetch_local spec + else if spec.type == "builtin-tarball" then fetch_builtin-tarball name + else if spec.type == "builtin-url" then fetch_builtin-url name + else + abort "ERROR: niv spec ${name} has unknown type ${builtins.toJSON spec.type}"; + + # If the environment variable NIV_OVERRIDE_${name} is set, then use + # the path directly as opposed to the fetched source. + replace = name: drv: + let + saneName = stringAsChars (c: if isNull (builtins.match "[a-zA-Z0-9]" c) then "_" else c) name; + ersatz = builtins.getEnv "NIV_OVERRIDE_${saneName}"; + in + if ersatz == "" then drv else + # this turns the string into an actual Nix path (for both absolute and + # relative paths) + if builtins.substring 0 1 ersatz == "/" then /. + ersatz else /. + builtins.getEnv "PWD" + "/${ersatz}"; + + # Ports of functions for older nix versions + + # a Nix version of mapAttrs if the built-in doesn't exist + mapAttrs = builtins.mapAttrs or ( + f: set: with builtins; + listToAttrs (map (attr: { name = attr; value = f attr set.${attr}; }) (attrNames set)) + ); + + # https://github.com/NixOS/nixpkgs/blob/0258808f5744ca980b9a1f24fe0b1e6f0fecee9c/lib/lists.nix#L295 + range = first: last: if first > last then [] else builtins.genList (n: first + n) (last - first + 1); + + # https://github.com/NixOS/nixpkgs/blob/0258808f5744ca980b9a1f24fe0b1e6f0fecee9c/lib/strings.nix#L257 + stringToCharacters = s: map (p: builtins.substring p 1 s) (range 0 (builtins.stringLength s - 1)); + + # https://github.com/NixOS/nixpkgs/blob/0258808f5744ca980b9a1f24fe0b1e6f0fecee9c/lib/strings.nix#L269 + stringAsChars = f: s: concatStrings (map f (stringToCharacters s)); + concatMapStrings = f: list: concatStrings (map f list); + concatStrings = builtins.concatStringsSep ""; + + # https://github.com/NixOS/nixpkgs/blob/8a9f58a375c401b96da862d969f66429def1d118/lib/attrsets.nix#L331 + optionalAttrs = cond: as: if cond then as else {}; + + # fetchTarball version that is compatible between all the versions of Nix + builtins_fetchTarball = { url, name ? null, sha256 }@attrs: + let + inherit (builtins) lessThan nixVersion fetchTarball; + in + if lessThan nixVersion "1.12" then + fetchTarball ({ inherit url; } // (optionalAttrs (!isNull name) { inherit name; })) + else + fetchTarball attrs; + + # fetchurl version that is compatible between all the versions of Nix + builtins_fetchurl = { url, name ? null, sha256 }@attrs: + let + inherit (builtins) lessThan nixVersion fetchurl; + in + if lessThan nixVersion "1.12" then + fetchurl ({ inherit url; } // (optionalAttrs (!isNull name) { inherit name; })) + else + fetchurl attrs; + + # Create the final "sources" from the config + mkSources = config: + mapAttrs ( + name: spec: + if builtins.hasAttr "outPath" spec + then abort + "The values in sources.json should not have an 'outPath' attribute" + else + spec // { outPath = replace name (fetch config.pkgs name spec); } + ) config.sources; + + # The "config" used by the fetchers + mkConfig = + { sourcesFile ? if builtins.pathExists ./sources.json then ./sources.json else null + , sources ? if isNull sourcesFile then {} else builtins.fromJSON (builtins.readFile sourcesFile) + , system ? builtins.currentSystem + , pkgs ? mkPkgs sources system + }: rec { + # The sources, i.e. the attribute set of spec name to spec + inherit sources; + + # The "pkgs" (evaluated nixpkgs) to use for e.g. non-builtin fetchers + inherit pkgs; + }; + +in +mkSources (mkConfig {}) // { __functor = _: settings: mkSources (mkConfig settings); } From fae80e5da9c558a4700e1f4d78169359ebd02d0f Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 27 Jan 2022 13:23:54 -0500 Subject: [PATCH 0613/2309] Fix zfec packaging --- default.nix | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/default.nix b/default.nix index 4c9ed1cc9..b6918ecb7 100644 --- a/default.nix +++ b/default.nix @@ -46,7 +46,21 @@ mach-nix.buildPythonPackage { providers = { # Through zfec 1.5.5 the wheel has an incorrect runtime dependency # declared on argparse, not available for recent versions of Python 3. - # Force mach-nix to use the sdist instead, side-stepping this issue. + # Force mach-nix to use the sdist instead. This allows us to apply a + # patch that removes the offending declaration. zfec = "sdist"; }; + + # Define certain overrides to the way Python dependencies are built. + _ = { + # Apply the argparse declaration fix to zfec sdist. + zfec.patches = with pkgs; [ + (fetchpatch { + name = "fix-argparse.patch"; + url = "https://github.com/tahoe-lafs/zfec/commit/c3e736a72cccf44b8e1fb7d6c276400204c6bc1e.patch"; + sha256 = "1md9i2fx1ya7mgcj9j01z58hs3q9pj4ch5is5b5kq4v86cf6x33x"; + }) + ]; + }; + } From c21ca210e35cb40e79105507a9e91675a9fcfef0 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 27 Jan 2022 13:23:59 -0500 Subject: [PATCH 0614/2309] a note about providers --- default.nix | 3 +++ 1 file changed, 3 insertions(+) diff --git a/default.nix b/default.nix index b6918ecb7..970fd75ca 100644 --- a/default.nix +++ b/default.nix @@ -43,6 +43,9 @@ mach-nix.buildPythonPackage { pyrsistent ''; + # Specify where mach-nix should find packages for our Python dependencies. + # There are some reasonable defaults so we only need to specify certain + # packages where the default configuration runs into some issue. providers = { # Through zfec 1.5.5 the wheel has an incorrect runtime dependency # declared on argparse, not available for recent versions of Python 3. From 86bcfaa14d3378802a441421649a42b8c7ac3cfd Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 27 Jan 2022 13:24:05 -0500 Subject: [PATCH 0615/2309] Update CircleCI configuration to the new packaging --- .circleci/config.yml | 21 +++++++++------------ nix/sources.json | 12 ++++++------ 2 files changed, 15 insertions(+), 18 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index a650313ed..6fa1106fc 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -39,10 +39,10 @@ workflows: - "centos-8": {} - - "nixos-19-09": + - "nixos-21-05": {} - - "nixos-21-05": + - "nixos-21-11": {} # Test against PyPy 2.7 @@ -441,15 +441,16 @@ jobs: image: "tahoelafsci/fedora:29-py" user: "nobody" - nixos-19-09: &NIXOS + nixos-21.05: &NIXOS docker: # Run in a highly Nix-capable environment. - <<: *DOCKERHUB_AUTH image: "nixorg/nix:circleci" environment: - NIX_PATH: "nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/nixos-19.09-small.tar.gz" - SOURCE: "nix/" + # Reference the name of a niv-managed nixpkgs source (see `niv show` and + # nix/sources.json) + NIXPKGS: "nixpkgs-21.05" steps: - "checkout" @@ -466,17 +467,13 @@ jobs: # build a couple simple little dependencies that don't take # advantage of multiple cores and we get a little speedup by doing # them in parallel. - nix-build --cores 3 --max-jobs 2 "$SOURCE" + nix-build --cores 3 --max-jobs 2 --argstr pkgsVersion "$NIXPKGS" - nixos-21-05: + nixos-21-11: <<: *NIXOS environment: - # Note this doesn't look more similar to the 19.09 NIX_PATH URL because - # there was some internal shuffling by the NixOS project about how they - # publish stable revisions. - NIX_PATH: "nixpkgs=https://github.com/NixOS/nixpkgs/archive/d32b07e6df276d78e3640eb43882b80c9b2b3459.tar.gz" - SOURCE: "nix/py3.nix" + NIXPKGS: "nixpkgs-21.11" typechecks: docker: diff --git a/nix/sources.json b/nix/sources.json index 1169911e2..e0235a3fb 100644 --- a/nix/sources.json +++ b/nix/sources.json @@ -23,23 +23,23 @@ "url": "https://github.com/nmattia/niv/archive/5830a4dd348d77e39a0f3c4c762ff2663b602d4c.tar.gz", "url_template": "https://github.com///archive/.tar.gz" }, - "nixpkgs": { - "branch": "release-20.03", + "nixpkgs-21.05": { + "branch": "nixos-21.05", "description": "Nix Packages collection", "homepage": "", "owner": "NixOS", "repo": "nixpkgs", - "rev": "eb73405ecceb1dc505b7cbbd234f8f94165e2696", - "sha256": "06k21wbyhhvq2f1xczszh3c2934p0m02by3l2ixvd6nkwrqklax7", + "rev": "0fd9ee1aa36ce865ad273f4f07fdc093adeb5c00", + "sha256": "1mr2qgv5r2nmf6s3gqpcjj76zpsca6r61grzmqngwm0xlh958smx", "type": "tarball", - "url": "https://github.com/NixOS/nixpkgs/archive/eb73405ecceb1dc505b7cbbd234f8f94165e2696.tar.gz", + "url": "https://github.com/NixOS/nixpkgs/archive/0fd9ee1aa36ce865ad273f4f07fdc093adeb5c00.tar.gz", "url_template": "https://github.com///archive/.tar.gz" }, "nixpkgs-21.11": { "branch": "nixos-21.11", "description": "Nix Packages collection", "homepage": "", - "owner": "nixos", + "owner": "NixOS", "repo": "nixpkgs", "rev": "6c4b9f1a2fd761e2d384ef86cff0d208ca27fdca", "sha256": "1yl5gj0mzczhl1j8sl8iqpwa1jzsgr12fdszw9rq13cdig2a2r5f", From b47457646c4a6e1c2a83577628b0e0b13ab39d77 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 27 Jan 2022 13:26:57 -0500 Subject: [PATCH 0616/2309] Correct naming of the CircleCI job --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 6fa1106fc..50191555f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -441,7 +441,7 @@ jobs: image: "tahoelafsci/fedora:29-py" user: "nobody" - nixos-21.05: &NIXOS + nixos-21-05: &NIXOS docker: # Run in a highly Nix-capable environment. - <<: *DOCKERHUB_AUTH From 9c964f4acd46d71d03a4b5e753b439bc9b63cc88 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 27 Jan 2022 13:52:10 -0500 Subject: [PATCH 0617/2309] generate the version info --- .circleci/config.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 50191555f..9d9b967f1 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -454,6 +454,12 @@ jobs: steps: - "checkout" + - "run": + name: "Generation version" + command: | + # The Nix package doesn't know how to do this part, unfortunately. + nix-shell -p python --run 'python setup.py update_version' + - "run": name: "Build and Test" command: | From 5cab1f7a4c147b6079583582231b021e5a837d7a Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 27 Jan 2022 13:57:09 -0500 Subject: [PATCH 0618/2309] Get Python this way? --- .circleci/config.yml | 2 +- .circleci/python.nix | 11 +++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) create mode 100644 .circleci/python.nix diff --git a/.circleci/config.yml b/.circleci/config.yml index 9d9b967f1..e4d6c9e93 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -458,7 +458,7 @@ jobs: name: "Generation version" command: | # The Nix package doesn't know how to do this part, unfortunately. - nix-shell -p python --run 'python setup.py update_version' + nix-shell .circleci/python.nix --run 'python setup.py update_version' - "run": name: "Build and Test" diff --git a/.circleci/python.nix b/.circleci/python.nix new file mode 100644 index 000000000..a830ee61b --- /dev/null +++ b/.circleci/python.nix @@ -0,0 +1,11 @@ +# Define a helper environment for incidental Python tasks required on CI. +let + sources = import ../nix/sources.nix; +in +{ pkgs ? import sources."nixpkgs-21.11" { } +}: +pkgs.mkShell { + buildInputs = [ + pkgs.python3 + ]; +} From dea4c7e131c097f875118f363c3c6e4472b8dc86 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 27 Jan 2022 13:59:32 -0500 Subject: [PATCH 0619/2309] get setuptools --- .circleci/python.nix | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.circleci/python.nix b/.circleci/python.nix index a830ee61b..6e3d79cc1 100644 --- a/.circleci/python.nix +++ b/.circleci/python.nix @@ -6,6 +6,8 @@ in }: pkgs.mkShell { buildInputs = [ - pkgs.python3 + (pkgs.python3.withPackages (ps: [ + ps.setuptools + ])) ]; } From 013e1810e4b0c94bf89701980f830490f1c45e22 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 27 Jan 2022 13:59:37 -0500 Subject: [PATCH 0620/2309] try to use a single nixpkgs in each job --- .circleci/config.yml | 6 ++++-- .circleci/python.nix | 3 ++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index e4d6c9e93..f76197bd7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -456,9 +456,11 @@ jobs: - "checkout" - "run": name: "Generation version" - command: | + command: >- # The Nix package doesn't know how to do this part, unfortunately. - nix-shell .circleci/python.nix --run 'python setup.py update_version' + nix-shell .circleci/python.nix + --argstr pkgsVersion "$NIXPKGS" + --run 'python setup.py update_version' - "run": name: "Build and Test" diff --git a/.circleci/python.nix b/.circleci/python.nix index 6e3d79cc1..ecaf9e27c 100644 --- a/.circleci/python.nix +++ b/.circleci/python.nix @@ -2,7 +2,8 @@ let sources = import ../nix/sources.nix; in -{ pkgs ? import sources."nixpkgs-21.11" { } +{ pkgsVersion +, pkgs ? import sources.${pkgsVersion} { } }: pkgs.mkShell { buildInputs = [ From 78c4b98b086241b43b08d8e7e2c531c1e63133f6 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 27 Jan 2022 14:01:40 -0500 Subject: [PATCH 0621/2309] that comment handles the >- yaml string type badly --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index f76197bd7..01aa75e80 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -455,9 +455,9 @@ jobs: steps: - "checkout" - "run": + # The Nix package doesn't know how to do this part, unfortunately. name: "Generation version" command: >- - # The Nix package doesn't know how to do this part, unfortunately. nix-shell .circleci/python.nix --argstr pkgsVersion "$NIXPKGS" --run 'python setup.py update_version' From 5b7f5a9f889c77c2946bfb0027e3a298f9338d33 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 27 Jan 2022 14:04:21 -0500 Subject: [PATCH 0622/2309] fix typo --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 01aa75e80..406a8f200 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -456,7 +456,7 @@ jobs: - "checkout" - "run": # The Nix package doesn't know how to do this part, unfortunately. - name: "Generation version" + name: "Generate version" command: >- nix-shell .circleci/python.nix --argstr pkgsVersion "$NIXPKGS" From b2acd0f7d0192bb9c03291e5fda56f9d76f3f43c Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 27 Jan 2022 14:05:59 -0500 Subject: [PATCH 0623/2309] >- and indentation changes don't interact well blackslashes are more likely to be understood, I guess --- .circleci/config.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 406a8f200..55c5730a5 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -457,9 +457,9 @@ jobs: - "run": # The Nix package doesn't know how to do this part, unfortunately. name: "Generate version" - command: >- - nix-shell .circleci/python.nix - --argstr pkgsVersion "$NIXPKGS" + command: | + nix-shell .circleci/python.nix \ + --argstr pkgsVersion "$NIXPKGS" \ --run 'python setup.py update_version' - "run": From 83a172210c1edf4298aedaed08c73371c22743ae Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 27 Jan 2022 14:22:35 -0500 Subject: [PATCH 0624/2309] Switch to Nix 2.3. mach-nix is not compatible with older versions. --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 55c5730a5..3752fb7c0 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -445,7 +445,7 @@ jobs: docker: # Run in a highly Nix-capable environment. - <<: *DOCKERHUB_AUTH - image: "nixorg/nix:circleci" + image: "nixos/nix:2.3.16" environment: # Reference the name of a niv-managed nixpkgs source (see `niv show` and From 5edd96ce6b018f93de2914e5c7a83d48e45f6998 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 27 Jan 2022 14:31:56 -0500 Subject: [PATCH 0625/2309] Change around environment management so we can install ssh too The new image does not come with it --- .circleci/config.yml | 14 +++++++++++--- .circleci/{python.nix => env.nix} | 11 +++++------ 2 files changed, 16 insertions(+), 9 deletions(-) rename .circleci/{python.nix => env.nix} (63%) diff --git a/.circleci/config.yml b/.circleci/config.yml index 3752fb7c0..499bb16b6 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -453,14 +453,22 @@ jobs: NIXPKGS: "nixpkgs-21.05" steps: + - "run": + name: "Install Basic Dependencies" + command: | + nix-env \ + -f .circleci/env.nix \ + --argstr pkgsVersion "$NIXPKGS" \ + --install \ + -A ssh python3 + - "checkout" + - "run": # The Nix package doesn't know how to do this part, unfortunately. name: "Generate version" command: | - nix-shell .circleci/python.nix \ - --argstr pkgsVersion "$NIXPKGS" \ - --run 'python setup.py update_version' + python setup.py update_version - "run": name: "Build and Test" diff --git a/.circleci/python.nix b/.circleci/env.nix similarity index 63% rename from .circleci/python.nix rename to .circleci/env.nix index ecaf9e27c..0225b00c8 100644 --- a/.circleci/python.nix +++ b/.circleci/env.nix @@ -5,10 +5,9 @@ in { pkgsVersion , pkgs ? import sources.${pkgsVersion} { } }: -pkgs.mkShell { - buildInputs = [ - (pkgs.python3.withPackages (ps: [ - ps.setuptools - ])) - ]; +{ + ssh = pkgs.openssh; + python = pkgs.python3.withPackages (ps: [ + ps.setuptools + ]); } From e7bba3dad0909c4c5585270de1deba8d53eae643 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 27 Jan 2022 14:36:59 -0500 Subject: [PATCH 0626/2309] cannot use the source before we do the checkout... --- .circleci/config.yml | 11 +++++------ .circleci/env.nix | 13 ------------- 2 files changed, 5 insertions(+), 19 deletions(-) delete mode 100644 .circleci/env.nix diff --git a/.circleci/config.yml b/.circleci/config.yml index 499bb16b6..21e560a89 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -450,17 +450,16 @@ jobs: environment: # Reference the name of a niv-managed nixpkgs source (see `niv show` and # nix/sources.json) - NIXPKGS: "nixpkgs-21.05" + NIXPKGS: "21.05" steps: - "run": name: "Install Basic Dependencies" command: | nix-env \ - -f .circleci/env.nix \ - --argstr pkgsVersion "$NIXPKGS" \ + -I nixpkgs=https://github.com/nixos/nixpkgs/archive/nixos-${NIXPKGS}.tar.gz\ --install \ - -A ssh python3 + -A git openssh python3 - "checkout" @@ -483,13 +482,13 @@ jobs: # build a couple simple little dependencies that don't take # advantage of multiple cores and we get a little speedup by doing # them in parallel. - nix-build --cores 3 --max-jobs 2 --argstr pkgsVersion "$NIXPKGS" + nix-build --cores 3 --max-jobs 2 --argstr pkgsVersion "nixpkgs-$NIXPKGS" nixos-21-11: <<: *NIXOS environment: - NIXPKGS: "nixpkgs-21.11" + NIXPKGS: "21.11" typechecks: docker: diff --git a/.circleci/env.nix b/.circleci/env.nix deleted file mode 100644 index 0225b00c8..000000000 --- a/.circleci/env.nix +++ /dev/null @@ -1,13 +0,0 @@ -# Define a helper environment for incidental Python tasks required on CI. -let - sources = import ../nix/sources.nix; -in -{ pkgsVersion -, pkgs ? import sources.${pkgsVersion} { } -}: -{ - ssh = pkgs.openssh; - python = pkgs.python3.withPackages (ps: [ - ps.setuptools - ]); -} From e4ed98fa64a55984098dc8e147d7eca150c8e366 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 27 Jan 2022 14:39:30 -0500 Subject: [PATCH 0627/2309] maybe this is where they may be found --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 21e560a89..ed45e3c3b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -459,7 +459,7 @@ jobs: nix-env \ -I nixpkgs=https://github.com/nixos/nixpkgs/archive/nixos-${NIXPKGS}.tar.gz\ --install \ - -A git openssh python3 + -A nixos.git nixos.openssh nixos.python3 - "checkout" From 7ee55d07e570af74aceb59fac88c6fccad13a2b1 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 27 Jan 2022 14:47:43 -0500 Subject: [PATCH 0628/2309] Use nix-env less wrong, maybe --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ed45e3c3b..294eacaf4 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -457,9 +457,9 @@ jobs: name: "Install Basic Dependencies" command: | nix-env \ - -I nixpkgs=https://github.com/nixos/nixpkgs/archive/nixos-${NIXPKGS}.tar.gz\ + --file https://github.com/nixos/nixpkgs/archive/nixos-${NIXPKGS}.tar.gz \ --install \ - -A nixos.git nixos.openssh nixos.python3 + -A git openssh python3 - "checkout" From 17d2119521b4adbb6d438e15e596f46494f663bf Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 27 Jan 2022 14:55:34 -0500 Subject: [PATCH 0629/2309] get setuptools in there --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 294eacaf4..de1966c86 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -459,7 +459,7 @@ jobs: nix-env \ --file https://github.com/nixos/nixpkgs/archive/nixos-${NIXPKGS}.tar.gz \ --install \ - -A git openssh python3 + -A git openssh 'python3.withPackages (ps: [ ps.setuptools ])' - "checkout" From a8033e2c2f734477aa9882f858477be7cb4266fb Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 27 Jan 2022 14:59:29 -0500 Subject: [PATCH 0630/2309] cannot get python env that way we don't need python until later anyway --- .circleci/config.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index de1966c86..2b72a4e78 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -459,7 +459,7 @@ jobs: nix-env \ --file https://github.com/nixos/nixpkgs/archive/nixos-${NIXPKGS}.tar.gz \ --install \ - -A git openssh 'python3.withPackages (ps: [ ps.setuptools ])' + -A git openssh - "checkout" @@ -467,7 +467,9 @@ jobs: # The Nix package doesn't know how to do this part, unfortunately. name: "Generate version" command: | - python setup.py update_version + nix-shell \ + -p 'python3.withPackages (ps: [ ps.setuptools ])' \ + --run 'python setup.py update_version' - "run": name: "Build and Test" From 0fb56c9a4890347a1124fcb98f6b459f46699e3a Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 27 Jan 2022 15:03:21 -0500 Subject: [PATCH 0631/2309] I checked, git is there. --- .circleci/config.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 2b72a4e78..11136e04e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -454,12 +454,14 @@ jobs: steps: - "run": + # The nixos/nix image does not include ssh. Install it so the + # `checkout` step will succeed. name: "Install Basic Dependencies" command: | nix-env \ --file https://github.com/nixos/nixpkgs/archive/nixos-${NIXPKGS}.tar.gz \ --install \ - -A git openssh + -A openssh - "checkout" From 136734c198d5bec305b1763aec41a36b321e3308 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 27 Jan 2022 15:09:52 -0500 Subject: [PATCH 0632/2309] try to use cachix --- .circleci/config.yml | 37 +++++++++++++++++++++++++++++++++++-- 1 file changed, 35 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 11136e04e..7153fd370 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -451,20 +451,33 @@ jobs: # Reference the name of a niv-managed nixpkgs source (see `niv show` and # nix/sources.json) NIXPKGS: "21.05" + # CACHIX_AUTH_TOKEN is manually set in the CircleCI web UI and allows us + # to push to CACHIX_NAME. + CACHIX_NAME: "tahoe-lafs-opensource" steps: - "run": # The nixos/nix image does not include ssh. Install it so the - # `checkout` step will succeed. + # `checkout` step will succeed. We also want cachix for + # Nix-friendly caching. name: "Install Basic Dependencies" command: | nix-env \ --file https://github.com/nixos/nixpkgs/archive/nixos-${NIXPKGS}.tar.gz \ --install \ - -A openssh + -A openssh cachix bash - "checkout" + - run: + name: "Cachix setup" + # Record the store paths that exist before we did much. There's no + # reason to cache these, they're either in the image or have to be + # retrieved before we can use cachix to restore from cache. + command: | + cachix use "${CACHIX_NAME}" + nix path-info --all > /tmp/store-path-pre-build + - "run": # The Nix package doesn't know how to do this part, unfortunately. name: "Generate version" @@ -488,6 +501,26 @@ jobs: # them in parallel. nix-build --cores 3 --max-jobs 2 --argstr pkgsVersion "nixpkgs-$NIXPKGS" + - run: + # Send any new store objects to cachix. + name: "Push to Cachix" + when: "always" + command: | + # Cribbed from + # https://circleci.com/blog/managing-secrets-when-you-have-pull-requests-from-outside-contributors/ + if [ -n "$CIRCLE_PR_NUMBER" ]; then + # I'm sure you're thinking "CIRCLE_PR_NUMBER must just be the + # number of the PR being built". Sorry, dear reader, you have + # guessed poorly. It is also conditionally set based on whether + # this is a PR from a fork or not. + # + # https://circleci.com/docs/2.0/env-vars/#built-in-environment-variables + echo "Skipping Cachix push for forked PR." + else + # https://docs.cachix.org/continuous-integration-setup/circleci.html + bash -c "comm -13 <(sort /tmp/store-path-pre-build | grep -v '\.drv$') <(nix path-info --all | grep -v '\.drv$' | sort) | cachix push $CACHIX_NAME" + fi + nixos-21-11: <<: *NIXOS From ccb6e65c0453ec38f582ae5c1163dee9ab49495e Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 27 Jan 2022 15:26:19 -0500 Subject: [PATCH 0633/2309] make sure CACHIX_NAME is set for both nixos jobs --- .circleci/config.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 7153fd370..8e860d497 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -451,9 +451,6 @@ jobs: # Reference the name of a niv-managed nixpkgs source (see `niv show` and # nix/sources.json) NIXPKGS: "21.05" - # CACHIX_AUTH_TOKEN is manually set in the CircleCI web UI and allows us - # to push to CACHIX_NAME. - CACHIX_NAME: "tahoe-lafs-opensource" steps: - "run": @@ -471,6 +468,11 @@ jobs: - run: name: "Cachix setup" + environment: + # CACHIX_AUTH_TOKEN is manually set in the CircleCI web UI and + # allows us to push to CACHIX_NAME. We only need this set for + # `cachix use` in this step. + CACHIX_NAME: "tahoe-lafs-opensource" # Record the store paths that exist before we did much. There's no # reason to cache these, they're either in the image or have to be # retrieved before we can use cachix to restore from cache. From f5e1af00c0689e60a54931fa0720f031a7e83f06 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 27 Jan 2022 15:35:23 -0500 Subject: [PATCH 0634/2309] try using parameters to avoid environment collision the `cachix push` later on also needs CACHIX_NAME so defining it on a single step is not great --- .circleci/config.yml | 47 +++++++++++++++++++++++--------------------- 1 file changed, 25 insertions(+), 22 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 8e860d497..1ef0e820d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -39,11 +39,11 @@ workflows: - "centos-8": {} - - "nixos-21-05": - {} + - "nixos": + nixpkgs: "21.05" - - "nixos-21-11": - {} + - "nixos": + nixpkgs: "21.11" # Test against PyPy 2.7 - "pypy27-buster": @@ -441,16 +441,24 @@ jobs: image: "tahoelafsci/fedora:29-py" user: "nobody" - nixos-21-05: &NIXOS + nixos: + parameters: + nixpkgs: + description: >- + Reference the name of a niv-managed nixpkgs source (see `niv show` + and nix/sources.json) + type: "string" + docker: # Run in a highly Nix-capable environment. - <<: *DOCKERHUB_AUTH image: "nixos/nix:2.3.16" environment: - # Reference the name of a niv-managed nixpkgs source (see `niv show` and - # nix/sources.json) - NIXPKGS: "21.05" + # CACHIX_AUTH_TOKEN is manually set in the CircleCI web UI and + # allows us to push to CACHIX_NAME. We only need this set for + # `cachix use` in this step. + CACHIX_NAME: "tahoe-lafs-opensource" steps: - "run": @@ -460,7 +468,7 @@ jobs: name: "Install Basic Dependencies" command: | nix-env \ - --file https://github.com/nixos/nixpkgs/archive/nixos-${NIXPKGS}.tar.gz \ + --file https://github.com/nixos/nixpkgs/archive/nixos-<>.tar.gz \ --install \ -A openssh cachix bash @@ -468,11 +476,6 @@ jobs: - run: name: "Cachix setup" - environment: - # CACHIX_AUTH_TOKEN is manually set in the CircleCI web UI and - # allows us to push to CACHIX_NAME. We only need this set for - # `cachix use` in this step. - CACHIX_NAME: "tahoe-lafs-opensource" # Record the store paths that exist before we did much. There's no # reason to cache these, they're either in the image or have to be # retrieved before we can use cachix to restore from cache. @@ -489,7 +492,7 @@ jobs: --run 'python setup.py update_version' - "run": - name: "Build and Test" + name: "Build" command: | # CircleCI build environment looks like it has a zillion and a # half cores. Don't let Nix autodetect this high core count @@ -501,7 +504,13 @@ jobs: # build a couple simple little dependencies that don't take # advantage of multiple cores and we get a little speedup by doing # them in parallel. - nix-build --cores 3 --max-jobs 2 --argstr pkgsVersion "nixpkgs-$NIXPKGS" + nix-build --cores 3 --max-jobs 2 --argstr pkgsVersion "nixpkgs-<>" + + - "run": + name: "Test" + command: | + # Let it go somewhat wild for the test suite itself + nix-build --cores 8 --argstr pkgsVersion "nixpkgs-<>" tests.nix - run: # Send any new store objects to cachix. @@ -523,12 +532,6 @@ jobs: bash -c "comm -13 <(sort /tmp/store-path-pre-build | grep -v '\.drv$') <(nix path-info --all | grep -v '\.drv$' | sort) | cachix push $CACHIX_NAME" fi - nixos-21-11: - <<: *NIXOS - - environment: - NIXPKGS: "21.11" - typechecks: docker: - <<: *DOCKERHUB_AUTH From 6154be1a96c967bacc5f507ccac4e6dbe003e737 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 27 Jan 2022 15:37:12 -0500 Subject: [PATCH 0635/2309] Give the NixOS job instantiations nice names --- .circleci/config.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 1ef0e820d..3b76c0fb9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -40,9 +40,11 @@ workflows: {} - "nixos": + name: "NixOS 21.05" nixpkgs: "21.05" - "nixos": + name: "NixOS 21.11" nixpkgs: "21.11" # Test against PyPy 2.7 From 60dd2ee413dcc5f2b2dc2bc484a9fc6a9a73e5d7 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 27 Jan 2022 16:17:37 -0500 Subject: [PATCH 0636/2309] Document the parameters and also accept an `extras` parameter --- default.nix | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/default.nix b/default.nix index 970fd75ca..d5acbbdd7 100644 --- a/default.nix +++ b/default.nix @@ -1,14 +1,27 @@ let sources = import nix/sources.nix; in -{ pkgsVersion ? "nixpkgs-21.11" -, pkgs ? import sources.${pkgsVersion} { } -, pypiData ? sources.pypi-deps-db -, pythonVersion ? "python37" -, mach-nix ? import sources.mach-nix { +{ + pkgsVersion ? "nixpkgs-21.11" # a string which choses a nixpkgs from the + # niv-managed sources data + +, pkgs ? import sources.${pkgsVersion} { } # nixpkgs itself + +, pypiData ? sources.pypi-deps-db # the pypi package database snapshot to use + # for dependency resolution + +, pythonVersion ? "python37" # a string chosing the python derivation from + # nixpkgs to target + +, extras ? [] # a list of strings identifying tahoe-lafs extras, the + # dependencies of which the resulting package will also depend + # on + +, mach-nix ? import sources.mach-nix { # the mach-nix package to use to build + # the tahoe-lafs package inherit pkgs pypiData; python = pythonVersion; - } +} }: # The project name, version, and most other metadata are automatically # extracted from the source. Some requirements are not properly extracted @@ -22,6 +35,9 @@ mach-nix.buildPythonPackage { # re-build when files that make no difference to the package have changed. src = pkgs.lib.cleanSource ./.; + # Select whichever package extras were requested. + inherit extras; + # Define some extra requirements that mach-nix does not automatically detect # from inspection of the source. We typically don't need to put version # constraints on any of these requirements. The pypi-deps-db we're From f03f5fb8d7d3205c25edc94baee2c8886b3092cb Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 27 Jan 2022 16:18:43 -0500 Subject: [PATCH 0637/2309] Add an expression for running the test suite --- default.nix | 6 +++++- tests.nix | 26 ++++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) create mode 100644 tests.nix diff --git a/default.nix b/default.nix index d5acbbdd7..044f59e7b 100644 --- a/default.nix +++ b/default.nix @@ -80,6 +80,10 @@ mach-nix.buildPythonPackage { sha256 = "1md9i2fx1ya7mgcj9j01z58hs3q9pj4ch5is5b5kq4v86cf6x33x"; }) ]; - }; + # Remove a click-default-group patch for a test suite problem which no + # longer applies because the project apparently no longer has a test suite + # in its source distribution. + click-default-group.patches = []; + }; } diff --git a/tests.nix b/tests.nix new file mode 100644 index 000000000..364407e87 --- /dev/null +++ b/tests.nix @@ -0,0 +1,26 @@ +let + sources = import nix/sources.nix; +in +# See default.nix for documentation about parameters. +{ pkgsVersion ? "nixpkgs-21.11" +, pkgs ? import sources.${pkgsVersion} { } +, pypiData ? sources.pypi-deps-db +, pythonVersion ? "python37" +, mach-nix ? import sources.mach-nix { + inherit pkgs pypiData; + python = pythonVersion; + } +}@args: +let + # Get the package with all of its test requirements. + tahoe-lafs = import ./. (args // { extras = [ "test" ]; }); + + # Put it into a Python environment. + python-env = pkgs.${pythonVersion}.withPackages (ps: [ + tahoe-lafs + ]); +in +# Make a derivation that runs the unit test suite. +pkgs.runCommand "tahoe-lafs-tests" { } '' + ${python-env}/bin/python -m twisted.trial -j $NIX_BUILD_CORES allmydata +'' From 16fd427b153551f139a39c5e9d371a2611da3a39 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 27 Jan 2022 16:27:10 -0500 Subject: [PATCH 0638/2309] Get undetected txi2p-tahoe test dependency into the test environment --- default.nix | 6 +++++- tests.nix | 12 +++++++++--- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/default.nix b/default.nix index 044f59e7b..03ab89a4e 100644 --- a/default.nix +++ b/default.nix @@ -28,7 +28,7 @@ in # and those cases are handled below. The version can only be extracted if # `setup.py update_version` has been run (this is not at all ideal but it # seems difficult to fix) - so for now just be sure to run that first. -mach-nix.buildPythonPackage { +mach-nix.buildPythonPackage rec { # Define the location of the Tahoe-LAFS source to be packaged. Clean up all # as many of the non-source files (eg the `.git` directory, `~` backup # files, nix's own `result` symlink, etc) as possible to avoid needing to @@ -86,4 +86,8 @@ mach-nix.buildPythonPackage { # in its source distribution. click-default-group.patches = []; }; + + passthru.meta.mach-nix = { + inherit providers _; + }; } diff --git a/tests.nix b/tests.nix index 364407e87..5b6eae497 100644 --- a/tests.nix +++ b/tests.nix @@ -16,9 +16,15 @@ let tahoe-lafs = import ./. (args // { extras = [ "test" ]; }); # Put it into a Python environment. - python-env = pkgs.${pythonVersion}.withPackages (ps: [ - tahoe-lafs - ]); + python-env = mach-nix.mkPython { + inherit (tahoe-lafs.meta.mach-nix) providers _; + packagesExtra = [ tahoe-lafs ]; + requirements = '' + # txi2p-tahoe is another dependency with an environment marker that + # mach-nix doesn't automatically pick up. + txi2p-tahoe + ''; + }; in # Make a derivation that runs the unit test suite. pkgs.runCommand "tahoe-lafs-tests" { } '' From f5de5fc1271dc434e044fc3c04f523a031b80b05 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 27 Jan 2022 16:41:38 -0500 Subject: [PATCH 0639/2309] produce and output so the build appears to be a success --- tests.nix | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests.nix b/tests.nix index 5b6eae497..3816d6ad8 100644 --- a/tests.nix +++ b/tests.nix @@ -29,4 +29,14 @@ in # Make a derivation that runs the unit test suite. pkgs.runCommand "tahoe-lafs-tests" { } '' ${python-env}/bin/python -m twisted.trial -j $NIX_BUILD_CORES allmydata + + # It's not cool to put the whole _trial_temp into $out because it has weird + # files in it we don't want in the store. Plus, even all of the less weird + # files are mostly just trash that's not meaningful if the test suite passes + # (which is the only way we get $out anyway). + # + # The build log itself is typically available from `nix-store --read-log` so + # we don't need to record that either. + echo "passed" >$out + '' From e4505cd7b88c981983229fc5f040bc02d4eb3f66 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 28 Jan 2022 09:56:57 -0500 Subject: [PATCH 0640/2309] change the strategy for building the test environment it's not clear to me if this is conceptually better or worse than what it replaces but it is about 25% faster --- default.nix | 9 +++++--- tests.nix | 61 ++++++++++++++++++++++++++++++++++++++++++++++------- 2 files changed, 59 insertions(+), 11 deletions(-) diff --git a/default.nix b/default.nix index 03ab89a4e..7abaa4c5a 100644 --- a/default.nix +++ b/default.nix @@ -13,9 +13,12 @@ in , pythonVersion ? "python37" # a string chosing the python derivation from # nixpkgs to target -, extras ? [] # a list of strings identifying tahoe-lafs extras, the - # dependencies of which the resulting package will also depend - # on +, extras ? [ "tor" "i2p" ] # a list of strings identifying tahoe-lafs extras, + # the dependencies of which the resulting package + # will also depend on. Include all of the runtime + # extras by default because the incremental cost of + # including them is a lot smaller than the cost of + # re-building the whole thing to add them. , mach-nix ? import sources.mach-nix { # the mach-nix package to use to build # the tahoe-lafs package diff --git a/tests.nix b/tests.nix index 3816d6ad8..53a8885c0 100644 --- a/tests.nix +++ b/tests.nix @@ -12,17 +12,62 @@ in } }@args: let - # Get the package with all of its test requirements. - tahoe-lafs = import ./. (args // { extras = [ "test" ]; }); + # We would like to know the test requirements but mach-nix does not directly + # expose this information to us. However, it is perfectly capable of + # determining it if we ask right... This is probably not meant to be a + # public mach-nix API but we pinned mach-nix so we can deal with mach-nix + # upgrade breakage in our own time. + mach-lib = import "${sources.mach-nix}/mach_nix/nix/lib.nix" { + inherit pkgs; + lib = pkgs.lib; + }; + tests_require = (mach-lib.extract "python37" ./. "extras_require" ).extras_require.test; - # Put it into a Python environment. + # Get the Tahoe-LAFS package itself. This does not include test + # requirements and we don't ask for test requirements so that we can just + # re-use the normal package if it is already built. + tahoe-lafs = import ./. args; + + # If we want to get tahoe-lafs into a Python environment with a bunch of + # *other* Python modules and let them interact in the usual way then we have + # to ask mach-nix for tahoe-lafs and those other Python modules in the same + # way - i.e., using `requirements`. The other tempting mechanism, + # `packagesExtra`, inserts an extra layer of Python environment and prevents + # normal interaction between Python modules (as well as usually producing + # file collisions in the packages that are both runtime and test + # dependencies). To get the tahoe-lafs we just built into the environment, + # put it into nixpkgs using an overlay and tell mach-nix to get tahoe-lafs + # from nixpkgs. + overridesPre = [(self: super: { inherit tahoe-lafs; })]; + providers = tahoe-lafs.meta.mach-nix.providers // { tahoe-lafs = "nixpkgs"; }; + + # Make the Python environment in which we can run the tests. python-env = mach-nix.mkPython { - inherit (tahoe-lafs.meta.mach-nix) providers _; - packagesExtra = [ tahoe-lafs ]; + # Get the packaging fixes we already know we need from putting together + # the runtime package. + inherit (tahoe-lafs.meta.mach-nix) _; + # Share the runtime package's provider configuration - combined with our + # own that causes the right tahoe-lafs to be picked up. + inherit providers overridesPre; requirements = '' - # txi2p-tahoe is another dependency with an environment marker that - # mach-nix doesn't automatically pick up. - txi2p-tahoe + # Here we pull in the Tahoe-LAFS package itself. + tahoe-lafs + + # Unfortunately mach-nix misses all of the Python dependencies of the + # tahoe-lafs satisfied from nixpkgs. Drag them in here. This gives a + # bit of a pyrrhic flavor to the whole endeavor but maybe mach-nix will + # fix this soon. + # + # https://github.com/DavHau/mach-nix/issues/123 + # https://github.com/DavHau/mach-nix/pull/386 + ${tahoe-lafs.requirements} + + # And then all of the test-only dependencies. + ${builtins.concatStringsSep "\n" tests_require} + + # txi2p-tahoe is another dependency with an environment marker that + # mach-nix doesn't automatically pick up. + txi2p-tahoe ''; }; in From d6e82d1d56e1a6787645c134de66f64b1048aa83 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 28 Jan 2022 10:37:43 -0500 Subject: [PATCH 0641/2309] explain this unfortunate cache step --- .circleci/config.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 3b76c0fb9..daf985567 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -530,7 +530,24 @@ jobs: # https://circleci.com/docs/2.0/env-vars/#built-in-environment-variables echo "Skipping Cachix push for forked PR." else + # If this *isn't* a build from a fork then we have the Cachix + # write key in our environment and we can push any new objects + # to Cachix. + # + # To decide what to push, we inspect the list of store objects + # that existed before and after we did most of our work. Any + # that are new after the work is probably a useful thing to have + # around so push it to the cache. We exclude all derivation + # objects (.drv files) because they're cheap to reconstruct and + # by the time you know their cache key you've already done all + # the work anyway. + # + # This shell expression for finding the objects and pushing them + # was from the Cachix docs: + # # https://docs.cachix.org/continuous-integration-setup/circleci.html + # + # but they seem to have removed it now. bash -c "comm -13 <(sort /tmp/store-path-pre-build | grep -v '\.drv$') <(nix path-info --all | grep -v '\.drv$' | sort) | cachix push $CACHIX_NAME" fi From 005a7622699c74c594ecdbe2b5be53cfa84ba8e9 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 28 Jan 2022 10:46:10 -0500 Subject: [PATCH 0642/2309] spelling --- default.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/default.nix b/default.nix index 7abaa4c5a..cecb5579a 100644 --- a/default.nix +++ b/default.nix @@ -2,7 +2,7 @@ let sources = import nix/sources.nix; in { - pkgsVersion ? "nixpkgs-21.11" # a string which choses a nixpkgs from the + pkgsVersion ? "nixpkgs-21.11" # a string which chooses a nixpkgs from the # niv-managed sources data , pkgs ? import sources.${pkgsVersion} { } # nixpkgs itself @@ -10,7 +10,7 @@ in , pypiData ? sources.pypi-deps-db # the pypi package database snapshot to use # for dependency resolution -, pythonVersion ? "python37" # a string chosing the python derivation from +, pythonVersion ? "python37" # a string choosing the python derivation from # nixpkgs to target , extras ? [ "tor" "i2p" ] # a list of strings identifying tahoe-lafs extras, From 9ba17ba8d1731ba004587144d0e5b0b63f870fcd Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 28 Jan 2022 10:46:13 -0500 Subject: [PATCH 0643/2309] explain sources.nix a bit --- default.nix | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/default.nix b/default.nix index cecb5579a..a8a7ba1c8 100644 --- a/default.nix +++ b/default.nix @@ -1,4 +1,23 @@ let + # sources.nix contains information about which versions of some of our + # dependencies we should use. since we use it to pin nixpkgs and the PyPI + # package database, roughly all the rest of our dependencies are *also* + # pinned - indirectly. + # + # sources.nix is managed using a tool called `niv`. as an example, to + # update to the most recent version of nixpkgs from the 21.11 maintenance + # release, in the top-level tahoe-lafs checkout directory you run: + # + # niv update nixpkgs-21.11 + # + # or, to update the PyPI package database -- which is necessary to make any + # newly released packages visible -- you likewise run: + # + # niv update pypi-deps-db + # + # niv also supports chosing a specific revision, following a different + # branch, etc. find complete documentation for the tool at + # https://github.com/nmattia/niv sources = import nix/sources.nix; in { From d23fdcdb8ab32bd11530db344b0448ecaae2b041 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 28 Jan 2022 12:03:17 -0500 Subject: [PATCH 0644/2309] Sketch of first IStorageServer test with HTTP server/client. --- src/allmydata/storage_client.py | 38 ++++++++++++ src/allmydata/test/test_istorageserver.py | 71 +++++++++++++++++++++-- 2 files changed, 103 insertions(+), 6 deletions(-) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 526e4e70d..13b4dfd01 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -75,6 +75,7 @@ from allmydata.util.observer import ObserverList from allmydata.util.rrefutil import add_version_to_remote_reference from allmydata.util.hashutil import permute_server_hash from allmydata.util.dictutil import BytesKeyDict, UnicodeKeyDict +from allmydata.storage.http_client import StorageClient # who is responsible for de-duplication? @@ -1024,3 +1025,40 @@ class _StorageServer(object): shnum, reason, ).addErrback(log.err, "Error from remote call to advise_corrupt_share") + + +# WORK IN PROGRESS, for now it doesn't actually implement whole thing. +@implementer(IStorageServer) # type: ignore +@attr.s +class _HTTPStorageServer(object): + """ + Talk to remote storage server over HTTP. + """ + _http_client = attr.ib(type=StorageClient) + + @staticmethod + def from_http_client(http_client): # type: (StorageClient) -> _HTTPStorageServer + """ + Create an ``IStorageServer`` from a HTTP ``StorageClient``. + """ + return _HTTPStorageServer(_http_client=http_client) + + def get_version(self): + return self._http_client.get_version() + + def allocate_buckets( + self, + storage_index, + renew_secret, + cancel_secret, + sharenums, + allocated_size, + canary, + ): + pass + + def get_buckets( + self, + storage_index, + ): + pass diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index a17264713..fb9765624 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -1,6 +1,8 @@ """ Tests for the ``IStorageServer`` interface. +Keep in mind that ``IStorageServer`` is actually the storage _client_ interface. + Note that for performance, in the future we might want the same node to be reused across tests, so each test should be careful to generate unique storage indexes. @@ -22,6 +24,10 @@ from random import Random from twisted.internet.defer import inlineCallbacks, returnValue from twisted.internet.task import Clock +from twisted.internet import reactor +from twisted.web.server import Site +from hyperlink import DecodedURL +from treq.api import get_global_pool as get_treq_pool from foolscap.api import Referenceable, RemoteException @@ -29,6 +35,10 @@ from allmydata.interfaces import IStorageServer # really, IStorageClient from .common_system import SystemTestMixin from .common import AsyncTestCase from allmydata.storage.server import StorageServer # not a IStorageServer!! +from allmydata.storage.http_server import HTTPServer +from allmydata.storage.http_client import StorageClient +from allmydata.util.iputil import allocate_tcp_port + # Use random generator with known seed, so results are reproducible if tests # are run in the same order. @@ -998,11 +1008,11 @@ class IStorageServerMutableAPIsTestsMixin(object): self.assertEqual(lease2.get_expiration_time() - initial_expiration_time, 167) -class _FoolscapMixin(SystemTestMixin): - """Run tests on Foolscap version of ``IStorageServer.""" +class _SharedMixin(SystemTestMixin): + """Base class for Foolscap and HTTP mixins.""" - def _get_native_server(self): - return next(iter(self.clients[0].storage_broker.get_known_servers())) + def _get_istorage_server(self): + raise NotImplementedError("implement in subclass") @inlineCallbacks def setUp(self): @@ -1010,8 +1020,6 @@ class _FoolscapMixin(SystemTestMixin): self.basedir = "test_istorageserver/" + self.id() yield SystemTestMixin.setUp(self) yield self.set_up_nodes(1) - self.storage_client = self._get_native_server().get_storage_server() - self.assertTrue(IStorageServer.providedBy(self.storage_client)) self.server = None for s in self.clients[0].services: if isinstance(s, StorageServer): @@ -1021,6 +1029,7 @@ class _FoolscapMixin(SystemTestMixin): self._clock = Clock() self._clock.advance(123456) self.server._clock = self._clock + self.storage_client = self._get_istorage_server() def fake_time(self): """Return the current fake, test-controlled, time.""" @@ -1035,6 +1044,25 @@ class _FoolscapMixin(SystemTestMixin): AsyncTestCase.tearDown(self) yield SystemTestMixin.tearDown(self) + @inlineCallbacks + def disconnect(self): + """ + Disconnect and then reconnect with a new ``IStorageServer``. + """ + raise NotImplementedError("implement in subclass") + + +class _FoolscapMixin(_SharedMixin): + """Run tests on Foolscap version of ``IStorageServer``.""" + + def _get_native_server(self): + return next(iter(self.clients[0].storage_broker.get_known_servers())) + + def _get_istorage_server(self): + client = self._get_native_server().get_storage_server() + self.assertTrue(IStorageServer.providedBy(client)) + return client + @inlineCallbacks def disconnect(self): """ @@ -1046,12 +1074,43 @@ class _FoolscapMixin(SystemTestMixin): assert self.storage_client is not current +class _HTTPMixin(_SharedMixin): + """Run tests on the HTTP version of ``IStorageServer``.""" + + def _get_istorage_server(self): + swissnum = b"1234" + self._http_storage_server = HTTPServer(self.server, swissnum) + self._port_number = allocate_tcp_port() + self._listening_port = reactor.listenTCP( + self._port_number, Site(self._http_storage_server.get_resource()), + interface="127.0.0.1" + ) + return StorageClient( + DecodedURL.from_text("http://127.0.0.1:{}".format(self._port_number)), + swissnum + ) + # Eventually should also: + # self.assertTrue(IStorageServer.providedBy(client)) + + @inlineCallbacks + def tearDown(self): + yield _SharedMixin.tearDown(self) + self._listening_port.stopListening() + yield get_treq_pool().closeCachedConnections() + + class FoolscapSharedAPIsTests( _FoolscapMixin, IStorageServerSharedAPIsTestsMixin, AsyncTestCase ): """Foolscap-specific tests for shared ``IStorageServer`` APIs.""" +class HTTPSharedAPIsTests( + _HTTPMixin, IStorageServerSharedAPIsTestsMixin, AsyncTestCase +): + """HTTP-specific tests for shared ``IStorageServer`` APIs.""" + + class FoolscapImmutableAPIsTests( _FoolscapMixin, IStorageServerImmutableAPIsTestsMixin, AsyncTestCase ): From e672029e6d1a1b2e6e209d6e5f6eca5f23842a40 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 31 Jan 2022 10:49:43 -0500 Subject: [PATCH 0645/2309] First HTTP test passes. --- src/allmydata/test/test_istorageserver.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index fb9765624..11758bf15 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -26,8 +26,9 @@ from twisted.internet.defer import inlineCallbacks, returnValue from twisted.internet.task import Clock from twisted.internet import reactor from twisted.web.server import Site +from twisted.web.client import HTTPConnectionPool from hyperlink import DecodedURL -from treq.api import get_global_pool as get_treq_pool +from treq.api import set_global_pool as set_treq_pool from foolscap.api import Referenceable, RemoteException @@ -1078,6 +1079,7 @@ class _HTTPMixin(_SharedMixin): """Run tests on the HTTP version of ``IStorageServer``.""" def _get_istorage_server(self): + set_treq_pool(HTTPConnectionPool(reactor, persistent=False)) swissnum = b"1234" self._http_storage_server = HTTPServer(self.server, swissnum) self._port_number = allocate_tcp_port() @@ -1095,8 +1097,7 @@ class _HTTPMixin(_SharedMixin): @inlineCallbacks def tearDown(self): yield _SharedMixin.tearDown(self) - self._listening_port.stopListening() - yield get_treq_pool().closeCachedConnections() + yield self._listening_port.stopListening() class FoolscapSharedAPIsTests( From 03bc39ed771f2c0446a511821d280d9772cfce2a Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 31 Jan 2022 11:30:41 -0500 Subject: [PATCH 0646/2309] Try to fix nix builds. --- default.nix | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/default.nix b/default.nix index a8a7ba1c8..095c54578 100644 --- a/default.nix +++ b/default.nix @@ -73,12 +73,13 @@ mach-nix.buildPythonPackage rec { # file. Tell it about them here. setuptools_rust - # mach-nix does not yet parse environment markers correctly. It misses - # all of our requirements which have an environment marker. Duplicate them - # here. + # mach-nix does not yet parse environment markers (e.g. "python > '3.0'") + # correctly. It misses all of our requirements which have an environment marker. + # Duplicate them here. foolscap eliot pyrsistent + collections-extended ''; # Specify where mach-nix should find packages for our Python dependencies. From 66abe5dfca17b67e22474210d72eee92dcede3c5 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 31 Jan 2022 12:02:52 -0500 Subject: [PATCH 0647/2309] First passing immutable-API-over-HTTP IStorageServer tests. --- src/allmydata/storage_client.py | 68 +++++++++++++++++++++-- src/allmydata/test/test_istorageserver.py | 20 +++++-- 2 files changed, 79 insertions(+), 9 deletions(-) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 13b4dfd01..ca977c9d9 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -40,7 +40,7 @@ if PY2: from six import ensure_text import re, time, hashlib - +from os import urandom # On Python 2 this will be the backport. from configparser import NoSectionError @@ -75,7 +75,7 @@ from allmydata.util.observer import ObserverList from allmydata.util.rrefutil import add_version_to_remote_reference from allmydata.util.hashutil import permute_server_hash from allmydata.util.dictutil import BytesKeyDict, UnicodeKeyDict -from allmydata.storage.http_client import StorageClient +from allmydata.storage.http_client import StorageClient, StorageClientImmutables # who is responsible for de-duplication? @@ -1027,6 +1027,48 @@ class _StorageServer(object): ).addErrback(log.err, "Error from remote call to advise_corrupt_share") + +@attr.s +class _FakeRemoteReference(object): + """ + Emulate a Foolscap RemoteReference, calling a local object instead. + """ + local_object = attr.ib(type=object) + + def callRemote(self, action, *args, **kwargs): + return getattr(self.local_object, action)(*args, **kwargs) + + +@attr.s +class _HTTPBucketWriter(object): + """ + Emulate a ``RIBucketWriter``. + """ + client = attr.ib(type=StorageClientImmutables) + storage_index = attr.ib(type=bytes) + share_number = attr.ib(type=int) + upload_secret = attr.ib(type=bytes) + finished = attr.ib(type=bool, default=False) + + def abort(self): + pass # TODO in later ticket + + @defer.inlineCallbacks + def write(self, offset, data): + result = yield self.client.write_share_chunk( + self.storage_index, self.share_number, self.upload_secret, offset, data + ) + if result.finished: + self.finished = True + defer.returnValue(None) + + def close(self): + # A no-op in HTTP protocol. + if not self.finished: + return defer.fail(RuntimeError("You didn't finish writing?!")) + return defer.succeed(None) + + # WORK IN PROGRESS, for now it doesn't actually implement whole thing. @implementer(IStorageServer) # type: ignore @attr.s @@ -1041,11 +1083,12 @@ class _HTTPStorageServer(object): """ Create an ``IStorageServer`` from a HTTP ``StorageClient``. """ - return _HTTPStorageServer(_http_client=http_client) + return _HTTPStorageServer(http_client=http_client) def get_version(self): return self._http_client.get_version() + @defer.inlineCallbacks def allocate_buckets( self, storage_index, @@ -1055,7 +1098,24 @@ class _HTTPStorageServer(object): allocated_size, canary, ): - pass + upload_secret = urandom(20) + immutable_client = StorageClientImmutables(self._http_client) + result = immutable_client.create( + storage_index, sharenums, allocated_size, upload_secret, renew_secret, + cancel_secret + ) + result = yield result + defer.returnValue( + (result.already_have, { + share_num: _FakeRemoteReference(_HTTPBucketWriter( + client=immutable_client, + storage_index=storage_index, + share_number=share_num, + upload_secret=upload_secret + )) + for share_num in result.allocated + }) + ) def get_buckets( self, diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index 11758bf15..72c07ab82 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -39,6 +39,7 @@ from allmydata.storage.server import StorageServer # not a IStorageServer!! from allmydata.storage.http_server import HTTPServer from allmydata.storage.http_client import StorageClient from allmydata.util.iputil import allocate_tcp_port +from allmydata.storage_client import _HTTPStorageServer # Use random generator with known seed, so results are reproducible if tests @@ -1084,12 +1085,15 @@ class _HTTPMixin(_SharedMixin): self._http_storage_server = HTTPServer(self.server, swissnum) self._port_number = allocate_tcp_port() self._listening_port = reactor.listenTCP( - self._port_number, Site(self._http_storage_server.get_resource()), - interface="127.0.0.1" + self._port_number, + Site(self._http_storage_server.get_resource()), + interface="127.0.0.1", ) - return StorageClient( - DecodedURL.from_text("http://127.0.0.1:{}".format(self._port_number)), - swissnum + return _HTTPStorageServer.from_http_client( + StorageClient( + DecodedURL.from_text("http://127.0.0.1:{}".format(self._port_number)), + swissnum, + ) ) # Eventually should also: # self.assertTrue(IStorageServer.providedBy(client)) @@ -1118,6 +1122,12 @@ class FoolscapImmutableAPIsTests( """Foolscap-specific tests for immutable ``IStorageServer`` APIs.""" +class HTTPImmutableAPIsTests( + _HTTPMixin, IStorageServerImmutableAPIsTestsMixin, AsyncTestCase +): + """HTTP-specific tests for immutable ``IStorageServer`` APIs.""" + + class FoolscapMutableAPIsTests( _FoolscapMixin, IStorageServerMutableAPIsTestsMixin, AsyncTestCase ): From 5dfaa82ed294fe097fdd75e3b576b0ae7724e346 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 1 Feb 2022 09:47:51 -0500 Subject: [PATCH 0648/2309] Skip tests that don't pass. --- src/allmydata/test/test_istorageserver.py | 25 +++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index 72c07ab82..65a67c586 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -21,6 +21,7 @@ if PY2: # fmt: on from random import Random +from unittest import SkipTest from twisted.internet.defer import inlineCallbacks, returnValue from twisted.internet.task import Clock @@ -1013,11 +1014,18 @@ class IStorageServerMutableAPIsTestsMixin(object): class _SharedMixin(SystemTestMixin): """Base class for Foolscap and HTTP mixins.""" + SKIP_TESTS = set() + def _get_istorage_server(self): raise NotImplementedError("implement in subclass") @inlineCallbacks def setUp(self): + if self._testMethodName in self.SKIP_TESTS: + raise SkipTest( + "Test {} is still not supported".format(self._testMethodName) + ) + AsyncTestCase.setUp(self) self.basedir = "test_istorageserver/" + self.id() yield SystemTestMixin.setUp(self) @@ -1127,6 +1135,23 @@ class HTTPImmutableAPIsTests( ): """HTTP-specific tests for immutable ``IStorageServer`` APIs.""" + # These will start passing in future PRs as HTTP protocol is implemented. + SKIP_TESTS = { + "test_abort", + "test_add_lease_renewal", + "test_add_new_lease", + "test_advise_corrupt_share", + "test_allocate_buckets_repeat", + "test_bucket_advise_corrupt_share", + "test_disconnection", + "test_get_buckets_skips_unfinished_buckets", + "test_matching_overlapping_writes", + "test_non_matching_overlapping_writes", + "test_read_bucket_at_offset", + "test_written_shares_are_readable", + "test_written_shares_are_allocated", + } + class FoolscapMutableAPIsTests( _FoolscapMixin, IStorageServerMutableAPIsTestsMixin, AsyncTestCase From 5cda7ad8b9221df9e5208a6592d39014354ded24 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 1 Feb 2022 09:52:04 -0500 Subject: [PATCH 0649/2309] News file. --- newsfragments/3868.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3868.minor diff --git a/newsfragments/3868.minor b/newsfragments/3868.minor new file mode 100644 index 000000000..e69de29bb From c2e524ddb8fd3b655aad3577a6524b08f359ce4a Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 1 Feb 2022 09:55:13 -0500 Subject: [PATCH 0650/2309] Make mypy happy. --- src/allmydata/test/test_istorageserver.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index 65a67c586..96d4f687f 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -19,6 +19,8 @@ if PY2: # fmt: off from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 # fmt: on +else: + from typing import Set from random import Random from unittest import SkipTest @@ -1014,7 +1016,7 @@ class IStorageServerMutableAPIsTestsMixin(object): class _SharedMixin(SystemTestMixin): """Base class for Foolscap and HTTP mixins.""" - SKIP_TESTS = set() + SKIP_TESTS = set() # type: Set[str] def _get_istorage_server(self): raise NotImplementedError("implement in subclass") From c72e7b0585edf515506a66c1b6b150315b81757d Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 1 Feb 2022 10:20:23 -0500 Subject: [PATCH 0651/2309] Implement HTTP share listing endpoint. --- docs/proposed/http-storage-node-protocol.rst | 4 +-- src/allmydata/storage/http_client.py | 29 ++++++++++++++---- src/allmydata/storage/http_server.py | 16 ++++++++++ src/allmydata/test/test_storage_http.py | 31 ++++++++++++++++++++ 4 files changed, 72 insertions(+), 8 deletions(-) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index 560220d00..f47a50d3e 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -630,8 +630,8 @@ Reading ``GET /v1/immutable/:storage_index/shares`` !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! -Retrieve a list indicating all shares available for the indicated storage index. -For example:: +Retrieve a list (semantically, a set) indicating all shares available for the +indicated storage index. For example:: [1, 5] diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index d4837d4ab..8a1d03192 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -136,6 +136,7 @@ class UploadProgress(object): """ Progress of immutable upload, per the server. """ + # True when upload has finished. finished = attr.ib(type=bool) # Remaining ranges to upload. @@ -221,7 +222,7 @@ class StorageClientImmutables(object): headers=Headers( { "content-range": [ - ContentRange("bytes", offset, offset+len(data)).to_header() + ContentRange("bytes", offset, offset + len(data)).to_header() ] } ), @@ -268,11 +269,7 @@ class StorageClientImmutables(object): "GET", url, headers=Headers( - { - "range": [ - Range("bytes", [(offset, offset + length)]).to_header() - ] - } + {"range": [Range("bytes", [(offset, offset + length)]).to_header()]} ), ) if response.code == http.PARTIAL_CONTENT: @@ -282,3 +279,23 @@ class StorageClientImmutables(object): raise ClientException( response.code, ) + + @inlineCallbacks + def list_shares(self, storage_index): # type: (bytes,) -> Deferred[Set[int]] + """ + Return the set of shares for a given storage index. + """ + url = self._client._url( + "/v1/immutable/{}/shares".format(_encode_si(storage_index)) + ) + response = yield self._client._request( + "GET", + url, + ) + if response.code == http.OK: + body = yield response.content() + returnValue(set(loads(body))) + else: + raise ClientException( + response.code, + ) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index d79e9a38b..f4ba865ca 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -255,6 +255,22 @@ class HTTPServer(object): required.append({"begin": start, "end": end}) return self._cbor(request, {"required": required}) + @_authorized_route( + _app, + set(), + "/v1/immutable//shares", + methods=["GET"], + ) + def list_shares(self, request, authorization, storage_index): + """ + List shares for the given storage index. + """ + storage_index = si_a2b(storage_index.encode("ascii")) + + # TODO in future ticket, handle KeyError as 404 + share_numbers = list(self._storage_server.get_buckets(storage_index).keys()) + return self._cbor(request, share_numbers) + @_authorized_route( _app, set(), diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index dcefc9950..4ddfff62e 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -382,6 +382,37 @@ class ImmutableHTTPAPITests(SyncTestCase): ) self.assertEqual(downloaded, expected_data[offset : offset + length]) + def test_list_shares(self): + """ + Once a share is finished uploading, it's possible to list it. + """ + im_client = StorageClientImmutables(self.http.client) + upload_secret = urandom(32) + lease_secret = urandom(32) + storage_index = b"".join(bytes([i]) for i in range(16)) + result_of( + im_client.create( + storage_index, {1, 2, 3}, 10, upload_secret, lease_secret, lease_secret + ) + ) + + # Initially there are no shares: + self.assertEqual(result_of(im_client.list_shares(storage_index)), set()) + + # Upload shares 1 and 3: + for share_number in [1, 3]: + progress = result_of(im_client.write_share_chunk( + storage_index, + share_number, + upload_secret, + 0, + b"0123456789", + )) + self.assertTrue(progress.finished) + + # Now shares 1 and 3 exist: + self.assertEqual(result_of(im_client.list_shares(storage_index)), {1, 3}) + def test_multiple_shares_uploaded_to_different_place(self): """ If a storage index has multiple shares, uploads to different shares are From 48a9bf745724c5c5333e46ad48f3ae6c3771c8fc Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 1 Feb 2022 10:25:13 -0500 Subject: [PATCH 0652/2309] Hook up more IStorageServer tests that can now pass with HTTP. --- src/allmydata/storage_client.py | 33 +++++++++++++++++++++-- src/allmydata/test/test_istorageserver.py | 2 -- 2 files changed, 31 insertions(+), 4 deletions(-) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index ca977c9d9..e7dbb27a8 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -1042,7 +1042,7 @@ class _FakeRemoteReference(object): @attr.s class _HTTPBucketWriter(object): """ - Emulate a ``RIBucketWriter``. + Emulate a ``RIBucketWriter``, but use HTTP protocol underneath. """ client = attr.ib(type=StorageClientImmutables) storage_index = attr.ib(type=bytes) @@ -1069,6 +1069,25 @@ class _HTTPBucketWriter(object): return defer.succeed(None) + +@attr.s +class _HTTPBucketReader(object): + """ + Emulate a ``RIBucketReader``. + """ + client = attr.ib(type=StorageClientImmutables) + storage_index = attr.ib(type=bytes) + share_number = attr.ib(type=int) + + def read(self, offset, length): + return self.client.read_share_chunk( + self.storage_index, self.share_number, offset, length + ) + + def advise_corrupt_share(self, reason): + pass # TODO in later ticket + + # WORK IN PROGRESS, for now it doesn't actually implement whole thing. @implementer(IStorageServer) # type: ignore @attr.s @@ -1117,8 +1136,18 @@ class _HTTPStorageServer(object): }) ) + @defer.inlineCallbacks def get_buckets( self, storage_index, ): - pass + immutable_client = StorageClientImmutables(self._http_client) + share_numbers = yield immutable_client.list_shares( + storage_index + ) + defer.returnValue({ + share_num: _FakeRemoteReference(_HTTPBucketReader( + immutable_client, storage_index, share_num + )) + for share_num in share_numbers + }) diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index 96d4f687f..dc2aa0efb 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -1149,8 +1149,6 @@ class HTTPImmutableAPIsTests( "test_get_buckets_skips_unfinished_buckets", "test_matching_overlapping_writes", "test_non_matching_overlapping_writes", - "test_read_bucket_at_offset", - "test_written_shares_are_readable", "test_written_shares_are_allocated", } From 7da506d5d0adbae71fefe87d6981b79540324caf Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 1 Feb 2022 10:26:42 -0500 Subject: [PATCH 0653/2309] News file. --- newsfragments/3871.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3871.minor diff --git a/newsfragments/3871.minor b/newsfragments/3871.minor new file mode 100644 index 000000000..e69de29bb From 0fbf746e27767c548219b23d25d883ca40a6ab82 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 1 Feb 2022 10:30:27 -0500 Subject: [PATCH 0654/2309] Skip on Python 2. --- src/allmydata/test/test_istorageserver.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index 96d4f687f..9b21c6480 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -1089,6 +1089,11 @@ class _FoolscapMixin(_SharedMixin): class _HTTPMixin(_SharedMixin): """Run tests on the HTTP version of ``IStorageServer``.""" + def setUp(self): + if PY2: + self.skipTest("Not going to bother supporting Python 2") + return _SharedMixin.setUp(self) + def _get_istorage_server(self): set_treq_pool(HTTPConnectionPool(reactor, persistent=False)) swissnum = b"1234" From 70d0bd0597b46015c45489ba57b5b566abe5e395 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 1 Feb 2022 10:41:12 -0500 Subject: [PATCH 0655/2309] Test and document what happens for non-existent storage index. --- docs/proposed/http-storage-node-protocol.rst | 3 +++ src/allmydata/storage/http_server.py | 2 -- src/allmydata/test/test_storage_http.py | 8 ++++++++ 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index f47a50d3e..9c09eb362 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -635,6 +635,9 @@ indicated storage index. For example:: [1, 5] +An unknown storage index results in empty list, so that lack of existence of +storage index is not leaked. + ``GET /v1/immutable/:storage_index/:share_number`` !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index f4ba865ca..f885baa22 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -266,8 +266,6 @@ class HTTPServer(object): List shares for the given storage index. """ storage_index = si_a2b(storage_index.encode("ascii")) - - # TODO in future ticket, handle KeyError as 404 share_numbers = list(self._storage_server.get_buckets(storage_index).keys()) return self._cbor(request, share_numbers) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 4ddfff62e..b2def5581 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -413,6 +413,14 @@ class ImmutableHTTPAPITests(SyncTestCase): # Now shares 1 and 3 exist: self.assertEqual(result_of(im_client.list_shares(storage_index)), {1, 3}) + def test_list_shares_unknown_storage_index(self): + """ + Listing unknown storage index's shares results in empty list of shares. + """ + im_client = StorageClientImmutables(self.http.client) + storage_index = b"".join(bytes([i]) for i in range(16)) + self.assertEqual(result_of(im_client.list_shares(storage_index)), set()) + def test_multiple_shares_uploaded_to_different_place(self): """ If a storage index has multiple shares, uploads to different shares are From 7a1f8e64f10f75f26b31d95f5f1ff69b73a64901 Mon Sep 17 00:00:00 2001 From: fenn-cs Date: Wed, 2 Feb 2022 01:33:22 +0100 Subject: [PATCH 0656/2309] remove code-markup around commands Signed-off-by: fenn-cs --- docs/release-checklist.rst | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/docs/release-checklist.rst b/docs/release-checklist.rst index e2002b1fa..9588fd1a5 100644 --- a/docs/release-checklist.rst +++ b/docs/release-checklist.rst @@ -53,16 +53,16 @@ previously generated files. Get into the release directory and install dependencies by running -- ``cd ../tahoe-release-x.x.x`` (assuming you are still in your original clone) -- ``python -m venv venv`` -- ``./venv/bin/pip install --editable .[test]`` +- cd ../tahoe-release-x.x.x (assuming you are still in your original clone) +- python -m venv venv +- ./venv/bin/pip install --editable .[test] Create Branch and Apply Updates ``````````````````````````````` - Create a branch for the release/candidate (e.g. ``XXXX.release-1.16.0``) -- run ``tox -e news`` to produce a new NEWS.txt file (this does a commit) +- run tox -e news to produce a new NEWS.txt file (this does a commit) - create the news for the release - newsfragments/.minor @@ -112,7 +112,7 @@ they will need to evaluate which contributors' signatures they trust. - (all steps above are completed) - sign the release - - ``git tag -s -u 0xE34E62D06D0E69CFCA4179FFBDE0D31D68666A7A -m "release Tahoe-LAFS-1.16.0rc0" tahoe-lafs-1.16.0rc0`` + - git tag -s -u 0xE34E62D06D0E69CFCA4179FFBDE0D31D68666A7A -m "release Tahoe-LAFS-1.16.0rc0" tahoe-lafs-1.16.0rc0 .. note:: - Replace the key-id above with your own, which can simply be your email if it's attached to your fingerprint. @@ -122,11 +122,11 @@ they will need to evaluate which contributors' signatures they trust. - these should all pass: - - ``tox -e py27,codechecks,docs,integration`` + - tox -e py27,codechecks,docs,integration - these can fail (ideally they should not of course): - - ``tox -e deprecations,upcoming-deprecations`` + - tox -e deprecations,upcoming-deprecations - clone to a clean, local checkout (to avoid extra files being included in the release) @@ -138,7 +138,7 @@ they will need to evaluate which contributors' signatures they trust. - tox -e tarballs - Confirm that release tarballs exist by runnig: - - ``ls dist/ | grep 1.16.0rc0`` + - ls dist/ | grep 1.16.0rc0 - inspect and test the tarballs @@ -147,8 +147,8 @@ they will need to evaluate which contributors' signatures they trust. - when satisfied, sign the tarballs: - - ``gpg --pinentry=loopback --armor --detach-sign dist/tahoe_lafs-1.16.0rc0-py2.py3-none-any.whl`` - - ``gpg --pinentry=loopback --armor --detach-sign dist/tahoe_lafs-1.16.0rc0.tar.gz`` + - gpg --pinentry=loopback --armor --detach-sign dist/tahoe_lafs-1.16.0rc0-py2.py3-none-any.whl + - gpg --pinentry=loopback --armor --detach-sign dist/tahoe_lafs-1.16.0rc0.tar.gz Privileged Contributor From aebb5056de20f59f2362431b83514b2a62634f42 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 2 Feb 2022 11:00:16 -0500 Subject: [PATCH 0657/2309] Don't use real reactor in these tests. --- src/allmydata/test/test_storage_http.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index b2def5581..982e22859 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -23,6 +23,7 @@ from treq.testing import StubTreq from klein import Klein from hyperlink import DecodedURL from collections_extended import RangeMap +from twisted.internet.task import Clock from .common import SyncTestCase from ..storage.server import StorageServer @@ -230,8 +231,11 @@ class HttpTestFixture(Fixture): """ def _setUp(self): + self.clock = Clock() self.tempdir = self.useFixture(TempDir()) - self.storage_server = StorageServer(self.tempdir.path, b"\x00" * 20) + self.storage_server = StorageServer( + self.tempdir.path, b"\x00" * 20, clock=self.clock + ) self.http_server = HTTPServer(self.storage_server, SWISSNUM_FOR_TEST) self.client = StorageClient( DecodedURL.from_text("http://127.0.0.1"), @@ -401,13 +405,15 @@ class ImmutableHTTPAPITests(SyncTestCase): # Upload shares 1 and 3: for share_number in [1, 3]: - progress = result_of(im_client.write_share_chunk( + progress = result_of( + im_client.write_share_chunk( storage_index, share_number, upload_secret, 0, b"0123456789", - )) + ) + ) self.assertTrue(progress.finished) # Now shares 1 and 3 exist: From f0c00fcbe41dcde78790dddbcac599699faa5d9c Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 2 Feb 2022 11:04:16 -0500 Subject: [PATCH 0658/2309] News file. --- newsfragments/3860.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3860.minor diff --git a/newsfragments/3860.minor b/newsfragments/3860.minor new file mode 100644 index 000000000..e69de29bb From bceed6e19984418745c077d0f04993fb994b296c Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 2 Feb 2022 11:52:31 -0500 Subject: [PATCH 0659/2309] More bucket allocation logic. --- src/allmydata/storage/http_server.py | 72 +++++++++++++------------ src/allmydata/test/test_storage_http.py | 50 +++++++++++++++++ 2 files changed, 89 insertions(+), 33 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index f885baa22..50aa6ae9e 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -128,10 +128,15 @@ class StorageIndexUploads(object): """ # Map share number to BucketWriter - shares = attr.ib() # type: Dict[int,BucketWriter] + shares = attr.ib(factory=dict) # type: Dict[int,BucketWriter] - # The upload key. - upload_secret = attr.ib() # type: bytes + # Mape share number to the upload secret (different shares might have + # different upload secrets). + upload_secrets = attr.ib(factory=dict) # type: Dict[int,bytes] + + def add_upload(self, share_number, upload_secret, bucket): + self.shares[share_number] = bucket + self.upload_secrets[share_number] = upload_secret class HTTPServer(object): @@ -179,39 +184,40 @@ class HTTPServer(object): def allocate_buckets(self, request, authorization, storage_index): """Allocate buckets.""" storage_index = si_a2b(storage_index.encode("ascii")) - info = loads(request.content.read()) upload_secret = authorization[Secrets.UPLOAD] + info = loads(request.content.read()) if storage_index in self._uploads: - # Pre-existing upload. - in_progress = self._uploads[storage_index] - if timing_safe_compare(in_progress.upload_secret, upload_secret): - # Same session. - # TODO add BucketWriters only for new shares that don't already have buckets; see the HTTP spec for details. - # The backend code may already implement this logic. - pass - else: - # TODO Fail, since the secret doesnt match. - pass - else: - # New upload. - already_got, sharenum_to_bucket = self._storage_server.allocate_buckets( - storage_index, - renew_secret=authorization[Secrets.LEASE_RENEW], - cancel_secret=authorization[Secrets.LEASE_CANCEL], - sharenums=info["share-numbers"], - allocated_size=info["allocated-size"], - ) - self._uploads[storage_index] = StorageIndexUploads( - shares=sharenum_to_bucket, upload_secret=authorization[Secrets.UPLOAD] - ) - return self._cbor( - request, - { - "already-have": set(already_got), - "allocated": set(sharenum_to_bucket), - }, - ) + for share_number in info["share-numbers"]: + in_progress = self._uploads[storage_index] + # For pre-existing upload, make sure password matches. + if ( + share_number in in_progress.upload_secrets + and not timing_safe_compare( + in_progress.upload_secrets[share_number], upload_secret + ) + ): + request.setResponseCode(http.UNAUTHORIZED) + return b"" + + already_got, sharenum_to_bucket = self._storage_server.allocate_buckets( + storage_index, + renew_secret=authorization[Secrets.LEASE_RENEW], + cancel_secret=authorization[Secrets.LEASE_CANCEL], + sharenums=info["share-numbers"], + allocated_size=info["allocated-size"], + ) + uploads = self._uploads.setdefault(storage_index, StorageIndexUploads()) + for share_number, bucket in sharenum_to_bucket.items(): + uploads.add_upload(share_number, upload_secret, bucket) + + return self._cbor( + request, + { + "already-have": set(already_got), + "allocated": set(sharenum_to_bucket), + }, + ) @_authorized_route( _app, diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 982e22859..39f07a54a 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -24,6 +24,7 @@ from klein import Klein from hyperlink import DecodedURL from collections_extended import RangeMap from twisted.internet.task import Clock +from twisted.web import http from .common import SyncTestCase from ..storage.server import StorageServer @@ -386,6 +387,55 @@ class ImmutableHTTPAPITests(SyncTestCase): ) self.assertEqual(downloaded, expected_data[offset : offset + length]) + def test_allocate_buckets_second_time_wrong_upload_key(self): + """ + If allocate buckets endpoint is called second time with wrong upload + key on the same shares, the result is an error. + """ + im_client = StorageClientImmutables(self.http.client) + + # Create a upload: + upload_secret = urandom(32) + lease_secret = urandom(32) + storage_index = b"".join(bytes([i]) for i in range(16)) + result_of( + im_client.create( + storage_index, {1, 2, 3}, 100, upload_secret, lease_secret, lease_secret + ) + ) + with self.assertRaises(ClientException) as e: + result_of( + im_client.create( + storage_index, {2, 3}, 100, b"x" * 32, lease_secret, lease_secret + ) + ) + self.assertEqual(e.exception.args[0], http.UNAUTHORIZED) + + def test_allocate_buckets_second_time_different_shares(self): + """ + If allocate buckets endpoint is called second time with different + upload key on different shares, that creates the buckets. + """ + im_client = StorageClientImmutables(self.http.client) + + # Create a upload: + upload_secret = urandom(32) + lease_secret = urandom(32) + storage_index = b"".join(bytes([i]) for i in range(16)) + result_of( + im_client.create( + storage_index, {1, 2, 3}, 100, upload_secret, lease_secret, lease_secret + ) + ) + + # Add same shares: + created2 = result_of( + im_client.create( + storage_index, {4, 6}, 100, b"x" * 2, lease_secret, lease_secret + ) + ) + self.assertEqual(created2.allocated, {4, 6}) + def test_list_shares(self): """ Once a share is finished uploading, it's possible to list it. From 39fe48b1746d5d854cdd87129112430332265bba Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 2 Feb 2022 12:55:41 -0500 Subject: [PATCH 0660/2309] More passing IStorageServer tests. --- src/allmydata/storage_client.py | 10 ++++++---- src/allmydata/test/test_istorageserver.py | 3 --- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index e7dbb27a8..665cfd513 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -1094,15 +1094,18 @@ class _HTTPBucketReader(object): class _HTTPStorageServer(object): """ Talk to remote storage server over HTTP. + + The same upload key is used for all communication. """ _http_client = attr.ib(type=StorageClient) + _upload_secret = attr.ib(type=bytes) @staticmethod def from_http_client(http_client): # type: (StorageClient) -> _HTTPStorageServer """ Create an ``IStorageServer`` from a HTTP ``StorageClient``. """ - return _HTTPStorageServer(http_client=http_client) + return _HTTPStorageServer(http_client=http_client, upload_secret=urandom(20)) def get_version(self): return self._http_client.get_version() @@ -1117,10 +1120,9 @@ class _HTTPStorageServer(object): allocated_size, canary, ): - upload_secret = urandom(20) immutable_client = StorageClientImmutables(self._http_client) result = immutable_client.create( - storage_index, sharenums, allocated_size, upload_secret, renew_secret, + storage_index, sharenums, allocated_size, self._upload_secret, renew_secret, cancel_secret ) result = yield result @@ -1130,7 +1132,7 @@ class _HTTPStorageServer(object): client=immutable_client, storage_index=storage_index, share_number=share_num, - upload_secret=upload_secret + upload_secret=self._upload_secret )) for share_num in result.allocated }) diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index cf1d977d8..679b0f964 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -1148,13 +1148,10 @@ class HTTPImmutableAPIsTests( "test_add_lease_renewal", "test_add_new_lease", "test_advise_corrupt_share", - "test_allocate_buckets_repeat", "test_bucket_advise_corrupt_share", "test_disconnection", - "test_get_buckets_skips_unfinished_buckets", "test_matching_overlapping_writes", "test_non_matching_overlapping_writes", - "test_written_shares_are_allocated", } From 1dfc0bde3679c7eaee8617b2dd9008ef8ae37053 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 3 Feb 2022 12:43:49 -0500 Subject: [PATCH 0661/2309] Use better method to listen on random port. --- src/allmydata/test/test_istorageserver.py | 39 +++++++++++++++-------- 1 file changed, 25 insertions(+), 14 deletions(-) diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index 9b21c6480..2a875b120 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -25,9 +25,10 @@ else: from random import Random from unittest import SkipTest -from twisted.internet.defer import inlineCallbacks, returnValue +from twisted.internet.defer import inlineCallbacks, returnValue, succeed from twisted.internet.task import Clock from twisted.internet import reactor +from twisted.internet.endpoints import serverFromString from twisted.web.server import Site from twisted.web.client import HTTPConnectionPool from hyperlink import DecodedURL @@ -37,11 +38,10 @@ from foolscap.api import Referenceable, RemoteException from allmydata.interfaces import IStorageServer # really, IStorageClient from .common_system import SystemTestMixin -from .common import AsyncTestCase +from .common import AsyncTestCase, SameProcessStreamEndpointAssigner from allmydata.storage.server import StorageServer # not a IStorageServer!! from allmydata.storage.http_server import HTTPServer from allmydata.storage.http_client import StorageClient -from allmydata.util.iputil import allocate_tcp_port from allmydata.storage_client import _HTTPStorageServer @@ -1029,6 +1029,11 @@ class _SharedMixin(SystemTestMixin): ) AsyncTestCase.setUp(self) + + self._port_assigner = SameProcessStreamEndpointAssigner() + self._port_assigner.setUp() + self.addCleanup(self._port_assigner.tearDown) + self.basedir = "test_istorageserver/" + self.id() yield SystemTestMixin.setUp(self) yield self.set_up_nodes(1) @@ -1041,7 +1046,7 @@ class _SharedMixin(SystemTestMixin): self._clock = Clock() self._clock.advance(123456) self.server._clock = self._clock - self.storage_client = self._get_istorage_server() + self.storage_client = yield self._get_istorage_server() def fake_time(self): """Return the current fake, test-controlled, time.""" @@ -1073,7 +1078,7 @@ class _FoolscapMixin(_SharedMixin): def _get_istorage_server(self): client = self._get_native_server().get_storage_server() self.assertTrue(IStorageServer.providedBy(client)) - return client + return succeed(client) @inlineCallbacks def disconnect(self): @@ -1094,20 +1099,26 @@ class _HTTPMixin(_SharedMixin): self.skipTest("Not going to bother supporting Python 2") return _SharedMixin.setUp(self) + @inlineCallbacks def _get_istorage_server(self): set_treq_pool(HTTPConnectionPool(reactor, persistent=False)) swissnum = b"1234" self._http_storage_server = HTTPServer(self.server, swissnum) - self._port_number = allocate_tcp_port() - self._listening_port = reactor.listenTCP( - self._port_number, - Site(self._http_storage_server.get_resource()), - interface="127.0.0.1", + + # Listen on randomly assigned port: + tcp_address, endpoint_string = self._port_assigner.assign(reactor) + _, host, port = tcp_address.split(":") + port = int(port) + endpoint = serverFromString(reactor, endpoint_string) + self._listening_port = yield endpoint.listen( + Site(self._http_storage_server.get_resource()) ) - return _HTTPStorageServer.from_http_client( - StorageClient( - DecodedURL.from_text("http://127.0.0.1:{}".format(self._port_number)), - swissnum, + returnValue( + _HTTPStorageServer.from_http_client( + StorageClient( + DecodedURL().replace(scheme="http", host=host, port=port), + swissnum, + ) ) ) # Eventually should also: From 23c8bde9d597b83cc3f89ac9a781f3e385961cea Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 3 Feb 2022 12:44:55 -0500 Subject: [PATCH 0662/2309] Nicer cleanup. --- src/allmydata/test/test_istorageserver.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index 2a875b120..cfd81feda 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -1113,6 +1113,7 @@ class _HTTPMixin(_SharedMixin): self._listening_port = yield endpoint.listen( Site(self._http_storage_server.get_resource()) ) + self.addCleanup(self._listening_port.stopListening) returnValue( _HTTPStorageServer.from_http_client( StorageClient( @@ -1124,11 +1125,6 @@ class _HTTPMixin(_SharedMixin): # Eventually should also: # self.assertTrue(IStorageServer.providedBy(client)) - @inlineCallbacks - def tearDown(self): - yield _SharedMixin.tearDown(self) - yield self._listening_port.stopListening() - class FoolscapSharedAPIsTests( _FoolscapMixin, IStorageServerSharedAPIsTestsMixin, AsyncTestCase From 6b3722d3f664d9274ce70812a485eb99a8e5f5a2 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 3 Feb 2022 12:50:29 -0500 Subject: [PATCH 0663/2309] Avoid using possibly-private API. --- src/allmydata/test/test_istorageserver.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index cfd81feda..7ed8a1b65 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -30,9 +30,9 @@ from twisted.internet.task import Clock from twisted.internet import reactor from twisted.internet.endpoints import serverFromString from twisted.web.server import Site -from twisted.web.client import HTTPConnectionPool +from twisted.web.client import Agent, HTTPConnectionPool from hyperlink import DecodedURL -from treq.api import set_global_pool as set_treq_pool +from treq.client import HTTPClient from foolscap.api import Referenceable, RemoteException @@ -1101,24 +1101,29 @@ class _HTTPMixin(_SharedMixin): @inlineCallbacks def _get_istorage_server(self): - set_treq_pool(HTTPConnectionPool(reactor, persistent=False)) swissnum = b"1234" - self._http_storage_server = HTTPServer(self.server, swissnum) + http_storage_server = HTTPServer(self.server, swissnum) # Listen on randomly assigned port: tcp_address, endpoint_string = self._port_assigner.assign(reactor) _, host, port = tcp_address.split(":") port = int(port) endpoint = serverFromString(reactor, endpoint_string) - self._listening_port = yield endpoint.listen( - Site(self._http_storage_server.get_resource()) + listening_port = yield endpoint.listen(Site(http_storage_server.get_resource())) + self.addCleanup(listening_port.stopListening) + + # Create HTTP client with non-persistent connections, so we don't leak + # state across tests: + treq_client = HTTPClient( + Agent(reactor, HTTPConnectionPool(reactor, persistent=False)) ) - self.addCleanup(self._listening_port.stopListening) + returnValue( _HTTPStorageServer.from_http_client( StorageClient( DecodedURL().replace(scheme="http", host=host, port=port), swissnum, + treq=treq_client, ) ) ) From c2c3411dc40155e31743089d99465ad9041cdafc Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 3 Feb 2022 12:57:48 -0500 Subject: [PATCH 0664/2309] Try to fix Python 2. --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5ae70a3bb..f65890d37 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -217,7 +217,7 @@ jobs: fetch-depth: 0 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v1 + uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} @@ -277,7 +277,7 @@ jobs: fetch-depth: 0 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v1 + uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} From 7454929be0d761b87909f04e26a9cf61d85a5702 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 4 Feb 2022 09:26:25 -0500 Subject: [PATCH 0665/2309] Less code duplication. --- src/allmydata/storage/http_client.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 8a1d03192..2dc133b52 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -238,7 +238,7 @@ class StorageClientImmutables(object): raise ClientException( response.code, ) - body = loads((yield response.content())) + body = yield _decode_cbor(response) remaining = RangeMap() for chunk in body["required"]: remaining.set(True, chunk["begin"], chunk["end"]) @@ -293,8 +293,8 @@ class StorageClientImmutables(object): url, ) if response.code == http.OK: - body = yield response.content() - returnValue(set(loads(body))) + body = yield _decode_cbor(response) + returnValue(set(body)) else: raise ClientException( response.code, From 5e3a31166d7782e8b84094b58fe4bca40faf1995 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 4 Feb 2022 09:26:58 -0500 Subject: [PATCH 0666/2309] Better explanation. --- src/allmydata/storage_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index e7dbb27a8..cf5fb65a2 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -1073,7 +1073,7 @@ class _HTTPBucketWriter(object): @attr.s class _HTTPBucketReader(object): """ - Emulate a ``RIBucketReader``. + Emulate a ``RIBucketReader``, but use HTTP protocol underneath. """ client = attr.ib(type=StorageClientImmutables) storage_index = attr.ib(type=bytes) From 83d8f2eb78f1c9da1eadf97429e9f0eac12deafd Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 4 Feb 2022 09:27:29 -0500 Subject: [PATCH 0667/2309] Remove incorrect editorial. --- docs/proposed/http-storage-node-protocol.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index 9c09eb362..b00b327e3 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -635,8 +635,7 @@ indicated storage index. For example:: [1, 5] -An unknown storage index results in empty list, so that lack of existence of -storage index is not leaked. +An unknown storage index results in an empty list. ``GET /v1/immutable/:storage_index/:share_number`` !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! From ce2468cdffb56705c7f9edde4581b7b675716117 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 4 Feb 2022 09:53:39 -0500 Subject: [PATCH 0668/2309] Validate inputs automatically as part of parsing. --- src/allmydata/storage/http_server.py | 27 +++++++++------ src/allmydata/test/test_storage_http.py | 45 +++++++++++++++++++++++++ 2 files changed, 62 insertions(+), 10 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 50aa6ae9e..cf8284e3e 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -23,6 +23,7 @@ from klein import Klein from twisted.web import http import attr from werkzeug.http import parse_range_header, parse_content_range_header +from werkzeug.routing import BaseConverter # TODO Make sure to use pure Python versions? from cbor2 import dumps, loads @@ -32,6 +33,7 @@ from .http_common import swissnum_auth_header, Secrets from .common import si_a2b from .immutable import BucketWriter from ..util.hashutil import timing_safe_compare +from ..util.base32 import rfc3548_alphabet class ClientSecretsException(Exception): @@ -139,12 +141,22 @@ class StorageIndexUploads(object): self.upload_secrets[share_number] = upload_secret +class StorageIndexConverter(BaseConverter): + """Parser/validator for storage index URL path segments.""" + + regex = "[" + str(rfc3548_alphabet, "ascii") + "]{26}" + + def to_python(self, value): + return si_a2b(value.encode("ascii")) + + class HTTPServer(object): """ A HTTP interface to the storage server. """ _app = Klein() + _app.url_map.converters["storage_index"] = StorageIndexConverter def __init__( self, storage_server, swissnum @@ -178,12 +190,11 @@ class HTTPServer(object): @_authorized_route( _app, {Secrets.LEASE_RENEW, Secrets.LEASE_CANCEL, Secrets.UPLOAD}, - "/v1/immutable/", + "/v1/immutable/", methods=["POST"], ) def allocate_buckets(self, request, authorization, storage_index): """Allocate buckets.""" - storage_index = si_a2b(storage_index.encode("ascii")) upload_secret = authorization[Secrets.UPLOAD] info = loads(request.content.read()) @@ -222,12 +233,11 @@ class HTTPServer(object): @_authorized_route( _app, {Secrets.UPLOAD}, - "/v1/immutable//", + "/v1/immutable//", methods=["PATCH"], ) def write_share_data(self, request, authorization, storage_index, share_number): """Write data to an in-progress immutable upload.""" - storage_index = si_a2b(storage_index.encode("ascii")) content_range = parse_content_range_header(request.getHeader("content-range")) # TODO in https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3860 # 1. Malformed header should result in error 416 @@ -236,7 +246,7 @@ class HTTPServer(object): # 4. Impossible range should resul tin error 416 offset = content_range.start - # TODO basic checks on validity of start, offset, and content-range in general. also of share_number. + # TODO basic checks on validity of start, offset, and content-range in general. # TODO basic check that body isn't infinite. require content-length? or maybe we should require content-range (it's optional now)? if so, needs to be rflected in protocol spec. data = request.content.read() @@ -264,34 +274,31 @@ class HTTPServer(object): @_authorized_route( _app, set(), - "/v1/immutable//shares", + "/v1/immutable//shares", methods=["GET"], ) def list_shares(self, request, authorization, storage_index): """ List shares for the given storage index. """ - storage_index = si_a2b(storage_index.encode("ascii")) share_numbers = list(self._storage_server.get_buckets(storage_index).keys()) return self._cbor(request, share_numbers) @_authorized_route( _app, set(), - "/v1/immutable//", + "/v1/immutable//", methods=["GET"], ) def read_share_chunk(self, request, authorization, storage_index, share_number): """Read a chunk for an already uploaded immutable.""" # TODO in https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3860 - # 1. basic checks on validity on storage index, share number # 2. missing range header should have response code 200 and return whole thing # 3. malformed range header should result in error? or return everything? # 4. non-bytes range results in error # 5. ranges make sense semantically (positive, etc.) # 6. multiple ranges fails with error # 7. missing end of range means "to the end of share" - storage_index = si_a2b(storage_index.encode("ascii")) range_header = parse_range_header(request.getHeader("range")) offset, end = range_header.ranges[0] assert end != None # TODO support this case diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 39f07a54a..f78d83b27 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -25,6 +25,8 @@ from hyperlink import DecodedURL from collections_extended import RangeMap from twisted.internet.task import Clock from twisted.web import http +from werkzeug import routing +from werkzeug.exceptions import NotFound as WNotFound from .common import SyncTestCase from ..storage.server import StorageServer @@ -34,6 +36,7 @@ from ..storage.http_server import ( Secrets, ClientSecretsException, _authorized_route, + StorageIndexConverter, ) from ..storage.http_client import ( StorageClient, @@ -42,6 +45,7 @@ from ..storage.http_client import ( ImmutableCreateResult, UploadProgress, ) +from ..storage.common import si_b2a def _post_process(params): @@ -148,6 +152,47 @@ class ExtractSecretsTests(SyncTestCase): _extract_secrets(["lease-cancel-secret eA=="], {Secrets.LEASE_RENEW}) +class RouteConverterTests(SyncTestCase): + """Tests for custom werkzeug path segment converters.""" + + adapter = routing.Map( + [ + routing.Rule( + "//", endpoint="si", methods=["GET"] + ) + ], + converters={"storage_index": StorageIndexConverter}, + ).bind("example.com", "/") + + @given(storage_index=st.binary(min_size=16, max_size=16)) + def test_good_storage_index_is_parsed(self, storage_index): + """ + A valid storage index is accepted and parsed back out by + StorageIndexConverter. + """ + self.assertEqual( + self.adapter.match( + "/{}/".format(str(si_b2a(storage_index), "ascii")), method="GET" + ), + ("si", {"storage_index": storage_index}), + ) + + def test_long_storage_index_is_not_parsed(self): + """An overly long storage_index string is not parsed.""" + with self.assertRaises(WNotFound): + self.adapter.match("/{}/".format("a" * 27), method="GET") + + def test_short_storage_index_is_not_parsed(self): + """An overly short storage_index string is not parsed.""" + with self.assertRaises(WNotFound): + self.adapter.match("/{}/".format("a" * 25), method="GET") + + def test_bad_characters_storage_index_is_not_parsed(self): + """An overly short storage_index string is not parsed.""" + with self.assertRaises(WNotFound): + self.adapter.match("/{}_/".format("a" * 25), method="GET") + + # TODO should be actual swissnum SWISSNUM_FOR_TEST = b"abcd" From 7107a85fba4bb7c8e6998d48442a1caac987279a Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 8 Feb 2022 10:19:37 -0500 Subject: [PATCH 0669/2309] Refactor client, separating low-level and high-level concerns. --- src/allmydata/storage/http_client.py | 35 ++++++++++++++++--------- src/allmydata/test/test_storage_http.py | 18 ++++++++----- 2 files changed, 33 insertions(+), 20 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 2dc133b52..3b1dbfb30 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -68,7 +68,7 @@ class ImmutableCreateResult(object): class StorageClient(object): """ - HTTP client that talks to the HTTP storage server. + Low-level HTTP client that talks to the HTTP storage server. """ def __init__( @@ -78,7 +78,7 @@ class StorageClient(object): self._swissnum = swissnum self._treq = treq - def _url(self, path): + def relative_url(self, path): """Get a URL relative to the base URL.""" return self._base_url.click(path) @@ -92,7 +92,7 @@ class StorageClient(object): ) return headers - def _request( + def request( self, method, url, @@ -120,13 +120,22 @@ class StorageClient(object): ) return self._treq.request(method, url, headers=headers, **kwargs) + +class StorageClientGeneral(object): + """ + High-level HTTP APIs that aren't immutable- or mutable-specific. + """ + + def __init__(self, client): # type: (StorageClient) -> None + self._client = client + @inlineCallbacks def get_version(self): """ Return the version metadata for the server. """ - url = self._url("/v1/version") - response = yield self._request("GET", url) + url = self._client.relative_url("/v1/version") + response = yield self._client.request("GET", url) decoded_response = yield _decode_cbor(response) returnValue(decoded_response) @@ -174,11 +183,11 @@ class StorageClientImmutables(object): Result fires when creating the storage index succeeded, if creating the storage index failed the result will fire with an exception. """ - url = self._client._url("/v1/immutable/" + _encode_si(storage_index)) + url = self._client.relative_url("/v1/immutable/" + _encode_si(storage_index)) message = dumps( {"share-numbers": share_numbers, "allocated-size": allocated_size} ) - response = yield self._client._request( + response = yield self._client.request( "POST", url, lease_renew_secret=lease_renew_secret, @@ -211,10 +220,10 @@ class StorageClientImmutables(object): whether the _complete_ share (i.e. all chunks, not just this one) has been uploaded. """ - url = self._client._url( + url = self._client.relative_url( "/v1/immutable/{}/{}".format(_encode_si(storage_index), share_number) ) - response = yield self._client._request( + response = yield self._client.request( "PATCH", url, upload_secret=upload_secret, @@ -262,10 +271,10 @@ class StorageClientImmutables(object): the HTTP protocol will be simplified, see https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3777 """ - url = self._client._url( + url = self._client.relative_url( "/v1/immutable/{}/{}".format(_encode_si(storage_index), share_number) ) - response = yield self._client._request( + response = yield self._client.request( "GET", url, headers=Headers( @@ -285,10 +294,10 @@ class StorageClientImmutables(object): """ Return the set of shares for a given storage index. """ - url = self._client._url( + url = self._client.relative_url( "/v1/immutable/{}/shares".format(_encode_si(storage_index)) ) - response = yield self._client._request( + response = yield self._client.request( "GET", url, ) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index f78d83b27..41391845f 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -44,6 +44,7 @@ from ..storage.http_client import ( StorageClientImmutables, ImmutableCreateResult, UploadProgress, + StorageClientGeneral, ) from ..storage.common import si_b2a @@ -253,7 +254,7 @@ class RoutingTests(SyncTestCase): """ # Without secret, get a 400 error. response = result_of( - self.client._request( + self.client.request( "GET", "http://127.0.0.1/upload_secret", ) @@ -262,7 +263,7 @@ class RoutingTests(SyncTestCase): # With secret, we're good. response = result_of( - self.client._request( + self.client.request( "GET", "http://127.0.0.1/upload_secret", upload_secret=b"MAGIC" ) ) @@ -307,10 +308,12 @@ class GenericHTTPAPITests(SyncTestCase): If the wrong swissnum is used, an ``Unauthorized`` response code is returned. """ - client = StorageClient( - DecodedURL.from_text("http://127.0.0.1"), - b"something wrong", - treq=StubTreq(self.http.http_server.get_resource()), + client = StorageClientGeneral( + StorageClient( + DecodedURL.from_text("http://127.0.0.1"), + b"something wrong", + treq=StubTreq(self.http.http_server.get_resource()), + ) ) with self.assertRaises(ClientException) as e: result_of(client.get_version()) @@ -323,7 +326,8 @@ class GenericHTTPAPITests(SyncTestCase): We ignore available disk space and max immutable share size, since that might change across calls. """ - version = result_of(self.http.client.get_version()) + client = StorageClientGeneral(self.http.client) + version = result_of(client.get_version()) version[b"http://allmydata.org/tahoe/protocols/storage/v1"].pop( b"available-space" ) From d38183335eb7ca8bbcb3d9b1a7cd57a898061ae6 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 8 Feb 2022 10:46:55 -0500 Subject: [PATCH 0670/2309] Handle bad Content-Range headers. --- src/allmydata/storage/http_client.py | 6 ++- src/allmydata/storage/http_server.py | 12 +++--- src/allmydata/test/test_storage_http.py | 57 +++++++++++++++++++++++++ 3 files changed, 68 insertions(+), 7 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 3b1dbfb30..475aa2330 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -48,7 +48,11 @@ def _encode_si(si): # type: (bytes) -> str class ClientException(Exception): - """An unexpected error.""" + """An unexpected response code from the server.""" + + def __init__(self, code, *additional_args): + Exception.__init__(self, code, *additional_args) + self.code = code def _decode_cbor(response): diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index cf8284e3e..ad1b8b2ac 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -239,14 +239,14 @@ class HTTPServer(object): def write_share_data(self, request, authorization, storage_index, share_number): """Write data to an in-progress immutable upload.""" content_range = parse_content_range_header(request.getHeader("content-range")) - # TODO in https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3860 - # 1. Malformed header should result in error 416 - # 2. Non-bytes unit should result in error 416 - # 3. Missing header means full upload in one request - # 4. Impossible range should resul tin error 416 + if content_range is None or content_range.units != "bytes": + # TODO Missing header means full upload in one request + request.setResponseCode(http.REQUESTED_RANGE_NOT_SATISFIABLE) + return b"" + offset = content_range.start - # TODO basic checks on validity of start, offset, and content-range in general. + # TODO basic check that body isn't infinite. require content-length? or maybe we should require content-range (it's optional now)? if so, needs to be rflected in protocol spec. data = request.content.read() diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 41391845f..7e459ee2a 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -25,6 +25,7 @@ from hyperlink import DecodedURL from collections_extended import RangeMap from twisted.internet.task import Clock from twisted.web import http +from twisted.web.http_headers import Headers from werkzeug import routing from werkzeug.exceptions import NotFound as WNotFound @@ -291,6 +292,24 @@ class HttpTestFixture(Fixture): ) +class StorageClientWithHeadersOverride(object): + """Wrap ``StorageClient`` and override sent headers.""" + + def __init__(self, storage_client, add_headers): + self.storage_client = storage_client + self.add_headers = add_headers + + def __getattr__(self, attr): + return getattr(self.storage_client, attr) + + def request(self, *args, headers=None, **kwargs): + if headers is None: + headers = Headers() + for key, value in self.add_headers.items(): + headers.setRawHeaders(key, [value]) + return self.storage_client.request(*args, headers=headers, **kwargs) + + class GenericHTTPAPITests(SyncTestCase): """ Tests of HTTP client talking to the HTTP server, for generic HTTP API @@ -518,6 +537,44 @@ class ImmutableHTTPAPITests(SyncTestCase): # Now shares 1 and 3 exist: self.assertEqual(result_of(im_client.list_shares(storage_index)), {1, 3}) + def test_upload_bad_content_range(self): + """ + Malformed or invalid Content-Range headers to the immutable upload + endpoint result in a 416 error. + """ + im_client = StorageClientImmutables(self.http.client) + upload_secret = urandom(32) + lease_secret = urandom(32) + storage_index = b"0" * 16 + result_of( + im_client.create( + storage_index, {1}, 10, upload_secret, lease_secret, lease_secret + ) + ) + + def check_invalid(bad_content_range_value): + client = StorageClientImmutables( + StorageClientWithHeadersOverride( + self.http.client, {"content-range": bad_content_range_value} + ) + ) + with self.assertRaises(ClientException) as e: + result_of( + client.write_share_chunk( + storage_index, + 1, + upload_secret, + 0, + b"0123456789", + ) + ) + self.assertEqual(e.exception.code, http.REQUESTED_RANGE_NOT_SATISFIABLE) + + check_invalid("not a valid content-range header at all") + check_invalid("bytes -1-9/10") + check_invalid("bytes 0--9/10") + check_invalid("teapots 0-9/10") + def test_list_shares_unknown_storage_index(self): """ Listing unknown storage index's shares results in empty list of shares. From ecb1a3c5a0bd63a6c36be5251b4881771ab65586 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 9 Feb 2022 12:25:47 -0500 Subject: [PATCH 0671/2309] Just require content-range for simplicity. --- docs/proposed/http-storage-node-protocol.rst | 2 +- src/allmydata/storage/http_server.py | 9 +++------ 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index b00b327e3..4d8d60560 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -540,7 +540,7 @@ Rejected designs for upload secrets: Write data for the indicated share. The share number must belong to the storage index. The request body is the raw share data (i.e., ``application/octet-stream``). -*Content-Range* requests are encouraged for large transfers to allow partially complete uploads to be resumed. +*Content-Range* requests are required; for large transfers this allows partially complete uploads to be resumed. For example, a 1MiB share can be divided in to eight separate 128KiB chunks. Each chunk can be uploaded in a separate request. diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index ad1b8b2ac..708b99380 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -240,16 +240,13 @@ class HTTPServer(object): """Write data to an in-progress immutable upload.""" content_range = parse_content_range_header(request.getHeader("content-range")) if content_range is None or content_range.units != "bytes": - # TODO Missing header means full upload in one request request.setResponseCode(http.REQUESTED_RANGE_NOT_SATISFIABLE) return b"" - + offset = content_range.start - - # TODO basic check that body isn't infinite. require content-length? or maybe we should require content-range (it's optional now)? if so, needs to be rflected in protocol spec. - - data = request.content.read() + # TODO limit memory usage + data = request.content.read(content_range.stop - content_range.start + 1) try: bucket = self._uploads[storage_index].shares[share_number] except (KeyError, IndexError): From 72bac785ee3b21d4d7bd9cb8f71df76ae3fd35a5 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 9 Feb 2022 12:27:08 -0500 Subject: [PATCH 0672/2309] Done elsewhere. --- src/allmydata/test/test_storage_http.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 7e459ee2a..ae316708c 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -615,13 +615,6 @@ class ImmutableHTTPAPITests(SyncTestCase): TBD in https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3860 """ - def test_upload_offset_cannot_be_negative(self): - """ - A negative upload offset will be rejected. - - TBD in https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3860 - """ - def test_mismatching_upload_fails(self): """ If an uploaded chunk conflicts with an already uploaded chunk, a From 95d7548629e6a96a2a2e6f57dbc8dac2015caeaf Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 9 Feb 2022 12:30:38 -0500 Subject: [PATCH 0673/2309] Upload to non-existent place. --- src/allmydata/storage/http_server.py | 4 +-- src/allmydata/test/test_storage_http.py | 33 +++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 708b99380..6f9033380 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -250,8 +250,8 @@ class HTTPServer(object): try: bucket = self._uploads[storage_index].shares[share_number] except (KeyError, IndexError): - # TODO return 404 - raise + request.setResponseCode(http.NOT_FOUND) + return b"" finished = bucket.write(offset, data) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index ae316708c..7a5c24905 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -583,6 +583,39 @@ class ImmutableHTTPAPITests(SyncTestCase): storage_index = b"".join(bytes([i]) for i in range(16)) self.assertEqual(result_of(im_client.list_shares(storage_index)), set()) + def test_upload_non_existent_storage_index(self): + """ + Uploading to a non-existent storage index or share number results in + 404. + """ + im_client = StorageClientImmutables(self.http.client) + upload_secret = urandom(32) + lease_secret = urandom(32) + storage_index = b"".join(bytes([i]) for i in range(16)) + result_of( + im_client.create( + storage_index, {1}, 10, upload_secret, lease_secret, lease_secret + ) + ) + + def unknown_check(storage_index, share_number): + with self.assertRaises(ClientException) as e: + result_of( + im_client.write_share_chunk( + storage_index, + share_number, + upload_secret, + 0, + b"0123456789", + ) + ) + self.assertEqual(e.exception.code, http.NOT_FOUND) + + # Wrong share number: + unknown_check(storage_index, 7) + # Wrong storage index: + unknown_check(b"X" * 16, 7) + def test_multiple_shares_uploaded_to_different_place(self): """ If a storage index has multiple shares, uploads to different shares are From 8c739343f3f21d3b8ad112a87eca582e03638967 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 9 Feb 2022 12:37:49 -0500 Subject: [PATCH 0674/2309] Reduce duplication. --- src/allmydata/test/test_storage_http.py | 104 +++++++++--------------- 1 file changed, 39 insertions(+), 65 deletions(-) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 7a5c24905..2bad160e4 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -373,6 +373,28 @@ class ImmutableHTTPAPITests(SyncTestCase): self.skipTest("Not going to bother supporting Python 2") super(ImmutableHTTPAPITests, self).setUp() self.http = self.useFixture(HttpTestFixture()) + self.im_client = StorageClientImmutables(self.http.client) + + def create_upload(self, share_numbers, length): + """ + Create a write bucket on server, return: + + (upload_secret, lease_secret, storage_index, result) + """ + upload_secret = urandom(32) + lease_secret = urandom(32) + storage_index = urandom(16) + created = result_of( + self.im_client.create( + storage_index, + share_numbers, + length, + upload_secret, + lease_secret, + lease_secret, + ) + ) + return (upload_secret, lease_secret, storage_index, created) def test_upload_can_be_downloaded(self): """ @@ -386,17 +408,8 @@ class ImmutableHTTPAPITests(SyncTestCase): length = 100 expected_data = b"".join(bytes([i]) for i in range(100)) - im_client = StorageClientImmutables(self.http.client) - # Create a upload: - upload_secret = urandom(32) - lease_secret = urandom(32) - storage_index = b"".join(bytes([i]) for i in range(16)) - created = result_of( - im_client.create( - storage_index, {1}, 100, upload_secret, lease_secret, lease_secret - ) - ) + (upload_secret, _, storage_index, created) = self.create_upload({1}, 100) self.assertEqual( created, ImmutableCreateResult(already_have=set(), allocated={1}) ) @@ -407,7 +420,7 @@ class ImmutableHTTPAPITests(SyncTestCase): # Three writes: 10-19, 30-39, 50-59. This allows for a bunch of holes. def write(offset, length): remaining.empty(offset, offset + length) - return im_client.write_share_chunk( + return self.im_client.write_share_chunk( storage_index, 1, upload_secret, @@ -451,7 +464,7 @@ class ImmutableHTTPAPITests(SyncTestCase): # We can now read: for offset, length in [(0, 100), (10, 19), (99, 1), (49, 200)]: downloaded = result_of( - im_client.read_share_chunk(storage_index, 1, offset, length) + self.im_client.read_share_chunk(storage_index, 1, offset, length) ) self.assertEqual(downloaded, expected_data[offset : offset + length]) @@ -460,20 +473,13 @@ class ImmutableHTTPAPITests(SyncTestCase): If allocate buckets endpoint is called second time with wrong upload key on the same shares, the result is an error. """ - im_client = StorageClientImmutables(self.http.client) - # Create a upload: - upload_secret = urandom(32) - lease_secret = urandom(32) - storage_index = b"".join(bytes([i]) for i in range(16)) - result_of( - im_client.create( - storage_index, {1, 2, 3}, 100, upload_secret, lease_secret, lease_secret - ) + (upload_secret, lease_secret, storage_index, _) = self.create_upload( + {1, 2, 3}, 100 ) with self.assertRaises(ClientException) as e: result_of( - im_client.create( + self.im_client.create( storage_index, {2, 3}, 100, b"x" * 32, lease_secret, lease_secret ) ) @@ -484,21 +490,14 @@ class ImmutableHTTPAPITests(SyncTestCase): If allocate buckets endpoint is called second time with different upload key on different shares, that creates the buckets. """ - im_client = StorageClientImmutables(self.http.client) - # Create a upload: - upload_secret = urandom(32) - lease_secret = urandom(32) - storage_index = b"".join(bytes([i]) for i in range(16)) - result_of( - im_client.create( - storage_index, {1, 2, 3}, 100, upload_secret, lease_secret, lease_secret - ) + (upload_secret, lease_secret, storage_index, created) = self.create_upload( + {1, 2, 3}, 100 ) # Add same shares: created2 = result_of( - im_client.create( + self.im_client.create( storage_index, {4, 6}, 100, b"x" * 2, lease_secret, lease_secret ) ) @@ -508,23 +507,15 @@ class ImmutableHTTPAPITests(SyncTestCase): """ Once a share is finished uploading, it's possible to list it. """ - im_client = StorageClientImmutables(self.http.client) - upload_secret = urandom(32) - lease_secret = urandom(32) - storage_index = b"".join(bytes([i]) for i in range(16)) - result_of( - im_client.create( - storage_index, {1, 2, 3}, 10, upload_secret, lease_secret, lease_secret - ) - ) + (upload_secret, _, storage_index, created) = self.create_upload({1, 2, 3}, 10) # Initially there are no shares: - self.assertEqual(result_of(im_client.list_shares(storage_index)), set()) + self.assertEqual(result_of(self.im_client.list_shares(storage_index)), set()) # Upload shares 1 and 3: for share_number in [1, 3]: progress = result_of( - im_client.write_share_chunk( + self.im_client.write_share_chunk( storage_index, share_number, upload_secret, @@ -535,22 +526,14 @@ class ImmutableHTTPAPITests(SyncTestCase): self.assertTrue(progress.finished) # Now shares 1 and 3 exist: - self.assertEqual(result_of(im_client.list_shares(storage_index)), {1, 3}) + self.assertEqual(result_of(self.im_client.list_shares(storage_index)), {1, 3}) def test_upload_bad_content_range(self): """ Malformed or invalid Content-Range headers to the immutable upload endpoint result in a 416 error. """ - im_client = StorageClientImmutables(self.http.client) - upload_secret = urandom(32) - lease_secret = urandom(32) - storage_index = b"0" * 16 - result_of( - im_client.create( - storage_index, {1}, 10, upload_secret, lease_secret, lease_secret - ) - ) + (upload_secret, _, storage_index, created) = self.create_upload({1}, 10) def check_invalid(bad_content_range_value): client = StorageClientImmutables( @@ -579,29 +562,20 @@ class ImmutableHTTPAPITests(SyncTestCase): """ Listing unknown storage index's shares results in empty list of shares. """ - im_client = StorageClientImmutables(self.http.client) storage_index = b"".join(bytes([i]) for i in range(16)) - self.assertEqual(result_of(im_client.list_shares(storage_index)), set()) + self.assertEqual(result_of(self.im_client.list_shares(storage_index)), set()) def test_upload_non_existent_storage_index(self): """ Uploading to a non-existent storage index or share number results in 404. """ - im_client = StorageClientImmutables(self.http.client) - upload_secret = urandom(32) - lease_secret = urandom(32) - storage_index = b"".join(bytes([i]) for i in range(16)) - result_of( - im_client.create( - storage_index, {1}, 10, upload_secret, lease_secret, lease_secret - ) - ) + (upload_secret, _, storage_index, _) = self.create_upload({1}, 10) def unknown_check(storage_index, share_number): with self.assertRaises(ClientException) as e: result_of( - im_client.write_share_chunk( + self.im_client.write_share_chunk( storage_index, share_number, upload_secret, From faacde4e32a68543d6e7e92ea80a705e44208afa Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 9 Feb 2022 12:41:32 -0500 Subject: [PATCH 0675/2309] Conflicting writes. --- src/allmydata/storage/http_server.py | 10 +++++---- src/allmydata/test/test_storage_http.py | 27 +++++++++++++++++++++++-- 2 files changed, 31 insertions(+), 6 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 6f9033380..4613a73c3 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -31,7 +31,7 @@ from cbor2 import dumps, loads from .server import StorageServer from .http_common import swissnum_auth_header, Secrets from .common import si_a2b -from .immutable import BucketWriter +from .immutable import BucketWriter, ConflictingWriteError from ..util.hashutil import timing_safe_compare from ..util.base32 import rfc3548_alphabet @@ -253,9 +253,11 @@ class HTTPServer(object): request.setResponseCode(http.NOT_FOUND) return b"" - finished = bucket.write(offset, data) - - # TODO if raises ConflictingWriteError, return HTTP CONFLICT code. + try: + finished = bucket.write(offset, data) + except ConflictingWriteError: + request.setResponseCode(http.CONFLICT) + return b"" if finished: bucket.close() diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 2bad160e4..225784cb6 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -626,9 +626,32 @@ class ImmutableHTTPAPITests(SyncTestCase): """ If an uploaded chunk conflicts with an already uploaded chunk, a CONFLICT error is returned. - - TBD in https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3860 """ + (upload_secret, _, storage_index, created) = self.create_upload({1}, 100) + + # Write: + result_of( + self.im_client.write_share_chunk( + storage_index, + 1, + upload_secret, + 0, + b"0" * 10, + ) + ) + + # Conflicting write: + with self.assertRaises(ClientException) as e: + result_of( + self.im_client.write_share_chunk( + storage_index, + 1, + upload_secret, + 0, + b"0123456789", + ) + ) + self.assertEqual(e.exception.code, http.NOT_FOUND) def test_read_of_wrong_storage_index_fails(self): """ From bae5d58ab97468e6fdefc8570a1f448ff3c30631 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 9 Feb 2022 13:07:34 -0500 Subject: [PATCH 0676/2309] Another test. --- src/allmydata/test/test_storage_http.py | 29 +++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 225784cb6..e61a023bd 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -594,9 +594,34 @@ class ImmutableHTTPAPITests(SyncTestCase): """ If a storage index has multiple shares, uploads to different shares are stored separately and can be downloaded separately. - - TBD in https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3860 """ + (upload_secret, _, storage_index, _) = self.create_upload({1, 2}, 10) + result_of( + self.im_client.write_share_chunk( + storage_index, + 1, + upload_secret, + 0, + b"1" * 10, + ) + ) + result_of( + self.im_client.write_share_chunk( + storage_index, + 2, + upload_secret, + 0, + b"2" * 10, + ) + ) + self.assertEqual( + result_of(self.im_client.read_share_chunk(storage_index, 1, 0, 10)), + b"1" * 10, + ) + self.assertEqual( + result_of(self.im_client.read_share_chunk(storage_index, 2, 0, 10)), + b"2" * 10, + ) def test_bucket_allocated_with_new_shares(self): """ From 45ee5e33460fa06f3c557c17b9697c27f09c4aa4 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 9 Feb 2022 13:08:34 -0500 Subject: [PATCH 0677/2309] Done elsewhere. --- src/allmydata/test/test_storage_http.py | 24 ------------------------ 1 file changed, 24 deletions(-) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index e61a023bd..ac52303f6 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -623,30 +623,6 @@ class ImmutableHTTPAPITests(SyncTestCase): b"2" * 10, ) - def test_bucket_allocated_with_new_shares(self): - """ - If some shares already exist, allocating shares indicates only the new - ones were created. - - TBD in https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3860 - """ - - def test_bucket_allocation_new_upload_secret(self): - """ - If a bucket was allocated with one upload secret, and a different upload - key is used to allocate the bucket again, the second allocation fails. - - TBD in https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3860 - """ - - def test_upload_with_wrong_upload_secret_fails(self): - """ - Uploading with a key that doesn't match the one used to allocate the - bucket will fail. - - TBD in https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3860 - """ - def test_mismatching_upload_fails(self): """ If an uploaded chunk conflicts with an already uploaded chunk, a From 5d9e0c9bca14ace590b842250efa69bb092d0b48 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 9 Feb 2022 13:14:27 -0500 Subject: [PATCH 0678/2309] Not found tests and implementation. --- src/allmydata/storage/http_server.py | 9 ++++-- src/allmydata/test/test_storage_http.py | 41 ++++++++++++++++++++++--- 2 files changed, 44 insertions(+), 6 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 4613a73c3..e7aa12f55 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -302,8 +302,13 @@ class HTTPServer(object): offset, end = range_header.ranges[0] assert end != None # TODO support this case - # TODO if not found, 404 - bucket = self._storage_server.get_buckets(storage_index)[share_number] + try: + bucket = self._storage_server.get_buckets(storage_index)[share_number] + except KeyError: + request.setResponseCode(http.NOT_FOUND) + return b"" + + # TODO limit memory usage data = bucket.read(offset, end - offset) request.setResponseCode(http.PARTIAL_CONTENT) # TODO set content-range on response. We we need to expand the diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index ac52303f6..b96eebb96 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -654,19 +654,52 @@ class ImmutableHTTPAPITests(SyncTestCase): ) self.assertEqual(e.exception.code, http.NOT_FOUND) + def upload(self, share_number): + """ + Create a share, return (storage_index). + """ + (upload_secret, _, storage_index, _) = self.create_upload({share_number}, 26) + result_of( + self.im_client.write_share_chunk( + storage_index, + share_number, + upload_secret, + 0, + b"abcdefghijklmnopqrstuvwxyz", + ) + ) + return storage_index + def test_read_of_wrong_storage_index_fails(self): """ Reading from unknown storage index results in 404. - - TBD in https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3860 """ + with self.assertRaises(ClientException) as e: + result_of( + self.im_client.read_share_chunk( + b"1" * 16, + 1, + 0, + 10, + ) + ) + self.assertEqual(e.exception.code, http.NOT_FOUND) def test_read_of_wrong_share_number_fails(self): """ Reading from unknown storage index results in 404. - - TBD in https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3860 """ + storage_index = self.upload(1) + with self.assertRaises(ClientException) as e: + result_of( + self.im_client.read_share_chunk( + storage_index, + 7, # different share number + 0, + 10, + ) + ) + self.assertEqual(e.exception.code, http.NOT_FOUND) def test_read_with_negative_offset_fails(self): """ From 7db1ddd8750d429d5a77ef3e3c423f99fe89fbad Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 10 Feb 2022 13:12:17 -0500 Subject: [PATCH 0679/2309] Implement Range header validation. --- docs/proposed/http-storage-node-protocol.rst | 2 +- src/allmydata/storage/http_server.py | 15 +++++--- src/allmydata/test/test_storage_http.py | 40 ++++++++++++++++---- 3 files changed, 42 insertions(+), 15 deletions(-) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index 4d8d60560..315546b8a 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -644,7 +644,7 @@ Read a contiguous sequence of bytes from one share in one bucket. The response body is the raw share data (i.e., ``application/octet-stream``). The ``Range`` header may be used to request exactly one ``bytes`` range, in which case the response code will be 206 (partial content). Interpretation and response behavior is as specified in RFC 7233 § 4.1. -Multiple ranges in a single request are *not* supported. +Multiple ranges in a single request are *not* supported; open-ended ranges are also not supported. Discussion `````````` diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index e7aa12f55..89c9ca6fa 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -291,16 +291,19 @@ class HTTPServer(object): ) def read_share_chunk(self, request, authorization, storage_index, share_number): """Read a chunk for an already uploaded immutable.""" - # TODO in https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3860 # 2. missing range header should have response code 200 and return whole thing - # 3. malformed range header should result in error? or return everything? - # 4. non-bytes range results in error - # 5. ranges make sense semantically (positive, etc.) - # 6. multiple ranges fails with error # 7. missing end of range means "to the end of share" range_header = parse_range_header(request.getHeader("range")) + if ( + range_header is None + or range_header.units != "bytes" + or len(range_header.ranges) > 1 # more than one range + or range_header.ranges[0][1] is None # range without end + ): + request.setResponseCode(http.REQUESTED_RANGE_NOT_SATISFIABLE) + return b"" + offset, end = range_header.ranges[0] - assert end != None # TODO support this case try: bucket = self._storage_server.get_buckets(storage_index)[share_number] diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index b96eebb96..f115d632c 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -703,14 +703,38 @@ class ImmutableHTTPAPITests(SyncTestCase): def test_read_with_negative_offset_fails(self): """ - The offset for reads cannot be negative. - - TBD in https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3860 + Malformed or unsupported Range headers result in 416 (requested range + not satisfiable) error. """ + storage_index = self.upload(1) - def test_read_with_negative_length_fails(self): - """ - The length for reads cannot be negative. + def check_bad_range(bad_range_value): + client = StorageClientImmutables( + StorageClientWithHeadersOverride( + self.http.client, {"range": bad_range_value} + ) + ) + + with self.assertRaises(ClientException) as e: + result_of( + client.read_share_chunk( + storage_index, + 1, + 0, + 10, + ) + ) + self.assertEqual(e.exception.code, http.REQUESTED_RANGE_NOT_SATISFIABLE) + + check_bad_range("molluscs=0-9") + check_bad_range("bytes=-2-9") + check_bad_range("bytes=0--10") + check_bad_range("bytes=-300-") + check_bad_range("bytes=") + # Multiple ranges are currently unsupported, even if they're + # semantically valid under HTTP: + check_bad_range("bytes=0-5, 6-7") + # Ranges without an end are currently unsupported, even if they're + # semantically valid under HTTP. + check_bad_range("bytes=0-") - TBD in https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3860 - """ From 416af7328cd0a89930c719882ab36ce4f3e74839 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 10 Feb 2022 13:31:09 -0500 Subject: [PATCH 0680/2309] Support lack of Range header. --- src/allmydata/storage/http_server.py | 26 ++++++++++++------ src/allmydata/test/test_storage_http.py | 36 ++++++++++++++++++++----- 2 files changed, 47 insertions(+), 15 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 89c9ca6fa..88de18f12 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -291,8 +291,24 @@ class HTTPServer(object): ) def read_share_chunk(self, request, authorization, storage_index, share_number): """Read a chunk for an already uploaded immutable.""" - # 2. missing range header should have response code 200 and return whole thing - # 7. missing end of range means "to the end of share" + try: + bucket = self._storage_server.get_buckets(storage_index)[share_number] + except KeyError: + request.setResponseCode(http.NOT_FOUND) + return b"" + + if request.getHeader("range") is None: + # Return the whole thing. + start = 0 + while True: + # TODO should probably yield to event loop occasionally... + data = bucket.read(start, start + 65536) + if not data: + request.finish() + return + request.write(data) + start += len(data) + range_header = parse_range_header(request.getHeader("range")) if ( range_header is None @@ -305,12 +321,6 @@ class HTTPServer(object): offset, end = range_header.ranges[0] - try: - bucket = self._storage_server.get_buckets(storage_index)[share_number] - except KeyError: - request.setResponseCode(http.NOT_FOUND) - return b"" - # TODO limit memory usage data = bucket.read(offset, end - offset) request.setResponseCode(http.PARTIAL_CONTENT) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index f115d632c..bbda00bf6 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -46,6 +46,7 @@ from ..storage.http_client import ( ImmutableCreateResult, UploadProgress, StorageClientGeneral, + _encode_si, ) from ..storage.common import si_b2a @@ -654,21 +655,26 @@ class ImmutableHTTPAPITests(SyncTestCase): ) self.assertEqual(e.exception.code, http.NOT_FOUND) - def upload(self, share_number): + def upload(self, share_number, data_length=26): """ - Create a share, return (storage_index). + Create a share, return (storage_index, uploaded_data). """ - (upload_secret, _, storage_index, _) = self.create_upload({share_number}, 26) + uploaded_data = (b"abcdefghijklmnopqrstuvwxyz" * ((data_length // 26) + 1))[ + :data_length + ] + (upload_secret, _, storage_index, _) = self.create_upload( + {share_number}, data_length + ) result_of( self.im_client.write_share_chunk( storage_index, share_number, upload_secret, 0, - b"abcdefghijklmnopqrstuvwxyz", + uploaded_data, ) ) - return storage_index + return storage_index, uploaded_data def test_read_of_wrong_storage_index_fails(self): """ @@ -689,7 +695,7 @@ class ImmutableHTTPAPITests(SyncTestCase): """ Reading from unknown storage index results in 404. """ - storage_index = self.upload(1) + storage_index, _ = self.upload(1) with self.assertRaises(ClientException) as e: result_of( self.im_client.read_share_chunk( @@ -706,7 +712,7 @@ class ImmutableHTTPAPITests(SyncTestCase): Malformed or unsupported Range headers result in 416 (requested range not satisfiable) error. """ - storage_index = self.upload(1) + storage_index, _ = self.upload(1) def check_bad_range(bad_range_value): client = StorageClientImmutables( @@ -738,3 +744,19 @@ class ImmutableHTTPAPITests(SyncTestCase): # semantically valid under HTTP. check_bad_range("bytes=0-") + @given(data_length=st.integers(min_value=1, max_value=300000)) + def test_read_with_no_range(self, data_length): + """ + A read with no range returns the whole immutable. + """ + storage_index, uploaded_data = self.upload(1, data_length) + response = result_of( + self.http.client.request( + "GET", + self.http.client.relative_url( + "/v1/immutable/{}/1".format(_encode_si(storage_index)) + ), + ) + ) + self.assertEqual(response.code, http.OK) + self.assertEqual(result_of(response.content()), uploaded_data) From aa68be645f80dde5a52dc5a32d74304921e97288 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 10 Feb 2022 13:47:56 -0500 Subject: [PATCH 0681/2309] Return Content-Range in responses. --- src/allmydata/storage/http_server.py | 12 +++++------ src/allmydata/test/test_storage_http.py | 27 +++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 6 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 88de18f12..bb4ef6c00 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -24,6 +24,7 @@ from twisted.web import http import attr from werkzeug.http import parse_range_header, parse_content_range_header from werkzeug.routing import BaseConverter +from werkzeug.datastructures import ContentRange # TODO Make sure to use pure Python versions? from cbor2 import dumps, loads @@ -323,11 +324,10 @@ class HTTPServer(object): # TODO limit memory usage data = bucket.read(offset, end - offset) + request.setResponseCode(http.PARTIAL_CONTENT) - # TODO set content-range on response. We we need to expand the - # BucketReader interface to return share's length. - # - # request.setHeader( - # "content-range", range_header.make_content_range(share_length).to_header() - # ) + request.setHeader( + "content-range", + ContentRange("bytes", offset, offset + len(data)).to_header(), + ) return data diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index bbda00bf6..c864f923c 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -760,3 +760,30 @@ class ImmutableHTTPAPITests(SyncTestCase): ) self.assertEqual(response.code, http.OK) self.assertEqual(result_of(response.content()), uploaded_data) + + def test_validate_content_range_response_to_read(self): + """ + The server responds to ranged reads with an appropriate Content-Range + header. + """ + storage_index, _ = self.upload(1, 26) + + def check_range(requested_range, expected_response): + headers = Headers() + headers.setRawHeaders("range", [requested_range]) + response = result_of( + self.http.client.request( + "GET", + self.http.client.relative_url( + "/v1/immutable/{}/1".format(_encode_si(storage_index)) + ), + headers=headers, + ) + ) + self.assertEqual( + response.headers.getRawHeaders("content-range"), [expected_response] + ) + + check_range("bytes=0-10", "bytes 0-10/*") + # Can't go beyond the end of the immutable! + check_range("bytes=10-100", "bytes 10-25/*") From fa2f142bc9ecd507312614ab134e53a74f7c8ce4 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 10 Feb 2022 13:50:09 -0500 Subject: [PATCH 0682/2309] Another ticket. --- src/allmydata/storage/http_server.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index bb4ef6c00..491fcd39b 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -247,6 +247,7 @@ class HTTPServer(object): offset = content_range.start # TODO limit memory usage + # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3872 data = request.content.read(content_range.stop - content_range.start + 1) try: bucket = self._uploads[storage_index].shares[share_number] @@ -303,6 +304,7 @@ class HTTPServer(object): start = 0 while True: # TODO should probably yield to event loop occasionally... + # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3872 data = bucket.read(start, start + 65536) if not data: request.finish() @@ -323,6 +325,7 @@ class HTTPServer(object): offset, end = range_header.ranges[0] # TODO limit memory usage + # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3872 data = bucket.read(offset, end - offset) request.setResponseCode(http.PARTIAL_CONTENT) From b049d4a792efa69c8f82ef0800380694b41cd5f3 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 10 Feb 2022 13:52:47 -0500 Subject: [PATCH 0683/2309] Fix get_version with new API. --- src/allmydata/storage_client.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index abced41b3..528711539 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -75,7 +75,9 @@ from allmydata.util.observer import ObserverList from allmydata.util.rrefutil import add_version_to_remote_reference from allmydata.util.hashutil import permute_server_hash from allmydata.util.dictutil import BytesKeyDict, UnicodeKeyDict -from allmydata.storage.http_client import StorageClient, StorageClientImmutables +from allmydata.storage.http_client import ( + StorageClient, StorageClientImmutables, StorageClientGeneral, +) # who is responsible for de-duplication? @@ -1108,7 +1110,7 @@ class _HTTPStorageServer(object): return _HTTPStorageServer(http_client=http_client, upload_secret=urandom(20)) def get_version(self): - return self._http_client.get_version() + return StorageClientGeneral(self._http_client).get_version() @defer.inlineCallbacks def allocate_buckets( From 7466ee25a8d94017735a0380fc232fd8ec472421 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 10 Feb 2022 13:57:57 -0500 Subject: [PATCH 0684/2309] Don't send header if it makes no sense to do so. --- src/allmydata/storage/http_server.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 491fcd39b..f89a156a3 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -329,8 +329,11 @@ class HTTPServer(object): data = bucket.read(offset, end - offset) request.setResponseCode(http.PARTIAL_CONTENT) - request.setHeader( - "content-range", - ContentRange("bytes", offset, offset + len(data)).to_header(), - ) + if len(data): + # For empty bodies the content-range header makes no sense since + # the end of the range is inclusive. + request.setHeader( + "content-range", + ContentRange("bytes", offset, offset + len(data)).to_header(), + ) return data From abf3048ab347ae737344db9bc4701b1452f83ad1 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 10 Feb 2022 17:07:21 -0500 Subject: [PATCH 0685/2309] More passing HTTP IStorageServer tests. --- src/allmydata/storage_client.py | 10 ++++++++-- src/allmydata/test/test_istorageserver.py | 2 -- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 528711539..17a6f79a5 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -58,7 +58,7 @@ from twisted.plugin import ( from eliot import ( log_call, ) -from foolscap.api import eventually +from foolscap.api import eventually, RemoteException from foolscap.reconnector import ( ReconnectionInfo, ) @@ -77,6 +77,7 @@ from allmydata.util.hashutil import permute_server_hash from allmydata.util.dictutil import BytesKeyDict, UnicodeKeyDict from allmydata.storage.http_client import ( StorageClient, StorageClientImmutables, StorageClientGeneral, + ClientException as HTTPClientException, ) @@ -1037,8 +1038,13 @@ class _FakeRemoteReference(object): """ local_object = attr.ib(type=object) + @defer.inlineCallbacks def callRemote(self, action, *args, **kwargs): - return getattr(self.local_object, action)(*args, **kwargs) + try: + result = yield getattr(self.local_object, action)(*args, **kwargs) + return result + except HTTPClientException as e: + raise RemoteException(e.args) @attr.s diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index f3083a4bd..95261ddb2 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -1162,8 +1162,6 @@ class HTTPImmutableAPIsTests( "test_advise_corrupt_share", "test_bucket_advise_corrupt_share", "test_disconnection", - "test_matching_overlapping_writes", - "test_non_matching_overlapping_writes", } From 5aa00abc3da0624541fe9e74aa61355cd59af8c0 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 11 Feb 2022 15:02:14 -0500 Subject: [PATCH 0686/2309] Use the correct API (since direct returns break Python 2 imports) --- src/allmydata/storage_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 17a6f79a5..c2e26b4b6 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -1042,7 +1042,7 @@ class _FakeRemoteReference(object): def callRemote(self, action, *args, **kwargs): try: result = yield getattr(self.local_object, action)(*args, **kwargs) - return result + defer.returnValue(result) except HTTPClientException as e: raise RemoteException(e.args) From 0639f2c16c70b51b5e5a965aedaaa0c220830ec8 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 14 Feb 2022 10:35:43 -0500 Subject: [PATCH 0687/2309] Try to switch to modern Python 3 world. Temporarily switch image building to always happen. --- ...ckerfile.centos => Dockerfile.oraclelinux} | 2 +- .circleci/config.yml | 205 +++++++----------- 2 files changed, 74 insertions(+), 133 deletions(-) rename .circleci/{Dockerfile.centos => Dockerfile.oraclelinux} (97%) diff --git a/.circleci/Dockerfile.centos b/.circleci/Dockerfile.oraclelinux similarity index 97% rename from .circleci/Dockerfile.centos rename to .circleci/Dockerfile.oraclelinux index 9070d71d9..ee31643b4 100644 --- a/.circleci/Dockerfile.centos +++ b/.circleci/Dockerfile.oraclelinux @@ -1,5 +1,5 @@ ARG TAG -FROM centos:${TAG} +FROM oraclelinux:${TAG} ARG PYTHON_VERSION ENV WHEELHOUSE_PATH /tmp/wheelhouse diff --git a/.circleci/config.yml b/.circleci/config.yml index daf985567..ac62d9ed9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -15,28 +15,20 @@ workflows: ci: jobs: # Start with jobs testing various platforms. - - "debian-9": - {} - "debian-10": + {} + - "debian-11": requires: - - "debian-9" + - "debian-10" - "ubuntu-20-04": {} - "ubuntu-18-04": requires: - "ubuntu-20-04" - - "ubuntu-16-04": - requires: - - "ubuntu-20-04" - - "fedora-29": - {} - - "fedora-28": - requires: - - "fedora-29" - - - "centos-8": + # Equivalent to RHEL 8; CentOS 8 is dead. + - "oraclelinux-8": {} - "nixos": @@ -47,9 +39,9 @@ workflows: name: "NixOS 21.11" nixpkgs: "21.11" - # Test against PyPy 2.7 - - "pypy27-buster": - {} + # Eventually, test against PyPy 3.8 + #- "pypy27-buster": + # {} # Test against Python 3: - "python37": @@ -74,7 +66,7 @@ workflows: requires: # If the unit test suite doesn't pass, don't bother running the # integration tests. - - "debian-9" + - "debian-10" - "typechecks": {} @@ -85,13 +77,13 @@ workflows: # Build the Docker images used by the ci jobs. This makes the ci jobs # faster and takes various spurious failures out of the critical path. triggers: - # Build once a day - - schedule: - cron: "0 0 * * *" - filters: - branches: - only: - - "master" + # # Build once a day + # - schedule: + # cron: "0 0 * * *" + # filters: + # branches: + # only: + # - "master" jobs: # Every job that pushes a Docker image from Docker Hub needs to provide @@ -104,22 +96,19 @@ workflows: # https://app.circleci.com/settings/organization/github/tahoe-lafs/contexts - "build-image-debian-10": &DOCKERHUB_CONTEXT context: "dockerhub-auth" - - "build-image-debian-9": - <<: *DOCKERHUB_CONTEXT - - "build-image-ubuntu-16-04": + - "build-image-debian-11": <<: *DOCKERHUB_CONTEXT - "build-image-ubuntu-18-04": <<: *DOCKERHUB_CONTEXT - "build-image-ubuntu-20-04": <<: *DOCKERHUB_CONTEXT - - "build-image-fedora-28": + - "build-image-fedora-35": <<: *DOCKERHUB_CONTEXT - - "build-image-fedora-29": - <<: *DOCKERHUB_CONTEXT - - "build-image-centos-8": - <<: *DOCKERHUB_CONTEXT - - "build-image-pypy27-buster": + - "build-image-oraclelinux-8": <<: *DOCKERHUB_CONTEXT + # Restore later as PyPy38 + #- "build-image-pypy27-buster": + # <<: *DOCKERHUB_CONTEXT - "build-image-python37-ubuntu": <<: *DOCKERHUB_CONTEXT @@ -150,7 +139,7 @@ jobs: lint: docker: - <<: *DOCKERHUB_AUTH - image: "circleci/python:2" + image: "cimg/python:3.9" steps: - "checkout" @@ -168,7 +157,7 @@ jobs: codechecks3: docker: - <<: *DOCKERHUB_AUTH - image: "circleci/python:3" + image: "cimg/python:3.9" steps: - "checkout" @@ -186,7 +175,7 @@ jobs: pyinstaller: docker: - <<: *DOCKERHUB_AUTH - image: "circleci/python:2" + image: "cimg/python:3.9" steps: - "checkout" @@ -209,10 +198,10 @@ jobs: command: | dist/Tahoe-LAFS/tahoe --version - debian-9: &DEBIAN + debian-10: &DEBIAN docker: - <<: *DOCKERHUB_AUTH - image: "tahoelafsci/debian:9-py2.7" + image: "tahoelafsci/debian:10-py3.7" user: "nobody" environment: &UTF_8_ENVIRONMENT @@ -226,7 +215,7 @@ jobs: # filenames and argv). LANG: "en_US.UTF-8" # Select a tox environment to run for this job. - TAHOE_LAFS_TOX_ENVIRONMENT: "py27" + TAHOE_LAFS_TOX_ENVIRONMENT: "py37" # Additional arguments to pass to tox. TAHOE_LAFS_TOX_ARGS: "" # The path in which test artifacts will be placed. @@ -299,24 +288,32 @@ jobs: <<: *DEBIAN docker: - <<: *DOCKERHUB_AUTH - image: "tahoelafsci/debian:10-py2.7" + image: "tahoelafsci/debian:10-py3.7" user: "nobody" - pypy27-buster: + debian-11: <<: *DEBIAN docker: - <<: *DOCKERHUB_AUTH - image: "tahoelafsci/pypy:buster-py2" + image: "tahoelafsci/debian:11-py3.9" user: "nobody" - environment: - <<: *UTF_8_ENVIRONMENT - # We don't do coverage since it makes PyPy far too slow: - TAHOE_LAFS_TOX_ENVIRONMENT: "pypy27" - # Since we didn't collect it, don't upload it. - UPLOAD_COVERAGE: "" + TAHOE_LAFS_TOX_ENVIRONMENT: "py39" + # Restore later using PyPy3.8 + # pypy27-buster: + # <<: *DEBIAN + # docker: + # - <<: *DOCKERHUB_AUTH + # image: "tahoelafsci/pypy:buster-py2" + # user: "nobody" + # environment: + # <<: *UTF_8_ENVIRONMENT + # # We don't do coverage since it makes PyPy far too slow: + # TAHOE_LAFS_TOX_ENVIRONMENT: "pypy27" + # # Since we didn't collect it, don't upload it. + # UPLOAD_COVERAGE: "" c-locale: <<: *DEBIAN @@ -364,23 +361,6 @@ jobs: - run: *SETUP_VIRTUALENV - run: *RUN_TESTS - - ubuntu-16-04: - <<: *DEBIAN - docker: - - <<: *DOCKERHUB_AUTH - image: "tahoelafsci/ubuntu:16.04-py2.7" - user: "nobody" - - - ubuntu-18-04: &UBUNTU_18_04 - <<: *DEBIAN - docker: - - <<: *DOCKERHUB_AUTH - image: "tahoelafsci/ubuntu:18.04-py2.7" - user: "nobody" - - python37: <<: *UBUNTU_18_04 docker: @@ -405,10 +385,10 @@ jobs: user: "nobody" - centos-8: &RHEL_DERIV + oracelinux-8: &RHEL_DERIV docker: - <<: *DOCKERHUB_AUTH - image: "tahoelafsci/centos:8-py2" + image: "tahoelafsci/oraclelinux:8-py3.8" user: "nobody" environment: *UTF_8_ENVIRONMENT @@ -427,20 +407,11 @@ jobs: - store_artifacts: *STORE_OTHER_ARTIFACTS - run: *SUBMIT_COVERAGE - - fedora-28: + fedora-35: <<: *RHEL_DERIV docker: - <<: *DOCKERHUB_AUTH - image: "tahoelafsci/fedora:28-py" - user: "nobody" - - - fedora-29: - <<: *RHEL_DERIV - docker: - - <<: *DOCKERHUB_AUTH - image: "tahoelafsci/fedora:29-py" + image: "tahoelafsci/fedora:35-py3.9" user: "nobody" nixos: @@ -554,7 +525,7 @@ jobs: typechecks: docker: - <<: *DOCKERHUB_AUTH - image: "tahoelafsci/ubuntu:18.04-py3" + image: "tahoelafsci/ubuntu:18.04-py3.7" steps: - "checkout" @@ -566,7 +537,7 @@ jobs: docs: docker: - <<: *DOCKERHUB_AUTH - image: "tahoelafsci/ubuntu:18.04-py3" + image: "tahoelafsci/ubuntu:18.04-py3.7" steps: - "checkout" @@ -589,8 +560,8 @@ jobs: image: "cimg/base:2022.01" environment: - DISTRO: "tahoelafsci/:foo-py2" - TAG: "tahoelafsci/distro:-py2" + DISTRO: "tahoelafsci/:foo-py3.9" + TAG: "tahoelafsci/distro:-py3.9" PYTHON_VERSION: "tahoelafsci/distro:tag-py Date: Mon, 14 Feb 2022 10:40:38 -0500 Subject: [PATCH 0688/2309] Fix some references. --- .circleci/config.yml | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ac62d9ed9..b178d945b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -43,10 +43,6 @@ workflows: #- "pypy27-buster": # {} - # Test against Python 3: - - "python37": - {} - # Other assorted tasks and configurations - "lint": {} @@ -361,8 +357,8 @@ jobs: - run: *SETUP_VIRTUALENV - run: *RUN_TESTS - python37: - <<: *UBUNTU_18_04 + ubuntu-18-04: &UBUNTU_18_04 + <<: *DEBIAN docker: - <<: *DOCKERHUB_AUTH image: "tahoelafsci/ubuntu:18.04-py3.7" From d4810ce5b865ddcf4cfe3138b3a401e4034a3c67 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 14 Feb 2022 10:42:58 -0500 Subject: [PATCH 0689/2309] Get rid of duplicate. --- .circleci/config.yml | 9 --------- 1 file changed, 9 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index b178d945b..c90ce16b7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -279,15 +279,6 @@ jobs: /tmp/venv/bin/codecov fi - - debian-10: - <<: *DEBIAN - docker: - - <<: *DOCKERHUB_AUTH - image: "tahoelafsci/debian:10-py3.7" - user: "nobody" - - debian-11: <<: *DEBIAN docker: From f0a81e1095a4a1ea90ba819e4e7b99a0b57f3617 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 14 Feb 2022 10:44:44 -0500 Subject: [PATCH 0690/2309] Fix typo. --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index c90ce16b7..0dbcc1314 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -372,7 +372,7 @@ jobs: user: "nobody" - oracelinux-8: &RHEL_DERIV + oraclelinux-8: &RHEL_DERIV docker: - <<: *DOCKERHUB_AUTH image: "tahoelafsci/oraclelinux:8-py3.8" From 5935d9977697c6ae1e5ffc4f6618084071b33c5b Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 14 Feb 2022 10:45:56 -0500 Subject: [PATCH 0691/2309] Fix name. --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 0dbcc1314..511cee87c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -591,7 +591,7 @@ jobs: TAG: "11" PYTHON_VERSION: "3.9" - build-image-python37-ubuntu: + build-image-ubuntu-18.04: <<: *BUILD_IMAGE environment: From 4e133eb759daafce648fce593248d7930effe235 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 14 Feb 2022 10:46:55 -0500 Subject: [PATCH 0692/2309] Fix name. --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 511cee87c..31df8d6a1 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -591,7 +591,7 @@ jobs: TAG: "11" PYTHON_VERSION: "3.9" - build-image-ubuntu-18.04: + build-image-ubuntu-18-04: <<: *BUILD_IMAGE environment: From 19a3d2acf7b6fc1871d277219724f6cdb23ff016 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 14 Feb 2022 10:49:17 -0500 Subject: [PATCH 0693/2309] Fix some more. --- .circleci/config.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 31df8d6a1..2b588fbb7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -105,8 +105,6 @@ workflows: # Restore later as PyPy38 #- "build-image-pypy27-buster": # <<: *DOCKERHUB_CONTEXT - - "build-image-python37-ubuntu": - <<: *DOCKERHUB_CONTEXT jobs: @@ -609,7 +607,7 @@ jobs: PYTHON_VERSION: "3.9" - build-image-oracelinux: + build-image-oraclelinux-8: <<: *BUILD_IMAGE environment: From 6a17c07158d56c94f2a129509d326daff779be57 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 14 Feb 2022 10:52:33 -0500 Subject: [PATCH 0694/2309] Drop unnecessary install. --- .circleci/Dockerfile.oraclelinux | 1 - 1 file changed, 1 deletion(-) diff --git a/.circleci/Dockerfile.oraclelinux b/.circleci/Dockerfile.oraclelinux index ee31643b4..cf4c009d2 100644 --- a/.circleci/Dockerfile.oraclelinux +++ b/.circleci/Dockerfile.oraclelinux @@ -13,7 +13,6 @@ RUN yum install --assumeyes \ sudo \ make automake gcc gcc-c++ \ python${PYTHON_VERSION} \ - python${PYTHON_VERSION}-devel \ libffi-devel \ openssl-devel \ libyaml \ From 13d23e3baffceab6d726ddad5a37e579f4217c01 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 14 Feb 2022 10:57:44 -0500 Subject: [PATCH 0695/2309] The terminal is a lie. --- .circleci/Dockerfile.debian | 2 +- .circleci/Dockerfile.ubuntu | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/Dockerfile.debian b/.circleci/Dockerfile.debian index 96c54736c..f12f19551 100644 --- a/.circleci/Dockerfile.debian +++ b/.circleci/Dockerfile.debian @@ -1,7 +1,7 @@ ARG TAG FROM debian:${TAG} ARG PYTHON_VERSION - +ENV DEBIAN_FRONTEND noninteractive ENV WHEELHOUSE_PATH /tmp/wheelhouse ENV VIRTUALENV_PATH /tmp/venv # This will get updated by the CircleCI checkout step. diff --git a/.circleci/Dockerfile.ubuntu b/.circleci/Dockerfile.ubuntu index 2fcc60f5a..22689f0c1 100644 --- a/.circleci/Dockerfile.ubuntu +++ b/.circleci/Dockerfile.ubuntu @@ -1,7 +1,7 @@ ARG TAG FROM ubuntu:${TAG} ARG PYTHON_VERSION - +ENV DEBIAN_FRONTEND noninteractive ENV WHEELHOUSE_PATH /tmp/wheelhouse ENV VIRTUALENV_PATH /tmp/venv # This will get updated by the CircleCI checkout step. From 0928a7993a588298e723c98bbe5d216ad713306f Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 14 Feb 2022 11:02:25 -0500 Subject: [PATCH 0696/2309] Rip out Python 2. --- .github/workflows/ci.yml | 32 +------------------------------- 1 file changed, 1 insertion(+), 31 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f65890d37..a4d8daca5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -38,14 +38,11 @@ jobs: - windows-latest - ubuntu-latest python-version: - - 2.7 - 3.7 - 3.8 - 3.9 include: # On macOS don't bother with 3.7-3.8, just to get faster builds. - - os: macos-10.15 - python-version: 2.7 - os: macos-latest python-version: 3.9 @@ -108,25 +105,6 @@ jobs: # Action for this, as of Jan 2021 it does not support Python coverage # files - only lcov files. Therefore, we use coveralls-python, the # coveralls.io-supplied Python reporter, for this. - # - # It is coveralls-python 1.x that has maintained compatibility - # with Python 2, while coveralls-python 3.x is compatible with - # Python 3. Sadly we can't use them both in the same workflow. - # - # The two versions of coveralls-python are somewhat mutually - # incompatible. Mixing these two different versions when - # reporting coverage to coveralls.io will lead to grief, since - # they get job IDs in different fashion. If we use both - # versions of coveralls in the same workflow, the finalizing - # step will be able to mark only part of the jobs as done, and - # the other part will be left hanging, never marked as done: it - # does not matter if we make an API call or `coveralls --finish` - # to indicate that CI has finished running. - # - # So we try to use the newer coveralls-python that is available - # via Python 3 (which is present in GitHub Actions tool cache, - # even when we're running Python 2.7 tests) throughout this - # workflow. - name: "Report Coverage to Coveralls" run: | pip3 install --upgrade coveralls==3.0.1 @@ -179,13 +157,10 @@ jobs: - windows-latest - ubuntu-latest python-version: - - 2.7 - 3.7 - 3.9 include: # On macOS don't bother with 3.7, just to get faster builds. - - os: macos-10.15 - python-version: 2.7 - os: macos-latest python-version: 3.9 @@ -242,12 +217,7 @@ jobs: - name: Display tool versions run: python misc/build_helpers/show-tool-versions.py - - name: Run "Python 2 integration tests" - if: ${{ matrix.python-version == '2.7' }} - run: tox -e integration - - name: Run "Python 3 integration tests" - if: ${{ matrix.python-version != '2.7' }} run: tox -e integration3 - name: Upload eliot.log in case of failure @@ -267,7 +237,7 @@ jobs: - windows-latest - ubuntu-latest python-version: - - 2.7 + - 3.9 steps: From 34fe6a41edd3c4468bfd78969b14cfed5d890a8d Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 14 Feb 2022 11:05:31 -0500 Subject: [PATCH 0697/2309] Fix Fedora package name. --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 2b588fbb7..cd26a5de8 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -396,7 +396,7 @@ jobs: <<: *RHEL_DERIV docker: - <<: *DOCKERHUB_AUTH - image: "tahoelafsci/fedora:35-py3.9" + image: "tahoelafsci/fedora:35-py3" user: "nobody" nixos: @@ -621,7 +621,7 @@ jobs: environment: DISTRO: "fedora" TAG: "35" - PYTHON_VERSION: "3.9" + PYTHON_VERSION: "3" # build-image-pypy27-buster: # <<: *BUILD_IMAGE From cd33e1cfb384df56cc59a46258c781f708f901c7 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 14 Feb 2022 11:10:56 -0500 Subject: [PATCH 0698/2309] Rip out more Python 2 stuff. --- Makefile | 8 ++++---- docs/release-checklist.rst | 2 +- tox.ini | 15 +-------------- 3 files changed, 6 insertions(+), 19 deletions(-) diff --git a/Makefile b/Makefile index 33a40df02..fa8351535 100644 --- a/Makefile +++ b/Makefile @@ -17,7 +17,7 @@ PYTHON=python export PYTHON PYFLAKES=flake8 export PYFLAKES -VIRTUAL_ENV=./.tox/py27 +VIRTUAL_ENV=./.tox/py37 SOURCES=src/allmydata static misc setup.py APPNAME=tahoe-lafs TEST_SUITE=allmydata @@ -33,9 +33,9 @@ default: ## Run all tests and code reports test: .tox/create-venvs.log # Run codechecks first since it takes the least time to report issues early. - tox --develop -e codechecks + tox --develop -e codechecks3 # Run all the test environments in parallel to reduce run-time - tox --develop -p auto -e 'py27,py37,pypy27' + tox --develop -p auto -e 'py37' .PHONY: test-venv-coverage ## Run all tests with coverage collection and reporting. test-venv-coverage: @@ -136,7 +136,7 @@ count-lines: # Here is a list of testing tools that can be run with 'python' from a # virtualenv in which Tahoe has been installed. There used to be Makefile # targets for each, but the exact path to a suitable python is now up to the -# developer. But as a hint, after running 'tox', ./.tox/py27/bin/python will +# developer. But as a hint, after running 'tox', ./.tox/py37/bin/python will # probably work. # src/allmydata/test/bench_dirnode.py diff --git a/docs/release-checklist.rst b/docs/release-checklist.rst index 9588fd1a5..5697ab95f 100644 --- a/docs/release-checklist.rst +++ b/docs/release-checklist.rst @@ -122,7 +122,7 @@ they will need to evaluate which contributors' signatures they trust. - these should all pass: - - tox -e py27,codechecks,docs,integration + - tox -e py37,codechecks3,docs,integration - these can fail (ideally they should not of course): diff --git a/tox.ini b/tox.ini index 34d555aa7..3125548f4 100644 --- a/tox.ini +++ b/tox.ini @@ -7,7 +7,6 @@ # the tox-gh-actions package. [gh-actions] python = - 2.7: py27-coverage,codechecks 3.7: py37-coverage,typechecks,codechecks3 3.8: py38-coverage 3.9: py39-coverage @@ -17,7 +16,7 @@ python = twisted = 1 [tox] -envlist = typechecks,codechecks,codechecks3,py{27,37,38,39}-{coverage},pypy27,pypy3,integration,integration3 +envlist = typechecks,codechecks3,py{37,38,39}-{coverage},pypy27,pypy3,integration,integration3 minversion = 2.4 [testenv] @@ -110,18 +109,6 @@ commands = coverage report -# Once 2.7 is dropped, this can be removed. It just does flake8 with Python 2 -# since that can give different results than flake8 on Python 3. -[testenv:codechecks] -basepython = python2.7 -setenv = - # If no positional arguments are given, try to run the checks on the - # entire codebase, including various pieces of supporting code. - DEFAULT_FILES=src integration static misc setup.py -commands = - flake8 {posargs:{env:DEFAULT_FILES}} - - [testenv:codechecks3] basepython = python3 deps = From 9428c5d45b14c432d0f369643c567cdc3a8f5e18 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 14 Feb 2022 11:29:19 -0500 Subject: [PATCH 0699/2309] We can use modern PyInstaller. --- tox.ini | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tox.ini b/tox.ini index 3125548f4..e42958b28 100644 --- a/tox.ini +++ b/tox.ini @@ -218,9 +218,7 @@ extras = deps = {[testenv]deps} packaging - # PyInstaller 4.0 drops Python 2 support. When we finish porting to - # Python 3 we can reconsider this constraint. - pyinstaller < 4.0 + pyinstaller pefile ; platform_system == "Windows" # Setting PYTHONHASHSEED to a known value assists with reproducible builds. # See https://pyinstaller.readthedocs.io/en/stable/advanced-topics.html#creating-a-reproducible-build From d54294a6ec039a0bdbb6aeac138eec29996b6e5e Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 14 Feb 2022 11:30:09 -0500 Subject: [PATCH 0700/2309] News files. --- newsfragments/3327.minor | 0 newsfragments/3873.incompat | 1 + 2 files changed, 1 insertion(+) create mode 100644 newsfragments/3327.minor create mode 100644 newsfragments/3873.incompat diff --git a/newsfragments/3327.minor b/newsfragments/3327.minor new file mode 100644 index 000000000..e69de29bb diff --git a/newsfragments/3873.incompat b/newsfragments/3873.incompat new file mode 100644 index 000000000..7e71b1504 --- /dev/null +++ b/newsfragments/3873.incompat @@ -0,0 +1 @@ +Dropped support for Python 2. \ No newline at end of file From 77e7f80a1a93bec0683588e5ce19eff2df638543 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 14 Feb 2022 11:30:19 -0500 Subject: [PATCH 0701/2309] Try to update to Python 3. --- misc/build_helpers/run-deprecations.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/misc/build_helpers/run-deprecations.py b/misc/build_helpers/run-deprecations.py index f99cf90aa..2ad335bd1 100644 --- a/misc/build_helpers/run-deprecations.py +++ b/misc/build_helpers/run-deprecations.py @@ -26,10 +26,10 @@ python run-deprecations.py [--warnings=STDERRFILE] [--package=PYTHONPACKAGE ] CO class RunPP(protocol.ProcessProtocol): def outReceived(self, data): self.stdout.write(data) - sys.stdout.write(data) + sys.stdout.write(str(data, sys.stdout.encoding)) def errReceived(self, data): self.stderr.write(data) - sys.stderr.write(data) + sys.stderr.write(str(data, sys.stdout.encoding)) def processEnded(self, reason): signal = reason.value.signal rc = reason.value.exitCode @@ -100,17 +100,19 @@ def run_command(main): pp.stdout.seek(0) for line in pp.stdout.readlines(): + line = str(line, sys.stdout.encoding) if match(line): add(line) # includes newline pp.stderr.seek(0) for line in pp.stderr.readlines(): + line = str(line, sys.stdout.encoding) if match(line): add(line) if warnings: if config["warnings"]: - with open(config["warnings"], "wb") as f: + with open(config["warnings"], "w") as f: print("".join(warnings), file=f) print("ERROR: %d deprecation warnings found" % len(warnings)) sys.exit(1) From 0bcfc58c2279dac8c94293aa8c0ea7fa14d9ce31 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 14 Feb 2022 11:30:24 -0500 Subject: [PATCH 0702/2309] Various version fixes. --- .circleci/config.yml | 27 +++++---------------------- 1 file changed, 5 insertions(+), 22 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index cd26a5de8..9050c30b4 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -44,8 +44,6 @@ workflows: # {} # Other assorted tasks and configurations - - "lint": - {} - "codechecks3": {} - "pyinstaller": @@ -130,24 +128,6 @@ jobs: # Since this job is never scheduled this step is never run so the # actual value here is irrelevant. - lint: - docker: - - <<: *DOCKERHUB_AUTH - image: "cimg/python:3.9" - - steps: - - "checkout" - - - run: - name: "Install tox" - command: | - pip install --user tox - - - run: - name: "Static-ish code checks" - command: | - ~/.local/bin/tox -e codechecks - codechecks3: docker: - <<: *DOCKERHUB_AUTH @@ -368,7 +348,8 @@ jobs: - <<: *DOCKERHUB_AUTH image: "tahoelafsci/ubuntu:20.04" user: "nobody" - + environment: + TAHOE_LAFS_TOX_ENVIRONMENT: "py39" oraclelinux-8: &RHEL_DERIV docker: @@ -376,7 +357,9 @@ jobs: image: "tahoelafsci/oraclelinux:8-py3.8" user: "nobody" - environment: *UTF_8_ENVIRONMENT + environment: + <<: *UTF_8_ENVIRONMENT + TAHOE_LAFS_TOX_ENVIRONMENT: "py38" # pip cannot install packages if the working directory is not readable. # We want to run a lot of steps as nobody instead of as root. From 3ccd051473f364cc37cbc1c94658cb55f2ac4320 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 14 Feb 2022 11:40:03 -0500 Subject: [PATCH 0703/2309] Correct image. --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 9050c30b4..bf0cbd8b4 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -346,7 +346,7 @@ jobs: <<: *DEBIAN docker: - <<: *DOCKERHUB_AUTH - image: "tahoelafsci/ubuntu:20.04" + image: "tahoelafsci/ubuntu:20.04-py3.9" user: "nobody" environment: TAHOE_LAFS_TOX_ENVIRONMENT: "py39" From d976524bb09debd3f396da2a7d0b54d3dbcff3f3 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 14 Feb 2022 11:41:51 -0500 Subject: [PATCH 0704/2309] Let's see if this is necessary any more. --- tox.ini | 3 --- 1 file changed, 3 deletions(-) diff --git a/tox.ini b/tox.ini index e42958b28..dcbcc3ab6 100644 --- a/tox.ini +++ b/tox.ini @@ -211,9 +211,6 @@ commands = sphinx-build -W -b html -d {toxinidir}/docs/_build/doctrees {toxinidir}/docs {toxinidir}/docs/_build/html [testenv:pyinstaller] -# We override this to pass --no-use-pep517 because pyinstaller (3.4, at least) -# is broken when this feature is enabled. -install_command = python -m pip install --no-use-pep517 {opts} {packages} extras = deps = {[testenv]deps} From dfe7de54a25868f8cdb636f03371d991ce9a031c Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 15 Feb 2022 09:59:04 -0500 Subject: [PATCH 0705/2309] Upgrade some versions. --- tox.ini | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/tox.ini b/tox.ini index dcbcc3ab6..3916b807c 100644 --- a/tox.ini +++ b/tox.ini @@ -34,12 +34,10 @@ deps = # happening at the time. The versions selected here are just the current # versions at the time. Bumping them to keep up with future releases is # fine as long as those releases are known to actually work. - # - # For now these are versions that support Python 2. - pip==20.3.4 - setuptools==44.1.1 - wheel==0.36.2 - subunitreporter==19.3.2 + pip==22.0.3 + setuptools==60.9.1 + wheel==0.37.1 + subunitreporter==22.2.0 # As an exception, we don't pin certifi because it contains CA # certificates which necessarily change over time. Pinning this is # guaranteed to cause things to break eventually as old certificates From 1fd8603673f6dc89fdd5a25f55fff5dfd82a46eb Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 15 Feb 2022 10:07:04 -0500 Subject: [PATCH 0706/2309] Use modern Docker version (with bugfixes for modern distributions). --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index bf0cbd8b4..ac054b7eb 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -535,6 +535,7 @@ jobs: steps: - "checkout" - "setup_remote_docker" + version: "20.10.11" - run: name: "Log in to Dockerhub" command: | From 4315f6a641893022b8c1c3d1c7e670cb56f9746c Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 15 Feb 2022 10:12:31 -0500 Subject: [PATCH 0707/2309] Run on Python 3. --- pyinstaller.spec | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pyinstaller.spec b/pyinstaller.spec index 875629c13..eece50757 100644 --- a/pyinstaller.spec +++ b/pyinstaller.spec @@ -11,7 +11,10 @@ import struct import sys -if not hasattr(sys, 'real_prefix'): +try: + import allmydata + del allmydata +except ImportError: sys.exit("Please run inside a virtualenv with Tahoe-LAFS installed.") From 95c32ef2eec363b369a3439aa1eb39e7a6aefe7d Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 15 Feb 2022 10:13:35 -0500 Subject: [PATCH 0708/2309] Fix syntax. --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ac054b7eb..afb82f79a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -534,7 +534,7 @@ jobs: steps: - "checkout" - - "setup_remote_docker" + - setup_remote_docker: version: "20.10.11" - run: name: "Log in to Dockerhub" From 7ea106a018a92f13ba73fce9c8ac9691315b4284 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 15 Feb 2022 10:19:56 -0500 Subject: [PATCH 0709/2309] Switch back to building Docker images on a schedule. --- .circleci/config.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index afb82f79a..7a4be90ab 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -71,13 +71,13 @@ workflows: # Build the Docker images used by the ci jobs. This makes the ci jobs # faster and takes various spurious failures out of the critical path. triggers: - # # Build once a day - # - schedule: - # cron: "0 0 * * *" - # filters: - # branches: - # only: - # - "master" + # Build once a day + - schedule: + cron: "0 0 * * *" + filters: + branches: + only: + - "master" jobs: # Every job that pushes a Docker image from Docker Hub needs to provide From be2590f9b8b95fccc795e0df47a89df58e882cc9 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 15 Feb 2022 10:20:29 -0500 Subject: [PATCH 0710/2309] Python 2 is now unsupported. --- README.rst | 9 ++++----- setup.py | 38 ++++++++++---------------------------- 2 files changed, 14 insertions(+), 33 deletions(-) diff --git a/README.rst b/README.rst index 0b73b520e..317378fae 100644 --- a/README.rst +++ b/README.rst @@ -53,12 +53,11 @@ For more detailed instructions, read `Installing Tahoe-LAFS `__ to learn how to set up your first Tahoe-LAFS node. -🐍 Python 3 Support --------------------- +🐍 Python 2 +----------- -Python 3 support has been introduced starting with Tahoe-LAFS 1.16.0, alongside Python 2. -System administrators are advised to start running Tahoe on Python 3 and should expect Python 2 support to be dropped in a future version. -Please, feel free to file issues if you run into bugs while running Tahoe on Python 3. +Python 3.7 or later is now required. +If you are still using Python 2.7, use Tahoe-LAFS version 1.17.1. 🤖 Issues diff --git a/setup.py b/setup.py index c38c0bfec..da4950d2e 100644 --- a/setup.py +++ b/setup.py @@ -55,8 +55,7 @@ install_requires = [ # * foolscap >= 0.12.6 has an i2p.sam_endpoint() that takes kwargs # * foolscap 0.13.2 drops i2p support completely # * foolscap >= 21.7 is necessary for Python 3 with i2p support. - "foolscap == 0.13.1 ; python_version < '3.0'", - "foolscap >= 21.7.0 ; python_version > '3.0'", + "foolscap >= 21.7.0", # * cryptography 2.6 introduced some ed25519 APIs we rely on. Note that # Twisted[conch] also depends on cryptography and Twisted[tls] @@ -106,16 +105,10 @@ install_requires = [ # for 'tahoe invite' and 'tahoe join' "magic-wormhole >= 0.10.2", - # Eliot is contemplating dropping Python 2 support. Stick to a version we - # know works on Python 2.7. - "eliot ~= 1.7 ; python_version < '3.0'", - # On Python 3, we want a new enough version to support custom JSON encoders. - "eliot >= 1.13.0 ; python_version > '3.0'", + # We want a new enough version to support custom JSON encoders. + "eliot >= 1.13.0", - # Pyrsistent 0.17.0 (which we use by way of Eliot) has dropped - # Python 2 entirely; stick to the version known to work for us. - "pyrsistent < 0.17.0 ; python_version < '3.0'", - "pyrsistent ; python_version > '3.0'", + "pyrsistent", # A great way to define types of values. "attrs >= 18.2.0", @@ -135,14 +128,8 @@ install_requires = [ # Linux distribution detection: "distro >= 1.4.0", - # Backported configparser for Python 2: - "configparser ; python_version < '3.0'", - - # For the RangeMap datastructure. Need 2.0.2 at least for bugfixes. Python - # 2 doesn't actually need this, since HTTP storage protocol isn't supported - # there, so we just pick whatever version so that code imports. - "collections-extended >= 2.0.2 ; python_version > '3.0'", - "collections-extended ; python_version < '3.0'", + # For the RangeMap datastructure. Need 2.0.2 at least for bugfixes. + "collections-extended >= 2.0.2", # HTTP server and client "klein", @@ -201,8 +188,7 @@ trove_classifiers=[ "Natural Language :: English", "Programming Language :: C", "Programming Language :: Python", - "Programming Language :: Python :: 2", - "Programming Language :: Python :: 2.7", + "Programming Language :: Python :: 3", "Topic :: Utilities", "Topic :: System :: Systems Administration", "Topic :: System :: Filesystems", @@ -229,7 +215,7 @@ def run_command(args, cwd=None): use_shell = sys.platform == "win32" try: p = subprocess.Popen(args, stdout=subprocess.PIPE, cwd=cwd, shell=use_shell) - except EnvironmentError as e: # if this gives a SyntaxError, note that Tahoe-LAFS requires Python 2.7+ + except EnvironmentError as e: # if this gives a SyntaxError, note that Tahoe-LAFS requires Python 3.7+ print("Warning: unable to run %r." % (" ".join(args),)) print(e) return None @@ -380,8 +366,8 @@ setup(name="tahoe-lafs", # also set in __init__.py package_dir = {'':'src'}, packages=find_packages('src') + ['allmydata.test.plugins'], classifiers=trove_classifiers, - # We support Python 2.7, and Python 3.7 or later. - python_requires=">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*, !=3.6.*", + # We support Python 3.7 or later. + python_requires=">=3.7", install_requires=install_requires, extras_require={ # Duplicate the Twisted pywin32 dependency here. See @@ -400,10 +386,6 @@ setup(name="tahoe-lafs", # also set in __init__.py "tox", "pytest", "pytest-twisted", - # XXX: decorator isn't a direct dependency, but pytest-twisted - # depends on decorator, and decorator 5.x isn't compatible with - # Python 2.7. - "decorator < 5", "hypothesis >= 3.6.1", "towncrier", "testtools", From bd907289468858d1a8e0d0a46a383ffb03516ce2 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 15 Feb 2022 10:26:54 -0500 Subject: [PATCH 0711/2309] Re-add missing environment. --- .circleci/config.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 7a4be90ab..b1d73e89f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -264,6 +264,7 @@ jobs: image: "tahoelafsci/debian:11-py3.9" user: "nobody" environment: + <<: *UTF_8_ENVIRONMENT TAHOE_LAFS_TOX_ENVIRONMENT: "py39" # Restore later using PyPy3.8 @@ -349,6 +350,7 @@ jobs: image: "tahoelafsci/ubuntu:20.04-py3.9" user: "nobody" environment: + <<: *UTF_8_ENVIRONMENT TAHOE_LAFS_TOX_ENVIRONMENT: "py39" oraclelinux-8: &RHEL_DERIV From 3255f93a5c1f94af75d05a2d07bf8851d74df369 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 15 Feb 2022 10:47:22 -0500 Subject: [PATCH 0712/2309] Try newer version of Chutney. --- integration/conftest.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/integration/conftest.py b/integration/conftest.py index ef5c518a8..e284b5cba 100644 --- a/integration/conftest.py +++ b/integration/conftest.py @@ -462,10 +462,8 @@ def chutney(reactor, temp_dir): ) pytest_twisted.blockon(proto.done) - # XXX: Here we reset Chutney to the last revision known to work - # with Python 2, as a workaround for Chutney moving to Python 3. - # When this is no longer necessary, we will have to drop this and - # add '--depth=1' back to the above 'git clone' subprocess. + # XXX: Here we reset Chutney to a specific revision known to work, + # since there are no stability guarantees or releases yet. proto = _DumpOutputProtocol(None) reactor.spawnProcess( proto, @@ -473,7 +471,7 @@ def chutney(reactor, temp_dir): ( 'git', '-C', chutney_dir, 'reset', '--hard', - '99bd06c7554b9113af8c0877b6eca4ceb95dcbaa' + 'c825cba0bcd813c644c6ac069deeb7347d3200ee' ), env=environ, ) From 6190399aef2f61f318bb5a0c78e5a1e9f8a7e335 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 15 Feb 2022 14:33:00 -0500 Subject: [PATCH 0713/2309] Just codechecks. --- .circleci/config.yml | 6 +++--- Makefile | 2 +- docs/release-checklist.rst | 2 +- tox.ini | 6 +++--- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index b1d73e89f..cf0c66aff 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -44,7 +44,7 @@ workflows: # {} # Other assorted tasks and configurations - - "codechecks3": + - "codechecks": {} - "pyinstaller": {} @@ -128,7 +128,7 @@ jobs: # Since this job is never scheduled this step is never run so the # actual value here is irrelevant. - codechecks3: + codechecks: docker: - <<: *DOCKERHUB_AUTH image: "cimg/python:3.9" @@ -144,7 +144,7 @@ jobs: - run: name: "Static-ish code checks" command: | - ~/.local/bin/tox -e codechecks3 + ~/.local/bin/tox -e codechecks pyinstaller: docker: diff --git a/Makefile b/Makefile index fa8351535..5cbd863a3 100644 --- a/Makefile +++ b/Makefile @@ -33,7 +33,7 @@ default: ## Run all tests and code reports test: .tox/create-venvs.log # Run codechecks first since it takes the least time to report issues early. - tox --develop -e codechecks3 + tox --develop -e codechecks # Run all the test environments in parallel to reduce run-time tox --develop -p auto -e 'py37' .PHONY: test-venv-coverage diff --git a/docs/release-checklist.rst b/docs/release-checklist.rst index 5697ab95f..aa5531b59 100644 --- a/docs/release-checklist.rst +++ b/docs/release-checklist.rst @@ -122,7 +122,7 @@ they will need to evaluate which contributors' signatures they trust. - these should all pass: - - tox -e py37,codechecks3,docs,integration + - tox -e py37,codechecks,docs,integration - these can fail (ideally they should not of course): diff --git a/tox.ini b/tox.ini index 3916b807c..525b5428c 100644 --- a/tox.ini +++ b/tox.ini @@ -7,7 +7,7 @@ # the tox-gh-actions package. [gh-actions] python = - 3.7: py37-coverage,typechecks,codechecks3 + 3.7: py37-coverage,typechecks,codechecks 3.8: py38-coverage 3.9: py39-coverage pypy-3.7: pypy3 @@ -16,7 +16,7 @@ python = twisted = 1 [tox] -envlist = typechecks,codechecks3,py{37,38,39}-{coverage},pypy27,pypy3,integration,integration3 +envlist = typechecks,codechecks,py{37,38,39}-{coverage},pypy27,pypy3,integration,integration3 minversion = 2.4 [testenv] @@ -107,7 +107,7 @@ commands = coverage report -[testenv:codechecks3] +[testenv:codechecks] basepython = python3 deps = # Newer versions of PyLint have buggy configuration From b7b71b2de7a6526564f5d509e29d6168ae1eb776 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 15 Feb 2022 14:33:37 -0500 Subject: [PATCH 0714/2309] Clarify. --- newsfragments/3873.incompat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/newsfragments/3873.incompat b/newsfragments/3873.incompat index 7e71b1504..da8a5fb0e 100644 --- a/newsfragments/3873.incompat +++ b/newsfragments/3873.incompat @@ -1 +1 @@ -Dropped support for Python 2. \ No newline at end of file +Python 3.7 or later is now required; Python 2 is no longer supported. \ No newline at end of file From 21e288a4d0e4b9078edd17589fd72eeda4f5a079 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 15 Feb 2022 14:35:18 -0500 Subject: [PATCH 0715/2309] Technically don't support 3.10 yet. --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index da4950d2e..8bb2b57aa 100644 --- a/setup.py +++ b/setup.py @@ -366,8 +366,8 @@ setup(name="tahoe-lafs", # also set in __init__.py package_dir = {'':'src'}, packages=find_packages('src') + ['allmydata.test.plugins'], classifiers=trove_classifiers, - # We support Python 3.7 or later. - python_requires=">=3.7", + # We support Python 3.7 or later. 3.10 is not supported quite yet. + python_requires=">=3.7, <3.10", install_requires=install_requires, extras_require={ # Duplicate the Twisted pywin32 dependency here. See From 510102dab1f6a5b38a14bffd789bfd5188c8b6fa Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 15 Feb 2022 14:37:18 -0500 Subject: [PATCH 0716/2309] Maybe we can use modern tor now. --- .github/workflows/ci.yml | 7 +------ newsfragments/3744.minor | 0 2 files changed, 1 insertion(+), 6 deletions(-) create mode 100644 newsfragments/3744.minor diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a4d8daca5..a0f7889b5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -170,15 +170,10 @@ jobs: if: matrix.os == 'ubuntu-latest' run: sudo apt install tor - # TODO: See https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3744. - # We have to use an older version of Tor for running integration - # tests on macOS. - name: Install Tor [macOS, ${{ matrix.python-version }} ] if: ${{ contains(matrix.os, 'macos') }} run: | - brew extract --version 0.4.5.8 tor homebrew/cask - brew install tor@0.4.5.8 - brew link --overwrite tor@0.4.5.8 + brew install tor - name: Install Tor [Windows] if: matrix.os == 'windows-latest' diff --git a/newsfragments/3744.minor b/newsfragments/3744.minor new file mode 100644 index 000000000..e69de29bb From 3a859e3cac580370682c5124744dd486d580600c Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 15 Feb 2022 14:51:55 -0500 Subject: [PATCH 0717/2309] Try a version that matches Ubuntu's. --- .github/workflows/ci.yml | 7 ++++++- newsfragments/3744.minor | 0 2 files changed, 6 insertions(+), 1 deletion(-) delete mode 100644 newsfragments/3744.minor diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a0f7889b5..8e6b0fd67 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -170,10 +170,15 @@ jobs: if: matrix.os == 'ubuntu-latest' run: sudo apt install tor + # TODO: See https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3744. + # We have to use an older version of Tor for running integration + # tests on macOS. - name: Install Tor [macOS, ${{ matrix.python-version }} ] if: ${{ contains(matrix.os, 'macos') }} run: | - brew install tor + brew extract --version 0.4.2.7 tor homebrew/cask + brew install tor@0.4.2.7 + brew link --overwrite tor@0.4.2.7 - name: Install Tor [Windows] if: matrix.os == 'windows-latest' diff --git a/newsfragments/3744.minor b/newsfragments/3744.minor deleted file mode 100644 index e69de29bb..000000000 From 84094b5ca00bd1af8168030f48678748d9ba4faa Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 16 Feb 2022 09:31:12 -0500 Subject: [PATCH 0718/2309] Try version that Windows uses. --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8e6b0fd67..3a12f51f2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -176,9 +176,9 @@ jobs: - name: Install Tor [macOS, ${{ matrix.python-version }} ] if: ${{ contains(matrix.os, 'macos') }} run: | - brew extract --version 0.4.2.7 tor homebrew/cask - brew install tor@0.4.2.7 - brew link --overwrite tor@0.4.2.7 + brew extract --version 0.4.6.9 tor homebrew/cask + brew install tor@0.4.6.9 + brew link --overwrite tor@0.4.6.9 - name: Install Tor [Windows] if: matrix.os == 'windows-latest' From 2928a480ff317f4acd50c0572c69d4b0b566e70e Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 16 Feb 2022 21:46:24 -0700 Subject: [PATCH 0719/2309] RSA key-size is not configurable, it's 2048bits --- src/allmydata/client.py | 32 ++++----------------- src/allmydata/crypto/rsa.py | 8 ++---- src/allmydata/nodemaker.py | 4 +-- src/allmydata/test/common.py | 3 -- src/allmydata/test/common_system.py | 5 ---- src/allmydata/test/mutable/test_problems.py | 3 +- src/allmydata/test/mutable/util.py | 13 ++++----- src/allmydata/test/no_network.py | 2 -- 8 files changed, 16 insertions(+), 54 deletions(-) diff --git a/src/allmydata/client.py b/src/allmydata/client.py index 645e157b6..56ecdc6ed 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -168,29 +168,12 @@ class SecretHolder(object): class KeyGenerator(object): """I create RSA keys for mutable files. Each call to generate() returns a - single keypair. The keysize is specified first by the keysize= argument - to generate(), then with a default set by set_default_keysize(), then - with a built-in default of 2048 bits.""" - def __init__(self): - self.default_keysize = 2048 + single keypair.""" - def set_default_keysize(self, keysize): - """Call this to override the size of the RSA keys created for new - mutable files which don't otherwise specify a size. This will affect - all subsequent calls to generate() without a keysize= argument. The - default size is 2048 bits. Test cases should call this method once - during setup, to cause me to create smaller keys, so the unit tests - run faster.""" - self.default_keysize = keysize - - def generate(self, keysize=None): + def generate(self): """I return a Deferred that fires with a (verifyingkey, signingkey) - pair. I accept a keysize in bits (2048 bit keys are standard, smaller - keys are used for testing). If you do not provide a keysize, I will - use my default, which is set by a call to set_default_keysize(). If - set_default_keysize() has never been called, I will create 2048 bit - keys.""" - keysize = keysize or self.default_keysize + pair. The returned key will be 2048 bit""" + keysize = 2048 # RSA key generation for a 2048 bit key takes between 0.8 and 3.2 # secs signer, verifier = rsa.create_signing_keypair(keysize) @@ -993,9 +976,6 @@ class _Client(node.Node, pollmixin.PollMixin): helper_furlfile = self.config.get_private_path("helper.furl").encode(get_filesystem_encoding()) self.tub.registerReference(self.helper, furlFile=helper_furlfile) - def set_default_mutable_keysize(self, keysize): - self._key_generator.set_default_keysize(keysize) - def _get_tempdir(self): """ Determine the path to the directory where temporary files for this node @@ -1096,8 +1076,8 @@ class _Client(node.Node, pollmixin.PollMixin): def create_immutable_dirnode(self, children, convergence=None): return self.nodemaker.create_immutable_directory(children, convergence) - def create_mutable_file(self, contents=None, keysize=None, version=None): - return self.nodemaker.create_mutable_file(contents, keysize, + def create_mutable_file(self, contents=None, version=None): + return self.nodemaker.create_mutable_file(contents, version=version) def upload(self, uploadable, reactor=None): diff --git a/src/allmydata/crypto/rsa.py b/src/allmydata/crypto/rsa.py index d290388da..95cf01413 100644 --- a/src/allmydata/crypto/rsa.py +++ b/src/allmydata/crypto/rsa.py @@ -81,13 +81,9 @@ def create_signing_keypair_from_string(private_key_der): raise ValueError( "Private Key did not decode to an RSA key" ) - if priv_key.key_size < 2048: + if priv_key.key_size != 2048: raise ValueError( - "Private Key is smaller than 2048 bits" - ) - if priv_key.key_size > (2048 * 8): - raise ValueError( - "Private Key is unreasonably large" + "Private Key must be 2048 bits" ) return priv_key, priv_key.public_key() diff --git a/src/allmydata/nodemaker.py b/src/allmydata/nodemaker.py index 6b0b77c5c..23ba4b451 100644 --- a/src/allmydata/nodemaker.py +++ b/src/allmydata/nodemaker.py @@ -126,12 +126,12 @@ class NodeMaker(object): return self._create_dirnode(filenode) return None - def create_mutable_file(self, contents=None, keysize=None, version=None): + def create_mutable_file(self, contents=None, version=None): if version is None: version = self.mutable_file_default n = MutableFileNode(self.storage_broker, self.secret_holder, self.default_encoding_parameters, self.history) - d = self.key_generator.generate(keysize) + d = self.key_generator.generate() d.addCallback(n.create_with_keys, contents, version=version) d.addCallback(lambda res: n) return d diff --git a/src/allmydata/test/common.py b/src/allmydata/test/common.py index 4d58fd0e5..b652b2e48 100644 --- a/src/allmydata/test/common.py +++ b/src/allmydata/test/common.py @@ -133,9 +133,6 @@ from subprocess import ( PIPE, ) -TEST_RSA_KEY_SIZE = 522 -TEST_RSA_KEY_SIZE = 2048 - EMPTY_CLIENT_CONFIG = config_from_string( "/dev/null", "tub.port", diff --git a/src/allmydata/test/common_system.py b/src/allmydata/test/common_system.py index 0c424136a..9851d2b91 100644 --- a/src/allmydata/test/common_system.py +++ b/src/allmydata/test/common_system.py @@ -34,7 +34,6 @@ from twisted.python.filepath import ( ) from .common import ( - TEST_RSA_KEY_SIZE, SameProcessStreamEndpointAssigner, ) @@ -736,7 +735,6 @@ class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin): c = yield client.create_client(basedirs[0]) c.setServiceParent(self.sparent) self.clients.append(c) - c.set_default_mutable_keysize(TEST_RSA_KEY_SIZE) with open(os.path.join(basedirs[0],"private","helper.furl"), "r") as f: helper_furl = f.read() @@ -754,7 +752,6 @@ class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin): c = yield client.create_client(basedirs[i]) c.setServiceParent(self.sparent) self.clients.append(c) - c.set_default_mutable_keysize(TEST_RSA_KEY_SIZE) log.msg("STARTING") yield self.wait_for_connections() log.msg("CONNECTED") @@ -838,7 +835,6 @@ class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin): def _stopped(res): new_c = yield client.create_client(self.getdir("client%d" % num)) self.clients[num] = new_c - new_c.set_default_mutable_keysize(TEST_RSA_KEY_SIZE) new_c.setServiceParent(self.sparent) d.addCallback(_stopped) d.addCallback(lambda res: self.wait_for_connections()) @@ -877,7 +873,6 @@ class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin): c = yield client.create_client(basedir.path) self.clients.append(c) - c.set_default_mutable_keysize(TEST_RSA_KEY_SIZE) self.numclients += 1 if add_to_sparent: c.setServiceParent(self.sparent) diff --git a/src/allmydata/test/mutable/test_problems.py b/src/allmydata/test/mutable/test_problems.py index 4bcb8161b..d3a779905 100644 --- a/src/allmydata/test/mutable/test_problems.py +++ b/src/allmydata/test/mutable/test_problems.py @@ -26,7 +26,6 @@ from allmydata.mutable.common import \ NotEnoughServersError from allmydata.mutable.publish import MutableData from allmydata.storage.common import storage_index_to_dir -from ..common import TEST_RSA_KEY_SIZE from ..no_network import GridTestMixin from .. import common_util as testutil from ..common_util import DevNullDictionary @@ -219,7 +218,7 @@ class Problems(GridTestMixin, AsyncTestCase, testutil.ShouldFailMixin): # use #467 static-server-selection to disable permutation and force # the choice of server for share[0]. - d = nm.key_generator.generate(TEST_RSA_KEY_SIZE) + d = nm.key_generator.generate() def _got_key(keypair): (pubkey, privkey) = keypair nm.key_generator = SameKeyGenerator(pubkey, privkey) diff --git a/src/allmydata/test/mutable/util.py b/src/allmydata/test/mutable/util.py index dac61a6e3..bed350652 100644 --- a/src/allmydata/test/mutable/util.py +++ b/src/allmydata/test/mutable/util.py @@ -25,7 +25,6 @@ from allmydata.storage_client import StorageFarmBroker from allmydata.mutable.layout import MDMFSlotReadProxy from allmydata.mutable.publish import MutableData from ..common import ( - TEST_RSA_KEY_SIZE, EMPTY_CLIENT_CONFIG, ) @@ -287,7 +286,7 @@ def make_storagebroker_with_peers(peers): return storage_broker -def make_nodemaker(s=None, num_peers=10, keysize=TEST_RSA_KEY_SIZE): +def make_nodemaker(s=None, num_peers=10): """ Make a ``NodeMaker`` connected to some number of fake storage servers. @@ -298,20 +297,20 @@ def make_nodemaker(s=None, num_peers=10, keysize=TEST_RSA_KEY_SIZE): the node maker. """ storage_broker = make_storagebroker(s, num_peers) - return make_nodemaker_with_storage_broker(storage_broker, keysize) + return make_nodemaker_with_storage_broker(storage_broker) -def make_nodemaker_with_peers(peers, keysize=TEST_RSA_KEY_SIZE): +def make_nodemaker_with_peers(peers): """ Make a ``NodeMaker`` connected to the given storage servers. :param list peers: The storage servers to associate with the node maker. """ storage_broker = make_storagebroker_with_peers(peers) - return make_nodemaker_with_storage_broker(storage_broker, keysize) + return make_nodemaker_with_storage_broker(storage_broker) -def make_nodemaker_with_storage_broker(storage_broker, keysize): +def make_nodemaker_with_storage_broker(storage_broker): """ Make a ``NodeMaker`` using the given storage broker. @@ -319,8 +318,6 @@ def make_nodemaker_with_storage_broker(storage_broker, keysize): """ sh = client.SecretHolder(b"lease secret", b"convergence secret") keygen = client.KeyGenerator() - if keysize: - keygen.set_default_keysize(keysize) nodemaker = NodeMaker(storage_broker, sh, None, None, None, {"k": 3, "n": 10}, SDMF_VERSION, keygen) diff --git a/src/allmydata/test/no_network.py b/src/allmydata/test/no_network.py index 97cb371e6..ed742e624 100644 --- a/src/allmydata/test/no_network.py +++ b/src/allmydata/test/no_network.py @@ -61,7 +61,6 @@ from allmydata.storage_client import ( _StorageServer, ) from .common import ( - TEST_RSA_KEY_SIZE, SameProcessStreamEndpointAssigner, ) @@ -393,7 +392,6 @@ class NoNetworkGrid(service.MultiService): if not c: c = yield create_no_network_client(clientdir) - c.set_default_mutable_keysize(TEST_RSA_KEY_SIZE) c.nodeid = clientid c.short_nodeid = b32encode(clientid).lower()[:8] From 025a3bb455f92ba92191eba5b010e9ace8bd27f1 Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 16 Feb 2022 22:00:02 -0700 Subject: [PATCH 0720/2309] news --- newsfragments/3828.feature | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 newsfragments/3828.feature diff --git a/newsfragments/3828.feature b/newsfragments/3828.feature new file mode 100644 index 000000000..f498421c8 --- /dev/null +++ b/newsfragments/3828.feature @@ -0,0 +1,8 @@ +Mutables' RSA keys are spec'd at 2048 bits + +Some code existed to allow tests to shorten this and it's +conceptually possible a modified client produced mutables +with different key-sizes. However, the spec says that they +must be 2048 bits. If you happen to have a capability with +a key-size different from 2048 you may use 1.17.1 or earlier +to read the content. From a7e4add602b390f2e7f482725182ce8e62475ac2 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 22 Feb 2022 11:25:13 -0500 Subject: [PATCH 0721/2309] Simplify. --- .github/workflows/ci.yml | 6 ++---- tox.ini | 15 +++------------ 2 files changed, 5 insertions(+), 16 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3a12f51f2..5f33dcd96 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -176,9 +176,7 @@ jobs: - name: Install Tor [macOS, ${{ matrix.python-version }} ] if: ${{ contains(matrix.os, 'macos') }} run: | - brew extract --version 0.4.6.9 tor homebrew/cask - brew install tor@0.4.6.9 - brew link --overwrite tor@0.4.6.9 + brew install tor - name: Install Tor [Windows] if: matrix.os == 'windows-latest' @@ -218,7 +216,7 @@ jobs: run: python misc/build_helpers/show-tool-versions.py - name: Run "Python 3 integration tests" - run: tox -e integration3 + run: tox -e integration - name: Upload eliot.log in case of failure uses: actions/upload-artifact@v1 diff --git a/tox.ini b/tox.ini index 525b5428c..52c199d41 100644 --- a/tox.ini +++ b/tox.ini @@ -16,7 +16,7 @@ python = twisted = 1 [tox] -envlist = typechecks,codechecks,py{37,38,39}-{coverage},pypy27,pypy3,integration,integration3 +envlist = typechecks,codechecks,py{37,38,39}-{coverage},pypy27,pypy3,integration minversion = 2.4 [testenv] @@ -86,21 +86,12 @@ commands = coverage: coverage report [testenv:integration] -setenv = - COVERAGE_PROCESS_START=.coveragerc -commands = - # NOTE: 'run with "py.test --keep-tempdir -s -v integration/" to debug failures' - py.test --timeout=1800 --coverage -v {posargs:integration} - coverage combine - coverage report - - -[testenv:integration3] basepython = python3 setenv = COVERAGE_PROCESS_START=.coveragerc + # Without this, temporary file paths are too long on macOS, breaking tor + commands = - python --version # NOTE: 'run with "py.test --keep-tempdir -s -v integration/" to debug failures' py.test --timeout=1800 --coverage -v {posargs:integration} coverage combine From 52f0e18d6bfe2cc7e5e53d1b6cd77ecc2bb78e7f Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 22 Feb 2022 11:26:00 -0500 Subject: [PATCH 0722/2309] Fix for overly-long temporary paths for unix sockets on macOS. --- tox.ini | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 52c199d41..c0bc159da 100644 --- a/tox.ini +++ b/tox.ini @@ -87,10 +87,13 @@ commands = [testenv:integration] basepython = python3 +platform = mylinux: linux + mymacos: darwin + mywindows: win32 setenv = COVERAGE_PROCESS_START=.coveragerc # Without this, temporary file paths are too long on macOS, breaking tor - + mymacos: TMPDIR=/tmp commands = # NOTE: 'run with "py.test --keep-tempdir -s -v integration/" to debug failures' py.test --timeout=1800 --coverage -v {posargs:integration} From 5647b4aee0ce88b8e8761588453dc608f0307d25 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 22 Feb 2022 11:40:16 -0500 Subject: [PATCH 0723/2309] Try to fix macOS another way. --- .github/workflows/ci.yml | 5 +++++ tox.ini | 2 -- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5f33dcd96..90778c6ba 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -216,6 +216,11 @@ jobs: run: python misc/build_helpers/show-tool-versions.py - name: Run "Python 3 integration tests" + env: + # On macOS this is necessary to ensure unix socket paths for tor + # aren't too long. On Windows tox won't pass it through so it has no + # effect. On Linux it doesn't make a difference one way or another. + TMPDIR: "/tmp" run: tox -e integration - name: Upload eliot.log in case of failure diff --git a/tox.ini b/tox.ini index c0bc159da..9a28b7b30 100644 --- a/tox.ini +++ b/tox.ini @@ -92,8 +92,6 @@ platform = mylinux: linux mywindows: win32 setenv = COVERAGE_PROCESS_START=.coveragerc - # Without this, temporary file paths are too long on macOS, breaking tor - mymacos: TMPDIR=/tmp commands = # NOTE: 'run with "py.test --keep-tempdir -s -v integration/" to debug failures' py.test --timeout=1800 --coverage -v {posargs:integration} From 82eb0d686e704dbdbb82ade998c6739ef78a6b96 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 22 Feb 2022 11:18:45 -0700 Subject: [PATCH 0724/2309] Update newsfragments/3828.feature Co-authored-by: Jean-Paul Calderone --- newsfragments/3828.feature | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/newsfragments/3828.feature b/newsfragments/3828.feature index f498421c8..d396439b0 100644 --- a/newsfragments/3828.feature +++ b/newsfragments/3828.feature @@ -1,4 +1,4 @@ -Mutables' RSA keys are spec'd at 2048 bits +The implementation of SDMF and MDMF (mutables) now requires RSA keys to be exactly 2048 bits, aligning them with the specification. Some code existed to allow tests to shorten this and it's conceptually possible a modified client produced mutables From c6ee41ab5c9413369fc697ce425ba64adc2cb417 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 23 Feb 2022 10:49:39 -0500 Subject: [PATCH 0725/2309] Add PyPy3. --- .github/workflows/ci.yml | 4 ++++ tox.ini | 6 ++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 90778c6ba..b06a7534e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -41,10 +41,14 @@ jobs: - 3.7 - 3.8 - 3.9 + - pypy-3.7 + - pypy-3.8 include: # On macOS don't bother with 3.7-3.8, just to get faster builds. - os: macos-latest python-version: 3.9 + - os: macos-latest + python-version: pypy3.8 steps: # See https://github.com/actions/checkout. A fetch-depth of 0 diff --git a/tox.ini b/tox.ini index 9a28b7b30..f628b8568 100644 --- a/tox.ini +++ b/tox.ini @@ -10,13 +10,15 @@ python = 3.7: py37-coverage,typechecks,codechecks 3.8: py38-coverage 3.9: py39-coverage - pypy-3.7: pypy3 + pypy-3.7: pypy37 + pypy-3.8: pypy38 + pypy-3.9: pypy39 [pytest] twisted = 1 [tox] -envlist = typechecks,codechecks,py{37,38,39}-{coverage},pypy27,pypy3,integration +envlist = typechecks,codechecks,py{37,38,39}-{coverage},pypy27,pypy37,pypy38,pypy39,integration minversion = 2.4 [testenv] From e5c49c890b651f64a7b92c023f59eca945cc057c Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 23 Feb 2022 10:50:41 -0500 Subject: [PATCH 0726/2309] News file. --- newsfragments/3697.minor | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/3697.minor diff --git a/newsfragments/3697.minor b/newsfragments/3697.minor new file mode 100644 index 000000000..356b42748 --- /dev/null +++ b/newsfragments/3697.minor @@ -0,0 +1 @@ +Added support for PyPy3 (3.7 and 3.8). \ No newline at end of file From f81cd6e595ff071a9e8e3d0aa0e491730a1e2dca Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 23 Feb 2022 10:57:36 -0500 Subject: [PATCH 0727/2309] Use an option also available on PyPy3. --- src/allmydata/test/test_runner.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/allmydata/test/test_runner.py b/src/allmydata/test/test_runner.py index 44c7e1bee..3eb6b8a34 100644 --- a/src/allmydata/test/test_runner.py +++ b/src/allmydata/test/test_runner.py @@ -203,10 +203,10 @@ class BinTahoe(common_util.SignalMixin, unittest.TestCase): # but on Windows we parse the whole command line string ourselves so # we have to have our own implementation of skipping these options. - # -t is a harmless option that warns about tabs so we can add it + # -B is a harmless option that prevents writing bytecode so we can add it # without impacting other behavior noticably. - out, err, returncode = run_bintahoe([u"--version"], python_options=[u"-t"]) - self.assertEqual(returncode, 0) + out, err, returncode = run_bintahoe([u"--version"], python_options=[u"-B"]) + self.assertEqual(returncode, 0, f"Out:\n{out}\nErr:\n{err}") self.assertTrue(out.startswith(allmydata.__appname__ + '/')) def test_help_eliot_destinations(self): From 9d9ec698e0b4fc0c67546a2fb6db3737ddfda169 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 23 Feb 2022 11:07:56 -0500 Subject: [PATCH 0728/2309] Add support for Python 3.10. --- .github/workflows/ci.yml | 3 +++ newsfragments/3697.minor | 2 +- setup.py | 4 ++-- tox.ini | 3 ++- 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b06a7534e..2d0290b86 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -41,6 +41,7 @@ jobs: - 3.7 - 3.8 - 3.9 + - 3.10 - pypy-3.7 - pypy-3.8 include: @@ -49,6 +50,8 @@ jobs: python-version: 3.9 - os: macos-latest python-version: pypy3.8 + - os: macos-latest + python-version: 3.10 steps: # See https://github.com/actions/checkout. A fetch-depth of 0 diff --git a/newsfragments/3697.minor b/newsfragments/3697.minor index 356b42748..8bc959086 100644 --- a/newsfragments/3697.minor +++ b/newsfragments/3697.minor @@ -1 +1 @@ -Added support for PyPy3 (3.7 and 3.8). \ No newline at end of file +Added support for Python 3.10 and PyPy3 (3.7 and 3.8). \ No newline at end of file diff --git a/setup.py b/setup.py index 8bb2b57aa..5285b5d08 100644 --- a/setup.py +++ b/setup.py @@ -366,8 +366,8 @@ setup(name="tahoe-lafs", # also set in __init__.py package_dir = {'':'src'}, packages=find_packages('src') + ['allmydata.test.plugins'], classifiers=trove_classifiers, - # We support Python 3.7 or later. 3.10 is not supported quite yet. - python_requires=">=3.7, <3.10", + # We support Python 3.7 or later. 3.11 is not supported yet. + python_requires=">=3.7, <3.11", install_requires=install_requires, extras_require={ # Duplicate the Twisted pywin32 dependency here. See diff --git a/tox.ini b/tox.ini index f628b8568..57489df89 100644 --- a/tox.ini +++ b/tox.ini @@ -10,6 +10,7 @@ python = 3.7: py37-coverage,typechecks,codechecks 3.8: py38-coverage 3.9: py39-coverage + 3.10: py310-coverage pypy-3.7: pypy37 pypy-3.8: pypy38 pypy-3.9: pypy39 @@ -18,7 +19,7 @@ python = twisted = 1 [tox] -envlist = typechecks,codechecks,py{37,38,39}-{coverage},pypy27,pypy37,pypy38,pypy39,integration +envlist = typechecks,codechecks,py{37,38,39,310}-{coverage},pypy27,pypy37,pypy38,pypy39,integration minversion = 2.4 [testenv] From abe4c6a1f020899892266020f9a0d7f44cb21d00 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 23 Feb 2022 11:09:49 -0500 Subject: [PATCH 0729/2309] Only support PyPy3 on Linux. --- .github/workflows/ci.yml | 11 ++++++----- newsfragments/3697.minor | 2 +- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2d0290b86..c67bdb007 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -42,17 +42,18 @@ jobs: - 3.8 - 3.9 - 3.10 - - pypy-3.7 - - pypy-3.8 include: # On macOS don't bother with 3.7-3.8, just to get faster builds. - os: macos-latest python-version: 3.9 - - os: macos-latest - python-version: pypy3.8 - os: macos-latest python-version: 3.10 - + # We only support PyPy on Linux at the moment. + - os: ubuntu-latest + python-version: pypy-3.7 + - os: ubuntu-latest + python-version: pypy-3.8 + steps: # See https://github.com/actions/checkout. A fetch-depth of 0 # fetches all tags and branches. diff --git a/newsfragments/3697.minor b/newsfragments/3697.minor index 8bc959086..0977d8a6f 100644 --- a/newsfragments/3697.minor +++ b/newsfragments/3697.minor @@ -1 +1 @@ -Added support for Python 3.10 and PyPy3 (3.7 and 3.8). \ No newline at end of file +Added support for Python 3.10. Added support for PyPy3 (3.7 and 3.8, on Linux only). \ No newline at end of file From 5ac3cb644f366747537327d2acb1a8728f0025ba Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 23 Feb 2022 11:11:04 -0500 Subject: [PATCH 0730/2309] These are not numbers. --- .github/workflows/ci.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c67bdb007..0327014ca 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -38,21 +38,21 @@ jobs: - windows-latest - ubuntu-latest python-version: - - 3.7 - - 3.8 - - 3.9 - - 3.10 + - "3.7" + - "3.8" + - "3.9" + - "3.10" include: # On macOS don't bother with 3.7-3.8, just to get faster builds. - os: macos-latest - python-version: 3.9 + python-version: "3.9" - os: macos-latest - python-version: 3.10 + python-version: "3.10" # We only support PyPy on Linux at the moment. - os: ubuntu-latest - python-version: pypy-3.7 + python-version: "pypy-3.7" - os: ubuntu-latest - python-version: pypy-3.8 + python-version: "pypy-3.8" steps: # See https://github.com/actions/checkout. A fetch-depth of 0 From 32cbc7b9dfe09492f85ed84c77b472448c42861a Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 2 Mar 2022 10:35:41 -0500 Subject: [PATCH 0731/2309] Function for getting SPKI hash. --- src/allmydata/storage/http_common.py | 22 ++++++++----- src/allmydata/test/test_storage_http.py | 42 +++++++++++++++++++++++++ 2 files changed, 57 insertions(+), 7 deletions(-) diff --git a/src/allmydata/storage/http_common.py b/src/allmydata/storage/http_common.py index af4224bd0..f570d45d7 100644 --- a/src/allmydata/storage/http_common.py +++ b/src/allmydata/storage/http_common.py @@ -1,15 +1,12 @@ """ Common HTTP infrastructure for the storge server. """ -from future.utils import PY2 - -if PY2: - # fmt: off - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - # fmt: on - from enum import Enum from base64 import b64encode +from hashlib import sha256 + +from cryptography.x509 import Certificate +from cryptography.hazmat.primitives.serialization import Encoding, PublicFormat def swissnum_auth_header(swissnum): # type: (bytes) -> bytes @@ -23,3 +20,14 @@ class Secrets(Enum): LEASE_RENEW = "lease-renew-secret" LEASE_CANCEL = "lease-cancel-secret" UPLOAD = "upload-secret" + + +def get_spki_hash(certificate: Certificate) -> bytes: + """ + Get the public key hash, as per RFC 7469: base64 of sha256 of the public + key encoded in DER + Subject Public Key Info format. + """ + public_key_bytes = certificate.public_key().public_bytes( + Encoding.DER, PublicFormat.SubjectPublicKeyInfo + ) + return b64encode(sha256(public_key_bytes).digest()).strip() diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 982e22859..e9c9e83f5 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -24,6 +24,7 @@ from klein import Klein from hyperlink import DecodedURL from collections_extended import RangeMap from twisted.internet.task import Clock +from cryptography.x509 import load_pem_x509_certificate from .common import SyncTestCase from ..storage.server import StorageServer @@ -41,6 +42,47 @@ from ..storage.http_client import ( ImmutableCreateResult, UploadProgress, ) +from ..storage.http_common import get_spki_hash + + +class HTTPFurlTests(SyncTestCase): + """Tests for HTTP furls.""" + + def test_spki_hash(self): + """The output of ``get_spki_hash()`` matches the semantics of RFC 7469. + + The expected hash was generated using Appendix A instructions in the + RFC:: + + openssl x509 -noout -in certificate.pem -pubkey | \ + openssl asn1parse -noout -inform pem -out public.key + openssl dgst -sha256 -binary public.key | openssl enc -base64 + """ + expected_hash = b"JIj6ezHkdSBlHhrnezAgIC/mrVQHy4KAFyL+8ZNPGPM=" + certificate_text = b"""\ +-----BEGIN CERTIFICATE----- +MIIDWTCCAkECFCf+I+3oEhTfqt+6ruH4qQ4Wst1DMA0GCSqGSIb3DQEBCwUAMGkx +CzAJBgNVBAYTAlpaMRAwDgYDVQQIDAdOb3doZXJlMRQwEgYDVQQHDAtFeGFtcGxl +dG93bjEcMBoGA1UECgwTRGVmYXVsdCBDb21wYW55IEx0ZDEUMBIGA1UEAwwLZXhh +bXBsZS5jb20wHhcNMjIwMzAyMTUyNTQ3WhcNMjMwMzAyMTUyNTQ3WjBpMQswCQYD +VQQGEwJaWjEQMA4GA1UECAwHTm93aGVyZTEUMBIGA1UEBwwLRXhhbXBsZXRvd24x +HDAaBgNVBAoME0RlZmF1bHQgQ29tcGFueSBMdGQxFDASBgNVBAMMC2V4YW1wbGUu +Y29tMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAv9vqtA8Toy9D6xLG +q41iUafSiAXnuirWxML2ct/LAcGJzATg6JctmJxxZQL7vkmaFFPBF6Y39bOGbbEC +M2iQYn2Qemj5fl3IzKTnYLqzryGM0ZwwnNbPyetSe/sksAIYRLzn49d6l+AHR+Dj +GyvoLzIyGUTn41MTDafMNtPgWx1i+65lFW3GHYpEmugu4bjeUPizNja2LrqwvwFu +YXwmKxbIMdioCoRvDGX9SI3/euFstuR4rbOEUDxniYRF5g6reP8UMF30zJzF5j0k +yDg8Z5b1XpKFNZAeyRYxcs9wJCqVlP6BLPDnvNVpMXodnWLeTK+r6YWvGadGVufk +YNC1PwIDAQABMA0GCSqGSIb3DQEBCwUAA4IBAQByrhn78GSS3dJ0pJ6czmhMX5wH ++fauCtt1+Wbn+ctTodTycS+pfULO4gG7wRzhl8KNoOqLmWMjyA2A3mon8kdkD+0C +i8McpoPaGS2wQcqC28Ud6kP9YO81YFyTl4nHVKQ0nmplT+eoLDTCIWMVxHHzxIgs +2ybUluAc+THSjpGxB6kWSAJeg3N+f2OKr+07Yg9LiQ2b8y0eZarpiuuuXCzWeWrQ +PudP0aniyq/gbPhxq0tYF628IBvhDAnr/2kqEmVF2TDr2Sm/Y3PDBuPY6MeIxjnr +ox5zO3LrQmQw11OaIAs2/kviKAoKTFFxeyYcpS5RuKNDZfHQCXlLwt9bySxG +-----END CERTIFICATE----- +""" + certificate = load_pem_x509_certificate(certificate_text) + self.assertEqual(get_spki_hash(certificate), expected_hash) def _post_process(params): From afe6b68ede0796bcbe56dc18bc3b771d437b7586 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 2 Mar 2022 10:38:05 -0500 Subject: [PATCH 0732/2309] News file. --- newsfragments/3875.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3875.minor diff --git a/newsfragments/3875.minor b/newsfragments/3875.minor new file mode 100644 index 000000000..e69de29bb From 7146cff227ae7cf11a25961d90201d5661b65c8f Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 2 Mar 2022 10:40:39 -0500 Subject: [PATCH 0733/2309] Sketch of TLS listening and furl construction for the HTTP storage server. --- src/allmydata/storage/http_server.py | 59 +++++++++++++++++++++++----- 1 file changed, 49 insertions(+), 10 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index f885baa22..687c9f2c9 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -7,28 +7,27 @@ from __future__ import division from __future__ import print_function from __future__ import unicode_literals -from future.utils import PY2 - -if PY2: - # fmt: off - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - # fmt: on -else: - from typing import Dict, List, Set - +from typing import Dict, List, Set, Tuple, Optional +from pathlib import Path from functools import wraps from base64 import b64decode from klein import Klein from twisted.web import http +from twisted.internet.interfaces import IListeningPort +from twisted.internet.defer import Deferred +from twisted.internet.endpoints import quoteStringArgument, serverFromString +from twisted.web.server import Site import attr from werkzeug.http import parse_range_header, parse_content_range_header +from hyperlink import DecodedURL +from cryptography.x509 import load_pem_x509_certificate # TODO Make sure to use pure Python versions? from cbor2 import dumps, loads from .server import StorageServer -from .http_common import swissnum_auth_header, Secrets +from .http_common import swissnum_auth_header, Secrets, get_spki_hash from .common import si_a2b from .immutable import BucketWriter from ..util.hashutil import timing_safe_compare @@ -301,3 +300,43 @@ class HTTPServer(object): # "content-range", range_header.make_content_range(share_length).to_header() # ) return data + + +def listen_tls( + server: HTTPServer, + hostname: str, + port: int, + private_key_path: Path, + cert_path: Path, + interface: Optional[str], +) -> Deferred[Tuple[DecodedURL, IListeningPort]]: + """ + Start a HTTPS storage server on the given port, return the fURL and the + listening port. + + The hostname is the external IP or hostname clients will connect to; it + does not modify what interfaces the server listens on. To set the + listening interface, use the ``interface`` argument. + """ + endpoint_string = "ssl:privateKey={}:certKey={}:port={}".format( + quoteStringArgument(str(private_key_path)), + quoteStringArgument(str(cert_path)), + port, + ) + if interface is not None: + endpoint_string += ":interface={}".format(quoteStringArgument(interface)) + endpoint = serverFromString(endpoint_string) + + def build_furl(listening_port: IListeningPort) -> DecodedURL: + furl = DecodedURL() + furl.fragment = "v=1" # HTTP-based + furl.host = hostname + furl.port = listening_port.getHost().port + furl.path = (server._swissnum,) + furl.user = get_spki_hash(load_pem_x509_certificate(cert_path.read_bytes())) + furl.scheme = "pb" + return furl + + return endpoint.listen(Site(server.get_resource())).addCallback( + lambda listening_port: (build_furl(listening_port), listening_port) + ) From 9f4f6668c0110e153879c950460a0bdfcde27a03 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 7 Mar 2022 08:21:58 -0500 Subject: [PATCH 0734/2309] Tweaks. --- src/allmydata/storage/http_client.py | 36 +++++++++++++++------------- src/allmydata/storage/http_server.py | 3 ++- 2 files changed, 21 insertions(+), 18 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 2dc133b52..b532c292e 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -7,22 +7,7 @@ from __future__ import division from __future__ import print_function from __future__ import unicode_literals -from future.utils import PY2 - -if PY2: - # fmt: off - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - # fmt: on - from collections import defaultdict - - Optional = Set = defaultdict( - lambda: None - ) # some garbage to just make this module import -else: - # typing module not available in Python 2, and we only do type checking in - # Python 3 anyway. - from typing import Union, Set, Optional - from treq.testing import StubTreq +from typing import Union, Set, Optional from base64 import b64encode @@ -37,6 +22,8 @@ from twisted.web import http from twisted.internet.defer import inlineCallbacks, returnValue, fail, Deferred from hyperlink import DecodedURL import treq +from treq.client import HTTPClient +from treq.testing import StubTreq from .http_common import swissnum_auth_header, Secrets from .common import si_b2a @@ -73,11 +60,26 @@ class StorageClient(object): def __init__( self, url, swissnum, treq=treq - ): # type: (DecodedURL, bytes, Union[treq,StubTreq]) -> None + ): # type: (DecodedURL, bytes, Union[treq,StubTreq,HTTPClient]) -> None + """ + The URL is a HTTPS URL ("http://..."). To construct from a furl, use + ``StorageClient.from_furl()``. + """ + assert url.to_text().startswith("https://") self._base_url = url self._swissnum = swissnum self._treq = treq + @classmethod + def from_furl(cls, furl: DecodedURL) -> "StorageClient": + """ + Create a ``StorageClient`` for the given furl. + """ + assert furl.fragment == "v=1" + assert furl.scheme == "pb" + swissnum = furl.path[0].encode("ascii") + certificate_hash = furl.user.encode("ascii") + def _url(self, path): """Get a URL relative to the base URL.""" return self._base_url.click(path) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 687c9f2c9..da9eea22f 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -303,6 +303,7 @@ class HTTPServer(object): def listen_tls( + reactor, server: HTTPServer, hostname: str, port: int, @@ -325,7 +326,7 @@ def listen_tls( ) if interface is not None: endpoint_string += ":interface={}".format(quoteStringArgument(interface)) - endpoint = serverFromString(endpoint_string) + endpoint = serverFromString(reactor, endpoint_string) def build_furl(listening_port: IListeningPort) -> DecodedURL: furl = DecodedURL() From 60bcd5fe9f41b811cb2630424935977c6e1db674 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 7 Mar 2022 08:25:12 -0500 Subject: [PATCH 0735/2309] Address review comments. --- src/allmydata/test/test_storage_http.py | 47 +++++++++++++------------ 1 file changed, 25 insertions(+), 22 deletions(-) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index c864f923c..c15749fc8 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -374,7 +374,7 @@ class ImmutableHTTPAPITests(SyncTestCase): self.skipTest("Not going to bother supporting Python 2") super(ImmutableHTTPAPITests, self).setUp() self.http = self.useFixture(HttpTestFixture()) - self.im_client = StorageClientImmutables(self.http.client) + self.imm_client = StorageClientImmutables(self.http.client) def create_upload(self, share_numbers, length): """ @@ -386,7 +386,7 @@ class ImmutableHTTPAPITests(SyncTestCase): lease_secret = urandom(32) storage_index = urandom(16) created = result_of( - self.im_client.create( + self.imm_client.create( storage_index, share_numbers, length, @@ -407,7 +407,7 @@ class ImmutableHTTPAPITests(SyncTestCase): that's already done in test_storage.py. """ length = 100 - expected_data = b"".join(bytes([i]) for i in range(100)) + expected_data = bytes(range(100)) # Create a upload: (upload_secret, _, storage_index, created) = self.create_upload({1}, 100) @@ -421,7 +421,7 @@ class ImmutableHTTPAPITests(SyncTestCase): # Three writes: 10-19, 30-39, 50-59. This allows for a bunch of holes. def write(offset, length): remaining.empty(offset, offset + length) - return self.im_client.write_share_chunk( + return self.imm_client.write_share_chunk( storage_index, 1, upload_secret, @@ -465,7 +465,7 @@ class ImmutableHTTPAPITests(SyncTestCase): # We can now read: for offset, length in [(0, 100), (10, 19), (99, 1), (49, 200)]: downloaded = result_of( - self.im_client.read_share_chunk(storage_index, 1, offset, length) + self.imm_client.read_share_chunk(storage_index, 1, offset, length) ) self.assertEqual(downloaded, expected_data[offset : offset + length]) @@ -480,7 +480,7 @@ class ImmutableHTTPAPITests(SyncTestCase): ) with self.assertRaises(ClientException) as e: result_of( - self.im_client.create( + self.imm_client.create( storage_index, {2, 3}, 100, b"x" * 32, lease_secret, lease_secret ) ) @@ -498,7 +498,7 @@ class ImmutableHTTPAPITests(SyncTestCase): # Add same shares: created2 = result_of( - self.im_client.create( + self.imm_client.create( storage_index, {4, 6}, 100, b"x" * 2, lease_secret, lease_secret ) ) @@ -511,12 +511,12 @@ class ImmutableHTTPAPITests(SyncTestCase): (upload_secret, _, storage_index, created) = self.create_upload({1, 2, 3}, 10) # Initially there are no shares: - self.assertEqual(result_of(self.im_client.list_shares(storage_index)), set()) + self.assertEqual(result_of(self.imm_client.list_shares(storage_index)), set()) # Upload shares 1 and 3: for share_number in [1, 3]: progress = result_of( - self.im_client.write_share_chunk( + self.imm_client.write_share_chunk( storage_index, share_number, upload_secret, @@ -527,7 +527,7 @@ class ImmutableHTTPAPITests(SyncTestCase): self.assertTrue(progress.finished) # Now shares 1 and 3 exist: - self.assertEqual(result_of(self.im_client.list_shares(storage_index)), {1, 3}) + self.assertEqual(result_of(self.imm_client.list_shares(storage_index)), {1, 3}) def test_upload_bad_content_range(self): """ @@ -563,8 +563,8 @@ class ImmutableHTTPAPITests(SyncTestCase): """ Listing unknown storage index's shares results in empty list of shares. """ - storage_index = b"".join(bytes([i]) for i in range(16)) - self.assertEqual(result_of(self.im_client.list_shares(storage_index)), set()) + storage_index = bytes(range(16)) + self.assertEqual(result_of(self.imm_client.list_shares(storage_index)), set()) def test_upload_non_existent_storage_index(self): """ @@ -576,7 +576,7 @@ class ImmutableHTTPAPITests(SyncTestCase): def unknown_check(storage_index, share_number): with self.assertRaises(ClientException) as e: result_of( - self.im_client.write_share_chunk( + self.imm_client.write_share_chunk( storage_index, share_number, upload_secret, @@ -598,7 +598,7 @@ class ImmutableHTTPAPITests(SyncTestCase): """ (upload_secret, _, storage_index, _) = self.create_upload({1, 2}, 10) result_of( - self.im_client.write_share_chunk( + self.imm_client.write_share_chunk( storage_index, 1, upload_secret, @@ -607,7 +607,7 @@ class ImmutableHTTPAPITests(SyncTestCase): ) ) result_of( - self.im_client.write_share_chunk( + self.imm_client.write_share_chunk( storage_index, 2, upload_secret, @@ -616,11 +616,11 @@ class ImmutableHTTPAPITests(SyncTestCase): ) ) self.assertEqual( - result_of(self.im_client.read_share_chunk(storage_index, 1, 0, 10)), + result_of(self.imm_client.read_share_chunk(storage_index, 1, 0, 10)), b"1" * 10, ) self.assertEqual( - result_of(self.im_client.read_share_chunk(storage_index, 2, 0, 10)), + result_of(self.imm_client.read_share_chunk(storage_index, 2, 0, 10)), b"2" * 10, ) @@ -633,7 +633,7 @@ class ImmutableHTTPAPITests(SyncTestCase): # Write: result_of( - self.im_client.write_share_chunk( + self.imm_client.write_share_chunk( storage_index, 1, upload_secret, @@ -645,7 +645,7 @@ class ImmutableHTTPAPITests(SyncTestCase): # Conflicting write: with self.assertRaises(ClientException) as e: result_of( - self.im_client.write_share_chunk( + self.imm_client.write_share_chunk( storage_index, 1, upload_secret, @@ -666,7 +666,7 @@ class ImmutableHTTPAPITests(SyncTestCase): {share_number}, data_length ) result_of( - self.im_client.write_share_chunk( + self.imm_client.write_share_chunk( storage_index, share_number, upload_secret, @@ -682,7 +682,7 @@ class ImmutableHTTPAPITests(SyncTestCase): """ with self.assertRaises(ClientException) as e: result_of( - self.im_client.read_share_chunk( + self.imm_client.read_share_chunk( b"1" * 16, 1, 0, @@ -698,7 +698,7 @@ class ImmutableHTTPAPITests(SyncTestCase): storage_index, _ = self.upload(1) with self.assertRaises(ClientException) as e: result_of( - self.im_client.read_share_chunk( + self.imm_client.read_share_chunk( storage_index, 7, # different share number 0, @@ -732,9 +732,12 @@ class ImmutableHTTPAPITests(SyncTestCase): ) self.assertEqual(e.exception.code, http.REQUESTED_RANGE_NOT_SATISFIABLE) + # Bad unit check_bad_range("molluscs=0-9") + # Negative offsets check_bad_range("bytes=-2-9") check_bad_range("bytes=0--10") + # Negative offset no endpoint check_bad_range("bytes=-300-") check_bad_range("bytes=") # Multiple ranges are currently unsupported, even if they're From 4efa65d3dbfdd844e14537b6dd18683c8ed89092 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 7 Mar 2022 08:29:26 -0500 Subject: [PATCH 0736/2309] Typo. --- src/allmydata/storage/http_server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index f89a156a3..9048359b1 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -133,7 +133,7 @@ class StorageIndexUploads(object): # Map share number to BucketWriter shares = attr.ib(factory=dict) # type: Dict[int,BucketWriter] - # Mape share number to the upload secret (different shares might have + # Map share number to the upload secret (different shares might have # different upload secrets). upload_secrets = attr.ib(factory=dict) # type: Dict[int,bytes] From 87ab56426ad1634ef97cd2ca3805cc07038cb2a4 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 7 Mar 2022 08:38:31 -0500 Subject: [PATCH 0737/2309] Validate another edge case of bad storage index. --- src/allmydata/storage/http_server.py | 8 ++++++-- src/allmydata/test/test_storage_http.py | 7 ++++++- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 9048359b1..0e1969593 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -18,12 +18,13 @@ else: from functools import wraps from base64 import b64decode +import binascii from klein import Klein from twisted.web import http import attr from werkzeug.http import parse_range_header, parse_content_range_header -from werkzeug.routing import BaseConverter +from werkzeug.routing import BaseConverter, ValidationError from werkzeug.datastructures import ContentRange # TODO Make sure to use pure Python versions? @@ -148,7 +149,10 @@ class StorageIndexConverter(BaseConverter): regex = "[" + str(rfc3548_alphabet, "ascii") + "]{26}" def to_python(self, value): - return si_a2b(value.encode("ascii")) + try: + return si_a2b(value.encode("ascii")) + except (AssertionError, binascii.Error, ValueError): + raise ValidationError("Invalid storage index") class HTTPServer(object): diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index c15749fc8..14f4437b6 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -191,10 +191,15 @@ class RouteConverterTests(SyncTestCase): self.adapter.match("/{}/".format("a" * 25), method="GET") def test_bad_characters_storage_index_is_not_parsed(self): - """An overly short storage_index string is not parsed.""" + """A storage_index string with bad characters is not parsed.""" with self.assertRaises(WNotFound): self.adapter.match("/{}_/".format("a" * 25), method="GET") + def test_invalid_storage_index_is_not_parsed(self): + """An invalid storage_index string is not parsed.""" + with self.assertRaises(WNotFound): + self.adapter.match("/nomd2a65ylxjbqzsw7gcfh4ivr/", method="GET") + # TODO should be actual swissnum SWISSNUM_FOR_TEST = b"abcd" From 8586028af82d4403d8d5e5b94f63bdbed84d758a Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 7 Mar 2022 09:13:21 -0500 Subject: [PATCH 0738/2309] News file. --- newsfragments/3876.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3876.minor diff --git a/newsfragments/3876.minor b/newsfragments/3876.minor new file mode 100644 index 000000000..e69de29bb From 7721c134f2815e50743398d21b95c6eb9c82cc64 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 7 Mar 2022 09:28:21 -0500 Subject: [PATCH 0739/2309] Change the semantics of HTTP bucket creation so that it's possible to have a different upload secret per upload. --- docs/proposed/http-storage-node-protocol.rst | 4 +- src/allmydata/storage/http_server.py | 19 ++--- src/allmydata/storage_client.py | 10 ++- src/allmydata/test/test_storage_http.py | 76 ++++++++++++++------ 4 files changed, 69 insertions(+), 40 deletions(-) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index 315546b8a..0f534f0c5 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -493,8 +493,8 @@ Handling repeat calls: * If the same API call is repeated with the same upload secret, the response is the same and no change is made to server state. This is necessary to ensure retries work in the face of lost responses from the server. * If the API calls is with a different upload secret, this implies a new client, perhaps because the old client died. - In order to prevent storage servers from being able to mess with each other, this API call will fail, because the secret doesn't match. - The use case of restarting upload from scratch if the client dies can be implemented by having the client persist the upload secret. + Or it may happen because the client wants to upload a different share number than a previous client. + New shares will be created, existing shares will be unchanged, regardless of whether the upload secret matches or not. Discussion `````````` diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 0e1969593..9b158ecfd 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -203,18 +203,13 @@ class HTTPServer(object): upload_secret = authorization[Secrets.UPLOAD] info = loads(request.content.read()) - if storage_index in self._uploads: - for share_number in info["share-numbers"]: - in_progress = self._uploads[storage_index] - # For pre-existing upload, make sure password matches. - if ( - share_number in in_progress.upload_secrets - and not timing_safe_compare( - in_progress.upload_secrets[share_number], upload_secret - ) - ): - request.setResponseCode(http.UNAUTHORIZED) - return b"" + # We do NOT validate the upload secret for existing bucket uploads. + # Another upload may be happening in parallel, with a different upload + # key. That's fine! If a client tries to _write_ to that upload, they + # need to have an upload key. That does mean we leak the existence of + # these parallel uploads, but if you know storage index you can + # download them once upload finishes, so it's not a big deal to leak + # that information. already_got, sharenum_to_bucket = self._storage_server.allocate_buckets( storage_index, diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index c2e26b4b6..2c7e13890 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -1102,18 +1102,15 @@ class _HTTPBucketReader(object): class _HTTPStorageServer(object): """ Talk to remote storage server over HTTP. - - The same upload key is used for all communication. """ _http_client = attr.ib(type=StorageClient) - _upload_secret = attr.ib(type=bytes) @staticmethod def from_http_client(http_client): # type: (StorageClient) -> _HTTPStorageServer """ Create an ``IStorageServer`` from a HTTP ``StorageClient``. """ - return _HTTPStorageServer(http_client=http_client, upload_secret=urandom(20)) + return _HTTPStorageServer(http_client=http_client) def get_version(self): return StorageClientGeneral(self._http_client).get_version() @@ -1128,9 +1125,10 @@ class _HTTPStorageServer(object): allocated_size, canary, ): + upload_secret = urandom(20) immutable_client = StorageClientImmutables(self._http_client) result = immutable_client.create( - storage_index, sharenums, allocated_size, self._upload_secret, renew_secret, + storage_index, sharenums, allocated_size, upload_secret, renew_secret, cancel_secret ) result = yield result @@ -1140,7 +1138,7 @@ class _HTTPStorageServer(object): client=immutable_client, storage_index=storage_index, share_number=share_num, - upload_secret=self._upload_secret + upload_secret=upload_secret )) for share_num in result.allocated }) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 14f4437b6..ae003c65f 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -474,41 +474,77 @@ class ImmutableHTTPAPITests(SyncTestCase): ) self.assertEqual(downloaded, expected_data[offset : offset + length]) - def test_allocate_buckets_second_time_wrong_upload_key(self): - """ - If allocate buckets endpoint is called second time with wrong upload - key on the same shares, the result is an error. - """ - # Create a upload: - (upload_secret, lease_secret, storage_index, _) = self.create_upload( - {1, 2, 3}, 100 - ) - with self.assertRaises(ClientException) as e: - result_of( - self.imm_client.create( - storage_index, {2, 3}, 100, b"x" * 32, lease_secret, lease_secret - ) - ) - self.assertEqual(e.exception.args[0], http.UNAUTHORIZED) - def test_allocate_buckets_second_time_different_shares(self): """ If allocate buckets endpoint is called second time with different - upload key on different shares, that creates the buckets. + upload key on potentially different shares, that creates the buckets on + those shares that are different. """ # Create a upload: (upload_secret, lease_secret, storage_index, created) = self.create_upload( {1, 2, 3}, 100 ) - # Add same shares: + # Write half of share 1 + result_of( + self.imm_client.write_share_chunk( + storage_index, + 1, + upload_secret, + 0, + b"a" * 50, + ) + ) + + # Add same shares with a different upload key share 1 overlaps with + # existing shares, this call shouldn't overwrite the existing + # work-in-progress. + upload_secret2 = b"x" * 2 created2 = result_of( self.imm_client.create( - storage_index, {4, 6}, 100, b"x" * 2, lease_secret, lease_secret + storage_index, + {1, 4, 6}, + 100, + upload_secret2, + lease_secret, + lease_secret, ) ) self.assertEqual(created2.allocated, {4, 6}) + # Write second half of share 1 + self.assertTrue( + result_of( + self.imm_client.write_share_chunk( + storage_index, + 1, + upload_secret, + 50, + b"b" * 50, + ) + ).finished + ) + + # The upload of share 1 succeeded, demonstrating that second create() + # call didn't overwrite work-in-progress. + downloaded = result_of( + self.imm_client.read_share_chunk(storage_index, 1, 0, 100) + ) + self.assertEqual(downloaded, b"a" * 50 + b"b" * 50) + + # We can successfully upload the shares created with the second upload secret. + self.assertTrue( + result_of( + self.imm_client.write_share_chunk( + storage_index, + 4, + upload_secret2, + 0, + b"x" * 100, + ) + ).finished + ) + def test_list_shares(self): """ Once a share is finished uploading, it's possible to list it. From 1d007cc573c6603323deaa6bf4a012ae8ed0fe33 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 7 Mar 2022 13:21:36 -0500 Subject: [PATCH 0740/2309] News file. --- newsfragments/3877.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3877.minor diff --git a/newsfragments/3877.minor b/newsfragments/3877.minor new file mode 100644 index 000000000..e69de29bb From 52038739950a0c7a7c82dfcdd5e7d2c083d15e6a Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 8 Mar 2022 10:13:37 -0500 Subject: [PATCH 0741/2309] Refactor to unify data structure logic. --- src/allmydata/storage/http_server.py | 114 +++++++++++++++++++++------ 1 file changed, 91 insertions(+), 23 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 0e1969593..d0653a97c 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -138,9 +138,69 @@ class StorageIndexUploads(object): # different upload secrets). upload_secrets = attr.ib(factory=dict) # type: Dict[int,bytes] - def add_upload(self, share_number, upload_secret, bucket): - self.shares[share_number] = bucket - self.upload_secrets[share_number] = upload_secret + +@attr.s +class UploadsInProgress(object): + """ + Keep track of uploads for storage indexes. + """ + + # Map storage index to corresponding uploads-in-progress + _uploads = attr.ib(type=Dict[bytes, StorageIndexUploads], factory=dict) + + def add_write_bucket( + self, + storage_index: bytes, + share_number: int, + upload_secret: bytes, + bucket: BucketWriter, + ): + """Add a new ``BucketWriter`` to be tracked. + + TODO 3877 how does a timed-out BucketWriter get removed?! + """ + si_uploads = self._uploads.setdefault(storage_index, StorageIndexUploads()) + si_uploads.shares[share_number] = bucket + si_uploads.upload_secrets[share_number] = upload_secret + + def get_write_bucket( + self, storage_index: bytes, share_number: int, upload_secret: bytes + ) -> BucketWriter: + """Get the given in-progress immutable share upload.""" + try: + # TODO 3877 check the upload secret matches given one + return self._uploads[storage_index].shares[share_number] + except (KeyError, IndexError): + raise _HTTPError(http.NOT_FOUND) + + def remove_write_bucket(self, storage_index: bytes, share_number: int): + """Stop tracking the given ``BucketWriter``.""" + uploads_index = self._uploads[storage_index] + uploads_index.shares.pop(share_number) + uploads_index.upload_secrets.pop(share_number) + if not uploads_index.shares: + self._uploads.pop(storage_index) + + def validate_upload_secret( + self, storage_index: bytes, share_number: int, upload_secret: bytes + ): + """ + Raise an unauthorized-HTTP-response exception if the given + storage_index+share_number have a different upload secret than the + given one. + + If the given upload doesn't exist at all, nothing happens. + """ + if storage_index in self._uploads: + try: + in_progress = self._uploads[storage_index] + except KeyError: + return + # For pre-existing upload, make sure password matches. + if share_number in in_progress.upload_secrets and not timing_safe_compare( + in_progress.upload_secrets[share_number], upload_secret + ): + raise _HTTPError(http.UNAUTHORIZED) class StorageIndexConverter(BaseConverter): @@ -155,6 +215,15 @@ class StorageIndexConverter(BaseConverter): raise ValidationError("Invalid storage index") +class _HTTPError(Exception): + """ + Raise from ``HTTPServer`` endpoint to return the given HTTP response code. + """ + + def __init__(self, code: int): + self.code = code + + class HTTPServer(object): """ A HTTP interface to the storage server. @@ -163,13 +232,19 @@ class HTTPServer(object): _app = Klein() _app.url_map.converters["storage_index"] = StorageIndexConverter + @_app.handle_errors(_HTTPError) + def _http_error(self, request, failure): + """Handle ``_HTTPError`` exceptions.""" + request.setResponseCode(failure.value.code) + return b"" + def __init__( self, storage_server, swissnum ): # type: (StorageServer, bytes) -> None self._storage_server = storage_server self._swissnum = swissnum # Maps storage index to StorageIndexUploads: - self._uploads = {} # type: Dict[bytes,StorageIndexUploads] + self._uploads = UploadsInProgress() def get_resource(self): """Return twisted.web ``Resource`` for this object.""" @@ -203,18 +278,10 @@ class HTTPServer(object): upload_secret = authorization[Secrets.UPLOAD] info = loads(request.content.read()) - if storage_index in self._uploads: - for share_number in info["share-numbers"]: - in_progress = self._uploads[storage_index] - # For pre-existing upload, make sure password matches. - if ( - share_number in in_progress.upload_secrets - and not timing_safe_compare( - in_progress.upload_secrets[share_number], upload_secret - ) - ): - request.setResponseCode(http.UNAUTHORIZED) - return b"" + for share_number in info["share-numbers"]: + self._uploads.validate_upload_secret( + storage_index, share_number, upload_secret + ) already_got, sharenum_to_bucket = self._storage_server.allocate_buckets( storage_index, @@ -223,9 +290,10 @@ class HTTPServer(object): sharenums=info["share-numbers"], allocated_size=info["allocated-size"], ) - uploads = self._uploads.setdefault(storage_index, StorageIndexUploads()) for share_number, bucket in sharenum_to_bucket.items(): - uploads.add_upload(share_number, upload_secret, bucket) + self._uploads.add_write_bucket( + storage_index, share_number, upload_secret, bucket + ) return self._cbor( request, @@ -250,14 +318,14 @@ class HTTPServer(object): offset = content_range.start + # TODO 3877 test for checking upload secret + # TODO limit memory usage # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3872 data = request.content.read(content_range.stop - content_range.start + 1) - try: - bucket = self._uploads[storage_index].shares[share_number] - except (KeyError, IndexError): - request.setResponseCode(http.NOT_FOUND) - return b"" + bucket = self._uploads.get_write_bucket( + storage_index, share_number, authorization[Secrets.UPLOAD] + ) try: finished = bucket.write(offset, data) From c642218173a969397edda4dc4f1cb45f6b8a7e9f Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 8 Mar 2022 10:41:56 -0500 Subject: [PATCH 0742/2309] Sketch of aborting uploads. --- src/allmydata/storage/http_client.py | 23 ++++++++++++++++++- src/allmydata/storage/http_server.py | 27 +++++++++++++++++++++++ src/allmydata/storage_client.py | 3 ++- src/allmydata/test/test_istorageserver.py | 6 ++--- 4 files changed, 54 insertions(+), 5 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 475aa2330..d83ecbdff 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -161,7 +161,7 @@ class StorageClientImmutables(object): APIs for interacting with immutables. """ - def __init__(self, client): # type: (StorageClient) -> None + def __init__(self, client: StorageClient): self._client = client @inlineCallbacks @@ -208,6 +208,27 @@ class StorageClientImmutables(object): ) ) + @inlineCallbacks + def abort_upload( + self, storage_index: bytes, share_number: int, upload_secret: bytes + ) -> Deferred[None]: + """Abort the upload.""" + url = self._client.relative_url( + "/v1/immutable/{}/{}/abort".format(_encode_si(storage_index), share_number) + ) + response = yield self._client.request( + "PUT", + url, + upload_secret=upload_secret, + ) + + if response.code == http.OK: + return + else: + raise ClientException( + response.code, + ) + @inlineCallbacks def write_share_chunk( self, storage_index, share_number, upload_secret, offset, data diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index d0653a97c..1acf08a81 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -303,6 +303,33 @@ class HTTPServer(object): }, ) + @_authorized_route( + _app, + {Secrets.UPLOAD}, + "/v1/immutable///abort", + methods=["PUT"], + ) + def abort_share_upload(self, request, authorization, storage_index, share_number): + """Abort an in-progress immutable share upload.""" + try: + bucket = self._uploads.get_write_bucket( + storage_index, share_number, authorization[Secrets.UPLOAD] + ) + except _HTTPError: + # TODO 3877 If 404, check if this was already uploaded, in which case return 405 + # TODO 3877 write tests for 404 cases? + raise + + # TODO 3877 test for checking upload secret + + # Abort the upload: + bucket.abort() + # Stop tracking the bucket, so we can create a new one later if a + # client requests it: + self._uploads.remove_write_bucket(storage_index, share_number) + + return b"" + @_authorized_route( _app, {Secrets.UPLOAD}, diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index c2e26b4b6..9e47ba3ff 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -1059,7 +1059,8 @@ class _HTTPBucketWriter(object): finished = attr.ib(type=bool, default=False) def abort(self): - pass # TODO in later ticket + return self.client.abort_upload(self.storage_index, self.share_number, + self.upload_secret) @defer.inlineCallbacks def write(self, offset, data): diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index 95261ddb2..668eeecc5 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -176,8 +176,9 @@ class IStorageServerImmutableAPIsTestsMixin(object): canary=Referenceable(), ) - # Bucket 1 is fully written in one go. - yield allocated[0].callRemote("write", 0, b"1" * 1024) + # Bucket 1 get some data written (but not all, or HTTP implicitly + # finishes the upload) + yield allocated[0].callRemote("write", 0, b"1" * 1023) # Disconnect or abort, depending on the test: yield abort_or_disconnect(allocated[0]) @@ -1156,7 +1157,6 @@ class HTTPImmutableAPIsTests( # These will start passing in future PRs as HTTP protocol is implemented. SKIP_TESTS = { - "test_abort", "test_add_lease_renewal", "test_add_new_lease", "test_advise_corrupt_share", From 92b952a5fe87202358cb133316db40b4a9c8429d Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 9 Mar 2022 10:22:00 -0500 Subject: [PATCH 0743/2309] Authenticate writes! --- src/allmydata/storage/http_server.py | 4 +--- src/allmydata/test/test_storage_http.py | 15 +++++++++++++++ 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 1acf08a81..91b492663 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -167,8 +167,8 @@ class UploadsInProgress(object): self, storage_index: bytes, share_number: int, upload_secret: bytes ) -> BucketWriter: """Get the given in-progress immutable share upload.""" + self.validate_upload_secret(storage_index, share_number, upload_secret) try: - # TODO 3877 check the upload secret matches given one return self._uploads[storage_index].shares[share_number] except (KeyError, IndexError): raise _HTTPError(http.NOT_FOUND) @@ -345,8 +345,6 @@ class HTTPServer(object): offset = content_range.start - # TODO 3877 test for checking upload secret - # TODO limit memory usage # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3872 data = request.content.read(content_range.stop - content_range.start + 1) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 14f4437b6..b3e1ab0ad 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -474,6 +474,21 @@ class ImmutableHTTPAPITests(SyncTestCase): ) self.assertEqual(downloaded, expected_data[offset : offset + length]) + def test_write_with_wrong_upload_key(self): + """A write with the wrong upload key fails.""" + (upload_secret, _, storage_index, _) = self.create_upload({1}, 100) + with self.assertRaises(ClientException) as e: + result_of( + self.imm_client.write_share_chunk( + storage_index, + 1, + upload_secret + b"X", + 0, + b"123", + ) + ) + self.assertEqual(e.exception.args[0], http.UNAUTHORIZED) + def test_allocate_buckets_second_time_wrong_upload_key(self): """ If allocate buckets endpoint is called second time with wrong upload From f47741afb163fa4680c7914c349ae00d7f6e622f Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 9 Mar 2022 10:45:21 -0500 Subject: [PATCH 0744/2309] Correct behavior on timed out immutable uploads. --- src/allmydata/storage/http_server.py | 38 +++++++++----------- src/allmydata/test/test_storage_http.py | 47 +++++++++++++++++++++++++ 2 files changed, 63 insertions(+), 22 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 91b492663..57e2b3e82 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -2,19 +2,7 @@ HTTP server for storage. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals - -from future.utils import PY2 - -if PY2: - # fmt: off - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - # fmt: on -else: - from typing import Dict, List, Set +from typing import Dict, List, Set, Tuple from functools import wraps from base64 import b64decode @@ -148,6 +136,9 @@ class UploadsInProgress(object): # Map storage index to corresponding uploads-in-progress _uploads = attr.ib(type=Dict[bytes, StorageIndexUploads], factory=dict) + # Map BucketWriter to (storage index, share number) + _bucketwriters = attr.ib(type=Dict[BucketWriter, Tuple[bytes, int]], factory=dict) + def add_write_bucket( self, storage_index: bytes, @@ -155,13 +146,11 @@ class UploadsInProgress(object): upload_secret: bytes, bucket: BucketWriter, ): - """Add a new ``BucketWriter`` to be tracked. - - TODO 3877 how does a timed-out BucketWriter get removed?! - """ + """Add a new ``BucketWriter`` to be tracked.""" si_uploads = self._uploads.setdefault(storage_index, StorageIndexUploads()) si_uploads.shares[share_number] = bucket si_uploads.upload_secrets[share_number] = upload_secret + self._bucketwriters[bucket] = (storage_index, share_number) def get_write_bucket( self, storage_index: bytes, share_number: int, upload_secret: bytes @@ -173,8 +162,9 @@ class UploadsInProgress(object): except (KeyError, IndexError): raise _HTTPError(http.NOT_FOUND) - def remove_write_bucket(self, storage_index: bytes, share_number: int): + def remove_write_bucket(self, bucket: BucketWriter): """Stop tracking the given ``BucketWriter``.""" + storage_index, share_number = self._bucketwriters.pop(bucket) uploads_index = self._uploads[storage_index] uploads_index.shares.pop(share_number) uploads_index.upload_secrets.pop(share_number) @@ -246,6 +236,12 @@ class HTTPServer(object): # Maps storage index to StorageIndexUploads: self._uploads = UploadsInProgress() + # When an upload finishes successfully, gets aborted, or times out, + # make sure it gets removed from our tracking datastructure: + self._storage_server.register_bucket_writer_close_handler( + self._uploads.remove_write_bucket + ) + def get_resource(self): """Return twisted.web ``Resource`` for this object.""" return self._app.resource() @@ -322,11 +318,9 @@ class HTTPServer(object): # TODO 3877 test for checking upload secret - # Abort the upload: + # Abort the upload; this should close it which will eventually result + # in self._uploads.remove_write_bucket() being called. bucket.abort() - # Stop tracking the bucket, so we can create a new one later if a - # client requests it: - self._uploads.remove_write_bucket(storage_index, share_number) return b"" diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index b3e1ab0ad..6788bc657 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -810,3 +810,50 @@ class ImmutableHTTPAPITests(SyncTestCase): check_range("bytes=0-10", "bytes 0-10/*") # Can't go beyond the end of the immutable! check_range("bytes=10-100", "bytes 10-25/*") + + def test_timed_out_upload_allows_reupload(self): + """ + If an in-progress upload times out, it is cancelled, allowing a new + upload to occur. + """ + # Start an upload: + (upload_secret, _, storage_index, _) = self.create_upload({1}, 100) + result_of( + self.imm_client.write_share_chunk( + storage_index, + 1, + upload_secret, + 0, + b"123", + ) + ) + + # Now, time passes, the in-progress upload should disappear... + self.http.clock.advance(30 * 60 + 1) + + # Now we can create a new share with the same storage index without + # complaint: + upload_secret = urandom(32) + lease_secret = urandom(32) + created = result_of( + self.imm_client.create( + storage_index, + {1}, + 100, + upload_secret, + lease_secret, + lease_secret, + ) + ) + self.assertEqual(created.allocated, {1}) + + # And write to it, too: + result_of( + self.imm_client.write_share_chunk( + storage_index, + 1, + upload_secret, + 0, + b"ABC", + ) + ) From 4fc7ef75288b935ff1ada9418b4535a7f319aed5 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 9 Mar 2022 10:57:05 -0500 Subject: [PATCH 0745/2309] Basic HTTP test for aborts. --- src/allmydata/test/test_storage_http.py | 32 +++++++++++++++++++++---- 1 file changed, 28 insertions(+), 4 deletions(-) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 6788bc657..117a7037a 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -813,8 +813,30 @@ class ImmutableHTTPAPITests(SyncTestCase): def test_timed_out_upload_allows_reupload(self): """ - If an in-progress upload times out, it is cancelled, allowing a new - upload to occur. + If an in-progress upload times out, it is cancelled altogether, + allowing a new upload to occur. + """ + self._test_abort_or_timed_out_upload_to_existing_storage_index( + lambda **kwargs: self.http.clock.advance(30 * 60 + 1) + ) + + def test_abort_upload_allows_reupload(self): + """ + If an in-progress upload is aborted, it is cancelled altogether, + allowing a new upload to occur. + """ + + def abort(storage_index, share_number, upload_secret): + return result_of( + self.imm_client.abort_upload(storage_index, share_number, upload_secret) + ) + + self._test_abort_or_timed_out_upload_to_existing_storage_index(abort) + + def _test_abort_or_timed_out_upload_to_existing_storage_index(self, cancel_upload): + """Start uploading to an existing storage index that then times out or aborts. + + Re-uploading should work. """ # Start an upload: (upload_secret, _, storage_index, _) = self.create_upload({1}, 100) @@ -828,8 +850,10 @@ class ImmutableHTTPAPITests(SyncTestCase): ) ) - # Now, time passes, the in-progress upload should disappear... - self.http.clock.advance(30 * 60 + 1) + # Now, the upload is cancelled somehow: + cancel_upload( + storage_index=storage_index, upload_secret=upload_secret, share_number=1 + ) # Now we can create a new share with the same storage index without # complaint: From ef4f912a68912bfe92800eca760058468cf39095 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 9 Mar 2022 11:11:39 -0500 Subject: [PATCH 0746/2309] Less error-prone testing assertion, and fix a testing bug. --- src/allmydata/test/test_storage_http.py | 46 +++++++++++++++---------- 1 file changed, 28 insertions(+), 18 deletions(-) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 117a7037a..46914dbf2 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -15,6 +15,7 @@ if PY2: # fmt: on from base64 import b64encode +from contextlib import contextmanager from os import urandom from hypothesis import assume, given, strategies as st @@ -316,6 +317,20 @@ class StorageClientWithHeadersOverride(object): return self.storage_client.request(*args, headers=headers, **kwargs) +@contextmanager +def assert_fails_with_http_code(test_case: SyncTestCase, code: int): + """ + Context manager that asserts the code fails with the given HTTP response + code. + """ + with test_case.assertRaises(ClientException) as e: + try: + yield + finally: + pass + test_case.assertEqual(e.exception.code, code) + + class GenericHTTPAPITests(SyncTestCase): """ Tests of HTTP client talking to the HTTP server, for generic HTTP API @@ -340,9 +355,8 @@ class GenericHTTPAPITests(SyncTestCase): treq=StubTreq(self.http.http_server.get_resource()), ) ) - with self.assertRaises(ClientException) as e: + with assert_fails_with_http_code(self, http.UNAUTHORIZED): result_of(client.get_version()) - self.assertEqual(e.exception.args[0], 401) def test_version(self): """ @@ -477,7 +491,7 @@ class ImmutableHTTPAPITests(SyncTestCase): def test_write_with_wrong_upload_key(self): """A write with the wrong upload key fails.""" (upload_secret, _, storage_index, _) = self.create_upload({1}, 100) - with self.assertRaises(ClientException) as e: + with assert_fails_with_http_code(self, http.UNAUTHORIZED): result_of( self.imm_client.write_share_chunk( storage_index, @@ -487,7 +501,6 @@ class ImmutableHTTPAPITests(SyncTestCase): b"123", ) ) - self.assertEqual(e.exception.args[0], http.UNAUTHORIZED) def test_allocate_buckets_second_time_wrong_upload_key(self): """ @@ -498,13 +511,12 @@ class ImmutableHTTPAPITests(SyncTestCase): (upload_secret, lease_secret, storage_index, _) = self.create_upload( {1, 2, 3}, 100 ) - with self.assertRaises(ClientException) as e: + with assert_fails_with_http_code(self, http.UNAUTHORIZED): result_of( self.imm_client.create( storage_index, {2, 3}, 100, b"x" * 32, lease_secret, lease_secret ) ) - self.assertEqual(e.exception.args[0], http.UNAUTHORIZED) def test_allocate_buckets_second_time_different_shares(self): """ @@ -562,7 +574,9 @@ class ImmutableHTTPAPITests(SyncTestCase): self.http.client, {"content-range": bad_content_range_value} ) ) - with self.assertRaises(ClientException) as e: + with assert_fails_with_http_code( + self, http.REQUESTED_RANGE_NOT_SATISFIABLE + ): result_of( client.write_share_chunk( storage_index, @@ -572,7 +586,6 @@ class ImmutableHTTPAPITests(SyncTestCase): b"0123456789", ) ) - self.assertEqual(e.exception.code, http.REQUESTED_RANGE_NOT_SATISFIABLE) check_invalid("not a valid content-range header at all") check_invalid("bytes -1-9/10") @@ -594,7 +607,7 @@ class ImmutableHTTPAPITests(SyncTestCase): (upload_secret, _, storage_index, _) = self.create_upload({1}, 10) def unknown_check(storage_index, share_number): - with self.assertRaises(ClientException) as e: + with assert_fails_with_http_code(self, http.NOT_FOUND): result_of( self.imm_client.write_share_chunk( storage_index, @@ -604,7 +617,6 @@ class ImmutableHTTPAPITests(SyncTestCase): b"0123456789", ) ) - self.assertEqual(e.exception.code, http.NOT_FOUND) # Wrong share number: unknown_check(storage_index, 7) @@ -663,7 +675,7 @@ class ImmutableHTTPAPITests(SyncTestCase): ) # Conflicting write: - with self.assertRaises(ClientException) as e: + with assert_fails_with_http_code(self, http.CONFLICT): result_of( self.imm_client.write_share_chunk( storage_index, @@ -673,7 +685,6 @@ class ImmutableHTTPAPITests(SyncTestCase): b"0123456789", ) ) - self.assertEqual(e.exception.code, http.NOT_FOUND) def upload(self, share_number, data_length=26): """ @@ -700,7 +711,7 @@ class ImmutableHTTPAPITests(SyncTestCase): """ Reading from unknown storage index results in 404. """ - with self.assertRaises(ClientException) as e: + with assert_fails_with_http_code(self, http.NOT_FOUND): result_of( self.imm_client.read_share_chunk( b"1" * 16, @@ -709,14 +720,13 @@ class ImmutableHTTPAPITests(SyncTestCase): 10, ) ) - self.assertEqual(e.exception.code, http.NOT_FOUND) def test_read_of_wrong_share_number_fails(self): """ Reading from unknown storage index results in 404. """ storage_index, _ = self.upload(1) - with self.assertRaises(ClientException) as e: + with assert_fails_with_http_code(self, http.NOT_FOUND): result_of( self.imm_client.read_share_chunk( storage_index, @@ -725,7 +735,6 @@ class ImmutableHTTPAPITests(SyncTestCase): 10, ) ) - self.assertEqual(e.exception.code, http.NOT_FOUND) def test_read_with_negative_offset_fails(self): """ @@ -741,7 +750,9 @@ class ImmutableHTTPAPITests(SyncTestCase): ) ) - with self.assertRaises(ClientException) as e: + with assert_fails_with_http_code( + self, http.REQUESTED_RANGE_NOT_SATISFIABLE + ): result_of( client.read_share_chunk( storage_index, @@ -750,7 +761,6 @@ class ImmutableHTTPAPITests(SyncTestCase): 10, ) ) - self.assertEqual(e.exception.code, http.REQUESTED_RANGE_NOT_SATISFIABLE) # Bad unit check_bad_range("molluscs=0-9") From 86769c19bf306da3cae583e6d30c39d34e603c33 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 9 Mar 2022 11:19:23 -0500 Subject: [PATCH 0747/2309] Finish abort logic and tests. --- src/allmydata/storage/http_server.py | 16 ++++-- src/allmydata/test/test_storage_http.py | 68 +++++++++++++++++++++++++ 2 files changed, 79 insertions(+), 5 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 57e2b3e82..7659513e1 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -311,13 +311,19 @@ class HTTPServer(object): bucket = self._uploads.get_write_bucket( storage_index, share_number, authorization[Secrets.UPLOAD] ) - except _HTTPError: - # TODO 3877 If 404, check if this was already uploaded, in which case return 405 - # TODO 3877 write tests for 404 cases? + except _HTTPError as e: + if e.code == http.NOT_FOUND: + # It may be we've already uploaded this, in which case error + # should be method not allowed (405). + try: + self._storage_server.get_buckets(storage_index)[share_number] + except KeyError: + pass + else: + # Already uploaded, so we can't abort. + raise _HTTPError(http.NOT_ALLOWED) raise - # TODO 3877 test for checking upload secret - # Abort the upload; this should close it which will eventually result # in self._uploads.remove_write_bucket() being called. bucket.abort() diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 46914dbf2..97a81e250 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -891,3 +891,71 @@ class ImmutableHTTPAPITests(SyncTestCase): b"ABC", ) ) + + def test_unknown_aborts(self): + """ + Aborting aborts with unknown storage index or share number will 404. + """ + (upload_secret, _, storage_index, _) = self.create_upload({1}, 100) + + for si, num in [(storage_index, 3), (b"x" * 16, 1)]: + with assert_fails_with_http_code(self, http.NOT_FOUND): + result_of(self.imm_client.abort_upload(si, num, upload_secret)) + + def test_unauthorized_abort(self): + """ + An abort with the wrong key will return an unauthorized error, and will + not abort the upload. + """ + (upload_secret, _, storage_index, _) = self.create_upload({1}, 100) + + # Failed to abort becaues wrong upload secret: + with assert_fails_with_http_code(self, http.UNAUTHORIZED): + result_of( + self.imm_client.abort_upload(storage_index, 1, upload_secret + b"X") + ) + + # We can still write to it: + result_of( + self.imm_client.write_share_chunk( + storage_index, + 1, + upload_secret, + 0, + b"ABC", + ) + ) + + def test_too_late_abort(self): + """ + An abort of an already-fully-uploaded immutable will result in 405 + error and will not affect the immutable. + """ + uploaded_data = b"123" + (upload_secret, _, storage_index, _) = self.create_upload({0}, 3) + result_of( + self.imm_client.write_share_chunk( + storage_index, + 0, + upload_secret, + 0, + uploaded_data, + ) + ) + + # Can't abort, we finished upload: + with assert_fails_with_http_code(self, http.NOT_ALLOWED): + result_of(self.imm_client.abort_upload(storage_index, 0, upload_secret)) + + # Abort didn't prevent reading: + self.assertEqual( + uploaded_data, + result_of( + self.imm_client.read_share_chunk( + storage_index, + 0, + 0, + 3, + ) + ), + ) From edb9eda53b4daac49c8582c9a82513fd809b18cc Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 9 Mar 2022 12:41:10 -0500 Subject: [PATCH 0748/2309] Clarify. --- src/allmydata/test/test_storage_http.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 2103c5b65..b2da8e9d7 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -489,7 +489,10 @@ class ImmutableHTTPAPITests(SyncTestCase): self.assertEqual(downloaded, expected_data[offset : offset + length]) def test_write_with_wrong_upload_key(self): - """A write with the wrong upload key fails.""" + """ + A write with an upload key that is different than the original upload + key will fail. + """ (upload_secret, _, storage_index, _) = self.create_upload({1}, 100) with assert_fails_with_http_code(self, http.UNAUTHORIZED): result_of( From 5d51aac0d309a83f77f677c41b0da08ca27c991e Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 9 Mar 2022 12:41:40 -0500 Subject: [PATCH 0749/2309] Clarify. --- src/allmydata/test/test_storage_http.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index b2da8e9d7..e062864e2 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -934,7 +934,8 @@ class ImmutableHTTPAPITests(SyncTestCase): def test_unknown_aborts(self): """ - Aborting aborts with unknown storage index or share number will 404. + Aborting uploads with an unknown storage index or share number will + result 404 HTTP response code. """ (upload_secret, _, storage_index, _) = self.create_upload({1}, 100) From e598fbbc85589be52168d954b47f3c6974be31e3 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 9 Mar 2022 12:42:24 -0500 Subject: [PATCH 0750/2309] Get rid of redundant code. --- src/allmydata/storage/http_server.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 357426000..6a43dec8b 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -182,10 +182,7 @@ class UploadsInProgress(object): If the given upload doesn't exist at all, nothing happens. """ if storage_index in self._uploads: - try: - in_progress = self._uploads[storage_index] - except KeyError: - return + in_progress = self._uploads[storage_index] # For pre-existing upload, make sure password matches. if share_number in in_progress.upload_secrets and not timing_safe_compare( in_progress.upload_secrets[share_number], upload_secret From ba604b8231d85e0ec16709cdbeced3e9014910a1 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 9 Mar 2022 12:44:24 -0500 Subject: [PATCH 0751/2309] News file. --- newsfragments/3879.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3879.minor diff --git a/newsfragments/3879.minor b/newsfragments/3879.minor new file mode 100644 index 000000000..e69de29bb From 636ab017d47b7668d9326a064055a2cdfbb78b78 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 9 Mar 2022 12:47:14 -0500 Subject: [PATCH 0752/2309] Disconnection is purely a Foolscap concern. --- src/allmydata/test/test_istorageserver.py | 57 ++++++++++------------- 1 file changed, 25 insertions(+), 32 deletions(-) diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index 668eeecc5..fea14df79 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -194,20 +194,6 @@ class IStorageServerImmutableAPIsTestsMixin(object): ) yield allocated[0].callRemote("write", 0, b"2" * 1024) - def test_disconnection(self): - """ - If we disconnect in the middle of writing to a bucket, all data is - wiped, and it's even possible to write different data to the bucket. - - (In the real world one shouldn't do that, but writing different data is - a good way to test that the original data really was wiped.) - - HTTP protocol should skip this test, since disconnection is meaningless - concept; this is more about testing implicit contract the Foolscap - implementation depends on doesn't change as we refactor things. - """ - return self.abort_or_disconnect_half_way(lambda _: self.disconnect()) - @inlineCallbacks def test_written_shares_are_allocated(self): """ @@ -1062,13 +1048,6 @@ class _SharedMixin(SystemTestMixin): AsyncTestCase.tearDown(self) yield SystemTestMixin.tearDown(self) - @inlineCallbacks - def disconnect(self): - """ - Disconnect and then reconnect with a new ``IStorageServer``. - """ - raise NotImplementedError("implement in subclass") - class _FoolscapMixin(_SharedMixin): """Run tests on Foolscap version of ``IStorageServer``.""" @@ -1081,16 +1060,6 @@ class _FoolscapMixin(_SharedMixin): self.assertTrue(IStorageServer.providedBy(client)) return succeed(client) - @inlineCallbacks - def disconnect(self): - """ - Disconnect and then reconnect with a new ``IStorageServer``. - """ - current = self.storage_client - yield self.bounce_client(0) - self.storage_client = self._get_native_server().get_storage_server() - assert self.storage_client is not current - class _HTTPMixin(_SharedMixin): """Run tests on the HTTP version of ``IStorageServer``.""" @@ -1149,6 +1118,31 @@ class FoolscapImmutableAPIsTests( ): """Foolscap-specific tests for immutable ``IStorageServer`` APIs.""" + def test_disconnection(self): + """ + If we disconnect in the middle of writing to a bucket, all data is + wiped, and it's even possible to write different data to the bucket. + + (In the real world one shouldn't do that, but writing different data is + a good way to test that the original data really was wiped.) + + HTTP protocol doesn't need this test, since disconnection is a + meaningless concept; this is more about testing the implicit contract + the Foolscap implementation depends on doesn't change as we refactor + things. + """ + return self.abort_or_disconnect_half_way(lambda _: self.disconnect()) + + @inlineCallbacks + def disconnect(self): + """ + Disconnect and then reconnect with a new ``IStorageServer``. + """ + current = self.storage_client + yield self.bounce_client(0) + self.storage_client = self._get_native_server().get_storage_server() + assert self.storage_client is not current + class HTTPImmutableAPIsTests( _HTTPMixin, IStorageServerImmutableAPIsTestsMixin, AsyncTestCase @@ -1161,7 +1155,6 @@ class HTTPImmutableAPIsTests( "test_add_new_lease", "test_advise_corrupt_share", "test_bucket_advise_corrupt_share", - "test_disconnection", } From aee0f7dc69d850c4d971b25bbb3c1db2cb269b8e Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 9 Mar 2022 13:10:13 -0500 Subject: [PATCH 0753/2309] Sketch of lease renewal implementation. --- src/allmydata/storage/http_client.py | 33 ++++++++++++++++++----- src/allmydata/storage/http_server.py | 24 +++++++++++++++++ src/allmydata/storage_client.py | 11 ++++++++ src/allmydata/test/test_istorageserver.py | 2 -- 4 files changed, 62 insertions(+), 8 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index d83ecbdff..1610f5433 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -310,9 +310,7 @@ class StorageClientImmutables(object): body = yield response.content() returnValue(body) else: - raise ClientException( - response.code, - ) + raise ClientException(response.code) @inlineCallbacks def list_shares(self, storage_index): # type: (bytes,) -> Deferred[Set[int]] @@ -330,6 +328,29 @@ class StorageClientImmutables(object): body = yield _decode_cbor(response) returnValue(set(body)) else: - raise ClientException( - response.code, - ) + raise ClientException(response.code) + + @inlineCallbacks + def add_or_renew_lease( + self, storage_index: bytes, renew_secret: bytes, cancel_secret: bytes + ): + """ + Add or renew a lease. + + If the renewal secret matches an existing lease, it is renewed. + Otherwise a new lease is added. + """ + url = self._client.relative_url( + "/v1/lease/{}".format(_encode_si(storage_index)) + ) + response = yield self._client.request( + "PUT", + url, + lease_renew_secret=renew_secret, + lease_cancel_secret=cancel_secret, + ) + + if response.code == http.NO_CONTENT: + return + else: + raise ClientException(response.code) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 6a43dec8b..37bb80d8f 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -434,3 +434,27 @@ class HTTPServer(object): ContentRange("bytes", offset, offset + len(data)).to_header(), ) return data + + @_authorized_route( + _app, + {Secrets.LEASE_RENEW, Secrets.LEASE_CANCEL}, + "/v1/lease/", + methods=["PUT"], + ) + def add_or_renew_lease(self, request, authorization, storage_index): + """Update the lease for an immutable share.""" + # TODO 3879 write direct test for success case + + # Checking of the renewal secret is done by the backend. + try: + self._storage_server.add_lease( + storage_index, + authorization[Secrets.LEASE_RENEW], + authorization[Secrets.LEASE_CANCEL], + ) + except IndexError: + # TODO 3879 write test for this case + raise + + request.setResponseCode(http.NO_CONTENT) + return b"" diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 0d3159b55..ac74f6c67 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -1160,3 +1160,14 @@ class _HTTPStorageServer(object): )) for share_num in share_numbers }) + + def add_lease( + self, + storage_index, + renew_secret, + cancel_secret, + ): + immutable_client = StorageClientImmutables(self._http_client) + return immutable_client.add_or_renew_lease( + storage_index, renew_secret, cancel_secret + ) diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index fea14df79..e85047081 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -1151,8 +1151,6 @@ class HTTPImmutableAPIsTests( # These will start passing in future PRs as HTTP protocol is implemented. SKIP_TESTS = { - "test_add_lease_renewal", - "test_add_new_lease", "test_advise_corrupt_share", "test_bucket_advise_corrupt_share", } From f7366833475d1e6432de34b94fd42e546b9ad74b Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 9 Mar 2022 13:21:05 -0500 Subject: [PATCH 0754/2309] Finish testing and implementing lease renewal. --- src/allmydata/storage/http_server.py | 17 ++++---- src/allmydata/test/test_storage_http.py | 55 +++++++++++++++++++++++++ 2 files changed, 62 insertions(+), 10 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 37bb80d8f..2deb03dfd 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -443,18 +443,15 @@ class HTTPServer(object): ) def add_or_renew_lease(self, request, authorization, storage_index): """Update the lease for an immutable share.""" - # TODO 3879 write direct test for success case + if not self._storage_server.get_buckets(storage_index): + raise _HTTPError(http.NOT_FOUND) # Checking of the renewal secret is done by the backend. - try: - self._storage_server.add_lease( - storage_index, - authorization[Secrets.LEASE_RENEW], - authorization[Secrets.LEASE_CANCEL], - ) - except IndexError: - # TODO 3879 write test for this case - raise + self._storage_server.add_lease( + storage_index, + authorization[Secrets.LEASE_RENEW], + authorization[Secrets.LEASE_CANCEL], + ) request.setResponseCode(http.NO_CONTENT) return b"" diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index e062864e2..31505a00f 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -1000,3 +1000,58 @@ class ImmutableHTTPAPITests(SyncTestCase): ) ), ) + + def test_lease_renew_and_add(self): + """ + It's possible the renew the lease on an uploaded immutable, by using + the same renewal secret, or add a new lease by choosing a different + renewal secret. + """ + # Create immutable: + (upload_secret, lease_secret, storage_index, _) = self.create_upload({0}, 100) + result_of( + self.imm_client.write_share_chunk( + storage_index, + 0, + upload_secret, + 0, + b"A" * 100, + ) + ) + + [lease] = self.http.storage_server.get_leases(storage_index) + initial_expiration_time = lease.get_expiration_time() + + # Time passes: + self.http.clock.advance(167) + + # We renew the lease: + result_of( + self.imm_client.add_or_renew_lease( + storage_index, lease_secret, lease_secret + ) + ) + + # More time passes: + self.http.clock.advance(10) + + # We create a new lease: + lease_secret2 = urandom(32) + result_of( + self.imm_client.add_or_renew_lease( + storage_index, lease_secret2, lease_secret2 + ) + ) + + [lease1, lease2] = self.http.storage_server.get_leases(storage_index) + self.assertEqual(lease1.get_expiration_time(), initial_expiration_time + 167) + self.assertEqual(lease2.get_expiration_time(), initial_expiration_time + 177) + + def test_lease_on_unknown_storage_index(self): + """ + An attempt to renew an unknown storage index will result in a HTTP 404. + """ + storage_index = urandom(16) + secret = b"A" * 32 + with assert_fails_with_http_code(self, http.NOT_FOUND): + result_of(self.imm_client.add_or_renew_lease(storage_index, secret, secret)) From 922ee4feb117956a1a766c1f3644b669cb75d45f Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 10 Mar 2022 11:09:45 -0500 Subject: [PATCH 0755/2309] Sketch of advise_corrupt_share support for immutables. --- docs/proposed/http-storage-node-protocol.rst | 4 ++- src/allmydata/storage/http_client.py | 28 ++++++++++++++++++++ src/allmydata/storage/http_server.py | 19 +++++++++++++ src/allmydata/storage/server.py | 5 ++-- src/allmydata/storage_client.py | 28 ++++++++++++++++---- src/allmydata/test/test_istorageserver.py | 6 ----- 6 files changed, 76 insertions(+), 14 deletions(-) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index 0f534f0c5..33a9c0b0e 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -615,7 +615,7 @@ From RFC 7231:: !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! Advise the server the data read from the indicated share was corrupt. -The request body includes an human-meaningful string with details about the corruption. +The request body includes an human-meaningful (Unicode) string with details about the corruption. It also includes potentially important details about the share. For example:: @@ -624,6 +624,8 @@ For example:: .. share-type, storage-index, and share-number are inferred from the URL +The response code is OK, or 404 not found if the share couldn't be found. + Reading ~~~~~~~ diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 1610f5433..d0ae4b584 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -354,3 +354,31 @@ class StorageClientImmutables(object): return else: raise ClientException(response.code) + + @inlineCallbacks + def advise_corrupt_share( + self, + storage_index: bytes, + share_number: int, + reason: str, + ): + """Indicate a share has been corrupted, with a human-readable message.""" + assert isinstance(reason, str) + url = self._client.relative_url( + "/v1/immutable/{}/{}/corrupt".format( + _encode_si(storage_index), share_number + ) + ) + message = dumps({"reason": reason}) + response = yield self._client.request( + "POST", + url, + data=message, + headers=Headers({"content-type": ["application/cbor"]}), + ) + if response.code == http.OK: + return + else: + raise ClientException( + response.code, + ) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 2deb03dfd..5d357b846 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -455,3 +455,22 @@ class HTTPServer(object): request.setResponseCode(http.NO_CONTENT) return b"" + + @_authorized_route( + _app, + set(), + "/v1/immutable///corrupt", + methods=["POST"], + ) + def advise_corrupt_share(self, request, authorization, storage_index, share_number): + """Indicate that given share is corrupt, with a text reason.""" + # TODO 3879 test success path + try: + bucket = self._storage_server.get_buckets(storage_index)[share_number] + except KeyError: + # TODO 3879 test this path + raise _HTTPError(http.NOT_FOUND) + + info = loads(request.content.read()) + bucket.advise_corrupt_share(info["reason"].encode("utf-8")) + return b"" diff --git a/src/allmydata/storage/server.py b/src/allmydata/storage/server.py index 0add9806b..7ef7b4d37 100644 --- a/src/allmydata/storage/server.py +++ b/src/allmydata/storage/server.py @@ -743,8 +743,9 @@ class StorageServer(service.MultiService): def advise_corrupt_share(self, share_type, storage_index, shnum, reason): - # This is a remote API, I believe, so this has to be bytes for legacy - # protocol backwards compatibility reasons. + # Previously this had to be bytes for legacy protocol backwards + # compatibility reasons. Now that Foolscap layer has been abstracted + # out, we can probably refactor this to be unicode... assert isinstance(share_type, bytes) assert isinstance(reason, bytes), "%r is not bytes" % (reason,) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index ac74f6c67..55b6cfb05 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -77,7 +77,7 @@ from allmydata.util.hashutil import permute_server_hash from allmydata.util.dictutil import BytesKeyDict, UnicodeKeyDict from allmydata.storage.http_client import ( StorageClient, StorageClientImmutables, StorageClientGeneral, - ClientException as HTTPClientException, + ClientException as HTTPClientException ) @@ -1094,7 +1094,10 @@ class _HTTPBucketReader(object): ) def advise_corrupt_share(self, reason): - pass # TODO in later ticket + return self.client.advise_corrupt_share( + self.storage_index, self.share_number, + str(reason, "utf-8", errors="backslashreplace") + ) # WORK IN PROGRESS, for now it doesn't actually implement whole thing. @@ -1124,7 +1127,7 @@ class _HTTPStorageServer(object): cancel_secret, sharenums, allocated_size, - canary, + canary ): upload_secret = urandom(20) immutable_client = StorageClientImmutables(self._http_client) @@ -1148,7 +1151,7 @@ class _HTTPStorageServer(object): @defer.inlineCallbacks def get_buckets( self, - storage_index, + storage_index ): immutable_client = StorageClientImmutables(self._http_client) share_numbers = yield immutable_client.list_shares( @@ -1165,9 +1168,24 @@ class _HTTPStorageServer(object): self, storage_index, renew_secret, - cancel_secret, + cancel_secret ): immutable_client = StorageClientImmutables(self._http_client) return immutable_client.add_or_renew_lease( storage_index, renew_secret, cancel_secret ) + + def advise_corrupt_share( + self, + share_type, + storage_index, + shnum, + reason: bytes + ): + if share_type == b"immutable": + imm_client = StorageClientImmutables(self._http_client) + return imm_client.advise_corrupt_share( + storage_index, shnum, str(reason, "utf-8", errors="backslashreplace") + ) + else: + raise NotImplementedError() # future tickets diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index e85047081..253ff6046 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -1149,12 +1149,6 @@ class HTTPImmutableAPIsTests( ): """HTTP-specific tests for immutable ``IStorageServer`` APIs.""" - # These will start passing in future PRs as HTTP protocol is implemented. - SKIP_TESTS = { - "test_advise_corrupt_share", - "test_bucket_advise_corrupt_share", - } - class FoolscapMutableAPIsTests( _FoolscapMixin, IStorageServerMutableAPIsTestsMixin, AsyncTestCase From 7e25b43dbaff26449fe5dbef0b2f2d32a3ada883 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 10 Mar 2022 11:28:48 -0500 Subject: [PATCH 0756/2309] Direct unit tests for advising share is corrupt. --- src/allmydata/storage/http_server.py | 2 -- src/allmydata/test/test_storage_http.py | 32 +++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 5d357b846..d122b95b4 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -464,11 +464,9 @@ class HTTPServer(object): ) def advise_corrupt_share(self, request, authorization, storage_index, share_number): """Indicate that given share is corrupt, with a text reason.""" - # TODO 3879 test success path try: bucket = self._storage_server.get_buckets(storage_index)[share_number] except KeyError: - # TODO 3879 test this path raise _HTTPError(http.NOT_FOUND) info = loads(request.content.read()) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 31505a00f..70a9f1c16 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -1055,3 +1055,35 @@ class ImmutableHTTPAPITests(SyncTestCase): secret = b"A" * 32 with assert_fails_with_http_code(self, http.NOT_FOUND): result_of(self.imm_client.add_or_renew_lease(storage_index, secret, secret)) + + def test_advise_corrupt_share(self): + """ + Advising share was corrupted succeeds from HTTP client's perspective, + and calls appropriate method on server. + """ + corrupted = [] + self.http.storage_server.advise_corrupt_share = lambda *args: corrupted.append( + args + ) + + storage_index, _ = self.upload(13) + reason = "OHNO \u1235" + result_of(self.imm_client.advise_corrupt_share(storage_index, 13, reason)) + + self.assertEqual( + corrupted, [(b"immutable", storage_index, 13, reason.encode("utf-8"))] + ) + + def test_advise_corrupt_share_unknown(self): + """ + Advising an unknown share was corrupted results in 404. + """ + storage_index, _ = self.upload(13) + reason = "OHNO \u1235" + result_of(self.imm_client.advise_corrupt_share(storage_index, 13, reason)) + + for (si, share_number) in [(storage_index, 11), (urandom(16), 13)]: + with assert_fails_with_http_code(self, http.NOT_FOUND): + result_of( + self.imm_client.advise_corrupt_share(si, share_number, reason) + ) From 5baf63219da4b73270b7f304e0f48e36045d05d8 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 10 Mar 2022 17:40:35 -0500 Subject: [PATCH 0757/2309] Always use UTF-8 for corruption reports. --- newsfragments/3879.minor | 1 + src/allmydata/storage/server.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/newsfragments/3879.minor b/newsfragments/3879.minor index e69de29bb..ca3f24f94 100644 --- a/newsfragments/3879.minor +++ b/newsfragments/3879.minor @@ -0,0 +1 @@ +Share corruption reports stored on disk are now always encoded in UTF-8. \ No newline at end of file diff --git a/src/allmydata/storage/server.py b/src/allmydata/storage/server.py index 7ef7b4d37..9d1a3d6a4 100644 --- a/src/allmydata/storage/server.py +++ b/src/allmydata/storage/server.py @@ -778,7 +778,7 @@ class StorageServer(service.MultiService): si_s, shnum, ) - with open(report_path, "w") as f: + with open(report_path, "w", encoding="utf-8") as f: f.write(report) return None From e4b4dc418a0ca2bb27a01f7c748842744262e72a Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 14 Mar 2022 10:15:43 -0400 Subject: [PATCH 0758/2309] Address review comments. --- docs/proposed/http-storage-node-protocol.rst | 11 ++++++----- newsfragments/{3879.minor => 3879.incompat} | 0 2 files changed, 6 insertions(+), 5 deletions(-) rename newsfragments/{3879.minor => 3879.incompat} (100%) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index 33a9c0b0e..2ceb3c03a 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -614,17 +614,18 @@ From RFC 7231:: ``POST /v1/immutable/:storage_index/:share_number/corrupt`` !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! -Advise the server the data read from the indicated share was corrupt. -The request body includes an human-meaningful (Unicode) string with details about the corruption. -It also includes potentially important details about the share. +Advise the server the data read from the indicated share was corrupt. The +request body includes an human-meaningful text string with details about the +corruption. It also includes potentially important details about the share. For example:: - {"reason": "expected hash abcd, got hash efgh"} + {"reason": u"expected hash abcd, got hash efgh"} .. share-type, storage-index, and share-number are inferred from the URL -The response code is OK, or 404 not found if the share couldn't be found. +The response code is OK (200) by default, or NOT FOUND (404) if the share +couldn't be found. Reading ~~~~~~~ diff --git a/newsfragments/3879.minor b/newsfragments/3879.incompat similarity index 100% rename from newsfragments/3879.minor rename to newsfragments/3879.incompat From f815083b4dfbadf0aab35ca32960b27c0b70c058 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 14 Mar 2022 10:20:28 -0400 Subject: [PATCH 0759/2309] News file. --- newsfragments/3881.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3881.minor diff --git a/newsfragments/3881.minor b/newsfragments/3881.minor new file mode 100644 index 000000000..e69de29bb From e55c3e8acf7708f1f548a12d393b9914558fbdbb Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 14 Mar 2022 10:35:39 -0400 Subject: [PATCH 0760/2309] Check for CBOR content-encoding header in client. --- src/allmydata/storage/http_client.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index d0ae4b584..7458f9271 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -58,8 +58,14 @@ class ClientException(Exception): def _decode_cbor(response): """Given HTTP response, return decoded CBOR body.""" if response.code > 199 and response.code < 300: - return treq.content(response).addCallback(loads) - return fail(ClientException(response.code, response.phrase)) + if response.headers.getRawHeaders("content-type") == ["application/cbor"]: + # TODO limit memory usage + # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3872 + return treq.content(response).addCallback(loads) + else: + raise ClientException(-1, "Server didn't send CBOR") + else: + return fail(ClientException(response.code, response.phrase)) @attr.s From b8ab3dd6a7c7c9bdc90894fd7b0112318464d32c Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 14 Mar 2022 10:53:22 -0400 Subject: [PATCH 0761/2309] Server handles Accept headers. --- src/allmydata/storage/http_server.py | 35 ++++++++++++++++--------- src/allmydata/test/test_storage_http.py | 11 ++++++++ 2 files changed, 34 insertions(+), 12 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index d122b95b4..79eb35e56 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -11,7 +11,11 @@ import binascii from klein import Klein from twisted.web import http import attr -from werkzeug.http import parse_range_header, parse_content_range_header +from werkzeug.http import ( + parse_range_header, + parse_content_range_header, + parse_accept_header, +) from werkzeug.routing import BaseConverter, ValidationError from werkzeug.datastructures import ContentRange @@ -243,20 +247,27 @@ class HTTPServer(object): """Return twisted.web ``Resource`` for this object.""" return self._app.resource() - def _cbor(self, request, data): - """Return CBOR-encoded data.""" - # TODO Might want to optionally send JSON someday, based on Accept - # headers, see https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3861 - request.setHeader("Content-Type", "application/cbor") - # TODO if data is big, maybe want to use a temporary file eventually... - return dumps(data) + def _send_encoded(self, request, data): + """Return encoded data, by default using CBOR.""" + cbor_mime = "application/cbor" + accept_headers = request.requestHeaders.getRawHeaders("accept") or [cbor_mime] + accept = parse_accept_header(accept_headers[0]) + if accept.best == cbor_mime: + request.setHeader("Content-Type", cbor_mime) + # TODO if data is big, maybe want to use a temporary file eventually... + # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3872 + return dumps(data) + else: + # TODO Might want to optionally send JSON someday: + # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3861 + raise _HTTPError(http.NOT_ACCEPTABLE) ##### Generic APIs ##### @_authorized_route(_app, set(), "/v1/version", methods=["GET"]) def version(self, request, authorization): """Return version information.""" - return self._cbor(request, self._storage_server.get_version()) + return self._send_encoded(request, self._storage_server.get_version()) ##### Immutable APIs ##### @@ -291,7 +302,7 @@ class HTTPServer(object): storage_index, share_number, upload_secret, bucket ) - return self._cbor( + return self._send_encoded( request, { "already-have": set(already_got), @@ -367,7 +378,7 @@ class HTTPServer(object): required = [] for start, end, _ in bucket.required_ranges().ranges(): required.append({"begin": start, "end": end}) - return self._cbor(request, {"required": required}) + return self._send_encoded(request, {"required": required}) @_authorized_route( _app, @@ -380,7 +391,7 @@ class HTTPServer(object): List shares for the given storage index. """ share_numbers = list(self._storage_server.get_buckets(storage_index).keys()) - return self._cbor(request, share_numbers) + return self._send_encoded(request, share_numbers) @_authorized_route( _app, diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 70a9f1c16..c20012a9b 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -358,6 +358,17 @@ class GenericHTTPAPITests(SyncTestCase): with assert_fails_with_http_code(self, http.UNAUTHORIZED): result_of(client.get_version()) + def test_unsupported_mime_type(self): + """ + The client can request mime types other than CBOR, and if they are + unsupported a NOT ACCEPTABLE (406) error will be returned. + """ + client = StorageClientGeneral( + StorageClientWithHeadersOverride(self.http.client, {"accept": "image/gif"}) + ) + with assert_fails_with_http_code(self, http.NOT_ACCEPTABLE): + result_of(client.get_version()) + def test_version(self): """ The client can return the version. From 1e108f8445aad556a79c7dbe817de2d51624b112 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 14 Mar 2022 11:01:09 -0400 Subject: [PATCH 0762/2309] Don't use a custom parser. --- src/allmydata/storage/http_client.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 7458f9271..5733c1514 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -32,6 +32,7 @@ import attr from cbor2 import loads, dumps from collections_extended import RangeMap from werkzeug.datastructures import Range, ContentRange +from werkzeug.http import parse_options_header from twisted.web.http_headers import Headers from twisted.web import http from twisted.internet.defer import inlineCallbacks, returnValue, fail, Deferred @@ -58,7 +59,10 @@ class ClientException(Exception): def _decode_cbor(response): """Given HTTP response, return decoded CBOR body.""" if response.code > 199 and response.code < 300: - if response.headers.getRawHeaders("content-type") == ["application/cbor"]: + content_type = parse_options_header( + (response.headers.getRawHeaders("content-type") or [None])[0] + )[0] + if content_type == "application/cbor": # TODO limit memory usage # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3872 return treq.content(response).addCallback(loads) From 13fd3b3685ab2255931353387bd07139bd1165cb Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 14 Mar 2022 11:01:20 -0400 Subject: [PATCH 0763/2309] Get rid of Python 2 crud. --- src/allmydata/storage/http_client.py | 23 ++--------------------- 1 file changed, 2 insertions(+), 21 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 5733c1514..2790f1f7e 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -2,27 +2,8 @@ HTTP client that talks to the HTTP storage server. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals - -from future.utils import PY2 - -if PY2: - # fmt: off - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - # fmt: on - from collections import defaultdict - - Optional = Set = defaultdict( - lambda: None - ) # some garbage to just make this module import -else: - # typing module not available in Python 2, and we only do type checking in - # Python 3 anyway. - from typing import Union, Set, Optional - from treq.testing import StubTreq +from typing import Union, Set, Optional +from treq.testing import StubTreq from base64 import b64encode From fef332754b28672659cd42ce1b5a47d677b3ef41 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 14 Mar 2022 11:09:40 -0400 Subject: [PATCH 0764/2309] Switch to shared utility so server can use it too. --- src/allmydata/storage/http_client.py | 7 ++----- src/allmydata/storage/http_common.py | 21 +++++++++++++++------ src/allmydata/test/test_storage_http.py | 20 ++++++++++++++++++++ 3 files changed, 37 insertions(+), 11 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 2790f1f7e..ace97508c 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -13,14 +13,13 @@ import attr from cbor2 import loads, dumps from collections_extended import RangeMap from werkzeug.datastructures import Range, ContentRange -from werkzeug.http import parse_options_header from twisted.web.http_headers import Headers from twisted.web import http from twisted.internet.defer import inlineCallbacks, returnValue, fail, Deferred from hyperlink import DecodedURL import treq -from .http_common import swissnum_auth_header, Secrets +from .http_common import swissnum_auth_header, Secrets, get_content_type from .common import si_b2a @@ -40,9 +39,7 @@ class ClientException(Exception): def _decode_cbor(response): """Given HTTP response, return decoded CBOR body.""" if response.code > 199 and response.code < 300: - content_type = parse_options_header( - (response.headers.getRawHeaders("content-type") or [None])[0] - )[0] + content_type = get_content_type(response.headers) if content_type == "application/cbor": # TODO limit memory usage # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3872 diff --git a/src/allmydata/storage/http_common.py b/src/allmydata/storage/http_common.py index af4224bd0..fdf637180 100644 --- a/src/allmydata/storage/http_common.py +++ b/src/allmydata/storage/http_common.py @@ -1,15 +1,24 @@ """ Common HTTP infrastructure for the storge server. """ -from future.utils import PY2 - -if PY2: - # fmt: off - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - # fmt: on from enum import Enum from base64 import b64encode +from typing import Optional + +from werkzeug.http import parse_options_header +from twisted.web.http_headers import Headers + + +def get_content_type(headers: Headers) -> Optional[str]: + """ + Get the content type from the HTTP ``Content-Type`` header. + + Returns ``None`` if no content-type was set. + """ + values = headers.getRawHeaders("content-type") or [None] + content_type = parse_options_header(values[0])[0] or None + return content_type def swissnum_auth_header(swissnum): # type: (bytes) -> bytes diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index c20012a9b..af90d58a9 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -49,9 +49,29 @@ from ..storage.http_client import ( StorageClientGeneral, _encode_si, ) +from ..storage.http_common import get_content_type from ..storage.common import si_b2a +class HTTPUtilities(SyncTestCase): + """Tests for HTTP common utilities.""" + + def test_get_content_type(self): + """``get_content_type()`` extracts the content-type from the header.""" + + def assert_header_values_result(values, expected_content_type): + headers = Headers() + if values: + headers.setRawHeaders("Content-Type", values) + content_type = get_content_type(headers) + self.assertEqual(content_type, expected_content_type) + + assert_header_values_result(["text/html"], "text/html") + assert_header_values_result([], None) + assert_header_values_result(["text/plain", "application/json"], "text/plain") + assert_header_values_result(["text/html;encoding=utf-8"], "text/html") + + def _post_process(params): secret_types, secrets = params secrets = {t: s for (t, s) in zip(secret_types, secrets)} From b6073b11c2706a6941754bc5ab36ebe5b7960afa Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 14 Mar 2022 11:16:09 -0400 Subject: [PATCH 0765/2309] Refactor to check HTTP content-type of request body. --- src/allmydata/storage/http_server.py | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 79eb35e56..673800321 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -2,7 +2,7 @@ HTTP server for storage. """ -from typing import Dict, List, Set, Tuple +from typing import Dict, List, Set, Tuple, Any from functools import wraps from base64 import b64decode @@ -23,7 +23,7 @@ from werkzeug.datastructures import ContentRange from cbor2 import dumps, loads from .server import StorageServer -from .http_common import swissnum_auth_header, Secrets +from .http_common import swissnum_auth_header, Secrets, get_content_type from .common import si_a2b from .immutable import BucketWriter, ConflictingWriteError from ..util.hashutil import timing_safe_compare @@ -248,7 +248,9 @@ class HTTPServer(object): return self._app.resource() def _send_encoded(self, request, data): - """Return encoded data, by default using CBOR.""" + """ + Return encoded data as the HTTP body response, by default using CBOR. + """ cbor_mime = "application/cbor" accept_headers = request.requestHeaders.getRawHeaders("accept") or [cbor_mime] accept = parse_accept_header(accept_headers[0]) @@ -262,6 +264,18 @@ class HTTPServer(object): # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3861 raise _HTTPError(http.NOT_ACCEPTABLE) + def _read_encoded(self, request) -> Any: + """ + Read encoded request body data, decoding it with CBOR by default. + """ + content_type = get_content_type(request.requestHeaders) + if content_type == "application/cbor": + # TODO limit memory usage, client could send arbitrarily large data... + # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3872 + return loads(request.content.read()) + else: + raise _HTTPError(http.UNSUPPORTED_MEDIA_TYPE) + ##### Generic APIs ##### @_authorized_route(_app, set(), "/v1/version", methods=["GET"]) @@ -280,7 +294,7 @@ class HTTPServer(object): def allocate_buckets(self, request, authorization, storage_index): """Allocate buckets.""" upload_secret = authorization[Secrets.UPLOAD] - info = loads(request.content.read()) + info = self._read_encoded(request) # We do NOT validate the upload secret for existing bucket uploads. # Another upload may be happening in parallel, with a different upload @@ -480,6 +494,6 @@ class HTTPServer(object): except KeyError: raise _HTTPError(http.NOT_FOUND) - info = loads(request.content.read()) + info = self._read_encoded(request) bucket.advise_corrupt_share(info["reason"].encode("utf-8")) return b"" From 722f8e9598a60c870396e2a00164a835e4e485db Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 14 Mar 2022 11:17:06 -0400 Subject: [PATCH 0766/2309] Expand docs. --- src/allmydata/storage/http_server.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 673800321..0bd9f5dfc 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -249,7 +249,10 @@ class HTTPServer(object): def _send_encoded(self, request, data): """ - Return encoded data as the HTTP body response, by default using CBOR. + Return encoded data suitable for writing as the HTTP body response, by + default using CBOR. + + Also sets the appropriate ``Content-Type`` header on the response. """ cbor_mime = "application/cbor" accept_headers = request.requestHeaders.getRawHeaders("accept") or [cbor_mime] From 106cc708a0a1d989c168ef4ad105be5b6a9b954e Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 14 Mar 2022 11:18:53 -0400 Subject: [PATCH 0767/2309] Use a constant. --- src/allmydata/storage/http_client.py | 8 ++++---- src/allmydata/storage/http_common.py | 2 ++ src/allmydata/storage/http_server.py | 13 +++++++------ 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index ace97508c..f38459cee 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -19,7 +19,7 @@ from twisted.internet.defer import inlineCallbacks, returnValue, fail, Deferred from hyperlink import DecodedURL import treq -from .http_common import swissnum_auth_header, Secrets, get_content_type +from .http_common import swissnum_auth_header, Secrets, get_content_type, CBOR_MIME_TYPE from .common import si_b2a @@ -40,7 +40,7 @@ def _decode_cbor(response): """Given HTTP response, return decoded CBOR body.""" if response.code > 199 and response.code < 300: content_type = get_content_type(response.headers) - if content_type == "application/cbor": + if content_type == CBOR_MIME_TYPE: # TODO limit memory usage # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3872 return treq.content(response).addCallback(loads) @@ -186,7 +186,7 @@ class StorageClientImmutables(object): lease_cancel_secret=lease_cancel_secret, upload_secret=upload_secret, data=message, - headers=Headers({"content-type": ["application/cbor"]}), + headers=Headers({"content-type": [CBOR_MIME_TYPE]}), ) decoded_response = yield _decode_cbor(response) returnValue( @@ -362,7 +362,7 @@ class StorageClientImmutables(object): "POST", url, data=message, - headers=Headers({"content-type": ["application/cbor"]}), + headers=Headers({"content-type": [CBOR_MIME_TYPE]}), ) if response.code == http.OK: return diff --git a/src/allmydata/storage/http_common.py b/src/allmydata/storage/http_common.py index fdf637180..8313846c9 100644 --- a/src/allmydata/storage/http_common.py +++ b/src/allmydata/storage/http_common.py @@ -9,6 +9,8 @@ from typing import Optional from werkzeug.http import parse_options_header from twisted.web.http_headers import Headers +CBOR_MIME_TYPE = "application/cbor" + def get_content_type(headers: Headers) -> Optional[str]: """ diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 0bd9f5dfc..f648d8331 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -23,7 +23,7 @@ from werkzeug.datastructures import ContentRange from cbor2 import dumps, loads from .server import StorageServer -from .http_common import swissnum_auth_header, Secrets, get_content_type +from .http_common import swissnum_auth_header, Secrets, get_content_type, CBOR_MIME_TYPE from .common import si_a2b from .immutable import BucketWriter, ConflictingWriteError from ..util.hashutil import timing_safe_compare @@ -254,11 +254,12 @@ class HTTPServer(object): Also sets the appropriate ``Content-Type`` header on the response. """ - cbor_mime = "application/cbor" - accept_headers = request.requestHeaders.getRawHeaders("accept") or [cbor_mime] + accept_headers = request.requestHeaders.getRawHeaders("accept") or [ + CBOR_MIME_TYPE + ] accept = parse_accept_header(accept_headers[0]) - if accept.best == cbor_mime: - request.setHeader("Content-Type", cbor_mime) + if accept.best == CBOR_MIME_TYPE: + request.setHeader("Content-Type", CBOR_MIME_TYPE) # TODO if data is big, maybe want to use a temporary file eventually... # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3872 return dumps(data) @@ -272,7 +273,7 @@ class HTTPServer(object): Read encoded request body data, decoding it with CBOR by default. """ content_type = get_content_type(request.requestHeaders) - if content_type == "application/cbor": + if content_type == CBOR_MIME_TYPE: # TODO limit memory usage, client could send arbitrarily large data... # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3872 return loads(request.content.read()) From 0aa8089d81e29284b865ab5da510b8e1d173de7f Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 14 Mar 2022 11:20:23 -0400 Subject: [PATCH 0768/2309] Explicitly tell the server that the client accepts CBOR. --- src/allmydata/storage/http_client.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index f38459cee..6b32c13f7 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -99,6 +99,8 @@ class StorageClient(object): into corresponding HTTP headers. """ headers = self._get_headers(headers) + + # Add secrets: for secret, value in [ (Secrets.LEASE_RENEW, lease_renew_secret), (Secrets.LEASE_CANCEL, lease_cancel_secret), @@ -110,6 +112,10 @@ class StorageClient(object): "X-Tahoe-Authorization", b"%s %s" % (secret.value.encode("ascii"), b64encode(value).strip()), ) + + # Note we can accept CBOR: + headers.addRawHeader("Accept", CBOR_MIME_TYPE) + return self._treq.request(method, url, headers=headers, **kwargs) From fae9556e3dec59bd4db1bf0ecb7fb9ddd1ca5e71 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 14 Mar 2022 11:28:54 -0400 Subject: [PATCH 0769/2309] Centralize client serialization logic too. --- src/allmydata/storage/http_client.py | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 6b32c13f7..41a2dd0b8 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -92,11 +92,15 @@ class StorageClient(object): lease_cancel_secret=None, upload_secret=None, headers=None, + message_to_serialize=None, **kwargs ): """ Like ``treq.request()``, but with optional secrets that get translated into corresponding HTTP headers. + + If ``message_to_serialize`` is set, it will be serialized (by default + with CBOR) and set as the request body. """ headers = self._get_headers(headers) @@ -116,6 +120,13 @@ class StorageClient(object): # Note we can accept CBOR: headers.addRawHeader("Accept", CBOR_MIME_TYPE) + # If there's a request message, serialize it and set the Content-Type + # header: + if message_to_serialize is not None: + assert "data" not in kwargs + kwargs["data"] = dumps(message_to_serialize) + headers.addRawHeader("Content-Type", CBOR_MIME_TYPE) + return self._treq.request(method, url, headers=headers, **kwargs) @@ -182,17 +193,15 @@ class StorageClientImmutables(object): storage index failed the result will fire with an exception. """ url = self._client.relative_url("/v1/immutable/" + _encode_si(storage_index)) - message = dumps( - {"share-numbers": share_numbers, "allocated-size": allocated_size} - ) + message = {"share-numbers": share_numbers, "allocated-size": allocated_size} + response = yield self._client.request( "POST", url, lease_renew_secret=lease_renew_secret, lease_cancel_secret=lease_cancel_secret, upload_secret=upload_secret, - data=message, - headers=Headers({"content-type": [CBOR_MIME_TYPE]}), + message_to_serialize=message, ) decoded_response = yield _decode_cbor(response) returnValue( @@ -363,13 +372,8 @@ class StorageClientImmutables(object): _encode_si(storage_index), share_number ) ) - message = dumps({"reason": reason}) - response = yield self._client.request( - "POST", - url, - data=message, - headers=Headers({"content-type": [CBOR_MIME_TYPE]}), - ) + message = {"reason": reason} + response = yield self._client.request("POST", url, message_to_serialize=message) if response.code == http.OK: return else: From 7de3d93b0eba77e4df5e4947f46727391ba0d6ff Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 18 Mar 2022 10:12:51 -0400 Subject: [PATCH 0770/2309] Switch to TypeError. --- src/allmydata/storage/http_client.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 41a2dd0b8..99275ae24 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -123,7 +123,11 @@ class StorageClient(object): # If there's a request message, serialize it and set the Content-Type # header: if message_to_serialize is not None: - assert "data" not in kwargs + if "data" in kwargs: + raise TypeError( + "Can't use both `message_to_serialize` and `data` " + "as keyword arguments at the same time" + ) kwargs["data"] = dumps(message_to_serialize) headers.addRawHeader("Content-Type", CBOR_MIME_TYPE) From b83b6adfbc46d104151450d76f4b48e428161685 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 18 Mar 2022 10:41:07 -0400 Subject: [PATCH 0771/2309] remove py2 compat boilerplate --- src/allmydata/stats.py | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/src/allmydata/stats.py b/src/allmydata/stats.py index 13ed8817c..b893be6ff 100644 --- a/src/allmydata/stats.py +++ b/src/allmydata/stats.py @@ -1,17 +1,7 @@ """ Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals - -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - from time import clock as process_time -else: - from time import process_time +from time import process_time import time from twisted.application import service From 32e88e580b42f698f5540a44aacd2698093c9bca Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 18 Mar 2022 10:41:33 -0400 Subject: [PATCH 0772/2309] switch to a deque this makes more of the data structure someone else's responsibility and probably improves performance too (but I didn't measure) --- src/allmydata/stats.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/allmydata/stats.py b/src/allmydata/stats.py index b893be6ff..a123ff1d8 100644 --- a/src/allmydata/stats.py +++ b/src/allmydata/stats.py @@ -1,6 +1,8 @@ """ Ported to Python 3. """ + +from collections import deque from time import process_time import time @@ -26,7 +28,7 @@ class CPUUsageMonitor(service.MultiService): # up. self.initial_cpu = 0.0 # just in case eventually(self._set_initial_cpu) - self.samples = [] + self.samples: list[tuple[float, float]] = deque([], self.HISTORY_LENGTH) # we provide 1min, 5min, and 15min moving averages TimerService(self.POLL_INTERVAL, self.check).setServiceParent(self) @@ -37,8 +39,6 @@ class CPUUsageMonitor(service.MultiService): now_wall = time.time() now_cpu = process_time() self.samples.append( (now_wall, now_cpu) ) - while len(self.samples) > self.HISTORY_LENGTH+1: - self.samples.pop(0) def _average_N_minutes(self, size): if len(self.samples) < size+1: From ce381f3e39db35249bca42f97cff7e6fa74a1aed Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 18 Mar 2022 10:42:40 -0400 Subject: [PATCH 0773/2309] pull the default initial_cpu value out to class scope also add python 3 syntax type annotations --- src/allmydata/stats.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/allmydata/stats.py b/src/allmydata/stats.py index a123ff1d8..60a24ec02 100644 --- a/src/allmydata/stats.py +++ b/src/allmydata/stats.py @@ -16,8 +16,9 @@ from allmydata.interfaces import IStatsProducer @implementer(IStatsProducer) class CPUUsageMonitor(service.MultiService): - HISTORY_LENGTH = 15 - POLL_INTERVAL = 60 # type: float + HISTORY_LENGTH: int = 15 + POLL_INTERVAL: float = 60 + initial_cpu: float = 0.0 def __init__(self): service.MultiService.__init__(self) @@ -26,7 +27,6 @@ class CPUUsageMonitor(service.MultiService): # rest of the program will be run by the child process, after twistd # forks. Instead, set self.initial_cpu as soon as the reactor starts # up. - self.initial_cpu = 0.0 # just in case eventually(self._set_initial_cpu) self.samples: list[tuple[float, float]] = deque([], self.HISTORY_LENGTH) # we provide 1min, 5min, and 15min moving averages From b56f79c7ad52d85cccf0a83abed406d890a5fd83 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 18 Mar 2022 10:43:17 -0400 Subject: [PATCH 0774/2309] read initial cpu usage value at service start time This removes the Foolscap dependency. --- src/allmydata/stats.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/src/allmydata/stats.py b/src/allmydata/stats.py index 60a24ec02..a2d208560 100644 --- a/src/allmydata/stats.py +++ b/src/allmydata/stats.py @@ -9,7 +9,6 @@ import time from twisted.application import service from twisted.application.internet import TimerService from zope.interface import implementer -from foolscap.api import eventually from allmydata.util import log, dictutil from allmydata.interfaces import IStatsProducer @@ -22,18 +21,13 @@ class CPUUsageMonitor(service.MultiService): def __init__(self): service.MultiService.__init__(self) - # we don't use process_time() here, because the constructor is run by - # the twistd parent process (as it loads the .tac file), whereas the - # rest of the program will be run by the child process, after twistd - # forks. Instead, set self.initial_cpu as soon as the reactor starts - # up. - eventually(self._set_initial_cpu) self.samples: list[tuple[float, float]] = deque([], self.HISTORY_LENGTH) # we provide 1min, 5min, and 15min moving averages TimerService(self.POLL_INTERVAL, self.check).setServiceParent(self) - def _set_initial_cpu(self): + def startService(self): self.initial_cpu = process_time() + return super().startService() def check(self): now_wall = time.time() From e17f4e68048fa08c0a9c9c29016da931d894b143 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 18 Mar 2022 10:43:54 -0400 Subject: [PATCH 0775/2309] news fragment --- newsfragments/3883.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3883.minor diff --git a/newsfragments/3883.minor b/newsfragments/3883.minor new file mode 100644 index 000000000..e69de29bb From 9bcf241f10875fb2f5d7d592073d1770a9082408 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 18 Mar 2022 11:40:30 -0400 Subject: [PATCH 0776/2309] Use environment variables we expect most runners to have. --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index cf0c66aff..75e2ab749 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -398,6 +398,7 @@ jobs: image: "nixos/nix:2.3.16" environment: + <<: *UTF_8_ENVIRONMENT # CACHIX_AUTH_TOKEN is manually set in the CircleCI web UI and # allows us to push to CACHIX_NAME. We only need this set for # `cachix use` in this step. From 62ac5bd0841ccfdb76341e17944c716d19f1ad09 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 18 Mar 2022 11:40:54 -0400 Subject: [PATCH 0777/2309] News file. --- newsfragments/3882.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3882.minor diff --git a/newsfragments/3882.minor b/newsfragments/3882.minor new file mode 100644 index 000000000..e69de29bb From 211343eca81a126c97df8d43e03995aacfbd644b Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 18 Mar 2022 11:51:39 -0400 Subject: [PATCH 0778/2309] Set the Hypothesis profile in more robust way. --- .circleci/config.yml | 1 - tests.nix | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 75e2ab749..cf0c66aff 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -398,7 +398,6 @@ jobs: image: "nixos/nix:2.3.16" environment: - <<: *UTF_8_ENVIRONMENT # CACHIX_AUTH_TOKEN is manually set in the CircleCI web UI and # allows us to push to CACHIX_NAME. We only need this set for # `cachix use` in this step. diff --git a/tests.nix b/tests.nix index 53a8885c0..dd477c273 100644 --- a/tests.nix +++ b/tests.nix @@ -73,6 +73,7 @@ let in # Make a derivation that runs the unit test suite. pkgs.runCommand "tahoe-lafs-tests" { } '' + export TAHOE_LAFS_HYPOTHESIS_PROFILE=ci ${python-env}/bin/python -m twisted.trial -j $NIX_BUILD_CORES allmydata # It's not cool to put the whole _trial_temp into $out because it has weird From 98634ae5cb3f706ccf8d7a3abaa1c43588d36793 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 18 Mar 2022 12:36:48 -0400 Subject: [PATCH 0779/2309] allow the correct number of samples in also speed up the test with some shorter intervals --- src/allmydata/stats.py | 2 +- src/allmydata/test/test_stats.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/allmydata/stats.py b/src/allmydata/stats.py index a2d208560..6e8de47e9 100644 --- a/src/allmydata/stats.py +++ b/src/allmydata/stats.py @@ -21,7 +21,7 @@ class CPUUsageMonitor(service.MultiService): def __init__(self): service.MultiService.__init__(self) - self.samples: list[tuple[float, float]] = deque([], self.HISTORY_LENGTH) + self.samples: list[tuple[float, float]] = deque([], self.HISTORY_LENGTH + 1) # we provide 1min, 5min, and 15min moving averages TimerService(self.POLL_INTERVAL, self.check).setServiceParent(self) diff --git a/src/allmydata/test/test_stats.py b/src/allmydata/test/test_stats.py index e56f9d444..6fe690f1f 100644 --- a/src/allmydata/test/test_stats.py +++ b/src/allmydata/test/test_stats.py @@ -17,7 +17,7 @@ from allmydata.util import pollmixin import allmydata.test.common_util as testutil class FasterMonitor(CPUUsageMonitor): - POLL_INTERVAL = 0.1 + POLL_INTERVAL = 0.01 class CPUUsage(unittest.TestCase, pollmixin.PollMixin, testutil.StallMixin): @@ -36,9 +36,9 @@ class CPUUsage(unittest.TestCase, pollmixin.PollMixin, testutil.StallMixin): def _poller(): return bool(len(m.samples) == m.HISTORY_LENGTH+1) d = self.poll(_poller) - # pause one more second, to make sure that the history-trimming code - # is exercised - d.addCallback(self.stall, 1.0) + # pause a couple more intervals, to make sure that the history-trimming + # code is exercised + d.addCallback(self.stall, FasterMonitor.POLL_INTERVAL * 2) def _check(res): s = m.get_stats() self.failUnless("cpu_monitor.1min_avg" in s) From 5310747eaa69d4bedf5e89383de58bc6692a5827 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 23 Mar 2022 16:33:29 -0400 Subject: [PATCH 0780/2309] Start hooking up end-to-end tests with TLS, fixing bugs along the way. At this point the issue is that the client fails certificate validation (which is expected lacking the pinning validation logic, which should be added next). --- src/allmydata/storage/http_client.py | 6 +++-- src/allmydata/storage/http_common.py | 2 +- src/allmydata/storage/http_server.py | 22 +++++++++++----- src/allmydata/test/certs/domain.crt | 19 ++++++++++++++ src/allmydata/test/certs/private.key | 27 ++++++++++++++++++++ src/allmydata/test/test_istorageserver.py | 31 ++++++++++++----------- 6 files changed, 82 insertions(+), 25 deletions(-) create mode 100644 src/allmydata/test/certs/domain.crt create mode 100644 src/allmydata/test/certs/private.key diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 572da506c..cbb111d18 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -3,7 +3,6 @@ HTTP client that talks to the HTTP storage server. """ from typing import Union, Set, Optional -from treq.testing import StubTreq from base64 import b64encode @@ -77,7 +76,7 @@ class StorageClient(object): self._treq = treq @classmethod - def from_furl(cls, furl: DecodedURL) -> "StorageClient": + def from_furl(cls, furl: DecodedURL, treq=treq) -> "StorageClient": """ Create a ``StorageClient`` for the given furl. """ @@ -86,6 +85,9 @@ class StorageClient(object): swissnum = furl.path[0].encode("ascii") certificate_hash = furl.user.encode("ascii") + https_url = DecodedURL().replace(scheme="https", host=furl.host, port=furl.port) + return cls(https_url, swissnum, treq) + def relative_url(self, path): """Get a URL relative to the base URL.""" return self._base_url.click(path) diff --git a/src/allmydata/storage/http_common.py b/src/allmydata/storage/http_common.py index c5e087b5b..11ab880c7 100644 --- a/src/allmydata/storage/http_common.py +++ b/src/allmydata/storage/http_common.py @@ -48,4 +48,4 @@ def get_spki_hash(certificate: Certificate) -> bytes: public_key_bytes = certificate.public_key().public_bytes( Encoding.DER, PublicFormat.SubjectPublicKeyInfo ) - return b64encode(sha256(public_key_bytes).digest()).strip() + return b64encode(sha256(public_key_bytes).digest()).strip().rstrip(b"=") diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index c3228ea04..80b34daa9 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -534,6 +534,8 @@ def listen_tls( The hostname is the external IP or hostname clients will connect to; it does not modify what interfaces the server listens on. To set the listening interface, use the ``interface`` argument. + + Port can be 0 to choose a random port. """ endpoint_string = "ssl:privateKey={}:certKey={}:port={}".format( quoteStringArgument(str(private_key_path)), @@ -545,13 +547,19 @@ def listen_tls( endpoint = serverFromString(reactor, endpoint_string) def build_furl(listening_port: IListeningPort) -> DecodedURL: - furl = DecodedURL() - furl.fragment = "v=1" # HTTP-based - furl.host = hostname - furl.port = listening_port.getHost().port - furl.path = (server._swissnum,) - furl.user = get_spki_hash(load_pem_x509_certificate(cert_path.read_bytes())) - furl.scheme = "pb" + furl = DecodedURL().replace( + fragment="v=1", # HTTP-based + host=hostname, + port=listening_port.getHost().port, + path=(str(server._swissnum, "ascii"),), + userinfo=[ + str( + get_spki_hash(load_pem_x509_certificate(cert_path.read_bytes())), + "ascii", + ) + ], + scheme="pb", + ) return furl return endpoint.listen(Site(server.get_resource())).addCallback( diff --git a/src/allmydata/test/certs/domain.crt b/src/allmydata/test/certs/domain.crt new file mode 100644 index 000000000..8932b44e7 --- /dev/null +++ b/src/allmydata/test/certs/domain.crt @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIDCzCCAfMCFHrs4pMBs35SlU3ZGMnVY5qp5MfZMA0GCSqGSIb3DQEBCwUAMEIx +CzAJBgNVBAYTAlhYMRUwEwYDVQQHDAxEZWZhdWx0IENpdHkxHDAaBgNVBAoME0Rl +ZmF1bHQgQ29tcGFueSBMdGQwHhcNMjIwMzIzMjAwNTM0WhcNMjIwNDIyMjAwNTM0 +WjBCMQswCQYDVQQGEwJYWDEVMBMGA1UEBwwMRGVmYXVsdCBDaXR5MRwwGgYDVQQK +DBNEZWZhdWx0IENvbXBhbnkgTHRkMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIB +CgKCAQEAzj0C3W4OiugEb3nr7NVQfrgzL3Tet5ze8pJCew0lIsDNrZiESF/Lvnrc +VcQtjraC3ySZO3rLDLhCwALLC1TVw3lp2ou+02kYtfJJVr1XyEcVCpsxx89u/W5B +VgMyBMoVZLS2BoBA1652XZnphvgm8n/daf9HH2Y4ifRDTIl1Bgl+4XtqlCKNFGYO +7zeadViKeI3fJOyzQaT+WTONQRWtAMP6c/n6VnlrdNughUEM+X0HqwVmr7UgatuA +LrJpfAkfKfOTZ70p2iOvegBH5ryR3InsB/O0/fp2+esNCEU3pfXWzg8ACif+ZQJc +AukUpQ4iJB7r1AK6iTZUqHnFgu9gYwIDAQABMA0GCSqGSIb3DQEBCwUAA4IBAQCv +BzVSinez5lcSWWCRve/TlMePEJK5d7OFcc90n7kmn+rkEYrel3a7Q+ctJxY9AKYG +A9X1AMDSH9z3243KFNaRJ1xKg0Mg8J/BLN9iphM5AnAuiAkCqs8VbD4hF4hZHLMZ +BVNyuLSdo+lBzbS57/Lz+lUWcxrXR5qgsEWSbjP+SrsDQKODfyoxKuU0XrxmNLd2 +dTswbZKsqXBs80/T1jHwJjJXLp6YUsZqN1TGYtk8hEcE7bGaC3n7WhRjBP1WghNl +OG7FFRPte2w5seQRgkrBodLb9OCkhU4xfdyLnFICqkQHQAqIdXksEvir9uGY/yjC +zxAh60LEEQz6Kz29jYog +-----END CERTIFICATE----- diff --git a/src/allmydata/test/certs/private.key b/src/allmydata/test/certs/private.key new file mode 100644 index 000000000..b4b5ed580 --- /dev/null +++ b/src/allmydata/test/certs/private.key @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEpAIBAAKCAQEAzj0C3W4OiugEb3nr7NVQfrgzL3Tet5ze8pJCew0lIsDNrZiE +SF/LvnrcVcQtjraC3ySZO3rLDLhCwALLC1TVw3lp2ou+02kYtfJJVr1XyEcVCpsx +x89u/W5BVgMyBMoVZLS2BoBA1652XZnphvgm8n/daf9HH2Y4ifRDTIl1Bgl+4Xtq +lCKNFGYO7zeadViKeI3fJOyzQaT+WTONQRWtAMP6c/n6VnlrdNughUEM+X0HqwVm +r7UgatuALrJpfAkfKfOTZ70p2iOvegBH5ryR3InsB/O0/fp2+esNCEU3pfXWzg8A +Cif+ZQJcAukUpQ4iJB7r1AK6iTZUqHnFgu9gYwIDAQABAoIBAG71rGDuIazif+Bq +PGDDs/c5q3BQ9LLdF6Zywonp3J0CFqbbc/BsefYVrA4I6mnqECd2TWsO+cfyKxeb +aRrDne75l9YZcaXU2ZKqtIKShHQgqlV2giX6mMCJXWWlenfRMglooLaGslxYZR6e +/GG9iVbXLI0m52EhYjH21W6MVgXUhsrDoI16pB87zk7jFZzyNsjRU5+bwr4L2jed +A1PseE6AI2kIpJCl8IIu6hRhVwjr8MIkaAtI3G8WmSAru99apHNttf6sgB2kcq2r +Qp1uXEXNVFQiJqcwMPOlWZ5X0kMIBxmFe10MkJbmCUoE/jPqO90XN2jyZPSONZU8 +4yqd9GECgYEA5qQ+tfgOJFJ86t103pwkPm0+PxuQUTXry5ETnez+wxqqwbDxrEHi +MQoPZuVXPkbQ6g80KSpdI7AkFvu6BzcNlgpOI3gLZ30PHTF4HJTP01fKfbbVhg8N +WJS0yUh+kQDrGVcZbIbB5Q1vS5hu8ftk5ukns5BdFf/NS7fBfU8b3K0CgYEA5Onm +V2D1D9kdhTjta8f0d0s6+TdHoV86SRbkAEgnqzwlHWV17LlpXQic6iwokfS4TQSl ++1Z23Dt+OhLm/N0N3LgCxBhzTMnWGdy+w9co4GifwqR6T72JAxGOVoqIWk2cVpa5 +8qJx0eAFXqcvpIASEoxYrdoKFUh60mAiQE6JQ08CgYB8wCoLUviTPOrEPrSQE/Sm +r4ATsl0FEB1SJk5uBVpnPW1PBt4xRhGKZN6f0Ty3OqaVc1PLUFbAju12YQHmFSkM +Ftbc6HmCqGocaD2HeBZRQhMMnHAx6sJVP1np5YRP+icvtaTSxrDpq7KfOPwJdujE +3SfUQCmZVJs+cU3+8WMooQKBgQCdvvl2eWAm/a00IxipT2+NzY/kMU3xTFg0Ccww +zYhYnefNrB9pdBPBgq/vR2LlwchHes6OtvTNq0m+50u6MPLeiQeO7nJ2FhiuVco3 +1staaX6+eO24iZojPTPjOy/fWuBDYzbcl0jsIf5RTdCtAXxyv7hUhY6xP/Mzif/Q +ZM5+TQKBgQCF7pB6yLIAp6YJQ4uqOVbC2bMLr6tVWaNdEykiDd9zQkoacw/DPX1Y +FKehY/9EKraJ4t6d2/BBpJQyuIU4/gz8QMvjqQGP3NIfVeqBAPYo/nTYKOK0PSxB +Kl28Axxz6rjEeK4BixOES5PXuq2nNJXT8OSPYZQxQdTHstCWOP4Z6g== +-----END RSA PRIVATE KEY----- diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index 253ff6046..7b3810f21 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -24,12 +24,11 @@ else: from random import Random from unittest import SkipTest +from pathlib import Path from twisted.internet.defer import inlineCallbacks, returnValue, succeed from twisted.internet.task import Clock from twisted.internet import reactor -from twisted.internet.endpoints import serverFromString -from twisted.web.server import Site from twisted.web.client import Agent, HTTPConnectionPool from hyperlink import DecodedURL from treq.client import HTTPClient @@ -40,7 +39,7 @@ from allmydata.interfaces import IStorageServer # really, IStorageClient from .common_system import SystemTestMixin from .common import AsyncTestCase, SameProcessStreamEndpointAssigner from allmydata.storage.server import StorageServer # not a IStorageServer!! -from allmydata.storage.http_server import HTTPServer +from allmydata.storage.http_server import HTTPServer, listen_tls from allmydata.storage.http_client import StorageClient from allmydata.storage_client import _HTTPStorageServer @@ -1074,27 +1073,29 @@ class _HTTPMixin(_SharedMixin): swissnum = b"1234" http_storage_server = HTTPServer(self.server, swissnum) - # Listen on randomly assigned port: - tcp_address, endpoint_string = self._port_assigner.assign(reactor) - _, host, port = tcp_address.split(":") - port = int(port) - endpoint = serverFromString(reactor, endpoint_string) - listening_port = yield endpoint.listen(Site(http_storage_server.get_resource())) + # Listen on randomly assigned port, using self-signed cert we generated + # manually: + certs_dir = Path(__file__).parent / "certs" + furl, listening_port = yield listen_tls( + reactor, + http_storage_server, + "127.0.0.1", + 0, + certs_dir / "private.key", + certs_dir / "domain.crt", + interface="127.0.0.1", + ) self.addCleanup(listening_port.stopListening) # Create HTTP client with non-persistent connections, so we don't leak # state across tests: treq_client = HTTPClient( - Agent(reactor, HTTPConnectionPool(reactor, persistent=False)) + Agent(reactor, pool=HTTPConnectionPool(reactor, persistent=False)) ) returnValue( _HTTPStorageServer.from_http_client( - StorageClient( - DecodedURL().replace(scheme="http", host=host, port=port), - swissnum, - treq=treq_client, - ) + StorageClient.from_furl(furl, treq_client) ) ) # Eventually should also: From c737bcdb6f59249d09eaa0587dfd76609dd66e8b Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 23 Mar 2022 18:40:12 -0400 Subject: [PATCH 0781/2309] use the generic version of the correct types for `samples` --- src/allmydata/stats.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/allmydata/stats.py b/src/allmydata/stats.py index 6e8de47e9..f6361b074 100644 --- a/src/allmydata/stats.py +++ b/src/allmydata/stats.py @@ -5,6 +5,7 @@ Ported to Python 3. from collections import deque from time import process_time import time +from typing import Deque, Tuple from twisted.application import service from twisted.application.internet import TimerService @@ -21,7 +22,7 @@ class CPUUsageMonitor(service.MultiService): def __init__(self): service.MultiService.__init__(self) - self.samples: list[tuple[float, float]] = deque([], self.HISTORY_LENGTH + 1) + self.samples: Deque[Tuple[float, float]] = deque([], self.HISTORY_LENGTH + 1) # we provide 1min, 5min, and 15min moving averages TimerService(self.POLL_INTERVAL, self.check).setServiceParent(self) From be0ff08275695aad99c8e3cceba6700b52b73be3 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 23 Mar 2022 17:18:06 -0400 Subject: [PATCH 0782/2309] Possibly correct, but communicating, end-to-end TLS with some amount of validation logic. Still untested! --- src/allmydata/storage/http_client.py | 109 +++++++++++++++++++++- src/allmydata/test/test_istorageserver.py | 10 +- src/allmydata/test/test_storage_http.py | 2 +- 3 files changed, 109 insertions(+), 12 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index cbb111d18..fc27e1e30 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -14,13 +14,26 @@ from collections_extended import RangeMap from werkzeug.datastructures import Range, ContentRange from twisted.web.http_headers import Headers from twisted.web import http +from twisted.web.iweb import IPolicyForHTTPS from twisted.internet.defer import inlineCallbacks, returnValue, fail, Deferred +from twisted.internet.interfaces import IOpenSSLClientConnectionCreator +from twisted.internet.ssl import CertificateOptions +from twisted.internet import reactor +from twisted.web.client import Agent, HTTPConnectionPool +from zope.interface import implementer from hyperlink import DecodedURL import treq from treq.client import HTTPClient from treq.testing import StubTreq +from OpenSSL import SSL -from .http_common import swissnum_auth_header, Secrets, get_content_type, CBOR_MIME_TYPE +from .http_common import ( + swissnum_auth_header, + Secrets, + get_content_type, + CBOR_MIME_TYPE, + get_spki_hash, +) from .common import si_b2a @@ -59,6 +72,86 @@ class ImmutableCreateResult(object): allocated = attr.ib(type=Set[int]) +class _TLSContextFactory(CertificateOptions): + """ + Create a context that validates the way Tahoe-LAFS wants to: based on a + pinned certificate hash, rather than a certificate authority. + + Originally implemented as part of Foolscap. + """ + + def getContext(self, expected_spki_hash: bytes) -> SSL.Context: + def always_validate(conn, cert, errno, depth, preverify_ok): + # This function is called to validate the certificate received by + # the other end. OpenSSL calls it multiple times, each time it + # see something funny, to ask if it should proceed. + + # We do not care about certificate authorities or revocation + # lists, we just want to know that the certificate has a valid + # signature and follow the chain back to one which is + # self-signed. We need to protect against forged signatures, but + # not the usual TLS concerns about invalid CAs or revoked + # certificates. + + # these constants are from openssl-0.9.7g/crypto/x509/x509_vfy.h + # and do not appear to be exposed by pyopenssl. Ick. + things_are_ok = ( + 0, # X509_V_OK + 9, # X509_V_ERR_CERT_NOT_YET_VALID + 10, # X509_V_ERR_CERT_HAS_EXPIRED + 18, # X509_V_ERR_DEPTH_ZERO_SELF_SIGNED_CERT + 19, # X509_V_ERR_SELF_SIGNED_CERT_IN_CHAIN + ) + # TODO can we do this once instead of multiple times? + if ( + errno in things_are_ok + and get_spki_hash(cert.to_cryptography()) == expected_spki_hash + ): + return 1 + # TODO: log the details of the error, because otherwise they get + # lost in the PyOpenSSL exception that will eventually be raised + # (possibly OpenSSL.SSL.Error: certificate verify failed) + + # I think that X509_V_ERR_CERT_SIGNATURE_FAILURE is the most + # obvious sign of hostile attack. + return 0 + + ctx = CertificateOptions.getContext(self) + + # VERIFY_PEER means we ask the the other end for their certificate. + ctx.set_verify(SSL.VERIFY_PEER, always_validate) + return ctx + + +@implementer(IPolicyForHTTPS) +@implementer(IOpenSSLClientConnectionCreator) +@attr.s +class _StorageClientHTTPSPolicy: + """ + A HTTPS policy that: + + 1. Makes sure the SPKI hash of the certificate matches a known hash (NEEDS TEST). + 2. The certificate hasn't expired. (NEEDS TEST) + 3. The server has a private key that matches the certificate (NEEDS TEST). + + I.e. pinning-based validation. + """ + + expected_spki_hash = attr.ib(type=bytes) + + # IPolicyForHTTPS + def creatorForNetloc(self, hostname, port): + return self + + # IOpenSSLClientConnectionCreator + def clientConnectionForTLS(self, tlsProtocol): + connection = SSL.Connection( + _TLSContextFactory().getContext(self.expected_spki_hash), None + ) + connection.set_app_data(tlsProtocol) + return connection + + class StorageClient(object): """ Low-level HTTP client that talks to the HTTP storage server. @@ -76,17 +169,27 @@ class StorageClient(object): self._treq = treq @classmethod - def from_furl(cls, furl: DecodedURL, treq=treq) -> "StorageClient": + def from_furl(cls, furl: DecodedURL, persistent: bool = True) -> "StorageClient": """ Create a ``StorageClient`` for the given furl. + + ``persistent`` indicates whether to use persistent HTTP connections. """ assert furl.fragment == "v=1" assert furl.scheme == "pb" swissnum = furl.path[0].encode("ascii") certificate_hash = furl.user.encode("ascii") + treq_client = HTTPClient( + Agent( + reactor, + _StorageClientHTTPSPolicy(expected_spki_hash=certificate_hash), + pool=HTTPConnectionPool(reactor, persistent=persistent), + ) + ) + https_url = DecodedURL().replace(scheme="https", host=furl.host, port=furl.port) - return cls(https_url, swissnum, treq) + return cls(https_url, swissnum, treq_client) def relative_url(self, path): """Get a URL relative to the base URL.""" diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index 7b3810f21..272e63764 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -29,9 +29,6 @@ from pathlib import Path from twisted.internet.defer import inlineCallbacks, returnValue, succeed from twisted.internet.task import Clock from twisted.internet import reactor -from twisted.web.client import Agent, HTTPConnectionPool -from hyperlink import DecodedURL -from treq.client import HTTPClient from foolscap.api import Referenceable, RemoteException @@ -1089,15 +1086,12 @@ class _HTTPMixin(_SharedMixin): # Create HTTP client with non-persistent connections, so we don't leak # state across tests: - treq_client = HTTPClient( - Agent(reactor, pool=HTTPConnectionPool(reactor, persistent=False)) - ) - returnValue( _HTTPStorageServer.from_http_client( - StorageClient.from_furl(furl, treq_client) + StorageClient.from_furl(furl, persistent=False) ) ) + # Eventually should also: # self.assertTrue(IStorageServer.providedBy(client)) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 2d17658ce..0630eeaa0 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -67,7 +67,7 @@ class HTTPFurlTests(SyncTestCase): openssl asn1parse -noout -inform pem -out public.key openssl dgst -sha256 -binary public.key | openssl enc -base64 """ - expected_hash = b"JIj6ezHkdSBlHhrnezAgIC/mrVQHy4KAFyL+8ZNPGPM=" + expected_hash = b"JIj6ezHkdSBlHhrnezAgIC/mrVQHy4KAFyL+8ZNPGPM" certificate_text = b"""\ -----BEGIN CERTIFICATE----- MIIDWTCCAkECFCf+I+3oEhTfqt+6ruH4qQ4Wst1DMA0GCSqGSIb3DQEBCwUAMGkx From e50d88f46d07b36926a053265767cb668e1ee8b2 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 25 Mar 2022 10:45:54 -0400 Subject: [PATCH 0783/2309] Technically this doesn't matter, because it's client-side, but it's good habit. --- src/allmydata/storage/http_client.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index fc27e1e30..37bf29901 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -35,6 +35,7 @@ from .http_common import ( get_spki_hash, ) from .common import si_b2a +from ..util.hashutil import timing_safe_compare def _encode_si(si): # type: (bytes) -> str @@ -103,9 +104,8 @@ class _TLSContextFactory(CertificateOptions): 19, # X509_V_ERR_SELF_SIGNED_CERT_IN_CHAIN ) # TODO can we do this once instead of multiple times? - if ( - errno in things_are_ok - and get_spki_hash(cert.to_cryptography()) == expected_spki_hash + if errno in things_are_ok and timing_safe_compare( + get_spki_hash(cert.to_cryptography()), expected_spki_hash ): return 1 # TODO: log the details of the error, because otherwise they get From 9240d9d657dd4c25d080fb4a4711ba5810f33c2a Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 25 Mar 2022 10:46:14 -0400 Subject: [PATCH 0784/2309] Expand the comment. --- src/allmydata/storage/http_server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 80b34daa9..8cc7972b8 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -548,7 +548,7 @@ def listen_tls( def build_furl(listening_port: IListeningPort) -> DecodedURL: furl = DecodedURL().replace( - fragment="v=1", # HTTP-based + fragment="v=1", # how we know this furl is HTTP-based host=hostname, port=listening_port.getHost().port, path=(str(server._swissnum, "ascii"),), From 6f86675766741f5fe9f8b9762a4434a83421f88d Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 25 Mar 2022 10:59:16 -0400 Subject: [PATCH 0785/2309] Split into its own file. --- src/allmydata/test/test_storage_http.py | 43 +------------------- src/allmydata/test/test_storage_https.py | 52 ++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 42 deletions(-) create mode 100644 src/allmydata/test/test_storage_https.py diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 0630eeaa0..eb0c61f9a 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -27,12 +27,11 @@ from collections_extended import RangeMap from twisted.internet.task import Clock from twisted.web import http from twisted.web.http_headers import Headers -from cryptography.x509 import load_pem_x509_certificate from werkzeug import routing from werkzeug.exceptions import NotFound as WNotFound from .common import SyncTestCase -from ..storage.http_common import get_content_type, get_spki_hash +from ..storage.http_common import get_content_type from ..storage.common import si_b2a from ..storage.server import StorageServer from ..storage.http_server import ( @@ -54,46 +53,6 @@ from ..storage.http_client import ( ) -class HTTPFurlTests(SyncTestCase): - """Tests for HTTP furls.""" - - def test_spki_hash(self): - """The output of ``get_spki_hash()`` matches the semantics of RFC 7469. - - The expected hash was generated using Appendix A instructions in the - RFC:: - - openssl x509 -noout -in certificate.pem -pubkey | \ - openssl asn1parse -noout -inform pem -out public.key - openssl dgst -sha256 -binary public.key | openssl enc -base64 - """ - expected_hash = b"JIj6ezHkdSBlHhrnezAgIC/mrVQHy4KAFyL+8ZNPGPM" - certificate_text = b"""\ ------BEGIN CERTIFICATE----- -MIIDWTCCAkECFCf+I+3oEhTfqt+6ruH4qQ4Wst1DMA0GCSqGSIb3DQEBCwUAMGkx -CzAJBgNVBAYTAlpaMRAwDgYDVQQIDAdOb3doZXJlMRQwEgYDVQQHDAtFeGFtcGxl -dG93bjEcMBoGA1UECgwTRGVmYXVsdCBDb21wYW55IEx0ZDEUMBIGA1UEAwwLZXhh -bXBsZS5jb20wHhcNMjIwMzAyMTUyNTQ3WhcNMjMwMzAyMTUyNTQ3WjBpMQswCQYD -VQQGEwJaWjEQMA4GA1UECAwHTm93aGVyZTEUMBIGA1UEBwwLRXhhbXBsZXRvd24x -HDAaBgNVBAoME0RlZmF1bHQgQ29tcGFueSBMdGQxFDASBgNVBAMMC2V4YW1wbGUu -Y29tMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAv9vqtA8Toy9D6xLG -q41iUafSiAXnuirWxML2ct/LAcGJzATg6JctmJxxZQL7vkmaFFPBF6Y39bOGbbEC -M2iQYn2Qemj5fl3IzKTnYLqzryGM0ZwwnNbPyetSe/sksAIYRLzn49d6l+AHR+Dj -GyvoLzIyGUTn41MTDafMNtPgWx1i+65lFW3GHYpEmugu4bjeUPizNja2LrqwvwFu -YXwmKxbIMdioCoRvDGX9SI3/euFstuR4rbOEUDxniYRF5g6reP8UMF30zJzF5j0k -yDg8Z5b1XpKFNZAeyRYxcs9wJCqVlP6BLPDnvNVpMXodnWLeTK+r6YWvGadGVufk -YNC1PwIDAQABMA0GCSqGSIb3DQEBCwUAA4IBAQByrhn78GSS3dJ0pJ6czmhMX5wH -+fauCtt1+Wbn+ctTodTycS+pfULO4gG7wRzhl8KNoOqLmWMjyA2A3mon8kdkD+0C -i8McpoPaGS2wQcqC28Ud6kP9YO81YFyTl4nHVKQ0nmplT+eoLDTCIWMVxHHzxIgs -2ybUluAc+THSjpGxB6kWSAJeg3N+f2OKr+07Yg9LiQ2b8y0eZarpiuuuXCzWeWrQ -PudP0aniyq/gbPhxq0tYF628IBvhDAnr/2kqEmVF2TDr2Sm/Y3PDBuPY6MeIxjnr -ox5zO3LrQmQw11OaIAs2/kviKAoKTFFxeyYcpS5RuKNDZfHQCXlLwt9bySxG ------END CERTIFICATE----- -""" - certificate = load_pem_x509_certificate(certificate_text) - self.assertEqual(get_spki_hash(certificate), expected_hash) - - class HTTPUtilities(SyncTestCase): """Tests for HTTP common utilities.""" diff --git a/src/allmydata/test/test_storage_https.py b/src/allmydata/test/test_storage_https.py new file mode 100644 index 000000000..ebb605d04 --- /dev/null +++ b/src/allmydata/test/test_storage_https.py @@ -0,0 +1,52 @@ +""" +Tests for the TLS part of the HTTP Storage Protocol. + +More broadly, these are tests for HTTPS usage as replacement for Foolscap's +server authentication logic, which may one day apply outside of HTTP Storage +Protocol. +""" + +from cryptography.x509 import load_pem_x509_certificate + +from .common import SyncTestCase +from ..storage.http_common import get_spki_hash + + +class HTTPFurlTests(SyncTestCase): + """Tests for HTTP furls.""" + + def test_spki_hash(self): + """The output of ``get_spki_hash()`` matches the semantics of RFC 7469. + + The expected hash was generated using Appendix A instructions in the + RFC:: + + openssl x509 -noout -in certificate.pem -pubkey | \ + openssl asn1parse -noout -inform pem -out public.key + openssl dgst -sha256 -binary public.key | openssl enc -base64 + """ + expected_hash = b"JIj6ezHkdSBlHhrnezAgIC/mrVQHy4KAFyL+8ZNPGPM" + certificate_text = b"""\ +-----BEGIN CERTIFICATE----- +MIIDWTCCAkECFCf+I+3oEhTfqt+6ruH4qQ4Wst1DMA0GCSqGSIb3DQEBCwUAMGkx +CzAJBgNVBAYTAlpaMRAwDgYDVQQIDAdOb3doZXJlMRQwEgYDVQQHDAtFeGFtcGxl +dG93bjEcMBoGA1UECgwTRGVmYXVsdCBDb21wYW55IEx0ZDEUMBIGA1UEAwwLZXhh +bXBsZS5jb20wHhcNMjIwMzAyMTUyNTQ3WhcNMjMwMzAyMTUyNTQ3WjBpMQswCQYD +VQQGEwJaWjEQMA4GA1UECAwHTm93aGVyZTEUMBIGA1UEBwwLRXhhbXBsZXRvd24x +HDAaBgNVBAoME0RlZmF1bHQgQ29tcGFueSBMdGQxFDASBgNVBAMMC2V4YW1wbGUu +Y29tMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAv9vqtA8Toy9D6xLG +q41iUafSiAXnuirWxML2ct/LAcGJzATg6JctmJxxZQL7vkmaFFPBF6Y39bOGbbEC +M2iQYn2Qemj5fl3IzKTnYLqzryGM0ZwwnNbPyetSe/sksAIYRLzn49d6l+AHR+Dj +GyvoLzIyGUTn41MTDafMNtPgWx1i+65lFW3GHYpEmugu4bjeUPizNja2LrqwvwFu +YXwmKxbIMdioCoRvDGX9SI3/euFstuR4rbOEUDxniYRF5g6reP8UMF30zJzF5j0k +yDg8Z5b1XpKFNZAeyRYxcs9wJCqVlP6BLPDnvNVpMXodnWLeTK+r6YWvGadGVufk +YNC1PwIDAQABMA0GCSqGSIb3DQEBCwUAA4IBAQByrhn78GSS3dJ0pJ6czmhMX5wH ++fauCtt1+Wbn+ctTodTycS+pfULO4gG7wRzhl8KNoOqLmWMjyA2A3mon8kdkD+0C +i8McpoPaGS2wQcqC28Ud6kP9YO81YFyTl4nHVKQ0nmplT+eoLDTCIWMVxHHzxIgs +2ybUluAc+THSjpGxB6kWSAJeg3N+f2OKr+07Yg9LiQ2b8y0eZarpiuuuXCzWeWrQ +PudP0aniyq/gbPhxq0tYF628IBvhDAnr/2kqEmVF2TDr2Sm/Y3PDBuPY6MeIxjnr +ox5zO3LrQmQw11OaIAs2/kviKAoKTFFxeyYcpS5RuKNDZfHQCXlLwt9bySxG +-----END CERTIFICATE----- +""" + certificate = load_pem_x509_certificate(certificate_text) + self.assertEqual(get_spki_hash(certificate), expected_hash) From 712f4f138546f315640d1e28c4955e01d28f2a2c Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 25 Mar 2022 13:49:11 -0400 Subject: [PATCH 0786/2309] Sketch of first HTTPS client logic test. --- src/allmydata/test/test_storage_https.py | 160 ++++++++++++++++++++++- 1 file changed, 155 insertions(+), 5 deletions(-) diff --git a/src/allmydata/test/test_storage_https.py b/src/allmydata/test/test_storage_https.py index ebb605d04..7f6b4c039 100644 --- a/src/allmydata/test/test_storage_https.py +++ b/src/allmydata/test/test_storage_https.py @@ -6,14 +6,30 @@ server authentication logic, which may one day apply outside of HTTP Storage Protocol. """ -from cryptography.x509 import load_pem_x509_certificate +import datetime +from functools import wraps +from contextlib import asynccontextmanager -from .common import SyncTestCase +from cryptography import x509 +from cryptography.x509.oid import NameOID +from cryptography.hazmat.primitives.asymmetric import rsa +from cryptography.hazmat.primitives import serialization, hashes + +from twisted.internet.endpoints import quoteStringArgument, serverFromString +from twisted.internet import reactor +from twisted.internet.defer import Deferred +from twisted.web.server import Site +from twisted.web.static import Data +from twisted.web.client import Agent, HTTPConnectionPool +from treq.client import HTTPClient + +from .common import SyncTestCase, AsyncTestCase from ..storage.http_common import get_spki_hash +from ..storage.http_client import _StorageClientHTTPSPolicy -class HTTPFurlTests(SyncTestCase): - """Tests for HTTP furls.""" +class HTTPSFurlTests(SyncTestCase): + """Tests for HTTPS furls.""" def test_spki_hash(self): """The output of ``get_spki_hash()`` matches the semantics of RFC 7469. @@ -48,5 +64,139 @@ PudP0aniyq/gbPhxq0tYF628IBvhDAnr/2kqEmVF2TDr2Sm/Y3PDBuPY6MeIxjnr ox5zO3LrQmQw11OaIAs2/kviKAoKTFFxeyYcpS5RuKNDZfHQCXlLwt9bySxG -----END CERTIFICATE----- """ - certificate = load_pem_x509_certificate(certificate_text) + certificate = x509.load_pem_x509_certificate(certificate_text) self.assertEqual(get_spki_hash(certificate), expected_hash) + + +def async_to_deferred(f): + """ + Wrap an async function to return a Deferred instead. + """ + + @wraps(f) + def not_async(*args, **kwargs): + return Deferred.fromCoroutine(f(*args, **kwargs)) + + return not_async + + +class PinningHTTPSValidation(AsyncTestCase): + """ + Test client-side validation logic of HTTPS certificates that uses + Tahoe-LAFS's pinning-based scheme instead of the traditional certificate + authority scheme. + + https://cryptography.io/en/latest/x509/tutorial/#creating-a-self-signed-certificate + """ + + # NEEDED TESTS + # + # Success case + # + # Failure cases: + # Self-signed cert has wrong hash. Cert+private key match each other. + # Self-signed cert has correct hash, but doesn't match private key, so is invalid cert. + # Self-signed cert has correct hash, is valid, but expired. + # Anonymous server, without certificate. + # Cert has correct hash, but is not self-signed. + # Certificate that isn't valid yet (i.e. from the future)? Or is that silly. + + def to_file(self, key_or_cert) -> str: + """ + Write the given key or cert to a temporary file on disk, return the + path. + """ + path = self.mktemp() + with open(path, "wb") as f: + if isinstance(key_or_cert, x509.Certificate): + data = key_or_cert.public_bytes(serialization.Encoding.PEM) + else: + data = key_or_cert.private_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PrivateFormat.TraditionalOpenSSL, + encryption_algorithm=serialization.NoEncryption(), + ) + f.write(data) + return path + + def generate_private_key(self): + """Create a RSA private key.""" + return rsa.generate_private_key(public_exponent=65537, key_size=2048) + + def generate_certificate(self, private_key, expires_days: int): + """Generate a certificate from a RSA private key.""" + subject = issuer = x509.Name( + [x509.NameAttribute(NameOID.ORGANIZATION_NAME, "Yoyodyne")] + ) + return ( + x509.CertificateBuilder() + .subject_name(subject) + .issuer_name(issuer) + .public_key(private_key.public_key()) + .serial_number(x509.random_serial_number()) + .not_valid_before(datetime.datetime.utcnow()) + .not_valid_after( + datetime.datetime.utcnow() + datetime.timedelta(days=expires_days) + ) + .add_extension( + x509.SubjectAlternativeName([x509.DNSName("localhost")]), + critical=False, + # Sign our certificate with our private key + ) + .sign(private_key, hashes.SHA256()) + ) + + @asynccontextmanager + async def listen(self, private_key_path, cert_path) -> str: + """ + Context manager that runs a HTTPS server with the given private key + and certificate. + + Returns a URL that will connect to the server. + """ + endpoint = serverFromString( + reactor, + "ssl:privateKey={}:certKey={}:port=0:interface=127.0.0.1".format( + quoteStringArgument(str(private_key_path)), + quoteStringArgument(str(cert_path)), + ), + ) + root = Data(b"YOYODYNE", "text/plain") + root.isLeaf = True + listening_port = await endpoint.listen(Site(root)) + try: + yield f"https://127.0.0.1:{listening_port.getHost().port}/" + finally: + await listening_port.stopListening() + + def request(self, url: str, expected_certificate: x509.Certificate): + """ + Send a HTTPS request to the given URL, ensuring that the given + certificate is the one used via SPKI-hash-based pinning comparison. + """ + # No persistent connections, so we don't have dirty reactor at the end + # of the test. + treq_client = HTTPClient( + Agent( + reactor, + _StorageClientHTTPSPolicy( + expected_spki_hash=get_spki_hash(expected_certificate) + ), + pool=HTTPConnectionPool(reactor, persistent=False), + ) + ) + return treq_client.get(url) + + @async_to_deferred + async def test_success(self): + """ + If all conditions are met, a TLS client using the Tahoe-LAFS policy can + connect to the server. + """ + private_key = self.generate_private_key() + certificate = self.generate_certificate(private_key, 10) + async with self.listen( + self.to_file(private_key), self.to_file(certificate) + ) as url: + response = await self.request(url, certificate) + self.assertEqual(await response.content(), b"YOYODYNE") From bd6e537891c34e9e7b0041238a11177f0b911a25 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 25 Mar 2022 13:52:25 -0400 Subject: [PATCH 0787/2309] Hacky fix. --- src/allmydata/test/test_storage_https.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/allmydata/test/test_storage_https.py b/src/allmydata/test/test_storage_https.py index 7f6b4c039..d4057760e 100644 --- a/src/allmydata/test/test_storage_https.py +++ b/src/allmydata/test/test_storage_https.py @@ -18,6 +18,7 @@ from cryptography.hazmat.primitives import serialization, hashes from twisted.internet.endpoints import quoteStringArgument, serverFromString from twisted.internet import reactor from twisted.internet.defer import Deferred +from twisted.internet.task import deferLater from twisted.web.server import Site from twisted.web.static import Data from twisted.web.client import Agent, HTTPConnectionPool @@ -168,6 +169,9 @@ class PinningHTTPSValidation(AsyncTestCase): yield f"https://127.0.0.1:{listening_port.getHost().port}/" finally: await listening_port.stopListening() + # Make sure all server connections are closed :( No idea why this + # is necessary when it's not for IStorageServer HTTPS tests. + await deferLater(reactor, 0.001) def request(self, url: str, expected_certificate: x509.Certificate): """ From 23ce58140573c9bb8c032bac864a76fa417a5274 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 25 Mar 2022 14:01:06 -0400 Subject: [PATCH 0788/2309] More tests; some still failing. --- src/allmydata/test/test_storage_https.py | 54 ++++++++++++++++++------ 1 file changed, 42 insertions(+), 12 deletions(-) diff --git a/src/allmydata/test/test_storage_https.py b/src/allmydata/test/test_storage_https.py index d4057760e..35077d66e 100644 --- a/src/allmydata/test/test_storage_https.py +++ b/src/allmydata/test/test_storage_https.py @@ -21,7 +21,7 @@ from twisted.internet.defer import Deferred from twisted.internet.task import deferLater from twisted.web.server import Site from twisted.web.static import Data -from twisted.web.client import Agent, HTTPConnectionPool +from twisted.web.client import Agent, HTTPConnectionPool, ResponseNeverReceived from treq.client import HTTPClient from .common import SyncTestCase, AsyncTestCase @@ -92,11 +92,8 @@ class PinningHTTPSValidation(AsyncTestCase): # NEEDED TESTS # - # Success case - # # Failure cases: - # Self-signed cert has wrong hash. Cert+private key match each other. - # Self-signed cert has correct hash, but doesn't match private key, so is invalid cert. + # DONE Self-signed cert has wrong hash. Cert+private key match each other. # Self-signed cert has correct hash, is valid, but expired. # Anonymous server, without certificate. # Cert has correct hash, but is not self-signed. @@ -124,21 +121,22 @@ class PinningHTTPSValidation(AsyncTestCase): """Create a RSA private key.""" return rsa.generate_private_key(public_exponent=65537, key_size=2048) - def generate_certificate(self, private_key, expires_days: int): + def generate_certificate( + self, private_key, expires_days: int = 10, org_name: str = "Yoyodyne" + ): """Generate a certificate from a RSA private key.""" subject = issuer = x509.Name( - [x509.NameAttribute(NameOID.ORGANIZATION_NAME, "Yoyodyne")] + [x509.NameAttribute(NameOID.ORGANIZATION_NAME, org_name)] ) + expires = datetime.datetime.utcnow() + datetime.timedelta(days=expires_days) return ( x509.CertificateBuilder() .subject_name(subject) .issuer_name(issuer) .public_key(private_key.public_key()) .serial_number(x509.random_serial_number()) - .not_valid_before(datetime.datetime.utcnow()) - .not_valid_after( - datetime.datetime.utcnow() + datetime.timedelta(days=expires_days) - ) + .not_valid_before(min(datetime.datetime.utcnow(), expires)) + .not_valid_after(expires) .add_extension( x509.SubjectAlternativeName([x509.DNSName("localhost")]), critical=False, @@ -198,9 +196,41 @@ class PinningHTTPSValidation(AsyncTestCase): connect to the server. """ private_key = self.generate_private_key() - certificate = self.generate_certificate(private_key, 10) + certificate = self.generate_certificate(private_key) async with self.listen( self.to_file(private_key), self.to_file(certificate) ) as url: response = await self.request(url, certificate) self.assertEqual(await response.content(), b"YOYODYNE") + + @async_to_deferred + async def test_server_certificate_has_wrong_hash(self): + """ + If the server's certificate hash doesn't match the hash the client + expects, the request to the server fails. + """ + private_key1 = self.generate_private_key() + certificate1 = self.generate_certificate(private_key1) + private_key2 = self.generate_private_key() + certificate2 = self.generate_certificate(private_key2) + + async with self.listen( + self.to_file(private_key1), self.to_file(certificate1) + ) as url: + with self.assertRaises(ResponseNeverReceived): + await self.request(url, certificate2) + + @async_to_deferred + async def test_server_certificate_expired(self): + """ + If the server's certificate has expired, the request to the server + fails even if the hash matches the one the client expects. + """ + private_key = self.generate_private_key() + certificate = self.generate_certificate(private_key, expires_days=-10) + + async with self.listen( + self.to_file(private_key), self.to_file(certificate) + ) as url: + with self.assertRaises(ResponseNeverReceived): + await self.request(url, certificate) From 638154b2ad2ac5b6eafc57f58303c7c21ac0d21b Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 25 Mar 2022 15:46:42 -0400 Subject: [PATCH 0789/2309] Cleanups. --- src/allmydata/storage/http_client.py | 22 +++++++++------------- src/allmydata/test/test_storage_https.py | 20 ++++++++------------ 2 files changed, 17 insertions(+), 25 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 37bf29901..4c8577e88 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -81,7 +81,11 @@ class _TLSContextFactory(CertificateOptions): Originally implemented as part of Foolscap. """ - def getContext(self, expected_spki_hash: bytes) -> SSL.Context: + def __init__(self, expected_spki_hash: bytes): + self.expected_spki_hash = expected_spki_hash + CertificateOptions.__init__(self) + + def getContext(self) -> SSL.Context: def always_validate(conn, cert, errno, depth, preverify_ok): # This function is called to validate the certificate received by # the other end. OpenSSL calls it multiple times, each time it @@ -105,15 +109,12 @@ class _TLSContextFactory(CertificateOptions): ) # TODO can we do this once instead of multiple times? if errno in things_are_ok and timing_safe_compare( - get_spki_hash(cert.to_cryptography()), expected_spki_hash + get_spki_hash(cert.to_cryptography()), self.expected_spki_hash ): return 1 # TODO: log the details of the error, because otherwise they get # lost in the PyOpenSSL exception that will eventually be raised # (possibly OpenSSL.SSL.Error: certificate verify failed) - - # I think that X509_V_ERR_CERT_SIGNATURE_FAILURE is the most - # obvious sign of hostile attack. return 0 ctx = CertificateOptions.getContext(self) @@ -128,13 +129,8 @@ class _TLSContextFactory(CertificateOptions): @attr.s class _StorageClientHTTPSPolicy: """ - A HTTPS policy that: - - 1. Makes sure the SPKI hash of the certificate matches a known hash (NEEDS TEST). - 2. The certificate hasn't expired. (NEEDS TEST) - 3. The server has a private key that matches the certificate (NEEDS TEST). - - I.e. pinning-based validation. + A HTTPS policy that ensures the SPKI hash of the public key matches a known + hash, i.e. pinning-based validation. """ expected_spki_hash = attr.ib(type=bytes) @@ -146,7 +142,7 @@ class _StorageClientHTTPSPolicy: # IOpenSSLClientConnectionCreator def clientConnectionForTLS(self, tlsProtocol): connection = SSL.Connection( - _TLSContextFactory().getContext(self.expected_spki_hash), None + _TLSContextFactory(self.expected_spki_hash).getContext(), None ) connection.set_app_data(tlsProtocol) return connection diff --git a/src/allmydata/test/test_storage_https.py b/src/allmydata/test/test_storage_https.py index 35077d66e..801fabc6b 100644 --- a/src/allmydata/test/test_storage_https.py +++ b/src/allmydata/test/test_storage_https.py @@ -90,15 +90,6 @@ class PinningHTTPSValidation(AsyncTestCase): https://cryptography.io/en/latest/x509/tutorial/#creating-a-self-signed-certificate """ - # NEEDED TESTS - # - # Failure cases: - # DONE Self-signed cert has wrong hash. Cert+private key match each other. - # Self-signed cert has correct hash, is valid, but expired. - # Anonymous server, without certificate. - # Cert has correct hash, but is not self-signed. - # Certificate that isn't valid yet (i.e. from the future)? Or is that silly. - def to_file(self, key_or_cert) -> str: """ Write the given key or cert to a temporary file on disk, return the @@ -224,7 +215,8 @@ class PinningHTTPSValidation(AsyncTestCase): async def test_server_certificate_expired(self): """ If the server's certificate has expired, the request to the server - fails even if the hash matches the one the client expects. + succeeds if the hash matches the one the client expects; expiration has + no effect. """ private_key = self.generate_private_key() certificate = self.generate_certificate(private_key, expires_days=-10) @@ -232,5 +224,9 @@ class PinningHTTPSValidation(AsyncTestCase): async with self.listen( self.to_file(private_key), self.to_file(certificate) ) as url: - with self.assertRaises(ResponseNeverReceived): - await self.request(url, certificate) + response = await self.request(url, certificate) + self.assertEqual(await response.content(), b"YOYODYNE") + + # TODO an obvious attack is a private key that doesn't match the + # certificate... but OpenSSL (quite rightly) won't let you listen with that + # so I don't know how to test that! From ae8a7eff438b60c8645aec357c5ba710f9da23c8 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 25 Mar 2022 15:52:31 -0400 Subject: [PATCH 0790/2309] Make mypy happy. --- src/allmydata/storage/http_server.py | 6 +++--- src/allmydata/test/test_storage_https.py | 2 +- tox.ini | 1 + 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 8cc7972b8..59728e1d3 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -552,12 +552,12 @@ def listen_tls( host=hostname, port=listening_port.getHost().port, path=(str(server._swissnum, "ascii"),), - userinfo=[ + userinfo=( str( get_spki_hash(load_pem_x509_certificate(cert_path.read_bytes())), "ascii", - ) - ], + ), + ), scheme="pb", ) return furl diff --git a/src/allmydata/test/test_storage_https.py b/src/allmydata/test/test_storage_https.py index 801fabc6b..19f469990 100644 --- a/src/allmydata/test/test_storage_https.py +++ b/src/allmydata/test/test_storage_https.py @@ -137,7 +137,7 @@ class PinningHTTPSValidation(AsyncTestCase): ) @asynccontextmanager - async def listen(self, private_key_path, cert_path) -> str: + async def listen(self, private_key_path, cert_path): """ Context manager that runs a HTTPS server with the given private key and certificate. diff --git a/tox.ini b/tox.ini index 57489df89..859cf18e0 100644 --- a/tox.ini +++ b/tox.ini @@ -141,6 +141,7 @@ deps = types-six types-PyYAML types-pkg_resources + types-pyOpenSSL git+https://github.com/warner/foolscap # Twisted 21.2.0 introduces some type hints which we are not yet # compatible with. From 4e58748c4a6fb4349ea60a4bd95033ac6f63549b Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 28 Mar 2022 11:27:32 -0400 Subject: [PATCH 0791/2309] Get constants from OpenSSL directly. --- src/allmydata/storage/http_client.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 4c8577e88..a4aaad1bc 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -26,6 +26,7 @@ import treq from treq.client import HTTPClient from treq.testing import StubTreq from OpenSSL import SSL +from cryptography.hazmat.bindings.openssl.binding import Binding from .http_common import ( swissnum_auth_header, @@ -37,6 +38,8 @@ from .http_common import ( from .common import si_b2a from ..util.hashutil import timing_safe_compare +_OPENSSL = Binding().lib + def _encode_si(si): # type: (bytes) -> str """Encode the storage index into Unicode string.""" @@ -88,8 +91,8 @@ class _TLSContextFactory(CertificateOptions): def getContext(self) -> SSL.Context: def always_validate(conn, cert, errno, depth, preverify_ok): # This function is called to validate the certificate received by - # the other end. OpenSSL calls it multiple times, each time it - # see something funny, to ask if it should proceed. + # the other end. OpenSSL calls it multiple times, for each errno + # for each certificate. # We do not care about certificate authorities or revocation # lists, we just want to know that the certificate has a valid @@ -97,15 +100,12 @@ class _TLSContextFactory(CertificateOptions): # self-signed. We need to protect against forged signatures, but # not the usual TLS concerns about invalid CAs or revoked # certificates. - - # these constants are from openssl-0.9.7g/crypto/x509/x509_vfy.h - # and do not appear to be exposed by pyopenssl. Ick. things_are_ok = ( - 0, # X509_V_OK - 9, # X509_V_ERR_CERT_NOT_YET_VALID - 10, # X509_V_ERR_CERT_HAS_EXPIRED - 18, # X509_V_ERR_DEPTH_ZERO_SELF_SIGNED_CERT - 19, # X509_V_ERR_SELF_SIGNED_CERT_IN_CHAIN + _OPENSSL.X509_V_OK, + _OPENSSL.X509_V_ERR_CERT_NOT_YET_VALID, + _OPENSSL.X509_V_ERR_CERT_HAS_EXPIRED, + _OPENSSL.X509_V_ERR_DEPTH_ZERO_SELF_SIGNED_CERT, + _OPENSSL.X509_V_ERR_SELF_SIGNED_CERT_IN_CHAIN, ) # TODO can we do this once instead of multiple times? if errno in things_are_ok and timing_safe_compare( From 119ba9468e57a22cac191743c14c7f9ff50c3790 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 28 Mar 2022 11:28:38 -0400 Subject: [PATCH 0792/2309] Not needed. --- src/allmydata/storage/http_client.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index a4aaad1bc..c0371ffa5 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -141,11 +141,9 @@ class _StorageClientHTTPSPolicy: # IOpenSSLClientConnectionCreator def clientConnectionForTLS(self, tlsProtocol): - connection = SSL.Connection( + return SSL.Connection( _TLSContextFactory(self.expected_spki_hash).getContext(), None ) - connection.set_app_data(tlsProtocol) - return connection class StorageClient(object): From da6838d6f9f65bcb1e7119ba980b773d210a358f Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 28 Mar 2022 11:35:45 -0400 Subject: [PATCH 0793/2309] Stop talking about furl, it's a NURL. --- docs/proposed/http-storage-node-protocol.rst | 20 ++++++++++---------- src/allmydata/storage/http_client.py | 18 +++++++++--------- src/allmydata/storage/http_server.py | 12 ++++++------ src/allmydata/test/test_istorageserver.py | 4 ++-- src/allmydata/test/test_storage_https.py | 4 ++-- 5 files changed, 29 insertions(+), 29 deletions(-) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index 2ceb3c03a..a6f0e2c36 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -35,10 +35,10 @@ Glossary (the storage service is an example of such an object) NURL - a self-authenticating URL-like string almost exactly like a fURL but without being tied to Foolscap + a self-authenticating URL-like string almost exactly like a NURL but without being tied to Foolscap swissnum - a short random string which is part of a fURL and which acts as a shared secret to authorize clients to use a storage service + a short random string which is part of a fURL/NURL and which acts as a shared secret to authorize clients to use a storage service lease state associated with a share informing a storage server of the duration of storage desired by a client @@ -211,15 +211,15 @@ To further clarify, consider this example. Alice operates a storage node. Alice generates a key pair and secures it properly. Alice generates a self-signed storage node certificate with the key pair. -Alice's storage node announces (to an introducer) a fURL containing (among other information) the SPKI hash. +Alice's storage node announces (to an introducer) a NURL containing (among other information) the SPKI hash. Imagine the SPKI hash is ``i5xb...``. -This results in a fURL of ``pb://i5xb...@example.com:443/g3m5...#v=1``. +This results in a NURL of ``pb://i5xb...@example.com:443/g3m5...#v=1``. Bob creates a client node pointed at the same introducer. Bob's client node receives the announcement from Alice's storage node (indirected through the introducer). -Bob's client node recognizes the fURL as referring to an HTTP-dialect server due to the ``v=1`` fragment. -Bob's client node can now perform a TLS handshake with a server at the address in the fURL location hints +Bob's client node recognizes the NURL as referring to an HTTP-dialect server due to the ``v=1`` fragment. +Bob's client node can now perform a TLS handshake with a server at the address in the NURL location hints (``example.com:443`` in this example). Following the above described validation procedures, Bob's client node can determine whether it has reached Alice's storage node or not. @@ -230,7 +230,7 @@ Additionally, by continuing to interact using TLS, Bob's client and Alice's storage node are assured of both **message authentication** and **message confidentiality**. -Bob's client further inspects the fURL for the *swissnum*. +Bob's client further inspects the NURL for the *swissnum*. When Bob's client issues HTTP requests to Alice's storage node it includes the *swissnum* in its requests. **Storage authorization** has been achieved. @@ -266,8 +266,8 @@ Generation of a new certificate allows for certain non-optimal conditions to be * The ``commonName`` of ``newpb_thingy`` may be changed to a more descriptive value. * A ``notValidAfter`` field with a timestamp in the past may be updated. -Storage nodes will announce a new fURL for this new HTTP-based server. -This fURL will be announced alongside their existing Foolscap-based server's fURL. +Storage nodes will announce a new NURL for this new HTTP-based server. +This NURL will be announced alongside their existing Foolscap-based server's fURL. Such an announcement will resemble this:: { @@ -312,7 +312,7 @@ The follow sequence of events is likely: #. The client uses the information in its cache to open a Foolscap connection to the storage server. Ideally, -the client would not rely on an update from the introducer to give it the GBS fURL for the updated storage server. +the client would not rely on an update from the introducer to give it the GBS NURL for the updated storage server. Therefore, when an updated client connects to a storage server using Foolscap, it should request the server's version information. diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index c0371ffa5..06b9b1145 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -155,24 +155,24 @@ class StorageClient(object): self, url, swissnum, treq=treq ): # type: (DecodedURL, bytes, Union[treq,StubTreq,HTTPClient]) -> None """ - The URL is a HTTPS URL ("https://..."). To construct from a furl, use - ``StorageClient.from_furl()``. + The URL is a HTTPS URL ("https://..."). To construct from a NURL, use + ``StorageClient.from_nurl()``. """ self._base_url = url self._swissnum = swissnum self._treq = treq @classmethod - def from_furl(cls, furl: DecodedURL, persistent: bool = True) -> "StorageClient": + def from_nurl(cls, nurl: DecodedURL, persistent: bool = True) -> "StorageClient": """ - Create a ``StorageClient`` for the given furl. + Create a ``StorageClient`` for the given NURL. ``persistent`` indicates whether to use persistent HTTP connections. """ - assert furl.fragment == "v=1" - assert furl.scheme == "pb" - swissnum = furl.path[0].encode("ascii") - certificate_hash = furl.user.encode("ascii") + assert nurl.fragment == "v=1" + assert nurl.scheme == "pb" + swissnum = nurl.path[0].encode("ascii") + certificate_hash = nurl.user.encode("ascii") treq_client = HTTPClient( Agent( @@ -182,7 +182,7 @@ class StorageClient(object): ) ) - https_url = DecodedURL().replace(scheme="https", host=furl.host, port=furl.port) + https_url = DecodedURL().replace(scheme="https", host=nurl.host, port=nurl.port) return cls(https_url, swissnum, treq_client) def relative_url(self, path): diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 59728e1d3..0374797c6 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -528,7 +528,7 @@ def listen_tls( interface: Optional[str], ) -> Deferred[Tuple[DecodedURL, IListeningPort]]: """ - Start a HTTPS storage server on the given port, return the fURL and the + Start a HTTPS storage server on the given port, return the NURL and the listening port. The hostname is the external IP or hostname clients will connect to; it @@ -546,9 +546,9 @@ def listen_tls( endpoint_string += ":interface={}".format(quoteStringArgument(interface)) endpoint = serverFromString(reactor, endpoint_string) - def build_furl(listening_port: IListeningPort) -> DecodedURL: - furl = DecodedURL().replace( - fragment="v=1", # how we know this furl is HTTP-based + def build_nurl(listening_port: IListeningPort) -> DecodedURL: + nurl = DecodedURL().replace( + fragment="v=1", # how we know this NURL is HTTP-based (i.e. not Foolscap) host=hostname, port=listening_port.getHost().port, path=(str(server._swissnum, "ascii"),), @@ -560,8 +560,8 @@ def listen_tls( ), scheme="pb", ) - return furl + return nurl return endpoint.listen(Site(server.get_resource())).addCallback( - lambda listening_port: (build_furl(listening_port), listening_port) + lambda listening_port: (build_nurl(listening_port), listening_port) ) diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index 272e63764..7c5e64042 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -1073,7 +1073,7 @@ class _HTTPMixin(_SharedMixin): # Listen on randomly assigned port, using self-signed cert we generated # manually: certs_dir = Path(__file__).parent / "certs" - furl, listening_port = yield listen_tls( + nurl, listening_port = yield listen_tls( reactor, http_storage_server, "127.0.0.1", @@ -1088,7 +1088,7 @@ class _HTTPMixin(_SharedMixin): # state across tests: returnValue( _HTTPStorageServer.from_http_client( - StorageClient.from_furl(furl, persistent=False) + StorageClient.from_nurl(nurl, persistent=False) ) ) diff --git a/src/allmydata/test/test_storage_https.py b/src/allmydata/test/test_storage_https.py index 19f469990..f4242ae0c 100644 --- a/src/allmydata/test/test_storage_https.py +++ b/src/allmydata/test/test_storage_https.py @@ -29,8 +29,8 @@ from ..storage.http_common import get_spki_hash from ..storage.http_client import _StorageClientHTTPSPolicy -class HTTPSFurlTests(SyncTestCase): - """Tests for HTTPS furls.""" +class HTTPSNurlTests(SyncTestCase): + """Tests for HTTPS NURLs.""" def test_spki_hash(self): """The output of ``get_spki_hash()`` matches the semantics of RFC 7469. From 30a3b006a06c8f850c1008731323922147594200 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 30 Mar 2022 10:26:26 -0400 Subject: [PATCH 0794/2309] Include self-signed cert in package install. --- setup.py | 1 + src/allmydata/test/test_istorageserver.py | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/setup.py b/setup.py index 5285b5d08..1883032de 100644 --- a/setup.py +++ b/setup.py @@ -410,6 +410,7 @@ setup(name="tahoe-lafs", # also set in __init__.py "static/css/*.css", ], "allmydata": ["ported-modules.txt"], + "allmydata.test": ["certs/*"] }, include_package_data=True, setup_requires=setup_requires, diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index 7c5e64042..9a9a1d39a 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -1078,6 +1078,10 @@ class _HTTPMixin(_SharedMixin): http_storage_server, "127.0.0.1", 0, + # This is just a self-signed certificate with randomly generated + # private key; nothing at all special about it. You can regenerate + # with code in allmydata.test.test_storage_https or with openssl + # CLI, with no meaningful change to the test. certs_dir / "private.key", certs_dir / "domain.crt", interface="127.0.0.1", From 5972a13457f9b0104f77551681f81a3525992965 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 6 Apr 2022 09:34:17 -0400 Subject: [PATCH 0795/2309] Add reactor argument. --- src/allmydata/storage/http_client.py | 3 +-- src/allmydata/test/test_istorageserver.py | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 06b9b1145..766036427 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -18,7 +18,6 @@ from twisted.web.iweb import IPolicyForHTTPS from twisted.internet.defer import inlineCallbacks, returnValue, fail, Deferred from twisted.internet.interfaces import IOpenSSLClientConnectionCreator from twisted.internet.ssl import CertificateOptions -from twisted.internet import reactor from twisted.web.client import Agent, HTTPConnectionPool from zope.interface import implementer from hyperlink import DecodedURL @@ -163,7 +162,7 @@ class StorageClient(object): self._treq = treq @classmethod - def from_nurl(cls, nurl: DecodedURL, persistent: bool = True) -> "StorageClient": + def from_nurl(cls, nurl: DecodedURL, reactor, persistent: bool = True) -> "StorageClient": """ Create a ``StorageClient`` for the given NURL. diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index 9a9a1d39a..c5c515434 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -1092,7 +1092,7 @@ class _HTTPMixin(_SharedMixin): # state across tests: returnValue( _HTTPStorageServer.from_http_client( - StorageClient.from_nurl(nurl, persistent=False) + StorageClient.from_nurl(nurl, reactor, persistent=False) ) ) From 2e934574f0bf169db493a17b7abf7e3c554d4a2e Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 6 Apr 2022 09:37:18 -0400 Subject: [PATCH 0796/2309] Switch to URL-safe base64 for SPKI hash, for nicer usage in NURLs. --- src/allmydata/storage/http_common.py | 6 ++++-- src/allmydata/test/test_storage_https.py | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/allmydata/storage/http_common.py b/src/allmydata/storage/http_common.py index 11ab880c7..bd88f9fae 100644 --- a/src/allmydata/storage/http_common.py +++ b/src/allmydata/storage/http_common.py @@ -3,7 +3,7 @@ Common HTTP infrastructure for the storge server. """ from enum import Enum -from base64 import b64encode +from base64 import urlsafe_b64encode, b64encode from hashlib import sha256 from typing import Optional @@ -44,8 +44,10 @@ def get_spki_hash(certificate: Certificate) -> bytes: """ Get the public key hash, as per RFC 7469: base64 of sha256 of the public key encoded in DER + Subject Public Key Info format. + + We use the URL-safe base64 variant, since this is typically found in NURLs. """ public_key_bytes = certificate.public_key().public_bytes( Encoding.DER, PublicFormat.SubjectPublicKeyInfo ) - return b64encode(sha256(public_key_bytes).digest()).strip().rstrip(b"=") + return urlsafe_b64encode(sha256(public_key_bytes).digest()).strip().rstrip(b"=") diff --git a/src/allmydata/test/test_storage_https.py b/src/allmydata/test/test_storage_https.py index f4242ae0c..82e907f46 100644 --- a/src/allmydata/test/test_storage_https.py +++ b/src/allmydata/test/test_storage_https.py @@ -42,7 +42,7 @@ class HTTPSNurlTests(SyncTestCase): openssl asn1parse -noout -inform pem -out public.key openssl dgst -sha256 -binary public.key | openssl enc -base64 """ - expected_hash = b"JIj6ezHkdSBlHhrnezAgIC/mrVQHy4KAFyL+8ZNPGPM" + expected_hash = b"JIj6ezHkdSBlHhrnezAgIC_mrVQHy4KAFyL-8ZNPGPM" certificate_text = b"""\ -----BEGIN CERTIFICATE----- MIIDWTCCAkECFCf+I+3oEhTfqt+6ruH4qQ4Wst1DMA0GCSqGSIb3DQEBCwUAMGkx From 710fad4f8ac3d9cfa0f81fc391f88ac6532ec5e2 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 6 Apr 2022 11:10:42 -0400 Subject: [PATCH 0797/2309] Support broader range of server endpoints, and switch to more robust random port assignment. --- src/allmydata/storage/http_server.py | 51 +++++++++++++++-------- src/allmydata/test/test_istorageserver.py | 30 ++++--------- 2 files changed, 42 insertions(+), 39 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 0374797c6..a2cb58545 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -2,19 +2,21 @@ HTTP server for storage. """ -from typing import Dict, List, Set, Tuple, Any, Optional +from typing import Dict, List, Set, Tuple, Any from pathlib import Path from functools import wraps from base64 import b64decode import binascii +from zope.interface import implementer from klein import Klein from twisted.web import http -from twisted.internet.interfaces import IListeningPort +from twisted.internet.interfaces import IListeningPort, IStreamServerEndpoint from twisted.internet.defer import Deferred -from twisted.internet.endpoints import quoteStringArgument, serverFromString +from twisted.internet.ssl import CertificateOptions, Certificate, PrivateCertificate from twisted.web.server import Site +from twisted.protocols.tls import TLSMemoryBIOFactory import attr from werkzeug.http import ( parse_range_header, @@ -518,33 +520,48 @@ class HTTPServer(object): return b"" +@implementer(IStreamServerEndpoint) +@attr.s +class _TLSEndpointWrapper(object): + """ + Wrap an existing endpoint with the storage TLS policy. This is useful + because not all Tahoe-LAFS endpoints might be plain TCP+TLS, for example + there's Tor and i2p. + """ + + endpoint = attr.ib(type=IStreamServerEndpoint) + context_factory = attr.ib(type=CertificateOptions) + + def listen(self, factory): + return self.endpoint.listen( + TLSMemoryBIOFactory(self.context_factory, False, factory) + ) + + def listen_tls( - reactor, server: HTTPServer, hostname: str, - port: int, + endpoint: IStreamServerEndpoint, private_key_path: Path, cert_path: Path, - interface: Optional[str], ) -> Deferred[Tuple[DecodedURL, IListeningPort]]: """ Start a HTTPS storage server on the given port, return the NURL and the listening port. - The hostname is the external IP or hostname clients will connect to; it - does not modify what interfaces the server listens on. To set the - listening interface, use the ``interface`` argument. + The hostname is the external IP or hostname clients will connect to, used + to constrtuct the NURL; it does not modify what interfaces the server + listens on. - Port can be 0 to choose a random port. + This will likely need to be updated eventually to handle Tor/i2p. """ - endpoint_string = "ssl:privateKey={}:certKey={}:port={}".format( - quoteStringArgument(str(private_key_path)), - quoteStringArgument(str(cert_path)), - port, + certificate = Certificate.loadPEM(cert_path.read_bytes()).original + private_key = PrivateCertificate.loadPEM( + cert_path.read_bytes() + b"\n" + private_key_path.read_bytes() + ).privateKey.original + endpoint = _TLSEndpointWrapper( + endpoint, CertificateOptions(privateKey=private_key, certificate=certificate) ) - if interface is not None: - endpoint_string += ":interface={}".format(quoteStringArgument(interface)) - endpoint = serverFromString(reactor, endpoint_string) def build_nurl(listening_port: IListeningPort) -> DecodedURL: nurl = DecodedURL().replace( diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index c5c515434..495115231 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -8,19 +8,9 @@ reused across tests, so each test should be careful to generate unique storage indexes. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals +from future.utils import bchr -from future.utils import PY2, bchr - -if PY2: - # fmt: off - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - # fmt: on -else: - from typing import Set +from typing import Set from random import Random from unittest import SkipTest @@ -29,7 +19,7 @@ from pathlib import Path from twisted.internet.defer import inlineCallbacks, returnValue, succeed from twisted.internet.task import Clock from twisted.internet import reactor - +from twisted.internet.endpoints import serverFromString from foolscap.api import Referenceable, RemoteException from allmydata.interfaces import IStorageServer # really, IStorageClient @@ -1013,10 +1003,6 @@ class _SharedMixin(SystemTestMixin): AsyncTestCase.setUp(self) - self._port_assigner = SameProcessStreamEndpointAssigner() - self._port_assigner.setUp() - self.addCleanup(self._port_assigner.tearDown) - self.basedir = "test_istorageserver/" + self.id() yield SystemTestMixin.setUp(self) yield self.set_up_nodes(1) @@ -1061,8 +1047,9 @@ class _HTTPMixin(_SharedMixin): """Run tests on the HTTP version of ``IStorageServer``.""" def setUp(self): - if PY2: - self.skipTest("Not going to bother supporting Python 2") + self._port_assigner = SameProcessStreamEndpointAssigner() + self._port_assigner.setUp() + self.addCleanup(self._port_assigner.tearDown) return _SharedMixin.setUp(self) @inlineCallbacks @@ -1073,18 +1060,17 @@ class _HTTPMixin(_SharedMixin): # Listen on randomly assigned port, using self-signed cert we generated # manually: certs_dir = Path(__file__).parent / "certs" + _, endpoint_string = self._port_assigner.assign(reactor) nurl, listening_port = yield listen_tls( - reactor, http_storage_server, "127.0.0.1", - 0, + serverFromString(reactor, endpoint_string), # This is just a self-signed certificate with randomly generated # private key; nothing at all special about it. You can regenerate # with code in allmydata.test.test_storage_https or with openssl # CLI, with no meaningful change to the test. certs_dir / "private.key", certs_dir / "domain.crt", - interface="127.0.0.1", ) self.addCleanup(listening_port.stopListening) From eda5925548518c3ab30cf787e20504defef56750 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 6 Apr 2022 11:25:37 -0400 Subject: [PATCH 0798/2309] Get rid of another place where listen on port 0, and switch to FilePath only for now. --- src/allmydata/storage/http_server.py | 40 +++++++++++++++-------- src/allmydata/test/test_istorageserver.py | 8 ++--- src/allmydata/test/test_storage_https.py | 28 +++++++++------- 3 files changed, 47 insertions(+), 29 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index a2cb58545..d22a67995 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -3,7 +3,6 @@ HTTP server for storage. """ from typing import Dict, List, Set, Tuple, Any -from pathlib import Path from functools import wraps from base64 import b64decode @@ -17,6 +16,8 @@ from twisted.internet.defer import Deferred from twisted.internet.ssl import CertificateOptions, Certificate, PrivateCertificate from twisted.web.server import Site from twisted.protocols.tls import TLSMemoryBIOFactory +from twisted.python.filepath import FilePath + import attr from werkzeug.http import ( parse_range_header, @@ -524,14 +525,31 @@ class HTTPServer(object): @attr.s class _TLSEndpointWrapper(object): """ - Wrap an existing endpoint with the storage TLS policy. This is useful - because not all Tahoe-LAFS endpoints might be plain TCP+TLS, for example - there's Tor and i2p. + Wrap an existing endpoint with the server-side storage TLS policy. This is + useful because not all Tahoe-LAFS endpoints might be plain TCP+TLS, for + example there's Tor and i2p. """ endpoint = attr.ib(type=IStreamServerEndpoint) context_factory = attr.ib(type=CertificateOptions) + @classmethod + def from_paths( + cls, endpoint, private_key_path: FilePath, cert_path: FilePath + ) -> "_TLSEndpointWrapper": + """ + Create an endpoint with the given private key and certificate paths on + the filesystem. + """ + certificate = Certificate.loadPEM(cert_path.getContent()).original + private_key = PrivateCertificate.loadPEM( + cert_path.getContent() + b"\n" + private_key_path.getContent() + ).privateKey.original + certificate_options = CertificateOptions( + privateKey=private_key, certificate=certificate + ) + return cls(endpoint=endpoint, context_factory=certificate_options) + def listen(self, factory): return self.endpoint.listen( TLSMemoryBIOFactory(self.context_factory, False, factory) @@ -542,8 +560,8 @@ def listen_tls( server: HTTPServer, hostname: str, endpoint: IStreamServerEndpoint, - private_key_path: Path, - cert_path: Path, + private_key_path: FilePath, + cert_path: FilePath, ) -> Deferred[Tuple[DecodedURL, IListeningPort]]: """ Start a HTTPS storage server on the given port, return the NURL and the @@ -555,13 +573,7 @@ def listen_tls( This will likely need to be updated eventually to handle Tor/i2p. """ - certificate = Certificate.loadPEM(cert_path.read_bytes()).original - private_key = PrivateCertificate.loadPEM( - cert_path.read_bytes() + b"\n" + private_key_path.read_bytes() - ).privateKey.original - endpoint = _TLSEndpointWrapper( - endpoint, CertificateOptions(privateKey=private_key, certificate=certificate) - ) + endpoint = _TLSEndpointWrapper.from_paths(endpoint, private_key_path, cert_path) def build_nurl(listening_port: IListeningPort) -> DecodedURL: nurl = DecodedURL().replace( @@ -571,7 +583,7 @@ def listen_tls( path=(str(server._swissnum, "ascii"),), userinfo=( str( - get_spki_hash(load_pem_x509_certificate(cert_path.read_bytes())), + get_spki_hash(load_pem_x509_certificate(cert_path.getContent())), "ascii", ), ), diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index 495115231..bc7d5b853 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -14,12 +14,12 @@ from typing import Set from random import Random from unittest import SkipTest -from pathlib import Path from twisted.internet.defer import inlineCallbacks, returnValue, succeed from twisted.internet.task import Clock from twisted.internet import reactor from twisted.internet.endpoints import serverFromString +from twisted.python.filepath import FilePath from foolscap.api import Referenceable, RemoteException from allmydata.interfaces import IStorageServer # really, IStorageClient @@ -1059,7 +1059,7 @@ class _HTTPMixin(_SharedMixin): # Listen on randomly assigned port, using self-signed cert we generated # manually: - certs_dir = Path(__file__).parent / "certs" + certs_dir = FilePath(__file__).parent().child("certs") _, endpoint_string = self._port_assigner.assign(reactor) nurl, listening_port = yield listen_tls( http_storage_server, @@ -1069,8 +1069,8 @@ class _HTTPMixin(_SharedMixin): # private key; nothing at all special about it. You can regenerate # with code in allmydata.test.test_storage_https or with openssl # CLI, with no meaningful change to the test. - certs_dir / "private.key", - certs_dir / "domain.crt", + certs_dir.child("private.key"), + certs_dir.child("domain.crt"), ) self.addCleanup(listening_port.stopListening) diff --git a/src/allmydata/test/test_storage_https.py b/src/allmydata/test/test_storage_https.py index 82e907f46..0a5b73a96 100644 --- a/src/allmydata/test/test_storage_https.py +++ b/src/allmydata/test/test_storage_https.py @@ -15,18 +15,20 @@ from cryptography.x509.oid import NameOID from cryptography.hazmat.primitives.asymmetric import rsa from cryptography.hazmat.primitives import serialization, hashes -from twisted.internet.endpoints import quoteStringArgument, serverFromString +from twisted.internet.endpoints import serverFromString from twisted.internet import reactor from twisted.internet.defer import Deferred from twisted.internet.task import deferLater from twisted.web.server import Site from twisted.web.static import Data from twisted.web.client import Agent, HTTPConnectionPool, ResponseNeverReceived +from twisted.python.filepath import FilePath from treq.client import HTTPClient -from .common import SyncTestCase, AsyncTestCase +from .common import SyncTestCase, AsyncTestCase, SameProcessStreamEndpointAssigner from ..storage.http_common import get_spki_hash from ..storage.http_client import _StorageClientHTTPSPolicy +from ..storage.http_server import _TLSEndpointWrapper class HTTPSNurlTests(SyncTestCase): @@ -90,7 +92,13 @@ class PinningHTTPSValidation(AsyncTestCase): https://cryptography.io/en/latest/x509/tutorial/#creating-a-self-signed-certificate """ - def to_file(self, key_or_cert) -> str: + def setUp(self): + self._port_assigner = SameProcessStreamEndpointAssigner() + self._port_assigner.setUp() + self.addCleanup(self._port_assigner.tearDown) + return AsyncTestCase.setUp(self) + + def to_file(self, key_or_cert) -> FilePath: """ Write the given key or cert to a temporary file on disk, return the path. @@ -106,7 +114,7 @@ class PinningHTTPSValidation(AsyncTestCase): encryption_algorithm=serialization.NoEncryption(), ) f.write(data) - return path + return FilePath(path) def generate_private_key(self): """Create a RSA private key.""" @@ -137,19 +145,17 @@ class PinningHTTPSValidation(AsyncTestCase): ) @asynccontextmanager - async def listen(self, private_key_path, cert_path): + async def listen(self, private_key_path: FilePath, cert_path: FilePath): """ Context manager that runs a HTTPS server with the given private key and certificate. Returns a URL that will connect to the server. """ - endpoint = serverFromString( - reactor, - "ssl:privateKey={}:certKey={}:port=0:interface=127.0.0.1".format( - quoteStringArgument(str(private_key_path)), - quoteStringArgument(str(cert_path)), - ), + location_hint, endpoint_string = self._port_assigner.assign(reactor) + underlying_endpoint = serverFromString(reactor, endpoint_string) + endpoint = _TLSEndpointWrapper.from_paths( + underlying_endpoint, private_key_path, cert_path ) root = Data(b"YOYODYNE", "text/plain") root.isLeaf = True From ab1297cdd6b7144aa974a41ee0a1b565f2f5202f Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 6 Apr 2022 11:27:42 -0400 Subject: [PATCH 0799/2309] Link to ticket. --- src/allmydata/test/test_storage_https.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/allmydata/test/test_storage_https.py b/src/allmydata/test/test_storage_https.py index 0a5b73a96..35aebf3f2 100644 --- a/src/allmydata/test/test_storage_https.py +++ b/src/allmydata/test/test_storage_https.py @@ -74,6 +74,8 @@ ox5zO3LrQmQw11OaIAs2/kviKAoKTFFxeyYcpS5RuKNDZfHQCXlLwt9bySxG def async_to_deferred(f): """ Wrap an async function to return a Deferred instead. + + Maybe solution to https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3886 """ @wraps(f) From 5a82ea880baa84670cf9d7d3525fa128fb835a51 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 6 Apr 2022 11:31:26 -0400 Subject: [PATCH 0800/2309] More specific methods. --- src/allmydata/test/test_storage_https.py | 39 +++++++++++++++--------- 1 file changed, 25 insertions(+), 14 deletions(-) diff --git a/src/allmydata/test/test_storage_https.py b/src/allmydata/test/test_storage_https.py index 35aebf3f2..1d95c037d 100644 --- a/src/allmydata/test/test_storage_https.py +++ b/src/allmydata/test/test_storage_https.py @@ -100,24 +100,35 @@ class PinningHTTPSValidation(AsyncTestCase): self.addCleanup(self._port_assigner.tearDown) return AsyncTestCase.setUp(self) - def to_file(self, key_or_cert) -> FilePath: + def _temp_file_with_data(self, data: bytes) -> FilePath: """ - Write the given key or cert to a temporary file on disk, return the - path. + Write data to temporary file, return its path. """ path = self.mktemp() with open(path, "wb") as f: - if isinstance(key_or_cert, x509.Certificate): - data = key_or_cert.public_bytes(serialization.Encoding.PEM) - else: - data = key_or_cert.private_bytes( - encoding=serialization.Encoding.PEM, - format=serialization.PrivateFormat.TraditionalOpenSSL, - encryption_algorithm=serialization.NoEncryption(), - ) f.write(data) return FilePath(path) + def cert_to_file(self, cert) -> FilePath: + """ + Write the given certificate to a temporary file on disk, return the + path. + """ + return self._temp_file_with_data(cert.public_bytes(serialization.Encoding.PEM)) + + def private_key_to_file(self, private_key) -> FilePath: + """ + Write the given key to a temporary file on disk, return the + path. + """ + return self._temp_file_with_data( + private_key.private_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PrivateFormat.TraditionalOpenSSL, + encryption_algorithm=serialization.NoEncryption(), + ) + ) + def generate_private_key(self): """Create a RSA private key.""" return rsa.generate_private_key(public_exponent=65537, key_size=2048) @@ -197,7 +208,7 @@ class PinningHTTPSValidation(AsyncTestCase): private_key = self.generate_private_key() certificate = self.generate_certificate(private_key) async with self.listen( - self.to_file(private_key), self.to_file(certificate) + self.private_key_to_file(private_key), self.cert_to_file(certificate) ) as url: response = await self.request(url, certificate) self.assertEqual(await response.content(), b"YOYODYNE") @@ -214,7 +225,7 @@ class PinningHTTPSValidation(AsyncTestCase): certificate2 = self.generate_certificate(private_key2) async with self.listen( - self.to_file(private_key1), self.to_file(certificate1) + self.private_key_to_file(private_key1), self.cert_to_file(certificate1) ) as url: with self.assertRaises(ResponseNeverReceived): await self.request(url, certificate2) @@ -230,7 +241,7 @@ class PinningHTTPSValidation(AsyncTestCase): certificate = self.generate_certificate(private_key, expires_days=-10) async with self.listen( - self.to_file(private_key), self.to_file(certificate) + self.private_key_to_file(private_key), self.cert_to_file(certificate) ) as url: response = await self.request(url, certificate) self.assertEqual(await response.content(), b"YOYODYNE") From 22ebbba5acc80c542df36c4334f4a6b7ee03a081 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 6 Apr 2022 11:35:05 -0400 Subject: [PATCH 0801/2309] Extend testing. --- src/allmydata/test/test_storage_https.py | 32 +++++++++++++++++++++--- 1 file changed, 28 insertions(+), 4 deletions(-) diff --git a/src/allmydata/test/test_storage_https.py b/src/allmydata/test/test_storage_https.py index 1d95c037d..4d08e28b9 100644 --- a/src/allmydata/test/test_storage_https.py +++ b/src/allmydata/test/test_storage_https.py @@ -134,12 +134,17 @@ class PinningHTTPSValidation(AsyncTestCase): return rsa.generate_private_key(public_exponent=65537, key_size=2048) def generate_certificate( - self, private_key, expires_days: int = 10, org_name: str = "Yoyodyne" + self, + private_key, + expires_days: int = 10, + valid_in_days: int = 0, + org_name: str = "Yoyodyne", ): """Generate a certificate from a RSA private key.""" subject = issuer = x509.Name( [x509.NameAttribute(NameOID.ORGANIZATION_NAME, org_name)] ) + starts = datetime.datetime.utcnow() + datetime.timedelta(days=valid_in_days) expires = datetime.datetime.utcnow() + datetime.timedelta(days=expires_days) return ( x509.CertificateBuilder() @@ -147,7 +152,7 @@ class PinningHTTPSValidation(AsyncTestCase): .issuer_name(issuer) .public_key(private_key.public_key()) .serial_number(x509.random_serial_number()) - .not_valid_before(min(datetime.datetime.utcnow(), expires)) + .not_valid_before(min(starts, expires)) .not_valid_after(expires) .add_extension( x509.SubjectAlternativeName([x509.DNSName("localhost")]), @@ -246,6 +251,25 @@ class PinningHTTPSValidation(AsyncTestCase): response = await self.request(url, certificate) self.assertEqual(await response.content(), b"YOYODYNE") - # TODO an obvious attack is a private key that doesn't match the + @async_to_deferred + async def test_server_certificate_not_valid_yet(self): + """ + If the server's certificate is only valid starting in The Future, the + request to the server succeeds if the hash matches the one the client + expects; start time has no effect. + """ + private_key = self.generate_private_key() + certificate = self.generate_certificate( + private_key, expires_days=10, valid_in_days=5 + ) + + async with self.listen( + self.private_key_to_file(private_key), self.cert_to_file(certificate) + ) as url: + response = await self.request(url, certificate) + self.assertEqual(await response.content(), b"YOYODYNE") + + # A potential attack to test is a private key that doesn't match the # certificate... but OpenSSL (quite rightly) won't let you listen with that - # so I don't know how to test that! + # so I don't know how to test that! See + # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3884 From 423512ad0019f155c012b8223e07fa8dde8169f0 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 6 Apr 2022 11:48:17 -0400 Subject: [PATCH 0802/2309] Wait harder. --- src/allmydata/test/test_storage_https.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/test_storage_https.py b/src/allmydata/test/test_storage_https.py index 4d08e28b9..6877dae2a 100644 --- a/src/allmydata/test/test_storage_https.py +++ b/src/allmydata/test/test_storage_https.py @@ -184,7 +184,7 @@ class PinningHTTPSValidation(AsyncTestCase): await listening_port.stopListening() # Make sure all server connections are closed :( No idea why this # is necessary when it's not for IStorageServer HTTPS tests. - await deferLater(reactor, 0.001) + await deferLater(reactor, 0.01) def request(self, url: str, expected_certificate: x509.Certificate): """ From bdcf054de61d66b25a8a12aff4658b11d29a3a7b Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 8 Apr 2022 13:37:18 -0400 Subject: [PATCH 0803/2309] Switch to generating certs on the fly since Python packaging was being a pain. --- setup.py | 1 - src/allmydata/test/certs.py | 66 ++++++++++++++ src/allmydata/test/certs/domain.crt | 19 ---- src/allmydata/test/certs/private.key | 27 ------ src/allmydata/test/test_istorageserver.py | 20 +++-- src/allmydata/test/test_storage_https.py | 104 +++++----------------- 6 files changed, 101 insertions(+), 136 deletions(-) create mode 100644 src/allmydata/test/certs.py delete mode 100644 src/allmydata/test/certs/domain.crt delete mode 100644 src/allmydata/test/certs/private.key diff --git a/setup.py b/setup.py index 1883032de..5285b5d08 100644 --- a/setup.py +++ b/setup.py @@ -410,7 +410,6 @@ setup(name="tahoe-lafs", # also set in __init__.py "static/css/*.css", ], "allmydata": ["ported-modules.txt"], - "allmydata.test": ["certs/*"] }, include_package_data=True, setup_requires=setup_requires, diff --git a/src/allmydata/test/certs.py b/src/allmydata/test/certs.py new file mode 100644 index 000000000..9e6640386 --- /dev/null +++ b/src/allmydata/test/certs.py @@ -0,0 +1,66 @@ +"""Utilities for generating TLS certificates.""" + +import datetime + +from cryptography import x509 +from cryptography.x509.oid import NameOID +from cryptography.hazmat.primitives.asymmetric import rsa +from cryptography.hazmat.primitives import serialization, hashes + +from twisted.python.filepath import FilePath + + +def cert_to_file(path: FilePath, cert) -> FilePath: + """ + Write the given certificate to a file on disk. Returns the path. + """ + path.setContent(cert.public_bytes(serialization.Encoding.PEM)) + return path + + +def private_key_to_file(path: FilePath, private_key) -> FilePath: + """ + Write the given key to a file on disk. Returns the path. + """ + path.setContent( + private_key.private_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PrivateFormat.TraditionalOpenSSL, + encryption_algorithm=serialization.NoEncryption(), + ) + ) + return path + + +def generate_private_key(): + """Create a RSA private key.""" + return rsa.generate_private_key(public_exponent=65537, key_size=2048) + + +def generate_certificate( + private_key, + expires_days: int = 10, + valid_in_days: int = 0, + org_name: str = "Yoyodyne", +): + """Generate a certificate from a RSA private key.""" + subject = issuer = x509.Name( + [x509.NameAttribute(NameOID.ORGANIZATION_NAME, org_name)] + ) + starts = datetime.datetime.utcnow() + datetime.timedelta(days=valid_in_days) + expires = datetime.datetime.utcnow() + datetime.timedelta(days=expires_days) + return ( + x509.CertificateBuilder() + .subject_name(subject) + .issuer_name(issuer) + .public_key(private_key.public_key()) + .serial_number(x509.random_serial_number()) + .not_valid_before(min(starts, expires)) + .not_valid_after(expires) + .add_extension( + x509.SubjectAlternativeName([x509.DNSName("localhost")]), + critical=False, + # Sign our certificate with our private key + ) + .sign(private_key, hashes.SHA256()) + ) diff --git a/src/allmydata/test/certs/domain.crt b/src/allmydata/test/certs/domain.crt deleted file mode 100644 index 8932b44e7..000000000 --- a/src/allmydata/test/certs/domain.crt +++ /dev/null @@ -1,19 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIDCzCCAfMCFHrs4pMBs35SlU3ZGMnVY5qp5MfZMA0GCSqGSIb3DQEBCwUAMEIx -CzAJBgNVBAYTAlhYMRUwEwYDVQQHDAxEZWZhdWx0IENpdHkxHDAaBgNVBAoME0Rl -ZmF1bHQgQ29tcGFueSBMdGQwHhcNMjIwMzIzMjAwNTM0WhcNMjIwNDIyMjAwNTM0 -WjBCMQswCQYDVQQGEwJYWDEVMBMGA1UEBwwMRGVmYXVsdCBDaXR5MRwwGgYDVQQK -DBNEZWZhdWx0IENvbXBhbnkgTHRkMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIB -CgKCAQEAzj0C3W4OiugEb3nr7NVQfrgzL3Tet5ze8pJCew0lIsDNrZiESF/Lvnrc -VcQtjraC3ySZO3rLDLhCwALLC1TVw3lp2ou+02kYtfJJVr1XyEcVCpsxx89u/W5B -VgMyBMoVZLS2BoBA1652XZnphvgm8n/daf9HH2Y4ifRDTIl1Bgl+4XtqlCKNFGYO -7zeadViKeI3fJOyzQaT+WTONQRWtAMP6c/n6VnlrdNughUEM+X0HqwVmr7UgatuA -LrJpfAkfKfOTZ70p2iOvegBH5ryR3InsB/O0/fp2+esNCEU3pfXWzg8ACif+ZQJc -AukUpQ4iJB7r1AK6iTZUqHnFgu9gYwIDAQABMA0GCSqGSIb3DQEBCwUAA4IBAQCv -BzVSinez5lcSWWCRve/TlMePEJK5d7OFcc90n7kmn+rkEYrel3a7Q+ctJxY9AKYG -A9X1AMDSH9z3243KFNaRJ1xKg0Mg8J/BLN9iphM5AnAuiAkCqs8VbD4hF4hZHLMZ -BVNyuLSdo+lBzbS57/Lz+lUWcxrXR5qgsEWSbjP+SrsDQKODfyoxKuU0XrxmNLd2 -dTswbZKsqXBs80/T1jHwJjJXLp6YUsZqN1TGYtk8hEcE7bGaC3n7WhRjBP1WghNl -OG7FFRPte2w5seQRgkrBodLb9OCkhU4xfdyLnFICqkQHQAqIdXksEvir9uGY/yjC -zxAh60LEEQz6Kz29jYog ------END CERTIFICATE----- diff --git a/src/allmydata/test/certs/private.key b/src/allmydata/test/certs/private.key deleted file mode 100644 index b4b5ed580..000000000 --- a/src/allmydata/test/certs/private.key +++ /dev/null @@ -1,27 +0,0 @@ ------BEGIN RSA PRIVATE KEY----- -MIIEpAIBAAKCAQEAzj0C3W4OiugEb3nr7NVQfrgzL3Tet5ze8pJCew0lIsDNrZiE -SF/LvnrcVcQtjraC3ySZO3rLDLhCwALLC1TVw3lp2ou+02kYtfJJVr1XyEcVCpsx -x89u/W5BVgMyBMoVZLS2BoBA1652XZnphvgm8n/daf9HH2Y4ifRDTIl1Bgl+4Xtq -lCKNFGYO7zeadViKeI3fJOyzQaT+WTONQRWtAMP6c/n6VnlrdNughUEM+X0HqwVm -r7UgatuALrJpfAkfKfOTZ70p2iOvegBH5ryR3InsB/O0/fp2+esNCEU3pfXWzg8A -Cif+ZQJcAukUpQ4iJB7r1AK6iTZUqHnFgu9gYwIDAQABAoIBAG71rGDuIazif+Bq -PGDDs/c5q3BQ9LLdF6Zywonp3J0CFqbbc/BsefYVrA4I6mnqECd2TWsO+cfyKxeb -aRrDne75l9YZcaXU2ZKqtIKShHQgqlV2giX6mMCJXWWlenfRMglooLaGslxYZR6e -/GG9iVbXLI0m52EhYjH21W6MVgXUhsrDoI16pB87zk7jFZzyNsjRU5+bwr4L2jed -A1PseE6AI2kIpJCl8IIu6hRhVwjr8MIkaAtI3G8WmSAru99apHNttf6sgB2kcq2r -Qp1uXEXNVFQiJqcwMPOlWZ5X0kMIBxmFe10MkJbmCUoE/jPqO90XN2jyZPSONZU8 -4yqd9GECgYEA5qQ+tfgOJFJ86t103pwkPm0+PxuQUTXry5ETnez+wxqqwbDxrEHi -MQoPZuVXPkbQ6g80KSpdI7AkFvu6BzcNlgpOI3gLZ30PHTF4HJTP01fKfbbVhg8N -WJS0yUh+kQDrGVcZbIbB5Q1vS5hu8ftk5ukns5BdFf/NS7fBfU8b3K0CgYEA5Onm -V2D1D9kdhTjta8f0d0s6+TdHoV86SRbkAEgnqzwlHWV17LlpXQic6iwokfS4TQSl -+1Z23Dt+OhLm/N0N3LgCxBhzTMnWGdy+w9co4GifwqR6T72JAxGOVoqIWk2cVpa5 -8qJx0eAFXqcvpIASEoxYrdoKFUh60mAiQE6JQ08CgYB8wCoLUviTPOrEPrSQE/Sm -r4ATsl0FEB1SJk5uBVpnPW1PBt4xRhGKZN6f0Ty3OqaVc1PLUFbAju12YQHmFSkM -Ftbc6HmCqGocaD2HeBZRQhMMnHAx6sJVP1np5YRP+icvtaTSxrDpq7KfOPwJdujE -3SfUQCmZVJs+cU3+8WMooQKBgQCdvvl2eWAm/a00IxipT2+NzY/kMU3xTFg0Ccww -zYhYnefNrB9pdBPBgq/vR2LlwchHes6OtvTNq0m+50u6MPLeiQeO7nJ2FhiuVco3 -1staaX6+eO24iZojPTPjOy/fWuBDYzbcl0jsIf5RTdCtAXxyv7hUhY6xP/Mzif/Q -ZM5+TQKBgQCF7pB6yLIAp6YJQ4uqOVbC2bMLr6tVWaNdEykiDd9zQkoacw/DPX1Y -FKehY/9EKraJ4t6d2/BBpJQyuIU4/gz8QMvjqQGP3NIfVeqBAPYo/nTYKOK0PSxB -Kl28Axxz6rjEeK4BixOES5PXuq2nNJXT8OSPYZQxQdTHstCWOP4Z6g== ------END RSA PRIVATE KEY----- diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index bc7d5b853..3d6f610be 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -25,6 +25,12 @@ from foolscap.api import Referenceable, RemoteException from allmydata.interfaces import IStorageServer # really, IStorageClient from .common_system import SystemTestMixin from .common import AsyncTestCase, SameProcessStreamEndpointAssigner +from .certs import ( + generate_certificate, + generate_private_key, + private_key_to_file, + cert_to_file, +) from allmydata.storage.server import StorageServer # not a IStorageServer!! from allmydata.storage.http_server import HTTPServer, listen_tls from allmydata.storage.http_client import StorageClient @@ -1057,20 +1063,16 @@ class _HTTPMixin(_SharedMixin): swissnum = b"1234" http_storage_server = HTTPServer(self.server, swissnum) - # Listen on randomly assigned port, using self-signed cert we generated - # manually: - certs_dir = FilePath(__file__).parent().child("certs") + # Listen on randomly assigned port, using self-signed cert: + private_key = generate_private_key() + certificate = generate_certificate(private_key) _, endpoint_string = self._port_assigner.assign(reactor) nurl, listening_port = yield listen_tls( http_storage_server, "127.0.0.1", serverFromString(reactor, endpoint_string), - # This is just a self-signed certificate with randomly generated - # private key; nothing at all special about it. You can regenerate - # with code in allmydata.test.test_storage_https or with openssl - # CLI, with no meaningful change to the test. - certs_dir.child("private.key"), - certs_dir.child("domain.crt"), + private_key_to_file(FilePath(self.mktemp()), private_key), + cert_to_file(FilePath(self.mktemp()), certificate), ) self.addCleanup(listening_port.stopListening) diff --git a/src/allmydata/test/test_storage_https.py b/src/allmydata/test/test_storage_https.py index 6877dae2a..73c99725a 100644 --- a/src/allmydata/test/test_storage_https.py +++ b/src/allmydata/test/test_storage_https.py @@ -6,14 +6,10 @@ server authentication logic, which may one day apply outside of HTTP Storage Protocol. """ -import datetime from functools import wraps from contextlib import asynccontextmanager from cryptography import x509 -from cryptography.x509.oid import NameOID -from cryptography.hazmat.primitives.asymmetric import rsa -from cryptography.hazmat.primitives import serialization, hashes from twisted.internet.endpoints import serverFromString from twisted.internet import reactor @@ -26,6 +22,12 @@ from twisted.python.filepath import FilePath from treq.client import HTTPClient from .common import SyncTestCase, AsyncTestCase, SameProcessStreamEndpointAssigner +from .certs import ( + generate_certificate, + generate_private_key, + private_key_to_file, + cert_to_file, +) from ..storage.http_common import get_spki_hash from ..storage.http_client import _StorageClientHTTPSPolicy from ..storage.http_server import _TLSEndpointWrapper @@ -100,68 +102,6 @@ class PinningHTTPSValidation(AsyncTestCase): self.addCleanup(self._port_assigner.tearDown) return AsyncTestCase.setUp(self) - def _temp_file_with_data(self, data: bytes) -> FilePath: - """ - Write data to temporary file, return its path. - """ - path = self.mktemp() - with open(path, "wb") as f: - f.write(data) - return FilePath(path) - - def cert_to_file(self, cert) -> FilePath: - """ - Write the given certificate to a temporary file on disk, return the - path. - """ - return self._temp_file_with_data(cert.public_bytes(serialization.Encoding.PEM)) - - def private_key_to_file(self, private_key) -> FilePath: - """ - Write the given key to a temporary file on disk, return the - path. - """ - return self._temp_file_with_data( - private_key.private_bytes( - encoding=serialization.Encoding.PEM, - format=serialization.PrivateFormat.TraditionalOpenSSL, - encryption_algorithm=serialization.NoEncryption(), - ) - ) - - def generate_private_key(self): - """Create a RSA private key.""" - return rsa.generate_private_key(public_exponent=65537, key_size=2048) - - def generate_certificate( - self, - private_key, - expires_days: int = 10, - valid_in_days: int = 0, - org_name: str = "Yoyodyne", - ): - """Generate a certificate from a RSA private key.""" - subject = issuer = x509.Name( - [x509.NameAttribute(NameOID.ORGANIZATION_NAME, org_name)] - ) - starts = datetime.datetime.utcnow() + datetime.timedelta(days=valid_in_days) - expires = datetime.datetime.utcnow() + datetime.timedelta(days=expires_days) - return ( - x509.CertificateBuilder() - .subject_name(subject) - .issuer_name(issuer) - .public_key(private_key.public_key()) - .serial_number(x509.random_serial_number()) - .not_valid_before(min(starts, expires)) - .not_valid_after(expires) - .add_extension( - x509.SubjectAlternativeName([x509.DNSName("localhost")]), - critical=False, - # Sign our certificate with our private key - ) - .sign(private_key, hashes.SHA256()) - ) - @asynccontextmanager async def listen(self, private_key_path: FilePath, cert_path: FilePath): """ @@ -210,10 +150,11 @@ class PinningHTTPSValidation(AsyncTestCase): If all conditions are met, a TLS client using the Tahoe-LAFS policy can connect to the server. """ - private_key = self.generate_private_key() - certificate = self.generate_certificate(private_key) + private_key = generate_private_key() + certificate = generate_certificate(private_key) async with self.listen( - self.private_key_to_file(private_key), self.cert_to_file(certificate) + private_key_to_file(FilePath(self.mktemp()), private_key), + cert_to_file(FilePath(self.mktemp()), certificate), ) as url: response = await self.request(url, certificate) self.assertEqual(await response.content(), b"YOYODYNE") @@ -224,13 +165,14 @@ class PinningHTTPSValidation(AsyncTestCase): If the server's certificate hash doesn't match the hash the client expects, the request to the server fails. """ - private_key1 = self.generate_private_key() - certificate1 = self.generate_certificate(private_key1) - private_key2 = self.generate_private_key() - certificate2 = self.generate_certificate(private_key2) + private_key1 = generate_private_key() + certificate1 = generate_certificate(private_key1) + private_key2 = generate_private_key() + certificate2 = generate_certificate(private_key2) async with self.listen( - self.private_key_to_file(private_key1), self.cert_to_file(certificate1) + private_key_to_file(FilePath(self.mktemp()), private_key1), + cert_to_file(FilePath(self.mktemp()), certificate1), ) as url: with self.assertRaises(ResponseNeverReceived): await self.request(url, certificate2) @@ -242,11 +184,12 @@ class PinningHTTPSValidation(AsyncTestCase): succeeds if the hash matches the one the client expects; expiration has no effect. """ - private_key = self.generate_private_key() - certificate = self.generate_certificate(private_key, expires_days=-10) + private_key = generate_private_key() + certificate = generate_certificate(private_key, expires_days=-10) async with self.listen( - self.private_key_to_file(private_key), self.cert_to_file(certificate) + private_key_to_file(FilePath(self.mktemp()), private_key), + cert_to_file(FilePath(self.mktemp()), certificate), ) as url: response = await self.request(url, certificate) self.assertEqual(await response.content(), b"YOYODYNE") @@ -258,13 +201,14 @@ class PinningHTTPSValidation(AsyncTestCase): request to the server succeeds if the hash matches the one the client expects; start time has no effect. """ - private_key = self.generate_private_key() - certificate = self.generate_certificate( + private_key = generate_private_key() + certificate = generate_certificate( private_key, expires_days=10, valid_in_days=5 ) async with self.listen( - self.private_key_to_file(private_key), self.cert_to_file(certificate) + private_key_to_file(FilePath(self.mktemp()), private_key), + cert_to_file(FilePath(self.mktemp()), certificate), ) as url: response = await self.request(url, certificate) self.assertEqual(await response.content(), b"YOYODYNE") From e5b0e51f72cdb110179ec4b4a2527a8af55afb4b Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 11 Apr 2022 13:11:45 -0400 Subject: [PATCH 0804/2309] Server-side schema validation of CBOR. --- setup.py | 3 +- src/allmydata/storage/http_client.py | 4 ++- src/allmydata/storage/http_server.py | 38 +++++++++++++++++++++---- src/allmydata/test/test_storage_http.py | 30 +++++++++++++++++++ 4 files changed, 68 insertions(+), 7 deletions(-) diff --git a/setup.py b/setup.py index 5285b5d08..c84d0ecde 100644 --- a/setup.py +++ b/setup.py @@ -135,7 +135,8 @@ install_requires = [ "klein", "werkzeug", "treq", - "cbor2" + "cbor2", + "pycddl", ] setup_requires = [ diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 99275ae24..e9a593a3e 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -47,7 +47,9 @@ def _decode_cbor(response): else: raise ClientException(-1, "Server didn't send CBOR") else: - return fail(ClientException(response.code, response.phrase)) + return treq.content(response).addCallback( + lambda data: fail(ClientException(response.code, response.phrase, data)) + ) @attr.s diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index f648d8331..4bf552fc5 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -21,7 +21,7 @@ from werkzeug.datastructures import ContentRange # TODO Make sure to use pure Python versions? from cbor2 import dumps, loads - +from pycddl import Schema, ValidationError as CDDLValidationError from .server import StorageServer from .http_common import swissnum_auth_header, Secrets, get_content_type, CBOR_MIME_TYPE from .common import si_a2b @@ -215,6 +215,25 @@ class _HTTPError(Exception): self.code = code +# CDDL schemas. +# +# Tags are of the form #6.nnn, where the number is documented at +# https://www.iana.org/assignments/cbor-tags/cbor-tags.xhtml. Notably, #6.258 +# indicates a set. +_SCHEMAS = { + "allocate_buckets": Schema(""" + message = { + share-numbers: #6.258([* uint]) + allocated-size: uint + } + """), + "advise_corrupt_share": Schema(""" + message = { + reason: tstr + } + """) +} + class HTTPServer(object): """ A HTTP interface to the storage server. @@ -229,6 +248,12 @@ class HTTPServer(object): request.setResponseCode(failure.value.code) return b"" + @_app.handle_errors(CDDLValidationError) + def _cddl_validation_error(self, request, failure): + """Handle CDDL validation errors.""" + request.setResponseCode(http.BAD_REQUEST) + return str(failure.value).encode("utf-8") + def __init__( self, storage_server, swissnum ): # type: (StorageServer, bytes) -> None @@ -268,7 +293,7 @@ class HTTPServer(object): # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3861 raise _HTTPError(http.NOT_ACCEPTABLE) - def _read_encoded(self, request) -> Any: + def _read_encoded(self, request, schema: Schema) -> Any: """ Read encoded request body data, decoding it with CBOR by default. """ @@ -276,7 +301,10 @@ class HTTPServer(object): if content_type == CBOR_MIME_TYPE: # TODO limit memory usage, client could send arbitrarily large data... # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3872 - return loads(request.content.read()) + message = request.content.read() + schema.validate_cbor(message) + result = loads(message) + return result else: raise _HTTPError(http.UNSUPPORTED_MEDIA_TYPE) @@ -298,7 +326,7 @@ class HTTPServer(object): def allocate_buckets(self, request, authorization, storage_index): """Allocate buckets.""" upload_secret = authorization[Secrets.UPLOAD] - info = self._read_encoded(request) + info = self._read_encoded(request, _SCHEMAS["allocate_buckets"]) # We do NOT validate the upload secret for existing bucket uploads. # Another upload may be happening in parallel, with a different upload @@ -498,6 +526,6 @@ class HTTPServer(object): except KeyError: raise _HTTPError(http.NOT_FOUND) - info = self._read_encoded(request) + info = self._read_encoded(request, _SCHEMAS["advise_corrupt_share"]) bucket.advise_corrupt_share(info["reason"].encode("utf-8")) return b"" diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index af90d58a9..af868ddce 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -413,6 +413,36 @@ class GenericHTTPAPITests(SyncTestCase): ) self.assertEqual(version, expected_version) + def test_schema_validation(self): + """ + Ensure that schema validation is happening: invalid CBOR should result + in bad request response code (error 400). + + We don't bother checking every single request, the API on the + server-side is designed to require a schema, so it validates + everywhere. But we check at least one to ensure we get correct + response code on bad input, so we know validation happened. + """ + upload_secret = urandom(32) + lease_secret = urandom(32) + storage_index = urandom(16) + url = self.http.client.relative_url( + "/v1/immutable/" + _encode_si(storage_index) + ) + message = {"bad-message": "missing expected keys"} + + response = result_of( + self.http.client.request( + "POST", + url, + lease_renew_secret=lease_secret, + lease_cancel_secret=lease_secret, + upload_secret=upload_secret, + message_to_serialize=message, + ) + ) + self.assertEqual(response.code, http.BAD_REQUEST) + class ImmutableHTTPAPITests(SyncTestCase): """ From 07049c2ac8f7eddc54393015f285f72231db8502 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 11 Apr 2022 13:13:14 -0400 Subject: [PATCH 0805/2309] News file. --- newsfragments/3802.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3802.minor diff --git a/newsfragments/3802.minor b/newsfragments/3802.minor new file mode 100644 index 000000000..e69de29bb From dfad50b1c236aeac0c7944b636e81ef2c9855f67 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 11 Apr 2022 14:03:30 -0400 Subject: [PATCH 0806/2309] Better error. --- src/allmydata/storage/http_server.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 4bf552fc5..3876409b0 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -86,8 +86,8 @@ def _authorization_decorator(required_secrets): try: secrets = _extract_secrets(authorization, required_secrets) except ClientSecretsException: - request.setResponseCode(400) - return b"" + request.setResponseCode(http.BAD_REQUEST) + return b"Missing required secrets" return f(self, request, secrets, *args, **kwargs) return route From 4b20b67ce60834cf81499b8eff2e2b6abfd4eb86 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 11 Apr 2022 14:03:48 -0400 Subject: [PATCH 0807/2309] Client-side schema validation. --- src/allmydata/storage/http_client.py | 61 ++++++++++++++++++++++--- src/allmydata/test/test_storage_http.py | 30 +++++++++--- 2 files changed, 78 insertions(+), 13 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index e9a593a3e..3a758e592 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -11,6 +11,7 @@ import attr # TODO Make sure to import Python version? from cbor2 import loads, dumps +from pycddl import Schema from collections_extended import RangeMap from werkzeug.datastructures import Range, ContentRange from twisted.web.http_headers import Headers @@ -36,14 +37,62 @@ class ClientException(Exception): self.code = code -def _decode_cbor(response): +# Schemas for server responses. +# +# TODO usage of sets is inconsistent. Either use everywhere (and document in +# spec document) or use nowhere. +_SCHEMAS = { + "get_version": Schema( + """ + message = {'http://allmydata.org/tahoe/protocols/storage/v1' => { + 'maximum-immutable-share-size' => uint + 'maximum-mutable-share-size' => uint + 'available-space' => uint + 'tolerates-immutable-read-overrun' => bool + 'delete-mutable-shares-with-zero-length-writev' => bool + 'fills-holes-with-zero-bytes' => bool + 'prevents-read-past-end-of-share-data' => bool + } + 'application-version' => bstr + } + """ + ), + "allocate_buckets": Schema( + """ + message = { + already-have: #6.258([* uint]) + allocated: #6.258([* uint]) + } + """ + ), + "immutable_write_share_chunk": Schema( + """ + message = { + required: [* {begin: uint, end: uint}] + } + """ + ), + "list_shares": Schema( + """ + message = [* uint] + """ + ), +} + + +def _decode_cbor(response, schema: Schema): """Given HTTP response, return decoded CBOR body.""" + + def got_content(data): + schema.validate_cbor(data) + return loads(data) + if response.code > 199 and response.code < 300: content_type = get_content_type(response.headers) if content_type == CBOR_MIME_TYPE: # TODO limit memory usage # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3872 - return treq.content(response).addCallback(loads) + return treq.content(response).addCallback(got_content) else: raise ClientException(-1, "Server didn't send CBOR") else: @@ -151,7 +200,7 @@ class StorageClientGeneral(object): """ url = self._client.relative_url("/v1/version") response = yield self._client.request("GET", url) - decoded_response = yield _decode_cbor(response) + decoded_response = yield _decode_cbor(response, _SCHEMAS["get_version"]) returnValue(decoded_response) @@ -209,7 +258,7 @@ class StorageClientImmutables(object): upload_secret=upload_secret, message_to_serialize=message, ) - decoded_response = yield _decode_cbor(response) + decoded_response = yield _decode_cbor(response, _SCHEMAS["allocate_buckets"]) returnValue( ImmutableCreateResult( already_have=decoded_response["already-have"], @@ -281,7 +330,7 @@ class StorageClientImmutables(object): raise ClientException( response.code, ) - body = yield _decode_cbor(response) + body = yield _decode_cbor(response, _SCHEMAS["immutable_write_share_chunk"]) remaining = RangeMap() for chunk in body["required"]: remaining.set(True, chunk["begin"], chunk["end"]) @@ -334,7 +383,7 @@ class StorageClientImmutables(object): url, ) if response.code == http.OK: - body = yield _decode_cbor(response) + body = yield _decode_cbor(response, _SCHEMAS["list_shares"]) returnValue(set(body)) else: raise ClientException(response.code) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index af868ddce..4679880a0 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -18,6 +18,8 @@ from base64 import b64encode from contextlib import contextmanager from os import urandom +from cbor2 import dumps +from pycddl import ValidationError as CDDLValidationError from hypothesis import assume, given, strategies as st from fixtures import Fixture, TempDir from treq.testing import StubTreq @@ -49,7 +51,7 @@ from ..storage.http_client import ( StorageClientGeneral, _encode_si, ) -from ..storage.http_common import get_content_type +from ..storage.http_common import get_content_type, CBOR_MIME_TYPE from ..storage.common import si_b2a @@ -239,6 +241,12 @@ class TestApp(object): else: return "BAD: {}".format(authorization) + @_authorized_route(_app, set(), "/v1/version", methods=["GET"]) + def bad_version(self, request, authorization): + """Return version result that violates the expected schema.""" + request.setHeader("content-type", CBOR_MIME_TYPE) + return dumps({"garbage": 123}) + def result_of(d): """ @@ -257,15 +265,15 @@ def result_of(d): ) -class RoutingTests(SyncTestCase): +class CustomHTTPServerTests(SyncTestCase): """ - Tests for the HTTP routing infrastructure. + Tests that use a custom HTTP server. """ def setUp(self): if PY2: self.skipTest("Not going to bother supporting Python 2") - super(RoutingTests, self).setUp() + super(CustomHTTPServerTests, self).setUp() # Could be a fixture, but will only be used in this test class so not # going to bother: self._http_server = TestApp() @@ -277,8 +285,8 @@ class RoutingTests(SyncTestCase): def test_authorization_enforcement(self): """ - The requirement for secrets is enforced; if they are not given, a 400 - response code is returned. + The requirement for secrets is enforced by the ``_authorized_route`` + decorator; if they are not given, a 400 response code is returned. """ # Without secret, get a 400 error. response = result_of( @@ -298,6 +306,14 @@ class RoutingTests(SyncTestCase): self.assertEqual(response.code, 200) self.assertEqual(result_of(response.content()), b"GOOD SECRET") + def test_client_side_schema_validation(self): + """ + The client validates returned CBOR message against a schema. + """ + client = StorageClientGeneral(self.client) + with self.assertRaises(CDDLValidationError): + result_of(client.get_version()) + class HttpTestFixture(Fixture): """ @@ -413,7 +429,7 @@ class GenericHTTPAPITests(SyncTestCase): ) self.assertEqual(version, expected_version) - def test_schema_validation(self): + def test_server_side_schema_validation(self): """ Ensure that schema validation is happening: invalid CBOR should result in bad request response code (error 400). From f19bf8cf86d58457be4e7b4726626f3de5f29dc3 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 11 Apr 2022 15:04:55 -0400 Subject: [PATCH 0808/2309] Parameterize the options object to the `run_cli` helper --- src/allmydata/test/common_util.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/allmydata/test/common_util.py b/src/allmydata/test/common_util.py index d2d20916d..e63c3eef8 100644 --- a/src/allmydata/test/common_util.py +++ b/src/allmydata/test/common_util.py @@ -69,6 +69,9 @@ def run_cli_native(verb, *args, **kwargs): Most code should prefer ``run_cli_unicode`` which deals with all the necessary encoding considerations. + :param runner.Options options: The options instance to use to parse the + given arguments. + :param native_str verb: The command to run. For example, ``"create-node"``. @@ -88,6 +91,7 @@ def run_cli_native(verb, *args, **kwargs): matching native behavior. If True, stdout/stderr are returned as bytes. """ + options = kwargs.pop("options", runner.Options()) nodeargs = kwargs.pop("nodeargs", []) encoding = kwargs.pop("encoding", None) or getattr(sys.stdout, "encoding") or "utf-8" return_bytes = kwargs.pop("return_bytes", False) @@ -134,7 +138,7 @@ def run_cli_native(verb, *args, **kwargs): d.addCallback( partial( runner.parse_or_exit, - runner.Options(), + options, ), stdout=stdout, stderr=stderr, From dffcdf28543ced3bd214370bccbb5b78354d587d Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 11 Apr 2022 15:05:32 -0400 Subject: [PATCH 0809/2309] Clean up the Py2/Py3 boilerplate --- src/allmydata/test/cli/test_invite.py | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/src/allmydata/test/cli/test_invite.py b/src/allmydata/test/cli/test_invite.py index 20d012995..749898b77 100644 --- a/src/allmydata/test/cli/test_invite.py +++ b/src/allmydata/test/cli/test_invite.py @@ -1,24 +1,12 @@ """ -Ported to Pythn 3. +Tests for ``tahoe invite``. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals - -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import os import mock import json from os.path import join - -try: - from typing import Optional, Sequence -except ImportError: - pass +from typing import Optional, Sequence from twisted.trial import unittest from twisted.internet import defer From bc6dafa999fa9b7d2af8dbab36ed532b74919e6b Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 12 Apr 2022 11:01:04 -0400 Subject: [PATCH 0810/2309] Replace monkey-patching of wormhole with a parameter to run_cli --- src/allmydata/scripts/create_node.py | 5 +- src/allmydata/scripts/runner.py | 6 + src/allmydata/scripts/tahoe_invite.py | 6 +- src/allmydata/test/cli/test_invite.py | 508 +++++++++++++--------- src/allmydata/test/cli/wormholetesting.py | 304 +++++++++++++ 5 files changed, 614 insertions(+), 215 deletions(-) create mode 100644 src/allmydata/test/cli/wormholetesting.py diff --git a/src/allmydata/scripts/create_node.py b/src/allmydata/scripts/create_node.py index 4959ed391..5d9da518b 100644 --- a/src/allmydata/scripts/create_node.py +++ b/src/allmydata/scripts/create_node.py @@ -37,9 +37,6 @@ from allmydata.util.assertutil import precondition from allmydata.util.encodingutil import listdir_unicode, argv_to_unicode, quote_local_unicode_path, get_io_encoding from allmydata.util import fileutil, i2p_provider, iputil, tor_provider, jsonbytes as json -from wormhole import wormhole - - dummy_tac = """ import sys print("Nodes created by Tahoe-LAFS v1.11.0 or later cannot be run by") @@ -377,7 +374,7 @@ def _get_config_via_wormhole(config): relay_url = config.parent['wormhole-server'] print("Connecting to '{}'".format(relay_url), file=out) - wh = wormhole.create( + wh = config.parent.wormhole.create( appid=config.parent['wormhole-invite-appid'], relay_url=relay_url, reactor=reactor, diff --git a/src/allmydata/scripts/runner.py b/src/allmydata/scripts/runner.py index 145ee6464..a0d8a752b 100644 --- a/src/allmydata/scripts/runner.py +++ b/src/allmydata/scripts/runner.py @@ -58,11 +58,17 @@ process_control_commands = [ class Options(usage.Options): + """ + :ivar wormhole: An object exposing the magic-wormhole API (mainly a test + hook). + """ # unit tests can override these to point at StringIO instances stdin = sys.stdin stdout = sys.stdout stderr = sys.stderr + from wormhole import wormhole + subCommands = ( create_node.subCommands + admin.subCommands + process_control_commands diff --git a/src/allmydata/scripts/tahoe_invite.py b/src/allmydata/scripts/tahoe_invite.py index 09d4cbd59..c5f08f588 100644 --- a/src/allmydata/scripts/tahoe_invite.py +++ b/src/allmydata/scripts/tahoe_invite.py @@ -18,8 +18,6 @@ except ImportError: from twisted.python import usage from twisted.internet import defer, reactor -from wormhole import wormhole - from allmydata.util.encodingutil import argv_to_abspath from allmydata.util import jsonbytes as json from allmydata.scripts.common import get_default_nodedir, get_introducer_furl @@ -44,13 +42,15 @@ class InviteOptions(usage.Options): self['nick'] = args[0].strip() +wormhole = None + @defer.inlineCallbacks def _send_config_via_wormhole(options, config): out = options.stdout err = options.stderr relay_url = options.parent['wormhole-server'] print("Connecting to '{}'...".format(relay_url), file=out) - wh = wormhole.create( + wh = options.parent.wormhole.create( appid=options.parent['wormhole-invite-appid'], relay_url=relay_url, reactor=reactor, diff --git a/src/allmydata/test/cli/test_invite.py b/src/allmydata/test/cli/test_invite.py index 749898b77..c4bb6fd7e 100644 --- a/src/allmydata/test/cli/test_invite.py +++ b/src/allmydata/test/cli/test_invite.py @@ -2,62 +2,21 @@ Tests for ``tahoe invite``. """ -import os -import mock import json +import os from os.path import join from typing import Optional, Sequence -from twisted.trial import unittest from twisted.internet import defer +from twisted.trial import unittest + +from ...client import read_config +from ...scripts import runner +from ...util.jsonbytes import dumps_bytes from ..common_util import run_cli from ..no_network import GridTestMixin from .common import CLITestMixin -from ...client import ( - read_config, -) - -class _FakeWormhole(object): - - def __init__(self, outgoing_messages): - self.messages = [] - for o in outgoing_messages: - assert isinstance(o, bytes) - self._outgoing = outgoing_messages - - def get_code(self): - return defer.succeed(u"6-alarmist-tuba") - - def set_code(self, code): - self._code = code - - def get_welcome(self): - return defer.succeed( - { - u"welcome": {}, - } - ) - - def allocate_code(self): - return None - - def send_message(self, msg): - assert isinstance(msg, bytes) - self.messages.append(msg) - - def get_message(self): - return defer.succeed(self._outgoing.pop(0)) - - def close(self): - return defer.succeed(None) - - -def _create_fake_wormhole(outgoing_messages): - outgoing_messages = [ - m.encode("utf-8") if isinstance(m, str) else m - for m in outgoing_messages - ] - return _FakeWormhole(outgoing_messages) +from .wormholetesting import MemoryWormholeServer, memory_server class Join(GridTestMixin, CLITestMixin, unittest.TestCase): @@ -74,41 +33,52 @@ class Join(GridTestMixin, CLITestMixin, unittest.TestCase): successfully join after an invite """ node_dir = self.mktemp() + server = MemoryWormholeServer() + options = runner.Options() + options.wormhole = server + reactor = object() - with mock.patch('allmydata.scripts.create_node.wormhole') as w: - fake_wh = _create_fake_wormhole([ - json.dumps({u"abilities": {u"server-v1": {}}}), - json.dumps({ - u"shares-needed": 1, - u"shares-happy": 1, - u"shares-total": 1, - u"nickname": u"somethinghopefullyunique", - u"introducer": u"pb://foo", - }), - ]) - w.create = mock.Mock(return_value=fake_wh) + wormhole = server.create( + "tahoe-lafs.org/invite", + "ws://wormhole.tahoe-lafs.org:4000/v1", + reactor, + ) + code = yield wormhole.get_code() + messages = [ + {u"abilities": {u"server-v1": {}}}, + { + u"shares-needed": 1, + u"shares-happy": 1, + u"shares-total": 1, + u"nickname": u"somethinghopefullyunique", + u"introducer": u"pb://foo", + }, + ] + for msg in messages: + wormhole.send_message(dumps_bytes(msg)) - rc, out, err = yield run_cli( - "create-client", - "--join", "1-abysmal-ant", - node_dir, - ) + rc, out, err = yield run_cli( + "create-client", + "--join", code, + node_dir, + options=options, + ) - self.assertEqual(0, rc) + self.assertEqual(0, rc) - config = read_config(node_dir, u"") - self.assertIn( - "pb://foo", - set( - furl - for (furl, cache) - in config.get_introducer_configuration().values() - ), - ) + config = read_config(node_dir, u"") + self.assertIn( + "pb://foo", + set( + furl + for (furl, cache) + in config.get_introducer_configuration().values() + ), + ) - with open(join(node_dir, 'tahoe.cfg'), 'r') as f: - config = f.read() - self.assertIn(u"somethinghopefullyunique", config) + with open(join(node_dir, 'tahoe.cfg'), 'r') as f: + config = f.read() + self.assertIn(u"somethinghopefullyunique", config) @defer.inlineCallbacks def test_create_node_illegal_option(self): @@ -116,30 +86,41 @@ class Join(GridTestMixin, CLITestMixin, unittest.TestCase): Server sends JSON with unknown/illegal key """ node_dir = self.mktemp() + server = MemoryWormholeServer() + options = runner.Options() + options.wormhole = server + reactor = object() - with mock.patch('allmydata.scripts.create_node.wormhole') as w: - fake_wh = _create_fake_wormhole([ - json.dumps({u"abilities": {u"server-v1": {}}}), - json.dumps({ - u"shares-needed": 1, - u"shares-happy": 1, - u"shares-total": 1, - u"nickname": u"somethinghopefullyunique", - u"introducer": u"pb://foo", - u"something-else": u"not allowed", - }), - ]) - w.create = mock.Mock(return_value=fake_wh) + wormhole = server.create( + "tahoe-lafs.org/invite", + "ws://wormhole.tahoe-lafs.org:4000/v1", + reactor, + ) + code = yield wormhole.get_code() + messages = [ + {u"abilities": {u"server-v1": {}}}, + { + u"shares-needed": 1, + u"shares-happy": 1, + u"shares-total": 1, + u"nickname": u"somethinghopefullyunique", + u"introducer": u"pb://foo", + u"something-else": u"not allowed", + }, + ] + for msg in messages: + wormhole.send_message(dumps_bytes(msg)) - rc, out, err = yield run_cli( - "create-client", - "--join", "1-abysmal-ant", - node_dir, - ) + rc, out, err = yield run_cli( + "create-client", + "--join", code, + node_dir, + options=options, + ) - # should still succeed -- just ignores the not-whitelisted - # "something-else" option - self.assertEqual(0, rc) + # should still succeed -- just ignores the not-whitelisted + # "something-else" option + self.assertEqual(0, rc) class Invite(GridTestMixin, CLITestMixin, unittest.TestCase): @@ -156,7 +137,7 @@ class Invite(GridTestMixin, CLITestMixin, unittest.TestCase): intro_dir, ) - def _invite_success(self, extra_args=(), tahoe_config=None): + async def _invite_success(self, extra_args=(), tahoe_config=None): # type: (Sequence[bytes], Optional[bytes]) -> defer.Deferred """ Exercise an expected-success case of ``tahoe invite``. @@ -178,53 +159,82 @@ class Invite(GridTestMixin, CLITestMixin, unittest.TestCase): with open(join(intro_dir, "tahoe.cfg"), "wb") as fobj_cfg: fobj_cfg.write(tahoe_config) - with mock.patch('allmydata.scripts.tahoe_invite.wormhole') as w: - fake_wh = _create_fake_wormhole([ - json.dumps({u"abilities": {u"client-v1": {}}}), - ]) - w.create = mock.Mock(return_value=fake_wh) + wormhole_server, helper = memory_server() + options = runner.Options() + options.wormhole = wormhole_server + reactor = object() - extra_args = tuple(extra_args) - - d = run_cli( + async def server(): + # Run the server side of the invitation process using the CLI. + rc, out, err = await run_cli( "-d", intro_dir, "invite", - *(extra_args + ("foo",)) + *tuple(extra_args) + ("foo",), + options=options, ) - def done(result): - rc, out, err = result - self.assertEqual(2, len(fake_wh.messages)) - self.assertEqual( - json.loads(fake_wh.messages[0]), + async def client(): + # Run the client side of the invitation by manually pumping a + # message through the wormhole. + + # First, wait for the server to create the wormhole at all. + wormhole = await helper.wait_for_wormhole( + "tahoe-lafs.org/invite", + "ws://wormhole.tahoe-lafs.org:4000/v1", + ) + # Then read out its code and open the other side of the wormhole. + code = await wormhole.when_code() + other_end = wormhole_server.create( + "tahoe-lafs.org/invite", + "ws://wormhole.tahoe-lafs.org:4000/v1", + reactor, + ) + other_end.set_code(code) + + # Send a proper client abilities message. + other_end.send_message(dumps_bytes({u"abilities": {u"client-v1": {}}})) + + # Check the server's messages. First, it should announce its + # abilities correctly. + server_abilities = json.loads(await other_end.when_received()) + self.assertEqual( + server_abilities, + { + "abilities": { - "abilities": - { - "server-v1": {} - }, + "server-v1": {} }, - ) - invite = json.loads(fake_wh.messages[1]) - self.assertEqual( - invite["nickname"], "foo", - ) - self.assertEqual( - invite["introducer"], "pb://fooblam", - ) - return invite - d.addCallback(done) - return d + }, + ) + + # Second, it should have an invitation with a nickname and + # introducer furl. + invite = json.loads(await other_end.when_received()) + self.assertEqual( + invite["nickname"], "foo", + ) + self.assertEqual( + invite["introducer"], "pb://fooblam", + ) + return invite + + invite, _ = await defer.gatherResults(map( + defer.Deferred.fromCoroutine, + [client(), server()], + )) + return invite + @defer.inlineCallbacks def test_invite_success(self): """ successfully send an invite """ - invite = yield self._invite_success(( + invite = yield defer.Deferred.fromCoroutine(self._invite_success(( "--shares-needed", "1", "--shares-happy", "2", "--shares-total", "3", - )) + ))) self.assertEqual( invite["shares-needed"], "1", ) @@ -241,12 +251,12 @@ class Invite(GridTestMixin, CLITestMixin, unittest.TestCase): If ``--shares-{needed,happy,total}`` are not given on the command line then the invitation is generated using the configured values. """ - invite = yield self._invite_success(tahoe_config=b""" + invite = yield defer.Deferred.fromCoroutine(self._invite_success(tahoe_config=b""" [client] shares.needed = 2 shares.happy = 4 shares.total = 6 -""") +""")) self.assertEqual( invite["shares-needed"], "2", ) @@ -265,22 +275,20 @@ shares.total = 6 """ intro_dir = os.path.join(self.basedir, "introducer") - with mock.patch('allmydata.scripts.tahoe_invite.wormhole') as w: - fake_wh = _create_fake_wormhole([ - json.dumps({u"abilities": {u"client-v1": {}}}), - ]) - w.create = mock.Mock(return_value=fake_wh) + options = runner.Options() + options.wormhole = None - rc, out, err = yield run_cli( - "-d", intro_dir, - "invite", - "--shares-needed", "1", - "--shares-happy", "1", - "--shares-total", "1", - "foo", - ) - self.assertNotEqual(rc, 0) - self.assertIn(u"Can't find introducer FURL", out + err) + rc, out, err = yield run_cli( + "-d", intro_dir, + "invite", + "--shares-needed", "1", + "--shares-happy", "1", + "--shares-total", "1", + "foo", + options=options, + ) + self.assertNotEqual(rc, 0) + self.assertIn(u"Can't find introducer FURL", out + err) @defer.inlineCallbacks def test_invite_wrong_client_abilities(self): @@ -294,23 +302,51 @@ shares.total = 6 with open(join(priv_dir, "introducer.furl"), "w") as f: f.write("pb://fooblam\n") - with mock.patch('allmydata.scripts.tahoe_invite.wormhole') as w: - fake_wh = _create_fake_wormhole([ - json.dumps({u"abilities": {u"client-v9000": {}}}), - ]) - w.create = mock.Mock(return_value=fake_wh) + wormhole_server, helper = memory_server() + options = runner.Options() + options.wormhole = wormhole_server + reactor = object() - rc, out, err = yield run_cli( + async def server(): + rc, out, err = await run_cli( "-d", intro_dir, "invite", "--shares-needed", "1", "--shares-happy", "1", "--shares-total", "1", "foo", + options=options, ) self.assertNotEqual(rc, 0) self.assertIn(u"No 'client-v1' in abilities", out + err) + async def client(): + # Run the client side of the invitation by manually pumping a + # message through the wormhole. + + # First, wait for the server to create the wormhole at all. + wormhole = await helper.wait_for_wormhole( + "tahoe-lafs.org/invite", + "ws://wormhole.tahoe-lafs.org:4000/v1", + ) + # Then read out its code and open the other side of the wormhole. + code = await wormhole.when_code() + other_end = wormhole_server.create( + "tahoe-lafs.org/invite", + "ws://wormhole.tahoe-lafs.org:4000/v1", + reactor, + ) + other_end.set_code(code) + + # Send some surprising client abilities. + other_end.send_message(dumps_bytes({u"abilities": {u"client-v9000": {}}})) + + yield defer.gatherResults(map( + defer.Deferred.fromCoroutine, + [client(), server()], + )) + + @defer.inlineCallbacks def test_invite_no_client_abilities(self): """ @@ -323,23 +359,52 @@ shares.total = 6 with open(join(priv_dir, "introducer.furl"), "w") as f: f.write("pb://fooblam\n") - with mock.patch('allmydata.scripts.tahoe_invite.wormhole') as w: - fake_wh = _create_fake_wormhole([ - json.dumps({}), - ]) - w.create = mock.Mock(return_value=fake_wh) + wormhole_server, helper = memory_server() + options = runner.Options() + options.wormhole = wormhole_server + reactor = object() - rc, out, err = yield run_cli( + async def server(): + # Run the server side of the invitation process using the CLI. + rc, out, err = await run_cli( "-d", intro_dir, "invite", "--shares-needed", "1", "--shares-happy", "1", "--shares-total", "1", "foo", + options=options, ) self.assertNotEqual(rc, 0) self.assertIn(u"No 'abilities' from client", out + err) + async def client(): + # Run the client side of the invitation by manually pumping a + # message through the wormhole. + + # First, wait for the server to create the wormhole at all. + wormhole = await helper.wait_for_wormhole( + "tahoe-lafs.org/invite", + "ws://wormhole.tahoe-lafs.org:4000/v1", + ) + # Then read out its code and open the other side of the wormhole. + code = await wormhole.when_code() + other_end = wormhole_server.create( + "tahoe-lafs.org/invite", + "ws://wormhole.tahoe-lafs.org:4000/v1", + reactor, + ) + other_end.set_code(code) + + # Send a no-abilities message through to the server. + other_end.send_message(dumps_bytes({})) + + yield defer.gatherResults(map( + defer.Deferred.fromCoroutine, + [client(), server()], + )) + + @defer.inlineCallbacks def test_invite_wrong_server_abilities(self): """ @@ -352,26 +417,38 @@ shares.total = 6 with open(join(priv_dir, "introducer.furl"), "w") as f: f.write("pb://fooblam\n") - with mock.patch('allmydata.scripts.create_node.wormhole') as w: - fake_wh = _create_fake_wormhole([ - json.dumps({u"abilities": {u"server-v9000": {}}}), - json.dumps({ - "shares-needed": "1", - "shares-total": "1", - "shares-happy": "1", - "nickname": "foo", - "introducer": "pb://fooblam", - }), - ]) - w.create = mock.Mock(return_value=fake_wh) + wormhole_server = MemoryWormholeServer() + options = runner.Options() + options.wormhole = wormhole_server + reactor = object() - rc, out, err = yield run_cli( - "create-client", - "--join", "1-alarmist-tuba", - "foo", - ) - self.assertNotEqual(rc, 0) - self.assertIn("Expected 'server-v1' in server abilities", out + err) + wormhole = wormhole_server.create( + "tahoe-lafs.org/invite", + "ws://wormhole.tahoe-lafs.org:4000/v1", + reactor, + ) + code = yield wormhole.get_code() + messages = [ + {u"abilities": {u"server-v9000": {}}}, + { + "shares-needed": "1", + "shares-total": "1", + "shares-happy": "1", + "nickname": "foo", + "introducer": "pb://fooblam", + }, + ] + for msg in messages: + wormhole.send_message(dumps_bytes(msg)) + + rc, out, err = yield run_cli( + "create-client", + "--join", code, + "foo", + options=options, + ) + self.assertNotEqual(rc, 0) + self.assertIn("Expected 'server-v1' in server abilities", out + err) @defer.inlineCallbacks def test_invite_no_server_abilities(self): @@ -385,26 +462,38 @@ shares.total = 6 with open(join(priv_dir, "introducer.furl"), "w") as f: f.write("pb://fooblam\n") - with mock.patch('allmydata.scripts.create_node.wormhole') as w: - fake_wh = _create_fake_wormhole([ - json.dumps({}), - json.dumps({ - "shares-needed": "1", - "shares-total": "1", - "shares-happy": "1", - "nickname": "bar", - "introducer": "pb://fooblam", - }), - ]) - w.create = mock.Mock(return_value=fake_wh) + server = MemoryWormholeServer() + options = runner.Options() + options.wormhole = server + reactor = object() - rc, out, err = yield run_cli( - "create-client", - "--join", "1-alarmist-tuba", - "bar", - ) - self.assertNotEqual(rc, 0) - self.assertIn("Expected 'abilities' in server introduction", out + err) + wormhole = server.create( + "tahoe-lafs.org/invite", + "ws://wormhole.tahoe-lafs.org:4000/v1", + reactor, + ) + code = yield wormhole.get_code() + messages = [ + {}, + { + "shares-needed": "1", + "shares-total": "1", + "shares-happy": "1", + "nickname": "bar", + "introducer": "pb://fooblam", + }, + ] + for msg in messages: + wormhole.send_message(dumps_bytes(msg)) + + rc, out, err = yield run_cli( + "create-client", + "--join", code, + "bar", + options=options, + ) + self.assertNotEqual(rc, 0) + self.assertIn("Expected 'abilities' in server introduction", out + err) @defer.inlineCallbacks def test_invite_no_nick(self): @@ -413,13 +502,16 @@ shares.total = 6 """ intro_dir = os.path.join(self.basedir, "introducer") - with mock.patch('allmydata.scripts.tahoe_invite.wormhole'): - rc, out, err = yield run_cli( - "-d", intro_dir, - "invite", - "--shares-needed", "1", - "--shares-happy", "1", - "--shares-total", "1", - ) - self.assertTrue(rc) - self.assertIn(u"Provide a single argument", out + err) + options = runner.Options() + options.wormhole = None + + rc, out, err = yield run_cli( + "-d", intro_dir, + "invite", + "--shares-needed", "1", + "--shares-happy", "1", + "--shares-total", "1", + options=options, + ) + self.assertTrue(rc) + self.assertIn(u"Provide a single argument", out + err) diff --git a/src/allmydata/test/cli/wormholetesting.py b/src/allmydata/test/cli/wormholetesting.py new file mode 100644 index 000000000..b60980bff --- /dev/null +++ b/src/allmydata/test/cli/wormholetesting.py @@ -0,0 +1,304 @@ +""" +An in-memory implementation of some of the magic-wormhole interfaces for +use by automated tests. + +For example:: + + async def peerA(mw): + wormhole = mw.create("myapp", "wss://myserver", reactor) + code = await wormhole.get_code() + print(f"I have a code: {code}") + message = await wormhole.when_received() + print(f"I have a message: {message}") + + async def local_peerB(helper, mw): + peerA_wormhole = await helper.wait_for_wormhole("myapp", "wss://myserver") + code = await peerA_wormhole.when_code() + + peerB_wormhole = mw.create("myapp", "wss://myserver") + peerB_wormhole.set_code(code) + + peerB_wormhole.send_message("Hello, peer A") + + # Run peerA against local_peerB with pure in-memory message passing. + server, helper = memory_server() + run(gather(peerA(server), local_peerB(helper, server))) + + # Run peerA against a peerB somewhere out in the world, using a real + # wormhole relay server somewhere. + import wormhole + run(peerA(wormhole)) +""" + +from __future__ import annotations + +from typing import Iterator +from collections.abc import Awaitable +from inspect import getargspec +from itertools import count +from sys import stderr + +from attrs import frozen, define, field, Factory +from twisted.internet.defer import Deferred, DeferredQueue, succeed +from wormhole._interfaces import IWormhole +from wormhole.wormhole import create +from zope.interface import implementer + + +@define +class MemoryWormholeServer(object): + """ + A factory for in-memory wormholes. + + :ivar _apps: Wormhole state arranged by the application id and relay URL + it belongs to. + + :ivar _waiters: Observers waiting for a wormhole to be created for a + specific application id and relay URL combination. + """ + _apps: dict[tuple[str, str], _WormholeApp] = field(default=Factory(dict)) + _waiters: dict[tuple[str, str], Deferred] = field(default=Factory(dict)) + + def create( + self, + appid, + relay_url, + reactor, + versions={}, + delegate=None, + journal=None, + tor=None, + timing=None, + stderr=stderr, + _eventual_queue=None, + _enable_dilate=False, + ): + """ + Create a wormhole. It will be able to connect to other wormholes created + by this instance (and constrained by the normal appid/relay_url + rules). + """ + if tor is not None: + raise ValueError("Cannot deal with Tor right now.") + if _enable_dilate: + raise ValueError("Cannot deal with dilation right now.") + + key = (relay_url, appid) + wormhole = _MemoryWormhole(self._view(key)) + if key in self._waiters: + self._waiters.pop(key).callback(wormhole) + return wormhole + + def _view(self, key: tuple[str, str]) -> _WormholeServerView: + """ + Created a view onto this server's state that is limited by a certain + appid/relay_url pair. + """ + return _WormholeServerView(self, key) + + +@frozen +class TestingHelper(object): + """ + Provide extra functionality for interacting with an in-memory wormhole + implementation. + + This is intentionally a separate API so that it is not confused with + proper public interface of the real wormhole implementation. + """ + _server: MemoryWormholeServer + + async def wait_for_wormhole(self, appid: str, relay_url: str) -> IWormhole: + """ + Wait for a wormhole to appear at a specific location. + + :param appid: The appid that the resulting wormhole will have. + + :param relay_url: The URL of the relay at which the resulting wormhole + will presume to be created. + + :return: The first wormhole to be created which matches the given + parameters. + """ + key = relay_url, appid + if key in self._server._waiters: + raise ValueError(f"There is already a waiter for {key}") + d = Deferred() + self._server._waiters[key] = d + wormhole = await d + return wormhole + + +def _verify(): + """ + Roughly confirm that the in-memory wormhole creation function matches the + interface of the real implementation. + """ + # Poor man's interface verification. + + a = getargspec(create) + b = getargspec(MemoryWormholeServer.create) + # I know it has a `self` argument at the beginning. That's okay. + b = b._replace(args=b.args[1:]) + assert a == b, "{} != {}".format(a, b) + + +_verify() + + +@define +class _WormholeApp(object): + """ + Represent a collection of wormholes that belong to the same + appid/relay_url scope. + """ + wormholes: dict = field(default=Factory(dict)) + _waiting: dict = field(default=Factory(dict)) + _counter: Iterator[int] = field(default=Factory(count)) + + def allocate_code(self, wormhole, code): + """ + Allocate a new code for the given wormhole. + + This also associates the given wormhole with the code for future + lookup. + + Code generation logic is trivial and certainly not good enough for any + real use. It is sufficient for automated testing, though. + """ + if code is None: + code = "{}-persnickety-tardigrade".format(next(self._counter)) + self.wormholes.setdefault(code, []).append(wormhole) + try: + waiters = self._waiting.pop(code) + except KeyError: + pass + else: + for w in waiters: + w.callback(wormhole) + + return code + + def wait_for_wormhole(self, code: str) -> Awaitable[_MemoryWormhole]: + """ + Return a ``Deferred`` which fires with the next wormhole to be associated + with the given code. This is used to let the first end of a wormhole + rendezvous with the second end. + """ + d = Deferred() + self._waiting.setdefault(code, []).append(d) + return d + + +@frozen +class _WormholeServerView(object): + """ + Present an interface onto the server to be consumed by individual + wormholes. + """ + _server: MemoryWormholeServer + _key: tuple[str, str] + + def allocate_code(self, wormhole: _MemoryWormhole, code: str) -> str: + """ + Allocate a new code for the given wormhole in the scope associated with + this view. + """ + app = self._server._apps.setdefault(self._key, _WormholeApp()) + return app.allocate_code(wormhole, code) + + def wormhole_by_code(self, code, exclude): + """ + Retrieve all wormholes previously associated with a code. + """ + app = self._server._apps[self._key] + wormholes = app.wormholes[code] + try: + [wormhole] = list(wormhole for wormhole in wormholes if wormhole != exclude) + except ValueError: + return app.wait_for_wormhole(code) + return succeed(wormhole) + + +@implementer(IWormhole) +@define +class _MemoryWormhole(object): + """ + Represent one side of a wormhole as conceived by ``MemoryWormholeServer``. + """ + + _view: _WormholeServerView + _code: str = None + _payload: DeferredQueue = field(default=Factory(DeferredQueue)) + _waiting_for_code: list[Deferred] = field(default=Factory(list)) + _allocated: bool = False + + def allocate_code(self): + if self._code is not None: + raise ValueError( + "allocate_code used with a wormhole which already has a code" + ) + self._allocated = True + self._code = self._view.allocate_code(self, None) + waiters = self._waiting_for_code + self._waiting_for_code = None + for d in waiters: + d.callback(self._code) + + def set_code(self, code): + if self._code is None: + self._code = code + self._view.allocate_code(self, code) + else: + raise ValueError("set_code used with a wormhole which already has a code") + + def when_code(self): + if self._code is None: + d = Deferred() + self._waiting_for_code.append(d) + return d + return succeed(self._code) + + get_code = when_code + + def get_welcome(self): + return succeed("welcome") + + def send_message(self, payload): + self._payload.put(payload) + + def when_received(self): + if self._code is None: + raise ValueError( + "This implementation requires set_code or allocate_code " + "before when_received." + ) + d = self._view.wormhole_by_code(self._code, exclude=self) + + def got_wormhole(wormhole): + msg = wormhole._payload.get() + return msg + + d.addCallback(got_wormhole) + return d + + get_message = when_received + + def close(self): + pass + + # 0.9.2 compatibility + def get_code(self): + if self._code is None: + self.allocate_code() + return self.when_code() + + get = when_received + + +def memory_server() -> tuple[MemoryWormholeServer, TestingHelper]: + """ + Create a paired in-memory wormhole server and testing helper. + """ + server = MemoryWormholeServer() + return server, TestingHelper(server) From e35bab966351a7a24238430f25efb3dd24cdd89c Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 12 Apr 2022 11:01:35 -0400 Subject: [PATCH 0811/2309] news fragment --- newsfragments/3526.minor | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/3526.minor diff --git a/newsfragments/3526.minor b/newsfragments/3526.minor new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/newsfragments/3526.minor @@ -0,0 +1 @@ + From 1634f137be90eed8e7da7ce3964d45b7f0cc0651 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 12 Apr 2022 12:54:16 -0400 Subject: [PATCH 0812/2309] Use sets more widely in the schema. --- docs/proposed/http-storage-node-protocol.rst | 3 +++ src/allmydata/storage/http_client.py | 7 ++++--- src/allmydata/storage/http_server.py | 2 +- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index 2ceb3c03a..be543abb2 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -350,6 +350,9 @@ Because of the simple types used throughout and the equivalence described in `RFC 7049`_ these examples should be representative regardless of which of these two encodings is chosen. +For CBOR messages, any sequence that is semantically a set (i.e. no repeated values allowed, order doesn't matter, and is hashable in Python) should be sent as a set. +Tag 6.258 is used to indicate sets in CBOR. + HTTP Design ~~~~~~~~~~~ diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 3a758e592..e735a6369 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -39,8 +39,9 @@ class ClientException(Exception): # Schemas for server responses. # -# TODO usage of sets is inconsistent. Either use everywhere (and document in -# spec document) or use nowhere. +# Tags are of the form #6.nnn, where the number is documented at +# https://www.iana.org/assignments/cbor-tags/cbor-tags.xhtml. Notably, #6.258 +# indicates a set. _SCHEMAS = { "get_version": Schema( """ @@ -74,7 +75,7 @@ _SCHEMAS = { ), "list_shares": Schema( """ - message = [* uint] + message = #6.258([* uint]) """ ), } diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 3876409b0..54e60f913 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -436,7 +436,7 @@ class HTTPServer(object): """ List shares for the given storage index. """ - share_numbers = list(self._storage_server.get_buckets(storage_index).keys()) + share_numbers = set(self._storage_server.get_buckets(storage_index).keys()) return self._send_encoded(request, share_numbers) @_authorized_route( From b0fffabed0aa525610714864ed74f3dd6c7445e8 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 12 Apr 2022 14:10:02 -0400 Subject: [PATCH 0813/2309] remove unnecessary module-scope wormhole used this during testing so the other mock() calls wouldn't explode in a boring way --- src/allmydata/scripts/tahoe_invite.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/allmydata/scripts/tahoe_invite.py b/src/allmydata/scripts/tahoe_invite.py index c5f08f588..b62d6a463 100644 --- a/src/allmydata/scripts/tahoe_invite.py +++ b/src/allmydata/scripts/tahoe_invite.py @@ -42,8 +42,6 @@ class InviteOptions(usage.Options): self['nick'] = args[0].strip() -wormhole = None - @defer.inlineCallbacks def _send_config_via_wormhole(options, config): out = options.stdout From 71b5cd9e0d64643a907d365c8d9580384cb92d4b Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 12 Apr 2022 14:13:48 -0400 Subject: [PATCH 0814/2309] rewrite comment annotations with syntax --- src/allmydata/test/cli/test_invite.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/allmydata/test/cli/test_invite.py b/src/allmydata/test/cli/test_invite.py index c4bb6fd7e..50f446ae2 100644 --- a/src/allmydata/test/cli/test_invite.py +++ b/src/allmydata/test/cli/test_invite.py @@ -137,8 +137,7 @@ class Invite(GridTestMixin, CLITestMixin, unittest.TestCase): intro_dir, ) - async def _invite_success(self, extra_args=(), tahoe_config=None): - # type: (Sequence[bytes], Optional[bytes]) -> defer.Deferred + async def _invite_success(self, extra_args: Sequence[bytes] = (), tahoe_config: Optional[byte] = None) -> str: """ Exercise an expected-success case of ``tahoe invite``. From 0f61a1dab9e72152e938ea3e5279ed1a1ac65d9f Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 12 Apr 2022 14:33:11 -0400 Subject: [PATCH 0815/2309] Factor some duplication out of the test methods --- src/allmydata/test/cli/test_invite.py | 149 ++++++++++++-------------- 1 file changed, 70 insertions(+), 79 deletions(-) diff --git a/src/allmydata/test/cli/test_invite.py b/src/allmydata/test/cli/test_invite.py index 50f446ae2..5b4871944 100644 --- a/src/allmydata/test/cli/test_invite.py +++ b/src/allmydata/test/cli/test_invite.py @@ -4,8 +4,9 @@ Tests for ``tahoe invite``. import json import os +from functools import partial from os.path import join -from typing import Optional, Sequence +from typing import Awaitable, Callable, Optional, Sequence, TypeVar from twisted.internet import defer from twisted.trial import unittest @@ -16,7 +17,58 @@ from ...util.jsonbytes import dumps_bytes from ..common_util import run_cli from ..no_network import GridTestMixin from .common import CLITestMixin -from .wormholetesting import MemoryWormholeServer, memory_server +from .wormholetesting import IWormhole, MemoryWormholeServer, memory_server + + +async def open_wormhole() -> tuple[Callable, IWormhole, str]: + """ + Create a new in-memory wormhole server, open one end of a wormhole, and + return it and related info. + + :return: A three-tuple allowing use of the wormhole. The first element is + a callable like ``run_cli`` but which will run commands so that they + use the in-memory wormhole server instead of a real one. The second + element is the open wormhole. The third element is the wormhole's + code. + """ + server = MemoryWormholeServer() + options = runner.Options() + options.wormhole = server + reactor = object() + + wormhole = server.create( + "tahoe-lafs.org/invite", + "ws://wormhole.tahoe-lafs.org:4000/v1", + reactor, + ) + code = await wormhole.get_code() + + return (partial(run_cli, options=options), wormhole, code) + + +def send_messages(wormhole: IWormhole, messages: list[dict]) -> None: + """ + Send a list of message through a wormhole. + """ + for msg in messages: + wormhole.send_message(dumps_bytes(msg)) + + +A = TypeVar("A") +B = TypeVar("B") + +def concurrently( + client: Callable[[], Awaitable[A]], + server: Callable[[], Awaitable[B]], +) -> defer.Deferred[tuple[A, B]]: + """ + Run two asynchronous functions concurrently and asynchronously return a + tuple of both their results. + """ + return defer.gatherResults([ + defer.Deferred.fromCoroutine(client()), + defer.Deferred.fromCoroutine(server()), + ]) class Join(GridTestMixin, CLITestMixin, unittest.TestCase): @@ -33,18 +85,8 @@ class Join(GridTestMixin, CLITestMixin, unittest.TestCase): successfully join after an invite """ node_dir = self.mktemp() - server = MemoryWormholeServer() - options = runner.Options() - options.wormhole = server - reactor = object() - - wormhole = server.create( - "tahoe-lafs.org/invite", - "ws://wormhole.tahoe-lafs.org:4000/v1", - reactor, - ) - code = yield wormhole.get_code() - messages = [ + run_cli, wormhole, code = yield defer.Deferred.fromCoroutine(open_wormhole()) + send_messages(wormhole, [ {u"abilities": {u"server-v1": {}}}, { u"shares-needed": 1, @@ -53,15 +95,12 @@ class Join(GridTestMixin, CLITestMixin, unittest.TestCase): u"nickname": u"somethinghopefullyunique", u"introducer": u"pb://foo", }, - ] - for msg in messages: - wormhole.send_message(dumps_bytes(msg)) + ]) rc, out, err = yield run_cli( "create-client", "--join", code, node_dir, - options=options, ) self.assertEqual(0, rc) @@ -86,18 +125,8 @@ class Join(GridTestMixin, CLITestMixin, unittest.TestCase): Server sends JSON with unknown/illegal key """ node_dir = self.mktemp() - server = MemoryWormholeServer() - options = runner.Options() - options.wormhole = server - reactor = object() - - wormhole = server.create( - "tahoe-lafs.org/invite", - "ws://wormhole.tahoe-lafs.org:4000/v1", - reactor, - ) - code = yield wormhole.get_code() - messages = [ + run_cli, wormhole, code = yield defer.Deferred.fromCoroutine(open_wormhole()) + send_messages(wormhole, [ {u"abilities": {u"server-v1": {}}}, { u"shares-needed": 1, @@ -107,15 +136,12 @@ class Join(GridTestMixin, CLITestMixin, unittest.TestCase): u"introducer": u"pb://foo", u"something-else": u"not allowed", }, - ] - for msg in messages: - wormhole.send_message(dumps_bytes(msg)) + ]) rc, out, err = yield run_cli( "create-client", "--join", code, node_dir, - options=options, ) # should still succeed -- just ignores the not-whitelisted @@ -137,7 +163,7 @@ class Invite(GridTestMixin, CLITestMixin, unittest.TestCase): intro_dir, ) - async def _invite_success(self, extra_args: Sequence[bytes] = (), tahoe_config: Optional[byte] = None) -> str: + async def _invite_success(self, extra_args: Sequence[bytes] = (), tahoe_config: Optional[bytes] = None) -> str: """ Exercise an expected-success case of ``tahoe invite``. @@ -217,10 +243,7 @@ class Invite(GridTestMixin, CLITestMixin, unittest.TestCase): ) return invite - invite, _ = await defer.gatherResults(map( - defer.Deferred.fromCoroutine, - [client(), server()], - )) + invite, _ = await concurrently(client, server) return invite @@ -340,10 +363,7 @@ shares.total = 6 # Send some surprising client abilities. other_end.send_message(dumps_bytes({u"abilities": {u"client-v9000": {}}})) - yield defer.gatherResults(map( - defer.Deferred.fromCoroutine, - [client(), server()], - )) + yield concurrently(client, server) @defer.inlineCallbacks @@ -398,10 +418,7 @@ shares.total = 6 # Send a no-abilities message through to the server. other_end.send_message(dumps_bytes({})) - yield defer.gatherResults(map( - defer.Deferred.fromCoroutine, - [client(), server()], - )) + yield concurrently(client, server) @defer.inlineCallbacks @@ -416,18 +433,8 @@ shares.total = 6 with open(join(priv_dir, "introducer.furl"), "w") as f: f.write("pb://fooblam\n") - wormhole_server = MemoryWormholeServer() - options = runner.Options() - options.wormhole = wormhole_server - reactor = object() - - wormhole = wormhole_server.create( - "tahoe-lafs.org/invite", - "ws://wormhole.tahoe-lafs.org:4000/v1", - reactor, - ) - code = yield wormhole.get_code() - messages = [ + run_cli, wormhole, code = yield defer.Deferred.fromCoroutine(open_wormhole()) + send_messages(wormhole, [ {u"abilities": {u"server-v9000": {}}}, { "shares-needed": "1", @@ -436,15 +443,12 @@ shares.total = 6 "nickname": "foo", "introducer": "pb://fooblam", }, - ] - for msg in messages: - wormhole.send_message(dumps_bytes(msg)) + ]) rc, out, err = yield run_cli( "create-client", "--join", code, "foo", - options=options, ) self.assertNotEqual(rc, 0) self.assertIn("Expected 'server-v1' in server abilities", out + err) @@ -461,18 +465,8 @@ shares.total = 6 with open(join(priv_dir, "introducer.furl"), "w") as f: f.write("pb://fooblam\n") - server = MemoryWormholeServer() - options = runner.Options() - options.wormhole = server - reactor = object() - - wormhole = server.create( - "tahoe-lafs.org/invite", - "ws://wormhole.tahoe-lafs.org:4000/v1", - reactor, - ) - code = yield wormhole.get_code() - messages = [ + run_cli, wormhole, code = yield defer.Deferred.fromCoroutine(open_wormhole()) + send_messages(wormhole, [ {}, { "shares-needed": "1", @@ -481,15 +475,12 @@ shares.total = 6 "nickname": "bar", "introducer": "pb://fooblam", }, - ] - for msg in messages: - wormhole.send_message(dumps_bytes(msg)) + ]) rc, out, err = yield run_cli( "create-client", "--join", code, "bar", - options=options, ) self.assertNotEqual(rc, 0) self.assertIn("Expected 'abilities' in server introduction", out + err) From 2e8c51ac4e9736134d40dcb2d55d117836504041 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 13 Apr 2022 08:24:53 -0400 Subject: [PATCH 0816/2309] bump nixpkgs-21.11 and drop the special zfec handling the latest zfec release works fine without help --- default.nix | 14 -------------- nix/sources.json | 6 +++--- 2 files changed, 3 insertions(+), 17 deletions(-) diff --git a/default.nix b/default.nix index 095c54578..5f4db2c78 100644 --- a/default.nix +++ b/default.nix @@ -86,24 +86,10 @@ mach-nix.buildPythonPackage rec { # There are some reasonable defaults so we only need to specify certain # packages where the default configuration runs into some issue. providers = { - # Through zfec 1.5.5 the wheel has an incorrect runtime dependency - # declared on argparse, not available for recent versions of Python 3. - # Force mach-nix to use the sdist instead. This allows us to apply a - # patch that removes the offending declaration. - zfec = "sdist"; }; # Define certain overrides to the way Python dependencies are built. _ = { - # Apply the argparse declaration fix to zfec sdist. - zfec.patches = with pkgs; [ - (fetchpatch { - name = "fix-argparse.patch"; - url = "https://github.com/tahoe-lafs/zfec/commit/c3e736a72cccf44b8e1fb7d6c276400204c6bc1e.patch"; - sha256 = "1md9i2fx1ya7mgcj9j01z58hs3q9pj4ch5is5b5kq4v86cf6x33x"; - }) - ]; - # Remove a click-default-group patch for a test suite problem which no # longer applies because the project apparently no longer has a test suite # in its source distribution. diff --git a/nix/sources.json b/nix/sources.json index e0235a3fb..ced730ab7 100644 --- a/nix/sources.json +++ b/nix/sources.json @@ -41,10 +41,10 @@ "homepage": "", "owner": "NixOS", "repo": "nixpkgs", - "rev": "6c4b9f1a2fd761e2d384ef86cff0d208ca27fdca", - "sha256": "1yl5gj0mzczhl1j8sl8iqpwa1jzsgr12fdszw9rq13cdig2a2r5f", + "rev": "838eefb4f93f2306d4614aafb9b2375f315d917f", + "sha256": "1bm8cmh1wx4h8b4fhbs75hjci3gcrpi7k1m1pmiy3nc0gjim9vkg", "type": "tarball", - "url": "https://github.com/nixos/nixpkgs/archive/6c4b9f1a2fd761e2d384ef86cff0d208ca27fdca.tar.gz", + "url": "https://github.com/NixOS/nixpkgs/archive/838eefb4f93f2306d4614aafb9b2375f315d917f.tar.gz", "url_template": "https://github.com///archive/.tar.gz" }, "pypi-deps-db": { From cb91012f982a6a2602d5e51737f855ab4394b796 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 13 Apr 2022 08:25:11 -0400 Subject: [PATCH 0817/2309] bump the pypi db to a version including upcoming pycddl dependency --- nix/sources.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nix/sources.json b/nix/sources.json index ced730ab7..79eabe7a1 100644 --- a/nix/sources.json +++ b/nix/sources.json @@ -53,10 +53,10 @@ "homepage": "", "owner": "DavHau", "repo": "pypi-deps-db", - "rev": "0f6de8bf1f186c275af862ec9667abb95aae8542", - "sha256": "1ygw9pywyl4p25hx761d1sbwl3qjhm630fa36gdf6b649im4mx8y", + "rev": "76b8f1e44a8ec051b853494bcf3cc8453a294a6a", + "sha256": "18fgqyh4z578jjhk26n1xi2cw2l98vrqp962rgz9a6wa5yh1nm4x", "type": "tarball", - "url": "https://github.com/DavHau/pypi-deps-db/archive/0f6de8bf1f186c275af862ec9667abb95aae8542.tar.gz", + "url": "https://github.com/DavHau/pypi-deps-db/archive/76b8f1e44a8ec051b853494bcf3cc8453a294a6a.tar.gz", "url_template": "https://github.com///archive/.tar.gz" } } From 1c49b2375e6cd5f3d19e5d68b069f3e0bdbba135 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 13 Apr 2022 08:31:49 -0400 Subject: [PATCH 0818/2309] news fragment --- newsfragments/3889.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3889.minor diff --git a/newsfragments/3889.minor b/newsfragments/3889.minor new file mode 100644 index 000000000..e69de29bb From 7aab039a00154885a6cd554131797e19092060fa Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 13 Apr 2022 13:33:51 -0400 Subject: [PATCH 0819/2309] Improve docs. --- docs/proposed/http-storage-node-protocol.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index be543abb2..b57e9056a 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -350,8 +350,8 @@ Because of the simple types used throughout and the equivalence described in `RFC 7049`_ these examples should be representative regardless of which of these two encodings is chosen. -For CBOR messages, any sequence that is semantically a set (i.e. no repeated values allowed, order doesn't matter, and is hashable in Python) should be sent as a set. -Tag 6.258 is used to indicate sets in CBOR. +For CBOR messages, any sequence that is semantically a set (i.e. no repeated values allowed, order doesn't matter, and elements are hashable in Python) should be sent as a set. +Tag 6.258 is used to indicate sets in CBOR; see `the CBOR registry `_ for more details. HTTP Design ~~~~~~~~~~~ From 10f79ce8aa8ac07637ffcfc0af3a9003ff2736ba Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 13 Apr 2022 14:39:52 -0400 Subject: [PATCH 0820/2309] Use __future__.annotations in test_invite for generic builtins too --- src/allmydata/test/cli/test_invite.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/allmydata/test/cli/test_invite.py b/src/allmydata/test/cli/test_invite.py index 5b4871944..7d2250bac 100644 --- a/src/allmydata/test/cli/test_invite.py +++ b/src/allmydata/test/cli/test_invite.py @@ -2,6 +2,8 @@ Tests for ``tahoe invite``. """ +from __future__ import annotations + import json import os from functools import partial From ec5be01f38ffdca34930b2301a206d1de1ecb047 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 13 Apr 2022 14:50:38 -0400 Subject: [PATCH 0821/2309] more completely annotate types in the wormholetesting module --- src/allmydata/test/cli/wormholetesting.py | 51 ++++++++++++----------- 1 file changed, 26 insertions(+), 25 deletions(-) diff --git a/src/allmydata/test/cli/wormholetesting.py b/src/allmydata/test/cli/wormholetesting.py index b60980bff..715e82236 100644 --- a/src/allmydata/test/cli/wormholetesting.py +++ b/src/allmydata/test/cli/wormholetesting.py @@ -32,7 +32,7 @@ For example:: from __future__ import annotations -from typing import Iterator +from typing import Iterator, Optional, Sequence from collections.abc import Awaitable from inspect import getargspec from itertools import count @@ -44,6 +44,11 @@ from wormhole._interfaces import IWormhole from wormhole.wormhole import create from zope.interface import implementer +WormholeCode = str +WormholeMessage = bytes +AppId = str +RelayURL = str +ApplicationKey = tuple[RelayURL, AppId] @define class MemoryWormholeServer(object): @@ -56,8 +61,8 @@ class MemoryWormholeServer(object): :ivar _waiters: Observers waiting for a wormhole to be created for a specific application id and relay URL combination. """ - _apps: dict[tuple[str, str], _WormholeApp] = field(default=Factory(dict)) - _waiters: dict[tuple[str, str], Deferred] = field(default=Factory(dict)) + _apps: dict[ApplicationKey, _WormholeApp] = field(default=Factory(dict)) + _waiters: dict[ApplicationKey, Deferred] = field(default=Factory(dict)) def create( self, @@ -89,7 +94,7 @@ class MemoryWormholeServer(object): self._waiters.pop(key).callback(wormhole) return wormhole - def _view(self, key: tuple[str, str]) -> _WormholeServerView: + def _view(self, key: ApplicationKey) -> _WormholeServerView: """ Created a view onto this server's state that is limited by a certain appid/relay_url pair. @@ -108,7 +113,7 @@ class TestingHelper(object): """ _server: MemoryWormholeServer - async def wait_for_wormhole(self, appid: str, relay_url: str) -> IWormhole: + async def wait_for_wormhole(self, appid: AppId, relay_url: RelayURL) -> IWormhole: """ Wait for a wormhole to appear at a specific location. @@ -120,7 +125,7 @@ class TestingHelper(object): :return: The first wormhole to be created which matches the given parameters. """ - key = relay_url, appid + key = (relay_url, appid) if key in self._server._waiters: raise ValueError(f"There is already a waiter for {key}") d = Deferred() @@ -152,11 +157,11 @@ class _WormholeApp(object): Represent a collection of wormholes that belong to the same appid/relay_url scope. """ - wormholes: dict = field(default=Factory(dict)) - _waiting: dict = field(default=Factory(dict)) + wormholes: dict[WormholeCode, IWormhole] = field(default=Factory(dict)) + _waiting: dict[WormholeCode, Sequence[Deferred]] = field(default=Factory(dict)) _counter: Iterator[int] = field(default=Factory(count)) - def allocate_code(self, wormhole, code): + def allocate_code(self, wormhole: IWormhole, code: Optional[WormholeCode]) -> WormholeCode: """ Allocate a new code for the given wormhole. @@ -179,7 +184,7 @@ class _WormholeApp(object): return code - def wait_for_wormhole(self, code: str) -> Awaitable[_MemoryWormhole]: + def wait_for_wormhole(self, code: WormholeCode) -> Awaitable[_MemoryWormhole]: """ Return a ``Deferred`` which fires with the next wormhole to be associated with the given code. This is used to let the first end of a wormhole @@ -197,9 +202,9 @@ class _WormholeServerView(object): wormholes. """ _server: MemoryWormholeServer - _key: tuple[str, str] + _key: ApplicationKey - def allocate_code(self, wormhole: _MemoryWormhole, code: str) -> str: + def allocate_code(self, wormhole: _MemoryWormhole, code: Optional[WormholeCode]) -> WormholeCode: """ Allocate a new code for the given wormhole in the scope associated with this view. @@ -207,7 +212,7 @@ class _WormholeServerView(object): app = self._server._apps.setdefault(self._key, _WormholeApp()) return app.allocate_code(wormhole, code) - def wormhole_by_code(self, code, exclude): + def wormhole_by_code(self, code: WormholeCode, exclude: object) -> Deferred[IWormhole]: """ Retrieve all wormholes previously associated with a code. """ @@ -228,46 +233,42 @@ class _MemoryWormhole(object): """ _view: _WormholeServerView - _code: str = None + _code: Optional[WormholeCode] = None _payload: DeferredQueue = field(default=Factory(DeferredQueue)) _waiting_for_code: list[Deferred] = field(default=Factory(list)) - _allocated: bool = False - def allocate_code(self): + def allocate_code(self) -> None: if self._code is not None: raise ValueError( "allocate_code used with a wormhole which already has a code" ) - self._allocated = True self._code = self._view.allocate_code(self, None) waiters = self._waiting_for_code self._waiting_for_code = None for d in waiters: d.callback(self._code) - def set_code(self, code): + def set_code(self, code: WormholeCode) -> None: if self._code is None: self._code = code self._view.allocate_code(self, code) else: raise ValueError("set_code used with a wormhole which already has a code") - def when_code(self): + def when_code(self) -> Deferred[WormholeCode]: if self._code is None: d = Deferred() self._waiting_for_code.append(d) return d return succeed(self._code) - get_code = when_code - def get_welcome(self): return succeed("welcome") - def send_message(self, payload): + def send_message(self, payload: WormholeMessage) -> None: self._payload.put(payload) - def when_received(self): + def when_received(self) -> Deferred[WormholeMessage]: if self._code is None: raise ValueError( "This implementation requires set_code or allocate_code " @@ -284,11 +285,11 @@ class _MemoryWormhole(object): get_message = when_received - def close(self): + def close(self) -> None: pass # 0.9.2 compatibility - def get_code(self): + def get_code(self) -> Deferred[WormholeCode]: if self._code is None: self.allocate_code() return self.when_code() From 38e1e93a75356a9275f7fc6b23bd998bc55d012e Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 13 Apr 2022 15:42:10 -0400 Subject: [PATCH 0822/2309] factor the duplicate client logic out --- src/allmydata/test/cli/test_invite.py | 157 +++++++++++--------------- 1 file changed, 69 insertions(+), 88 deletions(-) diff --git a/src/allmydata/test/cli/test_invite.py b/src/allmydata/test/cli/test_invite.py index 7d2250bac..9f2607433 100644 --- a/src/allmydata/test/cli/test_invite.py +++ b/src/allmydata/test/cli/test_invite.py @@ -8,7 +8,7 @@ import json import os from functools import partial from os.path import join -from typing import Awaitable, Callable, Optional, Sequence, TypeVar +from typing import Awaitable, Callable, Optional, Sequence, TypeVar, Union from twisted.internet import defer from twisted.trial import unittest @@ -21,6 +21,12 @@ from ..no_network import GridTestMixin from .common import CLITestMixin from .wormholetesting import IWormhole, MemoryWormholeServer, memory_server +# Logically: +# JSONable = dict[str, Union[JSONable, None, int, float, str, list[JSONable]]] +# +# But practically: +JSONable = Union[dict, None, int, float, str, list] + async def open_wormhole() -> tuple[Callable, IWormhole, str]: """ @@ -48,7 +54,42 @@ async def open_wormhole() -> tuple[Callable, IWormhole, str]: return (partial(run_cli, options=options), wormhole, code) -def send_messages(wormhole: IWormhole, messages: list[dict]) -> None: +def make_simple_peer( + reactor, + server: MemoryWormholeServer, + helper: TestingHelper, + messages: Sequence[JSONable], +) -> Callable[[], Awaitable[IWormhole]]: + """ + Make a wormhole peer that just sends the given messages. + + The returned function returns an awaitable that fires with the peer's end + of the wormhole. + """ + async def peer() -> IWormhole: + # Run the client side of the invitation by manually pumping a + # message through the wormhole. + + # First, wait for the server to create the wormhole at all. + wormhole = await helper.wait_for_wormhole( + "tahoe-lafs.org/invite", + "ws://wormhole.tahoe-lafs.org:4000/v1", + ) + # Then read out its code and open the other side of the wormhole. + code = await wormhole.when_code() + other_end = server.create( + "tahoe-lafs.org/invite", + "ws://wormhole.tahoe-lafs.org:4000/v1", + reactor, + ) + other_end.set_code(code) + send_messages(other_end, messages) + return other_end + + return peer + + +def send_messages(wormhole: IWormhole, messages: Sequence[JSONable]) -> None: """ Send a list of message through a wormhole. """ @@ -200,55 +241,34 @@ class Invite(GridTestMixin, CLITestMixin, unittest.TestCase): options=options, ) - async def client(): - # Run the client side of the invitation by manually pumping a - # message through the wormhole. + # Send a proper client abilities message. + client = make_simple_peer(reactor, wormhole_server, helper, [{u"abilities": {u"client-v1": {}}}]) + other_end, _ = await concurrently(client, server) - # First, wait for the server to create the wormhole at all. - wormhole = await helper.wait_for_wormhole( - "tahoe-lafs.org/invite", - "ws://wormhole.tahoe-lafs.org:4000/v1", - ) - # Then read out its code and open the other side of the wormhole. - code = await wormhole.when_code() - other_end = wormhole_server.create( - "tahoe-lafs.org/invite", - "ws://wormhole.tahoe-lafs.org:4000/v1", - reactor, - ) - other_end.set_code(code) - - # Send a proper client abilities message. - other_end.send_message(dumps_bytes({u"abilities": {u"client-v1": {}}})) - - # Check the server's messages. First, it should announce its - # abilities correctly. - server_abilities = json.loads(await other_end.when_received()) - self.assertEqual( - server_abilities, + # Check the server's messages. First, it should announce its + # abilities correctly. + server_abilities = json.loads(await other_end.when_received()) + self.assertEqual( + server_abilities, + { + "abilities": { - "abilities": - { - "server-v1": {} - }, + "server-v1": {} }, - ) + }, + ) - # Second, it should have an invitation with a nickname and - # introducer furl. - invite = json.loads(await other_end.when_received()) - self.assertEqual( - invite["nickname"], "foo", - ) - self.assertEqual( - invite["introducer"], "pb://fooblam", - ) - return invite - - invite, _ = await concurrently(client, server) + # Second, it should have an invitation with a nickname and introducer + # furl. + invite = json.loads(await other_end.when_received()) + self.assertEqual( + invite["nickname"], "foo", + ) + self.assertEqual( + invite["introducer"], "pb://fooblam", + ) return invite - @defer.inlineCallbacks def test_invite_success(self): """ @@ -344,30 +364,10 @@ shares.total = 6 self.assertNotEqual(rc, 0) self.assertIn(u"No 'client-v1' in abilities", out + err) - async def client(): - # Run the client side of the invitation by manually pumping a - # message through the wormhole. - - # First, wait for the server to create the wormhole at all. - wormhole = await helper.wait_for_wormhole( - "tahoe-lafs.org/invite", - "ws://wormhole.tahoe-lafs.org:4000/v1", - ) - # Then read out its code and open the other side of the wormhole. - code = await wormhole.when_code() - other_end = wormhole_server.create( - "tahoe-lafs.org/invite", - "ws://wormhole.tahoe-lafs.org:4000/v1", - reactor, - ) - other_end.set_code(code) - - # Send some surprising client abilities. - other_end.send_message(dumps_bytes({u"abilities": {u"client-v9000": {}}})) - + # Send some surprising client abilities. + client = make_simple_peer(reactor, wormhole_server, helper, [{u"abilities": {u"client-v9000": {}}}]) yield concurrently(client, server) - @defer.inlineCallbacks def test_invite_no_client_abilities(self): """ @@ -399,27 +399,8 @@ shares.total = 6 self.assertNotEqual(rc, 0) self.assertIn(u"No 'abilities' from client", out + err) - async def client(): - # Run the client side of the invitation by manually pumping a - # message through the wormhole. - - # First, wait for the server to create the wormhole at all. - wormhole = await helper.wait_for_wormhole( - "tahoe-lafs.org/invite", - "ws://wormhole.tahoe-lafs.org:4000/v1", - ) - # Then read out its code and open the other side of the wormhole. - code = await wormhole.when_code() - other_end = wormhole_server.create( - "tahoe-lafs.org/invite", - "ws://wormhole.tahoe-lafs.org:4000/v1", - reactor, - ) - other_end.set_code(code) - - # Send a no-abilities message through to the server. - other_end.send_message(dumps_bytes({})) - + # Send a no-abilities message through to the server. + client = make_simple_peer(reactor, wormhole_server, helper, [{}]) yield concurrently(client, server) From 03674bd4526ce2374f5077d50cd6da603e78f48a Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 13 Apr 2022 16:01:32 -0400 Subject: [PATCH 0823/2309] use Tuple for type alias __future__.annotations only fixes py37/generic builtins in annotations syntax, not arbitrary expressions --- src/allmydata/test/cli/wormholetesting.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/cli/wormholetesting.py b/src/allmydata/test/cli/wormholetesting.py index 715e82236..0cee78a5a 100644 --- a/src/allmydata/test/cli/wormholetesting.py +++ b/src/allmydata/test/cli/wormholetesting.py @@ -32,7 +32,7 @@ For example:: from __future__ import annotations -from typing import Iterator, Optional, Sequence +from typing import Iterator, Optional, Sequence, Tuple from collections.abc import Awaitable from inspect import getargspec from itertools import count @@ -48,7 +48,7 @@ WormholeCode = str WormholeMessage = bytes AppId = str RelayURL = str -ApplicationKey = tuple[RelayURL, AppId] +ApplicationKey = Tuple[RelayURL, AppId] @define class MemoryWormholeServer(object): From f34e01649df46753cf4f129293b7b2fb2506a757 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 13 Apr 2022 18:35:18 -0400 Subject: [PATCH 0824/2309] some more fixes for mypy --- src/allmydata/test/cli/test_invite.py | 2 +- src/allmydata/test/cli/wormholetesting.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/allmydata/test/cli/test_invite.py b/src/allmydata/test/cli/test_invite.py index 9f2607433..07756eeed 100644 --- a/src/allmydata/test/cli/test_invite.py +++ b/src/allmydata/test/cli/test_invite.py @@ -19,7 +19,7 @@ from ...util.jsonbytes import dumps_bytes from ..common_util import run_cli from ..no_network import GridTestMixin from .common import CLITestMixin -from .wormholetesting import IWormhole, MemoryWormholeServer, memory_server +from .wormholetesting import IWormhole, MemoryWormholeServer, TestingHelper, memory_server # Logically: # JSONable = dict[str, Union[JSONable, None, int, float, str, list[JSONable]]] diff --git a/src/allmydata/test/cli/wormholetesting.py b/src/allmydata/test/cli/wormholetesting.py index 0cee78a5a..744f9d75a 100644 --- a/src/allmydata/test/cli/wormholetesting.py +++ b/src/allmydata/test/cli/wormholetesting.py @@ -32,7 +32,7 @@ For example:: from __future__ import annotations -from typing import Iterator, Optional, Sequence, Tuple +from typing import Iterator, Optional, List, Tuple from collections.abc import Awaitable from inspect import getargspec from itertools import count @@ -158,7 +158,7 @@ class _WormholeApp(object): appid/relay_url scope. """ wormholes: dict[WormholeCode, IWormhole] = field(default=Factory(dict)) - _waiting: dict[WormholeCode, Sequence[Deferred]] = field(default=Factory(dict)) + _waiting: dict[WormholeCode, List[Deferred]] = field(default=Factory(dict)) _counter: Iterator[int] = field(default=Factory(count)) def allocate_code(self, wormhole: IWormhole, code: Optional[WormholeCode]) -> WormholeCode: @@ -244,7 +244,7 @@ class _MemoryWormhole(object): ) self._code = self._view.allocate_code(self, None) waiters = self._waiting_for_code - self._waiting_for_code = None + self._waiting_for_code = [] for d in waiters: d.callback(self._code) From 2bc8cdf852f821ab264e442ec857b2426ff87724 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 14 Apr 2022 11:40:19 -0400 Subject: [PATCH 0825/2309] Drop Python 2. --- src/allmydata/test/test_storage_http.py | 23 ----------------------- 1 file changed, 23 deletions(-) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index cf1504f58..df781012e 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -2,18 +2,6 @@ Tests for HTTP storage client + server. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals - -from future.utils import PY2 - -if PY2: - # fmt: off - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - # fmt: on - from base64 import b64encode from contextlib import contextmanager from os import urandom @@ -108,11 +96,6 @@ class ExtractSecretsTests(SyncTestCase): Tests for ``_extract_secrets``. """ - def setUp(self): - if PY2: - self.skipTest("Not going to bother supporting Python 2") - super(ExtractSecretsTests, self).setUp() - @given(secrets_to_send=SECRETS_STRATEGY) def test_extract_secrets(self, secrets_to_send): """ @@ -271,8 +254,6 @@ class CustomHTTPServerTests(SyncTestCase): """ def setUp(self): - if PY2: - self.skipTest("Not going to bother supporting Python 2") super(CustomHTTPServerTests, self).setUp() # Could be a fixture, but will only be used in this test class so not # going to bother: @@ -374,8 +355,6 @@ class GenericHTTPAPITests(SyncTestCase): """ def setUp(self): - if PY2: - self.skipTest("Not going to bother supporting Python 2") super(GenericHTTPAPITests, self).setUp() self.http = self.useFixture(HttpTestFixture()) @@ -466,8 +445,6 @@ class ImmutableHTTPAPITests(SyncTestCase): """ def setUp(self): - if PY2: - self.skipTest("Not going to bother supporting Python 2") super(ImmutableHTTPAPITests, self).setUp() self.http = self.useFixture(HttpTestFixture()) self.imm_client = StorageClientImmutables(self.http.client) From 9db5a397e1d88929b775c464042fa7ffcb736501 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 14 Apr 2022 11:45:47 -0400 Subject: [PATCH 0826/2309] Minor type annotation improvements. --- src/allmydata/storage/http_client.py | 4 +++- src/allmydata/storage/http_common.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 6ff462d73..9a2774aef 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -2,6 +2,8 @@ HTTP client that talks to the HTTP storage server. """ +from __future__ import annotations + from typing import Union, Set, Optional from base64 import b64encode @@ -214,7 +216,7 @@ class StorageClient(object): self._treq = treq @classmethod - def from_nurl(cls, nurl: DecodedURL, reactor, persistent: bool = True) -> "StorageClient": + def from_nurl(cls, nurl: DecodedURL, reactor, persistent: bool = True) -> StorageClient: """ Create a ``StorageClient`` for the given NURL. diff --git a/src/allmydata/storage/http_common.py b/src/allmydata/storage/http_common.py index bd88f9fae..addd926d1 100644 --- a/src/allmydata/storage/http_common.py +++ b/src/allmydata/storage/http_common.py @@ -27,7 +27,7 @@ def get_content_type(headers: Headers) -> Optional[str]: return content_type -def swissnum_auth_header(swissnum): # type: (bytes) -> bytes +def swissnum_auth_header(swissnum: bytes) -> bytes: """Return value for ``Authentication`` header.""" return b"Tahoe-LAFS " + b64encode(swissnum).strip() From 4e0f912a100bbee6e684fd830bb4d0510a2545e6 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 14 Apr 2022 11:52:20 -0400 Subject: [PATCH 0827/2309] Comply with license. --- src/allmydata/storage/http_client.py | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 9a2774aef..65bcd1c4b 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -134,7 +134,28 @@ class _TLSContextFactory(CertificateOptions): Create a context that validates the way Tahoe-LAFS wants to: based on a pinned certificate hash, rather than a certificate authority. - Originally implemented as part of Foolscap. + Originally implemented as part of Foolscap. To comply with the license, + here's the original licensing terms: + + Copyright (c) 2006-2008 Brian Warner + + Permission is hereby granted, free of charge, to any person obtaining a + copy of this software and associated documentation files (the "Software"), + to deal in the Software without restriction, including without limitation + the rights to use, copy, modify, merge, publish, distribute, sublicense, + and/or sell copies of the Software, and to permit persons to whom the + Software is furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + DEALINGS IN THE SOFTWARE. """ def __init__(self, expected_spki_hash: bytes): From fc2807cccc773386b83e53d7eb02ee02ef326ba9 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 15 Apr 2022 09:08:16 -0400 Subject: [PATCH 0828/2309] Sketch of server-side read-test-write endpoint. --- src/allmydata/storage/http_common.py | 1 + src/allmydata/storage/http_server.py | 60 ++++++++++++++++++++++++---- 2 files changed, 54 insertions(+), 7 deletions(-) diff --git a/src/allmydata/storage/http_common.py b/src/allmydata/storage/http_common.py index addd926d1..123ce403b 100644 --- a/src/allmydata/storage/http_common.py +++ b/src/allmydata/storage/http_common.py @@ -38,6 +38,7 @@ class Secrets(Enum): LEASE_RENEW = "lease-renew-secret" LEASE_CANCEL = "lease-cancel-secret" UPLOAD = "upload-secret" + WRITE_ENABLER = "write-enabler" def get_spki_hash(certificate: Certificate) -> bytes: diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 7c4860d57..bcb4b22c9 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -239,19 +239,39 @@ class _HTTPError(Exception): # https://www.iana.org/assignments/cbor-tags/cbor-tags.xhtml. Notably, #6.258 # indicates a set. _SCHEMAS = { - "allocate_buckets": Schema(""" - message = { + "allocate_buckets": Schema( + """ + request = { share-numbers: #6.258([* uint]) allocated-size: uint } - """), - "advise_corrupt_share": Schema(""" - message = { + """ + ), + "advise_corrupt_share": Schema( + """ + request = { reason: tstr } - """) + """ + ), + "mutable_read_test_write": Schema( + """ + request = { + "test-write-vectors": { + * share_number: { + "test": [* {"offset": uint, "size": uint, "specimen": bstr}] + "write": [* {"offset": uint, "data": bstr}] + "new-length": uint + } + } + "read-vector": [* {"offset": uint, "size": uint}] + } + share_number = uint + """ + ), } + class HTTPServer(object): """ A HTTP interface to the storage server. @@ -537,7 +557,9 @@ class HTTPServer(object): "/v1/immutable///corrupt", methods=["POST"], ) - def advise_corrupt_share(self, request, authorization, storage_index, share_number): + def advise_corrupt_share_immutable( + self, request, authorization, storage_index, share_number + ): """Indicate that given share is corrupt, with a text reason.""" try: bucket = self._storage_server.get_buckets(storage_index)[share_number] @@ -548,6 +570,30 @@ class HTTPServer(object): bucket.advise_corrupt_share(info["reason"].encode("utf-8")) return b"" + ##### Immutable APIs ##### + + @_authorized_route( + _app, + {Secrets.LEASE_RENEW, Secrets.LEASE_CANCEL, Secrets.WRITE_ENABLER}, + "/v1/mutable//read-test-write", + methods=["POST"], + ) + def mutable_read_test_write(self, request, authorization, storage_index): + """Read/test/write combined operation for mutables.""" + rtw_request = self._read_encoded(request, _SCHEMAS["mutable_read_test_write"]) + secrets = ( + authorization[Secrets.WRITE_ENABLER], + authorization[Secrets.LEASE_RENEW], + authorization[Secrets.LEASE_CANCEL], + ) + success, read_data = self._storage_server.slot_testv_and_readv_and_writev( + storage_index, + secrets, + rtw_request["test-write-vectors"], + rtw_request["read-vectors"], + ) + return self._send_encoded(request, {"success": success, "data": read_data}) + @implementer(IStreamServerEndpoint) @attr.s From 58bd38120294a6709fc74190188d6ae4ae74a03b Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 15 Apr 2022 09:19:30 -0400 Subject: [PATCH 0829/2309] Switch to newer attrs API. --- src/allmydata/storage/http_client.py | 42 +++++++++++++--------------- 1 file changed, 20 insertions(+), 22 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 65bcd1c4b..57ca4dae9 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -8,7 +8,7 @@ from typing import Union, Set, Optional from base64 import b64encode -import attr +from attrs import define # TODO Make sure to import Python version? from cbor2 import loads, dumps @@ -121,12 +121,12 @@ def _decode_cbor(response, schema: Schema): ) -@attr.s +@define class ImmutableCreateResult(object): """Result of creating a storage index for an immutable.""" - already_have = attr.ib(type=Set[int]) - allocated = attr.ib(type=Set[int]) + already_have: Set[int] + allocated: Set[int] class _TLSContextFactory(CertificateOptions): @@ -200,14 +200,14 @@ class _TLSContextFactory(CertificateOptions): @implementer(IPolicyForHTTPS) @implementer(IOpenSSLClientConnectionCreator) -@attr.s +@define class _StorageClientHTTPSPolicy: """ A HTTPS policy that ensures the SPKI hash of the public key matches a known hash, i.e. pinning-based validation. """ - expected_spki_hash = attr.ib(type=bytes) + expected_spki_hash: bytes # IPolicyForHTTPS def creatorForNetloc(self, hostname, port): @@ -220,24 +220,22 @@ class _StorageClientHTTPSPolicy: ) +@define class StorageClient(object): """ Low-level HTTP client that talks to the HTTP storage server. """ - def __init__( - self, url, swissnum, treq=treq - ): # type: (DecodedURL, bytes, Union[treq,StubTreq,HTTPClient]) -> None - """ - The URL is a HTTPS URL ("https://..."). To construct from a NURL, use - ``StorageClient.from_nurl()``. - """ - self._base_url = url - self._swissnum = swissnum - self._treq = treq + # The URL is a HTTPS URL ("https://..."). To construct from a NURL, use + # ``StorageClient.from_nurl()``. + _base_url: DecodedURL + _swissnum: bytes + _treq: Union[treq, StubTreq, HTTPClient] @classmethod - def from_nurl(cls, nurl: DecodedURL, reactor, persistent: bool = True) -> StorageClient: + def from_nurl( + cls, nurl: DecodedURL, reactor, persistent: bool = True + ) -> StorageClient: """ Create a ``StorageClient`` for the given NURL. @@ -342,25 +340,25 @@ class StorageClientGeneral(object): returnValue(decoded_response) -@attr.s +@define class UploadProgress(object): """ Progress of immutable upload, per the server. """ # True when upload has finished. - finished = attr.ib(type=bool) + finished: bool # Remaining ranges to upload. - required = attr.ib(type=RangeMap) + required: RangeMap +@define class StorageClientImmutables(object): """ APIs for interacting with immutables. """ - def __init__(self, client: StorageClient): - self._client = client + _client: StorageClient @inlineCallbacks def create( From 186aa9abc4715ef42f7d7d1c7ecd95884bdeab45 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 15 Apr 2022 09:32:15 -0400 Subject: [PATCH 0830/2309] Make the utility reusable. --- src/allmydata/test/test_deferredutil.py | 28 +++++++++++++++++++ src/allmydata/test/test_storage_https.py | 17 +----------- src/allmydata/util/deferredutil.py | 35 +++++++++++++----------- 3 files changed, 48 insertions(+), 32 deletions(-) diff --git a/src/allmydata/test/test_deferredutil.py b/src/allmydata/test/test_deferredutil.py index 2a155089f..a37dfdd6f 100644 --- a/src/allmydata/test/test_deferredutil.py +++ b/src/allmydata/test/test_deferredutil.py @@ -129,3 +129,31 @@ class UntilTests(unittest.TestCase): self.assertEqual([1], counter) r1.callback(None) self.assertEqual([2], counter) + + +class AsyncToDeferred(unittest.TestCase): + """Tests for ``deferredutil.async_to_deferred.``""" + + def test_async_to_deferred_success(self): + """ + Normal results from a ``@async_to_deferred``-wrapped function get + turned into a ``Deferred`` with that value. + """ + @deferredutil.async_to_deferred + async def f(x, y): + return x + y + + result = f(1, y=2) + self.assertEqual(self.successResultOf(result), 3) + + def test_async_to_deferred_exception(self): + """ + Exceptions from a ``@async_to_deferred``-wrapped function get + turned into a ``Deferred`` with that value. + """ + @deferredutil.async_to_deferred + async def f(x, y): + return x/y + + result = f(1, 0) + self.assertIsInstance(self.failureResultOf(result).value, ZeroDivisionError) diff --git a/src/allmydata/test/test_storage_https.py b/src/allmydata/test/test_storage_https.py index 73c99725a..3b41e8308 100644 --- a/src/allmydata/test/test_storage_https.py +++ b/src/allmydata/test/test_storage_https.py @@ -6,14 +6,12 @@ server authentication logic, which may one day apply outside of HTTP Storage Protocol. """ -from functools import wraps from contextlib import asynccontextmanager from cryptography import x509 from twisted.internet.endpoints import serverFromString from twisted.internet import reactor -from twisted.internet.defer import Deferred from twisted.internet.task import deferLater from twisted.web.server import Site from twisted.web.static import Data @@ -31,6 +29,7 @@ from .certs import ( from ..storage.http_common import get_spki_hash from ..storage.http_client import _StorageClientHTTPSPolicy from ..storage.http_server import _TLSEndpointWrapper +from ..util.deferredutil import async_to_deferred class HTTPSNurlTests(SyncTestCase): @@ -73,20 +72,6 @@ ox5zO3LrQmQw11OaIAs2/kviKAoKTFFxeyYcpS5RuKNDZfHQCXlLwt9bySxG self.assertEqual(get_spki_hash(certificate), expected_hash) -def async_to_deferred(f): - """ - Wrap an async function to return a Deferred instead. - - Maybe solution to https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3886 - """ - - @wraps(f) - def not_async(*args, **kwargs): - return Deferred.fromCoroutine(f(*args, **kwargs)) - - return not_async - - class PinningHTTPSValidation(AsyncTestCase): """ Test client-side validation logic of HTTPS certificates that uses diff --git a/src/allmydata/util/deferredutil.py b/src/allmydata/util/deferredutil.py index ed2a11ee4..782663e8b 100644 --- a/src/allmydata/util/deferredutil.py +++ b/src/allmydata/util/deferredutil.py @@ -4,24 +4,13 @@ Utilities for working with Twisted Deferreds. Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals - -from future.utils import PY2 -if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - import time +from functools import wraps -try: - from typing import ( - Callable, - Any, - ) -except ImportError: - pass +from typing import ( + Callable, + Any, +) from foolscap.api import eventually from eliot.twisted import ( @@ -231,3 +220,17 @@ def until( yield action() if condition(): break + + +def async_to_deferred(f): + """ + Wrap an async function to return a Deferred instead. + + Maybe solution to https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3886 + """ + + @wraps(f) + def not_async(*args, **kwargs): + return defer.Deferred.fromCoroutine(f(*args, **kwargs)) + + return not_async From 24548dee0b37184cef975d4febe9ca4425920266 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 15 Apr 2022 09:56:06 -0400 Subject: [PATCH 0831/2309] Sketch of read/write APIs interface for mutables on client side. --- src/allmydata/storage/http_client.py | 134 +++++++++++++++++++++++++-- src/allmydata/storage/http_server.py | 2 +- 2 files changed, 129 insertions(+), 7 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 57ca4dae9..1bff34699 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -5,10 +5,10 @@ HTTP client that talks to the HTTP storage server. from __future__ import annotations from typing import Union, Set, Optional - +from enum import Enum from base64 import b64encode -from attrs import define +from attrs import define, field # TODO Make sure to import Python version? from cbor2 import loads, dumps @@ -39,6 +39,7 @@ from .http_common import ( ) from .common import si_b2a from ..util.hashutil import timing_safe_compare +from ..util.deferredutil import async_to_deferred _OPENSSL = Binding().lib @@ -64,7 +65,7 @@ class ClientException(Exception): _SCHEMAS = { "get_version": Schema( """ - message = {'http://allmydata.org/tahoe/protocols/storage/v1' => { + response = {'http://allmydata.org/tahoe/protocols/storage/v1' => { 'maximum-immutable-share-size' => uint 'maximum-mutable-share-size' => uint 'available-space' => uint @@ -79,7 +80,7 @@ _SCHEMAS = { ), "allocate_buckets": Schema( """ - message = { + response = { already-have: #6.258([* uint]) allocated: #6.258([* uint]) } @@ -87,16 +88,25 @@ _SCHEMAS = { ), "immutable_write_share_chunk": Schema( """ - message = { + response = { required: [* {begin: uint, end: uint}] } """ ), "list_shares": Schema( """ - message = #6.258([* uint]) + response = #6.258([* uint]) """ ), + "mutable_read_test_write": Schema( + """ + response = { + "success": bool, + "data": [* share_number: [* bstr]] + } + share_number = uint + """ + ), } @@ -571,3 +581,115 @@ class StorageClientImmutables(object): raise ClientException( response.code, ) + + +@define +class WriteVector: + """Data to write to a chunk.""" + + offset: int + data: bytes + + +class TestVectorOperator(Enum): + """Possible operators for test vectors.""" + + LT = b"lt" + LE = b"le" + EQ = b"eq" + NE = b"ne" + GE = b"ge" + GT = b"gt" + + +@define +class TestVector: + """Checks to make on a chunk before writing to it.""" + + offset: int + size: int + operator: TestVectorOperator = field(default=TestVectorOperator.EQ) + specimen: bytes + + +@define +class ReadVector: + """ + Reads to do on chunks, as part of a read/test/write operation. + """ + + offset: int + size: int + + +@define +class TestWriteVectors: + """Test and write vectors for a specific share.""" + + test_vectors: list[TestVector] + write_vectors: list[WriteVector] + new_length: Optional[int] = field(default=None) + + +@define +class ReadTestWriteResult: + """Result of sending read-test-write vectors.""" + + success: bool + # Map share numbers to reads corresponding to the request's list of + # ReadVectors: + reads: dict[int, list[bytes]] + + +@define +class StorageClientMutables: + """ + APIs for interacting with mutables. + """ + + _client: StorageClient + + @async_to_deferred + async def read_test_write_chunks( + storage_index: bytes, + write_enabled_secret: bytes, + lease_renew_secret: bytes, + lease_cancel_secret: bytes, + testwrite_vectors: dict[int, TestWriteVectors], + read_vector: list[ReadVector], + ) -> ReadTestWriteResult: + """ + Read, test, and possibly write chunks to a particular mutable storage + index. + + Reads are done before writes. + + Given a mapping between share numbers and test/write vectors, the tests + are done and if they are valid the writes are done. + """ + pass + + @async_to_deferred + async def read_share_chunk( + self, + storage_index: bytes, + share_number: int, + # TODO is this really optional? + # TODO if yes, test non-optional variants + offset: Optional[int], + length: Optional[int], + ) -> bytes: + """ + Download a chunk of data from a share. + + TODO https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3857 Failed + downloads should be transparently retried and redownloaded by the + implementation a few times so that if a failure percolates up, the + caller can assume the failure isn't a short-term blip. + + NOTE: the underlying HTTP protocol is much more flexible than this API, + so a future refactor may expand this in order to simplify the calling + code and perhaps download data more efficiently. But then again maybe + the HTTP protocol will be simplified, see + https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3777 + """ diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index bcb4b22c9..7f279580b 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -261,7 +261,7 @@ _SCHEMAS = { * share_number: { "test": [* {"offset": uint, "size": uint, "specimen": bstr}] "write": [* {"offset": uint, "data": bstr}] - "new-length": uint + "new-length": (uint // null) } } "read-vector": [* {"offset": uint, "size": uint}] From b0d547ee53540649e18ceb437b7508d22a12dbaa Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 18 Apr 2022 14:56:20 -0400 Subject: [PATCH 0832/2309] Progress on implementing client side of mutable writes. --- src/allmydata/storage/http_client.py | 33 +++++++++++++++++++++++++--- 1 file changed, 30 insertions(+), 3 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 1bff34699..8899614b8 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -8,7 +8,7 @@ from typing import Union, Set, Optional from enum import Enum from base64 import b64encode -from attrs import define, field +from attrs import define, field, asdict # TODO Make sure to import Python version? from cbor2 import loads, dumps @@ -288,6 +288,7 @@ class StorageClient(object): lease_renew_secret=None, lease_cancel_secret=None, upload_secret=None, + write_enabler_secret=None, headers=None, message_to_serialize=None, **kwargs @@ -306,6 +307,7 @@ class StorageClient(object): (Secrets.LEASE_RENEW, lease_renew_secret), (Secrets.LEASE_CANCEL, lease_cancel_secret), (Secrets.UPLOAD, upload_secret), + (Secrets.WRITE_ENABLER, write_enabler_secret), ]: if value is None: continue @@ -651,8 +653,9 @@ class StorageClientMutables: @async_to_deferred async def read_test_write_chunks( + self, storage_index: bytes, - write_enabled_secret: bytes, + write_enabler_secret: bytes, lease_renew_secret: bytes, lease_cancel_secret: bytes, testwrite_vectors: dict[int, TestWriteVectors], @@ -667,7 +670,31 @@ class StorageClientMutables: Given a mapping between share numbers and test/write vectors, the tests are done and if they are valid the writes are done. """ - pass + # TODO unit test all the things + url = self._client.relative_url( + "/v1/mutable/{}/read-test-write".format(_encode_si(storage_index)) + ) + message = { + "test-write-vectors": { + share_number: asdict(twv) + for (share_number, twv) in testwrite_vectors.items() + }, + "read-vector": [asdict(r) for r in read_vector], + } + response = yield self._client.request( + "POST", + url, + write_enabler_secret=write_enabler_secret, + lease_renew_secret=lease_renew_secret, + lease_cancel_secret=lease_cancel_secret, + message_to_serialize=message, + ) + if response.code == http.OK: + return _decode_cbor(response, _SCHEMAS["mutable_test_read_write"]) + else: + raise ClientException( + response.code, + ) @async_to_deferred async def read_share_chunk( From 2ca5e22af9788aa4fccd3b25ed5c3188ef049ecb Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 19 Apr 2022 11:47:22 -0400 Subject: [PATCH 0833/2309] Make mutable reading match immutable reading. --- docs/proposed/http-storage-node-protocol.rst | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index 9354bc185..3926d9f4a 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -743,11 +743,15 @@ For example:: [1, 5] -``GET /v1/mutable/:storage_index?share=:s0&share=:sN&offset=:o1&size=:z0&offset=:oN&size=:zN`` +``GET /v1/mutable/:storage_index/:share_number`` !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! -Read data from the indicated mutable shares. -Just like ``GET /v1/mutable/:storage_index``. +Read data from the indicated mutable shares, just like ``GET /v1/immutable/:storage_index`` + +The ``Range`` header may be used to request exactly one ``bytes`` range, in which case the response code will be 206 (partial content). +Interpretation and response behavior is as specified in RFC 7233 § 4.1. +Multiple ranges in a single request are *not* supported; open-ended ranges are also not supported. + ``POST /v1/mutable/:storage_index/:share_number/corrupt`` !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! From 898fe0bc0e48489668cf58981a18a252c9ed587f Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 19 Apr 2022 13:18:31 -0400 Subject: [PATCH 0834/2309] Closer to running end-to-end mutable tests. --- src/allmydata/storage/http_client.py | 107 ++++++++++++---------- src/allmydata/storage/http_server.py | 4 +- src/allmydata/storage_client.py | 54 ++++++++++- src/allmydata/test/test_istorageserver.py | 8 +- 4 files changed, 122 insertions(+), 51 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 8899614b8..52177f401 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -364,6 +364,46 @@ class UploadProgress(object): required: RangeMap +@inlineCallbacks +def read_share_chunk( + client: StorageClient, + share_type: str, + storage_index: bytes, + share_number: int, + offset: int, + length: int, +) -> Deferred[bytes]: + """ + Download a chunk of data from a share. + + TODO https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3857 Failed + downloads should be transparently retried and redownloaded by the + implementation a few times so that if a failure percolates up, the + caller can assume the failure isn't a short-term blip. + + NOTE: the underlying HTTP protocol is much more flexible than this API, + so a future refactor may expand this in order to simplify the calling + code and perhaps download data more efficiently. But then again maybe + the HTTP protocol will be simplified, see + https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3777 + """ + url = client.relative_url( + "/v1/{}/{}/{}".format(share_type, _encode_si(storage_index), share_number) + ) + response = yield client.request( + "GET", + url, + headers=Headers( + {"range": [Range("bytes", [(offset, offset + length)]).to_header()]} + ), + ) + if response.code == http.PARTIAL_CONTENT: + body = yield response.content() + returnValue(body) + else: + raise ClientException(response.code) + + @define class StorageClientImmutables(object): """ @@ -484,39 +524,15 @@ class StorageClientImmutables(object): remaining.set(True, chunk["begin"], chunk["end"]) returnValue(UploadProgress(finished=finished, required=remaining)) - @inlineCallbacks def read_share_chunk( self, storage_index, share_number, offset, length ): # type: (bytes, int, int, int) -> Deferred[bytes] """ Download a chunk of data from a share. - - TODO https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3857 Failed - downloads should be transparently retried and redownloaded by the - implementation a few times so that if a failure percolates up, the - caller can assume the failure isn't a short-term blip. - - NOTE: the underlying HTTP protocol is much more flexible than this API, - so a future refactor may expand this in order to simplify the calling - code and perhaps download data more efficiently. But then again maybe - the HTTP protocol will be simplified, see - https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3777 """ - url = self._client.relative_url( - "/v1/immutable/{}/{}".format(_encode_si(storage_index), share_number) + return read_share_chunk( + self._client, "immutable", storage_index, share_number, offset, length ) - response = yield self._client.request( - "GET", - url, - headers=Headers( - {"range": [Range("bytes", [(offset, offset + length)]).to_header()]} - ), - ) - if response.code == http.PARTIAL_CONTENT: - body = yield response.content() - returnValue(body) - else: - raise ClientException(response.code) @inlineCallbacks def list_shares(self, storage_index): # type: (bytes,) -> Deferred[Set[int]] @@ -610,7 +626,7 @@ class TestVector: offset: int size: int - operator: TestVectorOperator = field(default=TestVectorOperator.EQ) + operator: TestVectorOperator specimen: bytes @@ -632,6 +648,14 @@ class TestWriteVectors: write_vectors: list[WriteVector] new_length: Optional[int] = field(default=None) + def asdict(self) -> dict: + """Return dictionary suitable for sending over CBOR.""" + d = asdict(self) + d["test"] = d.pop("test_vectors") + d["write"] = d.pop("write_vectors") + d["new-length"] = d.pop("new_length") + return d + @define class ReadTestWriteResult: @@ -676,12 +700,12 @@ class StorageClientMutables: ) message = { "test-write-vectors": { - share_number: asdict(twv) + share_number: twv.asdict() for (share_number, twv) in testwrite_vectors.items() }, "read-vector": [asdict(r) for r in read_vector], } - response = yield self._client.request( + response = await self._client.request( "POST", url, write_enabler_secret=write_enabler_secret, @@ -692,31 +716,20 @@ class StorageClientMutables: if response.code == http.OK: return _decode_cbor(response, _SCHEMAS["mutable_test_read_write"]) else: - raise ClientException( - response.code, - ) + raise ClientException(response.code, (await response.content())) @async_to_deferred async def read_share_chunk( self, storage_index: bytes, share_number: int, - # TODO is this really optional? - # TODO if yes, test non-optional variants - offset: Optional[int], - length: Optional[int], + offset: int, + length: int, ) -> bytes: """ Download a chunk of data from a share. - - TODO https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3857 Failed - downloads should be transparently retried and redownloaded by the - implementation a few times so that if a failure percolates up, the - caller can assume the failure isn't a short-term blip. - - NOTE: the underlying HTTP protocol is much more flexible than this API, - so a future refactor may expand this in order to simplify the calling - code and perhaps download data more efficiently. But then again maybe - the HTTP protocol will be simplified, see - https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3777 """ + # TODO unit test all the things + return read_share_chunk( + self._client, "mutable", storage_index, share_number, offset, length + ) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 7f279580b..6def5aeeb 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -261,7 +261,7 @@ _SCHEMAS = { * share_number: { "test": [* {"offset": uint, "size": uint, "specimen": bstr}] "write": [* {"offset": uint, "data": bstr}] - "new-length": (uint // null) + "new-length": uint // null } } "read-vector": [* {"offset": uint, "size": uint}] @@ -590,7 +590,7 @@ class HTTPServer(object): storage_index, secrets, rtw_request["test-write-vectors"], - rtw_request["read-vectors"], + rtw_request["read-vector"], ) return self._send_encoded(request, {"success": success, "data": read_data}) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 55b6cfb05..afed0e274 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -77,7 +77,8 @@ from allmydata.util.hashutil import permute_server_hash from allmydata.util.dictutil import BytesKeyDict, UnicodeKeyDict from allmydata.storage.http_client import ( StorageClient, StorageClientImmutables, StorageClientGeneral, - ClientException as HTTPClientException + ClientException as HTTPClientException, StorageClientMutables, + ReadVector, TestWriteVectors, WriteVector, TestVector, TestVectorOperator ) @@ -1189,3 +1190,54 @@ class _HTTPStorageServer(object): ) else: raise NotImplementedError() # future tickets + + @defer.inlineCallbacks + def slot_readv(self, storage_index, shares, readv): + mutable_client = StorageClientMutables(self._http_client) + reads = {} + for share_number in shares: + share_reads = reads[share_number] = [] + for (offset, length) in readv: + d = mutable_client.read_share_chunk( + storage_index, share_number, offset, length + ) + share_reads.append(d) + result = { + share_number: [(yield d) for d in share_reads] + for (share_number, reads) in reads.items() + } + defer.returnValue(result) + + @defer.inlineCallbacks + def slot_testv_and_readv_and_writev( + self, + storage_index, + secrets, + tw_vectors, + r_vector, + ): + mutable_client = StorageClientMutables(self._http_client) + we_secret, lr_secret, lc_secret = secrets + client_tw_vectors = {} + for share_num, (test_vector, data_vector, new_length) in tw_vectors.items(): + client_test_vectors = [ + TestVector(offset=offset, size=size, operator=TestVectorOperator[op], specimen=specimen) + for (offset, size, op, specimen) in test_vector + ] + client_write_vectors = [ + WriteVector(offset=offset, data=data) for (offset, data) in data_vector + ] + client_tw_vectors[share_num] = TestWriteVectors( + test_vectors=client_test_vectors, + write_vectors=client_write_vectors, + new_length=new_length + ) + client_read_vectors = [ + ReadVector(offset=offset, size=size) + for (offset, size) in r_vector + ] + client_result = yield mutable_client.read_test_write_chunks( + storage_index, we_secret, lr_secret, lc_secret, client_tw_vectors, + client_read_vectors, + ) + defer.returnValue((client_result.success, client_result.reads)) diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index 3d6f610be..702c66952 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -1140,4 +1140,10 @@ class HTTPImmutableAPIsTests( class FoolscapMutableAPIsTests( _FoolscapMixin, IStorageServerMutableAPIsTestsMixin, AsyncTestCase ): - """Foolscap-specific tests for immutable ``IStorageServer`` APIs.""" + """Foolscap-specific tests for mutable ``IStorageServer`` APIs.""" + + +class HTTPMutableAPIsTests( + _HTTPMixin, IStorageServerMutableAPIsTestsMixin, AsyncTestCase +): + """HTTP-specific tests for mutable ``IStorageServer`` APIs.""" From f5c4513cd38c79ec2af2fdba18811f023134c42b Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 19 Apr 2022 13:35:09 -0400 Subject: [PATCH 0835/2309] A little closer to serialization and deserialization working correctly, with some tests passing. --- src/allmydata/storage/http_client.py | 20 ++++---------------- src/allmydata/storage/http_server.py | 11 +++++++++-- src/allmydata/storage_client.py | 6 +++--- 3 files changed, 16 insertions(+), 21 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 52177f401..7b80ec602 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -102,7 +102,7 @@ _SCHEMAS = { """ response = { "success": bool, - "data": [* share_number: [* bstr]] + "data": {* share_number: [* bstr]} } share_number = uint """ @@ -609,24 +609,12 @@ class WriteVector: data: bytes -class TestVectorOperator(Enum): - """Possible operators for test vectors.""" - - LT = b"lt" - LE = b"le" - EQ = b"eq" - NE = b"ne" - GE = b"ge" - GT = b"gt" - - @define class TestVector: """Checks to make on a chunk before writing to it.""" offset: int size: int - operator: TestVectorOperator specimen: bytes @@ -714,12 +702,12 @@ class StorageClientMutables: message_to_serialize=message, ) if response.code == http.OK: - return _decode_cbor(response, _SCHEMAS["mutable_test_read_write"]) + result = await _decode_cbor(response, _SCHEMAS["mutable_read_test_write"]) + return ReadTestWriteResult(success=result["success"], reads=result["data"]) else: raise ClientException(response.code, (await response.content())) - @async_to_deferred - async def read_share_chunk( + def read_share_chunk( self, storage_index: bytes, share_number: int, diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 6def5aeeb..3eae476b7 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -589,8 +589,15 @@ class HTTPServer(object): success, read_data = self._storage_server.slot_testv_and_readv_and_writev( storage_index, secrets, - rtw_request["test-write-vectors"], - rtw_request["read-vector"], + { + k: ( + [(d["offset"], d["size"], b"eq", d["specimen"]) for d in v["test"]], + [(d["offset"], d["data"]) for d in v["write"]], + v["new-length"], + ) + for (k, v) in rtw_request["test-write-vectors"].items() + }, + [(d["offset"], d["size"]) for d in rtw_request["read-vector"]], ) return self._send_encoded(request, {"success": success, "data": read_data}) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index afed0e274..5321efb7d 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -78,7 +78,7 @@ from allmydata.util.dictutil import BytesKeyDict, UnicodeKeyDict from allmydata.storage.http_client import ( StorageClient, StorageClientImmutables, StorageClientGeneral, ClientException as HTTPClientException, StorageClientMutables, - ReadVector, TestWriteVectors, WriteVector, TestVector, TestVectorOperator + ReadVector, TestWriteVectors, WriteVector, TestVector ) @@ -1221,8 +1221,8 @@ class _HTTPStorageServer(object): client_tw_vectors = {} for share_num, (test_vector, data_vector, new_length) in tw_vectors.items(): client_test_vectors = [ - TestVector(offset=offset, size=size, operator=TestVectorOperator[op], specimen=specimen) - for (offset, size, op, specimen) in test_vector + TestVector(offset=offset, size=size, specimen=specimen) + for (offset, size, specimen) in test_vector ] client_write_vectors = [ WriteVector(offset=offset, data=data) for (offset, data) in data_vector From 21c3c50e37114a7a3e7e6d02ebe08af1101c45f3 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 19 Apr 2022 15:07:57 -0400 Subject: [PATCH 0836/2309] Basic mutable read support. --- src/allmydata/storage/http_server.py | 43 ++++++++++++++++++++++++++++ src/allmydata/storage_client.py | 13 ++++----- 2 files changed, 49 insertions(+), 7 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 3eae476b7..dbb79cf2b 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -601,6 +601,49 @@ class HTTPServer(object): ) return self._send_encoded(request, {"success": success, "data": read_data}) + @_authorized_route( + _app, + set(), + "/v1/mutable//", + methods=["GET"], + ) + def read_mutable_chunk(self, request, authorization, storage_index, share_number): + """Read a chunk from a mutable.""" + if request.getHeader("range") is None: + # TODO in follow-up ticket + raise NotImplementedError() + + # TODO reduce duplication with immutable reads? + # TODO unit tests, perhaps shared if possible + range_header = parse_range_header(request.getHeader("range")) + if ( + range_header is None + or range_header.units != "bytes" + or len(range_header.ranges) > 1 # more than one range + or range_header.ranges[0][1] is None # range without end + ): + request.setResponseCode(http.REQUESTED_RANGE_NOT_SATISFIABLE) + return b"" + + offset, end = range_header.ranges[0] + + # TODO limit memory usage + # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3872 + data = self._storage_server.slot_readv( + storage_index, [share_number], [(offset, end - offset)] + )[share_number][0] + + # TODO reduce duplication? + request.setResponseCode(http.PARTIAL_CONTENT) + if len(data): + # For empty bodies the content-range header makes no sense since + # the end of the range is inclusive. + request.setHeader( + "content-range", + ContentRange("bytes", offset, offset + len(data)).to_header(), + ) + return data + @implementer(IStreamServerEndpoint) @attr.s diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 5321efb7d..857b19ed7 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -1195,18 +1195,17 @@ class _HTTPStorageServer(object): def slot_readv(self, storage_index, shares, readv): mutable_client = StorageClientMutables(self._http_client) reads = {} + # TODO if shares list is empty, that means list all shares, so we need + # to do a query to get that. + assert shares # TODO replace with call to list shares for share_number in shares: share_reads = reads[share_number] = [] for (offset, length) in readv: - d = mutable_client.read_share_chunk( + r = yield mutable_client.read_share_chunk( storage_index, share_number, offset, length ) - share_reads.append(d) - result = { - share_number: [(yield d) for d in share_reads] - for (share_number, reads) in reads.items() - } - defer.returnValue(result) + share_reads.append(r) + defer.returnValue(reads) @defer.inlineCallbacks def slot_testv_and_readv_and_writev( From f03feb0595c63c5cfbcd72cfa1243d9e7e375fe4 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 19 Apr 2022 15:08:07 -0400 Subject: [PATCH 0837/2309] TODOs for later. --- src/allmydata/storage/http_server.py | 1 + src/allmydata/test/test_istorageserver.py | 9 +++++++++ 2 files changed, 10 insertions(+) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index dbb79cf2b..b71877bbf 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -580,6 +580,7 @@ class HTTPServer(object): ) def mutable_read_test_write(self, request, authorization, storage_index): """Read/test/write combined operation for mutables.""" + # TODO unit tests rtw_request = self._read_encoded(request, _SCHEMAS["mutable_read_test_write"]) secrets = ( authorization[Secrets.WRITE_ENABLER], diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index 702c66952..e7b869713 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -1147,3 +1147,12 @@ class HTTPMutableAPIsTests( _HTTPMixin, IStorageServerMutableAPIsTestsMixin, AsyncTestCase ): """HTTP-specific tests for mutable ``IStorageServer`` APIs.""" + + # TODO will be implemented in later tickets + SKIP_TESTS = { + "test_STARAW_write_enabler_must_match", + "test_add_lease_renewal", + "test_add_new_lease", + "test_advise_corrupt_share", + "test_slot_readv_no_shares", + } From 5ccafb2a032cdb5a91abd12be6fb0756465711f4 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 19 Apr 2022 15:08:30 -0400 Subject: [PATCH 0838/2309] News file. --- newsfragments/3890.mi | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3890.mi diff --git a/newsfragments/3890.mi b/newsfragments/3890.mi new file mode 100644 index 000000000..e69de29bb From 72c59b5f1a9a2960e53379d3753c0ee1fd5b5de3 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 19 Apr 2022 15:09:02 -0400 Subject: [PATCH 0839/2309] Unused import. --- src/allmydata/storage/http_client.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 7b80ec602..09aada555 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -5,7 +5,6 @@ HTTP client that talks to the HTTP storage server. from __future__ import annotations from typing import Union, Set, Optional -from enum import Enum from base64 import b64encode from attrs import define, field, asdict From 3d710406ef0b222f29af023acc7dedcb156f41f5 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 26 Apr 2022 15:50:23 -0400 Subject: [PATCH 0840/2309] News file. --- newsfragments/3890.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3890.minor diff --git a/newsfragments/3890.minor b/newsfragments/3890.minor new file mode 100644 index 000000000..e69de29bb From 2d34b6a998f3f5eb454e915415e94e1387a1ee06 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 26 Apr 2022 16:29:00 -0400 Subject: [PATCH 0841/2309] Log the request and response in the server. --- src/allmydata/storage/http_server.py | 36 ++++++++++++++++++++++------ 1 file changed, 29 insertions(+), 7 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 7c4860d57..935390d10 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -8,6 +8,7 @@ from functools import wraps from base64 import b64decode import binascii +from eliot import start_action from zope.interface import implementer from klein import Klein from twisted.web import http @@ -83,8 +84,9 @@ def _extract_secrets( def _authorization_decorator(required_secrets): """ - Check the ``Authorization`` header, and extract ``X-Tahoe-Authorization`` - headers and pass them in. + 1. Check the ``Authorization`` header matches server swissnum. + 2. Extract ``X-Tahoe-Authorization`` headers and pass them in. + 3. Log the request and response. """ def decorator(f): @@ -106,7 +108,22 @@ def _authorization_decorator(required_secrets): except ClientSecretsException: request.setResponseCode(http.BAD_REQUEST) return b"Missing required secrets" - return f(self, request, secrets, *args, **kwargs) + with start_action( + action_type="allmydata:storage:http-server:request", + method=request.method, + path=request.path, + ) as ctx: + try: + result = f(self, request, secrets, *args, **kwargs) + except _HTTPError as e: + # This isn't an error necessarily for logging purposes, + # it's an implementation detail, an easier way to set + # response codes. + ctx.add_success_fields(response_code=e.code) + ctx.finish() + raise + ctx.add_success_fields(response_code=request.code) + return result return route @@ -239,19 +256,24 @@ class _HTTPError(Exception): # https://www.iana.org/assignments/cbor-tags/cbor-tags.xhtml. Notably, #6.258 # indicates a set. _SCHEMAS = { - "allocate_buckets": Schema(""" + "allocate_buckets": Schema( + """ message = { share-numbers: #6.258([* uint]) allocated-size: uint } - """), - "advise_corrupt_share": Schema(""" + """ + ), + "advise_corrupt_share": Schema( + """ message = { reason: tstr } - """) + """ + ), } + class HTTPServer(object): """ A HTTP interface to the storage server. From 2722e81d2b4c8bffeebbab3be9654f1cb40736ad Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 26 Apr 2022 16:33:48 -0400 Subject: [PATCH 0842/2309] News file. --- newsfragments/3880.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3880.minor diff --git a/newsfragments/3880.minor b/newsfragments/3880.minor new file mode 100644 index 000000000..e69de29bb From 49c16f2a1acf804a6cd2c9e66ab46e10c888d463 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 27 Apr 2022 08:38:22 -0400 Subject: [PATCH 0843/2309] Delete 3890.mi remove spurious news fragment --- newsfragments/3890.mi | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 newsfragments/3890.mi diff --git a/newsfragments/3890.mi b/newsfragments/3890.mi deleted file mode 100644 index e69de29bb..000000000 From e16eb6dddfcb46dc530ba5ad81c4710578b6c609 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 28 Apr 2022 11:46:37 -0400 Subject: [PATCH 0844/2309] Better type definitions. --- src/allmydata/storage/http_client.py | 32 ++++++++++++++-------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 09aada555..da350e0c6 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -4,10 +4,10 @@ HTTP client that talks to the HTTP storage server. from __future__ import annotations -from typing import Union, Set, Optional +from typing import Union, Optional, Sequence, Mapping from base64 import b64encode -from attrs import define, field, asdict +from attrs import define, asdict, frozen # TODO Make sure to import Python version? from cbor2 import loads, dumps @@ -134,8 +134,8 @@ def _decode_cbor(response, schema: Schema): class ImmutableCreateResult(object): """Result of creating a storage index for an immutable.""" - already_have: Set[int] - allocated: Set[int] + already_have: set[int] + allocated: set[int] class _TLSContextFactory(CertificateOptions): @@ -420,7 +420,7 @@ class StorageClientImmutables(object): upload_secret, lease_renew_secret, lease_cancel_secret, - ): # type: (bytes, Set[int], int, bytes, bytes, bytes) -> Deferred[ImmutableCreateResult] + ): # type: (bytes, set[int], int, bytes, bytes, bytes) -> Deferred[ImmutableCreateResult] """ Create a new storage index for an immutable. @@ -534,7 +534,7 @@ class StorageClientImmutables(object): ) @inlineCallbacks - def list_shares(self, storage_index): # type: (bytes,) -> Deferred[Set[int]] + def list_shares(self, storage_index): # type: (bytes,) -> Deferred[set[int]] """ Return the set of shares for a given storage index. """ @@ -600,7 +600,7 @@ class StorageClientImmutables(object): ) -@define +@frozen class WriteVector: """Data to write to a chunk.""" @@ -608,7 +608,7 @@ class WriteVector: data: bytes -@define +@frozen class TestVector: """Checks to make on a chunk before writing to it.""" @@ -617,7 +617,7 @@ class TestVector: specimen: bytes -@define +@frozen class ReadVector: """ Reads to do on chunks, as part of a read/test/write operation. @@ -627,13 +627,13 @@ class ReadVector: size: int -@define +@frozen class TestWriteVectors: """Test and write vectors for a specific share.""" - test_vectors: list[TestVector] - write_vectors: list[WriteVector] - new_length: Optional[int] = field(default=None) + test_vectors: Sequence[TestVector] + write_vectors: Sequence[WriteVector] + new_length: Optional[int] = None def asdict(self) -> dict: """Return dictionary suitable for sending over CBOR.""" @@ -644,17 +644,17 @@ class TestWriteVectors: return d -@define +@frozen class ReadTestWriteResult: """Result of sending read-test-write vectors.""" success: bool # Map share numbers to reads corresponding to the request's list of # ReadVectors: - reads: dict[int, list[bytes]] + reads: Mapping[int, Sequence[bytes]] -@define +@frozen class StorageClientMutables: """ APIs for interacting with mutables. From 76d0cfb770f5f886889a0d5731a6fdde9ab3665e Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 28 Apr 2022 11:49:21 -0400 Subject: [PATCH 0845/2309] Correct comment. --- src/allmydata/storage/http_server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index b71877bbf..0169d1463 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -570,7 +570,7 @@ class HTTPServer(object): bucket.advise_corrupt_share(info["reason"].encode("utf-8")) return b"" - ##### Immutable APIs ##### + ##### Mutable APIs ##### @_authorized_route( _app, From b8b1d7515a392984ee34c0d9af8d693fffa9089b Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 28 Apr 2022 11:59:50 -0400 Subject: [PATCH 0846/2309] We can at least be efficient when possible. --- src/allmydata/storage_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 857b19ed7..9c6d5faa2 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -1197,7 +1197,7 @@ class _HTTPStorageServer(object): reads = {} # TODO if shares list is empty, that means list all shares, so we need # to do a query to get that. - assert shares # TODO replace with call to list shares + assert shares # TODO replace with call to list shares if and only if it's empty for share_number in shares: share_reads = reads[share_number] = [] for (offset, length) in readv: From 5ce204ed8d514eeaf5344f5ec3a774311137702a Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 28 Apr 2022 12:18:58 -0400 Subject: [PATCH 0847/2309] Make queries run in parallel. --- src/allmydata/storage_client.py | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 9c6d5faa2..68164e697 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -1194,18 +1194,29 @@ class _HTTPStorageServer(object): @defer.inlineCallbacks def slot_readv(self, storage_index, shares, readv): mutable_client = StorageClientMutables(self._http_client) + pending_reads = {} reads = {} # TODO if shares list is empty, that means list all shares, so we need # to do a query to get that. assert shares # TODO replace with call to list shares if and only if it's empty + + # Start all the queries in parallel: for share_number in shares: - share_reads = reads[share_number] = [] - for (offset, length) in readv: - r = yield mutable_client.read_share_chunk( - storage_index, share_number, offset, length - ) - share_reads.append(r) - defer.returnValue(reads) + share_reads = defer.gatherResults( + [ + mutable_client.read_share_chunk( + storage_index, share_number, offset, length + ) + for (offset, length) in readv + ] + ) + pending_reads[share_number] = share_reads + + # Wait for all the queries to finish: + for share_number, pending_result in pending_reads.items(): + reads[share_number] = yield pending_result + + return reads @defer.inlineCallbacks def slot_testv_and_readv_and_writev( @@ -1239,4 +1250,4 @@ class _HTTPStorageServer(object): storage_index, we_secret, lr_secret, lc_secret, client_tw_vectors, client_read_vectors, ) - defer.returnValue((client_result.success, client_result.reads)) + return (client_result.success, client_result.reads) From 36e3beaa482df538eed024162c0302a71af24751 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 29 Apr 2022 10:03:43 -0400 Subject: [PATCH 0848/2309] Get rid of deprecations builder. --- .circleci/config.yml | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index cf0c66aff..79ce57ed0 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -48,8 +48,6 @@ workflows: {} - "pyinstaller": {} - - "deprecations": - {} - "c-locale": {} # Any locale other than C or UTF-8. @@ -297,20 +295,6 @@ jobs: # aka "Latin 1" LANG: "en_US.ISO-8859-1" - - deprecations: - <<: *DEBIAN - - environment: - <<: *UTF_8_ENVIRONMENT - # Select the deprecations tox environments. - TAHOE_LAFS_TOX_ENVIRONMENT: "deprecations,upcoming-deprecations" - # Put the logs somewhere we can report them. - TAHOE_LAFS_WARNINGS_LOG: "/tmp/artifacts/deprecation-warnings.log" - # The deprecations tox environments don't do coverage measurement. - UPLOAD_COVERAGE: "" - - integration: <<: *DEBIAN From bd631665f4c5d66d16574dbab036c76529a16bca Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 4 May 2022 10:21:16 -0400 Subject: [PATCH 0849/2309] Add logging of HTTP requests from client. --- src/allmydata/storage/http_client.py | 88 +++++++++++++++------------- 1 file changed, 48 insertions(+), 40 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index da350e0c6..b4bde6a95 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -4,6 +4,7 @@ HTTP client that talks to the HTTP storage server. from __future__ import annotations +from eliot import start_action, register_exception_extractor from typing import Union, Optional, Sequence, Mapping from base64 import b64encode @@ -55,6 +56,8 @@ class ClientException(Exception): Exception.__init__(self, code, *additional_args) self.code = code +register_exception_extractor(ClientException, lambda e: {"response_code": e.code}) + # Schemas for server responses. # @@ -280,6 +283,7 @@ class StorageClient(object): ) return headers + @inlineCallbacks def request( self, method, @@ -299,37 +303,40 @@ class StorageClient(object): If ``message_to_serialize`` is set, it will be serialized (by default with CBOR) and set as the request body. """ - headers = self._get_headers(headers) + with start_action(action_type="allmydata:storage:http-client:request", method=method, url=str(url)) as ctx: + headers = self._get_headers(headers) - # Add secrets: - for secret, value in [ - (Secrets.LEASE_RENEW, lease_renew_secret), - (Secrets.LEASE_CANCEL, lease_cancel_secret), - (Secrets.UPLOAD, upload_secret), - (Secrets.WRITE_ENABLER, write_enabler_secret), - ]: - if value is None: - continue - headers.addRawHeader( - "X-Tahoe-Authorization", - b"%s %s" % (secret.value.encode("ascii"), b64encode(value).strip()), - ) - - # Note we can accept CBOR: - headers.addRawHeader("Accept", CBOR_MIME_TYPE) - - # If there's a request message, serialize it and set the Content-Type - # header: - if message_to_serialize is not None: - if "data" in kwargs: - raise TypeError( - "Can't use both `message_to_serialize` and `data` " - "as keyword arguments at the same time" + # Add secrets: + for secret, value in [ + (Secrets.LEASE_RENEW, lease_renew_secret), + (Secrets.LEASE_CANCEL, lease_cancel_secret), + (Secrets.UPLOAD, upload_secret), + (Secrets.WRITE_ENABLER, write_enabler_secret), + ]: + if value is None: + continue + headers.addRawHeader( + "X-Tahoe-Authorization", + b"%s %s" % (secret.value.encode("ascii"), b64encode(value).strip()), ) - kwargs["data"] = dumps(message_to_serialize) - headers.addRawHeader("Content-Type", CBOR_MIME_TYPE) - return self._treq.request(method, url, headers=headers, **kwargs) + # Note we can accept CBOR: + headers.addRawHeader("Accept", CBOR_MIME_TYPE) + + # If there's a request message, serialize it and set the Content-Type + # header: + if message_to_serialize is not None: + if "data" in kwargs: + raise TypeError( + "Can't use both `message_to_serialize` and `data` " + "as keyword arguments at the same time" + ) + kwargs["data"] = dumps(message_to_serialize) + headers.addRawHeader("Content-Type", CBOR_MIME_TYPE) + + response = yield self._treq.request(method, url, headers=headers, **kwargs) + ctx.add_success_fields(response_code=response.code) + return response class StorageClientGeneral(object): @@ -538,18 +545,19 @@ class StorageClientImmutables(object): """ Return the set of shares for a given storage index. """ - url = self._client.relative_url( - "/v1/immutable/{}/shares".format(_encode_si(storage_index)) - ) - response = yield self._client.request( - "GET", - url, - ) - if response.code == http.OK: - body = yield _decode_cbor(response, _SCHEMAS["list_shares"]) - returnValue(set(body)) - else: - raise ClientException(response.code) + with start_action(action_type="allmydata:storage:http-client:immutable:list-shares", storage_index=storage_index) as ctx: + url = self._client.relative_url( + "/v1/immutable/{}/shares".format(_encode_si(storage_index)) + ) + response = yield self._client.request( + "GET", + url, + ) + if response.code == http.OK: + body = yield _decode_cbor(response, _SCHEMAS["list_shares"]) + return set(body) + else: + raise ClientException(response.code) @inlineCallbacks def add_or_renew_lease( From 113eeb0e5908887aac8b03bd59a23fdc6999a3c5 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 4 May 2022 10:21:55 -0400 Subject: [PATCH 0850/2309] News file. --- newsfragments/3891.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3891.minor diff --git a/newsfragments/3891.minor b/newsfragments/3891.minor new file mode 100644 index 000000000..e69de29bb From c1ce74f88d346d92299af11c11ab01789d368c4e Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 4 May 2022 11:03:14 -0400 Subject: [PATCH 0851/2309] Ability to list shares, enabling more of IStorageClient to run over HTTP. --- docs/proposed/http-storage-node-protocol.rst | 2 +- src/allmydata/storage/http_client.py | 20 ++++++++++++ src/allmydata/storage/http_server.py | 14 ++++++++ src/allmydata/storage/server.py | 34 +++++++++++++------- src/allmydata/storage_client.py | 5 +-- src/allmydata/test/test_istorageserver.py | 1 - 6 files changed, 60 insertions(+), 16 deletions(-) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index 3926d9f4a..693ce9290 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -738,7 +738,7 @@ Reading ``GET /v1/mutable/:storage_index/shares`` !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! -Retrieve a list indicating all shares available for the indicated storage index. +Retrieve a set indicating all shares available for the indicated storage index. For example:: [1, 5] diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index da350e0c6..5920d5a5b 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -106,6 +106,11 @@ _SCHEMAS = { share_number = uint """ ), + "mutable_list_shares": Schema( + """ + response = #6.258([* uint]) + """ + ), } @@ -720,3 +725,18 @@ class StorageClientMutables: return read_share_chunk( self._client, "mutable", storage_index, share_number, offset, length ) + + @async_to_deferred + async def list_shares(self, storage_index: bytes) -> set[int]: + """ + List the share numbers for a given storage index. + """ + # TODO unit test all the things + url = self._client.relative_url( + "/v1/mutable/{}/shares".format(_encode_si(storage_index)) + ) + response = await self._client.request("GET", url) + if response.code == http.OK: + return await _decode_cbor(response, _SCHEMAS["mutable_list_shares"]) + else: + raise ClientException(response.code) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 0169d1463..0b407a1c4 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -645,6 +645,20 @@ class HTTPServer(object): ) return data + @_authorized_route( + _app, + set(), + "/v1/mutable//shares", + methods=["GET"], + ) + def list_mutable_shares(self, request, authorization, storage_index): + """List mutable shares for a storage index.""" + try: + shares = self._storage_server.list_mutable_shares(storage_index) + except KeyError: + raise _HTTPError(http.NOT_FOUND) + return self._send_encoded(request, shares) + @implementer(IStreamServerEndpoint) @attr.s diff --git a/src/allmydata/storage/server.py b/src/allmydata/storage/server.py index 9d1a3d6a4..1a0255601 100644 --- a/src/allmydata/storage/server.py +++ b/src/allmydata/storage/server.py @@ -1,18 +1,9 @@ """ Ported to Python 3. """ -from __future__ import division -from __future__ import absolute_import -from __future__ import print_function -from __future__ import unicode_literals - -from future.utils import bytes_to_native_str, PY2 -if PY2: - # Omit open() to get native behavior where open("w") always accepts native - # strings. Omit bytes so we don't leak future's custom bytes. - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, pow, round, super, dict, list, object, range, str, max, min # noqa: F401 -else: - from typing import Dict, Tuple +from __future__ import annotations +from future.utils import bytes_to_native_str +from typing import Dict, Tuple import os, re @@ -699,6 +690,25 @@ class StorageServer(service.MultiService): self) return share + def list_mutable_shares(self, storage_index) -> set[int]: + """List all share numbers for the given mutable. + + Raises ``KeyError`` if the storage index is not known. + """ + # TODO unit test + si_dir = storage_index_to_dir(storage_index) + # shares exist if there is a file for them + bucketdir = os.path.join(self.sharedir, si_dir) + if not os.path.isdir(bucketdir): + raise KeyError("Not found") + result = set() + for sharenum_s in os.listdir(bucketdir): + try: + result.add(int(sharenum_s)) + except ValueError: + continue + return result + def slot_readv(self, storage_index, shares, readv): start = self._clock.seconds() self.count("readv") diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 68164e697..8b2f68a9e 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -1196,9 +1196,10 @@ class _HTTPStorageServer(object): mutable_client = StorageClientMutables(self._http_client) pending_reads = {} reads = {} - # TODO if shares list is empty, that means list all shares, so we need + # If shares list is empty, that means list all shares, so we need # to do a query to get that. - assert shares # TODO replace with call to list shares if and only if it's empty + if not shares: + shares = yield mutable_client.list_shares(storage_index) # Start all the queries in parallel: for share_number in shares: diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index e7b869713..d9fd13acb 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -1154,5 +1154,4 @@ class HTTPMutableAPIsTests( "test_add_lease_renewal", "test_add_new_lease", "test_advise_corrupt_share", - "test_slot_readv_no_shares", } From 852162ba0694f0405666d432d39b464f646eeca0 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 4 May 2022 11:03:35 -0400 Subject: [PATCH 0852/2309] More accurate docs. --- src/allmydata/storage/http_client.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 5920d5a5b..2db28dc72 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -380,16 +380,14 @@ def read_share_chunk( """ Download a chunk of data from a share. - TODO https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3857 Failed - downloads should be transparently retried and redownloaded by the - implementation a few times so that if a failure percolates up, the - caller can assume the failure isn't a short-term blip. + TODO https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3857 Failed downloads + should be transparently retried and redownloaded by the implementation a + few times so that if a failure percolates up, the caller can assume the + failure isn't a short-term blip. - NOTE: the underlying HTTP protocol is much more flexible than this API, - so a future refactor may expand this in order to simplify the calling - code and perhaps download data more efficiently. But then again maybe - the HTTP protocol will be simplified, see - https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3777 + NOTE: the underlying HTTP protocol is somewhat more flexible than this API, + insofar as it doesn't always require a range. In practice a range is + always provided by the current callers. """ url = client.relative_url( "/v1/{}/{}/{}".format(share_type, _encode_si(storage_index), share_number) @@ -717,7 +715,7 @@ class StorageClientMutables: share_number: int, offset: int, length: int, - ) -> bytes: + ) -> Deferred[bytes]: """ Download a chunk of data from a share. """ From 06029d2878b6ad6edc67ae79dc1f59124c6934f0 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 4 May 2022 11:25:13 -0400 Subject: [PATCH 0853/2309] Another end-to-end test passing (albeit with ugly implementation). --- src/allmydata/storage/http_client.py | 6 +++++ src/allmydata/storage/http_server.py | 33 ++++++++++++++--------- src/allmydata/test/test_istorageserver.py | 1 - 3 files changed, 26 insertions(+), 14 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 2db28dc72..0229bef03 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -706,6 +706,12 @@ class StorageClientMutables: if response.code == http.OK: result = await _decode_cbor(response, _SCHEMAS["mutable_read_test_write"]) return ReadTestWriteResult(success=result["success"], reads=result["data"]) + elif response.code == http.UNAUTHORIZED: + # TODO mabye we can fix this to be nicer at some point? Custom + # exception? + from foolscap.api import RemoteException + + raise RemoteException("Authorization failed") else: raise ClientException(response.code, (await response.content())) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 0b407a1c4..748790a72 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -46,6 +46,7 @@ from .common import si_a2b from .immutable import BucketWriter, ConflictingWriteError from ..util.hashutil import timing_safe_compare from ..util.base32 import rfc3548_alphabet +from allmydata.interfaces import BadWriteEnablerError class ClientSecretsException(Exception): @@ -587,19 +588,25 @@ class HTTPServer(object): authorization[Secrets.LEASE_RENEW], authorization[Secrets.LEASE_CANCEL], ) - success, read_data = self._storage_server.slot_testv_and_readv_and_writev( - storage_index, - secrets, - { - k: ( - [(d["offset"], d["size"], b"eq", d["specimen"]) for d in v["test"]], - [(d["offset"], d["data"]) for d in v["write"]], - v["new-length"], - ) - for (k, v) in rtw_request["test-write-vectors"].items() - }, - [(d["offset"], d["size"]) for d in rtw_request["read-vector"]], - ) + try: + success, read_data = self._storage_server.slot_testv_and_readv_and_writev( + storage_index, + secrets, + { + k: ( + [ + (d["offset"], d["size"], b"eq", d["specimen"]) + for d in v["test"] + ], + [(d["offset"], d["data"]) for d in v["write"]], + v["new-length"], + ) + for (k, v) in rtw_request["test-write-vectors"].items() + }, + [(d["offset"], d["size"]) for d in rtw_request["read-vector"]], + ) + except BadWriteEnablerError: + raise _HTTPError(http.UNAUTHORIZED) return self._send_encoded(request, {"success": success, "data": read_data}) @_authorized_route( diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index d9fd13acb..a3e75bbac 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -1150,7 +1150,6 @@ class HTTPMutableAPIsTests( # TODO will be implemented in later tickets SKIP_TESTS = { - "test_STARAW_write_enabler_must_match", "test_add_lease_renewal", "test_add_new_lease", "test_advise_corrupt_share", From 2833bec80e7e6ff069b4b6eee890f99942c3dfc4 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 5 May 2022 12:04:45 -0400 Subject: [PATCH 0854/2309] Unit test the new storage server backend API. --- src/allmydata/storage/server.py | 1 - src/allmydata/test/test_storage.py | 25 +++++++++++++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/src/allmydata/storage/server.py b/src/allmydata/storage/server.py index 1a0255601..b46303cd8 100644 --- a/src/allmydata/storage/server.py +++ b/src/allmydata/storage/server.py @@ -695,7 +695,6 @@ class StorageServer(service.MultiService): Raises ``KeyError`` if the storage index is not known. """ - # TODO unit test si_dir = storage_index_to_dir(storage_index) # shares exist if there is a file for them bucketdir = os.path.join(self.sharedir, si_dir) diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index b37f74c24..8f1ece401 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -1315,6 +1315,31 @@ class MutableServer(unittest.TestCase): self.failUnless(isinstance(readv_data, dict)) self.failUnlessEqual(len(readv_data), 0) + def test_list_mutable_shares(self): + """ + ``StorageServer.list_mutable_shares()`` returns a set of share numbers + for the given storage index, or raises ``KeyError`` if it does not exist at all. + """ + ss = self.create("test_list_mutable_shares") + + # Initially, nothing exists: + with self.assertRaises(KeyError): + ss.list_mutable_shares(b"si1") + + self.allocate(ss, b"si1", b"we1", b"le1", [0, 1, 4, 2], 12) + shares0_1_2_4 = ss.list_mutable_shares(b"si1") + + # Remove share 2, by setting size to 0: + secrets = (self.write_enabler(b"we1"), + self.renew_secret(b"le1"), + self.cancel_secret(b"le1")) + ss.slot_testv_and_readv_and_writev(b"si1", secrets, {2: ([], [], 0)}, []) + shares0_1_4 = ss.list_mutable_shares(b"si1") + self.assertEqual( + (shares0_1_2_4, shares0_1_4), + ({0, 1, 2, 4}, {0, 1, 4}) + ) + def test_bad_magic(self): ss = self.create("test_bad_magic") self.allocate(ss, b"si1", b"we1", next(self._lease_secret), set([0]), 10) From b3fed56c00d03599b4a8479e5de0a36c696c7a97 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 5 May 2022 12:11:09 -0400 Subject: [PATCH 0855/2309] Move Foolscap compatibility to a better place. --- src/allmydata/storage/http_client.py | 6 ------ src/allmydata/storage_client.py | 16 +++++++++++----- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 0229bef03..2db28dc72 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -706,12 +706,6 @@ class StorageClientMutables: if response.code == http.OK: result = await _decode_cbor(response, _SCHEMAS["mutable_read_test_write"]) return ReadTestWriteResult(success=result["success"], reads=result["data"]) - elif response.code == http.UNAUTHORIZED: - # TODO mabye we can fix this to be nicer at some point? Custom - # exception? - from foolscap.api import RemoteException - - raise RemoteException("Authorization failed") else: raise ClientException(response.code, (await response.content())) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 8b2f68a9e..cd489a307 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -50,6 +50,7 @@ from zope.interface import ( Interface, implementer, ) +from twisted.web import http from twisted.internet import defer from twisted.application import service from twisted.plugin import ( @@ -78,7 +79,7 @@ from allmydata.util.dictutil import BytesKeyDict, UnicodeKeyDict from allmydata.storage.http_client import ( StorageClient, StorageClientImmutables, StorageClientGeneral, ClientException as HTTPClientException, StorageClientMutables, - ReadVector, TestWriteVectors, WriteVector, TestVector + ReadVector, TestWriteVectors, WriteVector, TestVector, ClientException ) @@ -1247,8 +1248,13 @@ class _HTTPStorageServer(object): ReadVector(offset=offset, size=size) for (offset, size) in r_vector ] - client_result = yield mutable_client.read_test_write_chunks( - storage_index, we_secret, lr_secret, lc_secret, client_tw_vectors, - client_read_vectors, - ) + try: + client_result = yield mutable_client.read_test_write_chunks( + storage_index, we_secret, lr_secret, lc_secret, client_tw_vectors, + client_read_vectors, + ) + except ClientException as e: + if e.code == http.UNAUTHORIZED: + raise RemoteException("Unauthorized write, possibly you passed the wrong write enabler?") + raise return (client_result.success, client_result.reads) From 5b0762d3a3a8fa1eb98aa6cd3b5b4d14e53047a3 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 10 May 2022 13:59:58 -0400 Subject: [PATCH 0856/2309] Workaround for autobahn issues. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index c84d0ecde..2b4fd6988 100644 --- a/setup.py +++ b/setup.py @@ -114,7 +114,7 @@ install_requires = [ "attrs >= 18.2.0", # WebSocket library for twisted and asyncio - "autobahn >= 19.5.2", + "autobahn < 22.4.1", # remove this when https://github.com/crossbario/autobahn-python/issues/1566 is fixed # Support for Python 3 transition "future >= 0.18.2", From 6f5a0e43ebceb730c97598b4fbe49f65f052e44b Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 11 May 2022 10:41:36 -0400 Subject: [PATCH 0857/2309] Implement advise_corrupt_share for mutables. --- src/allmydata/storage/http_client.py | 51 ++++++++++++++++------- src/allmydata/storage/http_server.py | 20 +++++++++ src/allmydata/storage_client.py | 12 +++--- src/allmydata/test/test_istorageserver.py | 1 - 4 files changed, 64 insertions(+), 20 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 2db28dc72..f39ed7d1a 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -406,6 +406,30 @@ def read_share_chunk( raise ClientException(response.code) +@async_to_deferred +async def advise_corrupt_share( + client: StorageClient, + share_type: str, + storage_index: bytes, + share_number: int, + reason: str, +): + assert isinstance(reason, str) + url = client.relative_url( + "/v1/{}/{}/{}/corrupt".format( + share_type, _encode_si(storage_index), share_number + ) + ) + message = {"reason": reason} + response = await client.request("POST", url, message_to_serialize=message) + if response.code == http.OK: + return + else: + raise ClientException( + response.code, + ) + + @define class StorageClientImmutables(object): """ @@ -579,7 +603,6 @@ class StorageClientImmutables(object): else: raise ClientException(response.code) - @inlineCallbacks def advise_corrupt_share( self, storage_index: bytes, @@ -587,20 +610,9 @@ class StorageClientImmutables(object): reason: str, ): """Indicate a share has been corrupted, with a human-readable message.""" - assert isinstance(reason, str) - url = self._client.relative_url( - "/v1/immutable/{}/{}/corrupt".format( - _encode_si(storage_index), share_number - ) + return advise_corrupt_share( + self._client, "immutable", storage_index, share_number, reason ) - message = {"reason": reason} - response = yield self._client.request("POST", url, message_to_serialize=message) - if response.code == http.OK: - return - else: - raise ClientException( - response.code, - ) @frozen @@ -738,3 +750,14 @@ class StorageClientMutables: return await _decode_cbor(response, _SCHEMAS["mutable_list_shares"]) else: raise ClientException(response.code) + + def advise_corrupt_share( + self, + storage_index: bytes, + share_number: int, + reason: str, + ): + """Indicate a share has been corrupted, with a human-readable message.""" + return advise_corrupt_share( + self._client, "mutable", storage_index, share_number, reason + ) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 748790a72..102a33e90 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -666,6 +666,26 @@ class HTTPServer(object): raise _HTTPError(http.NOT_FOUND) return self._send_encoded(request, shares) + @_authorized_route( + _app, + set(), + "/v1/mutable///corrupt", + methods=["POST"], + ) + def advise_corrupt_share_mutable( + self, request, authorization, storage_index, share_number + ): + """Indicate that given share is corrupt, with a text reason.""" + # TODO unit test all the paths + if not self._storage_server._share_exists(storage_index, share_number): + raise _HTTPError(http.NOT_FOUND) + + info = self._read_encoded(request, _SCHEMAS["advise_corrupt_share"]) + self._storage_server.advise_corrupt_share( + b"mutable", storage_index, share_number, info["reason"].encode("utf-8") + ) + return b"" + @implementer(IStreamServerEndpoint) @attr.s diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index cd489a307..c83527600 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -1185,12 +1185,14 @@ class _HTTPStorageServer(object): reason: bytes ): if share_type == b"immutable": - imm_client = StorageClientImmutables(self._http_client) - return imm_client.advise_corrupt_share( - storage_index, shnum, str(reason, "utf-8", errors="backslashreplace") - ) + client = StorageClientImmutables(self._http_client) + elif share_type == b"mutable": + client = StorageClientMutables(self._http_client) else: - raise NotImplementedError() # future tickets + raise ValueError("Unknown share type") + return client.advise_corrupt_share( + storage_index, shnum, str(reason, "utf-8", errors="backslashreplace") + ) @defer.inlineCallbacks def slot_readv(self, storage_index, shares, readv): diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index a3e75bbac..70543cbf0 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -1152,5 +1152,4 @@ class HTTPMutableAPIsTests( SKIP_TESTS = { "test_add_lease_renewal", "test_add_new_lease", - "test_advise_corrupt_share", } From 7ae682af27d878f7b07e4a0e533efe105348da95 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 11 May 2022 10:41:56 -0400 Subject: [PATCH 0858/2309] News file. --- newsfragments/3893.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3893.minor diff --git a/newsfragments/3893.minor b/newsfragments/3893.minor new file mode 100644 index 000000000..e69de29bb From 4afe3eb224d6f26ac768aaccbca68c69035d57d4 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 11 May 2022 10:58:13 -0400 Subject: [PATCH 0859/2309] Clarify sets vs lists some more. --- docs/proposed/http-storage-node-protocol.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index 693ce9290..7e0b4a542 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -350,8 +350,10 @@ Because of the simple types used throughout and the equivalence described in `RFC 7049`_ these examples should be representative regardless of which of these two encodings is chosen. +The one exception is sets. For CBOR messages, any sequence that is semantically a set (i.e. no repeated values allowed, order doesn't matter, and elements are hashable in Python) should be sent as a set. Tag 6.258 is used to indicate sets in CBOR; see `the CBOR registry `_ for more details. +Sets will be represented as JSON lists in examples because JSON doesn't support sets. HTTP Design ~~~~~~~~~~~ @@ -739,7 +741,7 @@ Reading !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! Retrieve a set indicating all shares available for the indicated storage index. -For example:: +For example (this is shown as list, since it will be list for JSON, but will be set for CBOR):: [1, 5] From 07e16b80b5df22a5620d390cb34032a55e62e795 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 11 May 2022 11:00:05 -0400 Subject: [PATCH 0860/2309] Better name. --- src/allmydata/storage/http_server.py | 4 ++-- src/allmydata/storage/server.py | 2 +- src/allmydata/test/test_storage.py | 12 ++++++------ 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 748790a72..96e906e43 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -658,10 +658,10 @@ class HTTPServer(object): "/v1/mutable//shares", methods=["GET"], ) - def list_mutable_shares(self, request, authorization, storage_index): + def enumerate_mutable_shares(self, request, authorization, storage_index): """List mutable shares for a storage index.""" try: - shares = self._storage_server.list_mutable_shares(storage_index) + shares = self._storage_server.enumerate_mutable_shares(storage_index) except KeyError: raise _HTTPError(http.NOT_FOUND) return self._send_encoded(request, shares) diff --git a/src/allmydata/storage/server.py b/src/allmydata/storage/server.py index b46303cd8..ab7947bf9 100644 --- a/src/allmydata/storage/server.py +++ b/src/allmydata/storage/server.py @@ -690,7 +690,7 @@ class StorageServer(service.MultiService): self) return share - def list_mutable_shares(self, storage_index) -> set[int]: + def enumerate_mutable_shares(self, storage_index) -> set[int]: """List all share numbers for the given mutable. Raises ``KeyError`` if the storage index is not known. diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index 8f1ece401..9bc218fa6 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -1315,26 +1315,26 @@ class MutableServer(unittest.TestCase): self.failUnless(isinstance(readv_data, dict)) self.failUnlessEqual(len(readv_data), 0) - def test_list_mutable_shares(self): + def test_enumerate_mutable_shares(self): """ - ``StorageServer.list_mutable_shares()`` returns a set of share numbers + ``StorageServer.enumerate_mutable_shares()`` returns a set of share numbers for the given storage index, or raises ``KeyError`` if it does not exist at all. """ - ss = self.create("test_list_mutable_shares") + ss = self.create("test_enumerate_mutable_shares") # Initially, nothing exists: with self.assertRaises(KeyError): - ss.list_mutable_shares(b"si1") + ss.enumerate_mutable_shares(b"si1") self.allocate(ss, b"si1", b"we1", b"le1", [0, 1, 4, 2], 12) - shares0_1_2_4 = ss.list_mutable_shares(b"si1") + shares0_1_2_4 = ss.enumerate_mutable_shares(b"si1") # Remove share 2, by setting size to 0: secrets = (self.write_enabler(b"we1"), self.renew_secret(b"le1"), self.cancel_secret(b"le1")) ss.slot_testv_and_readv_and_writev(b"si1", secrets, {2: ([], [], 0)}, []) - shares0_1_4 = ss.list_mutable_shares(b"si1") + shares0_1_4 = ss.enumerate_mutable_shares(b"si1") self.assertEqual( (shares0_1_2_4, shares0_1_4), ({0, 1, 2, 4}, {0, 1, 4}) From 6d412a017c34fc6f6528c89b1443d9bdf7caee64 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 11 May 2022 11:00:46 -0400 Subject: [PATCH 0861/2309] Type annotation. --- src/allmydata/storage/server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/storage/server.py b/src/allmydata/storage/server.py index ab7947bf9..f1b835780 100644 --- a/src/allmydata/storage/server.py +++ b/src/allmydata/storage/server.py @@ -690,7 +690,7 @@ class StorageServer(service.MultiService): self) return share - def enumerate_mutable_shares(self, storage_index) -> set[int]: + def enumerate_mutable_shares(self, storage_index: bytes) -> set[int]: """List all share numbers for the given mutable. Raises ``KeyError`` if the storage index is not known. From 4b62ec082bed7ecc33612741dfbffc16868ca3ff Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 11 May 2022 11:11:24 -0400 Subject: [PATCH 0862/2309] Match Foolscap behavior for slot_readv of unknown storage index. --- src/allmydata/storage_client.py | 8 +++++++- src/allmydata/test/test_istorageserver.py | 16 ++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index cd489a307..83a1233f5 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -1200,7 +1200,13 @@ class _HTTPStorageServer(object): # If shares list is empty, that means list all shares, so we need # to do a query to get that. if not shares: - shares = yield mutable_client.list_shares(storage_index) + try: + shares = yield mutable_client.list_shares(storage_index) + except ClientException as e: + if e.code == http.NOT_FOUND: + shares = set() + else: + raise # Start all the queries in parallel: for share_number in shares: diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index a3e75bbac..66535ddda 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -854,6 +854,22 @@ class IStorageServerMutableAPIsTestsMixin(object): {0: [b"abcdefg"], 1: [b"0123456"], 2: [b"9876543"]}, ) + @inlineCallbacks + def test_slot_readv_unknown_storage_index(self): + """ + With unknown storage index, ``IStorageServer.slot_readv()`` TODO. + """ + storage_index = new_storage_index() + reads = yield self.storage_client.slot_readv( + storage_index, + shares=[], + readv=[(0, 7)], + ) + self.assertEqual( + reads, + {}, + ) + @inlineCallbacks def create_slot(self): """Create a slot with sharenum 0.""" From 457db8f992c62e8f5c5bc4cdcb6e99f097062f08 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 11 May 2022 11:17:57 -0400 Subject: [PATCH 0863/2309] Get rid of the "no such storage index" edge case, since it's not really necessary. --- src/allmydata/storage/http_server.py | 5 +---- src/allmydata/storage/server.py | 7 ++----- src/allmydata/storage_client.py | 8 +------- src/allmydata/test/test_storage.py | 12 ++++++------ 4 files changed, 10 insertions(+), 22 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 96e906e43..a1641f563 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -660,10 +660,7 @@ class HTTPServer(object): ) def enumerate_mutable_shares(self, request, authorization, storage_index): """List mutable shares for a storage index.""" - try: - shares = self._storage_server.enumerate_mutable_shares(storage_index) - except KeyError: - raise _HTTPError(http.NOT_FOUND) + shares = self._storage_server.enumerate_mutable_shares(storage_index) return self._send_encoded(request, shares) diff --git a/src/allmydata/storage/server.py b/src/allmydata/storage/server.py index f1b835780..bcf44dc30 100644 --- a/src/allmydata/storage/server.py +++ b/src/allmydata/storage/server.py @@ -691,15 +691,12 @@ class StorageServer(service.MultiService): return share def enumerate_mutable_shares(self, storage_index: bytes) -> set[int]: - """List all share numbers for the given mutable. - - Raises ``KeyError`` if the storage index is not known. - """ + """Return all share numbers for the given mutable.""" si_dir = storage_index_to_dir(storage_index) # shares exist if there is a file for them bucketdir = os.path.join(self.sharedir, si_dir) if not os.path.isdir(bucketdir): - raise KeyError("Not found") + return set() result = set() for sharenum_s in os.listdir(bucketdir): try: diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 83a1233f5..cd489a307 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -1200,13 +1200,7 @@ class _HTTPStorageServer(object): # If shares list is empty, that means list all shares, so we need # to do a query to get that. if not shares: - try: - shares = yield mutable_client.list_shares(storage_index) - except ClientException as e: - if e.code == http.NOT_FOUND: - shares = set() - else: - raise + shares = yield mutable_client.list_shares(storage_index) # Start all the queries in parallel: for share_number in shares: diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index 9bc218fa6..65d09de25 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -1317,14 +1317,14 @@ class MutableServer(unittest.TestCase): def test_enumerate_mutable_shares(self): """ - ``StorageServer.enumerate_mutable_shares()`` returns a set of share numbers - for the given storage index, or raises ``KeyError`` if it does not exist at all. + ``StorageServer.enumerate_mutable_shares()`` returns a set of share + numbers for the given storage index, or an empty set if it does not + exist at all. """ ss = self.create("test_enumerate_mutable_shares") # Initially, nothing exists: - with self.assertRaises(KeyError): - ss.enumerate_mutable_shares(b"si1") + empty = ss.enumerate_mutable_shares(b"si1") self.allocate(ss, b"si1", b"we1", b"le1", [0, 1, 4, 2], 12) shares0_1_2_4 = ss.enumerate_mutable_shares(b"si1") @@ -1336,8 +1336,8 @@ class MutableServer(unittest.TestCase): ss.slot_testv_and_readv_and_writev(b"si1", secrets, {2: ([], [], 0)}, []) shares0_1_4 = ss.enumerate_mutable_shares(b"si1") self.assertEqual( - (shares0_1_2_4, shares0_1_4), - ({0, 1, 2, 4}, {0, 1, 4}) + (empty, shares0_1_2_4, shares0_1_4), + (set(), {0, 1, 2, 4}, {0, 1, 4}) ) def test_bad_magic(self): From 821bac3ddf4bc829d4a0724404242974fa2f1d0a Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 11 May 2022 11:50:01 -0400 Subject: [PATCH 0864/2309] Test another lease edge case. --- src/allmydata/storage_client.py | 16 ++++++++++++---- src/allmydata/test/test_istorageserver.py | 15 +++++++++++++++ 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index c83527600..e8d0e003a 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -76,6 +76,7 @@ from allmydata.util.observer import ObserverList from allmydata.util.rrefutil import add_version_to_remote_reference from allmydata.util.hashutil import permute_server_hash from allmydata.util.dictutil import BytesKeyDict, UnicodeKeyDict +from allmydata.util.deferredutil import async_to_deferred from allmydata.storage.http_client import ( StorageClient, StorageClientImmutables, StorageClientGeneral, ClientException as HTTPClientException, StorageClientMutables, @@ -1166,16 +1167,23 @@ class _HTTPStorageServer(object): for share_num in share_numbers }) - def add_lease( + @async_to_deferred + async def add_lease( self, storage_index, renew_secret, cancel_secret ): immutable_client = StorageClientImmutables(self._http_client) - return immutable_client.add_or_renew_lease( - storage_index, renew_secret, cancel_secret - ) + try: + await immutable_client.add_or_renew_lease( + storage_index, renew_secret, cancel_secret + ) + except ClientException as e: + if e.code == http.NOT_FOUND: + # Silently do nothing, as is the case for the Foolscap client + return + raise def advise_corrupt_share( self, diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index abb2e0fc4..cee80f8fb 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -459,6 +459,21 @@ class IStorageServerImmutableAPIsTestsMixin(object): lease.get_expiration_time() - self.fake_time() > (31 * 24 * 60 * 60 - 10) ) + @inlineCallbacks + def test_add_lease_non_existent(self): + """ + If the storage index doesn't exist, adding the lease silently does nothing. + """ + storage_index = new_storage_index() + self.assertEqual(list(self.server.get_leases(storage_index)), []) + + renew_secret = new_secret() + cancel_secret = new_secret() + + # Add a lease: + yield self.storage_client.add_lease(storage_index, renew_secret, cancel_secret) + self.assertEqual(list(self.server.get_leases(storage_index)), []) + @inlineCallbacks def test_add_lease_renewal(self): """ From b8735c79daefb751b4d81ba7631a81b37904b732 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 11 May 2022 11:50:29 -0400 Subject: [PATCH 0865/2309] Fix docstring. --- src/allmydata/test/test_istorageserver.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index cee80f8fb..c0dd50590 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -872,7 +872,8 @@ class IStorageServerMutableAPIsTestsMixin(object): @inlineCallbacks def test_slot_readv_unknown_storage_index(self): """ - With unknown storage index, ``IStorageServer.slot_readv()`` TODO. + With unknown storage index, ``IStorageServer.slot_readv()`` returns + empty dict. """ storage_index = new_storage_index() reads = yield self.storage_client.slot_readv( From f3cf13154da2c03e39fa3249384ec6e01ef90735 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 11 May 2022 12:00:27 -0400 Subject: [PATCH 0866/2309] Setup HTTP lease APIs for immutables too. --- src/allmydata/storage/http_client.py | 50 +++++++++++------------ src/allmydata/storage/http_server.py | 2 +- src/allmydata/storage_client.py | 2 +- src/allmydata/test/test_istorageserver.py | 6 --- src/allmydata/test/test_storage_http.py | 7 ++-- 5 files changed, 31 insertions(+), 36 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index f39ed7d1a..167d2394a 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -355,6 +355,31 @@ class StorageClientGeneral(object): decoded_response = yield _decode_cbor(response, _SCHEMAS["get_version"]) returnValue(decoded_response) + @inlineCallbacks + def add_or_renew_lease( + self, storage_index: bytes, renew_secret: bytes, cancel_secret: bytes + ): + """ + Add or renew a lease. + + If the renewal secret matches an existing lease, it is renewed. + Otherwise a new lease is added. + """ + url = self._client.relative_url( + "/v1/lease/{}".format(_encode_si(storage_index)) + ) + response = yield self._client.request( + "PUT", + url, + lease_renew_secret=renew_secret, + lease_cancel_secret=cancel_secret, + ) + + if response.code == http.NO_CONTENT: + return + else: + raise ClientException(response.code) + @define class UploadProgress(object): @@ -578,31 +603,6 @@ class StorageClientImmutables(object): else: raise ClientException(response.code) - @inlineCallbacks - def add_or_renew_lease( - self, storage_index: bytes, renew_secret: bytes, cancel_secret: bytes - ): - """ - Add or renew a lease. - - If the renewal secret matches an existing lease, it is renewed. - Otherwise a new lease is added. - """ - url = self._client.relative_url( - "/v1/lease/{}".format(_encode_si(storage_index)) - ) - response = yield self._client.request( - "PUT", - url, - lease_renew_secret=renew_secret, - lease_cancel_secret=cancel_secret, - ) - - if response.code == http.NO_CONTENT: - return - else: - raise ClientException(response.code) - def advise_corrupt_share( self, storage_index: bytes, diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index db73b6a86..709c1fda5 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -539,7 +539,7 @@ class HTTPServer(object): ) def add_or_renew_lease(self, request, authorization, storage_index): """Update the lease for an immutable share.""" - if not self._storage_server.get_buckets(storage_index): + if not list(self._storage_server._get_bucket_shares(storage_index)): raise _HTTPError(http.NOT_FOUND) # Checking of the renewal secret is done by the backend. diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index e8d0e003a..c529c4513 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -1174,7 +1174,7 @@ class _HTTPStorageServer(object): renew_secret, cancel_secret ): - immutable_client = StorageClientImmutables(self._http_client) + immutable_client = StorageClientGeneral(self._http_client) try: await immutable_client.add_or_renew_lease( storage_index, renew_secret, cancel_secret diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index c0dd50590..39675336f 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -1179,9 +1179,3 @@ class HTTPMutableAPIsTests( _HTTPMixin, IStorageServerMutableAPIsTestsMixin, AsyncTestCase ): """HTTP-specific tests for mutable ``IStorageServer`` APIs.""" - - # TODO will be implemented in later tickets - SKIP_TESTS = { - "test_add_lease_renewal", - "test_add_new_lease", - } diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index df781012e..fcc2401f2 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -448,6 +448,7 @@ class ImmutableHTTPAPITests(SyncTestCase): super(ImmutableHTTPAPITests, self).setUp() self.http = self.useFixture(HttpTestFixture()) self.imm_client = StorageClientImmutables(self.http.client) + self.general_client = StorageClientGeneral(self.http.client) def create_upload(self, share_numbers, length): """ @@ -1081,7 +1082,7 @@ class ImmutableHTTPAPITests(SyncTestCase): # We renew the lease: result_of( - self.imm_client.add_or_renew_lease( + self.general_client.add_or_renew_lease( storage_index, lease_secret, lease_secret ) ) @@ -1092,7 +1093,7 @@ class ImmutableHTTPAPITests(SyncTestCase): # We create a new lease: lease_secret2 = urandom(32) result_of( - self.imm_client.add_or_renew_lease( + self.general_client.add_or_renew_lease( storage_index, lease_secret2, lease_secret2 ) ) @@ -1108,7 +1109,7 @@ class ImmutableHTTPAPITests(SyncTestCase): storage_index = urandom(16) secret = b"A" * 32 with assert_fails_with_http_code(self, http.NOT_FOUND): - result_of(self.imm_client.add_or_renew_lease(storage_index, secret, secret)) + result_of(self.general_client.add_or_renew_lease(storage_index, secret, secret)) def test_advise_corrupt_share(self): """ From a54b443f9d2658cb6e196570a7a74681a8bec44d Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 12 May 2022 09:44:30 -0400 Subject: [PATCH 0867/2309] It's not an immutable client anymore. --- src/allmydata/storage_client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index c529c4513..0f66e8e4a 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -1174,9 +1174,9 @@ class _HTTPStorageServer(object): renew_secret, cancel_secret ): - immutable_client = StorageClientGeneral(self._http_client) + client = StorageClientGeneral(self._http_client) try: - await immutable_client.add_or_renew_lease( + await client.add_or_renew_lease( storage_index, renew_secret, cancel_secret ) except ClientException as e: From b0b67826e8c9a026879c68c04c13d2cce9a6466e Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 13 May 2022 12:58:55 -0400 Subject: [PATCH 0868/2309] More verbose output is helpful when debugging. --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 859cf18e0..fc95a0469 100644 --- a/tox.ini +++ b/tox.ini @@ -97,7 +97,7 @@ setenv = COVERAGE_PROCESS_START=.coveragerc commands = # NOTE: 'run with "py.test --keep-tempdir -s -v integration/" to debug failures' - py.test --timeout=1800 --coverage -v {posargs:integration} + py.test --timeout=1800 --coverage -s -v {posargs:integration} coverage combine coverage report From 20b021809c0a2cf3bd2abf5991de5638b48134b9 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 13 May 2022 12:59:04 -0400 Subject: [PATCH 0869/2309] Fix(?) the intermittently failing test. --- integration/test_tor.py | 6 +++++- newsfragments/3895.minor | 0 2 files changed, 5 insertions(+), 1 deletion(-) create mode 100644 newsfragments/3895.minor diff --git a/integration/test_tor.py b/integration/test_tor.py index b0419f0d2..5b701287c 100644 --- a/integration/test_tor.py +++ b/integration/test_tor.py @@ -21,7 +21,8 @@ from . import util from twisted.python.filepath import ( FilePath, ) - +from twisted.internet.task import deferLater +from twisted.internet import reactor from allmydata.test.common import ( write_introducer, ) @@ -68,6 +69,9 @@ def test_onion_service_storage(reactor, request, temp_dir, flog_gatherer, tor_ne cap = proto.output.getvalue().strip().split()[-1] print("TEH CAP!", cap) + # For some reason a wait is needed, or sometimes the get fails... + yield deferLater(reactor, 2, lambda: None) + proto = util._CollectOutputProtocol(capture_stderr=False) reactor.spawnProcess( proto, diff --git a/newsfragments/3895.minor b/newsfragments/3895.minor new file mode 100644 index 000000000..e69de29bb From 757b4492d75c899d21809ff51f3383189c7eb3c7 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 13 May 2022 13:29:08 -0400 Subject: [PATCH 0870/2309] A more semantically correct fix. --- integration/test_tor.py | 16 ++++++++-------- integration/util.py | 9 +++++---- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/integration/test_tor.py b/integration/test_tor.py index 5b701287c..d17e0f5cf 100644 --- a/integration/test_tor.py +++ b/integration/test_tor.py @@ -21,8 +21,7 @@ from . import util from twisted.python.filepath import ( FilePath, ) -from twisted.internet.task import deferLater -from twisted.internet import reactor + from allmydata.test.common import ( write_introducer, ) @@ -41,8 +40,11 @@ if PY2: @pytest_twisted.inlineCallbacks def test_onion_service_storage(reactor, request, temp_dir, flog_gatherer, tor_network, tor_introducer_furl): - yield _create_anonymous_node(reactor, 'carol', 8008, request, temp_dir, flog_gatherer, tor_network, tor_introducer_furl) - yield _create_anonymous_node(reactor, 'dave', 8009, request, temp_dir, flog_gatherer, tor_network, tor_introducer_furl) + carol = yield _create_anonymous_node(reactor, 'carol', 8008, request, temp_dir, flog_gatherer, tor_network, tor_introducer_furl) + dave = yield _create_anonymous_node(reactor, 'dave', 8009, request, temp_dir, flog_gatherer, tor_network, tor_introducer_furl) + util.await_client_ready(carol, expected_number_of_servers=2) + util.await_client_ready(dave, expected_number_of_servers=2) + # ensure both nodes are connected to "a grid" by uploading # something via carol, and retrieve it using dave. gold_path = join(temp_dir, "gold") @@ -69,9 +71,6 @@ def test_onion_service_storage(reactor, request, temp_dir, flog_gatherer, tor_ne cap = proto.output.getvalue().strip().split()[-1] print("TEH CAP!", cap) - # For some reason a wait is needed, or sometimes the get fails... - yield deferLater(reactor, 2, lambda: None) - proto = util._CollectOutputProtocol(capture_stderr=False) reactor.spawnProcess( proto, @@ -147,5 +146,6 @@ shares.total = 2 f.write(node_config) print("running") - yield util._run_node(reactor, node_dir.path, request, None) + result = yield util._run_node(reactor, node_dir.path, request, None) print("okay, launched") + return result diff --git a/integration/util.py b/integration/util.py index 7c7a1efd2..0ec824f82 100644 --- a/integration/util.py +++ b/integration/util.py @@ -482,14 +482,15 @@ def web_post(tahoe, uri_fragment, **kwargs): return resp.content -def await_client_ready(tahoe, timeout=10, liveness=60*2): +def await_client_ready(tahoe, timeout=10, liveness=60*2, expected_number_of_servers=1): """ Uses the status API to wait for a client-type node (in `tahoe`, a `TahoeProcess` instance usually from a fixture e.g. `alice`) to be 'ready'. A client is deemed ready if: - it answers `http:///statistics/?t=json/` - - there is at least one storage-server connected + - there is at least one storage-server connected (configurable via + ``expected_number_of_servers``) - every storage-server has a "last_received_data" and it is within the last `liveness` seconds @@ -506,8 +507,8 @@ def await_client_ready(tahoe, timeout=10, liveness=60*2): time.sleep(1) continue - if len(js['servers']) == 0: - print("waiting because no servers at all") + if len(js['servers']) != expected_number_of_servers: + print("waiting because insufficient servers") time.sleep(1) continue server_times = [ From f752f547ba50e283a13abac8b9764cef305ad2a0 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 13 May 2022 13:30:47 -0400 Subject: [PATCH 0871/2309] More servers is fine. --- integration/util.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/integration/util.py b/integration/util.py index 0ec824f82..ad9249e45 100644 --- a/integration/util.py +++ b/integration/util.py @@ -482,7 +482,7 @@ def web_post(tahoe, uri_fragment, **kwargs): return resp.content -def await_client_ready(tahoe, timeout=10, liveness=60*2, expected_number_of_servers=1): +def await_client_ready(tahoe, timeout=10, liveness=60*2, minimum_number_of_servers=1): """ Uses the status API to wait for a client-type node (in `tahoe`, a `TahoeProcess` instance usually from a fixture e.g. `alice`) to be @@ -490,7 +490,7 @@ def await_client_ready(tahoe, timeout=10, liveness=60*2, expected_number_of_serv - it answers `http:///statistics/?t=json/` - there is at least one storage-server connected (configurable via - ``expected_number_of_servers``) + ``minimum_number_of_servers``) - every storage-server has a "last_received_data" and it is within the last `liveness` seconds @@ -507,7 +507,7 @@ def await_client_ready(tahoe, timeout=10, liveness=60*2, expected_number_of_serv time.sleep(1) continue - if len(js['servers']) != expected_number_of_servers: + if len(js['servers']) < minimum_number_of_servers: print("waiting because insufficient servers") time.sleep(1) continue From 69f1244c5a85914535008911b231b1483fdee953 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 13 May 2022 13:42:10 -0400 Subject: [PATCH 0872/2309] Fix keyword argument name. --- integration/test_tor.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/integration/test_tor.py b/integration/test_tor.py index d17e0f5cf..c78fa8098 100644 --- a/integration/test_tor.py +++ b/integration/test_tor.py @@ -42,8 +42,8 @@ if PY2: def test_onion_service_storage(reactor, request, temp_dir, flog_gatherer, tor_network, tor_introducer_furl): carol = yield _create_anonymous_node(reactor, 'carol', 8008, request, temp_dir, flog_gatherer, tor_network, tor_introducer_furl) dave = yield _create_anonymous_node(reactor, 'dave', 8009, request, temp_dir, flog_gatherer, tor_network, tor_introducer_furl) - util.await_client_ready(carol, expected_number_of_servers=2) - util.await_client_ready(dave, expected_number_of_servers=2) + util.await_client_ready(carol, minimum_number_of_servers=2) + util.await_client_ready(dave, minimum_number_of_servers=2) # ensure both nodes are connected to "a grid" by uploading # something via carol, and retrieve it using dave. From 3abf992321c62728cf194090380dc46c32dc0156 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 13 May 2022 14:05:53 -0400 Subject: [PATCH 0873/2309] Autobahn regression workaround. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index c84d0ecde..2b4fd6988 100644 --- a/setup.py +++ b/setup.py @@ -114,7 +114,7 @@ install_requires = [ "attrs >= 18.2.0", # WebSocket library for twisted and asyncio - "autobahn >= 19.5.2", + "autobahn < 22.4.1", # remove this when https://github.com/crossbario/autobahn-python/issues/1566 is fixed # Support for Python 3 transition "future >= 0.18.2", From da4deab167187ec4baf02f84da8e8ae7e03a6a8a Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 16 May 2022 11:19:46 -0400 Subject: [PATCH 0874/2309] Note version with fix. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 2b4fd6988..d07031cd9 100644 --- a/setup.py +++ b/setup.py @@ -114,7 +114,7 @@ install_requires = [ "attrs >= 18.2.0", # WebSocket library for twisted and asyncio - "autobahn < 22.4.1", # remove this when https://github.com/crossbario/autobahn-python/issues/1566 is fixed + "autobahn < 22.4.1", # remove this when 22.4.3 is released # Support for Python 3 transition "future >= 0.18.2", From d209065a6e680eb3e42e4e5bad89655d4e3d7ec0 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 16 May 2022 11:22:44 -0400 Subject: [PATCH 0875/2309] Fix type issue, and modernize slightly. --- src/allmydata/storage_client.py | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 0f66e8e4a..c63bfccff 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -5,10 +5,6 @@ the foolscap-based server implemented in src/allmydata/storage/*.py . Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals # roadmap: # @@ -34,14 +30,10 @@ from __future__ import unicode_literals # # 6: implement other sorts of IStorageClient classes: S3, etc -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from six import ensure_text - +from typing import Union import re, time, hashlib from os import urandom -# On Python 2 this will be the backport. from configparser import NoSectionError import attr @@ -1193,7 +1185,7 @@ class _HTTPStorageServer(object): reason: bytes ): if share_type == b"immutable": - client = StorageClientImmutables(self._http_client) + client : Union[StorageClientImmutables, StorageClientMutables] = StorageClientImmutables(self._http_client) elif share_type == b"mutable": client = StorageClientMutables(self._http_client) else: From 32a11662a277b976af08194a42358e727fdf8ee8 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 18 May 2022 12:55:55 -0400 Subject: [PATCH 0876/2309] Install a specific version. --- integration/install-tor.sh | 2 +- integration/test_tor.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/integration/install-tor.sh b/integration/install-tor.sh index 66fa64cb1..97a7f9465 100755 --- a/integration/install-tor.sh +++ b/integration/install-tor.sh @@ -791,4 +791,4 @@ keSPmmDrjl8cySCNsMo= EOF ${SUDO} apt-get --quiet update -${SUDO} apt-get --quiet --yes install tor deb.torproject.org-keyring +${SUDO} apt-get --quiet --yes install tor=0.4.4.5-1 deb.torproject.org-keyring diff --git a/integration/test_tor.py b/integration/test_tor.py index c78fa8098..e1b25e161 100644 --- a/integration/test_tor.py +++ b/integration/test_tor.py @@ -40,6 +40,7 @@ if PY2: @pytest_twisted.inlineCallbacks def test_onion_service_storage(reactor, request, temp_dir, flog_gatherer, tor_network, tor_introducer_furl): + import time; time.sleep(3) carol = yield _create_anonymous_node(reactor, 'carol', 8008, request, temp_dir, flog_gatherer, tor_network, tor_introducer_furl) dave = yield _create_anonymous_node(reactor, 'dave', 8009, request, temp_dir, flog_gatherer, tor_network, tor_introducer_furl) util.await_client_ready(carol, minimum_number_of_servers=2) From 04198cdb73f8db0a3e9d3aeab5554e9ce15e2750 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 18 May 2022 12:56:22 -0400 Subject: [PATCH 0877/2309] News file. --- newsfragments/3898.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3898.minor diff --git a/newsfragments/3898.minor b/newsfragments/3898.minor new file mode 100644 index 000000000..e69de29bb From d6abefb041b58df8b019a11abda47c8dc1d6efd2 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 18 May 2022 12:57:29 -0400 Subject: [PATCH 0878/2309] Temporary always build images. --- .circleci/config.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 79ce57ed0..c285263f3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -68,14 +68,14 @@ workflows: images: # Build the Docker images used by the ci jobs. This makes the ci jobs # faster and takes various spurious failures out of the critical path. - triggers: - # Build once a day - - schedule: - cron: "0 0 * * *" - filters: - branches: - only: - - "master" + # triggers: + # # Build once a day + # - schedule: + # cron: "0 0 * * *" + # filters: + # branches: + # only: + # - "master" jobs: # Every job that pushes a Docker image from Docker Hub needs to provide From 5ef8fa5b8958d31e4b79091e30b5df8c6b3d7487 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 18 May 2022 12:57:50 -0400 Subject: [PATCH 0879/2309] TEmporary only build the image we care about. --- .circleci/config.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index c285263f3..7bccb01ec 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -88,16 +88,16 @@ workflows: # https://app.circleci.com/settings/organization/github/tahoe-lafs/contexts - "build-image-debian-10": &DOCKERHUB_CONTEXT context: "dockerhub-auth" - - "build-image-debian-11": - <<: *DOCKERHUB_CONTEXT - - "build-image-ubuntu-18-04": - <<: *DOCKERHUB_CONTEXT - - "build-image-ubuntu-20-04": - <<: *DOCKERHUB_CONTEXT - - "build-image-fedora-35": - <<: *DOCKERHUB_CONTEXT - - "build-image-oraclelinux-8": - <<: *DOCKERHUB_CONTEXT + # - "build-image-debian-11": + # <<: *DOCKERHUB_CONTEXT + # - "build-image-ubuntu-18-04": + # <<: *DOCKERHUB_CONTEXT + # - "build-image-ubuntu-20-04": + # <<: *DOCKERHUB_CONTEXT + # - "build-image-fedora-35": + # <<: *DOCKERHUB_CONTEXT + # - "build-image-oraclelinux-8": + # <<: *DOCKERHUB_CONTEXT # Restore later as PyPy38 #- "build-image-pypy27-buster": # <<: *DOCKERHUB_CONTEXT From 33c43cb2b3da13916b2926e2c6f6692a413058f8 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 18 May 2022 13:01:57 -0400 Subject: [PATCH 0880/2309] Try a different variant. --- integration/install-tor.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration/install-tor.sh b/integration/install-tor.sh index 97a7f9465..e4ec45e78 100755 --- a/integration/install-tor.sh +++ b/integration/install-tor.sh @@ -791,4 +791,4 @@ keSPmmDrjl8cySCNsMo= EOF ${SUDO} apt-get --quiet update -${SUDO} apt-get --quiet --yes install tor=0.4.4.5-1 deb.torproject.org-keyring +${SUDO} apt-get --quiet --yes install tor=0.4.4.5 deb.torproject.org-keyring From 9bef8f4abdb4a2f61b6e57f5b8476ce53835b708 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 18 May 2022 13:07:40 -0400 Subject: [PATCH 0881/2309] This appears to be the alternative to latest version :( --- integration/install-tor.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration/install-tor.sh b/integration/install-tor.sh index e4ec45e78..9a8fc500e 100755 --- a/integration/install-tor.sh +++ b/integration/install-tor.sh @@ -791,4 +791,4 @@ keSPmmDrjl8cySCNsMo= EOF ${SUDO} apt-get --quiet update -${SUDO} apt-get --quiet --yes install tor=0.4.4.5 deb.torproject.org-keyring +${SUDO} apt-get --quiet --yes install tor=0.3.5.16-1 deb.torproject.org-keyring From 012693f6b2ae7e28daf543971d9c2341b891620f Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 18 May 2022 13:19:13 -0400 Subject: [PATCH 0882/2309] Build a different image for now. --- .circleci/config.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 7bccb01ec..0120c6b15 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -86,10 +86,10 @@ workflows: # Contexts are managed in the CircleCI web interface: # # https://app.circleci.com/settings/organization/github/tahoe-lafs/contexts - - "build-image-debian-10": &DOCKERHUB_CONTEXT - context: "dockerhub-auth" - # - "build-image-debian-11": - # <<: *DOCKERHUB_CONTEXT + # - "build-image-debian-10": &DOCKERHUB_CONTEXT + # context: "dockerhub-auth" + - "build-image-debian-11": + <<: *DOCKERHUB_CONTEXT # - "build-image-ubuntu-18-04": # <<: *DOCKERHUB_CONTEXT # - "build-image-ubuntu-20-04": From 28e10d127aaac62be063258b6d46c7e5451a761a Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 18 May 2022 13:20:37 -0400 Subject: [PATCH 0883/2309] Do integration tests with more modern image. --- .circleci/config.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 0120c6b15..7a21a7941 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -18,8 +18,7 @@ workflows: - "debian-10": {} - "debian-11": - requires: - - "debian-10" + {} - "ubuntu-20-04": {} @@ -58,7 +57,7 @@ workflows: requires: # If the unit test suite doesn't pass, don't bother running the # integration tests. - - "debian-10" + - "debian-11" - "typechecks": {} @@ -297,6 +296,10 @@ jobs: integration: <<: *DEBIAN + docker: + - <<: *DOCKERHUB_AUTH + image: "tahoelafsci/debian:11-py3.9" + user: "nobody" environment: <<: *UTF_8_ENVIRONMENT From 90a6cf18ac2960a73b0fd455353e828f1b4ebd54 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 18 May 2022 13:20:44 -0400 Subject: [PATCH 0884/2309] Just use system Tor, for more stability. --- .circleci/Dockerfile.debian | 8 +- integration/install-tor.sh | 794 ------------------------------------ 2 files changed, 2 insertions(+), 800 deletions(-) delete mode 100755 integration/install-tor.sh diff --git a/.circleci/Dockerfile.debian b/.circleci/Dockerfile.debian index f12f19551..abab1f4fa 100644 --- a/.circleci/Dockerfile.debian +++ b/.circleci/Dockerfile.debian @@ -18,15 +18,11 @@ RUN apt-get --quiet update && \ libffi-dev \ libssl-dev \ libyaml-dev \ - virtualenv + virtualenv \ + tor # Get the project source. This is better than it seems. CircleCI will # *update* this checkout on each job run, saving us more time per-job. COPY . ${BUILD_SRC_ROOT} RUN "${BUILD_SRC_ROOT}"/.circleci/prepare-image.sh "${WHEELHOUSE_PATH}" "${VIRTUALENV_PATH}" "${BUILD_SRC_ROOT}" "python${PYTHON_VERSION}" - -# Only the integration tests currently need this but it doesn't hurt to always -# have it present and it's simpler than building a whole extra image just for -# the integration tests. -RUN ${BUILD_SRC_ROOT}/integration/install-tor.sh diff --git a/integration/install-tor.sh b/integration/install-tor.sh deleted file mode 100755 index 9a8fc500e..000000000 --- a/integration/install-tor.sh +++ /dev/null @@ -1,794 +0,0 @@ -#!/bin/bash - -# https://vaneyckt.io/posts/safer_bash_scripts_with_set_euxo_pipefail/ -set -euxo pipefail - -CODENAME=$(lsb_release --short --codename) - -if [ "$(id -u)" != "0" ]; then - SUDO="sudo" -else - SUDO="" -fi - -# Script to install Tor -echo "deb http://deb.torproject.org/torproject.org ${CODENAME} main" | ${SUDO} tee -a /etc/apt/sources.list -echo "deb-src http://deb.torproject.org/torproject.org ${CODENAME} main" | ${SUDO} tee -a /etc/apt/sources.list - -# # Install Tor repo signing key -${SUDO} apt-key add - < Date: Wed, 18 May 2022 13:26:07 -0400 Subject: [PATCH 0885/2309] Make it work temporarily. --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 7a21a7941..8a231ea9d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -85,8 +85,8 @@ workflows: # Contexts are managed in the CircleCI web interface: # # https://app.circleci.com/settings/organization/github/tahoe-lafs/contexts - # - "build-image-debian-10": &DOCKERHUB_CONTEXT - # context: "dockerhub-auth" + - "build-image-debian-10": &DOCKERHUB_CONTEXT + context: "dockerhub-auth" - "build-image-debian-11": <<: *DOCKERHUB_CONTEXT # - "build-image-ubuntu-18-04": From 63e16166d7e0bb6c0bd802791a841880856c6609 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 18 May 2022 13:43:26 -0400 Subject: [PATCH 0886/2309] Restore default image building setup. --- .circleci/config.yml | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 8a231ea9d..051e690b7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -67,14 +67,14 @@ workflows: images: # Build the Docker images used by the ci jobs. This makes the ci jobs # faster and takes various spurious failures out of the critical path. - # triggers: - # # Build once a day - # - schedule: - # cron: "0 0 * * *" - # filters: - # branches: - # only: - # - "master" + triggers: + # Build once a day + - schedule: + cron: "0 0 * * *" + filters: + branches: + only: + - "master" jobs: # Every job that pushes a Docker image from Docker Hub needs to provide @@ -89,14 +89,14 @@ workflows: context: "dockerhub-auth" - "build-image-debian-11": <<: *DOCKERHUB_CONTEXT - # - "build-image-ubuntu-18-04": - # <<: *DOCKERHUB_CONTEXT - # - "build-image-ubuntu-20-04": - # <<: *DOCKERHUB_CONTEXT - # - "build-image-fedora-35": - # <<: *DOCKERHUB_CONTEXT - # - "build-image-oraclelinux-8": - # <<: *DOCKERHUB_CONTEXT + - "build-image-ubuntu-18-04": + <<: *DOCKERHUB_CONTEXT + - "build-image-ubuntu-20-04": + <<: *DOCKERHUB_CONTEXT + - "build-image-fedora-35": + <<: *DOCKERHUB_CONTEXT + - "build-image-oraclelinux-8": + <<: *DOCKERHUB_CONTEXT # Restore later as PyPy38 #- "build-image-pypy27-buster": # <<: *DOCKERHUB_CONTEXT From 02bbce81115eb0f4778f1a61f5a39f19d8f266c3 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 18 May 2022 13:44:18 -0400 Subject: [PATCH 0887/2309] Get rid of spurious sleep. --- integration/test_tor.py | 1 - 1 file changed, 1 deletion(-) diff --git a/integration/test_tor.py b/integration/test_tor.py index e1b25e161..c78fa8098 100644 --- a/integration/test_tor.py +++ b/integration/test_tor.py @@ -40,7 +40,6 @@ if PY2: @pytest_twisted.inlineCallbacks def test_onion_service_storage(reactor, request, temp_dir, flog_gatherer, tor_network, tor_introducer_furl): - import time; time.sleep(3) carol = yield _create_anonymous_node(reactor, 'carol', 8008, request, temp_dir, flog_gatherer, tor_network, tor_introducer_furl) dave = yield _create_anonymous_node(reactor, 'dave', 8009, request, temp_dir, flog_gatherer, tor_network, tor_introducer_furl) util.await_client_ready(carol, minimum_number_of_servers=2) From 928e61bf224717aba58e4c340f82e0621f040609 Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 18 May 2022 12:12:09 -0600 Subject: [PATCH 0888/2309] Log 'something' when we fail to instantiate a client --- newsfragments/3899.bugfix | 1 + src/allmydata/storage_client.py | 11 +++++++---- 2 files changed, 8 insertions(+), 4 deletions(-) create mode 100644 newsfragments/3899.bugfix diff --git a/newsfragments/3899.bugfix b/newsfragments/3899.bugfix new file mode 100644 index 000000000..a55239c38 --- /dev/null +++ b/newsfragments/3899.bugfix @@ -0,0 +1 @@ +Print a useful message when a storage-client cannot be matched to configuration diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 68164e697..9056b9e7a 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -667,6 +667,9 @@ def _storage_from_foolscap_plugin(node_config, config, announcement, get_rref): :param allmydata.node._Config node_config: The node configuration to pass to the plugin. + + :param dict announcement: The storage announcement for the storage + server we should build """ plugins = { plugin.name: plugin @@ -687,7 +690,8 @@ def _storage_from_foolscap_plugin(node_config, config, announcement, get_rref): option, get_rref, ) - raise AnnouncementNotMatched() + plugin_names = ", ".join(sorted(list(config.storage_plugins.keys()))) + raise AnnouncementNotMatched(plugin_names) @implementer(IServer) @@ -761,9 +765,8 @@ class NativeStorageServer(service.MultiService): # able to get the most up-to-date value. self.get_rref, ) - except AnnouncementNotMatched: - # Nope. - pass + except AnnouncementNotMatched as e: + print('No plugin for storage-server "{nickname}" from plugins: {plugins}'.format(nickname=ann.get("nickname", ""), plugins=e.args[0])) else: return _FoolscapStorage.from_announcement( self._server_id, From 21112fd22bb780e8fb06478a1e5b43cbc41fecf1 Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 18 May 2022 22:09:21 -0600 Subject: [PATCH 0889/2309] twisted new-logger, not print() --- src/allmydata/storage_client.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 9056b9e7a..3fc18a908 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -52,6 +52,7 @@ from zope.interface import ( ) from twisted.internet import defer from twisted.application import service +from twisted.logger import Logger from twisted.plugin import ( getPlugins, ) @@ -720,6 +721,7 @@ class NativeStorageServer(service.MultiService): }), "application-version": "unknown: no get_version()", }) + log = Logger() def __init__(self, server_id, ann, tub_maker, handler_overrides, node_config, config=StorageClientConfig()): service.MultiService.__init__(self) @@ -766,7 +768,11 @@ class NativeStorageServer(service.MultiService): self.get_rref, ) except AnnouncementNotMatched as e: - print('No plugin for storage-server "{nickname}" from plugins: {plugins}'.format(nickname=ann.get("nickname", ""), plugins=e.args[0])) + self.log.error( + 'No plugin for storage-server "{nickname}" from plugins: {plugins}', + nickname=ann.get("nickname", ""), + plugins=e.args[0], + ) else: return _FoolscapStorage.from_announcement( self._server_id, From 8c8ea4927f4c015f391913b29a47300f69cdefcd Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 20 May 2022 11:07:55 -0400 Subject: [PATCH 0890/2309] Switch to public API. --- src/allmydata/storage/http_server.py | 8 +++++--- src/allmydata/storage/server.py | 25 +++++++++++++++---------- src/allmydata/test/test_repairer.py | 2 +- src/allmydata/test/test_storage.py | 4 ++-- 4 files changed, 23 insertions(+), 16 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 709c1fda5..033f9ec4c 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -538,8 +538,8 @@ class HTTPServer(object): methods=["PUT"], ) def add_or_renew_lease(self, request, authorization, storage_index): - """Update the lease for an immutable share.""" - if not list(self._storage_server._get_bucket_shares(storage_index)): + """Update the lease for an immutable or mutable share.""" + if not list(self._storage_server.get_shares(storage_index)): raise _HTTPError(http.NOT_FOUND) # Checking of the renewal secret is done by the backend. @@ -674,7 +674,9 @@ class HTTPServer(object): ): """Indicate that given share is corrupt, with a text reason.""" # TODO unit test all the paths - if not self._storage_server._share_exists(storage_index, share_number): + if share_number not in { + shnum for (shnum, _) in self._storage_server.get_shares(storage_index) + }: raise _HTTPError(http.NOT_FOUND) info = self._read_encoded(request, _SCHEMAS["advise_corrupt_share"]) diff --git a/src/allmydata/storage/server.py b/src/allmydata/storage/server.py index bcf44dc30..07b82b4d8 100644 --- a/src/allmydata/storage/server.py +++ b/src/allmydata/storage/server.py @@ -3,7 +3,7 @@ Ported to Python 3. """ from __future__ import annotations from future.utils import bytes_to_native_str -from typing import Dict, Tuple +from typing import Dict, Tuple, Iterable import os, re @@ -321,7 +321,7 @@ class StorageServer(service.MultiService): # they asked about: this will save them a lot of work. Add or update # leases for all of them: if they want us to hold shares for this # file, they'll want us to hold leases for this file. - for (shnum, fn) in self._get_bucket_shares(storage_index): + for (shnum, fn) in self.get_shares(storage_index): alreadygot[shnum] = ShareFile(fn) if renew_leases: self._add_or_renew_leases(alreadygot.values(), lease_info) @@ -363,7 +363,7 @@ class StorageServer(service.MultiService): return set(alreadygot), bucketwriters def _iter_share_files(self, storage_index): - for shnum, filename in self._get_bucket_shares(storage_index): + for shnum, filename in self.get_shares(storage_index): with open(filename, 'rb') as f: header = f.read(32) if MutableShareFile.is_valid_header(header): @@ -416,10 +416,12 @@ class StorageServer(service.MultiService): """ self._call_on_bucket_writer_close.append(handler) - def _get_bucket_shares(self, storage_index): - """Return a list of (shnum, pathname) tuples for files that hold + def get_shares(self, storage_index) -> Iterable[(int, str)]: + """ + Return an iterable of (shnum, pathname) tuples for files that hold shares for this storage_index. In each tuple, 'shnum' will always be - the integer form of the last component of 'pathname'.""" + the integer form of the last component of 'pathname'. + """ storagedir = os.path.join(self.sharedir, storage_index_to_dir(storage_index)) try: for f in os.listdir(storagedir): @@ -431,12 +433,15 @@ class StorageServer(service.MultiService): pass def get_buckets(self, storage_index): + """ + Get ``BucketReaders`` for an immutable. + """ start = self._clock.seconds() self.count("get") si_s = si_b2a(storage_index) log.msg("storage: get_buckets %r" % si_s) bucketreaders = {} # k: sharenum, v: BucketReader - for shnum, filename in self._get_bucket_shares(storage_index): + for shnum, filename in self.get_shares(storage_index): bucketreaders[shnum] = BucketReader(self, filename, storage_index, shnum) self.add_latency("get", self._clock.seconds() - start) @@ -453,7 +458,7 @@ class StorageServer(service.MultiService): # since all shares get the same lease data, we just grab the leases # from the first share try: - shnum, filename = next(self._get_bucket_shares(storage_index)) + shnum, filename = next(self.get_shares(storage_index)) sf = ShareFile(filename) return sf.get_leases() except StopIteration: @@ -467,7 +472,7 @@ class StorageServer(service.MultiService): :return: An iterable of the leases attached to this slot. """ - for _, share_filename in self._get_bucket_shares(storage_index): + for _, share_filename in self.get_shares(storage_index): share = MutableShareFile(share_filename) return share.get_leases() return [] @@ -742,7 +747,7 @@ class StorageServer(service.MultiService): :return bool: ``True`` if a share with the given number exists at the given storage index, ``False`` otherwise. """ - for existing_sharenum, ignored in self._get_bucket_shares(storage_index): + for existing_sharenum, ignored in self.get_shares(storage_index): if existing_sharenum == shnum: return True return False diff --git a/src/allmydata/test/test_repairer.py b/src/allmydata/test/test_repairer.py index 88696000c..f9b93af72 100644 --- a/src/allmydata/test/test_repairer.py +++ b/src/allmydata/test/test_repairer.py @@ -717,7 +717,7 @@ class Repairer(GridTestMixin, unittest.TestCase, RepairTestMixin, ss = self.g.servers_by_number[0] # we want to delete the share corresponding to the server # we're making not-respond - share = next(ss._get_bucket_shares(self.c0_filenode.get_storage_index()))[0] + share = next(ss.get_shares(self.c0_filenode.get_storage_index()))[0] self.delete_shares_numbered(self.uri, [share]) return self.c0_filenode.check_and_repair(Monitor()) d.addCallback(_then) diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index 65d09de25..91d55790e 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -766,7 +766,7 @@ class Server(unittest.TestCase): writer.close() # It should have a lease granted at the current time. - shares = dict(ss._get_bucket_shares(storage_index)) + shares = dict(ss.get_shares(storage_index)) self.assertEqual( [first_lease], list( @@ -789,7 +789,7 @@ class Server(unittest.TestCase): writer.close() # The first share's lease expiration time is unchanged. - shares = dict(ss._get_bucket_shares(storage_index)) + shares = dict(ss.get_shares(storage_index)) self.assertEqual( [first_lease], list( From 12927d50bafdb37d7dd0a53443b5d61ee6364be7 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 20 May 2022 11:09:04 -0400 Subject: [PATCH 0891/2309] Type annotation improvements. --- src/allmydata/storage/http_client.py | 2 +- src/allmydata/storage/server.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 167d2394a..de9dfc518 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -358,7 +358,7 @@ class StorageClientGeneral(object): @inlineCallbacks def add_or_renew_lease( self, storage_index: bytes, renew_secret: bytes, cancel_secret: bytes - ): + ) -> Deferred[None]: """ Add or renew a lease. diff --git a/src/allmydata/storage/server.py b/src/allmydata/storage/server.py index 07b82b4d8..0a1999dfb 100644 --- a/src/allmydata/storage/server.py +++ b/src/allmydata/storage/server.py @@ -416,7 +416,7 @@ class StorageServer(service.MultiService): """ self._call_on_bucket_writer_close.append(handler) - def get_shares(self, storage_index) -> Iterable[(int, str)]: + def get_shares(self, storage_index) -> Iterable[tuple[int, str]]: """ Return an iterable of (shnum, pathname) tuples for files that hold shares for this storage_index. In each tuple, 'shnum' will always be From 63624eedec9d50dd93c2812482fb6fa977e1e096 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 20 May 2022 11:33:02 -0400 Subject: [PATCH 0892/2309] Reduce code duplication. --- src/allmydata/storage/http_server.py | 53 +++++++++++++--------------- 1 file changed, 25 insertions(+), 28 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 033f9ec4c..a4f67bb5e 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -2,6 +2,7 @@ HTTP server for storage. """ +from __future__ import annotations from typing import Dict, List, Set, Tuple, Any from functools import wraps @@ -273,6 +274,28 @@ _SCHEMAS = { } +# TODO unit tests? or rely on higher-level tests +def parse_range(request) -> tuple[int, int]: + """ + Parse the subset of ``Range`` headers we support: bytes only, only a single + range, the end must be explicitly specified. Raises a + ``_HTTPError(http.REQUESTED_RANGE_NOT_SATISFIABLE)`` if parsing is not + possible or the header isn't set. + + Returns tuple of (start_offset, end_offset). + """ + range_header = parse_range_header(request.getHeader("range")) + if ( + range_header is None + or range_header.units != "bytes" + or len(range_header.ranges) > 1 # more than one range + or range_header.ranges[0][1] is None # range without end + ): + raise _HTTPError(http.REQUESTED_RANGE_NOT_SATISFIABLE) + + return range_header.ranges[0] + + class HTTPServer(object): """ A HTTP interface to the storage server. @@ -505,17 +528,7 @@ class HTTPServer(object): request.write(data) start += len(data) - range_header = parse_range_header(request.getHeader("range")) - if ( - range_header is None - or range_header.units != "bytes" - or len(range_header.ranges) > 1 # more than one range - or range_header.ranges[0][1] is None # range without end - ): - request.setResponseCode(http.REQUESTED_RANGE_NOT_SATISFIABLE) - return b"" - - offset, end = range_header.ranges[0] + offset, end = parse_range(request) # TODO limit memory usage # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3872 @@ -617,23 +630,7 @@ class HTTPServer(object): ) def read_mutable_chunk(self, request, authorization, storage_index, share_number): """Read a chunk from a mutable.""" - if request.getHeader("range") is None: - # TODO in follow-up ticket - raise NotImplementedError() - - # TODO reduce duplication with immutable reads? - # TODO unit tests, perhaps shared if possible - range_header = parse_range_header(request.getHeader("range")) - if ( - range_header is None - or range_header.units != "bytes" - or len(range_header.ranges) > 1 # more than one range - or range_header.ranges[0][1] is None # range without end - ): - request.setResponseCode(http.REQUESTED_RANGE_NOT_SATISFIABLE) - return b"" - - offset, end = range_header.ranges[0] + offset, end = parse_range(request) # TODO limit memory usage # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3872 From 2313195c2b94db799c93083580b643c85fabef47 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 20 May 2022 11:43:42 -0400 Subject: [PATCH 0893/2309] Reduce duplication. --- src/allmydata/storage/http_server.py | 75 +++++++++++++--------------- 1 file changed, 36 insertions(+), 39 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index a4f67bb5e..2ff0c6908 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -3,7 +3,7 @@ HTTP server for storage. """ from __future__ import annotations -from typing import Dict, List, Set, Tuple, Any +from typing import Dict, List, Set, Tuple, Any, Callable from functools import wraps from base64 import b64decode @@ -275,14 +275,20 @@ _SCHEMAS = { # TODO unit tests? or rely on higher-level tests -def parse_range(request) -> tuple[int, int]: +def read_range(request, read_data: Callable[int, int, bytes]) -> bytes: """ - Parse the subset of ``Range`` headers we support: bytes only, only a single - range, the end must be explicitly specified. Raises a - ``_HTTPError(http.REQUESTED_RANGE_NOT_SATISFIABLE)`` if parsing is not - possible or the header isn't set. + Parse the ``Range`` header, read appropriately, return as result. - Returns tuple of (start_offset, end_offset). + Only parses a subset of ``Range`` headers that we support: must be set, + bytes only, only a single range, the end must be explicitly specified. + Raises a ``_HTTPError(http.REQUESTED_RANGE_NOT_SATISFIABLE)`` if parsing is + not possible or the header isn't set. + + Returns the bytes to return from the request handler, and sets appropriate + response headers. + + Takes a function that will do the actual reading given the start offset and + a length to read. """ range_header = parse_range_header(request.getHeader("range")) if ( @@ -293,7 +299,21 @@ def parse_range(request) -> tuple[int, int]: ): raise _HTTPError(http.REQUESTED_RANGE_NOT_SATISFIABLE) - return range_header.ranges[0] + offset, end = range_header.ranges[0] + + # TODO limit memory usage + # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3872 + data = read_data(offset, end - offset) + + request.setResponseCode(http.PARTIAL_CONTENT) + if len(data): + # For empty bodies the content-range header makes no sense since + # the end of the range is inclusive. + request.setHeader( + "content-range", + ContentRange("bytes", offset, offset + len(data)).to_header(), + ) + return data class HTTPServer(object): @@ -528,21 +548,7 @@ class HTTPServer(object): request.write(data) start += len(data) - offset, end = parse_range(request) - - # TODO limit memory usage - # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3872 - data = bucket.read(offset, end - offset) - - request.setResponseCode(http.PARTIAL_CONTENT) - if len(data): - # For empty bodies the content-range header makes no sense since - # the end of the range is inclusive. - request.setHeader( - "content-range", - ContentRange("bytes", offset, offset + len(data)).to_header(), - ) - return data + return read_range(request, bucket.read) @_authorized_route( _app, @@ -630,24 +636,15 @@ class HTTPServer(object): ) def read_mutable_chunk(self, request, authorization, storage_index, share_number): """Read a chunk from a mutable.""" - offset, end = parse_range(request) + if request.getHeader("range") is None: + raise NotImplementedError() # should be able to move shared implementation into read_range()... - # TODO limit memory usage - # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3872 - data = self._storage_server.slot_readv( - storage_index, [share_number], [(offset, end - offset)] - )[share_number][0] + def read_data(offset, length): + return self._storage_server.slot_readv( + storage_index, [share_number], [(offset, length)] + )[share_number][0] - # TODO reduce duplication? - request.setResponseCode(http.PARTIAL_CONTENT) - if len(data): - # For empty bodies the content-range header makes no sense since - # the end of the range is inclusive. - request.setHeader( - "content-range", - ContentRange("bytes", offset, offset + len(data)).to_header(), - ) - return data + return read_range(request, read_data) @_authorized_route( _app, From fd306b9a61b2b4806c948ca0f2b2e7fe1f447110 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 1 Jun 2022 13:54:54 -0400 Subject: [PATCH 0894/2309] Share more code across mutable and immutable reads. --- src/allmydata/storage/http_server.py | 38 +++++++++++++--------------- 1 file changed, 18 insertions(+), 20 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 2ff0c6908..b031cbb15 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -275,7 +275,7 @@ _SCHEMAS = { # TODO unit tests? or rely on higher-level tests -def read_range(request, read_data: Callable[int, int, bytes]) -> bytes: +def read_range(request, read_data: Callable[int, int, bytes]) -> Optional[bytes]: """ Parse the ``Range`` header, read appropriately, return as result. @@ -284,15 +284,28 @@ def read_range(request, read_data: Callable[int, int, bytes]) -> bytes: Raises a ``_HTTPError(http.REQUESTED_RANGE_NOT_SATISFIABLE)`` if parsing is not possible or the header isn't set. - Returns the bytes to return from the request handler, and sets appropriate - response headers. + Returns a result that should be returned from the request handler, and sets + appropriate response headers. Takes a function that will do the actual reading given the start offset and a length to read. """ + if request.getHeader("range") is None: + # Return the whole thing. + start = 0 + while True: + # TODO should probably yield to event loop occasionally... + # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3872 + data = read_data(start, start + 65536) + if not data: + request.finish() + return + request.write(data) + start += len(data) + range_header = parse_range_header(request.getHeader("range")) if ( - range_header is None + range_header is None # failed to parse or range_header.units != "bytes" or len(range_header.ranges) > 1 # more than one range or range_header.ranges[0][1] is None # range without end @@ -535,19 +548,6 @@ class HTTPServer(object): request.setResponseCode(http.NOT_FOUND) return b"" - if request.getHeader("range") is None: - # Return the whole thing. - start = 0 - while True: - # TODO should probably yield to event loop occasionally... - # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3872 - data = bucket.read(start, start + 65536) - if not data: - request.finish() - return - request.write(data) - start += len(data) - return read_range(request, bucket.read) @_authorized_route( @@ -636,9 +636,7 @@ class HTTPServer(object): ) def read_mutable_chunk(self, request, authorization, storage_index, share_number): """Read a chunk from a mutable.""" - if request.getHeader("range") is None: - raise NotImplementedError() # should be able to move shared implementation into read_range()... - + # TODO unit tests def read_data(offset, length): return self._storage_server.slot_readv( storage_index, [share_number], [(offset, length)] From f1384096fa26c97e5c3f88a9d8fb0212c7f34c16 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 3 Jun 2022 13:46:23 -0400 Subject: [PATCH 0895/2309] First unit test for mutables. --- src/allmydata/storage/http_client.py | 6 +- src/allmydata/storage/http_server.py | 3 +- src/allmydata/test/test_storage_http.py | 83 ++++++++++++++++++++++++- 3 files changed, 86 insertions(+), 6 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index de9dfc518..bf6104dea 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -7,7 +7,7 @@ from __future__ import annotations from typing import Union, Optional, Sequence, Mapping from base64 import b64encode -from attrs import define, asdict, frozen +from attrs import define, asdict, frozen, field # TODO Make sure to import Python version? from cbor2 import loads, dumps @@ -646,8 +646,8 @@ class ReadVector: class TestWriteVectors: """Test and write vectors for a specific share.""" - test_vectors: Sequence[TestVector] - write_vectors: Sequence[WriteVector] + test_vectors: Sequence[TestVector] = field(factory=list) + write_vectors: Sequence[WriteVector] = field(factory=list) new_length: Optional[int] = None def asdict(self) -> dict: diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index b031cbb15..9735a0626 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -263,7 +263,7 @@ _SCHEMAS = { * share_number: { "test": [* {"offset": uint, "size": uint, "specimen": bstr}] "write": [* {"offset": uint, "data": bstr}] - "new-length": uint // null + "new-length": uint / null } } "read-vector": [* {"offset": uint, "size": uint}] @@ -274,7 +274,6 @@ _SCHEMAS = { } -# TODO unit tests? or rely on higher-level tests def read_range(request, read_data: Callable[int, int, bytes]) -> Optional[bytes]: """ Parse the ``Range`` header, read appropriately, return as result. diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index fcc2401f2..37e3be8a7 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -40,6 +40,10 @@ from ..storage.http_client import ( UploadProgress, StorageClientGeneral, _encode_si, + StorageClientMutables, + TestWriteVectors, + WriteVector, + ReadVector, ) @@ -1109,7 +1113,9 @@ class ImmutableHTTPAPITests(SyncTestCase): storage_index = urandom(16) secret = b"A" * 32 with assert_fails_with_http_code(self, http.NOT_FOUND): - result_of(self.general_client.add_or_renew_lease(storage_index, secret, secret)) + result_of( + self.general_client.add_or_renew_lease(storage_index, secret, secret) + ) def test_advise_corrupt_share(self): """ @@ -1142,3 +1148,78 @@ class ImmutableHTTPAPITests(SyncTestCase): result_of( self.imm_client.advise_corrupt_share(si, share_number, reason) ) + + +class MutableHTTPAPIsTests(SyncTestCase): + """Tests for mutable APIs.""" + + def setUp(self): + super(MutableHTTPAPIsTests, self).setUp() + self.http = self.useFixture(HttpTestFixture()) + self.mut_client = StorageClientMutables(self.http.client) + + def create_upload(self, data=b"abcdef"): + """ + Utility that creates shares 0 and 1 with bodies + ``{data}-{share_number}``. + """ + write_secret = urandom(32) + lease_secret = urandom(32) + storage_index = urandom(16) + result_of( + self.mut_client.read_test_write_chunks( + storage_index, + write_secret, + lease_secret, + lease_secret, + { + 0: TestWriteVectors( + write_vectors=[WriteVector(offset=0, data=data + b"-0")] + ), + 1: TestWriteVectors( + write_vectors=[ + WriteVector(offset=0, data=data), + WriteVector(offset=len(data), data=b"-1"), + ] + ), + }, + [ReadVector(0, len(data) + 2)], + ) + ) + return storage_index, write_secret, lease_secret + + def test_upload_can_be_downloaded(self): + """ + Written data can be read, both by the combo operation and a direct + read. + """ + storage_index, _, _ = self.create_upload() + data0 = result_of(self.mut_client.read_share_chunk(storage_index, 0, 1, 7)) + data1 = result_of(self.mut_client.read_share_chunk(storage_index, 1, 0, 8)) + self.assertEqual((data0, data1), (b"bcdef-0", b"abcdef-1")) + + def test_read_before_write(self): + """In combo read/test/write operation, reads happen before writes.""" + + def test_conditional_upload(self): + pass + + def test_list_shares(self): + pass + + def test_wrong_write_enabler(self): + pass + + # TODO refactor reads tests so they're shared + + def test_lease_renew_and_add(self): + pass + + def test_lease_on_unknown_storage_index(self): + pass + + def test_advise_corrupt_share(self): + pass + + def test_advise_corrupt_share_unknown(self): + pass From 3e67d2d7890ceb7f537fdfeda89072e1d99c1835 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 6 Jun 2022 09:50:36 -0400 Subject: [PATCH 0896/2309] More tests. --- src/allmydata/test/test_storage_http.py | 78 +++++++++++++++++++++++-- 1 file changed, 72 insertions(+), 6 deletions(-) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 37e3be8a7..6cf2f883b 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -44,6 +44,8 @@ from ..storage.http_client import ( TestWriteVectors, WriteVector, ReadVector, + ReadTestWriteResult, + TestVector, ) @@ -1183,15 +1185,14 @@ class MutableHTTPAPIsTests(SyncTestCase): ] ), }, - [ReadVector(0, len(data) + 2)], + [], ) ) return storage_index, write_secret, lease_secret - def test_upload_can_be_downloaded(self): + def test_write_can_be_read(self): """ - Written data can be read, both by the combo operation and a direct - read. + Written data can be read using ``read_share_chunk``. """ storage_index, _, _ = self.create_upload() data0 = result_of(self.mut_client.read_share_chunk(storage_index, 0, 1, 7)) @@ -1200,9 +1201,74 @@ class MutableHTTPAPIsTests(SyncTestCase): def test_read_before_write(self): """In combo read/test/write operation, reads happen before writes.""" + storage_index, write_secret, lease_secret = self.create_upload() + result = result_of( + self.mut_client.read_test_write_chunks( + storage_index, + write_secret, + lease_secret, + lease_secret, + { + 0: TestWriteVectors( + write_vectors=[WriteVector(offset=1, data=b"XYZ")] + ), + }, + [ReadVector(0, 8)], + ) + ) + # Reads are from before the write: + self.assertEqual( + result, + ReadTestWriteResult( + success=True, reads={0: [b"abcdef-0"], 1: [b"abcdef-1"]} + ), + ) + # But the write did happen: + data0 = result_of(self.mut_client.read_share_chunk(storage_index, 0, 0, 8)) + data1 = result_of(self.mut_client.read_share_chunk(storage_index, 1, 0, 8)) + self.assertEqual((data0, data1), (b"aXYZef-0", b"abcdef-1")) - def test_conditional_upload(self): - pass + def test_conditional_write(self): + """Uploads only happen if the test passes.""" + storage_index, write_secret, lease_secret = self.create_upload() + result_failed = result_of( + self.mut_client.read_test_write_chunks( + storage_index, + write_secret, + lease_secret, + lease_secret, + { + 0: TestWriteVectors( + test_vectors=[TestVector(1, 4, b"FAIL")], + write_vectors=[WriteVector(offset=1, data=b"XYZ")], + ), + }, + [], + ) + ) + self.assertFalse(result_failed.success) + + # This time the test matches: + result = result_of( + self.mut_client.read_test_write_chunks( + storage_index, + write_secret, + lease_secret, + lease_secret, + { + 0: TestWriteVectors( + test_vectors=[TestVector(1, 4, b"bcde")], + write_vectors=[WriteVector(offset=1, data=b"XYZ")], + ), + }, + [ReadVector(0, 8)], + ) + ) + self.assertTrue(result.success) + self.assertEqual( + result_of(self.mut_client.read_share_chunk(storage_index, 0, 0, 8)), + b"aXYZef-0", + ) def test_list_shares(self): pass From 797f34aec32eefbe495d9936d955e7ab4bdc3f1a Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 6 Jun 2022 09:59:12 -0400 Subject: [PATCH 0897/2309] More tests. --- src/allmydata/storage/http_client.py | 3 --- src/allmydata/test/test_storage_http.py | 35 +++++++++++++++++++++++-- 2 files changed, 33 insertions(+), 5 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index bf6104dea..9711e748d 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -696,7 +696,6 @@ class StorageClientMutables: Given a mapping between share numbers and test/write vectors, the tests are done and if they are valid the writes are done. """ - # TODO unit test all the things url = self._client.relative_url( "/v1/mutable/{}/read-test-write".format(_encode_si(storage_index)) ) @@ -731,7 +730,6 @@ class StorageClientMutables: """ Download a chunk of data from a share. """ - # TODO unit test all the things return read_share_chunk( self._client, "mutable", storage_index, share_number, offset, length ) @@ -741,7 +739,6 @@ class StorageClientMutables: """ List the share numbers for a given storage index. """ - # TODO unit test all the things url = self._client.relative_url( "/v1/mutable/{}/shares".format(_encode_si(storage_index)) ) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 6cf2f883b..65aa12e40 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -1271,10 +1271,41 @@ class MutableHTTPAPIsTests(SyncTestCase): ) def test_list_shares(self): - pass + """``list_shares()`` returns the shares for a given storage index.""" + storage_index, _, _ = self.create_upload() + self.assertEqual(result_of(self.mut_client.list_shares(storage_index)), {0, 1}) + + def test_non_existent_list_shares(self): + """A non-existent storage index errors when shares are listed.""" + with self.assertRaises(ClientException) as exc: + result_of(self.mut_client.list_shares(urandom(32))) + self.assertEqual(exc.exception.code, http.NOT_FOUND) def test_wrong_write_enabler(self): - pass + """Writes with the wrong write enabler fail, and are not processed.""" + storage_index, write_secret, lease_secret = self.create_upload() + with self.assertRaises(ClientException) as exc: + result_of( + self.mut_client.read_test_write_chunks( + storage_index, + urandom(32), + lease_secret, + lease_secret, + { + 0: TestWriteVectors( + write_vectors=[WriteVector(offset=1, data=b"XYZ")] + ), + }, + [ReadVector(0, 8)], + ) + ) + self.assertEqual(exc.exception.code, http.UNAUTHORIZED) + + # The write did not happen: + self.assertEqual( + result_of(self.mut_client.read_share_chunk(storage_index, 0, 0, 8)), + b"abcdef-0", + ) # TODO refactor reads tests so they're shared From e6efb62fd19eef08f14916438240f70bc197a4c3 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 6 Jun 2022 10:25:06 -0400 Subject: [PATCH 0898/2309] Refactor immutable tests so they can shared with mutables. --- src/allmydata/test/test_storage_http.py | 460 +++++++++++++----------- 1 file changed, 246 insertions(+), 214 deletions(-) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 65aa12e40..fc79fbe34 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -5,7 +5,7 @@ Tests for HTTP storage client + server. from base64 import b64encode from contextlib import contextmanager from os import urandom - +from typing import Union, Callable, Tuple from cbor2 import dumps from pycddl import ValidationError as CDDLValidationError from hypothesis import assume, given, strategies as st @@ -787,141 +787,6 @@ class ImmutableHTTPAPITests(SyncTestCase): ) ) - def upload(self, share_number, data_length=26): - """ - Create a share, return (storage_index, uploaded_data). - """ - uploaded_data = (b"abcdefghijklmnopqrstuvwxyz" * ((data_length // 26) + 1))[ - :data_length - ] - (upload_secret, _, storage_index, _) = self.create_upload( - {share_number}, data_length - ) - result_of( - self.imm_client.write_share_chunk( - storage_index, - share_number, - upload_secret, - 0, - uploaded_data, - ) - ) - return storage_index, uploaded_data - - def test_read_of_wrong_storage_index_fails(self): - """ - Reading from unknown storage index results in 404. - """ - with assert_fails_with_http_code(self, http.NOT_FOUND): - result_of( - self.imm_client.read_share_chunk( - b"1" * 16, - 1, - 0, - 10, - ) - ) - - def test_read_of_wrong_share_number_fails(self): - """ - Reading from unknown storage index results in 404. - """ - storage_index, _ = self.upload(1) - with assert_fails_with_http_code(self, http.NOT_FOUND): - result_of( - self.imm_client.read_share_chunk( - storage_index, - 7, # different share number - 0, - 10, - ) - ) - - def test_read_with_negative_offset_fails(self): - """ - Malformed or unsupported Range headers result in 416 (requested range - not satisfiable) error. - """ - storage_index, _ = self.upload(1) - - def check_bad_range(bad_range_value): - client = StorageClientImmutables( - StorageClientWithHeadersOverride( - self.http.client, {"range": bad_range_value} - ) - ) - - with assert_fails_with_http_code( - self, http.REQUESTED_RANGE_NOT_SATISFIABLE - ): - result_of( - client.read_share_chunk( - storage_index, - 1, - 0, - 10, - ) - ) - - # Bad unit - check_bad_range("molluscs=0-9") - # Negative offsets - check_bad_range("bytes=-2-9") - check_bad_range("bytes=0--10") - # Negative offset no endpoint - check_bad_range("bytes=-300-") - check_bad_range("bytes=") - # Multiple ranges are currently unsupported, even if they're - # semantically valid under HTTP: - check_bad_range("bytes=0-5, 6-7") - # Ranges without an end are currently unsupported, even if they're - # semantically valid under HTTP. - check_bad_range("bytes=0-") - - @given(data_length=st.integers(min_value=1, max_value=300000)) - def test_read_with_no_range(self, data_length): - """ - A read with no range returns the whole immutable. - """ - storage_index, uploaded_data = self.upload(1, data_length) - response = result_of( - self.http.client.request( - "GET", - self.http.client.relative_url( - "/v1/immutable/{}/1".format(_encode_si(storage_index)) - ), - ) - ) - self.assertEqual(response.code, http.OK) - self.assertEqual(result_of(response.content()), uploaded_data) - - def test_validate_content_range_response_to_read(self): - """ - The server responds to ranged reads with an appropriate Content-Range - header. - """ - storage_index, _ = self.upload(1, 26) - - def check_range(requested_range, expected_response): - headers = Headers() - headers.setRawHeaders("range", [requested_range]) - response = result_of( - self.http.client.request( - "GET", - self.http.client.relative_url( - "/v1/immutable/{}/1".format(_encode_si(storage_index)) - ), - headers=headers, - ) - ) - self.assertEqual( - response.headers.getRawHeaders("content-range"), [expected_response] - ) - - check_range("bytes=0-10", "bytes 0-10/*") - # Can't go beyond the end of the immutable! - check_range("bytes=10-100", "bytes 10-25/*") - def test_timed_out_upload_allows_reupload(self): """ If an in-progress upload times out, it is cancelled altogether, @@ -1062,52 +927,6 @@ class ImmutableHTTPAPITests(SyncTestCase): ), ) - def test_lease_renew_and_add(self): - """ - It's possible the renew the lease on an uploaded immutable, by using - the same renewal secret, or add a new lease by choosing a different - renewal secret. - """ - # Create immutable: - (upload_secret, lease_secret, storage_index, _) = self.create_upload({0}, 100) - result_of( - self.imm_client.write_share_chunk( - storage_index, - 0, - upload_secret, - 0, - b"A" * 100, - ) - ) - - [lease] = self.http.storage_server.get_leases(storage_index) - initial_expiration_time = lease.get_expiration_time() - - # Time passes: - self.http.clock.advance(167) - - # We renew the lease: - result_of( - self.general_client.add_or_renew_lease( - storage_index, lease_secret, lease_secret - ) - ) - - # More time passes: - self.http.clock.advance(10) - - # We create a new lease: - lease_secret2 = urandom(32) - result_of( - self.general_client.add_or_renew_lease( - storage_index, lease_secret2, lease_secret2 - ) - ) - - [lease1, lease2] = self.http.storage_server.get_leases(storage_index) - self.assertEqual(lease1.get_expiration_time(), initial_expiration_time + 167) - self.assertEqual(lease2.get_expiration_time(), initial_expiration_time + 177) - def test_lease_on_unknown_storage_index(self): """ An attempt to renew an unknown storage index will result in a HTTP 404. @@ -1119,38 +938,6 @@ class ImmutableHTTPAPITests(SyncTestCase): self.general_client.add_or_renew_lease(storage_index, secret, secret) ) - def test_advise_corrupt_share(self): - """ - Advising share was corrupted succeeds from HTTP client's perspective, - and calls appropriate method on server. - """ - corrupted = [] - self.http.storage_server.advise_corrupt_share = lambda *args: corrupted.append( - args - ) - - storage_index, _ = self.upload(13) - reason = "OHNO \u1235" - result_of(self.imm_client.advise_corrupt_share(storage_index, 13, reason)) - - self.assertEqual( - corrupted, [(b"immutable", storage_index, 13, reason.encode("utf-8"))] - ) - - def test_advise_corrupt_share_unknown(self): - """ - Advising an unknown share was corrupted results in 404. - """ - storage_index, _ = self.upload(13) - reason = "OHNO \u1235" - result_of(self.imm_client.advise_corrupt_share(storage_index, 13, reason)) - - for (si, share_number) in [(storage_index, 11), (urandom(16), 13)]: - with assert_fails_with_http_code(self, http.NOT_FOUND): - result_of( - self.imm_client.advise_corrupt_share(si, share_number, reason) - ) - class MutableHTTPAPIsTests(SyncTestCase): """Tests for mutable APIs.""" @@ -1320,3 +1107,248 @@ class MutableHTTPAPIsTests(SyncTestCase): def test_advise_corrupt_share_unknown(self): pass + + +class SharedImmutableMutableTestsMixin: + """ + Shared tests for mutables and immutables where the API is the same. + """ + + KIND: str # either "mutable" or "immutable" + general_client: StorageClientGeneral + client: Union[StorageClientImmutables, StorageClientMutables] + clientFactory: Callable[ + StorageClient, Union[StorageClientImmutables, StorageClientMutables] + ] + + def upload(self, share_number: int, data_length=26) -> Tuple[bytes, bytes, bytes]: + """ + Create a share, return (storage_index, uploaded_data, lease secret). + """ + raise NotImplementedError + + def test_advise_corrupt_share(self): + """ + Advising share was corrupted succeeds from HTTP client's perspective, + and calls appropriate method on server. + """ + corrupted = [] + self.http.storage_server.advise_corrupt_share = lambda *args: corrupted.append( + args + ) + + storage_index, _, _ = self.upload(13) + reason = "OHNO \u1235" + result_of(self.client.advise_corrupt_share(storage_index, 13, reason)) + + self.assertEqual( + corrupted, + [(self.KIND.encode("ascii"), storage_index, 13, reason.encode("utf-8"))], + ) + + def test_advise_corrupt_share_unknown(self): + """ + Advising an unknown share was corrupted results in 404. + """ + storage_index, _, _ = self.upload(13) + reason = "OHNO \u1235" + result_of(self.client.advise_corrupt_share(storage_index, 13, reason)) + + for (si, share_number) in [(storage_index, 11), (urandom(16), 13)]: + with assert_fails_with_http_code(self, http.NOT_FOUND): + result_of(self.client.advise_corrupt_share(si, share_number, reason)) + + def test_lease_renew_and_add(self): + """ + It's possible the renew the lease on an uploaded immutable, by using + the same renewal secret, or add a new lease by choosing a different + renewal secret. + """ + # Create a storage index: + storage_index, _, lease_secret = self.upload(0) + + [lease] = self.http.storage_server.get_leases(storage_index) + initial_expiration_time = lease.get_expiration_time() + + # Time passes: + self.http.clock.advance(167) + + # We renew the lease: + result_of( + self.general_client.add_or_renew_lease( + storage_index, lease_secret, lease_secret + ) + ) + + # More time passes: + self.http.clock.advance(10) + + # We create a new lease: + lease_secret2 = urandom(32) + result_of( + self.general_client.add_or_renew_lease( + storage_index, lease_secret2, lease_secret2 + ) + ) + + [lease1, lease2] = self.http.storage_server.get_leases(storage_index) + self.assertEqual(lease1.get_expiration_time(), initial_expiration_time + 167) + self.assertEqual(lease2.get_expiration_time(), initial_expiration_time + 177) + + def test_read_of_wrong_storage_index_fails(self): + """ + Reading from unknown storage index results in 404. + """ + with assert_fails_with_http_code(self, http.NOT_FOUND): + result_of( + self.client.read_share_chunk( + b"1" * 16, + 1, + 0, + 10, + ) + ) + + def test_read_of_wrong_share_number_fails(self): + """ + Reading from unknown storage index results in 404. + """ + storage_index, _, _ = self.upload(1) + with assert_fails_with_http_code(self, http.NOT_FOUND): + result_of( + self.client.read_share_chunk( + storage_index, + 7, # different share number + 0, + 10, + ) + ) + + def test_read_with_negative_offset_fails(self): + """ + Malformed or unsupported Range headers result in 416 (requested range + not satisfiable) error. + """ + storage_index, _, _ = self.upload(1) + + def check_bad_range(bad_range_value): + client = StorageClientImmutables( + StorageClientWithHeadersOverride( + self.http.client, {"range": bad_range_value} + ) + ) + + with assert_fails_with_http_code( + self, http.REQUESTED_RANGE_NOT_SATISFIABLE + ): + result_of( + client.read_share_chunk( + storage_index, + 1, + 0, + 10, + ) + ) + + # Bad unit + check_bad_range("molluscs=0-9") + # Negative offsets + check_bad_range("bytes=-2-9") + check_bad_range("bytes=0--10") + # Negative offset no endpoint + check_bad_range("bytes=-300-") + check_bad_range("bytes=") + # Multiple ranges are currently unsupported, even if they're + # semantically valid under HTTP: + check_bad_range("bytes=0-5, 6-7") + # Ranges without an end are currently unsupported, even if they're + # semantically valid under HTTP. + check_bad_range("bytes=0-") + + @given(data_length=st.integers(min_value=1, max_value=300000)) + def test_read_with_no_range(self, data_length): + """ + A read with no range returns the whole immutable. + """ + storage_index, uploaded_data, _ = self.upload(1, data_length) + response = result_of( + self.http.client.request( + "GET", + self.http.client.relative_url( + "/v1/{}/{}/1".format(self.KIND, _encode_si(storage_index)) + ), + ) + ) + self.assertEqual(response.code, http.OK) + self.assertEqual(result_of(response.content()), uploaded_data) + + def test_validate_content_range_response_to_read(self): + """ + The server responds to ranged reads with an appropriate Content-Range + header. + """ + storage_index, _, _ = self.upload(1, 26) + + def check_range(requested_range, expected_response): + headers = Headers() + headers.setRawHeaders("range", [requested_range]) + response = result_of( + self.http.client.request( + "GET", + self.http.client.relative_url( + "/v1/{}/{}/1".format(self.KIND, _encode_si(storage_index)) + ), + headers=headers, + ) + ) + self.assertEqual( + response.headers.getRawHeaders("content-range"), [expected_response] + ) + + check_range("bytes=0-10", "bytes 0-10/*") + # Can't go beyond the end of the immutable! + check_range("bytes=10-100", "bytes 10-25/*") + + +class ImmutableSharedTests(SharedImmutableMutableTestsMixin, SyncTestCase): + """Shared tests, running on immutables.""" + + KIND = "immutable" + clientFactory = StorageClientImmutables + + def setUp(self): + super(ImmutableSharedTests, self).setUp() + self.http = self.useFixture(HttpTestFixture()) + self.client = self.clientFactory(self.http.client) + self.general_client = StorageClientGeneral(self.http.client) + + def upload(self, share_number, data_length=26): + """ + Create a share, return (storage_index, uploaded_data). + """ + uploaded_data = (b"abcdefghijklmnopqrstuvwxyz" * ((data_length // 26) + 1))[ + :data_length + ] + upload_secret = urandom(32) + lease_secret = urandom(32) + storage_index = urandom(16) + result_of( + self.client.create( + storage_index, + {share_number}, + data_length, + upload_secret, + lease_secret, + lease_secret, + ) + ) + result_of( + self.client.write_share_chunk( + storage_index, + share_number, + upload_secret, + 0, + uploaded_data, + ) + ) + return storage_index, uploaded_data, lease_secret From 85774ced9526df771ed4b0ec14bc4fe83eaeb1dd Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 6 Jun 2022 10:56:37 -0400 Subject: [PATCH 0899/2309] Run shared tests on mutables too, with appropriate fixes to the tests and the server. --- src/allmydata/storage/http_server.py | 12 ++-- src/allmydata/test/test_storage_http.py | 82 +++++++++++++++++-------- 2 files changed, 64 insertions(+), 30 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 9735a0626..46023be72 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -599,7 +599,6 @@ class HTTPServer(object): ) def mutable_read_test_write(self, request, authorization, storage_index): """Read/test/write combined operation for mutables.""" - # TODO unit tests rtw_request = self._read_encoded(request, _SCHEMAS["mutable_read_test_write"]) secrets = ( authorization[Secrets.WRITE_ENABLER], @@ -635,11 +634,13 @@ class HTTPServer(object): ) def read_mutable_chunk(self, request, authorization, storage_index, share_number): """Read a chunk from a mutable.""" - # TODO unit tests def read_data(offset, length): - return self._storage_server.slot_readv( - storage_index, [share_number], [(offset, length)] - )[share_number][0] + try: + return self._storage_server.slot_readv( + storage_index, [share_number], [(offset, length)] + )[share_number][0] + except KeyError: + raise _HTTPError(http.NOT_FOUND) return read_range(request, read_data) @@ -664,7 +665,6 @@ class HTTPServer(object): self, request, authorization, storage_index, share_number ): """Indicate that given share is corrupt, with a text reason.""" - # TODO unit test all the paths if share_number not in { shnum for (shnum, _) in self._storage_server.get_shares(storage_index) }: diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index fc79fbe34..7ed4cd235 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -5,7 +5,7 @@ Tests for HTTP storage client + server. from base64 import b64encode from contextlib import contextmanager from os import urandom -from typing import Union, Callable, Tuple +from typing import Union, Callable, Tuple, Iterable from cbor2 import dumps from pycddl import ValidationError as CDDLValidationError from hypothesis import assume, given, strategies as st @@ -23,6 +23,7 @@ from werkzeug.exceptions import NotFound as WNotFound from .common import SyncTestCase from ..storage.http_common import get_content_type, CBOR_MIME_TYPE from ..storage.common import si_b2a +from ..storage.lease import LeaseInfo from ..storage.server import StorageServer from ..storage.http_server import ( HTTPServer, @@ -1094,20 +1095,6 @@ class MutableHTTPAPIsTests(SyncTestCase): b"abcdef-0", ) - # TODO refactor reads tests so they're shared - - def test_lease_renew_and_add(self): - pass - - def test_lease_on_unknown_storage_index(self): - pass - - def test_advise_corrupt_share(self): - pass - - def test_advise_corrupt_share_unknown(self): - pass - class SharedImmutableMutableTestsMixin: """ @@ -1127,6 +1114,10 @@ class SharedImmutableMutableTestsMixin: """ raise NotImplementedError + def get_leases(self, storage_index: bytes) -> Iterable[LeaseInfo]: + """Get leases for the storage index.""" + raise NotImplementedError() + def test_advise_corrupt_share(self): """ Advising share was corrupted succeeds from HTTP client's perspective, @@ -1160,14 +1151,14 @@ class SharedImmutableMutableTestsMixin: def test_lease_renew_and_add(self): """ - It's possible the renew the lease on an uploaded immutable, by using - the same renewal secret, or add a new lease by choosing a different - renewal secret. + It's possible the renew the lease on an uploaded mutable/immutable, by + using the same renewal secret, or add a new lease by choosing a + different renewal secret. """ # Create a storage index: storage_index, _, lease_secret = self.upload(0) - [lease] = self.http.storage_server.get_leases(storage_index) + [lease] = self.get_leases(storage_index) initial_expiration_time = lease.get_expiration_time() # Time passes: @@ -1191,7 +1182,7 @@ class SharedImmutableMutableTestsMixin: ) ) - [lease1, lease2] = self.http.storage_server.get_leases(storage_index) + [lease1, lease2] = self.get_leases(storage_index) self.assertEqual(lease1.get_expiration_time(), initial_expiration_time + 167) self.assertEqual(lease2.get_expiration_time(), initial_expiration_time + 177) @@ -1232,7 +1223,7 @@ class SharedImmutableMutableTestsMixin: storage_index, _, _ = self.upload(1) def check_bad_range(bad_range_value): - client = StorageClientImmutables( + client = self.clientFactory( StorageClientWithHeadersOverride( self.http.client, {"range": bad_range_value} ) @@ -1268,7 +1259,7 @@ class SharedImmutableMutableTestsMixin: @given(data_length=st.integers(min_value=1, max_value=300000)) def test_read_with_no_range(self, data_length): """ - A read with no range returns the whole immutable. + A read with no range returns the whole mutable/immutable. """ storage_index, uploaded_data, _ = self.upload(1, data_length) response = result_of( @@ -1306,7 +1297,7 @@ class SharedImmutableMutableTestsMixin: ) check_range("bytes=0-10", "bytes 0-10/*") - # Can't go beyond the end of the immutable! + # Can't go beyond the end of the mutable/immutable! check_range("bytes=10-100", "bytes 10-25/*") @@ -1324,7 +1315,7 @@ class ImmutableSharedTests(SharedImmutableMutableTestsMixin, SyncTestCase): def upload(self, share_number, data_length=26): """ - Create a share, return (storage_index, uploaded_data). + Create a share, return (storage_index, uploaded_data, lease_secret). """ uploaded_data = (b"abcdefghijklmnopqrstuvwxyz" * ((data_length // 26) + 1))[ :data_length @@ -1352,3 +1343,46 @@ class ImmutableSharedTests(SharedImmutableMutableTestsMixin, SyncTestCase): ) ) return storage_index, uploaded_data, lease_secret + + def get_leases(self, storage_index): + return self.http.storage_server.get_leases(storage_index) + + +class MutableSharedTests(SharedImmutableMutableTestsMixin, SyncTestCase): + """Shared tests, running on mutables.""" + + KIND = "mutable" + clientFactory = StorageClientMutables + + def setUp(self): + super(MutableSharedTests, self).setUp() + self.http = self.useFixture(HttpTestFixture()) + self.client = self.clientFactory(self.http.client) + self.general_client = StorageClientGeneral(self.http.client) + + def upload(self, share_number, data_length=26): + """ + Create a share, return (storage_index, uploaded_data, lease_secret). + """ + data = (b"abcdefghijklmnopqrstuvwxyz" * ((data_length // 26) + 1))[:data_length] + write_secret = urandom(32) + lease_secret = urandom(32) + storage_index = urandom(16) + result_of( + self.client.read_test_write_chunks( + storage_index, + write_secret, + lease_secret, + lease_secret, + { + share_number: TestWriteVectors( + write_vectors=[WriteVector(offset=0, data=data)] + ), + }, + [], + ) + ) + return storage_index, data, lease_secret + + def get_leases(self, storage_index): + return self.http.storage_server.get_slot_leases(storage_index) From ca0f311861aaefb2ae532abc6299b668d166862e Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 6 Jun 2022 10:59:29 -0400 Subject: [PATCH 0900/2309] News file. --- newsfragments/3896.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3896.minor diff --git a/newsfragments/3896.minor b/newsfragments/3896.minor new file mode 100644 index 000000000..e69de29bb From c3a304e1cc0d2eadd62017cbdea367063ec5bed2 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 6 Jun 2022 11:00:07 -0400 Subject: [PATCH 0901/2309] Lint and mypy fixes. --- src/allmydata/storage/http_client.py | 2 +- src/allmydata/storage/http_server.py | 6 +++--- src/allmydata/test/test_storage_http.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 9711e748d..9203d02ab 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -586,7 +586,7 @@ class StorageClientImmutables(object): ) @inlineCallbacks - def list_shares(self, storage_index): # type: (bytes,) -> Deferred[set[int]] + def list_shares(self, storage_index: bytes) -> Deferred[set[int]]: """ Return the set of shares for a given storage index. """ diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 46023be72..bcad0e972 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -3,7 +3,7 @@ HTTP server for storage. """ from __future__ import annotations -from typing import Dict, List, Set, Tuple, Any, Callable +from typing import Dict, List, Set, Tuple, Any, Callable, Optional from functools import wraps from base64 import b64decode @@ -274,7 +274,7 @@ _SCHEMAS = { } -def read_range(request, read_data: Callable[int, int, bytes]) -> Optional[bytes]: +def read_range(request, read_data: Callable[[int, int], bytes]) -> Optional[bytes]: """ Parse the ``Range`` header, read appropriately, return as result. @@ -298,7 +298,7 @@ def read_range(request, read_data: Callable[int, int, bytes]) -> Optional[bytes] data = read_data(start, start + 65536) if not data: request.finish() - return + return None request.write(data) start += len(data) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 7ed4cd235..5e0b35d88 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -1105,7 +1105,7 @@ class SharedImmutableMutableTestsMixin: general_client: StorageClientGeneral client: Union[StorageClientImmutables, StorageClientMutables] clientFactory: Callable[ - StorageClient, Union[StorageClientImmutables, StorageClientMutables] + [StorageClient], Union[StorageClientImmutables, StorageClientMutables] ] def upload(self, share_number: int, data_length=26) -> Tuple[bytes, bytes, bytes]: From 528d902460ae5bddaf3f140743072b3324fca5b7 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 6 Jun 2022 11:15:25 -0400 Subject: [PATCH 0902/2309] News file. --- newsfragments/3900.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3900.minor diff --git a/newsfragments/3900.minor b/newsfragments/3900.minor new file mode 100644 index 000000000..e69de29bb From 8694543659a36007c0d7a0787808fa119df15931 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 6 Jun 2022 11:15:51 -0400 Subject: [PATCH 0903/2309] Work with Sphinx 5. --- docs/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index af05e5900..cc9a11166 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -63,7 +63,7 @@ release = u'1.x' # # This is also used if you do content translation via gettext catalogs. # Usually you set "language" from the command line for these cases. -language = None +language = "en" # There are two options for replacing |today|: either, you set today to some # non-false value, then it is used: From 00381bc24fefc3a831d4f253819d154609266423 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 8 Jun 2022 13:52:45 -0400 Subject: [PATCH 0904/2309] Correction now that it does more than what it did before. --- src/allmydata/storage/http_server.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index bcad0e972..63be2f270 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -276,7 +276,8 @@ _SCHEMAS = { def read_range(request, read_data: Callable[[int, int], bytes]) -> Optional[bytes]: """ - Parse the ``Range`` header, read appropriately, return as result. + Read an optional ``Range`` header, reads data appropriately via the given + callable, return as result. Only parses a subset of ``Range`` headers that we support: must be set, bytes only, only a single range, the end must be explicitly specified. From db426513558bd328fc803c74c382f2ae0a214b92 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 8 Jun 2022 13:55:47 -0400 Subject: [PATCH 0905/2309] Be more consistent and just always write to the request in `read_range`. --- src/allmydata/storage/http_server.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 63be2f270..543fceb98 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -274,21 +274,20 @@ _SCHEMAS = { } -def read_range(request, read_data: Callable[[int, int], bytes]) -> Optional[bytes]: +def read_range(request, read_data: Callable[[int, int], bytes]) -> None: """ Read an optional ``Range`` header, reads data appropriately via the given - callable, return as result. + callable, writes the data to the request. Only parses a subset of ``Range`` headers that we support: must be set, bytes only, only a single range, the end must be explicitly specified. Raises a ``_HTTPError(http.REQUESTED_RANGE_NOT_SATISFIABLE)`` if parsing is not possible or the header isn't set. - Returns a result that should be returned from the request handler, and sets - appropriate response headers. - Takes a function that will do the actual reading given the start offset and a length to read. + + The resulting data is written to the request. """ if request.getHeader("range") is None: # Return the whole thing. @@ -299,7 +298,7 @@ def read_range(request, read_data: Callable[[int, int], bytes]) -> Optional[byte data = read_data(start, start + 65536) if not data: request.finish() - return None + return request.write(data) start += len(data) @@ -326,7 +325,8 @@ def read_range(request, read_data: Callable[[int, int], bytes]) -> Optional[byte "content-range", ContentRange("bytes", offset, offset + len(data)).to_header(), ) - return data + request.write(data) + request.finish() class HTTPServer(object): From d37f187c078868833a98a362e528f559b97cdd94 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 8 Jun 2022 13:56:23 -0400 Subject: [PATCH 0906/2309] Lint fix. --- src/allmydata/storage/http_server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 543fceb98..06a6863fa 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -3,7 +3,7 @@ HTTP server for storage. """ from __future__ import annotations -from typing import Dict, List, Set, Tuple, Any, Callable, Optional +from typing import Dict, List, Set, Tuple, Any, Callable from functools import wraps from base64 import b64decode From 839aaea541a2d9504abe46a1b8ecb97e333f8971 Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 8 Jun 2022 21:26:40 -0600 Subject: [PATCH 0907/2309] let misconfigured servers show up, and display information about missing plugins --- src/allmydata/storage_client.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 3fc18a908..a3974d4b9 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -661,6 +661,19 @@ class AnnouncementNotMatched(Exception): """ +@attr.s(auto_exc=True) +class MissingPlugin(Exception): + """ + A particular plugin was request, but is missing + """ + + plugin_name = attr.ib() + nickname = attr.ib() + + def __str__(self): + return "Missing plugin '{}' for server '{}'".format(self.plugin_name, self.nickname) + + def _storage_from_foolscap_plugin(node_config, config, announcement, get_rref): """ Construct an ``IStorageServer`` from the most locally-preferred plugin @@ -682,7 +695,7 @@ def _storage_from_foolscap_plugin(node_config, config, announcement, get_rref): try: plugin = plugins[plugin_name] except KeyError: - raise ValueError("{} not installed".format(plugin_name)) + raise MissingPlugin(plugin_name, announcement.get(u"nickname", "")) for option in storage_options: if plugin_name == option[u"name"]: furl = option[u"storage-server-FURL"] @@ -773,6 +786,11 @@ class NativeStorageServer(service.MultiService): nickname=ann.get("nickname", ""), plugins=e.args[0], ) + except MissingPlugin as e: + self.log.failure("Missing plugin") + ns = _NullStorage() + ns.longname = ''.format(e.args[0]) + return ns else: return _FoolscapStorage.from_announcement( self._server_id, From 6116b04ff7bd7a910651fb53f94b437f5595243f Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 10 Jun 2022 14:08:53 -0600 Subject: [PATCH 0908/2309] ignore incorrectly packaged autobahn versions --- setup.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index c84d0ecde..b14893712 100644 --- a/setup.py +++ b/setup.py @@ -114,7 +114,9 @@ install_requires = [ "attrs >= 18.2.0", # WebSocket library for twisted and asyncio - "autobahn >= 19.5.2", + "autobahn >= 19.5.2, != 22.5.1, != 22.4.2, != 22.4.1" + # (the ignored versions above don't have autobahn.twisted.testing + # packaged properly) # Support for Python 3 transition "future >= 0.18.2", From e1daa192fbe8ff8168392ec9989664ddd4b3905a Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 21 Jun 2022 17:20:08 -0400 Subject: [PATCH 0909/2309] Sketch of protocol switcher experiment. --- src/allmydata/node.py | 2 + src/allmydata/protocol_switch.py | 84 ++++++++++++++++++++++++++++++++ 2 files changed, 86 insertions(+) create mode 100644 src/allmydata/protocol_switch.py diff --git a/src/allmydata/node.py b/src/allmydata/node.py index 3ac4c507b..0547d3fe6 100644 --- a/src/allmydata/node.py +++ b/src/allmydata/node.py @@ -51,6 +51,7 @@ from allmydata.util import configutil from allmydata.util.yamlutil import ( safe_load, ) +from .protocol_switch import FoolscapOrHttp from . import ( __full_version__, @@ -707,6 +708,7 @@ def create_tub(tub_options, default_connection_handlers, foolscap_connection_han the new Tub via `Tub.setOption` """ tub = Tub(**kwargs) + tub.negotiationClass = FoolscapOrHttp for (name, value) in list(tub_options.items()): tub.setOption(name, value) handlers = default_connection_handlers.copy() diff --git a/src/allmydata/protocol_switch.py b/src/allmydata/protocol_switch.py new file mode 100644 index 000000000..59e1b609f --- /dev/null +++ b/src/allmydata/protocol_switch.py @@ -0,0 +1,84 @@ +""" +Support for listening with both HTTP and Foolscap on the same port. +""" + +from enum import Enum +from typing import Optional + +from twisted.internet.protocol import Protocol +from twisted.python.failure import Failure + +from foolscap.negotiate import Negotiation + +class ProtocolMode(Enum): + """Listening mode.""" + UNDECIDED = 0 + FOOLSCAP = 1 + HTTP = 2 + + +class PretendToBeNegotiation(type): + """😱""" + + def __instancecheck__(self, instance): + return (instance.__class__ == self) or isinstance(instance, Negotiation) + + +class FoolscapOrHttp(Protocol, metaclass=PretendToBeNegotiation): + """ + Based on initial query, decide whether we're talking Foolscap or HTTP. + + Pretends to be a ``foolscap.negotiate.Negotiation`` instance. + """ + _foolscap : Optional[Negotiation] = None + _protocol_mode : ProtocolMode = ProtocolMode.UNDECIDED + _buffer: bytes = b"" + + def __init__(self, *args, **kwargs): + self._foolscap = Negotiation(*args, **kwargs) + + def __setattr__(self, name, value): + if name in {"_foolscap", "_protocol_mode", "_buffer", "transport"}: + object.__setattr__(self, name, value) + else: + setattr(self._foolscap, name, value) + + def __getattr__(self, name): + return getattr(self._foolscap, name) + + def makeConnection(self, transport): + Protocol.makeConnection(self, transport) + self._foolscap.makeConnection(transport) + + def initServer(self, *args, **kwargs): + return self._foolscap.initServer(*args, **kwargs) + + def initClient(self, *args, **kwargs): + assert not self._buffer + self._protocol_mode = ProtocolMode.FOOLSCAP + return self._foolscap.initClient(*args, **kwargs) + + def dataReceived(self, data: bytes) -> None: + if self._protocol_mode == ProtocolMode.FOOLSCAP: + return self._foolscap.dataReceived(data) + if self._protocol_mode == ProtocolMode.HTTP: + raise NotImplementedError() + + # UNDECIDED mode. + self._buffer += data + if len(self._buffer) < 8: + return + + # Check if it looks like Foolscap request. If so, it can handle this + # and later data: + if self._buffer.startswith(b"GET /id/"): + self._protocol_mode = ProtocolMode.FOOLSCAP + buf, self._buffer = self._buffer, b"" + return self._foolscap.dataReceived(buf) + else: + self._protocol_mode = ProtocolMode.HTTP + raise NotImplementedError("") + + def connectionLost(self, reason: Failure) -> None: + if self._protocol_mode == ProtocolMode.FOOLSCAP: + return self._foolscap.connectionLost(reason) From 7910867be6b8154f2b10031f3790b1a8c5eba821 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 22 Jun 2022 10:23:23 -0400 Subject: [PATCH 0910/2309] It actually works(?!) now. --- src/allmydata/protocol_switch.py | 29 ++++++++++++++++++++--------- 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/src/allmydata/protocol_switch.py b/src/allmydata/protocol_switch.py index 59e1b609f..fa23738d2 100644 --- a/src/allmydata/protocol_switch.py +++ b/src/allmydata/protocol_switch.py @@ -10,8 +10,10 @@ from twisted.python.failure import Failure from foolscap.negotiate import Negotiation + class ProtocolMode(Enum): """Listening mode.""" + UNDECIDED = 0 FOOLSCAP = 1 HTTP = 2 @@ -30,15 +32,22 @@ class FoolscapOrHttp(Protocol, metaclass=PretendToBeNegotiation): Pretends to be a ``foolscap.negotiate.Negotiation`` instance. """ - _foolscap : Optional[Negotiation] = None - _protocol_mode : ProtocolMode = ProtocolMode.UNDECIDED + + _foolscap: Optional[Negotiation] = None + _protocol_mode: ProtocolMode = ProtocolMode.UNDECIDED _buffer: bytes = b"" def __init__(self, *args, **kwargs): self._foolscap = Negotiation(*args, **kwargs) def __setattr__(self, name, value): - if name in {"_foolscap", "_protocol_mode", "_buffer", "transport"}: + if name in { + "_foolscap", + "_protocol_mode", + "_buffer", + "transport", + "__class__", + }: object.__setattr__(self, name, value) else: setattr(self._foolscap, name, value) @@ -50,13 +59,15 @@ class FoolscapOrHttp(Protocol, metaclass=PretendToBeNegotiation): Protocol.makeConnection(self, transport) self._foolscap.makeConnection(transport) - def initServer(self, *args, **kwargs): - return self._foolscap.initServer(*args, **kwargs) - def initClient(self, *args, **kwargs): + # After creation, a Negotiation instance either has initClient() or + # initServer() called. SInce this is a client, we're never going to do + # HTTP. Relying on __getattr__/__setattr__ doesn't work, for some + # reason, so just mutate ourselves appropriately. assert not self._buffer - self._protocol_mode = ProtocolMode.FOOLSCAP - return self._foolscap.initClient(*args, **kwargs) + self.__class__ = Negotiation + self.__dict__ = self._foolscap.__dict__ + return self.initClient(*args, **kwargs) def dataReceived(self, data: bytes) -> None: if self._protocol_mode == ProtocolMode.FOOLSCAP: @@ -69,7 +80,7 @@ class FoolscapOrHttp(Protocol, metaclass=PretendToBeNegotiation): if len(self._buffer) < 8: return - # Check if it looks like Foolscap request. If so, it can handle this + # Check if it looks like a Foolscap request. If so, it can handle this # and later data: if self._buffer.startswith(b"GET /id/"): self._protocol_mode = ProtocolMode.FOOLSCAP From 7577d1e24ca8f8a93a11b3bd87deb251f40cbce8 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 22 Jun 2022 14:19:29 -0400 Subject: [PATCH 0911/2309] Sketch of HTTP support, still untested WIP. --- src/allmydata/client.py | 8 ++++++++ src/allmydata/node.py | 2 -- src/allmydata/protocol_switch.py | 24 ++++++++++++++++++++++-- 3 files changed, 30 insertions(+), 4 deletions(-) diff --git a/src/allmydata/client.py b/src/allmydata/client.py index 56ecdc6ed..ad5feb2ed 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -64,6 +64,7 @@ from allmydata.interfaces import ( from allmydata.nodemaker import NodeMaker from allmydata.blacklist import Blacklist from allmydata import node +from .protocol_switch import FoolscapOrHttp KiB=1024 @@ -818,6 +819,13 @@ class _Client(node.Node, pollmixin.PollMixin): if anonymous_storage_enabled(self.config): furl_file = self.config.get_private_path("storage.furl").encode(get_filesystem_encoding()) furl = self.tub.registerReference(FoolscapStorageServer(ss), furlFile=furl_file) + (_, _, swissnum) = furl.rpartition("/") + class FoolscapOrHttpWithCert(FoolscapOrHttp): + certificate = self.tub.myCertificate + storage_server = ss + swissnum = swissnum + self.tub.negotiationClass = FoolscapOrHttpWithCert + announcement["anonymous-storage-FURL"] = furl enabled_storage_servers = self._enable_storage_servers( diff --git a/src/allmydata/node.py b/src/allmydata/node.py index 0547d3fe6..3ac4c507b 100644 --- a/src/allmydata/node.py +++ b/src/allmydata/node.py @@ -51,7 +51,6 @@ from allmydata.util import configutil from allmydata.util.yamlutil import ( safe_load, ) -from .protocol_switch import FoolscapOrHttp from . import ( __full_version__, @@ -708,7 +707,6 @@ def create_tub(tub_options, default_connection_handlers, foolscap_connection_han the new Tub via `Tub.setOption` """ tub = Tub(**kwargs) - tub.negotiationClass = FoolscapOrHttp for (name, value) in list(tub_options.items()): tub.setOption(name, value) handlers = default_connection_handlers.copy() diff --git a/src/allmydata/protocol_switch.py b/src/allmydata/protocol_switch.py index fa23738d2..5a9589c17 100644 --- a/src/allmydata/protocol_switch.py +++ b/src/allmydata/protocol_switch.py @@ -7,9 +7,14 @@ from typing import Optional from twisted.internet.protocol import Protocol from twisted.python.failure import Failure +from twisted.internet.ssl import CertificateOptions +from twisted.web.server import Site +from twisted.protocols.tls import TLSMemoryBIOFactory from foolscap.negotiate import Negotiation +from .storage.http_server import HTTPServer + class ProtocolMode(Enum): """Listening mode.""" @@ -47,6 +52,7 @@ class FoolscapOrHttp(Protocol, metaclass=PretendToBeNegotiation): "_buffer", "transport", "__class__", + "_http", }: object.__setattr__(self, name, value) else: @@ -73,7 +79,7 @@ class FoolscapOrHttp(Protocol, metaclass=PretendToBeNegotiation): if self._protocol_mode == ProtocolMode.FOOLSCAP: return self._foolscap.dataReceived(data) if self._protocol_mode == ProtocolMode.HTTP: - raise NotImplementedError() + return self._http.dataReceived(data) # UNDECIDED mode. self._buffer += data @@ -83,12 +89,26 @@ class FoolscapOrHttp(Protocol, metaclass=PretendToBeNegotiation): # Check if it looks like a Foolscap request. If so, it can handle this # and later data: if self._buffer.startswith(b"GET /id/"): + # TODO or maybe just self.__class__ here too? self._protocol_mode = ProtocolMode.FOOLSCAP buf, self._buffer = self._buffer, b"" return self._foolscap.dataReceived(buf) else: self._protocol_mode = ProtocolMode.HTTP - raise NotImplementedError("") + + certificate_options = CertificateOptions( + privateKey=self.certificate.privateKey.original, + certificate=self.certificate.original, + ) + http_server = HTTPServer(self.storage_server, self.swissnum) + factory = TLSMemoryBIOFactory( + certificate_options, False, Site(http_server.get_resource()) + ) + protocol = factory.buildProtocol(self.transport.getPeer()) + protocol.makeConnection(self.transport) + protocol.dataReceived(self._buffer) + # TODO __getattr__ or maybe change the __class__ + self._http = protocol def connectionLost(self, reason: Failure) -> None: if self._protocol_mode == ProtocolMode.FOOLSCAP: From c5724c1d0a70ad1aa539aa0063a300bc359ddf21 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 22 Jun 2022 14:20:42 -0400 Subject: [PATCH 0912/2309] Clarify. --- src/allmydata/protocol_switch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/protocol_switch.py b/src/allmydata/protocol_switch.py index 5a9589c17..50a7b1476 100644 --- a/src/allmydata/protocol_switch.py +++ b/src/allmydata/protocol_switch.py @@ -107,7 +107,7 @@ class FoolscapOrHttp(Protocol, metaclass=PretendToBeNegotiation): protocol = factory.buildProtocol(self.transport.getPeer()) protocol.makeConnection(self.transport) protocol.dataReceived(self._buffer) - # TODO __getattr__ or maybe change the __class__ + # TODO maybe change the __class__ self._http = protocol def connectionLost(self, reason: Failure) -> None: From 1579530895c5e66997f95ff8424d26be73dec011 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 23 Jun 2022 07:59:43 -0400 Subject: [PATCH 0913/2309] Add working HTTP support. --- src/allmydata/client.py | 8 ++------ src/allmydata/node.py | 3 +++ src/allmydata/protocol_switch.py | 19 +++++++++++++++++++ 3 files changed, 24 insertions(+), 6 deletions(-) diff --git a/src/allmydata/client.py b/src/allmydata/client.py index ad5feb2ed..294684b58 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -64,7 +64,7 @@ from allmydata.interfaces import ( from allmydata.nodemaker import NodeMaker from allmydata.blacklist import Blacklist from allmydata import node -from .protocol_switch import FoolscapOrHttp +from .protocol_switch import update_foolscap_or_http_class KiB=1024 @@ -820,11 +820,7 @@ class _Client(node.Node, pollmixin.PollMixin): furl_file = self.config.get_private_path("storage.furl").encode(get_filesystem_encoding()) furl = self.tub.registerReference(FoolscapStorageServer(ss), furlFile=furl_file) (_, _, swissnum) = furl.rpartition("/") - class FoolscapOrHttpWithCert(FoolscapOrHttp): - certificate = self.tub.myCertificate - storage_server = ss - swissnum = swissnum - self.tub.negotiationClass = FoolscapOrHttpWithCert + update_foolscap_or_http_class(self.tub.negotiationClass, self.tub.myCertificate, ss, swissnum.encode("ascii")) announcement["anonymous-storage-FURL"] = furl diff --git a/src/allmydata/node.py b/src/allmydata/node.py index 3ac4c507b..93fa6a8e1 100644 --- a/src/allmydata/node.py +++ b/src/allmydata/node.py @@ -55,6 +55,8 @@ from allmydata.util.yamlutil import ( from . import ( __full_version__, ) +from .protocol_switch import create_foolscap_or_http_class + def _common_valid_config(): return configutil.ValidConfiguration({ @@ -707,6 +709,7 @@ def create_tub(tub_options, default_connection_handlers, foolscap_connection_han the new Tub via `Tub.setOption` """ tub = Tub(**kwargs) + tub.negotiationClass = create_foolscap_or_http_class() for (name, value) in list(tub_options.items()): tub.setOption(name, value) handlers = default_connection_handlers.copy() diff --git a/src/allmydata/protocol_switch.py b/src/allmydata/protocol_switch.py index 50a7b1476..bb1a59bef 100644 --- a/src/allmydata/protocol_switch.py +++ b/src/allmydata/protocol_switch.py @@ -14,6 +14,7 @@ from twisted.protocols.tls import TLSMemoryBIOFactory from foolscap.negotiate import Negotiation from .storage.http_server import HTTPServer +from .storage.server import StorageServer class ProtocolMode(Enum): @@ -38,6 +39,11 @@ class FoolscapOrHttp(Protocol, metaclass=PretendToBeNegotiation): Pretends to be a ``foolscap.negotiate.Negotiation`` instance. """ + # These three will be set by a subclass + swissnum: bytes + certificate = None # TODO figure out type + storage_server: StorageServer + _foolscap: Optional[Negotiation] = None _protocol_mode: ProtocolMode = ProtocolMode.UNDECIDED _buffer: bytes = b"" @@ -113,3 +119,16 @@ class FoolscapOrHttp(Protocol, metaclass=PretendToBeNegotiation): def connectionLost(self, reason: Failure) -> None: if self._protocol_mode == ProtocolMode.FOOLSCAP: return self._foolscap.connectionLost(reason) + + +def create_foolscap_or_http_class(): + class FoolscapOrHttpWithCert(FoolscapOrHttp): + pass + + return FoolscapOrHttpWithCert + + +def update_foolscap_or_http_class(cls, certificate, storage_server, swissnum): + cls.certificate = certificate + cls.storage_server = storage_server + cls.swissnum = swissnum From 04156db74ef28c63fb2273277f3f52b5f6d4883c Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 23 Jun 2022 12:32:43 -0400 Subject: [PATCH 0914/2309] Delay Negotiation.connectionMade so we don't create unnecessary timeouts. --- src/allmydata/protocol_switch.py | 34 ++++++++++++++------------------ 1 file changed, 15 insertions(+), 19 deletions(-) diff --git a/src/allmydata/protocol_switch.py b/src/allmydata/protocol_switch.py index bb1a59bef..2f834081b 100644 --- a/src/allmydata/protocol_switch.py +++ b/src/allmydata/protocol_switch.py @@ -3,10 +3,10 @@ Support for listening with both HTTP and Foolscap on the same port. """ from enum import Enum -from typing import Optional +from typing import Optional, Tuple from twisted.internet.protocol import Protocol -from twisted.python.failure import Failure +from twisted.internet.interfaces import ITransport from twisted.internet.ssl import CertificateOptions from twisted.web.server import Site from twisted.protocols.tls import TLSMemoryBIOFactory @@ -21,8 +21,7 @@ class ProtocolMode(Enum): """Listening mode.""" UNDECIDED = 0 - FOOLSCAP = 1 - HTTP = 2 + HTTP = 1 class PretendToBeNegotiation(type): @@ -67,9 +66,13 @@ class FoolscapOrHttp(Protocol, metaclass=PretendToBeNegotiation): def __getattr__(self, name): return getattr(self._foolscap, name) - def makeConnection(self, transport): - Protocol.makeConnection(self, transport) - self._foolscap.makeConnection(transport) + def _convert_to_negotiation(self) -> Tuple[bytes, ITransport]: + """Convert self to a ``Negotiation`` instance, return any buffered bytes""" + transport = self.transport + buf = self._buffer + self.__class__ = Negotiation # type: ignore + self.__dict__ = self._foolscap.__dict__ + return buf, transport def initClient(self, *args, **kwargs): # After creation, a Negotiation instance either has initClient() or @@ -77,13 +80,10 @@ class FoolscapOrHttp(Protocol, metaclass=PretendToBeNegotiation): # HTTP. Relying on __getattr__/__setattr__ doesn't work, for some # reason, so just mutate ourselves appropriately. assert not self._buffer - self.__class__ = Negotiation - self.__dict__ = self._foolscap.__dict__ + self._convert_to_negotiation() return self.initClient(*args, **kwargs) def dataReceived(self, data: bytes) -> None: - if self._protocol_mode == ProtocolMode.FOOLSCAP: - return self._foolscap.dataReceived(data) if self._protocol_mode == ProtocolMode.HTTP: return self._http.dataReceived(data) @@ -95,10 +95,10 @@ class FoolscapOrHttp(Protocol, metaclass=PretendToBeNegotiation): # Check if it looks like a Foolscap request. If so, it can handle this # and later data: if self._buffer.startswith(b"GET /id/"): - # TODO or maybe just self.__class__ here too? - self._protocol_mode = ProtocolMode.FOOLSCAP - buf, self._buffer = self._buffer, b"" - return self._foolscap.dataReceived(buf) + buf, transport = self._convert_to_negotiation() + self.makeConnection(transport) + self.dataReceived(buf) + return else: self._protocol_mode = ProtocolMode.HTTP @@ -116,10 +116,6 @@ class FoolscapOrHttp(Protocol, metaclass=PretendToBeNegotiation): # TODO maybe change the __class__ self._http = protocol - def connectionLost(self, reason: Failure) -> None: - if self._protocol_mode == ProtocolMode.FOOLSCAP: - return self._foolscap.connectionLost(reason) - def create_foolscap_or_http_class(): class FoolscapOrHttpWithCert(FoolscapOrHttp): From d86f8519dcdd14622eb5d695b880a77db2038e89 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 23 Jun 2022 12:41:01 -0400 Subject: [PATCH 0915/2309] Simplify implementation. --- src/allmydata/protocol_switch.py | 32 +++++++++----------------------- 1 file changed, 9 insertions(+), 23 deletions(-) diff --git a/src/allmydata/protocol_switch.py b/src/allmydata/protocol_switch.py index 2f834081b..11a35c324 100644 --- a/src/allmydata/protocol_switch.py +++ b/src/allmydata/protocol_switch.py @@ -2,7 +2,6 @@ Support for listening with both HTTP and Foolscap on the same port. """ -from enum import Enum from typing import Optional, Tuple from twisted.internet.protocol import Protocol @@ -17,13 +16,6 @@ from .storage.http_server import HTTPServer from .storage.server import StorageServer -class ProtocolMode(Enum): - """Listening mode.""" - - UNDECIDED = 0 - HTTP = 1 - - class PretendToBeNegotiation(type): """😱""" @@ -43,21 +35,16 @@ class FoolscapOrHttp(Protocol, metaclass=PretendToBeNegotiation): certificate = None # TODO figure out type storage_server: StorageServer - _foolscap: Optional[Negotiation] = None - _protocol_mode: ProtocolMode = ProtocolMode.UNDECIDED - _buffer: bytes = b"" - def __init__(self, *args, **kwargs): - self._foolscap = Negotiation(*args, **kwargs) + self._foolscap: Negotiation = Negotiation(*args, **kwargs) + self._buffer: bytes = b"" def __setattr__(self, name, value): if name in { "_foolscap", - "_protocol_mode", "_buffer", "transport", "__class__", - "_http", }: object.__setattr__(self, name, value) else: @@ -66,7 +53,7 @@ class FoolscapOrHttp(Protocol, metaclass=PretendToBeNegotiation): def __getattr__(self, name): return getattr(self._foolscap, name) - def _convert_to_negotiation(self) -> Tuple[bytes, ITransport]: + def _convert_to_negotiation(self) -> Tuple[bytes, Optional[ITransport]]: """Convert self to a ``Negotiation`` instance, return any buffered bytes""" transport = self.transport buf = self._buffer @@ -84,10 +71,11 @@ class FoolscapOrHttp(Protocol, metaclass=PretendToBeNegotiation): return self.initClient(*args, **kwargs) def dataReceived(self, data: bytes) -> None: - if self._protocol_mode == ProtocolMode.HTTP: - return self._http.dataReceived(data) + """Handle incoming data. - # UNDECIDED mode. + Once we've decided which protocol we are, update self.__class__, at + which point all methods will be called on the new class. + """ self._buffer += data if len(self._buffer) < 8: return @@ -100,8 +88,6 @@ class FoolscapOrHttp(Protocol, metaclass=PretendToBeNegotiation): self.dataReceived(buf) return else: - self._protocol_mode = ProtocolMode.HTTP - certificate_options = CertificateOptions( privateKey=self.certificate.privateKey.original, certificate=self.certificate.original, @@ -113,8 +99,8 @@ class FoolscapOrHttp(Protocol, metaclass=PretendToBeNegotiation): protocol = factory.buildProtocol(self.transport.getPeer()) protocol.makeConnection(self.transport) protocol.dataReceived(self._buffer) - # TODO maybe change the __class__ - self._http = protocol + self.__class__ = protocol.__class__ + self.__dict__ = protocol.__dict__ def create_foolscap_or_http_class(): From 026d63cd6a83f274fb1336945afe5145f0afc226 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 23 Jun 2022 12:41:47 -0400 Subject: [PATCH 0916/2309] Fix some mypy warnings. --- src/allmydata/protocol_switch.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/allmydata/protocol_switch.py b/src/allmydata/protocol_switch.py index 11a35c324..f3a624318 100644 --- a/src/allmydata/protocol_switch.py +++ b/src/allmydata/protocol_switch.py @@ -6,7 +6,7 @@ from typing import Optional, Tuple from twisted.internet.protocol import Protocol from twisted.internet.interfaces import ITransport -from twisted.internet.ssl import CertificateOptions +from twisted.internet.ssl import CertificateOptions, PrivateCertificate from twisted.web.server import Site from twisted.protocols.tls import TLSMemoryBIOFactory @@ -32,7 +32,7 @@ class FoolscapOrHttp(Protocol, metaclass=PretendToBeNegotiation): # These three will be set by a subclass swissnum: bytes - certificate = None # TODO figure out type + certificate: PrivateCertificate storage_server: StorageServer def __init__(self, *args, **kwargs): @@ -96,6 +96,7 @@ class FoolscapOrHttp(Protocol, metaclass=PretendToBeNegotiation): factory = TLSMemoryBIOFactory( certificate_options, False, Site(http_server.get_resource()) ) + assert self.transport is not None protocol = factory.buildProtocol(self.transport.getPeer()) protocol.makeConnection(self.transport) protocol.dataReceived(self._buffer) From d70f583172e409268cd83c58d935815ec70296b4 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 23 Jun 2022 12:43:46 -0400 Subject: [PATCH 0917/2309] More cleanups. --- src/allmydata/protocol_switch.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/src/allmydata/protocol_switch.py b/src/allmydata/protocol_switch.py index f3a624318..23d7dda84 100644 --- a/src/allmydata/protocol_switch.py +++ b/src/allmydata/protocol_switch.py @@ -30,7 +30,8 @@ class FoolscapOrHttp(Protocol, metaclass=PretendToBeNegotiation): Pretends to be a ``foolscap.negotiate.Negotiation`` instance. """ - # These three will be set by a subclass + # These three will be set by a subclass in update_foolscap_or_http_class() + # below. swissnum: bytes certificate: PrivateCertificate storage_server: StorageServer @@ -53,19 +54,18 @@ class FoolscapOrHttp(Protocol, metaclass=PretendToBeNegotiation): def __getattr__(self, name): return getattr(self._foolscap, name) - def _convert_to_negotiation(self) -> Tuple[bytes, Optional[ITransport]]: - """Convert self to a ``Negotiation`` instance, return any buffered bytes""" - transport = self.transport - buf = self._buffer + def _convert_to_negotiation(self): + """ + Convert self to a ``Negotiation`` instance, return any buffered + bytes and the transport if any. + """ self.__class__ = Negotiation # type: ignore self.__dict__ = self._foolscap.__dict__ - return buf, transport def initClient(self, *args, **kwargs): # After creation, a Negotiation instance either has initClient() or - # initServer() called. SInce this is a client, we're never going to do - # HTTP. Relying on __getattr__/__setattr__ doesn't work, for some - # reason, so just mutate ourselves appropriately. + # initServer() called. Since this is a client, we're never going to do + # HTTP, so we can immediately become a Negotiation instance. assert not self._buffer self._convert_to_negotiation() return self.initClient(*args, **kwargs) @@ -83,7 +83,9 @@ class FoolscapOrHttp(Protocol, metaclass=PretendToBeNegotiation): # Check if it looks like a Foolscap request. If so, it can handle this # and later data: if self._buffer.startswith(b"GET /id/"): - buf, transport = self._convert_to_negotiation() + transport = self.transport + buf = self._buffer + self._convert_to_negotiation() self.makeConnection(transport) self.dataReceived(buf) return From 0c99a9f7b0b14d340586d42cb589d3c888c3db1d Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 23 Jun 2022 12:44:17 -0400 Subject: [PATCH 0918/2309] Make it more accurate. --- src/allmydata/protocol_switch.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/allmydata/protocol_switch.py b/src/allmydata/protocol_switch.py index 23d7dda84..899c1258f 100644 --- a/src/allmydata/protocol_switch.py +++ b/src/allmydata/protocol_switch.py @@ -56,8 +56,7 @@ class FoolscapOrHttp(Protocol, metaclass=PretendToBeNegotiation): def _convert_to_negotiation(self): """ - Convert self to a ``Negotiation`` instance, return any buffered - bytes and the transport if any. + Convert self to a ``Negotiation`` instance. """ self.__class__ = Negotiation # type: ignore self.__dict__ = self._foolscap.__dict__ From eb1e48bcc367e78945024cc642a59693aa3ecf09 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 23 Jun 2022 12:47:33 -0400 Subject: [PATCH 0919/2309] Add a timeout. --- src/allmydata/protocol_switch.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/allmydata/protocol_switch.py b/src/allmydata/protocol_switch.py index 899c1258f..2d2590977 100644 --- a/src/allmydata/protocol_switch.py +++ b/src/allmydata/protocol_switch.py @@ -2,13 +2,14 @@ Support for listening with both HTTP and Foolscap on the same port. """ -from typing import Optional, Tuple +from typing import Optional from twisted.internet.protocol import Protocol -from twisted.internet.interfaces import ITransport +from twisted.internet.interfaces import IDelayedCall from twisted.internet.ssl import CertificateOptions, PrivateCertificate from twisted.web.server import Site from twisted.protocols.tls import TLSMemoryBIOFactory +from twisted.internet import reactor from foolscap.negotiate import Negotiation @@ -36,6 +37,8 @@ class FoolscapOrHttp(Protocol, metaclass=PretendToBeNegotiation): certificate: PrivateCertificate storage_server: StorageServer + _timeout: IDelayedCall + def __init__(self, *args, **kwargs): self._foolscap: Negotiation = Negotiation(*args, **kwargs) self._buffer: bytes = b"" @@ -69,6 +72,9 @@ class FoolscapOrHttp(Protocol, metaclass=PretendToBeNegotiation): self._convert_to_negotiation() return self.initClient(*args, **kwargs) + def connectionMade(self): + self._timeout = reactor.callLater(30, self.transport.abortConnection) + def dataReceived(self, data: bytes) -> None: """Handle incoming data. @@ -80,7 +86,8 @@ class FoolscapOrHttp(Protocol, metaclass=PretendToBeNegotiation): return # Check if it looks like a Foolscap request. If so, it can handle this - # and later data: + # and later data, otherwise assume HTTPS. + self._timeout.cancel() if self._buffer.startswith(b"GET /id/"): transport = self.transport buf = self._buffer From 01d8cc7ab66745d4371820334619f3ecd4ca2881 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 23 Jun 2022 12:49:07 -0400 Subject: [PATCH 0920/2309] Put the attribute on the correct object. --- src/allmydata/protocol_switch.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/allmydata/protocol_switch.py b/src/allmydata/protocol_switch.py index 2d2590977..d26bad745 100644 --- a/src/allmydata/protocol_switch.py +++ b/src/allmydata/protocol_switch.py @@ -44,12 +44,7 @@ class FoolscapOrHttp(Protocol, metaclass=PretendToBeNegotiation): self._buffer: bytes = b"" def __setattr__(self, name, value): - if name in { - "_foolscap", - "_buffer", - "transport", - "__class__", - }: + if name in {"_foolscap", "_buffer", "transport", "__class__", "_timeout"}: object.__setattr__(self, name, value) else: setattr(self._foolscap, name, value) From 1154371d22abd859a48efdee8eee9146b3164b1c Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 23 Jun 2022 12:51:07 -0400 Subject: [PATCH 0921/2309] Clarifying comments. --- src/allmydata/protocol_switch.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/allmydata/protocol_switch.py b/src/allmydata/protocol_switch.py index d26bad745..9b4e30671 100644 --- a/src/allmydata/protocol_switch.py +++ b/src/allmydata/protocol_switch.py @@ -84,6 +84,7 @@ class FoolscapOrHttp(Protocol, metaclass=PretendToBeNegotiation): # and later data, otherwise assume HTTPS. self._timeout.cancel() if self._buffer.startswith(b"GET /id/"): + # We're a Foolscap Negotiation server protocol instance: transport = self.transport buf = self._buffer self._convert_to_negotiation() @@ -91,6 +92,7 @@ class FoolscapOrHttp(Protocol, metaclass=PretendToBeNegotiation): self.dataReceived(buf) return else: + # We're a HTTPS protocol instance, serving the storage protocol: certificate_options = CertificateOptions( privateKey=self.certificate.privateKey.original, certificate=self.certificate.original, From bfd54dc6eadf4e012c3dbf32a2356243c0aa505c Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 27 Jun 2022 11:30:49 -0400 Subject: [PATCH 0922/2309] Switch to newer attrs API, for consistency across the module. --- src/allmydata/storage/http_server.py | 31 ++++++++++++---------------- 1 file changed, 13 insertions(+), 18 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 06a6863fa..ebd2323ef 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -19,7 +19,7 @@ from twisted.web.server import Site from twisted.protocols.tls import TLSMemoryBIOFactory from twisted.python.filepath import FilePath -import attr +from attrs import define, field from werkzeug.http import ( parse_range_header, parse_content_range_header, @@ -137,31 +137,31 @@ def _authorized_route(app, required_secrets, *route_args, **route_kwargs): return decorator -@attr.s +@define class StorageIndexUploads(object): """ In-progress upload to storage index. """ # Map share number to BucketWriter - shares = attr.ib(factory=dict) # type: Dict[int,BucketWriter] + shares: dict[int, BucketWriter] = field(factory=dict) # Map share number to the upload secret (different shares might have # different upload secrets). - upload_secrets = attr.ib(factory=dict) # type: Dict[int,bytes] + upload_secrets: dict[int, bytes] = field(factory=dict) -@attr.s +@define class UploadsInProgress(object): """ Keep track of uploads for storage indexes. """ # Map storage index to corresponding uploads-in-progress - _uploads = attr.ib(type=Dict[bytes, StorageIndexUploads], factory=dict) + _uploads: dict[bytes, StorageIndexUploads] = field(factory=dict) # Map BucketWriter to (storage index, share number) - _bucketwriters = attr.ib(type=Dict[BucketWriter, Tuple[bytes, int]], factory=dict) + _bucketwriters: dict[BucketWriter, Tuple[bytes, int]] = field(factory=dict) def add_write_bucket( self, @@ -445,10 +445,7 @@ class HTTPServer(object): return self._send_encoded( request, - { - "already-have": set(already_got), - "allocated": set(sharenum_to_bucket), - }, + {"already-have": set(already_got), "allocated": set(sharenum_to_bucket)}, ) @_authorized_route( @@ -635,6 +632,7 @@ class HTTPServer(object): ) def read_mutable_chunk(self, request, authorization, storage_index, share_number): """Read a chunk from a mutable.""" + def read_data(offset, length): try: return self._storage_server.slot_readv( @@ -646,10 +644,7 @@ class HTTPServer(object): return read_range(request, read_data) @_authorized_route( - _app, - set(), - "/v1/mutable//shares", - methods=["GET"], + _app, set(), "/v1/mutable//shares", methods=["GET"] ) def enumerate_mutable_shares(self, request, authorization, storage_index): """List mutable shares for a storage index.""" @@ -679,7 +674,7 @@ class HTTPServer(object): @implementer(IStreamServerEndpoint) -@attr.s +@define class _TLSEndpointWrapper(object): """ Wrap an existing endpoint with the server-side storage TLS policy. This is @@ -687,8 +682,8 @@ class _TLSEndpointWrapper(object): example there's Tor and i2p. """ - endpoint = attr.ib(type=IStreamServerEndpoint) - context_factory = attr.ib(type=CertificateOptions) + endpoint: IStreamServerEndpoint + context_factory: CertificateOptions @classmethod def from_paths( From 06eca79263382fab3b742d1d3243463735bc79f6 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 27 Jun 2022 14:03:05 -0400 Subject: [PATCH 0923/2309] Minimal streaming implementation. --- src/allmydata/storage/http_server.py | 55 ++++++++++++++++++------- src/allmydata/test/test_storage_http.py | 21 ++++++++-- 2 files changed, 59 insertions(+), 17 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index ebd2323ef..b8887bb4e 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -12,10 +12,15 @@ import binascii from zope.interface import implementer from klein import Klein from twisted.web import http -from twisted.internet.interfaces import IListeningPort, IStreamServerEndpoint +from twisted.web.server import NOT_DONE_YET +from twisted.internet.interfaces import ( + IListeningPort, + IStreamServerEndpoint, + IPullProducer, +) from twisted.internet.defer import Deferred from twisted.internet.ssl import CertificateOptions, Certificate, PrivateCertificate -from twisted.web.server import Site +from twisted.web.server import Site, Request from twisted.protocols.tls import TLSMemoryBIOFactory from twisted.python.filepath import FilePath @@ -274,7 +279,37 @@ _SCHEMAS = { } -def read_range(request, read_data: Callable[[int, int], bytes]) -> None: +@implementer(IPullProducer) +@define +class _ReadProducer: + """ + Producer that calls a read function, and writes to a request. + """ + + request: Request + read_data: Callable[[int, int], bytes] + result: Deferred + start: int = field(default=0) + + def resumeProducing(self): + data = self.read_data(self.start, self.start + 65536) + if not data: + self.request.unregisterProducer() + d = self.result + del self.result + d.callback(b"") + return + self.request.write(data) + self.start += len(data) + + def pauseProducing(self): + pass + + def stopProducing(self): + pass + + +def read_range(request: Request, read_data: Callable[[int, int], bytes]) -> None: """ Read an optional ``Range`` header, reads data appropriately via the given callable, writes the data to the request. @@ -290,17 +325,9 @@ def read_range(request, read_data: Callable[[int, int], bytes]) -> None: The resulting data is written to the request. """ if request.getHeader("range") is None: - # Return the whole thing. - start = 0 - while True: - # TODO should probably yield to event loop occasionally... - # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3872 - data = read_data(start, start + 65536) - if not data: - request.finish() - return - request.write(data) - start += len(data) + d = Deferred() + request.registerProducer(_ReadProducer(request, read_data, d), False) + return d range_header = parse_range_header(request.getHeader("range")) if ( diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 5e0b35d88..23d9bc276 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -6,6 +6,7 @@ from base64 import b64encode from contextlib import contextmanager from os import urandom from typing import Union, Callable, Tuple, Iterable +from time import sleep, time from cbor2 import dumps from pycddl import ValidationError as CDDLValidationError from hypothesis import assume, given, strategies as st @@ -14,7 +15,8 @@ from treq.testing import StubTreq from klein import Klein from hyperlink import DecodedURL from collections_extended import RangeMap -from twisted.internet.task import Clock +from twisted.internet.task import Clock, Cooperator +from twisted.internet import task from twisted.web import http from twisted.web.http_headers import Headers from werkzeug import routing @@ -316,10 +318,11 @@ class HttpTestFixture(Fixture): self.tempdir.path, b"\x00" * 20, clock=self.clock ) self.http_server = HTTPServer(self.storage_server, SWISSNUM_FOR_TEST) + self.treq = StubTreq(self.http_server.get_resource()) self.client = StorageClient( DecodedURL.from_text("http://127.0.0.1"), SWISSNUM_FOR_TEST, - treq=StubTreq(self.http_server.get_resource()), + treq=self.treq, ) @@ -1261,8 +1264,20 @@ class SharedImmutableMutableTestsMixin: """ A read with no range returns the whole mutable/immutable. """ + self.patch( + task, + "_theCooperator", + Cooperator(scheduler=lambda c: self.http.clock.callLater(0.000001, c)), + ) + + def result_of_with_flush(d): + for i in range(100): + self.http.clock.advance(0.001) + self.http.treq.flush() + return result_of(d) + storage_index, uploaded_data, _ = self.upload(1, data_length) - response = result_of( + response = result_of_with_flush( self.http.client.request( "GET", self.http.client.relative_url( From 6dd2b2d58357f30e7b663008e1f68ad798846f91 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 27 Jun 2022 14:44:51 -0400 Subject: [PATCH 0924/2309] More streaming, with tests passing again. --- src/allmydata/storage/http_server.py | 88 ++++++++++++++++++++----- src/allmydata/test/test_storage_http.py | 74 +++++++++++++-------- 2 files changed, 115 insertions(+), 47 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index b8887bb4e..a91b7963e 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -281,9 +281,10 @@ _SCHEMAS = { @implementer(IPullProducer) @define -class _ReadProducer: +class _ReadAllProducer: """ - Producer that calls a read function, and writes to a request. + Producer that calls a read function repeatedly to read all the data, and + writes to a request. """ request: Request @@ -292,7 +293,7 @@ class _ReadProducer: start: int = field(default=0) def resumeProducing(self): - data = self.read_data(self.start, self.start + 65536) + data = self.read_data(self.start, 65536) if not data: self.request.unregisterProducer() d = self.result @@ -309,6 +310,52 @@ class _ReadProducer: pass +@implementer(IPullProducer) +@define +class _ReadRangeProducer: + """ + Producer that calls a read function to read a range of data, and writes to + a request. + """ + + request: Request + read_data: Callable[[int, int], bytes] + result: Deferred + start: int + remaining: int + first_read: bool = field(default=True) + + def resumeProducing(self): + to_read = min(self.remaining, 65536) + data = self.read_data(self.start, to_read) + assert len(data) <= to_read + if self.first_read and data: + # For empty bodies the content-range header makes no sense since + # the end of the range is inclusive. + self.request.setHeader( + "content-range", + ContentRange("bytes", self.start, self.start + len(data)).to_header(), + ) + self.request.write(data) + + if not data or len(data) < to_read: + self.request.unregisterProducer() + d = self.result + del self.result + d.callback(b"") + return + + self.start += len(data) + self.remaining -= len(data) + assert self.remaining >= 0 + + def pauseProducing(self): + pass + + def stopProducing(self): + pass + + def read_range(request: Request, read_data: Callable[[int, int], bytes]) -> None: """ Read an optional ``Range`` header, reads data appropriately via the given @@ -324,9 +371,20 @@ def read_range(request: Request, read_data: Callable[[int, int], bytes]) -> None The resulting data is written to the request. """ + + def read_data_with_error_handling(offset: int, length: int) -> bytes: + try: + return read_data(offset, length) + except _HTTPError as e: + request.setResponseCode(e.code) + # Empty read means we're done. + return b"" + if request.getHeader("range") is None: d = Deferred() - request.registerProducer(_ReadProducer(request, read_data, d), False) + request.registerProducer( + _ReadAllProducer(request, read_data_with_error_handling, d), False + ) return d range_header = parse_range_header(request.getHeader("range")) @@ -339,21 +397,15 @@ def read_range(request: Request, read_data: Callable[[int, int], bytes]) -> None raise _HTTPError(http.REQUESTED_RANGE_NOT_SATISFIABLE) offset, end = range_header.ranges[0] - - # TODO limit memory usage - # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3872 - data = read_data(offset, end - offset) - request.setResponseCode(http.PARTIAL_CONTENT) - if len(data): - # For empty bodies the content-range header makes no sense since - # the end of the range is inclusive. - request.setHeader( - "content-range", - ContentRange("bytes", offset, offset + len(data)).to_header(), - ) - request.write(data) - request.finish() + d = Deferred() + request.registerProducer( + _ReadRangeProducer( + request, read_data_with_error_handling, d, offset, end - offset + ), + False, + ) + return d class HTTPServer(object): diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 23d9bc276..2382211df 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -10,7 +10,7 @@ from time import sleep, time from cbor2 import dumps from pycddl import ValidationError as CDDLValidationError from hypothesis import assume, given, strategies as st -from fixtures import Fixture, TempDir +from fixtures import Fixture, TempDir, MockPatch from treq.testing import StubTreq from klein import Klein from hyperlink import DecodedURL @@ -314,6 +314,12 @@ class HttpTestFixture(Fixture): def _setUp(self): self.clock = Clock() self.tempdir = self.useFixture(TempDir()) + self.mock = self.useFixture( + MockPatch( + "twisted.internet.task._theCooperator", + Cooperator(scheduler=lambda c: self.clock.callLater(0.000001, c)), + ) + ) self.storage_server = StorageServer( self.tempdir.path, b"\x00" * 20, clock=self.clock ) @@ -325,6 +331,12 @@ class HttpTestFixture(Fixture): treq=self.treq, ) + def result_of_with_flush(self, d): + for i in range(100): + self.clock.advance(0.001) + self.treq.flush() + return result_of(d) + class StorageClientWithHeadersOverride(object): """Wrap ``StorageClient`` and override sent headers.""" @@ -548,7 +560,7 @@ class ImmutableHTTPAPITests(SyncTestCase): # We can now read: for offset, length in [(0, 100), (10, 19), (99, 1), (49, 200)]: - downloaded = result_of( + downloaded = self.http.result_of_with_flush( self.imm_client.read_share_chunk(storage_index, 1, offset, length) ) self.assertEqual(downloaded, expected_data[offset : offset + length]) @@ -623,7 +635,7 @@ class ImmutableHTTPAPITests(SyncTestCase): # The upload of share 1 succeeded, demonstrating that second create() # call didn't overwrite work-in-progress. - downloaded = result_of( + downloaded = self.http.result_of_with_flush( self.imm_client.read_share_chunk(storage_index, 1, 0, 100) ) self.assertEqual(downloaded, b"a" * 50 + b"b" * 50) @@ -753,11 +765,15 @@ class ImmutableHTTPAPITests(SyncTestCase): ) ) self.assertEqual( - result_of(self.imm_client.read_share_chunk(storage_index, 1, 0, 10)), + self.http.result_of_with_flush( + self.imm_client.read_share_chunk(storage_index, 1, 0, 10) + ), b"1" * 10, ) self.assertEqual( - result_of(self.imm_client.read_share_chunk(storage_index, 2, 0, 10)), + self.http.result_of_with_flush( + self.imm_client.read_share_chunk(storage_index, 2, 0, 10) + ), b"2" * 10, ) @@ -921,7 +937,7 @@ class ImmutableHTTPAPITests(SyncTestCase): # Abort didn't prevent reading: self.assertEqual( uploaded_data, - result_of( + self.http.result_of_with_flush( self.imm_client.read_share_chunk( storage_index, 0, @@ -986,8 +1002,12 @@ class MutableHTTPAPIsTests(SyncTestCase): Written data can be read using ``read_share_chunk``. """ storage_index, _, _ = self.create_upload() - data0 = result_of(self.mut_client.read_share_chunk(storage_index, 0, 1, 7)) - data1 = result_of(self.mut_client.read_share_chunk(storage_index, 1, 0, 8)) + data0 = self.http.result_of_with_flush( + self.mut_client.read_share_chunk(storage_index, 0, 1, 7) + ) + data1 = self.http.result_of_with_flush( + self.mut_client.read_share_chunk(storage_index, 1, 0, 8) + ) self.assertEqual((data0, data1), (b"bcdef-0", b"abcdef-1")) def test_read_before_write(self): @@ -1015,8 +1035,12 @@ class MutableHTTPAPIsTests(SyncTestCase): ), ) # But the write did happen: - data0 = result_of(self.mut_client.read_share_chunk(storage_index, 0, 0, 8)) - data1 = result_of(self.mut_client.read_share_chunk(storage_index, 1, 0, 8)) + data0 = self.http.result_of_with_flush( + self.mut_client.read_share_chunk(storage_index, 0, 0, 8) + ) + data1 = self.http.result_of_with_flush( + self.mut_client.read_share_chunk(storage_index, 1, 0, 8) + ) self.assertEqual((data0, data1), (b"aXYZef-0", b"abcdef-1")) def test_conditional_write(self): @@ -1057,7 +1081,9 @@ class MutableHTTPAPIsTests(SyncTestCase): ) self.assertTrue(result.success) self.assertEqual( - result_of(self.mut_client.read_share_chunk(storage_index, 0, 0, 8)), + self.http.result_of_with_flush( + self.mut_client.read_share_chunk(storage_index, 0, 0, 8) + ), b"aXYZef-0", ) @@ -1094,7 +1120,9 @@ class MutableHTTPAPIsTests(SyncTestCase): # The write did not happen: self.assertEqual( - result_of(self.mut_client.read_share_chunk(storage_index, 0, 0, 8)), + self.http.result_of_with_flush( + self.mut_client.read_share_chunk(storage_index, 0, 0, 8) + ), b"abcdef-0", ) @@ -1194,7 +1222,7 @@ class SharedImmutableMutableTestsMixin: Reading from unknown storage index results in 404. """ with assert_fails_with_http_code(self, http.NOT_FOUND): - result_of( + self.http.result_of_with_flush( self.client.read_share_chunk( b"1" * 16, 1, @@ -1209,7 +1237,7 @@ class SharedImmutableMutableTestsMixin: """ storage_index, _, _ = self.upload(1) with assert_fails_with_http_code(self, http.NOT_FOUND): - result_of( + self.http.result_of_with_flush( self.client.read_share_chunk( storage_index, 7, # different share number @@ -1235,7 +1263,7 @@ class SharedImmutableMutableTestsMixin: with assert_fails_with_http_code( self, http.REQUESTED_RANGE_NOT_SATISFIABLE ): - result_of( + self.http.result_of_with_flush( client.read_share_chunk( storage_index, 1, @@ -1264,20 +1292,8 @@ class SharedImmutableMutableTestsMixin: """ A read with no range returns the whole mutable/immutable. """ - self.patch( - task, - "_theCooperator", - Cooperator(scheduler=lambda c: self.http.clock.callLater(0.000001, c)), - ) - - def result_of_with_flush(d): - for i in range(100): - self.http.clock.advance(0.001) - self.http.treq.flush() - return result_of(d) - storage_index, uploaded_data, _ = self.upload(1, data_length) - response = result_of_with_flush( + response = self.http.result_of_with_flush( self.http.client.request( "GET", self.http.client.relative_url( @@ -1298,7 +1314,7 @@ class SharedImmutableMutableTestsMixin: def check_range(requested_range, expected_response): headers = Headers() headers.setRawHeaders("range", [requested_range]) - response = result_of( + response = self.http.result_of_with_flush( self.http.client.request( "GET", self.http.client.relative_url( From 75f33022cd201f2477b86af9c22641f3c69a2188 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 27 Jun 2022 17:00:41 -0400 Subject: [PATCH 0925/2309] News file. --- newsfragments/3872.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3872.minor diff --git a/newsfragments/3872.minor b/newsfragments/3872.minor new file mode 100644 index 000000000..e69de29bb From efe9575d28dc18089525e9004159ddbe291997d0 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 29 Jun 2022 10:51:35 -0400 Subject: [PATCH 0926/2309] Nicer testing infrastructure so you don't have to switch back and forth between sync and async test APIs. --- src/allmydata/test/test_storage_http.py | 171 ++++++++++++++++-------- 1 file changed, 117 insertions(+), 54 deletions(-) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 2382211df..1f860cca0 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -1,5 +1,21 @@ """ Tests for HTTP storage client + server. + +The tests here are synchronous and don't involve running a real reactor. This +works, but has some caveats when it comes to testing HTTP endpoints: + +* Some HTTP endpoints are synchronous, some are not. +* For synchronous endpoints, the result is immediately available on the + ``Deferred`` coming out of ``StubTreq``. +* For asynchronous endpoints, you need to use ``StubTreq.flush()`` and + iterate the fake in-memory clock/reactor to advance time . + +So for HTTP endpoints, you should use ``HttpTestFixture.result_of_with_flush()`` +which handles both, and patches and moves forward the global Twisted +``Cooperator`` since that is used to drive pull producers. This is, +sadly, an internal implementation detail of Twisted being leaked to tests... + +For definitely synchronous calls, you can just use ``result_of()``. """ from base64 import b64encode @@ -332,10 +348,33 @@ class HttpTestFixture(Fixture): ) def result_of_with_flush(self, d): + """ + Like ``result_of``, but supports fake reactor and ``treq`` testing + infrastructure necessary to support asynchronous HTTP server endpoints. + """ + result = [] + error = [] + d.addCallbacks(result.append, error.append) + + # Check for synchronous HTTP endpoint handler: + if result: + return result[0] + if error: + error[0].raiseException() + + # OK, no result yet, probably async HTTP endpoint handler, so advance + # time, flush treq, and try again: for i in range(100): self.clock.advance(0.001) self.treq.flush() - return result_of(d) + if result: + return result[0] + if error: + error[0].raiseException() + raise RuntimeError( + "We expected given Deferred to have result already, but it wasn't. " + + "This is probably a test design issue." + ) class StorageClientWithHeadersOverride(object): @@ -393,7 +432,7 @@ class GenericHTTPAPITests(SyncTestCase): ) ) with assert_fails_with_http_code(self, http.UNAUTHORIZED): - result_of(client.get_version()) + self.http.result_of_with_flush(client.get_version()) def test_unsupported_mime_type(self): """ @@ -404,7 +443,7 @@ class GenericHTTPAPITests(SyncTestCase): StorageClientWithHeadersOverride(self.http.client, {"accept": "image/gif"}) ) with assert_fails_with_http_code(self, http.NOT_ACCEPTABLE): - result_of(client.get_version()) + self.http.result_of_with_flush(client.get_version()) def test_version(self): """ @@ -414,7 +453,7 @@ class GenericHTTPAPITests(SyncTestCase): might change across calls. """ client = StorageClientGeneral(self.http.client) - version = result_of(client.get_version()) + version = self.http.result_of_with_flush(client.get_version()) version[b"http://allmydata.org/tahoe/protocols/storage/v1"].pop( b"available-space" ) @@ -448,7 +487,7 @@ class GenericHTTPAPITests(SyncTestCase): ) message = {"bad-message": "missing expected keys"} - response = result_of( + response = self.http.result_of_with_flush( self.http.client.request( "POST", url, @@ -481,7 +520,7 @@ class ImmutableHTTPAPITests(SyncTestCase): upload_secret = urandom(32) lease_secret = urandom(32) storage_index = urandom(16) - created = result_of( + created = self.http.result_of_with_flush( self.imm_client.create( storage_index, share_numbers, @@ -525,35 +564,35 @@ class ImmutableHTTPAPITests(SyncTestCase): expected_data[offset : offset + length], ) - upload_progress = result_of(write(10, 10)) + upload_progress = self.http.result_of_with_flush(write(10, 10)) self.assertEqual( upload_progress, UploadProgress(finished=False, required=remaining) ) - upload_progress = result_of(write(30, 10)) + upload_progress = self.http.result_of_with_flush(write(30, 10)) self.assertEqual( upload_progress, UploadProgress(finished=False, required=remaining) ) - upload_progress = result_of(write(50, 10)) + upload_progress = self.http.result_of_with_flush(write(50, 10)) self.assertEqual( upload_progress, UploadProgress(finished=False, required=remaining) ) # Then, an overlapping write with matching data (15-35): - upload_progress = result_of(write(15, 20)) + upload_progress = self.http.result_of_with_flush(write(15, 20)) self.assertEqual( upload_progress, UploadProgress(finished=False, required=remaining) ) # Now fill in the holes: - upload_progress = result_of(write(0, 10)) + upload_progress = self.http.result_of_with_flush(write(0, 10)) self.assertEqual( upload_progress, UploadProgress(finished=False, required=remaining) ) - upload_progress = result_of(write(40, 10)) + upload_progress = self.http.result_of_with_flush(write(40, 10)) self.assertEqual( upload_progress, UploadProgress(finished=False, required=remaining) ) - upload_progress = result_of(write(60, 40)) + upload_progress = self.http.result_of_with_flush(write(60, 40)) self.assertEqual( upload_progress, UploadProgress(finished=True, required=RangeMap()) ) @@ -572,7 +611,7 @@ class ImmutableHTTPAPITests(SyncTestCase): """ (upload_secret, _, storage_index, _) = self.create_upload({1}, 100) with assert_fails_with_http_code(self, http.UNAUTHORIZED): - result_of( + self.http.result_of_with_flush( self.imm_client.write_share_chunk( storage_index, 1, @@ -594,7 +633,7 @@ class ImmutableHTTPAPITests(SyncTestCase): ) # Write half of share 1 - result_of( + self.http.result_of_with_flush( self.imm_client.write_share_chunk( storage_index, 1, @@ -608,7 +647,7 @@ class ImmutableHTTPAPITests(SyncTestCase): # existing shares, this call shouldn't overwrite the existing # work-in-progress. upload_secret2 = b"x" * 2 - created2 = result_of( + created2 = self.http.result_of_with_flush( self.imm_client.create( storage_index, {1, 4, 6}, @@ -622,7 +661,7 @@ class ImmutableHTTPAPITests(SyncTestCase): # Write second half of share 1 self.assertTrue( - result_of( + self.http.result_of_with_flush( self.imm_client.write_share_chunk( storage_index, 1, @@ -642,7 +681,7 @@ class ImmutableHTTPAPITests(SyncTestCase): # We can successfully upload the shares created with the second upload secret. self.assertTrue( - result_of( + self.http.result_of_with_flush( self.imm_client.write_share_chunk( storage_index, 4, @@ -660,11 +699,14 @@ class ImmutableHTTPAPITests(SyncTestCase): (upload_secret, _, storage_index, created) = self.create_upload({1, 2, 3}, 10) # Initially there are no shares: - self.assertEqual(result_of(self.imm_client.list_shares(storage_index)), set()) + self.assertEqual( + self.http.result_of_with_flush(self.imm_client.list_shares(storage_index)), + set(), + ) # Upload shares 1 and 3: for share_number in [1, 3]: - progress = result_of( + progress = self.http.result_of_with_flush( self.imm_client.write_share_chunk( storage_index, share_number, @@ -676,7 +718,10 @@ class ImmutableHTTPAPITests(SyncTestCase): self.assertTrue(progress.finished) # Now shares 1 and 3 exist: - self.assertEqual(result_of(self.imm_client.list_shares(storage_index)), {1, 3}) + self.assertEqual( + self.http.result_of_with_flush(self.imm_client.list_shares(storage_index)), + {1, 3}, + ) def test_upload_bad_content_range(self): """ @@ -694,7 +739,7 @@ class ImmutableHTTPAPITests(SyncTestCase): with assert_fails_with_http_code( self, http.REQUESTED_RANGE_NOT_SATISFIABLE ): - result_of( + self.http.result_of_with_flush( client.write_share_chunk( storage_index, 1, @@ -714,7 +759,10 @@ class ImmutableHTTPAPITests(SyncTestCase): Listing unknown storage index's shares results in empty list of shares. """ storage_index = bytes(range(16)) - self.assertEqual(result_of(self.imm_client.list_shares(storage_index)), set()) + self.assertEqual( + self.http.result_of_with_flush(self.imm_client.list_shares(storage_index)), + set(), + ) def test_upload_non_existent_storage_index(self): """ @@ -725,7 +773,7 @@ class ImmutableHTTPAPITests(SyncTestCase): def unknown_check(storage_index, share_number): with assert_fails_with_http_code(self, http.NOT_FOUND): - result_of( + self.http.result_of_with_flush( self.imm_client.write_share_chunk( storage_index, share_number, @@ -746,7 +794,7 @@ class ImmutableHTTPAPITests(SyncTestCase): stored separately and can be downloaded separately. """ (upload_secret, _, storage_index, _) = self.create_upload({1, 2}, 10) - result_of( + self.http.result_of_with_flush( self.imm_client.write_share_chunk( storage_index, 1, @@ -755,7 +803,7 @@ class ImmutableHTTPAPITests(SyncTestCase): b"1" * 10, ) ) - result_of( + self.http.result_of_with_flush( self.imm_client.write_share_chunk( storage_index, 2, @@ -785,7 +833,7 @@ class ImmutableHTTPAPITests(SyncTestCase): (upload_secret, _, storage_index, created) = self.create_upload({1}, 100) # Write: - result_of( + self.http.result_of_with_flush( self.imm_client.write_share_chunk( storage_index, 1, @@ -797,7 +845,7 @@ class ImmutableHTTPAPITests(SyncTestCase): # Conflicting write: with assert_fails_with_http_code(self, http.CONFLICT): - result_of( + self.http.result_of_with_flush( self.imm_client.write_share_chunk( storage_index, 1, @@ -823,7 +871,7 @@ class ImmutableHTTPAPITests(SyncTestCase): """ def abort(storage_index, share_number, upload_secret): - return result_of( + return self.http.result_of_with_flush( self.imm_client.abort_upload(storage_index, share_number, upload_secret) ) @@ -836,7 +884,7 @@ class ImmutableHTTPAPITests(SyncTestCase): """ # Start an upload: (upload_secret, _, storage_index, _) = self.create_upload({1}, 100) - result_of( + self.http.result_of_with_flush( self.imm_client.write_share_chunk( storage_index, 1, @@ -855,7 +903,7 @@ class ImmutableHTTPAPITests(SyncTestCase): # complaint: upload_secret = urandom(32) lease_secret = urandom(32) - created = result_of( + created = self.http.result_of_with_flush( self.imm_client.create( storage_index, {1}, @@ -868,7 +916,7 @@ class ImmutableHTTPAPITests(SyncTestCase): self.assertEqual(created.allocated, {1}) # And write to it, too: - result_of( + self.http.result_of_with_flush( self.imm_client.write_share_chunk( storage_index, 1, @@ -887,7 +935,9 @@ class ImmutableHTTPAPITests(SyncTestCase): for si, num in [(storage_index, 3), (b"x" * 16, 1)]: with assert_fails_with_http_code(self, http.NOT_FOUND): - result_of(self.imm_client.abort_upload(si, num, upload_secret)) + self.http.result_of_with_flush( + self.imm_client.abort_upload(si, num, upload_secret) + ) def test_unauthorized_abort(self): """ @@ -898,12 +948,12 @@ class ImmutableHTTPAPITests(SyncTestCase): # Failed to abort becaues wrong upload secret: with assert_fails_with_http_code(self, http.UNAUTHORIZED): - result_of( + self.http.result_of_with_flush( self.imm_client.abort_upload(storage_index, 1, upload_secret + b"X") ) # We can still write to it: - result_of( + self.http.result_of_with_flush( self.imm_client.write_share_chunk( storage_index, 1, @@ -920,7 +970,7 @@ class ImmutableHTTPAPITests(SyncTestCase): """ uploaded_data = b"123" (upload_secret, _, storage_index, _) = self.create_upload({0}, 3) - result_of( + self.http.result_of_with_flush( self.imm_client.write_share_chunk( storage_index, 0, @@ -932,7 +982,9 @@ class ImmutableHTTPAPITests(SyncTestCase): # Can't abort, we finished upload: with assert_fails_with_http_code(self, http.NOT_ALLOWED): - result_of(self.imm_client.abort_upload(storage_index, 0, upload_secret)) + self.http.result_of_with_flush( + self.imm_client.abort_upload(storage_index, 0, upload_secret) + ) # Abort didn't prevent reading: self.assertEqual( @@ -954,7 +1006,7 @@ class ImmutableHTTPAPITests(SyncTestCase): storage_index = urandom(16) secret = b"A" * 32 with assert_fails_with_http_code(self, http.NOT_FOUND): - result_of( + self.http.result_of_with_flush( self.general_client.add_or_renew_lease(storage_index, secret, secret) ) @@ -975,7 +1027,7 @@ class MutableHTTPAPIsTests(SyncTestCase): write_secret = urandom(32) lease_secret = urandom(32) storage_index = urandom(16) - result_of( + self.http.result_of_with_flush( self.mut_client.read_test_write_chunks( storage_index, write_secret, @@ -1013,7 +1065,7 @@ class MutableHTTPAPIsTests(SyncTestCase): def test_read_before_write(self): """In combo read/test/write operation, reads happen before writes.""" storage_index, write_secret, lease_secret = self.create_upload() - result = result_of( + result = self.http.result_of_with_flush( self.mut_client.read_test_write_chunks( storage_index, write_secret, @@ -1046,7 +1098,7 @@ class MutableHTTPAPIsTests(SyncTestCase): def test_conditional_write(self): """Uploads only happen if the test passes.""" storage_index, write_secret, lease_secret = self.create_upload() - result_failed = result_of( + result_failed = self.http.result_of_with_flush( self.mut_client.read_test_write_chunks( storage_index, write_secret, @@ -1064,7 +1116,7 @@ class MutableHTTPAPIsTests(SyncTestCase): self.assertFalse(result_failed.success) # This time the test matches: - result = result_of( + result = self.http.result_of_with_flush( self.mut_client.read_test_write_chunks( storage_index, write_secret, @@ -1090,19 +1142,22 @@ class MutableHTTPAPIsTests(SyncTestCase): def test_list_shares(self): """``list_shares()`` returns the shares for a given storage index.""" storage_index, _, _ = self.create_upload() - self.assertEqual(result_of(self.mut_client.list_shares(storage_index)), {0, 1}) + self.assertEqual( + self.http.result_of_with_flush(self.mut_client.list_shares(storage_index)), + {0, 1}, + ) def test_non_existent_list_shares(self): """A non-existent storage index errors when shares are listed.""" with self.assertRaises(ClientException) as exc: - result_of(self.mut_client.list_shares(urandom(32))) + self.http.result_of_with_flush(self.mut_client.list_shares(urandom(32))) self.assertEqual(exc.exception.code, http.NOT_FOUND) def test_wrong_write_enabler(self): """Writes with the wrong write enabler fail, and are not processed.""" storage_index, write_secret, lease_secret = self.create_upload() with self.assertRaises(ClientException) as exc: - result_of( + self.http.result_of_with_flush( self.mut_client.read_test_write_chunks( storage_index, urandom(32), @@ -1161,7 +1216,9 @@ class SharedImmutableMutableTestsMixin: storage_index, _, _ = self.upload(13) reason = "OHNO \u1235" - result_of(self.client.advise_corrupt_share(storage_index, 13, reason)) + self.http.result_of_with_flush( + self.client.advise_corrupt_share(storage_index, 13, reason) + ) self.assertEqual( corrupted, @@ -1174,11 +1231,15 @@ class SharedImmutableMutableTestsMixin: """ storage_index, _, _ = self.upload(13) reason = "OHNO \u1235" - result_of(self.client.advise_corrupt_share(storage_index, 13, reason)) + self.http.result_of_with_flush( + self.client.advise_corrupt_share(storage_index, 13, reason) + ) for (si, share_number) in [(storage_index, 11), (urandom(16), 13)]: with assert_fails_with_http_code(self, http.NOT_FOUND): - result_of(self.client.advise_corrupt_share(si, share_number, reason)) + self.http.result_of_with_flush( + self.client.advise_corrupt_share(si, share_number, reason) + ) def test_lease_renew_and_add(self): """ @@ -1196,7 +1257,7 @@ class SharedImmutableMutableTestsMixin: self.http.clock.advance(167) # We renew the lease: - result_of( + self.http.result_of_with_flush( self.general_client.add_or_renew_lease( storage_index, lease_secret, lease_secret ) @@ -1207,7 +1268,7 @@ class SharedImmutableMutableTestsMixin: # We create a new lease: lease_secret2 = urandom(32) - result_of( + self.http.result_of_with_flush( self.general_client.add_or_renew_lease( storage_index, lease_secret2, lease_secret2 ) @@ -1302,7 +1363,9 @@ class SharedImmutableMutableTestsMixin: ) ) self.assertEqual(response.code, http.OK) - self.assertEqual(result_of(response.content()), uploaded_data) + self.assertEqual( + self.http.result_of_with_flush(response.content()), uploaded_data + ) def test_validate_content_range_response_to_read(self): """ @@ -1354,7 +1417,7 @@ class ImmutableSharedTests(SharedImmutableMutableTestsMixin, SyncTestCase): upload_secret = urandom(32) lease_secret = urandom(32) storage_index = urandom(16) - result_of( + self.http.result_of_with_flush( self.client.create( storage_index, {share_number}, @@ -1364,7 +1427,7 @@ class ImmutableSharedTests(SharedImmutableMutableTestsMixin, SyncTestCase): lease_secret, ) ) - result_of( + self.http.result_of_with_flush( self.client.write_share_chunk( storage_index, share_number, @@ -1399,7 +1462,7 @@ class MutableSharedTests(SharedImmutableMutableTestsMixin, SyncTestCase): write_secret = urandom(32) lease_secret = urandom(32) storage_index = urandom(16) - result_of( + self.http.result_of_with_flush( self.client.read_test_write_chunks( storage_index, write_secret, From 520456bdc0411845715798ac72cd8a88686b798f Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 29 Jun 2022 11:26:25 -0400 Subject: [PATCH 0927/2309] Add streaming to CBOR results. --- src/allmydata/storage/http_server.py | 45 ++++++++++++++++++---------- 1 file changed, 30 insertions(+), 15 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index a91b7963e..f354fd837 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -3,16 +3,16 @@ HTTP server for storage. """ from __future__ import annotations -from typing import Dict, List, Set, Tuple, Any, Callable +from typing import Dict, List, Set, Tuple, Any, Callable from functools import wraps from base64 import b64decode import binascii +from tempfile import TemporaryFile from zope.interface import implementer from klein import Klein from twisted.web import http -from twisted.web.server import NOT_DONE_YET from twisted.internet.interfaces import ( IListeningPort, IStreamServerEndpoint, @@ -37,7 +37,7 @@ from cryptography.x509 import load_pem_x509_certificate # TODO Make sure to use pure Python versions? -from cbor2 import dumps, loads +from cbor2 import dump, loads from pycddl import Schema, ValidationError as CDDLValidationError from .server import StorageServer from .http_common import ( @@ -279,6 +279,10 @@ _SCHEMAS = { } +# Callabale that takes offset and length, returns the data at that range. +ReadData = Callable[[int, int], bytes] + + @implementer(IPullProducer) @define class _ReadAllProducer: @@ -288,10 +292,20 @@ class _ReadAllProducer: """ request: Request - read_data: Callable[[int, int], bytes] - result: Deferred + read_data: ReadData + result: Deferred = field(factory=Deferred) start: int = field(default=0) + @classmethod + def produce_to(cls, request: Request, read_data: ReadData) -> Deferred: + """ + Create and register the producer, returning ``Deferred`` that should be + returned from a HTTP server endpoint. + """ + producer = cls(request, read_data) + request.registerProducer(producer, False) + return producer.result + def resumeProducing(self): data = self.read_data(self.start, 65536) if not data: @@ -319,7 +333,7 @@ class _ReadRangeProducer: """ request: Request - read_data: Callable[[int, int], bytes] + read_data: ReadData result: Deferred start: int remaining: int @@ -356,7 +370,7 @@ class _ReadRangeProducer: pass -def read_range(request: Request, read_data: Callable[[int, int], bytes]) -> None: +def read_range(request: Request, read_data: ReadData) -> None: """ Read an optional ``Range`` header, reads data appropriately via the given callable, writes the data to the request. @@ -381,11 +395,7 @@ def read_range(request: Request, read_data: Callable[[int, int], bytes]) -> None return b"" if request.getHeader("range") is None: - d = Deferred() - request.registerProducer( - _ReadAllProducer(request, read_data_with_error_handling, d), False - ) - return d + return _ReadAllProducer.produce_to(request, read_data_with_error_handling) range_header = parse_range_header(request.getHeader("range")) if ( @@ -459,9 +469,14 @@ class HTTPServer(object): accept = parse_accept_header(accept_headers[0]) if accept.best == CBOR_MIME_TYPE: request.setHeader("Content-Type", CBOR_MIME_TYPE) - # TODO if data is big, maybe want to use a temporary file eventually... - # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3872 - return dumps(data) + f = TemporaryFile() + dump(data, f) + + def read_data(offset: int, length: int) -> bytes: + f.seek(offset) + return f.read(length) + + return _ReadAllProducer.produce_to(request, read_data) else: # TODO Might want to optionally send JSON someday: # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3861 From 0e8f2aa7024c75ba01943fb3f1fbce7160c8a799 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 29 Jun 2022 11:48:54 -0400 Subject: [PATCH 0928/2309] More memory usage reductions. --- src/allmydata/storage/http_server.py | 38 ++++++++++++++++--------- src/allmydata/test/test_storage_http.py | 9 ++++++ 2 files changed, 34 insertions(+), 13 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index f354fd837..98bd419c1 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -245,6 +245,8 @@ class _HTTPError(Exception): # Tags are of the form #6.nnn, where the number is documented at # https://www.iana.org/assignments/cbor-tags/cbor-tags.xhtml. Notably, #6.258 # indicates a set. +# +# TODO 3872 length limits in the schema. _SCHEMAS = { "allocate_buckets": Schema( """ @@ -485,12 +487,18 @@ class HTTPServer(object): def _read_encoded(self, request, schema: Schema) -> Any: """ Read encoded request body data, decoding it with CBOR by default. + + Somewhat arbitrarily, limit body size to 1MB; this may be too low, we + may want to customize per query type, but this is the starting point + for now. """ content_type = get_content_type(request.requestHeaders) if content_type == CBOR_MIME_TYPE: - # TODO limit memory usage, client could send arbitrarily large data... - # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3872 - message = request.content.read() + # Read 1 byte more than 1MB. We expect length to be 1MB or + # less; if it's more assume it's not a legitimate message. + message = request.content.read(1024 * 1024 + 1) + if len(message) > 1024 * 1024: + raise _HTTPError(http.REQUEST_ENTITY_TOO_LARGE) schema.validate_cbor(message) result = loads(message) return result @@ -586,20 +594,24 @@ class HTTPServer(object): request.setResponseCode(http.REQUESTED_RANGE_NOT_SATISFIABLE) return b"" - offset = content_range.start - - # TODO limit memory usage - # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3872 - data = request.content.read(content_range.stop - content_range.start + 1) bucket = self._uploads.get_write_bucket( storage_index, share_number, authorization[Secrets.UPLOAD] ) + offset = content_range.start + remaining = content_range.stop - content_range.start + finished = False - try: - finished = bucket.write(offset, data) - except ConflictingWriteError: - request.setResponseCode(http.CONFLICT) - return b"" + while remaining > 0: + data = request.content.read(min(remaining, 65536)) + assert data, "uploaded data length doesn't match range" + + try: + finished = bucket.write(offset, data) + except ConflictingWriteError: + request.setResponseCode(http.CONFLICT) + return b"" + remaining -= len(data) + offset += len(data) if finished: bucket.close() diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 1f860cca0..5418660c0 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -1139,6 +1139,15 @@ class MutableHTTPAPIsTests(SyncTestCase): b"aXYZef-0", ) + def test_too_large_write(self): + """ + Writing too large of a chunk results in a REQUEST ENTITY TOO LARGE http + error. + """ + with self.assertRaises(ClientException) as e: + self.create_upload(b"0123456789" * 1024 * 1024) + self.assertEqual(e.exception.code, http.REQUEST_ENTITY_TOO_LARGE) + def test_list_shares(self): """``list_shares()`` returns the shares for a given storage index.""" storage_index, _, _ = self.create_upload() From ab80c0f0a17affc87489cb29c031fb072803fb90 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 29 Jun 2022 14:04:42 -0400 Subject: [PATCH 0929/2309] Set some length limits on various queries lengths. --- src/allmydata/storage/http_server.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 98bd419c1..50e4ec946 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -246,12 +246,14 @@ class _HTTPError(Exception): # https://www.iana.org/assignments/cbor-tags/cbor-tags.xhtml. Notably, #6.258 # indicates a set. # -# TODO 3872 length limits in the schema. +# Somewhat arbitrary limits are set to reduce e.g. number of shares, number of +# vectors, etc.. These may need to be iterated on in future revisions of the +# code. _SCHEMAS = { "allocate_buckets": Schema( """ request = { - share-numbers: #6.258([* uint]) + share-numbers: #6.258([*30 uint]) allocated-size: uint } """ @@ -267,13 +269,15 @@ _SCHEMAS = { """ request = { "test-write-vectors": { - * share_number: { - "test": [* {"offset": uint, "size": uint, "specimen": bstr}] - "write": [* {"offset": uint, "data": bstr}] + ; TODO Add length limit here, after + ; https://github.com/anweiss/cddl/issues/128 is fixed + * share_number => { + "test": [*30 {"offset": uint, "size": uint, "specimen": bstr}] + "write": [*30 {"offset": uint, "data": bstr}] "new-length": uint / null } } - "read-vector": [* {"offset": uint, "size": uint}] + "read-vector": [*30 {"offset": uint, "size": uint}] } share_number = uint """ From bee46fae93494206c843633592ac04cbd65849b5 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 30 Jun 2022 13:48:33 -0400 Subject: [PATCH 0930/2309] Resource limits on the client side. --- src/allmydata/storage/http_client.py | 34 ++++++++++++++++++--- src/allmydata/test/test_storage_http.py | 40 +++++++++++++++++++++++++ 2 files changed, 70 insertions(+), 4 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 9203d02ab..b8bd0bf20 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -6,6 +6,7 @@ from __future__ import annotations from typing import Union, Optional, Sequence, Mapping from base64 import b64encode +from io import BytesIO from attrs import define, asdict, frozen, field @@ -114,6 +115,33 @@ _SCHEMAS = { } +@define +class _LengthLimitedCollector: + """ + Collect data using ``treq.collect()``, with limited length. + """ + + remaining_length: int + f: BytesIO = field(factory=BytesIO) + + def __call__(self, data: bytes): + if len(data) > self.remaining_length: + raise ValueError("Response length was too long") + self.f.write(data) + + +def limited_content(response, max_length: int = 30 * 1024 * 1024) -> Deferred: + """ + Like ``treq.content()``, but limit data read from the response to a set + length. If the response is longer than the max allowed length, the result + fails with a ``ValueError``. + """ + collector = _LengthLimitedCollector(max_length) + d = treq.collect(response, collector) + d.addCallback(lambda _: collector.f.getvalue()) + return d + + def _decode_cbor(response, schema: Schema): """Given HTTP response, return decoded CBOR body.""" @@ -124,9 +152,7 @@ def _decode_cbor(response, schema: Schema): if response.code > 199 and response.code < 300: content_type = get_content_type(response.headers) if content_type == CBOR_MIME_TYPE: - # TODO limit memory usage - # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3872 - return treq.content(response).addCallback(got_content) + return limited_content(response).addCallback(got_content) else: raise ClientException(-1, "Server didn't send CBOR") else: @@ -295,7 +321,7 @@ class StorageClient(object): write_enabler_secret=None, headers=None, message_to_serialize=None, - **kwargs + **kwargs, ): """ Like ``treq.request()``, but with optional secrets that get translated diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 5418660c0..915cd33f2 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -65,6 +65,7 @@ from ..storage.http_client import ( ReadVector, ReadTestWriteResult, TestVector, + limited_content, ) @@ -255,6 +256,11 @@ class TestApp(object): request.setHeader("content-type", CBOR_MIME_TYPE) return dumps({"garbage": 123}) + @_authorized_route(_app, set(), "/millionbytes", methods=["GET"]) + def million_bytes(self, request, authorization): + """Return 1,000,000 bytes.""" + return b"0123456789" * 100_000 + def result_of(d): """ @@ -320,6 +326,40 @@ class CustomHTTPServerTests(SyncTestCase): with self.assertRaises(CDDLValidationError): result_of(client.get_version()) + def test_limited_content_fits(self): + """ + ``http_client.limited_content()`` returns the body if it is less than + the max length. + """ + for at_least_length in (1_000_000, 1_000_001): + response = result_of( + self.client.request( + "GET", + "http://127.0.0.1/millionbytes", + ) + ) + + self.assertEqual( + result_of(limited_content(response, at_least_length)), + b"0123456789" * 100_000, + ) + + def test_limited_content_does_not_fit(self): + """ + If the body is longer than than max length, + ``http_client.limited_content()`` fails with a ``ValueError``. + """ + for too_short in (999_999, 10): + response = result_of( + self.client.request( + "GET", + "http://127.0.0.1/millionbytes", + ) + ) + + with self.assertRaises(ValueError): + result_of(limited_content(response, too_short)) + class HttpTestFixture(Fixture): """ From 451e68795cf5cfb02fabdf6baa870289b978a8f7 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 30 Jun 2022 13:54:58 -0400 Subject: [PATCH 0931/2309] Lints, better explanation. --- src/allmydata/test/test_storage_http.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 915cd33f2..3108ffae8 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -22,7 +22,6 @@ from base64 import b64encode from contextlib import contextmanager from os import urandom from typing import Union, Callable, Tuple, Iterable -from time import sleep, time from cbor2 import dumps from pycddl import ValidationError as CDDLValidationError from hypothesis import assume, given, strategies as st @@ -32,7 +31,6 @@ from klein import Klein from hyperlink import DecodedURL from collections_extended import RangeMap from twisted.internet.task import Clock, Cooperator -from twisted.internet import task from twisted.web import http from twisted.web.http_headers import Headers from werkzeug import routing @@ -370,6 +368,10 @@ class HttpTestFixture(Fixture): def _setUp(self): self.clock = Clock() self.tempdir = self.useFixture(TempDir()) + # The global Cooperator used by Twisted (a) used by pull producers in + # twisted.web, (b) is driven by a real reactor. We want to push time + # forward ourselves since we rely on pull producers in the HTTP storage + # server. self.mock = self.useFixture( MockPatch( "twisted.internet.task._theCooperator", From 03c515191edc519ad045191f18c5d558d4a19e35 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 30 Jun 2022 14:21:21 -0400 Subject: [PATCH 0932/2309] Better docs. --- src/allmydata/protocol_switch.py | 40 +++++++++++++++++++++++++------- 1 file changed, 32 insertions(+), 8 deletions(-) diff --git a/src/allmydata/protocol_switch.py b/src/allmydata/protocol_switch.py index 9b4e30671..20984c615 100644 --- a/src/allmydata/protocol_switch.py +++ b/src/allmydata/protocol_switch.py @@ -1,8 +1,15 @@ """ -Support for listening with both HTTP and Foolscap on the same port. -""" +Support for listening with both HTTPS and Foolscap on the same port. -from typing import Optional +The goal is to make the transition from Foolscap to HTTPS-based protocols as +simple as possible, with no extra configuration needed. Listening on the same +port means a user upgrading Tahoe-LAFS will automatically get HTTPS working +with no additional changes. + +Use ``create_foolscap_or_http_class()`` to create a new subclass per ``Tub``, +and then ``update_foolscap_or_http_class()`` to add the relevant information to +the subclass once it becomes available later in the configuration process. +""" from twisted.internet.protocol import Protocol from twisted.internet.interfaces import IDelayedCall @@ -17,18 +24,26 @@ from .storage.http_server import HTTPServer from .storage.server import StorageServer -class PretendToBeNegotiation(type): - """😱""" +class _PretendToBeNegotiation(type): + """ + Metaclass that allows ``_FoolscapOrHttps`` to pretend to be a ``Negotiation`` + instance, since Foolscap has some ``assert isinstance(protocol, + Negotiation`` checks. + """ def __instancecheck__(self, instance): return (instance.__class__ == self) or isinstance(instance, Negotiation) -class FoolscapOrHttp(Protocol, metaclass=PretendToBeNegotiation): +class _FoolscapOrHttps(Protocol, metaclass=_PretendToBeNegotiation): """ Based on initial query, decide whether we're talking Foolscap or HTTP. - Pretends to be a ``foolscap.negotiate.Negotiation`` instance. + Additionally, pretends to be a ``foolscap.negotiate.Negotiation`` instance, + since these are created by Foolscap's ``Tub``, by setting this to be the + tub's ``negotiationClass``. + + Do not use directly; this needs to be subclassed per ``Tub``. """ # These three will be set by a subclass in update_foolscap_or_http_class() @@ -110,13 +125,22 @@ class FoolscapOrHttp(Protocol, metaclass=PretendToBeNegotiation): def create_foolscap_or_http_class(): - class FoolscapOrHttpWithCert(FoolscapOrHttp): + """ + Create a new Foolscap-or-HTTPS protocol class for a specific ``Tub`` + instance. + """ + + class FoolscapOrHttpWithCert(_FoolscapOrHttps): pass return FoolscapOrHttpWithCert def update_foolscap_or_http_class(cls, certificate, storage_server, swissnum): + """ + Add the various parameters needed by a ``Tub``-specific + ``_FoolscapOrHttps`` subclass. + """ cls.certificate = certificate cls.storage_server = storage_server cls.swissnum = swissnum From d1bdce9682f9c6c8eefd1488db7c8c8bfc7cdf6b Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 30 Jun 2022 14:26:36 -0400 Subject: [PATCH 0933/2309] A nicer API. --- src/allmydata/client.py | 5 +++-- src/allmydata/node.py | 4 ++-- src/allmydata/protocol_switch.py | 32 +++++++++++++++++--------------- 3 files changed, 22 insertions(+), 19 deletions(-) diff --git a/src/allmydata/client.py b/src/allmydata/client.py index 294684b58..2f68c1cb4 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -64,7 +64,6 @@ from allmydata.interfaces import ( from allmydata.nodemaker import NodeMaker from allmydata.blacklist import Blacklist from allmydata import node -from .protocol_switch import update_foolscap_or_http_class KiB=1024 @@ -820,7 +819,9 @@ class _Client(node.Node, pollmixin.PollMixin): furl_file = self.config.get_private_path("storage.furl").encode(get_filesystem_encoding()) furl = self.tub.registerReference(FoolscapStorageServer(ss), furlFile=furl_file) (_, _, swissnum) = furl.rpartition("/") - update_foolscap_or_http_class(self.tub.negotiationClass, self.tub.myCertificate, ss, swissnum.encode("ascii")) + self.tub.negotiationClass.add_storage_server( + self.tub.myCertificate, ss, swissnum.encode("ascii") + ) announcement["anonymous-storage-FURL"] = furl diff --git a/src/allmydata/node.py b/src/allmydata/node.py index 93fa6a8e1..597221e9b 100644 --- a/src/allmydata/node.py +++ b/src/allmydata/node.py @@ -55,7 +55,7 @@ from allmydata.util.yamlutil import ( from . import ( __full_version__, ) -from .protocol_switch import create_foolscap_or_http_class +from .protocol_switch import support_foolscap_and_https def _common_valid_config(): @@ -709,7 +709,7 @@ def create_tub(tub_options, default_connection_handlers, foolscap_connection_han the new Tub via `Tub.setOption` """ tub = Tub(**kwargs) - tub.negotiationClass = create_foolscap_or_http_class() + support_foolscap_and_https(tub) for (name, value) in list(tub_options.items()): tub.setOption(name, value) handlers = default_connection_handlers.copy() diff --git a/src/allmydata/protocol_switch.py b/src/allmydata/protocol_switch.py index 20984c615..7623d68e5 100644 --- a/src/allmydata/protocol_switch.py +++ b/src/allmydata/protocol_switch.py @@ -6,9 +6,10 @@ simple as possible, with no extra configuration needed. Listening on the same port means a user upgrading Tahoe-LAFS will automatically get HTTPS working with no additional changes. -Use ``create_foolscap_or_http_class()`` to create a new subclass per ``Tub``, -and then ``update_foolscap_or_http_class()`` to add the relevant information to -the subclass once it becomes available later in the configuration process. +Use ``support_foolscap_and_https()`` to create a new subclass for a ``Tub`` +instance, and then ``add_storage_server()`` on the resulting class to add the +relevant information for a storage server once it becomes available later in +the configuration process. """ from twisted.internet.protocol import Protocol @@ -19,6 +20,7 @@ from twisted.protocols.tls import TLSMemoryBIOFactory from twisted.internet import reactor from foolscap.negotiate import Negotiation +from foolscap.api import Tub from .storage.http_server import HTTPServer from .storage.server import StorageServer @@ -54,6 +56,16 @@ class _FoolscapOrHttps(Protocol, metaclass=_PretendToBeNegotiation): _timeout: IDelayedCall + @classmethod + def add_storage_server(cls, certificate, storage_server, swissnum): + """ + Add the various parameters needed by a ``Tub``-specific + ``_FoolscapOrHttps`` subclass. + """ + cls.certificate = certificate + cls.storage_server = storage_server + cls.swissnum = swissnum + def __init__(self, *args, **kwargs): self._foolscap: Negotiation = Negotiation(*args, **kwargs) self._buffer: bytes = b"" @@ -124,7 +136,7 @@ class _FoolscapOrHttps(Protocol, metaclass=_PretendToBeNegotiation): self.__dict__ = protocol.__dict__ -def create_foolscap_or_http_class(): +def support_foolscap_and_https(tub: Tub): """ Create a new Foolscap-or-HTTPS protocol class for a specific ``Tub`` instance. @@ -133,14 +145,4 @@ def create_foolscap_or_http_class(): class FoolscapOrHttpWithCert(_FoolscapOrHttps): pass - return FoolscapOrHttpWithCert - - -def update_foolscap_or_http_class(cls, certificate, storage_server, swissnum): - """ - Add the various parameters needed by a ``Tub``-specific - ``_FoolscapOrHttps`` subclass. - """ - cls.certificate = certificate - cls.storage_server = storage_server - cls.swissnum = swissnum + tub.negotiationClass = FoolscapOrHttpWithCert # type: ignore From 03d9ff395cce0aeff1b1e08b80c8c073907cd3ad Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 30 Jun 2022 14:30:19 -0400 Subject: [PATCH 0934/2309] News file. --- newsfragments/3902.feature | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/3902.feature diff --git a/newsfragments/3902.feature b/newsfragments/3902.feature new file mode 100644 index 000000000..2477d0ae6 --- /dev/null +++ b/newsfragments/3902.feature @@ -0,0 +1 @@ +The new HTTPS-based storage server is now enabled transparently on the same port as the Foolscap server. This will not have any user-facing impact until the HTTPS storage protocol is supported in clients as well. \ No newline at end of file From 1798966f03c652396d434e14fb41e76607015cc4 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 30 Jun 2022 14:52:12 -0400 Subject: [PATCH 0935/2309] Store the tub on the subclass, since we'll want it (or rather its Listeners) for NURL construction. --- src/allmydata/client.py | 4 +--- src/allmydata/protocol_switch.py | 22 ++++++++++++++-------- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/src/allmydata/client.py b/src/allmydata/client.py index 2f68c1cb4..e737f93e6 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -819,9 +819,7 @@ class _Client(node.Node, pollmixin.PollMixin): furl_file = self.config.get_private_path("storage.furl").encode(get_filesystem_encoding()) furl = self.tub.registerReference(FoolscapStorageServer(ss), furlFile=furl_file) (_, _, swissnum) = furl.rpartition("/") - self.tub.negotiationClass.add_storage_server( - self.tub.myCertificate, ss, swissnum.encode("ascii") - ) + self.tub.negotiationClass.add_storage_server(ss, swissnum.encode("ascii")) announcement["anonymous-storage-FURL"] = furl diff --git a/src/allmydata/protocol_switch.py b/src/allmydata/protocol_switch.py index 7623d68e5..059339575 100644 --- a/src/allmydata/protocol_switch.py +++ b/src/allmydata/protocol_switch.py @@ -48,21 +48,25 @@ class _FoolscapOrHttps(Protocol, metaclass=_PretendToBeNegotiation): Do not use directly; this needs to be subclassed per ``Tub``. """ - # These three will be set by a subclass in update_foolscap_or_http_class() - # below. + # These will be set by support_foolscap_and_https() and add_storage_server(). + + # The swissnum for the storage_server. swissnum: bytes - certificate: PrivateCertificate + # The storage server we're exposing. storage_server: StorageServer + # The tub that created us: + tub: Tub + # The certificate for the endpoint: + certificate: PrivateCertificate _timeout: IDelayedCall @classmethod - def add_storage_server(cls, certificate, storage_server, swissnum): + def add_storage_server(cls, storage_server, swissnum): """ - Add the various parameters needed by a ``Tub``-specific - ``_FoolscapOrHttps`` subclass. + Add the various storage server-related attributes needed by a + ``Tub``-specific ``_FoolscapOrHttps`` subclass. """ - cls.certificate = certificate cls.storage_server = storage_server cls.swissnum = swissnum @@ -141,8 +145,10 @@ def support_foolscap_and_https(tub: Tub): Create a new Foolscap-or-HTTPS protocol class for a specific ``Tub`` instance. """ + the_tub = tub class FoolscapOrHttpWithCert(_FoolscapOrHttps): - pass + tub = the_tub + certificate = tub.myCertificate tub.negotiationClass = FoolscapOrHttpWithCert # type: ignore From 3db6080f6d62907e9eeaa8de5b6cd1a480b96e23 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 30 Jun 2022 15:18:22 -0400 Subject: [PATCH 0936/2309] Make the factories a class-level attribute. --- src/allmydata/protocol_switch.py | 45 ++++++++++++++++---------------- 1 file changed, 23 insertions(+), 22 deletions(-) diff --git a/src/allmydata/protocol_switch.py b/src/allmydata/protocol_switch.py index 059339575..21d896793 100644 --- a/src/allmydata/protocol_switch.py +++ b/src/allmydata/protocol_switch.py @@ -14,7 +14,7 @@ the configuration process. from twisted.internet.protocol import Protocol from twisted.internet.interfaces import IDelayedCall -from twisted.internet.ssl import CertificateOptions, PrivateCertificate +from twisted.internet.ssl import CertificateOptions from twisted.web.server import Site from twisted.protocols.tls import TLSMemoryBIOFactory from twisted.internet import reactor @@ -50,25 +50,35 @@ class _FoolscapOrHttps(Protocol, metaclass=_PretendToBeNegotiation): # These will be set by support_foolscap_and_https() and add_storage_server(). - # The swissnum for the storage_server. - swissnum: bytes - # The storage server we're exposing. - storage_server: StorageServer + # The HTTP storage server API we're exposing. + http_storage_server: HTTPServer + # The Twisted HTTPS protocol factory wrapping the storage server API: + https_factory: TLSMemoryBIOFactory # The tub that created us: tub: Tub - # The certificate for the endpoint: - certificate: PrivateCertificate + # This will be created by the instance in connectionMade(): _timeout: IDelayedCall @classmethod - def add_storage_server(cls, storage_server, swissnum): + def add_storage_server(cls, storage_server: StorageServer, swissnum): """ Add the various storage server-related attributes needed by a ``Tub``-specific ``_FoolscapOrHttps`` subclass. """ - cls.storage_server = storage_server - cls.swissnum = swissnum + # Tub.myCertificate is a twisted.internet.ssl.PrivateCertificate + # instance. + certificate_options = CertificateOptions( + privateKey=cls.tub.myCertificate.privateKey.original, + certificate=cls.tub.myCertificate.original, + ) + + cls.http_storage_server = HTTPServer(storage_server, swissnum) + cls.https_factory = TLSMemoryBIOFactory( + certificate_options, + False, + Site(cls.http_storage_server.get_resource()), + ) def __init__(self, *args, **kwargs): self._foolscap: Negotiation = Negotiation(*args, **kwargs) @@ -124,16 +134,8 @@ class _FoolscapOrHttps(Protocol, metaclass=_PretendToBeNegotiation): return else: # We're a HTTPS protocol instance, serving the storage protocol: - certificate_options = CertificateOptions( - privateKey=self.certificate.privateKey.original, - certificate=self.certificate.original, - ) - http_server = HTTPServer(self.storage_server, self.swissnum) - factory = TLSMemoryBIOFactory( - certificate_options, False, Site(http_server.get_resource()) - ) assert self.transport is not None - protocol = factory.buildProtocol(self.transport.getPeer()) + protocol = self.https_factory.buildProtocol(self.transport.getPeer()) protocol.makeConnection(self.transport) protocol.dataReceived(self._buffer) self.__class__ = protocol.__class__ @@ -147,8 +149,7 @@ def support_foolscap_and_https(tub: Tub): """ the_tub = tub - class FoolscapOrHttpWithCert(_FoolscapOrHttps): + class FoolscapOrHttpForTub(_FoolscapOrHttps): tub = the_tub - certificate = tub.myCertificate - tub.negotiationClass = FoolscapOrHttpWithCert # type: ignore + tub.negotiationClass = FoolscapOrHttpForTub # type: ignore From 70dfc4484173bb9592d02834e14bb85d8356a14c Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 30 Jun 2022 15:45:30 -0400 Subject: [PATCH 0937/2309] Fix for 3905. --- src/allmydata/storage/http_server.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 06a6863fa..f61030844 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -188,7 +188,12 @@ class UploadsInProgress(object): def remove_write_bucket(self, bucket: BucketWriter): """Stop tracking the given ``BucketWriter``.""" - storage_index, share_number = self._bucketwriters.pop(bucket) + try: + storage_index, share_number = self._bucketwriters.pop(bucket) + except KeyError: + # This is probably a BucketWriter created by Foolscap, so just + # ignore it. + return uploads_index = self._uploads[storage_index] uploads_index.shares.pop(share_number) uploads_index.upload_secrets.pop(share_number) From f2acf71998475bb8b5eef981f3c3b93e23432561 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 30 Jun 2022 15:58:52 -0400 Subject: [PATCH 0938/2309] Document next steps: NURL generation. --- src/allmydata/protocol_switch.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/allmydata/protocol_switch.py b/src/allmydata/protocol_switch.py index 21d896793..9f33560e7 100644 --- a/src/allmydata/protocol_switch.py +++ b/src/allmydata/protocol_switch.py @@ -66,6 +66,14 @@ class _FoolscapOrHttps(Protocol, metaclass=_PretendToBeNegotiation): Add the various storage server-related attributes needed by a ``Tub``-specific ``_FoolscapOrHttps`` subclass. """ + # TODO tub.locationHints will be in the format ["tcp:hostname:port"] + # (and maybe some other things we can ignore for now). We also have + # access to the certificate. Together, this should be sufficient to + # construct NURLs, one per hint. The code for NURls should be + # refactored out of http_server.py's build_nurl; that code might want + # to skip around for the future when we don't do foolscap, but for now + # this module will be main way we set up HTTPS. + # Tub.myCertificate is a twisted.internet.ssl.PrivateCertificate # instance. certificate_options = CertificateOptions( From 249f43184972d124d5144dccf52f3cb78662a523 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 5 Jul 2022 11:14:52 -0400 Subject: [PATCH 0939/2309] Use MonkeyPatch instead of MockPatch, since we're not mocking. --- src/allmydata/test/test_storage_http.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 3108ffae8..811cc2ac1 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -25,7 +25,7 @@ from typing import Union, Callable, Tuple, Iterable from cbor2 import dumps from pycddl import ValidationError as CDDLValidationError from hypothesis import assume, given, strategies as st -from fixtures import Fixture, TempDir, MockPatch +from fixtures import Fixture, TempDir, MonkeyPatch from treq.testing import StubTreq from klein import Klein from hyperlink import DecodedURL @@ -373,7 +373,7 @@ class HttpTestFixture(Fixture): # forward ourselves since we rely on pull producers in the HTTP storage # server. self.mock = self.useFixture( - MockPatch( + MonkeyPatch( "twisted.internet.task._theCooperator", Cooperator(scheduler=lambda c: self.clock.callLater(0.000001, c)), ) From 97d0ba23ebc48c3b5af378446b5adc3189b608a9 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 5 Jul 2022 11:21:46 -0400 Subject: [PATCH 0940/2309] Switch to hypothesis-based test. --- src/allmydata/test/test_storage_http.py | 31 ++++++++++++++++--------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 811cc2ac1..5c429af88 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -235,6 +235,13 @@ class RouteConverterTests(SyncTestCase): SWISSNUM_FOR_TEST = b"abcd" +def gen_bytes(length: int) -> bytes: + """Generate bytes to the given length.""" + result = (b"0123456789abcdef" * ((length // 16) + 1))[:length] + assert len(result) == length + return result + + class TestApp(object): """HTTP API for testing purposes.""" @@ -254,10 +261,10 @@ class TestApp(object): request.setHeader("content-type", CBOR_MIME_TYPE) return dumps({"garbage": 123}) - @_authorized_route(_app, set(), "/millionbytes", methods=["GET"]) - def million_bytes(self, request, authorization): - """Return 1,000,000 bytes.""" - return b"0123456789" * 100_000 + @_authorized_route(_app, set(), "/bytes/", methods=["GET"]) + def generate_bytes(self, request, authorization, length): + """Return bytes to the given length using ``gen_bytes()``.""" + return gen_bytes(length) def result_of(d): @@ -324,34 +331,36 @@ class CustomHTTPServerTests(SyncTestCase): with self.assertRaises(CDDLValidationError): result_of(client.get_version()) - def test_limited_content_fits(self): + @given(length=st.integers(min_value=1, max_value=1_000_000)) + def test_limited_content_fits(self, length): """ ``http_client.limited_content()`` returns the body if it is less than the max length. """ - for at_least_length in (1_000_000, 1_000_001): + for at_least_length in (length, length + 1, length + 1000): response = result_of( self.client.request( "GET", - "http://127.0.0.1/millionbytes", + f"http://127.0.0.1/bytes/{length}", ) ) self.assertEqual( result_of(limited_content(response, at_least_length)), - b"0123456789" * 100_000, + gen_bytes(length), ) - def test_limited_content_does_not_fit(self): + @given(length=st.integers(min_value=10, max_value=1_000_000)) + def test_limited_content_does_not_fit(self, length): """ If the body is longer than than max length, ``http_client.limited_content()`` fails with a ``ValueError``. """ - for too_short in (999_999, 10): + for too_short in (length - 1, 5): response = result_of( self.client.request( "GET", - "http://127.0.0.1/millionbytes", + f"http://127.0.0.1/bytes/{length}", ) ) From 1e6864ac0116b834be34ae556b80a7ca52f07e28 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 5 Jul 2022 11:30:01 -0400 Subject: [PATCH 0941/2309] Typo. --- src/allmydata/storage/http_server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 50e4ec946..ffba354bb 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -285,7 +285,7 @@ _SCHEMAS = { } -# Callabale that takes offset and length, returns the data at that range. +# Callable that takes offset and length, returns the data at that range. ReadData = Callable[[int, int], bytes] From 3270d24c45d1613b5418f6f189517b859b4afdaa Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 5 Jul 2022 11:30:48 -0400 Subject: [PATCH 0942/2309] Slight simplification. --- src/allmydata/storage/http_server.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index ffba354bb..c727b5e95 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -24,7 +24,7 @@ from twisted.web.server import Site, Request from twisted.protocols.tls import TLSMemoryBIOFactory from twisted.python.filepath import FilePath -from attrs import define, field +from attrs import define, field, Factory from werkzeug.http import ( parse_range_header, parse_content_range_header, @@ -149,11 +149,11 @@ class StorageIndexUploads(object): """ # Map share number to BucketWriter - shares: dict[int, BucketWriter] = field(factory=dict) + shares: dict[int, BucketWriter] = Factory(dict) # Map share number to the upload secret (different shares might have # different upload secrets). - upload_secrets: dict[int, bytes] = field(factory=dict) + upload_secrets: dict[int, bytes] = Factory(dict) @define @@ -163,10 +163,10 @@ class UploadsInProgress(object): """ # Map storage index to corresponding uploads-in-progress - _uploads: dict[bytes, StorageIndexUploads] = field(factory=dict) + _uploads: dict[bytes, StorageIndexUploads] = Factory(dict) # Map BucketWriter to (storage index, share number) - _bucketwriters: dict[BucketWriter, Tuple[bytes, int]] = field(factory=dict) + _bucketwriters: dict[BucketWriter, Tuple[bytes, int]] = Factory(dict) def add_write_bucket( self, @@ -299,7 +299,7 @@ class _ReadAllProducer: request: Request read_data: ReadData - result: Deferred = field(factory=Deferred) + result: Deferred = Factory(Deferred) start: int = field(default=0) @classmethod From 6e3ca256b9eaf4240a782abfcde887d360b10f10 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 5 Jul 2022 15:36:21 -0400 Subject: [PATCH 0943/2309] Some refactoring to handle edge cases better, in progress. --- src/allmydata/storage/http_server.py | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index c727b5e95..d55d12711 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -349,26 +349,35 @@ class _ReadRangeProducer: to_read = min(self.remaining, 65536) data = self.read_data(self.start, to_read) assert len(data) <= to_read - if self.first_read and data: + + if self.first_read and self.remaining > 0: # For empty bodies the content-range header makes no sense since # the end of the range is inclusive. self.request.setHeader( "content-range", - ContentRange("bytes", self.start, self.start + len(data)).to_header(), + ContentRange( + "bytes", self.start, self.start + self.remaining + ).to_header(), ) + self.first_read = False + + if not data and self.remaining > 0: + # Either data is missing locally (storage issue?) or a bug + pass # TODO abort. TODO test + + self.start += len(data) + self.remaining -= len(data) + assert self.remaining >= 0 + self.request.write(data) - if not data or len(data) < to_read: + if self.remaining == 0: self.request.unregisterProducer() d = self.result del self.result d.callback(b"") return - self.start += len(data) - self.remaining -= len(data) - assert self.remaining >= 0 - def pauseProducing(self): pass @@ -412,6 +421,8 @@ def read_range(request: Request, read_data: ReadData) -> None: ): raise _HTTPError(http.REQUESTED_RANGE_NOT_SATISFIABLE) + # TODO if end is beyond the end of the share, either return error, or maybe + # just return what we can... offset, end = range_header.ranges[0] request.setResponseCode(http.PARTIAL_CONTENT) d = Deferred() From 69c4dbf2b5e04cb3dd9e79ea9b98686178d777c4 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 5 Jul 2022 17:17:38 -0400 Subject: [PATCH 0944/2309] Fix tests and point to future work. --- src/allmydata/storage/http_server.py | 15 ++++++++++++--- src/allmydata/test/test_storage_http.py | 4 +++- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index d55d12711..9d90ba960 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -353,6 +353,11 @@ class _ReadRangeProducer: if self.first_read and self.remaining > 0: # For empty bodies the content-range header makes no sense since # the end of the range is inclusive. + # + # TODO this is wrong for requests that go beyond the end of the + # share. This will be fixed in + # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3907 by making that + # edge case not happen. self.request.setHeader( "content-range", ContentRange( @@ -362,8 +367,11 @@ class _ReadRangeProducer: self.first_read = False if not data and self.remaining > 0: - # Either data is missing locally (storage issue?) or a bug - pass # TODO abort. TODO test + # TODO Either data is missing locally (storage issue?) or a bug, + # abort response with error? Until + # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3907 is implemented + # we continue anyway. + pass self.start += len(data) self.remaining -= len(data) @@ -371,7 +379,8 @@ class _ReadRangeProducer: self.request.write(data) - if self.remaining == 0: + # TODO remove the second clause in https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3907 + if self.remaining == 0 or not data: self.request.unregisterProducer() d = self.result del self.result diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 5c429af88..4e44a9f96 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -1451,8 +1451,10 @@ class SharedImmutableMutableTestsMixin: ) check_range("bytes=0-10", "bytes 0-10/*") + check_range("bytes=3-17", "bytes 3-17/*") + # TODO re-enable in https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3907 # Can't go beyond the end of the mutable/immutable! - check_range("bytes=10-100", "bytes 10-25/*") + #check_range("bytes=10-100", "bytes 10-25/*") class ImmutableSharedTests(SharedImmutableMutableTestsMixin, SyncTestCase): From 5c5556d91505b659ce44e33c31e2ef82d4b079d1 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 6 Jul 2022 09:38:31 -0400 Subject: [PATCH 0945/2309] More robust usage. --- src/allmydata/storage/http_client.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index b8bd0bf20..0ccc3c4a1 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -18,7 +18,7 @@ from werkzeug.datastructures import Range, ContentRange from twisted.web.http_headers import Headers from twisted.web import http from twisted.web.iweb import IPolicyForHTTPS -from twisted.internet.defer import inlineCallbacks, returnValue, fail, Deferred +from twisted.internet.defer import inlineCallbacks, returnValue, fail, Deferred, succeed from twisted.internet.interfaces import IOpenSSLClientConnectionCreator from twisted.internet.ssl import CertificateOptions from twisted.web.client import Agent, HTTPConnectionPool @@ -137,7 +137,10 @@ def limited_content(response, max_length: int = 30 * 1024 * 1024) -> Deferred: fails with a ``ValueError``. """ collector = _LengthLimitedCollector(max_length) - d = treq.collect(response, collector) + # Make really sure everything gets called in Deferred context, treq might + # call collector directly... + d = succeed(None) + d.addCallback(lambda _: treq.collect(response, collector)) d.addCallback(lambda _: collector.f.getvalue()) return d From dac0080ea26cbbe83dfaaf06a777e7b5a554fa63 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 6 Jul 2022 09:40:46 -0400 Subject: [PATCH 0946/2309] Make sure we update remaining length, and update test to catch the edge case this fixes. --- src/allmydata/storage/http_client.py | 3 ++- src/allmydata/test/test_storage_http.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 0ccc3c4a1..daadebb28 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -125,7 +125,8 @@ class _LengthLimitedCollector: f: BytesIO = field(factory=BytesIO) def __call__(self, data: bytes): - if len(data) > self.remaining_length: + self.remaining_length -= len(data) + if self.remaining_length < 0: raise ValueError("Response length was too long") self.f.write(data) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 4e44a9f96..533771866 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -337,7 +337,7 @@ class CustomHTTPServerTests(SyncTestCase): ``http_client.limited_content()`` returns the body if it is less than the max length. """ - for at_least_length in (length, length + 1, length + 1000): + for at_least_length in (length, length + 1, length + 1000, length + 100_000): response = result_of( self.client.request( "GET", From fd8a385d1d70a52ecf26eade6c9f3933d73fef79 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 6 Jul 2022 09:46:59 -0400 Subject: [PATCH 0947/2309] Reformat with black. --- src/allmydata/test/test_storage_http.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 533771866..885750441 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -1454,7 +1454,7 @@ class SharedImmutableMutableTestsMixin: check_range("bytes=3-17", "bytes 3-17/*") # TODO re-enable in https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3907 # Can't go beyond the end of the mutable/immutable! - #check_range("bytes=10-100", "bytes 10-25/*") + # check_range("bytes=10-100", "bytes 10-25/*") class ImmutableSharedTests(SharedImmutableMutableTestsMixin, SyncTestCase): From 0b5132745ddfd8c2b14f66359535dc8e3c7a1eab Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 6 Jul 2022 09:47:08 -0400 Subject: [PATCH 0948/2309] A nicer interface. --- src/allmydata/storage/http_client.py | 18 ++++++++++++++---- src/allmydata/test/test_storage_http.py | 2 +- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index daadebb28..b8ba1641a 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -4,7 +4,7 @@ HTTP client that talks to the HTTP storage server. from __future__ import annotations -from typing import Union, Optional, Sequence, Mapping +from typing import Union, Optional, Sequence, Mapping, BinaryIO from base64 import b64encode from io import BytesIO @@ -131,25 +131,35 @@ class _LengthLimitedCollector: self.f.write(data) -def limited_content(response, max_length: int = 30 * 1024 * 1024) -> Deferred: +def limited_content(response, max_length: int = 30 * 1024 * 1024) -> Deferred[BinaryIO]: """ Like ``treq.content()``, but limit data read from the response to a set length. If the response is longer than the max allowed length, the result fails with a ``ValueError``. + + A potentially useful future improvement would be using a temporary file to + store the content; since filesystem buffering means that would use memory + for small responses and disk for large responses. """ collector = _LengthLimitedCollector(max_length) # Make really sure everything gets called in Deferred context, treq might # call collector directly... d = succeed(None) d.addCallback(lambda _: treq.collect(response, collector)) - d.addCallback(lambda _: collector.f.getvalue()) + + def done(_): + collector.f.seek(0) + return collector.f + + d.addCallback(done) return d def _decode_cbor(response, schema: Schema): """Given HTTP response, return decoded CBOR body.""" - def got_content(data): + def got_content(f: BinaryIO): + data = f.read() schema.validate_cbor(data) return loads(data) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 885750441..419052282 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -346,7 +346,7 @@ class CustomHTTPServerTests(SyncTestCase): ) self.assertEqual( - result_of(limited_content(response, at_least_length)), + result_of(limited_content(response, at_least_length)).read(), gen_bytes(length), ) From 87932e3444267a50c4a00700d356fda4057a9b14 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 6 Jul 2022 09:50:16 -0400 Subject: [PATCH 0949/2309] Correct type. --- src/allmydata/storage/http_server.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 9d90ba960..c53906218 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -4,7 +4,7 @@ HTTP server for storage. from __future__ import annotations -from typing import Dict, List, Set, Tuple, Any, Callable +from typing import Dict, List, Set, Tuple, Any, Callable, Union from functools import wraps from base64 import b64decode import binascii @@ -394,7 +394,7 @@ class _ReadRangeProducer: pass -def read_range(request: Request, read_data: ReadData) -> None: +def read_range(request: Request, read_data: ReadData) -> Union[Deferred, bytes]: """ Read an optional ``Range`` header, reads data appropriately via the given callable, writes the data to the request. From a24aefaebf8f0487b4a8cc981c7cb238d0aca1d2 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 15 Jul 2022 11:35:28 -0400 Subject: [PATCH 0950/2309] There can be up to 256 shares. --- src/allmydata/storage/http_server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index c53906218..a29742bab 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -253,7 +253,7 @@ _SCHEMAS = { "allocate_buckets": Schema( """ request = { - share-numbers: #6.258([*30 uint]) + share-numbers: #6.258([*256 uint]) allocated-size: uint } """ From b3ab2fdb81f60d909572fc8e1be7defe3885e1ca Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 15 Jul 2022 15:12:31 -0400 Subject: [PATCH 0951/2309] Fix grid manager CLI tests. --- src/allmydata/test/common_util.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/allmydata/test/common_util.py b/src/allmydata/test/common_util.py index e63c3eef8..abf675644 100644 --- a/src/allmydata/test/common_util.py +++ b/src/allmydata/test/common_util.py @@ -134,6 +134,7 @@ def run_cli_native(verb, *args, **kwargs): stdin = TextIOWrapper(BytesIO(stdin), encoding) stdout = TextIOWrapper(BytesIO(), encoding) stderr = TextIOWrapper(BytesIO(), encoding) + options.stdin = stdin d = defer.succeed(argv) d.addCallback( partial( From b4703ace930cadb276b627a3612816d4986f9231 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 18 Jul 2022 10:35:28 -0400 Subject: [PATCH 0952/2309] Some tweaks for Python 3. --- src/allmydata/cli/grid_manager.py | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/src/allmydata/cli/grid_manager.py b/src/allmydata/cli/grid_manager.py index b12839eae..4ef53887c 100644 --- a/src/allmydata/cli/grid_manager.py +++ b/src/allmydata/cli/grid_manager.py @@ -1,15 +1,8 @@ """ -Ported to Python 3. +A CLI for configuring a grid manager. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals - -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 +from typing import Optional from datetime import ( datetime, timedelta, @@ -224,7 +217,7 @@ def sign(ctx, name, expiry_days): f.write(certificate_data.encode("ascii")) -def _config_path_from_option(config): +def _config_path_from_option(config: str) -> Optional[FilePath]: """ :param str config: a path or - :returns: a FilePath instance or None From d84d366c72ddac5d15c27e739151fe8a8f91572d Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 18 Jul 2022 10:50:04 -0400 Subject: [PATCH 0953/2309] Some tweaks for Python 3 and modern attrs. --- src/allmydata/grid_manager.py | 62 ++++++++++++++--------------------- 1 file changed, 24 insertions(+), 38 deletions(-) diff --git a/src/allmydata/grid_manager.py b/src/allmydata/grid_manager.py index 8589e87c0..b6ae04136 100644 --- a/src/allmydata/grid_manager.py +++ b/src/allmydata/grid_manager.py @@ -1,21 +1,14 @@ """ Functions and classes relating to the Grid Manager internal state - -Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals - -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import sys from datetime import ( datetime, ) +from typing import Optional + +from twisted.python.filepath import FilePath from allmydata.crypto import ( ed25519, @@ -26,48 +19,41 @@ from allmydata.util import ( dictutil, ) -import attr +from attrs import define, asdict, Factory -@attr.s +@define class SignedCertificate(object): """ A signed certificate. """ # A JSON-encoded, UTF-8-encoded certificate. - certificate = attr.ib( - type=bytes, validator=attr.validators.instance_of(bytes) - ) + certificate : bytes + # The signature in base32. - signature = attr.ib( - type=bytes, - validator=attr.validators.instance_of(bytes) - ) + signature : bytes @classmethod def load(cls, file_like): data = json.load(file_like) return cls( - certificate=data["certificate"].encode("ascii"), + certificate=data["certificate"].encode("utf-8"), signature=data["signature"].encode("ascii") ) def asdict(self): - return attr.asdict(self) + return asdict(self) -@attr.s +@define class _GridManagerStorageServer(object): """ A Grid Manager's notion of a storage server """ - name = attr.ib() - public_key = attr.ib(validator=attr.validators.instance_of(ed25519.Ed25519PublicKey)) - certificates = attr.ib( - default=attr.Factory(list), - validator=attr.validators.instance_of(list), - ) + name : str + public_key : ed25519.Ed25519PublicKey + certificates : list = Factory(list) def add_certificate(self, certificate): """ @@ -75,9 +61,9 @@ class _GridManagerStorageServer(object): """ self.certificates.append(certificate) - def public_key_string(self): + def public_key_string(self) -> bytes: """ - :returns: the public key as a string + :returns: the public key as bytes. """ return ed25519.string_from_verifying_key(self.public_key) @@ -90,16 +76,16 @@ class _GridManagerStorageServer(object): } -@attr.s +@define class _GridManagerCertificate(object): """ Represents a single certificate for a single storage-server """ - filename = attr.ib() - index = attr.ib(validator=attr.validators.instance_of(int)) - expires = attr.ib(validator=attr.validators.instance_of(datetime)) - public_key = attr.ib(validator=attr.validators.instance_of(ed25519.Ed25519PublicKey)) + filename : str + index : int + expires : datetime + public_key : ed25519.Ed25519PublicKey def create_grid_manager(): @@ -113,7 +99,7 @@ def create_grid_manager(): ) -def _load_certificates_for(config_path, name, gm_key=None): +def _load_certificates_for(config_path: Optional[FilePath], name: str, gm_key=Optional[ed25519.Ed25519PublicKey]): """ Load any existing certificates for the given storage-server. @@ -122,7 +108,7 @@ def _load_certificates_for(config_path, name, gm_key=None): :param str name: the name of an existing storage-server - :param ed25519.VerifyingKey gm_key: an optional Grid Manager + :param ed25519.Ed25519PublicKey gm_key: an optional Grid Manager public key. If provided, certificates will be verified against it. :returns: list containing any known certificates (may be empty) @@ -159,7 +145,7 @@ def _load_certificates_for(config_path, name, gm_key=None): return certificates -def load_grid_manager(config_path): +def load_grid_manager(config_path: Optional[FilePath]): """ Load a Grid Manager from existing configuration. From 90188ce4d14b5e19c600ebc6b5734cd2bc267c34 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 18 Jul 2022 11:05:03 -0400 Subject: [PATCH 0954/2309] More Python 3 tweaks. --- src/allmydata/grid_manager.py | 4 ++-- src/allmydata/scripts/admin.py | 19 ++++++++----------- src/allmydata/scripts/types_.py | 2 -- 3 files changed, 10 insertions(+), 15 deletions(-) diff --git a/src/allmydata/grid_manager.py b/src/allmydata/grid_manager.py index b6ae04136..f502c413e 100644 --- a/src/allmydata/grid_manager.py +++ b/src/allmydata/grid_manager.py @@ -6,7 +6,7 @@ import sys from datetime import ( datetime, ) -from typing import Optional +from typing import Optional, Union from twisted.python.filepath import FilePath @@ -338,7 +338,7 @@ def save_grid_manager(file_path, grid_manager, create=True): f.write(b"\n") -def parse_grid_manager_certificate(gm_data): +def parse_grid_manager_certificate(gm_data: Union[str, bytes]): """ :param gm_data: some data that might be JSON that might be a valid Grid Manager Certificate diff --git a/src/allmydata/scripts/admin.py b/src/allmydata/scripts/admin.py index 36266f3dd..6a24614c0 100644 --- a/src/allmydata/scripts/admin.py +++ b/src/allmydata/scripts/admin.py @@ -10,12 +10,9 @@ from future.utils import PY2 if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 -from six import ensure_binary, ensure_str +from six import ensure_binary -try: - from allmydata.scripts.types_ import SubCommands -except ImportError: - pass +from typing import Union from twisted.python import usage from twisted.python.filepath import ( @@ -29,14 +26,12 @@ from allmydata.storage import ( crawler, expirer, ) -from twisted.python.filepath import FilePath - +from allmydata.scripts.types_ import SubCommands from allmydata.client import read_config from allmydata.grid_manager import ( parse_grid_manager_certificate, ) from allmydata.scripts.cli import _default_nodedir -from allmydata.scripts.common import BaseOptions from allmydata.util.encodingutil import argv_to_abspath from allmydata.util import jsonbytes @@ -127,8 +122,10 @@ class AddGridManagerCertOptions(BaseOptions): raise usage.UsageError( "Must provide --filename option" ) + + data : Union [bytes, str] if self['filename'] == '-': - print(ensure_str("reading certificate from stdin"), file=self.parent.parent.stderr) + print("reading certificate from stdin", file=self.parent.parent.stderr) data = self.parent.parent.stdin.read() if len(data) == 0: raise usage.UsageError( @@ -201,7 +198,7 @@ def add_grid_manager_cert(options): options['name'], cert_path.path, ) - print(ensure_str(msg), file=options.parent.parent.stderr) + print(msg, file=options.parent.parent.stderr) return 1 config.set_config("storage", "grid_management", "True") @@ -212,7 +209,7 @@ def add_grid_manager_cert(options): f.write(cert_bytes) cert_count = len(config.enumerate_section("grid_manager_certificates")) - print(ensure_str("There are now {} certificates").format(cert_count), + print("There are now {} certificates".format(cert_count), file=options.parent.parent.stderr) return 0 diff --git a/src/allmydata/scripts/types_.py b/src/allmydata/scripts/types_.py index 1bed6e11e..e2a5c2f1e 100644 --- a/src/allmydata/scripts/types_.py +++ b/src/allmydata/scripts/types_.py @@ -2,8 +2,6 @@ Type definitions used by modules in this package. """ -# Python 3 only - from typing import List, Tuple, Type, Sequence, Any from twisted.python.usage import Options From 6cf3bc75b65cbb90e3b4f7a577966df06f498fd0 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 18 Jul 2022 11:09:19 -0400 Subject: [PATCH 0955/2309] Some Python 3 cleanups. --- src/allmydata/crypto/ed25519.py | 35 +++++++++++---------------------- 1 file changed, 11 insertions(+), 24 deletions(-) diff --git a/src/allmydata/crypto/ed25519.py b/src/allmydata/crypto/ed25519.py index 098fa9758..e2d2ceb49 100644 --- a/src/allmydata/crypto/ed25519.py +++ b/src/allmydata/crypto/ed25519.py @@ -13,20 +13,7 @@ cut-and-pasteability. The base62 encoding is shorter than the base32 form, but the minor usability improvement is not worth the documentation and specification confusion of using a non-standard encoding. So we stick with base32. - -Ported to Python 3. ''' -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals - - -from future.utils import PY2 -if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - -import six from cryptography.exceptions import ( InvalidSignature, @@ -72,7 +59,7 @@ def verifying_key_from_signing_key(private_key): return private_key.public_key() -def sign_data(private_key, data): +def sign_data(private_key, data: bytes) -> bytes: """ Sign the given data using the given private key @@ -86,7 +73,7 @@ def sign_data(private_key, data): """ _validate_private_key(private_key) - if not isinstance(data, six.binary_type): + if not isinstance(data, bytes): raise ValueError('data must be bytes') return private_key.sign(data) @@ -110,7 +97,7 @@ def string_from_signing_key(private_key): return PRIVATE_KEY_PREFIX + b2a(raw_key_bytes) -def signing_keypair_from_string(private_key_bytes): +def signing_keypair_from_string(private_key_bytes: bytes): """ Load a signing keypair from a string of bytes (which includes the PRIVATE_KEY_PREFIX) @@ -118,7 +105,7 @@ def signing_keypair_from_string(private_key_bytes): :returns: a 2-tuple of (private_key, public_key) """ - if not isinstance(private_key_bytes, six.binary_type): + if not isinstance(private_key_bytes, bytes): raise ValueError('private_key_bytes must be bytes') private_key = Ed25519PrivateKey.from_private_bytes( @@ -127,7 +114,7 @@ def signing_keypair_from_string(private_key_bytes): return private_key, private_key.public_key() -def verify_signature(public_key, alleged_signature, data): +def verify_signature(public_key, alleged_signature: bytes, data: bytes): """ :param public_key: a verifying key @@ -139,10 +126,10 @@ def verify_signature(public_key, alleged_signature, data): :returns: None (or raises an exception). """ - if not isinstance(alleged_signature, six.binary_type): + if not isinstance(alleged_signature, bytes): raise ValueError('alleged_signature must be bytes') - if not isinstance(data, six.binary_type): + if not isinstance(data, bytes): raise ValueError('data must be bytes') _validate_public_key(public_key) @@ -159,7 +146,7 @@ def verifying_key_from_string(public_key_bytes): :returns: a public_key """ - if not isinstance(public_key_bytes, six.binary_type): + if not isinstance(public_key_bytes, bytes): raise ValueError('public_key_bytes must be bytes') return Ed25519PublicKey.from_public_bytes( @@ -167,7 +154,7 @@ def verifying_key_from_string(public_key_bytes): ) -def string_from_verifying_key(public_key): +def string_from_verifying_key(public_key) -> bytes: """ Encode a public key to a string of bytes @@ -183,7 +170,7 @@ def string_from_verifying_key(public_key): return PUBLIC_KEY_PREFIX + b2a(raw_key_bytes) -def _validate_public_key(public_key): +def _validate_public_key(public_key: Ed25519PublicKey): """ Internal helper. Verify that `public_key` is an appropriate object """ @@ -192,7 +179,7 @@ def _validate_public_key(public_key): return None -def _validate_private_key(private_key): +def _validate_private_key(private_key: Ed25519PrivateKey): """ Internal helper. Verify that `private_key` is an appropriate object """ From 07a3d1ecd64661ae590405cd6a9917652a6a8e15 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 18 Jul 2022 11:15:03 -0400 Subject: [PATCH 0956/2309] Python 3 tweaks. --- src/allmydata/storage_client.py | 1 - src/allmydata/test/cli/test_admin.py | 12 ++++-------- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index f2d50a574..91073579d 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -38,7 +38,6 @@ import re import time import hashlib -# On Python 2 this will be the backport. from configparser import NoSectionError import attr diff --git a/src/allmydata/test/cli/test_admin.py b/src/allmydata/test/cli/test_admin.py index ce59fa755..9bf12471f 100644 --- a/src/allmydata/test/cli/test_admin.py +++ b/src/allmydata/test/cli/test_admin.py @@ -6,16 +6,12 @@ from __future__ import division from __future__ import print_function from __future__ import unicode_literals -from future.utils import PY2, PY3 +from future.utils import PY2 if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 -from six import ensure_str # We're going to override stdin/stderr, so want to match their behavior on respective Python versions. -if PY3: - from io import StringIO -else: - from StringIO import StringIO +from io import StringIO from twisted.python.usage import ( UsageError, @@ -116,7 +112,7 @@ class AddCertificateOptions(SyncTestCase): """ When no data is passed to stdin an error is produced """ - self.tahoe.stdin = StringIO(ensure_str("")) + self.tahoe.stdin = StringIO("") self.tahoe.stderr = StringIO() # suppress message with self.assertRaises(UsageError) as ctx: @@ -159,7 +155,7 @@ class AddCertificateOptions(SyncTestCase): """ Unparseable data produces an error """ - self.tahoe.stdin = StringIO(ensure_str("{}")) + self.tahoe.stdin = StringIO("{}") self.tahoe.stderr = StringIO() # suppress message with self.assertRaises(UsageError) as ctx: From 8b0941acd26518e798130e70d6f4b05d3442c2d0 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 18 Jul 2022 11:20:46 -0400 Subject: [PATCH 0957/2309] Python 3 tweaks. --- src/allmydata/test/cli/test_grid_manager.py | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/src/allmydata/test/cli/test_grid_manager.py b/src/allmydata/test/cli/test_grid_manager.py index 03e50efdd..2ed738544 100644 --- a/src/allmydata/test/cli/test_grid_manager.py +++ b/src/allmydata/test/cli/test_grid_manager.py @@ -1,14 +1,6 @@ """ -Ported to Python 3. +Tests for the grid manager CLI. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals - -from future.utils import PY2, raise_ -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import os from io import ( @@ -56,7 +48,7 @@ class GridManagerCommandLine(TestCase): """Invoke a command with the runner and ensure it succeeded.""" result = self.runner.invoke(*args, **kwargs) if result.exception is not None: - raise_(*result.exc_info) + raise result.exc_info[1].with_traceback(result.exc_info[2]) self.assertEqual(result.exit_code, 0, result) return result From 011b942372782a14a495b7372ccfed7ff3b903af Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 18 Jul 2022 11:26:33 -0400 Subject: [PATCH 0958/2309] Python 3 tweaks. --- src/allmydata/test/test_grid_manager.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/src/allmydata/test/test_grid_manager.py b/src/allmydata/test/test_grid_manager.py index ded73e1e5..0f8df69d6 100644 --- a/src/allmydata/test/test_grid_manager.py +++ b/src/allmydata/test/test_grid_manager.py @@ -1,14 +1,6 @@ """ -Ported to Python 3. +Tests for the grid manager. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals - -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from datetime import ( timedelta, From 1bcca7f630884a66c7ae515496ca4765351a2a89 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 18 Jul 2022 11:29:20 -0400 Subject: [PATCH 0959/2309] Python 3 tweaks. --- src/allmydata/util/jsonbytes.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/allmydata/util/jsonbytes.py b/src/allmydata/util/jsonbytes.py index 159ae7990..1cc8c8ef0 100644 --- a/src/allmydata/util/jsonbytes.py +++ b/src/allmydata/util/jsonbytes.py @@ -9,7 +9,7 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function -from future.utils import PY2, PY3 +from future.utils import PY2 if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 @@ -108,10 +108,7 @@ def dumps_bytes(obj, *args, **kwargs): UTF-8 encoded Unicode strings. If True, non-UTF-8 bytes are quoted for human consumption. """ - result = dumps(obj, *args, **kwargs) - if PY3 or isinstance(result, str): - result = result.encode("utf-8") - return result + return dumps(obj, *args, **kwargs).encode("utf-8") # To make this module drop-in compatible with json module: From 49dfc8445cec28d6d903d0a15ee69c411b1b70a1 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 18 Jul 2022 14:12:12 -0400 Subject: [PATCH 0960/2309] Implementation of getting length of shares (albeit inefficiently for now). --- src/allmydata/storage/immutable.py | 8 ++++++++ src/allmydata/storage/mutable.py | 10 +++++----- src/allmydata/storage/server.py | 10 ++++++++++ src/allmydata/test/test_storage.py | 22 ++++++++++++++++++++++ 4 files changed, 45 insertions(+), 5 deletions(-) diff --git a/src/allmydata/storage/immutable.py b/src/allmydata/storage/immutable.py index 920bd3c5e..2c65304b8 100644 --- a/src/allmydata/storage/immutable.py +++ b/src/allmydata/storage/immutable.py @@ -199,8 +199,13 @@ class ShareFile(object): raise UnknownImmutableContainerVersionError(filename, version) self._num_leases = num_leases self._lease_offset = filesize - (num_leases * self.LEASE_SIZE) + self._length = filesize - 0xc - (num_leases * self.LEASE_SIZE) + self._data_offset = 0xc + def get_length(self): + return self._length + def unlink(self): os.unlink(self.home) @@ -544,6 +549,9 @@ class BucketReader(object): self.shnum, reason) + def get_length(self): + return self._share_file.get_length() + @implementer(RIBucketReader) class FoolscapBucketReader(Referenceable): # type: ignore # warner/foolscap#78 diff --git a/src/allmydata/storage/mutable.py b/src/allmydata/storage/mutable.py index bd59d96b8..9a99979e9 100644 --- a/src/allmydata/storage/mutable.py +++ b/src/allmydata/storage/mutable.py @@ -412,11 +412,11 @@ class MutableShareFile(object): datav.append(self._read_share_data(f, offset, length)) return datav -# def remote_get_length(self): -# f = open(self.home, 'rb') -# data_length = self._read_data_length(f) -# f.close() -# return data_length + def get_length(self): + f = open(self.home, 'rb') + data_length = self._read_data_length(f) + f.close() + return data_length def check_write_enabler(self, write_enabler, si_s): with open(self.home, 'rb+') as f: diff --git a/src/allmydata/storage/server.py b/src/allmydata/storage/server.py index 0a1999dfb..f452885d0 100644 --- a/src/allmydata/storage/server.py +++ b/src/allmydata/storage/server.py @@ -794,6 +794,16 @@ class StorageServer(service.MultiService): return None + def get_immutable_share_length(self, storage_index: bytes, share_number: int) -> int: + """Returns the length (in bytes) of an immutable.""" + return self.get_buckets(storage_index)[share_number].get_length() + + def get_mutable_share_length(self, storage_index: bytes, share_number: int) -> int: + """Returns the length (in bytes) of a mutable.""" + return MutableShareFile( + dict(self.get_shares(storage_index))[share_number] + ).get_length() + @implementer(RIStorageServer) class FoolscapStorageServer(Referenceable): # type: ignore # warner/foolscap#78 diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index 91d55790e..bb8d48d2f 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -688,6 +688,15 @@ class Server(unittest.TestCase): writer.abort() self.failUnlessEqual(ss.allocated_size(), 0) + def test_immutable_length(self): + """``get_immutable_share_length()`` returns the length of an immutable share.""" + ss = self.create("test_immutable_length") + _, writers = self.allocate(ss, b"allocate", [22], 75) + bucket = writers[22] + bucket.write(0, b"X" * 75) + bucket.close() + self.assertEqual(ss.get_immutable_share_length(b"allocate", 22), 75) + def test_allocate(self): ss = self.create("test_allocate") @@ -1340,6 +1349,19 @@ class MutableServer(unittest.TestCase): (set(), {0, 1, 2, 4}, {0, 1, 4}) ) + def test_mutable_share_length(self): + """``get_mutable_share_length()`` returns the length of the share.""" + ss = self.create("test_mutable_share_length") + self.allocate(ss, b"si1", b"we1", b"le1", [16], 23) + ss.slot_testv_and_readv_and_writev( + b"si1", (self.write_enabler(b"we1"), + self.renew_secret(b"le1"), + self.cancel_secret(b"le1")), + {16: ([], [(0, b"x" * 23)], None)}, + [] + ) + self.assertEqual(ss.get_mutable_share_length(b"si1", 16), 23) + def test_bad_magic(self): ss = self.create("test_bad_magic") self.allocate(ss, b"si1", b"we1", next(self._lease_secret), set([0]), 10) From b3aff5c43b73718de0ecffa6c39d5efac2f6d336 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 19 Jul 2022 14:37:46 -0400 Subject: [PATCH 0961/2309] More efficient implementations. --- src/allmydata/storage/server.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/allmydata/storage/server.py b/src/allmydata/storage/server.py index f452885d0..1b9b051eb 100644 --- a/src/allmydata/storage/server.py +++ b/src/allmydata/storage/server.py @@ -796,13 +796,16 @@ class StorageServer(service.MultiService): def get_immutable_share_length(self, storage_index: bytes, share_number: int) -> int: """Returns the length (in bytes) of an immutable.""" - return self.get_buckets(storage_index)[share_number].get_length() + si_dir = storage_index_to_dir(storage_index) + path = os.path.join(self.sharedir, si_dir, str(share_number)) + bucket = BucketReader(self, path, storage_index, share_number) + return bucket.get_length() def get_mutable_share_length(self, storage_index: bytes, share_number: int) -> int: """Returns the length (in bytes) of a mutable.""" - return MutableShareFile( - dict(self.get_shares(storage_index))[share_number] - ).get_length() + si_dir = storage_index_to_dir(storage_index) + path = os.path.join(self.sharedir, si_dir, str(share_number)) + return MutableShareFile(path).get_length() @implementer(RIStorageServer) From 1b8b71b3068e73486d406f6c79fbbdbf8ecaf261 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 19 Jul 2022 16:10:22 -0400 Subject: [PATCH 0962/2309] Content-Range headers are now checked (somewhat) and the server now sends correct headers when reading beyond the end. --- src/allmydata/storage/http_client.py | 29 ++++++++++++++++++++++-- src/allmydata/storage/http_server.py | 34 ++++++++++++++++++---------- src/allmydata/storage/server.py | 2 ++ 3 files changed, 51 insertions(+), 14 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index b8ba1641a..11c9ab2fc 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -7,6 +7,7 @@ from __future__ import annotations from typing import Union, Optional, Sequence, Mapping, BinaryIO from base64 import b64encode from io import BytesIO +from os import SEEK_END from attrs import define, asdict, frozen, field @@ -29,6 +30,7 @@ from treq.client import HTTPClient from treq.testing import StubTreq from OpenSSL import SSL from cryptography.hazmat.bindings.openssl.binding import Binding +from werkzeug.http import parse_content_range_header from .http_common import ( swissnum_auth_header, @@ -461,13 +463,36 @@ def read_share_chunk( "GET", url, headers=Headers( + # Ranges in HTTP are _inclusive_, Python's convention is exclusive, + # but Range constructor does that the conversion for us. {"range": [Range("bytes", [(offset, offset + length)]).to_header()]} ), ) if response.code == http.PARTIAL_CONTENT: - body = yield response.content() - returnValue(body) + content_range = parse_content_range_header( + response.headers.getRawHeaders("content-range")[0] + ) + supposed_length = content_range.stop - content_range.start + if supposed_length > length: + raise ValueError("Server sent more than we asked for?!") + # It might also send less than we asked for. That's (probably) OK, e.g. + # if we went past the end of the file. + body = yield limited_content(response, supposed_length) + body.seek(0, SEEK_END) + actual_length = body.tell() + if actual_length != supposed_length: + # Most likely a mutable that got changed out from under us, but + # concievably could be a bug... + raise ValueError( + f"Length of response sent from server ({actual_length}) " + + f"didn't match Content-Range header ({supposed_length})" + ) + body.seek(0) + returnValue(body.read()) else: + # Technically HTTP allows sending an OK with full body under these + # circumstances, but the server is not designed to do that so we ignore + # than possibility for now... raise ClientException(response.code) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index a29742bab..4eecf7f2f 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -352,12 +352,10 @@ class _ReadRangeProducer: if self.first_read and self.remaining > 0: # For empty bodies the content-range header makes no sense since - # the end of the range is inclusive. - # - # TODO this is wrong for requests that go beyond the end of the - # share. This will be fixed in - # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3907 by making that - # edge case not happen. + # the end of the range is inclusive. Actual conversion from + # Python's exclusive ranges to inclusive ranges is handled by + # werkzeug. The case where we're reading beyond the end of the + # share is handled by caller (read_range().) self.request.setHeader( "content-range", ContentRange( @@ -368,7 +366,7 @@ class _ReadRangeProducer: if not data and self.remaining > 0: # TODO Either data is missing locally (storage issue?) or a bug, - # abort response with error? Until + # abort response with error. Until # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3907 is implemented # we continue anyway. pass @@ -394,7 +392,9 @@ class _ReadRangeProducer: pass -def read_range(request: Request, read_data: ReadData) -> Union[Deferred, bytes]: +def read_range( + request: Request, read_data: ReadData, share_length: int +) -> Union[Deferred, bytes]: """ Read an optional ``Range`` header, reads data appropriately via the given callable, writes the data to the request. @@ -430,9 +430,12 @@ def read_range(request: Request, read_data: ReadData) -> Union[Deferred, bytes]: ): raise _HTTPError(http.REQUESTED_RANGE_NOT_SATISFIABLE) - # TODO if end is beyond the end of the share, either return error, or maybe - # just return what we can... offset, end = range_header.ranges[0] + # If we're being ask to read beyond the length of the share, just read + # less: + end = min(end, share_length) + # TODO when if end is now <= offset? + request.setResponseCode(http.PARTIAL_CONTENT) d = Deferred() request.registerProducer( @@ -675,7 +678,7 @@ class HTTPServer(object): request.setResponseCode(http.NOT_FOUND) return b"" - return read_range(request, bucket.read) + return read_range(request, bucket.read, bucket.get_length()) @_authorized_route( _app, @@ -763,6 +766,13 @@ class HTTPServer(object): def read_mutable_chunk(self, request, authorization, storage_index, share_number): """Read a chunk from a mutable.""" + try: + share_length = self._storage_server.get_mutable_share_length( + storage_index, share_number + ) + except KeyError: + raise _HTTPError(http.NOT_FOUND) + def read_data(offset, length): try: return self._storage_server.slot_readv( @@ -771,7 +781,7 @@ class HTTPServer(object): except KeyError: raise _HTTPError(http.NOT_FOUND) - return read_range(request, read_data) + return read_range(request, read_data, share_length) @_authorized_route( _app, set(), "/v1/mutable//shares", methods=["GET"] diff --git a/src/allmydata/storage/server.py b/src/allmydata/storage/server.py index 1b9b051eb..88b650bb9 100644 --- a/src/allmydata/storage/server.py +++ b/src/allmydata/storage/server.py @@ -805,6 +805,8 @@ class StorageServer(service.MultiService): """Returns the length (in bytes) of a mutable.""" si_dir = storage_index_to_dir(storage_index) path = os.path.join(self.sharedir, si_dir, str(share_number)) + if not os.path.exists(path): + raise KeyError("No such storage index or share number") return MutableShareFile(path).get_length() From 43c6af6fde66ca49aed735aba9d927ae44e86a2a Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 20 Jul 2022 11:28:14 -0400 Subject: [PATCH 0963/2309] More error handling for edge cases. --- src/allmydata/storage/http_server.py | 42 ++++++++++++++++++---------- 1 file changed, 28 insertions(+), 14 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 4eecf7f2f..82d3d4794 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -355,7 +355,7 @@ class _ReadRangeProducer: # the end of the range is inclusive. Actual conversion from # Python's exclusive ranges to inclusive ranges is handled by # werkzeug. The case where we're reading beyond the end of the - # share is handled by caller (read_range().) + # share is handled by the caller, read_range(). self.request.setHeader( "content-range", ContentRange( @@ -365,11 +365,24 @@ class _ReadRangeProducer: self.first_read = False if not data and self.remaining > 0: - # TODO Either data is missing locally (storage issue?) or a bug, - # abort response with error. Until - # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3907 is implemented - # we continue anyway. - pass + d, self.result = self.result, None + d.errback( + ValueError( + f"Should be {remaining} bytes left, but we got an empty read" + ) + ) + self.stopProducing() + return + + if len(data) > self.remaining: + d, self.result = self.result, None + d.errback( + ValueError( + f"Should be {remaining} bytes left, but we got more than that ({len(data)})!" + ) + ) + self.stopProducing() + return self.start += len(data) self.remaining -= len(data) @@ -377,19 +390,20 @@ class _ReadRangeProducer: self.request.write(data) - # TODO remove the second clause in https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3907 - if self.remaining == 0 or not data: - self.request.unregisterProducer() - d = self.result - del self.result - d.callback(b"") - return + if self.remaining == 0: + self.stopProducing() def pauseProducing(self): pass def stopProducing(self): - pass + if self.request is not None: + self.request.unregisterProducer() + self.request = None + if self.result is not None: + d = self.result + self.result = None + d.callback(b"") def read_range( From 69739f5f9bf28d6e35c8ceacafad4554056fabd5 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 20 Jul 2022 11:42:01 -0400 Subject: [PATCH 0964/2309] Handle case where requested range results in empty response. --- docs/proposed/http-storage-node-protocol.rst | 2 ++ src/allmydata/storage/http_client.py | 6 +++- src/allmydata/storage/http_server.py | 29 +++++++++----------- 3 files changed, 20 insertions(+), 17 deletions(-) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index 7e0b4a542..6a4e4136a 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -654,6 +654,8 @@ The ``Range`` header may be used to request exactly one ``bytes`` range, in whic Interpretation and response behavior is as specified in RFC 7233 § 4.1. Multiple ranges in a single request are *not* supported; open-ended ranges are also not supported. +If the response to a query is an empty range, the ``NO CONTENT`` (204) response code will be used. + Discussion `````````` diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 11c9ab2fc..236ec970f 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -468,6 +468,10 @@ def read_share_chunk( {"range": [Range("bytes", [(offset, offset + length)]).to_header()]} ), ) + + if response.code == http.NO_CONTENT: + return b"" + if response.code == http.PARTIAL_CONTENT: content_range = parse_content_range_header( response.headers.getRawHeaders("content-range")[0] @@ -488,7 +492,7 @@ def read_share_chunk( + f"didn't match Content-Range header ({supposed_length})" ) body.seek(0) - returnValue(body.read()) + return body.read() else: # Technically HTTP allows sending an OK with full body under these # circumstances, but the server is not designed to do that so we ignore diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 82d3d4794..cb55afffe 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -343,27 +343,12 @@ class _ReadRangeProducer: result: Deferred start: int remaining: int - first_read: bool = field(default=True) def resumeProducing(self): to_read = min(self.remaining, 65536) data = self.read_data(self.start, to_read) assert len(data) <= to_read - if self.first_read and self.remaining > 0: - # For empty bodies the content-range header makes no sense since - # the end of the range is inclusive. Actual conversion from - # Python's exclusive ranges to inclusive ranges is handled by - # werkzeug. The case where we're reading beyond the end of the - # share is handled by the caller, read_range(). - self.request.setHeader( - "content-range", - ContentRange( - "bytes", self.start, self.start + self.remaining - ).to_header(), - ) - self.first_read = False - if not data and self.remaining > 0: d, self.result = self.result, None d.errback( @@ -448,9 +433,21 @@ def read_range( # If we're being ask to read beyond the length of the share, just read # less: end = min(end, share_length) - # TODO when if end is now <= offset? + if offset >= end: + # Basically we'd need to return an empty body. However, the + # Content-Range header can't actually represent empty lengths... so + # (mis)use 204 response code to indicate that. + raise _HTTPError(http.NO_CONTENT) request.setResponseCode(http.PARTIAL_CONTENT) + + # Actual conversion from Python's exclusive ranges to inclusive ranges is + # handled by werkzeug. + request.setHeader( + "content-range", + ContentRange("bytes", offset, end).to_header(), + ) + d = Deferred() request.registerProducer( _ReadRangeProducer( From 92392501a7439c582599cc7cc36a6270926770c1 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 20 Jul 2022 11:47:15 -0400 Subject: [PATCH 0965/2309] Expand spec. --- docs/proposed/http-storage-node-protocol.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index 6a4e4136a..09b523c87 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -654,6 +654,9 @@ The ``Range`` header may be used to request exactly one ``bytes`` range, in whic Interpretation and response behavior is as specified in RFC 7233 § 4.1. Multiple ranges in a single request are *not* supported; open-ended ranges are also not supported. +If the response reads beyond the end fo the data, the response may be shorter than then requested range. +The resulting ``Content-Range`` header will be consistent with the returned data. + If the response to a query is an empty range, the ``NO CONTENT`` (204) response code will be used. Discussion @@ -756,6 +759,11 @@ The ``Range`` header may be used to request exactly one ``bytes`` range, in whic Interpretation and response behavior is as specified in RFC 7233 § 4.1. Multiple ranges in a single request are *not* supported; open-ended ranges are also not supported. +If the response reads beyond the end fo the data, the response may be shorter than then requested range. +The resulting ``Content-Range`` header will be consistent with the returned data. + +If the response to a query is an empty range, the ``NO CONTENT`` (204) response code will be used. + ``POST /v1/mutable/:storage_index/:share_number/corrupt`` !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! From da8a36fac9af1f37eb34bf7ab0a46253e906980b Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 20 Jul 2022 12:07:46 -0400 Subject: [PATCH 0966/2309] Improve test coverage. --- src/allmydata/test/test_storage.py | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index bb8d48d2f..c3f2a35e1 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -689,13 +689,17 @@ class Server(unittest.TestCase): self.failUnlessEqual(ss.allocated_size(), 0) def test_immutable_length(self): - """``get_immutable_share_length()`` returns the length of an immutable share.""" + """ + ``get_immutable_share_length()`` returns the length of an immutable + share, as does ``BucketWriter.get_length()``.. + """ ss = self.create("test_immutable_length") _, writers = self.allocate(ss, b"allocate", [22], 75) bucket = writers[22] bucket.write(0, b"X" * 75) bucket.close() self.assertEqual(ss.get_immutable_share_length(b"allocate", 22), 75) + self.assertEqual(ss.get_buckets(b"allocate")[22].get_length(), 75) def test_allocate(self): ss = self.create("test_allocate") @@ -1362,6 +1366,26 @@ class MutableServer(unittest.TestCase): ) self.assertEqual(ss.get_mutable_share_length(b"si1", 16), 23) + def test_mutable_share_length_unknown(self): + """ + ``get_mutable_share_length()`` raises a ``KeyError`` on unknown shares. + """ + ss = self.create("test_mutable_share_length_unknown") + self.allocate(ss, b"si1", b"we1", b"le1", [16], 23) + ss.slot_testv_and_readv_and_writev( + b"si1", (self.write_enabler(b"we1"), + self.renew_secret(b"le1"), + self.cancel_secret(b"le1")), + {16: ([], [(0, b"x" * 23)], None)}, + [] + ) + with self.assertRaises(KeyError): + # Wrong share number. + ss.get_mutable_share_length(b"si1", 17) + with self.assertRaises(KeyError): + # Wrong storage index + ss.get_mutable_share_length(b"unknown", 16) + def test_bad_magic(self): ss = self.create("test_bad_magic") self.allocate(ss, b"si1", b"we1", next(self._lease_secret), set([0]), 10) From d85b20b62d92d9cde835e1fb2438660f5e725962 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 20 Jul 2022 12:47:18 -0400 Subject: [PATCH 0967/2309] Fix lint. --- src/allmydata/storage/http_server.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index cb55afffe..68d0740b1 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -353,7 +353,7 @@ class _ReadRangeProducer: d, self.result = self.result, None d.errback( ValueError( - f"Should be {remaining} bytes left, but we got an empty read" + f"Should be {self.remaining} bytes left, but we got an empty read" ) ) self.stopProducing() @@ -363,7 +363,7 @@ class _ReadRangeProducer: d, self.result = self.result, None d.errback( ValueError( - f"Should be {remaining} bytes left, but we got more than that ({len(data)})!" + f"Should be {self.remaining} bytes left, but we got more than that ({len(data)})!" ) ) self.stopProducing() From 3b7345205bbb68a79f079b7c619ca2a912c7c918 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 20 Jul 2022 14:24:10 -0400 Subject: [PATCH 0968/2309] News file. --- newsfragments/3709.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3709.minor diff --git a/newsfragments/3709.minor b/newsfragments/3709.minor new file mode 100644 index 000000000..e69de29bb From 11f4ebc0d90ed80f612d29945cd2436f43f658ea Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 20 Jul 2022 15:12:00 -0400 Subject: [PATCH 0969/2309] Hook up NURL generation to the new Foolscap/HTTPS protocol switch. --- src/allmydata/client.py | 16 ++++++++ src/allmydata/protocol_switch.py | 8 ---- src/allmydata/storage/http_server.py | 46 +++++++++++++++-------- src/allmydata/test/test_istorageserver.py | 33 +++------------- 4 files changed, 52 insertions(+), 51 deletions(-) diff --git a/src/allmydata/client.py b/src/allmydata/client.py index e737f93e6..3318bbfa4 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -37,6 +37,7 @@ import allmydata from allmydata.crypto import rsa, ed25519 from allmydata.crypto.util import remove_prefix from allmydata.storage.server import StorageServer, FoolscapStorageServer +from allmydata.storage.http_server import build_nurl from allmydata import storage_client from allmydata.immutable.upload import Uploader from allmydata.immutable.offloaded import Helper @@ -658,6 +659,12 @@ class _Client(node.Node, pollmixin.PollMixin): if webport: self.init_web(webport) # strports string + # TODO this may be the wrong location for now? but as temporary measure + # it allows us to get NURLs for testing in test_istorageserver.py Will + # eventually get fixed one way or another in + # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3901 + self.storage_nurls = [] + def init_stats_provider(self): self.stats_provider = StatsProvider(self) self.stats_provider.setServiceParent(self) @@ -820,6 +827,15 @@ class _Client(node.Node, pollmixin.PollMixin): furl = self.tub.registerReference(FoolscapStorageServer(ss), furlFile=furl_file) (_, _, swissnum) = furl.rpartition("/") self.tub.negotiationClass.add_storage_server(ss, swissnum.encode("ascii")) + for location_hint in self.tub.locationHints: + if location_hint.startswith("tcp:"): + _, hostname, port = location_hint.split(":") + port = int(port) + self.storage_nurls.append( + build_nurl( + hostname, port, swissnum, self.tub.myCertificate.original.to_cryptography() + ) + ) announcement["anonymous-storage-FURL"] = furl diff --git a/src/allmydata/protocol_switch.py b/src/allmydata/protocol_switch.py index 9f33560e7..21d896793 100644 --- a/src/allmydata/protocol_switch.py +++ b/src/allmydata/protocol_switch.py @@ -66,14 +66,6 @@ class _FoolscapOrHttps(Protocol, metaclass=_PretendToBeNegotiation): Add the various storage server-related attributes needed by a ``Tub``-specific ``_FoolscapOrHttps`` subclass. """ - # TODO tub.locationHints will be in the format ["tcp:hostname:port"] - # (and maybe some other things we can ignore for now). We also have - # access to the certificate. Together, this should be sufficient to - # construct NURLs, one per hint. The code for NURls should be - # refactored out of http_server.py's build_nurl; that code might want - # to skip around for the future when we don't do foolscap, but for now - # this module will be main way we set up HTTPS. - # Tub.myCertificate is a twisted.internet.ssl.PrivateCertificate # instance. certificate_options = CertificateOptions( diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index e2b754b0d..7f7c1c0ae 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -10,6 +10,7 @@ from base64 import b64decode import binascii from tempfile import TemporaryFile +from cryptography.x509 import Certificate from zope.interface import implementer from klein import Klein from twisted.web import http @@ -843,6 +844,29 @@ class _TLSEndpointWrapper(object): ) +def build_nurl( + hostname: str, port: int, swissnum: str, certificate: Certificate +) -> DecodedURL: + """ + Construct a HTTPS NURL, given the hostname, port, server swissnum, and x509 + certificate for the server. Clients can then connect to the server using + this NURL. + """ + return DecodedURL().replace( + fragment="v=1", # how we know this NURL is HTTP-based (i.e. not Foolscap) + host=hostname, + port=port, + path=(swissnum,), + userinfo=( + str( + get_spki_hash(certificate), + "ascii", + ), + ), + scheme="pb", + ) + + def listen_tls( server: HTTPServer, hostname: str, @@ -862,22 +886,14 @@ def listen_tls( """ endpoint = _TLSEndpointWrapper.from_paths(endpoint, private_key_path, cert_path) - def build_nurl(listening_port: IListeningPort) -> DecodedURL: - nurl = DecodedURL().replace( - fragment="v=1", # how we know this NURL is HTTP-based (i.e. not Foolscap) - host=hostname, - port=listening_port.getHost().port, - path=(str(server._swissnum, "ascii"),), - userinfo=( - str( - get_spki_hash(load_pem_x509_certificate(cert_path.getContent())), - "ascii", - ), - ), - scheme="pb", + def get_nurl(listening_port: IListeningPort) -> DecodedURL: + return build_nurl( + hostname, + listening_port.getHost().port, + str(server._swissnum, "ascii"), + load_pem_x509_certificate(cert_path.getContent()), ) - return nurl return endpoint.listen(Site(server.get_resource())).addCallback( - lambda listening_port: (build_nurl(listening_port), listening_port) + lambda listening_port: (get_nurl(listening_port), listening_port) ) diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index 39675336f..12a3cba55 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -1084,40 +1084,17 @@ class _FoolscapMixin(_SharedMixin): class _HTTPMixin(_SharedMixin): """Run tests on the HTTP version of ``IStorageServer``.""" - def setUp(self): - self._port_assigner = SameProcessStreamEndpointAssigner() - self._port_assigner.setUp() - self.addCleanup(self._port_assigner.tearDown) - return _SharedMixin.setUp(self) - - @inlineCallbacks def _get_istorage_server(self): - swissnum = b"1234" - http_storage_server = HTTPServer(self.server, swissnum) - - # Listen on randomly assigned port, using self-signed cert: - private_key = generate_private_key() - certificate = generate_certificate(private_key) - _, endpoint_string = self._port_assigner.assign(reactor) - nurl, listening_port = yield listen_tls( - http_storage_server, - "127.0.0.1", - serverFromString(reactor, endpoint_string), - private_key_to_file(FilePath(self.mktemp()), private_key), - cert_to_file(FilePath(self.mktemp()), certificate), - ) - self.addCleanup(listening_port.stopListening) + nurl = self.clients[0].storage_nurls[0] # Create HTTP client with non-persistent connections, so we don't leak # state across tests: - returnValue( - _HTTPStorageServer.from_http_client( - StorageClient.from_nurl(nurl, reactor, persistent=False) - ) + client: IStorageServer = _HTTPStorageServer.from_http_client( + StorageClient.from_nurl(nurl, reactor, persistent=False) ) + self.assertTrue(IStorageServer.providedBy(client)) - # Eventually should also: - # self.assertTrue(IStorageServer.providedBy(client)) + return succeed(client) class FoolscapSharedAPIsTests( From 981b693402929dbe24f4f48cc5a42bb7b95b3285 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 20 Jul 2022 15:25:22 -0400 Subject: [PATCH 0970/2309] Make HTTPS protocols work with the protocol switcher magic. --- src/allmydata/protocol_switch.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/allmydata/protocol_switch.py b/src/allmydata/protocol_switch.py index 21d896793..d3e68f860 100644 --- a/src/allmydata/protocol_switch.py +++ b/src/allmydata/protocol_switch.py @@ -138,6 +138,13 @@ class _FoolscapOrHttps(Protocol, metaclass=_PretendToBeNegotiation): protocol = self.https_factory.buildProtocol(self.transport.getPeer()) protocol.makeConnection(self.transport) protocol.dataReceived(self._buffer) + + # Update the factory so it knows we're transforming to a new + # protocol object (we'll do that next) + value = self.https_factory.protocols.pop(protocol) + self.https_factory.protocols[self] = value + + # Transform self into the TLS protocol 🪄 self.__class__ = protocol.__class__ self.__dict__ = protocol.__dict__ From 757e8c418c62864246e360be05c9cb25654d9c2a Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 22 Jul 2022 11:51:26 -0400 Subject: [PATCH 0971/2309] Fix typos. --- docs/proposed/http-storage-node-protocol.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index 09b523c87..3dac376ff 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -654,7 +654,7 @@ The ``Range`` header may be used to request exactly one ``bytes`` range, in whic Interpretation and response behavior is as specified in RFC 7233 § 4.1. Multiple ranges in a single request are *not* supported; open-ended ranges are also not supported. -If the response reads beyond the end fo the data, the response may be shorter than then requested range. +If the response reads beyond the end of the data, the response may be shorter than the requested range. The resulting ``Content-Range`` header will be consistent with the returned data. If the response to a query is an empty range, the ``NO CONTENT`` (204) response code will be used. @@ -759,7 +759,7 @@ The ``Range`` header may be used to request exactly one ``bytes`` range, in whic Interpretation and response behavior is as specified in RFC 7233 § 4.1. Multiple ranges in a single request are *not* supported; open-ended ranges are also not supported. -If the response reads beyond the end fo the data, the response may be shorter than then requested range. +If the response reads beyond the end of the data, the response may be shorter than the requested range. The resulting ``Content-Range`` header will be consistent with the returned data. If the response to a query is an empty range, the ``NO CONTENT`` (204) response code will be used. From 5cd9ccfc6ae78be55eca1931402fd512d6199787 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 22 Jul 2022 11:52:56 -0400 Subject: [PATCH 0972/2309] Slightly nicer handling for bad edge cases. --- src/allmydata/storage/http_client.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 236ec970f..a464d445a 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -474,8 +474,16 @@ def read_share_chunk( if response.code == http.PARTIAL_CONTENT: content_range = parse_content_range_header( - response.headers.getRawHeaders("content-range")[0] + response.headers.getRawHeaders("content-range")[0] or "" ) + if ( + content_range is None + or content_range.stop is None + or content_range.start is None + ): + raise ValueError( + "Content-Range was missing, invalid, or in format we don't support" + ) supposed_length = content_range.stop - content_range.start if supposed_length > length: raise ValueError("Server sent more than we asked for?!") From f671b47a6decb0f53b49697340012119286d0f91 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 22 Jul 2022 11:53:12 -0400 Subject: [PATCH 0973/2309] Fix typo. --- src/allmydata/storage/http_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index a464d445a..da272240d 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -504,7 +504,7 @@ def read_share_chunk( else: # Technically HTTP allows sending an OK with full body under these # circumstances, but the server is not designed to do that so we ignore - # than possibility for now... + # that possibility for now... raise ClientException(response.code) From 36b96a8776a39d77f017725f2ca4ccb9e4f8cf5c Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 22 Jul 2022 11:53:28 -0400 Subject: [PATCH 0974/2309] Fix typo. --- src/allmydata/storage/http_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index da272240d..a2dc5379f 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -494,7 +494,7 @@ def read_share_chunk( actual_length = body.tell() if actual_length != supposed_length: # Most likely a mutable that got changed out from under us, but - # concievably could be a bug... + # conceivably could be a bug... raise ValueError( f"Length of response sent from server ({actual_length}) " + f"didn't match Content-Range header ({supposed_length})" From 2b3a8ddeece0e11d095e8c350a2fef66f4deee20 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 22 Jul 2022 11:55:00 -0400 Subject: [PATCH 0975/2309] Docstring. --- src/allmydata/storage/mutable.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/allmydata/storage/mutable.py b/src/allmydata/storage/mutable.py index 9a99979e9..51c3a3c8b 100644 --- a/src/allmydata/storage/mutable.py +++ b/src/allmydata/storage/mutable.py @@ -413,6 +413,9 @@ class MutableShareFile(object): return datav def get_length(self): + """ + Return the length of the data in the share. + """ f = open(self.home, 'rb') data_length = self._read_data_length(f) f.close() From be963e2324c52801998796f0ba2cd931a4bf556f Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 22 Jul 2022 11:55:33 -0400 Subject: [PATCH 0976/2309] Docstrings. --- src/allmydata/storage/immutable.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/allmydata/storage/immutable.py b/src/allmydata/storage/immutable.py index 2c65304b8..0338af41c 100644 --- a/src/allmydata/storage/immutable.py +++ b/src/allmydata/storage/immutable.py @@ -204,6 +204,9 @@ class ShareFile(object): self._data_offset = 0xc def get_length(self): + """ + Return the length of the data in the share, if we're reading. + """ return self._length def unlink(self): @@ -549,9 +552,6 @@ class BucketReader(object): self.shnum, reason) - def get_length(self): - return self._share_file.get_length() - @implementer(RIBucketReader) class FoolscapBucketReader(Referenceable): # type: ignore # warner/foolscap#78 From 83f9c0788b1fca812048fe4cafa912f97d664501 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 22 Jul 2022 11:56:18 -0400 Subject: [PATCH 0977/2309] Use more direct API. --- src/allmydata/storage/server.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/allmydata/storage/server.py b/src/allmydata/storage/server.py index 88b650bb9..2bf99d74c 100644 --- a/src/allmydata/storage/server.py +++ b/src/allmydata/storage/server.py @@ -798,8 +798,7 @@ class StorageServer(service.MultiService): """Returns the length (in bytes) of an immutable.""" si_dir = storage_index_to_dir(storage_index) path = os.path.join(self.sharedir, si_dir, str(share_number)) - bucket = BucketReader(self, path, storage_index, share_number) - return bucket.get_length() + return ShareFile(path).get_length() def get_mutable_share_length(self, storage_index: bytes, share_number: int) -> int: """Returns the length (in bytes) of a mutable.""" From 94e0568653a2fc49e33ba4be8c1f86b82aa48737 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 22 Jul 2022 11:57:32 -0400 Subject: [PATCH 0978/2309] Actually we do need it. --- src/allmydata/storage/immutable.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/allmydata/storage/immutable.py b/src/allmydata/storage/immutable.py index 0338af41c..f7f5aebce 100644 --- a/src/allmydata/storage/immutable.py +++ b/src/allmydata/storage/immutable.py @@ -552,6 +552,12 @@ class BucketReader(object): self.shnum, reason) + def get_length(self): + """ + Return the length of the data in the share. + """ + return self._share_file.get_length() + @implementer(RIBucketReader) class FoolscapBucketReader(Referenceable): # type: ignore # warner/foolscap#78 From c14463ac6df6c1d0b4f3a44dd5548de8128c214f Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 25 Jul 2022 09:52:40 -0400 Subject: [PATCH 0979/2309] News file. --- newsfragments/3909.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3909.minor diff --git a/newsfragments/3909.minor b/newsfragments/3909.minor new file mode 100644 index 000000000..e69de29bb From 921e3a771248c24b54e77c9c8d44cc488d572906 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 25 Jul 2022 09:55:03 -0400 Subject: [PATCH 0980/2309] Don't use broken version of werkzeug. --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index d07031cd9..c3ee4eb90 100644 --- a/setup.py +++ b/setup.py @@ -133,7 +133,8 @@ install_requires = [ # HTTP server and client "klein", - "werkzeug", + # 2.2.0 has a bug: https://github.com/pallets/werkzeug/issues/2465 + "werkzeug != 2.2.0", "treq", "cbor2", "pycddl", From 671e829f4e5d7a58be75ff1c4ef673b7f7e2fb3d Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 27 Jul 2022 12:23:20 -0400 Subject: [PATCH 0981/2309] We need to pass in the furl here. --- integration/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration/util.py b/integration/util.py index f84a6aed4..5e644f19d 100644 --- a/integration/util.py +++ b/integration/util.py @@ -336,7 +336,7 @@ def _create_node(reactor, request, temp_dir, introducer_furl, flog_gatherer, nam config, u'node', u'log_gatherer.furl', - flog_gatherer, + flog_gatherer.furl, ) write_config(FilePath(config_path), config) created_d.addCallback(created) From 2999ca45798d466ff8ee8f94eef1ac841f4d93db Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 27 Jul 2022 12:23:34 -0400 Subject: [PATCH 0982/2309] It's bytes now. --- integration/test_servers_of_happiness.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration/test_servers_of_happiness.py b/integration/test_servers_of_happiness.py index 3376b91d0..4cbb94654 100644 --- a/integration/test_servers_of_happiness.py +++ b/integration/test_servers_of_happiness.py @@ -51,7 +51,7 @@ def test_upload_immutable(reactor, temp_dir, introducer_furl, flog_gatherer, sto yield proto.done assert False, "should raise exception" except util.ProcessFailed as e: - assert "UploadUnhappinessError" in e.output + assert b"UploadUnhappinessError" in e.output output = proto.output.getvalue() assert b"shares could be placed on only" in output From 106b67db5588364727d3f6add1b9942943b34f58 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 27 Jul 2022 12:23:40 -0400 Subject: [PATCH 0983/2309] It's bytes now. --- integration/test_grid_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration/test_grid_manager.py b/integration/test_grid_manager.py index c01773b96..704dee04b 100644 --- a/integration/test_grid_manager.py +++ b/integration/test_grid_manager.py @@ -239,7 +239,7 @@ def test_reject_storage_server(reactor, request, temp_dir, flog_gatherer, port_a ) assert False, "Should get a failure" except util.ProcessFailed as e: - assert 'UploadUnhappinessError' in e.output + assert b'UploadUnhappinessError' in e.output @pytest_twisted.inlineCallbacks From 02cb4105b3a182c82e8b2a2b66755dcac6385ac8 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 29 Jul 2022 09:43:37 -0400 Subject: [PATCH 0984/2309] A lot closer to passing grid manager integration tests. --- integration/test_grid_manager.py | 6 +++--- src/allmydata/cli/grid_manager.py | 4 ++++ src/allmydata/storage_client.py | 7 ++++--- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/integration/test_grid_manager.py b/integration/test_grid_manager.py index 704dee04b..63ee827b0 100644 --- a/integration/test_grid_manager.py +++ b/integration/test_grid_manager.py @@ -194,7 +194,7 @@ def test_reject_storage_server(reactor, request, temp_dir, flog_gatherer, port_a "storage0", pubkey_str, stdinBytes=gm_config, ) - assert json.loads(gm_config)['storage_servers'].keys() == ['storage0'] + assert json.loads(gm_config)['storage_servers'].keys() == {'storage0'} print("inserting certificate") cert = yield _run_gm( @@ -221,7 +221,7 @@ def test_reject_storage_server(reactor, request, temp_dir, flog_gatherer, port_a config = configutil.get_config(join(diana.process.node_dir, "tahoe.cfg")) config.add_section("grid_managers") - config.set("grid_managers", "test", ed25519.string_from_verifying_key(gm_pubkey)) + config.set("grid_managers", "test", str(ed25519.string_from_verifying_key(gm_pubkey), "ascii")) with open(join(diana.process.node_dir, "tahoe.cfg"), "w") as f: config.write(f) @@ -235,7 +235,7 @@ def test_reject_storage_server(reactor, request, temp_dir, flog_gatherer, port_a yield util.run_tahoe( reactor, request, "--node-directory", diana.process.node_dir, "put", "-", - stdin="some content\n" * 200, + stdin=b"some content\n" * 200, ) assert False, "Should get a failure" except util.ProcessFailed as e: diff --git a/src/allmydata/cli/grid_manager.py b/src/allmydata/cli/grid_manager.py index 4ef53887c..d3a11b62d 100644 --- a/src/allmydata/cli/grid_manager.py +++ b/src/allmydata/cli/grid_manager.py @@ -225,3 +225,7 @@ def _config_path_from_option(config: str) -> Optional[FilePath]: if config == "-": return None return FilePath(config) + + +if __name__ == '__main__': + grid_manager() diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 91073579d..7fe4d6bd2 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -37,8 +37,9 @@ from os import urandom import re import time import hashlib - +from io import StringIO from configparser import NoSectionError +import json import attr from zope.interface import ( @@ -67,7 +68,7 @@ from allmydata.interfaces import ( IFoolscapStoragePlugin, ) from allmydata.grid_manager import ( - create_grid_manager_verifier, + create_grid_manager_verifier, SignedCertificate ) from allmydata.crypto import ( ed25519, @@ -289,7 +290,7 @@ class StorageFarmBroker(service.MultiService): handler_overrides = server.get("connections", {}) gm_verifier = create_grid_manager_verifier( self.storage_client_config.grid_manager_keys, - server["ann"].get("grid-manager-certificates", []), + [SignedCertificate.load(StringIO(json.dumps(data))) for data in server["ann"].get("grid-manager-certificates", [])], "pub-{}".format(str(server_id, "ascii")), # server_id is v0- not pub-v0-key .. for reasons? ) From 822b652d99296a1d767f88446b2825e09caf8b65 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 29 Jul 2022 09:57:18 -0400 Subject: [PATCH 0985/2309] Improve factoring. --- src/allmydata/client.py | 21 +++++----------- src/allmydata/protocol_switch.py | 30 ++++++++++++++++++++--- src/allmydata/test/test_istorageserver.py | 2 +- 3 files changed, 34 insertions(+), 19 deletions(-) diff --git a/src/allmydata/client.py b/src/allmydata/client.py index 3318bbfa4..9938ec076 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -37,7 +37,6 @@ import allmydata from allmydata.crypto import rsa, ed25519 from allmydata.crypto.util import remove_prefix from allmydata.storage.server import StorageServer, FoolscapStorageServer -from allmydata.storage.http_server import build_nurl from allmydata import storage_client from allmydata.immutable.upload import Uploader from allmydata.immutable.offloaded import Helper @@ -660,10 +659,10 @@ class _Client(node.Node, pollmixin.PollMixin): self.init_web(webport) # strports string # TODO this may be the wrong location for now? but as temporary measure - # it allows us to get NURLs for testing in test_istorageserver.py Will - # eventually get fixed one way or another in + # it allows us to get NURLs for testing in test_istorageserver.py. This + # will eventually get fixed one way or another in # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3901 - self.storage_nurls = [] + self.storage_nurls = set() def init_stats_provider(self): self.stats_provider = StatsProvider(self) @@ -826,17 +825,9 @@ class _Client(node.Node, pollmixin.PollMixin): furl_file = self.config.get_private_path("storage.furl").encode(get_filesystem_encoding()) furl = self.tub.registerReference(FoolscapStorageServer(ss), furlFile=furl_file) (_, _, swissnum) = furl.rpartition("/") - self.tub.negotiationClass.add_storage_server(ss, swissnum.encode("ascii")) - for location_hint in self.tub.locationHints: - if location_hint.startswith("tcp:"): - _, hostname, port = location_hint.split(":") - port = int(port) - self.storage_nurls.append( - build_nurl( - hostname, port, swissnum, self.tub.myCertificate.original.to_cryptography() - ) - ) - + self.storage_nurls.update( + self.tub.negotiationClass.add_storage_server(ss, swissnum.encode("ascii")) + ) announcement["anonymous-storage-FURL"] = furl enabled_storage_servers = self._enable_storage_servers( diff --git a/src/allmydata/protocol_switch.py b/src/allmydata/protocol_switch.py index d3e68f860..f1fa6e061 100644 --- a/src/allmydata/protocol_switch.py +++ b/src/allmydata/protocol_switch.py @@ -12,6 +12,8 @@ relevant information for a storage server once it becomes available later in the configuration process. """ +from __future__ import annotations + from twisted.internet.protocol import Protocol from twisted.internet.interfaces import IDelayedCall from twisted.internet.ssl import CertificateOptions @@ -19,10 +21,11 @@ from twisted.web.server import Site from twisted.protocols.tls import TLSMemoryBIOFactory from twisted.internet import reactor +from hyperlink import DecodedURL from foolscap.negotiate import Negotiation from foolscap.api import Tub -from .storage.http_server import HTTPServer +from .storage.http_server import HTTPServer, build_nurl from .storage.server import StorageServer @@ -45,7 +48,9 @@ class _FoolscapOrHttps(Protocol, metaclass=_PretendToBeNegotiation): since these are created by Foolscap's ``Tub``, by setting this to be the tub's ``negotiationClass``. - Do not use directly; this needs to be subclassed per ``Tub``. + Do not use directly, use ``support_foolscap_and_https(tub)`` instead. The + way this class works is that a new subclass is created for a specific + ``Tub`` instance. """ # These will be set by support_foolscap_and_https() and add_storage_server(). @@ -61,10 +66,14 @@ class _FoolscapOrHttps(Protocol, metaclass=_PretendToBeNegotiation): _timeout: IDelayedCall @classmethod - def add_storage_server(cls, storage_server: StorageServer, swissnum): + def add_storage_server( + cls, storage_server: StorageServer, swissnum: bytes + ) -> set[DecodedURL]: """ Add the various storage server-related attributes needed by a ``Tub``-specific ``_FoolscapOrHttps`` subclass. + + Returns the resulting NURLs. """ # Tub.myCertificate is a twisted.internet.ssl.PrivateCertificate # instance. @@ -80,6 +89,21 @@ class _FoolscapOrHttps(Protocol, metaclass=_PretendToBeNegotiation): Site(cls.http_storage_server.get_resource()), ) + storage_nurls = set() + for location_hint in cls.tub.locationHints: + if location_hint.startswith("tcp:"): + _, hostname, port = location_hint.split(":") + port = int(port) + storage_nurls.add( + build_nurl( + hostname, + port, + str(swissnum, "ascii"), + cls.tub.myCertificate.original.to_cryptography(), + ) + ) + return storage_nurls + def __init__(self, *args, **kwargs): self._foolscap: Negotiation = Negotiation(*args, **kwargs) self._buffer: bytes = b"" diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index 12a3cba55..90159f1f8 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -1085,7 +1085,7 @@ class _HTTPMixin(_SharedMixin): """Run tests on the HTTP version of ``IStorageServer``.""" def _get_istorage_server(self): - nurl = self.clients[0].storage_nurls[0] + nurl = list(self.clients[0].storage_nurls)[0] # Create HTTP client with non-persistent connections, so we don't leak # state across tests: From 34518f9d0dcb8fcb91382beb1ead726b811c5dff Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 29 Jul 2022 10:01:09 -0400 Subject: [PATCH 0986/2309] Fix lints. --- src/allmydata/storage/http_server.py | 4 ++-- src/allmydata/test/test_istorageserver.py | 15 ++++----------- 2 files changed, 6 insertions(+), 13 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 98611e833..ca8917694 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -10,7 +10,7 @@ from base64 import b64decode import binascii from tempfile import TemporaryFile -from cryptography.x509 import Certificate +from cryptography.x509 import Certificate as CryptoCertificate from zope.interface import implementer from klein import Klein from twisted.web import http @@ -866,7 +866,7 @@ class _TLSEndpointWrapper(object): def build_nurl( - hostname: str, port: int, swissnum: str, certificate: Certificate + hostname: str, port: int, swissnum: str, certificate: CryptoCertificate ) -> DecodedURL: """ Construct a HTTPS NURL, given the hostname, port, server swissnum, and x509 diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index 90159f1f8..3328ea598 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -18,21 +18,14 @@ from unittest import SkipTest from twisted.internet.defer import inlineCallbacks, returnValue, succeed from twisted.internet.task import Clock from twisted.internet import reactor -from twisted.internet.endpoints import serverFromString -from twisted.python.filepath import FilePath from foolscap.api import Referenceable, RemoteException -from allmydata.interfaces import IStorageServer # really, IStorageClient +# A better name for this would be IStorageClient... +from allmydata.interfaces import IStorageServer + from .common_system import SystemTestMixin -from .common import AsyncTestCase, SameProcessStreamEndpointAssigner -from .certs import ( - generate_certificate, - generate_private_key, - private_key_to_file, - cert_to_file, -) +from .common import AsyncTestCase from allmydata.storage.server import StorageServer # not a IStorageServer!! -from allmydata.storage.http_server import HTTPServer, listen_tls from allmydata.storage.http_client import StorageClient from allmydata.storage_client import _HTTPStorageServer From 1cd2185be75e3d2e35307c43e59ab803ebb52ab3 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 29 Jul 2022 10:12:24 -0400 Subject: [PATCH 0987/2309] More cleanups. --- src/allmydata/protocol_switch.py | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/src/allmydata/protocol_switch.py b/src/allmydata/protocol_switch.py index f1fa6e061..2e9d404c5 100644 --- a/src/allmydata/protocol_switch.py +++ b/src/allmydata/protocol_switch.py @@ -48,21 +48,20 @@ class _FoolscapOrHttps(Protocol, metaclass=_PretendToBeNegotiation): since these are created by Foolscap's ``Tub``, by setting this to be the tub's ``negotiationClass``. - Do not use directly, use ``support_foolscap_and_https(tub)`` instead. The - way this class works is that a new subclass is created for a specific - ``Tub`` instance. + Do not instantiate directly, use ``support_foolscap_and_https(tub)`` + instead. The way this class works is that a new subclass is created for a + specific ``Tub`` instance. """ - # These will be set by support_foolscap_and_https() and add_storage_server(). + # These are class attributes; they will be set by + # support_foolscap_and_https() and add_storage_server(). - # The HTTP storage server API we're exposing. - http_storage_server: HTTPServer - # The Twisted HTTPS protocol factory wrapping the storage server API: + # The Twisted HTTPS protocol factory wrapping the storage server HTTP API: https_factory: TLSMemoryBIOFactory # The tub that created us: tub: Tub - # This will be created by the instance in connectionMade(): + # This is an instance attribute; it will be set in connectionMade(). _timeout: IDelayedCall @classmethod @@ -70,11 +69,17 @@ class _FoolscapOrHttps(Protocol, metaclass=_PretendToBeNegotiation): cls, storage_server: StorageServer, swissnum: bytes ) -> set[DecodedURL]: """ - Add the various storage server-related attributes needed by a - ``Tub``-specific ``_FoolscapOrHttps`` subclass. + Update a ``_FoolscapOrHttps`` subclass for a specific ``Tub`` instance + with the class attributes it requires for a specific storage server. Returns the resulting NURLs. """ + # We need to be a subclass: + assert cls != _FoolscapOrHttps + # The tub instance must already be set: + assert hasattr(cls, "tub") + assert isinstance(cls.tub, Tub) + # Tub.myCertificate is a twisted.internet.ssl.PrivateCertificate # instance. certificate_options = CertificateOptions( @@ -82,11 +87,11 @@ class _FoolscapOrHttps(Protocol, metaclass=_PretendToBeNegotiation): certificate=cls.tub.myCertificate.original, ) - cls.http_storage_server = HTTPServer(storage_server, swissnum) + http_storage_server = HTTPServer(storage_server, swissnum) cls.https_factory = TLSMemoryBIOFactory( certificate_options, False, - Site(cls.http_storage_server.get_resource()), + Site(http_storage_server.get_resource()), ) storage_nurls = set() From 533d2a7ac9576734825822f0621d8db58cb36606 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 29 Jul 2022 10:15:23 -0400 Subject: [PATCH 0988/2309] Note Tor and I2P support. --- src/allmydata/protocol_switch.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/allmydata/protocol_switch.py b/src/allmydata/protocol_switch.py index 2e9d404c5..5ab4761c6 100644 --- a/src/allmydata/protocol_switch.py +++ b/src/allmydata/protocol_switch.py @@ -107,6 +107,10 @@ class _FoolscapOrHttps(Protocol, metaclass=_PretendToBeNegotiation): cls.tub.myCertificate.original.to_cryptography(), ) ) + # TODO this is probably where we'll have to support Tor and I2P? + # See https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3888#comment:9 + # for discussion (there will be separate tickets added for those at + # some point.) return storage_nurls def __init__(self, *args, **kwargs): From d4c73f19fe6eaea7bba25c51e632aef441d7549e Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 29 Jul 2022 10:42:56 -0400 Subject: [PATCH 0989/2309] A unittest for the metaclass. --- src/allmydata/protocol_switch.py | 10 +++-- src/allmydata/test/test_protocol_switch.py | 43 ++++++++++++++++++++++ 2 files changed, 49 insertions(+), 4 deletions(-) create mode 100644 src/allmydata/test/test_protocol_switch.py diff --git a/src/allmydata/protocol_switch.py b/src/allmydata/protocol_switch.py index 5ab4761c6..5143cab6a 100644 --- a/src/allmydata/protocol_switch.py +++ b/src/allmydata/protocol_switch.py @@ -31,13 +31,15 @@ from .storage.server import StorageServer class _PretendToBeNegotiation(type): """ - Metaclass that allows ``_FoolscapOrHttps`` to pretend to be a ``Negotiation`` - instance, since Foolscap has some ``assert isinstance(protocol, - Negotiation`` checks. + Metaclass that allows ``_FoolscapOrHttps`` to pretend to be a + ``Negotiation`` instance, since Foolscap does some checks like + ``assert isinstance(protocol, tub.negotiationClass)`` in its internals, + and sometimes that ``protocol`` is a ``_FoolscapOrHttps`` instance, but + sometimes it's a ``Negotiation`` instance. """ def __instancecheck__(self, instance): - return (instance.__class__ == self) or isinstance(instance, Negotiation) + return issubclass(instance.__class__, self) or isinstance(instance, Negotiation) class _FoolscapOrHttps(Protocol, metaclass=_PretendToBeNegotiation): diff --git a/src/allmydata/test/test_protocol_switch.py b/src/allmydata/test/test_protocol_switch.py new file mode 100644 index 000000000..4906896dc --- /dev/null +++ b/src/allmydata/test/test_protocol_switch.py @@ -0,0 +1,43 @@ +""" +Unit tests for ``allmydata.protocol_switch``. + +By its nature, most of the testing needs to be end-to-end; essentially any test +that uses real Foolscap (``test_system.py``, integration tests) ensures +Foolscap still works. ``test_istorageserver.py`` tests the HTTP support. +""" + +from foolscap.negotiate import Negotiation + +from .common import TestCase +from ..protocol_switch import _PretendToBeNegotiation + + +class UtilityTests(TestCase): + """Tests for utilities in the protocol switch code.""" + + def test_metaclass(self): + """ + A class that has the ``_PretendToBeNegotiation`` metaclass will support + ``isinstance()``'s normal semantics on its own instances, but will also + indicate that ``Negotiation`` instances are its instances. + """ + + class Parent(metaclass=_PretendToBeNegotiation): + pass + + class Child(Parent): + pass + + class Other: + pass + + p = Parent() + self.assertIsInstance(p, Parent) + self.assertIsInstance(Negotiation(), Parent) + self.assertNotIsInstance(Other(), Parent) + + c = Child() + self.assertIsInstance(c, Child) + self.assertIsInstance(c, Parent) + self.assertIsInstance(Negotiation(), Child) + self.assertNotIsInstance(Other(), Child) From 8b3280bf319c68ba1eb957ee9048718070267c8d Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 29 Jul 2022 10:51:17 -0400 Subject: [PATCH 0990/2309] Simplify more. --- src/allmydata/protocol_switch.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/allmydata/protocol_switch.py b/src/allmydata/protocol_switch.py index 5143cab6a..158df32b5 100644 --- a/src/allmydata/protocol_switch.py +++ b/src/allmydata/protocol_switch.py @@ -63,9 +63,6 @@ class _FoolscapOrHttps(Protocol, metaclass=_PretendToBeNegotiation): # The tub that created us: tub: Tub - # This is an instance attribute; it will be set in connectionMade(). - _timeout: IDelayedCall - @classmethod def add_storage_server( cls, storage_server: StorageServer, swissnum: bytes @@ -117,7 +114,6 @@ class _FoolscapOrHttps(Protocol, metaclass=_PretendToBeNegotiation): def __init__(self, *args, **kwargs): self._foolscap: Negotiation = Negotiation(*args, **kwargs) - self._buffer: bytes = b"" def __setattr__(self, name, value): if name in {"_foolscap", "_buffer", "transport", "__class__", "_timeout"}: @@ -139,12 +135,15 @@ class _FoolscapOrHttps(Protocol, metaclass=_PretendToBeNegotiation): # After creation, a Negotiation instance either has initClient() or # initServer() called. Since this is a client, we're never going to do # HTTP, so we can immediately become a Negotiation instance. - assert not self._buffer + assert not hasattr(self, "_buffer") self._convert_to_negotiation() return self.initClient(*args, **kwargs) def connectionMade(self): - self._timeout = reactor.callLater(30, self.transport.abortConnection) + self._buffer: bytes = b"" + self._timeout: IDelayedCall = reactor.callLater( + 30, self.transport.abortConnection + ) def dataReceived(self, data: bytes) -> None: """Handle incoming data. From 709f139c85e00f452ca5dacb308fd3494eb1be4a Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 11 Aug 2022 15:51:30 -0400 Subject: [PATCH 0991/2309] Start refactoring to enable HTTP storage client. --- src/allmydata/storage_client.py | 183 ++++++++++++++++++++++++++------ 1 file changed, 151 insertions(+), 32 deletions(-) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index c63bfccff..a058ae828 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -30,6 +30,8 @@ Ported to Python 3. # # 6: implement other sorts of IStorageClient classes: S3, etc +from __future__ import annotations + from six import ensure_text from typing import Union import re, time, hashlib @@ -523,6 +525,45 @@ class IFoolscapStorageServer(Interface): """ +def _parse_announcement(server_id: bytes, furl: bytes, ann: dict) -> tuple[str, bytes, bytes, bytes, bytes]: + """ + Parse the furl and announcement, return: + + (nickname, permutation_seed, tubid, short_description, long_description) + """ + m = re.match(br'pb://(\w+)@', furl) + assert m, furl + tubid_s = m.group(1).lower() + tubid = base32.a2b(tubid_s) + if "permutation-seed-base32" in ann: + seed = ann["permutation-seed-base32"] + if isinstance(seed, str): + seed = seed.encode("utf-8") + ps = base32.a2b(seed) + elif re.search(br'^v0-[0-9a-zA-Z]{52}$', server_id): + ps = base32.a2b(server_id[3:]) + else: + log.msg("unable to parse serverid '%(server_id)s as pubkey, " + "hashing it to get permutation-seed, " + "may not converge with other clients", + server_id=server_id, + facility="tahoe.storage_broker", + level=log.UNUSUAL, umid="qu86tw") + ps = hashlib.sha256(server_id).digest() + permutation_seed = ps + + assert server_id + long_description = server_id + if server_id.startswith(b"v0-"): + # remove v0- prefix from abbreviated name + short_description = server_id[3:3+8] + else: + short_description = server_id[:8] + nickname = ann.get("nickname", "") + + return (nickname, permutation_seed, tubid, short_description, long_description) + + @implementer(IFoolscapStorageServer) @attr.s(frozen=True) class _FoolscapStorage(object): @@ -566,43 +607,13 @@ class _FoolscapStorage(object): The furl will be a Unicode string on Python 3; on Python 2 it will be either a native (bytes) string or a Unicode string. """ - furl = furl.encode("utf-8") - m = re.match(br'pb://(\w+)@', furl) - assert m, furl - tubid_s = m.group(1).lower() - tubid = base32.a2b(tubid_s) - if "permutation-seed-base32" in ann: - seed = ann["permutation-seed-base32"] - if isinstance(seed, str): - seed = seed.encode("utf-8") - ps = base32.a2b(seed) - elif re.search(br'^v0-[0-9a-zA-Z]{52}$', server_id): - ps = base32.a2b(server_id[3:]) - else: - log.msg("unable to parse serverid '%(server_id)s as pubkey, " - "hashing it to get permutation-seed, " - "may not converge with other clients", - server_id=server_id, - facility="tahoe.storage_broker", - level=log.UNUSUAL, umid="qu86tw") - ps = hashlib.sha256(server_id).digest() - permutation_seed = ps - - assert server_id - long_description = server_id - if server_id.startswith(b"v0-"): - # remove v0- prefix from abbreviated name - short_description = server_id[3:3+8] - else: - short_description = server_id[:8] - nickname = ann.get("nickname", "") - + (nickname, permutation_seed, tubid, short_description, long_description) = _parse_announcement(server_id, furl.encode("utf-8"), ann) return cls( nickname=nickname, permutation_seed=permutation_seed, tubid=tubid, storage_server=storage_server, - furl=furl, + furl=furl.encode("utf-8"), short_description=short_description, long_description=long_description, ) @@ -910,6 +921,114 @@ class NativeStorageServer(service.MultiService): # used when the broker wants us to hurry up self._reconnector.reset() + +@implementer(IServer) +class HTTPNativeStorageServer(service.MultiService): + """ + Like ``NativeStorageServer``, but for HTTP clients. + + The notion of being "connected" is less meaningful for HTTP; we just poll + occasionally, and if we've succeeded at last poll, we assume we're + "connected". + """ + + def __init__(self, server_id: bytes, announcement): + service.MultiService.__init__(self) + assert isinstance(server_id, bytes) + self._server_id = server_id + self.announcement = announcement + self._on_status_changed = ObserverList() + furl = announcement["anonymous-storage-FURL"].encode("utf-8") + self._nickname, self._permutation_seed, self._tubid, self._short_description, self._long_description = _parse_announcement(server_id, furl, announcement) + + def get_permutation_seed(self): + return self._permutation_seed + + def get_name(self): # keep methodname short + return self._name + + def get_longname(self): + return self._longname + + def get_tubid(self): + return self._tubid + + def get_lease_seed(self): + return self._lease_seed + + def get_foolscap_write_enabler_seed(self): + return self._tubid + + def get_nickname(self): + return self._nickname + + def on_status_changed(self, status_changed): + """ + :param status_changed: a callable taking a single arg (the + NativeStorageServer) that is notified when we become connected + """ + return self._on_status_changed.subscribe(status_changed) + + # Special methods used by copy.copy() and copy.deepcopy(). When those are + # used in allmydata.immutable.filenode to copy CheckResults during + # repair, we want it to treat the IServer instances as singletons, and + # not attempt to duplicate them.. + def __copy__(self): + return self + + def __deepcopy__(self, memodict): + return self + + def __repr__(self): + return "" % self.get_name() + + def get_serverid(self): + return self._server_id + + def get_version(self): + pass + + def get_announcement(self): + return self.announcement + + def get_connection_status(self): + pass + + def is_connected(self): + pass + + def get_available_space(self): + # TODO refactor into shared utility with NativeStorageServer + version = self.get_version() + if version is None: + return None + protocol_v1_version = version.get(b'http://allmydata.org/tahoe/protocols/storage/v1', BytesKeyDict()) + available_space = protocol_v1_version.get(b'available-space') + if available_space is None: + available_space = protocol_v1_version.get(b'maximum-immutable-share-size', None) + return available_space + + def start_connecting(self, trigger_cb): + pass + + def get_rref(self): + # TODO UH + pass + + def get_storage_server(self): + """ + See ``IServer.get_storage_server``. + """ + + def stop_connecting(self): + # used when this descriptor has been superceded by another + pass + + def try_to_connect(self): + # used when the broker wants us to hurry up + pass + + class UnknownServerTypeError(Exception): pass From c3e41588130e9d2c9d65965a9ea06d8f3503bd52 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 11 Aug 2022 15:55:14 -0400 Subject: [PATCH 0992/2309] Remove duplication. --- src/allmydata/storage_client.py | 27 ++++++++++++--------------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index a058ae828..e64f63413 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -695,6 +695,16 @@ def _storage_from_foolscap_plugin(node_config, config, announcement, get_rref): raise AnnouncementNotMatched() +def _available_space_from_version(version): + if version is None: + return None + protocol_v1_version = version.get(b'http://allmydata.org/tahoe/protocols/storage/v1', BytesKeyDict()) + available_space = protocol_v1_version.get(b'available-space') + if available_space is None: + available_space = protocol_v1_version.get(b'maximum-immutable-share-size', None) + return available_space + + @implementer(IServer) class NativeStorageServer(service.MultiService): """I hold information about a storage server that we want to connect to. @@ -853,13 +863,7 @@ class NativeStorageServer(service.MultiService): def get_available_space(self): version = self.get_version() - if version is None: - return None - protocol_v1_version = version.get(b'http://allmydata.org/tahoe/protocols/storage/v1', BytesKeyDict()) - available_space = protocol_v1_version.get(b'available-space') - if available_space is None: - available_space = protocol_v1_version.get(b'maximum-immutable-share-size', None) - return available_space + return _available_space_from_version(version) def start_connecting(self, trigger_cb): self._tub = self._tub_maker(self._handler_overrides) @@ -998,15 +1002,8 @@ class HTTPNativeStorageServer(service.MultiService): pass def get_available_space(self): - # TODO refactor into shared utility with NativeStorageServer version = self.get_version() - if version is None: - return None - protocol_v1_version = version.get(b'http://allmydata.org/tahoe/protocols/storage/v1', BytesKeyDict()) - available_space = protocol_v1_version.get(b'available-space') - if available_space is None: - available_space = protocol_v1_version.get(b'maximum-immutable-share-size', None) - return available_space + return _available_space_from_version(version) def start_connecting(self, trigger_cb): pass From c3b159a3fd98e63bcc0641c4c2a5dffe5e795a15 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 11 Aug 2022 16:12:57 -0400 Subject: [PATCH 0993/2309] Continue simplified sketch of HTTPNativeStorageServer. --- src/allmydata/storage_client.py | 32 ++++++++++++++++++++++++-------- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index e64f63413..3bcd8e6db 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -45,7 +45,7 @@ from zope.interface import ( implementer, ) from twisted.web import http -from twisted.internet import defer +from twisted.internet import defer, reactor from twisted.application import service from twisted.plugin import ( getPlugins, @@ -934,6 +934,9 @@ class HTTPNativeStorageServer(service.MultiService): The notion of being "connected" is less meaningful for HTTP; we just poll occasionally, and if we've succeeded at last poll, we assume we're "connected". + + TODO as first pass, just to get the proof-of-concept going, we will just + assume we're always connected after an initial successful HTTP request. """ def __init__(self, server_id: bytes, announcement): @@ -944,6 +947,13 @@ class HTTPNativeStorageServer(service.MultiService): self._on_status_changed = ObserverList() furl = announcement["anonymous-storage-FURL"].encode("utf-8") self._nickname, self._permutation_seed, self._tubid, self._short_description, self._long_description = _parse_announcement(server_id, furl, announcement) + self._istorage_server = _HTTPStorageServer.from_http_client( + StorageClient.from_nurl( + announcement["anonymous-storage-NURLs"][0], reactor + ) + ) + self._connection_status = connection_status.ConnectionStatus.unstarted() + self._version = None def get_permutation_seed(self): return self._permutation_seed @@ -984,29 +994,33 @@ class HTTPNativeStorageServer(service.MultiService): return self def __repr__(self): - return "" % self.get_name() + return "" % self.get_name() def get_serverid(self): return self._server_id def get_version(self): - pass + return self._version def get_announcement(self): return self.announcement def get_connection_status(self): - pass + return self._connection_status def is_connected(self): - pass + return self._connection_status.connected def get_available_space(self): version = self.get_version() return _available_space_from_version(version) def start_connecting(self, trigger_cb): - pass + self._istorage_server.get_version().addCallback(self._got_version) + + def _got_version(self, version): + self._version = version + self._connection_status = connection_status.ConnectionStatus(True, "connected", [], time.time(), time.time()) def get_rref(self): # TODO UH @@ -1016,13 +1030,15 @@ class HTTPNativeStorageServer(service.MultiService): """ See ``IServer.get_storage_server``. """ + if self.is_connected(): + return self._istorage_server + else: + return None def stop_connecting(self): - # used when this descriptor has been superceded by another pass def try_to_connect(self): - # used when the broker wants us to hurry up pass From 94be227aaaf7adfefc3457dbc7556b10c7b5f3c4 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 11 Aug 2022 16:15:21 -0400 Subject: [PATCH 0994/2309] Hopefully don't actually need that. --- src/allmydata/storage_client.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 3bcd8e6db..62cc047f2 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -1022,10 +1022,6 @@ class HTTPNativeStorageServer(service.MultiService): self._version = version self._connection_status = connection_status.ConnectionStatus(True, "connected", [], time.time(), time.time()) - def get_rref(self): - # TODO UH - pass - def get_storage_server(self): """ See ``IServer.get_storage_server``. From 9ad4e844e86302682dfd38e82b7e262231c21ad9 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 11 Aug 2022 16:16:17 -0400 Subject: [PATCH 0995/2309] Do status change notification. --- src/allmydata/storage_client.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 62cc047f2..254179559 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -937,6 +937,7 @@ class HTTPNativeStorageServer(service.MultiService): TODO as first pass, just to get the proof-of-concept going, we will just assume we're always connected after an initial successful HTTP request. + Might do polling as follow-up ticket, in which case add link to that here. """ def __init__(self, server_id: bytes, announcement): @@ -1021,6 +1022,7 @@ class HTTPNativeStorageServer(service.MultiService): def _got_version(self, version): self._version = version self._connection_status = connection_status.ConnectionStatus(True, "connected", [], time.time(), time.time()) + self._on_status_changed.notify(self) def get_storage_server(self): """ From f671fb04a18c5a3f20437eac3424db1f97fc5df4 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 11 Aug 2022 16:24:33 -0400 Subject: [PATCH 0996/2309] A lot closer to working end-to-end. --- src/allmydata/client.py | 6 +++--- src/allmydata/storage_client.py | 9 ++++++++- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/src/allmydata/client.py b/src/allmydata/client.py index 9938ec076..769554b3d 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -825,9 +825,9 @@ class _Client(node.Node, pollmixin.PollMixin): furl_file = self.config.get_private_path("storage.furl").encode(get_filesystem_encoding()) furl = self.tub.registerReference(FoolscapStorageServer(ss), furlFile=furl_file) (_, _, swissnum) = furl.rpartition("/") - self.storage_nurls.update( - self.tub.negotiationClass.add_storage_server(ss, swissnum.encode("ascii")) - ) + nurls = self.tub.negotiationClass.add_storage_server(ss, swissnum.encode("ascii")) + self.storage_nurls.update(nurls) + announcement["anonymous-storage-NURLs"] = [n.to_text() for n in nurls] announcement["anonymous-storage-FURL"] = furl enabled_storage_servers = self._enable_storage_servers( diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 254179559..ec03393a1 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -39,6 +39,7 @@ from os import urandom from configparser import NoSectionError import attr +from hyperlink import DecodedURL from zope.interface import ( Attribute, Interface, @@ -264,6 +265,12 @@ class StorageFarmBroker(service.MultiService): by the given announcement. """ assert isinstance(server_id, bytes) + # TODO use constant + if "anonymous-storage-NURLs" in server["ann"]: + print("HTTTTTTTPPPPPPPPPPPPPPPPPPPP") + s = HTTPNativeStorageServer(server_id, server["ann"]) + s.on_status_changed(lambda _: self._got_connection()) + return s handler_overrides = server.get("connections", {}) s = NativeStorageServer( server_id, @@ -950,7 +957,7 @@ class HTTPNativeStorageServer(service.MultiService): self._nickname, self._permutation_seed, self._tubid, self._short_description, self._long_description = _parse_announcement(server_id, furl, announcement) self._istorage_server = _HTTPStorageServer.from_http_client( StorageClient.from_nurl( - announcement["anonymous-storage-NURLs"][0], reactor + DecodedURL.from_text(announcement["anonymous-storage-NURLs"][0]), reactor ) ) self._connection_status = connection_status.ConnectionStatus.unstarted() From ad027aff7656abc098539f4159fa30638581ad35 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 12 Aug 2022 00:34:47 -0600 Subject: [PATCH 0997/2309] compare bytes to bytes --- src/allmydata/grid_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/grid_manager.py b/src/allmydata/grid_manager.py index f502c413e..d776fb7d0 100644 --- a/src/allmydata/grid_manager.py +++ b/src/allmydata/grid_manager.py @@ -466,7 +466,7 @@ def create_grid_manager_verifier(keys, certs, public_key, now_fn=None, bad_cert= now = now_fn() for cert in valid_certs: expires = datetime.utcfromtimestamp(cert['expires']) - if cert['public_key'].encode("ascii") == public_key: + if cert['public_key'] == public_key: if expires > now: # not-expired return True From cb065aefbd417cc1d546744ff746e25dbe35f999 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 12 Aug 2022 01:17:22 -0600 Subject: [PATCH 0998/2309] key is bytes --- src/allmydata/storage_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 7fe4d6bd2..49663f141 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -291,7 +291,7 @@ class StorageFarmBroker(service.MultiService): gm_verifier = create_grid_manager_verifier( self.storage_client_config.grid_manager_keys, [SignedCertificate.load(StringIO(json.dumps(data))) for data in server["ann"].get("grid-manager-certificates", [])], - "pub-{}".format(str(server_id, "ascii")), # server_id is v0- not pub-v0-key .. for reasons? + "pub-{}".format(str(server_id, "ascii")).encode("ascii"), # server_id is v0- not pub-v0-key .. for reasons? ) s = NativeStorageServer( From 4d779cfe0742fb9e3d75ca33fb3984d33e5e7586 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 12 Aug 2022 01:17:33 -0600 Subject: [PATCH 0999/2309] more assert --- src/allmydata/grid_manager.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/allmydata/grid_manager.py b/src/allmydata/grid_manager.py index d776fb7d0..0201eace5 100644 --- a/src/allmydata/grid_manager.py +++ b/src/allmydata/grid_manager.py @@ -466,7 +466,9 @@ def create_grid_manager_verifier(keys, certs, public_key, now_fn=None, bad_cert= now = now_fn() for cert in valid_certs: expires = datetime.utcfromtimestamp(cert['expires']) - if cert['public_key'] == public_key: + pc = cert['public_key'].encode('ascii') + assert type(pc) == type(public_key), "{} isn't {}".format(type(pc), type(public_key)) + if pc == public_key: if expires > now: # not-expired return True From 1676e9e7c5e6b9c4208c6b71e6379581ee17e047 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 12 Aug 2022 01:27:01 -0600 Subject: [PATCH 1000/2309] unused --- integration/test_servers_of_happiness.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/integration/test_servers_of_happiness.py b/integration/test_servers_of_happiness.py index 4cbb94654..3adc11340 100644 --- a/integration/test_servers_of_happiness.py +++ b/integration/test_servers_of_happiness.py @@ -13,8 +13,6 @@ if PY2: import sys from os.path import join -from twisted.internet.error import ProcessTerminated - from . import util import pytest_twisted From 9ff863e6cd47ad6c255491b2a6127b944b835f9a Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 12 Aug 2022 09:54:12 -0400 Subject: [PATCH 1001/2309] Fix lint. --- integration/test_servers_of_happiness.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/integration/test_servers_of_happiness.py b/integration/test_servers_of_happiness.py index 4cbb94654..b85eb8e5b 100644 --- a/integration/test_servers_of_happiness.py +++ b/integration/test_servers_of_happiness.py @@ -1,4 +1,4 @@ -""" +P""" Ported to Python 3. """ from __future__ import absolute_import @@ -13,8 +13,6 @@ if PY2: import sys from os.path import join -from twisted.internet.error import ProcessTerminated - from . import util import pytest_twisted From 0c6881e6150ffa590aec6775586de0d83d657017 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 12 Aug 2022 09:59:43 -0400 Subject: [PATCH 1002/2309] Fix race condition. --- integration/test_grid_manager.py | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/integration/test_grid_manager.py b/integration/test_grid_manager.py index 63ee827b0..866856be7 100644 --- a/integration/test_grid_manager.py +++ b/integration/test_grid_manager.py @@ -214,6 +214,9 @@ def test_reject_storage_server(reactor, request, temp_dir, flog_gatherer, port_a # re-start this storage server yield storage0.restart(reactor, request) + import time + time.sleep(1) + # now only one storage-server has the certificate .. configure # diana to have the grid-manager certificate @@ -231,15 +234,22 @@ def test_reject_storage_server(reactor, request, temp_dir, flog_gatherer, port_a # diana has happy=2 but should only find storage0 to be acceptable # to upload to) - try: - yield util.run_tahoe( - reactor, request, "--node-directory", diana.process.node_dir, - "put", "-", - stdin=b"some content\n" * 200, - ) - assert False, "Should get a failure" - except util.ProcessFailed as e: - assert b'UploadUnhappinessError' in e.output + # Takes a little bit of time for node to connect: + for i in range(10): + try: + yield util.run_tahoe( + reactor, request, "--node-directory", diana.process.node_dir, + "put", "-", + stdin=b"some content\n" * 200, + ) + assert False, "Should get a failure" + except util.ProcessFailed as e: + if b'UploadUnhappinessError' in e.output: + # We're done! We've succeeded. + return + time.sleep(0.2) + + assert False, "Failed to see one of out of two servers" @pytest_twisted.inlineCallbacks From 298600969af240a3caff30fb2f2735fa669606ee Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 12 Aug 2022 10:06:35 -0400 Subject: [PATCH 1003/2309] Fix typo. --- integration/test_servers_of_happiness.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration/test_servers_of_happiness.py b/integration/test_servers_of_happiness.py index b85eb8e5b..3adc11340 100644 --- a/integration/test_servers_of_happiness.py +++ b/integration/test_servers_of_happiness.py @@ -1,4 +1,4 @@ -P""" +""" Ported to Python 3. """ from __future__ import absolute_import From 09d778c2cfb4f888831835903daf6d77205ff5c7 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 12 Aug 2022 11:13:09 -0400 Subject: [PATCH 1004/2309] Allow nodes to disable the HTTPS storage protocol. --- src/allmydata/client.py | 7 ++++--- src/allmydata/node.py | 5 ++++- src/allmydata/storage_client.py | 4 ++-- src/allmydata/test/common_system.py | 20 +++++++++++++------- 4 files changed, 23 insertions(+), 13 deletions(-) diff --git a/src/allmydata/client.py b/src/allmydata/client.py index 769554b3d..d9fc20e92 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -825,9 +825,10 @@ class _Client(node.Node, pollmixin.PollMixin): furl_file = self.config.get_private_path("storage.furl").encode(get_filesystem_encoding()) furl = self.tub.registerReference(FoolscapStorageServer(ss), furlFile=furl_file) (_, _, swissnum) = furl.rpartition("/") - nurls = self.tub.negotiationClass.add_storage_server(ss, swissnum.encode("ascii")) - self.storage_nurls.update(nurls) - announcement["anonymous-storage-NURLs"] = [n.to_text() for n in nurls] + if hasattr(self.tub.negotiationClass, "add_storage_server"): + nurls = self.tub.negotiationClass.add_storage_server(ss, swissnum.encode("ascii")) + self.storage_nurls.update(nurls) + announcement["anonymous-storage-NURLs"] = [n.to_text() for n in nurls] announcement["anonymous-storage-FURL"] = furl enabled_storage_servers = self._enable_storage_servers( diff --git a/src/allmydata/node.py b/src/allmydata/node.py index 597221e9b..0ad68f2b7 100644 --- a/src/allmydata/node.py +++ b/src/allmydata/node.py @@ -64,6 +64,7 @@ def _common_valid_config(): "tcp", ), "node": ( + "force_foolscap", "log_gatherer.furl", "nickname", "reveal-ip-address", @@ -709,7 +710,6 @@ def create_tub(tub_options, default_connection_handlers, foolscap_connection_han the new Tub via `Tub.setOption` """ tub = Tub(**kwargs) - support_foolscap_and_https(tub) for (name, value) in list(tub_options.items()): tub.setOption(name, value) handlers = default_connection_handlers.copy() @@ -907,6 +907,9 @@ def create_main_tub(config, tub_options, handler_overrides=handler_overrides, certFile=certfile, ) + if not config.get_config("node", "force_foolscap", False): + support_foolscap_and_https(tub) + if portlocation is None: log.msg("Tub is not listening") else: diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index ec03393a1..3c2c7a1b8 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -102,8 +102,8 @@ class StorageClientConfig(object): :ivar preferred_peers: An iterable of the server-ids (``bytes``) of the storage servers where share placement is preferred, in order of - decreasing preference. See the *[client]peers.preferred* - documentation for details. + decreasing preference. See the *[client]peers.preferred* documentation + for details. :ivar dict[unicode, dict[unicode, unicode]] storage_plugins: A mapping from names of ``IFoolscapStoragePlugin`` configured in *tahoe.cfg* to the diff --git a/src/allmydata/test/common_system.py b/src/allmydata/test/common_system.py index 9851d2b91..75379bbf3 100644 --- a/src/allmydata/test/common_system.py +++ b/src/allmydata/test/common_system.py @@ -698,7 +698,7 @@ class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin): return f.read().strip() @inlineCallbacks - def set_up_nodes(self, NUMCLIENTS=5): + def set_up_nodes(self, NUMCLIENTS=5, force_foolscap=False): """ Create an introducer and ``NUMCLIENTS`` client nodes pointed at it. All of the nodes are running in this process. @@ -711,6 +711,9 @@ class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin): :param int NUMCLIENTS: The number of client nodes to create. + :param bool force_foolscap: Force clients to use Foolscap instead of e.g. + HTTPS when available. + :return: A ``Deferred`` that fires when the nodes have connected to each other. """ @@ -719,16 +722,16 @@ class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin): self.introducer = yield self._create_introducer() self.add_service(self.introducer) self.introweb_url = self._get_introducer_web() - yield self._set_up_client_nodes() + yield self._set_up_client_nodes(force_foolscap) @inlineCallbacks - def _set_up_client_nodes(self): + def _set_up_client_nodes(self, force_foolscap): q = self.introducer self.introducer_furl = q.introducer_url self.clients = [] basedirs = [] for i in range(self.numclients): - basedirs.append((yield self._set_up_client_node(i))) + basedirs.append((yield self._set_up_client_node(i, force_foolscap))) # start clients[0], wait for it's tub to be ready (at which point it # will have registered the helper furl). @@ -761,7 +764,7 @@ class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin): # and the helper-using webport self.helper_webish_url = self.clients[3].getServiceNamed("webish").getURL() - def _generate_config(self, which, basedir): + def _generate_config(self, which, basedir, force_foolscap=False): config = {} allclients = set(range(self.numclients)) @@ -787,6 +790,9 @@ class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin): if which in feature_matrix.get((section, feature), {which}): config.setdefault(section, {})[feature] = value + if force_foolscap: + config.setdefault("node", {})["force_foolscap"] = force_foolscap + setnode = partial(setconf, config, which, "node") sethelper = partial(setconf, config, which, "helper") @@ -811,14 +817,14 @@ class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin): return _render_config(config) - def _set_up_client_node(self, which): + def _set_up_client_node(self, which, force_foolscap): basedir = self.getdir("client%d" % (which,)) fileutil.make_dirs(os.path.join(basedir, "private")) if len(SYSTEM_TEST_CERTS) > (which + 1): f = open(os.path.join(basedir, "private", "node.pem"), "w") f.write(SYSTEM_TEST_CERTS[which + 1]) f.close() - config = self._generate_config(which, basedir) + config = self._generate_config(which, basedir, force_foolscap) fileutil.write(os.path.join(basedir, 'tahoe.cfg'), config) return basedir From e8609ac2df01038a7c51952149aaf4566b24e271 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 12 Aug 2022 11:24:41 -0400 Subject: [PATCH 1005/2309] test_istorageserver passes with both Foolscap and HTTP again. --- src/allmydata/storage_client.py | 14 +++++----- src/allmydata/test/test_istorageserver.py | 33 ++++++++++++----------- 2 files changed, 26 insertions(+), 21 deletions(-) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 3c2c7a1b8..e2a48e521 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -265,9 +265,8 @@ class StorageFarmBroker(service.MultiService): by the given announcement. """ assert isinstance(server_id, bytes) - # TODO use constant - if "anonymous-storage-NURLs" in server["ann"]: - print("HTTTTTTTPPPPPPPPPPPPPPPPPPPP") + # TODO use constant for anonymous-storage-NURLs + if len(server["ann"].get("anonymous-storage-NURLs", [])) > 0: s = HTTPNativeStorageServer(server_id, server["ann"]) s.on_status_changed(lambda _: self._got_connection()) return s @@ -955,10 +954,13 @@ class HTTPNativeStorageServer(service.MultiService): self._on_status_changed = ObserverList() furl = announcement["anonymous-storage-FURL"].encode("utf-8") self._nickname, self._permutation_seed, self._tubid, self._short_description, self._long_description = _parse_announcement(server_id, furl, announcement) + nurl = DecodedURL.from_text(announcement["anonymous-storage-NURLs"][0]) + # Tests don't want persistent HTTPS pool, since that leaves a dirty + # reactor. As a reasonable hack, disabling persistent connnections for + # localhost allows us to have passing tests while not reducing + # performance for real-world usage. self._istorage_server = _HTTPStorageServer.from_http_client( - StorageClient.from_nurl( - DecodedURL.from_text(announcement["anonymous-storage-NURLs"][0]), reactor - ) + StorageClient.from_nurl(nurl, reactor, nurl.host not in ("localhost", "127.0.0.1")) ) self._connection_status = connection_status.ConnectionStatus.unstarted() self._version = None diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index 3328ea598..81025d779 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -17,7 +17,6 @@ from unittest import SkipTest from twisted.internet.defer import inlineCallbacks, returnValue, succeed from twisted.internet.task import Clock -from twisted.internet import reactor from foolscap.api import Referenceable, RemoteException # A better name for this would be IStorageClient... @@ -26,8 +25,10 @@ from allmydata.interfaces import IStorageServer from .common_system import SystemTestMixin from .common import AsyncTestCase from allmydata.storage.server import StorageServer # not a IStorageServer!! -from allmydata.storage.http_client import StorageClient -from allmydata.storage_client import _HTTPStorageServer +from allmydata.storage_client import ( + NativeStorageServer, + HTTPNativeStorageServer, +) # Use random generator with known seed, so results are reproducible if tests @@ -1021,6 +1022,10 @@ class _SharedMixin(SystemTestMixin): """Base class for Foolscap and HTTP mixins.""" SKIP_TESTS = set() # type: Set[str] + FORCE_FOOLSCAP = False + + def _get_native_server(self): + return next(iter(self.clients[0].storage_broker.get_known_servers())) def _get_istorage_server(self): raise NotImplementedError("implement in subclass") @@ -1036,7 +1041,7 @@ class _SharedMixin(SystemTestMixin): self.basedir = "test_istorageserver/" + self.id() yield SystemTestMixin.setUp(self) - yield self.set_up_nodes(1) + yield self.set_up_nodes(1, self.FORCE_FOOLSCAP) self.server = None for s in self.clients[0].services: if isinstance(s, StorageServer): @@ -1065,11 +1070,12 @@ class _SharedMixin(SystemTestMixin): class _FoolscapMixin(_SharedMixin): """Run tests on Foolscap version of ``IStorageServer``.""" - def _get_native_server(self): - return next(iter(self.clients[0].storage_broker.get_known_servers())) + FORCE_FOOLSCAP = True def _get_istorage_server(self): - client = self._get_native_server().get_storage_server() + native_server = self._get_native_server() + assert isinstance(native_server, NativeStorageServer) + client = native_server.get_storage_server() self.assertTrue(IStorageServer.providedBy(client)) return succeed(client) @@ -1077,16 +1083,13 @@ class _FoolscapMixin(_SharedMixin): class _HTTPMixin(_SharedMixin): """Run tests on the HTTP version of ``IStorageServer``.""" + FORCE_FOOLSCAP = False + def _get_istorage_server(self): - nurl = list(self.clients[0].storage_nurls)[0] - - # Create HTTP client with non-persistent connections, so we don't leak - # state across tests: - client: IStorageServer = _HTTPStorageServer.from_http_client( - StorageClient.from_nurl(nurl, reactor, persistent=False) - ) + native_server = self._get_native_server() + assert isinstance(native_server, HTTPNativeStorageServer) + client = native_server.get_storage_server() self.assertTrue(IStorageServer.providedBy(client)) - return succeed(client) From 636b8a9e2de2504b347c5662a9b828636c120743 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 12 Aug 2022 11:28:03 -0400 Subject: [PATCH 1006/2309] Fix a bytes-vs-str bug. --- newsfragments/3913.minor | 0 src/allmydata/test/test_storage_web.py | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) create mode 100644 newsfragments/3913.minor diff --git a/newsfragments/3913.minor b/newsfragments/3913.minor new file mode 100644 index 000000000..e69de29bb diff --git a/src/allmydata/test/test_storage_web.py b/src/allmydata/test/test_storage_web.py index 5984b2892..b47c93849 100644 --- a/src/allmydata/test/test_storage_web.py +++ b/src/allmydata/test/test_storage_web.py @@ -161,7 +161,7 @@ class BucketCounter(unittest.TestCase, pollmixin.PollMixin): html = renderSynchronously(w) s = remove_tags(html) self.failUnlessIn(b"Total buckets: 0 (the number of", s) - self.failUnless(b"Next crawl in 59 minutes" in s or "Next crawl in 60 minutes" in s, s) + self.failUnless(b"Next crawl in 59 minutes" in s or b"Next crawl in 60 minutes" in s, s) d.addCallback(_check2) return d From 3fbc4d7eea3398075a6986416a585f993549cc65 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 12 Aug 2022 11:45:37 -0400 Subject: [PATCH 1007/2309] Let's make this a little clearer --- src/allmydata/scripts/tahoe_run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/scripts/tahoe_run.py b/src/allmydata/scripts/tahoe_run.py index 51be32ee3..dfdc97ea5 100644 --- a/src/allmydata/scripts/tahoe_run.py +++ b/src/allmydata/scripts/tahoe_run.py @@ -179,7 +179,7 @@ class DaemonizeTheRealService(Service, HookMixin): ) ) else: - self.stderr.write("\nUnknown error\n") + self.stderr.write("\nUnknown error, here's the traceback:\n") reason.printTraceback(self.stderr) reactor.stop() From 42e818f0a702738ceae40d033fa69d43a68e5657 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 12 Aug 2022 11:47:08 -0400 Subject: [PATCH 1008/2309] Refer to appropriate attributes, hopefully. --- src/allmydata/storage_client.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index e2a48e521..87041ff8b 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -968,17 +968,18 @@ class HTTPNativeStorageServer(service.MultiService): def get_permutation_seed(self): return self._permutation_seed - def get_name(self): # keep methodname short - return self._name + def get_name(self): + return self._short_description def get_longname(self): - return self._longname + return self._long_description def get_tubid(self): return self._tubid def get_lease_seed(self): - return self._lease_seed + # Apparently this is what Foolscap version above does?! + return self._tubid def get_foolscap_write_enabler_seed(self): return self._tubid From c4a32b65ff7ada59ed4de0324e2634f3dc50bc29 Mon Sep 17 00:00:00 2001 From: meejah Date: Sat, 13 Aug 2022 11:45:51 -0600 Subject: [PATCH 1009/2309] actually wait --- integration/grid.py | 1 + 1 file changed, 1 insertion(+) diff --git a/integration/grid.py b/integration/grid.py index 3cb16c929..eb25d9514 100644 --- a/integration/grid.py +++ b/integration/grid.py @@ -175,6 +175,7 @@ class StorageServer(object): reactor, self.process.node_dir, request, None, ) self.protocol = self.process.transport._protocol + yield await_client_ready(self.process) @inlineCallbacks From 06a5176626dbd14d41d8ab6c3307462be9cc279c Mon Sep 17 00:00:00 2001 From: meejah Date: Sat, 13 Aug 2022 11:46:02 -0600 Subject: [PATCH 1010/2309] happy-path grid-manager test --- integration/test_grid_manager.py | 70 ++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/integration/test_grid_manager.py b/integration/test_grid_manager.py index 63ee827b0..672700e15 100644 --- a/integration/test_grid_manager.py +++ b/integration/test_grid_manager.py @@ -242,6 +242,76 @@ def test_reject_storage_server(reactor, request, temp_dir, flog_gatherer, port_a assert b'UploadUnhappinessError' in e.output +@pytest_twisted.inlineCallbacks +def test_accept_storage_server(reactor, request, temp_dir, flog_gatherer, port_allocator): + """ + Successfully upload to a Grid Manager enabled Grid. + """ + grid = yield create_grid(reactor, request, temp_dir, flog_gatherer, port_allocator) + happy0 = yield grid.add_storage_node() + happy1 = yield grid.add_storage_node() + + gm_config = yield _run_gm( + reactor, "--config", "-", "create", + ) + gm_privkey_bytes = json.loads(gm_config)['private_key'].encode('ascii') + gm_privkey, gm_pubkey = ed25519.signing_keypair_from_string(gm_privkey_bytes) + + # create certificate for the storage-servers + servers = ( + ("happy0", happy0), + ("happy1", happy1), + ) + for st_name, st in servers: + pubkey_fname = join(st.process.node_dir, "node.pubkey") + with open(pubkey_fname, 'r') as f: + pubkey_str = f.read().strip() + + gm_config = yield _run_gm( + reactor, "--config", "-", "add", + st_name, pubkey_str, + stdinBytes=gm_config, + ) + assert json.loads(gm_config)['storage_servers'].keys() == {'happy0', 'happy1'} + + print("inserting certificates") + for st_name, st in servers: + cert = yield _run_gm( + reactor, "--config", "-", "sign", st_name, "1", + stdinBytes=gm_config, + ) + + yield util.run_tahoe( + reactor, request, "--node-directory", st.process.node_dir, + "admin", "add-grid-manager-cert", + "--name", "default", + "--filename", "-", + stdin=cert, + ) + + # re-start the storage servers + yield happy0.restart(reactor, request) + yield happy1.restart(reactor, request) + + # configure edna to have the grid-manager certificate + + edna = yield grid.add_client("edna", needed=2, happy=2, total=2) + + config = configutil.get_config(join(edna.process.node_dir, "tahoe.cfg")) + config.add_section("grid_managers") + config.set("grid_managers", "test", str(ed25519.string_from_verifying_key(gm_pubkey), "ascii")) + with open(join(edna.process.node_dir, "tahoe.cfg"), "w") as f: + config.write(f) + + yield edna.restart(reactor, request, servers=2) + + yield util.run_tahoe( + reactor, request, "--node-directory", edna.process.node_dir, + "put", "-", + stdin=b"some content\n" * 200, + ) + + @pytest_twisted.inlineCallbacks def test_identity(reactor, request, temp_dir): """ From 34dd39bfbf78e627a2ca05e457c444d0e0ee5e8e Mon Sep 17 00:00:00 2001 From: meejah Date: Sat, 13 Aug 2022 11:51:01 -0600 Subject: [PATCH 1011/2309] fix race with 'await_client_ready' instead --- integration/grid.py | 1 + integration/test_grid_manager.py | 28 +++++++++++----------------- 2 files changed, 12 insertions(+), 17 deletions(-) diff --git a/integration/grid.py b/integration/grid.py index eb25d9514..4e5d8a900 100644 --- a/integration/grid.py +++ b/integration/grid.py @@ -228,6 +228,7 @@ class Client(object): ) self.process = process self.protocol = self.process.transport._protocol + yield await_client_ready(self.process, minimum_number_of_servers=servers) # XXX add stop / start / restart diff --git a/integration/test_grid_manager.py b/integration/test_grid_manager.py index 35ea10c9f..b24149a3b 100644 --- a/integration/test_grid_manager.py +++ b/integration/test_grid_manager.py @@ -214,9 +214,6 @@ def test_reject_storage_server(reactor, request, temp_dir, flog_gatherer, port_a # re-start this storage server yield storage0.restart(reactor, request) - import time - time.sleep(1) - # now only one storage-server has the certificate .. configure # diana to have the grid-manager certificate @@ -234,20 +231,17 @@ def test_reject_storage_server(reactor, request, temp_dir, flog_gatherer, port_a # diana has happy=2 but should only find storage0 to be acceptable # to upload to) - # Takes a little bit of time for node to connect: - for i in range(10): - try: - yield util.run_tahoe( - reactor, request, "--node-directory", diana.process.node_dir, - "put", "-", - stdin=b"some content\n" * 200, - ) - assert False, "Should get a failure" - except util.ProcessFailed as e: - if b'UploadUnhappinessError' in e.output: - # We're done! We've succeeded. - return - time.sleep(0.2) + try: + yield util.run_tahoe( + reactor, request, "--node-directory", diana.process.node_dir, + "put", "-", + stdin=b"some content\n" * 200, + ) + assert False, "Should get a failure" + except util.ProcessFailed as e: + if b'UploadUnhappinessError' in e.output: + # We're done! We've succeeded. + return assert False, "Failed to see one of out of two servers" From 71b7e9b643930aa2504b1e38a131dd6def208a85 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 15 Aug 2022 10:08:50 -0400 Subject: [PATCH 1012/2309] Support comma-separated multi-location hints. --- src/allmydata/protocol_switch.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/allmydata/protocol_switch.py b/src/allmydata/protocol_switch.py index 158df32b5..89570436c 100644 --- a/src/allmydata/protocol_switch.py +++ b/src/allmydata/protocol_switch.py @@ -14,6 +14,8 @@ the configuration process. from __future__ import annotations +from itertools import chain + from twisted.internet.protocol import Protocol from twisted.internet.interfaces import IDelayedCall from twisted.internet.ssl import CertificateOptions @@ -94,7 +96,11 @@ class _FoolscapOrHttps(Protocol, metaclass=_PretendToBeNegotiation): ) storage_nurls = set() - for location_hint in cls.tub.locationHints: + # Individual hints can be in the form + # "tcp:host:port,tcp:host:port,tcp:host:port". + for location_hint in chain.from_iterable( + hints.split(",") for hints in cls.tub.locationHints + ): if location_hint.startswith("tcp:"): _, hostname, port = location_hint.split(":") port = int(port) From c1bcfab7f80d9a1f3e7b5f2e8c39dd292daedcd9 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 15 Aug 2022 11:38:02 -0400 Subject: [PATCH 1013/2309] Repeatedly poll status of server. --- src/allmydata/storage_client.py | 36 +++++++++++++++++++++++++-------- 1 file changed, 28 insertions(+), 8 deletions(-) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 87041ff8b..f9a6feb7d 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -46,6 +46,7 @@ from zope.interface import ( implementer, ) from twisted.web import http +from twisted.internet.task import LoopingCall from twisted.internet import defer, reactor from twisted.application import service from twisted.plugin import ( @@ -940,10 +941,6 @@ class HTTPNativeStorageServer(service.MultiService): The notion of being "connected" is less meaningful for HTTP; we just poll occasionally, and if we've succeeded at last poll, we assume we're "connected". - - TODO as first pass, just to get the proof-of-concept going, we will just - assume we're always connected after an initial successful HTTP request. - Might do polling as follow-up ticket, in which case add link to that here. """ def __init__(self, server_id: bytes, announcement): @@ -962,8 +959,10 @@ class HTTPNativeStorageServer(service.MultiService): self._istorage_server = _HTTPStorageServer.from_http_client( StorageClient.from_nurl(nurl, reactor, nurl.host not in ("localhost", "127.0.0.1")) ) + self._connection_status = connection_status.ConnectionStatus.unstarted() self._version = None + self._last_connect_time = None def get_permutation_seed(self): return self._permutation_seed @@ -1027,11 +1026,21 @@ class HTTPNativeStorageServer(service.MultiService): return _available_space_from_version(version) def start_connecting(self, trigger_cb): - self._istorage_server.get_version().addCallback(self._got_version) + self._lc = LoopingCall(self._connect) + self._lc.start(1, True) def _got_version(self, version): + self._last_connect_time = time.time() self._version = version - self._connection_status = connection_status.ConnectionStatus(True, "connected", [], time.time(), time.time()) + self._connection_status = connection_status.ConnectionStatus( + True, "connected", [], self._last_connect_time, self._last_connect_time + ) + self._on_status_changed.notify(self) + + def _failed_to_connect(self, reason): + self._connection_status = connection_status.ConnectionStatus( + False, f"failure: {reason}", [], self._last_connect_time, self._last_connect_time + ) self._on_status_changed.notify(self) def get_storage_server(self): @@ -1044,10 +1053,21 @@ class HTTPNativeStorageServer(service.MultiService): return None def stop_connecting(self): - pass + self._lc.stop() def try_to_connect(self): - pass + self._connect() + + def _connect(self): + return self._istorage_server.get_version().addCallbacks( + self._got_version, + self._failed_to_connect + ) + + def stopService(self): + service.MultiService.stopService(self) + self._lc.stop() + self._failed_to_connect("shut down") class UnknownServerTypeError(Exception): From 2e5662aa91cacf6ad36d1ea619ea08ea799591c7 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 16 Aug 2022 13:11:06 -0400 Subject: [PATCH 1014/2309] Temporarily enforce requirement that allocated size matches actual size of an immutable. --- src/allmydata/storage/immutable.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/allmydata/storage/immutable.py b/src/allmydata/storage/immutable.py index f7f5aebce..6fcca3871 100644 --- a/src/allmydata/storage/immutable.py +++ b/src/allmydata/storage/immutable.py @@ -419,14 +419,19 @@ class BucketWriter(object): self._already_written.set(True, offset, end) self.ss.add_latency("write", self._clock.seconds() - start) self.ss.count("write") + return self._is_finished() - # Return whether the whole thing has been written. See - # https://github.com/mlenzen/collections-extended/issues/169 and - # https://github.com/mlenzen/collections-extended/issues/172 for why - # it's done this way. + def _is_finished(self): + """ + Return whether the whole thing has been written. + """ return sum([mr.stop - mr.start for mr in self._already_written.ranges()]) == self._max_size def close(self): + # TODO this can't actually be enabled, because it's not backwards + # compatible. But it's useful for testing, so leaving it on until the + # branch is ready for merge. + assert self._is_finished() precondition(not self.closed) self._timeout.cancel() start = self._clock.seconds() From 556606271dbde1ccfe47ba29cb3767985286a7b4 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 16 Aug 2022 13:11:45 -0400 Subject: [PATCH 1015/2309] News file. --- newsfragments/3915.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3915.minor diff --git a/newsfragments/3915.minor b/newsfragments/3915.minor new file mode 100644 index 000000000..e69de29bb From d50c98a1e95b912c1cbd04d4bf932117176f5ac0 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 16 Aug 2022 14:34:40 -0400 Subject: [PATCH 1016/2309] Calculate URI extension size upfront, instead of hand-waving with a larger value. --- src/allmydata/immutable/encode.py | 18 ++++++++++++++++++ src/allmydata/immutable/layout.py | 16 +++++++--------- src/allmydata/immutable/upload.py | 27 ++++++++++++--------------- 3 files changed, 37 insertions(+), 24 deletions(-) diff --git a/src/allmydata/immutable/encode.py b/src/allmydata/immutable/encode.py index 42fc18077..c7887b7ba 100644 --- a/src/allmydata/immutable/encode.py +++ b/src/allmydata/immutable/encode.py @@ -624,6 +624,7 @@ class Encoder(object): for k in ('crypttext_root_hash', 'crypttext_hash', ): assert k in self.uri_extension_data + self.uri_extension_data uri_extension = uri.pack_extension(self.uri_extension_data) ed = {} for k,v in self.uri_extension_data.items(): @@ -694,3 +695,20 @@ class Encoder(object): return self.uri_extension_data def get_uri_extension_hash(self): return self.uri_extension_hash + + def get_uri_extension_size(self): + """ + Calculate the size of the URI extension that gets written at the end of + immutables. + + This may be done earlier than actual encoding, so e.g. we might not + know the crypttext hashes, but that's fine for our purposes since we + only care about the length. + """ + params = self.uri_extension_data.copy() + assert params + params["crypttext_hash"] = b"\x00" * 32 + params["crypttext_root_hash"] = b"\x00" * 32 + params["share_root_hash"] = b"\x00" * 32 + uri_extension = uri.pack_extension(params) + return len(uri_extension) diff --git a/src/allmydata/immutable/layout.py b/src/allmydata/immutable/layout.py index 79c886237..74af09a2b 100644 --- a/src/allmydata/immutable/layout.py +++ b/src/allmydata/immutable/layout.py @@ -90,7 +90,7 @@ FORCE_V2 = False # set briefly by unit tests to make small-sized V2 shares def make_write_bucket_proxy(rref, server, data_size, block_size, num_segments, - num_share_hashes, uri_extension_size_max): + num_share_hashes, uri_extension_size): # Use layout v1 for small files, so they'll be readable by older versions # (= 2**32 or data_size >= 2**32: @@ -233,8 +232,7 @@ class WriteBucketProxy(object): def put_uri_extension(self, data): offset = self._offsets['uri_extension'] assert isinstance(data, bytes) - precondition(len(data) <= self._uri_extension_size_max, - len(data), self._uri_extension_size_max) + precondition(len(data) == self._uri_extension_size) length = struct.pack(self.fieldstruct, len(data)) return self._write(offset, length+data) diff --git a/src/allmydata/immutable/upload.py b/src/allmydata/immutable/upload.py index cb332dfdf..6b9b48f6a 100644 --- a/src/allmydata/immutable/upload.py +++ b/src/allmydata/immutable/upload.py @@ -242,31 +242,26 @@ class UploadResults(object): def get_verifycapstr(self): return self._verifycapstr -# our current uri_extension is 846 bytes for small files, a few bytes -# more for larger ones (since the filesize is encoded in decimal in a -# few places). Ask for a little bit more just in case we need it. If -# the extension changes size, we can change EXTENSION_SIZE to -# allocate a more accurate amount of space. -EXTENSION_SIZE = 1000 -# TODO: actual extensions are closer to 419 bytes, so we can probably lower -# this. def pretty_print_shnum_to_servers(s): return ', '.join([ "sh%s: %s" % (k, '+'.join([idlib.shortnodeid_b2a(x) for x in v])) for k, v in s.items() ]) + class ServerTracker(object): def __init__(self, server, sharesize, blocksize, num_segments, num_share_hashes, storage_index, - bucket_renewal_secret, bucket_cancel_secret): + bucket_renewal_secret, bucket_cancel_secret, + uri_extension_size): self._server = server self.buckets = {} # k: shareid, v: IRemoteBucketWriter self.sharesize = sharesize + self.uri_extension_size = uri_extension_size wbp = layout.make_write_bucket_proxy(None, None, sharesize, blocksize, num_segments, num_share_hashes, - EXTENSION_SIZE) + uri_extension_size) self.wbp_class = wbp.__class__ # to create more of them self.allocated_size = wbp.get_allocated_size() self.blocksize = blocksize @@ -314,7 +309,7 @@ class ServerTracker(object): self.blocksize, self.num_segments, self.num_share_hashes, - EXTENSION_SIZE) + self.uri_extension_size) b[sharenum] = bp self.buckets.update(b) return (alreadygot, set(b.keys())) @@ -487,7 +482,7 @@ class Tahoe2ServerSelector(log.PrefixingLogMixin): def get_shareholders(self, storage_broker, secret_holder, storage_index, share_size, block_size, num_segments, total_shares, needed_shares, - min_happiness): + min_happiness, uri_extension_size): """ @return: (upload_trackers, already_serverids), where upload_trackers is a set of ServerTracker instances that have agreed to hold @@ -529,7 +524,8 @@ class Tahoe2ServerSelector(log.PrefixingLogMixin): # figure out how much space to ask for wbp = layout.make_write_bucket_proxy(None, None, share_size, 0, num_segments, - num_share_hashes, EXTENSION_SIZE) + num_share_hashes, + uri_extension_size) allocated_size = wbp.get_allocated_size() # decide upon the renewal/cancel secrets, to include them in the @@ -554,7 +550,7 @@ class Tahoe2ServerSelector(log.PrefixingLogMixin): def _create_server_tracker(server, renew, cancel): return ServerTracker( server, share_size, block_size, num_segments, num_share_hashes, - storage_index, renew, cancel, + storage_index, renew, cancel, uri_extension_size ) readonly_trackers, write_trackers = self._create_trackers( @@ -1326,7 +1322,8 @@ class CHKUploader(object): d = server_selector.get_shareholders(storage_broker, secret_holder, storage_index, share_size, block_size, - num_segments, n, k, desired) + num_segments, n, k, desired, + encoder.get_uri_extension_size()) def _done(res): self._server_selection_elapsed = time.time() - server_selection_started return res From 7aa97336a0fe7b8bda7de38e2c639b142e50f494 Mon Sep 17 00:00:00 2001 From: "Fon E. Noel NFEBE" Date: Wed, 17 Aug 2022 16:03:03 +0100 Subject: [PATCH 1017/2309] Refactor FakeWebTest & MemoryConsumerTest classes There are base test classes namely `SyncTestCase` and `AsyncTestCase` which we would like all test classes in this code base to extend. This commit refactors two test classes to use the `SyncTestCase` with the newer assert methods. Signed-off-by: Fon E. Noel NFEBE --- newsfragments/3916.minor | 0 src/allmydata/test/test_consumer.py | 24 +++++++++++++++--------- src/allmydata/test/test_testing.py | 7 ++++--- 3 files changed, 19 insertions(+), 12 deletions(-) create mode 100644 newsfragments/3916.minor diff --git a/newsfragments/3916.minor b/newsfragments/3916.minor new file mode 100644 index 000000000..e69de29bb diff --git a/src/allmydata/test/test_consumer.py b/src/allmydata/test/test_consumer.py index a689de462..234fc2594 100644 --- a/src/allmydata/test/test_consumer.py +++ b/src/allmydata/test/test_consumer.py @@ -14,11 +14,17 @@ if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from zope.interface import implementer -from twisted.trial.unittest import TestCase from twisted.internet.interfaces import IPushProducer, IPullProducer from allmydata.util.consumer import MemoryConsumer +from .common import ( + SyncTestCase, +) +from testtools.matchers import ( + Equals, +) + @implementer(IPushProducer) @implementer(IPullProducer) @@ -50,7 +56,7 @@ class Producer(object): self.consumer.unregisterProducer() -class MemoryConsumerTests(TestCase): +class MemoryConsumerTests(SyncTestCase): """Tests for MemoryConsumer.""" def test_push_producer(self): @@ -60,14 +66,14 @@ class MemoryConsumerTests(TestCase): consumer = MemoryConsumer() producer = Producer(consumer, [b"abc", b"def", b"ghi"]) consumer.registerProducer(producer, True) - self.assertEqual(consumer.chunks, [b"abc"]) + self.assertThat(consumer.chunks, Equals([b"abc"])) producer.iterate() producer.iterate() - self.assertEqual(consumer.chunks, [b"abc", b"def", b"ghi"]) - self.assertEqual(consumer.done, False) + self.assertThat(consumer.chunks, Equals([b"abc", b"def", b"ghi"])) + self.assertFalse(consumer.done) producer.iterate() - self.assertEqual(consumer.chunks, [b"abc", b"def", b"ghi"]) - self.assertEqual(consumer.done, True) + self.assertThat(consumer.chunks, Equals([b"abc", b"def", b"ghi"])) + self.assertTrue(consumer.done) def test_pull_producer(self): """ @@ -76,8 +82,8 @@ class MemoryConsumerTests(TestCase): consumer = MemoryConsumer() producer = Producer(consumer, [b"abc", b"def", b"ghi"]) consumer.registerProducer(producer, False) - self.assertEqual(consumer.chunks, [b"abc", b"def", b"ghi"]) - self.assertEqual(consumer.done, True) + self.assertThat(consumer.chunks, Equals([b"abc", b"def", b"ghi"])) + self.assertTrue(consumer.done) # download_to_data() is effectively tested by some of the filenode tests, e.g. diff --git a/src/allmydata/test/test_testing.py b/src/allmydata/test/test_testing.py index 527b235bd..3715d1aca 100644 --- a/src/allmydata/test/test_testing.py +++ b/src/allmydata/test/test_testing.py @@ -46,9 +46,10 @@ from hypothesis.strategies import ( binary, ) -from testtools import ( - TestCase, +from .common import ( + SyncTestCase, ) + from testtools.matchers import ( Always, Equals, @@ -61,7 +62,7 @@ from testtools.twistedsupport import ( ) -class FakeWebTest(TestCase): +class FakeWebTest(SyncTestCase): """ Test the WebUI verified-fakes infrastucture """ From c9084a2a45fb16cc90d4f6043017cbc57ba463a9 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 17 Aug 2022 12:49:06 -0400 Subject: [PATCH 1018/2309] Disable assertion we can't, sadly, enable. --- src/allmydata/storage/immutable.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/allmydata/storage/immutable.py b/src/allmydata/storage/immutable.py index 6fcca3871..a02fd3bb2 100644 --- a/src/allmydata/storage/immutable.py +++ b/src/allmydata/storage/immutable.py @@ -428,10 +428,9 @@ class BucketWriter(object): return sum([mr.stop - mr.start for mr in self._already_written.ranges()]) == self._max_size def close(self): - # TODO this can't actually be enabled, because it's not backwards - # compatible. But it's useful for testing, so leaving it on until the - # branch is ready for merge. - assert self._is_finished() + # This can't actually be enabled, because it's not backwards compatible + # with old Foolscap clients. + # assert self._is_finished() precondition(not self.closed) self._timeout.cancel() start = self._clock.seconds() From 9d03c476d196c78d7ca5ac36e57c3f1b6c1434b0 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 17 Aug 2022 12:49:45 -0400 Subject: [PATCH 1019/2309] Make sure we write all the bytes we say we're sending. --- src/allmydata/immutable/layout.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/allmydata/immutable/layout.py b/src/allmydata/immutable/layout.py index 74af09a2b..30ab985a8 100644 --- a/src/allmydata/immutable/layout.py +++ b/src/allmydata/immutable/layout.py @@ -118,6 +118,7 @@ class WriteBucketProxy(object): self._data_size = data_size self._block_size = block_size self._num_segments = num_segments + self._written_bytes = 0 effective_segments = mathutil.next_power_of_k(num_segments,2) self._segment_hash_size = (2*effective_segments - 1) * HASH_SIZE @@ -194,6 +195,11 @@ class WriteBucketProxy(object): return self._write(offset, data) def put_crypttext_hashes(self, hashes): + # plaintext_hash_tree precedes crypttext_hash_tree. It is not used, and + # so is not explicitly written, but we need to write everything, so + # fill it in with nulls. + self._write(self._offsets['plaintext_hash_tree'], b"\x00" * self._segment_hash_size) + offset = self._offsets['crypttext_hash_tree'] assert isinstance(hashes, list) data = b"".join(hashes) @@ -242,11 +248,12 @@ class WriteBucketProxy(object): # would reduce the foolscap CPU overhead per share, but wouldn't # reduce the number of round trips, so it might not be worth the # effort. - + self._written_bytes += len(data) return self._pipeline.add(len(data), self._rref.callRemote, "write", offset, data) def close(self): + assert self._written_bytes == self.get_allocated_size(), f"{self._written_bytes} != {self.get_allocated_size()}" d = self._pipeline.add(0, self._rref.callRemote, "close") d.addCallback(lambda ign: self._pipeline.flush()) return d From 3464637bbb1de4a739d87a14d95b0a300c326063 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 17 Aug 2022 12:54:26 -0400 Subject: [PATCH 1020/2309] Fix unit tests. --- src/allmydata/test/test_storage.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index c3f2a35e1..134609f81 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -463,7 +463,7 @@ class BucketProxy(unittest.TestCase): block_size=10, num_segments=5, num_share_hashes=3, - uri_extension_size_max=500) + uri_extension_size=500) self.failUnless(interfaces.IStorageBucketWriter.providedBy(bp), bp) def _do_test_readwrite(self, name, header_size, wbp_class, rbp_class): @@ -494,7 +494,7 @@ class BucketProxy(unittest.TestCase): block_size=25, num_segments=4, num_share_hashes=3, - uri_extension_size_max=len(uri_extension)) + uri_extension_size=len(uri_extension)) d = bp.put_header() d.addCallback(lambda res: bp.put_block(0, b"a"*25)) From cd81e5a01c82796fd3d69c93fb7f088ad6bf2a3b Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 17 Aug 2022 13:13:22 -0400 Subject: [PATCH 1021/2309] Hint for future debugging. --- src/allmydata/storage/immutable.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/allmydata/storage/immutable.py b/src/allmydata/storage/immutable.py index a02fd3bb2..0893513ae 100644 --- a/src/allmydata/storage/immutable.py +++ b/src/allmydata/storage/immutable.py @@ -397,7 +397,9 @@ class BucketWriter(object): """ Write data at given offset, return whether the upload is complete. """ - # Delay the timeout, since we received data: + # Delay the timeout, since we received data; if we get an + # AlreadyCancelled error, that means there's a bug in the client and + # write() was called after close(). self._timeout.reset(30 * 60) start = self._clock.seconds() precondition(not self.closed) From 92662d802cf0e43a5a5f684faba455de7a6cde53 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 17 Aug 2022 13:15:13 -0400 Subject: [PATCH 1022/2309] Don't drop a Deferred on the ground. --- src/allmydata/immutable/layout.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/allmydata/immutable/layout.py b/src/allmydata/immutable/layout.py index 30ab985a8..de390bda9 100644 --- a/src/allmydata/immutable/layout.py +++ b/src/allmydata/immutable/layout.py @@ -198,8 +198,11 @@ class WriteBucketProxy(object): # plaintext_hash_tree precedes crypttext_hash_tree. It is not used, and # so is not explicitly written, but we need to write everything, so # fill it in with nulls. - self._write(self._offsets['plaintext_hash_tree'], b"\x00" * self._segment_hash_size) + d = self._write(self._offsets['plaintext_hash_tree'], b"\x00" * self._segment_hash_size) + d.addCallback(lambda _: self._really_put_crypttext_hashes(hashes)) + return d + def _really_put_crypttext_hashes(self, hashes): offset = self._offsets['crypttext_hash_tree'] assert isinstance(hashes, list) data = b"".join(hashes) From bdb4aac0de1bd03fb8625e156135d4f0964e478c Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 17 Aug 2022 13:15:27 -0400 Subject: [PATCH 1023/2309] Pass in the missing argument. --- src/allmydata/test/test_upload.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/test_upload.py b/src/allmydata/test/test_upload.py index 8d5435e88..18192de6c 100644 --- a/src/allmydata/test/test_upload.py +++ b/src/allmydata/test/test_upload.py @@ -983,7 +983,7 @@ class EncodingParameters(GridTestMixin, unittest.TestCase, SetDEPMixin, num_segments = encoder.get_param("num_segments") d = selector.get_shareholders(broker, sh, storage_index, share_size, block_size, num_segments, - 10, 3, 4) + 10, 3, 4, encoder.get_uri_extension_size()) def _have_shareholders(upload_trackers_and_already_servers): (upload_trackers, already_servers) = upload_trackers_and_already_servers assert servers_to_break <= len(upload_trackers) From 488a04cb9b8af59171f02908fb4d6d00474865c9 Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 1 Sep 2022 17:42:06 -0600 Subject: [PATCH 1024/2309] exit when stdin closes --- src/allmydata/scripts/tahoe_run.py | 47 ++++++++++++++++++++++++++-- src/allmydata/test/test_runner.py | 49 ++++++++++++++++++++++++++++++ 2 files changed, 94 insertions(+), 2 deletions(-) diff --git a/src/allmydata/scripts/tahoe_run.py b/src/allmydata/scripts/tahoe_run.py index 51be32ee3..68578a2a1 100644 --- a/src/allmydata/scripts/tahoe_run.py +++ b/src/allmydata/scripts/tahoe_run.py @@ -20,7 +20,9 @@ from allmydata.scripts.common import BasedirOptions from twisted.scripts import twistd from twisted.python import usage from twisted.python.reflect import namedAny -from twisted.internet.defer import maybeDeferred +from twisted.internet.defer import maybeDeferred, Deferred +from twisted.internet.protocol import Protocol +from twisted.internet.stdio import StandardIO from twisted.application.service import Service from allmydata.scripts.default_nodedir import _default_nodedir @@ -148,6 +150,8 @@ class DaemonizeTheRealService(Service, HookMixin): def startService(self): + from twisted.internet import reactor + def start(): node_to_instance = { u"client": lambda: maybeDeferred(namedAny("allmydata.client.create_client"), self.basedir), @@ -187,12 +191,14 @@ class DaemonizeTheRealService(Service, HookMixin): def created(srv): srv.setServiceParent(self.parent) + # exiting on stdin-closed facilitates cleanup when run + # as a subprocess + on_stdin_close(reactor, reactor.stop) d.addCallback(created) d.addErrback(handle_config_error) d.addBoth(self._call_hook, 'running') return d - from twisted.internet import reactor reactor.callWhenRunning(start) @@ -206,6 +212,43 @@ class DaemonizeTahoeNodePlugin(object): return DaemonizeTheRealService(self.nodetype, self.basedir, so) +def on_stdin_close(reactor, fn): + """ + Arrange for the function `fn` to run when our stdin closes + """ + when_closed_d = Deferred() + + class WhenClosed(Protocol): + """ + Notify a Deferred when our connection is lost .. as this is passed + to twisted's StandardIO class, it is used to detect our parent + going away. + """ + + def connectionLost(self, reason): + when_closed_d.callback(None) + + def on_close(arg): + try: + fn() + except Exception: + # for our "exit" use-case, this will _mostly_ just be + # ReactorNotRunning (because we're already shutting down + # when our stdin closes) but no matter what "bad thing" + # happens we just want to ignore it. + pass + return arg + + when_closed_d.addBoth(on_close) + # we don't need to do anything with this instance because it gets + # hooked into the reactor and thus remembered + StandardIO( + proto=WhenClosed(), + reactor=reactor, + ) + return None + + def run(config, runApp=twistd.runApp): """ Runs a Tahoe-LAFS node in the foreground. diff --git a/src/allmydata/test/test_runner.py b/src/allmydata/test/test_runner.py index 3eb6b8a34..fdd31c37d 100644 --- a/src/allmydata/test/test_runner.py +++ b/src/allmydata/test/test_runner.py @@ -46,6 +46,9 @@ from twisted.internet.defer import ( inlineCallbacks, DeferredList, ) +from twisted.internet.testing import ( + MemoryReactorClock, +) from twisted.python.filepath import FilePath from twisted.python.runtime import ( platform, @@ -57,6 +60,9 @@ import allmydata from allmydata.scripts.runner import ( parse_options, ) +from allmydata.scripts.tahoe_run import ( + on_stdin_close, +) from .common import ( PIPE, @@ -621,3 +627,46 @@ class RunNode(common_util.SignalMixin, unittest.TestCase, pollmixin.PollMixin): # What's left is a perfect indicator that the process has exited and # we won't get blamed for leaving the reactor dirty. yield client_running + + +class OnStdinCloseTests(SyncTestCase): + """ + Tests for on_stdin_close + """ + + def test_close_called(self): + """ + our on-close method is called when stdin closes + """ + reactor = MemoryReactorClock() + called = [] + + def onclose(): + called.append(True) + on_stdin_close(reactor, onclose) + self.assertEqual(called, []) + + reader = list(reactor.readers)[0] + reader.loseConnection() + reactor.advance(1) # ProcessReader does a callLater(0, ..) + + self.assertEqual(called, [True]) + + def test_exception_ignored(self): + """ + an exception from or on-close function is ignored + """ + reactor = MemoryReactorClock() + called = [] + + def onclose(): + called.append(True) + raise RuntimeError("unexpected error") + on_stdin_close(reactor, onclose) + self.assertEqual(called, []) + + reader = list(reactor.readers)[0] + reader.loseConnection() + reactor.advance(1) # ProcessReader does a callLater(0, ..) + + self.assertEqual(called, [True]) From 1e6381ca7f45bea90dc3518abd7ad0c0f79d7670 Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 1 Sep 2022 17:44:01 -0600 Subject: [PATCH 1025/2309] news --- newsfragments/3921.feature | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 newsfragments/3921.feature diff --git a/newsfragments/3921.feature b/newsfragments/3921.feature new file mode 100644 index 000000000..f2c3a98bd --- /dev/null +++ b/newsfragments/3921.feature @@ -0,0 +1,5 @@ +Automatically exit when stdin is closed + +This facilitates subprocess management, specifically cleanup. +When a parent process is running tahoe and exits without time to do "proper" cleanup at least the stdin descriptor will be closed. +Subsequently "tahoe run" notices this and exits. \ No newline at end of file From 768829e993d957e6d4a78134fcd16cb5d2e92295 Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 1 Sep 2022 21:22:45 -0600 Subject: [PATCH 1026/2309] more robust --- src/allmydata/test/test_runner.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/test_runner.py b/src/allmydata/test/test_runner.py index fdd31c37d..8424bec6a 100644 --- a/src/allmydata/test/test_runner.py +++ b/src/allmydata/test/test_runner.py @@ -646,7 +646,8 @@ class OnStdinCloseTests(SyncTestCase): on_stdin_close(reactor, onclose) self.assertEqual(called, []) - reader = list(reactor.readers)[0] + for reader in reactor.getReaders(): + reader.loseConnection() reader.loseConnection() reactor.advance(1) # ProcessReader does a callLater(0, ..) @@ -665,7 +666,8 @@ class OnStdinCloseTests(SyncTestCase): on_stdin_close(reactor, onclose) self.assertEqual(called, []) - reader = list(reactor.readers)[0] + for reader in reactor.getReaders(): + reader.loseConnection() reader.loseConnection() reactor.advance(1) # ProcessReader does a callLater(0, ..) From 00c785ec7697167cbe8c37b9f50bd9be95690395 Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 1 Sep 2022 21:47:28 -0600 Subject: [PATCH 1027/2309] debug windows --- src/allmydata/test/test_runner.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/allmydata/test/test_runner.py b/src/allmydata/test/test_runner.py index 8424bec6a..74d7ac59f 100644 --- a/src/allmydata/test/test_runner.py +++ b/src/allmydata/test/test_runner.py @@ -646,6 +646,8 @@ class OnStdinCloseTests(SyncTestCase): on_stdin_close(reactor, onclose) self.assertEqual(called, []) + print("READERS", reactor.getReaders()) + for reader in reactor.getReaders(): reader.loseConnection() reader.loseConnection() From decb36a8f6f4d755b51c6f2b2e624d77dcf31899 Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 1 Sep 2022 22:20:07 -0600 Subject: [PATCH 1028/2309] refactor for Windows testing --- src/allmydata/scripts/tahoe_run.py | 6 +++--- src/allmydata/test/test_runner.py | 27 ++++++++++++++++----------- 2 files changed, 19 insertions(+), 14 deletions(-) diff --git a/src/allmydata/scripts/tahoe_run.py b/src/allmydata/scripts/tahoe_run.py index 68578a2a1..63dc351b1 100644 --- a/src/allmydata/scripts/tahoe_run.py +++ b/src/allmydata/scripts/tahoe_run.py @@ -241,12 +241,12 @@ def on_stdin_close(reactor, fn): when_closed_d.addBoth(on_close) # we don't need to do anything with this instance because it gets - # hooked into the reactor and thus remembered - StandardIO( + # hooked into the reactor and thus remembered .. but we return it + # for Windows testing purposes. + return StandardIO( proto=WhenClosed(), reactor=reactor, ) - return None def run(config, runApp=twistd.runApp): diff --git a/src/allmydata/test/test_runner.py b/src/allmydata/test/test_runner.py index 74d7ac59f..fc3ed9618 100644 --- a/src/allmydata/test/test_runner.py +++ b/src/allmydata/test/test_runner.py @@ -643,15 +643,18 @@ class OnStdinCloseTests(SyncTestCase): def onclose(): called.append(True) - on_stdin_close(reactor, onclose) + proto = on_stdin_close(reactor, onclose) self.assertEqual(called, []) - print("READERS", reactor.getReaders()) - - for reader in reactor.getReaders(): - reader.loseConnection() - reader.loseConnection() - reactor.advance(1) # ProcessReader does a callLater(0, ..) + # one Unix we can just close all the readers, correctly + # "simulating" a stdin close .. of course, Windows has to be + # difficult + if platform.isWindows(): + proto.loseConnection() + else: + for reader in reactor.getReaders(): + reader.loseConnection() + reactor.advance(1) # ProcessReader does a callLater(0, ..) self.assertEqual(called, [True]) @@ -668,9 +671,11 @@ class OnStdinCloseTests(SyncTestCase): on_stdin_close(reactor, onclose) self.assertEqual(called, []) - for reader in reactor.getReaders(): - reader.loseConnection() - reader.loseConnection() - reactor.advance(1) # ProcessReader does a callLater(0, ..) + if platform.isWindows(): + proto.loseConnection() + else: + for reader in reactor.getReaders(): + reader.loseConnection() + reactor.advance(1) # ProcessReader does a callLater(0, ..) self.assertEqual(called, [True]) From 711f6d39e7281cad447fd51ae06c16e8d3247384 Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 1 Sep 2022 22:29:19 -0600 Subject: [PATCH 1029/2309] missing proto --- src/allmydata/test/test_runner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/test_runner.py b/src/allmydata/test/test_runner.py index fc3ed9618..c4bdee3fb 100644 --- a/src/allmydata/test/test_runner.py +++ b/src/allmydata/test/test_runner.py @@ -668,7 +668,7 @@ class OnStdinCloseTests(SyncTestCase): def onclose(): called.append(True) raise RuntimeError("unexpected error") - on_stdin_close(reactor, onclose) + proto = on_stdin_close(reactor, onclose) self.assertEqual(called, []) if platform.isWindows(): From 869b15803c506256004fd47d6c72d5a8f61e0267 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 6 Sep 2022 08:46:09 -0400 Subject: [PATCH 1030/2309] assorted fixes --- docs/proposed/http-storage-node-protocol.rst | 52 +++++++++++--------- docs/specifications/url.rst | 2 + 2 files changed, 31 insertions(+), 23 deletions(-) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index 3dac376ff..8fe855be3 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -30,12 +30,12 @@ Glossary introducer a Tahoe-LAFS process at a known location configured to re-publish announcements about the location of storage servers - fURL + `fURL `_ a self-authenticating URL-like string which can be used to locate a remote object using the Foolscap protocol (the storage service is an example of such an object) - NURL - a self-authenticating URL-like string almost exactly like a NURL but without being tied to Foolscap + `NURL `_ + a self-authenticating URL-like string almost exactly like a fURL but without being tied to Foolscap swissnum a short random string which is part of a fURL/NURL and which acts as a shared secret to authorize clients to use a storage service @@ -580,24 +580,6 @@ Responses: the response is ``CONFLICT``. At this point the only thing to do is abort the upload and start from scratch (see below). -``PUT /v1/immutable/:storage_index/:share_number/abort`` -!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - -This cancels an *in-progress* upload. - -The request must include a ``X-Tahoe-Authorization`` header that includes the upload secret:: - - X-Tahoe-Authorization: upload-secret - -The response code: - -* When the upload is still in progress and therefore the abort has succeeded, - the response is ``OK``. - Future uploads can start from scratch with no pre-existing upload state stored on the server. -* If the uploaded has already finished, the response is 405 (Method Not Allowed) - and no change is made. - - Discussion `````````` @@ -616,6 +598,24 @@ From RFC 7231:: PATCH method defined in [RFC5789]). +``PUT /v1/immutable/:storage_index/:share_number/abort`` +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + +This cancels an *in-progress* upload. + +The request must include a ``X-Tahoe-Authorization`` header that includes the upload secret:: + + X-Tahoe-Authorization: upload-secret + +The response code: + +* When the upload is still in progress and therefore the abort has succeeded, + the response is ``OK``. + Future uploads can start from scratch with no pre-existing upload state stored on the server. +* If the uploaded has already finished, the response is 405 (Method Not Allowed) + and no change is made. + + ``POST /v1/immutable/:storage_index/:share_number/corrupt`` !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! @@ -625,7 +625,7 @@ corruption. It also includes potentially important details about the share. For example:: - {"reason": u"expected hash abcd, got hash efgh"} + {"reason": "expected hash abcd, got hash efgh"} .. share-type, storage-index, and share-number are inferred from the URL @@ -799,6 +799,7 @@ Immutable Data 200 OK + { "required": [ {"begin": 16, "end": 48 } ] } PATCH /v1/immutable/AAAAAAAAAAAAAAAA/7 Authorization: Tahoe-LAFS nurl-swissnum @@ -807,6 +808,7 @@ Immutable Data 200 OK + { "required": [ {"begin": 32, "end": 48 } ] } PATCH /v1/immutable/AAAAAAAAAAAAAAAA/7 Authorization: Tahoe-LAFS nurl-swissnum @@ -823,6 +825,7 @@ Immutable Data Range: bytes=0-47 200 OK + Content-Range: bytes 0-47/48 #. Renew the lease on all immutable shares in bucket ``AAAAAAAAAAAAAAAA``:: @@ -906,9 +909,12 @@ otherwise it will read a byte which won't match `b""`:: #. Download the contents of share number ``3``:: - GET /v1/mutable/BBBBBBBBBBBBBBBB?share=3&offset=0&size=10 + GET /v1/mutable/BBBBBBBBBBBBBBBB?share=3 Authorization: Tahoe-LAFS nurl-swissnum + Range: bytes=0-16 + 200 OK + Content-Range: bytes 0-15/16 #. Renew the lease on previously uploaded mutable share in slot ``BBBBBBBBBBBBBBBB``:: diff --git a/docs/specifications/url.rst b/docs/specifications/url.rst index 31fb05fad..421ac57f7 100644 --- a/docs/specifications/url.rst +++ b/docs/specifications/url.rst @@ -10,6 +10,8 @@ The intended audience for this document is Tahoe-LAFS maintainers and other deve Background ---------- +.. _fURLs: + Tahoe-LAFS first used Foolscap_ for network communication. Foolscap connection setup takes as an input a Foolscap URL or a *fURL*. A fURL includes three components: From 3b9eea5b8f1278bad158df336a9f617f8dee7895 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 6 Sep 2022 08:46:52 -0400 Subject: [PATCH 1031/2309] news fragment --- newsfragments/3922.documentation | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/3922.documentation diff --git a/newsfragments/3922.documentation b/newsfragments/3922.documentation new file mode 100644 index 000000000..d0232dd02 --- /dev/null +++ b/newsfragments/3922.documentation @@ -0,0 +1 @@ +Several minor errors in the Great Black Swamp proposed specification document have been fixed. \ No newline at end of file From 48283ea6f871925268638e392b0984c9163812a0 Mon Sep 17 00:00:00 2001 From: "Fon E. Noel NFEBE" Date: Wed, 7 Sep 2022 22:35:57 +0100 Subject: [PATCH 1032/2309] Refactor test_storage.py There are base test classes namely `SyncTestCase` and `AsyncTestCase` which we would like all test classes in this code base to extend. This commit extends the listed classes in test_storage.py to extend the above mentioned base classes: * UtilTests * BucketProxy * Server Signed-off-by: Fon E. Noel NFEBE --- newsfragments/3917.minor | 0 src/allmydata/test/test_storage.py | 75 +++++++++++++++++------------- 2 files changed, 43 insertions(+), 32 deletions(-) create mode 100644 newsfragments/3917.minor diff --git a/newsfragments/3917.minor b/newsfragments/3917.minor new file mode 100644 index 000000000..e69de29bb diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index c3f2a35e1..3f33d82ab 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -25,8 +25,16 @@ import shutil from functools import partial from uuid import uuid4 +from .common import ( + SyncTestCase, + AsyncTestCase, +) + from testtools.matchers import ( + Equals, + Contains, HasLength, + IsInstance, ) from twisted.trial import unittest @@ -92,23 +100,23 @@ from .strategies import ( ) -class UtilTests(unittest.TestCase): +class UtilTests(SyncTestCase): """Tests for allmydata.storage.common and .shares.""" def test_encoding(self): """b2a/a2b are the same as base32.""" s = b"\xFF HELLO \xF3" result = si_b2a(s) - self.assertEqual(base32.b2a(s), result) - self.assertEqual(si_a2b(result), s) + self.assertThat(base32.b2a(s), Equals(result)) + self.assertThat(si_a2b(result), Equals(s)) def test_storage_index_to_dir(self): """storage_index_to_dir creates a native string path.""" s = b"\xFF HELLO \xF3" path = storage_index_to_dir(s) parts = os.path.split(path) - self.assertEqual(parts[0], parts[1][:2]) - self.assertIsInstance(path, native_str) + self.assertThat(parts[0], Equals(parts[1][:2])) + self.assertThat(path, IsInstance(native_str)) def test_get_share_file_mutable(self): """A mutable share is identified by get_share_file().""" @@ -116,16 +124,16 @@ class UtilTests(unittest.TestCase): msf = MutableShareFile(path) msf.create(b"12", b"abc") # arbitrary values loaded = get_share_file(path) - self.assertIsInstance(loaded, MutableShareFile) - self.assertEqual(loaded.home, path) + self.assertThat(loaded, IsInstance(MutableShareFile)) + self.assertThat(loaded.home, Equals(path)) def test_get_share_file_immutable(self): """An immutable share is identified by get_share_file().""" path = self.mktemp() _ = ShareFile(path, max_size=1000, create=True) loaded = get_share_file(path) - self.assertIsInstance(loaded, ShareFile) - self.assertEqual(loaded.home, path) + self.assertThat(loaded, IsInstance(ShareFile)) + self.assertThat(loaded.home, Equals(path)) class FakeStatsProvider(object): @@ -135,7 +143,7 @@ class FakeStatsProvider(object): pass -class Bucket(unittest.TestCase): +class Bucket(SyncTestCase): def make_workdir(self, name): basedir = os.path.join("storage", "Bucket", name) incoming = os.path.join(basedir, "tmp", "bucket") @@ -178,9 +186,9 @@ class Bucket(unittest.TestCase): # now read from it br = BucketReader(self, bw.finalhome) - self.failUnlessEqual(br.read(0, 25), b"a"*25) - self.failUnlessEqual(br.read(25, 25), b"b"*25) - self.failUnlessEqual(br.read(50, 7), b"c"*7) + self.assertThat(br.read(0, 25), Equals(b"a"*25)) + self.assertThat(br.read(25, 25), Equals(b"b"*25)) + self.assertThat(br.read(50, 7), Equals(b"c"*7)) def test_write_past_size_errors(self): """Writing beyond the size of the bucket throws an exception.""" @@ -430,7 +438,7 @@ class RemoteBucket(object): return defer.maybeDeferred(_call) -class BucketProxy(unittest.TestCase): +class BucketProxy(SyncTestCase): def make_bucket(self, name, size): basedir = os.path.join("storage", "BucketProxy", name) incoming = os.path.join(basedir, "tmp", "bucket") @@ -513,7 +521,7 @@ class BucketProxy(unittest.TestCase): rb = RemoteBucket(FoolscapBucketReader(br)) server = NoNetworkServer(b"abc", None) rbp = rbp_class(rb, server, storage_index=b"") - self.failUnlessIn("to peer", repr(rbp)) + self.assertThat(repr(rbp), Contains("to peer")) self.failUnless(interfaces.IStorageBucketReader.providedBy(rbp), rbp) d1 = rbp.get_block_data(0, 25, 25) @@ -550,13 +558,16 @@ class BucketProxy(unittest.TestCase): return self._do_test_readwrite("test_readwrite_v2", 0x44, WriteBucketProxy_v2, ReadBucketProxy) -class Server(unittest.TestCase): +class Server(AsyncTestCase): def setUp(self): + super(Server, self).setUp() self.sparent = LoggingServiceParent() self.sparent.startService() self._lease_secret = itertools.count() + def tearDown(self): + super(Server, self).tearDown() return self.sparent.stopService() def workdir(self, name): @@ -586,14 +597,14 @@ class Server(unittest.TestCase): ss = self.create("test_declares_maximum_share_sizes") ver = ss.get_version() sv1 = ver[b'http://allmydata.org/tahoe/protocols/storage/v1'] - self.failUnlessIn(b'maximum-immutable-share-size', sv1) - self.failUnlessIn(b'maximum-mutable-share-size', sv1) + self.assertThat(sv1, Contains(b'maximum-immutable-share-size')) + self.assertThat(sv1, Contains(b'maximum-mutable-share-size')) def test_declares_available_space(self): ss = self.create("test_declares_available_space") ver = ss.get_version() sv1 = ver[b'http://allmydata.org/tahoe/protocols/storage/v1'] - self.failUnlessIn(b'available-space', sv1) + self.assertThat(sv1, Contains(b'available-space')) def allocate(self, ss, storage_index, sharenums, size, renew_leases=True): """ @@ -725,8 +736,8 @@ class Server(unittest.TestCase): self.failUnlessEqual(set(b.keys()), set([0,1,2])) self.failUnlessEqual(b[0].read(0, 25), b"%25d" % 0) b_str = str(b[0]) - self.failUnlessIn("BucketReader", b_str) - self.failUnlessIn("mfwgy33dmf2g 0", b_str) + self.assertThat(b_str, Contains("BucketReader")) + self.assertThat(b_str, Contains("mfwgy33dmf2g 0")) # now if we ask about writing again, the server should offer those # three buckets as already present. It should offer them even if we @@ -1216,21 +1227,21 @@ class Server(unittest.TestCase): b"This share smells funny.\n") reportdir = os.path.join(workdir, "corruption-advisories") reports = os.listdir(reportdir) - self.failUnlessEqual(len(reports), 1) + self.assertThat(reports, HasLength(1)) report_si0 = reports[0] - self.failUnlessIn(ensure_str(si0_s), report_si0) + self.assertThat(report_si0, Contains(ensure_str(si0_s))) f = open(os.path.join(reportdir, report_si0), "rb") report = f.read() f.close() - self.failUnlessIn(b"type: immutable", report) - self.failUnlessIn(b"storage_index: %s" % si0_s, report) - self.failUnlessIn(b"share_number: 0", report) - self.failUnlessIn(b"This share smells funny.", report) + self.assertThat(report, Contains(b"type: immutable")) + self.assertThat(report, Contains(b"storage_index: %s" % si0_s)) + self.assertThat(report, Contains(b"share_number: 0")) + self.assertThat(report, Contains(b"This share smells funny.")) # test the RIBucketWriter version too si1_s = base32.b2a(b"si1") already,writers = self.allocate(ss, b"si1", [1], 75) - self.failUnlessEqual(already, set()) + self.assertThat(already, Equals(set())) self.failUnlessEqual(set(writers.keys()), set([1])) writers[1].write(0, b"data") writers[1].close() @@ -1245,10 +1256,10 @@ class Server(unittest.TestCase): f = open(os.path.join(reportdir, report_si1), "rb") report = f.read() f.close() - self.failUnlessIn(b"type: immutable", report) - self.failUnlessIn(b"storage_index: %s" % si1_s, report) - self.failUnlessIn(b"share_number: 1", report) - self.failUnlessIn(b"This share tastes like dust.", report) + self.assertThat(report, Contains(b"type: immutable")) + self.assertThat(report, Contains(b"storage_index: %s" % si1_s)) + self.assertThat(report, Contains(b"share_number: 1")) + self.assertThat(report, Contains(b"This share tastes like dust.")) def test_advise_corruption_missing(self): """ From 9975fddd88d263a58e146b91b2c22b5b53500a85 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 8 Sep 2022 13:42:19 -0400 Subject: [PATCH 1033/2309] Get rid of garbage. --- src/allmydata/immutable/encode.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/allmydata/immutable/encode.py b/src/allmydata/immutable/encode.py index c7887b7ba..34a9c2472 100644 --- a/src/allmydata/immutable/encode.py +++ b/src/allmydata/immutable/encode.py @@ -624,7 +624,6 @@ class Encoder(object): for k in ('crypttext_root_hash', 'crypttext_hash', ): assert k in self.uri_extension_data - self.uri_extension_data uri_extension = uri.pack_extension(self.uri_extension_data) ed = {} for k,v in self.uri_extension_data.items(): From c82bb5f21c90e293c1507c71aa68fb4768b3abb6 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 8 Sep 2022 13:44:22 -0400 Subject: [PATCH 1034/2309] Use a more meaningful constant. --- src/allmydata/immutable/encode.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/allmydata/immutable/encode.py b/src/allmydata/immutable/encode.py index 34a9c2472..3c4440486 100644 --- a/src/allmydata/immutable/encode.py +++ b/src/allmydata/immutable/encode.py @@ -706,8 +706,8 @@ class Encoder(object): """ params = self.uri_extension_data.copy() assert params - params["crypttext_hash"] = b"\x00" * 32 - params["crypttext_root_hash"] = b"\x00" * 32 - params["share_root_hash"] = b"\x00" * 32 + params["crypttext_hash"] = b"\x00" * hashutil.CRYPTO_VAL_SIZE + params["crypttext_root_hash"] = b"\x00" * hashutil.CRYPTO_VAL_SIZE + params["share_root_hash"] = b"\x00" * hashutil.CRYPTO_VAL_SIZE uri_extension = uri.pack_extension(params) return len(uri_extension) From 6310774b8267d2deb0620c1766bc7f93694a3803 Mon Sep 17 00:00:00 2001 From: Florian Sesser Date: Thu, 8 Sep 2022 17:50:58 +0000 Subject: [PATCH 1035/2309] Add documentation on OpenMetrics statistics endpoint. references ticket:3786 --- docs/stats.rst | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/docs/stats.rst b/docs/stats.rst index 50642d816..c7d69e0d2 100644 --- a/docs/stats.rst +++ b/docs/stats.rst @@ -264,3 +264,18 @@ the "tahoe-conf" file for notes about configuration and installing these plugins into a Munin environment. .. _Munin: http://munin-monitoring.org/ + + +Scraping Stats Values in OpenMetrics Format +=========================================== + +Time Series DataBase (TSDB) software like Prometheus_ and VictoriaMetrics_ can +parse statistics from the e.g. http://localhost:3456/statistics?t=openmetrics +URL in OpenMetrics_ format. Software like Grafana_ can then be used to graph +and alert on these numbers. You can find a pre-configured dashboard for +Grafana at https://grafana.com/grafana/dashboards/16894-tahoe-lafs/. + +.. _OpenMetrics: https://openmetrics.io/ +.. _Prometheus: https://prometheus.io/ +.. _VictoriaMetrics: https://victoriametrics.com/ +.. _Grafana: https://grafana.com/ From ae21ab74a2afb8a0db234e76e2d09e76df0f5958 Mon Sep 17 00:00:00 2001 From: Florian Sesser Date: Thu, 8 Sep 2022 18:05:59 +0000 Subject: [PATCH 1036/2309] Add newsfragment for the added documentation. --- newsfragments/3786.minor | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/3786.minor diff --git a/newsfragments/3786.minor b/newsfragments/3786.minor new file mode 100644 index 000000000..ecd1a2c4e --- /dev/null +++ b/newsfragments/3786.minor @@ -0,0 +1 @@ +Added re-structured text documentation for the OpenMetrics format statistics endpoint. From 1058e50c50f47635646c13d20b264d1579cf3de4 Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 8 Sep 2022 16:30:30 -0600 Subject: [PATCH 1037/2309] close properly --- src/allmydata/test/test_runner.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/allmydata/test/test_runner.py b/src/allmydata/test/test_runner.py index c4bdee3fb..bce5b3c20 100644 --- a/src/allmydata/test/test_runner.py +++ b/src/allmydata/test/test_runner.py @@ -646,11 +646,12 @@ class OnStdinCloseTests(SyncTestCase): proto = on_stdin_close(reactor, onclose) self.assertEqual(called, []) - # one Unix we can just close all the readers, correctly + # on Unix we can just close all the readers, correctly # "simulating" a stdin close .. of course, Windows has to be # difficult if platform.isWindows(): - proto.loseConnection() + proto.writeConnectionLost() + proto.readConnectionLost() else: for reader in reactor.getReaders(): reader.loseConnection() @@ -672,7 +673,8 @@ class OnStdinCloseTests(SyncTestCase): self.assertEqual(called, []) if platform.isWindows(): - proto.loseConnection() + proto.writeConnectionLost() + proto.readConnectionLost() else: for reader in reactor.getReaders(): reader.loseConnection() From fbc8baa238f72720cfa840a9c227c670a5e2fa6e Mon Sep 17 00:00:00 2001 From: "Fon E. Noel NFEBE" Date: Wed, 14 Sep 2022 22:55:31 +0100 Subject: [PATCH 1038/2309] Refactor Server class in test_storage.py As a follow up to commit: 48283ea6f871925268638e392b0984c9163812a0 this refactor adds better methods and cleans up the test to be consistent with methods that used in classes that extend the `AsyncTestCase`. Signed-off-by: Fon E. Noel NFEBE --- src/allmydata/test/test_storage.py | 166 ++++++++++++++--------------- 1 file changed, 82 insertions(+), 84 deletions(-) diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index 3f33d82ab..d50ae1c18 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -25,13 +25,9 @@ import shutil from functools import partial from uuid import uuid4 -from .common import ( - SyncTestCase, - AsyncTestCase, -) - from testtools.matchers import ( Equals, + NotEquals, Contains, HasLength, IsInstance, @@ -88,7 +84,9 @@ from .common import ( ShouldFailMixin, FakeDisk, SyncTestCase, + AsyncTestCase, ) + from .common_util import FakeCanary from .common_storage import ( upload_immutable, @@ -346,16 +344,16 @@ class Bucket(SyncTestCase): # Now read from it. br = BucketReader(mockstorageserver, final) - self.failUnlessEqual(br.read(0, len(share_data)), share_data) + self.assertThat(br.read(0, len(share_data)), Equals(share_data)) # Read past the end of share data to get the cancel secret. read_length = len(share_data) + len(ownernumber) + len(renewsecret) + len(cancelsecret) result_of_read = br.read(0, read_length) - self.failUnlessEqual(result_of_read, share_data) + self.assertThat(result_of_read, Equals(share_data)) result_of_read = br.read(0, len(share_data)+1) - self.failUnlessEqual(result_of_read, share_data) + self.assertThat(result_of_read, Equals(share_data)) def _assert_timeout_only_after_30_minutes(self, clock, bw): """ @@ -591,7 +589,7 @@ class Server(AsyncTestCase): ss = self.create("test_declares_fixed_1528") ver = ss.get_version() sv1 = ver[b'http://allmydata.org/tahoe/protocols/storage/v1'] - self.failUnless(sv1.get(b'prevents-read-past-end-of-share-data'), sv1) + self.assertTrue(sv1.get(b'prevents-read-past-end-of-share-data'), sv1) def test_declares_maximum_share_sizes(self): ss = self.create("test_declares_maximum_share_sizes") @@ -634,8 +632,8 @@ class Server(AsyncTestCase): ss = self.create("test_large_share") already,writers = self.allocate(ss, b"allocate", [0], 2**32+2) - self.failUnlessEqual(already, set()) - self.failUnlessEqual(set(writers.keys()), set([0])) + self.assertThat(set(), Equals(already)) + self.assertThat(set([0]), Equals(set(writers.keys()))) shnum, bucket = list(writers.items())[0] # This test is going to hammer your filesystem if it doesn't make a sparse file for this. :-( @@ -644,7 +642,7 @@ class Server(AsyncTestCase): readers = ss.get_buckets(b"allocate") reader = readers[shnum] - self.failUnlessEqual(reader.read(2**32, 2), b"ab") + self.assertThat(b"ab", Equals(reader.read(2**32, 2))) def test_dont_overfill_dirs(self): """ @@ -670,7 +668,7 @@ class Server(AsyncTestCase): storedir = os.path.join(self.workdir("test_dont_overfill_dirs"), "shares") new_children_of_storedir = set(os.listdir(storedir)) - self.failUnlessEqual(children_of_storedir, new_children_of_storedir) + self.assertThat(new_children_of_storedir, Equals(children_of_storedir)) def test_remove_incoming(self): ss = self.create("test_remove_incoming") @@ -682,9 +680,9 @@ class Server(AsyncTestCase): incoming_bucket_dir = os.path.dirname(incoming_share_dir) incoming_prefix_dir = os.path.dirname(incoming_bucket_dir) incoming_dir = os.path.dirname(incoming_prefix_dir) - self.failIf(os.path.exists(incoming_bucket_dir), incoming_bucket_dir) - self.failIf(os.path.exists(incoming_prefix_dir), incoming_prefix_dir) - self.failUnless(os.path.exists(incoming_dir), incoming_dir) + self.assertFalse(os.path.exists(incoming_bucket_dir), incoming_bucket_dir) + self.assertFalse(os.path.exists(incoming_prefix_dir), incoming_prefix_dir) + self.assertTrue(os.path.exists(incoming_dir), incoming_dir) def test_abort(self): # remote_abort, when called on a writer, should make sure that @@ -692,12 +690,12 @@ class Server(AsyncTestCase): # server when accounting for space. ss = self.create("test_abort") already, writers = self.allocate(ss, b"allocate", [0, 1, 2], 150) - self.failIfEqual(ss.allocated_size(), 0) + self.assertThat(ss.allocated_size(), NotEquals(0)) # Now abort the writers. for writer in writers.values(): writer.abort() - self.failUnlessEqual(ss.allocated_size(), 0) + self.assertThat(ss.allocated_size(), Equals(0)) def test_immutable_length(self): """ @@ -709,20 +707,20 @@ class Server(AsyncTestCase): bucket = writers[22] bucket.write(0, b"X" * 75) bucket.close() - self.assertEqual(ss.get_immutable_share_length(b"allocate", 22), 75) - self.assertEqual(ss.get_buckets(b"allocate")[22].get_length(), 75) + self.assertThat(ss.get_immutable_share_length(b"allocate", 22), Equals(75)) + self.assertThat(ss.get_buckets(b"allocate")[22].get_length(), Equals(75)) def test_allocate(self): ss = self.create("test_allocate") - self.failUnlessEqual(ss.get_buckets(b"allocate"), {}) + self.assertThat(ss.get_buckets(b"allocate"), Equals({})) already,writers = self.allocate(ss, b"allocate", [0,1,2], 75) - self.failUnlessEqual(already, set()) - self.failUnlessEqual(set(writers.keys()), set([0,1,2])) + self.assertThat(already, Equals(set())) + self.assertThat(set(writers.keys()), Equals(set([0,1,2]))) # while the buckets are open, they should not count as readable - self.failUnlessEqual(ss.get_buckets(b"allocate"), {}) + self.assertThat(ss.get_buckets(b"allocate"), Equals({})) # close the buckets for i,wb in writers.items(): @@ -733,8 +731,8 @@ class Server(AsyncTestCase): # now they should be readable b = ss.get_buckets(b"allocate") - self.failUnlessEqual(set(b.keys()), set([0,1,2])) - self.failUnlessEqual(b[0].read(0, 25), b"%25d" % 0) + self.assertThat(set(b.keys()), Equals(set([0,1,2]))) + self.assertThat(b[0].read(0, 25), Equals(b"%25d" % 0)) b_str = str(b[0]) self.assertThat(b_str, Contains("BucketReader")) self.assertThat(b_str, Contains("mfwgy33dmf2g 0")) @@ -743,22 +741,22 @@ class Server(AsyncTestCase): # three buckets as already present. It should offer them even if we # don't ask about those specific ones. already,writers = self.allocate(ss, b"allocate", [2,3,4], 75) - self.failUnlessEqual(already, set([0,1,2])) - self.failUnlessEqual(set(writers.keys()), set([3,4])) + self.assertThat(already, Equals(set([0,1,2]))) + self.assertThat(set(writers.keys()), Equals(set([3,4]))) # while those two buckets are open for writing, the server should # refuse to offer them to uploaders already2,writers2 = self.allocate(ss, b"allocate", [2,3,4,5], 75) - self.failUnlessEqual(already2, set([0,1,2])) - self.failUnlessEqual(set(writers2.keys()), set([5])) + self.assertThat(already2, Equals(set([0,1,2]))) + self.assertThat(set(writers2.keys()), Equals(set([5]))) # aborting the writes should remove the tempfiles for i,wb in writers2.items(): wb.abort() already2,writers2 = self.allocate(ss, b"allocate", [2,3,4,5], 75) - self.failUnlessEqual(already2, set([0,1,2])) - self.failUnlessEqual(set(writers2.keys()), set([5])) + self.assertThat(already2, Equals(set([0,1,2]))) + self.assertThat(set(writers2.keys()), Equals(set([5]))) for i,wb in writers2.items(): wb.abort() @@ -814,13 +812,13 @@ class Server(AsyncTestCase): # The first share's lease expiration time is unchanged. shares = dict(ss.get_shares(storage_index)) - self.assertEqual( + self.assertThat( [first_lease], - list( + Equals(list( lease.get_grant_renew_time_time() for lease in ShareFile(shares[0]).get_leases() - ), + )), ) def test_bad_container_version(self): @@ -839,9 +837,9 @@ class Server(AsyncTestCase): e = self.failUnlessRaises(UnknownImmutableContainerVersionError, ss.get_buckets, b"si1") - self.assertEqual(e.filename, fn) - self.assertEqual(e.version, 0) - self.assertIn("had unexpected version 0", str(e)) + self.assertThat(e.filename, Equals(fn)) + self.assertThat(e.version, Equals(0)) + self.assertThat(str(e), Contains("had unexpected version 0")) def test_disconnect(self): # simulate a disconnection @@ -857,8 +855,8 @@ class Server(AsyncTestCase): allocated_size=75, canary=canary, ) - self.failUnlessEqual(already, set()) - self.failUnlessEqual(set(writers.keys()), set([0,1,2])) + self.assertThat(already, Equals(set())) + self.assertThat(set(writers.keys()), Equals(set([0,1,2]))) for (f,args,kwargs) in list(canary.disconnectors.values()): f(*args, **kwargs) del already @@ -866,8 +864,8 @@ class Server(AsyncTestCase): # that ought to delete the incoming shares already,writers = self.allocate(ss, b"disconnect", [0,1,2], 75) - self.failUnlessEqual(already, set()) - self.failUnlessEqual(set(writers.keys()), set([0,1,2])) + self.assertThat(already, Equals(set())) + self.assertThat(set(writers.keys()), Equals(set([0,1,2]))) def test_reserved_space_immutable_lease(self): """ @@ -965,22 +963,22 @@ class Server(AsyncTestCase): allocated_size=1000, canary=canary, ) - self.failUnlessEqual(len(writers), 3) + self.assertThat(writers, HasLength(3)) # now the StorageServer should have 3000 bytes provisionally # allocated, allowing only 2000 more to be claimed - self.failUnlessEqual(len(ss._server._bucket_writers), 3) + self.assertThat(ss._server._bucket_writers, HasLength(3)) # allocating 1001-byte shares only leaves room for one canary2 = FakeCanary() already2, writers2 = self.allocate(ss, b"vid2", [0,1,2], 1001, canary2) - self.failUnlessEqual(len(writers2), 1) - self.failUnlessEqual(len(ss._server._bucket_writers), 4) + self.assertThat(writers2, HasLength(1)) + self.assertThat(ss._server._bucket_writers, HasLength(4)) # we abandon the first set, so their provisional allocation should be # returned canary.disconnected() - self.failUnlessEqual(len(ss._server._bucket_writers), 1) + self.assertThat(ss._server._bucket_writers, HasLength(1)) # now we have a provisional allocation of 1001 bytes # and we close the second set, so their provisional allocation should @@ -989,7 +987,7 @@ class Server(AsyncTestCase): for bw in writers2.values(): bw.write(0, b"a"*25) bw.close() - self.failUnlessEqual(len(ss._server._bucket_writers), 0) + self.assertThat(ss._server._bucket_writers, HasLength(0)) # this also changes the amount reported as available by call_get_disk_stats allocated = 1001 + OVERHEAD + LEASE_SIZE @@ -1005,12 +1003,12 @@ class Server(AsyncTestCase): allocated_size=100, canary=canary3, ) - self.failUnlessEqual(len(writers3), 39) - self.failUnlessEqual(len(ss._server._bucket_writers), 39) + self.assertThat(writers3, HasLength(39)) + self.assertThat(ss._server._bucket_writers, HasLength(39)) canary3.disconnected() - self.failUnlessEqual(len(ss._server._bucket_writers), 0) + self.assertThat(ss._server._bucket_writers, HasLength(0)) ss._server.disownServiceParent() del ss @@ -1029,9 +1027,9 @@ class Server(AsyncTestCase): f.write(b"100") f.close() filelen = os.stat(filename)[stat.ST_SIZE] - self.failUnlessEqual(filelen, 100+3) + self.assertThat(filelen, Equals(100+3)) f2 = open(filename, "rb") - self.failUnlessEqual(f2.read(5), b"start") + self.assertThat(f2.read(5), Equals(b"start")) def create_bucket_5_shares( self, ss, storage_index, expected_already=0, expected_writers=5 @@ -1048,8 +1046,8 @@ class Server(AsyncTestCase): hashutil.my_cancel_secret_hash(b"%d" % next(self._lease_secret))) already, writers = ss.allocate_buckets(storage_index, rs, cs, sharenums, size) - self.failUnlessEqual(len(already), expected_already) - self.failUnlessEqual(len(writers), expected_writers) + self.assertThat(already, HasLength(expected_already)) + self.assertThat(writers, HasLength(expected_writers)) for wb in writers.values(): wb.close() return rs, cs @@ -1085,11 +1083,11 @@ class Server(AsyncTestCase): self.assertTrue(lease3.is_renew_secret(rs2a)) # add-lease on a missing storage index is silently ignored - self.assertIsNone(ss.add_lease(b"si18", b"", b"")) + self.assertThat(ss.add_lease(b"si18", b"", b""), Equals(None)) # check that si0 is readable readers = ss.get_buckets(b"si0") - self.failUnlessEqual(len(readers), 5) + self.assertThat(readers, HasLength(5)) # renew the first lease. Only the proper renew_secret should work ss.renew_lease(b"si0", rs0) @@ -1098,11 +1096,11 @@ class Server(AsyncTestCase): # check that si0 is still readable readers = ss.get_buckets(b"si0") - self.failUnlessEqual(len(readers), 5) + self.assertThat(readers, HasLength(5)) # There is no such method as remote_cancel_lease for now -- see # ticket #1528. - self.failIf(hasattr(FoolscapStorageServer(ss), 'remote_cancel_lease'), \ + self.assertFalse(hasattr(FoolscapStorageServer(ss), 'remote_cancel_lease'), \ "ss should not have a 'remote_cancel_lease' method/attribute") # test overlapping uploads @@ -1112,25 +1110,25 @@ class Server(AsyncTestCase): hashutil.my_cancel_secret_hash(b"%d" % next(self._lease_secret))) already,writers = ss.allocate_buckets(b"si3", rs3, cs3, sharenums, size) - self.failUnlessEqual(len(already), 0) - self.failUnlessEqual(len(writers), 5) + self.assertThat(already, HasLength(0)) + self.assertThat(writers, HasLength(5)) already2,writers2 = ss.allocate_buckets(b"si3", rs4, cs4, sharenums, size) - self.failUnlessEqual(len(already2), 0) - self.failUnlessEqual(len(writers2), 0) + self.assertThat(already2, HasLength(0)) + self.assertThat(writers2, HasLength(0)) for wb in writers.values(): wb.close() leases = list(ss.get_leases(b"si3")) - self.failUnlessEqual(len(leases), 1) + self.assertThat(leases, HasLength(1)) already3,writers3 = ss.allocate_buckets(b"si3", rs4, cs4, sharenums, size) - self.failUnlessEqual(len(already3), 5) - self.failUnlessEqual(len(writers3), 0) + self.assertThat(already3, HasLength(5)) + self.assertThat(writers3, HasLength(0)) leases = list(ss.get_leases(b"si3")) - self.failUnlessEqual(len(leases), 2) + self.assertThat(leases, HasLength(2)) def test_immutable_add_lease_renews(self): """ @@ -1144,7 +1142,7 @@ class Server(AsyncTestCase): # Start out with single lease created with bucket: renewal_secret, cancel_secret = self.create_bucket_5_shares(ss, b"si0") [lease] = ss.get_leases(b"si0") - self.assertEqual(lease.get_expiration_time(), 123 + DEFAULT_RENEWAL_TIME) + self.assertThat(lease.get_expiration_time(), Equals(123 + DEFAULT_RENEWAL_TIME)) # Time passes: clock.advance(123456) @@ -1152,7 +1150,7 @@ class Server(AsyncTestCase): # Adding a lease with matching renewal secret just renews it: ss.add_lease(b"si0", renewal_secret, cancel_secret) [lease] = ss.get_leases(b"si0") - self.assertEqual(lease.get_expiration_time(), 123 + 123456 + DEFAULT_RENEWAL_TIME) + self.assertThat(lease.get_expiration_time(), Equals(123 + 123456 + DEFAULT_RENEWAL_TIME)) def test_have_shares(self): """By default the StorageServer has no shares.""" @@ -1166,15 +1164,15 @@ class Server(AsyncTestCase): ss.setServiceParent(self.sparent) already,writers = self.allocate(ss, b"vid", [0,1,2], 75) - self.failUnlessEqual(already, set()) - self.failUnlessEqual(writers, {}) + self.assertThat(already, Equals(set())) + self.assertThat(writers, Equals({})) stats = ss.get_stats() - self.failUnlessEqual(stats["storage_server.accepting_immutable_shares"], 0) + self.assertThat(stats["storage_server.accepting_immutable_shares"], Equals(0)) if "storage_server.disk_avail" in stats: # Some platforms may not have an API to get disk stats. # But if there are stats, readonly_storage means disk_avail=0 - self.failUnlessEqual(stats["storage_server.disk_avail"], 0) + self.assertThat(stats["storage_server.disk_avail"], Equals(0)) def test_discard(self): # discard is really only used for other tests, but we test it anyways @@ -1183,8 +1181,8 @@ class Server(AsyncTestCase): ss.setServiceParent(self.sparent) already,writers = self.allocate(ss, b"vid", [0,1,2], 75) - self.failUnlessEqual(already, set()) - self.failUnlessEqual(set(writers.keys()), set([0,1,2])) + self.assertThat(already, Equals(set())) + self.assertThat(set(writers.keys()), Equals(set([0,1,2]))) for i,wb in writers.items(): wb.write(0, b"%25d" % i) wb.close() @@ -1192,8 +1190,8 @@ class Server(AsyncTestCase): # Since we write with some seeks, the data we read back will be all # zeros. b = ss.get_buckets(b"vid") - self.failUnlessEqual(set(b.keys()), set([0,1,2])) - self.failUnlessEqual(b[0].read(0, 25), b"\x00" * 25) + self.assertThat(set(b.keys()), Equals(set([0,1,2]))) + self.assertThat(b[0].read(0, 25), Equals(b"\x00" * 25)) def test_reserved_space_advise_corruption(self): """ @@ -1211,9 +1209,9 @@ class Server(AsyncTestCase): ss.advise_corrupt_share(b"immutable", b"si0", 0, b"This share smells funny.\n") - self.assertEqual( + self.assertThat( [], - os.listdir(ss.corruption_advisory_dir), + Equals(os.listdir(ss.corruption_advisory_dir)), ) def test_advise_corruption(self): @@ -1242,16 +1240,16 @@ class Server(AsyncTestCase): si1_s = base32.b2a(b"si1") already,writers = self.allocate(ss, b"si1", [1], 75) self.assertThat(already, Equals(set())) - self.failUnlessEqual(set(writers.keys()), set([1])) + self.assertThat(set(writers.keys()), Equals(set([1]))) writers[1].write(0, b"data") writers[1].close() b = ss.get_buckets(b"si1") - self.failUnlessEqual(set(b.keys()), set([1])) + self.assertThat(set(b.keys()), Equals(set([1]))) b[1].advise_corrupt_share(b"This share tastes like dust.\n") reports = os.listdir(reportdir) - self.failUnlessEqual(len(reports), 2) + self.assertThat(reports, HasLength(2)) report_si1 = [r for r in reports if bytes_to_native_str(si1_s) in r][0] f = open(os.path.join(reportdir, report_si1), "rb") report = f.read() @@ -1277,9 +1275,9 @@ class Server(AsyncTestCase): ss.advise_corrupt_share(b"immutable", b"si0", 1, b"This share smells funny.\n") - self.assertEqual( + self.assertThat( [], - os.listdir(ss.corruption_advisory_dir), + Equals(os.listdir(ss.corruption_advisory_dir)), ) From 373a5328293693612123e7e47be4c01e5de3746b Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 15 Sep 2022 09:36:56 -0400 Subject: [PATCH 1039/2309] Detect corrupted UEB length more consistently. --- src/allmydata/immutable/layout.py | 8 ++++---- src/allmydata/test/test_repairer.py | 6 ++++++ 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/src/allmydata/immutable/layout.py b/src/allmydata/immutable/layout.py index de390bda9..6679fc94c 100644 --- a/src/allmydata/immutable/layout.py +++ b/src/allmydata/immutable/layout.py @@ -495,10 +495,10 @@ class ReadBucketProxy(object): if len(data) != self._fieldsize: raise LayoutInvalid("not enough bytes to encode URI length -- should be %d bytes long, not %d " % (self._fieldsize, len(data),)) length = struct.unpack(self._fieldstruct, data)[0] - if length >= 2**31: - # URI extension blocks are around 419 bytes long, so this - # must be corrupted. Anyway, the foolscap interface schema - # for "read" will not allow >= 2**31 bytes length. + if length >= 2000: + # URI extension blocks are around 419 bytes long; in previous + # versions of the code 1000 was used as a default catchall. So + # 2000 or more must be corrupted. raise RidiculouslyLargeURIExtensionBlock(length) return self._read(offset+self._fieldsize, length) diff --git a/src/allmydata/test/test_repairer.py b/src/allmydata/test/test_repairer.py index f9b93af72..8545b1cf4 100644 --- a/src/allmydata/test/test_repairer.py +++ b/src/allmydata/test/test_repairer.py @@ -251,6 +251,12 @@ class Verifier(GridTestMixin, unittest.TestCase, RepairTestMixin): self.judge_invisible_corruption) def test_corrupt_ueb(self): + # Note that in some rare situations this might fail, specifically if + # the length of the UEB is corrupted to be a value that is bigger than + # the size but less than 2000, it might not get caught... But that's + # mostly because in that case it doesn't meaningfully corrupt it. See + # _get_uri_extension_the_old_way() in layout.py for where the 2000 + # number comes from. self.basedir = "repairer/Verifier/corrupt_ueb" return self._help_test_verify(common._corrupt_uri_extension, self.judge_invisible_corruption) From 8d5f08771a4f73f09611500c39627334f7273fc9 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 15 Sep 2022 09:45:46 -0400 Subject: [PATCH 1040/2309] Minimal check on parameters' contents. --- src/allmydata/immutable/encode.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/allmydata/immutable/encode.py b/src/allmydata/immutable/encode.py index 3c4440486..874492785 100644 --- a/src/allmydata/immutable/encode.py +++ b/src/allmydata/immutable/encode.py @@ -705,9 +705,13 @@ class Encoder(object): only care about the length. """ params = self.uri_extension_data.copy() - assert params params["crypttext_hash"] = b"\x00" * hashutil.CRYPTO_VAL_SIZE params["crypttext_root_hash"] = b"\x00" * hashutil.CRYPTO_VAL_SIZE params["share_root_hash"] = b"\x00" * hashutil.CRYPTO_VAL_SIZE + assert params.keys() == { + "codec_name", "codec_params", "size", "segment_size", "num_segments", + "needed_shares", "total_shares", "tail_codec_params", + "crypttext_hash", "crypttext_root_hash", "share_root_hash" + }, params.keys() uri_extension = uri.pack_extension(params) return len(uri_extension) From 00972ba3c6d8b9b83eb8c069ec1c9fa5768aaed3 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 15 Sep 2022 09:59:36 -0400 Subject: [PATCH 1041/2309] Match latest GBS spec. --- docs/specifications/url.rst | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/docs/specifications/url.rst b/docs/specifications/url.rst index 31fb05fad..a9e37a0ec 100644 --- a/docs/specifications/url.rst +++ b/docs/specifications/url.rst @@ -103,11 +103,8 @@ Version 1 The hash component of a version 1 NURL differs in three ways from the prior version. -1. The hash function used is SHA3-224 instead of SHA1. - The security of SHA1 `continues to be eroded`_. - Contrariwise SHA3 is currently the most recent addition to the SHA family by NIST. - The 224 bit instance is chosen to keep the output short and because it offers greater collision resistance than SHA1 was thought to offer even at its inception - (prior to security research showing actual collision resistance is lower). +1. The hash function used is SHA-256, to match RFC 7469. + The security of SHA1 `continues to be eroded`_; Latacora `SHA-2`_. 2. The hash is computed over the certificate's SPKI instead of the whole certificate. This allows certificate re-generation so long as the public key remains the same. This is useful to allow contact information to be updated or extension of validity period. @@ -140,7 +137,8 @@ Examples * ``pb://azEu8vlRpnEeYm0DySQDeNY3Z2iJXHC_bsbaAw@localhost:47877/64i4aokv4ej#v=1`` .. _`continues to be eroded`: https://en.wikipedia.org/wiki/SHA-1#Cryptanalysis_and_validation -.. _`explored by the web community`: https://www.imperialviolet.org/2011/05/04/pinning.html +.. _`SHA-2`: https://latacora.micro.blog/2018/04/03/cryptographic-right-answers.html +.. _`explored by the web community`: https://www.rfc-editor.org/rfc/rfc7469 .. _Foolscap: https://github.com/warner/foolscap .. [1] ``foolscap.furl.decode_furl`` is taken as the canonical definition of the syntax of a fURL. From 1759eacee3c4cbdb72d956bf1df2c35f7fc435bb Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 15 Sep 2022 10:09:25 -0400 Subject: [PATCH 1042/2309] No need to include NURL. --- docs/proposed/http-storage-node-protocol.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index 3dac376ff..b601a785b 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -409,8 +409,7 @@ For example:: "tolerates-immutable-read-overrun": true, "delete-mutable-shares-with-zero-length-writev": true, "fills-holes-with-zero-bytes": true, - "prevents-read-past-end-of-share-data": true, - "gbs-anonymous-storage-url": "pb://...#v=1" + "prevents-read-past-end-of-share-data": true }, "application-version": "1.13.0" } From 0d97847ef5c4625d972dd92e29f9a8187f97a6b2 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 15 Sep 2022 10:09:50 -0400 Subject: [PATCH 1043/2309] News file. --- newsfragments/3904.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3904.minor diff --git a/newsfragments/3904.minor b/newsfragments/3904.minor new file mode 100644 index 000000000..e69de29bb From b1aa93e02234bae93efac860a0078d5a1c089d2a Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 15 Sep 2022 10:34:59 -0400 Subject: [PATCH 1044/2309] Switch prefix. --- docs/proposed/http-storage-node-protocol.rst | 48 ++++++++++---------- src/allmydata/storage/http_client.py | 28 ++++++++---- src/allmydata/storage/http_server.py | 27 ++++++----- src/allmydata/test/test_storage_http.py | 8 ++-- 4 files changed, 61 insertions(+), 50 deletions(-) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index b601a785b..ec800367c 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -395,7 +395,7 @@ Encoding General ~~~~~~~ -``GET /v1/version`` +``GET /storage/v1/version`` !!!!!!!!!!!!!!!!!!! Retrieve information about the version of the storage server. @@ -414,7 +414,7 @@ For example:: "application-version": "1.13.0" } -``PUT /v1/lease/:storage_index`` +``PUT /storage/v1/lease/:storage_index`` !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! Either renew or create a new lease on the bucket addressed by ``storage_index``. @@ -467,7 +467,7 @@ Immutable Writing ~~~~~~~ -``POST /v1/immutable/:storage_index`` +``POST /storage/v1/immutable/:storage_index`` !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! Initialize an immutable storage index with some buckets. @@ -503,7 +503,7 @@ Handling repeat calls: Discussion `````````` -We considered making this ``POST /v1/immutable`` instead. +We considered making this ``POST /storage/v1/immutable`` instead. The motivation was to keep *storage index* out of the request URL. Request URLs have an elevated chance of being logged by something. We were concerned that having the *storage index* logged may increase some risks. @@ -538,7 +538,7 @@ Rejected designs for upload secrets: it must contain randomness. Randomness means there is no need to have a secret per share, since adding share-specific content to randomness doesn't actually make the secret any better. -``PATCH /v1/immutable/:storage_index/:share_number`` +``PATCH /storage/v1/immutable/:storage_index/:share_number`` !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! Write data for the indicated share. @@ -579,7 +579,7 @@ Responses: the response is ``CONFLICT``. At this point the only thing to do is abort the upload and start from scratch (see below). -``PUT /v1/immutable/:storage_index/:share_number/abort`` +``PUT /storage/v1/immutable/:storage_index/:share_number/abort`` !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! This cancels an *in-progress* upload. @@ -615,7 +615,7 @@ From RFC 7231:: PATCH method defined in [RFC5789]). -``POST /v1/immutable/:storage_index/:share_number/corrupt`` +``POST /storage/v1/immutable/:storage_index/:share_number/corrupt`` !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! Advise the server the data read from the indicated share was corrupt. The @@ -634,7 +634,7 @@ couldn't be found. Reading ~~~~~~~ -``GET /v1/immutable/:storage_index/shares`` +``GET /storage/v1/immutable/:storage_index/shares`` !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! Retrieve a list (semantically, a set) indicating all shares available for the @@ -644,7 +644,7 @@ indicated storage index. For example:: An unknown storage index results in an empty list. -``GET /v1/immutable/:storage_index/:share_number`` +``GET /storage/v1/immutable/:storage_index/:share_number`` !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! Read a contiguous sequence of bytes from one share in one bucket. @@ -685,7 +685,7 @@ Mutable Writing ~~~~~~~ -``POST /v1/mutable/:storage_index/read-test-write`` +``POST /storage/v1/mutable/:storage_index/read-test-write`` !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! General purpose read-test-and-write operation for mutable storage indexes. @@ -741,7 +741,7 @@ As a result, if there is no data at all, an empty bytestring is returned no matt Reading ~~~~~~~ -``GET /v1/mutable/:storage_index/shares`` +``GET /storage/v1/mutable/:storage_index/shares`` !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! Retrieve a set indicating all shares available for the indicated storage index. @@ -749,10 +749,10 @@ For example (this is shown as list, since it will be list for JSON, but will be [1, 5] -``GET /v1/mutable/:storage_index/:share_number`` +``GET /storage/v1/mutable/:storage_index/:share_number`` !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! -Read data from the indicated mutable shares, just like ``GET /v1/immutable/:storage_index`` +Read data from the indicated mutable shares, just like ``GET /storage/v1/immutable/:storage_index`` The ``Range`` header may be used to request exactly one ``bytes`` range, in which case the response code will be 206 (partial content). Interpretation and response behavior is as specified in RFC 7233 § 4.1. @@ -764,7 +764,7 @@ The resulting ``Content-Range`` header will be consistent with the returned data If the response to a query is an empty range, the ``NO CONTENT`` (204) response code will be used. -``POST /v1/mutable/:storage_index/:share_number/corrupt`` +``POST /storage/v1/mutable/:storage_index/:share_number/corrupt`` !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! Advise the server the data read from the indicated share was corrupt. @@ -778,7 +778,7 @@ Immutable Data 1. Create a bucket for storage index ``AAAAAAAAAAAAAAAA`` to hold two immutable shares, discovering that share ``1`` was already uploaded:: - POST /v1/immutable/AAAAAAAAAAAAAAAA + POST /storage/v1/immutable/AAAAAAAAAAAAAAAA Authorization: Tahoe-LAFS nurl-swissnum X-Tahoe-Authorization: lease-renew-secret efgh X-Tahoe-Authorization: lease-cancel-secret jjkl @@ -791,7 +791,7 @@ Immutable Data #. Upload the content for immutable share ``7``:: - PATCH /v1/immutable/AAAAAAAAAAAAAAAA/7 + PATCH /storage/v1/immutable/AAAAAAAAAAAAAAAA/7 Authorization: Tahoe-LAFS nurl-swissnum Content-Range: bytes 0-15/48 X-Tahoe-Authorization: upload-secret xyzf @@ -799,7 +799,7 @@ Immutable Data 200 OK - PATCH /v1/immutable/AAAAAAAAAAAAAAAA/7 + PATCH /storage/v1/immutable/AAAAAAAAAAAAAAAA/7 Authorization: Tahoe-LAFS nurl-swissnum Content-Range: bytes 16-31/48 X-Tahoe-Authorization: upload-secret xyzf @@ -807,7 +807,7 @@ Immutable Data 200 OK - PATCH /v1/immutable/AAAAAAAAAAAAAAAA/7 + PATCH /storage/v1/immutable/AAAAAAAAAAAAAAAA/7 Authorization: Tahoe-LAFS nurl-swissnum Content-Range: bytes 32-47/48 X-Tahoe-Authorization: upload-secret xyzf @@ -817,7 +817,7 @@ Immutable Data #. Download the content of the previously uploaded immutable share ``7``:: - GET /v1/immutable/AAAAAAAAAAAAAAAA?share=7 + GET /storage/v1/immutable/AAAAAAAAAAAAAAAA?share=7 Authorization: Tahoe-LAFS nurl-swissnum Range: bytes=0-47 @@ -826,7 +826,7 @@ Immutable Data #. Renew the lease on all immutable shares in bucket ``AAAAAAAAAAAAAAAA``:: - PUT /v1/lease/AAAAAAAAAAAAAAAA + PUT /storage/v1/lease/AAAAAAAAAAAAAAAA Authorization: Tahoe-LAFS nurl-swissnum X-Tahoe-Authorization: lease-cancel-secret jjkl X-Tahoe-Authorization: lease-renew-secret efgh @@ -841,7 +841,7 @@ The special test vector of size 1 but empty bytes will only pass if there is no existing share, otherwise it will read a byte which won't match `b""`:: - POST /v1/mutable/BBBBBBBBBBBBBBBB/read-test-write + POST /storage/v1/mutable/BBBBBBBBBBBBBBBB/read-test-write Authorization: Tahoe-LAFS nurl-swissnum X-Tahoe-Authorization: write-enabler abcd X-Tahoe-Authorization: lease-cancel-secret efgh @@ -873,7 +873,7 @@ otherwise it will read a byte which won't match `b""`:: #. Safely rewrite the contents of a known version of mutable share number ``3`` (or fail):: - POST /v1/mutable/BBBBBBBBBBBBBBBB/read-test-write + POST /storage/v1/mutable/BBBBBBBBBBBBBBBB/read-test-write Authorization: Tahoe-LAFS nurl-swissnum X-Tahoe-Authorization: write-enabler abcd X-Tahoe-Authorization: lease-cancel-secret efgh @@ -905,14 +905,14 @@ otherwise it will read a byte which won't match `b""`:: #. Download the contents of share number ``3``:: - GET /v1/mutable/BBBBBBBBBBBBBBBB?share=3&offset=0&size=10 + GET /storage/v1/mutable/BBBBBBBBBBBBBBBB?share=3&offset=0&size=10 Authorization: Tahoe-LAFS nurl-swissnum #. Renew the lease on previously uploaded mutable share in slot ``BBBBBBBBBBBBBBBB``:: - PUT /v1/lease/BBBBBBBBBBBBBBBB + PUT /storage/v1/lease/BBBBBBBBBBBBBBBB Authorization: Tahoe-LAFS nurl-swissnum X-Tahoe-Authorization: lease-cancel-secret efgh X-Tahoe-Authorization: lease-renew-secret ijkl diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index a2dc5379f..16d426dda 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -392,7 +392,7 @@ class StorageClientGeneral(object): """ Return the version metadata for the server. """ - url = self._client.relative_url("/v1/version") + url = self._client.relative_url("/storage/v1/version") response = yield self._client.request("GET", url) decoded_response = yield _decode_cbor(response, _SCHEMAS["get_version"]) returnValue(decoded_response) @@ -408,7 +408,7 @@ class StorageClientGeneral(object): Otherwise a new lease is added. """ url = self._client.relative_url( - "/v1/lease/{}".format(_encode_si(storage_index)) + "/storage/v1/lease/{}".format(_encode_si(storage_index)) ) response = yield self._client.request( "PUT", @@ -457,7 +457,9 @@ def read_share_chunk( always provided by the current callers. """ url = client.relative_url( - "/v1/{}/{}/{}".format(share_type, _encode_si(storage_index), share_number) + "/storage/v1/{}/{}/{}".format( + share_type, _encode_si(storage_index), share_number + ) ) response = yield client.request( "GET", @@ -518,7 +520,7 @@ async def advise_corrupt_share( ): assert isinstance(reason, str) url = client.relative_url( - "/v1/{}/{}/{}/corrupt".format( + "/storage/v1/{}/{}/{}/corrupt".format( share_type, _encode_si(storage_index), share_number ) ) @@ -563,7 +565,9 @@ class StorageClientImmutables(object): Result fires when creating the storage index succeeded, if creating the storage index failed the result will fire with an exception. """ - url = self._client.relative_url("/v1/immutable/" + _encode_si(storage_index)) + url = self._client.relative_url( + "/storage/v1/immutable/" + _encode_si(storage_index) + ) message = {"share-numbers": share_numbers, "allocated-size": allocated_size} response = yield self._client.request( @@ -588,7 +592,9 @@ class StorageClientImmutables(object): ) -> Deferred[None]: """Abort the upload.""" url = self._client.relative_url( - "/v1/immutable/{}/{}/abort".format(_encode_si(storage_index), share_number) + "/storage/v1/immutable/{}/{}/abort".format( + _encode_si(storage_index), share_number + ) ) response = yield self._client.request( "PUT", @@ -620,7 +626,9 @@ class StorageClientImmutables(object): been uploaded. """ url = self._client.relative_url( - "/v1/immutable/{}/{}".format(_encode_si(storage_index), share_number) + "/storage/v1/immutable/{}/{}".format( + _encode_si(storage_index), share_number + ) ) response = yield self._client.request( "PATCH", @@ -668,7 +676,7 @@ class StorageClientImmutables(object): Return the set of shares for a given storage index. """ url = self._client.relative_url( - "/v1/immutable/{}/shares".format(_encode_si(storage_index)) + "/storage/v1/immutable/{}/shares".format(_encode_si(storage_index)) ) response = yield self._client.request( "GET", @@ -774,7 +782,7 @@ class StorageClientMutables: are done and if they are valid the writes are done. """ url = self._client.relative_url( - "/v1/mutable/{}/read-test-write".format(_encode_si(storage_index)) + "/storage/v1/mutable/{}/read-test-write".format(_encode_si(storage_index)) ) message = { "test-write-vectors": { @@ -817,7 +825,7 @@ class StorageClientMutables: List the share numbers for a given storage index. """ url = self._client.relative_url( - "/v1/mutable/{}/shares".format(_encode_si(storage_index)) + "/storage/v1/mutable/{}/shares".format(_encode_si(storage_index)) ) response = await self._client.request("GET", url) if response.code == http.OK: diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 68d0740b1..2e9b57b13 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -545,7 +545,7 @@ class HTTPServer(object): ##### Generic APIs ##### - @_authorized_route(_app, set(), "/v1/version", methods=["GET"]) + @_authorized_route(_app, set(), "/storage/v1/version", methods=["GET"]) def version(self, request, authorization): """Return version information.""" return self._send_encoded(request, self._storage_server.get_version()) @@ -555,7 +555,7 @@ class HTTPServer(object): @_authorized_route( _app, {Secrets.LEASE_RENEW, Secrets.LEASE_CANCEL, Secrets.UPLOAD}, - "/v1/immutable/", + "/storage/v1/immutable/", methods=["POST"], ) def allocate_buckets(self, request, authorization, storage_index): @@ -591,7 +591,7 @@ class HTTPServer(object): @_authorized_route( _app, {Secrets.UPLOAD}, - "/v1/immutable///abort", + "/storage/v1/immutable///abort", methods=["PUT"], ) def abort_share_upload(self, request, authorization, storage_index, share_number): @@ -622,7 +622,7 @@ class HTTPServer(object): @_authorized_route( _app, {Secrets.UPLOAD}, - "/v1/immutable//", + "/storage/v1/immutable//", methods=["PATCH"], ) def write_share_data(self, request, authorization, storage_index, share_number): @@ -665,7 +665,7 @@ class HTTPServer(object): @_authorized_route( _app, set(), - "/v1/immutable//shares", + "/storage/v1/immutable//shares", methods=["GET"], ) def list_shares(self, request, authorization, storage_index): @@ -678,7 +678,7 @@ class HTTPServer(object): @_authorized_route( _app, set(), - "/v1/immutable//", + "/storage/v1/immutable//", methods=["GET"], ) def read_share_chunk(self, request, authorization, storage_index, share_number): @@ -694,7 +694,7 @@ class HTTPServer(object): @_authorized_route( _app, {Secrets.LEASE_RENEW, Secrets.LEASE_CANCEL}, - "/v1/lease/", + "/storage/v1/lease/", methods=["PUT"], ) def add_or_renew_lease(self, request, authorization, storage_index): @@ -715,7 +715,7 @@ class HTTPServer(object): @_authorized_route( _app, set(), - "/v1/immutable///corrupt", + "/storage/v1/immutable///corrupt", methods=["POST"], ) def advise_corrupt_share_immutable( @@ -736,7 +736,7 @@ class HTTPServer(object): @_authorized_route( _app, {Secrets.LEASE_RENEW, Secrets.LEASE_CANCEL, Secrets.WRITE_ENABLER}, - "/v1/mutable//read-test-write", + "/storage/v1/mutable//read-test-write", methods=["POST"], ) def mutable_read_test_write(self, request, authorization, storage_index): @@ -771,7 +771,7 @@ class HTTPServer(object): @_authorized_route( _app, set(), - "/v1/mutable//", + "/storage/v1/mutable//", methods=["GET"], ) def read_mutable_chunk(self, request, authorization, storage_index, share_number): @@ -795,7 +795,10 @@ class HTTPServer(object): return read_range(request, read_data, share_length) @_authorized_route( - _app, set(), "/v1/mutable//shares", methods=["GET"] + _app, + set(), + "/storage/v1/mutable//shares", + methods=["GET"], ) def enumerate_mutable_shares(self, request, authorization, storage_index): """List mutable shares for a storage index.""" @@ -805,7 +808,7 @@ class HTTPServer(object): @_authorized_route( _app, set(), - "/v1/mutable///corrupt", + "/storage/v1/mutable///corrupt", methods=["POST"], ) def advise_corrupt_share_mutable( diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 419052282..4a912cf6c 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -255,7 +255,7 @@ class TestApp(object): else: return "BAD: {}".format(authorization) - @_authorized_route(_app, set(), "/v1/version", methods=["GET"]) + @_authorized_route(_app, set(), "/storage/v1/version", methods=["GET"]) def bad_version(self, request, authorization): """Return version result that violates the expected schema.""" request.setHeader("content-type", CBOR_MIME_TYPE) @@ -534,7 +534,7 @@ class GenericHTTPAPITests(SyncTestCase): lease_secret = urandom(32) storage_index = urandom(16) url = self.http.client.relative_url( - "/v1/immutable/" + _encode_si(storage_index) + "/storage/v1/immutable/" + _encode_si(storage_index) ) message = {"bad-message": "missing expected keys"} @@ -1418,7 +1418,7 @@ class SharedImmutableMutableTestsMixin: self.http.client.request( "GET", self.http.client.relative_url( - "/v1/{}/{}/1".format(self.KIND, _encode_si(storage_index)) + "/storage/v1/{}/{}/1".format(self.KIND, _encode_si(storage_index)) ), ) ) @@ -1441,7 +1441,7 @@ class SharedImmutableMutableTestsMixin: self.http.client.request( "GET", self.http.client.relative_url( - "/v1/{}/{}/1".format(self.KIND, _encode_si(storage_index)) + "/storage/v1/{}/{}/1".format(self.KIND, _encode_si(storage_index)) ), headers=headers, ) From f5b374a7a2ad95232e8cddca3d9d334f4f4b6986 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 15 Sep 2022 10:56:11 -0400 Subject: [PATCH 1045/2309] Make sphinx happy. --- docs/proposed/http-storage-node-protocol.rst | 22 ++++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index ec800367c..a44408e6c 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -396,7 +396,7 @@ General ~~~~~~~ ``GET /storage/v1/version`` -!!!!!!!!!!!!!!!!!!! +!!!!!!!!!!!!!!!!!!!!!!!!!!! Retrieve information about the version of the storage server. Information is returned as an encoded mapping. @@ -415,7 +415,7 @@ For example:: } ``PUT /storage/v1/lease/:storage_index`` -!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! Either renew or create a new lease on the bucket addressed by ``storage_index``. @@ -468,7 +468,7 @@ Writing ~~~~~~~ ``POST /storage/v1/immutable/:storage_index`` -!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! Initialize an immutable storage index with some buckets. The buckets may have share data written to them once. @@ -539,7 +539,7 @@ Rejected designs for upload secrets: Randomness means there is no need to have a secret per share, since adding share-specific content to randomness doesn't actually make the secret any better. ``PATCH /storage/v1/immutable/:storage_index/:share_number`` -!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! Write data for the indicated share. The share number must belong to the storage index. @@ -580,7 +580,7 @@ Responses: At this point the only thing to do is abort the upload and start from scratch (see below). ``PUT /storage/v1/immutable/:storage_index/:share_number/abort`` -!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! This cancels an *in-progress* upload. @@ -616,7 +616,7 @@ From RFC 7231:: ``POST /storage/v1/immutable/:storage_index/:share_number/corrupt`` -!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! Advise the server the data read from the indicated share was corrupt. The request body includes an human-meaningful text string with details about the @@ -635,7 +635,7 @@ Reading ~~~~~~~ ``GET /storage/v1/immutable/:storage_index/shares`` -!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! Retrieve a list (semantically, a set) indicating all shares available for the indicated storage index. For example:: @@ -645,7 +645,7 @@ indicated storage index. For example:: An unknown storage index results in an empty list. ``GET /storage/v1/immutable/:storage_index/:share_number`` -!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! Read a contiguous sequence of bytes from one share in one bucket. The response body is the raw share data (i.e., ``application/octet-stream``). @@ -686,7 +686,7 @@ Writing ~~~~~~~ ``POST /storage/v1/mutable/:storage_index/read-test-write`` -!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! General purpose read-test-and-write operation for mutable storage indexes. A mutable storage index is also called a "slot" @@ -742,7 +742,7 @@ Reading ~~~~~~~ ``GET /storage/v1/mutable/:storage_index/shares`` -!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! Retrieve a set indicating all shares available for the indicated storage index. For example (this is shown as list, since it will be list for JSON, but will be set for CBOR):: @@ -765,7 +765,7 @@ If the response to a query is an empty range, the ``NO CONTENT`` (204) response ``POST /storage/v1/mutable/:storage_index/:share_number/corrupt`` -!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! Advise the server the data read from the indicated share was corrupt. Just like the immutable version. From 4a573ede3461510d6f2aa09f78d2791dea8393b9 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 15 Sep 2022 11:29:32 -0400 Subject: [PATCH 1046/2309] Download the actual data we need, instead of relying on bad reading-beyond-the-end semantics. --- src/allmydata/immutable/layout.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/allmydata/immutable/layout.py b/src/allmydata/immutable/layout.py index 6679fc94c..07b6b8b3b 100644 --- a/src/allmydata/immutable/layout.py +++ b/src/allmydata/immutable/layout.py @@ -17,8 +17,10 @@ from allmydata.interfaces import IStorageBucketWriter, IStorageBucketReader, \ FileTooLargeError, HASH_SIZE from allmydata.util import mathutil, observer, pipeline, log from allmydata.util.assertutil import precondition +from allmydata.util.deferredutil import async_to_deferred from allmydata.storage.server import si_b2a + class LayoutInvalid(Exception): """ There is something wrong with these bytes so they can't be interpreted as the kind of immutable file that I know how to download.""" @@ -311,8 +313,6 @@ class WriteBucketProxy_v2(WriteBucketProxy): @implementer(IStorageBucketReader) class ReadBucketProxy(object): - MAX_UEB_SIZE = 2000 # actual size is closer to 419, but varies by a few bytes - def __init__(self, rref, server, storage_index): self._rref = rref self._server = server @@ -389,10 +389,15 @@ class ReadBucketProxy(object): self._offsets[field] = offset return self._offsets - def _fetch_sharehashtree_and_ueb(self, offsets): + @async_to_deferred + async def _fetch_sharehashtree_and_ueb(self, offsets): + [ueb_length] = struct.unpack( + await self._read(offsets['share_hashes'], self._fieldsize), + self._fieldstruct + ) sharehashtree_size = offsets['uri_extension'] - offsets['share_hashes'] return self._read(offsets['share_hashes'], - self.MAX_UEB_SIZE+sharehashtree_size) + ueb_length + self._fieldsize +sharehashtree_size) def _parse_sharehashtree_and_ueb(self, data): sharehashtree_size = self._offsets['uri_extension'] - self._offsets['share_hashes'] From 444bc724c54a07ef4e0dddb53706e1e1d16091b3 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 16 Sep 2022 10:38:29 -0400 Subject: [PATCH 1047/2309] A better approach to MAX_UEB_SIZE: just delete the code since it's not used in practice. --- src/allmydata/immutable/layout.py | 59 ++++++------------------------- 1 file changed, 10 insertions(+), 49 deletions(-) diff --git a/src/allmydata/immutable/layout.py b/src/allmydata/immutable/layout.py index 07b6b8b3b..d552d43c4 100644 --- a/src/allmydata/immutable/layout.py +++ b/src/allmydata/immutable/layout.py @@ -17,7 +17,6 @@ from allmydata.interfaces import IStorageBucketWriter, IStorageBucketReader, \ FileTooLargeError, HASH_SIZE from allmydata.util import mathutil, observer, pipeline, log from allmydata.util.assertutil import precondition -from allmydata.util.deferredutil import async_to_deferred from allmydata.storage.server import si_b2a @@ -340,11 +339,6 @@ class ReadBucketProxy(object): # TODO: for small shares, read the whole bucket in _start() d = self._fetch_header() d.addCallback(self._parse_offsets) - # XXX The following two callbacks implement a slightly faster/nicer - # way to get the ueb and sharehashtree, but it requires that the - # storage server be >= v1.3.0. - # d.addCallback(self._fetch_sharehashtree_and_ueb) - # d.addCallback(self._parse_sharehashtree_and_ueb) def _fail_waiters(f): self._ready.fire(f) def _notify_waiters(result): @@ -389,34 +383,6 @@ class ReadBucketProxy(object): self._offsets[field] = offset return self._offsets - @async_to_deferred - async def _fetch_sharehashtree_and_ueb(self, offsets): - [ueb_length] = struct.unpack( - await self._read(offsets['share_hashes'], self._fieldsize), - self._fieldstruct - ) - sharehashtree_size = offsets['uri_extension'] - offsets['share_hashes'] - return self._read(offsets['share_hashes'], - ueb_length + self._fieldsize +sharehashtree_size) - - def _parse_sharehashtree_and_ueb(self, data): - sharehashtree_size = self._offsets['uri_extension'] - self._offsets['share_hashes'] - if len(data) < sharehashtree_size: - raise LayoutInvalid("share hash tree truncated -- should have at least %d bytes -- not %d" % (sharehashtree_size, len(data))) - if sharehashtree_size % (2+HASH_SIZE) != 0: - raise LayoutInvalid("share hash tree malformed -- should have an even multiple of %d bytes -- not %d" % (2+HASH_SIZE, sharehashtree_size)) - self._share_hashes = [] - for i in range(0, sharehashtree_size, 2+HASH_SIZE): - hashnum = struct.unpack(">H", data[i:i+2])[0] - hashvalue = data[i+2:i+2+HASH_SIZE] - self._share_hashes.append( (hashnum, hashvalue) ) - - i = self._offsets['uri_extension']-self._offsets['share_hashes'] - if len(data) < i+self._fieldsize: - raise LayoutInvalid("not enough bytes to encode URI length -- should be at least %d bytes long, not %d " % (i+self._fieldsize, len(data),)) - length = struct.unpack(self._fieldstruct, data[i:i+self._fieldsize])[0] - self._ueb_data = data[i+self._fieldsize:i+self._fieldsize+length] - def _get_block_data(self, unused, blocknum, blocksize, thisblocksize): offset = self._offsets['data'] + blocknum * blocksize return self._read(offset, thisblocksize) @@ -459,20 +425,18 @@ class ReadBucketProxy(object): else: return defer.succeed([]) - def _get_share_hashes(self, unused=None): - if hasattr(self, '_share_hashes'): - return self._share_hashes - return self._get_share_hashes_the_old_way() - def get_share_hashes(self): d = self._start_if_needed() d.addCallback(self._get_share_hashes) return d - def _get_share_hashes_the_old_way(self): + def _get_share_hashes(self, _ignore): """ Tahoe storage servers < v1.3.0 would return an error if you tried to read past the end of the share, so we need to use the offset and - read just that much.""" + read just that much. + + HTTP-based storage protocol also doesn't like reading past the end. + """ offset = self._offsets['share_hashes'] size = self._offsets['uri_extension'] - offset if size % (2+HASH_SIZE) != 0: @@ -490,10 +454,13 @@ class ReadBucketProxy(object): d.addCallback(_unpack_share_hashes) return d - def _get_uri_extension_the_old_way(self, unused=None): + def _get_uri_extension(self, unused=None): """ Tahoe storage servers < v1.3.0 would return an error if you tried to read past the end of the share, so we need to fetch the UEB size - and then read just that much.""" + and then read just that much. + + HTTP-based storage protocol also doesn't like reading past the end. + """ offset = self._offsets['uri_extension'] d = self._read(offset, self._fieldsize) def _got_length(data): @@ -510,12 +477,6 @@ class ReadBucketProxy(object): d.addCallback(_got_length) return d - def _get_uri_extension(self, unused=None): - if hasattr(self, '_ueb_data'): - return self._ueb_data - else: - return self._get_uri_extension_the_old_way() - def get_uri_extension(self): d = self._start_if_needed() d.addCallback(self._get_uri_extension) From fb532a71ef4da28c91594e8d05695e267b747137 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 13 Sep 2022 22:43:09 -0600 Subject: [PATCH 1048/2309] own pid-file checks --- setup.py | 3 ++ src/allmydata/scripts/tahoe_run.py | 36 ++++++++++----- src/allmydata/util/pid.py | 72 ++++++++++++++++++++++++++++++ 3 files changed, 99 insertions(+), 12 deletions(-) create mode 100644 src/allmydata/util/pid.py diff --git a/setup.py b/setup.py index c3ee4eb90..bd16a61ce 100644 --- a/setup.py +++ b/setup.py @@ -138,6 +138,9 @@ install_requires = [ "treq", "cbor2", "pycddl", + + # for pid-file support + "psutil", ] setup_requires = [ diff --git a/src/allmydata/scripts/tahoe_run.py b/src/allmydata/scripts/tahoe_run.py index 51be32ee3..21041f1ab 100644 --- a/src/allmydata/scripts/tahoe_run.py +++ b/src/allmydata/scripts/tahoe_run.py @@ -19,6 +19,7 @@ import os, sys from allmydata.scripts.common import BasedirOptions from twisted.scripts import twistd from twisted.python import usage +from twisted.python.filepath import FilePath from twisted.python.reflect import namedAny from twisted.internet.defer import maybeDeferred from twisted.application.service import Service @@ -27,6 +28,11 @@ from allmydata.scripts.default_nodedir import _default_nodedir from allmydata.util.encodingutil import listdir_unicode, quote_local_unicode_path from allmydata.util.configutil import UnknownConfigError from allmydata.util.deferredutil import HookMixin +from allmydata.util.pid import ( + check_pid_process, + cleanup_pidfile, + ProcessInTheWay, +) from allmydata.storage.crawler import ( MigratePickleFileError, ) @@ -35,28 +41,31 @@ from allmydata.node import ( PrivacyError, ) + def get_pidfile(basedir): """ Returns the path to the PID file. :param basedir: the node's base directory :returns: the path to the PID file """ - return os.path.join(basedir, u"twistd.pid") + return os.path.join(basedir, u"running.process") + def get_pid_from_pidfile(pidfile): """ Tries to read and return the PID stored in the node's PID file - (twistd.pid). + :param pidfile: try to read this PID file :returns: A numeric PID on success, ``None`` if PID file absent or inaccessible, ``-1`` if PID file invalid. """ try: with open(pidfile, "r") as f: - pid = f.read() + data = f.read().strip() except EnvironmentError: return None + pid, _ = data.split() try: pid = int(pid) except ValueError: @@ -64,6 +73,7 @@ def get_pid_from_pidfile(pidfile): return pid + def identify_node_type(basedir): """ :return unicode: None or one of: 'client' or 'introducer'. @@ -227,10 +237,8 @@ def run(config, runApp=twistd.runApp): print("%s is not a recognizable node directory" % quoted_basedir, file=err) return 1 - twistd_args = ["--nodaemon", "--rundir", basedir] - if sys.platform != "win32": - pidfile = get_pidfile(basedir) - twistd_args.extend(["--pidfile", pidfile]) + # we turn off Twisted's pid-file to use our own + twistd_args = ["--pidfile", None, "--nodaemon", "--rundir", basedir] twistd_args.extend(config.twistd_args) twistd_args.append("DaemonizeTahoeNode") # point at our DaemonizeTahoeNodePlugin @@ -246,12 +254,16 @@ def run(config, runApp=twistd.runApp): return 1 twistd_config.loadedPlugins = {"DaemonizeTahoeNode": DaemonizeTahoeNodePlugin(nodetype, basedir)} - # handle invalid PID file (twistd might not start otherwise) - if sys.platform != "win32" and get_pid_from_pidfile(pidfile) == -1: - print("found invalid PID file in %s - deleting it" % basedir, file=err) - os.remove(pidfile) + # before we try to run, check against our pidfile -- this will + # raise an exception if there appears to be a running process "in + # the way" + pidfile = FilePath(get_pidfile(config['basedir'])) + try: + check_pid_process(pidfile) + except ProcessInTheWay as e: + print("ERROR: {}".format(e)) + return 1 # We always pass --nodaemon so twistd.runApp does not daemonize. - print("running node in %s" % (quoted_basedir,), file=out) runApp(twistd_config) return 0 diff --git a/src/allmydata/util/pid.py b/src/allmydata/util/pid.py new file mode 100644 index 000000000..21e30aa87 --- /dev/null +++ b/src/allmydata/util/pid.py @@ -0,0 +1,72 @@ +import os +import psutil + + +class ProcessInTheWay(Exception): + """ + our pidfile points at a running process + """ + + +def check_pid_process(pidfile, find_process=None): + """ + If another instance appears to be running already, raise an + exception. Otherwise, write our PID + start time to the pidfile + and arrange to delete it upon exit. + + :param FilePath pidfile: the file to read/write our PID from. + + :param Callable find_process: None, or a custom way to get a + Process objet (usually for tests) + + :raises ProcessInTheWay: if a running process exists at our PID + """ + find_process = psutil.Process if find_process is None else find_process + # check if we have another instance running already + if pidfile.exists(): + with pidfile.open("r") as f: + content = f.read().decode("utf8").strip() + pid, starttime = content.split() + pid = int(pid) + starttime = float(starttime) + try: + # if any other process is running at that PID, let the + # user decide if this is another magic-older + # instance. Automated programs may use the start-time to + # help decide this (if the PID is merely recycled, the + # start-time won't match). + proc = find_process(pid) + raise ProcessInTheWay( + "A process is already running as PID {}".format(pid) + ) + except psutil.NoSuchProcess: + print( + "'{pidpath}' refers to {pid} that isn't running".format( + pidpath=pidfile.path, + pid=pid, + ) + ) + # nothing is running at that PID so it must be a stale file + pidfile.remove() + + # write our PID + start-time to the pid-file + pid = os.getpid() + starttime = find_process(pid).create_time() + with pidfile.open("w") as f: + f.write("{} {}\n".format(pid, starttime).encode("utf8")) + + +def cleanup_pidfile(pidfile): + """ + Safely remove the given pidfile + """ + + try: + pidfile.remove() + except Exception as e: + print( + "Couldn't remove '{pidfile}': {err}.".format( + pidfile=pidfile.path, + err=e, + ) + ) From 3bfb60c6f426cadc25bb201e6e59165cedd2b490 Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 15 Sep 2022 19:57:01 -0600 Subject: [PATCH 1049/2309] back to context-manager, simplify --- src/allmydata/scripts/tahoe_run.py | 15 +++++++++------ src/allmydata/test/cli/test_run.py | 20 +++++++++++--------- src/allmydata/util/pid.py | 29 +++++++++++++++++++++-------- 3 files changed, 41 insertions(+), 23 deletions(-) diff --git a/src/allmydata/scripts/tahoe_run.py b/src/allmydata/scripts/tahoe_run.py index 21041f1ab..07f5bf72c 100644 --- a/src/allmydata/scripts/tahoe_run.py +++ b/src/allmydata/scripts/tahoe_run.py @@ -30,8 +30,8 @@ from allmydata.util.configutil import UnknownConfigError from allmydata.util.deferredutil import HookMixin from allmydata.util.pid import ( check_pid_process, - cleanup_pidfile, ProcessInTheWay, + InvalidPidFile, ) from allmydata.storage.crawler import ( MigratePickleFileError, @@ -237,8 +237,13 @@ def run(config, runApp=twistd.runApp): print("%s is not a recognizable node directory" % quoted_basedir, file=err) return 1 - # we turn off Twisted's pid-file to use our own - twistd_args = ["--pidfile", None, "--nodaemon", "--rundir", basedir] + twistd_args = [ + # turn off Twisted's pid-file to use our own + "--pidfile", None, + # ensure twistd machinery does not daemonize. + "--nodaemon", + "--rundir", basedir, + ] twistd_args.extend(config.twistd_args) twistd_args.append("DaemonizeTahoeNode") # point at our DaemonizeTahoeNodePlugin @@ -254,9 +259,7 @@ def run(config, runApp=twistd.runApp): return 1 twistd_config.loadedPlugins = {"DaemonizeTahoeNode": DaemonizeTahoeNodePlugin(nodetype, basedir)} - # before we try to run, check against our pidfile -- this will - # raise an exception if there appears to be a running process "in - # the way" + # our own pid-style file contains PID and process creation time pidfile = FilePath(get_pidfile(config['basedir'])) try: check_pid_process(pidfile) diff --git a/src/allmydata/test/cli/test_run.py b/src/allmydata/test/cli/test_run.py index 28613e8c1..db01eb440 100644 --- a/src/allmydata/test/cli/test_run.py +++ b/src/allmydata/test/cli/test_run.py @@ -159,7 +159,7 @@ class RunTests(SyncTestCase): """ basedir = FilePath(self.mktemp()).asTextMode() basedir.makedirs() - basedir.child(u"twistd.pid").setContent(b"foo") + basedir.child(u"running.process").setContent(b"foo") basedir.child(u"tahoe-client.tac").setContent(b"") config = RunOptions() @@ -168,17 +168,19 @@ class RunTests(SyncTestCase): config['basedir'] = basedir.path config.twistd_args = [] - runs = [] - result_code = run(config, runApp=runs.append) + class DummyRunner: + runs = [] + _exitSignal = None + + def run(self): + self.runs.append(True) + + result_code = run(config, runner=DummyRunner()) self.assertThat( config.stderr.getvalue(), Contains("found invalid PID file in"), ) self.assertThat( - runs, - HasLength(1), - ) - self.assertThat( - result_code, - Equals(0), + DummyRunner.runs, + Equals([]) ) diff --git a/src/allmydata/util/pid.py b/src/allmydata/util/pid.py index 21e30aa87..3b488a2c2 100644 --- a/src/allmydata/util/pid.py +++ b/src/allmydata/util/pid.py @@ -1,5 +1,8 @@ import os import psutil +from contextlib import ( + contextmanager, +) class ProcessInTheWay(Exception): @@ -8,6 +11,13 @@ class ProcessInTheWay(Exception): """ +class InvalidPidFile(Exception): + """ + our pidfile isn't well-formed + """ + + +@contextmanager def check_pid_process(pidfile, find_process=None): """ If another instance appears to be running already, raise an @@ -26,9 +36,16 @@ def check_pid_process(pidfile, find_process=None): if pidfile.exists(): with pidfile.open("r") as f: content = f.read().decode("utf8").strip() - pid, starttime = content.split() - pid = int(pid) - starttime = float(starttime) + try: + pid, starttime = content.split() + pid = int(pid) + starttime = float(starttime) + except ValueError: + raise InvalidPidFile( + "found invalid PID file in {}".format( + pidfile + ) + ) try: # if any other process is running at that PID, let the # user decide if this is another magic-older @@ -55,11 +72,7 @@ def check_pid_process(pidfile, find_process=None): with pidfile.open("w") as f: f.write("{} {}\n".format(pid, starttime).encode("utf8")) - -def cleanup_pidfile(pidfile): - """ - Safely remove the given pidfile - """ + yield # setup completed, await cleanup try: pidfile.remove() From cad162bb8fb2d961c74f457be6e4495b00f0aeed Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 15 Sep 2022 19:59:18 -0600 Subject: [PATCH 1050/2309] should have pid-file on windows too, now --- src/allmydata/test/cli/test_run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/cli/test_run.py b/src/allmydata/test/cli/test_run.py index db01eb440..902e4011a 100644 --- a/src/allmydata/test/cli/test_run.py +++ b/src/allmydata/test/cli/test_run.py @@ -151,7 +151,7 @@ class RunTests(SyncTestCase): """ Tests for ``run``. """ - @skipIf(platform.isWindows(), "There are no PID files on Windows.") + def test_non_numeric_pid(self): """ If the pidfile exists but does not contain a numeric value, a complaint to From 0e0ebf6687280d0be5ae6a536a4f9d48958d03b7 Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 15 Sep 2022 20:06:32 -0600 Subject: [PATCH 1051/2309] more testing --- src/allmydata/test/cli/test_run.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/allmydata/test/cli/test_run.py b/src/allmydata/test/cli/test_run.py index 902e4011a..ecc81fe3f 100644 --- a/src/allmydata/test/cli/test_run.py +++ b/src/allmydata/test/cli/test_run.py @@ -20,6 +20,9 @@ from testtools import ( skipIf, ) +from hypothesis.strategies import text +from hypothesis import given + from testtools.matchers import ( Contains, Equals, @@ -44,6 +47,10 @@ from ...scripts.tahoe_run import ( RunOptions, run, ) +from ...util.pid import ( + check_pid_process, + InvalidPidFile, +) from ...scripts.runner import ( parse_options @@ -180,7 +187,18 @@ class RunTests(SyncTestCase): config.stderr.getvalue(), Contains("found invalid PID file in"), ) + # because the pidfile is invalid we shouldn't get to the + # .run() call itself. self.assertThat( DummyRunner.runs, Equals([]) ) + + @given(text()) + def test_pidfile_contents(self, content): + pidfile = FilePath("pidfile") + pidfile.setContent(content.encode("utf8")) + + with self.assertRaises(InvalidPidFile): + with check_pid_process(pidfile): + pass From e6adfc7726cc3e081d18b712e573ef265e49c3ca Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 15 Sep 2022 20:22:07 -0600 Subject: [PATCH 1052/2309] news --- newsfragments/3926.incompat | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 newsfragments/3926.incompat diff --git a/newsfragments/3926.incompat b/newsfragments/3926.incompat new file mode 100644 index 000000000..3f58b4ba8 --- /dev/null +++ b/newsfragments/3926.incompat @@ -0,0 +1,10 @@ +Record both the PID and the process creation-time + +A new kind of pidfile in `running.process` records both +the PID and the creation-time of the process. This facilitates +automatic discovery of a "stale" pidfile that points to a +currently-running process. If the recorded creation-time matches +the creation-time of the running process, then it is a still-running +`tahoe run` proecss. Otherwise, the file is stale. + +The `twistd.pid` file is no longer present. \ No newline at end of file From 6048d1d9a99e5f88cd423a9524bede823277709f Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 15 Sep 2022 21:13:30 -0600 Subject: [PATCH 1053/2309] in case hypothesis finds the magic --- src/allmydata/test/cli/test_run.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/allmydata/test/cli/test_run.py b/src/allmydata/test/cli/test_run.py index ecc81fe3f..7bf87eea9 100644 --- a/src/allmydata/test/cli/test_run.py +++ b/src/allmydata/test/cli/test_run.py @@ -12,6 +12,7 @@ from future.utils import PY2 if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 +import re from six.moves import ( StringIO, ) @@ -21,7 +22,7 @@ from testtools import ( ) from hypothesis.strategies import text -from hypothesis import given +from hypothesis import given, assume from testtools.matchers import ( Contains, @@ -194,8 +195,11 @@ class RunTests(SyncTestCase): Equals([]) ) + good_file_content_re = re.compile(r"\w[0-9]*\w[0-9]*\w") + @given(text()) def test_pidfile_contents(self, content): + assume(not self.good_file_content_re.match(content)) pidfile = FilePath("pidfile") pidfile.setContent(content.encode("utf8")) From 642b604753dd9b9af2c740e04e65e58bbae00299 Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 15 Sep 2022 21:51:56 -0600 Subject: [PATCH 1054/2309] use stdin-closing for pidfile cleanup too --- src/allmydata/scripts/tahoe_run.py | 1 + src/allmydata/test/cli/test_run.py | 12 +++--------- src/allmydata/util/pid.py | 17 +++++++++++------ 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/src/allmydata/scripts/tahoe_run.py b/src/allmydata/scripts/tahoe_run.py index 07f5bf72c..20d5c2bf1 100644 --- a/src/allmydata/scripts/tahoe_run.py +++ b/src/allmydata/scripts/tahoe_run.py @@ -30,6 +30,7 @@ from allmydata.util.configutil import UnknownConfigError from allmydata.util.deferredutil import HookMixin from allmydata.util.pid import ( check_pid_process, + cleanup_pidfile, ProcessInTheWay, InvalidPidFile, ) diff --git a/src/allmydata/test/cli/test_run.py b/src/allmydata/test/cli/test_run.py index 7bf87eea9..71085fddd 100644 --- a/src/allmydata/test/cli/test_run.py +++ b/src/allmydata/test/cli/test_run.py @@ -176,14 +176,8 @@ class RunTests(SyncTestCase): config['basedir'] = basedir.path config.twistd_args = [] - class DummyRunner: - runs = [] - _exitSignal = None - - def run(self): - self.runs.append(True) - - result_code = run(config, runner=DummyRunner()) + runs = [] + result_code = run(config, runApp=runs.append) self.assertThat( config.stderr.getvalue(), Contains("found invalid PID file in"), @@ -191,7 +185,7 @@ class RunTests(SyncTestCase): # because the pidfile is invalid we shouldn't get to the # .run() call itself. self.assertThat( - DummyRunner.runs, + runs, Equals([]) ) diff --git a/src/allmydata/util/pid.py b/src/allmydata/util/pid.py index 3b488a2c2..3ab955cb3 100644 --- a/src/allmydata/util/pid.py +++ b/src/allmydata/util/pid.py @@ -1,8 +1,5 @@ import os import psutil -from contextlib import ( - contextmanager, -) class ProcessInTheWay(Exception): @@ -17,7 +14,12 @@ class InvalidPidFile(Exception): """ -@contextmanager +class CannotRemovePidFile(Exception): + """ + something went wrong removing the pidfile + """ + + def check_pid_process(pidfile, find_process=None): """ If another instance appears to be running already, raise an @@ -72,12 +74,15 @@ def check_pid_process(pidfile, find_process=None): with pidfile.open("w") as f: f.write("{} {}\n".format(pid, starttime).encode("utf8")) - yield # setup completed, await cleanup +def cleanup_pidfile(pidfile): + """ + Safely clean up a PID-file + """ try: pidfile.remove() except Exception as e: - print( + raise CannotRemovePidFile( "Couldn't remove '{pidfile}': {err}.".format( pidfile=pidfile.path, err=e, From 82c72ddede1dbbe97365877186af27928a996c0b Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 15 Sep 2022 21:58:20 -0600 Subject: [PATCH 1055/2309] cleanup --- src/allmydata/test/cli/test_run.py | 14 ++------------ src/allmydata/util/pid.py | 4 ++-- 2 files changed, 4 insertions(+), 14 deletions(-) diff --git a/src/allmydata/test/cli/test_run.py b/src/allmydata/test/cli/test_run.py index 71085fddd..ae869e475 100644 --- a/src/allmydata/test/cli/test_run.py +++ b/src/allmydata/test/cli/test_run.py @@ -17,22 +17,14 @@ from six.moves import ( StringIO, ) -from testtools import ( - skipIf, -) - from hypothesis.strategies import text from hypothesis import given, assume from testtools.matchers import ( Contains, Equals, - HasLength, ) -from twisted.python.runtime import ( - platform, -) from twisted.python.filepath import ( FilePath, ) @@ -184,10 +176,8 @@ class RunTests(SyncTestCase): ) # because the pidfile is invalid we shouldn't get to the # .run() call itself. - self.assertThat( - runs, - Equals([]) - ) + self.assertThat(runs, Equals([])) + self.assertThat(result_code, Equals(1)) good_file_content_re = re.compile(r"\w[0-9]*\w[0-9]*\w") diff --git a/src/allmydata/util/pid.py b/src/allmydata/util/pid.py index 3ab955cb3..ff8129bbc 100644 --- a/src/allmydata/util/pid.py +++ b/src/allmydata/util/pid.py @@ -50,11 +50,11 @@ def check_pid_process(pidfile, find_process=None): ) try: # if any other process is running at that PID, let the - # user decide if this is another magic-older + # user decide if this is another legitimate # instance. Automated programs may use the start-time to # help decide this (if the PID is merely recycled, the # start-time won't match). - proc = find_process(pid) + find_process(pid) raise ProcessInTheWay( "A process is already running as PID {}".format(pid) ) From 228bbbc2fe791b83af0d495df44882a63456b59f Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 15 Sep 2022 22:39:59 -0600 Subject: [PATCH 1056/2309] new pid-file --- src/allmydata/test/cli_node_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/cli_node_api.py b/src/allmydata/test/cli_node_api.py index 410796be2..c324d5565 100644 --- a/src/allmydata/test/cli_node_api.py +++ b/src/allmydata/test/cli_node_api.py @@ -134,7 +134,7 @@ class CLINodeAPI(object): @property def twistd_pid_file(self): - return self.basedir.child(u"twistd.pid") + return self.basedir.child(u"running.process") @property def node_url_file(self): From 114d5e1ed8582fa130953227eced0528862ca381 Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 15 Sep 2022 23:08:46 -0600 Subject: [PATCH 1057/2309] pidfile on windows now --- src/allmydata/scripts/tahoe_run.py | 6 +++-- src/allmydata/test/test_runner.py | 36 ++++++++++++------------------ 2 files changed, 18 insertions(+), 24 deletions(-) diff --git a/src/allmydata/scripts/tahoe_run.py b/src/allmydata/scripts/tahoe_run.py index 20d5c2bf1..72b8e3eca 100644 --- a/src/allmydata/scripts/tahoe_run.py +++ b/src/allmydata/scripts/tahoe_run.py @@ -239,12 +239,14 @@ def run(config, runApp=twistd.runApp): return 1 twistd_args = [ - # turn off Twisted's pid-file to use our own - "--pidfile", None, # ensure twistd machinery does not daemonize. "--nodaemon", "--rundir", basedir, ] + if sys.platform != "win32": + # turn off Twisted's pid-file to use our own -- but only on + # windows, because twistd doesn't know about pidfiles there + twistd_args.extend(["--pidfile", None]) twistd_args.extend(config.twistd_args) twistd_args.append("DaemonizeTahoeNode") # point at our DaemonizeTahoeNodePlugin diff --git a/src/allmydata/test/test_runner.py b/src/allmydata/test/test_runner.py index 3eb6b8a34..9b6357f46 100644 --- a/src/allmydata/test/test_runner.py +++ b/src/allmydata/test/test_runner.py @@ -418,9 +418,7 @@ class RunNode(common_util.SignalMixin, unittest.TestCase, pollmixin.PollMixin): tahoe.active() - # We don't keep track of PIDs in files on Windows. - if not platform.isWindows(): - self.assertTrue(tahoe.twistd_pid_file.exists()) + self.assertTrue(tahoe.twistd_pid_file.exists()) self.assertTrue(tahoe.node_url_file.exists()) # rm this so we can detect when the second incarnation is ready @@ -493,9 +491,7 @@ class RunNode(common_util.SignalMixin, unittest.TestCase, pollmixin.PollMixin): # change on restart storage_furl = fileutil.read(tahoe.storage_furl_file.path) - # We don't keep track of PIDs in files on Windows. - if not platform.isWindows(): - self.assertTrue(tahoe.twistd_pid_file.exists()) + self.assertTrue(tahoe.twistd_pid_file.exists()) # rm this so we can detect when the second incarnation is ready tahoe.node_url_file.remove() @@ -513,21 +509,18 @@ class RunNode(common_util.SignalMixin, unittest.TestCase, pollmixin.PollMixin): fileutil.read(tahoe.storage_furl_file.path), ) - if not platform.isWindows(): - self.assertTrue( - tahoe.twistd_pid_file.exists(), - "PID file ({}) didn't exist when we expected it to. " - "These exist: {}".format( - tahoe.twistd_pid_file, - tahoe.twistd_pid_file.parent().listdir(), - ), - ) + self.assertTrue( + tahoe.twistd_pid_file.exists(), + "PID file ({}) didn't exist when we expected it to. " + "These exist: {}".format( + tahoe.twistd_pid_file, + tahoe.twistd_pid_file.parent().listdir(), + ), + ) yield tahoe.stop_and_wait() - if not platform.isWindows(): - # twistd.pid should be gone by now. - self.assertFalse(tahoe.twistd_pid_file.exists()) - + # twistd.pid should be gone by now. + self.assertFalse(tahoe.twistd_pid_file.exists()) def _remove(self, res, file): fileutil.remove(file) @@ -610,9 +603,8 @@ class RunNode(common_util.SignalMixin, unittest.TestCase, pollmixin.PollMixin): ), ) - if not platform.isWindows(): - # It should not be running. - self.assertFalse(tahoe.twistd_pid_file.exists()) + # It should not be running. + self.assertFalse(tahoe.twistd_pid_file.exists()) # Wait for the operation to *complete*. If we got this far it's # because we got the expected message so we can expect the "tahoe ..." From aef2e96139fc0afc610736b181e218dce2aa9b79 Mon Sep 17 00:00:00 2001 From: meejah Date: Sat, 17 Sep 2022 16:28:25 -0600 Subject: [PATCH 1058/2309] refactor: dispatch with our reactor, pass to tahoe_run --- src/allmydata/scripts/runner.py | 12 ++++-------- src/allmydata/scripts/tahoe_run.py | 2 +- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/src/allmydata/scripts/runner.py b/src/allmydata/scripts/runner.py index a0d8a752b..756c26f2c 100644 --- a/src/allmydata/scripts/runner.py +++ b/src/allmydata/scripts/runner.py @@ -47,11 +47,6 @@ if _default_nodedir: NODEDIR_HELP += " [default for most commands: " + quote_local_unicode_path(_default_nodedir) + "]" -# XXX all this 'dispatch' stuff needs to be unified + fixed up -_control_node_dispatch = { - "run": tahoe_run.run, -} - process_control_commands = [ ("run", None, tahoe_run.RunOptions, "run a node without daemonizing"), ] # type: SubCommands @@ -195,6 +190,7 @@ def parse_or_exit(config, argv, stdout, stderr): return config def dispatch(config, + reactor, stdin=sys.stdin, stdout=sys.stdout, stderr=sys.stderr): command = config.subCommand so = config.subOptions @@ -206,8 +202,8 @@ def dispatch(config, if command in create_dispatch: f = create_dispatch[command] - elif command in _control_node_dispatch: - f = _control_node_dispatch[command] + elif command == "run": + f = lambda config: tahoe_run.run(reactor, config) elif command in debug.dispatch: f = debug.dispatch[command] elif command in admin.dispatch: @@ -361,7 +357,7 @@ def _run_with_reactor(reactor, config, argv, stdout, stderr): stderr, ) d.addCallback(_maybe_enable_eliot_logging, reactor) - d.addCallback(dispatch, stdout=stdout, stderr=stderr) + d.addCallback(dispatch, reactor, stdout=stdout, stderr=stderr) def _show_exception(f): # when task.react() notices a non-SystemExit exception, it does # log.err() with the failure and then exits with rc=1. We want this diff --git a/src/allmydata/scripts/tahoe_run.py b/src/allmydata/scripts/tahoe_run.py index 72b8e3eca..dd4561a4b 100644 --- a/src/allmydata/scripts/tahoe_run.py +++ b/src/allmydata/scripts/tahoe_run.py @@ -217,7 +217,7 @@ class DaemonizeTahoeNodePlugin(object): return DaemonizeTheRealService(self.nodetype, self.basedir, so) -def run(config, runApp=twistd.runApp): +def run(reactor, config, runApp=twistd.runApp): """ Runs a Tahoe-LAFS node in the foreground. From 8b2cb79070edabd20fb9bdbb41de51458788e50a Mon Sep 17 00:00:00 2001 From: meejah Date: Sat, 17 Sep 2022 16:29:03 -0600 Subject: [PATCH 1059/2309] cleanup via reactor --- src/allmydata/scripts/tahoe_run.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/allmydata/scripts/tahoe_run.py b/src/allmydata/scripts/tahoe_run.py index dd4561a4b..a5b833233 100644 --- a/src/allmydata/scripts/tahoe_run.py +++ b/src/allmydata/scripts/tahoe_run.py @@ -269,6 +269,11 @@ def run(reactor, config, runApp=twistd.runApp): except ProcessInTheWay as e: print("ERROR: {}".format(e)) return 1 + else: + reactor.addSystemEventTrigger( + "during", "shutdown", + lambda: cleanup_pidfile(pidfile) + ) # We always pass --nodaemon so twistd.runApp does not daemonize. runApp(twistd_config) From 254a994eb53035b70a653b47b2951d6159634a23 Mon Sep 17 00:00:00 2001 From: meejah Date: Sat, 17 Sep 2022 16:41:17 -0600 Subject: [PATCH 1060/2309] flake8 --- src/allmydata/scripts/tahoe_run.py | 2 +- src/allmydata/test/test_runner.py | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/src/allmydata/scripts/tahoe_run.py b/src/allmydata/scripts/tahoe_run.py index a5b833233..7722fef51 100644 --- a/src/allmydata/scripts/tahoe_run.py +++ b/src/allmydata/scripts/tahoe_run.py @@ -266,7 +266,7 @@ def run(reactor, config, runApp=twistd.runApp): pidfile = FilePath(get_pidfile(config['basedir'])) try: check_pid_process(pidfile) - except ProcessInTheWay as e: + except (ProcessInTheWay, InvalidPidFile) as e: print("ERROR: {}".format(e)) return 1 else: diff --git a/src/allmydata/test/test_runner.py b/src/allmydata/test/test_runner.py index 9b6357f46..14d0dfb7f 100644 --- a/src/allmydata/test/test_runner.py +++ b/src/allmydata/test/test_runner.py @@ -47,9 +47,6 @@ from twisted.internet.defer import ( DeferredList, ) from twisted.python.filepath import FilePath -from twisted.python.runtime import ( - platform, -) from allmydata.util import fileutil, pollmixin from allmydata.util.encodingutil import unicode_to_argv from allmydata.test import common_util From fe80126e3fcffe56c171188c7cd5847f19bf6f7b Mon Sep 17 00:00:00 2001 From: meejah Date: Sun, 18 Sep 2022 22:39:25 -0600 Subject: [PATCH 1061/2309] fixups --- src/allmydata/scripts/tahoe_run.py | 2 +- src/allmydata/test/cli/test_run.py | 4 +++- src/allmydata/test/common_util.py | 1 + 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/allmydata/scripts/tahoe_run.py b/src/allmydata/scripts/tahoe_run.py index 7722fef51..6dfa726a3 100644 --- a/src/allmydata/scripts/tahoe_run.py +++ b/src/allmydata/scripts/tahoe_run.py @@ -267,7 +267,7 @@ def run(reactor, config, runApp=twistd.runApp): try: check_pid_process(pidfile) except (ProcessInTheWay, InvalidPidFile) as e: - print("ERROR: {}".format(e)) + print("ERROR: {}".format(e), file=err) return 1 else: reactor.addSystemEventTrigger( diff --git a/src/allmydata/test/cli/test_run.py b/src/allmydata/test/cli/test_run.py index ae869e475..6358b70dd 100644 --- a/src/allmydata/test/cli/test_run.py +++ b/src/allmydata/test/cli/test_run.py @@ -168,8 +168,10 @@ class RunTests(SyncTestCase): config['basedir'] = basedir.path config.twistd_args = [] + from twisted.internet import reactor + runs = [] - result_code = run(config, runApp=runs.append) + result_code = run(reactor, config, runApp=runs.append) self.assertThat( config.stderr.getvalue(), Contains("found invalid PID file in"), diff --git a/src/allmydata/test/common_util.py b/src/allmydata/test/common_util.py index e63c3eef8..b6d352ab1 100644 --- a/src/allmydata/test/common_util.py +++ b/src/allmydata/test/common_util.py @@ -145,6 +145,7 @@ def run_cli_native(verb, *args, **kwargs): ) d.addCallback( runner.dispatch, + reactor, stdin=stdin, stdout=stdout, stderr=stderr, From ef0b2aca1769dbdf11c5eb50b66c186d2ee9e22f Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 19 Sep 2022 10:12:11 -0400 Subject: [PATCH 1062/2309] Adjust NURL spec to new decisions. --- docs/specifications/url.rst | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/docs/specifications/url.rst b/docs/specifications/url.rst index 31fb05fad..1ce3b2a7f 100644 --- a/docs/specifications/url.rst +++ b/docs/specifications/url.rst @@ -47,27 +47,27 @@ This can be considered to expand to "**N**\ ew URLs" or "Authe\ **N**\ ticating The anticipated use for a **NURL** will still be to establish a TLS connection to a peer. The protocol run over that TLS connection could be Foolscap though it is more likely to be an HTTP-based protocol (such as GBS). +Unlike fURLs, only a single net-loc is included, for consistency with other forms of URLs. +As a result, multiple NURLs may be available for a single server. + Syntax ------ The EBNF for a NURL is as follows:: - nurl = scheme, hash, "@", net-loc-list, "/", swiss-number, [ version1 ] - - scheme = "pb://" + nurl = tcp-nurl | tor-nurl | i2p-nurl + tcp-nurl = "pb://", hash, "@", tcp-loc, "/", swiss-number, [ version1 ] + tor-nurl = "pb+tor://", hash, "@", tcp-loc, "/", swiss-number, [ version1 ] + i2p-nurl = "pb+i2p://", hash, "@", i2p-loc, "/", swiss-number, [ version1 ] hash = unreserved - net-loc-list = net-loc, [ { ",", net-loc } ] - net-loc = tcp-loc | tor-loc | i2p-loc - - tcp-loc = [ "tcp:" ], hostname, [ ":" port ] - tor-loc = "tor:", hostname, [ ":" port ] - i2p-loc = "i2p:", i2p-addr, [ ":" port ] - - i2p-addr = { unreserved }, ".i2p" + tcp-loc = hostname, [ ":" port ] hostname = domain | IPv4address | IPv6address + i2p-loc = i2p-addr, [ ":" port ] + i2p-addr = { unreserved }, ".i2p" + swiss-number = segment version1 = "#v=1" From 4b2725df006ae172f267072a8bcb222b6be6aad9 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 20 Sep 2022 10:09:43 -0400 Subject: [PATCH 1063/2309] Try to prevent leaking timeouts. --- src/allmydata/protocol_switch.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/allmydata/protocol_switch.py b/src/allmydata/protocol_switch.py index 89570436c..a17f3055c 100644 --- a/src/allmydata/protocol_switch.py +++ b/src/allmydata/protocol_switch.py @@ -151,6 +151,10 @@ class _FoolscapOrHttps(Protocol, metaclass=_PretendToBeNegotiation): 30, self.transport.abortConnection ) + def connectionLost(self, reason): + if self._timeout.active(): + self._timeout.cancel() + def dataReceived(self, data: bytes) -> None: """Handle incoming data. From 81c8e1c57b8b926ebb3a396f653d7149bd4f6577 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 20 Sep 2022 14:24:02 -0600 Subject: [PATCH 1064/2309] windows is special --- src/allmydata/test/test_runner.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/test_runner.py b/src/allmydata/test/test_runner.py index 14d0dfb7f..5d8143558 100644 --- a/src/allmydata/test/test_runner.py +++ b/src/allmydata/test/test_runner.py @@ -42,6 +42,7 @@ from twisted.trial import unittest from twisted.internet import reactor from twisted.python import usage +from twisted.python.runtime import platform from twisted.internet.defer import ( inlineCallbacks, DeferredList, @@ -516,8 +517,12 @@ class RunNode(common_util.SignalMixin, unittest.TestCase, pollmixin.PollMixin): ) yield tahoe.stop_and_wait() - # twistd.pid should be gone by now. - self.assertFalse(tahoe.twistd_pid_file.exists()) + # twistd.pid should be gone by now -- except on Windows, where + # killing a subprocess immediately exits with no chance for + # any shutdown code (that is, no Twisted shutdown hooks can + # run). + if not platform.isWindows(): + self.assertFalse(tahoe.twistd_pid_file.exists()) def _remove(self, res, file): fileutil.remove(file) From 6db1476dacc99c00a12d88b1c6af6a8aa76f3404 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 20 Sep 2022 14:44:21 -0600 Subject: [PATCH 1065/2309] comment typo --- src/allmydata/scripts/tahoe_run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/scripts/tahoe_run.py b/src/allmydata/scripts/tahoe_run.py index 6dfa726a3..721ced376 100644 --- a/src/allmydata/scripts/tahoe_run.py +++ b/src/allmydata/scripts/tahoe_run.py @@ -244,7 +244,7 @@ def run(reactor, config, runApp=twistd.runApp): "--rundir", basedir, ] if sys.platform != "win32": - # turn off Twisted's pid-file to use our own -- but only on + # turn off Twisted's pid-file to use our own -- but not on # windows, because twistd doesn't know about pidfiles there twistd_args.extend(["--pidfile", None]) twistd_args.extend(config.twistd_args) From 0eeb11c9cd45598fbe2e5bdccb4f9cf50fe222f3 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 20 Sep 2022 14:44:51 -0600 Subject: [PATCH 1066/2309] after shutdown --- src/allmydata/scripts/tahoe_run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/scripts/tahoe_run.py b/src/allmydata/scripts/tahoe_run.py index 721ced376..40c4a6612 100644 --- a/src/allmydata/scripts/tahoe_run.py +++ b/src/allmydata/scripts/tahoe_run.py @@ -271,7 +271,7 @@ def run(reactor, config, runApp=twistd.runApp): return 1 else: reactor.addSystemEventTrigger( - "during", "shutdown", + "after", "shutdown", lambda: cleanup_pidfile(pidfile) ) From 77bc83d341794afbd0c6884fb5e0e914dbe90632 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 20 Sep 2022 14:45:19 -0600 Subject: [PATCH 1067/2309] incorrectly removed --- src/allmydata/scripts/tahoe_run.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/allmydata/scripts/tahoe_run.py b/src/allmydata/scripts/tahoe_run.py index 40c4a6612..eb4bb0b66 100644 --- a/src/allmydata/scripts/tahoe_run.py +++ b/src/allmydata/scripts/tahoe_run.py @@ -276,5 +276,6 @@ def run(reactor, config, runApp=twistd.runApp): ) # We always pass --nodaemon so twistd.runApp does not daemonize. + print("running node in %s" % (quoted_basedir,), file=out) runApp(twistd_config) return 0 From 1f29cc9c29e42a472ce893259f5bdbf2a31c00e0 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 20 Sep 2022 14:50:46 -0600 Subject: [PATCH 1068/2309] windows special --- src/allmydata/test/test_runner.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/test_runner.py b/src/allmydata/test/test_runner.py index 5d8143558..cf6e9f3b5 100644 --- a/src/allmydata/test/test_runner.py +++ b/src/allmydata/test/test_runner.py @@ -605,8 +605,10 @@ class RunNode(common_util.SignalMixin, unittest.TestCase, pollmixin.PollMixin): ), ) - # It should not be running. - self.assertFalse(tahoe.twistd_pid_file.exists()) + # It should not be running (but windows shutdown can't run + # code so the PID file still exists there). + if not platform.isWindows(): + self.assertFalse(tahoe.twistd_pid_file.exists()) # Wait for the operation to *complete*. If we got this far it's # because we got the expected message so we can expect the "tahoe ..." From 5973196931d2143f68a34d9b01857339582ec5c0 Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 21 Sep 2022 19:00:27 -0600 Subject: [PATCH 1069/2309] refactor: use filelock and test it --- setup.py | 1 + src/allmydata/test/test_runner.py | 47 ++++++++++++ src/allmydata/util/pid.py | 117 ++++++++++++++++++------------ 3 files changed, 119 insertions(+), 46 deletions(-) diff --git a/setup.py b/setup.py index bd16a61ce..d99831347 100644 --- a/setup.py +++ b/setup.py @@ -141,6 +141,7 @@ install_requires = [ # for pid-file support "psutil", + "filelock", ] setup_requires = [ diff --git a/src/allmydata/test/test_runner.py b/src/allmydata/test/test_runner.py index cf6e9f3b5..5a8311649 100644 --- a/src/allmydata/test/test_runner.py +++ b/src/allmydata/test/test_runner.py @@ -50,6 +50,11 @@ from twisted.internet.defer import ( from twisted.python.filepath import FilePath from allmydata.util import fileutil, pollmixin from allmydata.util.encodingutil import unicode_to_argv +from allmydata.util.pid import ( + check_pid_process, + _pidfile_to_lockpath, + ProcessInTheWay, +) from allmydata.test import common_util import allmydata from allmydata.scripts.runner import ( @@ -617,3 +622,45 @@ class RunNode(common_util.SignalMixin, unittest.TestCase, pollmixin.PollMixin): # What's left is a perfect indicator that the process has exited and # we won't get blamed for leaving the reactor dirty. yield client_running + + +class PidFileLocking(SyncTestCase): + """ + Direct tests for allmydata.util.pid functions + """ + + def test_locking(self): + """ + Fail to create a pidfile if another process has the lock already. + """ + # this can't just be "our" process because the locking library + # allows the same process to acquire a lock multiple times. + pidfile = FilePath("foo") + lockfile = _pidfile_to_lockpath(pidfile) + + with open("code.py", "w") as f: + f.write( + "\n".join([ + "import filelock, time", + "with filelock.FileLock('{}', timeout=1):".format(lockfile.path), + " print('.', flush=True)", + " time.sleep(5)", + ]) + ) + proc = Popen( + [sys.executable, "code.py"], + stdout=PIPE, + stderr=PIPE, + start_new_session=True, + ) + # make sure our subprocess has had time to acquire the lock + # for sure (from the "." it prints) + self.assertThat( + proc.stdout.read(1), + Equals(b".") + ) + + # we should not be able to acuire this corresponding lock as well + with self.assertRaises(ProcessInTheWay): + check_pid_process(pidfile) + proc.terminate() diff --git a/src/allmydata/util/pid.py b/src/allmydata/util/pid.py index ff8129bbc..e256615d6 100644 --- a/src/allmydata/util/pid.py +++ b/src/allmydata/util/pid.py @@ -1,6 +1,10 @@ import os import psutil +# the docs are a little misleading, but this is either WindowsFileLock +# or UnixFileLock depending upon the platform we're currently on +from filelock import FileLock, Timeout + class ProcessInTheWay(Exception): """ @@ -20,6 +24,14 @@ class CannotRemovePidFile(Exception): """ +def _pidfile_to_lockpath(pidfile): + """ + internal helper. + :returns FilePath: a path to use for file-locking the given pidfile + """ + return pidfile.sibling("{}.lock".format(pidfile.basename())) + + def check_pid_process(pidfile, find_process=None): """ If another instance appears to be running already, raise an @@ -34,57 +46,70 @@ def check_pid_process(pidfile, find_process=None): :raises ProcessInTheWay: if a running process exists at our PID """ find_process = psutil.Process if find_process is None else find_process - # check if we have another instance running already - if pidfile.exists(): - with pidfile.open("r") as f: - content = f.read().decode("utf8").strip() - try: - pid, starttime = content.split() - pid = int(pid) - starttime = float(starttime) - except ValueError: - raise InvalidPidFile( - "found invalid PID file in {}".format( - pidfile - ) - ) - try: - # if any other process is running at that PID, let the - # user decide if this is another legitimate - # instance. Automated programs may use the start-time to - # help decide this (if the PID is merely recycled, the - # start-time won't match). - find_process(pid) - raise ProcessInTheWay( - "A process is already running as PID {}".format(pid) - ) - except psutil.NoSuchProcess: - print( - "'{pidpath}' refers to {pid} that isn't running".format( - pidpath=pidfile.path, - pid=pid, - ) - ) - # nothing is running at that PID so it must be a stale file - pidfile.remove() + lock_path = _pidfile_to_lockpath(pidfile) - # write our PID + start-time to the pid-file - pid = os.getpid() - starttime = find_process(pid).create_time() - with pidfile.open("w") as f: - f.write("{} {}\n".format(pid, starttime).encode("utf8")) + try: + # a short timeout is fine, this lock should only be active + # while someone is reading or deleting the pidfile .. and + # facilitates testing the locking itself. + with FileLock(lock_path.path, timeout=2): + # check if we have another instance running already + if pidfile.exists(): + with pidfile.open("r") as f: + content = f.read().decode("utf8").strip() + try: + pid, starttime = content.split() + pid = int(pid) + starttime = float(starttime) + except ValueError: + raise InvalidPidFile( + "found invalid PID file in {}".format( + pidfile + ) + ) + try: + # if any other process is running at that PID, let the + # user decide if this is another legitimate + # instance. Automated programs may use the start-time to + # help decide this (if the PID is merely recycled, the + # start-time won't match). + find_process(pid) + raise ProcessInTheWay( + "A process is already running as PID {}".format(pid) + ) + except psutil.NoSuchProcess: + print( + "'{pidpath}' refers to {pid} that isn't running".format( + pidpath=pidfile.path, + pid=pid, + ) + ) + # nothing is running at that PID so it must be a stale file + pidfile.remove() + + # write our PID + start-time to the pid-file + pid = os.getpid() + starttime = find_process(pid).create_time() + with pidfile.open("w") as f: + f.write("{} {}\n".format(pid, starttime).encode("utf8")) + except Timeout: + raise ProcessInTheWay( + "Another process is still locking {}".format(pidfile.path) + ) def cleanup_pidfile(pidfile): """ Safely clean up a PID-file """ - try: - pidfile.remove() - except Exception as e: - raise CannotRemovePidFile( - "Couldn't remove '{pidfile}': {err}.".format( - pidfile=pidfile.path, - err=e, + lock_path = _pidfile_to_lockpath(pidfile) + with FileLock(lock_path.path): + try: + pidfile.remove() + except Exception as e: + raise CannotRemovePidFile( + "Couldn't remove '{pidfile}': {err}.".format( + pidfile=pidfile.path, + err=e, + ) ) - ) From ea39e4ca6902daad125596f5e1e2b81989e9cb6b Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 21 Sep 2022 19:01:28 -0600 Subject: [PATCH 1070/2309] docstring --- src/allmydata/test/cli/test_run.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/allmydata/test/cli/test_run.py b/src/allmydata/test/cli/test_run.py index 6358b70dd..551164d3c 100644 --- a/src/allmydata/test/cli/test_run.py +++ b/src/allmydata/test/cli/test_run.py @@ -185,6 +185,9 @@ class RunTests(SyncTestCase): @given(text()) def test_pidfile_contents(self, content): + """ + invalid contents for a pidfile raise errors + """ assume(not self.good_file_content_re.match(content)) pidfile = FilePath("pidfile") pidfile.setContent(content.encode("utf8")) From 56775dde192c90b48fa85cfcb4a2651f5b264791 Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 21 Sep 2022 19:05:30 -0600 Subject: [PATCH 1071/2309] refactor: parsing in a function --- src/allmydata/scripts/tahoe_run.py | 6 +++--- src/allmydata/util/pid.py | 34 +++++++++++++++++++----------- 2 files changed, 25 insertions(+), 15 deletions(-) diff --git a/src/allmydata/scripts/tahoe_run.py b/src/allmydata/scripts/tahoe_run.py index eb4bb0b66..4d17492d4 100644 --- a/src/allmydata/scripts/tahoe_run.py +++ b/src/allmydata/scripts/tahoe_run.py @@ -29,6 +29,7 @@ from allmydata.util.encodingutil import listdir_unicode, quote_local_unicode_pat from allmydata.util.configutil import UnknownConfigError from allmydata.util.deferredutil import HookMixin from allmydata.util.pid import ( + parse_pidfile, check_pid_process, cleanup_pidfile, ProcessInTheWay, @@ -66,10 +67,9 @@ def get_pid_from_pidfile(pidfile): except EnvironmentError: return None - pid, _ = data.split() try: - pid = int(pid) - except ValueError: + pid, _ = parse_pidfile(pidfile) + except InvalidPidFile: return -1 return pid diff --git a/src/allmydata/util/pid.py b/src/allmydata/util/pid.py index e256615d6..1cb2cc45a 100644 --- a/src/allmydata/util/pid.py +++ b/src/allmydata/util/pid.py @@ -32,6 +32,27 @@ def _pidfile_to_lockpath(pidfile): return pidfile.sibling("{}.lock".format(pidfile.basename())) +def parse_pidfile(pidfile): + """ + :param FilePath pidfile: + :returns tuple: 2-tuple of pid, creation-time as int, float + :raises InvalidPidFile: on error + """ + with pidfile.open("r") as f: + content = f.read().decode("utf8").strip() + try: + pid, starttime = content.split() + pid = int(pid) + starttime = float(starttime) + except ValueError: + raise InvalidPidFile( + "found invalid PID file in {}".format( + pidfile + ) + ) + return pid, startime + + def check_pid_process(pidfile, find_process=None): """ If another instance appears to be running already, raise an @@ -55,18 +76,7 @@ def check_pid_process(pidfile, find_process=None): with FileLock(lock_path.path, timeout=2): # check if we have another instance running already if pidfile.exists(): - with pidfile.open("r") as f: - content = f.read().decode("utf8").strip() - try: - pid, starttime = content.split() - pid = int(pid) - starttime = float(starttime) - except ValueError: - raise InvalidPidFile( - "found invalid PID file in {}".format( - pidfile - ) - ) + pid, starttime = parse_pidfile(pidfile) try: # if any other process is running at that PID, let the # user decide if this is another legitimate From 390c8c52da3f9f0ea21ad789caf7c8f6ae9bbc74 Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 21 Sep 2022 19:23:30 -0600 Subject: [PATCH 1072/2309] formatting + typo --- newsfragments/3926.incompat | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/newsfragments/3926.incompat b/newsfragments/3926.incompat index 3f58b4ba8..674ad289c 100644 --- a/newsfragments/3926.incompat +++ b/newsfragments/3926.incompat @@ -1,10 +1,10 @@ -Record both the PID and the process creation-time +Record both the PID and the process creation-time: -A new kind of pidfile in `running.process` records both +a new kind of pidfile in `running.process` records both the PID and the creation-time of the process. This facilitates automatic discovery of a "stale" pidfile that points to a currently-running process. If the recorded creation-time matches the creation-time of the running process, then it is a still-running -`tahoe run` proecss. Otherwise, the file is stale. +`tahoe run` process. Otherwise, the file is stale. The `twistd.pid` file is no longer present. \ No newline at end of file From e111694b3e33e54db974acbd057d74380c6de4ce Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 21 Sep 2022 19:28:09 -0600 Subject: [PATCH 1073/2309] get rid of find_process= --- src/allmydata/util/pid.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/src/allmydata/util/pid.py b/src/allmydata/util/pid.py index 1cb2cc45a..d681d819e 100644 --- a/src/allmydata/util/pid.py +++ b/src/allmydata/util/pid.py @@ -53,7 +53,7 @@ def parse_pidfile(pidfile): return pid, startime -def check_pid_process(pidfile, find_process=None): +def check_pid_process(pidfile): """ If another instance appears to be running already, raise an exception. Otherwise, write our PID + start time to the pidfile @@ -61,12 +61,8 @@ def check_pid_process(pidfile, find_process=None): :param FilePath pidfile: the file to read/write our PID from. - :param Callable find_process: None, or a custom way to get a - Process objet (usually for tests) - :raises ProcessInTheWay: if a running process exists at our PID """ - find_process = psutil.Process if find_process is None else find_process lock_path = _pidfile_to_lockpath(pidfile) try: @@ -83,7 +79,7 @@ def check_pid_process(pidfile, find_process=None): # instance. Automated programs may use the start-time to # help decide this (if the PID is merely recycled, the # start-time won't match). - find_process(pid) + psutil.Process(pid) raise ProcessInTheWay( "A process is already running as PID {}".format(pid) ) @@ -98,8 +94,7 @@ def check_pid_process(pidfile, find_process=None): pidfile.remove() # write our PID + start-time to the pid-file - pid = os.getpid() - starttime = find_process(pid).create_time() + starttime = psutil.Process().create_time() with pidfile.open("w") as f: f.write("{} {}\n".format(pid, starttime).encode("utf8")) except Timeout: From 0a09d23525fc4be928fa288c1301327d6eaccf32 Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 21 Sep 2022 19:29:40 -0600 Subject: [PATCH 1074/2309] more docstring --- src/allmydata/util/pid.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/allmydata/util/pid.py b/src/allmydata/util/pid.py index d681d819e..f965d72ab 100644 --- a/src/allmydata/util/pid.py +++ b/src/allmydata/util/pid.py @@ -105,7 +105,8 @@ def check_pid_process(pidfile): def cleanup_pidfile(pidfile): """ - Safely clean up a PID-file + Remove the pidfile specified (respecting locks). If anything at + all goes wrong, `CannotRemovePidFile` is raised. """ lock_path = _pidfile_to_lockpath(pidfile) with FileLock(lock_path.path): From 6eebbda7c6c06732932c50a96ba1a5315c9d35f4 Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 21 Sep 2022 20:07:29 -0600 Subject: [PATCH 1075/2309] documentation, example code --- docs/check_running.py | 47 +++++++++++++++++++++++++++++++++++++++++++ docs/running.rst | 29 ++++++++++++++++++++++++++ 2 files changed, 76 insertions(+) create mode 100644 docs/check_running.py diff --git a/docs/check_running.py b/docs/check_running.py new file mode 100644 index 000000000..ecc55da34 --- /dev/null +++ b/docs/check_running.py @@ -0,0 +1,47 @@ + +import psutil +import filelock + + +def can_spawn_tahoe(pidfile): + """ + Determine if we can spawn a Tahoe-LAFS for the given pidfile. That + pidfile may be deleted if it is stale. + + :param pathlib.Path pidfile: the file to check, that is the Path + to "running.process" in a Tahoe-LAFS configuration directory + + :returns bool: True if we can spawn `tahoe run` here + """ + lockpath = pidfile.parent / (pidfile.name + ".lock") + with filelock.FileLock(lockpath): + try: + with pidfile.open("r") as f: + pid, create_time = f.read().strip().split(" ", 1) + except FileNotFoundError: + return True + + # somewhat interesting: we have a pidfile + pid = int(pid) + create_time = float(create_time) + + try: + proc = psutil.Process(pid) + # most interesting case: there _is_ a process running at the + # recorded PID -- but did it just happen to get that PID, or + # is it the very same one that wrote the file? + if create_time == proc.create_time(): + # _not_ stale! another intance is still running against + # this configuration + return False + + except psutil.NoSuchProcess: + pass + + # the file is stale + pidfile.unlink() + return True + + +from pathlib import Path +print("can spawn?", can_spawn_tahoe(Path("running.process"))) diff --git a/docs/running.rst b/docs/running.rst index 406c8200b..2cff59928 100644 --- a/docs/running.rst +++ b/docs/running.rst @@ -124,6 +124,35 @@ Tahoe-LAFS. .. _magic wormhole: https://magic-wormhole.io/ +Multiple Instances +------------------ + +Running multiple instances against the same configuration directory isn't supported. +This will lead to undefined behavior and could corrupt the configuration state. + +We attempt to avoid this situation with a "pidfile"-style file in the config directory called ``running.process``. +There may be a parallel file called ``running.process.lock`` in existence. + +The ``.lock`` file exists to make sure only one process modifies ``running.process`` at once. +The lock file is managed by the `lockfile `_ library. +If you wish to make use of ``running.process`` for any reason you should also lock it and follow the semantics of lockfile. + +If ``running.process` exists it file contains the PID and the creation-time of the process. +When no such file exists, there is no other process running on this configuration. +If there is a ``running.process`` file, it may be a leftover file or it may indicate that another process is running against this config. +To tell the difference, determine if the PID in the file exists currently. +If it does, check the creation-time of the process versus the one in the file. +If these match, there is another process currently running. +Otherwise, the file is stale -- it should be removed before starting Tahoe-LAFS. + +Some example Python code to check the above situations: + +.. literalinclude:: check_running.py + + + + + A note about small grids ------------------------ From 930f4029f370313222b5c0872754f1db16434029 Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 21 Sep 2022 20:07:46 -0600 Subject: [PATCH 1076/2309] properly write pid, create-time --- src/allmydata/util/pid.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/util/pid.py b/src/allmydata/util/pid.py index f965d72ab..1a833f285 100644 --- a/src/allmydata/util/pid.py +++ b/src/allmydata/util/pid.py @@ -94,9 +94,9 @@ def check_pid_process(pidfile): pidfile.remove() # write our PID + start-time to the pid-file - starttime = psutil.Process().create_time() + proc = psutil.Process() with pidfile.open("w") as f: - f.write("{} {}\n".format(pid, starttime).encode("utf8")) + f.write("{} {}\n".format(proc.pid, proc.create_time()).encode("utf8")) except Timeout: raise ProcessInTheWay( "Another process is still locking {}".format(pidfile.path) From 8474ecf83d46a25a473269f4f7907a5eb6e6e552 Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 21 Sep 2022 20:15:07 -0600 Subject: [PATCH 1077/2309] typo --- src/allmydata/util/pid.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/util/pid.py b/src/allmydata/util/pid.py index 1a833f285..c13dc32f3 100644 --- a/src/allmydata/util/pid.py +++ b/src/allmydata/util/pid.py @@ -50,7 +50,7 @@ def parse_pidfile(pidfile): pidfile ) ) - return pid, startime + return pid, starttime def check_pid_process(pidfile): From fedea9696412d7397f58c11ea04a9148c55f8fd8 Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 21 Sep 2022 20:26:14 -0600 Subject: [PATCH 1078/2309] less state --- src/allmydata/test/cli/test_run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/cli/test_run.py b/src/allmydata/test/cli/test_run.py index 551164d3c..e84f52096 100644 --- a/src/allmydata/test/cli/test_run.py +++ b/src/allmydata/test/cli/test_run.py @@ -168,7 +168,7 @@ class RunTests(SyncTestCase): config['basedir'] = basedir.path config.twistd_args = [] - from twisted.internet import reactor + reactor = MemoryReactor() runs = [] result_code = run(reactor, config, runApp=runs.append) From 8d8b0e6f01cdd8ab1f64593eda772a8b7db6c3d6 Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 21 Sep 2022 20:40:25 -0600 Subject: [PATCH 1079/2309] cleanup --- src/allmydata/scripts/tahoe_run.py | 6 +----- src/allmydata/util/pid.py | 1 - 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/src/allmydata/scripts/tahoe_run.py b/src/allmydata/scripts/tahoe_run.py index 4d17492d4..e22e8c307 100644 --- a/src/allmydata/scripts/tahoe_run.py +++ b/src/allmydata/scripts/tahoe_run.py @@ -62,13 +62,9 @@ def get_pid_from_pidfile(pidfile): inaccessible, ``-1`` if PID file invalid. """ try: - with open(pidfile, "r") as f: - data = f.read().strip() + pid, _ = parse_pidfile(pidfile) except EnvironmentError: return None - - try: - pid, _ = parse_pidfile(pidfile) except InvalidPidFile: return -1 diff --git a/src/allmydata/util/pid.py b/src/allmydata/util/pid.py index c13dc32f3..f12c201d1 100644 --- a/src/allmydata/util/pid.py +++ b/src/allmydata/util/pid.py @@ -1,4 +1,3 @@ -import os import psutil # the docs are a little misleading, but this is either WindowsFileLock From 4f5a1ac37222e51974bcb0a28b5ec9e0e6c0e944 Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 21 Sep 2022 23:36:23 -0600 Subject: [PATCH 1080/2309] naming? --- src/allmydata/test/test_runner.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/test_runner.py b/src/allmydata/test/test_runner.py index 5a8311649..962dffd1a 100644 --- a/src/allmydata/test/test_runner.py +++ b/src/allmydata/test/test_runner.py @@ -638,7 +638,7 @@ class PidFileLocking(SyncTestCase): pidfile = FilePath("foo") lockfile = _pidfile_to_lockpath(pidfile) - with open("code.py", "w") as f: + with open("other_lock.py", "w") as f: f.write( "\n".join([ "import filelock, time", @@ -648,7 +648,7 @@ class PidFileLocking(SyncTestCase): ]) ) proc = Popen( - [sys.executable, "code.py"], + [sys.executable, "other_lock.py"], stdout=PIPE, stderr=PIPE, start_new_session=True, From 8ebe331c358789f3af8dd9d64607ee63404077d7 Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 22 Sep 2022 00:11:20 -0600 Subject: [PATCH 1081/2309] maybe a newline helps --- src/allmydata/test/test_runner.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/test_runner.py b/src/allmydata/test/test_runner.py index 962dffd1a..3d8180c7a 100644 --- a/src/allmydata/test/test_runner.py +++ b/src/allmydata/test/test_runner.py @@ -643,7 +643,7 @@ class PidFileLocking(SyncTestCase): "\n".join([ "import filelock, time", "with filelock.FileLock('{}', timeout=1):".format(lockfile.path), - " print('.', flush=True)", + " print('.\n', flush=True)", " time.sleep(5)", ]) ) @@ -657,7 +657,7 @@ class PidFileLocking(SyncTestCase): # for sure (from the "." it prints) self.assertThat( proc.stdout.read(1), - Equals(b".") + Equals(b".\n") ) # we should not be able to acuire this corresponding lock as well From a182a2507987213b519bcb22c6f49eec0004830c Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 22 Sep 2022 21:43:20 -0600 Subject: [PATCH 1082/2309] backslashes --- src/allmydata/test/test_runner.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/allmydata/test/test_runner.py b/src/allmydata/test/test_runner.py index 3d8180c7a..e6b7b746f 100644 --- a/src/allmydata/test/test_runner.py +++ b/src/allmydata/test/test_runner.py @@ -641,9 +641,10 @@ class PidFileLocking(SyncTestCase): with open("other_lock.py", "w") as f: f.write( "\n".join([ - "import filelock, time", + "import filelock, time, sys", "with filelock.FileLock('{}', timeout=1):".format(lockfile.path), - " print('.\n', flush=True)", + " sys.stdout.write('.\\n')", + " sys.stdout.flush()", " time.sleep(5)", ]) ) @@ -656,7 +657,7 @@ class PidFileLocking(SyncTestCase): # make sure our subprocess has had time to acquire the lock # for sure (from the "." it prints) self.assertThat( - proc.stdout.read(1), + proc.stdout.read(2), Equals(b".\n") ) From 62b92585c62c44694e5db7a8769a772b2d712a07 Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 22 Sep 2022 23:57:19 -0600 Subject: [PATCH 1083/2309] simplify --- src/allmydata/test/test_runner.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/allmydata/test/test_runner.py b/src/allmydata/test/test_runner.py index e6b7b746f..5431fbaa9 100644 --- a/src/allmydata/test/test_runner.py +++ b/src/allmydata/test/test_runner.py @@ -656,12 +656,9 @@ class PidFileLocking(SyncTestCase): ) # make sure our subprocess has had time to acquire the lock # for sure (from the "." it prints) - self.assertThat( - proc.stdout.read(2), - Equals(b".\n") - ) + proc.stdout.read(2), - # we should not be able to acuire this corresponding lock as well + # acquiring the same lock should fail; it is locked by the subprocess with self.assertRaises(ProcessInTheWay): check_pid_process(pidfile) proc.terminate() From 7fdeb8797e8164f1b0fd15ddda4108417545e00d Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 23 Sep 2022 00:26:39 -0600 Subject: [PATCH 1084/2309] hardcoding bad --- src/allmydata/test/test_runner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/test_runner.py b/src/allmydata/test/test_runner.py index 5431fbaa9..f414ed8b3 100644 --- a/src/allmydata/test/test_runner.py +++ b/src/allmydata/test/test_runner.py @@ -635,7 +635,7 @@ class PidFileLocking(SyncTestCase): """ # this can't just be "our" process because the locking library # allows the same process to acquire a lock multiple times. - pidfile = FilePath("foo") + pidfile = FilePath(self.mktemp()) lockfile = _pidfile_to_lockpath(pidfile) with open("other_lock.py", "w") as f: From f2cfd96b5e3af0fe82a7bf1ef770cad08d3969cd Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 23 Sep 2022 01:04:58 -0600 Subject: [PATCH 1085/2309] typo, longer timeout --- src/allmydata/test/test_runner.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/test_runner.py b/src/allmydata/test/test_runner.py index f414ed8b3..b80891642 100644 --- a/src/allmydata/test/test_runner.py +++ b/src/allmydata/test/test_runner.py @@ -645,7 +645,7 @@ class PidFileLocking(SyncTestCase): "with filelock.FileLock('{}', timeout=1):".format(lockfile.path), " sys.stdout.write('.\\n')", " sys.stdout.flush()", - " time.sleep(5)", + " time.sleep(10)", ]) ) proc = Popen( @@ -656,7 +656,7 @@ class PidFileLocking(SyncTestCase): ) # make sure our subprocess has had time to acquire the lock # for sure (from the "." it prints) - proc.stdout.read(2), + proc.stdout.read(2) # acquiring the same lock should fail; it is locked by the subprocess with self.assertRaises(ProcessInTheWay): From 8991509f8c82642d75e3070ad7ae02bfe061977d Mon Sep 17 00:00:00 2001 From: meejah Date: Sun, 25 Sep 2022 00:16:40 -0600 Subject: [PATCH 1086/2309] blackslashes.... --- src/allmydata/test/test_runner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/test_runner.py b/src/allmydata/test/test_runner.py index b80891642..f8211ec02 100644 --- a/src/allmydata/test/test_runner.py +++ b/src/allmydata/test/test_runner.py @@ -642,7 +642,7 @@ class PidFileLocking(SyncTestCase): f.write( "\n".join([ "import filelock, time, sys", - "with filelock.FileLock('{}', timeout=1):".format(lockfile.path), + "with filelock.FileLock(r'{}', timeout=1):".format(lockfile.path), " sys.stdout.write('.\\n')", " sys.stdout.flush()", " time.sleep(10)", From d42c00ae9293dd18c9f1efd22e86984c4725f222 Mon Sep 17 00:00:00 2001 From: meejah Date: Sun, 25 Sep 2022 00:46:30 -0600 Subject: [PATCH 1087/2309] do all checks with lock --- docs/check_running.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/docs/check_running.py b/docs/check_running.py index ecc55da34..55aae0015 100644 --- a/docs/check_running.py +++ b/docs/check_running.py @@ -21,22 +21,22 @@ def can_spawn_tahoe(pidfile): except FileNotFoundError: return True - # somewhat interesting: we have a pidfile - pid = int(pid) - create_time = float(create_time) + # somewhat interesting: we have a pidfile + pid = int(pid) + create_time = float(create_time) - try: - proc = psutil.Process(pid) - # most interesting case: there _is_ a process running at the - # recorded PID -- but did it just happen to get that PID, or - # is it the very same one that wrote the file? - if create_time == proc.create_time(): - # _not_ stale! another intance is still running against - # this configuration - return False + try: + proc = psutil.Process(pid) + # most interesting case: there _is_ a process running at the + # recorded PID -- but did it just happen to get that PID, or + # is it the very same one that wrote the file? + if create_time == proc.create_time(): + # _not_ stale! another intance is still running against + # this configuration + return False - except psutil.NoSuchProcess: - pass + except psutil.NoSuchProcess: + pass # the file is stale pidfile.unlink() From d16d233872df95b8e3876e3aa32e0fdb30cc9f98 Mon Sep 17 00:00:00 2001 From: meejah Date: Sun, 25 Sep 2022 00:47:58 -0600 Subject: [PATCH 1088/2309] wording --- docs/running.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/running.rst b/docs/running.rst index 2cff59928..b487f4ae3 100644 --- a/docs/running.rst +++ b/docs/running.rst @@ -128,7 +128,7 @@ Multiple Instances ------------------ Running multiple instances against the same configuration directory isn't supported. -This will lead to undefined behavior and could corrupt the configuration state. +This will lead to undefined behavior and could corrupt the configuration or state. We attempt to avoid this situation with a "pidfile"-style file in the config directory called ``running.process``. There may be a parallel file called ``running.process.lock`` in existence. From 04b0c30c11343838b246ed276a1fdac230383594 Mon Sep 17 00:00:00 2001 From: meejah Date: Sun, 25 Sep 2022 14:08:05 -0600 Subject: [PATCH 1089/2309] clean up comments --- integration/test_grid_manager.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/integration/test_grid_manager.py b/integration/test_grid_manager.py index b24149a3b..d89f1e8f6 100644 --- a/integration/test_grid_manager.py +++ b/integration/test_grid_manager.py @@ -261,7 +261,7 @@ def test_accept_storage_server(reactor, request, temp_dir, flog_gatherer, port_a gm_privkey_bytes = json.loads(gm_config)['private_key'].encode('ascii') gm_privkey, gm_pubkey = ed25519.signing_keypair_from_string(gm_privkey_bytes) - # create certificate for the storage-servers + # create certificates for all storage-servers servers = ( ("happy0", happy0), ("happy1", happy1), @@ -278,7 +278,8 @@ def test_accept_storage_server(reactor, request, temp_dir, flog_gatherer, port_a ) assert json.loads(gm_config)['storage_servers'].keys() == {'happy0', 'happy1'} - print("inserting certificates") + # add the certificates from the grid-manager to the storage servers + print("inserting storage-server certificates") for st_name, st in servers: cert = yield _run_gm( reactor, "--config", "-", "sign", st_name, "1", @@ -297,8 +298,7 @@ def test_accept_storage_server(reactor, request, temp_dir, flog_gatherer, port_a yield happy0.restart(reactor, request) yield happy1.restart(reactor, request) - # configure edna to have the grid-manager certificate - + # configure edna (a client) to have the grid-manager certificate edna = yield grid.add_client("edna", needed=2, happy=2, total=2) config = configutil.get_config(join(edna.process.node_dir, "tahoe.cfg")) @@ -309,6 +309,7 @@ def test_accept_storage_server(reactor, request, temp_dir, flog_gatherer, port_a yield edna.restart(reactor, request, servers=2) + # confirm that Edna will upload to the GridManager-enabled Grid yield util.run_tahoe( reactor, request, "--node-directory", edna.process.node_dir, "put", "-", From af227fb31517b11c1117eb0749e7231afc9d9e62 Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 26 Sep 2022 00:02:40 -0600 Subject: [PATCH 1090/2309] coverage for grid-manager tests --- integration/test_grid_manager.py | 53 +++++++++++++++++--------------- 1 file changed, 29 insertions(+), 24 deletions(-) diff --git a/integration/test_grid_manager.py b/integration/test_grid_manager.py index d89f1e8f6..0136a11ac 100644 --- a/integration/test_grid_manager.py +++ b/integration/test_grid_manager.py @@ -28,16 +28,21 @@ import pytest_twisted @inlineCallbacks -def _run_gm(reactor, *args, **kwargs): +def _run_gm(reactor, request, *args, **kwargs): """ Run the grid-manager process, passing all arguments as extra CLI args. :returns: all process output """ + if request.config.getoption('coverage'): + base_args = ("-b", "-m", "coverage", "run", "-m", "allmydata.cli.grid_manager") + else: + base_args = ("-m", "allmydata.cli.grid_manager") + output, errput, exit_code = yield getProcessOutputAndValue( sys.executable, - ("-m", "allmydata.cli.grid_manager") + args, + base_args + args, reactor=reactor, **kwargs ) @@ -54,7 +59,7 @@ def test_create_certificate(reactor, request): """ The Grid Manager produces a valid, correctly-signed certificate. """ - gm_config = yield _run_gm(reactor, "--config", "-", "create") + gm_config = yield _run_gm(reactor, request, "--config", "-", "create") privkey_bytes = json.loads(gm_config)['private_key'].encode('ascii') privkey, pubkey = ed25519.signing_keypair_from_string(privkey_bytes) @@ -62,12 +67,12 @@ def test_create_certificate(reactor, request): # "actual" clients in the test-grid; we're just checking that the # Grid Manager signs this properly. gm_config = yield _run_gm( - reactor, "--config", "-", "add", + reactor, request, "--config", "-", "add", "zara", "pub-v0-kzug3ut2m7ziihf3ndpqlquuxeie4foyl36wn54myqc4wmiwe4ga", stdinBytes=gm_config, ) zara_cert_bytes = yield _run_gm( - reactor, "--config", "-", "sign", "zara", "1", + reactor, request, "--config", "-", "sign", "zara", "1", stdinBytes=gm_config, ) zara_cert = json.loads(zara_cert_bytes) @@ -86,16 +91,16 @@ def test_remove_client(reactor, request): A Grid Manager can add and successfully remove a client """ gm_config = yield _run_gm( - reactor, "--config", "-", "create", + reactor, request, "--config", "-", "create", ) gm_config = yield _run_gm( - reactor, "--config", "-", "add", + reactor, request, "--config", "-", "add", "zara", "pub-v0-kzug3ut2m7ziihf3ndpqlquuxeie4foyl36wn54myqc4wmiwe4ga", stdinBytes=gm_config, ) gm_config = yield _run_gm( - reactor, "--config", "-", "add", + reactor, request, "--config", "-", "add", "yakov", "pub-v0-kvxhb3nexybmipkrar2ztfrwp4uxxsmrjzkpzafit3ket4u5yldq", stdinBytes=gm_config, ) @@ -103,7 +108,7 @@ def test_remove_client(reactor, request): assert "yakov" in json.loads(gm_config)['storage_servers'] gm_config = yield _run_gm( - reactor, "--config", "-", "remove", + reactor, request, "--config", "-", "remove", "zara", stdinBytes=gm_config, ) @@ -117,18 +122,18 @@ def test_remove_last_client(reactor, request): A Grid Manager can remove all clients """ gm_config = yield _run_gm( - reactor, "--config", "-", "create", + reactor, request, "--config", "-", "create", ) gm_config = yield _run_gm( - reactor, "--config", "-", "add", + reactor, request, "--config", "-", "add", "zara", "pub-v0-kzug3ut2m7ziihf3ndpqlquuxeie4foyl36wn54myqc4wmiwe4ga", stdinBytes=gm_config, ) assert "zara" in json.loads(gm_config)['storage_servers'] gm_config = yield _run_gm( - reactor, "--config", "-", "remove", + reactor, request, "--config", "-", "remove", "zara", stdinBytes=gm_config, ) @@ -145,22 +150,22 @@ def test_add_remove_client_file(reactor, request, temp_dir): gmconfig = join(temp_dir, "gmtest") gmconfig_file = join(temp_dir, "gmtest", "config.json") yield _run_gm( - reactor, "--config", gmconfig, "create", + reactor, request, "--config", gmconfig, "create", ) yield _run_gm( - reactor, "--config", gmconfig, "add", + reactor, request, "--config", gmconfig, "add", "zara", "pub-v0-kzug3ut2m7ziihf3ndpqlquuxeie4foyl36wn54myqc4wmiwe4ga", ) yield _run_gm( - reactor, "--config", gmconfig, "add", + reactor, request, "--config", gmconfig, "add", "yakov", "pub-v0-kvxhb3nexybmipkrar2ztfrwp4uxxsmrjzkpzafit3ket4u5yldq", ) assert "zara" in json.load(open(gmconfig_file, "r"))['storage_servers'] assert "yakov" in json.load(open(gmconfig_file, "r"))['storage_servers'] yield _run_gm( - reactor, "--config", gmconfig, "remove", + reactor, request, "--config", gmconfig, "remove", "zara", ) assert "zara" not in json.load(open(gmconfig_file, "r"))['storage_servers'] @@ -179,7 +184,7 @@ def test_reject_storage_server(reactor, request, temp_dir, flog_gatherer, port_a _ = yield grid.add_storage_node() gm_config = yield _run_gm( - reactor, "--config", "-", "create", + reactor, request, "--config", "-", "create", ) gm_privkey_bytes = json.loads(gm_config)['private_key'].encode('ascii') gm_privkey, gm_pubkey = ed25519.signing_keypair_from_string(gm_privkey_bytes) @@ -190,7 +195,7 @@ def test_reject_storage_server(reactor, request, temp_dir, flog_gatherer, port_a pubkey_str = f.read().strip() gm_config = yield _run_gm( - reactor, "--config", "-", "add", + reactor, request, "--config", "-", "add", "storage0", pubkey_str, stdinBytes=gm_config, ) @@ -198,7 +203,7 @@ def test_reject_storage_server(reactor, request, temp_dir, flog_gatherer, port_a print("inserting certificate") cert = yield _run_gm( - reactor, "--config", "-", "sign", "storage0", "1", + reactor, request, "--config", "-", "sign", "storage0", "1", stdinBytes=gm_config, ) print(cert) @@ -256,7 +261,7 @@ def test_accept_storage_server(reactor, request, temp_dir, flog_gatherer, port_a happy1 = yield grid.add_storage_node() gm_config = yield _run_gm( - reactor, "--config", "-", "create", + reactor, request, "--config", "-", "create", ) gm_privkey_bytes = json.loads(gm_config)['private_key'].encode('ascii') gm_privkey, gm_pubkey = ed25519.signing_keypair_from_string(gm_privkey_bytes) @@ -272,7 +277,7 @@ def test_accept_storage_server(reactor, request, temp_dir, flog_gatherer, port_a pubkey_str = f.read().strip() gm_config = yield _run_gm( - reactor, "--config", "-", "add", + reactor, request, "--config", "-", "add", st_name, pubkey_str, stdinBytes=gm_config, ) @@ -282,7 +287,7 @@ def test_accept_storage_server(reactor, request, temp_dir, flog_gatherer, port_a print("inserting storage-server certificates") for st_name, st in servers: cert = yield _run_gm( - reactor, "--config", "-", "sign", st_name, "1", + reactor, request, "--config", "-", "sign", st_name, "1", stdinBytes=gm_config, ) @@ -324,12 +329,12 @@ def test_identity(reactor, request, temp_dir): """ gm_config = join(temp_dir, "test_identity") yield _run_gm( - reactor, "--config", gm_config, "create", + reactor, request, "--config", gm_config, "create", ) # ask the CLI for the grid-manager pubkey pubkey = yield _run_gm( - reactor, "--config", gm_config, "public-identity", + reactor, request, "--config", gm_config, "public-identity", ) alleged_pubkey = ed25519.verifying_key_from_string(pubkey.strip()) From 8250c5fdd54a909a17f24d99ae2ec89e78fb4600 Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 26 Sep 2022 15:40:55 -0600 Subject: [PATCH 1091/2309] edna -> freya --- integration/test_grid_manager.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/integration/test_grid_manager.py b/integration/test_grid_manager.py index 0136a11ac..1856ef435 100644 --- a/integration/test_grid_manager.py +++ b/integration/test_grid_manager.py @@ -303,20 +303,20 @@ def test_accept_storage_server(reactor, request, temp_dir, flog_gatherer, port_a yield happy0.restart(reactor, request) yield happy1.restart(reactor, request) - # configure edna (a client) to have the grid-manager certificate - edna = yield grid.add_client("edna", needed=2, happy=2, total=2) + # configure freya (a client) to have the grid-manager certificate + freya = yield grid.add_client("freya", needed=2, happy=2, total=2) - config = configutil.get_config(join(edna.process.node_dir, "tahoe.cfg")) + config = configutil.get_config(join(freya.process.node_dir, "tahoe.cfg")) config.add_section("grid_managers") config.set("grid_managers", "test", str(ed25519.string_from_verifying_key(gm_pubkey), "ascii")) - with open(join(edna.process.node_dir, "tahoe.cfg"), "w") as f: + with open(join(freya.process.node_dir, "tahoe.cfg"), "w") as f: config.write(f) - yield edna.restart(reactor, request, servers=2) + yield freya.restart(reactor, request, servers=2) - # confirm that Edna will upload to the GridManager-enabled Grid + # confirm that Freya will upload to the GridManager-enabled Grid yield util.run_tahoe( - reactor, request, "--node-directory", edna.process.node_dir, + reactor, request, "--node-directory", freya.process.node_dir, "put", "-", stdin=b"some content\n" * 200, ) From 4919b6d9066a10028e6548800a589929c3c094d9 Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 28 Sep 2022 09:34:36 -0600 Subject: [PATCH 1092/2309] typo Co-authored-by: Jean-Paul Calderone --- docs/running.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/running.rst b/docs/running.rst index b487f4ae3..29df15a3c 100644 --- a/docs/running.rst +++ b/docs/running.rst @@ -137,7 +137,7 @@ The ``.lock`` file exists to make sure only one process modifies ``running.proce The lock file is managed by the `lockfile `_ library. If you wish to make use of ``running.process`` for any reason you should also lock it and follow the semantics of lockfile. -If ``running.process` exists it file contains the PID and the creation-time of the process. +If ``running.process`` exists then it contains the PID and the creation-time of the process. When no such file exists, there is no other process running on this configuration. If there is a ``running.process`` file, it may be a leftover file or it may indicate that another process is running against this config. To tell the difference, determine if the PID in the file exists currently. From 7aae2f78575541799002645e85a3aeab9f8706c2 Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 28 Sep 2022 09:34:54 -0600 Subject: [PATCH 1093/2309] Clarify Co-authored-by: Jean-Paul Calderone --- docs/running.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/running.rst b/docs/running.rst index 29df15a3c..263448735 100644 --- a/docs/running.rst +++ b/docs/running.rst @@ -142,7 +142,7 @@ When no such file exists, there is no other process running on this configuratio If there is a ``running.process`` file, it may be a leftover file or it may indicate that another process is running against this config. To tell the difference, determine if the PID in the file exists currently. If it does, check the creation-time of the process versus the one in the file. -If these match, there is another process currently running. +If these match, there is another process currently running and using this config. Otherwise, the file is stale -- it should be removed before starting Tahoe-LAFS. Some example Python code to check the above situations: From a7398e13f7c82707738c3862cb085d7e2a055bb2 Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 28 Sep 2022 09:35:17 -0600 Subject: [PATCH 1094/2309] Update docs/check_running.py Co-authored-by: Jean-Paul Calderone --- docs/check_running.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/check_running.py b/docs/check_running.py index 55aae0015..2705f1721 100644 --- a/docs/check_running.py +++ b/docs/check_running.py @@ -38,9 +38,9 @@ def can_spawn_tahoe(pidfile): except psutil.NoSuchProcess: pass - # the file is stale - pidfile.unlink() - return True + # the file is stale + pidfile.unlink() + return True from pathlib import Path From ca522a5293c8f2e38e9b8d2071fc3865907f4177 Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 28 Sep 2022 10:07:44 -0600 Subject: [PATCH 1095/2309] sys.argv not inline --- src/allmydata/test/test_runner.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/test_runner.py b/src/allmydata/test/test_runner.py index f8211ec02..00c87ce08 100644 --- a/src/allmydata/test/test_runner.py +++ b/src/allmydata/test/test_runner.py @@ -642,14 +642,14 @@ class PidFileLocking(SyncTestCase): f.write( "\n".join([ "import filelock, time, sys", - "with filelock.FileLock(r'{}', timeout=1):".format(lockfile.path), + "with filelock.FileLock(sys.argv[1], timeout=1):", " sys.stdout.write('.\\n')", " sys.stdout.flush()", " time.sleep(10)", ]) ) proc = Popen( - [sys.executable, "other_lock.py"], + [sys.executable, "other_lock.py", lockfile.path], stdout=PIPE, stderr=PIPE, start_new_session=True, From bef71978b6e7181598fc30f1b94d56b0b7e6a7c5 Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 28 Sep 2022 10:08:13 -0600 Subject: [PATCH 1096/2309] don't need start_new_session --- src/allmydata/test/test_runner.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/allmydata/test/test_runner.py b/src/allmydata/test/test_runner.py index 00c87ce08..74e3f803e 100644 --- a/src/allmydata/test/test_runner.py +++ b/src/allmydata/test/test_runner.py @@ -652,7 +652,6 @@ class PidFileLocking(SyncTestCase): [sys.executable, "other_lock.py", lockfile.path], stdout=PIPE, stderr=PIPE, - start_new_session=True, ) # make sure our subprocess has had time to acquire the lock # for sure (from the "." it prints) From 2a3b110d53146d86709a8e161d90e65d6a07f0fe Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 30 Sep 2022 16:48:23 -0600 Subject: [PATCH 1097/2309] simple build automation --- Makefile | 44 ++++++++++++++++++++++++++++++++++++++++++++ setup.py | 4 ++++ 2 files changed, 48 insertions(+) diff --git a/Makefile b/Makefile index 5cbd863a3..6dd2b743b 100644 --- a/Makefile +++ b/Makefile @@ -224,3 +224,47 @@ src/allmydata/_version.py: .tox/create-venvs.log: tox.ini setup.py tox --notest -p all | tee -a "$(@)" + + +# Make a new release. TODO: +# - clean checkout necessary? garbage in tarball? +release: + @echo "Is checkout clean?" + git diff-files --quiet + git diff-index --quiet --cached HEAD -- + + @echo "Install required build software" + python3 -m pip install --editable .[build] + + @echo "Test README" + python3 setup.py check -r -s + + @echo "Update NEWS" + python3 -m towncrier build --yes --version `python3 misc/build_helpers/update-version.py --no-tag` + git add -u + git commit -m "update NEWS for release" + + @echo "Bump version and create tag" + python3 misc/build_helpers/update-version.py + + @echo "Build and sign wheel" + python3 setup.py bdist_wheel + gpg --pinentry=loopback -u meejah@meejah.ca --armor --detach-sign dist/tahoe_lafs-`git describe --abbrev=0`-py3-none-any.whl + ls dist/*`git describe --abbrev=0`* + + @echo "Build and sign source-dist" + python3 setup.py sdist + gpg --pinentry=loopback -u meejah@meejah.ca --armor --detach-sign dist/tahoe-lafs-`git describe --abbrev=0`.tar.gz + ls dist/*`git describe --abbrev=0`* + +release-test: + gpg --verify dist/tahoe-lafs-`git describe --abbrev=0`.tar.gz.asc + gpg --verify dist/tahoe_lafs-`git describe --abbrev=0`-py3-none-any.whl.asc + virtualenv testmf_venv + testmf_venv/bin/pip install dist/tahoe_lafs-`git describe --abbrev=0`-py3-none-any.whl + testmf_venv/bin/tahoe-lafs --version +# ... + rm -rf testmf_venv + +release-upload: + twine upload dist/tahoe_lafs-`git describe --abbrev=0`-py3-none-any.whl dist/tahoe_lafs-`git describe --abbrev=0`-py3-none-any.whl.asc dist/tahoe-lafs-`git describe --abbrev=0`.tar.gz dist/tahoe-lafs-`git describe --abbrev=0`.tar.gz.asc diff --git a/setup.py b/setup.py index d99831347..ffe23a7b5 100644 --- a/setup.py +++ b/setup.py @@ -380,6 +380,10 @@ setup(name="tahoe-lafs", # also set in __init__.py # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/2392 for some # discussion. ':sys_platform=="win32"': ["pywin32 != 226"], + "build": [ + "dulwich", + "gpg", + ], "test": [ "flake8", # Pin a specific pyflakes so we don't have different folks From 4b708d87bd0bd6d07517a113c065e0f0329b8d34 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 30 Sep 2022 16:53:48 -0600 Subject: [PATCH 1098/2309] wip --- Makefile | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/Makefile b/Makefile index 6dd2b743b..66f1819ad 100644 --- a/Makefile +++ b/Makefile @@ -239,6 +239,9 @@ release: @echo "Test README" python3 setup.py check -r -s +# XXX make branch, based on a ticket (provided how?) +# XXX or, specify that "make release" must run on such a branch "XXXX.tahoe-release" + @echo "Update NEWS" python3 -m towncrier build --yes --version `python3 misc/build_helpers/update-version.py --no-tag` git add -u @@ -249,22 +252,22 @@ release: @echo "Build and sign wheel" python3 setup.py bdist_wheel - gpg --pinentry=loopback -u meejah@meejah.ca --armor --detach-sign dist/tahoe_lafs-`git describe --abbrev=0`-py3-none-any.whl - ls dist/*`git describe --abbrev=0`* + gpg --pinentry=loopback -u meejah@meejah.ca --armor --detach-sign dist/tahoe_lafs-`git describe | cut -b 12-`-py3-none-any.whl + ls dist/*`git describe | cut -b 12-`* @echo "Build and sign source-dist" python3 setup.py sdist - gpg --pinentry=loopback -u meejah@meejah.ca --armor --detach-sign dist/tahoe-lafs-`git describe --abbrev=0`.tar.gz - ls dist/*`git describe --abbrev=0`* + gpg --pinentry=loopback -u meejah@meejah.ca --armor --detach-sign dist/tahoe-lafs-`git describe | cut -b 12-`.tar.gz + ls dist/*`git describe | cut -b 12-`* release-test: - gpg --verify dist/tahoe-lafs-`git describe --abbrev=0`.tar.gz.asc - gpg --verify dist/tahoe_lafs-`git describe --abbrev=0`-py3-none-any.whl.asc + gpg --verify dist/tahoe-lafs-`git describe | cut -b 12-`.tar.gz.asc + gpg --verify dist/tahoe_lafs-`git describe | cut -b 12-`-py3-none-any.whl.asc virtualenv testmf_venv - testmf_venv/bin/pip install dist/tahoe_lafs-`git describe --abbrev=0`-py3-none-any.whl + testmf_venv/bin/pip install dist/tahoe_lafs-`git describe | cut -b 12-`-py3-none-any.whl testmf_venv/bin/tahoe-lafs --version # ... rm -rf testmf_venv release-upload: - twine upload dist/tahoe_lafs-`git describe --abbrev=0`-py3-none-any.whl dist/tahoe_lafs-`git describe --abbrev=0`-py3-none-any.whl.asc dist/tahoe-lafs-`git describe --abbrev=0`.tar.gz dist/tahoe-lafs-`git describe --abbrev=0`.tar.gz.asc + twine upload dist/tahoe_lafs-`git describe | cut -b 12-`-py3-none-any.whl dist/tahoe_lafs-`git describe | cut -b 12-`-py3-none-any.whl.asc dist/tahoe-lafs-`git describe | cut -b 12-`.tar.gz dist/tahoe-lafs-`git describe | cut -b 12-`.tar.gz.asc From 4137d6ebb7b73de4782e0d332485684cf3585376 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 30 Sep 2022 17:20:19 -0600 Subject: [PATCH 1099/2309] proper smoke-test --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 66f1819ad..5ad676e86 100644 --- a/Makefile +++ b/Makefile @@ -260,13 +260,13 @@ release: gpg --pinentry=loopback -u meejah@meejah.ca --armor --detach-sign dist/tahoe-lafs-`git describe | cut -b 12-`.tar.gz ls dist/*`git describe | cut -b 12-`* +# basically just a bare-minimum smoke-test that it installs and runs release-test: gpg --verify dist/tahoe-lafs-`git describe | cut -b 12-`.tar.gz.asc gpg --verify dist/tahoe_lafs-`git describe | cut -b 12-`-py3-none-any.whl.asc virtualenv testmf_venv testmf_venv/bin/pip install dist/tahoe_lafs-`git describe | cut -b 12-`-py3-none-any.whl - testmf_venv/bin/tahoe-lafs --version -# ... + testmf_venv/bin/tahoe --version rm -rf testmf_venv release-upload: From 923f456d6e9f53ecb6db67c73a999f72027b2655 Mon Sep 17 00:00:00 2001 From: meejah Date: Sat, 1 Oct 2022 14:47:19 -0600 Subject: [PATCH 1100/2309] all upload steps --- Makefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Makefile b/Makefile index 5ad676e86..b68b788ca 100644 --- a/Makefile +++ b/Makefile @@ -270,4 +270,6 @@ release-test: rm -rf testmf_venv release-upload: + scp dist/*`git describe | cut -b 12-`* meejah@tahoe-lafs.org:/home/source/downloads + git push origin_push tahoe-lafs-`git describe | cut -b 12-` twine upload dist/tahoe_lafs-`git describe | cut -b 12-`-py3-none-any.whl dist/tahoe_lafs-`git describe | cut -b 12-`-py3-none-any.whl.asc dist/tahoe-lafs-`git describe | cut -b 12-`.tar.gz dist/tahoe-lafs-`git describe | cut -b 12-`.tar.gz.asc From c711b5b0a9c825d6e1bfb3a30437da380a63b422 Mon Sep 17 00:00:00 2001 From: meejah Date: Sun, 2 Oct 2022 13:33:05 -0600 Subject: [PATCH 1101/2309] clean docs --- Makefile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Makefile b/Makefile index b68b788ca..8b34fd0e1 100644 --- a/Makefile +++ b/Makefile @@ -233,6 +233,9 @@ release: git diff-files --quiet git diff-index --quiet --cached HEAD -- + @echo "Clean docs build area" + rm -rf docs/_build/ + @echo "Install required build software" python3 -m pip install --editable .[build] From 3d3dc187646f0ba2203f73eecf2c927147400884 Mon Sep 17 00:00:00 2001 From: meejah Date: Sun, 2 Oct 2022 14:34:42 -0600 Subject: [PATCH 1102/2309] better instructions --- Makefile | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 8b34fd0e1..c501ba3c5 100644 --- a/Makefile +++ b/Makefile @@ -226,8 +226,16 @@ src/allmydata/_version.py: tox --notest -p all | tee -a "$(@)" -# Make a new release. TODO: -# - clean checkout necessary? garbage in tarball? +# to make a new release: +# - create a ticket for the release in Trac +# - ensure local copy is up-to-date +# - create a branch like "XXXX.release" from up-to-date master +# - in the branch, run "make release" +# - run "make release-test" +# - perform any other sanity-checks on the release +# - run "make release-upload" +# Note that several commands below hard-code "meejah"; if you are +# someone else please adjust them. release: @echo "Is checkout clean?" git diff-files --quiet From a22be070b8ff9b4e05af9f28e61d209f64fcdeb2 Mon Sep 17 00:00:00 2001 From: meejah Date: Sun, 2 Oct 2022 14:51:29 -0600 Subject: [PATCH 1103/2309] version-updating script --- misc/build_helpers/update-version.py | 96 ++++++++++++++++++++++++++++ 1 file changed, 96 insertions(+) create mode 100644 misc/build_helpers/update-version.py diff --git a/misc/build_helpers/update-version.py b/misc/build_helpers/update-version.py new file mode 100644 index 000000000..38baf7c7c --- /dev/null +++ b/misc/build_helpers/update-version.py @@ -0,0 +1,96 @@ +# +# this updates the (tagged) version of the software +# +# Any "options" are hard-coded in here (e.g. the GnuPG key to use) +# + +author = "meejah " + + +import sys +import time +import itertools +from datetime import datetime +from packaging.version import Version + +from dulwich.repo import Repo +from dulwich.porcelain import ( + tag_list, + tag_create, + status, +) + +from twisted.internet.task import ( + react, +) +from twisted.internet.defer import ( + ensureDeferred, +) + + +def existing_tags(git): + versions = sorted( + Version(v.decode("utf8").lstrip("tahoe-lafs-")) + for v in tag_list(git) + if v.startswith(b"tahoe-lafs-") + ) + return versions + + +def create_new_version(git): + versions = existing_tags(git) + biggest = versions[-1] + + return Version( + "{}.{}.{}".format( + biggest.major, + biggest.minor + 1, + 0, + ) + ) + + +async def main(reactor): + git = Repo(".") + + st = status(git) + if any(st.staged.values()) or st.unstaged: + print("unclean checkout; aborting") + raise SystemExit(1) + + v = create_new_version(git) + if "--no-tag" in sys.argv: + print(v) + return + + print("Existing tags: {}".format("\n".join(str(x) for x in existing_tags(git)))) + print("New tag will be {}".format(v)) + + # the "tag time" is seconds from the epoch .. we quantize these to + # the start of the day in question, in UTC. + now = datetime.now() + s = now.utctimetuple() + ts = int( + time.mktime( + time.struct_time((s.tm_year, s.tm_mon, s.tm_mday, 0, 0, 0, 0, s.tm_yday, 0)) + ) + ) + tag_create( + repo=git, + tag="tahoe-lafs-{}".format(str(v)).encode("utf8"), + author=author.encode("utf8"), + message="Release {}".format(v).encode("utf8"), + annotated=True, + objectish=b"HEAD", + sign=author.encode("utf8"), + tag_time=ts, + tag_timezone=0, + ) + + print("Tag created locally, it is not pushed") + print("To push it run something like:") + print(" git push origin {}".format(v)) + + +if __name__ == "__main__": + react(lambda r: ensureDeferred(main(r))) From 1af48672e39ab6430583dbd38fcfe4fa61821d09 Mon Sep 17 00:00:00 2001 From: meejah Date: Sun, 2 Oct 2022 14:53:03 -0600 Subject: [PATCH 1104/2309] correct notes --- Makefile | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index c501ba3c5..c02184a36 100644 --- a/Makefile +++ b/Makefile @@ -250,14 +250,13 @@ release: @echo "Test README" python3 setup.py check -r -s -# XXX make branch, based on a ticket (provided how?) -# XXX or, specify that "make release" must run on such a branch "XXXX.tahoe-release" - @echo "Update NEWS" python3 -m towncrier build --yes --version `python3 misc/build_helpers/update-version.py --no-tag` git add -u git commit -m "update NEWS for release" +# note that this always bumps the "middle" number, e.g. from 1.17.1 -> 1.18.0 +# and produces a tag into the Git repository @echo "Bump version and create tag" python3 misc/build_helpers/update-version.py From 6bb46a832bfde84714d35625a256265e93688684 Mon Sep 17 00:00:00 2001 From: meejah Date: Sun, 2 Oct 2022 18:52:57 -0600 Subject: [PATCH 1105/2309] flake8 --- misc/build_helpers/update-version.py | 1 - 1 file changed, 1 deletion(-) diff --git a/misc/build_helpers/update-version.py b/misc/build_helpers/update-version.py index 38baf7c7c..75b22edae 100644 --- a/misc/build_helpers/update-version.py +++ b/misc/build_helpers/update-version.py @@ -9,7 +9,6 @@ author = "meejah " import sys import time -import itertools from datetime import datetime from packaging.version import Version From 402d80710caa5aa50f0a5bef79ae979a75dc3594 Mon Sep 17 00:00:00 2001 From: meejah Date: Sun, 2 Oct 2022 19:03:10 -0600 Subject: [PATCH 1106/2309] news --- newsfragments/3846.feature | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/3846.feature diff --git a/newsfragments/3846.feature b/newsfragments/3846.feature new file mode 100644 index 000000000..fd321eaf0 --- /dev/null +++ b/newsfragments/3846.feature @@ -0,0 +1 @@ +"make" based release automation From e8e43d2100f1e8dfc6bd421dffd3824e57b903d0 Mon Sep 17 00:00:00 2001 From: meejah Date: Sun, 2 Oct 2022 19:05:16 -0600 Subject: [PATCH 1107/2309] update NEWS for release --- NEWS.rst | 41 +++++++++++++++++++++++++++++++++++++ newsfragments/3327.minor | 0 newsfragments/3526.minor | 1 - newsfragments/3697.minor | 1 - newsfragments/3709.minor | 0 newsfragments/3786.minor | 1 - newsfragments/3788.minor | 0 newsfragments/3802.minor | 0 newsfragments/3816.minor | 0 newsfragments/3828.feature | 8 -------- newsfragments/3846.feature | 1 - newsfragments/3855.minor | 0 newsfragments/3858.minor | 0 newsfragments/3859.minor | 0 newsfragments/3860.minor | 0 newsfragments/3865.incompat | 1 - newsfragments/3867.minor | 0 newsfragments/3868.minor | 0 newsfragments/3871.minor | 0 newsfragments/3872.minor | 0 newsfragments/3873.incompat | 1 - newsfragments/3875.minor | 0 newsfragments/3876.minor | 0 newsfragments/3877.minor | 0 newsfragments/3879.incompat | 1 - newsfragments/3881.minor | 0 newsfragments/3882.minor | 0 newsfragments/3883.minor | 0 newsfragments/3889.minor | 0 newsfragments/3890.minor | 0 newsfragments/3891.minor | 0 newsfragments/3893.minor | 0 newsfragments/3895.minor | 0 newsfragments/3896.minor | 0 newsfragments/3898.minor | 0 newsfragments/3900.minor | 0 newsfragments/3909.minor | 0 newsfragments/3913.minor | 0 newsfragments/3915.minor | 0 newsfragments/3916.minor | 0 newsfragments/3926.incompat | 10 --------- 41 files changed, 41 insertions(+), 25 deletions(-) delete mode 100644 newsfragments/3327.minor delete mode 100644 newsfragments/3526.minor delete mode 100644 newsfragments/3697.minor delete mode 100644 newsfragments/3709.minor delete mode 100644 newsfragments/3786.minor delete mode 100644 newsfragments/3788.minor delete mode 100644 newsfragments/3802.minor delete mode 100644 newsfragments/3816.minor delete mode 100644 newsfragments/3828.feature delete mode 100644 newsfragments/3846.feature delete mode 100644 newsfragments/3855.minor delete mode 100644 newsfragments/3858.minor delete mode 100644 newsfragments/3859.minor delete mode 100644 newsfragments/3860.minor delete mode 100644 newsfragments/3865.incompat delete mode 100644 newsfragments/3867.minor delete mode 100644 newsfragments/3868.minor delete mode 100644 newsfragments/3871.minor delete mode 100644 newsfragments/3872.minor delete mode 100644 newsfragments/3873.incompat delete mode 100644 newsfragments/3875.minor delete mode 100644 newsfragments/3876.minor delete mode 100644 newsfragments/3877.minor delete mode 100644 newsfragments/3879.incompat delete mode 100644 newsfragments/3881.minor delete mode 100644 newsfragments/3882.minor delete mode 100644 newsfragments/3883.minor delete mode 100644 newsfragments/3889.minor delete mode 100644 newsfragments/3890.minor delete mode 100644 newsfragments/3891.minor delete mode 100644 newsfragments/3893.minor delete mode 100644 newsfragments/3895.minor delete mode 100644 newsfragments/3896.minor delete mode 100644 newsfragments/3898.minor delete mode 100644 newsfragments/3900.minor delete mode 100644 newsfragments/3909.minor delete mode 100644 newsfragments/3913.minor delete mode 100644 newsfragments/3915.minor delete mode 100644 newsfragments/3916.minor delete mode 100644 newsfragments/3926.incompat diff --git a/NEWS.rst b/NEWS.rst index 0f9194cc4..7b1fadb8a 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -5,6 +5,47 @@ User-Visible Changes in Tahoe-LAFS ================================== .. towncrier start line +Release 1.18.0 (2022-10-02) +''''''''''''''''''''''''''' + +Backwards Incompatible Changes +------------------------------ + +- Python 3.6 is no longer supported, as it has reached end-of-life and is no longer receiving security updates. (`#3865 `_) +- Python 3.7 or later is now required; Python 2 is no longer supported. (`#3873 `_) +- Share corruption reports stored on disk are now always encoded in UTF-8. (`#3879 `_) +- Record both the PID and the process creation-time: + + a new kind of pidfile in `running.process` records both + the PID and the creation-time of the process. This facilitates + automatic discovery of a "stale" pidfile that points to a + currently-running process. If the recorded creation-time matches + the creation-time of the running process, then it is a still-running + `tahoe run` process. Otherwise, the file is stale. + + The `twistd.pid` file is no longer present. (`#3926 `_) + + +Features +-------- + +- The implementation of SDMF and MDMF (mutables) now requires RSA keys to be exactly 2048 bits, aligning them with the specification. + + Some code existed to allow tests to shorten this and it's + conceptually possible a modified client produced mutables + with different key-sizes. However, the spec says that they + must be 2048 bits. If you happen to have a capability with + a key-size different from 2048 you may use 1.17.1 or earlier + to read the content. (`#3828 `_) +- "make" based release automation (`#3846 `_) + + +Misc/Other +---------- + +- `#3327 `_, `#3526 `_, `#3697 `_, `#3709 `_, `#3786 `_, `#3788 `_, `#3802 `_, `#3816 `_, `#3855 `_, `#3858 `_, `#3859 `_, `#3860 `_, `#3867 `_, `#3868 `_, `#3871 `_, `#3872 `_, `#3875 `_, `#3876 `_, `#3877 `_, `#3881 `_, `#3882 `_, `#3883 `_, `#3889 `_, `#3890 `_, `#3891 `_, `#3893 `_, `#3895 `_, `#3896 `_, `#3898 `_, `#3900 `_, `#3909 `_, `#3913 `_, `#3915 `_, `#3916 `_ + + Release 1.17.1 (2022-01-07) ''''''''''''''''''''''''''' diff --git a/newsfragments/3327.minor b/newsfragments/3327.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3526.minor b/newsfragments/3526.minor deleted file mode 100644 index 8b1378917..000000000 --- a/newsfragments/3526.minor +++ /dev/null @@ -1 +0,0 @@ - diff --git a/newsfragments/3697.minor b/newsfragments/3697.minor deleted file mode 100644 index 0977d8a6f..000000000 --- a/newsfragments/3697.minor +++ /dev/null @@ -1 +0,0 @@ -Added support for Python 3.10. Added support for PyPy3 (3.7 and 3.8, on Linux only). \ No newline at end of file diff --git a/newsfragments/3709.minor b/newsfragments/3709.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3786.minor b/newsfragments/3786.minor deleted file mode 100644 index ecd1a2c4e..000000000 --- a/newsfragments/3786.minor +++ /dev/null @@ -1 +0,0 @@ -Added re-structured text documentation for the OpenMetrics format statistics endpoint. diff --git a/newsfragments/3788.minor b/newsfragments/3788.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3802.minor b/newsfragments/3802.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3816.minor b/newsfragments/3816.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3828.feature b/newsfragments/3828.feature deleted file mode 100644 index d396439b0..000000000 --- a/newsfragments/3828.feature +++ /dev/null @@ -1,8 +0,0 @@ -The implementation of SDMF and MDMF (mutables) now requires RSA keys to be exactly 2048 bits, aligning them with the specification. - -Some code existed to allow tests to shorten this and it's -conceptually possible a modified client produced mutables -with different key-sizes. However, the spec says that they -must be 2048 bits. If you happen to have a capability with -a key-size different from 2048 you may use 1.17.1 or earlier -to read the content. diff --git a/newsfragments/3846.feature b/newsfragments/3846.feature deleted file mode 100644 index fd321eaf0..000000000 --- a/newsfragments/3846.feature +++ /dev/null @@ -1 +0,0 @@ -"make" based release automation diff --git a/newsfragments/3855.minor b/newsfragments/3855.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3858.minor b/newsfragments/3858.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3859.minor b/newsfragments/3859.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3860.minor b/newsfragments/3860.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3865.incompat b/newsfragments/3865.incompat deleted file mode 100644 index 59381b269..000000000 --- a/newsfragments/3865.incompat +++ /dev/null @@ -1 +0,0 @@ -Python 3.6 is no longer supported, as it has reached end-of-life and is no longer receiving security updates. \ No newline at end of file diff --git a/newsfragments/3867.minor b/newsfragments/3867.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3868.minor b/newsfragments/3868.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3871.minor b/newsfragments/3871.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3872.minor b/newsfragments/3872.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3873.incompat b/newsfragments/3873.incompat deleted file mode 100644 index da8a5fb0e..000000000 --- a/newsfragments/3873.incompat +++ /dev/null @@ -1 +0,0 @@ -Python 3.7 or later is now required; Python 2 is no longer supported. \ No newline at end of file diff --git a/newsfragments/3875.minor b/newsfragments/3875.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3876.minor b/newsfragments/3876.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3877.minor b/newsfragments/3877.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3879.incompat b/newsfragments/3879.incompat deleted file mode 100644 index ca3f24f94..000000000 --- a/newsfragments/3879.incompat +++ /dev/null @@ -1 +0,0 @@ -Share corruption reports stored on disk are now always encoded in UTF-8. \ No newline at end of file diff --git a/newsfragments/3881.minor b/newsfragments/3881.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3882.minor b/newsfragments/3882.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3883.minor b/newsfragments/3883.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3889.minor b/newsfragments/3889.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3890.minor b/newsfragments/3890.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3891.minor b/newsfragments/3891.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3893.minor b/newsfragments/3893.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3895.minor b/newsfragments/3895.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3896.minor b/newsfragments/3896.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3898.minor b/newsfragments/3898.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3900.minor b/newsfragments/3900.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3909.minor b/newsfragments/3909.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3913.minor b/newsfragments/3913.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3915.minor b/newsfragments/3915.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3916.minor b/newsfragments/3916.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3926.incompat b/newsfragments/3926.incompat deleted file mode 100644 index 674ad289c..000000000 --- a/newsfragments/3926.incompat +++ /dev/null @@ -1,10 +0,0 @@ -Record both the PID and the process creation-time: - -a new kind of pidfile in `running.process` records both -the PID and the creation-time of the process. This facilitates -automatic discovery of a "stale" pidfile that points to a -currently-running process. If the recorded creation-time matches -the creation-time of the running process, then it is a still-running -`tahoe run` process. Otherwise, the file is stale. - -The `twistd.pid` file is no longer present. \ No newline at end of file From a53420c1931b4ec9c6a40f5105a44d7d4ac0f846 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 3 Oct 2022 10:49:01 -0400 Subject: [PATCH 1108/2309] Use known working version of i2pd. --- integration/test_i2p.py | 2 +- newsfragments/3928.minor | 0 2 files changed, 1 insertion(+), 1 deletion(-) create mode 100644 newsfragments/3928.minor diff --git a/integration/test_i2p.py b/integration/test_i2p.py index f0b06f1e2..97abb40a5 100644 --- a/integration/test_i2p.py +++ b/integration/test_i2p.py @@ -55,7 +55,7 @@ def i2p_network(reactor, temp_dir, request): proto, which("docker"), ( - "docker", "run", "-p", "7656:7656", "purplei2p/i2pd", + "docker", "run", "-p", "7656:7656", "purplei2p/i2pd:release-2.43.0", # Bad URL for reseeds, so it can't talk to other routers. "--reseed.urls", "http://localhost:1/", ), diff --git a/newsfragments/3928.minor b/newsfragments/3928.minor new file mode 100644 index 000000000..e69de29bb From ec15d58e10356016130cb7eaf97681584540a611 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 3 Oct 2022 10:49:08 -0400 Subject: [PATCH 1109/2309] Actually clean up the container. --- integration/test_i2p.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration/test_i2p.py b/integration/test_i2p.py index 97abb40a5..15f9d73cf 100644 --- a/integration/test_i2p.py +++ b/integration/test_i2p.py @@ -63,7 +63,7 @@ def i2p_network(reactor, temp_dir, request): def cleanup(): try: - proto.transport.signalProcess("KILL") + proto.transport.signalProcess("INT") util.block_with_timeout(proto.exited, reactor) except ProcessExitedAlready: pass From b86f99f0ebaa5d0239d18498f59f412967bf0b27 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 3 Oct 2022 11:00:34 -0400 Subject: [PATCH 1110/2309] Make this more accurate given changes in spec. --- docs/specifications/url.rst | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/specifications/url.rst b/docs/specifications/url.rst index 39a830e5a..fe756208b 100644 --- a/docs/specifications/url.rst +++ b/docs/specifications/url.rst @@ -87,11 +87,13 @@ These differences are separated into distinct versions. Version 0 --------- -A Foolscap fURL is considered the canonical definition of a version 0 NURL. +In theory, a Foolscap fURL with a single netloc is considered the canonical definition of a version 0 NURL. Notably, the hash component is defined as the base32-encoded SHA1 hash of the DER form of an x509v3 certificate. A version 0 NURL is identified by the absence of the ``v=1`` fragment. +In practice, real world fURLs may have more than one netloc, so lack of version fragment will likely just involve dispatching the fURL to a different parser. + Examples ~~~~~~~~ @@ -119,7 +121,7 @@ The hash component of a version 1 NURL differs in three ways from the prior vers *all* certificate fields should be considered within the context of the relationship identified by the SPKI hash. 3. The hash is encoded using urlsafe-base64 (without padding) instead of base32. - This provides a more compact representation and minimizes the usability impacts of switching from a 160 bit hash to a 224 bit hash. + This provides a more compact representation and minimizes the usability impacts of switching from a 160 bit hash to a 256 bit hash. A version 1 NURL is identified by the presence of the ``v=1`` fragment. Though the length of the hash string (38 bytes) could also be used to differentiate it from a version 0 NURL, From b0fb72e379bcbfaabb2ae37452d9a68dd481cbea Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 3 Oct 2022 11:02:48 -0400 Subject: [PATCH 1111/2309] Link to design issue. --- src/allmydata/client.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/allmydata/client.py b/src/allmydata/client.py index 9938ec076..ac8b03e2f 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -591,6 +591,10 @@ def anonymous_storage_enabled(config): @implementer(IStatsProducer) class _Client(node.Node, pollmixin.PollMixin): + """ + This class should be refactored; see + https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3931 + """ STOREDIR = 'storage' NODETYPE = "client" @@ -661,7 +665,9 @@ class _Client(node.Node, pollmixin.PollMixin): # TODO this may be the wrong location for now? but as temporary measure # it allows us to get NURLs for testing in test_istorageserver.py. This # will eventually get fixed one way or another in - # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3901 + # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3901. See also + # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3931 for the bigger + # picture issue. self.storage_nurls = set() def init_stats_provider(self): From d753bb58da880a00724bd0a9c592803ee7983fca Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 3 Oct 2022 11:05:56 -0400 Subject: [PATCH 1112/2309] Better type for storage_nurls. --- src/allmydata/client.py | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/src/allmydata/client.py b/src/allmydata/client.py index ac8b03e2f..a31d05b9c 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -1,17 +1,9 @@ """ Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals - -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, max, min # noqa: F401 - # Don't use future str to prevent leaking future's newbytes into foolscap, which they break. - from past.builtins import unicode as str +from __future__ import annotations +from typing import Optional import os, stat, time, weakref from base64 import urlsafe_b64encode from functools import partial @@ -668,7 +660,7 @@ class _Client(node.Node, pollmixin.PollMixin): # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3901. See also # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3931 for the bigger # picture issue. - self.storage_nurls = set() + self.storage_nurls : Optional[set] = None def init_stats_provider(self): self.stats_provider = StatsProvider(self) @@ -831,8 +823,8 @@ class _Client(node.Node, pollmixin.PollMixin): furl_file = self.config.get_private_path("storage.furl").encode(get_filesystem_encoding()) furl = self.tub.registerReference(FoolscapStorageServer(ss), furlFile=furl_file) (_, _, swissnum) = furl.rpartition("/") - self.storage_nurls.update( - self.tub.negotiationClass.add_storage_server(ss, swissnum.encode("ascii")) + self.storage_nurls = self.tub.negotiationClass.add_storage_server( + ss, swissnum.encode("ascii") ) announcement["anonymous-storage-FURL"] = furl From d918135a0d016f579073dac7236a3aadfab76bbf Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 3 Oct 2022 11:10:36 -0400 Subject: [PATCH 1113/2309] Use parser instead of ad-hoc parser. --- src/allmydata/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/client.py b/src/allmydata/client.py index a31d05b9c..417dffed8 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -822,7 +822,7 @@ class _Client(node.Node, pollmixin.PollMixin): if anonymous_storage_enabled(self.config): furl_file = self.config.get_private_path("storage.furl").encode(get_filesystem_encoding()) furl = self.tub.registerReference(FoolscapStorageServer(ss), furlFile=furl_file) - (_, _, swissnum) = furl.rpartition("/") + (_, _, swissnum) = decode_furl(furl) self.storage_nurls = self.tub.negotiationClass.add_storage_server( ss, swissnum.encode("ascii") ) From 5d53cd4a170cd7315b594102f705fcb9e7eec55e Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 3 Oct 2022 11:16:30 -0400 Subject: [PATCH 1114/2309] Nicer API. --- src/allmydata/node.py | 8 +++++--- src/allmydata/protocol_switch.py | 20 ++++++++++++-------- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/src/allmydata/node.py b/src/allmydata/node.py index 597221e9b..d6cbc9e36 100644 --- a/src/allmydata/node.py +++ b/src/allmydata/node.py @@ -55,7 +55,7 @@ from allmydata.util.yamlutil import ( from . import ( __full_version__, ) -from .protocol_switch import support_foolscap_and_https +from .protocol_switch import create_tub_with_https_support def _common_valid_config(): @@ -708,8 +708,10 @@ def create_tub(tub_options, default_connection_handlers, foolscap_connection_han :param dict tub_options: every key-value pair in here will be set in the new Tub via `Tub.setOption` """ - tub = Tub(**kwargs) - support_foolscap_and_https(tub) + # We listen simulataneously for both Foolscap and HTTPS on the same port, + # so we have to create a special Foolscap Tub for that to work: + tub = create_tub_with_https_support(**kwargs) + for (name, value) in list(tub_options.items()): tub.setOption(name, value) handlers = default_connection_handlers.copy() diff --git a/src/allmydata/protocol_switch.py b/src/allmydata/protocol_switch.py index a17f3055c..2b4ce6da1 100644 --- a/src/allmydata/protocol_switch.py +++ b/src/allmydata/protocol_switch.py @@ -6,10 +6,11 @@ simple as possible, with no extra configuration needed. Listening on the same port means a user upgrading Tahoe-LAFS will automatically get HTTPS working with no additional changes. -Use ``support_foolscap_and_https()`` to create a new subclass for a ``Tub`` -instance, and then ``add_storage_server()`` on the resulting class to add the -relevant information for a storage server once it becomes available later in -the configuration process. +Use ``create_tub_with_https_support()`` creates a new ``Tub`` that has its +``negotiationClass`` modified to be a new subclass tied to that specific +``Tub`` instance. Calling ``tub.negotiationClass.add_storage_server(...)`` +then adds relevant information for a storage server once it becomes available +later in the configuration process. """ from __future__ import annotations @@ -193,14 +194,17 @@ class _FoolscapOrHttps(Protocol, metaclass=_PretendToBeNegotiation): self.__dict__ = protocol.__dict__ -def support_foolscap_and_https(tub: Tub): +def create_tub_with_https_support(**kwargs) -> Tub: """ - Create a new Foolscap-or-HTTPS protocol class for a specific ``Tub`` + Create a new Tub that also supports HTTPS. + + This involves creating a new protocol switch class for the specific ``Tub`` instance. """ - the_tub = tub + the_tub = Tub(**kwargs) class FoolscapOrHttpForTub(_FoolscapOrHttps): tub = the_tub - tub.negotiationClass = FoolscapOrHttpForTub # type: ignore + the_tub.negotiationClass = FoolscapOrHttpForTub # type: ignore + return the_tub From 3034f35c7b1e1748a5d4a76f73585ced7fc1e2ff Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 3 Oct 2022 11:21:54 -0400 Subject: [PATCH 1115/2309] Document type expectations. --- src/allmydata/storage/http_server.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 540675cc7..eefb9b906 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -4,7 +4,7 @@ HTTP server for storage. from __future__ import annotations -from typing import Dict, List, Set, Tuple, Any, Callable, Union +from typing import Dict, List, Set, Tuple, Any, Callable, Union, cast from functools import wraps from base64 import b64decode import binascii @@ -19,6 +19,7 @@ from twisted.internet.interfaces import ( IStreamServerEndpoint, IPullProducer, ) +from twisted.internet.address import IPv4Address, IPv6Address from twisted.internet.defer import Deferred from twisted.internet.ssl import CertificateOptions, Certificate, PrivateCertificate from twisted.web.server import Site, Request @@ -911,9 +912,10 @@ def listen_tls( endpoint = _TLSEndpointWrapper.from_paths(endpoint, private_key_path, cert_path) def get_nurl(listening_port: IListeningPort) -> DecodedURL: + address = cast(Union[IPv4Address, IPv6Address], listening_port.getHost()) return build_nurl( hostname, - listening_port.getHost().port, + address.port, str(server._swissnum, "ascii"), load_pem_x509_certificate(cert_path.getContent()), ) From 58247799c1e5f12a2692be0dc72325484a38a6f6 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 3 Oct 2022 11:27:19 -0400 Subject: [PATCH 1116/2309] Fix remaining references to refactored-out-of-existence API. --- src/allmydata/protocol_switch.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/protocol_switch.py b/src/allmydata/protocol_switch.py index 2b4ce6da1..b0af84c33 100644 --- a/src/allmydata/protocol_switch.py +++ b/src/allmydata/protocol_switch.py @@ -53,13 +53,13 @@ class _FoolscapOrHttps(Protocol, metaclass=_PretendToBeNegotiation): since these are created by Foolscap's ``Tub``, by setting this to be the tub's ``negotiationClass``. - Do not instantiate directly, use ``support_foolscap_and_https(tub)`` + Do not instantiate directly, use ``create_tub_with_https_support(...)`` instead. The way this class works is that a new subclass is created for a specific ``Tub`` instance. """ # These are class attributes; they will be set by - # support_foolscap_and_https() and add_storage_server(). + # create_tub_with_https_support() and add_storage_server(). # The Twisted HTTPS protocol factory wrapping the storage server HTTP API: https_factory: TLSMemoryBIOFactory From 795ec0b2dbc6e5f9d5de23fb64d8148b47025ccc Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 3 Oct 2022 11:52:07 -0400 Subject: [PATCH 1117/2309] Fix flake8 issue. --- setup.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/setup.py b/setup.py index d99831347..c8a9669cb 100644 --- a/setup.py +++ b/setup.py @@ -382,6 +382,9 @@ setup(name="tahoe-lafs", # also set in __init__.py ':sys_platform=="win32"': ["pywin32 != 226"], "test": [ "flake8", + # On Python 3.7, importlib_metadata v5 breaks flake8. + # https://github.com/python/importlib_metadata/issues/407 + "importlib_metadata<5; python_version < '3.8'", # Pin a specific pyflakes so we don't have different folks # disagreeing on what is or is not a lint issue. We can bump # this version from time to time, but we will do it From a063241609ad918fe5617a8aedb6abaa660d36a5 Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 3 Oct 2022 10:18:32 -0600 Subject: [PATCH 1118/2309] 1.18.0 release-notes --- relnotes.txt | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/relnotes.txt b/relnotes.txt index e9b298771..dd7cc9429 100644 --- a/relnotes.txt +++ b/relnotes.txt @@ -1,6 +1,6 @@ -ANNOUNCING Tahoe, the Least-Authority File Store, v1.17.1 +ANNOUNCING Tahoe, the Least-Authority File Store, v1.18.0 -The Tahoe-LAFS team is pleased to announce version 1.17.1 of +The Tahoe-LAFS team is pleased to announce version 1.18.0 of Tahoe-LAFS, an extremely reliable decentralized storage system. Get it with "pip install tahoe-lafs", or download a tarball here: @@ -15,10 +15,12 @@ unique security and fault-tolerance properties: https://tahoe-lafs.readthedocs.org/en/latest/about.html -The previous stable release of Tahoe-LAFS was v1.17.0, released on -December 6, 2021. +The previous stable release of Tahoe-LAFS was v1.17.1, released on +January 7, 2022. -This release fixes two Python3-releated regressions and 4 minor bugs. +This release drops support for Python 2 and for Python 3.6 and earlier. +twistd.pid is no longer used (in favour of one with pid + process creation time). +A collection of minor bugs and issues were also fixed. Please see ``NEWS.rst`` [1] for a complete list of changes. @@ -132,24 +134,23 @@ Of Fame" [13]. ACKNOWLEDGEMENTS -This is the nineteenth release of Tahoe-LAFS to be created -solely as a labor of love by volunteers. Thank you very much -to the team of "hackers in the public interest" who make -Tahoe-LAFS possible. +This is the twentieth release of Tahoe-LAFS to be created solely as a +labor of love by volunteers. Thank you very much to the team of +"hackers in the public interest" who make Tahoe-LAFS possible. meejah on behalf of the Tahoe-LAFS team -January 7, 2022 +October 1, 2022 Planet Earth -[1] https://github.com/tahoe-lafs/tahoe-lafs/blob/tahoe-lafs-1.17.1/NEWS.rst +[1] https://github.com/tahoe-lafs/tahoe-lafs/blob/tahoe-lafs-1.18.0/NEWS.rst [2] https://github.com/tahoe-lafs/tahoe-lafs/blob/master/docs/known_issues.rst [3] https://tahoe-lafs.org/trac/tahoe-lafs/wiki/RelatedProjects -[4] https://github.com/tahoe-lafs/tahoe-lafs/blob/tahoe-lafs-1.17.1/COPYING.GPL -[5] https://github.com/tahoe-lafs/tahoe-lafs/blob/tahoe-lafs-1.17.1/COPYING.TGPPL.rst -[6] https://tahoe-lafs.readthedocs.org/en/tahoe-lafs-1.17.1/INSTALL.html +[4] https://github.com/tahoe-lafs/tahoe-lafs/blob/tahoe-lafs-1.18.0/COPYING.GPL +[5] https://github.com/tahoe-lafs/tahoe-lafs/blob/tahoe-lafs-1.18.0/COPYING.TGPPL.rst +[6] https://tahoe-lafs.readthedocs.org/en/tahoe-lafs-1.18.0/INSTALL.html [7] https://lists.tahoe-lafs.org/mailman/listinfo/tahoe-dev [8] https://tahoe-lafs.org/trac/tahoe-lafs/roadmap [9] https://github.com/tahoe-lafs/tahoe-lafs/blob/master/CREDITS From 0e9ab8a0e3b6fe1058a2347270a5c6d3b6dfe060 Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 3 Oct 2022 10:18:58 -0600 Subject: [PATCH 1119/2309] missed release-notes --- newsfragments/3927.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3927.minor diff --git a/newsfragments/3927.minor b/newsfragments/3927.minor new file mode 100644 index 000000000..e69de29bb From c13be0c89b8df744ff46b1b163e4b9138451169c Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 4 Oct 2022 09:19:48 -0400 Subject: [PATCH 1120/2309] Try harder to cleanup. --- src/allmydata/test/test_storage_https.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/allmydata/test/test_storage_https.py b/src/allmydata/test/test_storage_https.py index 3b41e8308..bacb40290 100644 --- a/src/allmydata/test/test_storage_https.py +++ b/src/allmydata/test/test_storage_https.py @@ -198,6 +198,10 @@ class PinningHTTPSValidation(AsyncTestCase): response = await self.request(url, certificate) self.assertEqual(await response.content(), b"YOYODYNE") + # We keep getting TLSMemoryBIOProtocol being left around, so try harder + # to wait for it to finish. + await deferLater(reactor, 0.001) + # A potential attack to test is a private key that doesn't match the # certificate... but OpenSSL (quite rightly) won't let you listen with that # so I don't know how to test that! See From 8b2884cf3a1ce0d4d17c8483202b48055646b7ed Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 4 Oct 2022 09:44:30 -0400 Subject: [PATCH 1121/2309] Make changes work again. --- src/allmydata/node.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/allmydata/node.py b/src/allmydata/node.py index 6747a3c77..7d33d220a 100644 --- a/src/allmydata/node.py +++ b/src/allmydata/node.py @@ -698,7 +698,7 @@ def create_connection_handlers(config, i2p_provider, tor_provider): def create_tub(tub_options, default_connection_handlers, foolscap_connection_handlers, - handler_overrides={}, **kwargs): + handler_overrides={}, force_foolscap=False, **kwargs): """ Create a Tub with the right options and handlers. It will be ephemeral unless the caller provides certFile= in kwargs @@ -708,10 +708,16 @@ def create_tub(tub_options, default_connection_handlers, foolscap_connection_han :param dict tub_options: every key-value pair in here will be set in the new Tub via `Tub.setOption` + + :param bool force_foolscap: If True, only allow Foolscap, not just HTTPS + storage protocol. """ - # We listen simulataneously for both Foolscap and HTTPS on the same port, + # We listen simultaneously for both Foolscap and HTTPS on the same port, # so we have to create a special Foolscap Tub for that to work: - tub = create_tub_with_https_support(**kwargs) + if force_foolscap: + tub = Tub(**kwargs) + else: + tub = create_tub_with_https_support(**kwargs) for (name, value) in list(tub_options.items()): tub.setOption(name, value) @@ -907,11 +913,10 @@ def create_main_tub(config, tub_options, tub_options, default_connection_handlers, foolscap_connection_handlers, + force_foolscap=config.get_config("node", "force_foolscap", False), handler_overrides=handler_overrides, certFile=certfile, ) - if not config.get_config("node", "force_foolscap", False): - support_foolscap_and_https(tub) if portlocation is None: log.msg("Tub is not listening") From fd07c092edf9e0367a0f2c6d770273a4ba1f6a52 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 4 Oct 2022 10:30:07 -0400 Subject: [PATCH 1122/2309] close() is called while writes are still happening. --- src/allmydata/storage_client.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index f9a6feb7d..4ab818b9c 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -1211,7 +1211,7 @@ class _HTTPBucketWriter(object): storage_index = attr.ib(type=bytes) share_number = attr.ib(type=int) upload_secret = attr.ib(type=bytes) - finished = attr.ib(type=bool, default=False) + finished = attr.ib(type=defer.Deferred[bool], factory=defer.Deferred) def abort(self): return self.client.abort_upload(self.storage_index, self.share_number, @@ -1223,14 +1223,13 @@ class _HTTPBucketWriter(object): self.storage_index, self.share_number, self.upload_secret, offset, data ) if result.finished: - self.finished = True + self.finished.callback(True) defer.returnValue(None) def close(self): - # A no-op in HTTP protocol. - if not self.finished: - return defer.fail(RuntimeError("You didn't finish writing?!")) - return defer.succeed(None) + # We're not _really_ closed until all writes have succeeded and we + # finished writing all the data. + return self.finished From 1294baa82e71e1d4cd8c63fc2c3f6e3041062505 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 4 Oct 2022 10:30:27 -0400 Subject: [PATCH 1123/2309] LoopingCall may already have been stopped. --- src/allmydata/storage_client.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 4ab818b9c..a7d5edb11 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -1066,7 +1066,8 @@ class HTTPNativeStorageServer(service.MultiService): def stopService(self): service.MultiService.stopService(self) - self._lc.stop() + if self._lc.running: + self._lc.stop() self._failed_to_connect("shut down") From ea1d2486115b848ec5a8409eae328792e5d2a338 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 4 Oct 2022 10:51:43 -0400 Subject: [PATCH 1124/2309] These objects get stored in a context where they need to be hashed, sometimes. --- src/allmydata/storage/http_client.py | 11 +++++------ src/allmydata/storage_client.py | 5 ++--- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 16d426dda..1fe9a99fd 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -276,7 +276,7 @@ class _StorageClientHTTPSPolicy: ) -@define +@define(hash=True) class StorageClient(object): """ Low-level HTTP client that talks to the HTTP storage server. @@ -286,7 +286,7 @@ class StorageClient(object): # ``StorageClient.from_nurl()``. _base_url: DecodedURL _swissnum: bytes - _treq: Union[treq, StubTreq, HTTPClient] + _treq: Union[treq, StubTreq, HTTPClient] = field(eq=False) @classmethod def from_nurl( @@ -379,13 +379,12 @@ class StorageClient(object): return self._treq.request(method, url, headers=headers, **kwargs) +@define(hash=True) class StorageClientGeneral(object): """ High-level HTTP APIs that aren't immutable- or mutable-specific. """ - - def __init__(self, client): # type: (StorageClient) -> None - self._client = client + _client : StorageClient @inlineCallbacks def get_version(self): @@ -534,7 +533,7 @@ async def advise_corrupt_share( ) -@define +@define(hash=True) class StorageClientImmutables(object): """ APIs for interacting with immutables. diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index a7d5edb11..3b08f0b25 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -1187,7 +1187,7 @@ class _StorageServer(object): -@attr.s +@attr.s(hash=True) class _FakeRemoteReference(object): """ Emulate a Foolscap RemoteReference, calling a local object instead. @@ -1203,7 +1203,6 @@ class _FakeRemoteReference(object): raise RemoteException(e.args) -@attr.s class _HTTPBucketWriter(object): """ Emulate a ``RIBucketWriter``, but use HTTP protocol underneath. @@ -1234,7 +1233,7 @@ class _HTTPBucketWriter(object): -@attr.s +@attr.s(hash=True) class _HTTPBucketReader(object): """ Emulate a ``RIBucketReader``, but use HTTP protocol underneath. From 8190eea48924a095bf8c681fc3a7b9960d7ed839 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 4 Oct 2022 11:02:36 -0400 Subject: [PATCH 1125/2309] Fix bug introduced in previous commit. --- src/allmydata/storage_client.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 3b08f0b25..6d59b4f7d 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -1203,6 +1203,7 @@ class _FakeRemoteReference(object): raise RemoteException(e.args) +@attr.s class _HTTPBucketWriter(object): """ Emulate a ``RIBucketWriter``, but use HTTP protocol underneath. From 8b0ddf406e2863d0991f287032efbb203a15c8c4 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 4 Oct 2022 11:17:19 -0400 Subject: [PATCH 1126/2309] Make HTTP and Foolscap match in another edge case. --- src/allmydata/storage_client.py | 15 ++++++++++++-- src/allmydata/test/test_istorageserver.py | 24 +++++++++++++++++++++++ 2 files changed, 37 insertions(+), 2 deletions(-) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 6d59b4f7d..51b1eabca 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -45,6 +45,7 @@ from zope.interface import ( Interface, implementer, ) +from twisted.python.failure import Failure from twisted.web import http from twisted.internet.task import LoopingCall from twisted.internet import defer, reactor @@ -1233,6 +1234,16 @@ class _HTTPBucketWriter(object): return self.finished +def _ignore_404(failure: Failure) -> Union[Failure, None]: + """ + Useful for advise_corrupt_share(), since it swallows unknown share numbers + in Foolscap. + """ + if failure.check(HTTPClientException) and failure.value.code == http.NOT_FOUND: + return None + else: + return failure + @attr.s(hash=True) class _HTTPBucketReader(object): @@ -1252,7 +1263,7 @@ class _HTTPBucketReader(object): return self.client.advise_corrupt_share( self.storage_index, self.share_number, str(reason, "utf-8", errors="backslashreplace") - ) + ).addErrback(_ignore_404) # WORK IN PROGRESS, for now it doesn't actually implement whole thing. @@ -1352,7 +1363,7 @@ class _HTTPStorageServer(object): raise ValueError("Unknown share type") return client.advise_corrupt_share( storage_index, shnum, str(reason, "utf-8", errors="backslashreplace") - ) + ).addErrback(_ignore_404) @defer.inlineCallbacks def slot_readv(self, storage_index, shares, readv): diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index 81025d779..a0370bdb6 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -440,6 +440,17 @@ class IStorageServerImmutableAPIsTestsMixin(object): b"immutable", storage_index, 0, b"ono" ) + @inlineCallbacks + def test_advise_corrupt_share_unknown_share_number(self): + """ + Calling ``advise_corrupt_share()`` on an immutable share, with an + unknown share number, does not result in error. + """ + storage_index, _, _ = yield self.create_share() + yield self.storage_client.advise_corrupt_share( + b"immutable", storage_index, 999, b"ono" + ) + @inlineCallbacks def test_allocate_buckets_creates_lease(self): """ @@ -909,6 +920,19 @@ class IStorageServerMutableAPIsTestsMixin(object): b"mutable", storage_index, 0, b"ono" ) + @inlineCallbacks + def test_advise_corrupt_share_unknown_share_number(self): + """ + Calling ``advise_corrupt_share()`` on a mutable share with an unknown + share number does not result in error (other behavior is opaque at this + level of abstraction). + """ + secrets, storage_index = yield self.create_slot() + + yield self.storage_client.advise_corrupt_share( + b"mutable", storage_index, 999, b"ono" + ) + @inlineCallbacks def test_STARAW_create_lease(self): """ From 0d23237b11aea61241a75e4d19c6df394b9de0b2 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 13 Oct 2022 13:44:49 -0400 Subject: [PATCH 1127/2309] Some progress towards passing test_rref. --- src/allmydata/storage/http_client.py | 44 +++++++++++++++++++++---- src/allmydata/storage_client.py | 16 +++++---- src/allmydata/test/common_system.py | 2 ++ src/allmydata/test/test_storage_http.py | 5 +++ src/allmydata/test/test_system.py | 9 +++++ 5 files changed, 64 insertions(+), 12 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 1fe9a99fd..2589d4e41 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -20,7 +20,7 @@ from twisted.web.http_headers import Headers from twisted.web import http from twisted.web.iweb import IPolicyForHTTPS from twisted.internet.defer import inlineCallbacks, returnValue, fail, Deferred, succeed -from twisted.internet.interfaces import IOpenSSLClientConnectionCreator +from twisted.internet.interfaces import IOpenSSLClientConnectionCreator, IReactorTime from twisted.internet.ssl import CertificateOptions from twisted.web.client import Agent, HTTPConnectionPool from zope.interface import implementer @@ -282,15 +282,32 @@ class StorageClient(object): Low-level HTTP client that talks to the HTTP storage server. """ + # If True, we're doing unit testing. + TEST_MODE = False + + @classmethod + def start_test_mode(cls): + """Switch to testing mode. + + In testing mode we disable persistent HTTP queries and have shorter + timeouts, to make certain tests work, but don't change the actual + semantic work being done—given a fast server, everything would work the + same. + """ + cls.TEST_MODE = True + # The URL is a HTTPS URL ("https://..."). To construct from a NURL, use # ``StorageClient.from_nurl()``. _base_url: DecodedURL _swissnum: bytes _treq: Union[treq, StubTreq, HTTPClient] = field(eq=False) + _clock: IReactorTime @classmethod def from_nurl( - cls, nurl: DecodedURL, reactor, persistent: bool = True + cls, + nurl: DecodedURL, + reactor, ) -> StorageClient: """ Create a ``StorageClient`` for the given NURL. @@ -302,16 +319,23 @@ class StorageClient(object): swissnum = nurl.path[0].encode("ascii") certificate_hash = nurl.user.encode("ascii") + if cls.TEST_MODE: + pool = HTTPConnectionPool(reactor, persistent=False) + pool.retryAutomatically = False + pool.maxPersistentPerHost = 0 + else: + pool = HTTPConnectionPool(reactor) + treq_client = HTTPClient( Agent( reactor, _StorageClientHTTPSPolicy(expected_spki_hash=certificate_hash), - pool=HTTPConnectionPool(reactor, persistent=persistent), + pool=pool, ) ) https_url = DecodedURL().replace(scheme="https", host=nurl.host, port=nurl.port) - return cls(https_url, swissnum, treq_client) + return cls(https_url, swissnum, treq_client, reactor) def relative_url(self, path): """Get a URL relative to the base URL.""" @@ -376,7 +400,14 @@ class StorageClient(object): kwargs["data"] = dumps(message_to_serialize) headers.addRawHeader("Content-Type", CBOR_MIME_TYPE) - return self._treq.request(method, url, headers=headers, **kwargs) + result = self._treq.request(method, url, headers=headers, **kwargs) + + # If we're in test mode, we want an aggressive timeout, e.g. for + # test_rref in test_system.py. + if self.TEST_MODE: + result.addTimeout(1, self._clock) + + return result @define(hash=True) @@ -384,7 +415,8 @@ class StorageClientGeneral(object): """ High-level HTTP APIs that aren't immutable- or mutable-specific. """ - _client : StorageClient + + _client: StorageClient @inlineCallbacks def get_version(self): diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 51b1eabca..d492ee4cf 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -951,14 +951,18 @@ class HTTPNativeStorageServer(service.MultiService): self.announcement = announcement self._on_status_changed = ObserverList() furl = announcement["anonymous-storage-FURL"].encode("utf-8") - self._nickname, self._permutation_seed, self._tubid, self._short_description, self._long_description = _parse_announcement(server_id, furl, announcement) + ( + self._nickname, + self._permutation_seed, + self._tubid, + self._short_description, + self._long_description + ) = _parse_announcement(server_id, furl, announcement) + # TODO need some way to do equivalent of Happy Eyeballs for multiple NURLs? + # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3935 nurl = DecodedURL.from_text(announcement["anonymous-storage-NURLs"][0]) - # Tests don't want persistent HTTPS pool, since that leaves a dirty - # reactor. As a reasonable hack, disabling persistent connnections for - # localhost allows us to have passing tests while not reducing - # performance for real-world usage. self._istorage_server = _HTTPStorageServer.from_http_client( - StorageClient.from_nurl(nurl, reactor, nurl.host not in ("localhost", "127.0.0.1")) + StorageClient.from_nurl(nurl, reactor) ) self._connection_status = connection_status.ConnectionStatus.unstarted() diff --git a/src/allmydata/test/common_system.py b/src/allmydata/test/common_system.py index 75379bbf3..ef4b65529 100644 --- a/src/allmydata/test/common_system.py +++ b/src/allmydata/test/common_system.py @@ -28,6 +28,7 @@ from foolscap.api import flushEventualQueue from allmydata import client from allmydata.introducer.server import create_introducer from allmydata.util import fileutil, log, pollmixin +from allmydata.storage import http_client from twisted.python.filepath import ( FilePath, @@ -645,6 +646,7 @@ def _render_section_values(values): class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin): def setUp(self): + http_client.StorageClient.start_test_mode() self.port_assigner = SameProcessStreamEndpointAssigner() self.port_assigner.setUp() self.addCleanup(self.port_assigner.tearDown) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 4a912cf6c..819c94f83 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -291,6 +291,7 @@ class CustomHTTPServerTests(SyncTestCase): def setUp(self): super(CustomHTTPServerTests, self).setUp() + StorageClient.start_test_mode() # Could be a fixture, but will only be used in this test class so not # going to bother: self._http_server = TestApp() @@ -298,6 +299,7 @@ class CustomHTTPServerTests(SyncTestCase): DecodedURL.from_text("http://127.0.0.1"), SWISSNUM_FOR_TEST, treq=StubTreq(self._http_server._app.resource()), + clock=Clock() ) def test_authorization_enforcement(self): @@ -375,6 +377,7 @@ class HttpTestFixture(Fixture): """ def _setUp(self): + StorageClient.start_test_mode() self.clock = Clock() self.tempdir = self.useFixture(TempDir()) # The global Cooperator used by Twisted (a) used by pull producers in @@ -396,6 +399,7 @@ class HttpTestFixture(Fixture): DecodedURL.from_text("http://127.0.0.1"), SWISSNUM_FOR_TEST, treq=self.treq, + clock=self.clock, ) def result_of_with_flush(self, d): @@ -480,6 +484,7 @@ class GenericHTTPAPITests(SyncTestCase): DecodedURL.from_text("http://127.0.0.1"), b"something wrong", treq=StubTreq(self.http.http_server.get_resource()), + clock=self.http.clock, ) ) with assert_fails_with_http_code(self, http.UNAUTHORIZED): diff --git a/src/allmydata/test/test_system.py b/src/allmydata/test/test_system.py index d859a0e00..d94b4d163 100644 --- a/src/allmydata/test/test_system.py +++ b/src/allmydata/test/test_system.py @@ -1796,6 +1796,15 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase): class Connections(SystemTestMixin, unittest.TestCase): def test_rref(self): + # The way the listening port is created is via + # SameProcessStreamEndpointAssigner (allmydata.test.common), which then + # makes an endpoint string parsed by AdoptedServerPort. The latter does + # dup(fd), which results in the filedescriptor staying alive _until the + # test ends_. That means that when we disown the service, we still have + # the listening port there on the OS level! Just the resulting + # connections aren't handled. So this test relies on aggressive + # timeouts in the HTTP client and presumably some equivalent in + # Foolscap, since connection refused does _not_ happen. self.basedir = "system/Connections/rref" d = self.set_up_nodes(2) def _start(ign): From b80a215ae1dc80a3760049bec864fe227eee1654 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 13 Oct 2022 13:56:28 -0400 Subject: [PATCH 1128/2309] test_rref passes now. --- src/allmydata/storage_client.py | 8 ++++---- src/allmydata/test/common_system.py | 2 ++ src/allmydata/test/test_system.py | 3 ++- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index d492ee4cf..6f2106f87 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -1052,10 +1052,9 @@ class HTTPNativeStorageServer(service.MultiService): """ See ``IServer.get_storage_server``. """ - if self.is_connected(): - return self._istorage_server - else: + if self._connection_status.summary == "unstarted": return None + return self._istorage_server def stop_connecting(self): self._lc.stop() @@ -1070,10 +1069,11 @@ class HTTPNativeStorageServer(service.MultiService): ) def stopService(self): - service.MultiService.stopService(self) + result = service.MultiService.stopService(self) if self._lc.running: self._lc.stop() self._failed_to_connect("shut down") + return result class UnknownServerTypeError(Exception): diff --git a/src/allmydata/test/common_system.py b/src/allmydata/test/common_system.py index ef4b65529..ee345a0c0 100644 --- a/src/allmydata/test/common_system.py +++ b/src/allmydata/test/common_system.py @@ -21,6 +21,7 @@ from functools import partial from twisted.internet import reactor from twisted.internet import defer from twisted.internet.defer import inlineCallbacks +from twisted.internet.task import deferLater from twisted.application import service from foolscap.api import flushEventualQueue @@ -658,6 +659,7 @@ class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin): log.msg("shutting down SystemTest services") d = self.sparent.stopService() d.addBoth(flush_but_dont_ignore) + d.addBoth(lambda x: deferLater(reactor, 0.01, lambda: x)) return d def getdir(self, subdir): diff --git a/src/allmydata/test/test_system.py b/src/allmydata/test/test_system.py index d94b4d163..c6d2c6bb7 100644 --- a/src/allmydata/test/test_system.py +++ b/src/allmydata/test/test_system.py @@ -1821,9 +1821,10 @@ class Connections(SystemTestMixin, unittest.TestCase): # now shut down the server d.addCallback(lambda ign: self.clients[1].disownServiceParent()) + # and wait for the client to notice def _poll(): - return len(self.c0.storage_broker.get_connected_servers()) < 2 + return len(self.c0.storage_broker.get_connected_servers()) == 1 d.addCallback(lambda ign: self.poll(_poll)) def _down(ign): From 0f31e3cd4b054b17076ffeaa73cc412bc63191b3 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 13 Oct 2022 14:41:59 -0400 Subject: [PATCH 1129/2309] Leave HTTP off by default for now. --- src/allmydata/node.py | 8 ++++++-- src/allmydata/test/common_system.py | 5 ++--- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/allmydata/node.py b/src/allmydata/node.py index 7d33d220a..f572cf7d9 100644 --- a/src/allmydata/node.py +++ b/src/allmydata/node.py @@ -908,12 +908,16 @@ def create_main_tub(config, tub_options, # FIXME? "node.pem" was the CERTFILE option/thing certfile = config.get_private_path("node.pem") - tub = create_tub( tub_options, default_connection_handlers, foolscap_connection_handlers, - force_foolscap=config.get_config("node", "force_foolscap", False), + # TODO eventually we will want the default to be False, but for now we + # don't want to enable HTTP by default. + # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3934 + force_foolscap=config.get_config( + "node", "force_foolscap", default=True, boolean=True + ), handler_overrides=handler_overrides, certFile=certfile, ) diff --git a/src/allmydata/test/common_system.py b/src/allmydata/test/common_system.py index ee345a0c0..edeea5689 100644 --- a/src/allmydata/test/common_system.py +++ b/src/allmydata/test/common_system.py @@ -794,13 +794,13 @@ class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin): if which in feature_matrix.get((section, feature), {which}): config.setdefault(section, {})[feature] = value - if force_foolscap: - config.setdefault("node", {})["force_foolscap"] = force_foolscap + #config.setdefault("node", {})["force_foolscap"] = force_foolscap setnode = partial(setconf, config, which, "node") sethelper = partial(setconf, config, which, "helper") setnode("nickname", u"client %d \N{BLACK SMILING FACE}" % (which,)) + setnode("force_foolscap", str(force_foolscap)) tub_location_hint, tub_port_endpoint = self.port_assigner.assign(reactor) setnode("tub.port", tub_port_endpoint) @@ -818,7 +818,6 @@ class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin): " furl: %s\n") % self.introducer_furl iyaml_fn = os.path.join(basedir, "private", "introducers.yaml") fileutil.write(iyaml_fn, iyaml) - return _render_config(config) def _set_up_client_node(self, which, force_foolscap): From 42d38433436a0f7650704fd45383688f4eeb9ac1 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 14 Oct 2022 09:16:59 -0400 Subject: [PATCH 1130/2309] Run test_system with both Foolscap and HTTP storage protocols, plus some resulting cleanups. --- src/allmydata/test/common_system.py | 37 +++++++------ src/allmydata/test/test_istorageserver.py | 65 +++++++++-------------- src/allmydata/test/test_system.py | 17 +++++- 3 files changed, 63 insertions(+), 56 deletions(-) diff --git a/src/allmydata/test/common_system.py b/src/allmydata/test/common_system.py index edeea5689..96ab4e093 100644 --- a/src/allmydata/test/common_system.py +++ b/src/allmydata/test/common_system.py @@ -5,16 +5,7 @@ in ``allmydata.test.test_system``. Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals - -from future.utils import PY2 -if PY2: - # Don't import bytes since it causes issues on (so far unported) modules on Python 2. - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, dict, list, object, range, max, min, str # noqa: F401 - +from typing import Optional import os from functools import partial @@ -30,6 +21,10 @@ from allmydata import client from allmydata.introducer.server import create_introducer from allmydata.util import fileutil, log, pollmixin from allmydata.storage import http_client +from allmydata.storage_client import ( + NativeStorageServer, + HTTPNativeStorageServer, +) from twisted.python.filepath import ( FilePath, @@ -646,6 +641,11 @@ def _render_section_values(values): class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin): + # If set to True, use Foolscap for storage protocol. If set to False, HTTP + # will be used when possible. If set to None, this suggests a bug in the + # test code. + FORCE_FOOLSCAP_FOR_STORAGE : Optional[bool] = None + def setUp(self): http_client.StorageClient.start_test_mode() self.port_assigner = SameProcessStreamEndpointAssigner() @@ -702,7 +702,7 @@ class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin): return f.read().strip() @inlineCallbacks - def set_up_nodes(self, NUMCLIENTS=5, force_foolscap=False): + def set_up_nodes(self, NUMCLIENTS=5): """ Create an introducer and ``NUMCLIENTS`` client nodes pointed at it. All of the nodes are running in this process. @@ -715,18 +715,25 @@ class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin): :param int NUMCLIENTS: The number of client nodes to create. - :param bool force_foolscap: Force clients to use Foolscap instead of e.g. - HTTPS when available. - :return: A ``Deferred`` that fires when the nodes have connected to each other. """ + self.assertIn( + self.FORCE_FOOLSCAP_FOR_STORAGE, (True, False), + "You forgot to set FORCE_FOOLSCAP_FOR_STORAGE on {}".format(self.__class__) + ) self.numclients = NUMCLIENTS self.introducer = yield self._create_introducer() self.add_service(self.introducer) self.introweb_url = self._get_introducer_web() - yield self._set_up_client_nodes(force_foolscap) + yield self._set_up_client_nodes(self.FORCE_FOOLSCAP_FOR_STORAGE) + native_server = next(iter(self.clients[0].storage_broker.get_known_servers())) + if self.FORCE_FOOLSCAP_FOR_STORAGE: + expected_storage_server_class = NativeStorageServer + else: + expected_storage_server_class = HTTPNativeStorageServer + self.assertIsInstance(native_server, expected_storage_server_class) @inlineCallbacks def _set_up_client_nodes(self, force_foolscap): diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index a0370bdb6..a488622c7 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -1046,13 +1046,12 @@ class _SharedMixin(SystemTestMixin): """Base class for Foolscap and HTTP mixins.""" SKIP_TESTS = set() # type: Set[str] - FORCE_FOOLSCAP = False - - def _get_native_server(self): - return next(iter(self.clients[0].storage_broker.get_known_servers())) def _get_istorage_server(self): - raise NotImplementedError("implement in subclass") + native_server = next(iter(self.clients[0].storage_broker.get_known_servers())) + client = native_server.get_storage_server() + self.assertTrue(IStorageServer.providedBy(client)) + return client @inlineCallbacks def setUp(self): @@ -1065,7 +1064,7 @@ class _SharedMixin(SystemTestMixin): self.basedir = "test_istorageserver/" + self.id() yield SystemTestMixin.setUp(self) - yield self.set_up_nodes(1, self.FORCE_FOOLSCAP) + yield self.set_up_nodes(1) self.server = None for s in self.clients[0].services: if isinstance(s, StorageServer): @@ -1075,7 +1074,7 @@ class _SharedMixin(SystemTestMixin): self._clock = Clock() self._clock.advance(123456) self.server._clock = self._clock - self.storage_client = yield self._get_istorage_server() + self.storage_client = self._get_istorage_server() def fake_time(self): """Return the current fake, test-controlled, time.""" @@ -1091,49 +1090,29 @@ class _SharedMixin(SystemTestMixin): yield SystemTestMixin.tearDown(self) -class _FoolscapMixin(_SharedMixin): - """Run tests on Foolscap version of ``IStorageServer``.""" - - FORCE_FOOLSCAP = True - - def _get_istorage_server(self): - native_server = self._get_native_server() - assert isinstance(native_server, NativeStorageServer) - client = native_server.get_storage_server() - self.assertTrue(IStorageServer.providedBy(client)) - return succeed(client) - - -class _HTTPMixin(_SharedMixin): - """Run tests on the HTTP version of ``IStorageServer``.""" - - FORCE_FOOLSCAP = False - - def _get_istorage_server(self): - native_server = self._get_native_server() - assert isinstance(native_server, HTTPNativeStorageServer) - client = native_server.get_storage_server() - self.assertTrue(IStorageServer.providedBy(client)) - return succeed(client) - - class FoolscapSharedAPIsTests( - _FoolscapMixin, IStorageServerSharedAPIsTestsMixin, AsyncTestCase + _SharedMixin, IStorageServerSharedAPIsTestsMixin, AsyncTestCase ): """Foolscap-specific tests for shared ``IStorageServer`` APIs.""" + FORCE_FOOLSCAP_FOR_STORAGE = True + class HTTPSharedAPIsTests( - _HTTPMixin, IStorageServerSharedAPIsTestsMixin, AsyncTestCase + _SharedMixin, IStorageServerSharedAPIsTestsMixin, AsyncTestCase ): """HTTP-specific tests for shared ``IStorageServer`` APIs.""" + FORCE_FOOLSCAP_FOR_STORAGE = False + class FoolscapImmutableAPIsTests( - _FoolscapMixin, IStorageServerImmutableAPIsTestsMixin, AsyncTestCase + _SharedMixin, IStorageServerImmutableAPIsTestsMixin, AsyncTestCase ): """Foolscap-specific tests for immutable ``IStorageServer`` APIs.""" + FORCE_FOOLSCAP_FOR_STORAGE = True + def test_disconnection(self): """ If we disconnect in the middle of writing to a bucket, all data is @@ -1156,23 +1135,29 @@ class FoolscapImmutableAPIsTests( """ current = self.storage_client yield self.bounce_client(0) - self.storage_client = self._get_native_server().get_storage_server() + self.storage_client = self._get_istorage_server() assert self.storage_client is not current class HTTPImmutableAPIsTests( - _HTTPMixin, IStorageServerImmutableAPIsTestsMixin, AsyncTestCase + _SharedMixin, IStorageServerImmutableAPIsTestsMixin, AsyncTestCase ): """HTTP-specific tests for immutable ``IStorageServer`` APIs.""" + FORCE_FOOLSCAP_FOR_STORAGE = False + class FoolscapMutableAPIsTests( - _FoolscapMixin, IStorageServerMutableAPIsTestsMixin, AsyncTestCase + _SharedMixin, IStorageServerMutableAPIsTestsMixin, AsyncTestCase ): """Foolscap-specific tests for mutable ``IStorageServer`` APIs.""" + FORCE_FOOLSCAP_FOR_STORAGE = True + class HTTPMutableAPIsTests( - _HTTPMixin, IStorageServerMutableAPIsTestsMixin, AsyncTestCase + _SharedMixin, IStorageServerMutableAPIsTestsMixin, AsyncTestCase ): """HTTP-specific tests for mutable ``IStorageServer`` APIs.""" + + FORCE_FOOLSCAP_FOR_STORAGE = False diff --git a/src/allmydata/test/test_system.py b/src/allmydata/test/test_system.py index c6d2c6bb7..a83ff9488 100644 --- a/src/allmydata/test/test_system.py +++ b/src/allmydata/test/test_system.py @@ -117,7 +117,8 @@ class CountingDataUploadable(upload.Data): class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase): - + """Foolscap integration-y tests.""" + FORCE_FOOLSCAP_FOR_STORAGE = True timeout = 180 def test_connections(self): @@ -1794,6 +1795,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase): class Connections(SystemTestMixin, unittest.TestCase): + FORCE_FOOLSCAP_FOR_STORAGE = True def test_rref(self): # The way the listening port is created is via @@ -1834,3 +1836,16 @@ class Connections(SystemTestMixin, unittest.TestCase): self.assertEqual(storage_server, self.s1_storage_server) d.addCallback(_down) return d + + +class HTTPSystemTest(SystemTest): + """HTTP storage protocol variant of the system tests.""" + + FORCE_FOOLSCAP_FOR_STORAGE = False + + + +class HTTPConnections(Connections): + """HTTP storage protocol variant of the connections tests.""" + FORCE_FOOLSCAP_FOR_STORAGE = False + From e409262e86ff3639187bfa89f438b6e9db071228 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 14 Oct 2022 09:50:07 -0400 Subject: [PATCH 1131/2309] Fix some flakes. --- src/allmydata/test/test_istorageserver.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index a488622c7..9e7e7b6e1 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -15,7 +15,7 @@ from typing import Set from random import Random from unittest import SkipTest -from twisted.internet.defer import inlineCallbacks, returnValue, succeed +from twisted.internet.defer import inlineCallbacks, returnValue from twisted.internet.task import Clock from foolscap.api import Referenceable, RemoteException @@ -25,10 +25,6 @@ from allmydata.interfaces import IStorageServer from .common_system import SystemTestMixin from .common import AsyncTestCase from allmydata.storage.server import StorageServer # not a IStorageServer!! -from allmydata.storage_client import ( - NativeStorageServer, - HTTPNativeStorageServer, -) # Use random generator with known seed, so results are reproducible if tests From 0febc8745653992cbb53d98702c92edc24b7a516 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 14 Oct 2022 10:03:06 -0400 Subject: [PATCH 1132/2309] Don't include reactor in comparison. --- src/allmydata/storage/http_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 2589d4e41..40979d3cb 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -301,7 +301,7 @@ class StorageClient(object): _base_url: DecodedURL _swissnum: bytes _treq: Union[treq, StubTreq, HTTPClient] = field(eq=False) - _clock: IReactorTime + _clock: IReactorTime = field(eq=False) @classmethod def from_nurl( From f68c3978f616c5efecc15094aa83c363bd6db58d Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 14 Oct 2022 10:18:38 -0400 Subject: [PATCH 1133/2309] News fragment. --- newsfragments/3783.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3783.minor diff --git a/newsfragments/3783.minor b/newsfragments/3783.minor new file mode 100644 index 000000000..e69de29bb From 1a3e3a86c317c22a79790cc134102f6dc5b368ff Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 14 Oct 2022 11:27:04 -0400 Subject: [PATCH 1134/2309] Require latest pycddl, and work around a regression. --- newsfragments/3938.bugfix | 1 + setup.py | 2 +- src/allmydata/storage/http_client.py | 12 ++++++------ src/allmydata/storage/http_server.py | 12 +++++------- 4 files changed, 13 insertions(+), 14 deletions(-) create mode 100644 newsfragments/3938.bugfix diff --git a/newsfragments/3938.bugfix b/newsfragments/3938.bugfix new file mode 100644 index 000000000..c2778cfdf --- /dev/null +++ b/newsfragments/3938.bugfix @@ -0,0 +1 @@ +Work with (and require) newer versions of pycddl. \ No newline at end of file diff --git a/setup.py b/setup.py index 72478767c..768e44e29 100644 --- a/setup.py +++ b/setup.py @@ -137,7 +137,7 @@ install_requires = [ "werkzeug != 2.2.0", "treq", "cbor2", - "pycddl", + "pycddl >= 0.2", # for pid-file support "psutil", diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 16d426dda..420d3610f 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -83,35 +83,35 @@ _SCHEMAS = { "allocate_buckets": Schema( """ response = { - already-have: #6.258([* uint]) - allocated: #6.258([* uint]) + already-have: #6.258([0*256 uint]) + allocated: #6.258([0*256 uint]) } """ ), "immutable_write_share_chunk": Schema( """ response = { - required: [* {begin: uint, end: uint}] + required: [0* {begin: uint, end: uint}] } """ ), "list_shares": Schema( """ - response = #6.258([* uint]) + response = #6.258([0*256 uint]) """ ), "mutable_read_test_write": Schema( """ response = { "success": bool, - "data": {* share_number: [* bstr]} + "data": {0*256 share_number: [0* bstr]} } share_number = uint """ ), "mutable_list_shares": Schema( """ - response = #6.258([* uint]) + response = #6.258([0*256 uint]) """ ), } diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index eefb9b906..3902976ba 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -260,7 +260,7 @@ _SCHEMAS = { "allocate_buckets": Schema( """ request = { - share-numbers: #6.258([*256 uint]) + share-numbers: #6.258([0*256 uint]) allocated-size: uint } """ @@ -276,15 +276,13 @@ _SCHEMAS = { """ request = { "test-write-vectors": { - ; TODO Add length limit here, after - ; https://github.com/anweiss/cddl/issues/128 is fixed - * share_number => { - "test": [*30 {"offset": uint, "size": uint, "specimen": bstr}] - "write": [*30 {"offset": uint, "data": bstr}] + 0*256 share_number : { + "test": [0*30 {"offset": uint, "size": uint, "specimen": bstr}] + "write": [0*30 {"offset": uint, "data": bstr}] "new-length": uint / null } } - "read-vector": [*30 {"offset": uint, "size": uint}] + "read-vector": [0*30 {"offset": uint, "size": uint}] } share_number = uint """ From 46fbe3d0283695dc503fabb0b9f8c4ed9401cdcf Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 18 Oct 2022 17:32:23 -0400 Subject: [PATCH 1135/2309] bump pypi-deps-db for new pycddl version --- nix/sources.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nix/sources.json b/nix/sources.json index 79eabe7a1..950151416 100644 --- a/nix/sources.json +++ b/nix/sources.json @@ -53,10 +53,10 @@ "homepage": "", "owner": "DavHau", "repo": "pypi-deps-db", - "rev": "76b8f1e44a8ec051b853494bcf3cc8453a294a6a", - "sha256": "18fgqyh4z578jjhk26n1xi2cw2l98vrqp962rgz9a6wa5yh1nm4x", + "rev": "5fe7d2d1c85cd86d64f4f079eef3f1ff5653bcd6", + "sha256": "0pc6mj7rzvmhh303rvj5wf4hrksm4h2rf4fsvqs0ljjdmgxrqm3f", "type": "tarball", - "url": "https://github.com/DavHau/pypi-deps-db/archive/76b8f1e44a8ec051b853494bcf3cc8453a294a6a.tar.gz", + "url": "https://github.com/DavHau/pypi-deps-db/archive/5fe7d2d1c85cd86d64f4f079eef3f1ff5653bcd6.tar.gz", "url_template": "https://github.com///archive/.tar.gz" } } From 48ae729c0de57818d132763aa62e99faffd46556 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 2 Nov 2022 10:18:23 -0400 Subject: [PATCH 1136/2309] Don't reuse basedir across tests. --- src/allmydata/test/test_system.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/src/allmydata/test/test_system.py b/src/allmydata/test/test_system.py index a83ff9488..f03d795ba 100644 --- a/src/allmydata/test/test_system.py +++ b/src/allmydata/test/test_system.py @@ -121,8 +121,13 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase): FORCE_FOOLSCAP_FOR_STORAGE = True timeout = 180 + @property + def basedir(self): + return "system/SystemTest/{}-foolscap-{}".format( + self.id().split(".")[-1], self.FORCE_FOOLSCAP_FOR_STORAGE + ) + def test_connections(self): - self.basedir = "system/SystemTest/test_connections" d = self.set_up_nodes() self.extra_node = None d.addCallback(lambda res: self.add_extra_node(self.numclients)) @@ -150,11 +155,9 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase): del test_connections def test_upload_and_download_random_key(self): - self.basedir = "system/SystemTest/test_upload_and_download_random_key" return self._test_upload_and_download(convergence=None) def test_upload_and_download_convergent(self): - self.basedir = "system/SystemTest/test_upload_and_download_convergent" return self._test_upload_and_download(convergence=b"some convergence string") def _test_upload_and_download(self, convergence): @@ -517,7 +520,6 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase): def test_mutable(self): - self.basedir = "system/SystemTest/test_mutable" DATA = b"initial contents go here." # 25 bytes % 3 != 0 DATA_uploadable = MutableData(DATA) NEWDATA = b"new contents yay" @@ -747,7 +749,6 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase): # plaintext_hash check. def test_filesystem(self): - self.basedir = "system/SystemTest/test_filesystem" self.data = LARGE_DATA d = self.set_up_nodes() def _new_happy_semantics(ign): @@ -1714,7 +1715,6 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase): def test_filesystem_with_cli_in_subprocess(self): # We do this in a separate test so that test_filesystem doesn't skip if we can't run bin/tahoe. - self.basedir = "system/SystemTest/test_filesystem_with_cli_in_subprocess" d = self.set_up_nodes() def _new_happy_semantics(ign): for c in self.clients: @@ -1807,7 +1807,9 @@ class Connections(SystemTestMixin, unittest.TestCase): # connections aren't handled. So this test relies on aggressive # timeouts in the HTTP client and presumably some equivalent in # Foolscap, since connection refused does _not_ happen. - self.basedir = "system/Connections/rref" + self.basedir = "system/Connections/rref-foolscap-{}".format( + self.FORCE_FOOLSCAP_FOR_STORAGE + ) d = self.set_up_nodes(2) def _start(ign): self.c0 = self.clients[0] From e05136c2385d222bd50413054dc8ac2a9d60d243 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 2 Nov 2022 13:13:21 -0400 Subject: [PATCH 1137/2309] Less aggressive timeout, to try to make tests pass on CI. --- src/allmydata/storage/http_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index adc3e1525..e520088c3 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -405,7 +405,7 @@ class StorageClient(object): # If we're in test mode, we want an aggressive timeout, e.g. for # test_rref in test_system.py. if self.TEST_MODE: - result.addTimeout(1, self._clock) + result.addTimeout(5, self._clock) return result From db59eb12c092264f357c59afc3586dcb8259d0f8 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 2 Nov 2022 15:22:36 -0400 Subject: [PATCH 1138/2309] Increase timeout. --- .circleci/run-tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/run-tests.sh b/.circleci/run-tests.sh index 764651c40..854013c32 100755 --- a/.circleci/run-tests.sh +++ b/.circleci/run-tests.sh @@ -52,7 +52,7 @@ fi # This is primarily aimed at catching hangs on the PyPy job which runs for # about 21 minutes and then gets killed by CircleCI in a way that fails the # job and bypasses our "allowed failure" logic. -TIMEOUT="timeout --kill-after 1m 15m" +TIMEOUT="timeout --kill-after 1m 25m" # Run the test suite as a non-root user. This is the expected usage some # small areas of the test suite assume non-root privileges (such as unreadable From 262d9d85b97cb064da47c82bab22e62b48db6cd4 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 3 Nov 2022 14:14:21 -0400 Subject: [PATCH 1139/2309] Switch to using persistent connections in tests too. --- src/allmydata/storage/http_client.py | 34 +++++++++++++++------------- src/allmydata/test/common_system.py | 10 +++++++- src/allmydata/test/test_system.py | 3 +++ 3 files changed, 30 insertions(+), 17 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index e520088c3..96820d4a5 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -282,19 +282,20 @@ class StorageClient(object): Low-level HTTP client that talks to the HTTP storage server. """ - # If True, we're doing unit testing. - TEST_MODE = False + # If set, we're doing unit testing and we should call this with + # HTTPConnectionPool we create. + TEST_MODE_REGISTER_HTTP_POOL = None @classmethod - def start_test_mode(cls): + def start_test_mode(cls, callback): """Switch to testing mode. - In testing mode we disable persistent HTTP queries and have shorter - timeouts, to make certain tests work, but don't change the actual - semantic work being done—given a fast server, everything would work the - same. + In testing mode we register the pool with test system using the given + callback so it can Do Things, most notably killing off idle HTTP + connections at test shutdown and, in some tests, in the midddle of the + test. """ - cls.TEST_MODE = True + cls.TEST_MODE_REGISTER_HTTP_POOL = callback # The URL is a HTTPS URL ("https://..."). To construct from a NURL, use # ``StorageClient.from_nurl()``. @@ -318,13 +319,10 @@ class StorageClient(object): assert nurl.scheme == "pb" swissnum = nurl.path[0].encode("ascii") certificate_hash = nurl.user.encode("ascii") + pool = HTTPConnectionPool(reactor) - if cls.TEST_MODE: - pool = HTTPConnectionPool(reactor, persistent=False) - pool.retryAutomatically = False - pool.maxPersistentPerHost = 0 - else: - pool = HTTPConnectionPool(reactor) + if cls.TEST_MODE_REGISTER_HTTP_POOL is not None: + cls.TEST_MODE_REGISTER_HTTP_POOL(pool) treq_client = HTTPClient( Agent( @@ -403,8 +401,12 @@ class StorageClient(object): result = self._treq.request(method, url, headers=headers, **kwargs) # If we're in test mode, we want an aggressive timeout, e.g. for - # test_rref in test_system.py. - if self.TEST_MODE: + # test_rref in test_system.py. Unfortunately, test_rref results in the + # socket still listening(!), only without an HTTP server, due to limits + # in the relevant socket-binding test setup code. As a result, we don't + # get connection refused, the client will successfully connect. So we + # want a timeout so we notice that. + if self.TEST_MODE_REGISTER_HTTP_POOL is not None: result.addTimeout(5, self._clock) return result diff --git a/src/allmydata/test/common_system.py b/src/allmydata/test/common_system.py index 96ab4e093..f47aad3b6 100644 --- a/src/allmydata/test/common_system.py +++ b/src/allmydata/test/common_system.py @@ -647,7 +647,8 @@ class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin): FORCE_FOOLSCAP_FOR_STORAGE : Optional[bool] = None def setUp(self): - http_client.StorageClient.start_test_mode() + self._http_client_pools = [] + http_client.StorageClient.start_test_mode(self._http_client_pools.append) self.port_assigner = SameProcessStreamEndpointAssigner() self.port_assigner.setUp() self.addCleanup(self.port_assigner.tearDown) @@ -655,10 +656,17 @@ class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin): self.sparent = service.MultiService() self.sparent.startService() + def close_idle_http_connections(self): + """Close all HTTP client connections that are just hanging around.""" + return defer.gatherResults( + [pool.closeCachedConnections() for pool in self._http_client_pools] + ) + def tearDown(self): log.msg("shutting down SystemTest services") d = self.sparent.stopService() d.addBoth(flush_but_dont_ignore) + d.addBoth(lambda x: self.close_idle_http_connections().addCallback(lambda _: x)) d.addBoth(lambda x: deferLater(reactor, 0.01, lambda: x)) return d diff --git a/src/allmydata/test/test_system.py b/src/allmydata/test/test_system.py index f03d795ba..670ac5868 100644 --- a/src/allmydata/test/test_system.py +++ b/src/allmydata/test/test_system.py @@ -1826,6 +1826,9 @@ class Connections(SystemTestMixin, unittest.TestCase): # now shut down the server d.addCallback(lambda ign: self.clients[1].disownServiceParent()) + # kill any persistent http connections that might continue to work + d.addCallback(lambda ign: self.close_idle_http_connections()) + # and wait for the client to notice def _poll(): return len(self.c0.storage_broker.get_connected_servers()) == 1 From 8bebb09edd2026a77dd6f8081a1fe7c0069071b3 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 3 Nov 2022 14:38:59 -0400 Subject: [PATCH 1140/2309] Less test-specific way to make test_rref pass. --- src/allmydata/storage/http_client.py | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 96820d4a5..7fcf8114c 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -398,18 +398,7 @@ class StorageClient(object): kwargs["data"] = dumps(message_to_serialize) headers.addRawHeader("Content-Type", CBOR_MIME_TYPE) - result = self._treq.request(method, url, headers=headers, **kwargs) - - # If we're in test mode, we want an aggressive timeout, e.g. for - # test_rref in test_system.py. Unfortunately, test_rref results in the - # socket still listening(!), only without an HTTP server, due to limits - # in the relevant socket-binding test setup code. As a result, we don't - # get connection refused, the client will successfully connect. So we - # want a timeout so we notice that. - if self.TEST_MODE_REGISTER_HTTP_POOL is not None: - result.addTimeout(5, self._clock) - - return result + return self._treq.request(method, url, headers=headers, **kwargs) @define(hash=True) @@ -426,7 +415,12 @@ class StorageClientGeneral(object): Return the version metadata for the server. """ url = self._client.relative_url("/storage/v1/version") - response = yield self._client.request("GET", url) + result = self._client.request("GET", url) + # 1. Getting the version should never take particularly long. + # 2. Clients rely on the version command for liveness checks of servers. + result.addTimeout(5, self._client._clock) + + response = yield result decoded_response = yield _decode_cbor(response, _SCHEMAS["get_version"]) returnValue(decoded_response) From 1e50e96e2456910598862e64f7585a6dd47d59f2 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 3 Nov 2022 15:04:41 -0400 Subject: [PATCH 1141/2309] Update to new test API. --- src/allmydata/test/test_storage_http.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 819c94f83..25c21e03f 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -291,7 +291,9 @@ class CustomHTTPServerTests(SyncTestCase): def setUp(self): super(CustomHTTPServerTests, self).setUp() - StorageClient.start_test_mode() + StorageClient.start_test_mode( + lambda pool: self.addCleanup(pool.closeCachedConnections) + ) # Could be a fixture, but will only be used in this test class so not # going to bother: self._http_server = TestApp() @@ -299,7 +301,7 @@ class CustomHTTPServerTests(SyncTestCase): DecodedURL.from_text("http://127.0.0.1"), SWISSNUM_FOR_TEST, treq=StubTreq(self._http_server._app.resource()), - clock=Clock() + clock=Clock(), ) def test_authorization_enforcement(self): @@ -377,7 +379,9 @@ class HttpTestFixture(Fixture): """ def _setUp(self): - StorageClient.start_test_mode() + StorageClient.start_test_mode( + lambda pool: self.addCleanup(pool.closeCachedConnections) + ) self.clock = Clock() self.tempdir = self.useFixture(TempDir()) # The global Cooperator used by Twisted (a) used by pull producers in @@ -1446,7 +1450,9 @@ class SharedImmutableMutableTestsMixin: self.http.client.request( "GET", self.http.client.relative_url( - "/storage/v1/{}/{}/1".format(self.KIND, _encode_si(storage_index)) + "/storage/v1/{}/{}/1".format( + self.KIND, _encode_si(storage_index) + ) ), headers=headers, ) From 414b4635569145ed277bfe0e0e540d62430e0bb8 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 7 Nov 2022 09:23:04 -0500 Subject: [PATCH 1142/2309] Use built-in treq timeout feature. --- src/allmydata/storage/http_client.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 7fcf8114c..d6121aba2 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -415,12 +415,10 @@ class StorageClientGeneral(object): Return the version metadata for the server. """ url = self._client.relative_url("/storage/v1/version") - result = self._client.request("GET", url) # 1. Getting the version should never take particularly long. # 2. Clients rely on the version command for liveness checks of servers. - result.addTimeout(5, self._client._clock) - - response = yield result + # Thus, a short timeout. + response = yield self._client.request("GET", url, timeout=5) decoded_response = yield _decode_cbor(response, _SCHEMAS["get_version"]) returnValue(decoded_response) From c4772482ef19d5e1aeed99f01e38fab52a14786d Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 7 Nov 2022 11:19:00 -0500 Subject: [PATCH 1143/2309] WIP --- src/allmydata/storage/http_client.py | 33 +++++++++++++--- src/allmydata/test/test_storage_http.py | 51 ++++++++++++++++++++++++- 2 files changed, 78 insertions(+), 6 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 420d3610f..0bf68fdd3 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -20,8 +20,13 @@ from twisted.web.http_headers import Headers from twisted.web import http from twisted.web.iweb import IPolicyForHTTPS from twisted.internet.defer import inlineCallbacks, returnValue, fail, Deferred, succeed -from twisted.internet.interfaces import IOpenSSLClientConnectionCreator +from twisted.internet.interfaces import ( + IOpenSSLClientConnectionCreator, + IReactorTime, + IDelayedCall, +) from twisted.internet.ssl import CertificateOptions +from twisted.internet import reactor from twisted.web.client import Agent, HTTPConnectionPool from zope.interface import implementer from hyperlink import DecodedURL @@ -124,16 +129,20 @@ class _LengthLimitedCollector: """ remaining_length: int + timeout_on_silence: IDelayedCall f: BytesIO = field(factory=BytesIO) def __call__(self, data: bytes): + self.timeout_on_silence.reset(60) self.remaining_length -= len(data) if self.remaining_length < 0: raise ValueError("Response length was too long") self.f.write(data) -def limited_content(response, max_length: int = 30 * 1024 * 1024) -> Deferred[BinaryIO]: +def limited_content( + response, max_length: int = 30 * 1024 * 1024, clock: IReactorTime = reactor +) -> Deferred[BinaryIO]: """ Like ``treq.content()``, but limit data read from the response to a set length. If the response is longer than the max allowed length, the result @@ -142,11 +151,16 @@ def limited_content(response, max_length: int = 30 * 1024 * 1024) -> Deferred[Bi A potentially useful future improvement would be using a temporary file to store the content; since filesystem buffering means that would use memory for small responses and disk for large responses. + + This will time out if no data is received for 60 seconds; so long as a + trickle of data continues to arrive, it will continue to run. """ - collector = _LengthLimitedCollector(max_length) + d = succeed(None) + timeout = clock.callLater(60, d.cancel) + collector = _LengthLimitedCollector(max_length, timeout) + # Make really sure everything gets called in Deferred context, treq might # call collector directly... - d = succeed(None) d.addCallback(lambda _: treq.collect(response, collector)) def done(_): @@ -307,6 +321,8 @@ class StorageClient(object): reactor, _StorageClientHTTPSPolicy(expected_spki_hash=certificate_hash), pool=HTTPConnectionPool(reactor, persistent=persistent), + # TCP-level connection timeout + connectTimeout=5, ) ) @@ -337,6 +353,7 @@ class StorageClient(object): write_enabler_secret=None, headers=None, message_to_serialize=None, + timeout: Union[int, float] = 60, **kwargs, ): """ @@ -376,7 +393,9 @@ class StorageClient(object): kwargs["data"] = dumps(message_to_serialize) headers.addRawHeader("Content-Type", CBOR_MIME_TYPE) - return self._treq.request(method, url, headers=headers, **kwargs) + return self._treq.request( + method, url, headers=headers, timeout=timeout, **kwargs + ) class StorageClientGeneral(object): @@ -461,6 +480,9 @@ def read_share_chunk( share_type, _encode_si(storage_index), share_number ) ) + # The default timeout is for getting the response, so it doesn't include + # the time it takes to download the body... so we will will deal with that + # later. response = yield client.request( "GET", url, @@ -469,6 +491,7 @@ def read_share_chunk( # but Range constructor does that the conversion for us. {"range": [Range("bytes", [(offset, offset + length)]).to_header()]} ), + unbuffered=True, # Don't buffer the response in memory. ) if response.code == http.NO_CONTENT: diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 4a912cf6c..54a26da09 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -31,6 +31,8 @@ from klein import Klein from hyperlink import DecodedURL from collections_extended import RangeMap from twisted.internet.task import Clock, Cooperator +from twisted.internet.interfaces import IReactorTime +from twisted.internet.defer import CancelledError, Deferred from twisted.web import http from twisted.web.http_headers import Headers from werkzeug import routing @@ -245,6 +247,7 @@ def gen_bytes(length: int) -> bytes: class TestApp(object): """HTTP API for testing purposes.""" + clock: IReactorTime _app = Klein() _swissnum = SWISSNUM_FOR_TEST # Match what the test client is using @@ -266,6 +269,17 @@ class TestApp(object): """Return bytes to the given length using ``gen_bytes()``.""" return gen_bytes(length) + @_authorized_route(_app, set(), "/slowly_never_finish_result", methods=["GET"]) + def slowly_never_finish_result(self, request, authorization): + """ + Send data immediately, after 59 seconds, after another 59 seconds, and then + never again, without finishing the response. + """ + request.write(b"a") + self.clock.callLater(59, request.write, b"b") + self.clock.callLater(59 + 59, request.write, b"c") + return Deferred() + def result_of(d): """ @@ -299,6 +313,10 @@ class CustomHTTPServerTests(SyncTestCase): SWISSNUM_FOR_TEST, treq=StubTreq(self._http_server._app.resource()), ) + # We're using a Treq private API to get the reactor, alas, but only in + # a test, so not going to worry about it too much. This would be fixed + # if https://github.com/twisted/treq/issues/226 were ever fixed. + self._http_server.clock = self.client._treq._agent._memoryReactor def test_authorization_enforcement(self): """ @@ -367,6 +385,35 @@ class CustomHTTPServerTests(SyncTestCase): with self.assertRaises(ValueError): result_of(limited_content(response, too_short)) + def test_limited_content_silence_causes_timeout(self): + """ + ``http_client.limited_content() times out if it receives no data for 60 + seconds. + """ + response = result_of( + self.client.request( + "GET", + "http://127.0.0.1/slowly_never_finish_result", + ) + ) + + body_deferred = limited_content(response, 4, self._http_server.clock) + result = [] + error = [] + body_deferred.addCallbacks(result.append, error.append) + + for i in range(59 + 59 + 60): + self.assertEqual((result, error), ([], [])) + self._http_server.clock.advance(1) + # Push data between in-memory client and in-memory server: + self.client._treq._agent.flush() + + # After 59 (second write) + 59 (third write) + 60 seconds (quiescent + # timeout) the limited_content() response times out. + self.assertTrue(error) + with self.assertRaises(CancelledError): + error[0].raiseException() + class HttpTestFixture(Fixture): """ @@ -1441,7 +1488,9 @@ class SharedImmutableMutableTestsMixin: self.http.client.request( "GET", self.http.client.relative_url( - "/storage/v1/{}/{}/1".format(self.KIND, _encode_si(storage_index)) + "/storage/v1/{}/{}/1".format( + self.KIND, _encode_si(storage_index) + ) ), headers=headers, ) From f8b9607fc2c1062609eb3bcf42024ad7e81e729f Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 7 Nov 2022 11:26:11 -0500 Subject: [PATCH 1144/2309] Finish up limited_content() timeout code. --- src/allmydata/storage/http_client.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 56f7aa629..c76cd00b9 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -164,6 +164,7 @@ def limited_content( d.addCallback(lambda _: treq.collect(response, collector)) def done(_): + timeout.cancel() collector.f.seek(0) return collector.f @@ -539,7 +540,7 @@ def read_share_chunk( raise ValueError("Server sent more than we asked for?!") # It might also send less than we asked for. That's (probably) OK, e.g. # if we went past the end of the file. - body = yield limited_content(response, supposed_length) + body = yield limited_content(response, supposed_length, client._clock) body.seek(0, SEEK_END) actual_length = body.tell() if actual_length != supposed_length: From 2c911eeac1901fbc333d550e6923d225e6ed07cb Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 7 Nov 2022 11:28:36 -0500 Subject: [PATCH 1145/2309] Make sure everything is using the same clock. --- src/allmydata/test/test_storage_http.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 55bc8f79a..3ee955c3b 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -311,16 +311,18 @@ class CustomHTTPServerTests(SyncTestCase): # Could be a fixture, but will only be used in this test class so not # going to bother: self._http_server = TestApp() + treq = StubTreq(self._http_server._app.resource()) self.client = StorageClient( DecodedURL.from_text("http://127.0.0.1"), SWISSNUM_FOR_TEST, - treq=StubTreq(self._http_server._app.resource()), - clock=Clock(), + treq=treq, + # We're using a Treq private API to get the reactor, alas, but only + # in a test, so not going to worry about it too much. This would be + # fixed if https://github.com/twisted/treq/issues/226 were ever + # fixed. + clock=treq._agent._memoryReactor, ) - # We're using a Treq private API to get the reactor, alas, but only in - # a test, so not going to worry about it too much. This would be fixed - # if https://github.com/twisted/treq/issues/226 were ever fixed. - self._http_server.clock = self.client._treq._agent._memoryReactor + self._http_server.clock = self.client._clock def test_authorization_enforcement(self): """ From afd4f52ff74d4d3f73258ec9ac27e1dea3a928e5 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 7 Nov 2022 11:32:14 -0500 Subject: [PATCH 1146/2309] News file. --- newsfragments/3940.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3940.minor diff --git a/newsfragments/3940.minor b/newsfragments/3940.minor new file mode 100644 index 000000000..e69de29bb From 65a7945fd9de23ad34c5f17bbf7cfe898243b9e2 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 7 Nov 2022 11:39:45 -0500 Subject: [PATCH 1147/2309] Don't need a connection timeout since we have request-level timeouts. --- src/allmydata/storage/http_client.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index c76cd00b9..adf4eb7fa 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -343,8 +343,6 @@ class StorageClient(object): Agent( reactor, _StorageClientHTTPSPolicy(expected_spki_hash=certificate_hash), - # TCP-level connection timeout - connectTimeout=5, pool=pool, ) ) @@ -385,6 +383,8 @@ class StorageClient(object): If ``message_to_serialize`` is set, it will be serialized (by default with CBOR) and set as the request body. + + Default timeout is 60 seconds. """ headers = self._get_headers(headers) @@ -506,9 +506,9 @@ def read_share_chunk( share_type, _encode_si(storage_index), share_number ) ) - # The default timeout is for getting the response, so it doesn't include - # the time it takes to download the body... so we will will deal with that - # later. + # The default 60 second timeout is for getting the response, so it doesn't + # include the time it takes to download the body... so we will will deal + # with that later, via limited_content(). response = yield client.request( "GET", url, From 8d678fe3de4dacdf206e737ef130a91b92004656 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 7 Nov 2022 11:41:50 -0500 Subject: [PATCH 1148/2309] Increase timeout. --- src/allmydata/test/common_system.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/common_system.py b/src/allmydata/test/common_system.py index f47aad3b6..90990a8ca 100644 --- a/src/allmydata/test/common_system.py +++ b/src/allmydata/test/common_system.py @@ -667,7 +667,7 @@ class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin): d = self.sparent.stopService() d.addBoth(flush_but_dont_ignore) d.addBoth(lambda x: self.close_idle_http_connections().addCallback(lambda _: x)) - d.addBoth(lambda x: deferLater(reactor, 0.01, lambda: x)) + d.addBoth(lambda x: deferLater(reactor, 0.02, lambda: x)) return d def getdir(self, subdir): From 90f1eb6245d176bc2a9f32098be1971fb0857f51 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 8 Nov 2022 09:24:29 -0500 Subject: [PATCH 1149/2309] Fix the fURL and NURL links --- docs/proposed/http-storage-node-protocol.rst | 4 ++-- docs/specifications/url.rst | 6 ++++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index 8fe855be3..6643c08f2 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -30,11 +30,11 @@ Glossary introducer a Tahoe-LAFS process at a known location configured to re-publish announcements about the location of storage servers - `fURL `_ + :ref:`fURLs ` a self-authenticating URL-like string which can be used to locate a remote object using the Foolscap protocol (the storage service is an example of such an object) - `NURL `_ + :ref:`NURLs ` a self-authenticating URL-like string almost exactly like a fURL but without being tied to Foolscap swissnum diff --git a/docs/specifications/url.rst b/docs/specifications/url.rst index 421ac57f7..efc7ad76c 100644 --- a/docs/specifications/url.rst +++ b/docs/specifications/url.rst @@ -7,11 +7,11 @@ These are not to be confused with the URI-like capabilities Tahoe-LAFS uses to r An attempt is also made to outline the rationale for certain choices about these URLs. The intended audience for this document is Tahoe-LAFS maintainers and other developers interested in interoperating with Tahoe-LAFS or these URLs. +.. _furls: + Background ---------- -.. _fURLs: - Tahoe-LAFS first used Foolscap_ for network communication. Foolscap connection setup takes as an input a Foolscap URL or a *fURL*. A fURL includes three components: @@ -33,6 +33,8 @@ The client's use of the swissnum is what allows the server to authorize the clie .. _`swiss number`: http://wiki.erights.org/wiki/Swiss_number +.. _NURLs: + NURLs ----- From 630ad1a60673ef794bb589925351d6e3d4560378 Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 9 Nov 2022 10:40:15 -0700 Subject: [PATCH 1150/2309] Update docs/managed-grid.rst Co-authored-by: Jean-Paul Calderone --- docs/managed-grid.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/managed-grid.rst b/docs/managed-grid.rst index 3630b956b..7b216c8bd 100644 --- a/docs/managed-grid.rst +++ b/docs/managed-grid.rst @@ -45,7 +45,7 @@ the statement: "Grid Manager X suggests you use storage-server Y to upload shares to" (X and Y are public-keys). Such a certificate consists of: - - a version + - the version of the format the certificate conforms to (`1`) - the public-key of a storage-server - an expiry timestamp - a signature of the above From 588f1fdb67d34cf78d3178a54b1abc862bd2306b Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 9 Nov 2022 10:40:35 -0700 Subject: [PATCH 1151/2309] Update docs/managed-grid.rst Co-authored-by: Jean-Paul Calderone --- docs/managed-grid.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/managed-grid.rst b/docs/managed-grid.rst index 7b216c8bd..17f7f370f 100644 --- a/docs/managed-grid.rst +++ b/docs/managed-grid.rst @@ -78,7 +78,7 @@ ever available on stdout. The configuration is a JSON document. It is subject to change as Grid Manager evolves. It contains a version number in the -`grid_manager_config_version` key which should increment whenever the +`grid_manager_config_version` key which will increment whenever the document schema changes. From 87c2f9b13e6465b0c7026b6d705d3df8903fdff3 Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 9 Nov 2022 10:44:04 -0700 Subject: [PATCH 1152/2309] Update docs/managed-grid.rst Co-authored-by: Jean-Paul Calderone --- docs/managed-grid.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/managed-grid.rst b/docs/managed-grid.rst index 17f7f370f..df12399d3 100644 --- a/docs/managed-grid.rst +++ b/docs/managed-grid.rst @@ -333,5 +333,5 @@ uploads should now fail (so ``tahoe put`` will fail) because they won't use storage2 and thus can't "achieve happiness". A proposal to expose more information about Grid Manager and -certifcate status in the Welcome page is discussed in +certificate status in the Welcome page is discussed in https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3506 From 6da9e50a02b2ebb85fe5bf5e3871c1f1b9dc908c Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 9 Nov 2022 10:45:03 -0700 Subject: [PATCH 1153/2309] Update newsfragments/2916.feature Co-authored-by: Jean-Paul Calderone --- newsfragments/2916.feature | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/newsfragments/2916.feature b/newsfragments/2916.feature index 292d429ea..c65f473a4 100644 --- a/newsfragments/2916.feature +++ b/newsfragments/2916.feature @@ -1 +1 @@ -A new 'Grid Manager' specification and implementation \ No newline at end of file +Tahoe-LAFS now includes a new "Grid Manager" specification and implementation adding more options to control which storage servers a client will use for uploads. \ No newline at end of file From b2431f3a89e28b76a4e98883c674e59b7ebc25fd Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 9 Nov 2022 10:46:31 -0700 Subject: [PATCH 1154/2309] Update src/allmydata/cli/grid_manager.py Co-authored-by: Jean-Paul Calderone --- src/allmydata/cli/grid_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/cli/grid_manager.py b/src/allmydata/cli/grid_manager.py index 4ef53887c..220f091cd 100644 --- a/src/allmydata/cli/grid_manager.py +++ b/src/allmydata/cli/grid_manager.py @@ -52,7 +52,7 @@ def grid_manager(ctx, config): class Config(object): """ - Availble to all sub-commands as Click's context.obj + Available to all sub-commands as Click's context.obj """ _grid_manager = None From 039c1d8037c0a5266383449fa5753ab21705a47e Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 9 Nov 2022 10:52:42 -0700 Subject: [PATCH 1155/2309] Update src/allmydata/grid_manager.py Co-authored-by: Jean-Paul Calderone --- src/allmydata/grid_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/grid_manager.py b/src/allmydata/grid_manager.py index f502c413e..ff5ce735d 100644 --- a/src/allmydata/grid_manager.py +++ b/src/allmydata/grid_manager.py @@ -461,7 +461,7 @@ def create_grid_manager_verifier(keys, certs, public_key, now_fn=None, bad_cert= def validate(): """ - :returns: True if if *any* certificate is still valid for a server + :returns: True if *any* certificate is still valid for a server """ now = now_fn() for cert in valid_certs: From d1287df62990d7c096e1935718c2f048d1a2039d Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 15 Nov 2022 14:02:19 -0500 Subject: [PATCH 1156/2309] The short timeout should be specific to the storage client's needs. --- src/allmydata/storage/http_client.py | 5 +---- src/allmydata/storage_client.py | 8 ++++++-- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index d6121aba2..d468d2436 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -415,10 +415,7 @@ class StorageClientGeneral(object): Return the version metadata for the server. """ url = self._client.relative_url("/storage/v1/version") - # 1. Getting the version should never take particularly long. - # 2. Clients rely on the version command for liveness checks of servers. - # Thus, a short timeout. - response = yield self._client.request("GET", url, timeout=5) + response = yield self._client.request("GET", url) decoded_response = yield _decode_cbor(response, _SCHEMAS["get_version"]) returnValue(decoded_response) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 6f2106f87..140e29607 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -944,12 +944,13 @@ class HTTPNativeStorageServer(service.MultiService): "connected". """ - def __init__(self, server_id: bytes, announcement): + def __init__(self, server_id: bytes, announcement, reactor=reactor): service.MultiService.__init__(self) assert isinstance(server_id, bytes) self._server_id = server_id self.announcement = announcement self._on_status_changed = ObserverList() + self._reactor = reactor furl = announcement["anonymous-storage-FURL"].encode("utf-8") ( self._nickname, @@ -1063,7 +1064,10 @@ class HTTPNativeStorageServer(service.MultiService): self._connect() def _connect(self): - return self._istorage_server.get_version().addCallbacks( + result = self._istorage_server.get_version() + # Set a short timeout since we're relying on this for server liveness. + result.addTimeout(5, self._reactor) + result.addCallbacks( self._got_version, self._failed_to_connect ) From 6c80ad5290c634a3395a3c5a222a15f6ed9f0abe Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 15 Nov 2022 14:13:50 -0500 Subject: [PATCH 1157/2309] Not necessary. --- src/allmydata/storage/http_client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index d468d2436..f0b45742c 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -301,8 +301,8 @@ class StorageClient(object): # ``StorageClient.from_nurl()``. _base_url: DecodedURL _swissnum: bytes - _treq: Union[treq, StubTreq, HTTPClient] = field(eq=False) - _clock: IReactorTime = field(eq=False) + _treq: Union[treq, StubTreq, HTTPClient] + _clock: IReactorTime @classmethod def from_nurl( From d700163aecda5ff23b772c561b5f9a1992b45f82 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 15 Nov 2022 14:14:29 -0500 Subject: [PATCH 1158/2309] Remove no-longer-relevant comment. --- src/allmydata/storage/http_client.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index f0b45742c..cc26d4b37 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -312,8 +312,6 @@ class StorageClient(object): ) -> StorageClient: """ Create a ``StorageClient`` for the given NURL. - - ``persistent`` indicates whether to use persistent HTTP connections. """ assert nurl.fragment == "v=1" assert nurl.scheme == "pb" From 4aeb62b66c12e5d337d6ebeeb26cea8f3f3ff13d Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 15 Nov 2022 14:16:41 -0500 Subject: [PATCH 1159/2309] Use a constant. --- src/allmydata/client.py | 2 +- src/allmydata/storage_client.py | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/allmydata/client.py b/src/allmydata/client.py index aa03015fc..1e28bb98b 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -826,7 +826,7 @@ class _Client(node.Node, pollmixin.PollMixin): if hasattr(self.tub.negotiationClass, "add_storage_server"): nurls = self.tub.negotiationClass.add_storage_server(ss, swissnum.encode("ascii")) self.storage_nurls = nurls - announcement["anonymous-storage-NURLs"] = [n.to_text() for n in nurls] + announcement[storage_client.ANONYMOUS_STORAGE_NURLS] = [n.to_text() for n in nurls] announcement["anonymous-storage-FURL"] = furl enabled_storage_servers = self._enable_storage_servers( diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 140e29607..59d3406f1 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -80,6 +80,8 @@ from allmydata.storage.http_client import ( ReadVector, TestWriteVectors, WriteVector, TestVector, ClientException ) +ANONYMOUS_STORAGE_NURLS = "anonymous-storage-NURLs" + # who is responsible for de-duplication? # both? @@ -267,8 +269,7 @@ class StorageFarmBroker(service.MultiService): by the given announcement. """ assert isinstance(server_id, bytes) - # TODO use constant for anonymous-storage-NURLs - if len(server["ann"].get("anonymous-storage-NURLs", [])) > 0: + if len(server["ann"].get(ANONYMOUS_STORAGE_NURLS, [])) > 0: s = HTTPNativeStorageServer(server_id, server["ann"]) s.on_status_changed(lambda _: self._got_connection()) return s @@ -961,7 +962,7 @@ class HTTPNativeStorageServer(service.MultiService): ) = _parse_announcement(server_id, furl, announcement) # TODO need some way to do equivalent of Happy Eyeballs for multiple NURLs? # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3935 - nurl = DecodedURL.from_text(announcement["anonymous-storage-NURLs"][0]) + nurl = DecodedURL.from_text(announcement[ANONYMOUS_STORAGE_NURLS][0]) self._istorage_server = _HTTPStorageServer.from_http_client( StorageClient.from_nurl(nurl, reactor) ) From 8e4ac6903298e8081daf4d1947c569d02111d160 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 15 Nov 2022 14:21:31 -0500 Subject: [PATCH 1160/2309] Stop test mode when done. --- src/allmydata/storage/http_client.py | 5 +++++ src/allmydata/test/common_system.py | 1 + src/allmydata/test/test_storage_http.py | 2 ++ 3 files changed, 8 insertions(+) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index cc26d4b37..fed66bb75 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -297,6 +297,11 @@ class StorageClient(object): """ cls.TEST_MODE_REGISTER_HTTP_POOL = callback + @classmethod + def stop_test_mode(cls): + """Stop testing mode.""" + cls.TEST_MODE_REGISTER_HTTP_POOL = None + # The URL is a HTTPS URL ("https://..."). To construct from a NURL, use # ``StorageClient.from_nurl()``. _base_url: DecodedURL diff --git a/src/allmydata/test/common_system.py b/src/allmydata/test/common_system.py index 90990a8ca..af86440cc 100644 --- a/src/allmydata/test/common_system.py +++ b/src/allmydata/test/common_system.py @@ -649,6 +649,7 @@ class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin): def setUp(self): self._http_client_pools = [] http_client.StorageClient.start_test_mode(self._http_client_pools.append) + self.addCleanup(http_client.StorageClient.stop_test_mode) self.port_assigner = SameProcessStreamEndpointAssigner() self.port_assigner.setUp() self.addCleanup(self.port_assigner.tearDown) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 25c21e03f..87a6a2306 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -294,6 +294,7 @@ class CustomHTTPServerTests(SyncTestCase): StorageClient.start_test_mode( lambda pool: self.addCleanup(pool.closeCachedConnections) ) + self.addCleanup(StorageClient.stop_test_mode) # Could be a fixture, but will only be used in this test class so not # going to bother: self._http_server = TestApp() @@ -382,6 +383,7 @@ class HttpTestFixture(Fixture): StorageClient.start_test_mode( lambda pool: self.addCleanup(pool.closeCachedConnections) ) + self.addCleanup(StorageClient.stop_test_mode) self.clock = Clock() self.tempdir = self.useFixture(TempDir()) # The global Cooperator used by Twisted (a) used by pull producers in From fb52b4d302d6f717a4393a518ddbe8fb773e406c Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 15 Nov 2022 14:22:30 -0500 Subject: [PATCH 1161/2309] Delete some garbage. --- src/allmydata/test/common_system.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/allmydata/test/common_system.py b/src/allmydata/test/common_system.py index af86440cc..0c7d7f747 100644 --- a/src/allmydata/test/common_system.py +++ b/src/allmydata/test/common_system.py @@ -810,8 +810,6 @@ class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin): if which in feature_matrix.get((section, feature), {which}): config.setdefault(section, {})[feature] = value - #config.setdefault("node", {})["force_foolscap"] = force_foolscap - setnode = partial(setconf, config, which, "node") sethelper = partial(setconf, config, which, "helper") From f3fc4268309316e9200f251df64b27a7bca5f33e Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 15 Nov 2022 14:36:14 -0500 Subject: [PATCH 1162/2309] Switch to [storage] force_foolscap. --- src/allmydata/client.py | 1 + src/allmydata/node.py | 3 +-- src/allmydata/test/common_system.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/allmydata/client.py b/src/allmydata/client.py index 1e28bb98b..1a158a1aa 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -104,6 +104,7 @@ _client_config = configutil.ValidConfiguration( "reserved_space", "storage_dir", "plugins", + "force_foolscap", ), "sftpd": ( "accounts.file", diff --git a/src/allmydata/node.py b/src/allmydata/node.py index f572cf7d9..8266fe3fb 100644 --- a/src/allmydata/node.py +++ b/src/allmydata/node.py @@ -64,7 +64,6 @@ def _common_valid_config(): "tcp", ), "node": ( - "force_foolscap", "log_gatherer.furl", "nickname", "reveal-ip-address", @@ -916,7 +915,7 @@ def create_main_tub(config, tub_options, # don't want to enable HTTP by default. # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3934 force_foolscap=config.get_config( - "node", "force_foolscap", default=True, boolean=True + "storage", "force_foolscap", default=True, boolean=True ), handler_overrides=handler_overrides, certFile=certfile, diff --git a/src/allmydata/test/common_system.py b/src/allmydata/test/common_system.py index 0c7d7f747..d49e7831d 100644 --- a/src/allmydata/test/common_system.py +++ b/src/allmydata/test/common_system.py @@ -814,7 +814,7 @@ class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin): sethelper = partial(setconf, config, which, "helper") setnode("nickname", u"client %d \N{BLACK SMILING FACE}" % (which,)) - setnode("force_foolscap", str(force_foolscap)) + setconf(config, which, "storage", "force_foolscap", str(force_foolscap)) tub_location_hint, tub_port_endpoint = self.port_assigner.assign(reactor) setnode("tub.port", tub_port_endpoint) From 2a5e8e59715ec647387f77b83733d9541886544b Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 15 Nov 2022 15:02:15 -0500 Subject: [PATCH 1163/2309] Better cleanup. --- src/allmydata/storage_client.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 59d3406f1..8e9ad3656 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -970,6 +970,7 @@ class HTTPNativeStorageServer(service.MultiService): self._connection_status = connection_status.ConnectionStatus.unstarted() self._version = None self._last_connect_time = None + self._connecting_deferred = None def get_permutation_seed(self): return self._permutation_seed @@ -1060,20 +1061,30 @@ class HTTPNativeStorageServer(service.MultiService): def stop_connecting(self): self._lc.stop() + if self._connecting_deferred is not None: + self._connecting_deferred.cancel() def try_to_connect(self): self._connect() def _connect(self): result = self._istorage_server.get_version() + + def remove_connecting_deferred(result): + self._connecting_deferred = None + return result + # Set a short timeout since we're relying on this for server liveness. - result.addTimeout(5, self._reactor) - result.addCallbacks( + self._connecting_deferred = result.addTimeout(5, self._reactor).addBoth( + remove_connecting_deferred).addCallbacks( self._got_version, self._failed_to_connect ) def stopService(self): + if self._connecting_deferred is not None: + self._connecting_deferred.cancel() + result = service.MultiService.stopService(self) if self._lc.running: self._lc.stop() From a20943e10c7d1f4b30b383138f489e9c9dd1eb85 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 16 Nov 2022 09:33:01 -0500 Subject: [PATCH 1164/2309] As an experiment, see if this fixes failing CI. --- src/allmydata/test/common_system.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/common_system.py b/src/allmydata/test/common_system.py index d49e7831d..8bc25aacf 100644 --- a/src/allmydata/test/common_system.py +++ b/src/allmydata/test/common_system.py @@ -649,7 +649,7 @@ class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin): def setUp(self): self._http_client_pools = [] http_client.StorageClient.start_test_mode(self._http_client_pools.append) - self.addCleanup(http_client.StorageClient.stop_test_mode) + #self.addCleanup(http_client.StorageClient.stop_test_mode) self.port_assigner = SameProcessStreamEndpointAssigner() self.port_assigner.setUp() self.addCleanup(self.port_assigner.tearDown) From 9f5f287473d734932f348d77b89fb81838e5c3d1 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 16 Nov 2022 09:57:39 -0500 Subject: [PATCH 1165/2309] Nope, not helpful. --- src/allmydata/test/common_system.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/common_system.py b/src/allmydata/test/common_system.py index 8bc25aacf..d49e7831d 100644 --- a/src/allmydata/test/common_system.py +++ b/src/allmydata/test/common_system.py @@ -649,7 +649,7 @@ class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin): def setUp(self): self._http_client_pools = [] http_client.StorageClient.start_test_mode(self._http_client_pools.append) - #self.addCleanup(http_client.StorageClient.stop_test_mode) + self.addCleanup(http_client.StorageClient.stop_test_mode) self.port_assigner = SameProcessStreamEndpointAssigner() self.port_assigner.setUp() self.addCleanup(self.port_assigner.tearDown) From 2ab172ffca9c6faac1751709ce5db5d17e4e28db Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 16 Nov 2022 10:26:29 -0500 Subject: [PATCH 1166/2309] Try to set more aggressive timeouts when testing. --- src/allmydata/test/common_system.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/common_system.py b/src/allmydata/test/common_system.py index d49e7831d..e75021248 100644 --- a/src/allmydata/test/common_system.py +++ b/src/allmydata/test/common_system.py @@ -648,7 +648,7 @@ class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin): def setUp(self): self._http_client_pools = [] - http_client.StorageClient.start_test_mode(self._http_client_pools.append) + http_client.StorageClient.start_test_mode(self._got_new_http_connection_pool) self.addCleanup(http_client.StorageClient.stop_test_mode) self.port_assigner = SameProcessStreamEndpointAssigner() self.port_assigner.setUp() @@ -657,6 +657,23 @@ class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin): self.sparent = service.MultiService() self.sparent.startService() + def _got_new_http_connection_pool(self, pool): + # Register the pool for shutdown later: + self._http_client_pools.append(pool) + # Disable retries: + pool.retryAutomatically = False + # Make a much more aggressive timeout for connections, we're connecting + # locally after all... and also make sure it's lower than the delay we + # add in tearDown, to prevent dirty reactor issues. + getConnection = pool.getConnection + + def getConnectionWithTimeout(*args, **kwargs): + d = getConnection(*args, **kwargs) + d.addTimeout(0.05, reactor) + return d + + pool.getConnection = getConnectionWithTimeout + def close_idle_http_connections(self): """Close all HTTP client connections that are just hanging around.""" return defer.gatherResults( @@ -668,7 +685,7 @@ class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin): d = self.sparent.stopService() d.addBoth(flush_but_dont_ignore) d.addBoth(lambda x: self.close_idle_http_connections().addCallback(lambda _: x)) - d.addBoth(lambda x: deferLater(reactor, 0.02, lambda: x)) + d.addBoth(lambda x: deferLater(reactor, 0.1, lambda: x)) return d def getdir(self, subdir): From 35317373474c5170d9a15b5b9cd895ceb7222391 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 16 Nov 2022 10:36:11 -0500 Subject: [PATCH 1167/2309] Make timeouts less aggressive, CI machines are slow? --- src/allmydata/test/common_system.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/common_system.py b/src/allmydata/test/common_system.py index e75021248..8d3019935 100644 --- a/src/allmydata/test/common_system.py +++ b/src/allmydata/test/common_system.py @@ -669,7 +669,7 @@ class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin): def getConnectionWithTimeout(*args, **kwargs): d = getConnection(*args, **kwargs) - d.addTimeout(0.05, reactor) + d.addTimeout(1, reactor) return d pool.getConnection = getConnectionWithTimeout @@ -685,7 +685,7 @@ class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin): d = self.sparent.stopService() d.addBoth(flush_but_dont_ignore) d.addBoth(lambda x: self.close_idle_http_connections().addCallback(lambda _: x)) - d.addBoth(lambda x: deferLater(reactor, 0.1, lambda: x)) + d.addBoth(lambda x: deferLater(reactor, 2, lambda: x)) return d def getdir(self, subdir): From add510701c0809cf89494434c1dccdfc3271df47 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 16 Nov 2022 11:44:51 -0500 Subject: [PATCH 1168/2309] Run integration tests both with and without HTTP storage protocol. --- .github/workflows/ci.yml | 6 +++++- integration/util.py | 17 +++++++++-------- newsfragments/3937.minor | 0 src/allmydata/protocol_switch.py | 16 ++++++++++++++++ src/allmydata/testing/__init__.py | 18 ++++++++++++++++++ 5 files changed, 48 insertions(+), 9 deletions(-) create mode 100644 newsfragments/3937.minor diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0327014ca..26574066c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -229,7 +229,11 @@ jobs: # aren't too long. On Windows tox won't pass it through so it has no # effect. On Linux it doesn't make a difference one way or another. TMPDIR: "/tmp" - run: tox -e integration + run: | + # Run with Foolscap forced: + __TAHOE_INTEGRATION_FORCE_FOOLSCAP=1 tox -e integration + # Run with Foolscap not forced, which should result in HTTP being used. + __TAHOE_INTEGRATION_FORCE_FOOLSCAP=0 tox -e integration - name: Upload eliot.log in case of failure uses: actions/upload-artifact@v1 diff --git a/integration/util.py b/integration/util.py index ad9249e45..cde837218 100644 --- a/integration/util.py +++ b/integration/util.py @@ -1,14 +1,6 @@ """ Ported to Python 3. """ -from __future__ import unicode_literals -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import sys import time @@ -38,6 +30,7 @@ from allmydata.util.configutil import ( write_config, ) from allmydata import client +from allmydata.testing import foolscap_only_for_integration_testing import pytest_twisted @@ -300,6 +293,14 @@ def _create_node(reactor, request, temp_dir, introducer_furl, flog_gatherer, nam u'log_gatherer.furl', flog_gatherer, ) + force_foolscap = foolscap_only_for_integration_testing() + if force_foolscap is not None: + set_config( + config, + 'storage', + 'force_foolscap', + str(force_foolscap), + ) write_config(FilePath(config_path), config) created_d.addCallback(created) diff --git a/newsfragments/3937.minor b/newsfragments/3937.minor new file mode 100644 index 000000000..e69de29bb diff --git a/src/allmydata/protocol_switch.py b/src/allmydata/protocol_switch.py index b0af84c33..d88863fdb 100644 --- a/src/allmydata/protocol_switch.py +++ b/src/allmydata/protocol_switch.py @@ -30,6 +30,7 @@ from foolscap.api import Tub from .storage.http_server import HTTPServer, build_nurl from .storage.server import StorageServer +from .testing import foolscap_only_for_integration_testing class _PretendToBeNegotiation(type): @@ -170,6 +171,21 @@ class _FoolscapOrHttps(Protocol, metaclass=_PretendToBeNegotiation): # and later data, otherwise assume HTTPS. self._timeout.cancel() if self._buffer.startswith(b"GET /id/"): + if foolscap_only_for_integration_testing() == False: + # Tahoe will prefer HTTP storage protocol over Foolscap when possible. + # + # If this is branch is taken, we are running a test that should + # be using HTTP for the storage protocol. As such, we + # aggressively disable Foolscap to ensure that HTTP is in fact + # going to be used. If we hit this branch that means our + # expectation that HTTP will be used was wrong, suggesting a + # bug in either the code of the integration testing setup. + # + # This branch should never be hit in production! + self.transport.loseConnection() + print("FOOLSCAP IS DISABLED, I PITY THE FOOLS WHO SEE THIS MESSAGE") + return + # We're a Foolscap Negotiation server protocol instance: transport = self.transport buf = self._buffer diff --git a/src/allmydata/testing/__init__.py b/src/allmydata/testing/__init__.py index e69de29bb..119ae4101 100644 --- a/src/allmydata/testing/__init__.py +++ b/src/allmydata/testing/__init__.py @@ -0,0 +1,18 @@ +import os +from typing import Optional + + +def foolscap_only_for_integration_testing() -> Optional[bool]: + """ + Return whether HTTP storage protocol has been disabled / Foolscap + forced, for purposes of integration testing. + + This is determined by the __TAHOE_INTEGRATION_FORCE_FOOLSCAP environment + variable, which can be 1, 0, or not set, corresponding to results of + ``True``, ``False`` and ``None`` (i.e. default). + """ + force_foolscap = os.environ.get("__TAHOE_INTEGRATION_FORCE_FOOLSCAP") + if force_foolscap is None: + return None + + return bool(int(force_foolscap)) From 7afd821efc826f2ee644ed85369b0bc6b8dbb482 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 16 Nov 2022 13:28:26 -0500 Subject: [PATCH 1169/2309] Sigh --- src/allmydata/test/test_storage_https.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/allmydata/test/test_storage_https.py b/src/allmydata/test/test_storage_https.py index bacb40290..a9421c3e5 100644 --- a/src/allmydata/test/test_storage_https.py +++ b/src/allmydata/test/test_storage_https.py @@ -179,6 +179,10 @@ class PinningHTTPSValidation(AsyncTestCase): response = await self.request(url, certificate) self.assertEqual(await response.content(), b"YOYODYNE") + # We keep getting TLSMemoryBIOProtocol being left around, so try harder + # to wait for it to finish. + await deferLater(reactor, 0.01) + @async_to_deferred async def test_server_certificate_not_valid_yet(self): """ From 097d918a240ba291ebd6b00108f071362eefcbd3 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 16 Nov 2022 13:37:50 -0500 Subject: [PATCH 1170/2309] Sigh --- src/allmydata/test/test_storage_https.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/allmydata/test/test_storage_https.py b/src/allmydata/test/test_storage_https.py index bacb40290..a9421c3e5 100644 --- a/src/allmydata/test/test_storage_https.py +++ b/src/allmydata/test/test_storage_https.py @@ -179,6 +179,10 @@ class PinningHTTPSValidation(AsyncTestCase): response = await self.request(url, certificate) self.assertEqual(await response.content(), b"YOYODYNE") + # We keep getting TLSMemoryBIOProtocol being left around, so try harder + # to wait for it to finish. + await deferLater(reactor, 0.01) + @async_to_deferred async def test_server_certificate_not_valid_yet(self): """ From d182a2f1865002cc9a3167c1f585413ac6db4307 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 17 Nov 2022 11:01:12 -0500 Subject: [PATCH 1171/2309] Add the delay to appropriate test. --- src/allmydata/test/test_storage_https.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/allmydata/test/test_storage_https.py b/src/allmydata/test/test_storage_https.py index a9421c3e5..01431267f 100644 --- a/src/allmydata/test/test_storage_https.py +++ b/src/allmydata/test/test_storage_https.py @@ -144,6 +144,10 @@ class PinningHTTPSValidation(AsyncTestCase): response = await self.request(url, certificate) self.assertEqual(await response.content(), b"YOYODYNE") + # We keep getting TLSMemoryBIOProtocol being left around, so try harder + # to wait for it to finish. + await deferLater(reactor, 0.01) + @async_to_deferred async def test_server_certificate_has_wrong_hash(self): """ @@ -179,10 +183,6 @@ class PinningHTTPSValidation(AsyncTestCase): response = await self.request(url, certificate) self.assertEqual(await response.content(), b"YOYODYNE") - # We keep getting TLSMemoryBIOProtocol being left around, so try harder - # to wait for it to finish. - await deferLater(reactor, 0.01) - @async_to_deferred async def test_server_certificate_not_valid_yet(self): """ From 9b21f1da90a1b80414959822fec689040db75d40 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 17 Nov 2022 11:35:10 -0500 Subject: [PATCH 1172/2309] Increase how many statuses are stored. --- src/allmydata/history.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/history.py b/src/allmydata/history.py index b5cfb7318..06a22ab5d 100644 --- a/src/allmydata/history.py +++ b/src/allmydata/history.py @@ -20,7 +20,7 @@ class History(object): MAX_UPLOAD_STATUSES = 10 MAX_MAPUPDATE_STATUSES = 20 MAX_PUBLISH_STATUSES = 20 - MAX_RETRIEVE_STATUSES = 20 + MAX_RETRIEVE_STATUSES = 40 def __init__(self, stats_provider=None): self.stats_provider = stats_provider From 4c8e8a74a4920359617bb4471f97eb3817eed37a Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 17 Nov 2022 12:25:37 -0500 Subject: [PATCH 1173/2309] Not needed. --- src/allmydata/test/test_storage_https.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/allmydata/test/test_storage_https.py b/src/allmydata/test/test_storage_https.py index a9421c3e5..88435bf89 100644 --- a/src/allmydata/test/test_storage_https.py +++ b/src/allmydata/test/test_storage_https.py @@ -202,10 +202,6 @@ class PinningHTTPSValidation(AsyncTestCase): response = await self.request(url, certificate) self.assertEqual(await response.content(), b"YOYODYNE") - # We keep getting TLSMemoryBIOProtocol being left around, so try harder - # to wait for it to finish. - await deferLater(reactor, 0.001) - # A potential attack to test is a private key that doesn't match the # certificate... but OpenSSL (quite rightly) won't let you listen with that # so I don't know how to test that! See From 4c0c75a034568c621ca327b00e881075743254c5 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 18 Nov 2022 13:56:54 -0500 Subject: [PATCH 1174/2309] Fix DelayedCall leak in tests. --- src/allmydata/storage/http_client.py | 38 ++++++++++++++++--------- src/allmydata/test/test_storage_http.py | 8 ++++-- 2 files changed, 30 insertions(+), 16 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 4ed37f901..5b4ec9db8 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -26,7 +26,6 @@ from twisted.internet.interfaces import ( IDelayedCall, ) from twisted.internet.ssl import CertificateOptions -from twisted.internet import reactor from twisted.web.client import Agent, HTTPConnectionPool from zope.interface import implementer from hyperlink import DecodedURL @@ -141,7 +140,9 @@ class _LengthLimitedCollector: def limited_content( - response, max_length: int = 30 * 1024 * 1024, clock: IReactorTime = reactor + response, + clock: IReactorTime, + max_length: int = 30 * 1024 * 1024, ) -> Deferred[BinaryIO]: """ Like ``treq.content()``, but limit data read from the response to a set @@ -168,11 +169,10 @@ def limited_content( collector.f.seek(0) return collector.f - d.addCallback(done) - return d + return d.addCallback(done) -def _decode_cbor(response, schema: Schema): +def _decode_cbor(response, schema: Schema, clock: IReactorTime): """Given HTTP response, return decoded CBOR body.""" def got_content(f: BinaryIO): @@ -183,7 +183,7 @@ def _decode_cbor(response, schema: Schema): if response.code > 199 and response.code < 300: content_type = get_content_type(response.headers) if content_type == CBOR_MIME_TYPE: - return limited_content(response).addCallback(got_content) + return limited_content(response, clock).addCallback(got_content) else: raise ClientException(-1, "Server didn't send CBOR") else: @@ -439,7 +439,9 @@ class StorageClientGeneral(object): """ url = self._client.relative_url("/storage/v1/version") response = yield self._client.request("GET", url) - decoded_response = yield _decode_cbor(response, _SCHEMAS["get_version"]) + decoded_response = yield _decode_cbor( + response, _SCHEMAS["get_version"], self._client._clock + ) returnValue(decoded_response) @inlineCallbacks @@ -540,7 +542,7 @@ def read_share_chunk( raise ValueError("Server sent more than we asked for?!") # It might also send less than we asked for. That's (probably) OK, e.g. # if we went past the end of the file. - body = yield limited_content(response, supposed_length, client._clock) + body = yield limited_content(response, client._clock, supposed_length) body.seek(0, SEEK_END) actual_length = body.tell() if actual_length != supposed_length: @@ -627,7 +629,9 @@ class StorageClientImmutables(object): upload_secret=upload_secret, message_to_serialize=message, ) - decoded_response = yield _decode_cbor(response, _SCHEMAS["allocate_buckets"]) + decoded_response = yield _decode_cbor( + response, _SCHEMAS["allocate_buckets"], self._client._clock + ) returnValue( ImmutableCreateResult( already_have=decoded_response["already-have"], @@ -703,7 +707,9 @@ class StorageClientImmutables(object): raise ClientException( response.code, ) - body = yield _decode_cbor(response, _SCHEMAS["immutable_write_share_chunk"]) + body = yield _decode_cbor( + response, _SCHEMAS["immutable_write_share_chunk"], self._client._clock + ) remaining = RangeMap() for chunk in body["required"]: remaining.set(True, chunk["begin"], chunk["end"]) @@ -732,7 +738,9 @@ class StorageClientImmutables(object): url, ) if response.code == http.OK: - body = yield _decode_cbor(response, _SCHEMAS["list_shares"]) + body = yield _decode_cbor( + response, _SCHEMAS["list_shares"], self._client._clock + ) returnValue(set(body)) else: raise ClientException(response.code) @@ -849,7 +857,9 @@ class StorageClientMutables: message_to_serialize=message, ) if response.code == http.OK: - result = await _decode_cbor(response, _SCHEMAS["mutable_read_test_write"]) + result = await _decode_cbor( + response, _SCHEMAS["mutable_read_test_write"], self._client._clock + ) return ReadTestWriteResult(success=result["success"], reads=result["data"]) else: raise ClientException(response.code, (await response.content())) @@ -878,7 +888,9 @@ class StorageClientMutables: ) response = await self._client.request("GET", url) if response.code == http.OK: - return await _decode_cbor(response, _SCHEMAS["mutable_list_shares"]) + return await _decode_cbor( + response, _SCHEMAS["mutable_list_shares"], self._client._clock + ) else: raise ClientException(response.code) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index fa3532839..4f7174c06 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -371,7 +371,9 @@ class CustomHTTPServerTests(SyncTestCase): ) self.assertEqual( - result_of(limited_content(response, at_least_length)).read(), + result_of( + limited_content(response, self._http_server.clock, at_least_length) + ).read(), gen_bytes(length), ) @@ -390,7 +392,7 @@ class CustomHTTPServerTests(SyncTestCase): ) with self.assertRaises(ValueError): - result_of(limited_content(response, too_short)) + result_of(limited_content(response, self._http_server.clock, too_short)) def test_limited_content_silence_causes_timeout(self): """ @@ -404,7 +406,7 @@ class CustomHTTPServerTests(SyncTestCase): ) ) - body_deferred = limited_content(response, 4, self._http_server.clock) + body_deferred = limited_content(response, self._http_server.clock, 4) result = [] error = [] body_deferred.addCallbacks(result.append, error.append) From 8cfdae2ab4005943689a0713ba5bd8f3b0831d9b Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 18 Nov 2022 15:26:02 -0500 Subject: [PATCH 1175/2309] sigh --- src/allmydata/test/test_storage_https.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/allmydata/test/test_storage_https.py b/src/allmydata/test/test_storage_https.py index 01431267f..062eb5b0e 100644 --- a/src/allmydata/test/test_storage_https.py +++ b/src/allmydata/test/test_storage_https.py @@ -183,6 +183,10 @@ class PinningHTTPSValidation(AsyncTestCase): response = await self.request(url, certificate) self.assertEqual(await response.content(), b"YOYODYNE") + # We keep getting TLSMemoryBIOProtocol being left around, so try harder + # to wait for it to finish. + await deferLater(reactor, 0.01) + @async_to_deferred async def test_server_certificate_not_valid_yet(self): """ From 1d85a2c5cf40a4e52bbb024e2edba4fcd883caa1 Mon Sep 17 00:00:00 2001 From: "Fon E. Noel NFEBE" Date: Sun, 20 Nov 2022 14:02:49 +0100 Subject: [PATCH 1176/2309] Refactor more test_storage.py classes As a follow up to commit fbc8baa238f72720cfa840a9c227c670a5e2fa6e this refactor continues to remove deprecated methods and ensures test classes either extend `SyncTestCase` or `AsyncTestCase` Classes refactored: - `MutableServer` - `MDMFProxies` - `Stats` - `MutableShareFileTests` Signed-off-by: Fon E. Noel NFEBE --- src/allmydata/test/test_storage.py | 654 ++++++++++++++--------------- 1 file changed, 327 insertions(+), 327 deletions(-) diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index 920f2d935..3d35ec55f 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -470,7 +470,7 @@ class BucketProxy(SyncTestCase): num_segments=5, num_share_hashes=3, uri_extension_size=500) - self.failUnless(interfaces.IStorageBucketWriter.providedBy(bp), bp) + self.assertTrue(interfaces.IStorageBucketWriter.providedBy(bp), bp) def _do_test_readwrite(self, name, header_size, wbp_class, rbp_class): # Let's pretend each share has 100 bytes of data, and that there are @@ -520,7 +520,7 @@ class BucketProxy(SyncTestCase): server = NoNetworkServer(b"abc", None) rbp = rbp_class(rb, server, storage_index=b"") self.assertThat(repr(rbp), Contains("to peer")) - self.failUnless(interfaces.IStorageBucketReader.providedBy(rbp), rbp) + self.assertTrue(interfaces.IStorageBucketReader.providedBy(rbp), rbp) d1 = rbp.get_block_data(0, 25, 25) d1.addCallback(lambda res: self.failUnlessEqual(res, b"a"*25)) @@ -1281,7 +1281,7 @@ class Server(AsyncTestCase): ) -class MutableServer(unittest.TestCase): +class MutableServer(SyncTestCase): def setUp(self): self.sparent = LoggingServiceParent() @@ -1311,13 +1311,13 @@ class MutableServer(unittest.TestCase): def renew_secret(self, tag): if isinstance(tag, int): tag = b"%d" % (tag,) - assert isinstance(tag, bytes) + self.assertThat(tag, IsInstance(bytes)) return hashutil.tagged_hash(b"renew_blah", tag) def cancel_secret(self, tag): if isinstance(tag, int): tag = b"%d" % (tag,) - assert isinstance(tag, bytes) + self.assertThat(tag, IsInstance(bytes)) return hashutil.tagged_hash(b"cancel_blah", tag) def allocate(self, ss, storage_index, we_tag, lease_tag, sharenums, size): @@ -1333,9 +1333,9 @@ class MutableServer(unittest.TestCase): testandwritev, readv) (did_write, readv_data) = rc - self.failUnless(did_write) - self.failUnless(isinstance(readv_data, dict)) - self.failUnlessEqual(len(readv_data), 0) + self.assertTrue(did_write) + self.assertThat(readv_data, IsInstance(dict)) + self.assertThat(readv_data, HasLength(0)) def test_enumerate_mutable_shares(self): """ @@ -1357,9 +1357,9 @@ class MutableServer(unittest.TestCase): self.cancel_secret(b"le1")) ss.slot_testv_and_readv_and_writev(b"si1", secrets, {2: ([], [], 0)}, []) shares0_1_4 = ss.enumerate_mutable_shares(b"si1") - self.assertEqual( + self.assertThat( (empty, shares0_1_2_4, shares0_1_4), - (set(), {0, 1, 2, 4}, {0, 1, 4}) + Equals((set(), {0, 1, 2, 4}, {0, 1, 4})) ) def test_mutable_share_length(self): @@ -1373,7 +1373,7 @@ class MutableServer(unittest.TestCase): {16: ([], [(0, b"x" * 23)], None)}, [] ) - self.assertEqual(ss.get_mutable_share_length(b"si1", 16), 23) + self.assertThat(ss.get_mutable_share_length(b"si1", 16), Equals(23)) def test_mutable_share_length_unknown(self): """ @@ -1406,10 +1406,10 @@ class MutableServer(unittest.TestCase): read = ss.slot_readv e = self.failUnlessRaises(UnknownMutableContainerVersionError, read, b"si1", [0], [(0,10)]) - self.assertEqual(e.filename, fn) + self.assertThat(e.filename, Equals(fn)) self.assertTrue(e.version.startswith(b"BAD MAGIC")) - self.assertIn("had unexpected version", str(e)) - self.assertIn("BAD MAGIC", str(e)) + self.assertThat(str(e), Contains("had unexpected version")) + self.assertThat(str(e), Contains("BAD MAGIC")) def test_container_size(self): ss = self.create("test_container_size") @@ -1424,7 +1424,7 @@ class MutableServer(unittest.TestCase): answer = rstaraw(b"si1", secrets, {0: ([], [(0,data)], len(data)+12)}, []) - self.failUnlessEqual(answer, (True, {0:[],1:[],2:[]}) ) + self.assertThat(answer, Equals((True, {0:[],1:[],2:[]}))) # Trying to make the container too large (by sending a write vector # whose offset is too high) will raise an exception. @@ -1437,10 +1437,10 @@ class MutableServer(unittest.TestCase): answer = rstaraw(b"si1", secrets, {0: ([], [(0,data)], None)}, []) - self.failUnlessEqual(answer, (True, {0:[],1:[],2:[]}) ) + self.assertThat(answer, Equals((True, {0:[],1:[],2:[]}))) read_answer = read(b"si1", [0], [(0,10)]) - self.failUnlessEqual(read_answer, {0: [data[:10]]}) + self.assertThat(read_answer, Equals({0: [data[:10]]})) # Sending a new_length shorter than the current length truncates the # data. @@ -1448,7 +1448,7 @@ class MutableServer(unittest.TestCase): {0: ([], [], 9)}, []) read_answer = read(b"si1", [0], [(0,10)]) - self.failUnlessEqual(read_answer, {0: [data[:9]]}) + self.assertThat(read_answer, Equals({0: [data[:9]]})) # Sending a new_length longer than the current length doesn't change # the data. @@ -1457,7 +1457,7 @@ class MutableServer(unittest.TestCase): []) assert answer == (True, {0:[],1:[],2:[]}) read_answer = read(b"si1", [0], [(0, 20)]) - self.failUnlessEqual(read_answer, {0: [data[:9]]}) + self.assertThat(read_answer, Equals({0: [data[:9]]})) # Sending a write vector whose start is after the end of the current # data doesn't reveal "whatever was there last time" (palimpsest), @@ -1479,7 +1479,7 @@ class MutableServer(unittest.TestCase): answer = rstaraw(b"si1", secrets, {0: ([], [], None)}, [(20, 1980)]) - self.failUnlessEqual(answer, (True, {0:[b''],1:[b''],2:[b'']})) + self.assertThat(answer, Equals((True, {0:[b''],1:[b''],2:[b'']}))) # Then the extend the file by writing a vector which starts out past # the end... @@ -1492,22 +1492,22 @@ class MutableServer(unittest.TestCase): answer = rstaraw(b"si1", secrets, {0: ([], [], None)}, [(20, 30)]) - self.failUnlessEqual(answer, (True, {0:[b'\x00'*30],1:[b''],2:[b'']})) + self.assertThat(answer, Equals((True, {0:[b'\x00'*30],1:[b''],2:[b'']}))) # Also see if the server explicitly declares that it supports this # feature. ver = ss.get_version() storage_v1_ver = ver[b"http://allmydata.org/tahoe/protocols/storage/v1"] - self.failUnless(storage_v1_ver.get(b"fills-holes-with-zero-bytes")) + self.assertTrue(storage_v1_ver.get(b"fills-holes-with-zero-bytes")) # If the size is dropped to zero the share is deleted. answer = rstaraw(b"si1", secrets, {0: ([], [(0,data)], 0)}, []) - self.failUnlessEqual(answer, (True, {0:[],1:[],2:[]}) ) + self.assertThat(answer, Equals((True, {0:[],1:[],2:[]}))) read_answer = read(b"si1", [0], [(0,10)]) - self.failUnlessEqual(read_answer, {}) + self.assertThat(read_answer, Equals({})) def test_allocate(self): ss = self.create("test_allocate") @@ -1515,12 +1515,12 @@ class MutableServer(unittest.TestCase): set([0,1,2]), 100) read = ss.slot_readv - self.failUnlessEqual(read(b"si1", [0], [(0, 10)]), - {0: [b""]}) - self.failUnlessEqual(read(b"si1", [], [(0, 10)]), - {0: [b""], 1: [b""], 2: [b""]}) - self.failUnlessEqual(read(b"si1", [0], [(100, 10)]), - {0: [b""]}) + self.assertThat(read(b"si1", [0], [(0, 10)]), + Equals({0: [b""]})) + self.assertThat(read(b"si1", [], [(0, 10)]), + Equals({0: [b""], 1: [b""], 2: [b""]})) + self.assertThat(read(b"si1", [0], [(100, 10)]), + Equals({0: [b""]})) # try writing to one secrets = ( self.write_enabler(b"we1"), @@ -1531,19 +1531,19 @@ class MutableServer(unittest.TestCase): answer = write(b"si1", secrets, {0: ([], [(0,data)], None)}, []) - self.failUnlessEqual(answer, (True, {0:[],1:[],2:[]}) ) + self.assertThat(answer, Equals((True, {0:[],1:[],2:[]}))) - self.failUnlessEqual(read(b"si1", [0], [(0,20)]), - {0: [b"00000000001111111111"]}) - self.failUnlessEqual(read(b"si1", [0], [(95,10)]), - {0: [b"99999"]}) + self.assertThat(read(b"si1", [0], [(0,20)]), + Equals({0: [b"00000000001111111111"]})) + self.assertThat(read(b"si1", [0], [(95,10)]), + Equals({0: [b"99999"]})) #self.failUnlessEqual(s0.get_length(), 100) bad_secrets = (b"bad write enabler", secrets[1], secrets[2]) f = self.failUnlessRaises(BadWriteEnablerError, write, b"si1", bad_secrets, {}, []) - self.failUnlessIn("The write enabler was recorded by nodeid 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'.", str(f)) + self.assertThat(str(f), Contains("The write enabler was recorded by nodeid 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'.")) # this testv should fail answer = write(b"si1", secrets, @@ -1555,12 +1555,12 @@ class MutableServer(unittest.TestCase): }, [(0,12), (20,5)], ) - self.failUnlessEqual(answer, (False, - {0: [b"000000000011", b"22222"], + self.assertThat(answer, (False, + Equals({0: [b"000000000011", b"22222"], 1: [b"", b""], 2: [b"", b""], - })) - self.failUnlessEqual(read(b"si1", [0], [(0,100)]), {0: [data]}) + }))) + self.assertThat(read(b"si1", [0], [(0,100)]), Equals({0: [data]})) def test_operators(self): # test operators, the data we're comparing is '11111' in all cases. @@ -1587,8 +1587,8 @@ class MutableServer(unittest.TestCase): [(0, b"x"*100)], None, )}, [(10,5)]) - self.failUnlessEqual(answer, (False, {0: [b"11111"]})) - self.failUnlessEqual(read(b"si1", [0], [(0,100)]), {0: [data]}) + self.assertThat(answer, Equals((False, {0: [b"11111"]}))) + self.assertThat(read(b"si1", [0], [(0,100)]), Equals({0: [data]})) reset() answer = write(b"si1", secrets, {0: ([(10, 5, b"eq", b"11111"), @@ -1596,8 +1596,8 @@ class MutableServer(unittest.TestCase): [(0, b"y"*100)], None, )}, [(10,5)]) - self.failUnlessEqual(answer, (True, {0: [b"11111"]})) - self.failUnlessEqual(read(b"si1", [0], [(0,100)]), {0: [b"y"*100]}) + self.assertThat(answer, Equals((True, {0: [b"11111"]}))) + self.assertThat(read(b"si1", [0], [(0,100)]), Equals({0: [b"y"*100]})) reset() # finally, test some operators against empty shares @@ -1606,8 +1606,8 @@ class MutableServer(unittest.TestCase): [(0, b"x"*100)], None, )}, [(10,5)]) - self.failUnlessEqual(answer, (False, {0: [b"11111"]})) - self.failUnlessEqual(read(b"si1", [0], [(0,100)]), {0: [data]}) + self.assertThat(answer, Equals((False, {0: [b"11111"]}))) + self.assertThat(read(b"si1", [0], [(0,100)]), Equals({0: [data]})) reset() def test_readv(self): @@ -1624,12 +1624,12 @@ class MutableServer(unittest.TestCase): 1: ([], [(0,data[1])], None), 2: ([], [(0,data[2])], None), }, []) - self.failUnlessEqual(rc, (True, {})) + self.assertThat(rc, Equals((True, {}))) answer = read(b"si1", [], [(0, 10)]) - self.failUnlessEqual(answer, {0: [b"0"*10], + self.assertThat(answer, Equals({0: [b"0"*10], 1: [b"1"*10], - 2: [b"2"*10]}) + 2: [b"2"*10]})) def compare_leases_without_timestamps(self, leases_a, leases_b): """ @@ -1646,11 +1646,11 @@ class MutableServer(unittest.TestCase): # non-equal inputs (expiration timestamp aside). It seems # reasonably safe to use `renew` to make _one_ of the timestamps # equal to the other though. - self.assertEqual( + self.assertThat( a.renew(b.get_expiration_time()), - b, + Equals(b), ) - self.assertEqual(len(leases_a), len(leases_b)) + self.assertThat(len(leases_a), Equals(len(leases_b))) def test_leases(self): ss = self.create("test_leases") @@ -1662,7 +1662,7 @@ class MutableServer(unittest.TestCase): write = ss.slot_testv_and_readv_and_writev read = ss.slot_readv rc = write(b"si1", secrets(0), {0: ([], [(0,data)], None)}, []) - self.failUnlessEqual(rc, (True, {})) + self.assertThat(rc, Equals((True, {}))) # create a random non-numeric file in the bucket directory, to # exercise the code that's supposed to ignore those. @@ -1673,32 +1673,32 @@ class MutableServer(unittest.TestCase): f.close() s0 = MutableShareFile(os.path.join(bucket_dir, "0")) - self.failUnlessEqual(len(list(s0.get_leases())), 1) + self.assertThat(list(s0.get_leases()), HasLength(1)) # add-lease on a missing storage index is silently ignored - self.failUnlessEqual(ss.add_lease(b"si18", b"", b""), None) + self.assertThat(ss.add_lease(b"si18", b"", b""), Equals(None)) # re-allocate the slots and use the same secrets, that should update # the lease write(b"si1", secrets(0), {0: ([], [(0,data)], None)}, []) - self.failUnlessEqual(len(list(s0.get_leases())), 1) + self.assertThat(list(s0.get_leases()), HasLength(1)) # renew it directly ss.renew_lease(b"si1", secrets(0)[1]) - self.failUnlessEqual(len(list(s0.get_leases())), 1) + self.assertThat(list(s0.get_leases()), HasLength(1)) # now allocate them with a bunch of different secrets, to trigger the # extended lease code. Use add_lease for one of them. write(b"si1", secrets(1), {0: ([], [(0,data)], None)}, []) - self.failUnlessEqual(len(list(s0.get_leases())), 2) + self.assertThat(list(s0.get_leases()), HasLength(2)) secrets2 = secrets(2) ss.add_lease(b"si1", secrets2[1], secrets2[2]) - self.failUnlessEqual(len(list(s0.get_leases())), 3) + self.assertThat(list(s0.get_leases()), HasLength(3)) write(b"si1", secrets(3), {0: ([], [(0,data)], None)}, []) write(b"si1", secrets(4), {0: ([], [(0,data)], None)}, []) write(b"si1", secrets(5), {0: ([], [(0,data)], None)}, []) - self.failUnlessEqual(len(list(s0.get_leases())), 6) + self.assertThat(list(s0.get_leases()), HasLength(6)) all_leases = list(s0.get_leases()) # and write enough data to expand the container, forcing the server @@ -1728,15 +1728,15 @@ class MutableServer(unittest.TestCase): ss.renew_lease, b"si1", secrets(20)[1]) e_s = str(e) - self.failUnlessIn("Unable to renew non-existent lease", e_s) - self.failUnlessIn("I have leases accepted by nodeids:", e_s) - self.failUnlessIn("nodeids: 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' .", e_s) + self.assertThat(e_s, Contains("Unable to renew non-existent lease")) + self.assertThat(e_s, Contains("I have leases accepted by nodeids:")) + self.assertThat(e_s, Contains("nodeids: 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' .")) - self.assertEqual(all_leases, list(s0.get_leases())) + self.assertThat(all_leases, Equals(list(s0.get_leases()))) # reading shares should not modify the timestamp read(b"si1", [], [(0,200)]) - self.assertEqual(all_leases, list(s0.get_leases())) + self.assertThat(all_leases, Equals(list(s0.get_leases()))) write(b"si1", secrets(0), {0: ([], [(200, b"make me bigger")], None)}, []) @@ -1764,13 +1764,13 @@ class MutableServer(unittest.TestCase): write_enabler, renew_secret, cancel_secret = secrets(0) rc = write(b"si1", (write_enabler, renew_secret, cancel_secret), {0: ([], [(0,data)], None)}, []) - self.failUnlessEqual(rc, (True, {})) + self.assertThat(rc, Equals((True, {}))) bucket_dir = os.path.join(self.workdir("test_mutable_add_lease_renews"), "shares", storage_index_to_dir(b"si1")) s0 = MutableShareFile(os.path.join(bucket_dir, "0")) [lease] = s0.get_leases() - self.assertEqual(lease.get_expiration_time(), 235 + DEFAULT_RENEWAL_TIME) + self.assertThat(lease.get_expiration_time(), Equals(235 + DEFAULT_RENEWAL_TIME)) # Time passes... clock.advance(835) @@ -1778,8 +1778,8 @@ class MutableServer(unittest.TestCase): # Adding a lease renews it: ss.add_lease(b"si1", renew_secret, cancel_secret) [lease] = s0.get_leases() - self.assertEqual(lease.get_expiration_time(), - 235 + 835 + DEFAULT_RENEWAL_TIME) + self.assertThat(lease.get_expiration_time(), + Equals(235 + 835 + DEFAULT_RENEWAL_TIME)) def test_remove(self): ss = self.create("test_remove") @@ -1796,26 +1796,26 @@ class MutableServer(unittest.TestCase): []) # the answer should mention all the shares that existed before the # write - self.failUnlessEqual(answer, (True, {0:[],1:[],2:[]}) ) + self.assertThat(answer, Equals((True, {0:[],1:[],2:[]}))) # but a new read should show only sh1 and sh2 - self.failUnlessEqual(readv(b"si1", [], [(0,10)]), - {1: [b""], 2: [b""]}) + self.assertThat(readv(b"si1", [], [(0,10)]), + Equals({1: [b""], 2: [b""]})) # delete sh1 by setting its size to zero answer = writev(b"si1", secrets, {1: ([], [], 0)}, []) - self.failUnlessEqual(answer, (True, {1:[],2:[]}) ) - self.failUnlessEqual(readv(b"si1", [], [(0,10)]), - {2: [b""]}) + self.assertThat(answer, Equals((True, {1:[],2:[]}))) + self.assertThat(readv(b"si1", [], [(0,10)]), + Equals({2: [b""]})) # delete sh2 by setting its size to zero answer = writev(b"si1", secrets, {2: ([], [], 0)}, []) - self.failUnlessEqual(answer, (True, {2:[]}) ) - self.failUnlessEqual(readv(b"si1", [], [(0,10)]), - {}) + self.assertThat(answer, Equals((True, {2:[]}))) + self.assertThat(readv(b"si1", [], [(0,10)]), + Equals({})) # and the bucket directory should now be gone si = base32.b2a(b"si1") # note: this is a detail of the storage server implementation, and @@ -1824,8 +1824,8 @@ class MutableServer(unittest.TestCase): prefix = si[:2] prefixdir = os.path.join(self.workdir("test_remove"), "shares", prefix) bucketdir = os.path.join(prefixdir, si) - self.failUnless(os.path.exists(prefixdir), prefixdir) - self.failIf(os.path.exists(bucketdir), bucketdir) + self.assertThat(prefixdir, Contains(os.path.exists(prefixdir))) + self.assertFalse(os.path.exists(bucketdir), bucketdir) def test_writev_without_renew_lease(self): """ @@ -1854,7 +1854,7 @@ class MutableServer(unittest.TestCase): renew_leases=False, ) leases = list(ss.get_slot_leases(storage_index)) - self.assertEqual([], leases) + self.assertThat([], Equals(leases)) def test_get_slot_leases_empty_slot(self): """ @@ -1862,9 +1862,9 @@ class MutableServer(unittest.TestCase): shares, it returns an empty iterable. """ ss = self.create("test_get_slot_leases_empty_slot") - self.assertEqual( + self.assertThat( list(ss.get_slot_leases(b"si1")), - [], + Equals([]), ) def test_remove_non_present(self): @@ -1900,10 +1900,10 @@ class MutableServer(unittest.TestCase): ) self.assertTrue(testv_is_good) - self.assertEqual({}, read_data) + self.assertThat({}, Equals(read_data)) -class MDMFProxies(unittest.TestCase, ShouldFailMixin): +class MDMFProxies(SyncTestCase, ShouldFailMixin): def setUp(self): self.sparent = LoggingServiceParent() self._lease_secret = itertools.count() @@ -2084,7 +2084,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin): tws[0] = (testvs, [(0, data)], None) readv = [(0, 1)] results = write(storage_index, self.secrets, tws, readv) - self.failUnless(results[0]) + self.assertTrue(results[0]) def build_test_sdmf_share(self, empty=False): @@ -2150,7 +2150,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin): tws[0] = (testvs, [(0, share)], None) readv = [] results = write(storage_index, self.secrets, tws, readv) - self.failUnless(results[0]) + self.assertFalse(results[0]) def test_read(self): @@ -2160,8 +2160,8 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin): d = defer.succeed(None) def _check_block_and_salt(block_and_salt): (block, salt) = block_and_salt - self.failUnlessEqual(block, self.block) - self.failUnlessEqual(salt, self.salt) + self.assertThat(block, Equals(self.block)) + self.assertThat(salt, Equals(self.salt)) for i in range(6): d.addCallback(lambda ignored, i=i: @@ -2171,57 +2171,57 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin): d.addCallback(lambda ignored: mr.get_encprivkey()) d.addCallback(lambda encprivkey: - self.failUnlessEqual(self.encprivkey, encprivkey)) + self.assertThat(self.encprivkey, Equals(encprivkey))) d.addCallback(lambda ignored: mr.get_blockhashes()) d.addCallback(lambda blockhashes: - self.failUnlessEqual(self.block_hash_tree, blockhashes)) + self.assertThat(self.block_hash_tree, Equals(blockhashes))) d.addCallback(lambda ignored: mr.get_sharehashes()) d.addCallback(lambda sharehashes: - self.failUnlessEqual(self.share_hash_chain, sharehashes)) + self.assertThat(self.share_hash_chain, Equals(sharehashes))) d.addCallback(lambda ignored: mr.get_signature()) d.addCallback(lambda signature: - self.failUnlessEqual(signature, self.signature)) + self.assertThat(signature, Equals(self.signature))) d.addCallback(lambda ignored: mr.get_verification_key()) d.addCallback(lambda verification_key: - self.failUnlessEqual(verification_key, self.verification_key)) + self.assertThat(verification_key, Equals(self.verification_key))) d.addCallback(lambda ignored: mr.get_seqnum()) d.addCallback(lambda seqnum: - self.failUnlessEqual(seqnum, 0)) + self.assertThat(seqnum, Equals(0))) d.addCallback(lambda ignored: mr.get_root_hash()) d.addCallback(lambda root_hash: - self.failUnlessEqual(self.root_hash, root_hash)) + self.assertThat(self.root_hash, Equals(root_hash))) d.addCallback(lambda ignored: mr.get_seqnum()) d.addCallback(lambda seqnum: - self.failUnlessEqual(0, seqnum)) + self.assertThat(seqnum, Equals(0))) d.addCallback(lambda ignored: mr.get_encoding_parameters()) def _check_encoding_parameters(args): (k, n, segsize, datalen) = args - self.failUnlessEqual(k, 3) - self.failUnlessEqual(n, 10) - self.failUnlessEqual(segsize, 6) - self.failUnlessEqual(datalen, 36) + self.assertThat(k, Equals(3)) + self.assertThat(n, Equals(10)) + self.assertThat(segsize, Equals(6)) + self.assertThat(datalen, Equals(36)) d.addCallback(_check_encoding_parameters) d.addCallback(lambda ignored: mr.get_checkstring()) d.addCallback(lambda checkstring: - self.failUnlessEqual(checkstring, checkstring)) + self.assertThat(checkstring, Equals(checkstring))) return d @@ -2231,8 +2231,8 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin): d = mr.get_block_and_salt(5) def _check_tail_segment(results): block, salt = results - self.failUnlessEqual(len(block), 1) - self.failUnlessEqual(block, b"a") + self.assertThat(block, HasLength(1)) + self.assertThat(block, Equals(b"a")) d.addCallback(_check_tail_segment) return d @@ -2254,10 +2254,10 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin): d = mr.get_encoding_parameters() def _check_encoding_parameters(args): (k, n, segment_size, datalen) = args - self.failUnlessEqual(k, 3) - self.failUnlessEqual(n, 10) - self.failUnlessEqual(segment_size, 6) - self.failUnlessEqual(datalen, 36) + self.assertThat(k, Equals(3)) + self.assertThat(n, Equals(10)) + self.assertThat(segment_size, Equals(6)) + self.assertThat(datalen, Equals(36)) d.addCallback(_check_encoding_parameters) return d @@ -2267,7 +2267,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin): mr = MDMFSlotReadProxy(self.storage_server, b"si1", 0) d = mr.get_seqnum() d.addCallback(lambda seqnum: - self.failUnlessEqual(seqnum, 0)) + self.assertThat(seqnum, Equals(0))) return d @@ -2276,7 +2276,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin): mr = MDMFSlotReadProxy(self.storage_server, b"si1", 0) d = mr.get_root_hash() d.addCallback(lambda root_hash: - self.failUnlessEqual(root_hash, self.root_hash)) + self.assertThat(root_hash, Equals(self.root_hash))) return d @@ -2285,7 +2285,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin): mr = MDMFSlotReadProxy(self.storage_server, b"si1", 0) d = mr.get_checkstring() d.addCallback(lambda checkstring: - self.failUnlessEqual(checkstring, self.checkstring)) + self.assertThat(checkstring, Equals(self.checkstring))) return d @@ -2307,22 +2307,22 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin): mw.put_verification_key(self.verification_key) d = mw.finish_publishing() def _then(results): - self.failUnless(len(results), 2) + self.assertThat(results, HasLength(2)) result, readv = results - self.failUnless(result) - self.failIf(readv) + self.assertTrue(result) + self.assertFalse(readv) self.old_checkstring = mw.get_checkstring() mw.set_checkstring(b"") d.addCallback(_then) d.addCallback(lambda ignored: mw.finish_publishing()) def _then_again(results): - self.failUnlessEqual(len(results), 2) + self.assertThat(results, HasLength(2)) result, readvs = results - self.failIf(result) - self.failUnlessIn(0, readvs) + self.assertFalse(result) + self.assertThat(readvs, Contains(0)) readv = readvs[0][0] - self.failUnlessEqual(readv, self.old_checkstring) + self.assertThat(readv, Equals(self.old_checkstring)) d.addCallback(_then_again) # The checkstring remains the same for the rest of the process. return d @@ -2383,11 +2383,11 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin): def _check_success(results): result, readvs = results - self.failUnless(result) + self.assertTrue(result) def _check_failure(results): result, readvs = results - self.failIf(result) + self.assertFalse(result) def _write_share(mw): for i in range(6): @@ -2431,14 +2431,14 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin): # any point during the process, it should fail to write when we # tell it to write. def _check_failure(results): - self.failUnlessEqual(len(results), 2) + self.assertThat(results, Equals(2)) res, d = results - self.failIf(res) + self.assertFalse(res) def _check_success(results): - self.failUnlessEqual(len(results), 2) + self.assertThat(results, HasLength(2)) res, d = results - self.failUnless(results) + self.assertFalse(results) mw = self._make_new_mw(b"si1", 0) mw.set_checkstring(b"this is a lie") @@ -2495,100 +2495,100 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin): mw.put_verification_key(self.verification_key) d = mw.finish_publishing() def _check_publish(results): - self.failUnlessEqual(len(results), 2) + self.assertThat(results, HasLength(2)) result, ign = results - self.failUnless(result, "publish failed") + self.assertTrue(result, "publish failed") for i in range(6): - self.failUnlessEqual(read(b"si1", [0], [(expected_sharedata_offset + (i * written_block_size), written_block_size)]), - {0: [written_block]}) + self.assertThat(read(b"si1", [0], [(expected_sharedata_offset + (i * written_block_size), written_block_size)]), + Equals({0: [written_block]})) - self.failUnlessEqual(len(self.encprivkey), 7) - self.failUnlessEqual(read(b"si1", [0], [(expected_private_key_offset, 7)]), - {0: [self.encprivkey]}) + self.assertThat(self.encprivkey, HasLength(7)) + self.assertThat(read(b"si1", [0], [(expected_private_key_offset, 7)]), + Equals({0: [self.encprivkey]})) expected_block_hash_offset = expected_sharedata_offset + \ (6 * written_block_size) - self.failUnlessEqual(len(self.block_hash_tree_s), 32 * 6) - self.failUnlessEqual(read(b"si1", [0], [(expected_block_hash_offset, 32 * 6)]), - {0: [self.block_hash_tree_s]}) + self.assertThat(self.block_hash_tree_s, HasLength(32 * 6)) + self.assertThat(read(b"si1", [0], [(expected_block_hash_offset, 32 * 6)]), + Equals({0: [self.block_hash_tree_s]})) expected_share_hash_offset = expected_private_key_offset + len(self.encprivkey) - self.failUnlessEqual(read(b"si1", [0],[(expected_share_hash_offset, (32 + 2) * 6)]), - {0: [self.share_hash_chain_s]}) + self.assertThat(read(b"si1", [0],[(expected_share_hash_offset, (32 + 2) * 6)]), + Equals({0: [self.share_hash_chain_s]})) - self.failUnlessEqual(read(b"si1", [0], [(9, 32)]), - {0: [self.root_hash]}) + self.assertThat(read(b"si1", [0], [(9, 32)]), + Equals({0: [self.root_hash]})) expected_signature_offset = expected_share_hash_offset + \ len(self.share_hash_chain_s) - self.failUnlessEqual(len(self.signature), 9) - self.failUnlessEqual(read(b"si1", [0], [(expected_signature_offset, 9)]), - {0: [self.signature]}) + self.assertThat(self.signature, HasLength(9)) + self.assertThat(read(b"si1", [0], [(expected_signature_offset, 9)]), + Equals({0: [self.signature]})) expected_verification_key_offset = expected_signature_offset + len(self.signature) - self.failUnlessEqual(len(self.verification_key), 6) - self.failUnlessEqual(read(b"si1", [0], [(expected_verification_key_offset, 6)]), - {0: [self.verification_key]}) + self.assertThat(self.verification_key, HasLength(6)) + self.assertThat(read(b"si1", [0], [(expected_verification_key_offset, 6)]), + Equals({0: [self.verification_key]})) signable = mw.get_signable() verno, seq, roothash, k, n, segsize, datalen = \ struct.unpack(">BQ32sBBQQ", signable) - self.failUnlessEqual(verno, 1) - self.failUnlessEqual(seq, 0) - self.failUnlessEqual(roothash, self.root_hash) - self.failUnlessEqual(k, 3) - self.failUnlessEqual(n, 10) - self.failUnlessEqual(segsize, 6) - self.failUnlessEqual(datalen, 36) + self.assertThat(verno, Equals(1)) + self.assertThat(seq, Equals(0)) + self.assertThat(roothash, Equals(self.root_hash)) + self.assertThat(k, Equals(3)) + self.assertThat(n, Equals(10)) + self.assertThat(segsize, Equals(6)) + self.assertThat(datalen, Equals(36)) expected_eof_offset = expected_block_hash_offset + \ len(self.block_hash_tree_s) # Check the version number to make sure that it is correct. expected_version_number = struct.pack(">B", 1) - self.failUnlessEqual(read(b"si1", [0], [(0, 1)]), - {0: [expected_version_number]}) + self.assertThat(read(b"si1", [0], [(0, 1)]), + Equals({0: [expected_version_number]})) # Check the sequence number to make sure that it is correct expected_sequence_number = struct.pack(">Q", 0) - self.failUnlessEqual(read(b"si1", [0], [(1, 8)]), - {0: [expected_sequence_number]}) + self.assertThat(read(b"si1", [0], [(1, 8)]), + Equals({0: [expected_sequence_number]})) # Check that the encoding parameters (k, N, segement size, data # length) are what they should be. These are 3, 10, 6, 36 expected_k = struct.pack(">B", 3) - self.failUnlessEqual(read(b"si1", [0], [(41, 1)]), - {0: [expected_k]}) + self.assertThat(read(b"si1", [0], [(41, 1)]), + Equals({0: [expected_k]})) expected_n = struct.pack(">B", 10) - self.failUnlessEqual(read(b"si1", [0], [(42, 1)]), - {0: [expected_n]}) + self.assertThat(read(b"si1", [0], [(42, 1)]), + Equals({0: [expected_n]})) expected_segment_size = struct.pack(">Q", 6) - self.failUnlessEqual(read(b"si1", [0], [(43, 8)]), - {0: [expected_segment_size]}) + self.assertThat(read(b"si1", [0], [(43, 8)]), + Equals({0: [expected_segment_size]})) expected_data_length = struct.pack(">Q", 36) - self.failUnlessEqual(read(b"si1", [0], [(51, 8)]), - {0: [expected_data_length]}) + self.assertThat(read(b"si1", [0], [(51, 8)]), + Equals({0: [expected_data_length]})) expected_offset = struct.pack(">Q", expected_private_key_offset) - self.failUnlessEqual(read(b"si1", [0], [(59, 8)]), - {0: [expected_offset]}) + self.assertThat(read(b"si1", [0], [(59, 8)]), + Equals({0: [expected_offset]})) expected_offset = struct.pack(">Q", expected_share_hash_offset) - self.failUnlessEqual(read(b"si1", [0], [(67, 8)]), - {0: [expected_offset]}) + self.assertThat(read(b"si1", [0], [(67, 8)]), + Equals({0: [expected_offset]})) expected_offset = struct.pack(">Q", expected_signature_offset) - self.failUnlessEqual(read(b"si1", [0], [(75, 8)]), - {0: [expected_offset]}) + self.assertThat(read(b"si1", [0], [(75, 8)]), + Equals({0: [expected_offset]})) expected_offset = struct.pack(">Q", expected_verification_key_offset) - self.failUnlessEqual(read(b"si1", [0], [(83, 8)]), - {0: [expected_offset]}) + self.assertThat(read(b"si1", [0], [(83, 8)]), + Equals({0: [expected_offset]})) expected_offset = struct.pack(">Q", expected_verification_key_offset + len(self.verification_key)) - self.failUnlessEqual(read(b"si1", [0], [(91, 8)]), - {0: [expected_offset]}) + self.assertThat(read(b"si1", [0], [(91, 8)]), + Equals({0: [expected_offset]})) expected_offset = struct.pack(">Q", expected_sharedata_offset) - self.failUnlessEqual(read(b"si1", [0], [(99, 8)]), - {0: [expected_offset]}) + self.assertThat(read(b"si1", [0], [(99, 8)]), + Equals({0: [expected_offset]})) expected_offset = struct.pack(">Q", expected_block_hash_offset) - self.failUnlessEqual(read(b"si1", [0], [(107, 8)]), - {0: [expected_offset]}) + self.assertThat(read(b"si1", [0], [(107, 8)]), + Equals({0: [expected_offset]})) expected_offset = struct.pack(">Q", expected_eof_offset) - self.failUnlessEqual(read(b"si1", [0], [(115, 8)]), - {0: [expected_offset]}) + self.assertThat(read(b"si1", [0], [(115, 8)]), + Equals({0: [expected_offset]})) d.addCallback(_check_publish) return d @@ -2803,8 +2803,8 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin): mr = MDMFSlotReadProxy(self.storage_server, b"si1", 0) def _check_block_and_salt(block_and_salt): (block, salt) = block_and_salt - self.failUnlessEqual(block, self.block) - self.failUnlessEqual(salt, self.salt) + self.assertThat(block, Equals(self.block)) + self.assertThat(salt, Equals(self.salt)) for i in range(6): d.addCallback(lambda ignored, i=i: @@ -2814,52 +2814,52 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin): d.addCallback(lambda ignored: mr.get_encprivkey()) d.addCallback(lambda encprivkey: - self.failUnlessEqual(self.encprivkey, encprivkey)) + self.assertThat(self.encprivkey, Equals(encprivkey))) d.addCallback(lambda ignored: mr.get_blockhashes()) d.addCallback(lambda blockhashes: - self.failUnlessEqual(self.block_hash_tree, blockhashes)) + self.assertThat(self.block_hash_tree, Equals(blockhashes))) d.addCallback(lambda ignored: mr.get_sharehashes()) d.addCallback(lambda sharehashes: - self.failUnlessEqual(self.share_hash_chain, sharehashes)) + self.assertThat(self.share_hash_chain, Equals(sharehashes))) d.addCallback(lambda ignored: mr.get_signature()) d.addCallback(lambda signature: - self.failUnlessEqual(signature, self.signature)) + self.assertThat(signature, Equals(self.signature))) d.addCallback(lambda ignored: mr.get_verification_key()) d.addCallback(lambda verification_key: - self.failUnlessEqual(verification_key, self.verification_key)) + self.assertThat(verification_key, Equals(self.verification_key))) d.addCallback(lambda ignored: mr.get_seqnum()) d.addCallback(lambda seqnum: - self.failUnlessEqual(seqnum, 0)) + self.assertThat(seqnum, Equals(0))) d.addCallback(lambda ignored: mr.get_root_hash()) d.addCallback(lambda root_hash: - self.failUnlessEqual(self.root_hash, root_hash)) + self.assertThat(self.root_hash, Equals(root_hash))) d.addCallback(lambda ignored: mr.get_encoding_parameters()) def _check_encoding_parameters(args): (k, n, segsize, datalen) = args - self.failUnlessEqual(k, 3) - self.failUnlessEqual(n, 10) - self.failUnlessEqual(segsize, 6) - self.failUnlessEqual(datalen, 36) + self.assertThat(k, Equals(3)) + self.assertThat(n, Equals(10)) + self.assertThat(segsize, Equals(6)) + self.assertThat(datalen, Equals(36)) d.addCallback(_check_encoding_parameters) d.addCallback(lambda ignored: mr.get_checkstring()) d.addCallback(lambda checkstring: - self.failUnlessEqual(checkstring, mw.get_checkstring())) + self.assertThat(checkstring, Equals(mw.get_checkstring()))) return d @@ -2871,7 +2871,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin): mr = MDMFSlotReadProxy(self.storage_server, b"si1", 0) d = mr.is_sdmf() d.addCallback(lambda issdmf: - self.failUnless(issdmf)) + self.assertFalse(issdmf)) return d @@ -2884,7 +2884,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin): d.addCallback(lambda ignored: mr.is_sdmf()) d.addCallback(lambda issdmf: - self.failUnless(issdmf)) + self.assertTrue(issdmf)) # What do we need to read? # - The sharedata @@ -2897,51 +2897,51 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin): # bytes in size. The share is composed entirely of the # letter a. self.block contains 2 as, so 6 * self.block is # what we are looking for. - self.failUnlessEqual(block, self.block * 6) - self.failUnlessEqual(salt, self.salt) + self.assertThat(block, Equals(self.block * 6)) + self.assertThat(salt, Equals(self.salt)) d.addCallback(_check_block_and_salt) # - The blockhashes d.addCallback(lambda ignored: mr.get_blockhashes()) d.addCallback(lambda blockhashes: - self.failUnlessEqual(self.block_hash_tree, - blockhashes, + self.assertThat(self.block_hash_tree, + Equals(blockhashes), blockhashes)) # - The sharehashes d.addCallback(lambda ignored: mr.get_sharehashes()) d.addCallback(lambda sharehashes: - self.failUnlessEqual(self.share_hash_chain, - sharehashes)) + self.assertThat(self.share_hash_chain, + Equals(sharehashes))) # - The keys d.addCallback(lambda ignored: mr.get_encprivkey()) d.addCallback(lambda encprivkey: - self.failUnlessEqual(encprivkey, self.encprivkey, encprivkey)) + self.assertThat(encprivkey, self.encprivkey, Equals(encprivkey))) d.addCallback(lambda ignored: mr.get_verification_key()) d.addCallback(lambda verification_key: - self.failUnlessEqual(verification_key, - self.verification_key, + self.assertThat(verification_key, + Equals(self.verification_key), verification_key)) # - The signature d.addCallback(lambda ignored: mr.get_signature()) d.addCallback(lambda signature: - self.failUnlessEqual(signature, self.signature, signature)) + self.assertThat(signature, Equals(self.signature), signature)) # - The sequence number d.addCallback(lambda ignored: mr.get_seqnum()) d.addCallback(lambda seqnum: - self.failUnlessEqual(seqnum, 0, seqnum)) + self.assertThat(seqnum, Equals(0), seqnum)) # - The root hash d.addCallback(lambda ignored: mr.get_root_hash()) d.addCallback(lambda root_hash: - self.failUnlessEqual(root_hash, self.root_hash, root_hash)) + self.assertThat(root_hash, Equals(self.root_hash), root_hash)) return d @@ -2955,7 +2955,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin): d.addCallback(lambda ignored: mr.is_sdmf()) d.addCallback(lambda issdmf: - self.failUnless(issdmf)) + self.assertTrue(issdmf)) d.addCallback(lambda ignored: self.shouldFail(LayoutInvalid, "test bad segment", None, @@ -2983,8 +2983,8 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin): d.addCallback(lambda mr: mr.get_verinfo()) def _check_verinfo(verinfo): - self.failUnless(verinfo) - self.failUnlessEqual(len(verinfo), 9) + self.assertTrue(verinfo) + self.assertThat(verinfo, HasLength(9)) (seqnum, root_hash, salt_hash, @@ -2994,12 +2994,12 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin): n, prefix, offsets) = verinfo - self.failUnlessEqual(seqnum, 0) - self.failUnlessEqual(root_hash, self.root_hash) - self.failUnlessEqual(segsize, 6) - self.failUnlessEqual(datalen, 36) - self.failUnlessEqual(k, 3) - self.failUnlessEqual(n, 10) + self.assertThat(seqnum, Equals(0)) + self.assertThat(root_hash, Equals(self.root_hash)) + self.assertThat(segsize, Equals(6)) + self.assertThat(datalen, Equals(36)) + self.assertThat(k, Equals(3)) + self.assertThat(n, Equals(10)) expected_prefix = struct.pack(MDMFSIGNABLEHEADER, 1, seqnum, @@ -3008,8 +3008,8 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin): n, segsize, datalen) - self.failUnlessEqual(expected_prefix, prefix) - self.failUnlessEqual(self.rref.read_count, 0) + self.assertThat(expected_prefix, Equals(prefix)) + self.assertThat(self.rref.read_count, Equals(0)) d.addCallback(_check_verinfo) # This is not enough data to read a block and a share, so the # wrapper should attempt to read this from the remote server. @@ -3018,9 +3018,9 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin): mr.get_block_and_salt(0)) def _check_block_and_salt(block_and_salt): (block, salt) = block_and_salt - self.failUnlessEqual(block, self.block) - self.failUnlessEqual(salt, self.salt) - self.failUnlessEqual(self.rref.read_count, 1) + self.assertThat(block, Equals(self.block)) + self.assertThat(salt, Equals(self.salt)) + self.assertThat(self.rref.read_count, Equals(1)) # This should be enough data to read one block. d.addCallback(_make_mr, 123 + PRIVATE_KEY_SIZE + SIGNATURE_SIZE + VERIFICATION_KEY_SIZE + SHARE_HASH_CHAIN_SIZE + 140) d.addCallback(lambda mr: @@ -3044,8 +3044,8 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin): d.addCallback(lambda mr: mr.get_verinfo()) def _check_verinfo(verinfo): - self.failUnless(verinfo) - self.failUnlessEqual(len(verinfo), 9) + self.assertTrue(verinfo) + self.assertThat(verinfo, HasLength(9)) (seqnum, root_hash, salt, @@ -3055,13 +3055,13 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin): n, prefix, offsets) = verinfo - self.failUnlessEqual(seqnum, 0) - self.failUnlessEqual(root_hash, self.root_hash) - self.failUnlessEqual(salt, self.salt) - self.failUnlessEqual(segsize, 36) - self.failUnlessEqual(datalen, 36) - self.failUnlessEqual(k, 3) - self.failUnlessEqual(n, 10) + self.assertThat(seqnum, Equals(0)) + self.assertThat(root_hash, Equals(self.root_hash)) + self.assertThat(salt, Equals(self.salt)) + self.assertThat(segsize, Equals(36)) + self.assertThat(datalen, Equals(36)) + self.assertThat(k, Equals(3)) + self.assertThat(n, Equals(10)) expected_prefix = struct.pack(SIGNED_PREFIX, 0, seqnum, @@ -3071,8 +3071,8 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin): n, segsize, datalen) - self.failUnlessEqual(expected_prefix, prefix) - self.failUnlessEqual(self.rref.read_count, 0) + self.assertThat(expected_prefix, Equals(prefix)) + self.assertThat(self.rref.read_count, Equals(0)) d.addCallback(_check_verinfo) # This shouldn't be enough to read any share data. d.addCallback(_make_mr, 123) @@ -3080,11 +3080,11 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin): mr.get_block_and_salt(0)) def _check_block_and_salt(block_and_salt): (block, salt) = block_and_salt - self.failUnlessEqual(block, self.block * 6) - self.failUnlessEqual(salt, self.salt) + self.assertThat(block, Equals(self.block * 6)) + self.assertThat(salt, Equals(self.salt)) # TODO: Fix the read routine so that it reads only the data # that it has cached if it can't read all of it. - self.failUnlessEqual(self.rref.read_count, 2) + self.assertThat(self.rref.read_count, Equals(2)) # This should be enough to read share data. d.addCallback(_make_mr, self.offsets['share_data']) @@ -3106,12 +3106,12 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin): d.addCallback(lambda ignored: mr.get_encoding_parameters()) def _check_encoding_parameters(params): - self.failUnlessEqual(len(params), 4) + self.assertThat(params, HasLength(4)) k, n, segsize, datalen = params - self.failUnlessEqual(k, 3) - self.failUnlessEqual(n, 10) - self.failUnlessEqual(segsize, 0) - self.failUnlessEqual(datalen, 0) + self.assertThat(k, Equals(3)) + self.assertThat(n, Equals(10)) + self.assertThat(segsize, Equals(0)) + self.assertThat(datalen, Equals(0)) d.addCallback(_check_encoding_parameters) # We should not be able to fetch a block, since there are no @@ -3132,12 +3132,12 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin): d.addCallback(lambda ignored: mr.get_encoding_parameters()) def _check_encoding_parameters(params): - self.failUnlessEqual(len(params), 4) + self.assertThat(params, HasLength(4)) k, n, segsize, datalen = params - self.failUnlessEqual(k, 3) - self.failUnlessEqual(n, 10) - self.failUnlessEqual(segsize, 0) - self.failUnlessEqual(datalen, 0) + self.assertThat(k, Equals(3)) + self.assertThat(n, Equals(10)) + self.assertThat(segsize, Equals(0)) + self.assertThat(datalen, Equals(0)) d.addCallback(_check_encoding_parameters) # It does not make sense to get a block in this format, so we @@ -3157,8 +3157,8 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin): d.addCallback(lambda ignored: mr.get_verinfo()) def _check_verinfo(verinfo): - self.failUnless(verinfo) - self.failUnlessEqual(len(verinfo), 9) + self.assertTrue(verinfo) + self.assertThat(verinfo, HasLength(9)) (seqnum, root_hash, salt, @@ -3168,13 +3168,13 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin): n, prefix, offsets) = verinfo - self.failUnlessEqual(seqnum, 0) - self.failUnlessEqual(root_hash, self.root_hash) - self.failUnlessEqual(salt, self.salt) - self.failUnlessEqual(segsize, 36) - self.failUnlessEqual(datalen, 36) - self.failUnlessEqual(k, 3) - self.failUnlessEqual(n, 10) + self.assertThat(seqnum, Equals(0)) + self.assertThat(root_hash, Equals(self.root_hash)) + self.assertThat(salt, Equals(self.salt)) + self.assertThat(segsize, Equals(36)) + self.assertThat(datalen, Equals(36)) + self.assertThat(k, Equals(3)) + self.assertThat(n, Equals(10)) expected_prefix = struct.pack(">BQ32s16s BBQQ", 0, seqnum, @@ -3184,8 +3184,8 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin): n, segsize, datalen) - self.failUnlessEqual(prefix, expected_prefix) - self.failUnlessEqual(offsets, self.offsets) + self.assertThat(prefix, Equals(expected_prefix)) + self.assertThat(offsets, Equals(self.offsets)) d.addCallback(_check_verinfo) return d @@ -3197,8 +3197,8 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin): d.addCallback(lambda ignored: mr.get_verinfo()) def _check_verinfo(verinfo): - self.failUnless(verinfo) - self.failUnlessEqual(len(verinfo), 9) + self.assertThat(verinfo) + self.assertThat(verinfo, HasLength(9)) (seqnum, root_hash, IV, @@ -3208,13 +3208,13 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin): n, prefix, offsets) = verinfo - self.failUnlessEqual(seqnum, 0) - self.failUnlessEqual(root_hash, self.root_hash) - self.failIf(IV) - self.failUnlessEqual(segsize, 6) - self.failUnlessEqual(datalen, 36) - self.failUnlessEqual(k, 3) - self.failUnlessEqual(n, 10) + self.assertThat(seqnum, Equals(0)) + self.assertThat(root_hash, Equals(self.root_hash)) + self.assertFalse(IV) + self.assertThat(segsize, Equals(6)) + self.assertThat(datalen, Equals(36)) + self.assertThat(k, Equals(3)) + self.assertThat(n, Equals(10)) expected_prefix = struct.pack(">BQ32s BBQQ", 1, seqnum, @@ -3223,8 +3223,8 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin): n, segsize, datalen) - self.failUnlessEqual(prefix, expected_prefix) - self.failUnlessEqual(offsets, self.offsets) + self.assertThat(prefix, Equals(expected_prefix)) + self.assertThat(offsets, Equals(self.offsets)) d.addCallback(_check_verinfo) return d @@ -3260,15 +3260,15 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin): sdmfr.put_verification_key(self.verification_key) # Now check to make sure that nothing has been written yet. - self.failUnlessEqual(self.rref.write_count, 0) + self.assertThat(self.rref.write_count, Equals(0)) # Now finish publishing d = sdmfr.finish_publishing() def _then(ignored): - self.failUnlessEqual(self.rref.write_count, 1) + self.assertThat(self.rref.write_count, Equals(1)) read = self.ss.slot_readv - self.failUnlessEqual(read(b"si1", [0], [(0, len(data))]), - {0: [data]}) + self.assertThat(read(b"si1", [0], [(0, len(data))]), + Equals({0: [data]})) d.addCallback(_then) return d @@ -3304,11 +3304,11 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin): sdmfw.put_verification_key(self.verification_key) # We shouldn't have a checkstring yet - self.failUnlessEqual(sdmfw.get_checkstring(), b"") + self.assertThat(sdmfw.get_checkstring(), Equals(b"")) d = sdmfw.finish_publishing() def _then(results): - self.failIf(results[0]) + self.assertFalse(results[0]) # this is the correct checkstring self._expected_checkstring = results[1][0][0] return self._expected_checkstring @@ -3318,21 +3318,21 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin): d.addCallback(lambda ignored: sdmfw.get_checkstring()) d.addCallback(lambda checkstring: - self.failUnlessEqual(checkstring, self._expected_checkstring)) + self.assertThat(checkstring, Equals(self._expected_checkstring))) d.addCallback(lambda ignored: sdmfw.finish_publishing()) def _then_again(results): - self.failUnless(results[0]) + self.assertTrue(results[0]) read = self.ss.slot_readv - self.failUnlessEqual(read(b"si1", [0], [(1, 8)]), + self.assertThat(read(b"si1", [0], [(1, 8)]), {0: [struct.pack(">Q", 1)]}) - self.failUnlessEqual(read(b"si1", [0], [(9, len(data) - 9)]), - {0: [data[9:]]}) + self.assertThat(read(b"si1", [0], [(9, len(data) - 9)]), + Equals({0: [data[9:]]})) d.addCallback(_then_again) return d -class Stats(unittest.TestCase): +class Stats(SyncTestCase): def setUp(self): self.sparent = LoggingServiceParent() @@ -3364,57 +3364,57 @@ class Stats(unittest.TestCase): output = ss.get_latencies() - self.failUnlessEqual(sorted(output.keys()), - sorted(["allocate", "renew", "cancel", "write", "get"])) - self.failUnlessEqual(len(ss.latencies["allocate"]), 1000) - self.failUnless(abs(output["allocate"]["mean"] - 9500) < 1, output) - self.failUnless(abs(output["allocate"]["01_0_percentile"] - 9010) < 1, output) - self.failUnless(abs(output["allocate"]["10_0_percentile"] - 9100) < 1, output) - self.failUnless(abs(output["allocate"]["50_0_percentile"] - 9500) < 1, output) - self.failUnless(abs(output["allocate"]["90_0_percentile"] - 9900) < 1, output) - self.failUnless(abs(output["allocate"]["95_0_percentile"] - 9950) < 1, output) - self.failUnless(abs(output["allocate"]["99_0_percentile"] - 9990) < 1, output) - self.failUnless(abs(output["allocate"]["99_9_percentile"] - 9999) < 1, output) + self.assertThat(sorted(output.keys()), + Equals(sorted(["allocate", "renew", "cancel", "write", "get"]))) + self.assertThat(ss.latencies["allocate"], HasLength(1000)) + self.assertTrue(abs(output["allocate"]["mean"] - 9500) < 1, output) + self.assertTrue(abs(output["allocate"]["01_0_percentile"] - 9010) < 1, output) + self.assertTrue(abs(output["allocate"]["10_0_percentile"] - 9100) < 1, output) + self.assertTrue(abs(output["allocate"]["50_0_percentile"] - 9500) < 1, output) + self.assertTrue(abs(output["allocate"]["90_0_percentile"] - 9900) < 1, output) + self.assertTrue(abs(output["allocate"]["95_0_percentile"] - 9950) < 1, output) + self.assertTrue(abs(output["allocate"]["99_0_percentile"] - 9990) < 1, output) + self.assertTrue(abs(output["allocate"]["99_9_percentile"] - 9999) < 1, output) - self.failUnlessEqual(len(ss.latencies["renew"]), 1000) - self.failUnless(abs(output["renew"]["mean"] - 500) < 1, output) - self.failUnless(abs(output["renew"]["01_0_percentile"] - 10) < 1, output) - self.failUnless(abs(output["renew"]["10_0_percentile"] - 100) < 1, output) - self.failUnless(abs(output["renew"]["50_0_percentile"] - 500) < 1, output) - self.failUnless(abs(output["renew"]["90_0_percentile"] - 900) < 1, output) - self.failUnless(abs(output["renew"]["95_0_percentile"] - 950) < 1, output) - self.failUnless(abs(output["renew"]["99_0_percentile"] - 990) < 1, output) - self.failUnless(abs(output["renew"]["99_9_percentile"] - 999) < 1, output) + self.assertThat(ss.latencies["renew"], HasLength(1000)) + self.assertTrue(abs(output["renew"]["mean"] - 500) < 1, output) + self.assertTrue(abs(output["renew"]["01_0_percentile"] - 10) < 1, output) + self.assertTrue(abs(output["renew"]["10_0_percentile"] - 100) < 1, output) + self.assertTrue(abs(output["renew"]["50_0_percentile"] - 500) < 1, output) + self.assertTrue(abs(output["renew"]["90_0_percentile"] - 900) < 1, output) + self.assertTrue(abs(output["renew"]["95_0_percentile"] - 950) < 1, output) + self.assertTrue(abs(output["renew"]["99_0_percentile"] - 990) < 1, output) + self.assertTrue(abs(output["renew"]["99_9_percentile"] - 999) < 1, output) - self.failUnlessEqual(len(ss.latencies["write"]), 20) - self.failUnless(abs(output["write"]["mean"] - 9) < 1, output) - self.failUnless(output["write"]["01_0_percentile"] is None, output) - self.failUnless(abs(output["write"]["10_0_percentile"] - 2) < 1, output) - self.failUnless(abs(output["write"]["50_0_percentile"] - 10) < 1, output) - self.failUnless(abs(output["write"]["90_0_percentile"] - 18) < 1, output) - self.failUnless(abs(output["write"]["95_0_percentile"] - 19) < 1, output) - self.failUnless(output["write"]["99_0_percentile"] is None, output) - self.failUnless(output["write"]["99_9_percentile"] is None, output) + self.assertThat(ss.latencies["write"], HasLength(20)) + self.assertTrue(abs(output["write"]["mean"] - 9) < 1, output) + self.assertTrue(output["write"]["01_0_percentile"] is None, output) + self.assertTrue(abs(output["write"]["10_0_percentile"] - 2) < 1, output) + self.assertTrue(abs(output["write"]["50_0_percentile"] - 10) < 1, output) + self.assertTrue(abs(output["write"]["90_0_percentile"] - 18) < 1, output) + self.assertTrue(abs(output["write"]["95_0_percentile"] - 19) < 1, output) + self.assertTrue(output["write"]["99_0_percentile"] is None, output) + self.assertTrue(output["write"]["99_9_percentile"] is None, output) - self.failUnlessEqual(len(ss.latencies["cancel"]), 10) - self.failUnless(abs(output["cancel"]["mean"] - 9) < 1, output) - self.failUnless(output["cancel"]["01_0_percentile"] is None, output) - self.failUnless(abs(output["cancel"]["10_0_percentile"] - 2) < 1, output) - self.failUnless(abs(output["cancel"]["50_0_percentile"] - 10) < 1, output) - self.failUnless(abs(output["cancel"]["90_0_percentile"] - 18) < 1, output) - self.failUnless(output["cancel"]["95_0_percentile"] is None, output) - self.failUnless(output["cancel"]["99_0_percentile"] is None, output) - self.failUnless(output["cancel"]["99_9_percentile"] is None, output) + self.assertThat(ss.latencies["cancel"], HasLength(10)) + self.assertTrue(abs(output["cancel"]["mean"] - 9) < 1, output) + self.assertTrue(output["cancel"]["01_0_percentile"] is None, output) + self.assertTrue(abs(output["cancel"]["10_0_percentile"] - 2) < 1, output) + self.assertTrue(abs(output["cancel"]["50_0_percentile"] - 10) < 1, output) + self.assertTrue(abs(output["cancel"]["90_0_percentile"] - 18) < 1, output) + self.assertTrue(output["cancel"]["95_0_percentile"] is None, output) + self.assertTrue(output["cancel"]["99_0_percentile"] is None, output) + self.assertTrue(output["cancel"]["99_9_percentile"] is None, output) - self.failUnlessEqual(len(ss.latencies["get"]), 1) - self.failUnless(output["get"]["mean"] is None, output) - self.failUnless(output["get"]["01_0_percentile"] is None, output) - self.failUnless(output["get"]["10_0_percentile"] is None, output) - self.failUnless(output["get"]["50_0_percentile"] is None, output) - self.failUnless(output["get"]["90_0_percentile"] is None, output) - self.failUnless(output["get"]["95_0_percentile"] is None, output) - self.failUnless(output["get"]["99_0_percentile"] is None, output) - self.failUnless(output["get"]["99_9_percentile"] is None, output) + self.assertThat(ss.latencies["get"], HasLength(1)) + self.assertTrue(output["get"]["mean"] is None, output) + self.assertTrue(output["get"]["01_0_percentile"] is None, output) + self.assertTrue(output["get"]["10_0_percentile"] is None, output) + self.assertTrue(output["get"]["50_0_percentile"] is None, output) + self.assertTrue(output["get"]["90_0_percentile"] is None, output) + self.assertTrue(output["get"]["95_0_percentile"] is None, output) + self.assertTrue(output["get"]["99_0_percentile"] is None, output) + self.assertTrue(output["get"]["99_9_percentile"] is None, output) immutable_schemas = strategies.sampled_from(list(ALL_IMMUTABLE_SCHEMAS)) @@ -3557,7 +3557,7 @@ class ShareFileTests(unittest.TestCase): mutable_schemas = strategies.sampled_from(list(ALL_MUTABLE_SCHEMAS)) -class MutableShareFileTests(unittest.TestCase): +class MutableShareFileTests(SyncTestCase): """ Tests for allmydata.storage.mutable.MutableShareFile. """ From 3a613aee704d0d4231f70e82d46bcaed84960692 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 21 Nov 2022 12:24:50 -0500 Subject: [PATCH 1177/2309] Try a different approach to timeouts: dynamic, instead of hardcoded. --- src/allmydata/test/common_system.py | 30 +++++++++++++++++++++++- src/allmydata/test/test_storage_https.py | 21 ++++------------- 2 files changed, 34 insertions(+), 17 deletions(-) diff --git a/src/allmydata/test/common_system.py b/src/allmydata/test/common_system.py index 8d3019935..a6b239005 100644 --- a/src/allmydata/test/common_system.py +++ b/src/allmydata/test/common_system.py @@ -20,6 +20,7 @@ from foolscap.api import flushEventualQueue from allmydata import client from allmydata.introducer.server import create_introducer from allmydata.util import fileutil, log, pollmixin +from allmydata.util.deferredutil import async_to_deferred from allmydata.storage import http_client from allmydata.storage_client import ( NativeStorageServer, @@ -639,6 +640,33 @@ def _render_section_values(values): )) +@async_to_deferred +async def spin_until_cleanup_done(value=None, timeout=10): + """ + At the end of the test, spin until either a timeout is hit, or the reactor + has no more DelayedCalls. + + Make sure to register during setUp. + """ + def num_fds(): + if hasattr(reactor, "handles"): + # IOCP! + return len(reactor.handles) + else: + # Normal reactor + return len([r for r in reactor.getReaders() + if r.__class__.__name__ not in ("_UnixWaker", "_SIGCHLDWaker")] + ) + len(reactor.getWriters()) + + for i in range(timeout * 1000): + # There's a single DelayedCall for AsynchronousDeferredRunTest's + # timeout... + if (len(reactor.getDelayedCalls()) < 2 and num_fds() == 0): + break + await deferLater(reactor, 0.001) + return value + + class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin): # If set to True, use Foolscap for storage protocol. If set to False, HTTP @@ -685,7 +713,7 @@ class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin): d = self.sparent.stopService() d.addBoth(flush_but_dont_ignore) d.addBoth(lambda x: self.close_idle_http_connections().addCallback(lambda _: x)) - d.addBoth(lambda x: deferLater(reactor, 2, lambda: x)) + d.addBoth(spin_until_cleanup_done) return d def getdir(self, subdir): diff --git a/src/allmydata/test/test_storage_https.py b/src/allmydata/test/test_storage_https.py index 062eb5b0e..284c8cda8 100644 --- a/src/allmydata/test/test_storage_https.py +++ b/src/allmydata/test/test_storage_https.py @@ -12,7 +12,6 @@ from cryptography import x509 from twisted.internet.endpoints import serverFromString from twisted.internet import reactor -from twisted.internet.task import deferLater from twisted.web.server import Site from twisted.web.static import Data from twisted.web.client import Agent, HTTPConnectionPool, ResponseNeverReceived @@ -30,6 +29,7 @@ from ..storage.http_common import get_spki_hash from ..storage.http_client import _StorageClientHTTPSPolicy from ..storage.http_server import _TLSEndpointWrapper from ..util.deferredutil import async_to_deferred +from .common_system import spin_until_cleanup_done class HTTPSNurlTests(SyncTestCase): @@ -87,6 +87,10 @@ class PinningHTTPSValidation(AsyncTestCase): self.addCleanup(self._port_assigner.tearDown) return AsyncTestCase.setUp(self) + def tearDown(self): + AsyncTestCase.tearDown(self) + return spin_until_cleanup_done() + @asynccontextmanager async def listen(self, private_key_path: FilePath, cert_path: FilePath): """ @@ -107,9 +111,6 @@ class PinningHTTPSValidation(AsyncTestCase): yield f"https://127.0.0.1:{listening_port.getHost().port}/" finally: await listening_port.stopListening() - # Make sure all server connections are closed :( No idea why this - # is necessary when it's not for IStorageServer HTTPS tests. - await deferLater(reactor, 0.01) def request(self, url: str, expected_certificate: x509.Certificate): """ @@ -144,10 +145,6 @@ class PinningHTTPSValidation(AsyncTestCase): response = await self.request(url, certificate) self.assertEqual(await response.content(), b"YOYODYNE") - # We keep getting TLSMemoryBIOProtocol being left around, so try harder - # to wait for it to finish. - await deferLater(reactor, 0.01) - @async_to_deferred async def test_server_certificate_has_wrong_hash(self): """ @@ -183,10 +180,6 @@ class PinningHTTPSValidation(AsyncTestCase): response = await self.request(url, certificate) self.assertEqual(await response.content(), b"YOYODYNE") - # We keep getting TLSMemoryBIOProtocol being left around, so try harder - # to wait for it to finish. - await deferLater(reactor, 0.01) - @async_to_deferred async def test_server_certificate_not_valid_yet(self): """ @@ -206,10 +199,6 @@ class PinningHTTPSValidation(AsyncTestCase): response = await self.request(url, certificate) self.assertEqual(await response.content(), b"YOYODYNE") - # We keep getting TLSMemoryBIOProtocol being left around, so try harder - # to wait for it to finish. - await deferLater(reactor, 0.001) - # A potential attack to test is a private key that doesn't match the # certificate... but OpenSSL (quite rightly) won't let you listen with that # so I don't know how to test that! See From c80469b50bd6f97d98ab22b48ac4b6481020a1df Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 22 Nov 2022 11:55:56 -0500 Subject: [PATCH 1178/2309] Handle the Windows waker too. --- src/allmydata/test/common_system.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/common_system.py b/src/allmydata/test/common_system.py index a6b239005..ca2904b53 100644 --- a/src/allmydata/test/common_system.py +++ b/src/allmydata/test/common_system.py @@ -655,7 +655,7 @@ async def spin_until_cleanup_done(value=None, timeout=10): else: # Normal reactor return len([r for r in reactor.getReaders() - if r.__class__.__name__ not in ("_UnixWaker", "_SIGCHLDWaker")] + if r.__class__.__name__ not in ("_UnixWaker", "_SIGCHLDWaker", "_SocketWaker")] ) + len(reactor.getWriters()) for i in range(timeout * 1000): From c296071767a9c8d62e227aae6cdd824abfc7d331 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 22 Nov 2022 14:11:58 -0500 Subject: [PATCH 1179/2309] News file. --- newsfragments/3939.bugfix | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/3939.bugfix diff --git a/newsfragments/3939.bugfix b/newsfragments/3939.bugfix new file mode 100644 index 000000000..61fb4244a --- /dev/null +++ b/newsfragments/3939.bugfix @@ -0,0 +1 @@ +Uploading immutables will now use more bandwidth, which should allow for faster uploads in many cases. \ No newline at end of file From a4787ca45ebebf3c59216366b5edf3a56f548003 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 22 Nov 2022 14:12:14 -0500 Subject: [PATCH 1180/2309] Batch writes much more aggressively. --- src/allmydata/immutable/layout.py | 69 ++++++++++++++++++++----------- 1 file changed, 44 insertions(+), 25 deletions(-) diff --git a/src/allmydata/immutable/layout.py b/src/allmydata/immutable/layout.py index d552d43c4..863b1cb75 100644 --- a/src/allmydata/immutable/layout.py +++ b/src/allmydata/immutable/layout.py @@ -113,13 +113,14 @@ class WriteBucketProxy(object): fieldstruct = ">L" def __init__(self, rref, server, data_size, block_size, num_segments, - num_share_hashes, uri_extension_size, pipeline_size=50000): + num_share_hashes, uri_extension_size, batch_size=1_000_000): self._rref = rref self._server = server self._data_size = data_size self._block_size = block_size self._num_segments = num_segments self._written_bytes = 0 + self._to_write = b"" effective_segments = mathutil.next_power_of_k(num_segments,2) self._segment_hash_size = (2*effective_segments - 1) * HASH_SIZE @@ -130,11 +131,13 @@ class WriteBucketProxy(object): self._create_offsets(block_size, data_size) - # k=3, max_segment_size=128KiB gives us a typical segment of 43691 - # bytes. Setting the default pipeline_size to 50KB lets us get two - # segments onto the wire but not a third, which would keep the pipe - # filled. - self._pipeline = pipeline.Pipeline(pipeline_size) + # With a ~1MB batch size, max upload speed is 1MB * round-trip latency + # assuming the writing code waits for writes to finish, so 20MB/sec if + # latency is 50ms. In the US many people only have 1MB/sec upload speed + # as of 2022 (standard Comcast). For further discussion of how one + # might set batch sizes see + # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3787#comment:1. + self._batch_size = batch_size def get_allocated_size(self): return (self._offsets['uri_extension'] + self.fieldsize + @@ -179,7 +182,7 @@ class WriteBucketProxy(object): return "" % self._server.get_name() def put_header(self): - return self._write(0, self._offset_data) + return self._queue_write(0, self._offset_data) def put_block(self, segmentnum, data): offset = self._offsets['data'] + segmentnum * self._block_size @@ -193,13 +196,13 @@ class WriteBucketProxy(object): (self._block_size * (self._num_segments - 1))), len(data), self._block_size) - return self._write(offset, data) + return self._queue_write(offset, data) def put_crypttext_hashes(self, hashes): # plaintext_hash_tree precedes crypttext_hash_tree. It is not used, and # so is not explicitly written, but we need to write everything, so # fill it in with nulls. - d = self._write(self._offsets['plaintext_hash_tree'], b"\x00" * self._segment_hash_size) + d = self._queue_write(self._offsets['plaintext_hash_tree'], b"\x00" * self._segment_hash_size) d.addCallback(lambda _: self._really_put_crypttext_hashes(hashes)) return d @@ -212,7 +215,7 @@ class WriteBucketProxy(object): precondition(offset + len(data) <= self._offsets['block_hashes'], offset, len(data), offset+len(data), self._offsets['block_hashes']) - return self._write(offset, data) + return self._queue_write(offset, data) def put_block_hashes(self, blockhashes): offset = self._offsets['block_hashes'] @@ -223,7 +226,7 @@ class WriteBucketProxy(object): precondition(offset + len(data) <= self._offsets['share_hashes'], offset, len(data), offset+len(data), self._offsets['share_hashes']) - return self._write(offset, data) + return self._queue_write(offset, data) def put_share_hashes(self, sharehashes): # sharehashes is a list of (index, hash) tuples, so they get stored @@ -237,29 +240,45 @@ class WriteBucketProxy(object): precondition(offset + len(data) <= self._offsets['uri_extension'], offset, len(data), offset+len(data), self._offsets['uri_extension']) - return self._write(offset, data) + return self._queue_write(offset, data) def put_uri_extension(self, data): offset = self._offsets['uri_extension'] assert isinstance(data, bytes) precondition(len(data) == self._uri_extension_size) length = struct.pack(self.fieldstruct, len(data)) - return self._write(offset, length+data) + return self._queue_write(offset, length+data) - def _write(self, offset, data): - # use a Pipeline to pipeline several writes together. TODO: another - # speedup would be to coalesce small writes into a single call: this - # would reduce the foolscap CPU overhead per share, but wouldn't - # reduce the number of round trips, so it might not be worth the - # effort. - self._written_bytes += len(data) - return self._pipeline.add(len(data), - self._rref.callRemote, "write", offset, data) + def _queue_write(self, offset, data): + """ + This queues up small writes to be written in a single batched larger + write. + + Callers of this function are expected to queue the data in order, with + no holes. As such, the offset is technically unnecessary, but is used + to check the inputs. Possibly we should get rid of it. + """ + assert offset == self._written_bytes + len(self._to_write) + self._to_write += data + if len(self._to_write) >= self._batch_size: + return self._actually_write() + else: + return defer.succeed(False) + + def _actually_write(self): + """Actually write data to the server.""" + offset = self._written_bytes + data = self._to_write + self._written_bytes += len(self._to_write) + self._to_write = b"" + return self._rref.callRemote("write", offset, data) def close(self): - assert self._written_bytes == self.get_allocated_size(), f"{self._written_bytes} != {self.get_allocated_size()}" - d = self._pipeline.add(0, self._rref.callRemote, "close") - d.addCallback(lambda ign: self._pipeline.flush()) + assert self._written_bytes + len(self._to_write) == self.get_allocated_size(), ( + f"{self._written_bytes} + {len(self._to_write)} != {self.get_allocated_size()}" + ) + d = self._actually_write() + d.addCallback(lambda _: self._rref.callRemote("close")) return d def abort(self): From f638aec0af901ca763c2d5e4f034a3c45f2787bb Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 22 Nov 2022 14:22:54 -0500 Subject: [PATCH 1181/2309] Refactor to use BytesIO. --- src/allmydata/immutable/layout.py | 30 ++++++++++++------------------ 1 file changed, 12 insertions(+), 18 deletions(-) diff --git a/src/allmydata/immutable/layout.py b/src/allmydata/immutable/layout.py index 863b1cb75..477fdf159 100644 --- a/src/allmydata/immutable/layout.py +++ b/src/allmydata/immutable/layout.py @@ -1,21 +1,14 @@ """ Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals - -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import struct +from io import BytesIO from zope.interface import implementer from twisted.internet import defer from allmydata.interfaces import IStorageBucketWriter, IStorageBucketReader, \ FileTooLargeError, HASH_SIZE -from allmydata.util import mathutil, observer, pipeline, log +from allmydata.util import mathutil, observer, log from allmydata.util.assertutil import precondition from allmydata.storage.server import si_b2a @@ -120,7 +113,7 @@ class WriteBucketProxy(object): self._block_size = block_size self._num_segments = num_segments self._written_bytes = 0 - self._to_write = b"" + self._to_write = BytesIO() effective_segments = mathutil.next_power_of_k(num_segments,2) self._segment_hash_size = (2*effective_segments - 1) * HASH_SIZE @@ -258,9 +251,10 @@ class WriteBucketProxy(object): no holes. As such, the offset is technically unnecessary, but is used to check the inputs. Possibly we should get rid of it. """ - assert offset == self._written_bytes + len(self._to_write) - self._to_write += data - if len(self._to_write) >= self._batch_size: + queued_size = len(self._to_write.getbuffer()) + assert offset == self._written_bytes + queued_size + self._to_write.write(data) + if queued_size + len(data) >= self._batch_size: return self._actually_write() else: return defer.succeed(False) @@ -268,14 +262,14 @@ class WriteBucketProxy(object): def _actually_write(self): """Actually write data to the server.""" offset = self._written_bytes - data = self._to_write - self._written_bytes += len(self._to_write) - self._to_write = b"" + data = self._to_write.getvalue() + self._written_bytes += len(data) + self._to_write = BytesIO() return self._rref.callRemote("write", offset, data) def close(self): - assert self._written_bytes + len(self._to_write) == self.get_allocated_size(), ( - f"{self._written_bytes} + {len(self._to_write)} != {self.get_allocated_size()}" + assert self._written_bytes + len(self._to_write.getbuffer()) == self.get_allocated_size(), ( + f"{self._written_bytes} + {len(self._to_write.getbuffer())} != {self.get_allocated_size()}" ) d = self._actually_write() d.addCallback(lambda _: self._rref.callRemote("close")) From d86d578034030975593341e8730d2e9e74f2ba6c Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 22 Nov 2022 15:17:56 -0500 Subject: [PATCH 1182/2309] Refactor to make core data structure easier to test in isolation. --- src/allmydata/immutable/layout.py | 77 ++++++++++++++++++++++--------- 1 file changed, 55 insertions(+), 22 deletions(-) diff --git a/src/allmydata/immutable/layout.py b/src/allmydata/immutable/layout.py index 477fdf159..b9eb74d8f 100644 --- a/src/allmydata/immutable/layout.py +++ b/src/allmydata/immutable/layout.py @@ -2,8 +2,12 @@ Ported to Python 3. """ +from __future__ import annotations + import struct from io import BytesIO + +from attrs import define, field from zope.interface import implementer from twisted.internet import defer from allmydata.interfaces import IStorageBucketWriter, IStorageBucketReader, \ @@ -100,6 +104,43 @@ def make_write_bucket_proxy(rref, server, num_share_hashes, uri_extension_size) return wbp + +@define +class _WriteBuffer: + """ + Queue up small writes to be written in a single batched larger write. + """ + _batch_size: int + _to_write : BytesIO = field(factory=BytesIO) + _written_bytes : int = field(default=0) + + def queue_write(self, offset: int, data: bytes) -> bool: + """ + Queue a write. If the result is ``False``, no further action is needed + for now. If the result is some ``True``, it's time to call ``flush()`` + and do a real write. + + Callers of this function are expected to queue the data in order, with + no holes. As such, the offset is technically unnecessary, but is used + to check the inputs. Possibly we should get rid of it. + """ + assert offset == self.get_total_bytes_queued() + self._to_write.write(data) + return len(self._to_write.getbuffer()) >= self._batch_size + + def flush(self) -> tuple[int, bytes]: + """Return offset and data to be written.""" + offset = self._written_bytes + data = self._to_write.getvalue() + self._written_bytes += len(data) + self._to_write = BytesIO() + return (offset, data) + + def get_total_bytes_queued(self) -> int: + """Return how many bytes were written or queued in total.""" + return self._written_bytes + len(self._to_write.getbuffer()) + + @implementer(IStorageBucketWriter) class WriteBucketProxy(object): fieldsize = 4 @@ -112,8 +153,6 @@ class WriteBucketProxy(object): self._data_size = data_size self._block_size = block_size self._num_segments = num_segments - self._written_bytes = 0 - self._to_write = BytesIO() effective_segments = mathutil.next_power_of_k(num_segments,2) self._segment_hash_size = (2*effective_segments - 1) * HASH_SIZE @@ -130,7 +169,7 @@ class WriteBucketProxy(object): # as of 2022 (standard Comcast). For further discussion of how one # might set batch sizes see # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3787#comment:1. - self._batch_size = batch_size + self._write_buffer = _WriteBuffer(batch_size) def get_allocated_size(self): return (self._offsets['uri_extension'] + self.fieldsize + @@ -251,25 +290,19 @@ class WriteBucketProxy(object): no holes. As such, the offset is technically unnecessary, but is used to check the inputs. Possibly we should get rid of it. """ - queued_size = len(self._to_write.getbuffer()) - assert offset == self._written_bytes + queued_size - self._to_write.write(data) - if queued_size + len(data) >= self._batch_size: + if self._write_buffer.queue_write(offset, data): return self._actually_write() else: return defer.succeed(False) def _actually_write(self): - """Actually write data to the server.""" - offset = self._written_bytes - data = self._to_write.getvalue() - self._written_bytes += len(data) - self._to_write = BytesIO() + """Write data to the server.""" + offset, data = self._write_buffer.flush() return self._rref.callRemote("write", offset, data) def close(self): - assert self._written_bytes + len(self._to_write.getbuffer()) == self.get_allocated_size(), ( - f"{self._written_bytes} + {len(self._to_write.getbuffer())} != {self.get_allocated_size()}" + assert self._write_buffer.get_total_bytes_queued() == self.get_allocated_size(), ( + f"{self._written_buffer.get_total_bytes_queued()} != {self.get_allocated_size()}" ) d = self._actually_write() d.addCallback(lambda _: self._rref.callRemote("close")) @@ -384,16 +417,16 @@ class ReadBucketProxy(object): self._fieldsize = fieldsize self._fieldstruct = fieldstruct - for field in ( 'data', - 'plaintext_hash_tree', # UNUSED - 'crypttext_hash_tree', - 'block_hashes', - 'share_hashes', - 'uri_extension', - ): + for field_name in ( 'data', + 'plaintext_hash_tree', # UNUSED + 'crypttext_hash_tree', + 'block_hashes', + 'share_hashes', + 'uri_extension', + ): offset = struct.unpack(fieldstruct, data[x:x+fieldsize])[0] x += fieldsize - self._offsets[field] = offset + self._offsets[field_name] = offset return self._offsets def _get_block_data(self, unused, blocknum, blocksize, thisblocksize): From 62400d29b3819994e6c777f77cd0ec7e3ecb5def Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 23 Nov 2022 09:36:53 -0500 Subject: [PATCH 1183/2309] Seems like Ubuntu 22.04 has issues with Tor at the moment --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0327014ca..ad055da2f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -163,7 +163,7 @@ jobs: matrix: os: - windows-latest - - ubuntu-latest + - ubuntu-20.04 python-version: - 3.7 - 3.9 From 4fd92a915bb312a2e2bf4f185112570b8d32d393 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 23 Nov 2022 09:43:45 -0500 Subject: [PATCH 1184/2309] Install tor on any ubuntu version. --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ad055da2f..4e5c9a757 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -175,7 +175,7 @@ jobs: steps: - name: Install Tor [Ubuntu] - if: matrix.os == 'ubuntu-latest' + if: ${{ contains(matrix.os, 'ubuntu') }} run: sudo apt install tor # TODO: See https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3744. From 7f1d7d4f46847ea83a78d85dea649b45d78583dd Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 23 Nov 2022 09:53:07 -0500 Subject: [PATCH 1185/2309] Better explanation. --- src/allmydata/test/common_system.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/allmydata/test/common_system.py b/src/allmydata/test/common_system.py index ca2904b53..297046cc5 100644 --- a/src/allmydata/test/common_system.py +++ b/src/allmydata/test/common_system.py @@ -643,10 +643,16 @@ def _render_section_values(values): @async_to_deferred async def spin_until_cleanup_done(value=None, timeout=10): """ - At the end of the test, spin until either a timeout is hit, or the reactor - has no more DelayedCalls. + At the end of the test, spin until the reactor has no more DelayedCalls + and file descriptors (or equivalents) registered. This prevents dirty + reactor errors, while also not hard-coding a fixed amount of time, so it + can finish faster on faster computers. - Make sure to register during setUp. + There is also a timeout: if it takes more than 10 seconds (by default) for + the remaining reactor state to clean itself up, the presumption is that it + will never get cleaned up and the spinning stops. + + Make sure to run as last thing in tearDown. """ def num_fds(): if hasattr(reactor, "handles"): From 6c3e9e670de208e7b5d2dc37d192de2c3d464e80 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 23 Nov 2022 09:53:11 -0500 Subject: [PATCH 1186/2309] Link to issue. --- .github/workflows/ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4e5c9a757..99ac28926 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -163,6 +163,8 @@ jobs: matrix: os: - windows-latest + # 22.04 has some issue with Tor at the moment: + # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3943 - ubuntu-20.04 python-version: - 3.7 From d1deda5fdd00f0a9b03bda9a0c1be7e494c90089 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 23 Nov 2022 10:09:53 -0500 Subject: [PATCH 1187/2309] Unit tests for _WriteBuffer. --- src/allmydata/test/test_storage.py | 47 +++++++++++++++++++++++++----- 1 file changed, 39 insertions(+), 8 deletions(-) diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index 134609f81..f5762f616 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -3,14 +3,9 @@ Tests for allmydata.storage. Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals -from future.utils import native_str, PY2, bytes_to_native_str, bchr -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 +from __future__ import annotations +from future.utils import native_str, bytes_to_native_str, bchr from six import ensure_str from io import ( @@ -59,7 +54,7 @@ from allmydata.storage.common import storage_index_to_dir, \ si_b2a, si_a2b from allmydata.storage.lease import LeaseInfo from allmydata.immutable.layout import WriteBucketProxy, WriteBucketProxy_v2, \ - ReadBucketProxy + ReadBucketProxy, _WriteBuffer from allmydata.mutable.layout import MDMFSlotWriteProxy, MDMFSlotReadProxy, \ LayoutInvalid, MDMFSIGNABLEHEADER, \ SIGNED_PREFIX, MDMFHEADER, \ @@ -3746,3 +3741,39 @@ class LeaseInfoTests(SyncTestCase): info.to_mutable_data(), HasLength(info.mutable_size()), ) + + +class WriteBufferTests(SyncTestCase): + """Tests for ``_WriteBuffer``.""" + + @given( + small_writes=strategies.lists( + strategies.binary(min_size=1, max_size=20), + min_size=10, max_size=20), + batch_size=strategies.integers(min_value=5, max_value=10) + ) + def test_write_buffer(self, small_writes: list[bytes], batch_size: int): + """ + ``_WriteBuffer`` coalesces small writes into bigger writes based on + the batch size. + """ + offset = 0 + wb = _WriteBuffer(batch_size) + result = b"" + for data in small_writes: + should_flush = wb.queue_write(offset, data) + offset += len(data) + if should_flush: + flushed_offset, flushed_data = wb.flush() + self.assertEqual(flushed_offset, len(result)) + # The flushed data is in batch sizes, or closest approximation + # given queued inputs: + self.assertTrue(batch_size <= len(flushed_data) < batch_size + len(data)) + result += flushed_data + + # Final flush: + flushed_offset, flushed_data = wb.flush() + self.assertEqual(flushed_offset, len(result)) + result += flushed_data + + self.assertEqual(result, b"".join(small_writes)) From fd9e50adf1efcb0878a7dcc14bc0ac1d3a3c620c Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 23 Nov 2022 10:13:18 -0500 Subject: [PATCH 1188/2309] Simplify _WriteBuffer slightly. --- src/allmydata/immutable/layout.py | 10 +++------- src/allmydata/test/test_storage.py | 4 +--- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/src/allmydata/immutable/layout.py b/src/allmydata/immutable/layout.py index b9eb74d8f..cb41b0594 100644 --- a/src/allmydata/immutable/layout.py +++ b/src/allmydata/immutable/layout.py @@ -114,17 +114,12 @@ class _WriteBuffer: _to_write : BytesIO = field(factory=BytesIO) _written_bytes : int = field(default=0) - def queue_write(self, offset: int, data: bytes) -> bool: + def queue_write(self, data: bytes) -> bool: """ Queue a write. If the result is ``False``, no further action is needed for now. If the result is some ``True``, it's time to call ``flush()`` and do a real write. - - Callers of this function are expected to queue the data in order, with - no holes. As such, the offset is technically unnecessary, but is used - to check the inputs. Possibly we should get rid of it. """ - assert offset == self.get_total_bytes_queued() self._to_write.write(data) return len(self._to_write.getbuffer()) >= self._batch_size @@ -290,7 +285,8 @@ class WriteBucketProxy(object): no holes. As such, the offset is technically unnecessary, but is used to check the inputs. Possibly we should get rid of it. """ - if self._write_buffer.queue_write(offset, data): + assert offset == self._write_buffer.get_total_bytes_queued() + if self._write_buffer.queue_write(data): return self._actually_write() else: return defer.succeed(False) diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index f5762f616..820d4fd79 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -3757,12 +3757,10 @@ class WriteBufferTests(SyncTestCase): ``_WriteBuffer`` coalesces small writes into bigger writes based on the batch size. """ - offset = 0 wb = _WriteBuffer(batch_size) result = b"" for data in small_writes: - should_flush = wb.queue_write(offset, data) - offset += len(data) + should_flush = wb.queue_write(data) if should_flush: flushed_offset, flushed_data = wb.flush() self.assertEqual(flushed_offset, len(result)) From 37902802646d873e0cbcd4b508c59406d5e3967e Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 23 Nov 2022 10:15:19 -0500 Subject: [PATCH 1189/2309] Documentation. --- src/allmydata/immutable/encode.py | 2 ++ src/allmydata/immutable/layout.py | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/src/allmydata/immutable/encode.py b/src/allmydata/immutable/encode.py index 874492785..2b6602773 100644 --- a/src/allmydata/immutable/encode.py +++ b/src/allmydata/immutable/encode.py @@ -262,6 +262,8 @@ class Encoder(object): d.addCallback(lambda res: self.finish_hashing()) + # These calls have to happen in order, and waiting for previous one + # also ensures backpressure: d.addCallback(lambda res: self.send_crypttext_hash_tree_to_all_shareholders()) d.addCallback(lambda res: self.send_all_block_hash_trees()) diff --git a/src/allmydata/immutable/layout.py b/src/allmydata/immutable/layout.py index cb41b0594..562ca4470 100644 --- a/src/allmydata/immutable/layout.py +++ b/src/allmydata/immutable/layout.py @@ -138,6 +138,10 @@ class _WriteBuffer: @implementer(IStorageBucketWriter) class WriteBucketProxy(object): + """ + Note: The various ``put_`` methods need to be called in the order in which the + bytes will get written. + """ fieldsize = 4 fieldstruct = ">L" From 41533f162e4d78b2ece85a09e1cc9ecf79810f76 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 23 Nov 2022 10:20:32 -0500 Subject: [PATCH 1190/2309] Not used anymore. --- src/allmydata/test/test_pipeline.py | 198 ---------------------------- src/allmydata/util/pipeline.py | 149 --------------------- 2 files changed, 347 deletions(-) delete mode 100644 src/allmydata/test/test_pipeline.py delete mode 100644 src/allmydata/util/pipeline.py diff --git a/src/allmydata/test/test_pipeline.py b/src/allmydata/test/test_pipeline.py deleted file mode 100644 index 31d952836..000000000 --- a/src/allmydata/test/test_pipeline.py +++ /dev/null @@ -1,198 +0,0 @@ -""" -Tests for allmydata.util.pipeline. - -Ported to Python 3. -""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals - -from future.utils import PY2 -if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - -import gc - -from twisted.internet import defer -from twisted.trial import unittest -from twisted.python import log -from twisted.python.failure import Failure - -from allmydata.util import pipeline - - -class Pipeline(unittest.TestCase): - def pause(self, *args, **kwargs): - d = defer.Deferred() - self.calls.append( (d, args, kwargs) ) - return d - - def failUnlessCallsAre(self, expected): - #print(self.calls) - #print(expected) - self.failUnlessEqual(len(self.calls), len(expected), self.calls) - for i,c in enumerate(self.calls): - self.failUnlessEqual(c[1:], expected[i], str(i)) - - def test_basic(self): - self.calls = [] - finished = [] - p = pipeline.Pipeline(100) - - d = p.flush() # fires immediately - d.addCallbacks(finished.append, log.err) - self.failUnlessEqual(len(finished), 1) - finished = [] - - d = p.add(10, self.pause, "one") - # the call should start right away, and our return Deferred should - # fire right away - d.addCallbacks(finished.append, log.err) - self.failUnlessEqual(len(finished), 1) - self.failUnlessEqual(finished[0], None) - self.failUnlessCallsAre([ ( ("one",) , {} ) ]) - self.failUnlessEqual(p.gauge, 10) - - # pipeline: [one] - - finished = [] - d = p.add(20, self.pause, "two", kw=2) - # pipeline: [one, two] - - # the call and the Deferred should fire right away - d.addCallbacks(finished.append, log.err) - self.failUnlessEqual(len(finished), 1) - self.failUnlessEqual(finished[0], None) - self.failUnlessCallsAre([ ( ("one",) , {} ), - ( ("two",) , {"kw": 2} ), - ]) - self.failUnlessEqual(p.gauge, 30) - - self.calls[0][0].callback("one-result") - # pipeline: [two] - self.failUnlessEqual(p.gauge, 20) - - finished = [] - d = p.add(90, self.pause, "three", "posarg1") - # pipeline: [two, three] - flushed = [] - fd = p.flush() - fd.addCallbacks(flushed.append, log.err) - self.failUnlessEqual(flushed, []) - - # the call will be made right away, but the return Deferred will not, - # because the pipeline is now full. - d.addCallbacks(finished.append, log.err) - self.failUnlessEqual(len(finished), 0) - self.failUnlessCallsAre([ ( ("one",) , {} ), - ( ("two",) , {"kw": 2} ), - ( ("three", "posarg1"), {} ), - ]) - self.failUnlessEqual(p.gauge, 110) - - self.failUnlessRaises(pipeline.SingleFileError, p.add, 10, self.pause) - - # retiring either call will unblock the pipeline, causing the #3 - # Deferred to fire - self.calls[2][0].callback("three-result") - # pipeline: [two] - - self.failUnlessEqual(len(finished), 1) - self.failUnlessEqual(finished[0], None) - self.failUnlessEqual(flushed, []) - - # retiring call#2 will finally allow the flush() Deferred to fire - self.calls[1][0].callback("two-result") - self.failUnlessEqual(len(flushed), 1) - - def test_errors(self): - self.calls = [] - p = pipeline.Pipeline(100) - - d1 = p.add(200, self.pause, "one") - d2 = p.flush() - - finished = [] - d1.addBoth(finished.append) - self.failUnlessEqual(finished, []) - - flushed = [] - d2.addBoth(flushed.append) - self.failUnlessEqual(flushed, []) - - self.calls[0][0].errback(ValueError("oops")) - - self.failUnlessEqual(len(finished), 1) - f = finished[0] - self.failUnless(isinstance(f, Failure)) - self.failUnless(f.check(pipeline.PipelineError)) - self.failUnlessIn("PipelineError", str(f.value)) - self.failUnlessIn("ValueError", str(f.value)) - r = repr(f.value) - self.failUnless("ValueError" in r, r) - f2 = f.value.error - self.failUnless(f2.check(ValueError)) - - self.failUnlessEqual(len(flushed), 1) - f = flushed[0] - self.failUnless(isinstance(f, Failure)) - self.failUnless(f.check(pipeline.PipelineError)) - f2 = f.value.error - self.failUnless(f2.check(ValueError)) - - # now that the pipeline is in the failed state, any new calls will - # fail immediately - - d3 = p.add(20, self.pause, "two") - - finished = [] - d3.addBoth(finished.append) - self.failUnlessEqual(len(finished), 1) - f = finished[0] - self.failUnless(isinstance(f, Failure)) - self.failUnless(f.check(pipeline.PipelineError)) - r = repr(f.value) - self.failUnless("ValueError" in r, r) - f2 = f.value.error - self.failUnless(f2.check(ValueError)) - - d4 = p.flush() - flushed = [] - d4.addBoth(flushed.append) - self.failUnlessEqual(len(flushed), 1) - f = flushed[0] - self.failUnless(isinstance(f, Failure)) - self.failUnless(f.check(pipeline.PipelineError)) - f2 = f.value.error - self.failUnless(f2.check(ValueError)) - - def test_errors2(self): - self.calls = [] - p = pipeline.Pipeline(100) - - d1 = p.add(10, self.pause, "one") - d2 = p.add(20, self.pause, "two") - d3 = p.add(30, self.pause, "three") - d4 = p.flush() - - # one call fails, then the second one succeeds: make sure - # ExpandableDeferredList tolerates the second one - - flushed = [] - d4.addBoth(flushed.append) - self.failUnlessEqual(flushed, []) - - self.calls[0][0].errback(ValueError("oops")) - self.failUnlessEqual(len(flushed), 1) - f = flushed[0] - self.failUnless(isinstance(f, Failure)) - self.failUnless(f.check(pipeline.PipelineError)) - f2 = f.value.error - self.failUnless(f2.check(ValueError)) - - self.calls[1][0].callback("two-result") - self.calls[2][0].errback(ValueError("three-error")) - - del d1,d2,d3,d4 - gc.collect() # for PyPy diff --git a/src/allmydata/util/pipeline.py b/src/allmydata/util/pipeline.py deleted file mode 100644 index 31f5d5d49..000000000 --- a/src/allmydata/util/pipeline.py +++ /dev/null @@ -1,149 +0,0 @@ -""" -A pipeline of Deferreds. - -Ported to Python 3. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals - -from future.utils import PY2 -if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - -from twisted.internet import defer -from twisted.python.failure import Failure -from twisted.python import log -from allmydata.util.assertutil import precondition - - -class PipelineError(Exception): - """One of the pipelined messages returned an error. The received Failure - object is stored in my .error attribute.""" - def __init__(self, error): - self.error = error - - def __repr__(self): - return "" % (self.error,) - def __str__(self): - return "" % (self.error,) - -class SingleFileError(Exception): - """You are not permitted to add a job to a full pipeline.""" - - -class ExpandableDeferredList(defer.Deferred, object): - # like DeferredList(fireOnOneErrback=True) with a built-in - # gatherResults(), but you can add new Deferreds until you close it. This - # gives you a chance to add don't-complain-about-unhandled-error errbacks - # immediately after attachment, regardless of whether you actually end up - # wanting the list or not. - def __init__(self): - defer.Deferred.__init__(self) - self.resultsReceived = 0 - self.resultList = [] - self.failure = None - self.closed = False - - def addDeferred(self, d): - precondition(not self.closed, "don't call addDeferred() on a closed ExpandableDeferredList") - index = len(self.resultList) - self.resultList.append(None) - d.addCallbacks(self._cbDeferred, self._ebDeferred, - callbackArgs=(index,)) - return d - - def close(self): - self.closed = True - self.checkForFinished() - - def checkForFinished(self): - if not self.closed: - return - if self.called: - return - if self.failure: - self.errback(self.failure) - elif self.resultsReceived == len(self.resultList): - self.callback(self.resultList) - - def _cbDeferred(self, res, index): - self.resultList[index] = res - self.resultsReceived += 1 - self.checkForFinished() - return res - - def _ebDeferred(self, f): - self.failure = f - self.checkForFinished() - return f - - -class Pipeline(object): - """I manage a size-limited pipeline of Deferred operations, usually - callRemote() messages.""" - - def __init__(self, capacity): - self.capacity = capacity # how full we can be - self.gauge = 0 # how full we are - self.failure = None - self.waiting = [] # callers of add() who are blocked - self.unflushed = ExpandableDeferredList() - - def add(self, _size, _func, *args, **kwargs): - # We promise that all the Deferreds we return will fire in the order - # they were returned. To make it easier to keep this promise, we - # prohibit multiple outstanding calls to add() . - if self.waiting: - raise SingleFileError - if self.failure: - return defer.fail(self.failure) - self.gauge += _size - fd = defer.maybeDeferred(_func, *args, **kwargs) - fd.addBoth(self._call_finished, _size) - self.unflushed.addDeferred(fd) - fd.addErrback(self._eat_pipeline_errors) - fd.addErrback(log.err, "_eat_pipeline_errors didn't eat it") - if self.gauge < self.capacity: - return defer.succeed(None) - d = defer.Deferred() - self.waiting.append(d) - return d - - def flush(self): - if self.failure: - return defer.fail(self.failure) - d, self.unflushed = self.unflushed, ExpandableDeferredList() - d.close() - d.addErrback(self._flushed_error) - return d - - def _flushed_error(self, f): - precondition(self.failure) # should have been set by _call_finished - return self.failure - - def _call_finished(self, res, size): - self.gauge -= size - if isinstance(res, Failure): - res = Failure(PipelineError(res)) - if not self.failure: - self.failure = res - if self.failure: - while self.waiting: - d = self.waiting.pop(0) - d.errback(self.failure) - else: - while self.waiting and (self.gauge < self.capacity): - d = self.waiting.pop(0) - d.callback(None) - # the d.callback() might trigger a new call to add(), which - # will raise our gauge and might cause the pipeline to be - # filled. So the while() loop gets a chance to tell the - # caller to stop. - return res - - def _eat_pipeline_errors(self, f): - f.trap(PipelineError) - return None From 6b0fa64236fa8decc2b877163a437ff29d74052a Mon Sep 17 00:00:00 2001 From: "Fon E. Noel NFEBE" Date: Fri, 25 Nov 2022 12:15:58 +0100 Subject: [PATCH 1191/2309] Clean up test_storage.py after refactor This PR cleans up errorneous changes resulting from 1d85a2c5cf40a4e52bbb024e2edba4fcd883caa1 and adds a few improvements such as calling `super` implementations. Making sure classes with functions returning deferreds use `AsyncTestCase` Signed-off-by: Fon E. Noel NFEBE --- src/allmydata/test/test_storage.py | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index 3d35ec55f..f7d5ae919 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -436,7 +436,7 @@ class RemoteBucket(object): return defer.maybeDeferred(_call) -class BucketProxy(SyncTestCase): +class BucketProxy(AsyncTestCase): def make_bucket(self, name, size): basedir = os.path.join("storage", "BucketProxy", name) incoming = os.path.join(basedir, "tmp", "bucket") @@ -563,10 +563,7 @@ class Server(AsyncTestCase): self.sparent = LoggingServiceParent() self.sparent.startService() self._lease_secret = itertools.count() - - def tearDown(self): - super(Server, self).tearDown() - return self.sparent.stopService() + self.addCleanup(self.sparent.stopService()) def workdir(self, name): basedir = os.path.join("storage", "Server", name) @@ -1284,10 +1281,10 @@ class Server(AsyncTestCase): class MutableServer(SyncTestCase): def setUp(self): + super(MutableServer, self).setUp() self.sparent = LoggingServiceParent() self._lease_secret = itertools.count() - def tearDown(self): - return self.sparent.stopService() + self.addCleanup(self.sparent.stopService()) def workdir(self, name): basedir = os.path.join("storage", "MutableServer", name) @@ -1903,8 +1900,9 @@ class MutableServer(SyncTestCase): self.assertThat({}, Equals(read_data)) -class MDMFProxies(SyncTestCase, ShouldFailMixin): +class MDMFProxies(AsyncTestCase, ShouldFailMixin): def setUp(self): + super(MDMFProxies, self).setUp() self.sparent = LoggingServiceParent() self._lease_secret = itertools.count() self.ss = self.create("MDMFProxies storage test server") @@ -1935,6 +1933,7 @@ class MDMFProxies(SyncTestCase, ShouldFailMixin): def tearDown(self): + super(MDMFProxies, self).tearDown() self.sparent.stopService() shutil.rmtree(self.workdir("MDMFProxies storage test server")) @@ -2150,7 +2149,7 @@ class MDMFProxies(SyncTestCase, ShouldFailMixin): tws[0] = (testvs, [(0, share)], None) readv = [] results = write(storage_index, self.secrets, tws, readv) - self.assertFalse(results[0]) + self.assertTrue(results[0]) def test_read(self): @@ -2438,7 +2437,7 @@ class MDMFProxies(SyncTestCase, ShouldFailMixin): def _check_success(results): self.assertThat(results, HasLength(2)) res, d = results - self.assertFalse(results) + self.assertTrue(results) mw = self._make_new_mw(b"si1", 0) mw.set_checkstring(b"this is a lie") @@ -2918,7 +2917,7 @@ class MDMFProxies(SyncTestCase, ShouldFailMixin): d.addCallback(lambda ignored: mr.get_encprivkey()) d.addCallback(lambda encprivkey: - self.assertThat(encprivkey, self.encprivkey, Equals(encprivkey))) + self.assertThat(encprivkey, Equals(self.encprivkey), encprivkey)) d.addCallback(lambda ignored: mr.get_verification_key()) d.addCallback(lambda verification_key: @@ -3325,7 +3324,7 @@ class MDMFProxies(SyncTestCase, ShouldFailMixin): self.assertTrue(results[0]) read = self.ss.slot_readv self.assertThat(read(b"si1", [0], [(1, 8)]), - {0: [struct.pack(">Q", 1)]}) + Equals({0: [struct.pack(">Q", 1)]})) self.assertThat(read(b"si1", [0], [(9, len(data) - 9)]), Equals({0: [data[9:]]})) d.addCallback(_then_again) @@ -3335,10 +3334,10 @@ class MDMFProxies(SyncTestCase, ShouldFailMixin): class Stats(SyncTestCase): def setUp(self): + super(Stats, self).setUp() self.sparent = LoggingServiceParent() self._lease_secret = itertools.count() - def tearDown(self): - return self.sparent.stopService() + self.addCleanup(self.sparent.stopService()) def workdir(self, name): basedir = os.path.join("storage", "Server", name) @@ -3418,7 +3417,7 @@ class Stats(SyncTestCase): immutable_schemas = strategies.sampled_from(list(ALL_IMMUTABLE_SCHEMAS)) -class ShareFileTests(unittest.TestCase): +class ShareFileTests(SyncTestCase): """Tests for allmydata.storage.immutable.ShareFile.""" def get_sharefile(self, **kwargs): From 562111012e4418707ec141c8f488d36ea61325ae Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Sat, 26 Nov 2022 18:18:05 -0600 Subject: [PATCH 1192/2309] Give GITHUB_TOKEN just enough permissions to run the workflow --- .github/workflows/ci.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0327014ca..588e71747 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,6 +6,16 @@ on: - "master" pull_request: +# At the start of each workflow run, GitHub creates a unique +# GITHUB_TOKEN secret to use in the workflow. It is a good idea for +# this GITHUB_TOKEN to have the minimum of permissions. See: +# +# - https://docs.github.com/en/actions/security-guides/automatic-token-authentication +# - https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#permissions +# +permissions: + contents: read + # Control to what degree jobs in this workflow will run concurrently with # other instances of themselves. # From 9bd384ac2db0199c446ebcffefffb01cccf1e2de Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Sat, 26 Nov 2022 18:18:44 -0600 Subject: [PATCH 1193/2309] Add news fragment --- newsfragments/3944.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3944.minor diff --git a/newsfragments/3944.minor b/newsfragments/3944.minor new file mode 100644 index 000000000..e69de29bb From 5e6189e1159432e30b55340a9230d1ea317971ce Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Sat, 26 Nov 2022 18:25:19 -0600 Subject: [PATCH 1194/2309] Use newer version of actions/setup-python --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 588e71747..bd757fe08 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -73,7 +73,7 @@ jobs: fetch-depth: 0 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} @@ -208,7 +208,7 @@ jobs: fetch-depth: 0 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} @@ -268,7 +268,7 @@ jobs: fetch-depth: 0 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} From 23d8d1cb01682a13ad788bdf832513c1cddc63ed Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Sat, 26 Nov 2022 18:28:57 -0600 Subject: [PATCH 1195/2309] Use action/setup-python@v4's caching feature --- .github/workflows/ci.yml | 48 +++------------------------------------- 1 file changed, 3 insertions(+), 45 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bd757fe08..6c608e888 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -76,25 +76,7 @@ jobs: uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} - - # To use pip caching with GitHub Actions in an OS-independent - # manner, we need `pip cache dir` command, which became - # available since pip v20.1+. At the time of writing this, - # GitHub Actions offers pip v20.3.3 for both ubuntu-latest and - # windows-latest, and pip v20.3.1 for macos-latest. - - name: Get pip cache directory - id: pip-cache - run: | - echo "::set-output name=dir::$(pip cache dir)" - - # See https://github.com/actions/cache - - name: Use pip cache - uses: actions/cache@v2 - with: - path: ${{ steps.pip-cache.outputs.dir }} - key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }} - restore-keys: | - ${{ runner.os }}-pip- + cache: 'pip' # caching pip dependencies - name: Install Python packages run: | @@ -211,19 +193,7 @@ jobs: uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} - - - name: Get pip cache directory - id: pip-cache - run: | - echo "::set-output name=dir::$(pip cache dir)" - - - name: Use pip cache - uses: actions/cache@v2 - with: - path: ${{ steps.pip-cache.outputs.dir }} - key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }} - restore-keys: | - ${{ runner.os }}-pip- + cache: 'pip' # caching pip dependencies - name: Install Python packages run: | @@ -271,19 +241,7 @@ jobs: uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} - - - name: Get pip cache directory - id: pip-cache - run: | - echo "::set-output name=dir::$(pip cache dir)" - - - name: Use pip cache - uses: actions/cache@v2 - with: - path: ${{ steps.pip-cache.outputs.dir }} - key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }} - restore-keys: | - ${{ runner.os }}-pip- + cache: 'pip' # caching pip dependencies - name: Install Python packages run: | From 15881da348dfa9c9f92836f59175e3582fdab8cb Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Sat, 26 Nov 2022 18:37:46 -0600 Subject: [PATCH 1196/2309] Use newer version of actions/checkout --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6c608e888..4447e539c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -68,7 +68,7 @@ jobs: # See https://github.com/actions/checkout. A fetch-depth of 0 # fetches all tags and branches. - name: Check out Tahoe-LAFS sources - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: fetch-depth: 0 @@ -185,7 +185,7 @@ jobs: args: install tor - name: Check out Tahoe-LAFS sources - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: fetch-depth: 0 @@ -233,7 +233,7 @@ jobs: steps: - name: Check out Tahoe-LAFS sources - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: fetch-depth: 0 From 26d30979c0fc3345c78846aaf37db1a7f83610eb Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Sat, 26 Nov 2022 18:38:48 -0600 Subject: [PATCH 1197/2309] Use newer version of actions/upload-artifact --- .github/workflows/ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4447e539c..64a60bd04 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -90,13 +90,13 @@ jobs: run: python -m tox - name: Upload eliot.log - uses: actions/upload-artifact@v1 + uses: actions/upload-artifact@v3 with: name: eliot.log path: eliot.log - name: Upload trial log - uses: actions/upload-artifact@v1 + uses: actions/upload-artifact@v3 with: name: test.log path: _trial_temp/test.log @@ -212,7 +212,7 @@ jobs: run: tox -e integration - name: Upload eliot.log in case of failure - uses: actions/upload-artifact@v1 + uses: actions/upload-artifact@v3 if: failure() with: name: integration.eliot.json @@ -259,7 +259,7 @@ jobs: run: dist/Tahoe-LAFS/tahoe --version - name: Upload PyInstaller package - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 with: name: Tahoe-LAFS-${{ matrix.os }}-Python-${{ matrix.python-version }} path: dist/Tahoe-LAFS-*-*.* From 7715972429c34d4c6a684f184ab5f4ba1613df16 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Sat, 26 Nov 2022 18:40:19 -0600 Subject: [PATCH 1198/2309] Use newer version of crazy-max/ghaction-chocolatey --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 64a60bd04..169e981ed 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -180,7 +180,7 @@ jobs: - name: Install Tor [Windows] if: matrix.os == 'windows-latest' - uses: crazy-max/ghaction-chocolatey@v1 + uses: crazy-max/ghaction-chocolatey@v2 with: args: install tor From 2ab8e3e8d20087521dc4aa7ffb358e3f65a7a6aa Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 28 Nov 2022 10:02:56 -0500 Subject: [PATCH 1199/2309] Cancel timeout on failures too. --- src/allmydata/storage/http_client.py | 7 ++++++- src/allmydata/test/test_storage_http.py | 24 ++++++++++++++++++++++++ 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 5b4ec9db8..73fba9888 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -169,7 +169,12 @@ def limited_content( collector.f.seek(0) return collector.f - return d.addCallback(done) + def failed(f): + if timeout.active(): + timeout.cancel() + return f + + return d.addCallbacks(done, failed) def _decode_cbor(response, schema: Schema, clock: IReactorTime): diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 4f7174c06..8dbe18545 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -280,6 +280,14 @@ class TestApp(object): self.clock.callLater(59 + 59, request.write, b"c") return Deferred() + @_authorized_route(_app, set(), "/die_unfinished", methods=["GET"]) + def die(self, request, authorization): + """ + Dies half-way. + """ + request.transport.loseConnection() + return Deferred() + def result_of(d): """ @@ -423,6 +431,22 @@ class CustomHTTPServerTests(SyncTestCase): with self.assertRaises(CancelledError): error[0].raiseException() + def test_limited_content_cancels_timeout_on_failed_response(self): + """ + If the response fails somehow, the timeout is still cancelled. + """ + response = result_of( + self.client.request( + "GET", + "http://127.0.0.1/die", + ) + ) + + d = limited_content(response, self._http_server.clock, 4) + with self.assertRaises(ValueError): + result_of(d) + self.assertEqual(len(self._http_server.clock.getDelayedCalls()), 0) + class HttpTestFixture(Fixture): """ From 38d7430c570fd3ff9b2b3ea720706d6d3198fbfa Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 28 Nov 2022 10:03:42 -0500 Subject: [PATCH 1200/2309] Simplify. --- src/allmydata/storage/http_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 73fba9888..5abc44bdd 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -382,7 +382,7 @@ class StorageClient(object): write_enabler_secret=None, headers=None, message_to_serialize=None, - timeout: Union[int, float] = 60, + timeout: float = 60, **kwargs, ): """ From 0f4dc9129538dbbe8b88073c3a5047462f4209a2 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 28 Nov 2022 10:12:08 -0500 Subject: [PATCH 1201/2309] Refactor so internal attributes needn't leak. --- src/allmydata/storage/http_client.py | 63 +++++++++++++--------------- 1 file changed, 30 insertions(+), 33 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 5abc44bdd..79bf061c9 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -177,26 +177,6 @@ def limited_content( return d.addCallbacks(done, failed) -def _decode_cbor(response, schema: Schema, clock: IReactorTime): - """Given HTTP response, return decoded CBOR body.""" - - def got_content(f: BinaryIO): - data = f.read() - schema.validate_cbor(data) - return loads(data) - - if response.code > 199 and response.code < 300: - content_type = get_content_type(response.headers) - if content_type == CBOR_MIME_TYPE: - return limited_content(response, clock).addCallback(got_content) - else: - raise ClientException(-1, "Server didn't send CBOR") - else: - return treq.content(response).addCallback( - lambda data: fail(ClientException(response.code, response.phrase, data)) - ) - - @define class ImmutableCreateResult(object): """Result of creating a storage index for an immutable.""" @@ -428,6 +408,25 @@ class StorageClient(object): method, url, headers=headers, timeout=timeout, **kwargs ) + def decode_cbor(self, response, schema: Schema): + """Given HTTP response, return decoded CBOR body.""" + + def got_content(f: BinaryIO): + data = f.read() + schema.validate_cbor(data) + return loads(data) + + if response.code > 199 and response.code < 300: + content_type = get_content_type(response.headers) + if content_type == CBOR_MIME_TYPE: + return limited_content(response, self._clock).addCallback(got_content) + else: + raise ClientException(-1, "Server didn't send CBOR") + else: + return treq.content(response).addCallback( + lambda data: fail(ClientException(response.code, response.phrase, data)) + ) + @define(hash=True) class StorageClientGeneral(object): @@ -444,8 +443,8 @@ class StorageClientGeneral(object): """ url = self._client.relative_url("/storage/v1/version") response = yield self._client.request("GET", url) - decoded_response = yield _decode_cbor( - response, _SCHEMAS["get_version"], self._client._clock + decoded_response = yield self._client.decode_cbor( + response, _SCHEMAS["get_version"] ) returnValue(decoded_response) @@ -634,8 +633,8 @@ class StorageClientImmutables(object): upload_secret=upload_secret, message_to_serialize=message, ) - decoded_response = yield _decode_cbor( - response, _SCHEMAS["allocate_buckets"], self._client._clock + decoded_response = yield self._client.decode_cbor( + response, _SCHEMAS["allocate_buckets"] ) returnValue( ImmutableCreateResult( @@ -712,8 +711,8 @@ class StorageClientImmutables(object): raise ClientException( response.code, ) - body = yield _decode_cbor( - response, _SCHEMAS["immutable_write_share_chunk"], self._client._clock + body = yield self._client.decode_cbor( + response, _SCHEMAS["immutable_write_share_chunk"] ) remaining = RangeMap() for chunk in body["required"]: @@ -743,9 +742,7 @@ class StorageClientImmutables(object): url, ) if response.code == http.OK: - body = yield _decode_cbor( - response, _SCHEMAS["list_shares"], self._client._clock - ) + body = yield self._client.decode_cbor(response, _SCHEMAS["list_shares"]) returnValue(set(body)) else: raise ClientException(response.code) @@ -862,8 +859,8 @@ class StorageClientMutables: message_to_serialize=message, ) if response.code == http.OK: - result = await _decode_cbor( - response, _SCHEMAS["mutable_read_test_write"], self._client._clock + result = await self._client.decode_cbor( + response, _SCHEMAS["mutable_read_test_write"] ) return ReadTestWriteResult(success=result["success"], reads=result["data"]) else: @@ -893,8 +890,8 @@ class StorageClientMutables: ) response = await self._client.request("GET", url) if response.code == http.OK: - return await _decode_cbor( - response, _SCHEMAS["mutable_list_shares"], self._client._clock + return await self._client.decode_cbor( + response, _SCHEMAS["mutable_list_shares"] ) else: raise ClientException(response.code) From 3ba166c2cb939a58fdb16dad06cd0dbd1ad39961 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 28 Nov 2022 10:20:12 -0500 Subject: [PATCH 1202/2309] A bit more robust code. --- src/allmydata/test/common_system.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/allmydata/test/common_system.py b/src/allmydata/test/common_system.py index 297046cc5..01966824a 100644 --- a/src/allmydata/test/common_system.py +++ b/src/allmydata/test/common_system.py @@ -659,10 +659,11 @@ async def spin_until_cleanup_done(value=None, timeout=10): # IOCP! return len(reactor.handles) else: - # Normal reactor - return len([r for r in reactor.getReaders() - if r.__class__.__name__ not in ("_UnixWaker", "_SIGCHLDWaker", "_SocketWaker")] - ) + len(reactor.getWriters()) + # Normal reactor; having internal readers still registered is fine, + # that's not our code. + return len( + set(reactor.getReaders()) - set(reactor._internalReaders) + ) + len(reactor.getWriters()) for i in range(timeout * 1000): # There's a single DelayedCall for AsynchronousDeferredRunTest's From aa80c9ef4748cf10e3b448b298df8b589c35cafd Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 28 Nov 2022 10:21:59 -0500 Subject: [PATCH 1203/2309] Be more robust. --- src/allmydata/test/test_storage_https.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/test_storage_https.py b/src/allmydata/test/test_storage_https.py index 284c8cda8..a11b0eed5 100644 --- a/src/allmydata/test/test_storage_https.py +++ b/src/allmydata/test/test_storage_https.py @@ -12,6 +12,7 @@ from cryptography import x509 from twisted.internet.endpoints import serverFromString from twisted.internet import reactor +from twisted.internet.defer import maybeDeferred from twisted.web.server import Site from twisted.web.static import Data from twisted.web.client import Agent, HTTPConnectionPool, ResponseNeverReceived @@ -88,8 +89,8 @@ class PinningHTTPSValidation(AsyncTestCase): return AsyncTestCase.setUp(self) def tearDown(self): - AsyncTestCase.tearDown(self) - return spin_until_cleanup_done() + d = maybeDeferred(AsyncTestCase.tearDown, self) + return d.addCallback(lambda _: spin_until_cleanup_done()) @asynccontextmanager async def listen(self, private_key_path: FilePath, cert_path: FilePath): From c6fc82665c3fcf4dba9809be4405daba33f0d66c Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 29 Nov 2022 09:33:05 -0500 Subject: [PATCH 1204/2309] Pull `_make_storage_system` into a free function for easier testing --- src/allmydata/storage_client.py | 153 ++++++++++++++++++-------------- 1 file changed, 85 insertions(+), 68 deletions(-) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 410bfd28b..5dc4beb22 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -33,7 +33,7 @@ Ported to Python 3. from __future__ import annotations from six import ensure_text -from typing import Union +from typing import Union, Callable, Optional import re, time, hashlib from os import urandom from configparser import NoSectionError @@ -57,6 +57,7 @@ from twisted.plugin import ( from eliot import ( log_call, ) +from foolscap.ipb import IRemoteReference from foolscap.api import eventually, RemoteException from foolscap.reconnector import ( ReconnectionInfo, @@ -80,6 +81,9 @@ from allmydata.storage.http_client import ( ClientException as HTTPClientException, StorageClientMutables, ReadVector, TestWriteVectors, WriteVector, TestVector, ClientException ) +from .node import _Config + +_log = Logger() ANONYMOUS_STORAGE_NURLS = "anonymous-storage-NURLs" @@ -732,6 +736,85 @@ def _available_space_from_version(version): return available_space +def _make_storage_system( + node_config: _Config, + config: StorageClientConfig, + ann: dict, + server_id: bytes, + get_rref: Callable[[], Optional[IRemoteReference]], +) -> IFoolscapStorageServer: + """ + Create an object for interacting with the storage server described by + the given announcement. + + :param node_config: The node configuration to pass to any configured + storage plugins. + + :param config: Configuration specifying desired storage client behavior. + + :param ann: The storage announcement from the storage server we are meant + to communicate with. + + :param server_id: The unique identifier for the server. + + :param get_rref: A function which returns a remote reference to the + server-side object which implements this storage system, if one is + available (otherwise None). + + :return: An object enabling communication via Foolscap with the server + which generated the announcement. + """ + # Try to match the announcement against a plugin. + try: + furl, storage_server = _storage_from_foolscap_plugin( + node_config, + config, + ann, + # Pass in an accessor for our _rref attribute. The value of + # the attribute may change over time as connections are lost + # and re-established. The _StorageServer should always be + # able to get the most up-to-date value. + get_rref, + ) + except AnnouncementNotMatched as e: + _log.error( + 'No plugin for storage-server "{nickname}" from plugins: {plugins}', + nickname=ann.get("nickname", ""), + plugins=e.args[0], + ) + except MissingPlugin as e: + _log.failure("Missing plugin") + ns = _NullStorage() + ns.longname = ''.format(e.args[0]) + return ns + else: + return _FoolscapStorage.from_announcement( + server_id, + furl, + ann, + storage_server, + ) + + # Try to match the announcement against the anonymous access scheme. + try: + furl = ann[u"anonymous-storage-FURL"] + except KeyError: + # Nope + pass + else: + # See comment above for the _storage_from_foolscap_plugin case + # about passing in get_rref. + storage_server = _StorageServer(get_rref=get_rref) + return _FoolscapStorage.from_announcement( + server_id, + furl, + ann, + storage_server, + ) + + # Nothing matched so we can't talk to this server. + return _null_storage + @implementer(IServer) class NativeStorageServer(service.MultiService): """I hold information about a storage server that we want to connect to. @@ -758,7 +841,6 @@ class NativeStorageServer(service.MultiService): }), "application-version": "unknown: no get_version()", }) - log = Logger() def __init__(self, server_id, ann, tub_maker, handler_overrides, node_config, config=StorageClientConfig()): service.MultiService.__init__(self) @@ -768,7 +850,7 @@ class NativeStorageServer(service.MultiService): self._tub_maker = tub_maker self._handler_overrides = handler_overrides - self._storage = self._make_storage_system(node_config, config, ann) + self._storage = _make_storage_system(node_config, config, ann, self._server_id, self.get_rref) self.last_connect_time = None self.last_loss_time = None @@ -778,71 +860,6 @@ class NativeStorageServer(service.MultiService): self._trigger_cb = None self._on_status_changed = ObserverList() - def _make_storage_system(self, node_config, config, ann): - """ - :param allmydata.node._Config node_config: The node configuration to pass - to any configured storage plugins. - - :param StorageClientConfig config: Configuration specifying desired - storage client behavior. - - :param dict ann: The storage announcement from the storage server we - are meant to communicate with. - - :return IFoolscapStorageServer: An object enabling communication via - Foolscap with the server which generated the announcement. - """ - # Try to match the announcement against a plugin. - try: - furl, storage_server = _storage_from_foolscap_plugin( - node_config, - config, - ann, - # Pass in an accessor for our _rref attribute. The value of - # the attribute may change over time as connections are lost - # and re-established. The _StorageServer should always be - # able to get the most up-to-date value. - self.get_rref, - ) - except AnnouncementNotMatched as e: - self.log.error( - 'No plugin for storage-server "{nickname}" from plugins: {plugins}', - nickname=ann.get("nickname", ""), - plugins=e.args[0], - ) - except MissingPlugin as e: - self.log.failure("Missing plugin") - ns = _NullStorage() - ns.longname = ''.format(e.args[0]) - return ns - else: - return _FoolscapStorage.from_announcement( - self._server_id, - furl, - ann, - storage_server, - ) - - # Try to match the announcement against the anonymous access scheme. - try: - furl = ann[u"anonymous-storage-FURL"] - except KeyError: - # Nope - pass - else: - # See comment above for the _storage_from_foolscap_plugin case - # about passing in get_rref. - storage_server = _StorageServer(get_rref=self.get_rref) - return _FoolscapStorage.from_announcement( - self._server_id, - furl, - ann, - storage_server, - ) - - # Nothing matched so we can't talk to this server. - return _null_storage - def get_permutation_seed(self): return self._storage.permutation_seed def get_name(self): # keep methodname short From c4c9d1389ef10236227d5a58927cebbb0907c3a1 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 29 Nov 2022 09:47:10 -0500 Subject: [PATCH 1205/2309] Try (but fail) to demonstrate the longname behavior --- setup.py | 2 +- src/allmydata/storage_client.py | 8 ++++---- src/allmydata/test/test_storage_client.py | 20 +++++++++++++++++--- 3 files changed, 22 insertions(+), 8 deletions(-) diff --git a/setup.py b/setup.py index 78f7042a9..480cb0d88 100644 --- a/setup.py +++ b/setup.py @@ -111,7 +111,7 @@ install_requires = [ "pyrsistent", # A great way to define types of values. - "attrs >= 18.2.0", + "attrs >= 20.1.0", # WebSocket library for twisted and asyncio "autobahn >= 22.4.3", diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 5dc4beb22..2ecd3bc0c 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -39,6 +39,7 @@ from os import urandom from configparser import NoSectionError import attr +from attr import define from hyperlink import DecodedURL from zope.interface import ( Attribute, @@ -637,6 +638,7 @@ class _FoolscapStorage(object): @implementer(IFoolscapStorageServer) +@define class _NullStorage(object): """ Abstraction for *not* communicating with a storage server of a type with @@ -650,7 +652,7 @@ class _NullStorage(object): lease_seed = hashlib.sha256(b"").digest() name = "" - longname = "" + longname: str = "" def connect_to(self, tub, got_connection): return NonReconnector() @@ -784,9 +786,7 @@ def _make_storage_system( ) except MissingPlugin as e: _log.failure("Missing plugin") - ns = _NullStorage() - ns.longname = ''.format(e.args[0]) - return ns + return _NullStorage(''.format(e.args[0])) else: return _FoolscapStorage.from_announcement( server_id, diff --git a/src/allmydata/test/test_storage_client.py b/src/allmydata/test/test_storage_client.py index 1a84f35ec..0c05be2e6 100644 --- a/src/allmydata/test/test_storage_client.py +++ b/src/allmydata/test/test_storage_client.py @@ -159,7 +159,7 @@ class GetConnectionStatus(unittest.TestCase): self.assertTrue(IConnectionStatus.providedBy(connection_status)) -class UnrecognizedAnnouncement(unittest.TestCase): +class UnrecognizedAnnouncement(SyncTestCase): """ Tests for handling of announcements that aren't recognized and don't use *anonymous-storage-FURL*. @@ -169,9 +169,14 @@ class UnrecognizedAnnouncement(unittest.TestCase): an announcement generated by a storage server plugin which is not loaded in the client. """ + plugin_name = u"tahoe-lafs-testing-v1" ann = { - u"name": u"tahoe-lafs-testing-v1", - u"any-parameter": 12345, + u"storage-options": [ + { + u"name": plugin_name, + u"any-parameter": 12345, + }, + ], } server_id = b"abc" @@ -234,6 +239,15 @@ class UnrecognizedAnnouncement(unittest.TestCase): server.get_foolscap_write_enabler_seed() server.get_nickname() + def test_longname(self) -> None: + """ + ``NativeStorageServer.get_longname`` describes the missing plugin. + """ + server = self.native_storage_server() + self.assertThat( + server.get_longname(), + Equals(''.format(self.plugin_name)), + ) class PluginMatchedAnnouncement(SyncTestCase): From f5b24d51e909d4e5bc5a836fb2970ee025faf66f Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 29 Nov 2022 10:14:08 -0500 Subject: [PATCH 1206/2309] Add a test for missing Authorization --- src/allmydata/test/test_storage_http.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 8dbe18545..de60812e3 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -37,6 +37,7 @@ from twisted.web import http from twisted.web.http_headers import Headers from werkzeug import routing from werkzeug.exceptions import NotFound as WNotFound +from testtools.matchers import Equals from .common import SyncTestCase from ..storage.http_common import get_content_type, CBOR_MIME_TYPE @@ -555,6 +556,20 @@ class GenericHTTPAPITests(SyncTestCase): super(GenericHTTPAPITests, self).setUp() self.http = self.useFixture(HttpTestFixture()) + def test_missing_authentication(self) -> None: + """ + If nothing is given in the ``Authorization`` header at all an + ``Unauthorized`` response is returned. + """ + client = StubTreq(self.http.http_server.get_resource()) + response = self.http.result_of_with_flush( + client.request( + "GET", + "http://127.0.0.1/storage/v1/version", + ), + ) + self.assertThat(response.code, Equals(http.UNAUTHORIZED)) + def test_bad_authentication(self): """ If the wrong swissnum is used, an ``Unauthorized`` response code is From 920467dcea958dee101eec55e6cb67d7118e11ac Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 29 Nov 2022 10:19:01 -0500 Subject: [PATCH 1207/2309] Treat missing Authorization as the same as empty Authorization --- src/allmydata/storage/http_server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 3902976ba..96a491882 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -100,7 +100,7 @@ def _authorization_decorator(required_secrets): @wraps(f) def route(self, request, *args, **kwargs): if not timing_safe_compare( - request.requestHeaders.getRawHeaders("Authorization", [None])[0].encode( + request.requestHeaders.getRawHeaders("Authorization", [""])[0].encode( "utf-8" ), swissnum_auth_header(self._swissnum), From 57f13a2472c4fef1e99ffc4b8522a88d4be3c14c Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 29 Nov 2022 10:20:13 -0500 Subject: [PATCH 1208/2309] news fragment --- newsfragments/3942.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3942.minor diff --git a/newsfragments/3942.minor b/newsfragments/3942.minor new file mode 100644 index 000000000..e69de29bb From d7fe25f7c7ffc1e8eb9da45755085c5dd04c25d8 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 29 Nov 2022 10:49:20 -0500 Subject: [PATCH 1209/2309] Correct the assertion about how "not found" should be handled MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Behavior verified visually against a live client node: ``` ❯ curl -v 'http://localhost:3456/uri/URI:CHK:cmtcxq7hwxvfxan34yiev6ivhy:qvcekmjtoetdcw4kmi7b3rtblvgx7544crnwaqtiewemdliqsokq:1:1:1' * Trying 127.0.0.1:3456... * Connected to localhost (127.0.0.1) port 3456 (#0) > GET /uri/URI:CHK:cmtcxq7hwxvfxan34yiev6ivhy:qvcekmjtoetdcw4kmi7b3rtblvgx7544crnwaqtiewemdliqsokq:1:1:1 HTTP/1.1 > Host: localhost:3456 > User-Agent: curl/7.83.1 > Accept: */* > * Mark bundle as not supporting multiuse < HTTP/1.1 410 Gone < X-Frame-Options: DENY < Referrer-Policy: no-referrer < Server: TwistedWeb/22.10.0 < Date: Tue, 29 Nov 2022 15:39:47 GMT < Content-Type: text/plain;charset=utf-8 < Accept-Ranges: bytes < Content-Length: 294 < ETag: ui2tnwl5lltj5clzpyff42jdce- < NoSharesError: no shares could be found. Zero shares usually indicates a corrupt URI, or that no servers were connected, but it might also indicate severe corruption. You should perform a filecheck on this object to learn more. The full error message is: * Connection #0 to host localhost left intact no shares (need 1). Last failure: None ``` --- src/allmydata/test/test_testing.py | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/src/allmydata/test/test_testing.py b/src/allmydata/test/test_testing.py index 3715d1aca..07bebb7a1 100644 --- a/src/allmydata/test/test_testing.py +++ b/src/allmydata/test/test_testing.py @@ -9,18 +9,7 @@ """ Tests for the allmydata.testing helpers - -Ported to Python 3. - """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals - -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from twisted.internet.defer import ( inlineCallbacks, @@ -56,10 +45,12 @@ from testtools.matchers import ( IsInstance, MatchesStructure, AfterPreprocessing, + Contains, ) from testtools.twistedsupport import ( succeeded, ) +from twisted.web.http import GONE class FakeWebTest(SyncTestCase): @@ -144,7 +135,8 @@ class FakeWebTest(SyncTestCase): def test_download_missing(self): """ - Error if we download a capability that doesn't exist + The response to a request to download a capability that doesn't exist + is 410 (GONE). """ http_client = create_tahoe_treq_client() @@ -157,7 +149,11 @@ class FakeWebTest(SyncTestCase): resp, succeeded( MatchesStructure( - code=Equals(500) + code=Equals(GONE), + content=AfterPreprocessing( + lambda m: m(), + succeeded(Contains(b"No data for")), + ), ) ) ) From 02aeb68f1731c58e146ccefd1d8ea99ad364b73d Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 29 Nov 2022 10:51:07 -0500 Subject: [PATCH 1210/2309] Take care with str vs bytes in the implementation Also replace the intentional BAD_REQUEST with GONE for this case. --- src/allmydata/testing/web.py | 24 +++++++----------------- 1 file changed, 7 insertions(+), 17 deletions(-) diff --git a/src/allmydata/testing/web.py b/src/allmydata/testing/web.py index bb858b555..6538dc3a4 100644 --- a/src/allmydata/testing/web.py +++ b/src/allmydata/testing/web.py @@ -6,22 +6,13 @@ # This file is part of Tahoe-LAFS. # # See the docs/about.rst file for licensing information. -"""Test-helpers for clients that use the WebUI. - -Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals - -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - +Test-helpers for clients that use the WebUI. +""" import hashlib +from typing import Dict import attr @@ -147,7 +138,7 @@ class _FakeTahoeUriHandler(Resource, object): isLeaf = True - data = attr.ib(default=attr.Factory(dict)) + data: Dict[bytes, bytes] = attr.ib(default=attr.Factory(dict)) capability_generators = attr.ib(default=attr.Factory(dict)) def _generate_capability(self, kind): @@ -209,7 +200,7 @@ class _FakeTahoeUriHandler(Resource, object): capability = None for arg, value in uri.query: if arg == u"uri": - capability = value + capability = value.encode("utf-8") # it's legal to use the form "/uri/" if capability is None and request.postpath and request.postpath[0]: capability = request.postpath[0] @@ -221,10 +212,9 @@ class _FakeTahoeUriHandler(Resource, object): # the user gave us a capability; if our Grid doesn't have any # data for it, that's an error. - capability = capability.encode('ascii') if capability not in self.data: - request.setResponseCode(http.BAD_REQUEST) - return u"No data for '{}'".format(capability.decode('ascii')) + request.setResponseCode(http.GONE) + return u"No data for '{}'".format(capability.decode('ascii')).encode("utf-8") return self.data[capability] From 6c0e5f5807cf72d38b9fb4a9461d0fd085920360 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 29 Nov 2022 10:52:02 -0500 Subject: [PATCH 1211/2309] news fragment --- newsfragments/3874.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3874.minor diff --git a/newsfragments/3874.minor b/newsfragments/3874.minor new file mode 100644 index 000000000..e69de29bb From 4367e5a0fcfd5c905195b741eec727eb2416096d Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 30 Nov 2022 09:28:58 -0500 Subject: [PATCH 1212/2309] Bump the Twisted dependency so we can do this --- setup.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 768e44e29..a3b3d5b98 100644 --- a/setup.py +++ b/setup.py @@ -96,7 +96,9 @@ install_requires = [ # an sftp extra in Tahoe-LAFS, there is no point in having one. # * Twisted 19.10 introduces Site.getContentFile which we use to get # temporary upload files placed into a per-node temporary directory. - "Twisted[tls,conch] >= 19.10.0", + # * Twisted 22.8.0 added support for coroutine-returning functions in many + # places (mainly via `maybeDeferred`) + "Twisted[tls,conch] >= 22.8.0", "PyYAML >= 3.11", From 5cebe91406c5d9db2c4b5ce150f85a3fd50322e7 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 30 Nov 2022 09:29:57 -0500 Subject: [PATCH 1213/2309] update the module docstring --- src/allmydata/test/mutable/test_version.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/allmydata/test/mutable/test_version.py b/src/allmydata/test/mutable/test_version.py index d5c44f204..aa6fb539f 100644 --- a/src/allmydata/test/mutable/test_version.py +++ b/src/allmydata/test/mutable/test_version.py @@ -1,5 +1,6 @@ """ -Ported to Python 3. +Tests related to the way ``allmydata.mutable`` handles different versions +of data for an object. """ from __future__ import print_function from __future__ import absolute_import From 1acf8604eff5227ed372b81eac20bc08677a853a Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 30 Nov 2022 09:30:08 -0500 Subject: [PATCH 1214/2309] Remove the Py2/Py3 compatibility header --- src/allmydata/test/mutable/test_version.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/src/allmydata/test/mutable/test_version.py b/src/allmydata/test/mutable/test_version.py index aa6fb539f..669baa8db 100644 --- a/src/allmydata/test/mutable/test_version.py +++ b/src/allmydata/test/mutable/test_version.py @@ -2,17 +2,9 @@ Tests related to the way ``allmydata.mutable`` handles different versions of data for an object. """ -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division -from __future__ import unicode_literals - -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 +from io import StringIO import os -from six.moves import cStringIO as StringIO from twisted.internet import defer from ..common import AsyncTestCase From a11eeaf240d1fde831e571ad5b5df3ebeed97168 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 30 Nov 2022 09:30:37 -0500 Subject: [PATCH 1215/2309] Convert all of the asynchronous functions to use `async` and `await` --- src/allmydata/test/mutable/test_version.py | 546 +++++++++------------ 1 file changed, 228 insertions(+), 318 deletions(-) diff --git a/src/allmydata/test/mutable/test_version.py b/src/allmydata/test/mutable/test_version.py index 669baa8db..d14cc9295 100644 --- a/src/allmydata/test/mutable/test_version.py +++ b/src/allmydata/test/mutable/test_version.py @@ -5,8 +5,8 @@ of data for an object. from io import StringIO import os +from typing import Optional -from twisted.internet import defer from ..common import AsyncTestCase from testtools.matchers import ( Equals, @@ -40,343 +40,269 @@ class Version(GridTestMixin, AsyncTestCase, testutil.ShouldFailMixin, \ self.small_data = b"test data" * 10 # 90 B; SDMF - def do_upload_mdmf(self, data=None): + async def do_upload_mdmf(self, data: Optional[bytes] = None) -> MutableFileNode: if data is None: data = self.data - d = self.nm.create_mutable_file(MutableData(data), - version=MDMF_VERSION) - def _then(n): - self.assertThat(n, IsInstance(MutableFileNode)) - self.assertThat(n._protocol_version, Equals(MDMF_VERSION)) - self.mdmf_node = n - return n - d.addCallback(_then) - return d + n = await self.nm.create_mutable_file(MutableData(data), + version=MDMF_VERSION) + self.assertThat(n, IsInstance(MutableFileNode)) + self.assertThat(n._protocol_version, Equals(MDMF_VERSION)) + self.mdmf_node = n + return n - def do_upload_sdmf(self, data=None): + async def do_upload_sdmf(self, data: Optional[bytes] = None) -> MutableFileNode: if data is None: data = self.small_data - d = self.nm.create_mutable_file(MutableData(data)) - def _then(n): - self.assertThat(n, IsInstance(MutableFileNode)) - self.assertThat(n._protocol_version, Equals(SDMF_VERSION)) - self.sdmf_node = n - return n - d.addCallback(_then) - return d + n = await self.nm.create_mutable_file(MutableData(data)) + self.assertThat(n, IsInstance(MutableFileNode)) + self.assertThat(n._protocol_version, Equals(SDMF_VERSION)) + self.sdmf_node = n + return n - def do_upload_empty_sdmf(self): - d = self.nm.create_mutable_file(MutableData(b"")) - def _then(n): - self.assertThat(n, IsInstance(MutableFileNode)) - self.sdmf_zero_length_node = n - self.assertThat(n._protocol_version, Equals(SDMF_VERSION)) - return n - d.addCallback(_then) - return d + async def do_upload_empty_sdmf(self) -> MutableFileNode: + n = await self.nm.create_mutable_file(MutableData(b"")) + self.assertThat(n, IsInstance(MutableFileNode)) + self.sdmf_zero_length_node = n + self.assertThat(n._protocol_version, Equals(SDMF_VERSION)) + return n - def do_upload(self): - d = self.do_upload_mdmf() - d.addCallback(lambda ign: self.do_upload_sdmf()) - return d + async def do_upload(self) -> MutableFileNode: + await self.do_upload_mdmf() + return await self.do_upload_sdmf() - def test_debug(self): - d = self.do_upload_mdmf() - def _debug(n): - fso = debug.FindSharesOptions() - storage_index = base32.b2a(n.get_storage_index()) - fso.si_s = str(storage_index, "utf-8") # command-line options are unicode on Python 3 - fso.nodedirs = [os.path.dirname(abspath_expanduser_unicode(str(storedir))) - for (i,ss,storedir) - in self.iterate_servers()] - fso.stdout = StringIO() - fso.stderr = StringIO() - debug.find_shares(fso) - sharefiles = fso.stdout.getvalue().splitlines() - expected = self.nm.default_encoding_parameters["n"] - self.assertThat(sharefiles, HasLength(expected)) + async def test_debug(self) -> None: + n = await self.do_upload_mdmf() + fso = debug.FindSharesOptions() + storage_index = base32.b2a(n.get_storage_index()) + fso.si_s = str(storage_index, "utf-8") # command-line options are unicode on Python 3 + fso.nodedirs = [os.path.dirname(abspath_expanduser_unicode(str(storedir))) + for (i,ss,storedir) + in self.iterate_servers()] + fso.stdout = StringIO() + fso.stderr = StringIO() + debug.find_shares(fso) + sharefiles = fso.stdout.getvalue().splitlines() + expected = self.nm.default_encoding_parameters["n"] + self.assertThat(sharefiles, HasLength(expected)) - do = debug.DumpOptions() - do["filename"] = sharefiles[0] - do.stdout = StringIO() - debug.dump_share(do) - output = do.stdout.getvalue() - lines = set(output.splitlines()) - self.assertTrue("Mutable slot found:" in lines, output) - self.assertTrue(" share_type: MDMF" in lines, output) - self.assertTrue(" num_extra_leases: 0" in lines, output) - self.assertTrue(" MDMF contents:" in lines, output) - self.assertTrue(" seqnum: 1" in lines, output) - self.assertTrue(" required_shares: 3" in lines, output) - self.assertTrue(" total_shares: 10" in lines, output) - self.assertTrue(" segsize: 131073" in lines, output) - self.assertTrue(" datalen: %d" % len(self.data) in lines, output) - vcap = str(n.get_verify_cap().to_string(), "utf-8") - self.assertTrue(" verify-cap: %s" % vcap in lines, output) - cso = debug.CatalogSharesOptions() - cso.nodedirs = fso.nodedirs - cso.stdout = StringIO() - cso.stderr = StringIO() - debug.catalog_shares(cso) - shares = cso.stdout.getvalue().splitlines() - oneshare = shares[0] # all shares should be MDMF - self.failIf(oneshare.startswith("UNKNOWN"), oneshare) - self.assertTrue(oneshare.startswith("MDMF"), oneshare) - fields = oneshare.split() - self.assertThat(fields[0], Equals("MDMF")) - self.assertThat(fields[1].encode("ascii"), Equals(storage_index)) - self.assertThat(fields[2], Equals("3/10")) - self.assertThat(fields[3], Equals("%d" % len(self.data))) - self.assertTrue(fields[4].startswith("#1:"), fields[3]) - # the rest of fields[4] is the roothash, which depends upon - # encryption salts and is not constant. fields[5] is the - # remaining time on the longest lease, which is timing dependent. - # The rest of the line is the quoted pathname to the share. - d.addCallback(_debug) - return d + do = debug.DumpOptions() + do["filename"] = sharefiles[0] + do.stdout = StringIO() + debug.dump_share(do) + output = do.stdout.getvalue() + lines = set(output.splitlines()) + self.assertTrue("Mutable slot found:" in lines, output) + self.assertTrue(" share_type: MDMF" in lines, output) + self.assertTrue(" num_extra_leases: 0" in lines, output) + self.assertTrue(" MDMF contents:" in lines, output) + self.assertTrue(" seqnum: 1" in lines, output) + self.assertTrue(" required_shares: 3" in lines, output) + self.assertTrue(" total_shares: 10" in lines, output) + self.assertTrue(" segsize: 131073" in lines, output) + self.assertTrue(" datalen: %d" % len(self.data) in lines, output) + vcap = str(n.get_verify_cap().to_string(), "utf-8") + self.assertTrue(" verify-cap: %s" % vcap in lines, output) + cso = debug.CatalogSharesOptions() + cso.nodedirs = fso.nodedirs + cso.stdout = StringIO() + cso.stderr = StringIO() + debug.catalog_shares(cso) + shares = cso.stdout.getvalue().splitlines() + oneshare = shares[0] # all shares should be MDMF + self.failIf(oneshare.startswith("UNKNOWN"), oneshare) + self.assertTrue(oneshare.startswith("MDMF"), oneshare) + fields = oneshare.split() + self.assertThat(fields[0], Equals("MDMF")) + self.assertThat(fields[1].encode("ascii"), Equals(storage_index)) + self.assertThat(fields[2], Equals("3/10")) + self.assertThat(fields[3], Equals("%d" % len(self.data))) + self.assertTrue(fields[4].startswith("#1:"), fields[3]) + # the rest of fields[4] is the roothash, which depends upon + # encryption salts and is not constant. fields[5] is the + # remaining time on the longest lease, which is timing dependent. + # The rest of the line is the quoted pathname to the share. + + async def test_get_sequence_number(self) -> None: + await self.do_upload() + bv = await self.mdmf_node.get_best_readable_version() + self.assertThat(bv.get_sequence_number(), Equals(1)) + bv = await self.sdmf_node.get_best_readable_version() + self.assertThat(bv.get_sequence_number(), Equals(1)) - def test_get_sequence_number(self): - d = self.do_upload() - d.addCallback(lambda ign: self.mdmf_node.get_best_readable_version()) - d.addCallback(lambda bv: - self.assertThat(bv.get_sequence_number(), Equals(1))) - d.addCallback(lambda ignored: - self.sdmf_node.get_best_readable_version()) - d.addCallback(lambda bv: - self.assertThat(bv.get_sequence_number(), Equals(1))) # Now update. The sequence number in both cases should be 1 in # both cases. - def _do_update(ignored): - new_data = MutableData(b"foo bar baz" * 100000) - new_small_data = MutableData(b"foo bar baz" * 10) - d1 = self.mdmf_node.overwrite(new_data) - d2 = self.sdmf_node.overwrite(new_small_data) - dl = gatherResults([d1, d2]) - return dl - d.addCallback(_do_update) - d.addCallback(lambda ignored: - self.mdmf_node.get_best_readable_version()) - d.addCallback(lambda bv: - self.assertThat(bv.get_sequence_number(), Equals(2))) - d.addCallback(lambda ignored: - self.sdmf_node.get_best_readable_version()) - d.addCallback(lambda bv: - self.assertThat(bv.get_sequence_number(), Equals(2))) - return d + new_data = MutableData(b"foo bar baz" * 100000) + new_small_data = MutableData(b"foo bar baz" * 10) + d1 = self.mdmf_node.overwrite(new_data) + d2 = self.sdmf_node.overwrite(new_small_data) + await gatherResults([d1, d2]) + bv = await self.mdmf_node.get_best_readable_version() + self.assertThat(bv.get_sequence_number(), Equals(2)) + bv = await self.sdmf_node.get_best_readable_version() + self.assertThat(bv.get_sequence_number(), Equals(2)) - - def test_cap_after_upload(self): + async def test_cap_after_upload(self) -> None: # If we create a new mutable file and upload things to it, and # it's an MDMF file, we should get an MDMF cap back from that # file and should be able to use that. # That's essentially what MDMF node is, so just check that. - d = self.do_upload_mdmf() - def _then(ign): - mdmf_uri = self.mdmf_node.get_uri() - cap = uri.from_string(mdmf_uri) - self.assertTrue(isinstance(cap, uri.WriteableMDMFFileURI)) - readonly_mdmf_uri = self.mdmf_node.get_readonly_uri() - cap = uri.from_string(readonly_mdmf_uri) - self.assertTrue(isinstance(cap, uri.ReadonlyMDMFFileURI)) - d.addCallback(_then) - return d + await self.do_upload_mdmf() + mdmf_uri = self.mdmf_node.get_uri() + cap = uri.from_string(mdmf_uri) + self.assertTrue(isinstance(cap, uri.WriteableMDMFFileURI)) + readonly_mdmf_uri = self.mdmf_node.get_readonly_uri() + cap = uri.from_string(readonly_mdmf_uri) + self.assertTrue(isinstance(cap, uri.ReadonlyMDMFFileURI)) - def test_mutable_version(self): + async def test_mutable_version(self) -> None: # assert that getting parameters from the IMutableVersion object # gives us the same data as getting them from the filenode itself - d = self.do_upload() - d.addCallback(lambda ign: self.mdmf_node.get_best_mutable_version()) - def _check_mdmf(bv): - n = self.mdmf_node - self.assertThat(bv.get_writekey(), Equals(n.get_writekey())) - self.assertThat(bv.get_storage_index(), Equals(n.get_storage_index())) - self.assertFalse(bv.is_readonly()) - d.addCallback(_check_mdmf) - d.addCallback(lambda ign: self.sdmf_node.get_best_mutable_version()) - def _check_sdmf(bv): - n = self.sdmf_node - self.assertThat(bv.get_writekey(), Equals(n.get_writekey())) - self.assertThat(bv.get_storage_index(), Equals(n.get_storage_index())) - self.assertFalse(bv.is_readonly()) - d.addCallback(_check_sdmf) - return d + await self.do_upload() + bv = await self.mdmf_node.get_best_mutable_version() + n = self.mdmf_node + self.assertThat(bv.get_writekey(), Equals(n.get_writekey())) + self.assertThat(bv.get_storage_index(), Equals(n.get_storage_index())) + self.assertFalse(bv.is_readonly()) + + bv = await self.sdmf_node.get_best_mutable_version() + n = self.sdmf_node + self.assertThat(bv.get_writekey(), Equals(n.get_writekey())) + self.assertThat(bv.get_storage_index(), Equals(n.get_storage_index())) + self.assertFalse(bv.is_readonly()) - def test_get_readonly_version(self): - d = self.do_upload() - d.addCallback(lambda ign: self.mdmf_node.get_best_readable_version()) - d.addCallback(lambda bv: self.assertTrue(bv.is_readonly())) + async def test_get_readonly_version(self) -> None: + await self.do_upload() + bv = await self.mdmf_node.get_best_readable_version() + self.assertTrue(bv.is_readonly()) # Attempting to get a mutable version of a mutable file from a # filenode initialized with a readcap should return a readonly # version of that same node. - d.addCallback(lambda ign: self.mdmf_node.get_readonly()) - d.addCallback(lambda ro: ro.get_best_mutable_version()) - d.addCallback(lambda v: self.assertTrue(v.is_readonly())) + ro = self.mdmf_node.get_readonly() + v = await ro.get_best_mutable_version() + self.assertTrue(v.is_readonly()) - d.addCallback(lambda ign: self.sdmf_node.get_best_readable_version()) - d.addCallback(lambda bv: self.assertTrue(bv.is_readonly())) + bv = await self.sdmf_node.get_best_readable_version() + self.assertTrue(bv.is_readonly()) - d.addCallback(lambda ign: self.sdmf_node.get_readonly()) - d.addCallback(lambda ro: ro.get_best_mutable_version()) - d.addCallback(lambda v: self.assertTrue(v.is_readonly())) - return d + ro = self.sdmf_node.get_readonly() + v = await ro.get_best_mutable_version() + self.assertTrue(v.is_readonly()) - def test_toplevel_overwrite(self): + async def test_toplevel_overwrite(self) -> None: new_data = MutableData(b"foo bar baz" * 100000) new_small_data = MutableData(b"foo bar baz" * 10) - d = self.do_upload() - d.addCallback(lambda ign: self.mdmf_node.overwrite(new_data)) - d.addCallback(lambda ignored: - self.mdmf_node.download_best_version()) - d.addCallback(lambda data: - self.assertThat(data, Equals(b"foo bar baz" * 100000))) - d.addCallback(lambda ignored: - self.sdmf_node.overwrite(new_small_data)) - d.addCallback(lambda ignored: - self.sdmf_node.download_best_version()) - d.addCallback(lambda data: - self.assertThat(data, Equals(b"foo bar baz" * 10))) - return d + await self.do_upload() + await self.mdmf_node.overwrite(new_data) + data = await self.mdmf_node.download_best_version() + self.assertThat(data, Equals(b"foo bar baz" * 100000)) + await self.sdmf_node.overwrite(new_small_data) + data = await self.sdmf_node.download_best_version() + self.assertThat(data, Equals(b"foo bar baz" * 10)) - def test_toplevel_modify(self): - d = self.do_upload() + async def test_toplevel_modify(self) -> None: + await self.do_upload() def modifier(old_contents, servermap, first_time): return old_contents + b"modified" - d.addCallback(lambda ign: self.mdmf_node.modify(modifier)) - d.addCallback(lambda ignored: - self.mdmf_node.download_best_version()) - d.addCallback(lambda data: - self.assertThat(data, Contains(b"modified"))) - d.addCallback(lambda ignored: - self.sdmf_node.modify(modifier)) - d.addCallback(lambda ignored: - self.sdmf_node.download_best_version()) - d.addCallback(lambda data: - self.assertThat(data, Contains(b"modified"))) - return d + await self.mdmf_node.modify(modifier) + data = await self.mdmf_node.download_best_version() + self.assertThat(data, Contains(b"modified")) + await self.sdmf_node.modify(modifier) + data = await self.sdmf_node.download_best_version() + self.assertThat(data, Contains(b"modified")) - def test_version_modify(self): + async def test_version_modify(self) -> None: # TODO: When we can publish multiple versions, alter this test # to modify a version other than the best usable version, then # test to see that the best recoverable version is that. - d = self.do_upload() + await self.do_upload() def modifier(old_contents, servermap, first_time): return old_contents + b"modified" - d.addCallback(lambda ign: self.mdmf_node.modify(modifier)) - d.addCallback(lambda ignored: - self.mdmf_node.download_best_version()) - d.addCallback(lambda data: - self.assertThat(data, Contains(b"modified"))) - d.addCallback(lambda ignored: - self.sdmf_node.modify(modifier)) - d.addCallback(lambda ignored: - self.sdmf_node.download_best_version()) - d.addCallback(lambda data: - self.assertThat(data, Contains(b"modified"))) - return d + await self.mdmf_node.modify(modifier) + data = await self.mdmf_node.download_best_version() + self.assertThat(data, Contains(b"modified")) + await self.sdmf_node.modify(modifier) + data = await self.sdmf_node.download_best_version() + self.assertThat(data, Contains(b"modified")) - def test_download_version(self): - d = self.publish_multiple() + async def test_download_version(self) -> None: + await self.publish_multiple() # We want to have two recoverable versions on the grid. - d.addCallback(lambda res: - self._set_versions({0:0,2:0,4:0,6:0,8:0, - 1:1,3:1,5:1,7:1,9:1})) + self._set_versions({0:0,2:0,4:0,6:0,8:0, + 1:1,3:1,5:1,7:1,9:1}) # Now try to download each version. We should get the plaintext # associated with that version. - d.addCallback(lambda ignored: - self._fn.get_servermap(mode=MODE_READ)) - def _got_servermap(smap): - versions = smap.recoverable_versions() - assert len(versions) == 2 + smap = await self._fn.get_servermap(mode=MODE_READ) + versions = smap.recoverable_versions() + assert len(versions) == 2 - self.servermap = smap - self.version1, self.version2 = versions - assert self.version1 != self.version2 + self.servermap = smap + self.version1, self.version2 = versions + assert self.version1 != self.version2 - self.version1_seqnum = self.version1[0] - self.version2_seqnum = self.version2[0] - self.version1_index = self.version1_seqnum - 1 - self.version2_index = self.version2_seqnum - 1 + self.version1_seqnum = self.version1[0] + self.version2_seqnum = self.version2[0] + self.version1_index = self.version1_seqnum - 1 + self.version2_index = self.version2_seqnum - 1 - d.addCallback(_got_servermap) - d.addCallback(lambda ignored: - self._fn.download_version(self.servermap, self.version1)) - d.addCallback(lambda results: - self.assertThat(self.CONTENTS[self.version1_index], - Equals(results))) - d.addCallback(lambda ignored: - self._fn.download_version(self.servermap, self.version2)) - d.addCallback(lambda results: - self.assertThat(self.CONTENTS[self.version2_index], - Equals(results))) - return d + results = await self._fn.download_version(self.servermap, self.version1) + self.assertThat(self.CONTENTS[self.version1_index], + Equals(results)) + results = await self._fn.download_version(self.servermap, self.version2) + self.assertThat(self.CONTENTS[self.version2_index], + Equals(results)) - def test_download_nonexistent_version(self): - d = self.do_upload_mdmf() - d.addCallback(lambda ign: self.mdmf_node.get_servermap(mode=MODE_WRITE)) - def _set_servermap(servermap): - self.servermap = servermap - d.addCallback(_set_servermap) - d.addCallback(lambda ignored: - self.shouldFail(UnrecoverableFileError, "nonexistent version", - None, - self.mdmf_node.download_version, self.servermap, - "not a version")) - return d + async def test_download_nonexistent_version(self) -> None: + await self.do_upload_mdmf() + servermap = await self.mdmf_node.get_servermap(mode=MODE_WRITE) + await self.shouldFail(UnrecoverableFileError, "nonexistent version", + None, + self.mdmf_node.download_version, servermap, + "not a version") - def _test_partial_read(self, node, expected, modes, step): - d = node.get_best_readable_version() + async def _test_partial_read(self, node, expected, modes, step) -> None: + version = await node.get_best_readable_version() for (name, offset, length) in modes: - d.addCallback(self._do_partial_read, name, expected, offset, length) + version = await self._do_partial_read(version, name, expected, offset, length) # then read the whole thing, but only a few bytes at a time, and see # that the results are what we expect. - def _read_data(version): - c = consumer.MemoryConsumer() - d2 = defer.succeed(None) - for i in range(0, len(expected), step): - d2.addCallback(lambda ignored, i=i: version.read(c, i, step)) - d2.addCallback(lambda ignored: - self.assertThat(expected, Equals(b"".join(c.chunks)))) - return d2 - d.addCallback(_read_data) - return d - - def _do_partial_read(self, version, name, expected, offset, length): c = consumer.MemoryConsumer() - d = version.read(c, offset, length) + for i in range(0, len(expected), step): + await version.read(c, i, step) + self.assertThat(expected, Equals(b"".join(c.chunks))) + + async def _do_partial_read(self, version, name, expected, offset, length) -> None: + c = consumer.MemoryConsumer() + await version.read(c, offset, length) if length is None: expected_range = expected[offset:] else: expected_range = expected[offset:offset+length] - d.addCallback(lambda ignored: b"".join(c.chunks)) - def _check(results): - if results != expected_range: - print("read([%d]+%s) got %d bytes, not %d" % \ - (offset, length, len(results), len(expected_range))) - print("got: %s ... %s" % (results[:20], results[-20:])) - print("exp: %s ... %s" % (expected_range[:20], expected_range[-20:])) - self.fail("results[%s] != expected_range" % name) - return version # daisy-chained to next call - d.addCallback(_check) - return d + results = b"".join(c.chunks) + if results != expected_range: + print("read([%d]+%s) got %d bytes, not %d" % \ + (offset, length, len(results), len(expected_range))) + print("got: %s ... %s" % (results[:20], results[-20:])) + print("exp: %s ... %s" % (expected_range[:20], expected_range[-20:])) + self.fail("results[%s] != expected_range" % name) + return version # daisy-chained to next call - def test_partial_read_mdmf_0(self): + async def test_partial_read_mdmf_0(self) -> None: data = b"" - d = self.do_upload_mdmf(data=data) + result = await self.do_upload_mdmf(data=data) modes = [("all1", 0,0), ("all2", 0,None), ] - d.addCallback(self._test_partial_read, data, modes, 1) - return d + await self._test_partial_read(result, data, modes, 1) - def test_partial_read_mdmf_large(self): + async def test_partial_read_mdmf_large(self) -> None: segment_boundary = mathutil.next_multiple(128 * 1024, 3) modes = [("start_on_segment_boundary", segment_boundary, 50), ("ending_one_byte_after_segment_boundary", segment_boundary-50, 51), @@ -386,20 +312,18 @@ class Version(GridTestMixin, AsyncTestCase, testutil.ShouldFailMixin, \ ("complete_file1", 0, len(self.data)), ("complete_file2", 0, None), ] - d = self.do_upload_mdmf() - d.addCallback(self._test_partial_read, self.data, modes, 10000) - return d + result = await self.do_upload_mdmf() + await self._test_partial_read(result, self.data, modes, 10000) - def test_partial_read_sdmf_0(self): + async def test_partial_read_sdmf_0(self) -> None: data = b"" modes = [("all1", 0,0), ("all2", 0,None), ] - d = self.do_upload_sdmf(data=data) - d.addCallback(self._test_partial_read, data, modes, 1) - return d + result = await self.do_upload_sdmf(data=data) + await self._test_partial_read(result, data, modes, 1) - def test_partial_read_sdmf_2(self): + async def test_partial_read_sdmf_2(self) -> None: data = b"hi" modes = [("one_byte", 0, 1), ("last_byte", 1, 1), @@ -407,11 +331,10 @@ class Version(GridTestMixin, AsyncTestCase, testutil.ShouldFailMixin, \ ("complete_file", 0, 2), ("complete_file2", 0, None), ] - d = self.do_upload_sdmf(data=data) - d.addCallback(self._test_partial_read, data, modes, 1) - return d + result = await self.do_upload_sdmf(data=data) + await self._test_partial_read(result, data, modes, 1) - def test_partial_read_sdmf_90(self): + async def test_partial_read_sdmf_90(self) -> None: modes = [("start_at_middle", 50, 40), ("start_at_middle2", 50, None), ("zero_length_at_start", 0, 0), @@ -420,11 +343,10 @@ class Version(GridTestMixin, AsyncTestCase, testutil.ShouldFailMixin, \ ("complete_file1", 0, None), ("complete_file2", 0, 90), ] - d = self.do_upload_sdmf() - d.addCallback(self._test_partial_read, self.small_data, modes, 10) - return d + result = await self.do_upload_sdmf() + await self._test_partial_read(result, self.small_data, modes, 10) - def test_partial_read_sdmf_100(self): + async def test_partial_read_sdmf_100(self) -> None: data = b"test data "*10 modes = [("start_at_middle", 50, 50), ("start_at_middle2", 50, None), @@ -433,42 +355,30 @@ class Version(GridTestMixin, AsyncTestCase, testutil.ShouldFailMixin, \ ("complete_file1", 0, 100), ("complete_file2", 0, None), ] - d = self.do_upload_sdmf(data=data) - d.addCallback(self._test_partial_read, data, modes, 10) - return d + result = await self.do_upload_sdmf(data=data) + await self._test_partial_read(result, data, modes, 10) + async def _test_read_and_download(self, node, expected) -> None: + version = await node.get_best_readable_version() + c = consumer.MemoryConsumer() + await version.read(c) + self.assertThat(expected, Equals(b"".join(c.chunks))) - def _test_read_and_download(self, node, expected): - d = node.get_best_readable_version() - def _read_data(version): - c = consumer.MemoryConsumer() - c2 = consumer.MemoryConsumer() - d2 = defer.succeed(None) - d2.addCallback(lambda ignored: version.read(c)) - d2.addCallback(lambda ignored: - self.assertThat(expected, Equals(b"".join(c.chunks)))) + c2 = consumer.MemoryConsumer() + await version.read(c2, offset=0, size=len(expected)) + self.assertThat(expected, Equals(b"".join(c2.chunks))) - d2.addCallback(lambda ignored: version.read(c2, offset=0, - size=len(expected))) - d2.addCallback(lambda ignored: - self.assertThat(expected, Equals(b"".join(c2.chunks)))) - return d2 - d.addCallback(_read_data) - d.addCallback(lambda ignored: node.download_best_version()) - d.addCallback(lambda data: self.assertThat(expected, Equals(data))) - return d + data = await node.download_best_version() + self.assertThat(expected, Equals(data)) - def test_read_and_download_mdmf(self): - d = self.do_upload_mdmf() - d.addCallback(self._test_read_and_download, self.data) - return d + async def test_read_and_download_mdmf(self) -> None: + result = await self.do_upload_mdmf() + await self._test_read_and_download(result, self.data) - def test_read_and_download_sdmf(self): - d = self.do_upload_sdmf() - d.addCallback(self._test_read_and_download, self.small_data) - return d + async def test_read_and_download_sdmf(self) -> None: + result = await self.do_upload_sdmf() + await self._test_read_and_download(result, self.small_data) - def test_read_and_download_sdmf_zero_length(self): - d = self.do_upload_empty_sdmf() - d.addCallback(self._test_read_and_download, b"") - return d + async def test_read_and_download_sdmf_zero_length(self) -> None: + result = await self.do_upload_empty_sdmf() + await self._test_read_and_download(result, b"") From e72847115be571559167181d5209fa3dccfbd458 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 30 Nov 2022 09:37:26 -0500 Subject: [PATCH 1216/2309] news fragment --- newsfragments/3947.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3947.minor diff --git a/newsfragments/3947.minor b/newsfragments/3947.minor new file mode 100644 index 000000000..e69de29bb From 156954c621f7b39406831ca18bed00a2dedf8b70 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 30 Nov 2022 09:43:01 -0500 Subject: [PATCH 1217/2309] no longer any need to "daisy chain" this value --- src/allmydata/test/mutable/test_version.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/allmydata/test/mutable/test_version.py b/src/allmydata/test/mutable/test_version.py index d14cc9295..1d9467694 100644 --- a/src/allmydata/test/mutable/test_version.py +++ b/src/allmydata/test/mutable/test_version.py @@ -270,7 +270,7 @@ class Version(GridTestMixin, AsyncTestCase, testutil.ShouldFailMixin, \ async def _test_partial_read(self, node, expected, modes, step) -> None: version = await node.get_best_readable_version() for (name, offset, length) in modes: - version = await self._do_partial_read(version, name, expected, offset, length) + await self._do_partial_read(version, name, expected, offset, length) # then read the whole thing, but only a few bytes at a time, and see # that the results are what we expect. c = consumer.MemoryConsumer() @@ -292,7 +292,6 @@ class Version(GridTestMixin, AsyncTestCase, testutil.ShouldFailMixin, \ print("got: %s ... %s" % (results[:20], results[-20:])) print("exp: %s ... %s" % (expected_range[:20], expected_range[-20:])) self.fail("results[%s] != expected_range" % name) - return version # daisy-chained to next call async def test_partial_read_mdmf_0(self) -> None: data = b"" From 05dfa875a771e6ff27006b8fc13aad3dc1709b67 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 30 Nov 2022 09:46:13 -0500 Subject: [PATCH 1218/2309] Quite a mypy warning about formatting bytes into a string --- src/allmydata/test/mutable/test_version.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/mutable/test_version.py b/src/allmydata/test/mutable/test_version.py index 1d9467694..87050424b 100644 --- a/src/allmydata/test/mutable/test_version.py +++ b/src/allmydata/test/mutable/test_version.py @@ -289,8 +289,8 @@ class Version(GridTestMixin, AsyncTestCase, testutil.ShouldFailMixin, \ if results != expected_range: print("read([%d]+%s) got %d bytes, not %d" % \ (offset, length, len(results), len(expected_range))) - print("got: %s ... %s" % (results[:20], results[-20:])) - print("exp: %s ... %s" % (expected_range[:20], expected_range[-20:])) + print("got: %r ... %r" % (results[:20], results[-20:])) + print("exp: %r ... %r" % (expected_range[:20], expected_range[-20:])) self.fail("results[%s] != expected_range" % name) async def test_partial_read_mdmf_0(self) -> None: From b193ad3ed42527d9118cc7d82d142d648a21a5c1 Mon Sep 17 00:00:00 2001 From: "Fon E. Noel NFEBE" Date: Wed, 30 Nov 2022 16:53:20 +0100 Subject: [PATCH 1219/2309] Correct addCleanup reference Some test_storage.py classes contain calls to cleanup methods instead of references. This commit fixes that. Signed-off-by: Fon E. Noel NFEBE --- src/allmydata/test/test_storage.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index f7d5ae919..2234b5af2 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -563,7 +563,7 @@ class Server(AsyncTestCase): self.sparent = LoggingServiceParent() self.sparent.startService() self._lease_secret = itertools.count() - self.addCleanup(self.sparent.stopService()) + self.addCleanup(self.sparent.stopService) def workdir(self, name): basedir = os.path.join("storage", "Server", name) @@ -1284,7 +1284,7 @@ class MutableServer(SyncTestCase): super(MutableServer, self).setUp() self.sparent = LoggingServiceParent() self._lease_secret = itertools.count() - self.addCleanup(self.sparent.stopService()) + self.addCleanup(self.sparent.stopService) def workdir(self, name): basedir = os.path.join("storage", "MutableServer", name) @@ -3337,7 +3337,7 @@ class Stats(SyncTestCase): super(Stats, self).setUp() self.sparent = LoggingServiceParent() self._lease_secret = itertools.count() - self.addCleanup(self.sparent.stopService()) + self.addCleanup(self.sparent.stopService) def workdir(self, name): basedir = os.path.join("storage", "Server", name) From 1436eb0fb689d5f230e3cdd44b41579d152d26ad Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 1 Dec 2022 14:26:41 -0500 Subject: [PATCH 1220/2309] Better explanation. --- newsfragments/3939.bugfix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/newsfragments/3939.bugfix b/newsfragments/3939.bugfix index 61fb4244a..9d2071d32 100644 --- a/newsfragments/3939.bugfix +++ b/newsfragments/3939.bugfix @@ -1 +1 @@ -Uploading immutables will now use more bandwidth, which should allow for faster uploads in many cases. \ No newline at end of file +Uploading immutables will now better use available bandwidth, which should allow for faster uploads in many cases. \ No newline at end of file From 17dfda6b5aef1bcbb9b5baea2c0785fef6f833ca Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 1 Dec 2022 14:42:52 -0500 Subject: [PATCH 1221/2309] More direct API. --- src/allmydata/immutable/layout.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/immutable/layout.py b/src/allmydata/immutable/layout.py index 562ca4470..6e8cfe1d8 100644 --- a/src/allmydata/immutable/layout.py +++ b/src/allmydata/immutable/layout.py @@ -121,7 +121,7 @@ class _WriteBuffer: and do a real write. """ self._to_write.write(data) - return len(self._to_write.getbuffer()) >= self._batch_size + return self._to_write.tell() >= self._batch_size def flush(self) -> tuple[int, bytes]: """Return offset and data to be written.""" @@ -133,7 +133,7 @@ class _WriteBuffer: def get_total_bytes_queued(self) -> int: """Return how many bytes were written or queued in total.""" - return self._written_bytes + len(self._to_write.getbuffer()) + return self._written_bytes + self._to_write.tell() @implementer(IStorageBucketWriter) From d4c202307caff5d4cb580421de4ce44d389c9193 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 1 Dec 2022 14:43:49 -0500 Subject: [PATCH 1222/2309] Better method name. --- src/allmydata/immutable/layout.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/allmydata/immutable/layout.py b/src/allmydata/immutable/layout.py index 6e8cfe1d8..3321ca0b6 100644 --- a/src/allmydata/immutable/layout.py +++ b/src/allmydata/immutable/layout.py @@ -131,7 +131,7 @@ class _WriteBuffer: self._to_write = BytesIO() return (offset, data) - def get_total_bytes_queued(self) -> int: + def get_total_bytes(self) -> int: """Return how many bytes were written or queued in total.""" return self._written_bytes + self._to_write.tell() @@ -289,7 +289,7 @@ class WriteBucketProxy(object): no holes. As such, the offset is technically unnecessary, but is used to check the inputs. Possibly we should get rid of it. """ - assert offset == self._write_buffer.get_total_bytes_queued() + assert offset == self._write_buffer.get_total_bytes() if self._write_buffer.queue_write(data): return self._actually_write() else: @@ -301,7 +301,7 @@ class WriteBucketProxy(object): return self._rref.callRemote("write", offset, data) def close(self): - assert self._write_buffer.get_total_bytes_queued() == self.get_allocated_size(), ( + assert self._write_buffer.get_total_bytes() == self.get_allocated_size(), ( f"{self._written_buffer.get_total_bytes_queued()} != {self.get_allocated_size()}" ) d = self._actually_write() From 0ba58070cdf0e463c4731da46d64eddc27bd26d9 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 1 Dec 2022 14:45:39 -0500 Subject: [PATCH 1223/2309] Tweaks. --- src/allmydata/immutable/encode.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/immutable/encode.py b/src/allmydata/immutable/encode.py index 2b6602773..2414527ff 100644 --- a/src/allmydata/immutable/encode.py +++ b/src/allmydata/immutable/encode.py @@ -262,8 +262,8 @@ class Encoder(object): d.addCallback(lambda res: self.finish_hashing()) - # These calls have to happen in order, and waiting for previous one - # also ensures backpressure: + # These calls have to happen in order; layout.py now requires writes to + # be appended to the data written so far. d.addCallback(lambda res: self.send_crypttext_hash_tree_to_all_shareholders()) d.addCallback(lambda res: self.send_all_block_hash_trees()) From 8ed333b171f1a99da60948ddbd2eee84c93b0d78 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 1 Dec 2022 14:45:45 -0500 Subject: [PATCH 1224/2309] Correct explanation. --- src/allmydata/immutable/layout.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/immutable/layout.py b/src/allmydata/immutable/layout.py index 3321ca0b6..f86590057 100644 --- a/src/allmydata/immutable/layout.py +++ b/src/allmydata/immutable/layout.py @@ -162,7 +162,7 @@ class WriteBucketProxy(object): self._create_offsets(block_size, data_size) - # With a ~1MB batch size, max upload speed is 1MB * round-trip latency + # With a ~1MB batch size, max upload speed is 1MB/(round-trip latency) # assuming the writing code waits for writes to finish, so 20MB/sec if # latency is 50ms. In the US many people only have 1MB/sec upload speed # as of 2022 (standard Comcast). For further discussion of how one From c93ff23da7479defe3f05f8ef622ee68be1bb568 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 1 Dec 2022 14:54:28 -0500 Subject: [PATCH 1225/2309] Don't send empty string writes. --- src/allmydata/immutable/layout.py | 14 +++++++++++--- src/allmydata/test/test_storage.py | 2 ++ 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/src/allmydata/immutable/layout.py b/src/allmydata/immutable/layout.py index f86590057..9154f2f30 100644 --- a/src/allmydata/immutable/layout.py +++ b/src/allmydata/immutable/layout.py @@ -121,7 +121,7 @@ class _WriteBuffer: and do a real write. """ self._to_write.write(data) - return self._to_write.tell() >= self._batch_size + return self.get_queued_bytes() >= self._batch_size def flush(self) -> tuple[int, bytes]: """Return offset and data to be written.""" @@ -131,9 +131,13 @@ class _WriteBuffer: self._to_write = BytesIO() return (offset, data) + def get_queued_bytes(self) -> int: + """Return number of queued, unwritten bytes.""" + return self._to_write.tell() + def get_total_bytes(self) -> int: """Return how many bytes were written or queued in total.""" - return self._written_bytes + self._to_write.tell() + return self._written_bytes + self.get_queued_bytes() @implementer(IStorageBucketWriter) @@ -304,7 +308,11 @@ class WriteBucketProxy(object): assert self._write_buffer.get_total_bytes() == self.get_allocated_size(), ( f"{self._written_buffer.get_total_bytes_queued()} != {self.get_allocated_size()}" ) - d = self._actually_write() + if self._write_buffer.get_queued_bytes() > 0: + d = self._actually_write() + else: + # No data queued, don't send empty string write. + d = defer.succeed(True) d.addCallback(lambda _: self._rref.callRemote("close")) return d diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index 820d4fd79..9b9d2d8de 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -3770,7 +3770,9 @@ class WriteBufferTests(SyncTestCase): result += flushed_data # Final flush: + remaining_length = wb.get_queued_bytes() flushed_offset, flushed_data = wb.flush() + self.assertEqual(remaining_length, len(flushed_data)) self.assertEqual(flushed_offset, len(result)) result += flushed_data From 1eb4a4adf8ba6b6f3c9ac236753a020428e424c5 Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 1 Dec 2022 16:47:24 -0700 Subject: [PATCH 1226/2309] Update newsfragments/3921.feature Co-authored-by: Jean-Paul Calderone --- newsfragments/3921.feature | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/newsfragments/3921.feature b/newsfragments/3921.feature index f2c3a98bd..798aee817 100644 --- a/newsfragments/3921.feature +++ b/newsfragments/3921.feature @@ -1,4 +1,4 @@ -Automatically exit when stdin is closed +`tahoe run ...` will now exit when its stdin is closed. This facilitates subprocess management, specifically cleanup. When a parent process is running tahoe and exits without time to do "proper" cleanup at least the stdin descriptor will be closed. From 7ffcfcdb67213ef9002279f2a55b57a2789fa12d Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 1 Dec 2022 16:47:40 -0700 Subject: [PATCH 1227/2309] Update src/allmydata/test/test_runner.py Co-authored-by: Jean-Paul Calderone --- src/allmydata/test/test_runner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/test_runner.py b/src/allmydata/test/test_runner.py index bce5b3c20..9830487f3 100644 --- a/src/allmydata/test/test_runner.py +++ b/src/allmydata/test/test_runner.py @@ -661,7 +661,7 @@ class OnStdinCloseTests(SyncTestCase): def test_exception_ignored(self): """ - an exception from or on-close function is ignored + An exception from our on-close function is discarded. """ reactor = MemoryReactorClock() called = [] From 3d43cbccc9ef24fb3168e89eb63a93d682584417 Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 1 Dec 2022 17:01:38 -0700 Subject: [PATCH 1228/2309] log less-specific failures --- src/allmydata/scripts/tahoe_run.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/allmydata/scripts/tahoe_run.py b/src/allmydata/scripts/tahoe_run.py index 49507765e..aaf234b61 100644 --- a/src/allmydata/scripts/tahoe_run.py +++ b/src/allmydata/scripts/tahoe_run.py @@ -21,9 +21,11 @@ from twisted.scripts import twistd from twisted.python import usage from twisted.python.filepath import FilePath from twisted.python.reflect import namedAny +from twisted.python.failure import Failure from twisted.internet.defer import maybeDeferred, Deferred from twisted.internet.protocol import Protocol from twisted.internet.stdio import StandardIO +from twisted.internet.error import ReactorNotRunning from twisted.application.service import Service from allmydata.scripts.default_nodedir import _default_nodedir @@ -238,12 +240,15 @@ def on_stdin_close(reactor, fn): def on_close(arg): try: fn() + except ReactorNotRunning: + pass except Exception: - # for our "exit" use-case, this will _mostly_ just be + # for our "exit" use-case failures will _mostly_ just be # ReactorNotRunning (because we're already shutting down # when our stdin closes) but no matter what "bad thing" - # happens we just want to ignore it. - pass + # happens we just want to ignore it .. although other + # errors might be interesting so we'll log those + print(Failure()) return arg when_closed_d.addBoth(on_close) From 36ed554627dc393172d8532b6a66c1bcdbb22334 Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 1 Dec 2022 17:03:48 -0700 Subject: [PATCH 1229/2309] proto -> transport --- src/allmydata/test/test_runner.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/allmydata/test/test_runner.py b/src/allmydata/test/test_runner.py index 52ec84ae6..cd09e5aa0 100644 --- a/src/allmydata/test/test_runner.py +++ b/src/allmydata/test/test_runner.py @@ -644,15 +644,15 @@ class OnStdinCloseTests(SyncTestCase): def onclose(): called.append(True) - proto = on_stdin_close(reactor, onclose) + transport = on_stdin_close(reactor, onclose) self.assertEqual(called, []) # on Unix we can just close all the readers, correctly # "simulating" a stdin close .. of course, Windows has to be # difficult if platform.isWindows(): - proto.writeConnectionLost() - proto.readConnectionLost() + transport.writeConnectionLost() + transport.readConnectionLost() else: for reader in reactor.getReaders(): reader.loseConnection() @@ -670,12 +670,12 @@ class OnStdinCloseTests(SyncTestCase): def onclose(): called.append(True) raise RuntimeError("unexpected error") - proto = on_stdin_close(reactor, onclose) + transport = on_stdin_close(reactor, onclose) self.assertEqual(called, []) if platform.isWindows(): - proto.writeConnectionLost() - proto.readConnectionLost() + transport.writeConnectionLost() + transport.readConnectionLost() else: for reader in reactor.getReaders(): reader.loseConnection() From 20b3594d128e4a90b56ef516a713d44156d17122 Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 1 Dec 2022 17:05:58 -0700 Subject: [PATCH 1230/2309] exarkun wants a helper --- src/allmydata/test/test_runner.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/src/allmydata/test/test_runner.py b/src/allmydata/test/test_runner.py index cd09e5aa0..a84fa28f8 100644 --- a/src/allmydata/test/test_runner.py +++ b/src/allmydata/test/test_runner.py @@ -630,6 +630,15 @@ class RunNode(common_util.SignalMixin, unittest.TestCase, pollmixin.PollMixin): yield client_running +def _simulate_windows_stdin_close(stdio): + """ + on Unix we can just close all the readers, correctly "simulating" + a stdin close .. of course, Windows has to be difficult + """ + stdio.writeConnectionLost() + stdio.readConnectionLost() + + class OnStdinCloseTests(SyncTestCase): """ Tests for on_stdin_close @@ -647,12 +656,8 @@ class OnStdinCloseTests(SyncTestCase): transport = on_stdin_close(reactor, onclose) self.assertEqual(called, []) - # on Unix we can just close all the readers, correctly - # "simulating" a stdin close .. of course, Windows has to be - # difficult if platform.isWindows(): - transport.writeConnectionLost() - transport.readConnectionLost() + _simulate_windows_stdin_close(transport) else: for reader in reactor.getReaders(): reader.loseConnection() @@ -674,8 +679,7 @@ class OnStdinCloseTests(SyncTestCase): self.assertEqual(called, []) if platform.isWindows(): - transport.writeConnectionLost() - transport.readConnectionLost() + _simulate_windows_stdin_close(transport) else: for reader in reactor.getReaders(): reader.loseConnection() From 89b6a008d2277e17832acf3afb16c6e71f88715c Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 1 Dec 2022 23:24:24 -0700 Subject: [PATCH 1231/2309] since 'coverage report' is what fails with disk-space on windows, try turning it off --- tox.ini | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index fc95a0469..9b6dc8756 100644 --- a/tox.ini +++ b/tox.ini @@ -86,7 +86,7 @@ commands = coverage: python -b -m coverage run -m twisted.trial {env:TAHOE_LAFS_TRIAL_ARGS:--rterrors --reporter=timing} {posargs:{env:TEST_SUITE}} coverage: coverage combine coverage: coverage xml - coverage: coverage report +## coverage: coverage report [testenv:integration] basepython = python3 @@ -99,7 +99,7 @@ commands = # NOTE: 'run with "py.test --keep-tempdir -s -v integration/" to debug failures' py.test --timeout=1800 --coverage -s -v {posargs:integration} coverage combine - coverage report +## coverage report [testenv:codechecks] From 3d831f653ba20286ea94e512cf0e87882cbc9e26 Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 1 Dec 2022 23:58:53 -0700 Subject: [PATCH 1232/2309] cleanup --- .github/workflows/ci.yml | 2 +- tox.ini | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 163266613..15e7d8fa4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -63,7 +63,7 @@ jobs: python-version: "pypy-3.7" - os: ubuntu-latest python-version: "pypy-3.8" - + steps: # See https://github.com/actions/checkout. A fetch-depth of 0 # fetches all tags and branches. diff --git a/tox.ini b/tox.ini index 9b6dc8756..db4748033 100644 --- a/tox.ini +++ b/tox.ini @@ -86,7 +86,6 @@ commands = coverage: python -b -m coverage run -m twisted.trial {env:TAHOE_LAFS_TRIAL_ARGS:--rterrors --reporter=timing} {posargs:{env:TEST_SUITE}} coverage: coverage combine coverage: coverage xml -## coverage: coverage report [testenv:integration] basepython = python3 @@ -99,7 +98,6 @@ commands = # NOTE: 'run with "py.test --keep-tempdir -s -v integration/" to debug failures' py.test --timeout=1800 --coverage -s -v {posargs:integration} coverage combine -## coverage report [testenv:codechecks] From 347d11a83c3cb184a1f77cc1060f613a01cdb13f Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 2 Dec 2022 01:27:13 -0700 Subject: [PATCH 1233/2309] fix test, un-log error --- src/allmydata/test/test_storage_client.py | 25 ++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/src/allmydata/test/test_storage_client.py b/src/allmydata/test/test_storage_client.py index 0c05be2e6..04f2c7e29 100644 --- a/src/allmydata/test/test_storage_client.py +++ b/src/allmydata/test/test_storage_client.py @@ -77,6 +77,7 @@ from .common import ( UseNode, SameProcessStreamEndpointAssigner, MemoryIntroducerClient, + flush_logged_errors, ) from .common_web import ( do_http, @@ -92,6 +93,8 @@ from allmydata.storage_client import ( IFoolscapStorageServer, NativeStorageServer, StorageFarmBroker, + StorageClientConfig, + MissingPlugin, _FoolscapStorage, _NullStorage, ) @@ -159,7 +162,7 @@ class GetConnectionStatus(unittest.TestCase): self.assertTrue(IConnectionStatus.providedBy(connection_status)) -class UnrecognizedAnnouncement(SyncTestCase): +class UnrecognizedAnnouncement(unittest.TestCase): """ Tests for handling of announcements that aren't recognized and don't use *anonymous-storage-FURL*. @@ -183,7 +186,7 @@ class UnrecognizedAnnouncement(SyncTestCase): def _tub_maker(self, overrides): return Service() - def native_storage_server(self): + def native_storage_server(self, config=None): """ Make a ``NativeStorageServer`` out of an unrecognizable announcement. """ @@ -192,7 +195,8 @@ class UnrecognizedAnnouncement(SyncTestCase): self.ann, self._tub_maker, {}, - EMPTY_CLIENT_CONFIG, + node_config=EMPTY_CLIENT_CONFIG, + config=config or StorageClientConfig(), ) def test_no_exceptions(self): @@ -243,11 +247,18 @@ class UnrecognizedAnnouncement(SyncTestCase): """ ``NativeStorageServer.get_longname`` describes the missing plugin. """ - server = self.native_storage_server() - self.assertThat( - server.get_longname(), - Equals(''.format(self.plugin_name)), + server = self.native_storage_server( + StorageClientConfig( + storage_plugins={ + "nothing": {} + } + ) ) + self.assertEqual( + server.get_longname(), + '', + ) + self.flushLoggedErrors(MissingPlugin) class PluginMatchedAnnouncement(SyncTestCase): From 9619e286f4e0f50c20975f5789418eed09f4e350 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 2 Dec 2022 08:16:02 -0500 Subject: [PATCH 1234/2309] Switch the web testing double to BytesKeyDict This will catch more str/bytes errors by default than `dict` --- src/allmydata/testing/web.py | 3 ++- src/allmydata/util/dictutil.py | 34 +++++++--------------------------- 2 files changed, 9 insertions(+), 28 deletions(-) diff --git a/src/allmydata/testing/web.py b/src/allmydata/testing/web.py index 6538dc3a4..a687e5480 100644 --- a/src/allmydata/testing/web.py +++ b/src/allmydata/testing/web.py @@ -45,6 +45,7 @@ import allmydata.uri from allmydata.util import ( base32, ) +from ..util.dictutil import BytesKeyDict __all__ = ( @@ -138,7 +139,7 @@ class _FakeTahoeUriHandler(Resource, object): isLeaf = True - data: Dict[bytes, bytes] = attr.ib(default=attr.Factory(dict)) + data: BytesKeyDict[bytes, bytes] = attr.ib(default=attr.Factory(BytesKeyDict)) capability_generators = attr.ib(default=attr.Factory(dict)) def _generate_capability(self, kind): diff --git a/src/allmydata/util/dictutil.py b/src/allmydata/util/dictutil.py index 5971d26f6..0a7df0a38 100644 --- a/src/allmydata/util/dictutil.py +++ b/src/allmydata/util/dictutil.py @@ -1,21 +1,6 @@ """ Tools to mess with dicts. - -Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals - -from future.utils import PY2 -if PY2: - # IMPORTANT: We deliberately don't import dict. The issue is that we're - # subclassing dict, so we'd end up exposing Python 3 dict APIs to lots of - # code that doesn't support it. - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, list, object, range, str, max, min # noqa: F401 -from six import ensure_str - class DictOfSets(dict): def add(self, key, value): @@ -104,7 +89,7 @@ def _make_enforcing_override(K, method_name): raise TypeError("{} must be of type {}".format( repr(key), self.KEY_TYPE)) return getattr(dict, method_name)(self, key, *args, **kwargs) - f.__name__ = ensure_str(method_name) + f.__name__ = method_name setattr(K, method_name, f) for _method_name in ["__setitem__", "__getitem__", "setdefault", "get", @@ -113,18 +98,13 @@ for _method_name in ["__setitem__", "__getitem__", "setdefault", "get", del _method_name -if PY2: - # No need for enforcement, can use either bytes or unicode as keys and it's - # fine. - BytesKeyDict = UnicodeKeyDict = dict -else: - class BytesKeyDict(_TypedKeyDict): - """Keys should be bytes.""" +class BytesKeyDict(_TypedKeyDict): + """Keys should be bytes.""" - KEY_TYPE = bytes + KEY_TYPE = bytes - class UnicodeKeyDict(_TypedKeyDict): - """Keys should be unicode strings.""" +class UnicodeKeyDict(_TypedKeyDict): + """Keys should be unicode strings.""" - KEY_TYPE = str + KEY_TYPE = str From a84b278ecdaf3263030ad9817961d70cdfdfd741 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 2 Dec 2022 08:26:15 -0500 Subject: [PATCH 1235/2309] support older pythons --- src/allmydata/testing/web.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/allmydata/testing/web.py b/src/allmydata/testing/web.py index a687e5480..be7878d57 100644 --- a/src/allmydata/testing/web.py +++ b/src/allmydata/testing/web.py @@ -11,6 +11,8 @@ Test-helpers for clients that use the WebUI. """ +from __future__ import annotations + import hashlib from typing import Dict From b40d882fceb20fa102ca32540819555a08b2fdf1 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 2 Dec 2022 08:28:22 -0500 Subject: [PATCH 1236/2309] remove unused import --- src/allmydata/testing/web.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/allmydata/testing/web.py b/src/allmydata/testing/web.py index be7878d57..4af2603a8 100644 --- a/src/allmydata/testing/web.py +++ b/src/allmydata/testing/web.py @@ -14,7 +14,6 @@ Test-helpers for clients that use the WebUI. from __future__ import annotations import hashlib -from typing import Dict import attr From c6cc3708f45c9e581a719852d4f1082528941701 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 2 Dec 2022 08:38:46 -0500 Subject: [PATCH 1237/2309] Fixup the annotations a bit --- src/allmydata/testing/web.py | 2 +- src/allmydata/util/dictutil.py | 15 +++++++++++---- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/src/allmydata/testing/web.py b/src/allmydata/testing/web.py index 4af2603a8..72ecd7161 100644 --- a/src/allmydata/testing/web.py +++ b/src/allmydata/testing/web.py @@ -140,7 +140,7 @@ class _FakeTahoeUriHandler(Resource, object): isLeaf = True - data: BytesKeyDict[bytes, bytes] = attr.ib(default=attr.Factory(BytesKeyDict)) + data: BytesKeyDict[bytes] = attr.ib(default=attr.Factory(BytesKeyDict)) capability_generators = attr.ib(default=attr.Factory(dict)) def _generate_capability(self, kind): diff --git a/src/allmydata/util/dictutil.py b/src/allmydata/util/dictutil.py index 0a7df0a38..c436ab963 100644 --- a/src/allmydata/util/dictutil.py +++ b/src/allmydata/util/dictutil.py @@ -2,6 +2,10 @@ Tools to mess with dicts. """ +from __future__ import annotations + +from typing import TypeVar, Type + class DictOfSets(dict): def add(self, key, value): if key in self: @@ -64,7 +68,10 @@ class AuxValueDict(dict): self.auxilliary[key] = auxilliary -class _TypedKeyDict(dict): +K = TypeVar("K") +V = TypeVar("V") + +class _TypedKeyDict(dict[K, V]): """Dictionary that enforces key type. Doesn't override everything, but probably good enough to catch most @@ -73,7 +80,7 @@ class _TypedKeyDict(dict): Subclass and override KEY_TYPE. """ - KEY_TYPE = object + KEY_TYPE: Type[K] def __init__(self, *args, **kwargs): dict.__init__(self, *args, **kwargs) @@ -98,13 +105,13 @@ for _method_name in ["__setitem__", "__getitem__", "setdefault", "get", del _method_name -class BytesKeyDict(_TypedKeyDict): +class BytesKeyDict(_TypedKeyDict[bytes, V]): """Keys should be bytes.""" KEY_TYPE = bytes -class UnicodeKeyDict(_TypedKeyDict): +class UnicodeKeyDict(_TypedKeyDict[str, V]): """Keys should be unicode strings.""" KEY_TYPE = str From c542b8463707cd5720822de260f79b61c7582994 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 2 Dec 2022 08:47:07 -0500 Subject: [PATCH 1238/2309] remove the annotations everything is broken on older pythons --- src/allmydata/testing/web.py | 2 +- src/allmydata/util/dictutil.py | 15 ++++----------- 2 files changed, 5 insertions(+), 12 deletions(-) diff --git a/src/allmydata/testing/web.py b/src/allmydata/testing/web.py index 72ecd7161..4f68b3774 100644 --- a/src/allmydata/testing/web.py +++ b/src/allmydata/testing/web.py @@ -140,7 +140,7 @@ class _FakeTahoeUriHandler(Resource, object): isLeaf = True - data: BytesKeyDict[bytes] = attr.ib(default=attr.Factory(BytesKeyDict)) + data: BytesKeyDict = attr.ib(default=attr.Factory(BytesKeyDict)) capability_generators = attr.ib(default=attr.Factory(dict)) def _generate_capability(self, kind): diff --git a/src/allmydata/util/dictutil.py b/src/allmydata/util/dictutil.py index c436ab963..0a7df0a38 100644 --- a/src/allmydata/util/dictutil.py +++ b/src/allmydata/util/dictutil.py @@ -2,10 +2,6 @@ Tools to mess with dicts. """ -from __future__ import annotations - -from typing import TypeVar, Type - class DictOfSets(dict): def add(self, key, value): if key in self: @@ -68,10 +64,7 @@ class AuxValueDict(dict): self.auxilliary[key] = auxilliary -K = TypeVar("K") -V = TypeVar("V") - -class _TypedKeyDict(dict[K, V]): +class _TypedKeyDict(dict): """Dictionary that enforces key type. Doesn't override everything, but probably good enough to catch most @@ -80,7 +73,7 @@ class _TypedKeyDict(dict[K, V]): Subclass and override KEY_TYPE. """ - KEY_TYPE: Type[K] + KEY_TYPE = object def __init__(self, *args, **kwargs): dict.__init__(self, *args, **kwargs) @@ -105,13 +98,13 @@ for _method_name in ["__setitem__", "__getitem__", "setdefault", "get", del _method_name -class BytesKeyDict(_TypedKeyDict[bytes, V]): +class BytesKeyDict(_TypedKeyDict): """Keys should be bytes.""" KEY_TYPE = bytes -class UnicodeKeyDict(_TypedKeyDict[str, V]): +class UnicodeKeyDict(_TypedKeyDict): """Keys should be unicode strings.""" KEY_TYPE = str From 22a7aacb9da710d8112d5f40a9bf5f7e82b2e138 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 2 Dec 2022 10:36:41 -0700 Subject: [PATCH 1239/2309] flake8 --- src/allmydata/test/test_storage_client.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/allmydata/test/test_storage_client.py b/src/allmydata/test/test_storage_client.py index 04f2c7e29..a92e6723f 100644 --- a/src/allmydata/test/test_storage_client.py +++ b/src/allmydata/test/test_storage_client.py @@ -77,7 +77,6 @@ from .common import ( UseNode, SameProcessStreamEndpointAssigner, MemoryIntroducerClient, - flush_logged_errors, ) from .common_web import ( do_http, From 11fb194d74599dc3f27f31b14ba340acbd3a2615 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 9 Dec 2022 14:00:41 -0500 Subject: [PATCH 1240/2309] kick ci --- newsfragments/3942.minor | 1 + 1 file changed, 1 insertion(+) diff --git a/newsfragments/3942.minor b/newsfragments/3942.minor index e69de29bb..8b1378917 100644 --- a/newsfragments/3942.minor +++ b/newsfragments/3942.minor @@ -0,0 +1 @@ + From 88ee978d98aea740ddf357acf08c85768bd0e950 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 9 Dec 2022 14:06:24 -0500 Subject: [PATCH 1241/2309] Some features we depend on are broken in tox 4 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 768e44e29..8558abd02 100644 --- a/setup.py +++ b/setup.py @@ -396,7 +396,7 @@ setup(name="tahoe-lafs", # also set in __init__.py "pyflakes == 2.2.0", "coverage ~= 5.0", "mock", - "tox", + "tox ~= 3.0", "pytest", "pytest-twisted", "hypothesis >= 3.6.1", From 6485eb5186190a5e73eb55f05b66a42ffb6655ff Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 9 Dec 2022 14:07:38 -0500 Subject: [PATCH 1242/2309] Also constrain tox here --- .circleci/config.yml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 051e690b7..d7e4f2563 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -133,10 +133,10 @@ jobs: steps: - "checkout" - - run: + - run: &INSTALL_TOX name: "Install tox" command: | - pip install --user tox + pip install --user 'tox~=3.0' - run: name: "Static-ish code checks" @@ -152,9 +152,7 @@ jobs: - "checkout" - run: - name: "Install tox" - command: | - pip install --user tox + <<: *INSTALL_TOX - run: name: "Make PyInstaller executable" From f6a46c86d24b59110d148b2879c2d7e6647d0501 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 9 Dec 2022 14:11:59 -0500 Subject: [PATCH 1243/2309] Populate the wheelhouse with a working version of tox --- .circleci/populate-wheelhouse.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/populate-wheelhouse.sh b/.circleci/populate-wheelhouse.sh index 519a80cac..39bf4ae4c 100755 --- a/.circleci/populate-wheelhouse.sh +++ b/.circleci/populate-wheelhouse.sh @@ -9,7 +9,7 @@ BASIC_DEPS="pip wheel" # Python packages we need to support the test infrastructure. *Not* packages # Tahoe-LAFS itself (implementation or test suite) need. -TEST_DEPS="tox codecov" +TEST_DEPS="'tox~=3.0' codecov" # Python packages we need to generate test reports for CI infrastructure. # *Not* packages Tahoe-LAFS itself (implement or test suite) need. From 13aa000d0b4c42c550e42a7d85ad2a0035b7b56d Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 9 Dec 2022 14:06:24 -0500 Subject: [PATCH 1244/2309] Some features we depend on are broken in tox 4 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 768e44e29..8558abd02 100644 --- a/setup.py +++ b/setup.py @@ -396,7 +396,7 @@ setup(name="tahoe-lafs", # also set in __init__.py "pyflakes == 2.2.0", "coverage ~= 5.0", "mock", - "tox", + "tox ~= 3.0", "pytest", "pytest-twisted", "hypothesis >= 3.6.1", From 666cd24c2b07e5a4ea70100ec3a1554296c47507 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 9 Dec 2022 14:07:38 -0500 Subject: [PATCH 1245/2309] Also constrain tox here --- .circleci/config.yml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 051e690b7..d7e4f2563 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -133,10 +133,10 @@ jobs: steps: - "checkout" - - run: + - run: &INSTALL_TOX name: "Install tox" command: | - pip install --user tox + pip install --user 'tox~=3.0' - run: name: "Static-ish code checks" @@ -152,9 +152,7 @@ jobs: - "checkout" - run: - name: "Install tox" - command: | - pip install --user tox + <<: *INSTALL_TOX - run: name: "Make PyInstaller executable" From 43c044a11b8c98565dcc035608ec82a18affcf08 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 9 Dec 2022 14:13:29 -0500 Subject: [PATCH 1246/2309] build me the images --- .circleci/config.yml | 106 +++++++++++++++++++++---------------------- 1 file changed, 53 insertions(+), 53 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index d7e4f2563..89748c5aa 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -14,69 +14,69 @@ version: 2.1 workflows: ci: jobs: - # Start with jobs testing various platforms. - - "debian-10": - {} - - "debian-11": - {} + # # Start with jobs testing various platforms. + # - "debian-10": + # {} + # - "debian-11": + # {} - - "ubuntu-20-04": - {} - - "ubuntu-18-04": - requires: - - "ubuntu-20-04" + # - "ubuntu-20-04": + # {} + # - "ubuntu-18-04": + # requires: + # - "ubuntu-20-04" - # Equivalent to RHEL 8; CentOS 8 is dead. - - "oraclelinux-8": - {} + # # Equivalent to RHEL 8; CentOS 8 is dead. + # - "oraclelinux-8": + # {} - - "nixos": - name: "NixOS 21.05" - nixpkgs: "21.05" + # - "nixos": + # name: "NixOS 21.05" + # nixpkgs: "21.05" - - "nixos": - name: "NixOS 21.11" - nixpkgs: "21.11" + # - "nixos": + # name: "NixOS 21.11" + # nixpkgs: "21.11" - # Eventually, test against PyPy 3.8 - #- "pypy27-buster": - # {} + # # Eventually, test against PyPy 3.8 + # #- "pypy27-buster": + # # {} - # Other assorted tasks and configurations - - "codechecks": - {} - - "pyinstaller": - {} - - "c-locale": - {} - # Any locale other than C or UTF-8. - - "another-locale": - {} + # # Other assorted tasks and configurations + # - "codechecks": + # {} + # - "pyinstaller": + # {} + # - "c-locale": + # {} + # # Any locale other than C or UTF-8. + # - "another-locale": + # {} - - "integration": - requires: - # If the unit test suite doesn't pass, don't bother running the - # integration tests. - - "debian-11" + # - "integration": + # requires: + # # If the unit test suite doesn't pass, don't bother running the + # # integration tests. + # - "debian-11" - - "typechecks": - {} - - "docs": - {} + # - "typechecks": + # {} + # - "docs": + # {} - images: - # Build the Docker images used by the ci jobs. This makes the ci jobs - # faster and takes various spurious failures out of the critical path. - triggers: - # Build once a day - - schedule: - cron: "0 0 * * *" - filters: - branches: - only: - - "master" + # images: + # # Build the Docker images used by the ci jobs. This makes the ci jobs + # # faster and takes various spurious failures out of the critical path. + # triggers: + # # Build once a day + # - schedule: + # cron: "0 0 * * *" + # filters: + # branches: + # only: + # - "master" - jobs: + # jobs: # Every job that pushes a Docker image from Docker Hub needs to provide # credentials. Use this first job to define a yaml anchor that can be # used to supply a CircleCI job context which makes Docker Hub From e835ed538fe81876898b36cbccd5fd32bac75554 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 9 Dec 2022 14:18:40 -0500 Subject: [PATCH 1247/2309] Okay don't quote it then --- .circleci/populate-wheelhouse.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/populate-wheelhouse.sh b/.circleci/populate-wheelhouse.sh index 39bf4ae4c..857171979 100755 --- a/.circleci/populate-wheelhouse.sh +++ b/.circleci/populate-wheelhouse.sh @@ -9,7 +9,7 @@ BASIC_DEPS="pip wheel" # Python packages we need to support the test infrastructure. *Not* packages # Tahoe-LAFS itself (implementation or test suite) need. -TEST_DEPS="'tox~=3.0' codecov" +TEST_DEPS="tox~=3.0 codecov" # Python packages we need to generate test reports for CI infrastructure. # *Not* packages Tahoe-LAFS itself (implement or test suite) need. From d5380fe1569f6548df37002949b0169a55ca4151 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 9 Dec 2022 14:27:37 -0500 Subject: [PATCH 1248/2309] regular ci config --- .circleci/config.yml | 106 +++++++++++++++++++++---------------------- 1 file changed, 53 insertions(+), 53 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 89748c5aa..d7e4f2563 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -14,69 +14,69 @@ version: 2.1 workflows: ci: jobs: - # # Start with jobs testing various platforms. - # - "debian-10": - # {} - # - "debian-11": - # {} + # Start with jobs testing various platforms. + - "debian-10": + {} + - "debian-11": + {} - # - "ubuntu-20-04": - # {} - # - "ubuntu-18-04": - # requires: - # - "ubuntu-20-04" + - "ubuntu-20-04": + {} + - "ubuntu-18-04": + requires: + - "ubuntu-20-04" - # # Equivalent to RHEL 8; CentOS 8 is dead. - # - "oraclelinux-8": - # {} + # Equivalent to RHEL 8; CentOS 8 is dead. + - "oraclelinux-8": + {} - # - "nixos": - # name: "NixOS 21.05" - # nixpkgs: "21.05" + - "nixos": + name: "NixOS 21.05" + nixpkgs: "21.05" - # - "nixos": - # name: "NixOS 21.11" - # nixpkgs: "21.11" + - "nixos": + name: "NixOS 21.11" + nixpkgs: "21.11" - # # Eventually, test against PyPy 3.8 - # #- "pypy27-buster": - # # {} + # Eventually, test against PyPy 3.8 + #- "pypy27-buster": + # {} - # # Other assorted tasks and configurations - # - "codechecks": - # {} - # - "pyinstaller": - # {} - # - "c-locale": - # {} - # # Any locale other than C or UTF-8. - # - "another-locale": - # {} + # Other assorted tasks and configurations + - "codechecks": + {} + - "pyinstaller": + {} + - "c-locale": + {} + # Any locale other than C or UTF-8. + - "another-locale": + {} - # - "integration": - # requires: - # # If the unit test suite doesn't pass, don't bother running the - # # integration tests. - # - "debian-11" + - "integration": + requires: + # If the unit test suite doesn't pass, don't bother running the + # integration tests. + - "debian-11" - # - "typechecks": - # {} - # - "docs": - # {} + - "typechecks": + {} + - "docs": + {} - # images: - # # Build the Docker images used by the ci jobs. This makes the ci jobs - # # faster and takes various spurious failures out of the critical path. - # triggers: - # # Build once a day - # - schedule: - # cron: "0 0 * * *" - # filters: - # branches: - # only: - # - "master" + images: + # Build the Docker images used by the ci jobs. This makes the ci jobs + # faster and takes various spurious failures out of the critical path. + triggers: + # Build once a day + - schedule: + cron: "0 0 * * *" + filters: + branches: + only: + - "master" - # jobs: + jobs: # Every job that pushes a Docker image from Docker Hub needs to provide # credentials. Use this first job to define a yaml anchor that can be # used to supply a CircleCI job context which makes Docker Hub From ea0426318ea695def6a66593ae44d372b17d194e Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 12 Dec 2022 10:02:43 -0500 Subject: [PATCH 1249/2309] news fragment --- newsfragments/3950.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3950.minor diff --git a/newsfragments/3950.minor b/newsfragments/3950.minor new file mode 100644 index 000000000..e69de29bb From 98e25507df5fdde29b5047c7d325607cb3906b5a Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 12 Dec 2022 10:43:36 -0500 Subject: [PATCH 1250/2309] A different approach to forcing foolscap in integration tests. --- .github/workflows/ci.yml | 27 +++++++++++++-------------- integration/conftest.py | 16 +++++++--------- integration/util.py | 17 ++++++++--------- src/allmydata/protocol_switch.py | 16 ---------------- src/allmydata/testing/__init__.py | 18 ------------------ 5 files changed, 28 insertions(+), 66 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 41de7baed..afbe5c7a4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -161,19 +161,21 @@ jobs: strategy: fail-fast: false matrix: - os: - - windows-latest - # 22.04 has some issue with Tor at the moment: - # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3943 - - ubuntu-20.04 - python-version: - - 3.7 - - 3.9 include: - # On macOS don't bother with 3.7, just to get faster builds. - os: macos-latest python-version: 3.9 - + extra-tox-options: "" + - os: windows-latest + python-version: 3.10 + extra-tox-options: "" + # 22.04 has some issue with Tor at the moment: + # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3943 + - os: ubuntu-20.04 + python-version: 3.8 + extra-tox-options: "--force-foolscap integration/" + - os: ubuntu-20.04 + python-version: 3.10 + extra-tox-options: "" steps: - name: Install Tor [Ubuntu] @@ -232,10 +234,7 @@ jobs: # effect. On Linux it doesn't make a difference one way or another. TMPDIR: "/tmp" run: | - # Run with Foolscap forced: - __TAHOE_INTEGRATION_FORCE_FOOLSCAP=1 tox -e integration - # Run with Foolscap not forced, which should result in HTTP being used. - __TAHOE_INTEGRATION_FORCE_FOOLSCAP=0 tox -e integration + tox -e integration ${{ matrix.extra-tox-options }} - name: Upload eliot.log in case of failure uses: actions/upload-artifact@v1 diff --git a/integration/conftest.py b/integration/conftest.py index e284b5cba..5cbe9ad6b 100644 --- a/integration/conftest.py +++ b/integration/conftest.py @@ -1,15 +1,6 @@ """ Ported to Python 3. """ -from __future__ import unicode_literals -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - import sys import shutil from time import sleep @@ -66,6 +57,13 @@ def pytest_addoption(parser): "--coverage", action="store_true", dest="coverage", help="Collect coverage statistics", ) + parser.addoption( + "--force-foolscap", action="store_true", default=False, + dest="force_foolscap", + help=("If set, force Foolscap only for the storage protocol. " + + "Otherwise HTTP will be used.") + ) + @pytest.fixture(autouse=True, scope='session') def eliot_logging(): diff --git a/integration/util.py b/integration/util.py index cde837218..7d885ee6c 100644 --- a/integration/util.py +++ b/integration/util.py @@ -30,7 +30,6 @@ from allmydata.util.configutil import ( write_config, ) from allmydata import client -from allmydata.testing import foolscap_only_for_integration_testing import pytest_twisted @@ -293,14 +292,14 @@ def _create_node(reactor, request, temp_dir, introducer_furl, flog_gatherer, nam u'log_gatherer.furl', flog_gatherer, ) - force_foolscap = foolscap_only_for_integration_testing() - if force_foolscap is not None: - set_config( - config, - 'storage', - 'force_foolscap', - str(force_foolscap), - ) + force_foolscap = request.config.getoption("force_foolscap") + assert force_foolscap in (True, False) + set_config( + config, + 'storage', + 'force_foolscap', + str(force_foolscap), + ) write_config(FilePath(config_path), config) created_d.addCallback(created) diff --git a/src/allmydata/protocol_switch.py b/src/allmydata/protocol_switch.py index d88863fdb..b0af84c33 100644 --- a/src/allmydata/protocol_switch.py +++ b/src/allmydata/protocol_switch.py @@ -30,7 +30,6 @@ from foolscap.api import Tub from .storage.http_server import HTTPServer, build_nurl from .storage.server import StorageServer -from .testing import foolscap_only_for_integration_testing class _PretendToBeNegotiation(type): @@ -171,21 +170,6 @@ class _FoolscapOrHttps(Protocol, metaclass=_PretendToBeNegotiation): # and later data, otherwise assume HTTPS. self._timeout.cancel() if self._buffer.startswith(b"GET /id/"): - if foolscap_only_for_integration_testing() == False: - # Tahoe will prefer HTTP storage protocol over Foolscap when possible. - # - # If this is branch is taken, we are running a test that should - # be using HTTP for the storage protocol. As such, we - # aggressively disable Foolscap to ensure that HTTP is in fact - # going to be used. If we hit this branch that means our - # expectation that HTTP will be used was wrong, suggesting a - # bug in either the code of the integration testing setup. - # - # This branch should never be hit in production! - self.transport.loseConnection() - print("FOOLSCAP IS DISABLED, I PITY THE FOOLS WHO SEE THIS MESSAGE") - return - # We're a Foolscap Negotiation server protocol instance: transport = self.transport buf = self._buffer diff --git a/src/allmydata/testing/__init__.py b/src/allmydata/testing/__init__.py index 119ae4101..e69de29bb 100644 --- a/src/allmydata/testing/__init__.py +++ b/src/allmydata/testing/__init__.py @@ -1,18 +0,0 @@ -import os -from typing import Optional - - -def foolscap_only_for_integration_testing() -> Optional[bool]: - """ - Return whether HTTP storage protocol has been disabled / Foolscap - forced, for purposes of integration testing. - - This is determined by the __TAHOE_INTEGRATION_FORCE_FOOLSCAP environment - variable, which can be 1, 0, or not set, corresponding to results of - ``True``, ``False`` and ``None`` (i.e. default). - """ - force_foolscap = os.environ.get("__TAHOE_INTEGRATION_FORCE_FOOLSCAP") - if force_foolscap is None: - return None - - return bool(int(force_foolscap)) From c5c616afd5146f8cde9dddead3bdbeb092890992 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 12 Dec 2022 10:44:49 -0500 Subject: [PATCH 1251/2309] Garbage. --- src/allmydata/test/test_storage_https.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/allmydata/test/test_storage_https.py b/src/allmydata/test/test_storage_https.py index 3d2a31143..a11b0eed5 100644 --- a/src/allmydata/test/test_storage_https.py +++ b/src/allmydata/test/test_storage_https.py @@ -181,10 +181,6 @@ class PinningHTTPSValidation(AsyncTestCase): response = await self.request(url, certificate) self.assertEqual(await response.content(), b"YOYODYNE") - # We keep getting TLSMemoryBIOProtocol being left around, so try harder - # to wait for it to finish. - await deferLater(reactor, 0.01) - @async_to_deferred async def test_server_certificate_not_valid_yet(self): """ From 106df423be0e864100ba2d96cdb91558d206160a Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 12 Dec 2022 10:52:01 -0500 Subject: [PATCH 1252/2309] Another approach. --- .github/workflows/ci.yml | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2ffc260df..8c3eaf29e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -156,18 +156,18 @@ jobs: include: - os: macos-latest python-version: 3.9 - extra-tox-options: "" + force-foolscap: false - os: windows-latest python-version: 3.10 - extra-tox-options: "" + force-foolscap: false # 22.04 has some issue with Tor at the moment: # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3943 - os: ubuntu-20.04 python-version: 3.8 - extra-tox-options: "--force-foolscap integration/" + force-foolscap: true - os: ubuntu-20.04 python-version: 3.10 - extra-tox-options: "" + force-foolscap: false steps: - name: Install Tor [Ubuntu] @@ -208,14 +208,24 @@ jobs: run: python misc/build_helpers/show-tool-versions.py - name: Run "Python 3 integration tests" + if: "${{ !matrix.force-foolscap }}" env: # On macOS this is necessary to ensure unix socket paths for tor # aren't too long. On Windows tox won't pass it through so it has no # effect. On Linux it doesn't make a difference one way or another. TMPDIR: "/tmp" run: | - tox -e integration ${{ matrix.extra-tox-options }} + tox -e integration + - name: Run "Python 3 integration tests (force Foolscap)" + if: "${{ matrix.force-foolscap }}" + env: + # On macOS this is necessary to ensure unix socket paths for tor + # aren't too long. On Windows tox won't pass it through so it has no + # effect. On Linux it doesn't make a difference one way or another. + TMPDIR: "/tmp" + run: | + tox -e integration -- --force-foolscap integration/ - name: Upload eliot.log in case of failure uses: actions/upload-artifact@v3 if: failure() From 742b352861629a0d1f1b900c4c71d6e1ba22a0f2 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 12 Dec 2022 10:52:17 -0500 Subject: [PATCH 1253/2309] Whitespace. --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8c3eaf29e..e87337a1b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -226,6 +226,7 @@ jobs: TMPDIR: "/tmp" run: | tox -e integration -- --force-foolscap integration/ + - name: Upload eliot.log in case of failure uses: actions/upload-artifact@v3 if: failure() From d05a1313d1773d8f4bf7041d51f4bca1ab0de4b3 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 12 Dec 2022 10:54:23 -0500 Subject: [PATCH 1254/2309] Don't change versions for now, use strings so it'll be future compatible with 3.10. --- .github/workflows/ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e87337a1b..01f0890da 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -155,18 +155,18 @@ jobs: matrix: include: - os: macos-latest - python-version: 3.9 + python-version: "3.9" force-foolscap: false - os: windows-latest - python-version: 3.10 + python-version: "3.9" force-foolscap: false # 22.04 has some issue with Tor at the moment: # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3943 - os: ubuntu-20.04 - python-version: 3.8 + python-version: "3.7" force-foolscap: true - os: ubuntu-20.04 - python-version: 3.10 + python-version: "3.9" force-foolscap: false steps: From 366cbf90017874ec24d60bd0df3b3d9f75d79182 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 12 Dec 2022 10:55:07 -0500 Subject: [PATCH 1255/2309] Tox is bad? --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 01f0890da..37f41d06b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -225,7 +225,7 @@ jobs: # effect. On Linux it doesn't make a difference one way or another. TMPDIR: "/tmp" run: | - tox -e integration -- --force-foolscap integration/ + tox -e integration -- --force-foolscap,integration/ - name: Upload eliot.log in case of failure uses: actions/upload-artifact@v3 From 6a1f49551b6683f64b110c2060dcd309c21bdd8d Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 12 Dec 2022 11:05:09 -0500 Subject: [PATCH 1256/2309] No, that's not it. --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 37f41d06b..01f0890da 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -225,7 +225,7 @@ jobs: # effect. On Linux it doesn't make a difference one way or another. TMPDIR: "/tmp" run: | - tox -e integration -- --force-foolscap,integration/ + tox -e integration -- --force-foolscap integration/ - name: Upload eliot.log in case of failure uses: actions/upload-artifact@v3 From b8680750daa6af924d4da4549780f9cdbc224605 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 12 Dec 2022 11:47:32 -0500 Subject: [PATCH 1257/2309] pin it in more places --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 15e7d8fa4..c1e0c9391 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -80,7 +80,7 @@ jobs: - name: Install Python packages run: | - pip install --upgrade codecov tox tox-gh-actions setuptools + pip install --upgrade codecov "tox<4" tox-gh-actions setuptools pip list - name: Display tool versions @@ -199,7 +199,7 @@ jobs: - name: Install Python packages run: | - pip install --upgrade tox + pip install --upgrade "tox<4" pip list - name: Display tool versions @@ -247,7 +247,7 @@ jobs: - name: Install Python packages run: | - pip install --upgrade tox + pip install --upgrade "tox<4" pip list - name: Display tool versions From 16d14c8688fb6d09760001777d9bf0548b454633 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 12 Dec 2022 11:54:23 -0500 Subject: [PATCH 1258/2309] A minimal upload/download script. --- benchmarks/test_immutable.py | 47 ++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 benchmarks/test_immutable.py diff --git a/benchmarks/test_immutable.py b/benchmarks/test_immutable.py new file mode 100644 index 000000000..15ba0cb4f --- /dev/null +++ b/benchmarks/test_immutable.py @@ -0,0 +1,47 @@ +""" +First attempt at benchmarking immutable uploads and downloads. + +TODO Parameterization (pytest?) +- Foolscap vs not foolscap +- Number of nodes +- Data size +- Number of needed/happy/total shares. +""" + +from twisted.trial.unittest import TestCase + +from allmydata.util.deferredutil import async_to_deferred +from allmydata.util.consumer import MemoryConsumer +from allmydata.test.common_system import SystemTestMixin +from allmydata.immutable.upload import Data as UData + + +class ImmutableBenchmarks(SystemTestMixin, TestCase): + """Benchmarks for immutables.""" + + FORCE_FOOLSCAP_FOR_STORAGE = True + + @async_to_deferred + async def test_upload_and_download(self): + self.basedir = self.mktemp() + + DATA = b"Some data to upload\n" * 2000 + + # 3 nodes + await self.set_up_nodes(3) + + # 3 shares + for c in self.clients: + c.encoding_params["k"] = 3 + c.encoding_params["happy"] = 3 + c.encoding_params["n"] = 3 + + # 1. Upload: + uploader = self.clients[0].getServiceNamed("uploader") + results = await uploader.upload(UData(DATA, convergence=None)) + + # 2. Download: + uri = results.get_uri() + node = self.clients[1].create_node_from_uri(uri) + mc = await node.read(MemoryConsumer(), 0, None) + self.assertEqual(b"".join(mc.chunks), DATA) From a3e02f14c93f5e532f15b9c9c1ed20a516f2ac9d Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 12 Dec 2022 13:38:25 -0500 Subject: [PATCH 1259/2309] Add a little timing. --- benchmarks/test_immutable.py | 37 +++++++++++++++++++++++++----------- 1 file changed, 26 insertions(+), 11 deletions(-) diff --git a/benchmarks/test_immutable.py b/benchmarks/test_immutable.py index 15ba0cb4f..abf4bddf3 100644 --- a/benchmarks/test_immutable.py +++ b/benchmarks/test_immutable.py @@ -8,6 +8,9 @@ TODO Parameterization (pytest?) - Number of needed/happy/total shares. """ +from time import time +from contextlib import contextmanager + from twisted.trial.unittest import TestCase from allmydata.util.deferredutil import async_to_deferred @@ -16,16 +19,25 @@ from allmydata.test.common_system import SystemTestMixin from allmydata.immutable.upload import Data as UData +@contextmanager +def timeit(name): + start = time() + try: + yield + finally: + print(f"{name}: {time() - start:.3f}") + + class ImmutableBenchmarks(SystemTestMixin, TestCase): """Benchmarks for immutables.""" - FORCE_FOOLSCAP_FOR_STORAGE = True + FORCE_FOOLSCAP_FOR_STORAGE = False @async_to_deferred async def test_upload_and_download(self): self.basedir = self.mktemp() - - DATA = b"Some data to upload\n" * 2000 + + DATA = b"Some data to upload\n" * 10 # 3 nodes await self.set_up_nodes(3) @@ -36,12 +48,15 @@ class ImmutableBenchmarks(SystemTestMixin, TestCase): c.encoding_params["happy"] = 3 c.encoding_params["n"] = 3 - # 1. Upload: - uploader = self.clients[0].getServiceNamed("uploader") - results = await uploader.upload(UData(DATA, convergence=None)) + for i in range(5): + # 1. Upload: + with timeit("upload"): + uploader = self.clients[0].getServiceNamed("uploader") + results = await uploader.upload(UData(DATA, convergence=None)) - # 2. Download: - uri = results.get_uri() - node = self.clients[1].create_node_from_uri(uri) - mc = await node.read(MemoryConsumer(), 0, None) - self.assertEqual(b"".join(mc.chunks), DATA) + # 2. Download: + with timeit("download"): + uri = results.get_uri() + node = self.clients[1].create_node_from_uri(uri) + mc = await node.read(MemoryConsumer(), 0, None) + self.assertEqual(b"".join(mc.chunks), DATA) From 8282fce4cdd574ec4b8cc849070ed4ac2ee03cc8 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 13 Dec 2022 08:57:21 -0500 Subject: [PATCH 1260/2309] build the images again --- .circleci/config.yml | 106 +++++++++++++++++++++---------------------- 1 file changed, 53 insertions(+), 53 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index d7e4f2563..89748c5aa 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -14,69 +14,69 @@ version: 2.1 workflows: ci: jobs: - # Start with jobs testing various platforms. - - "debian-10": - {} - - "debian-11": - {} + # # Start with jobs testing various platforms. + # - "debian-10": + # {} + # - "debian-11": + # {} - - "ubuntu-20-04": - {} - - "ubuntu-18-04": - requires: - - "ubuntu-20-04" + # - "ubuntu-20-04": + # {} + # - "ubuntu-18-04": + # requires: + # - "ubuntu-20-04" - # Equivalent to RHEL 8; CentOS 8 is dead. - - "oraclelinux-8": - {} + # # Equivalent to RHEL 8; CentOS 8 is dead. + # - "oraclelinux-8": + # {} - - "nixos": - name: "NixOS 21.05" - nixpkgs: "21.05" + # - "nixos": + # name: "NixOS 21.05" + # nixpkgs: "21.05" - - "nixos": - name: "NixOS 21.11" - nixpkgs: "21.11" + # - "nixos": + # name: "NixOS 21.11" + # nixpkgs: "21.11" - # Eventually, test against PyPy 3.8 - #- "pypy27-buster": - # {} + # # Eventually, test against PyPy 3.8 + # #- "pypy27-buster": + # # {} - # Other assorted tasks and configurations - - "codechecks": - {} - - "pyinstaller": - {} - - "c-locale": - {} - # Any locale other than C or UTF-8. - - "another-locale": - {} + # # Other assorted tasks and configurations + # - "codechecks": + # {} + # - "pyinstaller": + # {} + # - "c-locale": + # {} + # # Any locale other than C or UTF-8. + # - "another-locale": + # {} - - "integration": - requires: - # If the unit test suite doesn't pass, don't bother running the - # integration tests. - - "debian-11" + # - "integration": + # requires: + # # If the unit test suite doesn't pass, don't bother running the + # # integration tests. + # - "debian-11" - - "typechecks": - {} - - "docs": - {} + # - "typechecks": + # {} + # - "docs": + # {} - images: - # Build the Docker images used by the ci jobs. This makes the ci jobs - # faster and takes various spurious failures out of the critical path. - triggers: - # Build once a day - - schedule: - cron: "0 0 * * *" - filters: - branches: - only: - - "master" + # images: + # # Build the Docker images used by the ci jobs. This makes the ci jobs + # # faster and takes various spurious failures out of the critical path. + # triggers: + # # Build once a day + # - schedule: + # cron: "0 0 * * *" + # filters: + # branches: + # only: + # - "master" - jobs: + # jobs: # Every job that pushes a Docker image from Docker Hub needs to provide # credentials. Use this first job to define a yaml anchor that can be # used to supply a CircleCI job context which makes Docker Hub From 815c998c3323829f066752be0f3f707e2716a490 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 13 Dec 2022 09:09:02 -0500 Subject: [PATCH 1261/2309] regular ci --- .circleci/config.yml | 106 +++++++++++++++++++++---------------------- 1 file changed, 53 insertions(+), 53 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 89748c5aa..d7e4f2563 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -14,69 +14,69 @@ version: 2.1 workflows: ci: jobs: - # # Start with jobs testing various platforms. - # - "debian-10": - # {} - # - "debian-11": - # {} + # Start with jobs testing various platforms. + - "debian-10": + {} + - "debian-11": + {} - # - "ubuntu-20-04": - # {} - # - "ubuntu-18-04": - # requires: - # - "ubuntu-20-04" + - "ubuntu-20-04": + {} + - "ubuntu-18-04": + requires: + - "ubuntu-20-04" - # # Equivalent to RHEL 8; CentOS 8 is dead. - # - "oraclelinux-8": - # {} + # Equivalent to RHEL 8; CentOS 8 is dead. + - "oraclelinux-8": + {} - # - "nixos": - # name: "NixOS 21.05" - # nixpkgs: "21.05" + - "nixos": + name: "NixOS 21.05" + nixpkgs: "21.05" - # - "nixos": - # name: "NixOS 21.11" - # nixpkgs: "21.11" + - "nixos": + name: "NixOS 21.11" + nixpkgs: "21.11" - # # Eventually, test against PyPy 3.8 - # #- "pypy27-buster": - # # {} + # Eventually, test against PyPy 3.8 + #- "pypy27-buster": + # {} - # # Other assorted tasks and configurations - # - "codechecks": - # {} - # - "pyinstaller": - # {} - # - "c-locale": - # {} - # # Any locale other than C or UTF-8. - # - "another-locale": - # {} + # Other assorted tasks and configurations + - "codechecks": + {} + - "pyinstaller": + {} + - "c-locale": + {} + # Any locale other than C or UTF-8. + - "another-locale": + {} - # - "integration": - # requires: - # # If the unit test suite doesn't pass, don't bother running the - # # integration tests. - # - "debian-11" + - "integration": + requires: + # If the unit test suite doesn't pass, don't bother running the + # integration tests. + - "debian-11" - # - "typechecks": - # {} - # - "docs": - # {} + - "typechecks": + {} + - "docs": + {} - # images: - # # Build the Docker images used by the ci jobs. This makes the ci jobs - # # faster and takes various spurious failures out of the critical path. - # triggers: - # # Build once a day - # - schedule: - # cron: "0 0 * * *" - # filters: - # branches: - # only: - # - "master" + images: + # Build the Docker images used by the ci jobs. This makes the ci jobs + # faster and takes various spurious failures out of the critical path. + triggers: + # Build once a day + - schedule: + cron: "0 0 * * *" + filters: + branches: + only: + - "master" - # jobs: + jobs: # Every job that pushes a Docker image from Docker Hub needs to provide # credentials. Use this first job to define a yaml anchor that can be # used to supply a CircleCI job context which makes Docker Hub From 5266adefce55b3ac565fe20e1c9b1e46d99ad76c Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 13 Dec 2022 10:57:07 -0500 Subject: [PATCH 1262/2309] Include CPU time. --- benchmarks/test_immutable.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/benchmarks/test_immutable.py b/benchmarks/test_immutable.py index abf4bddf3..abdd11035 100644 --- a/benchmarks/test_immutable.py +++ b/benchmarks/test_immutable.py @@ -8,7 +8,7 @@ TODO Parameterization (pytest?) - Number of needed/happy/total shares. """ -from time import time +from time import time, process_time from contextlib import contextmanager from twisted.trial.unittest import TestCase @@ -22,10 +22,11 @@ from allmydata.immutable.upload import Data as UData @contextmanager def timeit(name): start = time() + start_cpu = process_time() try: yield finally: - print(f"{name}: {time() - start:.3f}") + print(f"{name}: {time() - start:.3f} elapsed, {process_time() - start_cpu:.3f} CPU") class ImmutableBenchmarks(SystemTestMixin, TestCase): From be3ace7adebffaa5d410e3f8e248b6db08bd7d50 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 13 Dec 2022 15:39:04 -0500 Subject: [PATCH 1263/2309] News file. --- newsfragments/3954.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3954.minor diff --git a/newsfragments/3954.minor b/newsfragments/3954.minor new file mode 100644 index 000000000..e69de29bb From 6ae40a932d5504edc66176fc0fbdef45998dec77 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 13 Dec 2022 15:54:19 -0500 Subject: [PATCH 1264/2309] A much more reasonable number of HTTP connections. --- src/allmydata/storage/http_client.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 79bf061c9..90bda7fc0 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -323,6 +323,7 @@ class StorageClient(object): swissnum = nurl.path[0].encode("ascii") certificate_hash = nurl.user.encode("ascii") pool = HTTPConnectionPool(reactor) + pool.maxPersistentPerHost = 20 if cls.TEST_MODE_REGISTER_HTTP_POOL is not None: cls.TEST_MODE_REGISTER_HTTP_POOL(pool) From 2057f59950fcbd6576d530526d41f9835e42ec7c Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 14 Dec 2022 08:35:06 -0500 Subject: [PATCH 1265/2309] news fragment --- newsfragments/3953.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3953.minor diff --git a/newsfragments/3953.minor b/newsfragments/3953.minor new file mode 100644 index 000000000..e69de29bb From a1cb8893083d06da0c7f1bca760e3333334acac3 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 14 Dec 2022 08:35:10 -0500 Subject: [PATCH 1266/2309] Take typechecks and codechecks out of the GitHub Actions config There's a dedicated job on CircleCI. --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index db4748033..96eed4e40 100644 --- a/tox.ini +++ b/tox.ini @@ -7,7 +7,7 @@ # the tox-gh-actions package. [gh-actions] python = - 3.7: py37-coverage,typechecks,codechecks + 3.7: py37-coverage 3.8: py38-coverage 3.9: py39-coverage 3.10: py310-coverage From 2677f26455f2b91f13e8c453b91f43d9c08f0527 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 14 Dec 2022 08:46:39 -0500 Subject: [PATCH 1267/2309] news fragment --- newsfragments/3914.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3914.minor diff --git a/newsfragments/3914.minor b/newsfragments/3914.minor new file mode 100644 index 000000000..e69de29bb From 05c7450376bbbfc4cfe0fb977265b9a1365cf588 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 14 Dec 2022 08:47:05 -0500 Subject: [PATCH 1268/2309] Try to use an upcoming python-cryptography feature to avoid some costs If the key is the wrong number of bits then we don't care about any other validation results because we're just going to reject it. So, check that before applying other validation, if possible. This is untested since the version of python-cryptography that supports it is not released yet and I don't feel like setting up a Rust build tool chain at the moment. --- src/allmydata/crypto/rsa.py | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/src/allmydata/crypto/rsa.py b/src/allmydata/crypto/rsa.py index 95cf01413..96885cfa1 100644 --- a/src/allmydata/crypto/rsa.py +++ b/src/allmydata/crypto/rsa.py @@ -72,20 +72,39 @@ def create_signing_keypair_from_string(private_key_der): :returns: 2-tuple of (private_key, public_key) """ - priv_key = load_der_private_key( + load = partial( + load_der_private_key, private_key_der, password=None, backend=default_backend(), ) - if not isinstance(priv_key, rsa.RSAPrivateKey): + + try: + # Load it once without the potentially expensive OpenSSL validation + # checks. These have superlinear complexity. We *will* run them just + # below - but first we'll apply our own constant-time checks. + unsafe_priv_key = load(unsafe_skip_rsa_key_validation=True) + except TypeError: + # cryptography<39 does not support this parameter, so just load the + # key with validation... + unsafe_priv_key = load() + # But avoid *reloading* it since that will run the expensive + # validation *again*. + load = lambda: unsafe_priv_key + + if not isinstance(unsafe_priv_key, rsa.RSAPrivateKey): raise ValueError( "Private Key did not decode to an RSA key" ) - if priv_key.key_size != 2048: + if unsafe_priv_key.key_size != 2048: raise ValueError( "Private Key must be 2048 bits" ) - return priv_key, priv_key.public_key() + + # Now re-load it with OpenSSL's validation applied. + safe_priv_key = load() + + return safe_priv_key, safe_priv_key.public_key() def der_string_from_signing_key(private_key): From c014ad55b1aa42295db7295f7a7d99092fee39fd Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 14 Dec 2022 08:48:02 -0500 Subject: [PATCH 1269/2309] remove Python 2 boilerplate --- src/allmydata/crypto/rsa.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/src/allmydata/crypto/rsa.py b/src/allmydata/crypto/rsa.py index 96885cfa1..cdd9a6035 100644 --- a/src/allmydata/crypto/rsa.py +++ b/src/allmydata/crypto/rsa.py @@ -12,14 +12,9 @@ on any of their methods. Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals +from __future__ import annotations -from future.utils import PY2 -if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 +from functools import partial from cryptography.exceptions import InvalidSignature from cryptography.hazmat.backends import default_backend From 761bf9cb9c03313c2d378c7e08cda44468c05ac3 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 14 Dec 2022 09:49:50 -0500 Subject: [PATCH 1270/2309] See if we can get a triggered image build too --- .circleci/config.yml | 72 ++++++++++++++++++++++++++++---------------- 1 file changed, 46 insertions(+), 26 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index d7e4f2563..446c6b3a9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -11,8 +11,45 @@ # version: 2.1 +# A template that can be shared between the two different image-building +# workflows. +.images: &IMAGES + jobs: + # Every job that pushes a Docker image from Docker Hub needs to provide + # credentials. Use this first job to define a yaml anchor that can be + # used to supply a CircleCI job context which makes Docker Hub credentials + # available in the environment. + # + # Contexts are managed in the CircleCI web interface: + # + # https://app.circleci.com/settings/organization/github/tahoe-lafs/contexts + - "build-image-debian-10": &DOCKERHUB_CONTEXT + context: "dockerhub-auth" + - "build-image-debian-11": + <<: *DOCKERHUB_CONTEXT + - "build-image-ubuntu-18-04": + <<: *DOCKERHUB_CONTEXT + - "build-image-ubuntu-20-04": + <<: *DOCKERHUB_CONTEXT + - "build-image-fedora-35": + <<: *DOCKERHUB_CONTEXT + - "build-image-oraclelinux-8": + <<: *DOCKERHUB_CONTEXT + # Restore later as PyPy38 + #- "build-image-pypy27-buster": + # <<: *DOCKERHUB_CONTEXT + +parameters: + build-images: + default: false + type: "boolean" + run-tests: + default: true + type: "boolean" + workflows: ci: + when: "<< pipeline.parameters.run-tests >>" jobs: # Start with jobs testing various platforms. - "debian-10": @@ -64,7 +101,15 @@ workflows: - "docs": {} - images: + triggered-images: + <<: *IMAGES + + # Build as part of the workflow but only if requested. + when: "<< pipeline.parameters.build-images >>" + + scheduled-images: + <<: *IMAGES + # Build the Docker images used by the ci jobs. This makes the ci jobs # faster and takes various spurious failures out of the critical path. triggers: @@ -76,31 +121,6 @@ workflows: only: - "master" - jobs: - # Every job that pushes a Docker image from Docker Hub needs to provide - # credentials. Use this first job to define a yaml anchor that can be - # used to supply a CircleCI job context which makes Docker Hub - # credentials available in the environment. - # - # Contexts are managed in the CircleCI web interface: - # - # https://app.circleci.com/settings/organization/github/tahoe-lafs/contexts - - "build-image-debian-10": &DOCKERHUB_CONTEXT - context: "dockerhub-auth" - - "build-image-debian-11": - <<: *DOCKERHUB_CONTEXT - - "build-image-ubuntu-18-04": - <<: *DOCKERHUB_CONTEXT - - "build-image-ubuntu-20-04": - <<: *DOCKERHUB_CONTEXT - - "build-image-fedora-35": - <<: *DOCKERHUB_CONTEXT - - "build-image-oraclelinux-8": - <<: *DOCKERHUB_CONTEXT - # Restore later as PyPy38 - #- "build-image-pypy27-buster": - # <<: *DOCKERHUB_CONTEXT - jobs: dockerhub-auth-template: From 1d248f4bd2d9fd9313cf419f94ec8ebbed31fdd0 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 14 Dec 2022 09:56:16 -0500 Subject: [PATCH 1271/2309] comments --- .circleci/config.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 446c6b3a9..b7c6cdbee 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -40,9 +40,17 @@ version: 2.1 # <<: *DOCKERHUB_CONTEXT parameters: + # Control whether the image-building workflow runs as part of this pipeline. + # Generally we do not want this to run because we don't need our + # dependencies to move around all the time and because building the image + # takes a couple minutes. build-images: default: false type: "boolean" + + # Control whether the test-running workflow runs as part of this pipeline. + # Generally we do want this to run because running the tests is the primary + # purpose of this pipeline. run-tests: default: true type: "boolean" From d66d928fb4abf7a81dce91fdca2583c32cd64cf9 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 14 Dec 2022 10:07:07 -0500 Subject: [PATCH 1272/2309] Provide a helper for rebuilding the images --- .circleci/rebuild-images.sh | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 .circleci/rebuild-images.sh diff --git a/.circleci/rebuild-images.sh b/.circleci/rebuild-images.sh new file mode 100644 index 000000000..7ee17b8ff --- /dev/null +++ b/.circleci/rebuild-images.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +set -euo pipefail + +# Get your API token here: +# https://app.circleci.com/settings/user/tokens +API_TOKEN=$1 +shift + +# Name the branch you want to trigger the build for +BRANCH=$1 +shift + +curl \ + --verbose \ + --request POST \ + --url https://circleci.com/api/v2/project/gh/tahoe-lafs/tahoe-lafs/pipeline \ + --header 'Circle-Token: $API_TOKEN' \ + --header 'content-type: application/json' \ + --data '{"branch":"$BRANCH","parameters":{"build-images":true,"run-tests":false}}' From edb2a2120439106502ff43a004b22afbfe4e207f Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 14 Dec 2022 11:18:17 -0500 Subject: [PATCH 1273/2309] For now, just one share. Plus more docs. --- benchmarks/test_immutable.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/benchmarks/test_immutable.py b/benchmarks/test_immutable.py index abdd11035..2098030a8 100644 --- a/benchmarks/test_immutable.py +++ b/benchmarks/test_immutable.py @@ -1,5 +1,5 @@ """ -First attempt at benchmarking immutable uploads and downloads. +First attempt at benchmarking uploads and downloads. TODO Parameterization (pytest?) - Foolscap vs not foolscap @@ -32,22 +32,24 @@ def timeit(name): class ImmutableBenchmarks(SystemTestMixin, TestCase): """Benchmarks for immutables.""" + # To use HTTP, change to true: FORCE_FOOLSCAP_FOR_STORAGE = False @async_to_deferred - async def test_upload_and_download(self): + async def test_upload_and_download_immutables(self): self.basedir = self.mktemp() + # To test larger files, change this: DATA = b"Some data to upload\n" * 10 - # 3 nodes - await self.set_up_nodes(3) + # 1 node + await self.set_up_nodes(1) - # 3 shares + # 1 share for c in self.clients: - c.encoding_params["k"] = 3 - c.encoding_params["happy"] = 3 - c.encoding_params["n"] = 3 + c.encoding_params["k"] = 1 + c.encoding_params["happy"] = 1 + c.encoding_params["n"] = 1 for i in range(5): # 1. Upload: From 983521c17ad9c3926cd48f6d92dfc7240a9e75cb Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 14 Dec 2022 11:22:05 -0500 Subject: [PATCH 1274/2309] Mutable benchmark. --- benchmarks/test_immutable.py | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/benchmarks/test_immutable.py b/benchmarks/test_immutable.py index 2098030a8..ea7624fcc 100644 --- a/benchmarks/test_immutable.py +++ b/benchmarks/test_immutable.py @@ -17,6 +17,7 @@ from allmydata.util.deferredutil import async_to_deferred from allmydata.util.consumer import MemoryConsumer from allmydata.test.common_system import SystemTestMixin from allmydata.immutable.upload import Data as UData +from allmydata.mutable.publish import MutableData @contextmanager @@ -36,7 +37,7 @@ class ImmutableBenchmarks(SystemTestMixin, TestCase): FORCE_FOOLSCAP_FOR_STORAGE = False @async_to_deferred - async def test_upload_and_download_immutables(self): + async def test_upload_and_download_immutable(self): self.basedir = self.mktemp() # To test larger files, change this: @@ -63,3 +64,30 @@ class ImmutableBenchmarks(SystemTestMixin, TestCase): node = self.clients[1].create_node_from_uri(uri) mc = await node.read(MemoryConsumer(), 0, None) self.assertEqual(b"".join(mc.chunks), DATA) + + + @async_to_deferred + async def test_upload_and_download_mutable(self): + self.basedir = self.mktemp() + + # To test larger files, change this: + DATA = b"Some data to upload\n" * 10 + + # 1 node + await self.set_up_nodes(1) + + # 1 share + for c in self.clients: + c.encoding_params["k"] = 1 + c.encoding_params["happy"] = 1 + c.encoding_params["n"] = 1 + + for i in range(5): + # 1. Upload: + with timeit("upload"): + result = await self.clients[0].create_mutable_file(MutableData(DATA)) + + # 2. Download: + with timeit("download"): + data = await result.download_best_version() + self.assertEqual(b"".join(mc.chunks), DATA) From 954ae2f2a4dc2dede706cdb1277803f80d51607b Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 14 Dec 2022 11:23:06 -0500 Subject: [PATCH 1275/2309] Make tests pass again. --- benchmarks/test_immutable.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/benchmarks/test_immutable.py b/benchmarks/test_immutable.py index ea7624fcc..7268b3399 100644 --- a/benchmarks/test_immutable.py +++ b/benchmarks/test_immutable.py @@ -43,8 +43,8 @@ class ImmutableBenchmarks(SystemTestMixin, TestCase): # To test larger files, change this: DATA = b"Some data to upload\n" * 10 - # 1 node - await self.set_up_nodes(1) + # 2 nodes + await self.set_up_nodes(2) # 1 share for c in self.clients: @@ -90,4 +90,4 @@ class ImmutableBenchmarks(SystemTestMixin, TestCase): # 2. Download: with timeit("download"): data = await result.download_best_version() - self.assertEqual(b"".join(mc.chunks), DATA) + self.assertEqual(data, DATA) From 146341de59654fd066f0eb2a6d455ebd6bc442d7 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 14 Dec 2022 11:24:47 -0500 Subject: [PATCH 1276/2309] Rename. --- benchmarks/{test_immutable.py => upload_download.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename benchmarks/{test_immutable.py => upload_download.py} (100%) diff --git a/benchmarks/test_immutable.py b/benchmarks/upload_download.py similarity index 100% rename from benchmarks/test_immutable.py rename to benchmarks/upload_download.py From 1d6a4e6d562cac42016475235daccee08677350c Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 14 Dec 2022 11:43:59 -0500 Subject: [PATCH 1277/2309] Minor cleanups. --- benchmarks/upload_download.py | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/benchmarks/upload_download.py b/benchmarks/upload_download.py index 7268b3399..11da26624 100644 --- a/benchmarks/upload_download.py +++ b/benchmarks/upload_download.py @@ -10,6 +10,10 @@ TODO Parameterization (pytest?) from time import time, process_time from contextlib import contextmanager +from tempfile import mkdtemp +import os + +import pytest from twisted.trial.unittest import TestCase @@ -27,7 +31,9 @@ def timeit(name): try: yield finally: - print(f"{name}: {time() - start:.3f} elapsed, {process_time() - start_cpu:.3f} CPU") + print( + f"{name}: {time() - start:.3f} elapsed, {process_time() - start_cpu:.3f} CPU" + ) class ImmutableBenchmarks(SystemTestMixin, TestCase): @@ -37,11 +43,9 @@ class ImmutableBenchmarks(SystemTestMixin, TestCase): FORCE_FOOLSCAP_FOR_STORAGE = False @async_to_deferred - async def test_upload_and_download_immutable(self): - self.basedir = self.mktemp() - - # To test larger files, change this: - DATA = b"Some data to upload\n" * 10 + async def setUp(self): + SystemTestMixin.setUp(self) + self.basedir = os.path.join(mkdtemp(), "nodes") # 2 nodes await self.set_up_nodes(2) @@ -52,6 +56,11 @@ class ImmutableBenchmarks(SystemTestMixin, TestCase): c.encoding_params["happy"] = 1 c.encoding_params["n"] = 1 + @async_to_deferred + async def test_upload_and_download_immutable(self): + # To test larger files, change this: + DATA = b"Some data to upload\n" * 10 + for i in range(5): # 1. Upload: with timeit("upload"): @@ -65,11 +74,8 @@ class ImmutableBenchmarks(SystemTestMixin, TestCase): mc = await node.read(MemoryConsumer(), 0, None) self.assertEqual(b"".join(mc.chunks), DATA) - @async_to_deferred async def test_upload_and_download_mutable(self): - self.basedir = self.mktemp() - # To test larger files, change this: DATA = b"Some data to upload\n" * 10 From d3e89cc8fdd63472fa3b273b04998a2be48547d6 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 14 Dec 2022 11:44:17 -0500 Subject: [PATCH 1278/2309] Add print(). --- benchmarks/upload_download.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/benchmarks/upload_download.py b/benchmarks/upload_download.py index 11da26624..053aa6955 100644 --- a/benchmarks/upload_download.py +++ b/benchmarks/upload_download.py @@ -56,6 +56,8 @@ class ImmutableBenchmarks(SystemTestMixin, TestCase): c.encoding_params["happy"] = 1 c.encoding_params["n"] = 1 + print() + @async_to_deferred async def test_upload_and_download_immutable(self): # To test larger files, change this: From 9e06cc09d800521d1abf9a0120115c0c24ec5c90 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 14 Dec 2022 11:45:32 -0500 Subject: [PATCH 1279/2309] Make it line up. --- benchmarks/upload_download.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/benchmarks/upload_download.py b/benchmarks/upload_download.py index 053aa6955..bf0f95838 100644 --- a/benchmarks/upload_download.py +++ b/benchmarks/upload_download.py @@ -65,7 +65,7 @@ class ImmutableBenchmarks(SystemTestMixin, TestCase): for i in range(5): # 1. Upload: - with timeit("upload"): + with timeit(" upload"): uploader = self.clients[0].getServiceNamed("uploader") results = await uploader.upload(UData(DATA, convergence=None)) @@ -92,7 +92,7 @@ class ImmutableBenchmarks(SystemTestMixin, TestCase): for i in range(5): # 1. Upload: - with timeit("upload"): + with timeit(" upload"): result = await self.clients[0].create_mutable_file(MutableData(DATA)) # 2. Download: From 55c1bae13e93c25f9a455667b18e71921ce31d74 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 14 Dec 2022 11:45:56 -0500 Subject: [PATCH 1280/2309] Instructions. --- benchmarks/upload_download.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/benchmarks/upload_download.py b/benchmarks/upload_download.py index bf0f95838..14447b550 100644 --- a/benchmarks/upload_download.py +++ b/benchmarks/upload_download.py @@ -1,6 +1,10 @@ """ First attempt at benchmarking uploads and downloads. +To run: + +$ pytest benchmarks/upload_download.py -s -v -Wignore + TODO Parameterization (pytest?) - Foolscap vs not foolscap - Number of nodes From f22988e9df21909bc612d2df6b8d28515166145c Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 14 Dec 2022 11:46:01 -0500 Subject: [PATCH 1281/2309] Unused. --- benchmarks/upload_download.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/benchmarks/upload_download.py b/benchmarks/upload_download.py index 14447b550..1ea015aeb 100644 --- a/benchmarks/upload_download.py +++ b/benchmarks/upload_download.py @@ -17,8 +17,6 @@ from contextlib import contextmanager from tempfile import mkdtemp import os -import pytest - from twisted.trial.unittest import TestCase from allmydata.util.deferredutil import async_to_deferred From 4cc4670e0545840e07317b2487ac461443fa0f7d Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 14 Dec 2022 11:46:30 -0500 Subject: [PATCH 1282/2309] Correct docs. --- benchmarks/upload_download.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmarks/upload_download.py b/benchmarks/upload_download.py index 1ea015aeb..2690a89e2 100644 --- a/benchmarks/upload_download.py +++ b/benchmarks/upload_download.py @@ -41,7 +41,7 @@ def timeit(name): class ImmutableBenchmarks(SystemTestMixin, TestCase): """Benchmarks for immutables.""" - # To use HTTP, change to true: + # To use Foolscap, change to True: FORCE_FOOLSCAP_FOR_STORAGE = False @async_to_deferred From 46954613dcb3e382f43233c3c7f2e6def2cffcbf Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 14 Dec 2022 11:48:36 -0500 Subject: [PATCH 1283/2309] Don't need setup. --- benchmarks/upload_download.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/benchmarks/upload_download.py b/benchmarks/upload_download.py index 2690a89e2..bd1b08e7a 100644 --- a/benchmarks/upload_download.py +++ b/benchmarks/upload_download.py @@ -83,15 +83,6 @@ class ImmutableBenchmarks(SystemTestMixin, TestCase): # To test larger files, change this: DATA = b"Some data to upload\n" * 10 - # 1 node - await self.set_up_nodes(1) - - # 1 share - for c in self.clients: - c.encoding_params["k"] = 1 - c.encoding_params["happy"] = 1 - c.encoding_params["n"] = 1 - for i in range(5): # 1. Upload: with timeit(" upload"): From 0ce7331c2118a6c92f25dee33e960303ff08b34e Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 14 Dec 2022 11:56:35 -0500 Subject: [PATCH 1284/2309] News file. --- newsfragments/3952.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3952.minor diff --git a/newsfragments/3952.minor b/newsfragments/3952.minor new file mode 100644 index 000000000..e69de29bb From 793033caa8004851c7d35f9378101972a84f849f Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 14 Dec 2022 12:44:11 -0500 Subject: [PATCH 1285/2309] Fix quoting on the helper --- .circleci/rebuild-images.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) mode change 100644 => 100755 .circleci/rebuild-images.sh diff --git a/.circleci/rebuild-images.sh b/.circleci/rebuild-images.sh old mode 100644 new mode 100755 index 7ee17b8ff..901651905 --- a/.circleci/rebuild-images.sh +++ b/.circleci/rebuild-images.sh @@ -15,6 +15,6 @@ curl \ --verbose \ --request POST \ --url https://circleci.com/api/v2/project/gh/tahoe-lafs/tahoe-lafs/pipeline \ - --header 'Circle-Token: $API_TOKEN' \ - --header 'content-type: application/json' \ - --data '{"branch":"$BRANCH","parameters":{"build-images":true,"run-tests":false}}' + --header "Circle-Token: $API_TOKEN" \ + --header "content-type: application/json" \ + --data '{"branch":"'"$BRANCH"'","parameters":{"build-images":true,"run-tests":false}}' From f053ef371a2162a9ac9c833694e534c8a0cdfad4 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 14 Dec 2022 12:47:50 -0500 Subject: [PATCH 1286/2309] Get rid of the scheduled image building workflow. --- .circleci/config.yml | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index b7c6cdbee..722ad390f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -109,26 +109,12 @@ workflows: - "docs": {} - triggered-images: + images: <<: *IMAGES # Build as part of the workflow but only if requested. when: "<< pipeline.parameters.build-images >>" - scheduled-images: - <<: *IMAGES - - # Build the Docker images used by the ci jobs. This makes the ci jobs - # faster and takes various spurious failures out of the critical path. - triggers: - # Build once a day - - schedule: - cron: "0 0 * * *" - filters: - branches: - only: - - "master" - jobs: dockerhub-auth-template: From 63fd24d0607cf4a9440f4837f58347e1caab6300 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 14 Dec 2022 12:48:53 -0500 Subject: [PATCH 1287/2309] Note how you can get this parameter set --- .circleci/config.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 722ad390f..480926825 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -44,6 +44,10 @@ parameters: # Generally we do not want this to run because we don't need our # dependencies to move around all the time and because building the image # takes a couple minutes. + # + # An easy way to trigger a pipeline with this set to true is with the + # rebuild-images.sh tool in this directory. You can also do so via the + # CircleCI web UI. build-images: default: false type: "boolean" From 8c8a04fa1bbc0e95b78e487d3734504f50be8120 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 14 Dec 2022 13:24:36 -0500 Subject: [PATCH 1288/2309] news fragment --- newsfragments/3958.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3958.minor diff --git a/newsfragments/3958.minor b/newsfragments/3958.minor new file mode 100644 index 000000000..e69de29bb From 96347e22e27b0614ba5e9797a401129c5bdb8101 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 15 Dec 2022 13:14:49 -0500 Subject: [PATCH 1289/2309] Make a test demonstrating the problem. --- src/allmydata/test/test_system.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/allmydata/test/test_system.py b/src/allmydata/test/test_system.py index 670ac5868..33b0284da 100644 --- a/src/allmydata/test/test_system.py +++ b/src/allmydata/test/test_system.py @@ -477,9 +477,10 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase): def _corrupt_mutable_share(self, filename, which): msf = MutableShareFile(filename) - datav = msf.readv([ (0, 1000000) ]) + # Read more than share length: + datav = msf.readv([ (0, 10_000_000) ]) final_share = datav[0] - assert len(final_share) < 1000000 # ought to be truncated + assert len(final_share) < 10_000_000 # ought to be truncated pieces = mutable_layout.unpack_share(final_share) (seqnum, root_hash, IV, k, N, segsize, datalen, verification_key, signature, share_hash_chain, block_hash_tree, @@ -524,7 +525,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase): DATA_uploadable = MutableData(DATA) NEWDATA = b"new contents yay" NEWDATA_uploadable = MutableData(NEWDATA) - NEWERDATA = b"this is getting old" + NEWERDATA = b"this is getting old" * 1_000_000 NEWERDATA_uploadable = MutableData(NEWERDATA) d = self.set_up_nodes() From 1d3464a430a465b4b5ef568b33a6138c0a7a495c Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 19 Dec 2022 13:37:20 -0500 Subject: [PATCH 1290/2309] Add end-to-end MDMF test. --- src/allmydata/test/test_system.py | 37 ++++++++++++++++++++++--------- 1 file changed, 26 insertions(+), 11 deletions(-) diff --git a/src/allmydata/test/test_system.py b/src/allmydata/test/test_system.py index 33b0284da..55bf0ed8d 100644 --- a/src/allmydata/test/test_system.py +++ b/src/allmydata/test/test_system.py @@ -34,7 +34,7 @@ from allmydata.util.encodingutil import quote_output, unicode_to_argv from allmydata.util.fileutil import abspath_expanduser_unicode from allmydata.util.consumer import MemoryConsumer, download_to_data from allmydata.interfaces import IDirectoryNode, IFileNode, \ - NoSuchChildError, NoSharesError + NoSuchChildError, NoSharesError, SDMF_VERSION, MDMF_VERSION from allmydata.monitor import Monitor from allmydata.mutable.common import NotWriteableError from allmydata.mutable import layout as mutable_layout @@ -520,7 +520,13 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase): msf.writev( [(0, final_share)], None) - def test_mutable(self): + def test_mutable_sdmf(self): + return self._test_mutable(SDMF_VERSION) + + def test_mutable_mdmf(self): + return self._test_mutable(MDMF_VERSION) + + def _test_mutable(self, mutable_version): DATA = b"initial contents go here." # 25 bytes % 3 != 0 DATA_uploadable = MutableData(DATA) NEWDATA = b"new contents yay" @@ -533,7 +539,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase): def _create_mutable(res): c = self.clients[0] log.msg("starting create_mutable_file") - d1 = c.create_mutable_file(DATA_uploadable) + d1 = c.create_mutable_file(DATA_uploadable, mutable_version) def _done(res): log.msg("DONE: %s" % (res,)) self._mutable_node_1 = res @@ -555,27 +561,33 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase): filename) self.failUnlessEqual(rc, 0) try: + share_type = 'SDMF' if mutable_version == SDMF_VERSION else 'MDMF' self.failUnless("Mutable slot found:\n" in output) - self.failUnless("share_type: SDMF\n" in output) + self.assertIn(f"share_type: {share_type}\n", output) peerid = idlib.nodeid_b2a(self.clients[client_num].nodeid) self.failUnless(" WE for nodeid: %s\n" % peerid in output) self.failUnless(" num_extra_leases: 0\n" in output) self.failUnless(" secrets are for nodeid: %s\n" % peerid in output) - self.failUnless(" SDMF contents:\n" in output) + self.failUnless(f" {share_type} contents:\n" in output) self.failUnless(" seqnum: 1\n" in output) self.failUnless(" required_shares: 3\n" in output) self.failUnless(" total_shares: 10\n" in output) - self.failUnless(" segsize: 27\n" in output, (output, filename)) + if mutable_version == SDMF_VERSION: + self.failUnless(" segsize: 27\n" in output, (output, filename)) self.failUnless(" datalen: 25\n" in output) # the exact share_hash_chain nodes depends upon the sharenum, # and is more of a hassle to compute than I want to deal with # now self.failUnless(" share_hash_chain: " in output) self.failUnless(" block_hash_tree: 1 nodes\n" in output) - expected = (" verify-cap: URI:SSK-Verifier:%s:" % - str(base32.b2a(storage_index), "ascii")) - self.failUnless(expected in output) + if mutable_version == SDMF_VERSION: + expected = (" verify-cap: URI:SSK-Verifier:%s:" % + str(base32.b2a(storage_index), "ascii")) + else: + expected = (" verify-cap: URI:MDMF-Verifier:%s" % + str(base32.b2a(storage_index), "ascii")) + self.assertIn(expected, output) except unittest.FailTest: print() print("dump-share output was:") @@ -695,7 +707,10 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase): # when we retrieve this, we should get three signature # failures (where we've mangled seqnum, R, and segsize). The # pubkey mangling - d.addCallback(_corrupt_shares) + + if mutable_version == SDMF_VERSION: + # TODO Corrupting shares in test_systm doesn't work for MDMF right now + d.addCallback(_corrupt_shares) d.addCallback(lambda res: self._newnode3.download_best_version()) d.addCallback(_check_download_5) @@ -703,7 +718,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase): def _check_empty_file(res): # make sure we can create empty files, this usually screws up the # segsize math - d1 = self.clients[2].create_mutable_file(MutableData(b"")) + d1 = self.clients[2].create_mutable_file(MutableData(b""), mutable_version) d1.addCallback(lambda newnode: newnode.download_best_version()) d1.addCallback(lambda res: self.failUnlessEqual(b"", res)) return d1 From 78e04cc82170f8139b67b419f6cc72e3e75bc477 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 21 Dec 2022 06:25:22 -0500 Subject: [PATCH 1291/2309] Modernize cachix usage; attempt to fix CircleCI conditional CIRCLE_PR_NUMBER documentation may just be wrong. It seems like maybe it is never set? Try inspecting the source repo value instead. --- .circleci/config.yml | 73 ++++++++++++-------------------------------- .circleci/lib.sh | 25 +++++++++++++++ 2 files changed, 44 insertions(+), 54 deletions(-) create mode 100644 .circleci/lib.sh diff --git a/.circleci/config.yml b/.circleci/config.yml index d7e4f2563..4dcf2a2db 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -380,7 +380,7 @@ jobs: docker: # Run in a highly Nix-capable environment. - <<: *DOCKERHUB_AUTH - image: "nixos/nix:2.3.16" + image: "nixos/nix:2.10.3" environment: # CACHIX_AUTH_TOKEN is manually set in the CircleCI web UI and @@ -390,27 +390,21 @@ jobs: steps: - "run": - # The nixos/nix image does not include ssh. Install it so the - # `checkout` step will succeed. We also want cachix for - # Nix-friendly caching. + # Get cachix for Nix-friendly caching. name: "Install Basic Dependencies" command: | + NIXPKGS="https://github.com/nixos/nixpkgs/archive/nixos-<>.tar.gz" nix-env \ - --file https://github.com/nixos/nixpkgs/archive/nixos-<>.tar.gz \ + --file $NIXPKGS \ --install \ - -A openssh cachix bash + -A cachix bash + # Activate it for "binary substitution". This sets up + # configuration tht lets Nix download something from the cache + # instead of building it locally, if possible. + cachix use "${CACHIX_NAME}" - "checkout" - - run: - name: "Cachix setup" - # Record the store paths that exist before we did much. There's no - # reason to cache these, they're either in the image or have to be - # retrieved before we can use cachix to restore from cache. - command: | - cachix use "${CACHIX_NAME}" - nix path-info --all > /tmp/store-path-pre-build - - "run": # The Nix package doesn't know how to do this part, unfortunately. name: "Generate version" @@ -432,50 +426,21 @@ jobs: # build a couple simple little dependencies that don't take # advantage of multiple cores and we get a little speedup by doing # them in parallel. - nix-build --cores 3 --max-jobs 2 --argstr pkgsVersion "nixpkgs-<>" + source .circleci/lib.sh + cache_if_able nix-build \ + --cores 3 \ + --max-jobs 2 \ + --argstr pkgsVersion "nixpkgs-<>" - "run": name: "Test" command: | # Let it go somewhat wild for the test suite itself - nix-build --cores 8 --argstr pkgsVersion "nixpkgs-<>" tests.nix - - - run: - # Send any new store objects to cachix. - name: "Push to Cachix" - when: "always" - command: | - # Cribbed from - # https://circleci.com/blog/managing-secrets-when-you-have-pull-requests-from-outside-contributors/ - if [ -n "$CIRCLE_PR_NUMBER" ]; then - # I'm sure you're thinking "CIRCLE_PR_NUMBER must just be the - # number of the PR being built". Sorry, dear reader, you have - # guessed poorly. It is also conditionally set based on whether - # this is a PR from a fork or not. - # - # https://circleci.com/docs/2.0/env-vars/#built-in-environment-variables - echo "Skipping Cachix push for forked PR." - else - # If this *isn't* a build from a fork then we have the Cachix - # write key in our environment and we can push any new objects - # to Cachix. - # - # To decide what to push, we inspect the list of store objects - # that existed before and after we did most of our work. Any - # that are new after the work is probably a useful thing to have - # around so push it to the cache. We exclude all derivation - # objects (.drv files) because they're cheap to reconstruct and - # by the time you know their cache key you've already done all - # the work anyway. - # - # This shell expression for finding the objects and pushing them - # was from the Cachix docs: - # - # https://docs.cachix.org/continuous-integration-setup/circleci.html - # - # but they seem to have removed it now. - bash -c "comm -13 <(sort /tmp/store-path-pre-build | grep -v '\.drv$') <(nix path-info --all | grep -v '\.drv$' | sort) | cachix push $CACHIX_NAME" - fi + source .circleci/lib.sh + cache_if_able nix-build \ + --cores 8 \ + --argstr pkgsVersion "nixpkgs-<>" \ + tests.nix typechecks: docker: diff --git a/.circleci/lib.sh b/.circleci/lib.sh new file mode 100644 index 000000000..f3fe07bae --- /dev/null +++ b/.circleci/lib.sh @@ -0,0 +1,25 @@ +# Run a command, enabling cache writes to cachix if possible. The command is +# accepted as a variable number of positional arguments (like argv). +function cache_if_able() { + # The `cachix watch-exec ...` does our cache population. When it sees + # something added to the store (I guess) it pushes it to the named cache. + # + # We can only *push* to it if we have a CACHIX_AUTH_TOKEN, though. + # in-repo jobs will get this from CircleCI configuration but jobs from + # forks may not. + if [ -v CACHIX_AUTH_TOKEN ]; then + echo "Cachix credentials present; will attempt to write to cache." + cachix watch-exec "${CACHIX_NAME}" -- "$@" + else + # If we're building a from a forked repository then we're allowed to + # not have the credentials (but it's also fine if the owner of the + # fork supplied their own). + if [ "${CIRCLE_PR_REPONAME}" == "https://github.com/tahoe-lafs/tahoe-lafs" ]; then + echo "Required credentials (CACHIX_AUTH_TOKEN) are missing." + return 1 + else + echo "Cachix credentials missing; will not attempt cache writes." + "$@" + fi + fi +} From 21af00bf83ff8b1f684d965d772c564d7af92e2b Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 21 Dec 2022 06:27:41 -0500 Subject: [PATCH 1292/2309] Report the CIRCLE_PR_REPONAME too, because who knows --- .circleci/lib.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/lib.sh b/.circleci/lib.sh index f3fe07bae..cc7ce5e97 100644 --- a/.circleci/lib.sh +++ b/.circleci/lib.sh @@ -7,6 +7,7 @@ function cache_if_able() { # We can only *push* to it if we have a CACHIX_AUTH_TOKEN, though. # in-repo jobs will get this from CircleCI configuration but jobs from # forks may not. + echo "Building PR from repo: ${CIRCLE_PR_REPONAME}" if [ -v CACHIX_AUTH_TOKEN ]; then echo "Cachix credentials present; will attempt to write to cache." cachix watch-exec "${CACHIX_NAME}" -- "$@" From 25eb3ca262e0a2bff842e8eff78284f0723faa42 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 21 Dec 2022 06:47:21 -0500 Subject: [PATCH 1293/2309] Switch to a variable observed in practice There is apparently no CIRCLE_PR_REPONAME set in the runtime environment, either, despite what the docs say. --- .circleci/lib.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/lib.sh b/.circleci/lib.sh index cc7ce5e97..7717cdb18 100644 --- a/.circleci/lib.sh +++ b/.circleci/lib.sh @@ -7,7 +7,7 @@ function cache_if_able() { # We can only *push* to it if we have a CACHIX_AUTH_TOKEN, though. # in-repo jobs will get this from CircleCI configuration but jobs from # forks may not. - echo "Building PR from repo: ${CIRCLE_PR_REPONAME}" + echo "Building PR from user/org: ${CIRCLE_PROJECT_USERNAME}" if [ -v CACHIX_AUTH_TOKEN ]; then echo "Cachix credentials present; will attempt to write to cache." cachix watch-exec "${CACHIX_NAME}" -- "$@" @@ -15,7 +15,7 @@ function cache_if_able() { # If we're building a from a forked repository then we're allowed to # not have the credentials (but it's also fine if the owner of the # fork supplied their own). - if [ "${CIRCLE_PR_REPONAME}" == "https://github.com/tahoe-lafs/tahoe-lafs" ]; then + if [ "${CIRCLE_PROJECT_USERNAME}" == "tahoe-lafs" ]; then echo "Required credentials (CACHIX_AUTH_TOKEN) are missing." return 1 else From 2da3d43b2e4e7a0b6dff7f2efd7a8bb675a00ced Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 21 Dec 2022 07:22:37 -0500 Subject: [PATCH 1294/2309] news fragment --- newsfragments/3870.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3870.minor diff --git a/newsfragments/3870.minor b/newsfragments/3870.minor new file mode 100644 index 000000000..e69de29bb From a71e873c21836898318512c61c45696a847f4134 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 20 Dec 2022 14:07:12 -0500 Subject: [PATCH 1295/2309] pycddl 0.2 is broken, 0.3 is missing mmap() support. --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 8558abd02..dd50e0fcf 100644 --- a/setup.py +++ b/setup.py @@ -137,7 +137,8 @@ install_requires = [ "werkzeug != 2.2.0", "treq", "cbor2", - "pycddl >= 0.2", + # Need 0.4 to be able to pass in mmap() + "pycddl >= 0.4", # for pid-file support "psutil", From 6d2e797581ed214488da9562e09b78c7dd7299a3 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 21 Dec 2022 09:16:18 -0500 Subject: [PATCH 1296/2309] News file. --- newsfragments/3956.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3956.minor diff --git a/newsfragments/3956.minor b/newsfragments/3956.minor new file mode 100644 index 000000000..e69de29bb From 1a4dcc70e26ce3e720180fb2572e02def7fca351 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 21 Dec 2022 09:24:31 -0500 Subject: [PATCH 1297/2309] Support large mutable uploads in a memory-efficient manner. --- src/allmydata/storage/http_server.py | 60 ++++++++++++++++++------- src/allmydata/test/test_storage_http.py | 47 ++++++++++++------- 2 files changed, 74 insertions(+), 33 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 3902976ba..d76948d93 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -9,6 +9,10 @@ from functools import wraps from base64 import b64decode import binascii from tempfile import TemporaryFile +from os import SEEK_END, SEEK_SET +from io import BytesIO +import mmap +import sys from cryptography.x509 import Certificate as CryptoCertificate from zope.interface import implementer @@ -39,7 +43,7 @@ from cryptography.x509 import load_pem_x509_certificate # TODO Make sure to use pure Python versions? -from cbor2 import dump, loads +import cbor2 from pycddl import Schema, ValidationError as CDDLValidationError from .server import StorageServer from .http_common import ( @@ -515,7 +519,7 @@ class HTTPServer(object): if accept.best == CBOR_MIME_TYPE: request.setHeader("Content-Type", CBOR_MIME_TYPE) f = TemporaryFile() - dump(data, f) + cbor2.dump(data, f) def read_data(offset: int, length: int) -> bytes: f.seek(offset) @@ -527,27 +531,47 @@ class HTTPServer(object): # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3861 raise _HTTPError(http.NOT_ACCEPTABLE) - def _read_encoded(self, request, schema: Schema) -> Any: + def _read_encoded( + self, request, schema: Schema, max_size: int = 1024 * 1024 + ) -> Any: """ Read encoded request body data, decoding it with CBOR by default. - Somewhat arbitrarily, limit body size to 1MB; this may be too low, we - may want to customize per query type, but this is the starting point - for now. + Somewhat arbitrarily, limit body size to 1MiB by default. """ content_type = get_content_type(request.requestHeaders) - if content_type == CBOR_MIME_TYPE: - # Read 1 byte more than 1MB. We expect length to be 1MB or - # less; if it's more assume it's not a legitimate message. - message = request.content.read(1024 * 1024 + 1) - if len(message) > 1024 * 1024: - raise _HTTPError(http.REQUEST_ENTITY_TOO_LARGE) - schema.validate_cbor(message) - result = loads(message) - return result - else: + if content_type != CBOR_MIME_TYPE: raise _HTTPError(http.UNSUPPORTED_MEDIA_TYPE) + # Make sure it's not too large: + request.content.seek(SEEK_END, 0) + if request.content.tell() > max_size: + raise _HTTPError(http.REQUEST_ENTITY_TOO_LARGE) + request.content.seek(SEEK_SET, 0) + + # We don't want to load the whole message into memory, cause it might + # be quite large. The CDDL validator takes a read-only bytes-like + # thing. Luckily, for large request bodies twisted.web will buffer the + # data in a file, so we can use mmap() to get a memory view. The CDDL + # validator will not make a copy, so it won't increase memory usage + # beyond that. + try: + fd = request.content.fileno() + except (ValueError, OSError): + fd = -1 + if fd > 0: + # It's a file, so we can use mmap() to save memory. + message = mmap.mmap(fd, 0, access=mmap.ACCESS_READ) + else: + message = request.content.read() + schema.validate_cbor(message) + + # The CBOR parser will allocate more memory, but at least we can feed + # it the file-like object, so that if it's large it won't be make two + # copies. + request.content.seek(SEEK_SET, 0) + return cbor2.load(request.content) + ##### Generic APIs ##### @_authorized_route(_app, set(), "/storage/v1/version", methods=["GET"]) @@ -746,7 +770,9 @@ class HTTPServer(object): ) def mutable_read_test_write(self, request, authorization, storage_index): """Read/test/write combined operation for mutables.""" - rtw_request = self._read_encoded(request, _SCHEMAS["mutable_read_test_write"]) + rtw_request = self._read_encoded( + request, _SCHEMAS["mutable_read_test_write"], max_size=2**48 + ) secrets = ( authorization[Secrets.WRITE_ENABLER], authorization[Secrets.LEASE_RENEW], diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 8dbe18545..bc2e10eb6 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -1186,18 +1186,42 @@ class MutableHTTPAPIsTests(SyncTestCase): ) return storage_index, write_secret, lease_secret - def test_write_can_be_read(self): + def test_write_can_be_read_small_data(self): + """ + Small written data can be read using ``read_share_chunk``. + """ + self.write_can_be_read(b"abcdef") + + def test_write_can_be_read_large_data(self): + """ + Large written data (50MB) can be read using ``read_share_chunk``. + """ + self.write_can_be_read(b"abcdefghij" * 5 * 1024 * 1024) + + def write_can_be_read(self, data): """ Written data can be read using ``read_share_chunk``. """ - storage_index, _, _ = self.create_upload() - data0 = self.http.result_of_with_flush( - self.mut_client.read_share_chunk(storage_index, 0, 1, 7) + lease_secret = urandom(32) + storage_index = urandom(16) + self.http.result_of_with_flush( + self.mut_client.read_test_write_chunks( + storage_index, + urandom(32), + lease_secret, + lease_secret, + { + 0: TestWriteVectors( + write_vectors=[WriteVector(offset=0, data=data)] + ), + }, + [], + ) ) - data1 = self.http.result_of_with_flush( - self.mut_client.read_share_chunk(storage_index, 1, 0, 8) + read_data = self.http.result_of_with_flush( + self.mut_client.read_share_chunk(storage_index, 0, 0, len(data)) ) - self.assertEqual((data0, data1), (b"bcdef-0", b"abcdef-1")) + self.assertEqual(read_data, data) def test_read_before_write(self): """In combo read/test/write operation, reads happen before writes.""" @@ -1276,15 +1300,6 @@ class MutableHTTPAPIsTests(SyncTestCase): b"aXYZef-0", ) - def test_too_large_write(self): - """ - Writing too large of a chunk results in a REQUEST ENTITY TOO LARGE http - error. - """ - with self.assertRaises(ClientException) as e: - self.create_upload(b"0123456789" * 1024 * 1024) - self.assertEqual(e.exception.code, http.REQUEST_ENTITY_TOO_LARGE) - def test_list_shares(self): """``list_shares()`` returns the shares for a given storage index.""" storage_index, _, _ = self.create_upload() From 54da6eb60a35a53dc981eca3f5172c5fae6faf38 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 21 Dec 2022 09:34:25 -0500 Subject: [PATCH 1298/2309] Remove unneeded imports. --- src/allmydata/storage/http_server.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index d76948d93..47fac879f 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -10,9 +10,7 @@ from base64 import b64decode import binascii from tempfile import TemporaryFile from os import SEEK_END, SEEK_SET -from io import BytesIO import mmap -import sys from cryptography.x509 import Certificate as CryptoCertificate from zope.interface import implementer From d1b464d0d871b9c09b0f0312136a33bbc08239df Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 21 Dec 2022 09:35:10 -0500 Subject: [PATCH 1299/2309] Writing large files can involve many writes. --- src/allmydata/storage/http_server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 47fac879f..6d22c92df 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -280,7 +280,7 @@ _SCHEMAS = { "test-write-vectors": { 0*256 share_number : { "test": [0*30 {"offset": uint, "size": uint, "specimen": bstr}] - "write": [0*30 {"offset": uint, "data": bstr}] + "write": [* {"offset": uint, "data": bstr}] "new-length": uint / null } } From 29a5f7a076a5cb47e100749cef4cce93d35acdd0 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 21 Dec 2022 17:14:08 -0500 Subject: [PATCH 1300/2309] start of a test vector thingy --- integration/test_vectors.py | 84 +++++++++++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) create mode 100644 integration/test_vectors.py diff --git a/integration/test_vectors.py b/integration/test_vectors.py new file mode 100644 index 000000000..d7fe214a1 --- /dev/null +++ b/integration/test_vectors.py @@ -0,0 +1,84 @@ +""" +Verify certain results against test vectors with well-known results. +""" + +from hashlib import sha256 +from itertools import product + +import vectors + +CONVERGENCE_SECRETS = [ + b"aaaaaaaaaaaaaaaa", + b"bbbbbbbbbbbbbbbb", + b"abcdefghijklmnop", + b"hello world stuf", + b"\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + sha256(b"Hello world").digest()[:16], +] + +ONE_KB = sha256(b"Hello world").digest() * 32 +assert length(ONE_KB) == 1024 + +OBJECT_DATA = [ + b"a" * 1024, + b"b" * 2048, + b"c" * 4096, + (ONE_KB * 8)[:-1], + (ONE_KB * 8) + b"z", + (ONE_KB * 128)[:-1], + (ONE_KB * 128) + b"z", +] + +ZFEC_PARAMS = [ + (1, 1), + (1, 3), + (2, 3), + (3, 10), + (71, 255), + (101, 256), +] + +@parametrize('convergence', CONVERGENCE_SECRETS) +def test_convergence(convergence): + assert isinstance(convergence, bytes), "Convergence secret must be bytes" + assert len(convergence) == 16, "Convergence secret must by 16 bytes" + + +@parametrize('daata', OBJECT_DATA) +def test_data(data): + assert isinstance(data, bytes), "Object data must be bytes." + + +@parametrize('params', ZFEC_PARAMS) +@parametrize('convergence', CONVERGENCE_SECRETS) +@parametrize('data', OBJECT_DATA) +def test_chk_capability(alice, params, convergence, data): + # rewrite alice's config to match params and convergence + needed, total = params + config = read_config(alice.path, "tub.port") + config.set_config("client", "shares.happy", 1) + config.set_config("client", "shares.needed", str(needed)) + config.set_config("client", "shares.happy", str(total)) + + # restart alice + alice.kill() + yield util._run_node(reactor, alice.path, request, None) + + # upload data as a CHK + actual = upload(alice, data) + + # compare the resulting cap to the expected result + expected = vectors.immutable[params, convergence, digest(data)] + assert actual == expected + +def test_generate(alice): + caps = {} + for params, secret, data in product(ZFEC_PARAMS, CONVERGENCE_SECRETS, OBJECT_DATA): + caps[fec, secret, sha256(data).hexdigest()] = create_immutable(params, secret, data) + print(dump(caps)) + +def create_immutable(alice, params, secret, data): + tempfile = str(tmpdir.join("file")) + with tempfile.open("wb") as f: + f.write(data) + actual = cli(alice, "put", str(datafile)) From 49b513fefc0fafa1686645c36e8be84792fd59bf Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 22 Dec 2022 10:51:59 -0500 Subject: [PATCH 1301/2309] Get basic generation working, apparently --- integration/_vectors_chk.yaml | 4 + integration/conftest.py | 8 +- integration/test_vectors.py | 147 ++++++++++++++++++++++++---------- integration/util.py | 38 +++++++-- integration/vectors.py | 7 ++ 5 files changed, 149 insertions(+), 55 deletions(-) create mode 100644 integration/_vectors_chk.yaml create mode 100644 integration/vectors.py diff --git a/integration/_vectors_chk.yaml b/integration/_vectors_chk.yaml new file mode 100644 index 000000000..070d9b4be --- /dev/null +++ b/integration/_vectors_chk.yaml @@ -0,0 +1,4 @@ +? 1/1,0c0beacef8877bbf2416eb00f2b5dc96354e26dd1df5517320459b1236860f8c,2edc986847e209b4016e141a6dc8716d3207350f416969382d431539bf292e4a +: URI:CHK:mri732rh3meyh4drikau3a24ba:6tsj5wvcp6szdhmrbu5bea57wduoza64y6nd2lm7aleqpsxjm5la:1:1:1024 +? 1/3,0c0beacef8877bbf2416eb00f2b5dc96354e26dd1df5517320459b1236860f8c,2edc986847e209b4016e141a6dc8716d3207350f416969382d431539bf292e4a +: URI:CHK:ycvogzi6wllnq2bkx3t6zdtwju:um42l4yen7jiwdfgirvedtty3tt3xuhjiyxzqoourvughtxjar3q:1:3:1024 diff --git a/integration/conftest.py b/integration/conftest.py index e284b5cba..1c43aecfa 100644 --- a/integration/conftest.py +++ b/integration/conftest.py @@ -49,7 +49,6 @@ from .util import ( await_client_ready, TahoeProcess, cli, - _run_node, generate_ssh_key, block_with_timeout, ) @@ -359,7 +358,7 @@ def alice_sftp_client_key_path(temp_dir): # typically), but for convenience sake for testing we'll put it inside node. return join(temp_dir, "alice", "private", "ssh_client_rsa_key") -@pytest.fixture(scope='session') +@pytest.fixture(scope='function') @log_call(action_type=u"integration:alice", include_args=[], include_result=False) def alice( reactor, @@ -410,10 +409,9 @@ alice-key ssh-rsa {ssh_public_key} {rwcap} """.format(rwcap=rwcap, ssh_public_key=ssh_public_key)) # 4. Restart the node with new SFTP config. - process.kill() - pytest_twisted.blockon(_run_node(reactor, process.node_dir, request, None)) - + pytest_twisted.blockon(process.restart_async(reactor, request)) await_client_ready(process) + print(f"Alice pid: {process.transport.pid}") return process diff --git a/integration/test_vectors.py b/integration/test_vectors.py index d7fe214a1..7753ac18d 100644 --- a/integration/test_vectors.py +++ b/integration/test_vectors.py @@ -2,83 +2,142 @@ Verify certain results against test vectors with well-known results. """ +from typing import TypeVar, Iterator, Awaitable, Callable + +from tempfile import NamedTemporaryFile from hashlib import sha256 from itertools import product +from yaml import safe_dump -import vectors +from pytest import mark +from pytest_twisted import ensureDeferred + +from . import vectors +from .util import cli, await_client_ready +from allmydata.client import read_config +from allmydata.util import base32 CONVERGENCE_SECRETS = [ b"aaaaaaaaaaaaaaaa", - b"bbbbbbbbbbbbbbbb", - b"abcdefghijklmnop", - b"hello world stuf", - b"\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", - sha256(b"Hello world").digest()[:16], + # b"bbbbbbbbbbbbbbbb", + # b"abcdefghijklmnop", + # b"hello world stuf", + # b"\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + # sha256(b"Hello world").digest()[:16], ] ONE_KB = sha256(b"Hello world").digest() * 32 -assert length(ONE_KB) == 1024 +assert len(ONE_KB) == 1024 OBJECT_DATA = [ b"a" * 1024, - b"b" * 2048, - b"c" * 4096, - (ONE_KB * 8)[:-1], - (ONE_KB * 8) + b"z", - (ONE_KB * 128)[:-1], - (ONE_KB * 128) + b"z", + # b"b" * 2048, + # b"c" * 4096, + # (ONE_KB * 8)[:-1], + # (ONE_KB * 8) + b"z", + # (ONE_KB * 128)[:-1], + # (ONE_KB * 128) + b"z", ] ZFEC_PARAMS = [ (1, 1), (1, 3), - (2, 3), - (3, 10), - (71, 255), - (101, 256), + # (2, 3), + # (3, 10), + # (71, 255), + # (101, 256), ] -@parametrize('convergence', CONVERGENCE_SECRETS) +@mark.parametrize('convergence', CONVERGENCE_SECRETS) def test_convergence(convergence): assert isinstance(convergence, bytes), "Convergence secret must be bytes" assert len(convergence) == 16, "Convergence secret must by 16 bytes" -@parametrize('daata', OBJECT_DATA) +@mark.parametrize('data', OBJECT_DATA) def test_data(data): assert isinstance(data, bytes), "Object data must be bytes." - -@parametrize('params', ZFEC_PARAMS) -@parametrize('convergence', CONVERGENCE_SECRETS) -@parametrize('data', OBJECT_DATA) -def test_chk_capability(alice, params, convergence, data): +@mark.parametrize('params', ZFEC_PARAMS) +@mark.parametrize('convergence', CONVERGENCE_SECRETS) +@mark.parametrize('data', OBJECT_DATA) +@ensureDeferred +async def test_chk_capability(alice, params, convergence, data): # rewrite alice's config to match params and convergence - needed, total = params - config = read_config(alice.path, "tub.port") - config.set_config("client", "shares.happy", 1) - config.set_config("client", "shares.needed", str(needed)) - config.set_config("client", "shares.happy", str(total)) - - # restart alice - alice.kill() - yield util._run_node(reactor, alice.path, request, None) + await reconfigure(alice, params, convergence) # upload data as a CHK - actual = upload(alice, data) + actual = upload_immutable(alice, data) # compare the resulting cap to the expected result - expected = vectors.immutable[params, convergence, digest(data)] + expected = vectors.chk[key(params, convergence, data)] assert actual == expected -def test_generate(alice): - caps = {} - for params, secret, data in product(ZFEC_PARAMS, CONVERGENCE_SECRETS, OBJECT_DATA): - caps[fec, secret, sha256(data).hexdigest()] = create_immutable(params, secret, data) - print(dump(caps)) -def create_immutable(alice, params, secret, data): - tempfile = str(tmpdir.join("file")) - with tempfile.open("wb") as f: +α = TypeVar("α") +β = TypeVar("β") + +async def asyncfoldr( + i: Iterator[Awaitable[α]], + f: Callable[[α, β], β], + initial: β, +) -> β: + result = initial + async for a in i: + result = f(a, result) + return result + +def insert(item: tuple[α, β], d: dict[α, β]) -> dict[α, β]: + d[item[0]] = item[1] + return d + +@ensureDeferred +async def test_generate(reactor, request, alice): + results = await asyncfoldr( + generate(reactor, request, alice), + insert, + {}, + ) + with vectors.CHK_PATH.open("w") as f: + f.write(safe_dump(results)) + + +async def reconfigure(reactor, request, alice, params, convergence): + needed, total = params + config = read_config(alice.node_dir, "tub.port") + config.set_config("client", "shares.happy", str(1)) + config.set_config("client", "shares.needed", str(needed)) + config.set_config("client", "shares.total", str(total)) + config.write_private_config("convergence", base32.b2a(convergence)) + + # restart alice + print(f"Restarting {alice.node_dir} for ZFEC reconfiguration") + await alice.restart_async(reactor, request) + print("Restarted. Waiting for ready state.") + await_client_ready(alice) + print("Ready.") + + +async def generate(reactor, request, alice): + node_key = (None, None) + for params, secret, data in product(ZFEC_PARAMS, CONVERGENCE_SECRETS, OBJECT_DATA): + if node_key != (params, secret): + await reconfigure(reactor, request, alice, params, secret) + node_key = (params, secret) + + yield key(params, secret, data), upload_immutable(alice, data) + + +def key(params, secret, data): + return f"{params[0]}/{params[1]},{digest(secret)},{digest(data)}" + + +def upload_immutable(alice, data): + with NamedTemporaryFile() as f: f.write(data) - actual = cli(alice, "put", str(datafile)) + f.flush() + return cli(alice, "put", "--format=chk", f.name).decode("utf-8").strip() + + +def digest(bs): + return sha256(bs).hexdigest() diff --git a/integration/util.py b/integration/util.py index ad9249e45..c2394375a 100644 --- a/integration/util.py +++ b/integration/util.py @@ -142,7 +142,18 @@ class _MagicTextProtocol(ProcessProtocol): sys.stdout.write(data) -def _cleanup_tahoe_process(tahoe_transport, exited): +def _cleanup_tahoe_process_async(tahoe_transport, allow_missing): + if tahoe_transport.pid is None: + if allow_missing: + print("Process already cleaned up and that's okay.") + return + else: + raise ValueError("Process is not running") + print("signaling {} with TERM".format(tahoe_transport.pid)) + tahoe_transport.signalProcess('TERM') + + +def _cleanup_tahoe_process(tahoe_transport, exited, allow_missing=False): """ Terminate the given process with a kill signal (SIGKILL on POSIX, TerminateProcess on Windows). @@ -154,13 +165,11 @@ def _cleanup_tahoe_process(tahoe_transport, exited): """ from twisted.internet import reactor try: - print("signaling {} with TERM".format(tahoe_transport.pid)) - tahoe_transport.signalProcess('TERM') + _cleanup_tahoe_process_async(tahoe_transport, allow_missing=allow_missing) + except ProcessExitedAlready: print("signaled, blocking on exit") block_with_timeout(exited, reactor) print("exited, goodbye") - except ProcessExitedAlready: - pass def _tahoe_runner_optional_coverage(proto, reactor, request, other_args): @@ -207,8 +216,25 @@ class TahoeProcess(object): def kill(self): """Kill the process, block until it's done.""" + print(f"TahoeProcess.kill({self.transport.pid} / {self.node_dir})") _cleanup_tahoe_process(self.transport, self.transport.exited) + def kill_async(self): + """ + Kill the process, return a Deferred that fires when it's done. + """ + print(f"TahoeProcess.kill_async({self.transport.pid} / {self.node_dir})") + _cleanup_tahoe_process_async(self.transport, allow_missing=False) + return self.transport.exited + + def restart_async(self, reactor, request): + d = self.kill_async() + d.addCallback(lambda ignored: _run_node(reactor, self.node_dir, request, None)) + def got_new_process(proc): + self._process_transport = proc.transport + d.addCallback(got_new_process) + return d + def __str__(self): return "".format(self._node_dir) @@ -238,7 +264,7 @@ def _run_node(reactor, node_dir, request, magic_text, finalize=True): transport.exited = protocol.exited if finalize: - request.addfinalizer(partial(_cleanup_tahoe_process, transport, protocol.exited)) + request.addfinalizer(partial(_cleanup_tahoe_process, transport, protocol.exited, allow_missing=True)) # XXX abusing the Deferred; should use .when_magic_seen() pattern diff --git a/integration/vectors.py b/integration/vectors.py new file mode 100644 index 000000000..53e581a1e --- /dev/null +++ b/integration/vectors.py @@ -0,0 +1,7 @@ +from yaml import safe_load +from pathlib import Path + +CHK_PATH = Path(__file__).parent / "_vectors_chk.yaml" + +with CHK_PATH.open() as f: + chk = safe_load(f) From aa58faddaf4d1affec22ea780161ec43a7644bf4 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 22 Dec 2022 11:04:48 -0500 Subject: [PATCH 1302/2309] Pass the right number of args to reconfigure --- integration/test_vectors.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/integration/test_vectors.py b/integration/test_vectors.py index 7753ac18d..d9999f1d6 100644 --- a/integration/test_vectors.py +++ b/integration/test_vectors.py @@ -62,9 +62,9 @@ def test_data(data): @mark.parametrize('convergence', CONVERGENCE_SECRETS) @mark.parametrize('data', OBJECT_DATA) @ensureDeferred -async def test_chk_capability(alice, params, convergence, data): +async def test_chk_capability(reactor, request, alice, params, convergence, data): # rewrite alice's config to match params and convergence - await reconfigure(alice, params, convergence) + await reconfigure(reactor, request, alice, params, convergence) # upload data as a CHK actual = upload_immutable(alice, data) @@ -91,6 +91,7 @@ def insert(item: tuple[α, β], d: dict[α, β]) -> dict[α, β]: d[item[0]] = item[1] return d + @ensureDeferred async def test_generate(reactor, request, alice): results = await asyncfoldr( From 1ae98c1830dc659f1717fcdf3bc0dc539e8df67e Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 22 Dec 2022 11:05:07 -0500 Subject: [PATCH 1303/2309] Switch back to session scope for Alice --- integration/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration/conftest.py b/integration/conftest.py index 1c43aecfa..e82eecae6 100644 --- a/integration/conftest.py +++ b/integration/conftest.py @@ -358,7 +358,7 @@ def alice_sftp_client_key_path(temp_dir): # typically), but for convenience sake for testing we'll put it inside node. return join(temp_dir, "alice", "private", "ssh_client_rsa_key") -@pytest.fixture(scope='function') +@pytest.fixture(scope='session') @log_call(action_type=u"integration:alice", include_args=[], include_result=False) def alice( reactor, From 97b397870baf201924dac243e0e43ed7e8b7a386 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 22 Dec 2022 11:35:37 -0500 Subject: [PATCH 1304/2309] Generate a whole mess of vectors --- integration/_vectors_chk.yaml | 500 ++++++++++++++++++++++++++++++++++ integration/test_vectors.py | 30 +- 2 files changed, 515 insertions(+), 15 deletions(-) diff --git a/integration/_vectors_chk.yaml b/integration/_vectors_chk.yaml index 070d9b4be..81a2a70c4 100644 --- a/integration/_vectors_chk.yaml +++ b/integration/_vectors_chk.yaml @@ -1,4 +1,504 @@ ? 1/1,0c0beacef8877bbf2416eb00f2b5dc96354e26dd1df5517320459b1236860f8c,2edc986847e209b4016e141a6dc8716d3207350f416969382d431539bf292e4a : URI:CHK:mri732rh3meyh4drikau3a24ba:6tsj5wvcp6szdhmrbu5bea57wduoza64y6nd2lm7aleqpsxjm5la:1:1:1024 +? 1/1,0c0beacef8877bbf2416eb00f2b5dc96354e26dd1df5517320459b1236860f8c,3abc94a93a42d0eee5c8dda0315f9f1343e2ba36b552ab512c435fd4989c1ac6 +: URI:CHK:e5tpbcmalcq6nn2zhd3qvg4chy:emzsvv2xnkhhrj2oatds5hf2cney25awi56ybeq4ofiyehzyakua:1:1:4096 +? 1/1,0c0beacef8877bbf2416eb00f2b5dc96354e26dd1df5517320459b1236860f8c,77f3b5f9437275607c3b5eef5f67d18d6a9835155cfee75acd9d2c043dbb8528 +: URI:CHK:nldstwombnjucmrhzok557b2gi:6vxghlkmxq3fmrqx5bjs5bwqxdh7b2krsfse2wnfympv5djgk4cq:1:1:8193 +? 1/1,0c0beacef8877bbf2416eb00f2b5dc96354e26dd1df5517320459b1236860f8c,80c58fc1932767b113819b033e1874d8de05e8809534590db9d01510abc8ed6b +: URI:CHK:3o3ksfxjqakmjhscpnkmdabisy:ailkrgaw5fcor7ywro225jp52i5mfoigbsvoaaqbdbbgdxrvmfta:1:1:2048 +? 1/1,0c0beacef8877bbf2416eb00f2b5dc96354e26dd1df5517320459b1236860f8c,8454e5bb164935d6a8da749d78cf91eb0556897d44273053b67777aed1d418e8 +: URI:CHK:x4cvhgcrxpvqf6k3ldgjnt7tiy:2ydvfx4sh6wbi6ucf5atocfwkvxcshtkbqonkdr3oc5unppixbmq:1:1:131071 +? 1/1,0c0beacef8877bbf2416eb00f2b5dc96354e26dd1df5517320459b1236860f8c,8a6a6bff19a6f768864a69d6c6c97e6a25385793201eefe2505650978aef9193 +: URI:CHK:6wip7b2rqy5kmx7mixsq35zzja:wjgohcfhurrjiv3wd7pc7g7lp5stvl23ynv4k26ehcswhlrsmkiq:1:1:131073 +? 1/1,0c0beacef8877bbf2416eb00f2b5dc96354e26dd1df5517320459b1236860f8c,caccfbaa894a6cf59e3d677a146924604f8edcd07ed1d61cd1d1569ce4bf3ec5 +: URI:CHK:mpzclo7ghftme56qnm2ehien5y:uoqact6oasexnzh3tkm4oqtvhfcqtl6vkrjzbp5vucxg2ptel2ma:1:1:8191 +? 1/1,2e61afd25d3ca76a0ee265dbc22ba5e9d844dc93ac5f2e37199a34fcbdcb5a52,2edc986847e209b4016e141a6dc8716d3207350f416969382d431539bf292e4a +: URI:CHK:7bkmeibwcejlkd67uxb5ggnbnq:osf4xaukorh2zdhhipyg2lpuxdcn3mm4zukrxlqwtnnoglydwzka:1:1:1024 +? 1/1,2e61afd25d3ca76a0ee265dbc22ba5e9d844dc93ac5f2e37199a34fcbdcb5a52,3abc94a93a42d0eee5c8dda0315f9f1343e2ba36b552ab512c435fd4989c1ac6 +: URI:CHK:6ix53x4efg2ijovglu3j47nehu:rsfmmi27da2fnw4qauav2lddvaoht7lhl6zebhmpyefzdonpk3ya:1:1:4096 +? 1/1,2e61afd25d3ca76a0ee265dbc22ba5e9d844dc93ac5f2e37199a34fcbdcb5a52,77f3b5f9437275607c3b5eef5f67d18d6a9835155cfee75acd9d2c043dbb8528 +: URI:CHK:bzf2m5gq2awaq6e2izbjrtwb4i:wfi2rysyt7k3xyfcbyu3oargpbqfg7e4afrr4zk6sacizu4cuecq:1:1:8193 +? 1/1,2e61afd25d3ca76a0ee265dbc22ba5e9d844dc93ac5f2e37199a34fcbdcb5a52,80c58fc1932767b113819b033e1874d8de05e8809534590db9d01510abc8ed6b +: URI:CHK:qocnrwjq2go6ae6rlixxojjj5a:btbm2djqqqim6ptrlvpsrqhf7tgkidizrz5nv4hlo7nbhemhuwyq:1:1:2048 +? 1/1,2e61afd25d3ca76a0ee265dbc22ba5e9d844dc93ac5f2e37199a34fcbdcb5a52,8454e5bb164935d6a8da749d78cf91eb0556897d44273053b67777aed1d418e8 +: URI:CHK:tnjp7mhj2y25vhmuivzwa5a2v4:tn3nquzibiqds3vjjjxjlgxryfz4z57wwyekplexvcqqccbkzylq:1:1:131071 +? 1/1,2e61afd25d3ca76a0ee265dbc22ba5e9d844dc93ac5f2e37199a34fcbdcb5a52,8a6a6bff19a6f768864a69d6c6c97e6a25385793201eefe2505650978aef9193 +: URI:CHK:bkpgz4emfogefvsgkf6qrvqdpa:qbinjbmhnawpv4k7kqll5xc7gjcyfevrqigg3jflqowc726pyiba:1:1:131073 +? 1/1,2e61afd25d3ca76a0ee265dbc22ba5e9d844dc93ac5f2e37199a34fcbdcb5a52,caccfbaa894a6cf59e3d677a146924604f8edcd07ed1d61cd1d1569ce4bf3ec5 +: URI:CHK:weepfzfu52fyaj3sl2grlj7x6e:iwpntrlq7kd3iokdcxkzj67srwbxgm7db5ujagxihiqn4hfr2stq:1:1:8191 +? 1/1,43a1e8995c86ee7f559e5d187d1657296662ca180b394146a6e1dd0df9158a30,2edc986847e209b4016e141a6dc8716d3207350f416969382d431539bf292e4a +: URI:CHK:nnv4vrtlxmzkurfzvonj22leua:ywcyijrfnwykraku56dq7v3o4ts3xsxqfgmk3kgzwhq2cpest4pa:1:1:1024 +? 1/1,43a1e8995c86ee7f559e5d187d1657296662ca180b394146a6e1dd0df9158a30,3abc94a93a42d0eee5c8dda0315f9f1343e2ba36b552ab512c435fd4989c1ac6 +: URI:CHK:wlszole5m2emf6wbhp3lnlyfxq:irdttvny74gxdrcotzfjslfq7p24kyiigyrm5shhw7zyj3hsiuba:1:1:4096 +? 1/1,43a1e8995c86ee7f559e5d187d1657296662ca180b394146a6e1dd0df9158a30,77f3b5f9437275607c3b5eef5f67d18d6a9835155cfee75acd9d2c043dbb8528 +: URI:CHK:jq2q3dk343h6cmzbbozlc4pdua:grtvc4pgd5cbzedghyk7erpal2m7ururlzv5ui2kc2lbwszk6feq:1:1:8193 +? 1/1,43a1e8995c86ee7f559e5d187d1657296662ca180b394146a6e1dd0df9158a30,80c58fc1932767b113819b033e1874d8de05e8809534590db9d01510abc8ed6b +: URI:CHK:7sfwjhsji4vmf6dxauh32mxc3u:vnoepbsyapzfslimfdw6udn5khw4xspjn4fhpvkw6h2zigbv2qja:1:1:2048 +? 1/1,43a1e8995c86ee7f559e5d187d1657296662ca180b394146a6e1dd0df9158a30,8454e5bb164935d6a8da749d78cf91eb0556897d44273053b67777aed1d418e8 +: URI:CHK:vuyye2vhfat5fjbgig7y5jokr4:sibrvhclkne5jtmc6teoiri2onhh7htflsdixb6i2vccg3hhxfpa:1:1:131071 +? 1/1,43a1e8995c86ee7f559e5d187d1657296662ca180b394146a6e1dd0df9158a30,8a6a6bff19a6f768864a69d6c6c97e6a25385793201eefe2505650978aef9193 +: URI:CHK:hfjcytrorzrycjzjsxfa6hioha:pxtxr26bokroy2etwcvs3hcv3quil3cywfmlvg45eqdaff7r4y4a:1:1:131073 +? 1/1,43a1e8995c86ee7f559e5d187d1657296662ca180b394146a6e1dd0df9158a30,caccfbaa894a6cf59e3d677a146924604f8edcd07ed1d61cd1d1569ce4bf3ec5 +: URI:CHK:thoqlkvbsnpiodlnxdoabwgu7a:oufaztxpozbheudq5aiou6h25c675knx55fnwvd56qt7sqwhyrmq:1:1:8191 +? 1/1,be45cb2605bf36bebde684841a28f0fd43c69850a3dce5fedba69928ee3a8991,2edc986847e209b4016e141a6dc8716d3207350f416969382d431539bf292e4a +: URI:CHK:rkspaiyphipftpmtcshm4x2qwq:xiefohxijxssgeednc6hvd77dqnrw7zhppvcubfghf5ci432woqq:1:1:1024 +? 1/1,be45cb2605bf36bebde684841a28f0fd43c69850a3dce5fedba69928ee3a8991,3abc94a93a42d0eee5c8dda0315f9f1343e2ba36b552ab512c435fd4989c1ac6 +: URI:CHK:kybvknf7jzu7g6gn5ngme4sb2e:56krx24234cbjger2b2q5mukghs4we72xz6va6whzriuepqoy2kq:1:1:4096 +? 1/1,be45cb2605bf36bebde684841a28f0fd43c69850a3dce5fedba69928ee3a8991,77f3b5f9437275607c3b5eef5f67d18d6a9835155cfee75acd9d2c043dbb8528 +: URI:CHK:nrpahak2l4xgfokz5ehrribtuy:mx5ix7in5anfrhldds3qkuy65rembijqi2r72wfcl6djotse2pha:1:1:8193 +? 1/1,be45cb2605bf36bebde684841a28f0fd43c69850a3dce5fedba69928ee3a8991,80c58fc1932767b113819b033e1874d8de05e8809534590db9d01510abc8ed6b +: URI:CHK:bv5i26hs2lltigwv3q6jad2j3m:u6m5xityfho4ecyywathvckxmgzfiz3t7kd6niszbsxq65ayzpmq:1:1:2048 +? 1/1,be45cb2605bf36bebde684841a28f0fd43c69850a3dce5fedba69928ee3a8991,8454e5bb164935d6a8da749d78cf91eb0556897d44273053b67777aed1d418e8 +: URI:CHK:donvstm35rcmks6cekutcijq5e:b5rnhrvzkvhas7ax3cm2smcodjvvvfgbnsf7yfg3ypaqce6zrezq:1:1:131071 +? 1/1,be45cb2605bf36bebde684841a28f0fd43c69850a3dce5fedba69928ee3a8991,8a6a6bff19a6f768864a69d6c6c97e6a25385793201eefe2505650978aef9193 +: URI:CHK:5fqqgakv2li3d4iqy2hvfqhaxi:76y5lu7uqwuuji3gtcivzca7hgxs23hyvzompgd43zle5flnuwya:1:1:131073 +? 1/1,be45cb2605bf36bebde684841a28f0fd43c69850a3dce5fedba69928ee3a8991,caccfbaa894a6cf59e3d677a146924604f8edcd07ed1d61cd1d1569ce4bf3ec5 +: URI:CHK:adrurnek5f244enbpslfhb3fni:w2ht6yczgmy7yo7ga7epem76bnt272a3g5vxnblsanl5xmkxiayq:1:1:8191 +? 1/1,cd82b3962c8796095a288e29ed94f98017cc681d17ea9ea8861c4034b13a3b6f,2edc986847e209b4016e141a6dc8716d3207350f416969382d431539bf292e4a +: URI:CHK:os7rlg55kwbtgdibianzgoekwe:ug6ju36skjxtwasbqssasl73krp63c46o3ayr4gsj4l2zyzb4p7q:1:1:1024 +? 1/1,cd82b3962c8796095a288e29ed94f98017cc681d17ea9ea8861c4034b13a3b6f,3abc94a93a42d0eee5c8dda0315f9f1343e2ba36b552ab512c435fd4989c1ac6 +: URI:CHK:wevlzjefiuecq734iz6h3j6vym:p2krrasfxrdi2o3uws6afrwod7b2fbqpt2di4m6h5p3uovbzpkca:1:1:4096 +? 1/1,cd82b3962c8796095a288e29ed94f98017cc681d17ea9ea8861c4034b13a3b6f,77f3b5f9437275607c3b5eef5f67d18d6a9835155cfee75acd9d2c043dbb8528 +: URI:CHK:ivgaoi6yhfe7emqkfdhz7x2npu:nfcqouqdcwv5d2otuknokpxgdggxuvu7f4m4ct3utbxit6wz5caq:1:1:8193 +? 1/1,cd82b3962c8796095a288e29ed94f98017cc681d17ea9ea8861c4034b13a3b6f,80c58fc1932767b113819b033e1874d8de05e8809534590db9d01510abc8ed6b +: URI:CHK:vbk4tahxlqulrqha3vyckpppm4:wzmzru7z74apg4vyrqhabvbrrrpew7n5ltxha5bwbddfghse7mva:1:1:2048 +? 1/1,cd82b3962c8796095a288e29ed94f98017cc681d17ea9ea8861c4034b13a3b6f,8454e5bb164935d6a8da749d78cf91eb0556897d44273053b67777aed1d418e8 +: URI:CHK:jyyidgcdadxfb2qdcrkvwultau:jktpx4gzdsh25bd42k3p74p7eujhnh5zre4wvjp7gorue4whldia:1:1:131071 +? 1/1,cd82b3962c8796095a288e29ed94f98017cc681d17ea9ea8861c4034b13a3b6f,8a6a6bff19a6f768864a69d6c6c97e6a25385793201eefe2505650978aef9193 +: URI:CHK:f2bren65rc4l7zrcnbrwocttty:giph2rn7u4dkkbuiruqfrxgb4ar6xyhsxtjdeeeuwwnc6vrxalsq:1:1:131073 +? 1/1,cd82b3962c8796095a288e29ed94f98017cc681d17ea9ea8861c4034b13a3b6f,caccfbaa894a6cf59e3d677a146924604f8edcd07ed1d61cd1d1569ce4bf3ec5 +: URI:CHK:oskjs2mq2pp7w75rnnb7b36ynu:qujhwnvdlpcvx5ccxwbhke5pbd2bn63vmp3ftugqm2n6u7qteoda:1:1:8191 +? 1/1,f39dac6cbaba535e2c207cd0cd8f154974223c848f727f98b3564cea569b41cf,2edc986847e209b4016e141a6dc8716d3207350f416969382d431539bf292e4a +: URI:CHK:72l476fpask7kz53frtmw3t6pa:yf35zcretagoeaer5mwpjvy35vba35b2anuy3kprcwfbddbdappa:1:1:1024 +? 1/1,f39dac6cbaba535e2c207cd0cd8f154974223c848f727f98b3564cea569b41cf,3abc94a93a42d0eee5c8dda0315f9f1343e2ba36b552ab512c435fd4989c1ac6 +: URI:CHK:5nmi5kbb5p5hycwlfn653c7s2i:cb3xoljokodml6ecspwj5llvdhu632x25mow3d7jbznuvwwsm7xa:1:1:4096 +? 1/1,f39dac6cbaba535e2c207cd0cd8f154974223c848f727f98b3564cea569b41cf,77f3b5f9437275607c3b5eef5f67d18d6a9835155cfee75acd9d2c043dbb8528 +: URI:CHK:d7yl3rivi3icf3awe6s26wwcp4:6ijmbhqfo26oywcky6enlkmbvopvkc5sotdtw5rnkrewtukxzaja:1:1:8193 +? 1/1,f39dac6cbaba535e2c207cd0cd8f154974223c848f727f98b3564cea569b41cf,80c58fc1932767b113819b033e1874d8de05e8809534590db9d01510abc8ed6b +: URI:CHK:ej5bvkhdqclpvoh4hddn3fc72i:6sxjptm4ozclmm6fzplpt46fgtcxvmmjdkmwhntf3h2jiouoftfa:1:1:2048 +? 1/1,f39dac6cbaba535e2c207cd0cd8f154974223c848f727f98b3564cea569b41cf,8454e5bb164935d6a8da749d78cf91eb0556897d44273053b67777aed1d418e8 +: URI:CHK:4fkhrkyi4srb2jiir4qdm32zj4:khjfhoz7c24my4dt5xm5hxgttxmid6ekbmst26b6rmkk3n4lubzq:1:1:131071 +? 1/1,f39dac6cbaba535e2c207cd0cd8f154974223c848f727f98b3564cea569b41cf,8a6a6bff19a6f768864a69d6c6c97e6a25385793201eefe2505650978aef9193 +: URI:CHK:wbsbqw3os3sjc2yhdj6qashad4:bubdew37qtm7inja5dsrzu6vlfdmheegq65foijd56cy54vh3ihq:1:1:131073 +? 1/1,f39dac6cbaba535e2c207cd0cd8f154974223c848f727f98b3564cea569b41cf,caccfbaa894a6cf59e3d677a146924604f8edcd07ed1d61cd1d1569ce4bf3ec5 +: URI:CHK:ovgqdsshkjg3q2qwjfupb2ysfa:hdjonr7dpz662jin3dqn35rt74lsnyp6ox2qvzpc67nzrd7m6c7q:1:1:8191 ? 1/3,0c0beacef8877bbf2416eb00f2b5dc96354e26dd1df5517320459b1236860f8c,2edc986847e209b4016e141a6dc8716d3207350f416969382d431539bf292e4a : URI:CHK:ycvogzi6wllnq2bkx3t6zdtwju:um42l4yen7jiwdfgirvedtty3tt3xuhjiyxzqoourvughtxjar3q:1:3:1024 +? 1/3,0c0beacef8877bbf2416eb00f2b5dc96354e26dd1df5517320459b1236860f8c,3abc94a93a42d0eee5c8dda0315f9f1343e2ba36b552ab512c435fd4989c1ac6 +: URI:CHK:3yjtuv5h2g45g2cncg6hka3euu:bqqhp6u73ldawial7rtrfomx2qkmboyvqve6ywo2jpvfbu7zptga:1:3:4096 +? 1/3,0c0beacef8877bbf2416eb00f2b5dc96354e26dd1df5517320459b1236860f8c,77f3b5f9437275607c3b5eef5f67d18d6a9835155cfee75acd9d2c043dbb8528 +: URI:CHK:gqndvjhqwsgyawfnmpfsr4x6sy:d6txdlbset3s536cdbqhthxxq4r2u4a2eud53ccphc7kmssaa7jq:1:3:8193 +? 1/3,0c0beacef8877bbf2416eb00f2b5dc96354e26dd1df5517320459b1236860f8c,80c58fc1932767b113819b033e1874d8de05e8809534590db9d01510abc8ed6b +: URI:CHK:lfysy2xpqyaz5lx5velu67ifyu:uiarh7kalq2nxtt5nphmdrrg6zsnqzhf4fzdvauzppsls4p5fyka:1:3:2048 +? 1/3,0c0beacef8877bbf2416eb00f2b5dc96354e26dd1df5517320459b1236860f8c,8454e5bb164935d6a8da749d78cf91eb0556897d44273053b67777aed1d418e8 +: URI:CHK:wmdpxzpdjbw3jljibi7k7g4pnu:2jma3twdad5ojmh6e7ysh2kgdhoz7jmanmocxudzzkzbpav2ca5a:1:3:131071 +? 1/3,0c0beacef8877bbf2416eb00f2b5dc96354e26dd1df5517320459b1236860f8c,8a6a6bff19a6f768864a69d6c6c97e6a25385793201eefe2505650978aef9193 +: URI:CHK:pfskrgoirfzk724lpjv4bxtuuu:chq3s4q4wqdj33vpkc47r55niakgzcxncdksp5dhp7aw63zv5u3a:1:3:131073 +? 1/3,0c0beacef8877bbf2416eb00f2b5dc96354e26dd1df5517320459b1236860f8c,caccfbaa894a6cf59e3d677a146924604f8edcd07ed1d61cd1d1569ce4bf3ec5 +: URI:CHK:wfrvklrwkxjbtj6vqbgttlgb5e:utbh3nrqv236xkzmt2fyx3ej2cfnj4pcw532lnfcutsvhgv72opq:1:3:8191 +? 1/3,2e61afd25d3ca76a0ee265dbc22ba5e9d844dc93ac5f2e37199a34fcbdcb5a52,2edc986847e209b4016e141a6dc8716d3207350f416969382d431539bf292e4a +: URI:CHK:cogbisdsvqpkynu6ontf4vlgaq:2biqzdoxqweid7si5y5zb37mf4pzme7tnnn4hw5dunzdvikogdwq:1:3:1024 +? 1/3,2e61afd25d3ca76a0ee265dbc22ba5e9d844dc93ac5f2e37199a34fcbdcb5a52,3abc94a93a42d0eee5c8dda0315f9f1343e2ba36b552ab512c435fd4989c1ac6 +: URI:CHK:yh6cn346w44f7odadasfsyouaa:zut5qrxxzs255gf6a3nefa7qcrxo3bfv72i3xlncmhnd5gzhv4ia:1:3:4096 +? 1/3,2e61afd25d3ca76a0ee265dbc22ba5e9d844dc93ac5f2e37199a34fcbdcb5a52,77f3b5f9437275607c3b5eef5f67d18d6a9835155cfee75acd9d2c043dbb8528 +: URI:CHK:5fjy2fkocl5gye4dgmvqu5kcoi:6oayuhip53s4jhnvjv374mrn5hqaxxzpu62kxdf3sa2jn36y6mzq:1:3:8193 +? 1/3,2e61afd25d3ca76a0ee265dbc22ba5e9d844dc93ac5f2e37199a34fcbdcb5a52,80c58fc1932767b113819b033e1874d8de05e8809534590db9d01510abc8ed6b +: URI:CHK:zp3nc6tyuyhwszywmqehybc6oe:sd32yfuuyyq52ppee6yviivsqrglvgitijl3mweydhpu4zm5hgca:1:3:2048 +? 1/3,2e61afd25d3ca76a0ee265dbc22ba5e9d844dc93ac5f2e37199a34fcbdcb5a52,8454e5bb164935d6a8da749d78cf91eb0556897d44273053b67777aed1d418e8 +: URI:CHK:sexguezmb55m4qbgxb5getu7qu:5ud5zz6kvutmu6mca6b2nicerk7pofigc3ysosyngs5nkctyimaq:1:3:131071 +? 1/3,2e61afd25d3ca76a0ee265dbc22ba5e9d844dc93ac5f2e37199a34fcbdcb5a52,8a6a6bff19a6f768864a69d6c6c97e6a25385793201eefe2505650978aef9193 +: URI:CHK:o6tv2j53qsfosjidti6rr65txm:5gca453tox7fp3qwz43if6t5upl32ialclbko3p562mkizqq3b2a:1:3:131073 +? 1/3,2e61afd25d3ca76a0ee265dbc22ba5e9d844dc93ac5f2e37199a34fcbdcb5a52,caccfbaa894a6cf59e3d677a146924604f8edcd07ed1d61cd1d1569ce4bf3ec5 +: URI:CHK:o7px44ar5tvwjzqghpw46jxoqu:vdaug57fb57cybqhmhbutjamnc5uclvgzhjzebqrhpdpw4vv6znq:1:3:8191 +? 1/3,43a1e8995c86ee7f559e5d187d1657296662ca180b394146a6e1dd0df9158a30,2edc986847e209b4016e141a6dc8716d3207350f416969382d431539bf292e4a +: URI:CHK:pbfcdvxhbikxd2hcc43oel3v3e:y5txpmiptmoz36ionmki3p6krmdbiqasw2v3wdq4ia5lmrhh33lq:1:3:1024 +? 1/3,43a1e8995c86ee7f559e5d187d1657296662ca180b394146a6e1dd0df9158a30,3abc94a93a42d0eee5c8dda0315f9f1343e2ba36b552ab512c435fd4989c1ac6 +: URI:CHK:vkdc7iae4pgkdomhbbb554kv6a:g5xkatxv2entfssfh5eqgexb5j3qs6jvohofjqo5p5erlxlx2dga:1:3:4096 +? 1/3,43a1e8995c86ee7f559e5d187d1657296662ca180b394146a6e1dd0df9158a30,77f3b5f9437275607c3b5eef5f67d18d6a9835155cfee75acd9d2c043dbb8528 +: URI:CHK:mipcizv4soceb3kepaeyaw4k2y:ysg7t5umiqz4cluwk3coqg3yl44c4clegtju3udwffeasrwovddq:1:3:8193 +? 1/3,43a1e8995c86ee7f559e5d187d1657296662ca180b394146a6e1dd0df9158a30,80c58fc1932767b113819b033e1874d8de05e8809534590db9d01510abc8ed6b +: URI:CHK:3yhotahhjha4j6acgc2e2vbktm:khrftxrlbudn7c25f2oidos7or4wdmagmgwzditz232c3b7ea6wq:1:3:2048 +? 1/3,43a1e8995c86ee7f559e5d187d1657296662ca180b394146a6e1dd0df9158a30,8454e5bb164935d6a8da749d78cf91eb0556897d44273053b67777aed1d418e8 +: URI:CHK:6yzsaazkkblikql63nyglc77pq:blsru6a6cu3k4y4nbllgkxidwcehmo7ey2qvidvi6vqywo2j6geq:1:3:131071 +? 1/3,43a1e8995c86ee7f559e5d187d1657296662ca180b394146a6e1dd0df9158a30,8a6a6bff19a6f768864a69d6c6c97e6a25385793201eefe2505650978aef9193 +: URI:CHK:gpdcbuf22tbrrtycwa2l2hclii:6r3r4ocqforegcetf3wpohwkqk4t5b7cyywx3uzqdfeb4dg7iofa:1:3:131073 +? 1/3,43a1e8995c86ee7f559e5d187d1657296662ca180b394146a6e1dd0df9158a30,caccfbaa894a6cf59e3d677a146924604f8edcd07ed1d61cd1d1569ce4bf3ec5 +: URI:CHK:o5jeaynpscalnainlggkedq2ee:3px5gi7msjc3bjm32lxunmen3ugvnl42erh46xg26ae2dcgghg6q:1:3:8191 +? 1/3,be45cb2605bf36bebde684841a28f0fd43c69850a3dce5fedba69928ee3a8991,2edc986847e209b4016e141a6dc8716d3207350f416969382d431539bf292e4a +: URI:CHK:a4ebyitlpldquuq6ucvnha4ixa:4pdnqdib5vswqko3if6ap6rmrgwcixapaqbe33ctsyghgcm3uofa:1:3:1024 +? 1/3,be45cb2605bf36bebde684841a28f0fd43c69850a3dce5fedba69928ee3a8991,3abc94a93a42d0eee5c8dda0315f9f1343e2ba36b552ab512c435fd4989c1ac6 +: URI:CHK:frqacd3kdebc4ur5evzyqwckfq:hbpilmiqtojoedq3zoyiv3pyb27fktpbm6jsln4acfvgadcvh5ia:1:3:4096 +? 1/3,be45cb2605bf36bebde684841a28f0fd43c69850a3dce5fedba69928ee3a8991,77f3b5f9437275607c3b5eef5f67d18d6a9835155cfee75acd9d2c043dbb8528 +: URI:CHK:f6irlornvt557fsjtznb7ebkii:u4i4sab7naw35fisspioirnmdpt4fb6vnsxxakmwqlxhpnr6upcq:1:3:8193 +? 1/3,be45cb2605bf36bebde684841a28f0fd43c69850a3dce5fedba69928ee3a8991,80c58fc1932767b113819b033e1874d8de05e8809534590db9d01510abc8ed6b +: URI:CHK:lff7klhpboefzcfvfub6zwe724:x435m7yjcwul6elf3jaq44flu5jr6fafqzs5vsbi27rovugkbgwq:1:3:2048 +? 1/3,be45cb2605bf36bebde684841a28f0fd43c69850a3dce5fedba69928ee3a8991,8454e5bb164935d6a8da749d78cf91eb0556897d44273053b67777aed1d418e8 +: URI:CHK:7hmarhc2mjvjan2omvipkpzpca:jppvg5d66dtc2cxqgspzq7xejgidwleek2t5i2dygucdjrxotbgq:1:3:131071 +? 1/3,be45cb2605bf36bebde684841a28f0fd43c69850a3dce5fedba69928ee3a8991,8a6a6bff19a6f768864a69d6c6c97e6a25385793201eefe2505650978aef9193 +: URI:CHK:qeiyu3hniuxlhnwm32ftlnv7ma:7gwkm46s2jqwpqlf5fizlgkh766i7oxjfrmxmxs6g2ypagmo4tnq:1:3:131073 +? 1/3,be45cb2605bf36bebde684841a28f0fd43c69850a3dce5fedba69928ee3a8991,caccfbaa894a6cf59e3d677a146924604f8edcd07ed1d61cd1d1569ce4bf3ec5 +: URI:CHK:ivmnumxa5bru43k2snafpcxk6e:77k2cwnqzq7tsf7rui53ar44s6w5x55626rc24pv7ilca33fehyq:1:3:8191 +? 1/3,cd82b3962c8796095a288e29ed94f98017cc681d17ea9ea8861c4034b13a3b6f,2edc986847e209b4016e141a6dc8716d3207350f416969382d431539bf292e4a +: URI:CHK:vcvq26lwgxvjaxgd7a62qnuqem:vllbr4klrxc4w63vzvfpoqpajmephgkjqlh4bawr67q3i7i2q7cq:1:3:1024 +? 1/3,cd82b3962c8796095a288e29ed94f98017cc681d17ea9ea8861c4034b13a3b6f,3abc94a93a42d0eee5c8dda0315f9f1343e2ba36b552ab512c435fd4989c1ac6 +: URI:CHK:k6gfriaqplqeui6xheccfyodim:kbnovfz4pcq6jdjcmgnbfpaqjo4x6e4tzj3qyeeyulrs6apelprq:1:3:4096 +? 1/3,cd82b3962c8796095a288e29ed94f98017cc681d17ea9ea8861c4034b13a3b6f,77f3b5f9437275607c3b5eef5f67d18d6a9835155cfee75acd9d2c043dbb8528 +: URI:CHK:af2unzjbnqmlaaicbssikxm2ba:szxxxayjygrtz72yrha3hsj2ei6xvyosdkdxpx7tuzd4rcg2siwa:1:3:8193 +? 1/3,cd82b3962c8796095a288e29ed94f98017cc681d17ea9ea8861c4034b13a3b6f,80c58fc1932767b113819b033e1874d8de05e8809534590db9d01510abc8ed6b +: URI:CHK:xsiowqs5kymdejztz3zgzjrfpu:t43dbkeimvxiokkl6oz6x5hluidc2siweq4pa66pkgj6zi443y4a:1:3:2048 +? 1/3,cd82b3962c8796095a288e29ed94f98017cc681d17ea9ea8861c4034b13a3b6f,8454e5bb164935d6a8da749d78cf91eb0556897d44273053b67777aed1d418e8 +: URI:CHK:7jve3xr3dkptw7w7bc7ubokqzy:rrr4imn7sc7jpcv4gpkbuaob6buwclz7ubcsc4hckwvfaodq4wpa:1:3:131071 +? 1/3,cd82b3962c8796095a288e29ed94f98017cc681d17ea9ea8861c4034b13a3b6f,8a6a6bff19a6f768864a69d6c6c97e6a25385793201eefe2505650978aef9193 +: URI:CHK:nv4gljql7eyef5yrpx5xfkyez4:hykk52owibpmfmjrgtmi3wzvrpx7xtp3xz6beaieuthhxsob7hra:1:3:131073 +? 1/3,cd82b3962c8796095a288e29ed94f98017cc681d17ea9ea8861c4034b13a3b6f,caccfbaa894a6cf59e3d677a146924604f8edcd07ed1d61cd1d1569ce4bf3ec5 +: URI:CHK:3yntwrjfnabw7tnmh3rkejzhha:fenyosagg43ltfzczjzqmp3y4jmt2y4q6b2m6pg6oliijpxeacjq:1:3:8191 +? 1/3,f39dac6cbaba535e2c207cd0cd8f154974223c848f727f98b3564cea569b41cf,2edc986847e209b4016e141a6dc8716d3207350f416969382d431539bf292e4a +: URI:CHK:35mmxquvlxib7ev3it3dn2zuta:c2ep3r5wapqze6234pep5w6ssf2oj2k2rppnyzfvuvnuuch3eekq:1:3:1024 +? 1/3,f39dac6cbaba535e2c207cd0cd8f154974223c848f727f98b3564cea569b41cf,3abc94a93a42d0eee5c8dda0315f9f1343e2ba36b552ab512c435fd4989c1ac6 +: URI:CHK:qprhw4ebrdbje4pn7qvqenyo4q:mi2fnz4fkjvgzrgzzprh3d3s6xzvlqzwwbo4iuybqtl3uv6qdsha:1:3:4096 +? 1/3,f39dac6cbaba535e2c207cd0cd8f154974223c848f727f98b3564cea569b41cf,77f3b5f9437275607c3b5eef5f67d18d6a9835155cfee75acd9d2c043dbb8528 +: URI:CHK:jeo5ldffp4n3ldsd2cajazhmsy:2mewlw2mcx6nqs7ushmk7ipfkceppkil2wvbvloa7ub47igpnirq:1:3:8193 +? 1/3,f39dac6cbaba535e2c207cd0cd8f154974223c848f727f98b3564cea569b41cf,80c58fc1932767b113819b033e1874d8de05e8809534590db9d01510abc8ed6b +: URI:CHK:hjiajko6mh74umf3k5wcdx3tra:feznuxdbn4mv42hkyp6di55ddpfaxreac4kwrh4fxjvkgrz3cwsa:1:3:2048 +? 1/3,f39dac6cbaba535e2c207cd0cd8f154974223c848f727f98b3564cea569b41cf,8454e5bb164935d6a8da749d78cf91eb0556897d44273053b67777aed1d418e8 +: URI:CHK:mxvgpptt5g4cmmlwduo2dbuhvi:uubgdwc2h4rsudgqft6tedeybltzzlxp4qwqxkqjvwqws3wsixoa:1:3:131071 +? 1/3,f39dac6cbaba535e2c207cd0cd8f154974223c848f727f98b3564cea569b41cf,8a6a6bff19a6f768864a69d6c6c97e6a25385793201eefe2505650978aef9193 +: URI:CHK:bblnb2r5keyy5ybguji4xuyf74:z32ynttnjxh3nqtf4krlrqptu4vmajxpce3q22igc4dr5nx3y2ra:1:3:131073 +? 1/3,f39dac6cbaba535e2c207cd0cd8f154974223c848f727f98b3564cea569b41cf,caccfbaa894a6cf59e3d677a146924604f8edcd07ed1d61cd1d1569ce4bf3ec5 +: URI:CHK:6woltewcikjtsbbucyipnjcaay:avs4cqqyjvjj4awkxvsifc7fsmtbvbp35pqtyo4jejnvdm75uvjq:1:3:8191 +? 101/256,0c0beacef8877bbf2416eb00f2b5dc96354e26dd1df5517320459b1236860f8c,2edc986847e209b4016e141a6dc8716d3207350f416969382d431539bf292e4a +: URI:CHK:mftnvh5l3brshwoupvu6kz35cy:sncfokby2tjykbr3zgi3i6lhgerg4kz7fsrslrlxdy7upwkzq6lq:101:256:1024 +? 101/256,0c0beacef8877bbf2416eb00f2b5dc96354e26dd1df5517320459b1236860f8c,3abc94a93a42d0eee5c8dda0315f9f1343e2ba36b552ab512c435fd4989c1ac6 +: URI:CHK:sshu77h6opnto3jnngot3lwzl4:4cjhp6u3i2uzdzrjitje3inlhs6gnlgwc6wgi6vtedc2grgvcpza:101:256:4096 +? 101/256,0c0beacef8877bbf2416eb00f2b5dc96354e26dd1df5517320459b1236860f8c,77f3b5f9437275607c3b5eef5f67d18d6a9835155cfee75acd9d2c043dbb8528 +: URI:CHK:fhan3wrsj22np5sp34ezma3cuu:4xprdxp4mhsowp74snsn52rtkuxdyrfdckyygr2ineo6xjuog2qa:101:256:8193 +? 101/256,0c0beacef8877bbf2416eb00f2b5dc96354e26dd1df5517320459b1236860f8c,80c58fc1932767b113819b033e1874d8de05e8809534590db9d01510abc8ed6b +: URI:CHK:4wimykgotkgywu7d26obnszwoa:zl2e6fnyrn72z27yzsal734srjz6erg227z2pacvxqe5yzy3ssjq:101:256:2048 +? 101/256,0c0beacef8877bbf2416eb00f2b5dc96354e26dd1df5517320459b1236860f8c,8454e5bb164935d6a8da749d78cf91eb0556897d44273053b67777aed1d418e8 +: URI:CHK:gkbzvojuugtuj3m55ib376eazy:473bxxwpfa4zahetztea2gadrrfdv26rrhfeaakogqvk3ngay5gq:101:256:131071 +? 101/256,0c0beacef8877bbf2416eb00f2b5dc96354e26dd1df5517320459b1236860f8c,8a6a6bff19a6f768864a69d6c6c97e6a25385793201eefe2505650978aef9193 +: URI:CHK:tez3j5xoszuttu3uhvgbztk4zu:b6s66mdiopowshfrdtycnumaacko63uszz6ghnzmyzo7a5n5a3tq:101:256:131073 +? 101/256,0c0beacef8877bbf2416eb00f2b5dc96354e26dd1df5517320459b1236860f8c,caccfbaa894a6cf59e3d677a146924604f8edcd07ed1d61cd1d1569ce4bf3ec5 +: URI:CHK:f5zsbt3wuawsgu5allwqvfuhai:djhrg2jjdubbyi6lol62fwyfcryawnnw6ivqt76rlw5slz4w2kda:101:256:8191 +? 101/256,2e61afd25d3ca76a0ee265dbc22ba5e9d844dc93ac5f2e37199a34fcbdcb5a52,2edc986847e209b4016e141a6dc8716d3207350f416969382d431539bf292e4a +: URI:CHK:f7wgsiwpd2ieoe3iy5fg5uq4bq:nt7zovrykadj6jirc7guq3hyfesdd77rzucez7awdswg2kbfga5q:101:256:1024 +? 101/256,2e61afd25d3ca76a0ee265dbc22ba5e9d844dc93ac5f2e37199a34fcbdcb5a52,3abc94a93a42d0eee5c8dda0315f9f1343e2ba36b552ab512c435fd4989c1ac6 +: URI:CHK:y6j4jiy3asjv66qdacozkgiduq:acqcevrjrfbtjbe5d4rl4y566lv2ype4dsyoznedbrqzjy4pxe6a:101:256:4096 +? 101/256,2e61afd25d3ca76a0ee265dbc22ba5e9d844dc93ac5f2e37199a34fcbdcb5a52,77f3b5f9437275607c3b5eef5f67d18d6a9835155cfee75acd9d2c043dbb8528 +: URI:CHK:ldeo5hbywzenytsqqauftpthoy:7lye6cutm2btoen2rspz7pqipxrhynpk2bvecrvhvwsfcuaed3fa:101:256:8193 +? 101/256,2e61afd25d3ca76a0ee265dbc22ba5e9d844dc93ac5f2e37199a34fcbdcb5a52,80c58fc1932767b113819b033e1874d8de05e8809534590db9d01510abc8ed6b +: URI:CHK:ts2nlbfh6i7cvg4y5hywshzb7e:cyceefzvc2pdcqjyzxq7vnvgsxkoaok7kfwyy6f2q4ahxr6inkaa:101:256:2048 +? 101/256,2e61afd25d3ca76a0ee265dbc22ba5e9d844dc93ac5f2e37199a34fcbdcb5a52,8454e5bb164935d6a8da749d78cf91eb0556897d44273053b67777aed1d418e8 +: URI:CHK:ds7rbmi4ewkc7u2diopc463bsy:n4xbp2xjogncvhzaph62u5p3lt3263cosz7o5zh4fdl32x7ggmqq:101:256:131071 +? 101/256,2e61afd25d3ca76a0ee265dbc22ba5e9d844dc93ac5f2e37199a34fcbdcb5a52,8a6a6bff19a6f768864a69d6c6c97e6a25385793201eefe2505650978aef9193 +: URI:CHK:xruphp3lx7spgpjmrbzcw3h63u:5txd3djukdpxgimev3ahhej7zftglfk65dqfgosgwd7lqna6t2ga:101:256:131073 +? 101/256,2e61afd25d3ca76a0ee265dbc22ba5e9d844dc93ac5f2e37199a34fcbdcb5a52,caccfbaa894a6cf59e3d677a146924604f8edcd07ed1d61cd1d1569ce4bf3ec5 +: URI:CHK:uzhqk67vyyixgbs4uppciyfdmm:3oll2cjamxozrur3dyz3ga2puswrwitk7lmwkpb3gxuexe3mi2aq:101:256:8191 +? 101/256,43a1e8995c86ee7f559e5d187d1657296662ca180b394146a6e1dd0df9158a30,2edc986847e209b4016e141a6dc8716d3207350f416969382d431539bf292e4a +: URI:CHK:ihyeqq5peg2pdyw45ct5mmj4ay:2rzhndv6o4knlvo4po6rvbnknlcitasseea3mirqjor5c7quh5ra:101:256:1024 +? 101/256,43a1e8995c86ee7f559e5d187d1657296662ca180b394146a6e1dd0df9158a30,3abc94a93a42d0eee5c8dda0315f9f1343e2ba36b552ab512c435fd4989c1ac6 +: URI:CHK:mvk5f2mghudk7xnknqddhvnl4m:c74myeugtix7e7l6uwybtrdtfselha2qrcc7hhbymqr6brtg4cea:101:256:4096 +? 101/256,43a1e8995c86ee7f559e5d187d1657296662ca180b394146a6e1dd0df9158a30,77f3b5f9437275607c3b5eef5f67d18d6a9835155cfee75acd9d2c043dbb8528 +: URI:CHK:4f5moqibmc776kn2eyy6lewgrm:e6fjp7worf4ejwj3bhwdbxpwwds73xxlluqnq2h7jeink3k3hlzq:101:256:8193 +? 101/256,43a1e8995c86ee7f559e5d187d1657296662ca180b394146a6e1dd0df9158a30,80c58fc1932767b113819b033e1874d8de05e8809534590db9d01510abc8ed6b +: URI:CHK:xy354pqtzbmlmzwby3hei7kqxu:oj7en424yabutdnsgvak5npd7uzaxk7n35wdnw2ehpaswf6sba2q:101:256:2048 +? 101/256,43a1e8995c86ee7f559e5d187d1657296662ca180b394146a6e1dd0df9158a30,8454e5bb164935d6a8da749d78cf91eb0556897d44273053b67777aed1d418e8 +: URI:CHK:fpjk7xgei2hctfwaz7u7icym3q:li7vkv74rtx5atvidmeg5qgku7quz2opuc3kzsenhdlynzqswgna:101:256:131071 +? 101/256,43a1e8995c86ee7f559e5d187d1657296662ca180b394146a6e1dd0df9158a30,8a6a6bff19a6f768864a69d6c6c97e6a25385793201eefe2505650978aef9193 +: URI:CHK:edmto6grvkwbg3f2qbd4hucmmu:n4xb35itwdp66p73tyjddk6pi3w4nqxzzeactdpnl4rrmiwcbs6q:101:256:131073 +? 101/256,43a1e8995c86ee7f559e5d187d1657296662ca180b394146a6e1dd0df9158a30,caccfbaa894a6cf59e3d677a146924604f8edcd07ed1d61cd1d1569ce4bf3ec5 +: URI:CHK:oj6tnrpdt76cjhtlab2kk2edhe:jsgpmpguhzrqxautcpmkwldbdl5u4vfvjthefua3dikf5e6r2tfq:101:256:8191 +? 101/256,be45cb2605bf36bebde684841a28f0fd43c69850a3dce5fedba69928ee3a8991,2edc986847e209b4016e141a6dc8716d3207350f416969382d431539bf292e4a +: URI:CHK:6tv464bg2dhui7ux7s3qrbizwq:zcew6imlkwdka6rwiqloi6gmadq7nrl4s3sbwjadgb2pqci7r7la:101:256:1024 +? 101/256,be45cb2605bf36bebde684841a28f0fd43c69850a3dce5fedba69928ee3a8991,3abc94a93a42d0eee5c8dda0315f9f1343e2ba36b552ab512c435fd4989c1ac6 +: URI:CHK:uqx46dwiak4sbzdhtw26syqvmm:qmd6gzzuoahnnigl4qfutllbz6gwqw3dwxa5imfnpgvctfquoqtq:101:256:4096 +? 101/256,be45cb2605bf36bebde684841a28f0fd43c69850a3dce5fedba69928ee3a8991,77f3b5f9437275607c3b5eef5f67d18d6a9835155cfee75acd9d2c043dbb8528 +: URI:CHK:a2stcgnpwl3cyedubvsc2534zu:huxjlll2ciq2fszm5gsc2qbyp7ch4gc3eli4bgs5qyhn6f2nbdfq:101:256:8193 +? 101/256,be45cb2605bf36bebde684841a28f0fd43c69850a3dce5fedba69928ee3a8991,80c58fc1932767b113819b033e1874d8de05e8809534590db9d01510abc8ed6b +: URI:CHK:sonrxjr7bble64yaygtpjdwifa:su7ciagahmf4a3aeuxavwcjlo7ou6zcyyoe6ef3qqi2o7fnljiaq:101:256:2048 +? 101/256,be45cb2605bf36bebde684841a28f0fd43c69850a3dce5fedba69928ee3a8991,8454e5bb164935d6a8da749d78cf91eb0556897d44273053b67777aed1d418e8 +: URI:CHK:z7fjyntzsb47tpjc5wxj7ehl6i:5yfcwqbvgvfsercgpaxkcsdxzptzc6iehqhbf6rgxibsggbkmika:101:256:131071 +? 101/256,be45cb2605bf36bebde684841a28f0fd43c69850a3dce5fedba69928ee3a8991,8a6a6bff19a6f768864a69d6c6c97e6a25385793201eefe2505650978aef9193 +: URI:CHK:wnyvsaz7sfbi47zbvjq6veskzq:ju2bh3k2wfr6xxalreb2uzq6xbsmllrt23hdufo7inr2hzrbkv3q:101:256:131073 +? 101/256,be45cb2605bf36bebde684841a28f0fd43c69850a3dce5fedba69928ee3a8991,caccfbaa894a6cf59e3d677a146924604f8edcd07ed1d61cd1d1569ce4bf3ec5 +: URI:CHK:kxm6xaqm7e7ncfmcrzxypdamhu:b2oybfq6w4cuw4rmtfcwvmc2mkp5fchbf5aq3cixynwwuwrh7lkq:101:256:8191 +? 101/256,cd82b3962c8796095a288e29ed94f98017cc681d17ea9ea8861c4034b13a3b6f,2edc986847e209b4016e141a6dc8716d3207350f416969382d431539bf292e4a +: URI:CHK:5ezlshdnhxu2pwle42lvtixc44:ubwznjuovofoehgcfje3cpfstfntnw7h23jfa6obtfbbm5octi6q:101:256:1024 +? 101/256,cd82b3962c8796095a288e29ed94f98017cc681d17ea9ea8861c4034b13a3b6f,3abc94a93a42d0eee5c8dda0315f9f1343e2ba36b552ab512c435fd4989c1ac6 +: URI:CHK:jjsarrteexsv6hrqglslumloxa:xvgoz5m2ka6crduwh2gsoxsbvf2u7yb2upwp7obwwlcndqedpnha:101:256:4096 +? 101/256,cd82b3962c8796095a288e29ed94f98017cc681d17ea9ea8861c4034b13a3b6f,77f3b5f9437275607c3b5eef5f67d18d6a9835155cfee75acd9d2c043dbb8528 +: URI:CHK:b6fbcvqshq4jbg42jwioif4yga:mtrgqr6wmholpqn6wxt6hfny2zysmqnsubex57q5bakieos6jotq:101:256:8193 +? 101/256,cd82b3962c8796095a288e29ed94f98017cc681d17ea9ea8861c4034b13a3b6f,80c58fc1932767b113819b033e1874d8de05e8809534590db9d01510abc8ed6b +: URI:CHK:csywucom45snjtrhuvwr55jx2a:pnjbpjtgroscazbbhhzzgxtvqsnqduey6muvizzvdjy5b6phwqka:101:256:2048 +? 101/256,cd82b3962c8796095a288e29ed94f98017cc681d17ea9ea8861c4034b13a3b6f,8454e5bb164935d6a8da749d78cf91eb0556897d44273053b67777aed1d418e8 +: URI:CHK:5quoy4xiiwnxqekhtqe4y4icmi:mkz2pbuen6n3wb4tqjngmu5t645nnaaye4zs2o6iagojcrtrchua:101:256:131071 +? 101/256,cd82b3962c8796095a288e29ed94f98017cc681d17ea9ea8861c4034b13a3b6f,8a6a6bff19a6f768864a69d6c6c97e6a25385793201eefe2505650978aef9193 +: URI:CHK:adzihjhpwm5vfq7y7tjikjgruq:su3erjgmgbzf2g6xlhnvrzy3w76v4kdiakedizgp4zkehhn6ztiq:101:256:131073 +? 101/256,cd82b3962c8796095a288e29ed94f98017cc681d17ea9ea8861c4034b13a3b6f,caccfbaa894a6cf59e3d677a146924604f8edcd07ed1d61cd1d1569ce4bf3ec5 +: URI:CHK:42cyabs6gdfuzqpsg6unlmxq7a:u7tit3edtyazrapvzvklpf3o2mykepnc3v66lsndellpxz3dciia:101:256:8191 +? 101/256,f39dac6cbaba535e2c207cd0cd8f154974223c848f727f98b3564cea569b41cf,2edc986847e209b4016e141a6dc8716d3207350f416969382d431539bf292e4a +: URI:CHK:g23jpnnqww4v3au5dai2fu5jxy:tghgrgfjbfu37zl3l4keahps74tlsgwbidvxbtge2o6pcbjxjjeq:101:256:1024 +? 101/256,f39dac6cbaba535e2c207cd0cd8f154974223c848f727f98b3564cea569b41cf,3abc94a93a42d0eee5c8dda0315f9f1343e2ba36b552ab512c435fd4989c1ac6 +: URI:CHK:k6igwbutc3cggihl7itzfcltbe:ejbikbgtcbbcfkuhtbbtdcj4jr666jjancn35vgeipnvor72xq2a:101:256:4096 +? 101/256,f39dac6cbaba535e2c207cd0cd8f154974223c848f727f98b3564cea569b41cf,77f3b5f9437275607c3b5eef5f67d18d6a9835155cfee75acd9d2c043dbb8528 +: URI:CHK:ywnywmjsazdgg7nf3dchrlgwoi:sp3xlizgz6gnye2kcdxunm3tual5cmcsckrz6rybuxj2atnrao5a:101:256:8193 +? 101/256,f39dac6cbaba535e2c207cd0cd8f154974223c848f727f98b3564cea569b41cf,80c58fc1932767b113819b033e1874d8de05e8809534590db9d01510abc8ed6b +: URI:CHK:dkxeqnu42nhmh6z24t7ya6rwla:uobek7aktnlksbwoumrn6jlhhmbk2rikjyouchambre4fqskw5na:101:256:2048 +? 101/256,f39dac6cbaba535e2c207cd0cd8f154974223c848f727f98b3564cea569b41cf,8454e5bb164935d6a8da749d78cf91eb0556897d44273053b67777aed1d418e8 +: URI:CHK:63dhrxqcqa5hr5uu2hazaq3mxq:do2zijkco5j4kbrj4tf3f6ykdqhakhlmixs3elbsokaztrbsokgq:101:256:131071 +? 101/256,f39dac6cbaba535e2c207cd0cd8f154974223c848f727f98b3564cea569b41cf,8a6a6bff19a6f768864a69d6c6c97e6a25385793201eefe2505650978aef9193 +: URI:CHK:64dvle2iefzyvpqcytkugy2kda:oildbyluuth6xthq24lwxbumpnishztheazac55f6penotwsztqq:101:256:131073 +? 101/256,f39dac6cbaba535e2c207cd0cd8f154974223c848f727f98b3564cea569b41cf,caccfbaa894a6cf59e3d677a146924604f8edcd07ed1d61cd1d1569ce4bf3ec5 +: URI:CHK:yupjn3fl7xgygpga2hgkijjp7e:juqszewyjp7jttlnltyz3ahp7rtywkmxzr2f5wr6jsvzffni6ukq:101:256:8191 +? 2/3,0c0beacef8877bbf2416eb00f2b5dc96354e26dd1df5517320459b1236860f8c,2edc986847e209b4016e141a6dc8716d3207350f416969382d431539bf292e4a +: URI:CHK:jdm6ytmx2i3ya2bllsdzxurjtu:iqr6tyysaseegzfurhuywy3mbbwkbbsov5bt3fo6oazpyv7olvda:2:3:1024 +? 2/3,0c0beacef8877bbf2416eb00f2b5dc96354e26dd1df5517320459b1236860f8c,3abc94a93a42d0eee5c8dda0315f9f1343e2ba36b552ab512c435fd4989c1ac6 +: URI:CHK:7wz5dhkyob3gmkydzgbptk5qku:cl6ovq5a3km7rpfhyb3putg656sp57lhnp7aexbocsbbahdxnfia:2:3:4096 +? 2/3,0c0beacef8877bbf2416eb00f2b5dc96354e26dd1df5517320459b1236860f8c,77f3b5f9437275607c3b5eef5f67d18d6a9835155cfee75acd9d2c043dbb8528 +: URI:CHK:x6mwrib5rq7rovhqqkdrfj3fei:wjiher75xpe646ccus4etlz6gmsxslkkretchupdqgr4o6uzuvia:2:3:8193 +? 2/3,0c0beacef8877bbf2416eb00f2b5dc96354e26dd1df5517320459b1236860f8c,80c58fc1932767b113819b033e1874d8de05e8809534590db9d01510abc8ed6b +: URI:CHK:o3o2nxtuo6mm4iapnfqzllckcm:epp2ynxr2ok2po3ihvdpimh3q5e2y3r62gfygj7qgloowu5wlika:2:3:2048 +? 2/3,0c0beacef8877bbf2416eb00f2b5dc96354e26dd1df5517320459b1236860f8c,8454e5bb164935d6a8da749d78cf91eb0556897d44273053b67777aed1d418e8 +: URI:CHK:33g3nzjobrl5unjffmqegkgzri:hhj3t5tmc4zdx7n4zgriyie4ujtdoj4epcesmqf2ubjti5cs2r2a:2:3:131071 +? 2/3,0c0beacef8877bbf2416eb00f2b5dc96354e26dd1df5517320459b1236860f8c,8a6a6bff19a6f768864a69d6c6c97e6a25385793201eefe2505650978aef9193 +: URI:CHK:kasnq2fdmif3hnxqhofujik6c4:texnwykka5fnzwb4324xtlv5x2looo6kkfpb4f2vmr6zqgrusk7a:2:3:131073 +? 2/3,0c0beacef8877bbf2416eb00f2b5dc96354e26dd1df5517320459b1236860f8c,caccfbaa894a6cf59e3d677a146924604f8edcd07ed1d61cd1d1569ce4bf3ec5 +: URI:CHK:dwsrkeq2zmixvntzulpqethvvi:tkjxibzjnkzjcns7ofvkw74o4x3r3dx7ic6ocbakrr4t2lfh3nea:2:3:8191 +? 2/3,2e61afd25d3ca76a0ee265dbc22ba5e9d844dc93ac5f2e37199a34fcbdcb5a52,2edc986847e209b4016e141a6dc8716d3207350f416969382d431539bf292e4a +: URI:CHK:ggdnrfd7oqwgmaq3mlzyxbc6fq:5h66csimxvecntptd36qrn6jb7g7yfogpbgfz5pj765xn6wj3utq:2:3:1024 +? 2/3,2e61afd25d3ca76a0ee265dbc22ba5e9d844dc93ac5f2e37199a34fcbdcb5a52,3abc94a93a42d0eee5c8dda0315f9f1343e2ba36b552ab512c435fd4989c1ac6 +: URI:CHK:appba4iq3vzl267nedzhfpjgny:pt7gakv7red2udejvzdlirsgci44b6chzndv7aksbm62xm5p7dwa:2:3:4096 +? 2/3,2e61afd25d3ca76a0ee265dbc22ba5e9d844dc93ac5f2e37199a34fcbdcb5a52,77f3b5f9437275607c3b5eef5f67d18d6a9835155cfee75acd9d2c043dbb8528 +: URI:CHK:fmllehy7ds6zu2yufgamneuhnm:6q7tu3fwv2kqknrffnz3g3juc3eckjprj62hrvk2sz7no77q3fgq:2:3:8193 +? 2/3,2e61afd25d3ca76a0ee265dbc22ba5e9d844dc93ac5f2e37199a34fcbdcb5a52,80c58fc1932767b113819b033e1874d8de05e8809534590db9d01510abc8ed6b +: URI:CHK:cmpwnp5m42dvuzbiewpawtamqy:llxkfyxees27paf2sjyfvo5ppiw6mefcnlk4enrf3upzxfntzhca:2:3:2048 +? 2/3,2e61afd25d3ca76a0ee265dbc22ba5e9d844dc93ac5f2e37199a34fcbdcb5a52,8454e5bb164935d6a8da749d78cf91eb0556897d44273053b67777aed1d418e8 +: URI:CHK:53ev7ja4kbovkcfvrjctskexgm:nkutoibua7cnw42ysil3aynwrfhlqdayidotcisizu4loo6dmjka:2:3:131071 +? 2/3,2e61afd25d3ca76a0ee265dbc22ba5e9d844dc93ac5f2e37199a34fcbdcb5a52,8a6a6bff19a6f768864a69d6c6c97e6a25385793201eefe2505650978aef9193 +: URI:CHK:t5oode47lyq6gievqcjxedw43a:rkfywj4x64hnhn2ldq4asgquecuq7sbz2pmd3ig5oigd6j7jtoga:2:3:131073 +? 2/3,2e61afd25d3ca76a0ee265dbc22ba5e9d844dc93ac5f2e37199a34fcbdcb5a52,caccfbaa894a6cf59e3d677a146924604f8edcd07ed1d61cd1d1569ce4bf3ec5 +: URI:CHK:du6mixt5gg4xjhz6rbnxwrataa:sab52dir4oc7rytzthdmfr4f4yvs22ewpf7tt6k6ac4ngpmcvukq:2:3:8191 +? 2/3,43a1e8995c86ee7f559e5d187d1657296662ca180b394146a6e1dd0df9158a30,2edc986847e209b4016e141a6dc8716d3207350f416969382d431539bf292e4a +: URI:CHK:ld3thziutpaqv25nbtoqebhtru:ri2obsvzl2etyuv3qnchn2wvw5mh5rjuvbjaqkqbvyprn7xwamoq:2:3:1024 +? 2/3,43a1e8995c86ee7f559e5d187d1657296662ca180b394146a6e1dd0df9158a30,3abc94a93a42d0eee5c8dda0315f9f1343e2ba36b552ab512c435fd4989c1ac6 +: URI:CHK:pz3mga5pvqd4bpsz7pbyna2qtq:w3ptt7xwwgywod3u2rpgx7iggvm2kpwl7d2ryr3u7mcaxgqotlaq:2:3:4096 +? 2/3,43a1e8995c86ee7f559e5d187d1657296662ca180b394146a6e1dd0df9158a30,77f3b5f9437275607c3b5eef5f67d18d6a9835155cfee75acd9d2c043dbb8528 +: URI:CHK:mhrrcvp5qcxxlktw4hj6bqfnkq:jj76kg5tr7q5m5fracpquhozfq34ttv3uqclskvoeftvfbpjdyqq:2:3:8193 +? 2/3,43a1e8995c86ee7f559e5d187d1657296662ca180b394146a6e1dd0df9158a30,80c58fc1932767b113819b033e1874d8de05e8809534590db9d01510abc8ed6b +: URI:CHK:lmx2nobpjuljsz4guu7jl55mnu:pbl3dskb3qhmayowi5aoszhlorphm3xjl7xgofzosdbug2ap6h7a:2:3:2048 +? 2/3,43a1e8995c86ee7f559e5d187d1657296662ca180b394146a6e1dd0df9158a30,8454e5bb164935d6a8da749d78cf91eb0556897d44273053b67777aed1d418e8 +: URI:CHK:tykhb62tv32cxeb3t2ievvyiha:cbgakfrdmnuwwmumqtwg53gm6agujd56lep3rnf6glc4y5afqmia:2:3:131071 +? 2/3,43a1e8995c86ee7f559e5d187d1657296662ca180b394146a6e1dd0df9158a30,8a6a6bff19a6f768864a69d6c6c97e6a25385793201eefe2505650978aef9193 +: URI:CHK:3expkq235v4ugmard7riwcdcp4:sh2yanw7j4lg5hz7fna5n2tw4yigw3fk3qiinkmmdhz7wnitg6lq:2:3:131073 +? 2/3,43a1e8995c86ee7f559e5d187d1657296662ca180b394146a6e1dd0df9158a30,caccfbaa894a6cf59e3d677a146924604f8edcd07ed1d61cd1d1569ce4bf3ec5 +: URI:CHK:rubvwrpa4pgdmlpwlfefhkte24:bbqakzizzwk7wvelfhnvcsuxcuirzyplcqc5qqujrqfny3wrswna:2:3:8191 +? 2/3,be45cb2605bf36bebde684841a28f0fd43c69850a3dce5fedba69928ee3a8991,2edc986847e209b4016e141a6dc8716d3207350f416969382d431539bf292e4a +: URI:CHK:nvndhmpesezx4hr426lcrfopqy:gnx56qcnolpy2erpayo6lil5qzsomfbhv3quofsa654ks4sta7lq:2:3:1024 +? 2/3,be45cb2605bf36bebde684841a28f0fd43c69850a3dce5fedba69928ee3a8991,3abc94a93a42d0eee5c8dda0315f9f1343e2ba36b552ab512c435fd4989c1ac6 +: URI:CHK:zwbvg2jvulmv34ixtwwfgugk7m:cd2rwx4oo7fikddpjkhswnwofr4s6mj2texiuv34xp5722kpfyiq:2:3:4096 +? 2/3,be45cb2605bf36bebde684841a28f0fd43c69850a3dce5fedba69928ee3a8991,77f3b5f9437275607c3b5eef5f67d18d6a9835155cfee75acd9d2c043dbb8528 +: URI:CHK:t3m3hhlt3cbtobinz7bupsuphm:hrpiylbj3r4drrnupj7v54d4bwauz4s4llosxkawpm6fetwflqoa:2:3:8193 +? 2/3,be45cb2605bf36bebde684841a28f0fd43c69850a3dce5fedba69928ee3a8991,80c58fc1932767b113819b033e1874d8de05e8809534590db9d01510abc8ed6b +: URI:CHK:mbdlys4nttnwkehseindeoz4aa:jqutmu5qr2jppkdbbvlf7d33ugumaraoxi2mu6d4tocmbjnw5cqa:2:3:2048 +? 2/3,be45cb2605bf36bebde684841a28f0fd43c69850a3dce5fedba69928ee3a8991,8454e5bb164935d6a8da749d78cf91eb0556897d44273053b67777aed1d418e8 +: URI:CHK:y56rgygxem4dycq6jbqn6jdjdm:c3vw5i5avlm5fsqmathclfy3o7k4abiinp3lqzgvzndv6sbfawbq:2:3:131071 +? 2/3,be45cb2605bf36bebde684841a28f0fd43c69850a3dce5fedba69928ee3a8991,8a6a6bff19a6f768864a69d6c6c97e6a25385793201eefe2505650978aef9193 +: URI:CHK:wbpyopk35tm7z7foczudou4e2a:34wamdd2la4chje42ncfu7zj6vypwygmtpad6mfakbw3agycfgoa:2:3:131073 +? 2/3,be45cb2605bf36bebde684841a28f0fd43c69850a3dce5fedba69928ee3a8991,caccfbaa894a6cf59e3d677a146924604f8edcd07ed1d61cd1d1569ce4bf3ec5 +: URI:CHK:hatspmrspkgpedybcqrij62hj4:buo3twcstwlmasnjywjqvdk3r3auzkbwzrsuitzgucayw3tkskxq:2:3:8191 +? 2/3,cd82b3962c8796095a288e29ed94f98017cc681d17ea9ea8861c4034b13a3b6f,2edc986847e209b4016e141a6dc8716d3207350f416969382d431539bf292e4a +: URI:CHK:sxliom42kylmiggxfb2xzetj24:sqly2tkyr4kommagjccbjmswsnaioirorvqqtsucg6pmrqi4mpzq:2:3:1024 +? 2/3,cd82b3962c8796095a288e29ed94f98017cc681d17ea9ea8861c4034b13a3b6f,3abc94a93a42d0eee5c8dda0315f9f1343e2ba36b552ab512c435fd4989c1ac6 +: URI:CHK:u4m3m5gjga5tbiefdyxmhucina:tweiddczb6675d3vrkp4fuu5xrvml5xvzj322qyahr3camurw4ra:2:3:4096 +? 2/3,cd82b3962c8796095a288e29ed94f98017cc681d17ea9ea8861c4034b13a3b6f,77f3b5f9437275607c3b5eef5f67d18d6a9835155cfee75acd9d2c043dbb8528 +: URI:CHK:s2ixjohdpmwnr3c66tjfaflng4:p2i4pvbvf3z7aguin56uerbifvlhgzwhchkanhf6je4qclodd2jq:2:3:8193 +? 2/3,cd82b3962c8796095a288e29ed94f98017cc681d17ea9ea8861c4034b13a3b6f,80c58fc1932767b113819b033e1874d8de05e8809534590db9d01510abc8ed6b +: URI:CHK:367h226jbryhxhdouhcumna7cq:fhiul4h7pgwcrzdpluzhuxvedfijruxjaqbmpkwrh54xyhxljqga:2:3:2048 +? 2/3,cd82b3962c8796095a288e29ed94f98017cc681d17ea9ea8861c4034b13a3b6f,8454e5bb164935d6a8da749d78cf91eb0556897d44273053b67777aed1d418e8 +: URI:CHK:5jaiz63t6k7bktjwp4vbr7m2fq:wtpdpfokxepmokcpwtypg7fd7jmr73fffw5h3kbzb3fcjbrjgv6a:2:3:131071 +? 2/3,cd82b3962c8796095a288e29ed94f98017cc681d17ea9ea8861c4034b13a3b6f,8a6a6bff19a6f768864a69d6c6c97e6a25385793201eefe2505650978aef9193 +: URI:CHK:x4hpdnr5lrq7uqvm42mijivhle:oba26veckkqz23umeyp2cd5vaoock3d6uzhxxlwn2w2aztj4oi7q:2:3:131073 +? 2/3,cd82b3962c8796095a288e29ed94f98017cc681d17ea9ea8861c4034b13a3b6f,caccfbaa894a6cf59e3d677a146924604f8edcd07ed1d61cd1d1569ce4bf3ec5 +: URI:CHK:ovqefeaxuvitgzbzlpkdnmxo64:3y2jxab2o2iclexkqlxnlk4xfw6jvbikjh5ho3okdvpasv6cduuq:2:3:8191 +? 2/3,f39dac6cbaba535e2c207cd0cd8f154974223c848f727f98b3564cea569b41cf,2edc986847e209b4016e141a6dc8716d3207350f416969382d431539bf292e4a +: URI:CHK:d5upd7gpfxw62sppsntkw73svu:i4fr2ejfbkulxxim6j2tpduhhywr45edoooms7k5ou55q4be4lka:2:3:1024 +? 2/3,f39dac6cbaba535e2c207cd0cd8f154974223c848f727f98b3564cea569b41cf,3abc94a93a42d0eee5c8dda0315f9f1343e2ba36b552ab512c435fd4989c1ac6 +: URI:CHK:itfpgduf4gr5qonqisl4fupb4u:aynkfv4qke6jqpn6pe7jcad65az2wnxp6u3sxq3chsw2trzxtieq:2:3:4096 +? 2/3,f39dac6cbaba535e2c207cd0cd8f154974223c848f727f98b3564cea569b41cf,77f3b5f9437275607c3b5eef5f67d18d6a9835155cfee75acd9d2c043dbb8528 +: URI:CHK:2qkptbxaidg7vloyuhvyooflui:y2cxicn3skyne2jnwyah63wtngle2ffjn62xvkbr2xktbc3lreiq:2:3:8193 +? 2/3,f39dac6cbaba535e2c207cd0cd8f154974223c848f727f98b3564cea569b41cf,80c58fc1932767b113819b033e1874d8de05e8809534590db9d01510abc8ed6b +: URI:CHK:oyj7oyyiaihipvpxrav4np7474:snjty23mprvvoja3uwg2eu5ivz4ajt4zsvevfub2wozjjj55ytoq:2:3:2048 +? 2/3,f39dac6cbaba535e2c207cd0cd8f154974223c848f727f98b3564cea569b41cf,8454e5bb164935d6a8da749d78cf91eb0556897d44273053b67777aed1d418e8 +: URI:CHK:4qaxpw4bt36qmn3hy6cpkedyay:pjw5hyzdq3ebiykd2c7talnofrfh733w77tm6hafs22c7ov5e4ia:2:3:131071 +? 2/3,f39dac6cbaba535e2c207cd0cd8f154974223c848f727f98b3564cea569b41cf,8a6a6bff19a6f768864a69d6c6c97e6a25385793201eefe2505650978aef9193 +: URI:CHK:tcgo7l6rhvwm22y5bgko7c5zy4:by2qfi7uqiyeufy5d5kkcxmeqxeha7k7gtcwadtae5y6ahru3hya:2:3:131073 +? 2/3,f39dac6cbaba535e2c207cd0cd8f154974223c848f727f98b3564cea569b41cf,caccfbaa894a6cf59e3d677a146924604f8edcd07ed1d61cd1d1569ce4bf3ec5 +: URI:CHK:vnzr4qsjx6tgneo4y45vsqlgji:eyw6lmtxs73laj2ozuejm2ya6kdtrn6bgnsrg6miz34znxipsh4a:2:3:8191 +? 3/10,0c0beacef8877bbf2416eb00f2b5dc96354e26dd1df5517320459b1236860f8c,2edc986847e209b4016e141a6dc8716d3207350f416969382d431539bf292e4a +: URI:CHK:dx7tvyr2fc4u7lxjc6kehq2svq:tiy4qh2g6lqejxcaym3rr7ymkdkinn4qised6kgxloj7sptsqu4a:3:10:1024 +? 3/10,0c0beacef8877bbf2416eb00f2b5dc96354e26dd1df5517320459b1236860f8c,3abc94a93a42d0eee5c8dda0315f9f1343e2ba36b552ab512c435fd4989c1ac6 +: URI:CHK:yo75evk4cte3b7rdw72zxvl5ye:ex6h7ff7nclucjtsqwgwu33qgmb67t4ezbrki4zbgurwn2ct6bbq:3:10:4096 +? 3/10,0c0beacef8877bbf2416eb00f2b5dc96354e26dd1df5517320459b1236860f8c,77f3b5f9437275607c3b5eef5f67d18d6a9835155cfee75acd9d2c043dbb8528 +: URI:CHK:tq5s76lgxpirmo57bs2vom3dfi:c4drrjewq2vj7mu3pie4tmhkbubbr7jxdy7lb6dytjvxrxvglldq:3:10:8193 +? 3/10,0c0beacef8877bbf2416eb00f2b5dc96354e26dd1df5517320459b1236860f8c,80c58fc1932767b113819b033e1874d8de05e8809534590db9d01510abc8ed6b +: URI:CHK:sojahkxprwqsymyxj4cbkvah6m:sveaqceazsm7olekkh7vk75kywi2mlqymok4qzjmrfkmhn3wygha:3:10:2048 +? 3/10,0c0beacef8877bbf2416eb00f2b5dc96354e26dd1df5517320459b1236860f8c,8454e5bb164935d6a8da749d78cf91eb0556897d44273053b67777aed1d418e8 +: URI:CHK:7mdyutwroo5ebxt5rvwlrmuoza:rlwrxlb6kupswgmil4r5arm3n24qq7l7nniiebelng5jve7pc74q:3:10:131071 +? 3/10,0c0beacef8877bbf2416eb00f2b5dc96354e26dd1df5517320459b1236860f8c,8a6a6bff19a6f768864a69d6c6c97e6a25385793201eefe2505650978aef9193 +: URI:CHK:p4do46g2k6mwkm4b5gybbf4qia:anhyucmmnsxyccwpvjc3cbncii652s7qglfxlswpp6lllya35u2q:3:10:131073 +? 3/10,0c0beacef8877bbf2416eb00f2b5dc96354e26dd1df5517320459b1236860f8c,caccfbaa894a6cf59e3d677a146924604f8edcd07ed1d61cd1d1569ce4bf3ec5 +: URI:CHK:eofaxrbscvampoxze3revacvgm:hnelkvqvffvaba7z2q2ryxevqyyu7hmitkldo6gc63oburi6it5q:3:10:8191 +? 3/10,2e61afd25d3ca76a0ee265dbc22ba5e9d844dc93ac5f2e37199a34fcbdcb5a52,2edc986847e209b4016e141a6dc8716d3207350f416969382d431539bf292e4a +: URI:CHK:waolb5bl2gcd5hbbiurwwipdlq:bs7ixrhwbivs2krsv2aicv5p2t6iiluj4angimza3zr3h2my726a:3:10:1024 +? 3/10,2e61afd25d3ca76a0ee265dbc22ba5e9d844dc93ac5f2e37199a34fcbdcb5a52,3abc94a93a42d0eee5c8dda0315f9f1343e2ba36b552ab512c435fd4989c1ac6 +: URI:CHK:wwnxsu5dp5y42dbdgbvpxcju6e:qongonvlsucqyndshzqbmipcqzje7pz7g6l7457zrdm2ldf5apha:3:10:4096 +? 3/10,2e61afd25d3ca76a0ee265dbc22ba5e9d844dc93ac5f2e37199a34fcbdcb5a52,77f3b5f9437275607c3b5eef5f67d18d6a9835155cfee75acd9d2c043dbb8528 +: URI:CHK:pxxzoxraxumzo5v26la3y2kony:fkv6rny5mllqsudxcimodqt43cmxowap6m7onsvzjocxcwkodx4q:3:10:8193 +? 3/10,2e61afd25d3ca76a0ee265dbc22ba5e9d844dc93ac5f2e37199a34fcbdcb5a52,80c58fc1932767b113819b033e1874d8de05e8809534590db9d01510abc8ed6b +: URI:CHK:4cuh76ehoxr7at5irtqazsgdvy:3oitt3kd3wklugjilii2vests73ci6el5tpnqhso23y5pjjetbta:3:10:2048 +? 3/10,2e61afd25d3ca76a0ee265dbc22ba5e9d844dc93ac5f2e37199a34fcbdcb5a52,8454e5bb164935d6a8da749d78cf91eb0556897d44273053b67777aed1d418e8 +: URI:CHK:u2ooocrxsdb3ts3wjkybbfuoeq:2q2g2g4mcqaj66ipbr5p2eotjqzpjfxhli6zrni2bje7luboxhha:3:10:131071 +? 3/10,2e61afd25d3ca76a0ee265dbc22ba5e9d844dc93ac5f2e37199a34fcbdcb5a52,8a6a6bff19a6f768864a69d6c6c97e6a25385793201eefe2505650978aef9193 +: URI:CHK:qfwww2o2kn3hz2iywqzxgqy24e:2pzpb6pdnqcgz4thlbtwx2dmvva6mvqvwc7p327c3gp2a2nkg2tq:3:10:131073 +? 3/10,2e61afd25d3ca76a0ee265dbc22ba5e9d844dc93ac5f2e37199a34fcbdcb5a52,caccfbaa894a6cf59e3d677a146924604f8edcd07ed1d61cd1d1569ce4bf3ec5 +: URI:CHK:az4h34ljherrtom5cy3bmihr4m:7apbwgroxfzsxszxe2626ih6onueaqgmcb4wcmrgzv3qadfazzla:3:10:8191 +? 3/10,43a1e8995c86ee7f559e5d187d1657296662ca180b394146a6e1dd0df9158a30,2edc986847e209b4016e141a6dc8716d3207350f416969382d431539bf292e4a +: URI:CHK:x55owxzhsezfoayaxe7jpwnove:vawdgtqpxyntgy5i2po2twgelrynkfcjgwm7publnlbdp7hpqmfa:3:10:1024 +? 3/10,43a1e8995c86ee7f559e5d187d1657296662ca180b394146a6e1dd0df9158a30,3abc94a93a42d0eee5c8dda0315f9f1343e2ba36b552ab512c435fd4989c1ac6 +: URI:CHK:fe64krzyaeff3d4teunjbetkzy:27hrywwaffqiqcgfkmzwbot3iamotr3bey2l5kaladmdmxuaz5ka:3:10:4096 +? 3/10,43a1e8995c86ee7f559e5d187d1657296662ca180b394146a6e1dd0df9158a30,77f3b5f9437275607c3b5eef5f67d18d6a9835155cfee75acd9d2c043dbb8528 +: URI:CHK:s7x2mswk33trx5lkgkvbdssihi:y5uabvo36gvtfi4rdjnaiyg3xlfh5bx5hpswivpd3zxzjupjs5qa:3:10:8193 +? 3/10,43a1e8995c86ee7f559e5d187d1657296662ca180b394146a6e1dd0df9158a30,80c58fc1932767b113819b033e1874d8de05e8809534590db9d01510abc8ed6b +: URI:CHK:urr3adnkm3lsisfshb4tgchvie:nrfj2cbr56abogv4pvmgb32il5zckaun3j2uv45aa76wtdrqwitq:3:10:2048 +? 3/10,43a1e8995c86ee7f559e5d187d1657296662ca180b394146a6e1dd0df9158a30,8454e5bb164935d6a8da749d78cf91eb0556897d44273053b67777aed1d418e8 +: URI:CHK:acepdcmzrafgvur6jmcgywuecm:p64sqhkx5272djlrvyvu44odn4s2fw4afslqcqlxu65issvtsusq:3:10:131071 +? 3/10,43a1e8995c86ee7f559e5d187d1657296662ca180b394146a6e1dd0df9158a30,8a6a6bff19a6f768864a69d6c6c97e6a25385793201eefe2505650978aef9193 +: URI:CHK:plgiizqfrqsor5y7up2js5ovg4:cpj2kpiojy6an4spsppcxrz6uchywccw2jgzmch3lo7u6cvxnlda:3:10:131073 +? 3/10,43a1e8995c86ee7f559e5d187d1657296662ca180b394146a6e1dd0df9158a30,caccfbaa894a6cf59e3d677a146924604f8edcd07ed1d61cd1d1569ce4bf3ec5 +: URI:CHK:7refz6urxegdrnczkoewjjikvq:wd2uftpusakntdgzcnvvbrnxr47hsylfxawvt73xapndf2lrevfq:3:10:8191 +? 3/10,be45cb2605bf36bebde684841a28f0fd43c69850a3dce5fedba69928ee3a8991,2edc986847e209b4016e141a6dc8716d3207350f416969382d431539bf292e4a +: URI:CHK:2kia6xptokio2pksshbgmhw4vu:trstbnyzthh3ncdciksentnxaqebnjv76lujt37td3h62yk7wd2a:3:10:1024 +? 3/10,be45cb2605bf36bebde684841a28f0fd43c69850a3dce5fedba69928ee3a8991,3abc94a93a42d0eee5c8dda0315f9f1343e2ba36b552ab512c435fd4989c1ac6 +: URI:CHK:7767v4lygmnqbrjaiczlsf7ehy:ig3ev7lcogswoyfwelcg4ap5xh7rzhjpn645pv7u6xcjxlzmbhta:3:10:4096 +? 3/10,be45cb2605bf36bebde684841a28f0fd43c69850a3dce5fedba69928ee3a8991,77f3b5f9437275607c3b5eef5f67d18d6a9835155cfee75acd9d2c043dbb8528 +: URI:CHK:66c63jd3qlgpmca5eyc7m6unqy:4lh75xz5tksdvx3xt5pmeoy3h3amaklgcsdtcgvjncx26rd6ysfq:3:10:8193 +? 3/10,be45cb2605bf36bebde684841a28f0fd43c69850a3dce5fedba69928ee3a8991,80c58fc1932767b113819b033e1874d8de05e8809534590db9d01510abc8ed6b +: URI:CHK:usekh3kmiilhnx3ciy4gknu4fe:steov6hzrp24t5bhqjb4an5buspol4phbhhjekvjmqf2j5f4pnuq:3:10:2048 +? 3/10,be45cb2605bf36bebde684841a28f0fd43c69850a3dce5fedba69928ee3a8991,8454e5bb164935d6a8da749d78cf91eb0556897d44273053b67777aed1d418e8 +: URI:CHK:2lzrturd4ni5idui57aiaminiy:qxaigpe2tyriywdgykw5i3jbfuxfoscgg7cqxkfkvl4276iys5vq:3:10:131071 +? 3/10,be45cb2605bf36bebde684841a28f0fd43c69850a3dce5fedba69928ee3a8991,8a6a6bff19a6f768864a69d6c6c97e6a25385793201eefe2505650978aef9193 +: URI:CHK:5kt7pcf7lhddi4t6rgvg4wimye:zhvvii4ym22svvm6dqcbjubtqtxiuicsu3m73stkbgi6aerv4vbq:3:10:131073 +? 3/10,be45cb2605bf36bebde684841a28f0fd43c69850a3dce5fedba69928ee3a8991,caccfbaa894a6cf59e3d677a146924604f8edcd07ed1d61cd1d1569ce4bf3ec5 +: URI:CHK:v3ggpeotvq7yasm7xv7njll4ua:suq7wstq553hzexpkrt6p2catu5vxffq2vv2bzgp4wxqxsrd3mha:3:10:8191 +? 3/10,cd82b3962c8796095a288e29ed94f98017cc681d17ea9ea8861c4034b13a3b6f,2edc986847e209b4016e141a6dc8716d3207350f416969382d431539bf292e4a +: URI:CHK:4yxb4zy46aj7jynfxmxsuzoyvi:2r7rzhsw7klo3nsm7nrgv7ixfsusap6iwdyzx422vnkvuhb3k3tq:3:10:1024 +? 3/10,cd82b3962c8796095a288e29ed94f98017cc681d17ea9ea8861c4034b13a3b6f,3abc94a93a42d0eee5c8dda0315f9f1343e2ba36b552ab512c435fd4989c1ac6 +: URI:CHK:s2i62qbkrqsipvtk5inri5gm2q:5fi5h47qtagp7ds3qzhp7exdu4ftldaddmefhthca7v5iqvsvn5q:3:10:4096 +? 3/10,cd82b3962c8796095a288e29ed94f98017cc681d17ea9ea8861c4034b13a3b6f,77f3b5f9437275607c3b5eef5f67d18d6a9835155cfee75acd9d2c043dbb8528 +: URI:CHK:du5bm55q3wescfhtizl67t4nd4:egmwrekadv6uc53zjgyq6da5flzd4ml6mgjhnuihn7nthmlkzhvq:3:10:8193 +? 3/10,cd82b3962c8796095a288e29ed94f98017cc681d17ea9ea8861c4034b13a3b6f,80c58fc1932767b113819b033e1874d8de05e8809534590db9d01510abc8ed6b +: URI:CHK:6w4d7jjpxdmwedvowsjcze6uzm:iywt6ln36v3bdfmzcxofh44tp24osotxypwes6teitlzio3m72ba:3:10:2048 +? 3/10,cd82b3962c8796095a288e29ed94f98017cc681d17ea9ea8861c4034b13a3b6f,8454e5bb164935d6a8da749d78cf91eb0556897d44273053b67777aed1d418e8 +: URI:CHK:j444wto2zemwoa3fv6ybqvciky:ap7zqb4zuffilxberhbb4jp4dh7ymugkrgnrf5k6uhwsxcxdppcq:3:10:131071 +? 3/10,cd82b3962c8796095a288e29ed94f98017cc681d17ea9ea8861c4034b13a3b6f,8a6a6bff19a6f768864a69d6c6c97e6a25385793201eefe2505650978aef9193 +: URI:CHK:o7nfpsjh4ubatgoezv2ovy5roq:qntmp2flklkymn54bjburj6amrr465uzqrtlneibkir2feh34f6a:3:10:131073 +? 3/10,cd82b3962c8796095a288e29ed94f98017cc681d17ea9ea8861c4034b13a3b6f,caccfbaa894a6cf59e3d677a146924604f8edcd07ed1d61cd1d1569ce4bf3ec5 +: URI:CHK:3jx4hsxurmutlzshxircbzg4wy:wtmbkrrvf3z4bg3jt4lpml5bc246tvhguwsrrzltcqcgd7um7dlq:3:10:8191 +? 3/10,f39dac6cbaba535e2c207cd0cd8f154974223c848f727f98b3564cea569b41cf,2edc986847e209b4016e141a6dc8716d3207350f416969382d431539bf292e4a +: URI:CHK:7lsdyuu2ka2enhnyug4ytklhzi:pk2h7sbj3sjrauwskmvtbacyp3i57awiofg4uxxsbvfgxfibtoha:3:10:1024 +? 3/10,f39dac6cbaba535e2c207cd0cd8f154974223c848f727f98b3564cea569b41cf,3abc94a93a42d0eee5c8dda0315f9f1343e2ba36b552ab512c435fd4989c1ac6 +: URI:CHK:sg35empm5gtsica4j4tsdhjihy:sqehx4q76sx4npafchg2u6lntgwm3jzqv3in2rbg7q6atr3lxdua:3:10:4096 +? 3/10,f39dac6cbaba535e2c207cd0cd8f154974223c848f727f98b3564cea569b41cf,77f3b5f9437275607c3b5eef5f67d18d6a9835155cfee75acd9d2c043dbb8528 +: URI:CHK:et5wbi3r6aufs33azuqko672k4:ahkwexqrtrkyyy4c3ixsmsz6aqhtyctynz2m6qgsfx6nk7jzn2oq:3:10:8193 +? 3/10,f39dac6cbaba535e2c207cd0cd8f154974223c848f727f98b3564cea569b41cf,80c58fc1932767b113819b033e1874d8de05e8809534590db9d01510abc8ed6b +: URI:CHK:s77i5y6ckuqgb36sihcsnutavu:idmwswj5p7z5w5elulxu5t6ulb7zt7ovaktrcypbq7k24ef5ekva:3:10:2048 +? 3/10,f39dac6cbaba535e2c207cd0cd8f154974223c848f727f98b3564cea569b41cf,8454e5bb164935d6a8da749d78cf91eb0556897d44273053b67777aed1d418e8 +: URI:CHK:3eirnemfrtnvkczhthm3ngxg2i:3uq3x2uaskwsykos2h7of6iwtyfp6nabgibk3ockphhdhrepcxia:3:10:131071 +? 3/10,f39dac6cbaba535e2c207cd0cd8f154974223c848f727f98b3564cea569b41cf,8a6a6bff19a6f768864a69d6c6c97e6a25385793201eefe2505650978aef9193 +: URI:CHK:uuwk7eo4w3nfxz4hunif5vcc5e:tneaxz6kr6cardyconuslivifzdb67re4stmtfvrd6dp7bbk4iea:3:10:131073 +? 3/10,f39dac6cbaba535e2c207cd0cd8f154974223c848f727f98b3564cea569b41cf,caccfbaa894a6cf59e3d677a146924604f8edcd07ed1d61cd1d1569ce4bf3ec5 +: URI:CHK:4selpjhb6m2jqpixg3fzfti2yi:mvkil3khmwm5om7ucneo675chan4vx7cja27m55qj4pktvbv4snq:3:10:8191 +? 71/255,0c0beacef8877bbf2416eb00f2b5dc96354e26dd1df5517320459b1236860f8c,2edc986847e209b4016e141a6dc8716d3207350f416969382d431539bf292e4a +: URI:CHK:2wvrzaobtfkoiqeqwryjzvxatq:7zzaycbdhepzaqtdrku2sbb572h64ggwstu47osa2gn4ol3eglmq:71:255:1024 +? 71/255,0c0beacef8877bbf2416eb00f2b5dc96354e26dd1df5517320459b1236860f8c,3abc94a93a42d0eee5c8dda0315f9f1343e2ba36b552ab512c435fd4989c1ac6 +: URI:CHK:7wi5xn2uwerkjzqs5ndn4eimti:llrws4uig2fpi3rtla6lk7m4t6zudjhszdfjgqtihghfrvs2robq:71:255:4096 +? 71/255,0c0beacef8877bbf2416eb00f2b5dc96354e26dd1df5517320459b1236860f8c,77f3b5f9437275607c3b5eef5f67d18d6a9835155cfee75acd9d2c043dbb8528 +: URI:CHK:ybc7ijc45jwunkecolewxrmyyu:d4m3oh7vkyavap7rditzuarwjmmkot7iz4rhqpsks37sxq5gmcpq:71:255:8193 +? 71/255,0c0beacef8877bbf2416eb00f2b5dc96354e26dd1df5517320459b1236860f8c,80c58fc1932767b113819b033e1874d8de05e8809534590db9d01510abc8ed6b +: URI:CHK:ts5yfcjoq3jwahsp4yuqahqhq4:bbeyk64tjbj732zga3gsz46n24lgxte6ydyni3tphoircaifiiya:71:255:2048 +? 71/255,0c0beacef8877bbf2416eb00f2b5dc96354e26dd1df5517320459b1236860f8c,8454e5bb164935d6a8da749d78cf91eb0556897d44273053b67777aed1d418e8 +: URI:CHK:hzhnspwdfp55wt5gw6fiixujlq:eok5auvypt77mv2ic2225shisclyuhhmtyssh5mwpyrdtgfndyua:71:255:131071 +? 71/255,0c0beacef8877bbf2416eb00f2b5dc96354e26dd1df5517320459b1236860f8c,8a6a6bff19a6f768864a69d6c6c97e6a25385793201eefe2505650978aef9193 +: URI:CHK:mxpwc2imfzik6lxb6vyg6h7kv4:2ai2kkkwscucexcplsp52au62xdqclngfixippolxul2i4u3tgpa:71:255:131073 +? 71/255,0c0beacef8877bbf2416eb00f2b5dc96354e26dd1df5517320459b1236860f8c,caccfbaa894a6cf59e3d677a146924604f8edcd07ed1d61cd1d1569ce4bf3ec5 +: URI:CHK:rc7jm3hj4qwt73wk7jmpabofdy:ohfz4s5ezjnan33wef3zav4rkqnb67ikbzyz63ctlofnkmwfujna:71:255:8191 +? 71/255,2e61afd25d3ca76a0ee265dbc22ba5e9d844dc93ac5f2e37199a34fcbdcb5a52,2edc986847e209b4016e141a6dc8716d3207350f416969382d431539bf292e4a +: URI:CHK:e25n6qpypv3n6l3pwnkydfceay:rbridd4pcj47dhnal3jg36zrhjnus4egzn2lb6myfpegj6rqii7a:71:255:1024 +? 71/255,2e61afd25d3ca76a0ee265dbc22ba5e9d844dc93ac5f2e37199a34fcbdcb5a52,3abc94a93a42d0eee5c8dda0315f9f1343e2ba36b552ab512c435fd4989c1ac6 +: URI:CHK:dyhyxipdc5p7s7x2ntmxdjsrb4:bk4c6hjk2qvvcxsen76dqkvn2ckvehv4z5ogzlv2csyjubbwxe2q:71:255:4096 +? 71/255,2e61afd25d3ca76a0ee265dbc22ba5e9d844dc93ac5f2e37199a34fcbdcb5a52,77f3b5f9437275607c3b5eef5f67d18d6a9835155cfee75acd9d2c043dbb8528 +: URI:CHK:ggpxln6r5e6jmcnfr723awlah4:e6ypwv2uo37tcjn23mp3rerjrula4sob2wxylhz4ttyf45kfoa6a:71:255:8193 +? 71/255,2e61afd25d3ca76a0ee265dbc22ba5e9d844dc93ac5f2e37199a34fcbdcb5a52,80c58fc1932767b113819b033e1874d8de05e8809534590db9d01510abc8ed6b +: URI:CHK:dcde7b4d6mxxf5kvece5w6nxry:rxteuxaxpt4kyd6ym4grkkfiwgd2c6soet3cx46arvubbtmmjkbq:71:255:2048 +? 71/255,2e61afd25d3ca76a0ee265dbc22ba5e9d844dc93ac5f2e37199a34fcbdcb5a52,8454e5bb164935d6a8da749d78cf91eb0556897d44273053b67777aed1d418e8 +: URI:CHK:vulnb4ns3vovkvlu3536vxsvkm:6vww2a2eypjb4ctdd2uh4zylnxb575edg3gsiobgirgenhw6f3ja:71:255:131071 +? 71/255,2e61afd25d3ca76a0ee265dbc22ba5e9d844dc93ac5f2e37199a34fcbdcb5a52,8a6a6bff19a6f768864a69d6c6c97e6a25385793201eefe2505650978aef9193 +: URI:CHK:6hk7efj3jyb7g6ewpb6alua6ve:i6oq2hul7a2yoinddn2nfwfarh6ytzu6ernedkevp5vgtyms7dja:71:255:131073 +? 71/255,2e61afd25d3ca76a0ee265dbc22ba5e9d844dc93ac5f2e37199a34fcbdcb5a52,caccfbaa894a6cf59e3d677a146924604f8edcd07ed1d61cd1d1569ce4bf3ec5 +: URI:CHK:ehixpgvrjy6yr2wqlw6n3cqcui:rrrejfiwxefb2u6uicmxx3qr7j5z3ox5n7boklrly25op5iwflta:71:255:8191 +? 71/255,43a1e8995c86ee7f559e5d187d1657296662ca180b394146a6e1dd0df9158a30,2edc986847e209b4016e141a6dc8716d3207350f416969382d431539bf292e4a +: URI:CHK:wxmzfzvgaqm3xfzgaqrlndgolm:6t3tqcphsutigxnnzyp2xx3afklcpeytnh543mpqptquvqqahwea:71:255:1024 +? 71/255,43a1e8995c86ee7f559e5d187d1657296662ca180b394146a6e1dd0df9158a30,3abc94a93a42d0eee5c8dda0315f9f1343e2ba36b552ab512c435fd4989c1ac6 +: URI:CHK:r3q3q4f4zbsipxauq6knb5dfoq:lkae5mflammzxkoc7gllt6qqyg5gqdxeejmxtgcscblqh5tv5obq:71:255:4096 +? 71/255,43a1e8995c86ee7f559e5d187d1657296662ca180b394146a6e1dd0df9158a30,77f3b5f9437275607c3b5eef5f67d18d6a9835155cfee75acd9d2c043dbb8528 +: URI:CHK:l63y2aolbcvdcm5s4i2xa6mlbq:pjlht25xggvtstoynmi5tq3x3wvgtzmjym3lcpulh7keg6mc366q:71:255:8193 +? 71/255,43a1e8995c86ee7f559e5d187d1657296662ca180b394146a6e1dd0df9158a30,80c58fc1932767b113819b033e1874d8de05e8809534590db9d01510abc8ed6b +: URI:CHK:rhsd2socwnvt6nrnhtwpic23am:x52gdwnvpuh63oo7l2tclnclt64k7u4upyadaaifpvlnirnqy7lq:71:255:2048 +? 71/255,43a1e8995c86ee7f559e5d187d1657296662ca180b394146a6e1dd0df9158a30,8454e5bb164935d6a8da749d78cf91eb0556897d44273053b67777aed1d418e8 +: URI:CHK:zwes34ucxchxajmf4fjaw5lzeu:eh3hvvilxud5za7j4x6u7wnvarmtsuj2f2g5mdqibrys2yd7qb6a:71:255:131071 +? 71/255,43a1e8995c86ee7f559e5d187d1657296662ca180b394146a6e1dd0df9158a30,8a6a6bff19a6f768864a69d6c6c97e6a25385793201eefe2505650978aef9193 +: URI:CHK:eootdbcym74eidk3a2jrzag42i:cityydyejtqqjqdfnb4o3f64f46udr6hxzzm3rdyabh3o3buvcha:71:255:131073 +? 71/255,43a1e8995c86ee7f559e5d187d1657296662ca180b394146a6e1dd0df9158a30,caccfbaa894a6cf59e3d677a146924604f8edcd07ed1d61cd1d1569ce4bf3ec5 +: URI:CHK:inm7itgqlu5djoeqeud55rkbka:jxacqjjtdxeevrntievnwa2kq452vsm3ur3sb3z32tr7uuw4fpiq:71:255:8191 +? 71/255,be45cb2605bf36bebde684841a28f0fd43c69850a3dce5fedba69928ee3a8991,2edc986847e209b4016e141a6dc8716d3207350f416969382d431539bf292e4a +: URI:CHK:khbldourqhs52sky6pnolmaiyy:qq4ixgnxeauqmiaftuvb6spviyn4xubdoycetc62ckjq26ycx22q:71:255:1024 +? 71/255,be45cb2605bf36bebde684841a28f0fd43c69850a3dce5fedba69928ee3a8991,3abc94a93a42d0eee5c8dda0315f9f1343e2ba36b552ab512c435fd4989c1ac6 +: URI:CHK:e4etimwsvsbgwlamrcmpa3bxwq:pdjv5a3wrc4licma2tlmfpcp6maawimzqlkawva2plnmwo5gwlna:71:255:4096 +? 71/255,be45cb2605bf36bebde684841a28f0fd43c69850a3dce5fedba69928ee3a8991,77f3b5f9437275607c3b5eef5f67d18d6a9835155cfee75acd9d2c043dbb8528 +: URI:CHK:zpoi7kp6bqhbuthfhnzr7lrnza:xlumj77xnyz2ddbfviww2fjyca55sx5oj2hbqx6yiwq3eqorkytq:71:255:8193 +? 71/255,be45cb2605bf36bebde684841a28f0fd43c69850a3dce5fedba69928ee3a8991,80c58fc1932767b113819b033e1874d8de05e8809534590db9d01510abc8ed6b +: URI:CHK:rtdvpxir3vbvq44mhoepfz7bvi:prpry2f452mrtkarcf7n6vpjw64hhyeshe5srrwzk42q5dd3iczq:71:255:2048 +? 71/255,be45cb2605bf36bebde684841a28f0fd43c69850a3dce5fedba69928ee3a8991,8454e5bb164935d6a8da749d78cf91eb0556897d44273053b67777aed1d418e8 +: URI:CHK:qnm225ycss3xthcme232ivbxd4:nxhjpm6rfztd2v5ucg476zphaps3xcfzfmaxsxlr25mvz2funi4a:71:255:131071 +? 71/255,be45cb2605bf36bebde684841a28f0fd43c69850a3dce5fedba69928ee3a8991,8a6a6bff19a6f768864a69d6c6c97e6a25385793201eefe2505650978aef9193 +: URI:CHK:yrsvl5gbjqeuiaqr76lnzdnahq:sxheuay55a245bcy4b5gmyhteakwkbrixmae6gnilhekvzmzwxda:71:255:131073 +? 71/255,be45cb2605bf36bebde684841a28f0fd43c69850a3dce5fedba69928ee3a8991,caccfbaa894a6cf59e3d677a146924604f8edcd07ed1d61cd1d1569ce4bf3ec5 +: URI:CHK:gzan4jdk5xfrp2q6chohk3wiv4:a4tcumb7vz7wms3a6ibaxm4u7jorks5thnhnfs5ajdpy4fwdgyqq:71:255:8191 +? 71/255,cd82b3962c8796095a288e29ed94f98017cc681d17ea9ea8861c4034b13a3b6f,2edc986847e209b4016e141a6dc8716d3207350f416969382d431539bf292e4a +: URI:CHK:q53ik6mdwijpptijuzgt6beuh4:rcfwmqzfuw5xs3b3a5mc6blzdln6vw3ojz6obz24jxb4rkukroqa:71:255:1024 +? 71/255,cd82b3962c8796095a288e29ed94f98017cc681d17ea9ea8861c4034b13a3b6f,3abc94a93a42d0eee5c8dda0315f9f1343e2ba36b552ab512c435fd4989c1ac6 +: URI:CHK:turjtbsb74zqbilacgdarkwxka:kusxr4ooeyyg5nofyoccufnjgxi3em2jm43ox3kur3m3ctv43kkq:71:255:4096 +? 71/255,cd82b3962c8796095a288e29ed94f98017cc681d17ea9ea8861c4034b13a3b6f,77f3b5f9437275607c3b5eef5f67d18d6a9835155cfee75acd9d2c043dbb8528 +: URI:CHK:utveoddo7fuhcpspizfdcynnn4:f5wiofdi4ehjwsdutwcnmhhdnohnrbhbbruxmxavplt2doxaks4a:71:255:8193 +? 71/255,cd82b3962c8796095a288e29ed94f98017cc681d17ea9ea8861c4034b13a3b6f,80c58fc1932767b113819b033e1874d8de05e8809534590db9d01510abc8ed6b +: URI:CHK:2gad4sxrilmhighbreeoy4vugu:qyoa2bnpqd6xludkft5jhsbzaoq3mowx72tp7k5yz3j3ydc7depa:71:255:2048 +? 71/255,cd82b3962c8796095a288e29ed94f98017cc681d17ea9ea8861c4034b13a3b6f,8454e5bb164935d6a8da749d78cf91eb0556897d44273053b67777aed1d418e8 +: URI:CHK:f2kefs2nfjjvcgsomrripeebli:3zvxpr2kcqi6npf3zu4lzi3ywzypebg5tmvzf5j3ifrhjtyjbioq:71:255:131071 +? 71/255,cd82b3962c8796095a288e29ed94f98017cc681d17ea9ea8861c4034b13a3b6f,8a6a6bff19a6f768864a69d6c6c97e6a25385793201eefe2505650978aef9193 +: URI:CHK:twi4p7gxwecvr5ypoa3i2mtrru:kcurtyehuorsxfqruoddhwenjufazrnjn7daxa2rexc72i7hr3jq:71:255:131073 +? 71/255,cd82b3962c8796095a288e29ed94f98017cc681d17ea9ea8861c4034b13a3b6f,caccfbaa894a6cf59e3d677a146924604f8edcd07ed1d61cd1d1569ce4bf3ec5 +: URI:CHK:bzenvzdzxlbysdvxy2fdcwzkp4:yagzttmvppme5rm5xvkebk7zerz5evkcrxoafjotozeuou5l6oza:71:255:8191 +? 71/255,f39dac6cbaba535e2c207cd0cd8f154974223c848f727f98b3564cea569b41cf,2edc986847e209b4016e141a6dc8716d3207350f416969382d431539bf292e4a +: URI:CHK:zwtwx7r77yseqnx6gjqnzbuu4i:ie237xmuehuz6iarwfx7ih44d6zipwqwtecck3ybmdwylqczasua:71:255:1024 +? 71/255,f39dac6cbaba535e2c207cd0cd8f154974223c848f727f98b3564cea569b41cf,3abc94a93a42d0eee5c8dda0315f9f1343e2ba36b552ab512c435fd4989c1ac6 +: URI:CHK:22lyxndvtgmgjofwsgr2r4xdka:tyw2xibpmksbyeazzqrpd4jkvoycxj45mbe22mmi4wgzrt4dvlfq:71:255:4096 +? 71/255,f39dac6cbaba535e2c207cd0cd8f154974223c848f727f98b3564cea569b41cf,77f3b5f9437275607c3b5eef5f67d18d6a9835155cfee75acd9d2c043dbb8528 +: URI:CHK:mihdgaisquzpfy26sfkzi4iuwa:5lrp47rkwoxc3pwbi7irpruzrec6672hrmva47owcygce3exycoa:71:255:8193 +? 71/255,f39dac6cbaba535e2c207cd0cd8f154974223c848f727f98b3564cea569b41cf,80c58fc1932767b113819b033e1874d8de05e8809534590db9d01510abc8ed6b +: URI:CHK:wu6hee63ij6hickphc5pus4jry:rei6yq4qqr6wa7bxt2udxcpqbzdn7zhwupor66iyj4sjphw4dqwa:71:255:2048 +? 71/255,f39dac6cbaba535e2c207cd0cd8f154974223c848f727f98b3564cea569b41cf,8454e5bb164935d6a8da749d78cf91eb0556897d44273053b67777aed1d418e8 +: URI:CHK:l3yx4gafcpaoctzfhiykbqwg7e:o6pgeihvxatl5z4sae6xyr3vjlqwinjvm4orj3wklzo7atoixciq:71:255:131071 +? 71/255,f39dac6cbaba535e2c207cd0cd8f154974223c848f727f98b3564cea569b41cf,8a6a6bff19a6f768864a69d6c6c97e6a25385793201eefe2505650978aef9193 +: URI:CHK:rxarznyu7f7n2bfdk73q7ko77y:irlqlbsduaaotjw4hn4yg5n4dve36rpindy5zgtc7zf4yl2giy2a:71:255:131073 +? 71/255,f39dac6cbaba535e2c207cd0cd8f154974223c848f727f98b3564cea569b41cf,caccfbaa894a6cf59e3d677a146924604f8edcd07ed1d61cd1d1569ce4bf3ec5 +: URI:CHK:aikq3v6hwp5bad5mv6myoc76ze:ufih4yqr7jtjptsrfvdhcf5pq3x7drtxzpddzyatvex2sreed2dq:71:255:8191 diff --git a/integration/test_vectors.py b/integration/test_vectors.py index d9999f1d6..750c082af 100644 --- a/integration/test_vectors.py +++ b/integration/test_vectors.py @@ -19,11 +19,11 @@ from allmydata.util import base32 CONVERGENCE_SECRETS = [ b"aaaaaaaaaaaaaaaa", - # b"bbbbbbbbbbbbbbbb", - # b"abcdefghijklmnop", - # b"hello world stuf", - # b"\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", - # sha256(b"Hello world").digest()[:16], + b"bbbbbbbbbbbbbbbb", + b"abcdefghijklmnop", + b"hello world stuf", + b"\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + sha256(b"Hello world").digest()[:16], ] ONE_KB = sha256(b"Hello world").digest() * 32 @@ -31,21 +31,21 @@ assert len(ONE_KB) == 1024 OBJECT_DATA = [ b"a" * 1024, - # b"b" * 2048, - # b"c" * 4096, - # (ONE_KB * 8)[:-1], - # (ONE_KB * 8) + b"z", - # (ONE_KB * 128)[:-1], - # (ONE_KB * 128) + b"z", + b"b" * 2048, + b"c" * 4096, + (ONE_KB * 8)[:-1], + (ONE_KB * 8) + b"z", + (ONE_KB * 128)[:-1], + (ONE_KB * 128) + b"z", ] ZFEC_PARAMS = [ (1, 1), (1, 3), - # (2, 3), - # (3, 10), - # (71, 255), - # (101, 256), + (2, 3), + (3, 10), + (71, 255), + (101, 256), ] @mark.parametrize('convergence', CONVERGENCE_SECRETS) From daad22d1b1bdb4209e9a2f4cdeff1bd27221586e Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 22 Dec 2022 12:13:12 -0500 Subject: [PATCH 1305/2309] comments --- integration/vectors.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/integration/vectors.py b/integration/vectors.py index 53e581a1e..b83aaad3a 100644 --- a/integration/vectors.py +++ b/integration/vectors.py @@ -1,7 +1,15 @@ +""" +A module that loads pre-generated test vectors. + +:ivar CHK_PATH: The path of the file containing CHK test vectors. + +:ivar chk: The CHK test vectors. +""" + from yaml import safe_load from pathlib import Path -CHK_PATH = Path(__file__).parent / "_vectors_chk.yaml" +CHK_PATH: Path = Path(__file__).parent / "_vectors_chk.yaml" with CHK_PATH.open() as f: - chk = safe_load(f) + chk: dict[str, str] = safe_load(f) From 43388ee7116cf082070df95dccae099735b30af2 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 22 Dec 2022 16:52:00 -0500 Subject: [PATCH 1306/2309] Comments and minor factoring improvements and such --- integration/test_vectors.py | 141 ++++++++++++++++++++++++++++++------ 1 file changed, 119 insertions(+), 22 deletions(-) diff --git a/integration/test_vectors.py b/integration/test_vectors.py index 750c082af..7b939f5a7 100644 --- a/integration/test_vectors.py +++ b/integration/test_vectors.py @@ -2,6 +2,8 @@ Verify certain results against test vectors with well-known results. """ +from __future__ import annotations + from typing import TypeVar, Iterator, Awaitable, Callable from tempfile import NamedTemporaryFile @@ -17,16 +19,19 @@ from .util import cli, await_client_ready from allmydata.client import read_config from allmydata.util import base32 +def digest(bs: bytes) -> str: + return sha256(bs).hexdigest() + CONVERGENCE_SECRETS = [ b"aaaaaaaaaaaaaaaa", b"bbbbbbbbbbbbbbbb", b"abcdefghijklmnop", b"hello world stuf", b"\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", - sha256(b"Hello world").digest()[:16], + digest(b"Hello world")[:16], ] -ONE_KB = sha256(b"Hello world").digest() * 32 +ONE_KB = digest(b"Hello world") * 32 assert len(ONE_KB) == 1024 OBJECT_DATA = [ @@ -48,26 +53,43 @@ ZFEC_PARAMS = [ (101, 256), ] -@mark.parametrize('convergence', CONVERGENCE_SECRETS) -def test_convergence(convergence): +@mark.parametrize('convergence_idx', range(len(CONVERGENCE_SECRETS))) +def test_convergence(convergence_idx): + """ + Convergence secrets are 16 bytes. + """ + convergence = CONVERGENCE_SECRETS[convergence_idx] assert isinstance(convergence, bytes), "Convergence secret must be bytes" assert len(convergence) == 16, "Convergence secret must by 16 bytes" -@mark.parametrize('data', OBJECT_DATA) -def test_data(data): +@mark.parametrize('data_idx', range(len(OBJECT_DATA))) +def test_data(data_idx): + """ + Plaintext data is bytes. + """ + data = OBJECT_DATA[data_idx] assert isinstance(data, bytes), "Object data must be bytes." -@mark.parametrize('params', ZFEC_PARAMS) -@mark.parametrize('convergence', CONVERGENCE_SECRETS) -@mark.parametrize('data', OBJECT_DATA) +@mark.parametrize('params_idx', range(len(ZFEC_PARAMS))) +@mark.parametrize('convergence_idx', range(len(CONVERGENCE_SECRETS))) +@mark.parametrize('data_idx', range(len(OBJECT_DATA))) @ensureDeferred -async def test_chk_capability(reactor, request, alice, params, convergence, data): +async def test_chk_capability(reactor, request, alice, params_idx, convergence_idx, data_idx): + """ + The CHK capability that results from uploading certain well-known data + with certain well-known parameters results in exactly the previously + computed value. + """ + params = ZFEC_PARAMS[params_idx] + convergence = CONVERGENCE_SECRETS[convergence_idx] + data = OBJECT_DATA[data_idx] + # rewrite alice's config to match params and convergence await reconfigure(reactor, request, alice, params, convergence) # upload data as a CHK - actual = upload_immutable(alice, data) + actual = upload(alice, "chk", data) # compare the resulting cap to the expected result expected = vectors.chk[key(params, convergence, data)] @@ -82,18 +104,47 @@ async def asyncfoldr( f: Callable[[α, β], β], initial: β, ) -> β: + """ + Right fold over an async iterator. + + :param i: The async iterator. + :param f: The function to fold. + :param initial: The starting value. + + :return: The result of the fold. + """ result = initial async for a in i: result = f(a, result) return result + def insert(item: tuple[α, β], d: dict[α, β]) -> dict[α, β]: + """ + In-place add an item to a dictionary. + + If the key is already present, replace the value. + + :param item: A tuple of the key and value. + :param d: The dictionary to modify. + + :return: The dictionary. + """ d[item[0]] = item[1] return d @ensureDeferred -async def test_generate(reactor, request, alice): +async def skiptest_generate(reactor, request, alice): + """ + This is a helper for generating the test vectors. + + You can re-generate the test vectors by fixing the name of the test and + running it. Normally this test doesn't run because it ran once and we + captured its output. Other tests run against that output and we want them + to run against the results produced originally, not a possibly + ever-changing set of outputs. + """ results = await asyncfoldr( generate(reactor, request, alice), insert, @@ -103,7 +154,21 @@ async def test_generate(reactor, request, alice): f.write(safe_dump(results)) -async def reconfigure(reactor, request, alice, params, convergence): +async def reconfigure(reactor, request, alice: TahoeProcess, params: tuple[int, int], convergence: bytes) -> None: + """ + Reconfigure a Tahoe-LAFS node with different ZFEC parameters and + convergence secret. + + :param reactor: A reactor to use to restart the process. + :param request: The pytest request object to use to arrange process + cleanup. + :param alice: The Tahoe-LAFS node to reconfigure. + :param params: The ``needed`` and ``total`` ZFEC encoding parameters. + :param convergence: The convergence secret. + + :return: ``None`` after the node configuration has been rewritten, the + node has been restarted, and the node is ready to provide service. + """ needed, total = params config = read_config(alice.node_dir, "tub.port") config.set_config("client", "shares.happy", str(1)) @@ -119,26 +184,58 @@ async def reconfigure(reactor, request, alice, params, convergence): print("Ready.") -async def generate(reactor, request, alice): +async def generate(reactor, request, alice: TahoeProcess) -> AsyncGenerator[tuple[str, str], None]: + """ + Generate all of the test vectors using the given node. + + :param reactor: The reactor to use to restart the Tahoe-LAFS node when it + needs to be reconfigured. + + :param request: The pytest request object to use to arrange process + cleanup. + + :param alice: The Tahoe-LAFS node to use to generate the test vectors. + + :return: The yield values are two-tuples describing a test vector. The + first element is a string describing a case and the second element is + the CHK capability for that case. + """ node_key = (None, None) for params, secret, data in product(ZFEC_PARAMS, CONVERGENCE_SECRETS, OBJECT_DATA): if node_key != (params, secret): await reconfigure(reactor, request, alice, params, secret) node_key = (params, secret) - yield key(params, secret, data), upload_immutable(alice, data) + yield key(params, secret, data), upload(alice, "chk", data) -def key(params, secret, data): +def key(params: tuple[int, int], secret: bytes, data: bytes) -> str: + """ + Construct the key describing the case defined by the given parameters. + + :param params: The ``needed`` and ``total`` ZFEC encoding parameters. + :param secret: The convergence secret. + :param data: The plaintext data. + + :return: A distinct string for the given inputs, but shorter. This is + suitable for use as, eg, a key in a dictionary. + """ return f"{params[0]}/{params[1]},{digest(secret)},{digest(data)}" -def upload_immutable(alice, data): +def upload(alice: TahoeProcess, fmt: str, data: bytes) -> str: + """ + Upload the given data to the given node. + + :param alice: The node to upload to. + + :param fmt: The name of the format for the upload. CHK, SDMF, or MDMF. + + :param data: The data to upload. + + :return: The capability for the uploaded data. + """ with NamedTemporaryFile() as f: f.write(data) f.flush() - return cli(alice, "put", "--format=chk", f.name).decode("utf-8").strip() - - -def digest(bs): - return sha256(bs).hexdigest() + return cli(alice, "put", f"--format={fmt}", f.name).decode("utf-8").strip() From 8a427203019dc1ba2da1f0d3c1d0f589e90612a1 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 22 Dec 2022 17:02:42 -0500 Subject: [PATCH 1307/2309] Move some general utility functions into the util module --- integration/test_vectors.py | 103 ++++-------------------------------- integration/util.py | 99 ++++++++++++++++++++++++++++++---- 2 files changed, 99 insertions(+), 103 deletions(-) diff --git a/integration/test_vectors.py b/integration/test_vectors.py index 7b939f5a7..042421e6f 100644 --- a/integration/test_vectors.py +++ b/integration/test_vectors.py @@ -4,9 +4,7 @@ Verify certain results against test vectors with well-known results. from __future__ import annotations -from typing import TypeVar, Iterator, Awaitable, Callable - -from tempfile import NamedTemporaryFile +from typing import AsyncGenerator from hashlib import sha256 from itertools import product from yaml import safe_dump @@ -15,13 +13,16 @@ from pytest import mark from pytest_twisted import ensureDeferred from . import vectors -from .util import cli, await_client_ready -from allmydata.client import read_config -from allmydata.util import base32 +from .util import reconfigure, upload, asyncfoldr, insert, TahoeProcess -def digest(bs: bytes) -> str: +def digest(bs: bytes) -> bytes: + return sha256(bs).digest() + + +def hexdigest(bs: bytes) -> str: return sha256(bs).hexdigest() + CONVERGENCE_SECRETS = [ b"aaaaaaaaaaaaaaaa", b"bbbbbbbbbbbbbbbb", @@ -96,44 +97,6 @@ async def test_chk_capability(reactor, request, alice, params_idx, convergence_i assert actual == expected -α = TypeVar("α") -β = TypeVar("β") - -async def asyncfoldr( - i: Iterator[Awaitable[α]], - f: Callable[[α, β], β], - initial: β, -) -> β: - """ - Right fold over an async iterator. - - :param i: The async iterator. - :param f: The function to fold. - :param initial: The starting value. - - :return: The result of the fold. - """ - result = initial - async for a in i: - result = f(a, result) - return result - - -def insert(item: tuple[α, β], d: dict[α, β]) -> dict[α, β]: - """ - In-place add an item to a dictionary. - - If the key is already present, replace the value. - - :param item: A tuple of the key and value. - :param d: The dictionary to modify. - - :return: The dictionary. - """ - d[item[0]] = item[1] - return d - - @ensureDeferred async def skiptest_generate(reactor, request, alice): """ @@ -154,36 +117,6 @@ async def skiptest_generate(reactor, request, alice): f.write(safe_dump(results)) -async def reconfigure(reactor, request, alice: TahoeProcess, params: tuple[int, int], convergence: bytes) -> None: - """ - Reconfigure a Tahoe-LAFS node with different ZFEC parameters and - convergence secret. - - :param reactor: A reactor to use to restart the process. - :param request: The pytest request object to use to arrange process - cleanup. - :param alice: The Tahoe-LAFS node to reconfigure. - :param params: The ``needed`` and ``total`` ZFEC encoding parameters. - :param convergence: The convergence secret. - - :return: ``None`` after the node configuration has been rewritten, the - node has been restarted, and the node is ready to provide service. - """ - needed, total = params - config = read_config(alice.node_dir, "tub.port") - config.set_config("client", "shares.happy", str(1)) - config.set_config("client", "shares.needed", str(needed)) - config.set_config("client", "shares.total", str(total)) - config.write_private_config("convergence", base32.b2a(convergence)) - - # restart alice - print(f"Restarting {alice.node_dir} for ZFEC reconfiguration") - await alice.restart_async(reactor, request) - print("Restarted. Waiting for ready state.") - await_client_ready(alice) - print("Ready.") - - async def generate(reactor, request, alice: TahoeProcess) -> AsyncGenerator[tuple[str, str], None]: """ Generate all of the test vectors using the given node. @@ -220,22 +153,4 @@ def key(params: tuple[int, int], secret: bytes, data: bytes) -> str: :return: A distinct string for the given inputs, but shorter. This is suitable for use as, eg, a key in a dictionary. """ - return f"{params[0]}/{params[1]},{digest(secret)},{digest(data)}" - - -def upload(alice: TahoeProcess, fmt: str, data: bytes) -> str: - """ - Upload the given data to the given node. - - :param alice: The node to upload to. - - :param fmt: The name of the format for the upload. CHK, SDMF, or MDMF. - - :param data: The data to upload. - - :return: The capability for the uploaded data. - """ - with NamedTemporaryFile() as f: - f.write(data) - f.flush() - return cli(alice, "put", f"--format={fmt}", f.name).decode("utf-8").strip() + return f"{params[0]}/{params[1]},{hexdigest(secret)},{hexdigest(data)}" diff --git a/integration/util.py b/integration/util.py index c2394375a..0dc63b2d5 100644 --- a/integration/util.py +++ b/integration/util.py @@ -1,15 +1,9 @@ """ -Ported to Python 3. +General functionality useful for the implementation of integration tests. """ -from __future__ import unicode_literals -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 +from typing import TypeVar, Iterator, Awaitable, Callable +from tempfile import NamedTemporaryFile import sys import time import json @@ -32,6 +26,7 @@ import requests from paramiko.rsakey import RSAKey from boltons.funcutils import wraps +from allmydata.util import base32 from allmydata.util.configutil import ( get_config, set_config, @@ -598,3 +593,89 @@ def run_in_thread(f): def test(*args, **kwargs): return deferToThread(lambda: f(*args, **kwargs)) return test + + +def upload(alice: TahoeProcess, fmt: str, data: bytes) -> str: + """ + Upload the given data to the given node. + + :param alice: The node to upload to. + + :param fmt: The name of the format for the upload. CHK, SDMF, or MDMF. + + :param data: The data to upload. + + :return: The capability for the uploaded data. + """ + with NamedTemporaryFile() as f: + f.write(data) + f.flush() + return cli(alice, "put", f"--format={fmt}", f.name).decode("utf-8").strip() + + +α = TypeVar("α") +β = TypeVar("β") + +async def asyncfoldr( + i: Iterator[Awaitable[α]], + f: Callable[[α, β], β], + initial: β, +) -> β: + """ + Right fold over an async iterator. + + :param i: The async iterator. + :param f: The function to fold. + :param initial: The starting value. + + :return: The result of the fold. + """ + result = initial + async for a in i: + result = f(a, result) + return result + + +def insert(item: tuple[α, β], d: dict[α, β]) -> dict[α, β]: + """ + In-place add an item to a dictionary. + + If the key is already present, replace the value. + + :param item: A tuple of the key and value. + :param d: The dictionary to modify. + + :return: The dictionary. + """ + d[item[0]] = item[1] + return d + + +async def reconfigure(reactor, request, node: TahoeProcess, params: tuple[int, int], convergence: bytes) -> None: + """ + Reconfigure a Tahoe-LAFS node with different ZFEC parameters and + convergence secret. + + :param reactor: A reactor to use to restart the process. + :param request: The pytest request object to use to arrange process + cleanup. + :param node: The Tahoe-LAFS node to reconfigure. + :param params: The ``needed`` and ``total`` ZFEC encoding parameters. + :param convergence: The convergence secret. + + :return: ``None`` after the node configuration has been rewritten, the + node has been restarted, and the node is ready to provide service. + """ + needed, total = params + config = node.get_config() + config.set_config("client", "shares.happy", str(1)) + config.set_config("client", "shares.needed", str(needed)) + config.set_config("client", "shares.total", str(total)) + config.write_private_config("convergence", base32.b2a(convergence)) + + # restart the node + print(f"Restarting {node.node_dir} for ZFEC reconfiguration") + await node.restart_async(reactor, request) + print("Restarted. Waiting for ready state.") + await_client_ready(node) + print("Ready.") From 5af6fc0f9db49d60fee70d6ee14dac926d8a7191 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 22 Dec 2022 20:53:49 -0500 Subject: [PATCH 1308/2309] reconfigure() only needs to restart the node if something changed --- integration/test_vectors.py | 2 +- integration/util.py | 45 ++++++++++++++++++++++++++----------- 2 files changed, 33 insertions(+), 14 deletions(-) diff --git a/integration/test_vectors.py b/integration/test_vectors.py index 042421e6f..252360fa2 100644 --- a/integration/test_vectors.py +++ b/integration/test_vectors.py @@ -87,7 +87,7 @@ async def test_chk_capability(reactor, request, alice, params_idx, convergence_i data = OBJECT_DATA[data_idx] # rewrite alice's config to match params and convergence - await reconfigure(reactor, request, alice, params, convergence) + await reconfigure(reactor, request, alice, (1,) + params, convergence) # upload data as a CHK actual = upload(alice, "chk", data) diff --git a/integration/util.py b/integration/util.py index 0dc63b2d5..2db6ac391 100644 --- a/integration/util.py +++ b/integration/util.py @@ -651,31 +651,50 @@ def insert(item: tuple[α, β], d: dict[α, β]) -> dict[α, β]: return d -async def reconfigure(reactor, request, node: TahoeProcess, params: tuple[int, int], convergence: bytes) -> None: +async def reconfigure(reactor, request, node: TahoeProcess, params: tuple[int, int, int], convergence: bytes) -> None: """ Reconfigure a Tahoe-LAFS node with different ZFEC parameters and convergence secret. + If the current configuration is different from the specified + configuration, the node will be restarted so it takes effect. + :param reactor: A reactor to use to restart the process. :param request: The pytest request object to use to arrange process cleanup. :param node: The Tahoe-LAFS node to reconfigure. - :param params: The ``needed`` and ``total`` ZFEC encoding parameters. + :param params: The ``happy``, ``needed``, and ``total`` ZFEC encoding + parameters. :param convergence: The convergence secret. :return: ``None`` after the node configuration has been rewritten, the node has been restarted, and the node is ready to provide service. """ - needed, total = params + happy, needed, total = params config = node.get_config() - config.set_config("client", "shares.happy", str(1)) - config.set_config("client", "shares.needed", str(needed)) - config.set_config("client", "shares.total", str(total)) - config.write_private_config("convergence", base32.b2a(convergence)) - # restart the node - print(f"Restarting {node.node_dir} for ZFEC reconfiguration") - await node.restart_async(reactor, request) - print("Restarted. Waiting for ready state.") - await_client_ready(node) - print("Ready.") + changed = False + cur_happy = int(config.get_config("client", "shares.happy")) + cur_needed = int(config.get_config("client", "shares.needed")) + cur_total = int(config.get_config("client", "shares.total")) + + if (happy, needed, total) != (cur_happy, cur_needed, cur_total): + changed = True + config.set_config("client", "shares.happy", str(happy)) + config.set_config("client", "shares.needed", str(needed)) + config.set_config("client", "shares.total", str(total)) + + cur_convergence = config.get_private_config("convergence").encode("ascii") + if base32.a2b(cur_convergence) != convergence: + changed = True + config.write_private_config("convergence", base32.b2a(convergence)) + + if changed: + # restart the node + print(f"Restarting {node.node_dir} for ZFEC reconfiguration") + await node.restart_async(reactor, request) + print("Restarted. Waiting for ready state.") + await_client_ready(node) + print("Ready.") + else: + print("Config unchanged, not restarting.") From 39b3f19c0e9f1ea280525e7c930fb59eda8188f3 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 26 Dec 2022 12:06:34 -0500 Subject: [PATCH 1309/2309] Put the generator inputs into the output file This should make it easier for other implementations to use the test data, I think. Also put a version in there so we can change inputs in the future but still talk about results meaningfully. And some other minor refactoring --- integration/_vectors_chk.yaml | 504 ---------------------------------- integration/test_vectors.py | 132 ++++++--- integration/test_vectors.yaml | 429 +++++++++++++++++++++++++++++ integration/vectors.py | 9 +- 4 files changed, 525 insertions(+), 549 deletions(-) delete mode 100644 integration/_vectors_chk.yaml create mode 100644 integration/test_vectors.yaml diff --git a/integration/_vectors_chk.yaml b/integration/_vectors_chk.yaml deleted file mode 100644 index 81a2a70c4..000000000 --- a/integration/_vectors_chk.yaml +++ /dev/null @@ -1,504 +0,0 @@ -? 1/1,0c0beacef8877bbf2416eb00f2b5dc96354e26dd1df5517320459b1236860f8c,2edc986847e209b4016e141a6dc8716d3207350f416969382d431539bf292e4a -: URI:CHK:mri732rh3meyh4drikau3a24ba:6tsj5wvcp6szdhmrbu5bea57wduoza64y6nd2lm7aleqpsxjm5la:1:1:1024 -? 1/1,0c0beacef8877bbf2416eb00f2b5dc96354e26dd1df5517320459b1236860f8c,3abc94a93a42d0eee5c8dda0315f9f1343e2ba36b552ab512c435fd4989c1ac6 -: URI:CHK:e5tpbcmalcq6nn2zhd3qvg4chy:emzsvv2xnkhhrj2oatds5hf2cney25awi56ybeq4ofiyehzyakua:1:1:4096 -? 1/1,0c0beacef8877bbf2416eb00f2b5dc96354e26dd1df5517320459b1236860f8c,77f3b5f9437275607c3b5eef5f67d18d6a9835155cfee75acd9d2c043dbb8528 -: URI:CHK:nldstwombnjucmrhzok557b2gi:6vxghlkmxq3fmrqx5bjs5bwqxdh7b2krsfse2wnfympv5djgk4cq:1:1:8193 -? 1/1,0c0beacef8877bbf2416eb00f2b5dc96354e26dd1df5517320459b1236860f8c,80c58fc1932767b113819b033e1874d8de05e8809534590db9d01510abc8ed6b -: URI:CHK:3o3ksfxjqakmjhscpnkmdabisy:ailkrgaw5fcor7ywro225jp52i5mfoigbsvoaaqbdbbgdxrvmfta:1:1:2048 -? 1/1,0c0beacef8877bbf2416eb00f2b5dc96354e26dd1df5517320459b1236860f8c,8454e5bb164935d6a8da749d78cf91eb0556897d44273053b67777aed1d418e8 -: URI:CHK:x4cvhgcrxpvqf6k3ldgjnt7tiy:2ydvfx4sh6wbi6ucf5atocfwkvxcshtkbqonkdr3oc5unppixbmq:1:1:131071 -? 1/1,0c0beacef8877bbf2416eb00f2b5dc96354e26dd1df5517320459b1236860f8c,8a6a6bff19a6f768864a69d6c6c97e6a25385793201eefe2505650978aef9193 -: URI:CHK:6wip7b2rqy5kmx7mixsq35zzja:wjgohcfhurrjiv3wd7pc7g7lp5stvl23ynv4k26ehcswhlrsmkiq:1:1:131073 -? 1/1,0c0beacef8877bbf2416eb00f2b5dc96354e26dd1df5517320459b1236860f8c,caccfbaa894a6cf59e3d677a146924604f8edcd07ed1d61cd1d1569ce4bf3ec5 -: URI:CHK:mpzclo7ghftme56qnm2ehien5y:uoqact6oasexnzh3tkm4oqtvhfcqtl6vkrjzbp5vucxg2ptel2ma:1:1:8191 -? 1/1,2e61afd25d3ca76a0ee265dbc22ba5e9d844dc93ac5f2e37199a34fcbdcb5a52,2edc986847e209b4016e141a6dc8716d3207350f416969382d431539bf292e4a -: URI:CHK:7bkmeibwcejlkd67uxb5ggnbnq:osf4xaukorh2zdhhipyg2lpuxdcn3mm4zukrxlqwtnnoglydwzka:1:1:1024 -? 1/1,2e61afd25d3ca76a0ee265dbc22ba5e9d844dc93ac5f2e37199a34fcbdcb5a52,3abc94a93a42d0eee5c8dda0315f9f1343e2ba36b552ab512c435fd4989c1ac6 -: URI:CHK:6ix53x4efg2ijovglu3j47nehu:rsfmmi27da2fnw4qauav2lddvaoht7lhl6zebhmpyefzdonpk3ya:1:1:4096 -? 1/1,2e61afd25d3ca76a0ee265dbc22ba5e9d844dc93ac5f2e37199a34fcbdcb5a52,77f3b5f9437275607c3b5eef5f67d18d6a9835155cfee75acd9d2c043dbb8528 -: URI:CHK:bzf2m5gq2awaq6e2izbjrtwb4i:wfi2rysyt7k3xyfcbyu3oargpbqfg7e4afrr4zk6sacizu4cuecq:1:1:8193 -? 1/1,2e61afd25d3ca76a0ee265dbc22ba5e9d844dc93ac5f2e37199a34fcbdcb5a52,80c58fc1932767b113819b033e1874d8de05e8809534590db9d01510abc8ed6b -: URI:CHK:qocnrwjq2go6ae6rlixxojjj5a:btbm2djqqqim6ptrlvpsrqhf7tgkidizrz5nv4hlo7nbhemhuwyq:1:1:2048 -? 1/1,2e61afd25d3ca76a0ee265dbc22ba5e9d844dc93ac5f2e37199a34fcbdcb5a52,8454e5bb164935d6a8da749d78cf91eb0556897d44273053b67777aed1d418e8 -: URI:CHK:tnjp7mhj2y25vhmuivzwa5a2v4:tn3nquzibiqds3vjjjxjlgxryfz4z57wwyekplexvcqqccbkzylq:1:1:131071 -? 1/1,2e61afd25d3ca76a0ee265dbc22ba5e9d844dc93ac5f2e37199a34fcbdcb5a52,8a6a6bff19a6f768864a69d6c6c97e6a25385793201eefe2505650978aef9193 -: URI:CHK:bkpgz4emfogefvsgkf6qrvqdpa:qbinjbmhnawpv4k7kqll5xc7gjcyfevrqigg3jflqowc726pyiba:1:1:131073 -? 1/1,2e61afd25d3ca76a0ee265dbc22ba5e9d844dc93ac5f2e37199a34fcbdcb5a52,caccfbaa894a6cf59e3d677a146924604f8edcd07ed1d61cd1d1569ce4bf3ec5 -: URI:CHK:weepfzfu52fyaj3sl2grlj7x6e:iwpntrlq7kd3iokdcxkzj67srwbxgm7db5ujagxihiqn4hfr2stq:1:1:8191 -? 1/1,43a1e8995c86ee7f559e5d187d1657296662ca180b394146a6e1dd0df9158a30,2edc986847e209b4016e141a6dc8716d3207350f416969382d431539bf292e4a -: URI:CHK:nnv4vrtlxmzkurfzvonj22leua:ywcyijrfnwykraku56dq7v3o4ts3xsxqfgmk3kgzwhq2cpest4pa:1:1:1024 -? 1/1,43a1e8995c86ee7f559e5d187d1657296662ca180b394146a6e1dd0df9158a30,3abc94a93a42d0eee5c8dda0315f9f1343e2ba36b552ab512c435fd4989c1ac6 -: URI:CHK:wlszole5m2emf6wbhp3lnlyfxq:irdttvny74gxdrcotzfjslfq7p24kyiigyrm5shhw7zyj3hsiuba:1:1:4096 -? 1/1,43a1e8995c86ee7f559e5d187d1657296662ca180b394146a6e1dd0df9158a30,77f3b5f9437275607c3b5eef5f67d18d6a9835155cfee75acd9d2c043dbb8528 -: URI:CHK:jq2q3dk343h6cmzbbozlc4pdua:grtvc4pgd5cbzedghyk7erpal2m7ururlzv5ui2kc2lbwszk6feq:1:1:8193 -? 1/1,43a1e8995c86ee7f559e5d187d1657296662ca180b394146a6e1dd0df9158a30,80c58fc1932767b113819b033e1874d8de05e8809534590db9d01510abc8ed6b -: URI:CHK:7sfwjhsji4vmf6dxauh32mxc3u:vnoepbsyapzfslimfdw6udn5khw4xspjn4fhpvkw6h2zigbv2qja:1:1:2048 -? 1/1,43a1e8995c86ee7f559e5d187d1657296662ca180b394146a6e1dd0df9158a30,8454e5bb164935d6a8da749d78cf91eb0556897d44273053b67777aed1d418e8 -: URI:CHK:vuyye2vhfat5fjbgig7y5jokr4:sibrvhclkne5jtmc6teoiri2onhh7htflsdixb6i2vccg3hhxfpa:1:1:131071 -? 1/1,43a1e8995c86ee7f559e5d187d1657296662ca180b394146a6e1dd0df9158a30,8a6a6bff19a6f768864a69d6c6c97e6a25385793201eefe2505650978aef9193 -: URI:CHK:hfjcytrorzrycjzjsxfa6hioha:pxtxr26bokroy2etwcvs3hcv3quil3cywfmlvg45eqdaff7r4y4a:1:1:131073 -? 1/1,43a1e8995c86ee7f559e5d187d1657296662ca180b394146a6e1dd0df9158a30,caccfbaa894a6cf59e3d677a146924604f8edcd07ed1d61cd1d1569ce4bf3ec5 -: URI:CHK:thoqlkvbsnpiodlnxdoabwgu7a:oufaztxpozbheudq5aiou6h25c675knx55fnwvd56qt7sqwhyrmq:1:1:8191 -? 1/1,be45cb2605bf36bebde684841a28f0fd43c69850a3dce5fedba69928ee3a8991,2edc986847e209b4016e141a6dc8716d3207350f416969382d431539bf292e4a -: URI:CHK:rkspaiyphipftpmtcshm4x2qwq:xiefohxijxssgeednc6hvd77dqnrw7zhppvcubfghf5ci432woqq:1:1:1024 -? 1/1,be45cb2605bf36bebde684841a28f0fd43c69850a3dce5fedba69928ee3a8991,3abc94a93a42d0eee5c8dda0315f9f1343e2ba36b552ab512c435fd4989c1ac6 -: URI:CHK:kybvknf7jzu7g6gn5ngme4sb2e:56krx24234cbjger2b2q5mukghs4we72xz6va6whzriuepqoy2kq:1:1:4096 -? 1/1,be45cb2605bf36bebde684841a28f0fd43c69850a3dce5fedba69928ee3a8991,77f3b5f9437275607c3b5eef5f67d18d6a9835155cfee75acd9d2c043dbb8528 -: URI:CHK:nrpahak2l4xgfokz5ehrribtuy:mx5ix7in5anfrhldds3qkuy65rembijqi2r72wfcl6djotse2pha:1:1:8193 -? 1/1,be45cb2605bf36bebde684841a28f0fd43c69850a3dce5fedba69928ee3a8991,80c58fc1932767b113819b033e1874d8de05e8809534590db9d01510abc8ed6b -: URI:CHK:bv5i26hs2lltigwv3q6jad2j3m:u6m5xityfho4ecyywathvckxmgzfiz3t7kd6niszbsxq65ayzpmq:1:1:2048 -? 1/1,be45cb2605bf36bebde684841a28f0fd43c69850a3dce5fedba69928ee3a8991,8454e5bb164935d6a8da749d78cf91eb0556897d44273053b67777aed1d418e8 -: URI:CHK:donvstm35rcmks6cekutcijq5e:b5rnhrvzkvhas7ax3cm2smcodjvvvfgbnsf7yfg3ypaqce6zrezq:1:1:131071 -? 1/1,be45cb2605bf36bebde684841a28f0fd43c69850a3dce5fedba69928ee3a8991,8a6a6bff19a6f768864a69d6c6c97e6a25385793201eefe2505650978aef9193 -: URI:CHK:5fqqgakv2li3d4iqy2hvfqhaxi:76y5lu7uqwuuji3gtcivzca7hgxs23hyvzompgd43zle5flnuwya:1:1:131073 -? 1/1,be45cb2605bf36bebde684841a28f0fd43c69850a3dce5fedba69928ee3a8991,caccfbaa894a6cf59e3d677a146924604f8edcd07ed1d61cd1d1569ce4bf3ec5 -: URI:CHK:adrurnek5f244enbpslfhb3fni:w2ht6yczgmy7yo7ga7epem76bnt272a3g5vxnblsanl5xmkxiayq:1:1:8191 -? 1/1,cd82b3962c8796095a288e29ed94f98017cc681d17ea9ea8861c4034b13a3b6f,2edc986847e209b4016e141a6dc8716d3207350f416969382d431539bf292e4a -: URI:CHK:os7rlg55kwbtgdibianzgoekwe:ug6ju36skjxtwasbqssasl73krp63c46o3ayr4gsj4l2zyzb4p7q:1:1:1024 -? 1/1,cd82b3962c8796095a288e29ed94f98017cc681d17ea9ea8861c4034b13a3b6f,3abc94a93a42d0eee5c8dda0315f9f1343e2ba36b552ab512c435fd4989c1ac6 -: URI:CHK:wevlzjefiuecq734iz6h3j6vym:p2krrasfxrdi2o3uws6afrwod7b2fbqpt2di4m6h5p3uovbzpkca:1:1:4096 -? 1/1,cd82b3962c8796095a288e29ed94f98017cc681d17ea9ea8861c4034b13a3b6f,77f3b5f9437275607c3b5eef5f67d18d6a9835155cfee75acd9d2c043dbb8528 -: URI:CHK:ivgaoi6yhfe7emqkfdhz7x2npu:nfcqouqdcwv5d2otuknokpxgdggxuvu7f4m4ct3utbxit6wz5caq:1:1:8193 -? 1/1,cd82b3962c8796095a288e29ed94f98017cc681d17ea9ea8861c4034b13a3b6f,80c58fc1932767b113819b033e1874d8de05e8809534590db9d01510abc8ed6b -: URI:CHK:vbk4tahxlqulrqha3vyckpppm4:wzmzru7z74apg4vyrqhabvbrrrpew7n5ltxha5bwbddfghse7mva:1:1:2048 -? 1/1,cd82b3962c8796095a288e29ed94f98017cc681d17ea9ea8861c4034b13a3b6f,8454e5bb164935d6a8da749d78cf91eb0556897d44273053b67777aed1d418e8 -: URI:CHK:jyyidgcdadxfb2qdcrkvwultau:jktpx4gzdsh25bd42k3p74p7eujhnh5zre4wvjp7gorue4whldia:1:1:131071 -? 1/1,cd82b3962c8796095a288e29ed94f98017cc681d17ea9ea8861c4034b13a3b6f,8a6a6bff19a6f768864a69d6c6c97e6a25385793201eefe2505650978aef9193 -: URI:CHK:f2bren65rc4l7zrcnbrwocttty:giph2rn7u4dkkbuiruqfrxgb4ar6xyhsxtjdeeeuwwnc6vrxalsq:1:1:131073 -? 1/1,cd82b3962c8796095a288e29ed94f98017cc681d17ea9ea8861c4034b13a3b6f,caccfbaa894a6cf59e3d677a146924604f8edcd07ed1d61cd1d1569ce4bf3ec5 -: URI:CHK:oskjs2mq2pp7w75rnnb7b36ynu:qujhwnvdlpcvx5ccxwbhke5pbd2bn63vmp3ftugqm2n6u7qteoda:1:1:8191 -? 1/1,f39dac6cbaba535e2c207cd0cd8f154974223c848f727f98b3564cea569b41cf,2edc986847e209b4016e141a6dc8716d3207350f416969382d431539bf292e4a -: URI:CHK:72l476fpask7kz53frtmw3t6pa:yf35zcretagoeaer5mwpjvy35vba35b2anuy3kprcwfbddbdappa:1:1:1024 -? 1/1,f39dac6cbaba535e2c207cd0cd8f154974223c848f727f98b3564cea569b41cf,3abc94a93a42d0eee5c8dda0315f9f1343e2ba36b552ab512c435fd4989c1ac6 -: URI:CHK:5nmi5kbb5p5hycwlfn653c7s2i:cb3xoljokodml6ecspwj5llvdhu632x25mow3d7jbznuvwwsm7xa:1:1:4096 -? 1/1,f39dac6cbaba535e2c207cd0cd8f154974223c848f727f98b3564cea569b41cf,77f3b5f9437275607c3b5eef5f67d18d6a9835155cfee75acd9d2c043dbb8528 -: URI:CHK:d7yl3rivi3icf3awe6s26wwcp4:6ijmbhqfo26oywcky6enlkmbvopvkc5sotdtw5rnkrewtukxzaja:1:1:8193 -? 1/1,f39dac6cbaba535e2c207cd0cd8f154974223c848f727f98b3564cea569b41cf,80c58fc1932767b113819b033e1874d8de05e8809534590db9d01510abc8ed6b -: URI:CHK:ej5bvkhdqclpvoh4hddn3fc72i:6sxjptm4ozclmm6fzplpt46fgtcxvmmjdkmwhntf3h2jiouoftfa:1:1:2048 -? 1/1,f39dac6cbaba535e2c207cd0cd8f154974223c848f727f98b3564cea569b41cf,8454e5bb164935d6a8da749d78cf91eb0556897d44273053b67777aed1d418e8 -: URI:CHK:4fkhrkyi4srb2jiir4qdm32zj4:khjfhoz7c24my4dt5xm5hxgttxmid6ekbmst26b6rmkk3n4lubzq:1:1:131071 -? 1/1,f39dac6cbaba535e2c207cd0cd8f154974223c848f727f98b3564cea569b41cf,8a6a6bff19a6f768864a69d6c6c97e6a25385793201eefe2505650978aef9193 -: URI:CHK:wbsbqw3os3sjc2yhdj6qashad4:bubdew37qtm7inja5dsrzu6vlfdmheegq65foijd56cy54vh3ihq:1:1:131073 -? 1/1,f39dac6cbaba535e2c207cd0cd8f154974223c848f727f98b3564cea569b41cf,caccfbaa894a6cf59e3d677a146924604f8edcd07ed1d61cd1d1569ce4bf3ec5 -: URI:CHK:ovgqdsshkjg3q2qwjfupb2ysfa:hdjonr7dpz662jin3dqn35rt74lsnyp6ox2qvzpc67nzrd7m6c7q:1:1:8191 -? 1/3,0c0beacef8877bbf2416eb00f2b5dc96354e26dd1df5517320459b1236860f8c,2edc986847e209b4016e141a6dc8716d3207350f416969382d431539bf292e4a -: URI:CHK:ycvogzi6wllnq2bkx3t6zdtwju:um42l4yen7jiwdfgirvedtty3tt3xuhjiyxzqoourvughtxjar3q:1:3:1024 -? 1/3,0c0beacef8877bbf2416eb00f2b5dc96354e26dd1df5517320459b1236860f8c,3abc94a93a42d0eee5c8dda0315f9f1343e2ba36b552ab512c435fd4989c1ac6 -: URI:CHK:3yjtuv5h2g45g2cncg6hka3euu:bqqhp6u73ldawial7rtrfomx2qkmboyvqve6ywo2jpvfbu7zptga:1:3:4096 -? 1/3,0c0beacef8877bbf2416eb00f2b5dc96354e26dd1df5517320459b1236860f8c,77f3b5f9437275607c3b5eef5f67d18d6a9835155cfee75acd9d2c043dbb8528 -: URI:CHK:gqndvjhqwsgyawfnmpfsr4x6sy:d6txdlbset3s536cdbqhthxxq4r2u4a2eud53ccphc7kmssaa7jq:1:3:8193 -? 1/3,0c0beacef8877bbf2416eb00f2b5dc96354e26dd1df5517320459b1236860f8c,80c58fc1932767b113819b033e1874d8de05e8809534590db9d01510abc8ed6b -: URI:CHK:lfysy2xpqyaz5lx5velu67ifyu:uiarh7kalq2nxtt5nphmdrrg6zsnqzhf4fzdvauzppsls4p5fyka:1:3:2048 -? 1/3,0c0beacef8877bbf2416eb00f2b5dc96354e26dd1df5517320459b1236860f8c,8454e5bb164935d6a8da749d78cf91eb0556897d44273053b67777aed1d418e8 -: URI:CHK:wmdpxzpdjbw3jljibi7k7g4pnu:2jma3twdad5ojmh6e7ysh2kgdhoz7jmanmocxudzzkzbpav2ca5a:1:3:131071 -? 1/3,0c0beacef8877bbf2416eb00f2b5dc96354e26dd1df5517320459b1236860f8c,8a6a6bff19a6f768864a69d6c6c97e6a25385793201eefe2505650978aef9193 -: URI:CHK:pfskrgoirfzk724lpjv4bxtuuu:chq3s4q4wqdj33vpkc47r55niakgzcxncdksp5dhp7aw63zv5u3a:1:3:131073 -? 1/3,0c0beacef8877bbf2416eb00f2b5dc96354e26dd1df5517320459b1236860f8c,caccfbaa894a6cf59e3d677a146924604f8edcd07ed1d61cd1d1569ce4bf3ec5 -: URI:CHK:wfrvklrwkxjbtj6vqbgttlgb5e:utbh3nrqv236xkzmt2fyx3ej2cfnj4pcw532lnfcutsvhgv72opq:1:3:8191 -? 1/3,2e61afd25d3ca76a0ee265dbc22ba5e9d844dc93ac5f2e37199a34fcbdcb5a52,2edc986847e209b4016e141a6dc8716d3207350f416969382d431539bf292e4a -: URI:CHK:cogbisdsvqpkynu6ontf4vlgaq:2biqzdoxqweid7si5y5zb37mf4pzme7tnnn4hw5dunzdvikogdwq:1:3:1024 -? 1/3,2e61afd25d3ca76a0ee265dbc22ba5e9d844dc93ac5f2e37199a34fcbdcb5a52,3abc94a93a42d0eee5c8dda0315f9f1343e2ba36b552ab512c435fd4989c1ac6 -: URI:CHK:yh6cn346w44f7odadasfsyouaa:zut5qrxxzs255gf6a3nefa7qcrxo3bfv72i3xlncmhnd5gzhv4ia:1:3:4096 -? 1/3,2e61afd25d3ca76a0ee265dbc22ba5e9d844dc93ac5f2e37199a34fcbdcb5a52,77f3b5f9437275607c3b5eef5f67d18d6a9835155cfee75acd9d2c043dbb8528 -: URI:CHK:5fjy2fkocl5gye4dgmvqu5kcoi:6oayuhip53s4jhnvjv374mrn5hqaxxzpu62kxdf3sa2jn36y6mzq:1:3:8193 -? 1/3,2e61afd25d3ca76a0ee265dbc22ba5e9d844dc93ac5f2e37199a34fcbdcb5a52,80c58fc1932767b113819b033e1874d8de05e8809534590db9d01510abc8ed6b -: URI:CHK:zp3nc6tyuyhwszywmqehybc6oe:sd32yfuuyyq52ppee6yviivsqrglvgitijl3mweydhpu4zm5hgca:1:3:2048 -? 1/3,2e61afd25d3ca76a0ee265dbc22ba5e9d844dc93ac5f2e37199a34fcbdcb5a52,8454e5bb164935d6a8da749d78cf91eb0556897d44273053b67777aed1d418e8 -: URI:CHK:sexguezmb55m4qbgxb5getu7qu:5ud5zz6kvutmu6mca6b2nicerk7pofigc3ysosyngs5nkctyimaq:1:3:131071 -? 1/3,2e61afd25d3ca76a0ee265dbc22ba5e9d844dc93ac5f2e37199a34fcbdcb5a52,8a6a6bff19a6f768864a69d6c6c97e6a25385793201eefe2505650978aef9193 -: URI:CHK:o6tv2j53qsfosjidti6rr65txm:5gca453tox7fp3qwz43if6t5upl32ialclbko3p562mkizqq3b2a:1:3:131073 -? 1/3,2e61afd25d3ca76a0ee265dbc22ba5e9d844dc93ac5f2e37199a34fcbdcb5a52,caccfbaa894a6cf59e3d677a146924604f8edcd07ed1d61cd1d1569ce4bf3ec5 -: URI:CHK:o7px44ar5tvwjzqghpw46jxoqu:vdaug57fb57cybqhmhbutjamnc5uclvgzhjzebqrhpdpw4vv6znq:1:3:8191 -? 1/3,43a1e8995c86ee7f559e5d187d1657296662ca180b394146a6e1dd0df9158a30,2edc986847e209b4016e141a6dc8716d3207350f416969382d431539bf292e4a -: URI:CHK:pbfcdvxhbikxd2hcc43oel3v3e:y5txpmiptmoz36ionmki3p6krmdbiqasw2v3wdq4ia5lmrhh33lq:1:3:1024 -? 1/3,43a1e8995c86ee7f559e5d187d1657296662ca180b394146a6e1dd0df9158a30,3abc94a93a42d0eee5c8dda0315f9f1343e2ba36b552ab512c435fd4989c1ac6 -: URI:CHK:vkdc7iae4pgkdomhbbb554kv6a:g5xkatxv2entfssfh5eqgexb5j3qs6jvohofjqo5p5erlxlx2dga:1:3:4096 -? 1/3,43a1e8995c86ee7f559e5d187d1657296662ca180b394146a6e1dd0df9158a30,77f3b5f9437275607c3b5eef5f67d18d6a9835155cfee75acd9d2c043dbb8528 -: URI:CHK:mipcizv4soceb3kepaeyaw4k2y:ysg7t5umiqz4cluwk3coqg3yl44c4clegtju3udwffeasrwovddq:1:3:8193 -? 1/3,43a1e8995c86ee7f559e5d187d1657296662ca180b394146a6e1dd0df9158a30,80c58fc1932767b113819b033e1874d8de05e8809534590db9d01510abc8ed6b -: URI:CHK:3yhotahhjha4j6acgc2e2vbktm:khrftxrlbudn7c25f2oidos7or4wdmagmgwzditz232c3b7ea6wq:1:3:2048 -? 1/3,43a1e8995c86ee7f559e5d187d1657296662ca180b394146a6e1dd0df9158a30,8454e5bb164935d6a8da749d78cf91eb0556897d44273053b67777aed1d418e8 -: URI:CHK:6yzsaazkkblikql63nyglc77pq:blsru6a6cu3k4y4nbllgkxidwcehmo7ey2qvidvi6vqywo2j6geq:1:3:131071 -? 1/3,43a1e8995c86ee7f559e5d187d1657296662ca180b394146a6e1dd0df9158a30,8a6a6bff19a6f768864a69d6c6c97e6a25385793201eefe2505650978aef9193 -: URI:CHK:gpdcbuf22tbrrtycwa2l2hclii:6r3r4ocqforegcetf3wpohwkqk4t5b7cyywx3uzqdfeb4dg7iofa:1:3:131073 -? 1/3,43a1e8995c86ee7f559e5d187d1657296662ca180b394146a6e1dd0df9158a30,caccfbaa894a6cf59e3d677a146924604f8edcd07ed1d61cd1d1569ce4bf3ec5 -: URI:CHK:o5jeaynpscalnainlggkedq2ee:3px5gi7msjc3bjm32lxunmen3ugvnl42erh46xg26ae2dcgghg6q:1:3:8191 -? 1/3,be45cb2605bf36bebde684841a28f0fd43c69850a3dce5fedba69928ee3a8991,2edc986847e209b4016e141a6dc8716d3207350f416969382d431539bf292e4a -: URI:CHK:a4ebyitlpldquuq6ucvnha4ixa:4pdnqdib5vswqko3if6ap6rmrgwcixapaqbe33ctsyghgcm3uofa:1:3:1024 -? 1/3,be45cb2605bf36bebde684841a28f0fd43c69850a3dce5fedba69928ee3a8991,3abc94a93a42d0eee5c8dda0315f9f1343e2ba36b552ab512c435fd4989c1ac6 -: URI:CHK:frqacd3kdebc4ur5evzyqwckfq:hbpilmiqtojoedq3zoyiv3pyb27fktpbm6jsln4acfvgadcvh5ia:1:3:4096 -? 1/3,be45cb2605bf36bebde684841a28f0fd43c69850a3dce5fedba69928ee3a8991,77f3b5f9437275607c3b5eef5f67d18d6a9835155cfee75acd9d2c043dbb8528 -: URI:CHK:f6irlornvt557fsjtznb7ebkii:u4i4sab7naw35fisspioirnmdpt4fb6vnsxxakmwqlxhpnr6upcq:1:3:8193 -? 1/3,be45cb2605bf36bebde684841a28f0fd43c69850a3dce5fedba69928ee3a8991,80c58fc1932767b113819b033e1874d8de05e8809534590db9d01510abc8ed6b -: URI:CHK:lff7klhpboefzcfvfub6zwe724:x435m7yjcwul6elf3jaq44flu5jr6fafqzs5vsbi27rovugkbgwq:1:3:2048 -? 1/3,be45cb2605bf36bebde684841a28f0fd43c69850a3dce5fedba69928ee3a8991,8454e5bb164935d6a8da749d78cf91eb0556897d44273053b67777aed1d418e8 -: URI:CHK:7hmarhc2mjvjan2omvipkpzpca:jppvg5d66dtc2cxqgspzq7xejgidwleek2t5i2dygucdjrxotbgq:1:3:131071 -? 1/3,be45cb2605bf36bebde684841a28f0fd43c69850a3dce5fedba69928ee3a8991,8a6a6bff19a6f768864a69d6c6c97e6a25385793201eefe2505650978aef9193 -: URI:CHK:qeiyu3hniuxlhnwm32ftlnv7ma:7gwkm46s2jqwpqlf5fizlgkh766i7oxjfrmxmxs6g2ypagmo4tnq:1:3:131073 -? 1/3,be45cb2605bf36bebde684841a28f0fd43c69850a3dce5fedba69928ee3a8991,caccfbaa894a6cf59e3d677a146924604f8edcd07ed1d61cd1d1569ce4bf3ec5 -: URI:CHK:ivmnumxa5bru43k2snafpcxk6e:77k2cwnqzq7tsf7rui53ar44s6w5x55626rc24pv7ilca33fehyq:1:3:8191 -? 1/3,cd82b3962c8796095a288e29ed94f98017cc681d17ea9ea8861c4034b13a3b6f,2edc986847e209b4016e141a6dc8716d3207350f416969382d431539bf292e4a -: URI:CHK:vcvq26lwgxvjaxgd7a62qnuqem:vllbr4klrxc4w63vzvfpoqpajmephgkjqlh4bawr67q3i7i2q7cq:1:3:1024 -? 1/3,cd82b3962c8796095a288e29ed94f98017cc681d17ea9ea8861c4034b13a3b6f,3abc94a93a42d0eee5c8dda0315f9f1343e2ba36b552ab512c435fd4989c1ac6 -: URI:CHK:k6gfriaqplqeui6xheccfyodim:kbnovfz4pcq6jdjcmgnbfpaqjo4x6e4tzj3qyeeyulrs6apelprq:1:3:4096 -? 1/3,cd82b3962c8796095a288e29ed94f98017cc681d17ea9ea8861c4034b13a3b6f,77f3b5f9437275607c3b5eef5f67d18d6a9835155cfee75acd9d2c043dbb8528 -: URI:CHK:af2unzjbnqmlaaicbssikxm2ba:szxxxayjygrtz72yrha3hsj2ei6xvyosdkdxpx7tuzd4rcg2siwa:1:3:8193 -? 1/3,cd82b3962c8796095a288e29ed94f98017cc681d17ea9ea8861c4034b13a3b6f,80c58fc1932767b113819b033e1874d8de05e8809534590db9d01510abc8ed6b -: URI:CHK:xsiowqs5kymdejztz3zgzjrfpu:t43dbkeimvxiokkl6oz6x5hluidc2siweq4pa66pkgj6zi443y4a:1:3:2048 -? 1/3,cd82b3962c8796095a288e29ed94f98017cc681d17ea9ea8861c4034b13a3b6f,8454e5bb164935d6a8da749d78cf91eb0556897d44273053b67777aed1d418e8 -: URI:CHK:7jve3xr3dkptw7w7bc7ubokqzy:rrr4imn7sc7jpcv4gpkbuaob6buwclz7ubcsc4hckwvfaodq4wpa:1:3:131071 -? 1/3,cd82b3962c8796095a288e29ed94f98017cc681d17ea9ea8861c4034b13a3b6f,8a6a6bff19a6f768864a69d6c6c97e6a25385793201eefe2505650978aef9193 -: URI:CHK:nv4gljql7eyef5yrpx5xfkyez4:hykk52owibpmfmjrgtmi3wzvrpx7xtp3xz6beaieuthhxsob7hra:1:3:131073 -? 1/3,cd82b3962c8796095a288e29ed94f98017cc681d17ea9ea8861c4034b13a3b6f,caccfbaa894a6cf59e3d677a146924604f8edcd07ed1d61cd1d1569ce4bf3ec5 -: URI:CHK:3yntwrjfnabw7tnmh3rkejzhha:fenyosagg43ltfzczjzqmp3y4jmt2y4q6b2m6pg6oliijpxeacjq:1:3:8191 -? 1/3,f39dac6cbaba535e2c207cd0cd8f154974223c848f727f98b3564cea569b41cf,2edc986847e209b4016e141a6dc8716d3207350f416969382d431539bf292e4a -: URI:CHK:35mmxquvlxib7ev3it3dn2zuta:c2ep3r5wapqze6234pep5w6ssf2oj2k2rppnyzfvuvnuuch3eekq:1:3:1024 -? 1/3,f39dac6cbaba535e2c207cd0cd8f154974223c848f727f98b3564cea569b41cf,3abc94a93a42d0eee5c8dda0315f9f1343e2ba36b552ab512c435fd4989c1ac6 -: URI:CHK:qprhw4ebrdbje4pn7qvqenyo4q:mi2fnz4fkjvgzrgzzprh3d3s6xzvlqzwwbo4iuybqtl3uv6qdsha:1:3:4096 -? 1/3,f39dac6cbaba535e2c207cd0cd8f154974223c848f727f98b3564cea569b41cf,77f3b5f9437275607c3b5eef5f67d18d6a9835155cfee75acd9d2c043dbb8528 -: URI:CHK:jeo5ldffp4n3ldsd2cajazhmsy:2mewlw2mcx6nqs7ushmk7ipfkceppkil2wvbvloa7ub47igpnirq:1:3:8193 -? 1/3,f39dac6cbaba535e2c207cd0cd8f154974223c848f727f98b3564cea569b41cf,80c58fc1932767b113819b033e1874d8de05e8809534590db9d01510abc8ed6b -: URI:CHK:hjiajko6mh74umf3k5wcdx3tra:feznuxdbn4mv42hkyp6di55ddpfaxreac4kwrh4fxjvkgrz3cwsa:1:3:2048 -? 1/3,f39dac6cbaba535e2c207cd0cd8f154974223c848f727f98b3564cea569b41cf,8454e5bb164935d6a8da749d78cf91eb0556897d44273053b67777aed1d418e8 -: URI:CHK:mxvgpptt5g4cmmlwduo2dbuhvi:uubgdwc2h4rsudgqft6tedeybltzzlxp4qwqxkqjvwqws3wsixoa:1:3:131071 -? 1/3,f39dac6cbaba535e2c207cd0cd8f154974223c848f727f98b3564cea569b41cf,8a6a6bff19a6f768864a69d6c6c97e6a25385793201eefe2505650978aef9193 -: URI:CHK:bblnb2r5keyy5ybguji4xuyf74:z32ynttnjxh3nqtf4krlrqptu4vmajxpce3q22igc4dr5nx3y2ra:1:3:131073 -? 1/3,f39dac6cbaba535e2c207cd0cd8f154974223c848f727f98b3564cea569b41cf,caccfbaa894a6cf59e3d677a146924604f8edcd07ed1d61cd1d1569ce4bf3ec5 -: URI:CHK:6woltewcikjtsbbucyipnjcaay:avs4cqqyjvjj4awkxvsifc7fsmtbvbp35pqtyo4jejnvdm75uvjq:1:3:8191 -? 101/256,0c0beacef8877bbf2416eb00f2b5dc96354e26dd1df5517320459b1236860f8c,2edc986847e209b4016e141a6dc8716d3207350f416969382d431539bf292e4a -: URI:CHK:mftnvh5l3brshwoupvu6kz35cy:sncfokby2tjykbr3zgi3i6lhgerg4kz7fsrslrlxdy7upwkzq6lq:101:256:1024 -? 101/256,0c0beacef8877bbf2416eb00f2b5dc96354e26dd1df5517320459b1236860f8c,3abc94a93a42d0eee5c8dda0315f9f1343e2ba36b552ab512c435fd4989c1ac6 -: URI:CHK:sshu77h6opnto3jnngot3lwzl4:4cjhp6u3i2uzdzrjitje3inlhs6gnlgwc6wgi6vtedc2grgvcpza:101:256:4096 -? 101/256,0c0beacef8877bbf2416eb00f2b5dc96354e26dd1df5517320459b1236860f8c,77f3b5f9437275607c3b5eef5f67d18d6a9835155cfee75acd9d2c043dbb8528 -: URI:CHK:fhan3wrsj22np5sp34ezma3cuu:4xprdxp4mhsowp74snsn52rtkuxdyrfdckyygr2ineo6xjuog2qa:101:256:8193 -? 101/256,0c0beacef8877bbf2416eb00f2b5dc96354e26dd1df5517320459b1236860f8c,80c58fc1932767b113819b033e1874d8de05e8809534590db9d01510abc8ed6b -: URI:CHK:4wimykgotkgywu7d26obnszwoa:zl2e6fnyrn72z27yzsal734srjz6erg227z2pacvxqe5yzy3ssjq:101:256:2048 -? 101/256,0c0beacef8877bbf2416eb00f2b5dc96354e26dd1df5517320459b1236860f8c,8454e5bb164935d6a8da749d78cf91eb0556897d44273053b67777aed1d418e8 -: URI:CHK:gkbzvojuugtuj3m55ib376eazy:473bxxwpfa4zahetztea2gadrrfdv26rrhfeaakogqvk3ngay5gq:101:256:131071 -? 101/256,0c0beacef8877bbf2416eb00f2b5dc96354e26dd1df5517320459b1236860f8c,8a6a6bff19a6f768864a69d6c6c97e6a25385793201eefe2505650978aef9193 -: URI:CHK:tez3j5xoszuttu3uhvgbztk4zu:b6s66mdiopowshfrdtycnumaacko63uszz6ghnzmyzo7a5n5a3tq:101:256:131073 -? 101/256,0c0beacef8877bbf2416eb00f2b5dc96354e26dd1df5517320459b1236860f8c,caccfbaa894a6cf59e3d677a146924604f8edcd07ed1d61cd1d1569ce4bf3ec5 -: URI:CHK:f5zsbt3wuawsgu5allwqvfuhai:djhrg2jjdubbyi6lol62fwyfcryawnnw6ivqt76rlw5slz4w2kda:101:256:8191 -? 101/256,2e61afd25d3ca76a0ee265dbc22ba5e9d844dc93ac5f2e37199a34fcbdcb5a52,2edc986847e209b4016e141a6dc8716d3207350f416969382d431539bf292e4a -: URI:CHK:f7wgsiwpd2ieoe3iy5fg5uq4bq:nt7zovrykadj6jirc7guq3hyfesdd77rzucez7awdswg2kbfga5q:101:256:1024 -? 101/256,2e61afd25d3ca76a0ee265dbc22ba5e9d844dc93ac5f2e37199a34fcbdcb5a52,3abc94a93a42d0eee5c8dda0315f9f1343e2ba36b552ab512c435fd4989c1ac6 -: URI:CHK:y6j4jiy3asjv66qdacozkgiduq:acqcevrjrfbtjbe5d4rl4y566lv2ype4dsyoznedbrqzjy4pxe6a:101:256:4096 -? 101/256,2e61afd25d3ca76a0ee265dbc22ba5e9d844dc93ac5f2e37199a34fcbdcb5a52,77f3b5f9437275607c3b5eef5f67d18d6a9835155cfee75acd9d2c043dbb8528 -: URI:CHK:ldeo5hbywzenytsqqauftpthoy:7lye6cutm2btoen2rspz7pqipxrhynpk2bvecrvhvwsfcuaed3fa:101:256:8193 -? 101/256,2e61afd25d3ca76a0ee265dbc22ba5e9d844dc93ac5f2e37199a34fcbdcb5a52,80c58fc1932767b113819b033e1874d8de05e8809534590db9d01510abc8ed6b -: URI:CHK:ts2nlbfh6i7cvg4y5hywshzb7e:cyceefzvc2pdcqjyzxq7vnvgsxkoaok7kfwyy6f2q4ahxr6inkaa:101:256:2048 -? 101/256,2e61afd25d3ca76a0ee265dbc22ba5e9d844dc93ac5f2e37199a34fcbdcb5a52,8454e5bb164935d6a8da749d78cf91eb0556897d44273053b67777aed1d418e8 -: URI:CHK:ds7rbmi4ewkc7u2diopc463bsy:n4xbp2xjogncvhzaph62u5p3lt3263cosz7o5zh4fdl32x7ggmqq:101:256:131071 -? 101/256,2e61afd25d3ca76a0ee265dbc22ba5e9d844dc93ac5f2e37199a34fcbdcb5a52,8a6a6bff19a6f768864a69d6c6c97e6a25385793201eefe2505650978aef9193 -: URI:CHK:xruphp3lx7spgpjmrbzcw3h63u:5txd3djukdpxgimev3ahhej7zftglfk65dqfgosgwd7lqna6t2ga:101:256:131073 -? 101/256,2e61afd25d3ca76a0ee265dbc22ba5e9d844dc93ac5f2e37199a34fcbdcb5a52,caccfbaa894a6cf59e3d677a146924604f8edcd07ed1d61cd1d1569ce4bf3ec5 -: URI:CHK:uzhqk67vyyixgbs4uppciyfdmm:3oll2cjamxozrur3dyz3ga2puswrwitk7lmwkpb3gxuexe3mi2aq:101:256:8191 -? 101/256,43a1e8995c86ee7f559e5d187d1657296662ca180b394146a6e1dd0df9158a30,2edc986847e209b4016e141a6dc8716d3207350f416969382d431539bf292e4a -: URI:CHK:ihyeqq5peg2pdyw45ct5mmj4ay:2rzhndv6o4knlvo4po6rvbnknlcitasseea3mirqjor5c7quh5ra:101:256:1024 -? 101/256,43a1e8995c86ee7f559e5d187d1657296662ca180b394146a6e1dd0df9158a30,3abc94a93a42d0eee5c8dda0315f9f1343e2ba36b552ab512c435fd4989c1ac6 -: URI:CHK:mvk5f2mghudk7xnknqddhvnl4m:c74myeugtix7e7l6uwybtrdtfselha2qrcc7hhbymqr6brtg4cea:101:256:4096 -? 101/256,43a1e8995c86ee7f559e5d187d1657296662ca180b394146a6e1dd0df9158a30,77f3b5f9437275607c3b5eef5f67d18d6a9835155cfee75acd9d2c043dbb8528 -: URI:CHK:4f5moqibmc776kn2eyy6lewgrm:e6fjp7worf4ejwj3bhwdbxpwwds73xxlluqnq2h7jeink3k3hlzq:101:256:8193 -? 101/256,43a1e8995c86ee7f559e5d187d1657296662ca180b394146a6e1dd0df9158a30,80c58fc1932767b113819b033e1874d8de05e8809534590db9d01510abc8ed6b -: URI:CHK:xy354pqtzbmlmzwby3hei7kqxu:oj7en424yabutdnsgvak5npd7uzaxk7n35wdnw2ehpaswf6sba2q:101:256:2048 -? 101/256,43a1e8995c86ee7f559e5d187d1657296662ca180b394146a6e1dd0df9158a30,8454e5bb164935d6a8da749d78cf91eb0556897d44273053b67777aed1d418e8 -: URI:CHK:fpjk7xgei2hctfwaz7u7icym3q:li7vkv74rtx5atvidmeg5qgku7quz2opuc3kzsenhdlynzqswgna:101:256:131071 -? 101/256,43a1e8995c86ee7f559e5d187d1657296662ca180b394146a6e1dd0df9158a30,8a6a6bff19a6f768864a69d6c6c97e6a25385793201eefe2505650978aef9193 -: URI:CHK:edmto6grvkwbg3f2qbd4hucmmu:n4xb35itwdp66p73tyjddk6pi3w4nqxzzeactdpnl4rrmiwcbs6q:101:256:131073 -? 101/256,43a1e8995c86ee7f559e5d187d1657296662ca180b394146a6e1dd0df9158a30,caccfbaa894a6cf59e3d677a146924604f8edcd07ed1d61cd1d1569ce4bf3ec5 -: URI:CHK:oj6tnrpdt76cjhtlab2kk2edhe:jsgpmpguhzrqxautcpmkwldbdl5u4vfvjthefua3dikf5e6r2tfq:101:256:8191 -? 101/256,be45cb2605bf36bebde684841a28f0fd43c69850a3dce5fedba69928ee3a8991,2edc986847e209b4016e141a6dc8716d3207350f416969382d431539bf292e4a -: URI:CHK:6tv464bg2dhui7ux7s3qrbizwq:zcew6imlkwdka6rwiqloi6gmadq7nrl4s3sbwjadgb2pqci7r7la:101:256:1024 -? 101/256,be45cb2605bf36bebde684841a28f0fd43c69850a3dce5fedba69928ee3a8991,3abc94a93a42d0eee5c8dda0315f9f1343e2ba36b552ab512c435fd4989c1ac6 -: URI:CHK:uqx46dwiak4sbzdhtw26syqvmm:qmd6gzzuoahnnigl4qfutllbz6gwqw3dwxa5imfnpgvctfquoqtq:101:256:4096 -? 101/256,be45cb2605bf36bebde684841a28f0fd43c69850a3dce5fedba69928ee3a8991,77f3b5f9437275607c3b5eef5f67d18d6a9835155cfee75acd9d2c043dbb8528 -: URI:CHK:a2stcgnpwl3cyedubvsc2534zu:huxjlll2ciq2fszm5gsc2qbyp7ch4gc3eli4bgs5qyhn6f2nbdfq:101:256:8193 -? 101/256,be45cb2605bf36bebde684841a28f0fd43c69850a3dce5fedba69928ee3a8991,80c58fc1932767b113819b033e1874d8de05e8809534590db9d01510abc8ed6b -: URI:CHK:sonrxjr7bble64yaygtpjdwifa:su7ciagahmf4a3aeuxavwcjlo7ou6zcyyoe6ef3qqi2o7fnljiaq:101:256:2048 -? 101/256,be45cb2605bf36bebde684841a28f0fd43c69850a3dce5fedba69928ee3a8991,8454e5bb164935d6a8da749d78cf91eb0556897d44273053b67777aed1d418e8 -: URI:CHK:z7fjyntzsb47tpjc5wxj7ehl6i:5yfcwqbvgvfsercgpaxkcsdxzptzc6iehqhbf6rgxibsggbkmika:101:256:131071 -? 101/256,be45cb2605bf36bebde684841a28f0fd43c69850a3dce5fedba69928ee3a8991,8a6a6bff19a6f768864a69d6c6c97e6a25385793201eefe2505650978aef9193 -: URI:CHK:wnyvsaz7sfbi47zbvjq6veskzq:ju2bh3k2wfr6xxalreb2uzq6xbsmllrt23hdufo7inr2hzrbkv3q:101:256:131073 -? 101/256,be45cb2605bf36bebde684841a28f0fd43c69850a3dce5fedba69928ee3a8991,caccfbaa894a6cf59e3d677a146924604f8edcd07ed1d61cd1d1569ce4bf3ec5 -: URI:CHK:kxm6xaqm7e7ncfmcrzxypdamhu:b2oybfq6w4cuw4rmtfcwvmc2mkp5fchbf5aq3cixynwwuwrh7lkq:101:256:8191 -? 101/256,cd82b3962c8796095a288e29ed94f98017cc681d17ea9ea8861c4034b13a3b6f,2edc986847e209b4016e141a6dc8716d3207350f416969382d431539bf292e4a -: URI:CHK:5ezlshdnhxu2pwle42lvtixc44:ubwznjuovofoehgcfje3cpfstfntnw7h23jfa6obtfbbm5octi6q:101:256:1024 -? 101/256,cd82b3962c8796095a288e29ed94f98017cc681d17ea9ea8861c4034b13a3b6f,3abc94a93a42d0eee5c8dda0315f9f1343e2ba36b552ab512c435fd4989c1ac6 -: URI:CHK:jjsarrteexsv6hrqglslumloxa:xvgoz5m2ka6crduwh2gsoxsbvf2u7yb2upwp7obwwlcndqedpnha:101:256:4096 -? 101/256,cd82b3962c8796095a288e29ed94f98017cc681d17ea9ea8861c4034b13a3b6f,77f3b5f9437275607c3b5eef5f67d18d6a9835155cfee75acd9d2c043dbb8528 -: URI:CHK:b6fbcvqshq4jbg42jwioif4yga:mtrgqr6wmholpqn6wxt6hfny2zysmqnsubex57q5bakieos6jotq:101:256:8193 -? 101/256,cd82b3962c8796095a288e29ed94f98017cc681d17ea9ea8861c4034b13a3b6f,80c58fc1932767b113819b033e1874d8de05e8809534590db9d01510abc8ed6b -: URI:CHK:csywucom45snjtrhuvwr55jx2a:pnjbpjtgroscazbbhhzzgxtvqsnqduey6muvizzvdjy5b6phwqka:101:256:2048 -? 101/256,cd82b3962c8796095a288e29ed94f98017cc681d17ea9ea8861c4034b13a3b6f,8454e5bb164935d6a8da749d78cf91eb0556897d44273053b67777aed1d418e8 -: URI:CHK:5quoy4xiiwnxqekhtqe4y4icmi:mkz2pbuen6n3wb4tqjngmu5t645nnaaye4zs2o6iagojcrtrchua:101:256:131071 -? 101/256,cd82b3962c8796095a288e29ed94f98017cc681d17ea9ea8861c4034b13a3b6f,8a6a6bff19a6f768864a69d6c6c97e6a25385793201eefe2505650978aef9193 -: URI:CHK:adzihjhpwm5vfq7y7tjikjgruq:su3erjgmgbzf2g6xlhnvrzy3w76v4kdiakedizgp4zkehhn6ztiq:101:256:131073 -? 101/256,cd82b3962c8796095a288e29ed94f98017cc681d17ea9ea8861c4034b13a3b6f,caccfbaa894a6cf59e3d677a146924604f8edcd07ed1d61cd1d1569ce4bf3ec5 -: URI:CHK:42cyabs6gdfuzqpsg6unlmxq7a:u7tit3edtyazrapvzvklpf3o2mykepnc3v66lsndellpxz3dciia:101:256:8191 -? 101/256,f39dac6cbaba535e2c207cd0cd8f154974223c848f727f98b3564cea569b41cf,2edc986847e209b4016e141a6dc8716d3207350f416969382d431539bf292e4a -: URI:CHK:g23jpnnqww4v3au5dai2fu5jxy:tghgrgfjbfu37zl3l4keahps74tlsgwbidvxbtge2o6pcbjxjjeq:101:256:1024 -? 101/256,f39dac6cbaba535e2c207cd0cd8f154974223c848f727f98b3564cea569b41cf,3abc94a93a42d0eee5c8dda0315f9f1343e2ba36b552ab512c435fd4989c1ac6 -: URI:CHK:k6igwbutc3cggihl7itzfcltbe:ejbikbgtcbbcfkuhtbbtdcj4jr666jjancn35vgeipnvor72xq2a:101:256:4096 -? 101/256,f39dac6cbaba535e2c207cd0cd8f154974223c848f727f98b3564cea569b41cf,77f3b5f9437275607c3b5eef5f67d18d6a9835155cfee75acd9d2c043dbb8528 -: URI:CHK:ywnywmjsazdgg7nf3dchrlgwoi:sp3xlizgz6gnye2kcdxunm3tual5cmcsckrz6rybuxj2atnrao5a:101:256:8193 -? 101/256,f39dac6cbaba535e2c207cd0cd8f154974223c848f727f98b3564cea569b41cf,80c58fc1932767b113819b033e1874d8de05e8809534590db9d01510abc8ed6b -: URI:CHK:dkxeqnu42nhmh6z24t7ya6rwla:uobek7aktnlksbwoumrn6jlhhmbk2rikjyouchambre4fqskw5na:101:256:2048 -? 101/256,f39dac6cbaba535e2c207cd0cd8f154974223c848f727f98b3564cea569b41cf,8454e5bb164935d6a8da749d78cf91eb0556897d44273053b67777aed1d418e8 -: URI:CHK:63dhrxqcqa5hr5uu2hazaq3mxq:do2zijkco5j4kbrj4tf3f6ykdqhakhlmixs3elbsokaztrbsokgq:101:256:131071 -? 101/256,f39dac6cbaba535e2c207cd0cd8f154974223c848f727f98b3564cea569b41cf,8a6a6bff19a6f768864a69d6c6c97e6a25385793201eefe2505650978aef9193 -: URI:CHK:64dvle2iefzyvpqcytkugy2kda:oildbyluuth6xthq24lwxbumpnishztheazac55f6penotwsztqq:101:256:131073 -? 101/256,f39dac6cbaba535e2c207cd0cd8f154974223c848f727f98b3564cea569b41cf,caccfbaa894a6cf59e3d677a146924604f8edcd07ed1d61cd1d1569ce4bf3ec5 -: URI:CHK:yupjn3fl7xgygpga2hgkijjp7e:juqszewyjp7jttlnltyz3ahp7rtywkmxzr2f5wr6jsvzffni6ukq:101:256:8191 -? 2/3,0c0beacef8877bbf2416eb00f2b5dc96354e26dd1df5517320459b1236860f8c,2edc986847e209b4016e141a6dc8716d3207350f416969382d431539bf292e4a -: URI:CHK:jdm6ytmx2i3ya2bllsdzxurjtu:iqr6tyysaseegzfurhuywy3mbbwkbbsov5bt3fo6oazpyv7olvda:2:3:1024 -? 2/3,0c0beacef8877bbf2416eb00f2b5dc96354e26dd1df5517320459b1236860f8c,3abc94a93a42d0eee5c8dda0315f9f1343e2ba36b552ab512c435fd4989c1ac6 -: URI:CHK:7wz5dhkyob3gmkydzgbptk5qku:cl6ovq5a3km7rpfhyb3putg656sp57lhnp7aexbocsbbahdxnfia:2:3:4096 -? 2/3,0c0beacef8877bbf2416eb00f2b5dc96354e26dd1df5517320459b1236860f8c,77f3b5f9437275607c3b5eef5f67d18d6a9835155cfee75acd9d2c043dbb8528 -: URI:CHK:x6mwrib5rq7rovhqqkdrfj3fei:wjiher75xpe646ccus4etlz6gmsxslkkretchupdqgr4o6uzuvia:2:3:8193 -? 2/3,0c0beacef8877bbf2416eb00f2b5dc96354e26dd1df5517320459b1236860f8c,80c58fc1932767b113819b033e1874d8de05e8809534590db9d01510abc8ed6b -: URI:CHK:o3o2nxtuo6mm4iapnfqzllckcm:epp2ynxr2ok2po3ihvdpimh3q5e2y3r62gfygj7qgloowu5wlika:2:3:2048 -? 2/3,0c0beacef8877bbf2416eb00f2b5dc96354e26dd1df5517320459b1236860f8c,8454e5bb164935d6a8da749d78cf91eb0556897d44273053b67777aed1d418e8 -: URI:CHK:33g3nzjobrl5unjffmqegkgzri:hhj3t5tmc4zdx7n4zgriyie4ujtdoj4epcesmqf2ubjti5cs2r2a:2:3:131071 -? 2/3,0c0beacef8877bbf2416eb00f2b5dc96354e26dd1df5517320459b1236860f8c,8a6a6bff19a6f768864a69d6c6c97e6a25385793201eefe2505650978aef9193 -: URI:CHK:kasnq2fdmif3hnxqhofujik6c4:texnwykka5fnzwb4324xtlv5x2looo6kkfpb4f2vmr6zqgrusk7a:2:3:131073 -? 2/3,0c0beacef8877bbf2416eb00f2b5dc96354e26dd1df5517320459b1236860f8c,caccfbaa894a6cf59e3d677a146924604f8edcd07ed1d61cd1d1569ce4bf3ec5 -: URI:CHK:dwsrkeq2zmixvntzulpqethvvi:tkjxibzjnkzjcns7ofvkw74o4x3r3dx7ic6ocbakrr4t2lfh3nea:2:3:8191 -? 2/3,2e61afd25d3ca76a0ee265dbc22ba5e9d844dc93ac5f2e37199a34fcbdcb5a52,2edc986847e209b4016e141a6dc8716d3207350f416969382d431539bf292e4a -: URI:CHK:ggdnrfd7oqwgmaq3mlzyxbc6fq:5h66csimxvecntptd36qrn6jb7g7yfogpbgfz5pj765xn6wj3utq:2:3:1024 -? 2/3,2e61afd25d3ca76a0ee265dbc22ba5e9d844dc93ac5f2e37199a34fcbdcb5a52,3abc94a93a42d0eee5c8dda0315f9f1343e2ba36b552ab512c435fd4989c1ac6 -: URI:CHK:appba4iq3vzl267nedzhfpjgny:pt7gakv7red2udejvzdlirsgci44b6chzndv7aksbm62xm5p7dwa:2:3:4096 -? 2/3,2e61afd25d3ca76a0ee265dbc22ba5e9d844dc93ac5f2e37199a34fcbdcb5a52,77f3b5f9437275607c3b5eef5f67d18d6a9835155cfee75acd9d2c043dbb8528 -: URI:CHK:fmllehy7ds6zu2yufgamneuhnm:6q7tu3fwv2kqknrffnz3g3juc3eckjprj62hrvk2sz7no77q3fgq:2:3:8193 -? 2/3,2e61afd25d3ca76a0ee265dbc22ba5e9d844dc93ac5f2e37199a34fcbdcb5a52,80c58fc1932767b113819b033e1874d8de05e8809534590db9d01510abc8ed6b -: URI:CHK:cmpwnp5m42dvuzbiewpawtamqy:llxkfyxees27paf2sjyfvo5ppiw6mefcnlk4enrf3upzxfntzhca:2:3:2048 -? 2/3,2e61afd25d3ca76a0ee265dbc22ba5e9d844dc93ac5f2e37199a34fcbdcb5a52,8454e5bb164935d6a8da749d78cf91eb0556897d44273053b67777aed1d418e8 -: URI:CHK:53ev7ja4kbovkcfvrjctskexgm:nkutoibua7cnw42ysil3aynwrfhlqdayidotcisizu4loo6dmjka:2:3:131071 -? 2/3,2e61afd25d3ca76a0ee265dbc22ba5e9d844dc93ac5f2e37199a34fcbdcb5a52,8a6a6bff19a6f768864a69d6c6c97e6a25385793201eefe2505650978aef9193 -: URI:CHK:t5oode47lyq6gievqcjxedw43a:rkfywj4x64hnhn2ldq4asgquecuq7sbz2pmd3ig5oigd6j7jtoga:2:3:131073 -? 2/3,2e61afd25d3ca76a0ee265dbc22ba5e9d844dc93ac5f2e37199a34fcbdcb5a52,caccfbaa894a6cf59e3d677a146924604f8edcd07ed1d61cd1d1569ce4bf3ec5 -: URI:CHK:du6mixt5gg4xjhz6rbnxwrataa:sab52dir4oc7rytzthdmfr4f4yvs22ewpf7tt6k6ac4ngpmcvukq:2:3:8191 -? 2/3,43a1e8995c86ee7f559e5d187d1657296662ca180b394146a6e1dd0df9158a30,2edc986847e209b4016e141a6dc8716d3207350f416969382d431539bf292e4a -: URI:CHK:ld3thziutpaqv25nbtoqebhtru:ri2obsvzl2etyuv3qnchn2wvw5mh5rjuvbjaqkqbvyprn7xwamoq:2:3:1024 -? 2/3,43a1e8995c86ee7f559e5d187d1657296662ca180b394146a6e1dd0df9158a30,3abc94a93a42d0eee5c8dda0315f9f1343e2ba36b552ab512c435fd4989c1ac6 -: URI:CHK:pz3mga5pvqd4bpsz7pbyna2qtq:w3ptt7xwwgywod3u2rpgx7iggvm2kpwl7d2ryr3u7mcaxgqotlaq:2:3:4096 -? 2/3,43a1e8995c86ee7f559e5d187d1657296662ca180b394146a6e1dd0df9158a30,77f3b5f9437275607c3b5eef5f67d18d6a9835155cfee75acd9d2c043dbb8528 -: URI:CHK:mhrrcvp5qcxxlktw4hj6bqfnkq:jj76kg5tr7q5m5fracpquhozfq34ttv3uqclskvoeftvfbpjdyqq:2:3:8193 -? 2/3,43a1e8995c86ee7f559e5d187d1657296662ca180b394146a6e1dd0df9158a30,80c58fc1932767b113819b033e1874d8de05e8809534590db9d01510abc8ed6b -: URI:CHK:lmx2nobpjuljsz4guu7jl55mnu:pbl3dskb3qhmayowi5aoszhlorphm3xjl7xgofzosdbug2ap6h7a:2:3:2048 -? 2/3,43a1e8995c86ee7f559e5d187d1657296662ca180b394146a6e1dd0df9158a30,8454e5bb164935d6a8da749d78cf91eb0556897d44273053b67777aed1d418e8 -: URI:CHK:tykhb62tv32cxeb3t2ievvyiha:cbgakfrdmnuwwmumqtwg53gm6agujd56lep3rnf6glc4y5afqmia:2:3:131071 -? 2/3,43a1e8995c86ee7f559e5d187d1657296662ca180b394146a6e1dd0df9158a30,8a6a6bff19a6f768864a69d6c6c97e6a25385793201eefe2505650978aef9193 -: URI:CHK:3expkq235v4ugmard7riwcdcp4:sh2yanw7j4lg5hz7fna5n2tw4yigw3fk3qiinkmmdhz7wnitg6lq:2:3:131073 -? 2/3,43a1e8995c86ee7f559e5d187d1657296662ca180b394146a6e1dd0df9158a30,caccfbaa894a6cf59e3d677a146924604f8edcd07ed1d61cd1d1569ce4bf3ec5 -: URI:CHK:rubvwrpa4pgdmlpwlfefhkte24:bbqakzizzwk7wvelfhnvcsuxcuirzyplcqc5qqujrqfny3wrswna:2:3:8191 -? 2/3,be45cb2605bf36bebde684841a28f0fd43c69850a3dce5fedba69928ee3a8991,2edc986847e209b4016e141a6dc8716d3207350f416969382d431539bf292e4a -: URI:CHK:nvndhmpesezx4hr426lcrfopqy:gnx56qcnolpy2erpayo6lil5qzsomfbhv3quofsa654ks4sta7lq:2:3:1024 -? 2/3,be45cb2605bf36bebde684841a28f0fd43c69850a3dce5fedba69928ee3a8991,3abc94a93a42d0eee5c8dda0315f9f1343e2ba36b552ab512c435fd4989c1ac6 -: URI:CHK:zwbvg2jvulmv34ixtwwfgugk7m:cd2rwx4oo7fikddpjkhswnwofr4s6mj2texiuv34xp5722kpfyiq:2:3:4096 -? 2/3,be45cb2605bf36bebde684841a28f0fd43c69850a3dce5fedba69928ee3a8991,77f3b5f9437275607c3b5eef5f67d18d6a9835155cfee75acd9d2c043dbb8528 -: URI:CHK:t3m3hhlt3cbtobinz7bupsuphm:hrpiylbj3r4drrnupj7v54d4bwauz4s4llosxkawpm6fetwflqoa:2:3:8193 -? 2/3,be45cb2605bf36bebde684841a28f0fd43c69850a3dce5fedba69928ee3a8991,80c58fc1932767b113819b033e1874d8de05e8809534590db9d01510abc8ed6b -: URI:CHK:mbdlys4nttnwkehseindeoz4aa:jqutmu5qr2jppkdbbvlf7d33ugumaraoxi2mu6d4tocmbjnw5cqa:2:3:2048 -? 2/3,be45cb2605bf36bebde684841a28f0fd43c69850a3dce5fedba69928ee3a8991,8454e5bb164935d6a8da749d78cf91eb0556897d44273053b67777aed1d418e8 -: URI:CHK:y56rgygxem4dycq6jbqn6jdjdm:c3vw5i5avlm5fsqmathclfy3o7k4abiinp3lqzgvzndv6sbfawbq:2:3:131071 -? 2/3,be45cb2605bf36bebde684841a28f0fd43c69850a3dce5fedba69928ee3a8991,8a6a6bff19a6f768864a69d6c6c97e6a25385793201eefe2505650978aef9193 -: URI:CHK:wbpyopk35tm7z7foczudou4e2a:34wamdd2la4chje42ncfu7zj6vypwygmtpad6mfakbw3agycfgoa:2:3:131073 -? 2/3,be45cb2605bf36bebde684841a28f0fd43c69850a3dce5fedba69928ee3a8991,caccfbaa894a6cf59e3d677a146924604f8edcd07ed1d61cd1d1569ce4bf3ec5 -: URI:CHK:hatspmrspkgpedybcqrij62hj4:buo3twcstwlmasnjywjqvdk3r3auzkbwzrsuitzgucayw3tkskxq:2:3:8191 -? 2/3,cd82b3962c8796095a288e29ed94f98017cc681d17ea9ea8861c4034b13a3b6f,2edc986847e209b4016e141a6dc8716d3207350f416969382d431539bf292e4a -: URI:CHK:sxliom42kylmiggxfb2xzetj24:sqly2tkyr4kommagjccbjmswsnaioirorvqqtsucg6pmrqi4mpzq:2:3:1024 -? 2/3,cd82b3962c8796095a288e29ed94f98017cc681d17ea9ea8861c4034b13a3b6f,3abc94a93a42d0eee5c8dda0315f9f1343e2ba36b552ab512c435fd4989c1ac6 -: URI:CHK:u4m3m5gjga5tbiefdyxmhucina:tweiddczb6675d3vrkp4fuu5xrvml5xvzj322qyahr3camurw4ra:2:3:4096 -? 2/3,cd82b3962c8796095a288e29ed94f98017cc681d17ea9ea8861c4034b13a3b6f,77f3b5f9437275607c3b5eef5f67d18d6a9835155cfee75acd9d2c043dbb8528 -: URI:CHK:s2ixjohdpmwnr3c66tjfaflng4:p2i4pvbvf3z7aguin56uerbifvlhgzwhchkanhf6je4qclodd2jq:2:3:8193 -? 2/3,cd82b3962c8796095a288e29ed94f98017cc681d17ea9ea8861c4034b13a3b6f,80c58fc1932767b113819b033e1874d8de05e8809534590db9d01510abc8ed6b -: URI:CHK:367h226jbryhxhdouhcumna7cq:fhiul4h7pgwcrzdpluzhuxvedfijruxjaqbmpkwrh54xyhxljqga:2:3:2048 -? 2/3,cd82b3962c8796095a288e29ed94f98017cc681d17ea9ea8861c4034b13a3b6f,8454e5bb164935d6a8da749d78cf91eb0556897d44273053b67777aed1d418e8 -: URI:CHK:5jaiz63t6k7bktjwp4vbr7m2fq:wtpdpfokxepmokcpwtypg7fd7jmr73fffw5h3kbzb3fcjbrjgv6a:2:3:131071 -? 2/3,cd82b3962c8796095a288e29ed94f98017cc681d17ea9ea8861c4034b13a3b6f,8a6a6bff19a6f768864a69d6c6c97e6a25385793201eefe2505650978aef9193 -: URI:CHK:x4hpdnr5lrq7uqvm42mijivhle:oba26veckkqz23umeyp2cd5vaoock3d6uzhxxlwn2w2aztj4oi7q:2:3:131073 -? 2/3,cd82b3962c8796095a288e29ed94f98017cc681d17ea9ea8861c4034b13a3b6f,caccfbaa894a6cf59e3d677a146924604f8edcd07ed1d61cd1d1569ce4bf3ec5 -: URI:CHK:ovqefeaxuvitgzbzlpkdnmxo64:3y2jxab2o2iclexkqlxnlk4xfw6jvbikjh5ho3okdvpasv6cduuq:2:3:8191 -? 2/3,f39dac6cbaba535e2c207cd0cd8f154974223c848f727f98b3564cea569b41cf,2edc986847e209b4016e141a6dc8716d3207350f416969382d431539bf292e4a -: URI:CHK:d5upd7gpfxw62sppsntkw73svu:i4fr2ejfbkulxxim6j2tpduhhywr45edoooms7k5ou55q4be4lka:2:3:1024 -? 2/3,f39dac6cbaba535e2c207cd0cd8f154974223c848f727f98b3564cea569b41cf,3abc94a93a42d0eee5c8dda0315f9f1343e2ba36b552ab512c435fd4989c1ac6 -: URI:CHK:itfpgduf4gr5qonqisl4fupb4u:aynkfv4qke6jqpn6pe7jcad65az2wnxp6u3sxq3chsw2trzxtieq:2:3:4096 -? 2/3,f39dac6cbaba535e2c207cd0cd8f154974223c848f727f98b3564cea569b41cf,77f3b5f9437275607c3b5eef5f67d18d6a9835155cfee75acd9d2c043dbb8528 -: URI:CHK:2qkptbxaidg7vloyuhvyooflui:y2cxicn3skyne2jnwyah63wtngle2ffjn62xvkbr2xktbc3lreiq:2:3:8193 -? 2/3,f39dac6cbaba535e2c207cd0cd8f154974223c848f727f98b3564cea569b41cf,80c58fc1932767b113819b033e1874d8de05e8809534590db9d01510abc8ed6b -: URI:CHK:oyj7oyyiaihipvpxrav4np7474:snjty23mprvvoja3uwg2eu5ivz4ajt4zsvevfub2wozjjj55ytoq:2:3:2048 -? 2/3,f39dac6cbaba535e2c207cd0cd8f154974223c848f727f98b3564cea569b41cf,8454e5bb164935d6a8da749d78cf91eb0556897d44273053b67777aed1d418e8 -: URI:CHK:4qaxpw4bt36qmn3hy6cpkedyay:pjw5hyzdq3ebiykd2c7talnofrfh733w77tm6hafs22c7ov5e4ia:2:3:131071 -? 2/3,f39dac6cbaba535e2c207cd0cd8f154974223c848f727f98b3564cea569b41cf,8a6a6bff19a6f768864a69d6c6c97e6a25385793201eefe2505650978aef9193 -: URI:CHK:tcgo7l6rhvwm22y5bgko7c5zy4:by2qfi7uqiyeufy5d5kkcxmeqxeha7k7gtcwadtae5y6ahru3hya:2:3:131073 -? 2/3,f39dac6cbaba535e2c207cd0cd8f154974223c848f727f98b3564cea569b41cf,caccfbaa894a6cf59e3d677a146924604f8edcd07ed1d61cd1d1569ce4bf3ec5 -: URI:CHK:vnzr4qsjx6tgneo4y45vsqlgji:eyw6lmtxs73laj2ozuejm2ya6kdtrn6bgnsrg6miz34znxipsh4a:2:3:8191 -? 3/10,0c0beacef8877bbf2416eb00f2b5dc96354e26dd1df5517320459b1236860f8c,2edc986847e209b4016e141a6dc8716d3207350f416969382d431539bf292e4a -: URI:CHK:dx7tvyr2fc4u7lxjc6kehq2svq:tiy4qh2g6lqejxcaym3rr7ymkdkinn4qised6kgxloj7sptsqu4a:3:10:1024 -? 3/10,0c0beacef8877bbf2416eb00f2b5dc96354e26dd1df5517320459b1236860f8c,3abc94a93a42d0eee5c8dda0315f9f1343e2ba36b552ab512c435fd4989c1ac6 -: URI:CHK:yo75evk4cte3b7rdw72zxvl5ye:ex6h7ff7nclucjtsqwgwu33qgmb67t4ezbrki4zbgurwn2ct6bbq:3:10:4096 -? 3/10,0c0beacef8877bbf2416eb00f2b5dc96354e26dd1df5517320459b1236860f8c,77f3b5f9437275607c3b5eef5f67d18d6a9835155cfee75acd9d2c043dbb8528 -: URI:CHK:tq5s76lgxpirmo57bs2vom3dfi:c4drrjewq2vj7mu3pie4tmhkbubbr7jxdy7lb6dytjvxrxvglldq:3:10:8193 -? 3/10,0c0beacef8877bbf2416eb00f2b5dc96354e26dd1df5517320459b1236860f8c,80c58fc1932767b113819b033e1874d8de05e8809534590db9d01510abc8ed6b -: URI:CHK:sojahkxprwqsymyxj4cbkvah6m:sveaqceazsm7olekkh7vk75kywi2mlqymok4qzjmrfkmhn3wygha:3:10:2048 -? 3/10,0c0beacef8877bbf2416eb00f2b5dc96354e26dd1df5517320459b1236860f8c,8454e5bb164935d6a8da749d78cf91eb0556897d44273053b67777aed1d418e8 -: URI:CHK:7mdyutwroo5ebxt5rvwlrmuoza:rlwrxlb6kupswgmil4r5arm3n24qq7l7nniiebelng5jve7pc74q:3:10:131071 -? 3/10,0c0beacef8877bbf2416eb00f2b5dc96354e26dd1df5517320459b1236860f8c,8a6a6bff19a6f768864a69d6c6c97e6a25385793201eefe2505650978aef9193 -: URI:CHK:p4do46g2k6mwkm4b5gybbf4qia:anhyucmmnsxyccwpvjc3cbncii652s7qglfxlswpp6lllya35u2q:3:10:131073 -? 3/10,0c0beacef8877bbf2416eb00f2b5dc96354e26dd1df5517320459b1236860f8c,caccfbaa894a6cf59e3d677a146924604f8edcd07ed1d61cd1d1569ce4bf3ec5 -: URI:CHK:eofaxrbscvampoxze3revacvgm:hnelkvqvffvaba7z2q2ryxevqyyu7hmitkldo6gc63oburi6it5q:3:10:8191 -? 3/10,2e61afd25d3ca76a0ee265dbc22ba5e9d844dc93ac5f2e37199a34fcbdcb5a52,2edc986847e209b4016e141a6dc8716d3207350f416969382d431539bf292e4a -: URI:CHK:waolb5bl2gcd5hbbiurwwipdlq:bs7ixrhwbivs2krsv2aicv5p2t6iiluj4angimza3zr3h2my726a:3:10:1024 -? 3/10,2e61afd25d3ca76a0ee265dbc22ba5e9d844dc93ac5f2e37199a34fcbdcb5a52,3abc94a93a42d0eee5c8dda0315f9f1343e2ba36b552ab512c435fd4989c1ac6 -: URI:CHK:wwnxsu5dp5y42dbdgbvpxcju6e:qongonvlsucqyndshzqbmipcqzje7pz7g6l7457zrdm2ldf5apha:3:10:4096 -? 3/10,2e61afd25d3ca76a0ee265dbc22ba5e9d844dc93ac5f2e37199a34fcbdcb5a52,77f3b5f9437275607c3b5eef5f67d18d6a9835155cfee75acd9d2c043dbb8528 -: URI:CHK:pxxzoxraxumzo5v26la3y2kony:fkv6rny5mllqsudxcimodqt43cmxowap6m7onsvzjocxcwkodx4q:3:10:8193 -? 3/10,2e61afd25d3ca76a0ee265dbc22ba5e9d844dc93ac5f2e37199a34fcbdcb5a52,80c58fc1932767b113819b033e1874d8de05e8809534590db9d01510abc8ed6b -: URI:CHK:4cuh76ehoxr7at5irtqazsgdvy:3oitt3kd3wklugjilii2vests73ci6el5tpnqhso23y5pjjetbta:3:10:2048 -? 3/10,2e61afd25d3ca76a0ee265dbc22ba5e9d844dc93ac5f2e37199a34fcbdcb5a52,8454e5bb164935d6a8da749d78cf91eb0556897d44273053b67777aed1d418e8 -: URI:CHK:u2ooocrxsdb3ts3wjkybbfuoeq:2q2g2g4mcqaj66ipbr5p2eotjqzpjfxhli6zrni2bje7luboxhha:3:10:131071 -? 3/10,2e61afd25d3ca76a0ee265dbc22ba5e9d844dc93ac5f2e37199a34fcbdcb5a52,8a6a6bff19a6f768864a69d6c6c97e6a25385793201eefe2505650978aef9193 -: URI:CHK:qfwww2o2kn3hz2iywqzxgqy24e:2pzpb6pdnqcgz4thlbtwx2dmvva6mvqvwc7p327c3gp2a2nkg2tq:3:10:131073 -? 3/10,2e61afd25d3ca76a0ee265dbc22ba5e9d844dc93ac5f2e37199a34fcbdcb5a52,caccfbaa894a6cf59e3d677a146924604f8edcd07ed1d61cd1d1569ce4bf3ec5 -: URI:CHK:az4h34ljherrtom5cy3bmihr4m:7apbwgroxfzsxszxe2626ih6onueaqgmcb4wcmrgzv3qadfazzla:3:10:8191 -? 3/10,43a1e8995c86ee7f559e5d187d1657296662ca180b394146a6e1dd0df9158a30,2edc986847e209b4016e141a6dc8716d3207350f416969382d431539bf292e4a -: URI:CHK:x55owxzhsezfoayaxe7jpwnove:vawdgtqpxyntgy5i2po2twgelrynkfcjgwm7publnlbdp7hpqmfa:3:10:1024 -? 3/10,43a1e8995c86ee7f559e5d187d1657296662ca180b394146a6e1dd0df9158a30,3abc94a93a42d0eee5c8dda0315f9f1343e2ba36b552ab512c435fd4989c1ac6 -: URI:CHK:fe64krzyaeff3d4teunjbetkzy:27hrywwaffqiqcgfkmzwbot3iamotr3bey2l5kaladmdmxuaz5ka:3:10:4096 -? 3/10,43a1e8995c86ee7f559e5d187d1657296662ca180b394146a6e1dd0df9158a30,77f3b5f9437275607c3b5eef5f67d18d6a9835155cfee75acd9d2c043dbb8528 -: URI:CHK:s7x2mswk33trx5lkgkvbdssihi:y5uabvo36gvtfi4rdjnaiyg3xlfh5bx5hpswivpd3zxzjupjs5qa:3:10:8193 -? 3/10,43a1e8995c86ee7f559e5d187d1657296662ca180b394146a6e1dd0df9158a30,80c58fc1932767b113819b033e1874d8de05e8809534590db9d01510abc8ed6b -: URI:CHK:urr3adnkm3lsisfshb4tgchvie:nrfj2cbr56abogv4pvmgb32il5zckaun3j2uv45aa76wtdrqwitq:3:10:2048 -? 3/10,43a1e8995c86ee7f559e5d187d1657296662ca180b394146a6e1dd0df9158a30,8454e5bb164935d6a8da749d78cf91eb0556897d44273053b67777aed1d418e8 -: URI:CHK:acepdcmzrafgvur6jmcgywuecm:p64sqhkx5272djlrvyvu44odn4s2fw4afslqcqlxu65issvtsusq:3:10:131071 -? 3/10,43a1e8995c86ee7f559e5d187d1657296662ca180b394146a6e1dd0df9158a30,8a6a6bff19a6f768864a69d6c6c97e6a25385793201eefe2505650978aef9193 -: URI:CHK:plgiizqfrqsor5y7up2js5ovg4:cpj2kpiojy6an4spsppcxrz6uchywccw2jgzmch3lo7u6cvxnlda:3:10:131073 -? 3/10,43a1e8995c86ee7f559e5d187d1657296662ca180b394146a6e1dd0df9158a30,caccfbaa894a6cf59e3d677a146924604f8edcd07ed1d61cd1d1569ce4bf3ec5 -: URI:CHK:7refz6urxegdrnczkoewjjikvq:wd2uftpusakntdgzcnvvbrnxr47hsylfxawvt73xapndf2lrevfq:3:10:8191 -? 3/10,be45cb2605bf36bebde684841a28f0fd43c69850a3dce5fedba69928ee3a8991,2edc986847e209b4016e141a6dc8716d3207350f416969382d431539bf292e4a -: URI:CHK:2kia6xptokio2pksshbgmhw4vu:trstbnyzthh3ncdciksentnxaqebnjv76lujt37td3h62yk7wd2a:3:10:1024 -? 3/10,be45cb2605bf36bebde684841a28f0fd43c69850a3dce5fedba69928ee3a8991,3abc94a93a42d0eee5c8dda0315f9f1343e2ba36b552ab512c435fd4989c1ac6 -: URI:CHK:7767v4lygmnqbrjaiczlsf7ehy:ig3ev7lcogswoyfwelcg4ap5xh7rzhjpn645pv7u6xcjxlzmbhta:3:10:4096 -? 3/10,be45cb2605bf36bebde684841a28f0fd43c69850a3dce5fedba69928ee3a8991,77f3b5f9437275607c3b5eef5f67d18d6a9835155cfee75acd9d2c043dbb8528 -: URI:CHK:66c63jd3qlgpmca5eyc7m6unqy:4lh75xz5tksdvx3xt5pmeoy3h3amaklgcsdtcgvjncx26rd6ysfq:3:10:8193 -? 3/10,be45cb2605bf36bebde684841a28f0fd43c69850a3dce5fedba69928ee3a8991,80c58fc1932767b113819b033e1874d8de05e8809534590db9d01510abc8ed6b -: URI:CHK:usekh3kmiilhnx3ciy4gknu4fe:steov6hzrp24t5bhqjb4an5buspol4phbhhjekvjmqf2j5f4pnuq:3:10:2048 -? 3/10,be45cb2605bf36bebde684841a28f0fd43c69850a3dce5fedba69928ee3a8991,8454e5bb164935d6a8da749d78cf91eb0556897d44273053b67777aed1d418e8 -: URI:CHK:2lzrturd4ni5idui57aiaminiy:qxaigpe2tyriywdgykw5i3jbfuxfoscgg7cqxkfkvl4276iys5vq:3:10:131071 -? 3/10,be45cb2605bf36bebde684841a28f0fd43c69850a3dce5fedba69928ee3a8991,8a6a6bff19a6f768864a69d6c6c97e6a25385793201eefe2505650978aef9193 -: URI:CHK:5kt7pcf7lhddi4t6rgvg4wimye:zhvvii4ym22svvm6dqcbjubtqtxiuicsu3m73stkbgi6aerv4vbq:3:10:131073 -? 3/10,be45cb2605bf36bebde684841a28f0fd43c69850a3dce5fedba69928ee3a8991,caccfbaa894a6cf59e3d677a146924604f8edcd07ed1d61cd1d1569ce4bf3ec5 -: URI:CHK:v3ggpeotvq7yasm7xv7njll4ua:suq7wstq553hzexpkrt6p2catu5vxffq2vv2bzgp4wxqxsrd3mha:3:10:8191 -? 3/10,cd82b3962c8796095a288e29ed94f98017cc681d17ea9ea8861c4034b13a3b6f,2edc986847e209b4016e141a6dc8716d3207350f416969382d431539bf292e4a -: URI:CHK:4yxb4zy46aj7jynfxmxsuzoyvi:2r7rzhsw7klo3nsm7nrgv7ixfsusap6iwdyzx422vnkvuhb3k3tq:3:10:1024 -? 3/10,cd82b3962c8796095a288e29ed94f98017cc681d17ea9ea8861c4034b13a3b6f,3abc94a93a42d0eee5c8dda0315f9f1343e2ba36b552ab512c435fd4989c1ac6 -: URI:CHK:s2i62qbkrqsipvtk5inri5gm2q:5fi5h47qtagp7ds3qzhp7exdu4ftldaddmefhthca7v5iqvsvn5q:3:10:4096 -? 3/10,cd82b3962c8796095a288e29ed94f98017cc681d17ea9ea8861c4034b13a3b6f,77f3b5f9437275607c3b5eef5f67d18d6a9835155cfee75acd9d2c043dbb8528 -: URI:CHK:du5bm55q3wescfhtizl67t4nd4:egmwrekadv6uc53zjgyq6da5flzd4ml6mgjhnuihn7nthmlkzhvq:3:10:8193 -? 3/10,cd82b3962c8796095a288e29ed94f98017cc681d17ea9ea8861c4034b13a3b6f,80c58fc1932767b113819b033e1874d8de05e8809534590db9d01510abc8ed6b -: URI:CHK:6w4d7jjpxdmwedvowsjcze6uzm:iywt6ln36v3bdfmzcxofh44tp24osotxypwes6teitlzio3m72ba:3:10:2048 -? 3/10,cd82b3962c8796095a288e29ed94f98017cc681d17ea9ea8861c4034b13a3b6f,8454e5bb164935d6a8da749d78cf91eb0556897d44273053b67777aed1d418e8 -: URI:CHK:j444wto2zemwoa3fv6ybqvciky:ap7zqb4zuffilxberhbb4jp4dh7ymugkrgnrf5k6uhwsxcxdppcq:3:10:131071 -? 3/10,cd82b3962c8796095a288e29ed94f98017cc681d17ea9ea8861c4034b13a3b6f,8a6a6bff19a6f768864a69d6c6c97e6a25385793201eefe2505650978aef9193 -: URI:CHK:o7nfpsjh4ubatgoezv2ovy5roq:qntmp2flklkymn54bjburj6amrr465uzqrtlneibkir2feh34f6a:3:10:131073 -? 3/10,cd82b3962c8796095a288e29ed94f98017cc681d17ea9ea8861c4034b13a3b6f,caccfbaa894a6cf59e3d677a146924604f8edcd07ed1d61cd1d1569ce4bf3ec5 -: URI:CHK:3jx4hsxurmutlzshxircbzg4wy:wtmbkrrvf3z4bg3jt4lpml5bc246tvhguwsrrzltcqcgd7um7dlq:3:10:8191 -? 3/10,f39dac6cbaba535e2c207cd0cd8f154974223c848f727f98b3564cea569b41cf,2edc986847e209b4016e141a6dc8716d3207350f416969382d431539bf292e4a -: URI:CHK:7lsdyuu2ka2enhnyug4ytklhzi:pk2h7sbj3sjrauwskmvtbacyp3i57awiofg4uxxsbvfgxfibtoha:3:10:1024 -? 3/10,f39dac6cbaba535e2c207cd0cd8f154974223c848f727f98b3564cea569b41cf,3abc94a93a42d0eee5c8dda0315f9f1343e2ba36b552ab512c435fd4989c1ac6 -: URI:CHK:sg35empm5gtsica4j4tsdhjihy:sqehx4q76sx4npafchg2u6lntgwm3jzqv3in2rbg7q6atr3lxdua:3:10:4096 -? 3/10,f39dac6cbaba535e2c207cd0cd8f154974223c848f727f98b3564cea569b41cf,77f3b5f9437275607c3b5eef5f67d18d6a9835155cfee75acd9d2c043dbb8528 -: URI:CHK:et5wbi3r6aufs33azuqko672k4:ahkwexqrtrkyyy4c3ixsmsz6aqhtyctynz2m6qgsfx6nk7jzn2oq:3:10:8193 -? 3/10,f39dac6cbaba535e2c207cd0cd8f154974223c848f727f98b3564cea569b41cf,80c58fc1932767b113819b033e1874d8de05e8809534590db9d01510abc8ed6b -: URI:CHK:s77i5y6ckuqgb36sihcsnutavu:idmwswj5p7z5w5elulxu5t6ulb7zt7ovaktrcypbq7k24ef5ekva:3:10:2048 -? 3/10,f39dac6cbaba535e2c207cd0cd8f154974223c848f727f98b3564cea569b41cf,8454e5bb164935d6a8da749d78cf91eb0556897d44273053b67777aed1d418e8 -: URI:CHK:3eirnemfrtnvkczhthm3ngxg2i:3uq3x2uaskwsykos2h7of6iwtyfp6nabgibk3ockphhdhrepcxia:3:10:131071 -? 3/10,f39dac6cbaba535e2c207cd0cd8f154974223c848f727f98b3564cea569b41cf,8a6a6bff19a6f768864a69d6c6c97e6a25385793201eefe2505650978aef9193 -: URI:CHK:uuwk7eo4w3nfxz4hunif5vcc5e:tneaxz6kr6cardyconuslivifzdb67re4stmtfvrd6dp7bbk4iea:3:10:131073 -? 3/10,f39dac6cbaba535e2c207cd0cd8f154974223c848f727f98b3564cea569b41cf,caccfbaa894a6cf59e3d677a146924604f8edcd07ed1d61cd1d1569ce4bf3ec5 -: URI:CHK:4selpjhb6m2jqpixg3fzfti2yi:mvkil3khmwm5om7ucneo675chan4vx7cja27m55qj4pktvbv4snq:3:10:8191 -? 71/255,0c0beacef8877bbf2416eb00f2b5dc96354e26dd1df5517320459b1236860f8c,2edc986847e209b4016e141a6dc8716d3207350f416969382d431539bf292e4a -: URI:CHK:2wvrzaobtfkoiqeqwryjzvxatq:7zzaycbdhepzaqtdrku2sbb572h64ggwstu47osa2gn4ol3eglmq:71:255:1024 -? 71/255,0c0beacef8877bbf2416eb00f2b5dc96354e26dd1df5517320459b1236860f8c,3abc94a93a42d0eee5c8dda0315f9f1343e2ba36b552ab512c435fd4989c1ac6 -: URI:CHK:7wi5xn2uwerkjzqs5ndn4eimti:llrws4uig2fpi3rtla6lk7m4t6zudjhszdfjgqtihghfrvs2robq:71:255:4096 -? 71/255,0c0beacef8877bbf2416eb00f2b5dc96354e26dd1df5517320459b1236860f8c,77f3b5f9437275607c3b5eef5f67d18d6a9835155cfee75acd9d2c043dbb8528 -: URI:CHK:ybc7ijc45jwunkecolewxrmyyu:d4m3oh7vkyavap7rditzuarwjmmkot7iz4rhqpsks37sxq5gmcpq:71:255:8193 -? 71/255,0c0beacef8877bbf2416eb00f2b5dc96354e26dd1df5517320459b1236860f8c,80c58fc1932767b113819b033e1874d8de05e8809534590db9d01510abc8ed6b -: URI:CHK:ts5yfcjoq3jwahsp4yuqahqhq4:bbeyk64tjbj732zga3gsz46n24lgxte6ydyni3tphoircaifiiya:71:255:2048 -? 71/255,0c0beacef8877bbf2416eb00f2b5dc96354e26dd1df5517320459b1236860f8c,8454e5bb164935d6a8da749d78cf91eb0556897d44273053b67777aed1d418e8 -: URI:CHK:hzhnspwdfp55wt5gw6fiixujlq:eok5auvypt77mv2ic2225shisclyuhhmtyssh5mwpyrdtgfndyua:71:255:131071 -? 71/255,0c0beacef8877bbf2416eb00f2b5dc96354e26dd1df5517320459b1236860f8c,8a6a6bff19a6f768864a69d6c6c97e6a25385793201eefe2505650978aef9193 -: URI:CHK:mxpwc2imfzik6lxb6vyg6h7kv4:2ai2kkkwscucexcplsp52au62xdqclngfixippolxul2i4u3tgpa:71:255:131073 -? 71/255,0c0beacef8877bbf2416eb00f2b5dc96354e26dd1df5517320459b1236860f8c,caccfbaa894a6cf59e3d677a146924604f8edcd07ed1d61cd1d1569ce4bf3ec5 -: URI:CHK:rc7jm3hj4qwt73wk7jmpabofdy:ohfz4s5ezjnan33wef3zav4rkqnb67ikbzyz63ctlofnkmwfujna:71:255:8191 -? 71/255,2e61afd25d3ca76a0ee265dbc22ba5e9d844dc93ac5f2e37199a34fcbdcb5a52,2edc986847e209b4016e141a6dc8716d3207350f416969382d431539bf292e4a -: URI:CHK:e25n6qpypv3n6l3pwnkydfceay:rbridd4pcj47dhnal3jg36zrhjnus4egzn2lb6myfpegj6rqii7a:71:255:1024 -? 71/255,2e61afd25d3ca76a0ee265dbc22ba5e9d844dc93ac5f2e37199a34fcbdcb5a52,3abc94a93a42d0eee5c8dda0315f9f1343e2ba36b552ab512c435fd4989c1ac6 -: URI:CHK:dyhyxipdc5p7s7x2ntmxdjsrb4:bk4c6hjk2qvvcxsen76dqkvn2ckvehv4z5ogzlv2csyjubbwxe2q:71:255:4096 -? 71/255,2e61afd25d3ca76a0ee265dbc22ba5e9d844dc93ac5f2e37199a34fcbdcb5a52,77f3b5f9437275607c3b5eef5f67d18d6a9835155cfee75acd9d2c043dbb8528 -: URI:CHK:ggpxln6r5e6jmcnfr723awlah4:e6ypwv2uo37tcjn23mp3rerjrula4sob2wxylhz4ttyf45kfoa6a:71:255:8193 -? 71/255,2e61afd25d3ca76a0ee265dbc22ba5e9d844dc93ac5f2e37199a34fcbdcb5a52,80c58fc1932767b113819b033e1874d8de05e8809534590db9d01510abc8ed6b -: URI:CHK:dcde7b4d6mxxf5kvece5w6nxry:rxteuxaxpt4kyd6ym4grkkfiwgd2c6soet3cx46arvubbtmmjkbq:71:255:2048 -? 71/255,2e61afd25d3ca76a0ee265dbc22ba5e9d844dc93ac5f2e37199a34fcbdcb5a52,8454e5bb164935d6a8da749d78cf91eb0556897d44273053b67777aed1d418e8 -: URI:CHK:vulnb4ns3vovkvlu3536vxsvkm:6vww2a2eypjb4ctdd2uh4zylnxb575edg3gsiobgirgenhw6f3ja:71:255:131071 -? 71/255,2e61afd25d3ca76a0ee265dbc22ba5e9d844dc93ac5f2e37199a34fcbdcb5a52,8a6a6bff19a6f768864a69d6c6c97e6a25385793201eefe2505650978aef9193 -: URI:CHK:6hk7efj3jyb7g6ewpb6alua6ve:i6oq2hul7a2yoinddn2nfwfarh6ytzu6ernedkevp5vgtyms7dja:71:255:131073 -? 71/255,2e61afd25d3ca76a0ee265dbc22ba5e9d844dc93ac5f2e37199a34fcbdcb5a52,caccfbaa894a6cf59e3d677a146924604f8edcd07ed1d61cd1d1569ce4bf3ec5 -: URI:CHK:ehixpgvrjy6yr2wqlw6n3cqcui:rrrejfiwxefb2u6uicmxx3qr7j5z3ox5n7boklrly25op5iwflta:71:255:8191 -? 71/255,43a1e8995c86ee7f559e5d187d1657296662ca180b394146a6e1dd0df9158a30,2edc986847e209b4016e141a6dc8716d3207350f416969382d431539bf292e4a -: URI:CHK:wxmzfzvgaqm3xfzgaqrlndgolm:6t3tqcphsutigxnnzyp2xx3afklcpeytnh543mpqptquvqqahwea:71:255:1024 -? 71/255,43a1e8995c86ee7f559e5d187d1657296662ca180b394146a6e1dd0df9158a30,3abc94a93a42d0eee5c8dda0315f9f1343e2ba36b552ab512c435fd4989c1ac6 -: URI:CHK:r3q3q4f4zbsipxauq6knb5dfoq:lkae5mflammzxkoc7gllt6qqyg5gqdxeejmxtgcscblqh5tv5obq:71:255:4096 -? 71/255,43a1e8995c86ee7f559e5d187d1657296662ca180b394146a6e1dd0df9158a30,77f3b5f9437275607c3b5eef5f67d18d6a9835155cfee75acd9d2c043dbb8528 -: URI:CHK:l63y2aolbcvdcm5s4i2xa6mlbq:pjlht25xggvtstoynmi5tq3x3wvgtzmjym3lcpulh7keg6mc366q:71:255:8193 -? 71/255,43a1e8995c86ee7f559e5d187d1657296662ca180b394146a6e1dd0df9158a30,80c58fc1932767b113819b033e1874d8de05e8809534590db9d01510abc8ed6b -: URI:CHK:rhsd2socwnvt6nrnhtwpic23am:x52gdwnvpuh63oo7l2tclnclt64k7u4upyadaaifpvlnirnqy7lq:71:255:2048 -? 71/255,43a1e8995c86ee7f559e5d187d1657296662ca180b394146a6e1dd0df9158a30,8454e5bb164935d6a8da749d78cf91eb0556897d44273053b67777aed1d418e8 -: URI:CHK:zwes34ucxchxajmf4fjaw5lzeu:eh3hvvilxud5za7j4x6u7wnvarmtsuj2f2g5mdqibrys2yd7qb6a:71:255:131071 -? 71/255,43a1e8995c86ee7f559e5d187d1657296662ca180b394146a6e1dd0df9158a30,8a6a6bff19a6f768864a69d6c6c97e6a25385793201eefe2505650978aef9193 -: URI:CHK:eootdbcym74eidk3a2jrzag42i:cityydyejtqqjqdfnb4o3f64f46udr6hxzzm3rdyabh3o3buvcha:71:255:131073 -? 71/255,43a1e8995c86ee7f559e5d187d1657296662ca180b394146a6e1dd0df9158a30,caccfbaa894a6cf59e3d677a146924604f8edcd07ed1d61cd1d1569ce4bf3ec5 -: URI:CHK:inm7itgqlu5djoeqeud55rkbka:jxacqjjtdxeevrntievnwa2kq452vsm3ur3sb3z32tr7uuw4fpiq:71:255:8191 -? 71/255,be45cb2605bf36bebde684841a28f0fd43c69850a3dce5fedba69928ee3a8991,2edc986847e209b4016e141a6dc8716d3207350f416969382d431539bf292e4a -: URI:CHK:khbldourqhs52sky6pnolmaiyy:qq4ixgnxeauqmiaftuvb6spviyn4xubdoycetc62ckjq26ycx22q:71:255:1024 -? 71/255,be45cb2605bf36bebde684841a28f0fd43c69850a3dce5fedba69928ee3a8991,3abc94a93a42d0eee5c8dda0315f9f1343e2ba36b552ab512c435fd4989c1ac6 -: URI:CHK:e4etimwsvsbgwlamrcmpa3bxwq:pdjv5a3wrc4licma2tlmfpcp6maawimzqlkawva2plnmwo5gwlna:71:255:4096 -? 71/255,be45cb2605bf36bebde684841a28f0fd43c69850a3dce5fedba69928ee3a8991,77f3b5f9437275607c3b5eef5f67d18d6a9835155cfee75acd9d2c043dbb8528 -: URI:CHK:zpoi7kp6bqhbuthfhnzr7lrnza:xlumj77xnyz2ddbfviww2fjyca55sx5oj2hbqx6yiwq3eqorkytq:71:255:8193 -? 71/255,be45cb2605bf36bebde684841a28f0fd43c69850a3dce5fedba69928ee3a8991,80c58fc1932767b113819b033e1874d8de05e8809534590db9d01510abc8ed6b -: URI:CHK:rtdvpxir3vbvq44mhoepfz7bvi:prpry2f452mrtkarcf7n6vpjw64hhyeshe5srrwzk42q5dd3iczq:71:255:2048 -? 71/255,be45cb2605bf36bebde684841a28f0fd43c69850a3dce5fedba69928ee3a8991,8454e5bb164935d6a8da749d78cf91eb0556897d44273053b67777aed1d418e8 -: URI:CHK:qnm225ycss3xthcme232ivbxd4:nxhjpm6rfztd2v5ucg476zphaps3xcfzfmaxsxlr25mvz2funi4a:71:255:131071 -? 71/255,be45cb2605bf36bebde684841a28f0fd43c69850a3dce5fedba69928ee3a8991,8a6a6bff19a6f768864a69d6c6c97e6a25385793201eefe2505650978aef9193 -: URI:CHK:yrsvl5gbjqeuiaqr76lnzdnahq:sxheuay55a245bcy4b5gmyhteakwkbrixmae6gnilhekvzmzwxda:71:255:131073 -? 71/255,be45cb2605bf36bebde684841a28f0fd43c69850a3dce5fedba69928ee3a8991,caccfbaa894a6cf59e3d677a146924604f8edcd07ed1d61cd1d1569ce4bf3ec5 -: URI:CHK:gzan4jdk5xfrp2q6chohk3wiv4:a4tcumb7vz7wms3a6ibaxm4u7jorks5thnhnfs5ajdpy4fwdgyqq:71:255:8191 -? 71/255,cd82b3962c8796095a288e29ed94f98017cc681d17ea9ea8861c4034b13a3b6f,2edc986847e209b4016e141a6dc8716d3207350f416969382d431539bf292e4a -: URI:CHK:q53ik6mdwijpptijuzgt6beuh4:rcfwmqzfuw5xs3b3a5mc6blzdln6vw3ojz6obz24jxb4rkukroqa:71:255:1024 -? 71/255,cd82b3962c8796095a288e29ed94f98017cc681d17ea9ea8861c4034b13a3b6f,3abc94a93a42d0eee5c8dda0315f9f1343e2ba36b552ab512c435fd4989c1ac6 -: URI:CHK:turjtbsb74zqbilacgdarkwxka:kusxr4ooeyyg5nofyoccufnjgxi3em2jm43ox3kur3m3ctv43kkq:71:255:4096 -? 71/255,cd82b3962c8796095a288e29ed94f98017cc681d17ea9ea8861c4034b13a3b6f,77f3b5f9437275607c3b5eef5f67d18d6a9835155cfee75acd9d2c043dbb8528 -: URI:CHK:utveoddo7fuhcpspizfdcynnn4:f5wiofdi4ehjwsdutwcnmhhdnohnrbhbbruxmxavplt2doxaks4a:71:255:8193 -? 71/255,cd82b3962c8796095a288e29ed94f98017cc681d17ea9ea8861c4034b13a3b6f,80c58fc1932767b113819b033e1874d8de05e8809534590db9d01510abc8ed6b -: URI:CHK:2gad4sxrilmhighbreeoy4vugu:qyoa2bnpqd6xludkft5jhsbzaoq3mowx72tp7k5yz3j3ydc7depa:71:255:2048 -? 71/255,cd82b3962c8796095a288e29ed94f98017cc681d17ea9ea8861c4034b13a3b6f,8454e5bb164935d6a8da749d78cf91eb0556897d44273053b67777aed1d418e8 -: URI:CHK:f2kefs2nfjjvcgsomrripeebli:3zvxpr2kcqi6npf3zu4lzi3ywzypebg5tmvzf5j3ifrhjtyjbioq:71:255:131071 -? 71/255,cd82b3962c8796095a288e29ed94f98017cc681d17ea9ea8861c4034b13a3b6f,8a6a6bff19a6f768864a69d6c6c97e6a25385793201eefe2505650978aef9193 -: URI:CHK:twi4p7gxwecvr5ypoa3i2mtrru:kcurtyehuorsxfqruoddhwenjufazrnjn7daxa2rexc72i7hr3jq:71:255:131073 -? 71/255,cd82b3962c8796095a288e29ed94f98017cc681d17ea9ea8861c4034b13a3b6f,caccfbaa894a6cf59e3d677a146924604f8edcd07ed1d61cd1d1569ce4bf3ec5 -: URI:CHK:bzenvzdzxlbysdvxy2fdcwzkp4:yagzttmvppme5rm5xvkebk7zerz5evkcrxoafjotozeuou5l6oza:71:255:8191 -? 71/255,f39dac6cbaba535e2c207cd0cd8f154974223c848f727f98b3564cea569b41cf,2edc986847e209b4016e141a6dc8716d3207350f416969382d431539bf292e4a -: URI:CHK:zwtwx7r77yseqnx6gjqnzbuu4i:ie237xmuehuz6iarwfx7ih44d6zipwqwtecck3ybmdwylqczasua:71:255:1024 -? 71/255,f39dac6cbaba535e2c207cd0cd8f154974223c848f727f98b3564cea569b41cf,3abc94a93a42d0eee5c8dda0315f9f1343e2ba36b552ab512c435fd4989c1ac6 -: URI:CHK:22lyxndvtgmgjofwsgr2r4xdka:tyw2xibpmksbyeazzqrpd4jkvoycxj45mbe22mmi4wgzrt4dvlfq:71:255:4096 -? 71/255,f39dac6cbaba535e2c207cd0cd8f154974223c848f727f98b3564cea569b41cf,77f3b5f9437275607c3b5eef5f67d18d6a9835155cfee75acd9d2c043dbb8528 -: URI:CHK:mihdgaisquzpfy26sfkzi4iuwa:5lrp47rkwoxc3pwbi7irpruzrec6672hrmva47owcygce3exycoa:71:255:8193 -? 71/255,f39dac6cbaba535e2c207cd0cd8f154974223c848f727f98b3564cea569b41cf,80c58fc1932767b113819b033e1874d8de05e8809534590db9d01510abc8ed6b -: URI:CHK:wu6hee63ij6hickphc5pus4jry:rei6yq4qqr6wa7bxt2udxcpqbzdn7zhwupor66iyj4sjphw4dqwa:71:255:2048 -? 71/255,f39dac6cbaba535e2c207cd0cd8f154974223c848f727f98b3564cea569b41cf,8454e5bb164935d6a8da749d78cf91eb0556897d44273053b67777aed1d418e8 -: URI:CHK:l3yx4gafcpaoctzfhiykbqwg7e:o6pgeihvxatl5z4sae6xyr3vjlqwinjvm4orj3wklzo7atoixciq:71:255:131071 -? 71/255,f39dac6cbaba535e2c207cd0cd8f154974223c848f727f98b3564cea569b41cf,8a6a6bff19a6f768864a69d6c6c97e6a25385793201eefe2505650978aef9193 -: URI:CHK:rxarznyu7f7n2bfdk73q7ko77y:irlqlbsduaaotjw4hn4yg5n4dve36rpindy5zgtc7zf4yl2giy2a:71:255:131073 -? 71/255,f39dac6cbaba535e2c207cd0cd8f154974223c848f727f98b3564cea569b41cf,caccfbaa894a6cf59e3d677a146924604f8edcd07ed1d61cd1d1569ce4bf3ec5 -: URI:CHK:aikq3v6hwp5bad5mv6myoc76ze:ufih4yqr7jtjptsrfvdhcf5pq3x7drtxzpddzyatvex2sreed2dq:71:255:8191 diff --git a/integration/test_vectors.py b/integration/test_vectors.py index 252360fa2..e1eb33644 100644 --- a/integration/test_vectors.py +++ b/integration/test_vectors.py @@ -4,7 +4,7 @@ Verify certain results against test vectors with well-known results. from __future__ import annotations -from typing import AsyncGenerator +from typing import AsyncGenerator, Iterator from hashlib import sha256 from itertools import product from yaml import safe_dump @@ -23,26 +23,39 @@ def hexdigest(bs: bytes) -> str: return sha256(bs).hexdigest() +# Just a couple convergence secrets. The only thing we do with this value is +# feed it into a tagged hash. It certainly makes a difference to the output +# but the hash should destroy any structure in the input so it doesn't seem +# like there's a reason to test a lot of different values. CONVERGENCE_SECRETS = [ b"aaaaaaaaaaaaaaaa", - b"bbbbbbbbbbbbbbbb", - b"abcdefghijklmnop", - b"hello world stuf", - b"\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", digest(b"Hello world")[:16], ] -ONE_KB = digest(b"Hello world") * 32 -assert len(ONE_KB) == 1024 -OBJECT_DATA = [ - b"a" * 1024, - b"b" * 2048, - b"c" * 4096, - (ONE_KB * 8)[:-1], - (ONE_KB * 8) + b"z", - (ONE_KB * 128)[:-1], - (ONE_KB * 128) + b"z", +# Exercise at least a handful of different sizes, trying to cover: +# +# 1. Some cases smaller than one "segment" (128k). +# This covers shrinking of some parameters to match data size. +# +# 2. Some cases right on the edges of integer segment multiples. +# Because boundaries are tricky. +# +# 4. Some cases that involve quite a few segments. +# This exercises merkle tree construction more thoroughly. +# +# See ``stretch`` for construction of the actual test data. + +SEGMENT_SIZE = 128 * 1024 +OBJECT_DESCRIPTIONS = [ + (b"a", 1024), + (b"c", 4096), + (digest(b"foo"), SEGMENT_SIZE - 1), + (digest(b"bar"), SEGMENT_SIZE + 1), + (digest(b"baz"), SEGMENT_SIZE * 16 - 1), + (digest(b"quux"), SEGMENT_SIZE * 16 + 1), + (digest(b"foobar"), SEGMENT_SIZE * 64 - 1), + (digest(b"barbaz"), SEGMENT_SIZE * 64 + 1), ] ZFEC_PARAMS = [ @@ -64,17 +77,9 @@ def test_convergence(convergence_idx): assert len(convergence) == 16, "Convergence secret must by 16 bytes" -@mark.parametrize('data_idx', range(len(OBJECT_DATA))) -def test_data(data_idx): - """ - Plaintext data is bytes. - """ - data = OBJECT_DATA[data_idx] - assert isinstance(data, bytes), "Object data must be bytes." - @mark.parametrize('params_idx', range(len(ZFEC_PARAMS))) @mark.parametrize('convergence_idx', range(len(CONVERGENCE_SECRETS))) -@mark.parametrize('data_idx', range(len(OBJECT_DATA))) +@mark.parametrize('data_idx', range(len(OBJECT_DESCRIPTIONS))) @ensureDeferred async def test_chk_capability(reactor, request, alice, params_idx, convergence_idx, data_idx): """ @@ -82,9 +87,11 @@ async def test_chk_capability(reactor, request, alice, params_idx, convergence_i with certain well-known parameters results in exactly the previously computed value. """ - params = ZFEC_PARAMS[params_idx] - convergence = CONVERGENCE_SECRETS[convergence_idx] - data = OBJECT_DATA[data_idx] + key, params, convergence, data = load_case( + params_idx, + convergence_idx, + data_idx, + ) # rewrite alice's config to match params and convergence await reconfigure(reactor, request, alice, (1,) + params, convergence) @@ -93,12 +100,12 @@ async def test_chk_capability(reactor, request, alice, params_idx, convergence_i actual = upload(alice, "chk", data) # compare the resulting cap to the expected result - expected = vectors.chk[key(params, convergence, data)] + expected = vectors.chk[key] assert actual == expected @ensureDeferred -async def skiptest_generate(reactor, request, alice): +async def test_generate(reactor, request, alice): """ This is a helper for generating the test vectors. @@ -108,16 +115,34 @@ async def skiptest_generate(reactor, request, alice): to run against the results produced originally, not a possibly ever-changing set of outputs. """ + space = product( + range(len(ZFEC_PARAMS)), + range(len(CONVERGENCE_SECRETS)), + range(len(OBJECT_DESCRIPTIONS)), + ) results = await asyncfoldr( - generate(reactor, request, alice), + generate(reactor, request, alice, space), insert, {}, ) with vectors.CHK_PATH.open("w") as f: - f.write(safe_dump(results)) + f.write(safe_dump({ + "version": "2022-12-26", + "params": { + "zfec": ZFEC_PARAMS, + "convergence": CONVERGENCE_SECRETS, + "objects": OBJECT_DESCRIPTIONS, + }, + "vector": results, + })) -async def generate(reactor, request, alice: TahoeProcess) -> AsyncGenerator[tuple[str, str], None]: +async def generate( + reactor, + request, + alice: TahoeProcess, + space: Iterator[int, int, int], +) -> AsyncGenerator[tuple[str, str], None]: """ Generate all of the test vectors using the given node. @@ -133,24 +158,47 @@ async def generate(reactor, request, alice: TahoeProcess) -> AsyncGenerator[tupl first element is a string describing a case and the second element is the CHK capability for that case. """ + # Share placement doesn't affect the resulting capability. For maximum + # reliability, be happy if we can put shares anywhere + happy = 1 node_key = (None, None) - for params, secret, data in product(ZFEC_PARAMS, CONVERGENCE_SECRETS, OBJECT_DATA): + for params_idx, secret_idx, data_idx in space: + key, params, secret, data = load_case(params_idx, secret_idx, data_idx) if node_key != (params, secret): - await reconfigure(reactor, request, alice, params, secret) + await reconfigure(reactor, request, alice, (happy,) + params, secret) node_key = (params, secret) - yield key(params, secret, data), upload(alice, "chk", data) + yield key, upload(alice, "chk", data) -def key(params: tuple[int, int], secret: bytes, data: bytes) -> str: +def key(params: int, secret: int, data: int) -> str: """ Construct the key describing the case defined by the given parameters. - :param params: The ``needed`` and ``total`` ZFEC encoding parameters. - :param secret: The convergence secret. - :param data: The plaintext data. + The parameters are indexes into the test data for a certain case. - :return: A distinct string for the given inputs, but shorter. This is - suitable for use as, eg, a key in a dictionary. + :return: A distinct string for the given inputs. """ - return f"{params[0]}/{params[1]},{hexdigest(secret)},{hexdigest(data)}" + return f"{params}-{secret}-{data}" + + +def stretch(seed: bytes, size: int) -> bytes: + """ + Given a simple description of a byte string, return the byte string + itself. + """ + assert isinstance(seed, bytes) + assert isinstance(size, int) + assert size > 0 + assert len(seed) > 0 + + multiples = size // len(seed) + 1 + return (seed * multiples)[:size] + + +def load_case(params_idx: int, convergence_idx: int, data_idx: int) -> tuple[str, tuple[int, int], bytes, bytes]: + params = ZFEC_PARAMS[params_idx] + convergence = CONVERGENCE_SECRETS[convergence_idx] + data = stretch(*OBJECT_DESCRIPTIONS[data_idx]) + key = (params_idx, convergence_idx, data_idx) + return (key, params, convergence, data) diff --git a/integration/test_vectors.yaml b/integration/test_vectors.yaml new file mode 100644 index 000000000..a2803c4ab --- /dev/null +++ b/integration/test_vectors.yaml @@ -0,0 +1,429 @@ +params: + convergence: + - !!binary | + YWFhYWFhYWFhYWFhYWFhYQ== + - !!binary | + ZOyIygCyaOW6GjVnihtTFg== + objects: + - - !!binary | + YQ== + - 1024 + - - !!binary | + Yw== + - 4096 + - - !!binary | + LCa0a2j/xo/5m0U8HTBBNBNCLXBkg7+g+YpeiGJm564= + - 131071 + - - !!binary | + /N4rLtula/QIYB+3If6bXDONEO5CnqBPrlURto+/j7k= + - 131073 + - - !!binary | + uqWglk0zIPvAxqkiFARTyFE+okq4/QV3A0gEqWckgJY= + - 2097151 + - - !!binary | + BTBX/ampNfLU+ox7xipBGiaSbgC0kcB8Gy7BkJB4oKI= + - 2097153 + - - !!binary | + w6uP8Tcg6K2QR905Rms8iXTlksL6OD1KOWBxTK7wxPI= + - 8388607 + - - !!binary | + yPi3JHKKbWaEEG5eZOlM6BHJll0Z3UTdBzz4bPQ7wjg= + - 8388609 + zfec: + - - 1 + - 1 + - - 1 + - 3 + - - 2 + - 3 + - - 3 + - 10 + - - 71 + - 255 + - - 101 + - 256 +vector: + ? - 0 + - 0 + - 0 + : URI:CHK:mri732rh3meyh4drikau3a24ba:6tsj5wvcp6szdhmrbu5bea57wduoza64y6nd2lm7aleqpsxjm5la:1:1:1024 + ? - 0 + - 0 + - 1 + : URI:CHK:e5tpbcmalcq6nn2zhd3qvg4chy:emzsvv2xnkhhrj2oatds5hf2cney25awi56ybeq4ofiyehzyakua:1:1:4096 + ? - 0 + - 0 + - 2 + : URI:CHK:ok3slnjd3e56za3iot74audhl4:2mjvrb455yldouoxwzbx4sbsysowipqje6ifa4pmbzqahj5j4mnq:1:1:131071 + ? - 0 + - 0 + - 3 + : URI:CHK:worle55uksa2uqqeebm4yxnihu:vta76jbmejt2pxx4prqa75xawpdtx42cmzzhappeetzjksym37wq:1:1:131073 + ? - 0 + - 0 + - 4 + : URI:CHK:6kccrgbtmmprqe4jcfi7vf6v74:wpivl2evi25yfl4tzbbj6vp6nzk4vxl6lbongkac7vl3escvopsa:1:1:2097151 + ? - 0 + - 0 + - 5 + : URI:CHK:ofgig3loex6pev2eymmvohv7wq:yasbfedqnueaajdcavuba7kxbdqzn7e6zx3y6cbvucgx433vlzcq:1:1:2097153 + ? - 0 + - 0 + - 6 + : URI:CHK:scwepfpkvhuc3fc5jkpmrodsfi:wui2ixaxyac76mbxtr6diw2ajky75wehk57f2kqgytddnpo4rozq:1:1:8388607 + ? - 0 + - 0 + - 7 + : URI:CHK:5eeprb6lfxclwt7fieskd2euly:ffjgjbrxfl6d2ug2a63iaesxtrqa5pk3yaagfcldqvo7x4ok7ykq:1:1:8388609 + ? - 0 + - 1 + - 0 + : URI:CHK:nnv4vrtlxmzkurfzvonj22leua:ywcyijrfnwykraku56dq7v3o4ts3xsxqfgmk3kgzwhq2cpest4pa:1:1:1024 + ? - 0 + - 1 + - 1 + : URI:CHK:wlszole5m2emf6wbhp3lnlyfxq:irdttvny74gxdrcotzfjslfq7p24kyiigyrm5shhw7zyj3hsiuba:1:1:4096 + ? - 0 + - 1 + - 2 + : URI:CHK:4ewm23jvdtm2i5xf4gck26wg5y:7cujxxc34mkfkmwhbemqtuixuektknmhhxlmujcekibf5amfwwga:1:1:131071 + ? - 0 + - 1 + - 3 + : URI:CHK:heczpdxphw5frp3ri5sh2hpqei:6wflx6lphy5mhtpfb2abznoa3yk27ynbqbzcecs362miu72vkowq:1:1:131073 + ? - 0 + - 1 + - 4 + : URI:CHK:mz5siv27pqttsl2f4vcqmxju64:rvxhulxtufho6pdbwj7rifneb6pei5fl4fqptcce7jdkbyrjfaya:1:1:2097151 + ? - 0 + - 1 + - 5 + : URI:CHK:shbt5viqjzuewblgt6qeijry6a:je3omw53tmmluz6fvqupx4uh4jaejc3fcvfjt56rfzokzhgdczvq:1:1:2097153 + ? - 0 + - 1 + - 6 + : URI:CHK:n2642ofjfr5qljptdwjbwjz6fq:uwtjrewrev42yvez7br2574xu5vleallwnyddjaxqquvouv3dyra:1:1:8388607 + ? - 0 + - 1 + - 7 + : URI:CHK:nktoeeuydph4acj2fux4axyaj4:g3fvjwanenwsgdcn3oxnau5gtbdmzbnbevlrzrb5qe4yukuwhejq:1:1:8388609 + ? - 1 + - 0 + - 0 + : URI:CHK:ycvogzi6wllnq2bkx3t6zdtwju:um42l4yen7jiwdfgirvedtty3tt3xuhjiyxzqoourvughtxjar3q:1:3:1024 + ? - 1 + - 0 + - 1 + : URI:CHK:3yjtuv5h2g45g2cncg6hka3euu:bqqhp6u73ldawial7rtrfomx2qkmboyvqve6ywo2jpvfbu7zptga:1:3:4096 + ? - 1 + - 0 + - 2 + : URI:CHK:annnqmu72p7iels5loqwlxt2zq:s7wv3jfi2hlpcvp4dnloc4eex6vre42kwiel46achaie5n5uodqa:1:3:131071 + ? - 1 + - 0 + - 3 + : URI:CHK:rhkln7unkktot72mit5dmuqbdy:ilo6u6hugipdimyrzrvlam47xsmp3ur2lwnrtbecmvocb2664zxq:1:3:131073 + ? - 1 + - 0 + - 4 + : URI:CHK:7yjcwwt6454lbv3pni5mjofsxe:y5nwpzwmvpvr3gqxnykjixprpxw3w6qyqmszf7ijyxnm3wl6f5oa:1:3:2097151 + ? - 1 + - 0 + - 5 + : URI:CHK:r7wva6tisq6m5zszgr7seu77cm:oqlgdw3hi72qtahpsi3h3yryxpqdshagvt6xnobsppkwp7ic4cia:1:3:2097153 + ? - 1 + - 0 + - 6 + : URI:CHK:mz7pzh24u6flkfmfvvokdwmriu:5mzo57de2ywsdqkdvm7557gcikkpjqbmj65mk235t2xyoejtpbta:1:3:8388607 + ? - 1 + - 0 + - 7 + : URI:CHK:54sq6lrqwqlpxicg747gdls6ri:2aaxyrdytn7r74my36ek434hlbhe6glrgr2ic5vvpc5bbdxmvo6a:1:3:8388609 + ? - 1 + - 1 + - 0 + : URI:CHK:pbfcdvxhbikxd2hcc43oel3v3e:y5txpmiptmoz36ionmki3p6krmdbiqasw2v3wdq4ia5lmrhh33lq:1:3:1024 + ? - 1 + - 1 + - 1 + : URI:CHK:vkdc7iae4pgkdomhbbb554kv6a:g5xkatxv2entfssfh5eqgexb5j3qs6jvohofjqo5p5erlxlx2dga:1:3:4096 + ? - 1 + - 1 + - 2 + : URI:CHK:zvla2jyu5fqlb2s63zftyfjnla:yqspaexhew55cvv7w2m6czezhzkqnyk5fea4gvnplc5za4xlvbva:1:3:131071 + ? - 1 + - 1 + - 3 + : URI:CHK:3idf2xqm5wnw2q3nbva5vmpaiq:7nmkh5q55omjloezibvyzvveq4gqeaivei5ypfm5vfuugwiz6hpa:1:3:131073 + ? - 1 + - 1 + - 4 + : URI:CHK:o7xdokumtcukutzgmmpfme4nnm:nn3s5gau5dychy6wlodwriuit4wyavfze6bo4icywcbhdb4ccxla:1:3:2097151 + ? - 1 + - 1 + - 5 + : URI:CHK:vws2xrl2nch2hsptqb36yqqu3e:fhujsv5rkhyiqstrktjvfy5235c7gcwekyd3gux3bv2glsgstjga:1:3:2097153 + ? - 1 + - 1 + - 6 + : URI:CHK:5anac46dpfsjumutzsoovvtlpe:yuc3gqe7m6s6xp7k3uodcg2g6k2bac32gt5amu7gdx7k43qnudxa:1:3:8388607 + ? - 1 + - 1 + - 7 + : URI:CHK:fb6g3fkfbtlwzheujzvc2dn7dq:644je55ri6q5halshuxfesjz2afhqtebetuzqqbfecp6yhqwbb5q:1:3:8388609 + ? - 2 + - 0 + - 0 + : URI:CHK:jdm6ytmx2i3ya2bllsdzxurjtu:iqr6tyysaseegzfurhuywy3mbbwkbbsov5bt3fo6oazpyv7olvda:2:3:1024 + ? - 2 + - 0 + - 1 + : URI:CHK:7wz5dhkyob3gmkydzgbptk5qku:cl6ovq5a3km7rpfhyb3putg656sp57lhnp7aexbocsbbahdxnfia:2:3:4096 + ? - 2 + - 0 + - 2 + : URI:CHK:aagw6esdm2msphvgnr3rlzg43e:zqxtrlee6hh3vtfg6ihtssjqvr27tpvi6gkmngnhiguttjz7yyja:2:3:131071 + ? - 2 + - 0 + - 3 + : URI:CHK:e6uhixvbm3g5amrzdzd3mnmsqm:4n5aafrb64sqpfnhcdpdfrk5gh2qznprjky6q6jybuuqy2q6z6mq:2:3:131073 + ? - 2 + - 0 + - 4 + : URI:CHK:abfmzuiczjwt6e67f3uoyg5i7q:rk7ie3afnp3xgvnxu5e2vioq476xqwslqxgyx3i5xxevsjhkcvba:2:3:2097151 + ? - 2 + - 0 + - 5 + : URI:CHK:34bzejdzpqinmuzdmqenkidf74:pcecbl4ulygiadgdagonbgqmpb4lomjz4v4vqyssr56reyib4q2q:2:3:2097153 + ? - 2 + - 0 + - 6 + : URI:CHK:ej56vl65kawat62wsxuvbcd2wy:tv65cgghaddq2p7dyvimmff24l34wsx6mhzd3dyprfvrdo52qtma:2:3:8388607 + ? - 2 + - 0 + - 7 + : URI:CHK:ufshkmuewwql2xsyiecbjhh7x4:qoywwygjqrmifowob3mvwkhfskk6geomyw5e3qlgqzui3kymu6nq:2:3:8388609 + ? - 2 + - 1 + - 0 + : URI:CHK:ld3thziutpaqv25nbtoqebhtru:ri2obsvzl2etyuv3qnchn2wvw5mh5rjuvbjaqkqbvyprn7xwamoq:2:3:1024 + ? - 2 + - 1 + - 1 + : URI:CHK:pz3mga5pvqd4bpsz7pbyna2qtq:w3ptt7xwwgywod3u2rpgx7iggvm2kpwl7d2ryr3u7mcaxgqotlaq:2:3:4096 + ? - 2 + - 1 + - 2 + : URI:CHK:rsh7ysvsgbwh5g3xuj6z7zky3y:ykg5rhzpzqkvdy53qqyghfbcusibynfylggbhypo3iibsrrzvlkq:2:3:131071 + ? - 2 + - 1 + - 3 + : URI:CHK:7xqejfqfej3u3zp2qymqd2iqeq:po4t2tzkh4d34ku6gdhlocadfwdyweiyckyqz57zdi7ndt4ikj3q:2:3:131073 + ? - 2 + - 1 + - 4 + : URI:CHK:kzhg5zp4hcs75eboack4kqpcdm:g2cwnhpfwxrv3c4lrmpjyj327a274oa5qt4isu4avjixbu7vblrq:2:3:2097151 + ? - 2 + - 1 + - 5 + : URI:CHK:6sf35smixwvpapf2zw7sllxet4:ks5zd3hkd7ppwobhnymezckszexpychngnkipxfvmlcng5fgjbea:2:3:2097153 + ? - 2 + - 1 + - 6 + : URI:CHK:dnmjqjku5vyrkeaulae4bt6juu:d47airwaqedeji2omacwsp6yfqp5mpurj776gmzvq43c6dovtdca:2:3:8388607 + ? - 2 + - 1 + - 7 + : URI:CHK:bfjedwcwjehjzpwjsgwkfk7rja:jpwea2sgz4hfohqab642yj4cmrh64w6nfo7lv3mtacfzicurqkva:2:3:8388609 + ? - 3 + - 0 + - 0 + : URI:CHK:dx7tvyr2fc4u7lxjc6kehq2svq:tiy4qh2g6lqejxcaym3rr7ymkdkinn4qised6kgxloj7sptsqu4a:3:10:1024 + ? - 3 + - 0 + - 1 + : URI:CHK:yo75evk4cte3b7rdw72zxvl5ye:ex6h7ff7nclucjtsqwgwu33qgmb67t4ezbrki4zbgurwn2ct6bbq:3:10:4096 + ? - 3 + - 0 + - 2 + : URI:CHK:4gokef54smahrbfr4kq3jhc4zq:owpwwfp5gof2vhly5u6jdnbsfuwwwhqkazpsbeg3nldxv5pse2iq:3:10:131071 + ? - 3 + - 0 + - 3 + : URI:CHK:7vfgl5cv4nlzqx35z4uthjv36y:nnueftbzxfz6u5yjxwwofaxzzft7xss5wzfh66rrcwv2zwrm63sa:3:10:131073 + ? - 3 + - 0 + - 4 + : URI:CHK:cy7fmjsldeakhfd4psbmghmqyi:6a6uvyai4jkzz6hj5yjugm6uie5etvymcudgiwwjh47apz636zwa:3:10:2097151 + ? - 3 + - 0 + - 5 + : URI:CHK:ptsofqwylmkvzmuvrw5n34j3ma:ky2fs7xrlke64w6kfmhzsuilzxbhfrwwzkxih4rykpbxrr3bxhiq:3:10:2097153 + ? - 3 + - 0 + - 6 + : URI:CHK:tyt4hfvn44igztqh5zeiusggcm:ex3gzzn7byvhajithwbo7c3p7qwqxmnactaxxsxsxowkafyjrf7q:3:10:8388607 + ? - 3 + - 0 + - 7 + : URI:CHK:xymheose4rdlspgydkzr4nqkre:z3pfrvpq5fdpkoybhdxppwbzrt6ejf26xh6emzlce2sgquljginq:3:10:8388609 + ? - 3 + - 1 + - 0 + : URI:CHK:x55owxzhsezfoayaxe7jpwnove:vawdgtqpxyntgy5i2po2twgelrynkfcjgwm7publnlbdp7hpqmfa:3:10:1024 + ? - 3 + - 1 + - 1 + : URI:CHK:fe64krzyaeff3d4teunjbetkzy:27hrywwaffqiqcgfkmzwbot3iamotr3bey2l5kaladmdmxuaz5ka:3:10:4096 + ? - 3 + - 1 + - 2 + : URI:CHK:gvajllsonkuscfemygbnqhq2re:uwyilm5a7so4blhsaielnf34u2qbaqmudd73opjkgodgg3okeaga:3:10:131071 + ? - 3 + - 1 + - 3 + : URI:CHK:zdmicwopo4p4h4wbfcbnwcrvyi:6qn75anpvs5gls27f4lybisis3udvjfjhatxiny7c72bcbtuztia:3:10:131073 + ? - 3 + - 1 + - 4 + : URI:CHK:bv6qthmetlhdnwc5tfjqamp3yq:ehz4ttd4g7ktkxvbovt562wfedc6jgnt5c6af7wxgp7jbwfwhoaa:3:10:2097151 + ? - 3 + - 1 + - 5 + : URI:CHK:n7ogyjbo5jigvxgel5ll6q4vbe:3yjb3zq5hcdavv7ruefawal6euyvjx3lx7quslvasjellv63cxya:3:10:2097153 + ? - 3 + - 1 + - 6 + : URI:CHK:d2hbvcmbex7fm3qu22yj4qnkh4:2xwqxbawwgn773hht6etox3oypvqqjv2orktnthfo2e7vibko7ha:3:10:8388607 + ? - 3 + - 1 + - 7 + : URI:CHK:ccxkyfl2qtqyhihduarpxbdcci:e64be3i2t25selbpc5y2zj443gkdo65chs2o4tpqb7axud5lnxta:3:10:8388609 + ? - 4 + - 0 + - 0 + : URI:CHK:2wvrzaobtfkoiqeqwryjzvxatq:7zzaycbdhepzaqtdrku2sbb572h64ggwstu47osa2gn4ol3eglmq:71:255:1024 + ? - 4 + - 0 + - 1 + : URI:CHK:7wi5xn2uwerkjzqs5ndn4eimti:llrws4uig2fpi3rtla6lk7m4t6zudjhszdfjgqtihghfrvs2robq:71:255:4096 + ? - 4 + - 0 + - 2 + : URI:CHK:6nvs23bhrpiwiz5prqdtvztujy:wlg7g522rpdoitpm4qwhmctrjhnh4zfloiq6uq4tsvaoawg4slpq:71:255:131071 + ? - 4 + - 0 + - 3 + : URI:CHK:62dtbeowncjj62jnwbggaufvbu:6r2sapg2cm6dvmylodyxabrj63a736uouzsqyacnimgo4svnktva:71:255:131073 + ? - 4 + - 0 + - 4 + : URI:CHK:hk3a5ync7y3dnwowqjqoa34eae:y4f5b2xqa3lslh2vti5fdbho2syqhbj2p6f3enlhxsjbfpt7zuta:71:255:2097151 + ? - 4 + - 0 + - 5 + : URI:CHK:qa4hwhwtd3cpvut74biixlv6ye:uuj4ujg3mcf3lqpuvvwf5pszcvviyb4cm2jjicu7ugiypojjie4q:71:255:2097153 + ? - 4 + - 0 + - 6 + : URI:CHK:r7sv6u3ihzys754twjltuzzpza:tpyoxinubkl4kvg6hipls6ezun7t3yyqj6ix3vp6tvnzqirzzl4a:71:255:8388607 + ? - 4 + - 0 + - 7 + : URI:CHK:3esfsjn7csgnyqmq5afbgtinay:xsrhdomrbrtzftg6u4hgipm5tkaomumbdlxi5hpvpvawe3bjikua:71:255:8388609 + ? - 4 + - 1 + - 0 + : URI:CHK:wxmzfzvgaqm3xfzgaqrlndgolm:6t3tqcphsutigxnnzyp2xx3afklcpeytnh543mpqptquvqqahwea:71:255:1024 + ? - 4 + - 1 + - 1 + : URI:CHK:r3q3q4f4zbsipxauq6knb5dfoq:lkae5mflammzxkoc7gllt6qqyg5gqdxeejmxtgcscblqh5tv5obq:71:255:4096 + ? - 4 + - 1 + - 2 + : URI:CHK:rdrzetasw4w7tuweqpev65d5vq:gzzb2v5fzrduk6kx5xx27mvo6dgnd65ym5lm7re53in7xuudhsxa:71:255:131071 + ? - 4 + - 1 + - 3 + : URI:CHK:e2bjefibvz4tgu6jgf66gw74bq:mugtu5bx7h5gcivevmh2gmwoc2kkhmobrzshkuj2dgrtm3siysfq:71:255:131073 + ? - 4 + - 1 + - 4 + : URI:CHK:gy7bci6yvllxzvhh45khcwp3u4:qsfjfmcl64zey4k4o5cfnh2i3mzboahf7bhtggceszrulsv537vq:71:255:2097151 + ? - 4 + - 1 + - 5 + : URI:CHK:pcta7uk5cpioxzv5nxxqsogw3q:gnu7fx56k3j72bbevirn4kdu27yfpvttzl3qk2zypwqt7tw4rijq:71:255:2097153 + ? - 4 + - 1 + - 6 + : URI:CHK:znxyqo2qglpkhjafnuesu4foge:trau477bj5u5wjfpy4oq5e3aziajgbpyfjdv4svyf27m7atpa34a:71:255:8388607 + ? - 4 + - 1 + - 7 + : URI:CHK:mlzs2hbztak5fxjkrkuuvnpdpe:3myvisewm5uklimp2xucrwep75sm2rizfi2sq5drhlqjszkpdyeq:71:255:8388609 + ? - 5 + - 0 + - 0 + : URI:CHK:mftnvh5l3brshwoupvu6kz35cy:sncfokby2tjykbr3zgi3i6lhgerg4kz7fsrslrlxdy7upwkzq6lq:101:256:1024 + ? - 5 + - 0 + - 1 + : URI:CHK:sshu77h6opnto3jnngot3lwzl4:4cjhp6u3i2uzdzrjitje3inlhs6gnlgwc6wgi6vtedc2grgvcpza:101:256:4096 + ? - 5 + - 0 + - 2 + : URI:CHK:kl2jayrstlf5q3gax7o65ybivy:mnn5doti4wllssdg35ymojuiv4hd2ir7i6asurgms3ea67watm7a:101:256:131071 + ? - 5 + - 0 + - 3 + : URI:CHK:a6wuekh5mynr4c74ma3b2gswfa:b6lovwnbv3523n5kaxga3ruxa2dvxef6h4a2jnl6lasn4fwvvdba:101:256:131073 + ? - 5 + - 0 + - 4 + : URI:CHK:pmuo6vpcodol76sexq7s2ojej4:irp6rxnkgwbn3h4g5fzgjpqgsvhvycxobr3xrk3zr44r64wimgqa:101:256:2097151 + ? - 5 + - 0 + - 5 + : URI:CHK:fh4boug3wkciqhkgpusjsjyzbe:sy5lwooxchqu5mrzwr3mgc7qtwfjgmpz2srdmwpaip4iuj3tn4vq:101:256:2097153 + ? - 5 + - 0 + - 6 + : URI:CHK:asmjgajvwpwvylxdcf5rm2gyd4:imwtdi4eqq3wm4bvk6npho3ta2fewceqzl4f7wyyabg5i4s6h3mq:101:256:8388607 + ? - 5 + - 0 + - 7 + : URI:CHK:yrznm2uuybfbda6jlj7tqs3yum:zy45lpc7ilfzgltxxqvy55wcokiay2fbbbv2wdvu6ubjluh4gf4a:101:256:8388609 + ? - 5 + - 1 + - 0 + : URI:CHK:ihyeqq5peg2pdyw45ct5mmj4ay:2rzhndv6o4knlvo4po6rvbnknlcitasseea3mirqjor5c7quh5ra:101:256:1024 + ? - 5 + - 1 + - 1 + : URI:CHK:mvk5f2mghudk7xnknqddhvnl4m:c74myeugtix7e7l6uwybtrdtfselha2qrcc7hhbymqr6brtg4cea:101:256:4096 + ? - 5 + - 1 + - 2 + : URI:CHK:rvytwcc4sbmmkrdwh6b23ckv5u:klk5nqpbv37pr2qavcebcugfnikarjdefjlseo3ru5uv7guai22q:101:256:131071 + ? - 5 + - 1 + - 3 + : URI:CHK:ocqx55dopakqnzyui6q6euz4gm:wklziyoctwrz5zjfg5wfejz4bnaagtg57fgjlspl4ba6ij6n2obq:101:256:131073 + ? - 5 + - 1 + - 4 + : URI:CHK:lf6qub3bkdhfozpdis4bmorj7q:zte3u3ohbuom3iqjjx4odrjylcpw5myw4n4kwf4zcaif36vngf6q:101:256:2097151 + ? - 5 + - 1 + - 5 + : URI:CHK:a2qoww5xx6so5uftwg4msdtrpy:h7hwsjco5i3jkd7icytrtsuzcjvjzlnkr2wwau2tjlj5wehjrokq:101:256:2097153 + ? - 5 + - 1 + - 6 + : URI:CHK:tg5w4m6hi5ezeapuix3lrrbbtq:7kerk5plqpsd2upnole3ifxoegxdux7prazbvzeu2kyilc6idk7a:101:256:8388607 + ? - 5 + - 1 + - 7 + : URI:CHK:ioridipksj6o7ph3bcf37vslmi:pzhkwm4ry67ohj6dcs7zwnsjrvfqzam6tuitrns5vuo3uoomau6q:101:256:8388609 diff --git a/integration/vectors.py b/integration/vectors.py index b83aaad3a..e60c4497a 100644 --- a/integration/vectors.py +++ b/integration/vectors.py @@ -9,7 +9,10 @@ A module that loads pre-generated test vectors. from yaml import safe_load from pathlib import Path -CHK_PATH: Path = Path(__file__).parent / "_vectors_chk.yaml" +CHK_PATH: Path = Path(__file__).parent / "test_vectors.yaml" -with CHK_PATH.open() as f: - chk: dict[str, str] = safe_load(f) +try: + with CHK_PATH.open() as f: + chk: dict[str, str] = safe_load(f) +except FileNotFoundError: + chk = {} From 4a39c4b7ecd5c11cbe35dd74172bac0c65b97167 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 26 Dec 2022 17:08:30 -0500 Subject: [PATCH 1310/2309] Add SDMF and MDMF --- integration/test_vectors.py | 102 ++++- integration/test_vectors.yaml | 679 +++++++++++++++------------------- 2 files changed, 376 insertions(+), 405 deletions(-) diff --git a/integration/test_vectors.py b/integration/test_vectors.py index e1eb33644..261970b17 100644 --- a/integration/test_vectors.py +++ b/integration/test_vectors.py @@ -4,11 +4,14 @@ Verify certain results against test vectors with well-known results. from __future__ import annotations +from time import sleep from typing import AsyncGenerator, Iterator from hashlib import sha256 from itertools import product from yaml import safe_dump +from attrs import frozen + from pytest import mark from pytest_twisted import ensureDeferred @@ -23,6 +26,10 @@ def hexdigest(bs: bytes) -> str: return sha256(bs).hexdigest() +# Sometimes upload fail spuriously... +RETRIES = 3 + + # Just a couple convergence secrets. The only thing we do with this value is # feed it into a tagged hash. It certainly makes a difference to the output # but the hash should destroy any structure in the input so it doesn't seem @@ -58,13 +65,34 @@ OBJECT_DESCRIPTIONS = [ (digest(b"barbaz"), SEGMENT_SIZE * 64 + 1), ] +# CHK have a max of 256 shares. SDMF / MDMF have a max of 255 shares! +# Represent max symbolically and resolve it when we know what format we're +# dealing with. +MAX_SHARES = "max" + +# SDMF and MDMF encode share counts (N and k) into the share itself as an +# unsigned byte. They could have encoded (share count - 1) to fit the full +# range supported by ZFEC into the unsigned byte - but they don't. So 256 is +# inaccessible to those formats and we set the upper bound at 255. +MAX_SHARES_MAP = { + "chk": 256, + "sdmf": 255, + "mdmf": 255, +} + ZFEC_PARAMS = [ (1, 1), (1, 3), (2, 3), (3, 10), (71, 255), - (101, 256), + (101, MAX_SHARES), +] + +FORMATS = [ + "chk", + "sdmf", + "mdmf", ] @mark.parametrize('convergence_idx', range(len(CONVERGENCE_SECRETS))) @@ -80,27 +108,29 @@ def test_convergence(convergence_idx): @mark.parametrize('params_idx', range(len(ZFEC_PARAMS))) @mark.parametrize('convergence_idx', range(len(CONVERGENCE_SECRETS))) @mark.parametrize('data_idx', range(len(OBJECT_DESCRIPTIONS))) +@mark.parametrize('fmt_idx', range(len(FORMATS))) @ensureDeferred -async def test_chk_capability(reactor, request, alice, params_idx, convergence_idx, data_idx): +async def test_capability(reactor, request, alice, params_idx, convergence_idx, data_idx, fmt_idx): """ - The CHK capability that results from uploading certain well-known data + The capability that results from uploading certain well-known data with certain well-known parameters results in exactly the previously computed value. """ - key, params, convergence, data = load_case( + case = load_case( params_idx, convergence_idx, data_idx, + fmt_idx, ) # rewrite alice's config to match params and convergence - await reconfigure(reactor, request, alice, (1,) + params, convergence) + await reconfigure(reactor, request, alice, (1,) + case.params, case.convergence) - # upload data as a CHK - actual = upload(alice, "chk", data) + # upload data in the correct format + actual = upload(alice, case.fmt, case.data) # compare the resulting cap to the expected result - expected = vectors.chk[key] + expected = vectors.capabilities[case.key] assert actual == expected @@ -119,6 +149,7 @@ async def test_generate(reactor, request, alice): range(len(ZFEC_PARAMS)), range(len(CONVERGENCE_SECRETS)), range(len(OBJECT_DESCRIPTIONS)), + range(len(FORMATS)), ) results = await asyncfoldr( generate(reactor, request, alice, space), @@ -132,6 +163,7 @@ async def test_generate(reactor, request, alice): "zfec": ZFEC_PARAMS, "convergence": CONVERGENCE_SECRETS, "objects": OBJECT_DESCRIPTIONS, + "formats": FORMATS, }, "vector": results, })) @@ -141,7 +173,7 @@ async def generate( reactor, request, alice: TahoeProcess, - space: Iterator[int, int, int], + space: Iterator[int, int, int, int], ) -> AsyncGenerator[tuple[str, str], None]: """ Generate all of the test vectors using the given node. @@ -152,8 +184,14 @@ async def generate( :param request: The pytest request object to use to arrange process cleanup. + :param format: The name of the encryption/data format to use. + :param alice: The Tahoe-LAFS node to use to generate the test vectors. + :param space: An iterator of coordinates in the test vector space for + which to generate values. The elements of each tuple give indexes into + ZFEC_PARAMS, CONVERGENCE_SECRETS, OBJECT_DESCRIPTIONS, and FORMTS. + :return: The yield values are two-tuples describing a test vector. The first element is a string describing a case and the second element is the CHK capability for that case. @@ -162,16 +200,17 @@ async def generate( # reliability, be happy if we can put shares anywhere happy = 1 node_key = (None, None) - for params_idx, secret_idx, data_idx in space: - key, params, secret, data = load_case(params_idx, secret_idx, data_idx) - if node_key != (params, secret): - await reconfigure(reactor, request, alice, (happy,) + params, secret) - node_key = (params, secret) + for params_idx, secret_idx, data_idx, fmt_idx in space: + case = load_case(params_idx, secret_idx, data_idx, fmt_idx) + if node_key != (case.params, case.convergence): + await reconfigure(reactor, request, alice, (happy,) + case.params, case.convergence) + node_key = (case.params, case.convergence) - yield key, upload(alice, "chk", data) + cap = upload(alice, case.fmt, case.data) + yield case.key, cap -def key(params: int, secret: int, data: int) -> str: +def key(params: int, secret: int, data: int, fmt: int) -> str: """ Construct the key describing the case defined by the given parameters. @@ -179,7 +218,7 @@ def key(params: int, secret: int, data: int) -> str: :return: A distinct string for the given inputs. """ - return f"{params}-{secret}-{data}" + return f"{params}-{secret}-{data}-{fmt}" def stretch(seed: bytes, size: int) -> bytes: @@ -196,9 +235,32 @@ def stretch(seed: bytes, size: int) -> bytes: return (seed * multiples)[:size] -def load_case(params_idx: int, convergence_idx: int, data_idx: int) -> tuple[str, tuple[int, int], bytes, bytes]: +def load_case( + params_idx: int, + convergence_idx: int, + data_idx: int, + fmt_idx: int +) -> Case: + """ + :return: + """ params = ZFEC_PARAMS[params_idx] + fmt = FORMATS[fmt_idx] convergence = CONVERGENCE_SECRETS[convergence_idx] data = stretch(*OBJECT_DESCRIPTIONS[data_idx]) - key = (params_idx, convergence_idx, data_idx) - return (key, params, convergence, data) + if params[1] == MAX_SHARES: + params = (params[0], MAX_SHARES_MAP[fmt]) + k = key(params_idx, convergence_idx, data_idx, fmt_idx) + return Case(k, fmt, params, convergence, data) + + +@frozen +class Case: + """ + Represent one case for which we want/have a test vector. + """ + key: str + fmt: str + params: tuple[int, int] + convergence: bytes + data: bytes diff --git a/integration/test_vectors.yaml b/integration/test_vectors.yaml index a2803c4ab..11d5134df 100644 --- a/integration/test_vectors.yaml +++ b/integration/test_vectors.yaml @@ -4,6 +4,10 @@ params: YWFhYWFhYWFhYWFhYWFhYQ== - !!binary | ZOyIygCyaOW6GjVnihtTFg== + formats: + - chk + - sdmf + - mdmf objects: - - !!binary | YQ== @@ -41,389 +45,294 @@ params: - - 71 - 255 - - 101 - - 256 + - max vector: - ? - 0 - - 0 - - 0 - : URI:CHK:mri732rh3meyh4drikau3a24ba:6tsj5wvcp6szdhmrbu5bea57wduoza64y6nd2lm7aleqpsxjm5la:1:1:1024 - ? - 0 - - 0 - - 1 - : URI:CHK:e5tpbcmalcq6nn2zhd3qvg4chy:emzsvv2xnkhhrj2oatds5hf2cney25awi56ybeq4ofiyehzyakua:1:1:4096 - ? - 0 - - 0 - - 2 - : URI:CHK:ok3slnjd3e56za3iot74audhl4:2mjvrb455yldouoxwzbx4sbsysowipqje6ifa4pmbzqahj5j4mnq:1:1:131071 - ? - 0 - - 0 - - 3 - : URI:CHK:worle55uksa2uqqeebm4yxnihu:vta76jbmejt2pxx4prqa75xawpdtx42cmzzhappeetzjksym37wq:1:1:131073 - ? - 0 - - 0 - - 4 - : URI:CHK:6kccrgbtmmprqe4jcfi7vf6v74:wpivl2evi25yfl4tzbbj6vp6nzk4vxl6lbongkac7vl3escvopsa:1:1:2097151 - ? - 0 - - 0 - - 5 - : URI:CHK:ofgig3loex6pev2eymmvohv7wq:yasbfedqnueaajdcavuba7kxbdqzn7e6zx3y6cbvucgx433vlzcq:1:1:2097153 - ? - 0 - - 0 - - 6 - : URI:CHK:scwepfpkvhuc3fc5jkpmrodsfi:wui2ixaxyac76mbxtr6diw2ajky75wehk57f2kqgytddnpo4rozq:1:1:8388607 - ? - 0 - - 0 - - 7 - : URI:CHK:5eeprb6lfxclwt7fieskd2euly:ffjgjbrxfl6d2ug2a63iaesxtrqa5pk3yaagfcldqvo7x4ok7ykq:1:1:8388609 - ? - 0 - - 1 - - 0 - : URI:CHK:nnv4vrtlxmzkurfzvonj22leua:ywcyijrfnwykraku56dq7v3o4ts3xsxqfgmk3kgzwhq2cpest4pa:1:1:1024 - ? - 0 - - 1 - - 1 - : URI:CHK:wlszole5m2emf6wbhp3lnlyfxq:irdttvny74gxdrcotzfjslfq7p24kyiigyrm5shhw7zyj3hsiuba:1:1:4096 - ? - 0 - - 1 - - 2 - : URI:CHK:4ewm23jvdtm2i5xf4gck26wg5y:7cujxxc34mkfkmwhbemqtuixuektknmhhxlmujcekibf5amfwwga:1:1:131071 - ? - 0 - - 1 - - 3 - : URI:CHK:heczpdxphw5frp3ri5sh2hpqei:6wflx6lphy5mhtpfb2abznoa3yk27ynbqbzcecs362miu72vkowq:1:1:131073 - ? - 0 - - 1 - - 4 - : URI:CHK:mz5siv27pqttsl2f4vcqmxju64:rvxhulxtufho6pdbwj7rifneb6pei5fl4fqptcce7jdkbyrjfaya:1:1:2097151 - ? - 0 - - 1 - - 5 - : URI:CHK:shbt5viqjzuewblgt6qeijry6a:je3omw53tmmluz6fvqupx4uh4jaejc3fcvfjt56rfzokzhgdczvq:1:1:2097153 - ? - 0 - - 1 - - 6 - : URI:CHK:n2642ofjfr5qljptdwjbwjz6fq:uwtjrewrev42yvez7br2574xu5vleallwnyddjaxqquvouv3dyra:1:1:8388607 - ? - 0 - - 1 - - 7 - : URI:CHK:nktoeeuydph4acj2fux4axyaj4:g3fvjwanenwsgdcn3oxnau5gtbdmzbnbevlrzrb5qe4yukuwhejq:1:1:8388609 - ? - 1 - - 0 - - 0 - : URI:CHK:ycvogzi6wllnq2bkx3t6zdtwju:um42l4yen7jiwdfgirvedtty3tt3xuhjiyxzqoourvughtxjar3q:1:3:1024 - ? - 1 - - 0 - - 1 - : URI:CHK:3yjtuv5h2g45g2cncg6hka3euu:bqqhp6u73ldawial7rtrfomx2qkmboyvqve6ywo2jpvfbu7zptga:1:3:4096 - ? - 1 - - 0 - - 2 - : URI:CHK:annnqmu72p7iels5loqwlxt2zq:s7wv3jfi2hlpcvp4dnloc4eex6vre42kwiel46achaie5n5uodqa:1:3:131071 - ? - 1 - - 0 - - 3 - : URI:CHK:rhkln7unkktot72mit5dmuqbdy:ilo6u6hugipdimyrzrvlam47xsmp3ur2lwnrtbecmvocb2664zxq:1:3:131073 - ? - 1 - - 0 - - 4 - : URI:CHK:7yjcwwt6454lbv3pni5mjofsxe:y5nwpzwmvpvr3gqxnykjixprpxw3w6qyqmszf7ijyxnm3wl6f5oa:1:3:2097151 - ? - 1 - - 0 - - 5 - : URI:CHK:r7wva6tisq6m5zszgr7seu77cm:oqlgdw3hi72qtahpsi3h3yryxpqdshagvt6xnobsppkwp7ic4cia:1:3:2097153 - ? - 1 - - 0 - - 6 - : URI:CHK:mz7pzh24u6flkfmfvvokdwmriu:5mzo57de2ywsdqkdvm7557gcikkpjqbmj65mk235t2xyoejtpbta:1:3:8388607 - ? - 1 - - 0 - - 7 - : URI:CHK:54sq6lrqwqlpxicg747gdls6ri:2aaxyrdytn7r74my36ek434hlbhe6glrgr2ic5vvpc5bbdxmvo6a:1:3:8388609 - ? - 1 - - 1 - - 0 - : URI:CHK:pbfcdvxhbikxd2hcc43oel3v3e:y5txpmiptmoz36ionmki3p6krmdbiqasw2v3wdq4ia5lmrhh33lq:1:3:1024 - ? - 1 - - 1 - - 1 - : URI:CHK:vkdc7iae4pgkdomhbbb554kv6a:g5xkatxv2entfssfh5eqgexb5j3qs6jvohofjqo5p5erlxlx2dga:1:3:4096 - ? - 1 - - 1 - - 2 - : URI:CHK:zvla2jyu5fqlb2s63zftyfjnla:yqspaexhew55cvv7w2m6czezhzkqnyk5fea4gvnplc5za4xlvbva:1:3:131071 - ? - 1 - - 1 - - 3 - : URI:CHK:3idf2xqm5wnw2q3nbva5vmpaiq:7nmkh5q55omjloezibvyzvveq4gqeaivei5ypfm5vfuugwiz6hpa:1:3:131073 - ? - 1 - - 1 - - 4 - : URI:CHK:o7xdokumtcukutzgmmpfme4nnm:nn3s5gau5dychy6wlodwriuit4wyavfze6bo4icywcbhdb4ccxla:1:3:2097151 - ? - 1 - - 1 - - 5 - : URI:CHK:vws2xrl2nch2hsptqb36yqqu3e:fhujsv5rkhyiqstrktjvfy5235c7gcwekyd3gux3bv2glsgstjga:1:3:2097153 - ? - 1 - - 1 - - 6 - : URI:CHK:5anac46dpfsjumutzsoovvtlpe:yuc3gqe7m6s6xp7k3uodcg2g6k2bac32gt5amu7gdx7k43qnudxa:1:3:8388607 - ? - 1 - - 1 - - 7 - : URI:CHK:fb6g3fkfbtlwzheujzvc2dn7dq:644je55ri6q5halshuxfesjz2afhqtebetuzqqbfecp6yhqwbb5q:1:3:8388609 - ? - 2 - - 0 - - 0 - : URI:CHK:jdm6ytmx2i3ya2bllsdzxurjtu:iqr6tyysaseegzfurhuywy3mbbwkbbsov5bt3fo6oazpyv7olvda:2:3:1024 - ? - 2 - - 0 - - 1 - : URI:CHK:7wz5dhkyob3gmkydzgbptk5qku:cl6ovq5a3km7rpfhyb3putg656sp57lhnp7aexbocsbbahdxnfia:2:3:4096 - ? - 2 - - 0 - - 2 - : URI:CHK:aagw6esdm2msphvgnr3rlzg43e:zqxtrlee6hh3vtfg6ihtssjqvr27tpvi6gkmngnhiguttjz7yyja:2:3:131071 - ? - 2 - - 0 - - 3 - : URI:CHK:e6uhixvbm3g5amrzdzd3mnmsqm:4n5aafrb64sqpfnhcdpdfrk5gh2qznprjky6q6jybuuqy2q6z6mq:2:3:131073 - ? - 2 - - 0 - - 4 - : URI:CHK:abfmzuiczjwt6e67f3uoyg5i7q:rk7ie3afnp3xgvnxu5e2vioq476xqwslqxgyx3i5xxevsjhkcvba:2:3:2097151 - ? - 2 - - 0 - - 5 - : URI:CHK:34bzejdzpqinmuzdmqenkidf74:pcecbl4ulygiadgdagonbgqmpb4lomjz4v4vqyssr56reyib4q2q:2:3:2097153 - ? - 2 - - 0 - - 6 - : URI:CHK:ej56vl65kawat62wsxuvbcd2wy:tv65cgghaddq2p7dyvimmff24l34wsx6mhzd3dyprfvrdo52qtma:2:3:8388607 - ? - 2 - - 0 - - 7 - : URI:CHK:ufshkmuewwql2xsyiecbjhh7x4:qoywwygjqrmifowob3mvwkhfskk6geomyw5e3qlgqzui3kymu6nq:2:3:8388609 - ? - 2 - - 1 - - 0 - : URI:CHK:ld3thziutpaqv25nbtoqebhtru:ri2obsvzl2etyuv3qnchn2wvw5mh5rjuvbjaqkqbvyprn7xwamoq:2:3:1024 - ? - 2 - - 1 - - 1 - : URI:CHK:pz3mga5pvqd4bpsz7pbyna2qtq:w3ptt7xwwgywod3u2rpgx7iggvm2kpwl7d2ryr3u7mcaxgqotlaq:2:3:4096 - ? - 2 - - 1 - - 2 - : URI:CHK:rsh7ysvsgbwh5g3xuj6z7zky3y:ykg5rhzpzqkvdy53qqyghfbcusibynfylggbhypo3iibsrrzvlkq:2:3:131071 - ? - 2 - - 1 - - 3 - : URI:CHK:7xqejfqfej3u3zp2qymqd2iqeq:po4t2tzkh4d34ku6gdhlocadfwdyweiyckyqz57zdi7ndt4ikj3q:2:3:131073 - ? - 2 - - 1 - - 4 - : URI:CHK:kzhg5zp4hcs75eboack4kqpcdm:g2cwnhpfwxrv3c4lrmpjyj327a274oa5qt4isu4avjixbu7vblrq:2:3:2097151 - ? - 2 - - 1 - - 5 - : URI:CHK:6sf35smixwvpapf2zw7sllxet4:ks5zd3hkd7ppwobhnymezckszexpychngnkipxfvmlcng5fgjbea:2:3:2097153 - ? - 2 - - 1 - - 6 - : URI:CHK:dnmjqjku5vyrkeaulae4bt6juu:d47airwaqedeji2omacwsp6yfqp5mpurj776gmzvq43c6dovtdca:2:3:8388607 - ? - 2 - - 1 - - 7 - : URI:CHK:bfjedwcwjehjzpwjsgwkfk7rja:jpwea2sgz4hfohqab642yj4cmrh64w6nfo7lv3mtacfzicurqkva:2:3:8388609 - ? - 3 - - 0 - - 0 - : URI:CHK:dx7tvyr2fc4u7lxjc6kehq2svq:tiy4qh2g6lqejxcaym3rr7ymkdkinn4qised6kgxloj7sptsqu4a:3:10:1024 - ? - 3 - - 0 - - 1 - : URI:CHK:yo75evk4cte3b7rdw72zxvl5ye:ex6h7ff7nclucjtsqwgwu33qgmb67t4ezbrki4zbgurwn2ct6bbq:3:10:4096 - ? - 3 - - 0 - - 2 - : URI:CHK:4gokef54smahrbfr4kq3jhc4zq:owpwwfp5gof2vhly5u6jdnbsfuwwwhqkazpsbeg3nldxv5pse2iq:3:10:131071 - ? - 3 - - 0 - - 3 - : URI:CHK:7vfgl5cv4nlzqx35z4uthjv36y:nnueftbzxfz6u5yjxwwofaxzzft7xss5wzfh66rrcwv2zwrm63sa:3:10:131073 - ? - 3 - - 0 - - 4 - : URI:CHK:cy7fmjsldeakhfd4psbmghmqyi:6a6uvyai4jkzz6hj5yjugm6uie5etvymcudgiwwjh47apz636zwa:3:10:2097151 - ? - 3 - - 0 - - 5 - : URI:CHK:ptsofqwylmkvzmuvrw5n34j3ma:ky2fs7xrlke64w6kfmhzsuilzxbhfrwwzkxih4rykpbxrr3bxhiq:3:10:2097153 - ? - 3 - - 0 - - 6 - : URI:CHK:tyt4hfvn44igztqh5zeiusggcm:ex3gzzn7byvhajithwbo7c3p7qwqxmnactaxxsxsxowkafyjrf7q:3:10:8388607 - ? - 3 - - 0 - - 7 - : URI:CHK:xymheose4rdlspgydkzr4nqkre:z3pfrvpq5fdpkoybhdxppwbzrt6ejf26xh6emzlce2sgquljginq:3:10:8388609 - ? - 3 - - 1 - - 0 - : URI:CHK:x55owxzhsezfoayaxe7jpwnove:vawdgtqpxyntgy5i2po2twgelrynkfcjgwm7publnlbdp7hpqmfa:3:10:1024 - ? - 3 - - 1 - - 1 - : URI:CHK:fe64krzyaeff3d4teunjbetkzy:27hrywwaffqiqcgfkmzwbot3iamotr3bey2l5kaladmdmxuaz5ka:3:10:4096 - ? - 3 - - 1 - - 2 - : URI:CHK:gvajllsonkuscfemygbnqhq2re:uwyilm5a7so4blhsaielnf34u2qbaqmudd73opjkgodgg3okeaga:3:10:131071 - ? - 3 - - 1 - - 3 - : URI:CHK:zdmicwopo4p4h4wbfcbnwcrvyi:6qn75anpvs5gls27f4lybisis3udvjfjhatxiny7c72bcbtuztia:3:10:131073 - ? - 3 - - 1 - - 4 - : URI:CHK:bv6qthmetlhdnwc5tfjqamp3yq:ehz4ttd4g7ktkxvbovt562wfedc6jgnt5c6af7wxgp7jbwfwhoaa:3:10:2097151 - ? - 3 - - 1 - - 5 - : URI:CHK:n7ogyjbo5jigvxgel5ll6q4vbe:3yjb3zq5hcdavv7ruefawal6euyvjx3lx7quslvasjellv63cxya:3:10:2097153 - ? - 3 - - 1 - - 6 - : URI:CHK:d2hbvcmbex7fm3qu22yj4qnkh4:2xwqxbawwgn773hht6etox3oypvqqjv2orktnthfo2e7vibko7ha:3:10:8388607 - ? - 3 - - 1 - - 7 - : URI:CHK:ccxkyfl2qtqyhihduarpxbdcci:e64be3i2t25selbpc5y2zj443gkdo65chs2o4tpqb7axud5lnxta:3:10:8388609 - ? - 4 - - 0 - - 0 - : URI:CHK:2wvrzaobtfkoiqeqwryjzvxatq:7zzaycbdhepzaqtdrku2sbb572h64ggwstu47osa2gn4ol3eglmq:71:255:1024 - ? - 4 - - 0 - - 1 - : URI:CHK:7wi5xn2uwerkjzqs5ndn4eimti:llrws4uig2fpi3rtla6lk7m4t6zudjhszdfjgqtihghfrvs2robq:71:255:4096 - ? - 4 - - 0 - - 2 - : URI:CHK:6nvs23bhrpiwiz5prqdtvztujy:wlg7g522rpdoitpm4qwhmctrjhnh4zfloiq6uq4tsvaoawg4slpq:71:255:131071 - ? - 4 - - 0 - - 3 - : URI:CHK:62dtbeowncjj62jnwbggaufvbu:6r2sapg2cm6dvmylodyxabrj63a736uouzsqyacnimgo4svnktva:71:255:131073 - ? - 4 - - 0 - - 4 - : URI:CHK:hk3a5ync7y3dnwowqjqoa34eae:y4f5b2xqa3lslh2vti5fdbho2syqhbj2p6f3enlhxsjbfpt7zuta:71:255:2097151 - ? - 4 - - 0 - - 5 - : URI:CHK:qa4hwhwtd3cpvut74biixlv6ye:uuj4ujg3mcf3lqpuvvwf5pszcvviyb4cm2jjicu7ugiypojjie4q:71:255:2097153 - ? - 4 - - 0 - - 6 - : URI:CHK:r7sv6u3ihzys754twjltuzzpza:tpyoxinubkl4kvg6hipls6ezun7t3yyqj6ix3vp6tvnzqirzzl4a:71:255:8388607 - ? - 4 - - 0 - - 7 - : URI:CHK:3esfsjn7csgnyqmq5afbgtinay:xsrhdomrbrtzftg6u4hgipm5tkaomumbdlxi5hpvpvawe3bjikua:71:255:8388609 - ? - 4 - - 1 - - 0 - : URI:CHK:wxmzfzvgaqm3xfzgaqrlndgolm:6t3tqcphsutigxnnzyp2xx3afklcpeytnh543mpqptquvqqahwea:71:255:1024 - ? - 4 - - 1 - - 1 - : URI:CHK:r3q3q4f4zbsipxauq6knb5dfoq:lkae5mflammzxkoc7gllt6qqyg5gqdxeejmxtgcscblqh5tv5obq:71:255:4096 - ? - 4 - - 1 - - 2 - : URI:CHK:rdrzetasw4w7tuweqpev65d5vq:gzzb2v5fzrduk6kx5xx27mvo6dgnd65ym5lm7re53in7xuudhsxa:71:255:131071 - ? - 4 - - 1 - - 3 - : URI:CHK:e2bjefibvz4tgu6jgf66gw74bq:mugtu5bx7h5gcivevmh2gmwoc2kkhmobrzshkuj2dgrtm3siysfq:71:255:131073 - ? - 4 - - 1 - - 4 - : URI:CHK:gy7bci6yvllxzvhh45khcwp3u4:qsfjfmcl64zey4k4o5cfnh2i3mzboahf7bhtggceszrulsv537vq:71:255:2097151 - ? - 4 - - 1 - - 5 - : URI:CHK:pcta7uk5cpioxzv5nxxqsogw3q:gnu7fx56k3j72bbevirn4kdu27yfpvttzl3qk2zypwqt7tw4rijq:71:255:2097153 - ? - 4 - - 1 - - 6 - : URI:CHK:znxyqo2qglpkhjafnuesu4foge:trau477bj5u5wjfpy4oq5e3aziajgbpyfjdv4svyf27m7atpa34a:71:255:8388607 - ? - 4 - - 1 - - 7 - : URI:CHK:mlzs2hbztak5fxjkrkuuvnpdpe:3myvisewm5uklimp2xucrwep75sm2rizfi2sq5drhlqjszkpdyeq:71:255:8388609 - ? - 5 - - 0 - - 0 - : URI:CHK:mftnvh5l3brshwoupvu6kz35cy:sncfokby2tjykbr3zgi3i6lhgerg4kz7fsrslrlxdy7upwkzq6lq:101:256:1024 - ? - 5 - - 0 - - 1 - : URI:CHK:sshu77h6opnto3jnngot3lwzl4:4cjhp6u3i2uzdzrjitje3inlhs6gnlgwc6wgi6vtedc2grgvcpza:101:256:4096 - ? - 5 - - 0 - - 2 - : URI:CHK:kl2jayrstlf5q3gax7o65ybivy:mnn5doti4wllssdg35ymojuiv4hd2ir7i6asurgms3ea67watm7a:101:256:131071 - ? - 5 - - 0 - - 3 - : URI:CHK:a6wuekh5mynr4c74ma3b2gswfa:b6lovwnbv3523n5kaxga3ruxa2dvxef6h4a2jnl6lasn4fwvvdba:101:256:131073 - ? - 5 - - 0 - - 4 - : URI:CHK:pmuo6vpcodol76sexq7s2ojej4:irp6rxnkgwbn3h4g5fzgjpqgsvhvycxobr3xrk3zr44r64wimgqa:101:256:2097151 - ? - 5 - - 0 - - 5 - : URI:CHK:fh4boug3wkciqhkgpusjsjyzbe:sy5lwooxchqu5mrzwr3mgc7qtwfjgmpz2srdmwpaip4iuj3tn4vq:101:256:2097153 - ? - 5 - - 0 - - 6 - : URI:CHK:asmjgajvwpwvylxdcf5rm2gyd4:imwtdi4eqq3wm4bvk6npho3ta2fewceqzl4f7wyyabg5i4s6h3mq:101:256:8388607 - ? - 5 - - 0 - - 7 - : URI:CHK:yrznm2uuybfbda6jlj7tqs3yum:zy45lpc7ilfzgltxxqvy55wcokiay2fbbbv2wdvu6ubjluh4gf4a:101:256:8388609 - ? - 5 - - 1 - - 0 - : URI:CHK:ihyeqq5peg2pdyw45ct5mmj4ay:2rzhndv6o4knlvo4po6rvbnknlcitasseea3mirqjor5c7quh5ra:101:256:1024 - ? - 5 - - 1 - - 1 - : URI:CHK:mvk5f2mghudk7xnknqddhvnl4m:c74myeugtix7e7l6uwybtrdtfselha2qrcc7hhbymqr6brtg4cea:101:256:4096 - ? - 5 - - 1 - - 2 - : URI:CHK:rvytwcc4sbmmkrdwh6b23ckv5u:klk5nqpbv37pr2qavcebcugfnikarjdefjlseo3ru5uv7guai22q:101:256:131071 - ? - 5 - - 1 - - 3 - : URI:CHK:ocqx55dopakqnzyui6q6euz4gm:wklziyoctwrz5zjfg5wfejz4bnaagtg57fgjlspl4ba6ij6n2obq:101:256:131073 - ? - 5 - - 1 - - 4 - : URI:CHK:lf6qub3bkdhfozpdis4bmorj7q:zte3u3ohbuom3iqjjx4odrjylcpw5myw4n4kwf4zcaif36vngf6q:101:256:2097151 - ? - 5 - - 1 - - 5 - : URI:CHK:a2qoww5xx6so5uftwg4msdtrpy:h7hwsjco5i3jkd7icytrtsuzcjvjzlnkr2wwau2tjlj5wehjrokq:101:256:2097153 - ? - 5 - - 1 - - 6 - : URI:CHK:tg5w4m6hi5ezeapuix3lrrbbtq:7kerk5plqpsd2upnole3ifxoegxdux7prazbvzeu2kyilc6idk7a:101:256:8388607 - ? - 5 - - 1 - - 7 - : URI:CHK:ioridipksj6o7ph3bcf37vslmi:pzhkwm4ry67ohj6dcs7zwnsjrvfqzam6tuitrns5vuo3uoomau6q:101:256:8388609 + 0-0-0-0: URI:CHK:mri732rh3meyh4drikau3a24ba:6tsj5wvcp6szdhmrbu5bea57wduoza64y6nd2lm7aleqpsxjm5la:1:1:1024 + 0-0-0-1: URI:SSK:64ipvwidczoqhptiwxewux75a4:47pnj3eykp6jfx4db274qfkjtndo235p6l56ah54nq2ndasskgha + 0-0-0-2: URI:MDMF:2bncspmundtrdanr5ty5rswm5e:nt3azayg6eyebhjhuf67p2kahb3wf2pedmtitlubhjc34unjj46a + 0-0-1-0: URI:CHK:e5tpbcmalcq6nn2zhd3qvg4chy:emzsvv2xnkhhrj2oatds5hf2cney25awi56ybeq4ofiyehzyakua:1:1:4096 + 0-0-1-1: URI:SSK:rxaltidfitcuzkxpi5lupojd6y:l7bzwvlzlcqrpg7zecg3plk6cfvvgzoylyryxtbv56b3hwgz5lpa + 0-0-1-2: URI:MDMF:5mqmhjtvr52gqvqziy4kox7blm:tfvldj3lph5ww6thcewusaxjnzhajfeyuui5e264cqpgigxok5ra + 0-0-2-0: URI:CHK:ok3slnjd3e56za3iot74audhl4:2mjvrb455yldouoxwzbx4sbsysowipqje6ifa4pmbzqahj5j4mnq:1:1:131071 + 0-0-2-1: URI:SSK:a7xyeaedjdi7vej32o522r4sme:n7w6h57yuxcrlkgkhjbe6akf7lrnt6jd5gqa2u2nbxboechpbfxq + 0-0-2-2: URI:MDMF:qctbamqytlidaqc6hhj7kd5yqe:exdrlt2auxsl3cewufeunjxa4672zrsulx3wsdxgk3z4s2uvob7q + 0-0-3-0: URI:CHK:worle55uksa2uqqeebm4yxnihu:vta76jbmejt2pxx4prqa75xawpdtx42cmzzhappeetzjksym37wq:1:1:131073 + 0-0-3-1: URI:SSK:wvjziwdykuw7dcthuyvbozdc2y:lzbiyksn5phd6mw63r4ixllresqn6suluv6xpkzq2y77xplskmxq + 0-0-3-2: URI:MDMF:nk4lmqor4jif6jh4ptpevqvlsy:5ihzv32rcarelcingl2jgkqumokl4unf3vyzaxuht73vnnjkjnbq + 0-0-4-0: URI:CHK:6kccrgbtmmprqe4jcfi7vf6v74:wpivl2evi25yfl4tzbbj6vp6nzk4vxl6lbongkac7vl3escvopsa:1:1:2097151 + 0-0-4-1: URI:SSK:j4ikqxq54udpd6pffhzyh22gaq:davtelwhbu2off6h3xzgjneeaxu4ujtvstwxp7kdrlnlluq7r4pq + 0-0-4-2: URI:MDMF:5ya7auna2ojhb5qxrmy33ohtyi:olubk3gn2icihwa6zkyebqdqugdnhwczq5dziirkt7koexim2a4a + 0-0-5-0: URI:CHK:ofgig3loex6pev2eymmvohv7wq:yasbfedqnueaajdcavuba7kxbdqzn7e6zx3y6cbvucgx433vlzcq:1:1:2097153 + 0-0-5-1: URI:SSK:ze3nkp7unkx3omh2qka74x3zhq:4ltf23sdboqiwvkwhuvhk2ehbcqyvsenx6oxisc2z3mkqgidjteq + 0-0-5-2: URI:MDMF:pehv4ip7wvu5vigq425lokcxam:peuvelif4letrl7d3ou2k3pcfm7sv4d3kszjhjxpo4wwek67vnbq + 0-0-6-0: URI:CHK:scwepfpkvhuc3fc5jkpmrodsfi:wui2ixaxyac76mbxtr6diw2ajky75wehk57f2kqgytddnpo4rozq:1:1:8388607 + 0-0-6-1: URI:SSK:i6nerj3afbsbinhyi4yq32ttyi:hlgzk6pm2msqedm2ixhysvdewwlaiqn3xravycy2wxpikimo3gtq + 0-0-6-2: URI:MDMF:bpulrhen44p63tqelmtq64rafa:xhnlxbyabauxbafaecnynjznhliqcswjamym52kq4dj4jivuuv3a + 0-0-7-0: URI:CHK:5eeprb6lfxclwt7fieskd2euly:ffjgjbrxfl6d2ug2a63iaesxtrqa5pk3yaagfcldqvo7x4ok7ykq:1:1:8388609 + 0-0-7-1: URI:SSK:5tzpr45bx6pmqkhb6j4ofcgoyu:cjpggju6lfleijv7ucrmclvpqm5nrhdymqkwzjkopbyz42umyeeq + 0-0-7-2: URI:MDMF:xkqjvm2uc6ajoji4cvokmh337y:smvwzvyrxhde6j64l7s6dx3xiohgco4t744gnnuaizrcehovn3rq + 0-1-0-0: URI:CHK:nnv4vrtlxmzkurfzvonj22leua:ywcyijrfnwykraku56dq7v3o4ts3xsxqfgmk3kgzwhq2cpest4pa:1:1:1024 + 0-1-0-1: URI:SSK:sqylzhpnxjxlgytx6wtjpmgkle:jlol7tvkpo4esw6cfd4lzpyq2ra7vai6fn2fpt3qkglbimwunalq + 0-1-0-2: URI:MDMF:oewssqeva3bextinsbsvddke24:p2mquc646afacs4p73xwtcrtvapvzwmjffvek36w4gebgalufzwa + 0-1-1-0: URI:CHK:wlszole5m2emf6wbhp3lnlyfxq:irdttvny74gxdrcotzfjslfq7p24kyiigyrm5shhw7zyj3hsiuba:1:1:4096 + 0-1-1-1: URI:SSK:4rfhew7z7fig3ztoyfspnqkzhy:7bp5wg5agcrju66sh6zklithnf32nsso5kgyznl36vv3vrg74p2a + 0-1-1-2: URI:MDMF:mztzdkhjxeut4lzwynb6hgqzj4:r3ujkurraq52qjbp2gulbds47nlrjif3pq2whpwjw4dqn5r3mepa + 0-1-2-0: URI:CHK:4ewm23jvdtm2i5xf4gck26wg5y:7cujxxc34mkfkmwhbemqtuixuektknmhhxlmujcekibf5amfwwga:1:1:131071 + 0-1-2-1: URI:SSK:tcvykalplqpck5usnjljvhil2y:kgyz3qryyabm7ppxo7xn3q43h37yz5ygjl6mavthrunki6sjcdya + 0-1-2-2: URI:MDMF:tpke6qm6ra5dcv5ef34kjq334u:yexpy6cp3tl5bypjhs42r5hs6pxmcptk77wc43qfb2odd5wkm4ta + 0-1-3-0: URI:CHK:heczpdxphw5frp3ri5sh2hpqei:6wflx6lphy5mhtpfb2abznoa3yk27ynbqbzcecs362miu72vkowq:1:1:131073 + 0-1-3-1: URI:SSK:zmt4xsuswb24t7xb5ghc6x4zoq:khwlexwe3bfpkjl6k56niynndw6usr63ujpqsnyz6oplugnooqva + 0-1-3-2: URI:MDMF:wtxng6wxbjzasgmzjol2tyzp2y:cu3eni3wqffuqysghrb35cy5m3h6zkpnoasghejlnn3tutkv62ra + 0-1-4-0: URI:CHK:mz5siv27pqttsl2f4vcqmxju64:rvxhulxtufho6pdbwj7rifneb6pei5fl4fqptcce7jdkbyrjfaya:1:1:2097151 + 0-1-4-1: URI:SSK:dbjrnvd6ze3hfh6dpy5p776yjm:kopm44nxmgsapuphibtxy7avh5jzzsvq5uizg43lvfpk2bp5qgwq + 0-1-4-2: URI:MDMF:yo2yjrc7qlrl6xpkg46v5odx6q:2dh6tymdchl36aqrqayqqhhvxj6byaxshadc2johx6d2jxmemfuq + 0-1-5-0: URI:CHK:shbt5viqjzuewblgt6qeijry6a:je3omw53tmmluz6fvqupx4uh4jaejc3fcvfjt56rfzokzhgdczvq:1:1:2097153 + 0-1-5-1: URI:SSK:pjvcabyr7hlzvo3hgfwayomvyy:ioykumds24zhxwour2ktrbfr2g3iyephqz6s6lriu2bpjh5n234q + 0-1-5-2: URI:MDMF:cuy7igduaoj5c5o7sxud5dgc3i:wfuxxoehvu5xqyy6teljt4gporjazqqwewtcynowgrg7ylliu3ra + 0-1-6-0: URI:CHK:n2642ofjfr5qljptdwjbwjz6fq:uwtjrewrev42yvez7br2574xu5vleallwnyddjaxqquvouv3dyra:1:1:8388607 + 0-1-6-1: URI:SSK:vqzgr727yx6zei3ougcx73j5m4:osrlcedb3ws5lz6ofdcpf3erro3ydqvwedssjtvi5rzlb35mc2va + 0-1-6-2: URI:MDMF:mxyohlnkjvad4p3mk4ssqyqgcy:p32sx4fnl5thwjpszmalpdvvnrsaurfevrkkybiyy5sg66yrq7ka + 0-1-7-0: URI:CHK:nktoeeuydph4acj2fux4axyaj4:g3fvjwanenwsgdcn3oxnau5gtbdmzbnbevlrzrb5qe4yukuwhejq:1:1:8388609 + 0-1-7-1: URI:SSK:vmmmxar3clfixjsjswr5owjt6q:g5dsw44pkbg3lvbhrmpjki3oqqjvyyv4dkbejcm5svojxlgmmaya + 0-1-7-2: URI:MDMF:dedcqq5ryz7p6bup7vnqhlbp4y:dkiinwnqyizu7q3vrpordqetnwkoe74k4mplyjpw4dhwv4mgnj2a + 1-0-0-0: URI:CHK:ycvogzi6wllnq2bkx3t6zdtwju:um42l4yen7jiwdfgirvedtty3tt3xuhjiyxzqoourvughtxjar3q:1:3:1024 + 1-0-0-1: URI:SSK:3nlzd3o4goxnkms67hjezvnx3e:5cw77s7ywr6gmzoeu2dr62sl3mmzuk4uyn7tsqyb6ycc5uaglxtq + 1-0-0-2: URI:MDMF:ilkr2vh5k6ob3qe3cnrio222nq:4b4j6ysy6t6rfga63qauqdq6oavyhmqe6nusyccbbdzfm42fjfvq + 1-0-1-0: URI:CHK:3yjtuv5h2g45g2cncg6hka3euu:bqqhp6u73ldawial7rtrfomx2qkmboyvqve6ywo2jpvfbu7zptga:1:3:4096 + 1-0-1-1: URI:SSK:7gzsibil7wlnxvhf7jnnhugumi:s2tdz77gmlgge4726cjseflqiv6zqjtwolofk5rnnx7qa56f372a + 1-0-1-2: URI:MDMF:obvmtzzryj2gfxdp2o6y4iqyoq:vy7kr4euwkhvrpgjqjonsy4xfif6qy4mqxcw5r5tekdyt6o4rdka + 1-0-2-0: URI:CHK:annnqmu72p7iels5loqwlxt2zq:s7wv3jfi2hlpcvp4dnloc4eex6vre42kwiel46achaie5n5uodqa:1:3:131071 + 1-0-2-1: URI:SSK:sxos74s6bhwr3n25fxjabz3o7a:nugsx6go3a2pwne6p3hravnu6kcfjmzd5eboeewjaxyczczpllpa + 1-0-2-2: URI:MDMF:yxhafheawf6oogddsr3icqcrqe:dt4zatgywpz3tjicga6iiz7ce6ccqspkccjnwobihkmh35xnwaxa + 1-0-3-0: URI:CHK:rhkln7unkktot72mit5dmuqbdy:ilo6u6hugipdimyrzrvlam47xsmp3ur2lwnrtbecmvocb2664zxq:1:3:131073 + 1-0-3-1: URI:SSK:kikhu6vcfuhnmbkihqekq5ezeu:e6l2k5h3kwrpccg5koba3atxp2y23iyylwvrhvctlbsphtpywvjq + 1-0-3-2: URI:MDMF:hxdnms24wh2o54glbrhw4egolu:77kjhrpztiz7eapsipjb6toldabzca767lyvh47iut7insdhrp3a + 1-0-4-0: URI:CHK:7yjcwwt6454lbv3pni5mjofsxe:y5nwpzwmvpvr3gqxnykjixprpxw3w6qyqmszf7ijyxnm3wl6f5oa:1:3:2097151 + 1-0-4-1: URI:SSK:fanakrkirjczr7v2w5l6ovwt6u:uyrwhi4oqqehewjfomvris4gmbehrlidxb2drh2xold5entrheeq + 1-0-4-2: URI:MDMF:4dyxo6lvt5v5qjwpveolbnguve:h4ck2q6pd62pwb63dfewvfeyfck3gjqie66gj4pwwf2nk7an7l7a + 1-0-5-0: URI:CHK:r7wva6tisq6m5zszgr7seu77cm:oqlgdw3hi72qtahpsi3h3yryxpqdshagvt6xnobsppkwp7ic4cia:1:3:2097153 + 1-0-5-1: URI:SSK:agiakuvmrmzv6c34cfi4mdwdzy:qq5gymi6bvqvt7lykgdznmuoufety3a6rnyn4o22wk2xqirnbdrq + 1-0-5-2: URI:MDMF:pm6fmmvxu5zmypkbmpqt6qpawq:wrj4w27ldtgfw7djfhednxzti6rmqdyx74n5e45qcwe5euptcbrq + 1-0-6-0: URI:CHK:mz7pzh24u6flkfmfvvokdwmriu:5mzo57de2ywsdqkdvm7557gcikkpjqbmj65mk235t2xyoejtpbta:1:3:8388607 + 1-0-6-1: URI:SSK:q5jatla7dgcrykoeyy5hq2kne4:n2bgno4z5qy2f5uq3vrgyqq4lpa5o5bhuzgyy725qlksswkihrwa + 1-0-6-2: URI:MDMF:mijwsnxcbdcyvegro3dlfb2kvm:lsn2etmig274xnfwe6nu6wd5lu7mvdy55sqytwb2nzx6ko3xtqoa + 1-0-7-0: URI:CHK:54sq6lrqwqlpxicg747gdls6ri:2aaxyrdytn7r74my36ek434hlbhe6glrgr2ic5vvpc5bbdxmvo6a:1:3:8388609 + 1-0-7-1: URI:SSK:wusjn7evp3wutlijs3bomfmtka:wnig2pqxofw7socupyhexz4qiuwqcky5aq6tqoymdlk4mscm7tpa + 1-0-7-2: URI:MDMF:lpmgpoqsbjqekgh2cb2crlunda:de4l4ox64kx7lutli3h3iefxsolkqrzxdxqchnc74msobat4dhra + 1-1-0-0: URI:CHK:pbfcdvxhbikxd2hcc43oel3v3e:y5txpmiptmoz36ionmki3p6krmdbiqasw2v3wdq4ia5lmrhh33lq:1:3:1024 + 1-1-0-1: URI:SSK:wpto56y2mcqjz76yeq32dqg4za:7qwecvg265xwrflipssea3qvgdbce3xp52odm6f6dwxiunq3ipqq + 1-1-0-2: URI:MDMF:holxufnqnricjbf6pfxsajv6ua:iffvui7drjicjcuoul5wdxn5b56znrvjpaon4ntc5s6pml5pgqda + 1-1-1-0: URI:CHK:vkdc7iae4pgkdomhbbb554kv6a:g5xkatxv2entfssfh5eqgexb5j3qs6jvohofjqo5p5erlxlx2dga:1:3:4096 + 1-1-1-1: URI:SSK:j3okig3bvfg2tz3bf4yakmaqkq:m3fmgcxah23izgz77a3jl4bxzn6ziwga75drqkzhnuhazwhd4deq + 1-1-1-2: URI:MDMF:gnloxjxwuathfupvhmjfiml34u:fhqeulalg3xkaja7fcgukm2arguh2gipiizceftnkwnadqfv224q + 1-1-2-0: URI:CHK:zvla2jyu5fqlb2s63zftyfjnla:yqspaexhew55cvv7w2m6czezhzkqnyk5fea4gvnplc5za4xlvbva:1:3:131071 + 1-1-2-1: URI:SSK:m2a4klrarck66kxdaelqirusje:u2mpjlt7vrkhetfifxv77zhw6vxgifx4cwmp62i4qurhrrjtaefa + 1-1-2-2: URI:MDMF:ek7p7ocgap3c44njbqviux634m:6nfyyt5pper3wvqsp3rnubgploukfogejuo4po62qyi66fmutlaa + 1-1-3-0: URI:CHK:3idf2xqm5wnw2q3nbva5vmpaiq:7nmkh5q55omjloezibvyzvveq4gqeaivei5ypfm5vfuugwiz6hpa:1:3:131073 + 1-1-3-1: URI:SSK:ycfor2ol5iv6w6uukisf7xianq:dp32lujnfnpn64bhzfwkc7poewdkhluiz3bmsofka26fndxe5cyq + 1-1-3-2: URI:MDMF:46g6h67kbymmny5mxaqgoeq57e:r7dojkuirkw6svgesc3qnry5vkquam5ayuydoljdfsgirffkjcpa + 1-1-4-0: URI:CHK:o7xdokumtcukutzgmmpfme4nnm:nn3s5gau5dychy6wlodwriuit4wyavfze6bo4icywcbhdb4ccxla:1:3:2097151 + 1-1-4-1: URI:SSK:s3sfivcsgqqcv44rh62f7ybua4:zllpdgffhzci4ddbfyfx22yzimnlv4jmsbyf3lig7w3zlk3lowea + 1-1-4-2: URI:MDMF:ammiuctyg7v7caimclofugplqq:jvwrtrj6yx7xrtmpckcecbcsozbhb2jpprgs2nosii245yj766xq + 1-1-5-0: URI:CHK:vws2xrl2nch2hsptqb36yqqu3e:fhujsv5rkhyiqstrktjvfy5235c7gcwekyd3gux3bv2glsgstjga:1:3:2097153 + 1-1-5-1: URI:SSK:6jx3psk4ymb4m6xijdkmwyvn5a:pokhzlauyftwdfeh7fpx5vudee35tzh4eqg2nuzfv445565zge3q + 1-1-5-2: URI:MDMF:2kv3aehx33jpkmmdxuw56mlv54:papxsfb63bakhhcsmnvxqvfgtl3cwvxr4ecxgyemk7bewvdpzdva + 1-1-6-0: URI:CHK:5anac46dpfsjumutzsoovvtlpe:yuc3gqe7m6s6xp7k3uodcg2g6k2bac32gt5amu7gdx7k43qnudxa:1:3:8388607 + 1-1-6-1: URI:SSK:wawckuhibbpqpnuqpukfmeykjy:gz4qevawur75onpf6mif6vyqs52wr53624pbqirq7ioicw26ltdq + 1-1-6-2: URI:MDMF:g3ssdqnaayw2ibczy3stibpmdq:3pmbetun4gpewxspp2h6lkmnez6cn2yorxmfmeef4tpziq4yhiza + 1-1-7-0: URI:CHK:fb6g3fkfbtlwzheujzvc2dn7dq:644je55ri6q5halshuxfesjz2afhqtebetuzqqbfecp6yhqwbb5q:1:3:8388609 + 1-1-7-1: URI:SSK:mbxn22dpw3te2bmttgijzza4my:rdkopjhcyuya4mazhiazun5hppykjxpz6n3xmx4npn7x6lwfkw3q + 1-1-7-2: URI:MDMF:ennvchlb4z2h22fcxmfzqfwlt4:yq2hqxdlhenxnxl4pv4bz4cou2waygkaouk42iozctjrue7r27rq + 2-0-0-0: URI:CHK:jdm6ytmx2i3ya2bllsdzxurjtu:iqr6tyysaseegzfurhuywy3mbbwkbbsov5bt3fo6oazpyv7olvda:2:3:1024 + 2-0-0-1: URI:SSK:rfdsbtoh7ig76hmujuyooyvwri:iihu65yi7f2str7eump6iqv6reqatvrffd74tmaunbgzi733vkeq + 2-0-0-2: URI:MDMF:jm6yqftdmdoqxvyuqn5vnx2bsm:5gobkzionvrsjv24fgb4t3o36cgd2oyzwhizq2lwcj4hbrwl2u6q + 2-0-1-0: URI:CHK:7wz5dhkyob3gmkydzgbptk5qku:cl6ovq5a3km7rpfhyb3putg656sp57lhnp7aexbocsbbahdxnfia:2:3:4096 + 2-0-1-1: URI:SSK:utxhffimno7m4u76fscoxhzdym:avszduoxd57rjstwfi2fqatf47yibuy6hnc7z26pvsl53nirw47a + 2-0-1-2: URI:MDMF:fwb5mg3kz2nzmhdzfgg742khp4:fcytu2aox234opw2pipywkapnfl64afxzrhcmi7w7jveihrmoy4q + 2-0-2-0: URI:CHK:aagw6esdm2msphvgnr3rlzg43e:zqxtrlee6hh3vtfg6ihtssjqvr27tpvi6gkmngnhiguttjz7yyja:2:3:131071 + 2-0-2-1: URI:SSK:xgedi566hv3vocftdfywzvw4he:z5lmieqwlswkijomu4mks6qvtepl7e2hrbs7druekieymricw2ua + 2-0-2-2: URI:MDMF:esutc5ohuhcmj726xv66wihjhu:bcvsqlfbqepgwwnucfjeatlxredtblhf2isw6rgztgi4cgu7a7za + 2-0-3-0: URI:CHK:e6uhixvbm3g5amrzdzd3mnmsqm:4n5aafrb64sqpfnhcdpdfrk5gh2qznprjky6q6jybuuqy2q6z6mq:2:3:131073 + 2-0-3-1: URI:SSK:u7o4qtogmvnshwz3wbcslcxiiq:n5jxccr7wj443wjyjiof7grtkst3dxb7g23zhq2vcijzpzi2zpia + 2-0-3-2: URI:MDMF:nctwh42xkr5q5b5gwew4r22gne:zwlchalto2xquywhk7ux2r3dbhdq3btadrz36no2uftlgomjb63a + 2-0-4-0: URI:CHK:abfmzuiczjwt6e67f3uoyg5i7q:rk7ie3afnp3xgvnxu5e2vioq476xqwslqxgyx3i5xxevsjhkcvba:2:3:2097151 + 2-0-4-1: URI:SSK:7liwiw7qfs4f4hv5gpfih4a6gm:6u5homs7iry4ryebmj7ji4meedrxbn5vl5y5jd3rsfo6mp5egwra + 2-0-4-2: URI:MDMF:emh52u7gv3x6r6sc4lj54d6f5i:utoggiwnd5ptkgsw5c4qaftjzbh5w6shbvfc3pja4qfgsppdhieq + 2-0-5-0: URI:CHK:34bzejdzpqinmuzdmqenkidf74:pcecbl4ulygiadgdagonbgqmpb4lomjz4v4vqyssr56reyib4q2q:2:3:2097153 + 2-0-5-1: URI:SSK:kak4npmxsndcrbiry4nr6mwaku:qrilttfrkbyxaaxptpa3s4rydwv57mgarbxmtwz3ouli3b6d3mia + 2-0-5-2: URI:MDMF:htww4ycieablkeuu7nyw3q2phm:hvrth4e5sv6tjwjxsmjfzvsmnqbbwhgceyxz3cr5wuox6e5ajyfq + 2-0-6-0: URI:CHK:ej56vl65kawat62wsxuvbcd2wy:tv65cgghaddq2p7dyvimmff24l34wsx6mhzd3dyprfvrdo52qtma:2:3:8388607 + 2-0-6-1: URI:SSK:ocrry76nfkqgia643p32xfowxa:nnnc7xaknc55iibxhvskraifbx24a6h3q5dpmckkivmjx4eig4ua + 2-0-6-2: URI:MDMF:preyyqadzm55vijzfjqfd27xbm:jv4d74sxklayo2emq7ue6mapeqh7gldzyn7zz2eigosbti2luloq + 2-0-7-0: URI:CHK:ufshkmuewwql2xsyiecbjhh7x4:qoywwygjqrmifowob3mvwkhfskk6geomyw5e3qlgqzui3kymu6nq:2:3:8388609 + 2-0-7-1: URI:SSK:fevaztou3q56upx4uigz4byywe:sd2kfppinkyo5a6ji6y6ftdktuyam2ywg6mqzvrqsesgbjcyhijq + 2-0-7-2: URI:MDMF:d4oaonhqdbb7ct2mtcbguroxpy:bx4xmffvttca6j4sitb74phqppokcvkymx6yzqj2765uk23gssgq + 2-1-0-0: URI:CHK:ld3thziutpaqv25nbtoqebhtru:ri2obsvzl2etyuv3qnchn2wvw5mh5rjuvbjaqkqbvyprn7xwamoq:2:3:1024 + 2-1-0-1: URI:SSK:7lhegvunppwq5bmvay3nce5jt4:uuxdhl6ap4hpvpaeo3jgha727nl5o7ktbrxi6fai7dp55omhlspa + 2-1-0-2: URI:MDMF:ifs2eqqeokipwz4r5zwxbcacq4:7oqu6ex7yidy6bqotojqedik5r3vwkmepp27qez3a23nzofpjpqa + 2-1-1-0: URI:CHK:pz3mga5pvqd4bpsz7pbyna2qtq:w3ptt7xwwgywod3u2rpgx7iggvm2kpwl7d2ryr3u7mcaxgqotlaq:2:3:4096 + 2-1-1-1: URI:SSK:cezgyjtq5ugmh25mryojcdjutu:bu36zk7bsvghs5bm3zdkmllbrlq3vbvcoa7crmnz7ai2oiwjbyqq + 2-1-1-2: URI:MDMF:izdrqjjyk6nyfpoexeqnyhmxcy:54qffrqnpcvfq33qpu7dz3xbmd2dxbiritwtvy7eraujoknskymq + 2-1-2-0: URI:CHK:rsh7ysvsgbwh5g3xuj6z7zky3y:ykg5rhzpzqkvdy53qqyghfbcusibynfylggbhypo3iibsrrzvlkq:2:3:131071 + 2-1-2-1: URI:SSK:xnm6td5a2v65efzx4zyyr4gena:alzhu6j37of275k4u3uovxsi4u57zcarzzw7rozx5fskp4mnz64q + 2-1-2-2: URI:MDMF:76nwwbjf5eslzqf7gxmmu43tdm:7vwv5acs3yr54az5eaqwt5kv7sgrfdoimskxxxbhos5il3bcnbwq + 2-1-3-0: URI:CHK:7xqejfqfej3u3zp2qymqd2iqeq:po4t2tzkh4d34ku6gdhlocadfwdyweiyckyqz57zdi7ndt4ikj3q:2:3:131073 + 2-1-3-1: URI:SSK:uva7lh3b4j42mrcp5utf2lcyxi:3peveaddnwuup7vbouuy6urkyog2ovucim7yv6a66h7hjnrbei3q + 2-1-3-2: URI:MDMF:7lyl6pejhg2gimf5omvjmolckq:ufgtgmrd3k25tzj4zr7rbup3vh4fiwmwljkktymcuminqs3ino3a + 2-1-4-0: URI:CHK:kzhg5zp4hcs75eboack4kqpcdm:g2cwnhpfwxrv3c4lrmpjyj327a274oa5qt4isu4avjixbu7vblrq:2:3:2097151 + 2-1-4-1: URI:SSK:wzqmll7265ckpuaquuu4tipzhi:fyajlm7egfm7i6ijowmt4sgke37uspn7ay7v5mr2tpmnt3hh2m3a + 2-1-4-2: URI:MDMF:dloi4oc6aos4isw6rqhtxzd26e:fugp7o2rss7w26ud3d4o7daj6ors7sofec45q5j4gggvtonuucda + 2-1-5-0: URI:CHK:6sf35smixwvpapf2zw7sllxet4:ks5zd3hkd7ppwobhnymezckszexpychngnkipxfvmlcng5fgjbea:2:3:2097153 + 2-1-5-1: URI:SSK:7mq4vfi5j6jzpszjyboputuv3a:qartnhxp3ii4wyjk6h44vwrzlayj2qj4l3xmiiqj2ryhdnvdzora + 2-1-5-2: URI:MDMF:4j7vlr323wmo3vnmpeaemv4b44:hin5lmydp7b2odgu63uqpbxuron4vewnwle5g6yzgvpitq53tgyq + 2-1-6-0: URI:CHK:dnmjqjku5vyrkeaulae4bt6juu:d47airwaqedeji2omacwsp6yfqp5mpurj776gmzvq43c6dovtdca:2:3:8388607 + 2-1-6-1: URI:SSK:3erqoieayuam2jvu4echcj3azu:hk4ko4bj2nlhjh4kqxx7idxk3xb2vmksitkft43blilhyluyqt5q + 2-1-6-2: URI:MDMF:em5zwqit3eevv2orxfh2i3v4ku:6emaakoo52w5e2yzwufsje5zzkxiz7j6vjxwrbvkyvf24cthzwfa + 2-1-7-0: URI:CHK:bfjedwcwjehjzpwjsgwkfk7rja:jpwea2sgz4hfohqab642yj4cmrh64w6nfo7lv3mtacfzicurqkva:2:3:8388609 + 2-1-7-1: URI:SSK:7edtpkmasd6ehqcxhhfqaaodgu:trmtsw3g6dnvg2ojmmwt4mcvkcd3mzpdbwraqhrhzwboq63xccxq + 2-1-7-2: URI:MDMF:esyeaj2wof5udiw67sqxg2oja4:rggac3zexaunabzpvb36sqimouzchhcvaiigmsvmjrmgb62th5xq + 3-0-0-0: URI:CHK:dx7tvyr2fc4u7lxjc6kehq2svq:tiy4qh2g6lqejxcaym3rr7ymkdkinn4qised6kgxloj7sptsqu4a:3:10:1024 + 3-0-0-1: URI:SSK:6b2saxjadoy2ievtxh4ri7cspi:qmdtijkjdf4er4rooi3ka57oh33whelwecy3euksahu3vep5inaa + 3-0-0-2: URI:MDMF:c4yzz3zuapsmteiavjx56nfc7y:fia2oloiwm62klq3qh7tb5v5rbmwaol6tvpfd37f5ibapf4jyjmq + 3-0-1-0: URI:CHK:yo75evk4cte3b7rdw72zxvl5ye:ex6h7ff7nclucjtsqwgwu33qgmb67t4ezbrki4zbgurwn2ct6bbq:3:10:4096 + 3-0-1-1: URI:SSK:axmognjdoajpylni3kac6b6tlu:lpvdrbkq7qt4hvjkkxbmzdjfehnfp7y3e2qxylmb7v4w6fmyywka + 3-0-1-2: URI:MDMF:uvsqupyfamsvbitsenyexxkz2q:fb44nsshdrscvhuonh5fqmhnpbxugw4evyoxnunyronduv3skwsq + 3-0-2-0: URI:CHK:4gokef54smahrbfr4kq3jhc4zq:owpwwfp5gof2vhly5u6jdnbsfuwwwhqkazpsbeg3nldxv5pse2iq:3:10:131071 + 3-0-2-1: URI:SSK:2wxje7wkktur2ys4e73hholec4:enqfpbenh2sdcugbd22qb5tsqa2x43vzr73y666k3s4ktyl4oq4a + 3-0-2-2: URI:MDMF:2e2vrraewqlffijhfjjm7odqyq:wurnspwx7nqdxota5lzgmjdab4jdcfs6slb5qujljoegm4i3x2ha + 3-0-3-0: URI:CHK:7vfgl5cv4nlzqx35z4uthjv36y:nnueftbzxfz6u5yjxwwofaxzzft7xss5wzfh66rrcwv2zwrm63sa:3:10:131073 + 3-0-3-1: URI:SSK:uaxoqctux3abakjjyi3fojtcvm:y6gqrke2oi5w55pznh63ockqyq7txu5my7fw6mde6f6iw6uqkkjq + 3-0-3-2: URI:MDMF:ko74zoqcprygi5ipc775mavlsu:dbvql5htmnld455yxqbkml7csfku44cfzbuy5mfpxczbx7nisvva + 3-0-4-0: URI:CHK:cy7fmjsldeakhfd4psbmghmqyi:6a6uvyai4jkzz6hj5yjugm6uie5etvymcudgiwwjh47apz636zwa:3:10:2097151 + 3-0-4-1: URI:SSK:ix74qdf4oy6oigb5kcwkjl7lwq:vrbfidc5l3e2u76ztzeaezfem6w5budpmx6paph6vwlkukmmq4ra + 3-0-4-2: URI:MDMF:alpmmh4rfvcytolewejupt6sse:hmtxrp53i75v2mekiqbiw7hgxm2zm5sglcv4bq6v4zhbsr257wda + 3-0-5-0: URI:CHK:ptsofqwylmkvzmuvrw5n34j3ma:ky2fs7xrlke64w6kfmhzsuilzxbhfrwwzkxih4rykpbxrr3bxhiq:3:10:2097153 + 3-0-5-1: URI:SSK:pefhxppdqkfq77zsiwbdr7ye2m:whxta7t67iyvjsr6dia674hwibsr67vz5nwrxaeoedzal5qg6yga + 3-0-5-2: URI:MDMF:ybbbm5l3yr566ki4z4tpnqv4hm:temkyl3jg6xwk7o345pkovmx3mxglefpoof6fzqiiqxsjl4xw2ma + 3-0-6-0: URI:CHK:tyt4hfvn44igztqh5zeiusggcm:ex3gzzn7byvhajithwbo7c3p7qwqxmnactaxxsxsxowkafyjrf7q:3:10:8388607 + 3-0-6-1: URI:SSK:gitwhiiauwxo6xyur47gaevk2q:fsjra53xrguxomizdtftembdo47ypydbahq26jt67yvqylhuyzfq + 3-0-6-2: URI:MDMF:hp3uxpsp73lyy2rz7sznnt7qli:macxhhrmtqswkzcnyfna3dtanwi5xkvibtv7uitlma6sr7nkklga + 3-0-7-0: URI:CHK:xymheose4rdlspgydkzr4nqkre:z3pfrvpq5fdpkoybhdxppwbzrt6ejf26xh6emzlce2sgquljginq:3:10:8388609 + 3-0-7-1: URI:SSK:eunztljebzuw2ja6qafuqesuly:xoqie57c4eqr465nrwihhnhfjmn44qj2jgzn72typ2raph5vizsa + 3-0-7-2: URI:MDMF:ztkemleegq7saj4j3aebzohfju:lxz2564f3tjanxlfscvrjbdm3cqen2gp53w3owmnt7umov6fn2bq + 3-1-0-0: URI:CHK:x55owxzhsezfoayaxe7jpwnove:vawdgtqpxyntgy5i2po2twgelrynkfcjgwm7publnlbdp7hpqmfa:3:10:1024 + 3-1-0-1: URI:SSK:yffv3eigw2bil2zacthvbuecme:r56c6xlljx2ccaqf3dlesz6zi4wd56ekohiou6xu3z5rhquopjaq + 3-1-0-2: URI:MDMF:d5cou2vdhswmfh2tgw55q3ndpy:cpu4fzg4zijyhqdkw6ftghlfkczculab3nbqtzqf3nlezuuzkoiq + 3-1-1-0: URI:CHK:fe64krzyaeff3d4teunjbetkzy:27hrywwaffqiqcgfkmzwbot3iamotr3bey2l5kaladmdmxuaz5ka:3:10:4096 + 3-1-1-1: URI:SSK:hrcs4vs4ae2ji47fx2xe6jfvoy:ga7u6v4cundqk33s4jcnt54d4rihjwosrp5xfx6m3xvqbiardo7a + 3-1-1-2: URI:MDMF:frgfl4n5pppywn6ez72dxryomm:fu5gnu24mwhmldodkcde3b3nk6bu3f7amsc42rpbl6x7dafg4txq + 3-1-2-0: URI:CHK:gvajllsonkuscfemygbnqhq2re:uwyilm5a7so4blhsaielnf34u2qbaqmudd73opjkgodgg3okeaga:3:10:131071 + 3-1-2-1: URI:SSK:otczcexqybjxjobsnw2gpfh7ua:s2ijirfjs4wuzde4qszag6hmdygcmflwi5ywwvqblij5unccjrca + 3-1-2-2: URI:MDMF:7flmqsxaoo44d6xozhvchumi7e:cdy6in2tgiuwieuslgrvsahv4i2mtsapbuk3pztegzqd44phvj3q + 3-1-3-0: URI:CHK:zdmicwopo4p4h4wbfcbnwcrvyi:6qn75anpvs5gls27f4lybisis3udvjfjhatxiny7c72bcbtuztia:3:10:131073 + 3-1-3-1: URI:SSK:4dsb5diiy4c5ph5bilbeorb2y4:2i35prduaaaj7zdojoagjkzlrouk2wvgbqscmfevn6pq6hn2c3ca + 3-1-3-2: URI:MDMF:k5tkdlb3vxjmeq5ocn4k7nn7v4:3bjb4iilbnb2zs3pxyksyac2huau5yiizs43xbtglewlsq5zjswq + 3-1-4-0: URI:CHK:bv6qthmetlhdnwc5tfjqamp3yq:ehz4ttd4g7ktkxvbovt562wfedc6jgnt5c6af7wxgp7jbwfwhoaa:3:10:2097151 + 3-1-4-1: URI:SSK:u4h47atwwwxnildul6hstzjdpq:om4e6ogzgblvgxry3os2gm4qnauiypfjiax4paa4avvawnc7gnka + 3-1-4-2: URI:MDMF:jexmxy22l2s3f5s26q56oozbia:4wt3zuvm5afubpxugo5ieyzy3yhxurmra72tyou6n42aofjauoyq + 3-1-5-0: URI:CHK:n7ogyjbo5jigvxgel5ll6q4vbe:3yjb3zq5hcdavv7ruefawal6euyvjx3lx7quslvasjellv63cxya:3:10:2097153 + 3-1-5-1: URI:SSK:q3ehvqgpiydqljr5baecgfoade:e3qd3antznp3yrudd3vlvfhve57uebfowd6s6adjppiv3h5rexea + 3-1-5-2: URI:MDMF:mdq6e4jzzjcvnnzzdcw373up3y:luvwwba76brnwovmqwuzfdmiowp7suigojmfe7qbk3lkwncjzyhq + 3-1-6-0: URI:CHK:d2hbvcmbex7fm3qu22yj4qnkh4:2xwqxbawwgn773hht6etox3oypvqqjv2orktnthfo2e7vibko7ha:3:10:8388607 + 3-1-6-1: URI:SSK:nlk64f25lbnm42zbycv6ypchea:ziipysr3cwntri6txx634qx7xuaybt3af2fr7afu6bio44wyh2ca + 3-1-6-2: URI:MDMF:5rw4gxpvybgu75rzaes3vgwy7i:7na2hbjq5ejuxv7azwpxemym7ds4nynnzf2xoptpgwdpob5ytecq + 3-1-7-0: URI:CHK:ccxkyfl2qtqyhihduarpxbdcci:e64be3i2t25selbpc5y2zj443gkdo65chs2o4tpqb7axud5lnxta:3:10:8388609 + 3-1-7-1: URI:SSK:gg4bdwy462s4gbqm2omovufnri:zswcwabtlmd2qdjcvj7ltmkkuf7xuiw6vmut4ke4k5gdcl7oqjra + 3-1-7-2: URI:MDMF:7trroesakxfrt3gwoh3m3lifea:ksv6fa5dmxr3epewosbm6hjllfmmbmiznmnh6bgal3nhs4gss6wq + 4-0-0-0: URI:CHK:2wvrzaobtfkoiqeqwryjzvxatq:7zzaycbdhepzaqtdrku2sbb572h64ggwstu47osa2gn4ol3eglmq:71:255:1024 + 4-0-0-1: URI:SSK:hgef47gcwxn4einhqanzspdxsq:px63z6ezudouh7rcurhov7jscvi5p2ri4tjmx2p3sg44z5qgu7ta + 4-0-0-2: URI:MDMF:4y3lzdqx4dtimj4h4oh4pyhnte:x7xlolo4aemt3lmbha6ejjodxyh7wkhhgu6ua3kvdvpfz3t3647a + 4-0-1-0: URI:CHK:7wi5xn2uwerkjzqs5ndn4eimti:llrws4uig2fpi3rtla6lk7m4t6zudjhszdfjgqtihghfrvs2robq:71:255:4096 + 4-0-1-1: URI:SSK:5cseulzm26fheuvquntqqvyyri:j5tuxus63kag2tcfit5b3cgojgvg6hksydeumpnxrr5lcroyemlq + 4-0-1-2: URI:MDMF:tbhr6ffrepvshxvaxh3bvl6if4:cdlp3yeqvt7q4fw547j625j54eiluuebwyx6iikgqoyuscmlotea + 4-0-2-0: URI:CHK:6nvs23bhrpiwiz5prqdtvztujy:wlg7g522rpdoitpm4qwhmctrjhnh4zfloiq6uq4tsvaoawg4slpq:71:255:131071 + 4-0-2-1: URI:SSK:svlkrjk55ti5k3prcpazndiizu:mzewxyzfsfsaufod73awsi5ixcdcu4vnrlcb6y4pk4jzxp6agnwq + 4-0-2-2: URI:MDMF:mwcxdmqtjxrmtk73v6ib3syac4:u4yq4rkluijyy6uxv76enaynjrorwuik4ihljpqtglr4xinrjasa + 4-0-3-0: URI:CHK:62dtbeowncjj62jnwbggaufvbu:6r2sapg2cm6dvmylodyxabrj63a736uouzsqyacnimgo4svnktva:71:255:131073 + 4-0-3-1: URI:SSK:7ybfzwfadqryj65daupc5mtjim:mokqdwv37xdj42yg567jaauazuivcnvpmi4kxubmduwdgqj6qxaa + 4-0-3-2: URI:MDMF:nodj7mjiycqbf4zlb2l5sai6de:ozqrakvrodeb4uc4nndgtbnryacmpzg4e43x6adcvew5l2qvfa3a + 4-0-4-0: URI:CHK:hk3a5ync7y3dnwowqjqoa34eae:y4f5b2xqa3lslh2vti5fdbho2syqhbj2p6f3enlhxsjbfpt7zuta:71:255:2097151 + 4-0-4-1: URI:SSK:pwzgrbwxthdnu2gfjnbycmj5wa:jvhexrr3e4iejx4o3kri7rhhenpqt5h4dtdwev7r2noyrdahehtq + 4-0-4-2: URI:MDMF:sfras4yucfxpqakxx4cqbefvxq:3r7gltmdsev3vx2fvcj7mi34czdqpm5odx5m37yeqnpzek2ijakq + 4-0-5-0: URI:CHK:qa4hwhwtd3cpvut74biixlv6ye:uuj4ujg3mcf3lqpuvvwf5pszcvviyb4cm2jjicu7ugiypojjie4q:71:255:2097153 + 4-0-5-1: URI:SSK:xpwtwzrqhs4pjc4sinxzbulzn4:vamd7m2f7lrhn67ygcuwdsd4oqk5fqzhncr646ut72birecttw5q + 4-0-5-2: URI:MDMF:ud6fekwwtgoivsp6smbs3jweyq:rdnf6kuhxgiirwqthumyerqwuvezw4jtxtqglkseijzpedpugcjq + 4-0-6-0: URI:CHK:r7sv6u3ihzys754twjltuzzpza:tpyoxinubkl4kvg6hipls6ezun7t3yyqj6ix3vp6tvnzqirzzl4a:71:255:8388607 + 4-0-6-1: URI:SSK:fii277xjgusxszdo4jwhdiqwt4:vvx3em5ywpq7hcpbisuyuhpriifhnhttf3l3ntkdgkwtyhsj3s3q + 4-0-6-2: URI:MDMF:xufeawqo56oeitacojihy4hvhq:d4zi53idpc775d63lgvjojobfnmsbtewaty677qgw6aezzckacma + 4-0-7-0: URI:CHK:3esfsjn7csgnyqmq5afbgtinay:xsrhdomrbrtzftg6u4hgipm5tkaomumbdlxi5hpvpvawe3bjikua:71:255:8388609 + 4-0-7-1: URI:SSK:ih26p3vtz73m4vtafxffdkw42a:5rgjtx7z3fjrgwwzls7xmajoos6jkkeh7nnay7yhfdlfk4kyvi2a + 4-0-7-2: URI:MDMF:cj4oi3gmmlufchtmo7uniam6tu:xx5vppvu7py2h7rlknkjclqaal6oi2d2eeehga2cszmdx5egkrda + 4-1-0-0: URI:CHK:wxmzfzvgaqm3xfzgaqrlndgolm:6t3tqcphsutigxnnzyp2xx3afklcpeytnh543mpqptquvqqahwea:71:255:1024 + 4-1-0-1: URI:SSK:cbacdsnd5p6dnjjb7ds7ngs5ly:xrq3fhpz5m2yhhlsk7qon6ljcb7zmqqzzeeuf57jnmcpshfiwowa + 4-1-0-2: URI:MDMF:76zgyx5geis4pkrmbrixtmhjom:7r7ewpfiz3en25ffacqelntevvxx6oyldwh2b2l2yc25rxa7dqqa + 4-1-1-0: URI:CHK:r3q3q4f4zbsipxauq6knb5dfoq:lkae5mflammzxkoc7gllt6qqyg5gqdxeejmxtgcscblqh5tv5obq:71:255:4096 + 4-1-1-1: URI:SSK:icp52tuqpe5at2ugh36hsch2ki:ntip6j5ifu7ysm5m3rauy32rxdhfsubl7nsibpyhufatmxqnfxqa + 4-1-1-2: URI:MDMF:57cjkmrvqwi4a2oswkq7vzuhga:ez6div27j3azyfbsnycrgwxklylse3zexprqlt7tcd7hbihc2vxq + 4-1-2-0: URI:CHK:rdrzetasw4w7tuweqpev65d5vq:gzzb2v5fzrduk6kx5xx27mvo6dgnd65ym5lm7re53in7xuudhsxa:71:255:131071 + 4-1-2-1: URI:SSK:grmx2nfcvfnhh7iljncp2aff6e:sj2znocnowgarswtnmvfc3mkawlxahv2vm7nq365jt2uti2x5adq + 4-1-2-2: URI:MDMF:fnzfrawb63fofzivusnbmzrsem:s6ol6wea5nwt25bql33kwlwwkhgrh7ubfgqr2rz2foamz4yzbfda + 4-1-3-0: URI:CHK:e2bjefibvz4tgu6jgf66gw74bq:mugtu5bx7h5gcivevmh2gmwoc2kkhmobrzshkuj2dgrtm3siysfq:71:255:131073 + 4-1-3-1: URI:SSK:74zkfoe364z5wp43uanq4sjctm:uihfuthtjz5uaa22bn2ehbsqfsdga6xryztmce3hbl5vf2kzesuq + 4-1-3-2: URI:MDMF:posrxmsietgukgtamv5kjxi3qy:eo5w2yiwdmgo2xvozuitqxf5xfxleppdanh6t4w4ikzcift7hfca + 4-1-4-0: URI:CHK:gy7bci6yvllxzvhh45khcwp3u4:qsfjfmcl64zey4k4o5cfnh2i3mzboahf7bhtggceszrulsv537vq:71:255:2097151 + 4-1-4-1: URI:SSK:cmt4ks4vzuyeu5uh35wm3zirmu:5k3gawnpqcvokrmq72nddhuuho5sqhzqirkoi52uwyzdnidkbk2a + 4-1-4-2: URI:MDMF:2dhxtuv3mkrqc5dfdjulny57au:ptkxn6eemtivpczwoxoxvzivv7dyzn6q5z4unr3dmvdy25ymnxqq + 4-1-5-0: URI:CHK:pcta7uk5cpioxzv5nxxqsogw3q:gnu7fx56k3j72bbevirn4kdu27yfpvttzl3qk2zypwqt7tw4rijq:71:255:2097153 + 4-1-5-1: URI:SSK:aww7jpirjcubd2psmrnnx2vieu:m732exwqik55a7r6hf36vjyomi3w537am3e442xotme2iqdmsogq + 4-1-5-2: URI:MDMF:xmjottltglnlcccs247rmrylue:q2tgyaiuymz5z4m534a627uszdvi5mlrwytyvh7iefr4n4fw33gq + 4-1-6-0: URI:CHK:znxyqo2qglpkhjafnuesu4foge:trau477bj5u5wjfpy4oq5e3aziajgbpyfjdv4svyf27m7atpa34a:71:255:8388607 + 4-1-6-1: URI:SSK:qlbkffx3umudmmyf7pjaou42zy:whzwp6jz7stuw3ketnkvisr2n4t2qb7myaa3ex6q3stjfc5ixj3q + 4-1-6-2: URI:MDMF:2yozfuwqq4h6f45d56wqq7c7hy:4xokulr4nkgu5t72ndf6cdegitgaojyybv77k4btumlth3lypcma + 4-1-7-0: URI:CHK:mlzs2hbztak5fxjkrkuuvnpdpe:3myvisewm5uklimp2xucrwep75sm2rizfi2sq5drhlqjszkpdyeq:71:255:8388609 + 4-1-7-1: URI:SSK:jxz73uawqyauzohhk4q5xroyxe:d3w663ufccfoxxfdqw7dvezepyxzsotkz644l6a4izturxbhqmja + 4-1-7-2: URI:MDMF:gkxsktuwbhkqcwwxp52n3zj2na:lrwgbajbsspxpg6ov3otn52wk5rtln2v74n4xzozhr4ukg4mosga + 5-0-0-0: URI:CHK:mftnvh5l3brshwoupvu6kz35cy:sncfokby2tjykbr3zgi3i6lhgerg4kz7fsrslrlxdy7upwkzq6lq:101:256:1024 + 5-0-0-1: URI:SSK:un7ljbwxsvpcvyzxhnqyagjttq:ds52mgfwg3gw6fhhdoihrxwzaye4i4tpugddyx4r5reebbl2gpwa + 5-0-0-2: URI:MDMF:acn6rtbcds6uivf7a7o6z4tzri:jvvkm56qlueoz4pnbyj5fupxcw22cnuhnhyoeytt3tfp4ib5czda + 5-0-1-0: URI:CHK:sshu77h6opnto3jnngot3lwzl4:4cjhp6u3i2uzdzrjitje3inlhs6gnlgwc6wgi6vtedc2grgvcpza:101:256:4096 + 5-0-1-1: URI:SSK:n5nwguxxp3d6kh3i3ewrm77zxi:be3ivcy544d6s2ggzkw2ql4yeg5sf4feeimi7vzdglk5kilnkzga + 5-0-1-2: URI:MDMF:bvgdxytmeefqcx7x6pf3yki44a:p7sxdwfsl7qeklsrrijtubhput5pinm7x7jaeu2klt3vb4qytz6a + 5-0-2-0: URI:CHK:kl2jayrstlf5q3gax7o65ybivy:mnn5doti4wllssdg35ymojuiv4hd2ir7i6asurgms3ea67watm7a:101:256:131071 + 5-0-2-1: URI:SSK:7w3snvdpns7ea7evpbbvc43y6i:r7o7furnq3ma7esaqz65mqwwb4m3bbrndlyb5dbtb4atv2vvkksq + 5-0-2-2: URI:MDMF:iadp6yjiwyxgrzzedbiczb33iu:yv4yvkoizk5ox3ddfa2eyitemgkwqnd6jy67dxygkels6h74yxla + 5-0-3-0: URI:CHK:a6wuekh5mynr4c74ma3b2gswfa:b6lovwnbv3523n5kaxga3ruxa2dvxef6h4a2jnl6lasn4fwvvdba:101:256:131073 + 5-0-3-1: URI:SSK:6elghkg7p45xhyq474wqgipmi4:ud3wk2gts4n4j2eczhiuabhtf4wlolexnsfnmhgmw2gg4zpzncha + 5-0-3-2: URI:MDMF:gkeqwnqd57gudytjtim577en2a:5vr5lj7kmgfrl64sbsqnqgs5m66drexvh5dseejptni4j7bo3dwq + 5-0-4-0: URI:CHK:pmuo6vpcodol76sexq7s2ojej4:irp6rxnkgwbn3h4g5fzgjpqgsvhvycxobr3xrk3zr44r64wimgqa:101:256:2097151 + 5-0-4-1: URI:SSK:u24u3hw3qfmd2xki3hqsfpmwaq:le2qsy255qbusbxhdbpupofutipobez6da56ot6c7v6pndb7rw6q + 5-0-4-2: URI:MDMF:27obnopt2uprbfoltizn5nxhda:as4b7sendu2gu3qbttnnxk73revv3fw2nslc34flggnpidkg2gra + 5-0-5-0: URI:CHK:fh4boug3wkciqhkgpusjsjyzbe:sy5lwooxchqu5mrzwr3mgc7qtwfjgmpz2srdmwpaip4iuj3tn4vq:101:256:2097153 + 5-0-5-1: URI:SSK:p44gzfrmpjvwvv7rdioal3tcay:r5agbss2y72szn2rzqqninppyej2hip4aysp6uffv35e472nnbza + 5-0-5-2: URI:MDMF:3kdkkcykmxxm5uws4ro6b6xw4m:gomvjpqavm3vk6plkycpzxyhbx2t4g53vb62gm2tptpi3gcs5hea + 5-0-6-0: URI:CHK:asmjgajvwpwvylxdcf5rm2gyd4:imwtdi4eqq3wm4bvk6npho3ta2fewceqzl4f7wyyabg5i4s6h3mq:101:256:8388607 + 5-0-6-1: URI:SSK:sdtbn4trlyju5aimti6cfvhjxa:icvxmxt7vbab4mkchpw2q2pbz6pvftw3tyzu4rdjhysgw63ft47a + 5-0-6-2: URI:MDMF:a3zqfcnhd2vvyxk52sch2gjxii:nfaiz3lznarkiqpym6hzxjalcidvm3cb4pd4aizt4jote6ildzsq + 5-0-7-0: URI:CHK:yrznm2uuybfbda6jlj7tqs3yum:zy45lpc7ilfzgltxxqvy55wcokiay2fbbbv2wdvu6ubjluh4gf4a:101:256:8388609 + 5-0-7-1: URI:SSK:kl6hw2qywih3hb6snjo4ti3muy:dnwcw3wj26jx7t3z4zt37xhrmwui2gqtjjw537qjdsuropboe5vq + 5-0-7-2: URI:MDMF:sfs3c7wvdjxxvtb4h3lgn6qmve:6vwlktezoopvhd44f6qr35ffwnvxkbrexio5cdgtyol5h5luc3fq + 5-1-0-0: URI:CHK:ihyeqq5peg2pdyw45ct5mmj4ay:2rzhndv6o4knlvo4po6rvbnknlcitasseea3mirqjor5c7quh5ra:101:256:1024 + 5-1-0-1: URI:SSK:nsfj767mvco6cwxvfgcarl3ok4:rniviz4xd5aprg6bkozeqr5ubh5pm7qw3vimw35vlp335lff4bka + 5-1-0-2: URI:MDMF:jtjo4rpjsfsmj45anozydonuci:bk5beemlcpagv34l76wuijomr2s22stpdyszdhxrl3hbnfwxth7a + 5-1-1-0: URI:CHK:mvk5f2mghudk7xnknqddhvnl4m:c74myeugtix7e7l6uwybtrdtfselha2qrcc7hhbymqr6brtg4cea:101:256:4096 + 5-1-1-1: URI:SSK:a347ijcfid4ko4ndqmzukib3nm:f656xyo2ghvx2gp4gotwfbjtbztxv6qofn6egiyfsdpe2zitij5a + 5-1-1-2: URI:MDMF:3pmna4pbpcprn5bcrrprb2hlyi:h5ohpuh4xe5zwabi3xtgilcj4kykw7qhfrfsuy5dug6oessdrs5q + 5-1-2-0: URI:CHK:rvytwcc4sbmmkrdwh6b23ckv5u:klk5nqpbv37pr2qavcebcugfnikarjdefjlseo3ru5uv7guai22q:101:256:131071 + 5-1-2-1: URI:SSK:d7eaudatu6r47pp5zpimcjvcde:mit3ynp7gje26ejulvsxaessydzetkddscmgcktyzu4kdddtxceq + 5-1-2-2: URI:MDMF:hvsxlqkcu7m6srvudrdf667pp4:jh4cjypstjtkrsid3wwzaph52qbaxhx3aephrdwbxu6fd7unxkna + 5-1-3-0: URI:CHK:ocqx55dopakqnzyui6q6euz4gm:wklziyoctwrz5zjfg5wfejz4bnaagtg57fgjlspl4ba6ij6n2obq:101:256:131073 + 5-1-3-1: URI:SSK:6mxjz6yydgyeefq4mwvc56shti:wxw3d7e5jmtragswb42kgt4dffdjcpe2yzq2wvnx7rcwjuhjiy6q + 5-1-3-2: URI:MDMF:3mcv72otyu2ih3qgpt6tqizdai:j45sk3r4izog5vmae5f232q3ilnyoyoenavy6jlz4uytxjzl2jlq + 5-1-4-0: URI:CHK:lf6qub3bkdhfozpdis4bmorj7q:zte3u3ohbuom3iqjjx4odrjylcpw5myw4n4kwf4zcaif36vngf6q:101:256:2097151 + 5-1-4-1: URI:SSK:zbm3epfnv7pqdjtkn2d4igkora:yqei62xxffh5fpibml5uvptfw3e2rjf7po4l2nmh4nlocw6lczgq + 5-1-4-2: URI:MDMF:qgi3t2uwvvfjur7vi23h23xyce:k2mhiduzfdfkrsoo5jy4lhuxsjlqced6fz4tftu74hgqqjewy7tq + 5-1-5-0: URI:CHK:a2qoww5xx6so5uftwg4msdtrpy:h7hwsjco5i3jkd7icytrtsuzcjvjzlnkr2wwau2tjlj5wehjrokq:101:256:2097153 + 5-1-5-1: URI:SSK:vw3rhtwjip7bsfysysvwokkhqm:5ujsp6l2ywkqxmwebyznyszdtyttmzcymq5yz7lqjt54dnhezmaq + 5-1-5-2: URI:MDMF:qov2vzt2rv2woudodsblpban4a:upem3xr63i6kqeq3ku4dval7iwybsnnf2trisb7zus7igbuniyuq + 5-1-6-0: URI:CHK:tg5w4m6hi5ezeapuix3lrrbbtq:7kerk5plqpsd2upnole3ifxoegxdux7prazbvzeu2kyilc6idk7a:101:256:8388607 + 5-1-6-1: URI:SSK:xieqbrwa2s3vmxj4tg4lrf2qea:bphqgbbtytmbknp33wgtu4ezktpbkicoly5o3skiaadghcdxmlea + 5-1-6-2: URI:MDMF:qfrl5z67reetjmnufz4imgz5cq:aomtulzpd63u22be44vk3w2rxi3vp6cx4a7xnsjjohp7qkojyaza + 5-1-7-0: URI:CHK:ioridipksj6o7ph3bcf37vslmi:pzhkwm4ry67ohj6dcs7zwnsjrvfqzam6tuitrns5vuo3uoomau6q:101:256:8388609 + 5-1-7-1: URI:SSK:amruym46dleng2rjwhwxxgqyba:gkrjlcu2g2aycgdhp3alr754b5sm6vztoalqw3evhsfidgrv2saa + 5-1-7-2: URI:MDMF:h5gk2z5fz732jpvrjq6jmayk7m:dkbh4h5s2glge5mskgq6jzkzhhcq53xt5e4e7hv6em7woufi2q4a +version: '2022-12-26' From aecaaa2426565b94e2a10b96cf888a4d596c32b4 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 27 Dec 2022 09:01:33 -0500 Subject: [PATCH 1311/2309] in general, do not regenerate the test vectors --- integration/test_vectors.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration/test_vectors.py b/integration/test_vectors.py index 261970b17..97af016a1 100644 --- a/integration/test_vectors.py +++ b/integration/test_vectors.py @@ -135,7 +135,7 @@ async def test_capability(reactor, request, alice, params_idx, convergence_idx, @ensureDeferred -async def test_generate(reactor, request, alice): +async def skiptest_generate(reactor, request, alice): """ This is a helper for generating the test vectors. From e11b589eba01447645e635c5f326fba6e5ee405f Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 27 Dec 2022 09:02:43 -0500 Subject: [PATCH 1312/2309] typo --- integration/test_vectors.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration/test_vectors.py b/integration/test_vectors.py index 97af016a1..9273d408a 100644 --- a/integration/test_vectors.py +++ b/integration/test_vectors.py @@ -190,7 +190,7 @@ async def generate( :param space: An iterator of coordinates in the test vector space for which to generate values. The elements of each tuple give indexes into - ZFEC_PARAMS, CONVERGENCE_SECRETS, OBJECT_DESCRIPTIONS, and FORMTS. + ZFEC_PARAMS, CONVERGENCE_SECRETS, OBJECT_DESCRIPTIONS, and FORMATS. :return: The yield values are two-tuples describing a test vector. The first element is a string describing a case and the second element is From 6a1a2fb7058fede8c3d309feff46143e1b330a10 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 27 Dec 2022 09:03:01 -0500 Subject: [PATCH 1313/2309] we support other capability types now --- integration/test_vectors.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration/test_vectors.py b/integration/test_vectors.py index 9273d408a..a075d6e08 100644 --- a/integration/test_vectors.py +++ b/integration/test_vectors.py @@ -194,7 +194,7 @@ async def generate( :return: The yield values are two-tuples describing a test vector. The first element is a string describing a case and the second element is - the CHK capability for that case. + the capability for that case. """ # Share placement doesn't affect the resulting capability. For maximum # reliability, be happy if we can put shares anywhere From 13a9ed02026c7792851d475f367d9ee6534b59a5 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 27 Dec 2022 09:03:24 -0500 Subject: [PATCH 1314/2309] clarify what reliability we hope for --- integration/test_vectors.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration/test_vectors.py b/integration/test_vectors.py index a075d6e08..70fbe24a4 100644 --- a/integration/test_vectors.py +++ b/integration/test_vectors.py @@ -197,7 +197,7 @@ async def generate( the capability for that case. """ # Share placement doesn't affect the resulting capability. For maximum - # reliability, be happy if we can put shares anywhere + # reliability of this generator, be happy if we can put shares anywhere happy = 1 node_key = (None, None) for params_idx, secret_idx, data_idx, fmt_idx in space: From 3f8f715aa2a827a0c9a565c56b552ecf983c7057 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 27 Dec 2022 09:12:34 -0500 Subject: [PATCH 1315/2309] Be consistent between the test and the data source --- integration/test_vectors.py | 4 ++-- integration/vectors.py | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/integration/test_vectors.py b/integration/test_vectors.py index 70fbe24a4..9bb362947 100644 --- a/integration/test_vectors.py +++ b/integration/test_vectors.py @@ -130,7 +130,7 @@ async def test_capability(reactor, request, alice, params_idx, convergence_idx, actual = upload(alice, case.fmt, case.data) # compare the resulting cap to the expected result - expected = vectors.capabilities[case.key] + expected = vectors.capabilities["vector"][case.key] assert actual == expected @@ -156,7 +156,7 @@ async def skiptest_generate(reactor, request, alice): insert, {}, ) - with vectors.CHK_PATH.open("w") as f: + with vectors.DATA_PATH.open("w") as f: f.write(safe_dump({ "version": "2022-12-26", "params": { diff --git a/integration/vectors.py b/integration/vectors.py index e60c4497a..3224e5c6b 100644 --- a/integration/vectors.py +++ b/integration/vectors.py @@ -9,10 +9,10 @@ A module that loads pre-generated test vectors. from yaml import safe_load from pathlib import Path -CHK_PATH: Path = Path(__file__).parent / "test_vectors.yaml" +DATA_PATH: Path = Path(__file__).parent / "test_vectors.yaml" try: - with CHK_PATH.open() as f: - chk: dict[str, str] = safe_load(f) + with DATA_PATH.open() as f: + capabilities: dict[str, str] = safe_load(f) except FileNotFoundError: - chk = {} + capabilities = {} From 77e5357a04319d1fc8fb472be4cd810629956370 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 27 Dec 2022 09:12:56 -0500 Subject: [PATCH 1316/2309] note to self --- integration/util.py | 1 + 1 file changed, 1 insertion(+) diff --git a/integration/util.py b/integration/util.py index 2db6ac391..a2c854471 100644 --- a/integration/util.py +++ b/integration/util.py @@ -162,6 +162,7 @@ def _cleanup_tahoe_process(tahoe_transport, exited, allow_missing=False): try: _cleanup_tahoe_process_async(tahoe_transport, allow_missing=allow_missing) except ProcessExitedAlready: + # XXX is this wait logic right? print("signaled, blocking on exit") block_with_timeout(exited, reactor) print("exited, goodbye") From 40eff1523e446569ea9ed8a933f394dcfd15205d Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 2 Jan 2023 16:23:06 -0500 Subject: [PATCH 1317/2309] The retry logic was removed a few revisions ago The uploads failed because of the zfec parameters, not because of unreliable localhost networking that might go away when retried. --- integration/test_vectors.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/integration/test_vectors.py b/integration/test_vectors.py index 9bb362947..3d8b8ca85 100644 --- a/integration/test_vectors.py +++ b/integration/test_vectors.py @@ -26,10 +26,6 @@ def hexdigest(bs: bytes) -> str: return sha256(bs).hexdigest() -# Sometimes upload fail spuriously... -RETRIES = 3 - - # Just a couple convergence secrets. The only thing we do with this value is # feed it into a tagged hash. It certainly makes a difference to the output # but the hash should destroy any structure in the input so it doesn't seem From 15e22dcc52c73721f05a2bc48f84c9b5d21b005e Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 2 Jan 2023 19:29:13 -0500 Subject: [PATCH 1318/2309] Add `keypair` to `NodeMaker.create_mutable_file` Previously `NodeMaker` always took responsibility for generating a keypair to use. Now the caller may supply one. --- src/allmydata/crypto/rsa.py | 14 +++++--------- src/allmydata/nodemaker.py | 19 +++++++++---------- src/allmydata/test/mutable/test_filenode.py | 11 +++++++++++ src/allmydata/test/test_dirnode.py | 3 ++- src/allmydata/test/web/test_web.py | 5 ++++- 5 files changed, 31 insertions(+), 21 deletions(-) diff --git a/src/allmydata/crypto/rsa.py b/src/allmydata/crypto/rsa.py index 95cf01413..7ea4e6c13 100644 --- a/src/allmydata/crypto/rsa.py +++ b/src/allmydata/crypto/rsa.py @@ -9,17 +9,11 @@ features of any objects that `cryptography` documents. That is, the public and private keys are opaque objects; DO NOT depend on any of their methods. - -Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals -from future.utils import PY2 -if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 +from __future__ import annotations + +from typing import TypeVar from cryptography.exceptions import InvalidSignature from cryptography.hazmat.backends import default_backend @@ -30,6 +24,8 @@ from cryptography.hazmat.primitives.serialization import load_der_private_key, l from allmydata.crypto.error import BadSignature +PublicKey = TypeVar("PublicKey", bound=rsa.RSAPublicKey) +PrivateKey = TypeVar("PrivateKey", bound=rsa.RSAPrivateKey) # This is the value that was used by `pycryptopp`, and we must continue to use it for # both backwards compatibility and interoperability. diff --git a/src/allmydata/nodemaker.py b/src/allmydata/nodemaker.py index 23ba4b451..1b7ea5f45 100644 --- a/src/allmydata/nodemaker.py +++ b/src/allmydata/nodemaker.py @@ -1,17 +1,12 @@ """ -Ported to Python 3. +Create file nodes of various types. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 +from __future__ import annotations import weakref from zope.interface import implementer +from twisted.internet.defer import succeed from allmydata.util.assertutil import precondition from allmydata.interfaces import INodeMaker from allmydata.immutable.literal import LiteralFileNode @@ -22,6 +17,7 @@ from allmydata.mutable.publish import MutableData from allmydata.dirnode import DirectoryNode, pack_children from allmydata.unknown import UnknownNode from allmydata.blacklist import ProhibitedNode +from allmydata.crypto.rsa import PublicKey, PrivateKey from allmydata import uri @@ -126,12 +122,15 @@ class NodeMaker(object): return self._create_dirnode(filenode) return None - def create_mutable_file(self, contents=None, version=None): + def create_mutable_file(self, contents=None, version=None, keypair: tuple[PublicKey, PrivateKey] | None = None): if version is None: version = self.mutable_file_default n = MutableFileNode(self.storage_broker, self.secret_holder, self.default_encoding_parameters, self.history) - d = self.key_generator.generate() + if keypair is None: + d = self.key_generator.generate() + else: + d = succeed(keypair) d.addCallback(n.create_with_keys, contents, version=version) d.addCallback(lambda res: n) return d diff --git a/src/allmydata/test/mutable/test_filenode.py b/src/allmydata/test/mutable/test_filenode.py index 579734433..6c00e4420 100644 --- a/src/allmydata/test/mutable/test_filenode.py +++ b/src/allmydata/test/mutable/test_filenode.py @@ -30,6 +30,7 @@ from allmydata.mutable.publish import MutableData from ..test_download import PausingConsumer, PausingAndStoppingConsumer, \ StoppingConsumer, ImmediatelyStoppingConsumer from .. import common_util as testutil +from ...crypto.rsa import create_signing_keypair from .util import ( FakeStorage, make_nodemaker_with_peers, @@ -65,6 +66,16 @@ class Filenode(AsyncBrokenTestCase, testutil.ShouldFailMixin): d.addCallback(_created) return d + async def test_create_with_keypair(self): + """ + An SDMF can be created using a given keypair. + """ + (priv, pub) = create_signing_keypair(2048) + node = await self.nodemaker.create_mutable_file(keypair=(pub, priv)) + self.assertThat( + (node.get_privkey(), node.get_pubkey()), + Equals((priv, pub)), + ) def test_create_mdmf(self): d = self.nodemaker.create_mutable_file(version=MDMF_VERSION) diff --git a/src/allmydata/test/test_dirnode.py b/src/allmydata/test/test_dirnode.py index 67d331430..2319e3ce1 100644 --- a/src/allmydata/test/test_dirnode.py +++ b/src/allmydata/test/test_dirnode.py @@ -1619,7 +1619,8 @@ class FakeMutableFile(object): # type: ignore # incomplete implementation return defer.succeed(None) class FakeNodeMaker(NodeMaker): - def create_mutable_file(self, contents=b"", keysize=None, version=None): + def create_mutable_file(self, contents=b"", keysize=None, version=None, keypair=None): + assert keypair is None, "FakeNodeMaker does not support externally supplied keypairs" return defer.succeed(FakeMutableFile(contents)) class FakeClient2(_Client): # type: ignore # tahoe-lafs/ticket/3573 diff --git a/src/allmydata/test/web/test_web.py b/src/allmydata/test/web/test_web.py index 03cd6e560..4fc9b37e5 100644 --- a/src/allmydata/test/web/test_web.py +++ b/src/allmydata/test/web/test_web.py @@ -102,7 +102,10 @@ class FakeNodeMaker(NodeMaker): self.encoding_params, None, self.all_contents).init_from_cap(cap) def create_mutable_file(self, contents=b"", keysize=None, - version=SDMF_VERSION): + version=SDMF_VERSION, + keypair=None, + ): + assert keypair is None, "FakeNodeMaker does not support externally supplied keypairs" n = FakeMutableFileNode(None, None, self.encoding_params, None, self.all_contents) return n.create(contents, version=version) From 23f2d8b019578b4ae9706b2e143ca1bfe44f57b1 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 3 Jan 2023 10:28:32 -0500 Subject: [PATCH 1319/2309] add some type annotations to allmydata.crypto.rsa --- src/allmydata/crypto/rsa.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/allmydata/crypto/rsa.py b/src/allmydata/crypto/rsa.py index 7ea4e6c13..a4b2090a0 100644 --- a/src/allmydata/crypto/rsa.py +++ b/src/allmydata/crypto/rsa.py @@ -42,12 +42,12 @@ RSA_PADDING = padding.PSS( -def create_signing_keypair(key_size): +def create_signing_keypair(key_size: int) -> tuple[PrivateKey, PublicKey]: """ Create a new RSA signing (private) keypair from scratch. Can be used with `sign_data` function. - :param int key_size: length of key in bits + :param key_size: length of key in bits :returns: 2-tuple of (private_key, public_key) """ @@ -59,12 +59,12 @@ def create_signing_keypair(key_size): return priv_key, priv_key.public_key() -def create_signing_keypair_from_string(private_key_der): +def create_signing_keypair_from_string(private_key_der: bytes) -> tuple[PrivateKey, PublicKey]: """ Create an RSA signing (private) key from previously serialized private key bytes. - :param bytes private_key_der: blob as returned from `der_string_from_signing_keypair` + :param private_key_der: blob as returned from `der_string_from_signing_keypair` :returns: 2-tuple of (private_key, public_key) """ @@ -84,7 +84,7 @@ def create_signing_keypair_from_string(private_key_der): return priv_key, priv_key.public_key() -def der_string_from_signing_key(private_key): +def der_string_from_signing_key(private_key: PrivateKey) -> bytes: """ Serializes a given RSA private key to a DER string @@ -101,7 +101,7 @@ def der_string_from_signing_key(private_key): ) -def der_string_from_verifying_key(public_key): +def der_string_from_verifying_key(public_key: PublicKey) -> bytes: """ Serializes a given RSA public key to a DER string. @@ -117,7 +117,7 @@ def der_string_from_verifying_key(public_key): ) -def create_verifying_key_from_string(public_key_der): +def create_verifying_key_from_string(public_key_der: bytes) -> PublicKey: """ Create an RSA verifying key from a previously serialized public key @@ -133,12 +133,12 @@ def create_verifying_key_from_string(public_key_der): return pub_key -def sign_data(private_key, data): +def sign_data(private_key: PrivateKey, data: bytes) -> bytes: """ :param private_key: the private part of a keypair returned from `create_signing_keypair_from_string` or `create_signing_keypair` - :param bytes data: the bytes to sign + :param data: the bytes to sign :returns: bytes which are a signature of the bytes given as `data`. """ @@ -149,7 +149,7 @@ def sign_data(private_key, data): hashes.SHA256(), ) -def verify_signature(public_key, alleged_signature, data): +def verify_signature(public_key: PublicKey, alleged_signature: bytes, data: bytes) -> None: """ :param public_key: a verifying key, returned from `create_verifying_key_from_string` or `create_verifying_key_from_private_key` @@ -169,7 +169,7 @@ def verify_signature(public_key, alleged_signature, data): raise BadSignature() -def _validate_public_key(public_key): +def _validate_public_key(public_key: PublicKey) -> None: """ Internal helper. Checks that `public_key` is a valid cryptography object @@ -180,7 +180,7 @@ def _validate_public_key(public_key): ) -def _validate_private_key(private_key): +def _validate_private_key(private_key: PrivateKey) -> None: """ Internal helper. Checks that `public_key` is a valid cryptography object From f6d9c335261c3f7f9b1ef61fa26e7559a16af141 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 3 Jan 2023 10:28:59 -0500 Subject: [PATCH 1320/2309] Give slightly better error messages from rsa key validation failure --- src/allmydata/crypto/rsa.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/crypto/rsa.py b/src/allmydata/crypto/rsa.py index a4b2090a0..5acc59ab2 100644 --- a/src/allmydata/crypto/rsa.py +++ b/src/allmydata/crypto/rsa.py @@ -176,7 +176,7 @@ def _validate_public_key(public_key: PublicKey) -> None: """ if not isinstance(public_key, rsa.RSAPublicKey): raise ValueError( - "public_key must be an RSAPublicKey" + f"public_key must be an RSAPublicKey not {type(public_key)}" ) @@ -187,5 +187,5 @@ def _validate_private_key(private_key: PrivateKey) -> None: """ if not isinstance(private_key, rsa.RSAPrivateKey): raise ValueError( - "private_key must be an RSAPrivateKey" + f"private_key must be an RSAPrivateKey not {type(private_key)}" ) From 6b58b6667786ec712c48486d2711d70868c1fa2b Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 3 Jan 2023 10:32:03 -0500 Subject: [PATCH 1321/2309] Clean up some Python 2 remnants --- src/allmydata/web/filenode.py | 24 +++++++----------------- 1 file changed, 7 insertions(+), 17 deletions(-) diff --git a/src/allmydata/web/filenode.py b/src/allmydata/web/filenode.py index dd793888e..678078ba3 100644 --- a/src/allmydata/web/filenode.py +++ b/src/allmydata/web/filenode.py @@ -1,23 +1,13 @@ """ Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals +from __future__ import annotations -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, max, min # noqa: F401 - # Use native unicode() as str() to prevent leaking futurebytes in ways that - # break string formattin. - from past.builtins import unicode as str -from past.builtins import long from twisted.web import http, static from twisted.internet import defer from twisted.web.resource import ( - Resource, # note: Resource is an old-style class + Resource, ErrorPage, ) @@ -395,7 +385,7 @@ class FileDownloader(Resource, object): # list of (first,last) inclusive range tuples. filesize = self.filenode.get_size() - assert isinstance(filesize, (int,long)), filesize + assert isinstance(filesize, int), filesize try: # byte-ranges-specifier @@ -408,19 +398,19 @@ class FileDownloader(Resource, object): if first == '': # suffix-byte-range-spec - first = filesize - long(last) + first = filesize - int(last) last = filesize - 1 else: # byte-range-spec # first-byte-pos - first = long(first) + first = int(first) # last-byte-pos if last == '': last = filesize - 1 else: - last = long(last) + last = int(last) if last < first: raise ValueError @@ -456,7 +446,7 @@ class FileDownloader(Resource, object): b'attachment; filename="%s"' % self.filename) filesize = self.filenode.get_size() - assert isinstance(filesize, (int,long)), filesize + assert isinstance(filesize, int), filesize first, size = 0, None contentsize = filesize req.setHeader("accept-ranges", "bytes") From a58d8a567af39f0ab86a2d28794999a84c9b0cf0 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 3 Jan 2023 10:33:37 -0500 Subject: [PATCH 1322/2309] Clean up some more Python 2 remnants --- src/allmydata/mutable/common.py | 9 +- src/allmydata/mutable/filenode.py | 9 +- src/allmydata/mutable/retrieve.py | 10 +- src/allmydata/mutable/servermap.py | 20 +-- src/allmydata/test/_win_subprocess.py | 210 -------------------------- src/allmydata/test/common.py | 25 +-- 6 files changed, 13 insertions(+), 270 deletions(-) delete mode 100644 src/allmydata/test/_win_subprocess.py diff --git a/src/allmydata/mutable/common.py b/src/allmydata/mutable/common.py index 87951c7b2..a2e482d3c 100644 --- a/src/allmydata/mutable/common.py +++ b/src/allmydata/mutable/common.py @@ -1,14 +1,7 @@ """ Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals - -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 +from __future__ import annotations MODE_CHECK = "MODE_CHECK" # query all peers MODE_ANYTHING = "MODE_ANYTHING" # one recoverable version diff --git a/src/allmydata/mutable/filenode.py b/src/allmydata/mutable/filenode.py index cd8cb0dc7..99fdcc085 100644 --- a/src/allmydata/mutable/filenode.py +++ b/src/allmydata/mutable/filenode.py @@ -1,14 +1,7 @@ """ Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals - -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 +from __future__ import annotations import random diff --git a/src/allmydata/mutable/retrieve.py b/src/allmydata/mutable/retrieve.py index 32aaa72e5..efb2c0f85 100644 --- a/src/allmydata/mutable/retrieve.py +++ b/src/allmydata/mutable/retrieve.py @@ -1,15 +1,7 @@ """ Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals - -from future.utils import PY2 -if PY2: - # Don't import bytes and str, to prevent API leakage - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, dict, list, object, range, max, min # noqa: F401 +from __future__ import annotations import time diff --git a/src/allmydata/mutable/servermap.py b/src/allmydata/mutable/servermap.py index 211b1fc16..cd220ce0f 100644 --- a/src/allmydata/mutable/servermap.py +++ b/src/allmydata/mutable/servermap.py @@ -1,16 +1,8 @@ """ Ported to Python 3. """ -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division -from __future__ import unicode_literals +from __future__ import annotations -from future.utils import PY2 -if PY2: - # Doesn't import str to prevent API leakage on Python 2 - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, max, min # noqa: F401 -from past.builtins import unicode from six import ensure_str import sys, time, copy @@ -203,8 +195,8 @@ class ServerMap(object): (seqnum, root_hash, IV, segsize, datalength, k, N, prefix, offsets_tuple) = verinfo print("[%s]: sh#%d seq%d-%s %d-of-%d len%d" % - (unicode(server.get_name(), "utf-8"), shnum, - seqnum, unicode(base32.b2a(root_hash)[:4], "utf-8"), k, N, + (str(server.get_name(), "utf-8"), shnum, + seqnum, str(base32.b2a(root_hash)[:4], "utf-8"), k, N, datalength), file=out) if self._problems: print("%d PROBLEMS" % len(self._problems), file=out) @@ -276,7 +268,7 @@ class ServerMap(object): """Take a versionid, return a string that describes it.""" (seqnum, root_hash, IV, segsize, datalength, k, N, prefix, offsets_tuple) = verinfo - return "seq%d-%s" % (seqnum, unicode(base32.b2a(root_hash)[:4], "utf-8")) + return "seq%d-%s" % (seqnum, str(base32.b2a(root_hash)[:4], "utf-8")) def summarize_versions(self): """Return a string describing which versions we know about.""" @@ -824,7 +816,7 @@ class ServermapUpdater(object): def notify_server_corruption(self, server, shnum, reason): - if isinstance(reason, unicode): + if isinstance(reason, str): reason = reason.encode("utf-8") ss = server.get_storage_server() ss.advise_corrupt_share( @@ -879,7 +871,7 @@ class ServermapUpdater(object): # ok, it's a valid verinfo. Add it to the list of validated # versions. self.log(" found valid version %d-%s from %s-sh%d: %d-%d/%d/%d" - % (seqnum, unicode(base32.b2a(root_hash)[:4], "utf-8"), + % (seqnum, str(base32.b2a(root_hash)[:4], "utf-8"), ensure_str(server.get_name()), shnum, k, n, segsize, datalen), parent=lp) diff --git a/src/allmydata/test/_win_subprocess.py b/src/allmydata/test/_win_subprocess.py deleted file mode 100644 index bf9767e73..000000000 --- a/src/allmydata/test/_win_subprocess.py +++ /dev/null @@ -1,210 +0,0 @@ -""" -This module is only necessary on Python 2. Once Python 2 code is dropped, it -can be deleted. -""" - -from future.utils import PY3 -if PY3: - raise RuntimeError("Just use subprocess.Popen") - -# This is necessary to pacify flake8 on Python 3, while we're still supporting -# Python 2. -from past.builtins import unicode - -# -*- coding: utf-8 -*- - -## Copyright (C) 2021 Valentin Lab -## -## Redistribution and use in source and binary forms, with or without -## modification, are permitted provided that the following conditions -## are met: -## -## 1. Redistributions of source code must retain the above copyright -## notice, this list of conditions and the following disclaimer. -## -## 2. Redistributions in binary form must reproduce the above -## copyright notice, this list of conditions and the following -## disclaimer in the documentation and/or other materials provided -## with the distribution. -## -## 3. Neither the name of the copyright holder nor the names of its -## contributors may be used to endorse or promote products derived -## from this software without specific prior written permission. -## -## THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -## "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -## LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS -## FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE -## COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, -## INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -## (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -## SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) -## HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, -## STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -## ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED -## OF THE POSSIBILITY OF SUCH DAMAGE. -## - -## issue: https://bugs.python.org/issue19264 - -# See allmydata/windows/fixups.py -import sys -assert sys.platform == "win32" - -import os -import ctypes -import subprocess -import _subprocess -from ctypes import byref, windll, c_char_p, c_wchar_p, c_void_p, \ - Structure, sizeof, c_wchar, WinError -from ctypes.wintypes import BYTE, WORD, LPWSTR, BOOL, DWORD, LPVOID, \ - HANDLE - - -## -## Types -## - -CREATE_UNICODE_ENVIRONMENT = 0x00000400 -LPCTSTR = c_char_p -LPTSTR = c_wchar_p -LPSECURITY_ATTRIBUTES = c_void_p -LPBYTE = ctypes.POINTER(BYTE) - -class STARTUPINFOW(Structure): - _fields_ = [ - ("cb", DWORD), ("lpReserved", LPWSTR), - ("lpDesktop", LPWSTR), ("lpTitle", LPWSTR), - ("dwX", DWORD), ("dwY", DWORD), - ("dwXSize", DWORD), ("dwYSize", DWORD), - ("dwXCountChars", DWORD), ("dwYCountChars", DWORD), - ("dwFillAtrribute", DWORD), ("dwFlags", DWORD), - ("wShowWindow", WORD), ("cbReserved2", WORD), - ("lpReserved2", LPBYTE), ("hStdInput", HANDLE), - ("hStdOutput", HANDLE), ("hStdError", HANDLE), - ] - -LPSTARTUPINFOW = ctypes.POINTER(STARTUPINFOW) - - -class PROCESS_INFORMATION(Structure): - _fields_ = [ - ("hProcess", HANDLE), ("hThread", HANDLE), - ("dwProcessId", DWORD), ("dwThreadId", DWORD), - ] - -LPPROCESS_INFORMATION = ctypes.POINTER(PROCESS_INFORMATION) - - -class DUMMY_HANDLE(ctypes.c_void_p): - - def __init__(self, *a, **kw): - super(DUMMY_HANDLE, self).__init__(*a, **kw) - self.closed = False - - def Close(self): - if not self.closed: - windll.kernel32.CloseHandle(self) - self.closed = True - - def __int__(self): - return self.value - - -CreateProcessW = windll.kernel32.CreateProcessW -CreateProcessW.argtypes = [ - LPCTSTR, LPTSTR, LPSECURITY_ATTRIBUTES, - LPSECURITY_ATTRIBUTES, BOOL, DWORD, LPVOID, LPCTSTR, - LPSTARTUPINFOW, LPPROCESS_INFORMATION, -] -CreateProcessW.restype = BOOL - - -## -## Patched functions/classes -## - -def CreateProcess(executable, args, _p_attr, _t_attr, - inherit_handles, creation_flags, env, cwd, - startup_info): - """Create a process supporting unicode executable and args for win32 - - Python implementation of CreateProcess using CreateProcessW for Win32 - - """ - - si = STARTUPINFOW( - dwFlags=startup_info.dwFlags, - wShowWindow=startup_info.wShowWindow, - cb=sizeof(STARTUPINFOW), - ## XXXvlab: not sure of the casting here to ints. - hStdInput=int(startup_info.hStdInput), - hStdOutput=int(startup_info.hStdOutput), - hStdError=int(startup_info.hStdError), - ) - - wenv = None - if env is not None: - ## LPCWSTR seems to be c_wchar_p, so let's say CWSTR is c_wchar - env = (unicode("").join([ - unicode("%s=%s\0") % (k, v) - for k, v in env.items()])) + unicode("\0") - wenv = (c_wchar * len(env))() - wenv.value = env - - pi = PROCESS_INFORMATION() - creation_flags |= CREATE_UNICODE_ENVIRONMENT - - if CreateProcessW(executable, args, None, None, - inherit_handles, creation_flags, - wenv, cwd, byref(si), byref(pi)): - return (DUMMY_HANDLE(pi.hProcess), DUMMY_HANDLE(pi.hThread), - pi.dwProcessId, pi.dwThreadId) - raise WinError() - - -class Popen(subprocess.Popen): - """This superseeds Popen and corrects a bug in cPython 2.7 implem""" - - def _execute_child(self, args, executable, preexec_fn, close_fds, - cwd, env, universal_newlines, - startupinfo, creationflags, shell, to_close, - p2cread, p2cwrite, - c2pread, c2pwrite, - errread, errwrite): - """Code from part of _execute_child from Python 2.7 (9fbb65e) - - There are only 2 little changes concerning the construction of - the the final string in shell mode: we preempt the creation of - the command string when shell is True, because original function - will try to encode unicode args which we want to avoid to be able to - sending it as-is to ``CreateProcess``. - - """ - if not isinstance(args, subprocess.types.StringTypes): - args = subprocess.list2cmdline(args) - - if startupinfo is None: - startupinfo = subprocess.STARTUPINFO() - if shell: - startupinfo.dwFlags |= _subprocess.STARTF_USESHOWWINDOW - startupinfo.wShowWindow = _subprocess.SW_HIDE - comspec = os.environ.get("COMSPEC", unicode("cmd.exe")) - args = unicode('{} /c "{}"').format(comspec, args) - if (_subprocess.GetVersion() >= 0x80000000 or - os.path.basename(comspec).lower() == "command.com"): - w9xpopen = self._find_w9xpopen() - args = unicode('"%s" %s') % (w9xpopen, args) - creationflags |= _subprocess.CREATE_NEW_CONSOLE - - cp = _subprocess.CreateProcess - _subprocess.CreateProcess = CreateProcess - try: - super(Popen, self)._execute_child( - args, executable, - preexec_fn, close_fds, cwd, env, universal_newlines, - startupinfo, creationflags, False, to_close, p2cread, - p2cwrite, c2pread, c2pwrite, errread, errwrite, - ) - finally: - _subprocess.CreateProcess = cp diff --git a/src/allmydata/test/common.py b/src/allmydata/test/common.py index b652b2e48..5728b84ad 100644 --- a/src/allmydata/test/common.py +++ b/src/allmydata/test/common.py @@ -1,14 +1,8 @@ """ -Ported to Python 3. +Functionality related to a lot of the test suite. """ -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division -from __future__ import unicode_literals +from __future__ import annotations -from future.utils import PY2, native_str -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from past.builtins import chr as byteschr __all__ = [ @@ -117,19 +111,8 @@ from .eliotutil import ( ) from .common_util import ShouldFailMixin # noqa: F401 -if sys.platform == "win32" and PY2: - # Python 2.7 doesn't have good options for launching a process with - # non-ASCII in its command line. So use this alternative that does a - # better job. However, only use it on Windows because it doesn't work - # anywhere else. - from ._win_subprocess import ( - Popen, - ) -else: - from subprocess import ( - Popen, - ) from subprocess import ( + Popen, PIPE, ) @@ -298,7 +281,7 @@ class UseNode(object): plugin_config = attr.ib() storage_plugin = attr.ib() basedir = attr.ib(validator=attr.validators.instance_of(FilePath)) - introducer_furl = attr.ib(validator=attr.validators.instance_of(native_str), + introducer_furl = attr.ib(validator=attr.validators.instance_of(str), converter=six.ensure_str) node_config = attr.ib(default=attr.Factory(dict)) From 5bad92cfc533a4a236ffb1bec21acfd611aef40d Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 3 Jan 2023 10:34:39 -0500 Subject: [PATCH 1323/2309] Another Python 2 remnant cleanup --- src/allmydata/test/web/test_web.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/src/allmydata/test/web/test_web.py b/src/allmydata/test/web/test_web.py index 4fc9b37e5..c220b0a0b 100644 --- a/src/allmydata/test/web/test_web.py +++ b/src/allmydata/test/web/test_web.py @@ -1,14 +1,8 @@ """ -Ported to Python 3. +Tests for a bunch of web-related APIs. """ -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division -from __future__ import unicode_literals +from __future__ import annotations -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from six import ensure_binary import os.path, re, time From c7bb190290ce2b85cb78605599daeb8aefbc072b Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 3 Jan 2023 10:38:18 -0500 Subject: [PATCH 1324/2309] Factor some SSK "signature" key handling code into a more reusable shape This gives the test suite access to the derivation function so it can re-derive certain values to use as expected results to compare against actual results. --- src/allmydata/mutable/common.py | 33 ++++++++++++++++++++++++++++++ src/allmydata/mutable/filenode.py | 33 +++++++++++------------------- src/allmydata/mutable/retrieve.py | 7 ++++--- src/allmydata/mutable/servermap.py | 7 ++++--- 4 files changed, 53 insertions(+), 27 deletions(-) diff --git a/src/allmydata/mutable/common.py b/src/allmydata/mutable/common.py index a2e482d3c..33a1c2731 100644 --- a/src/allmydata/mutable/common.py +++ b/src/allmydata/mutable/common.py @@ -10,6 +10,9 @@ MODE_WRITE = "MODE_WRITE" # replace all shares, probably.. not for initial MODE_READ = "MODE_READ" MODE_REPAIR = "MODE_REPAIR" # query all peers, get the privkey +from allmydata.crypto import aes, rsa +from allmydata.util import hashutil + class NotWriteableError(Exception): pass @@ -61,3 +64,33 @@ class CorruptShareError(BadShareError): class UnknownVersionError(BadShareError): """The share we received was of a version we don't recognize.""" + + +def encrypt_privkey(writekey: bytes, privkey: rsa.PrivateKey) -> bytes: + """ + For SSK, encrypt a private ("signature") key using the writekey. + """ + encryptor = aes.create_encryptor(writekey) + crypttext = aes.encrypt_data(encryptor, privkey) + return crypttext + +def decrypt_privkey(writekey: bytes, enc_privkey: bytes) -> rsa.PrivateKey: + """ + The inverse of ``encrypt_privkey``. + """ + decryptor = aes.create_decryptor(writekey) + privkey = aes.decrypt_data(decryptor, enc_privkey) + return privkey + +def derive_mutable_keys(keypair: tuple[rsa.PublicKey, rsa.PrivateKey]) -> tuple[bytes, bytes, bytes]: + """ + Derive the SSK writekey, encrypted writekey, and fingerprint from the + public/private ("verification" / "signature") keypair. + """ + pubkey, privkey = keypair + pubkey_s = rsa.der_string_from_verifying_key(pubkey) + privkey_s = rsa.der_string_from_signing_key(privkey) + writekey = hashutil.ssk_writekey_hash(privkey_s) + encprivkey = encrypt_privkey(writekey, privkey_s) + fingerprint = hashutil.ssk_pubkey_fingerprint_hash(pubkey_s) + return writekey, encprivkey, fingerprint diff --git a/src/allmydata/mutable/filenode.py b/src/allmydata/mutable/filenode.py index 99fdcc085..00b31c52b 100644 --- a/src/allmydata/mutable/filenode.py +++ b/src/allmydata/mutable/filenode.py @@ -9,8 +9,6 @@ from zope.interface import implementer from twisted.internet import defer, reactor from foolscap.api import eventually -from allmydata.crypto import aes -from allmydata.crypto import rsa from allmydata.interfaces import IMutableFileNode, ICheckable, ICheckResults, \ NotEnoughSharesError, MDMF_VERSION, SDMF_VERSION, IMutableUploadable, \ IMutableFileVersion, IWriteable @@ -21,8 +19,14 @@ from allmydata.uri import WriteableSSKFileURI, ReadonlySSKFileURI, \ from allmydata.monitor import Monitor from allmydata.mutable.publish import Publish, MutableData,\ TransformingUploadable -from allmydata.mutable.common import MODE_READ, MODE_WRITE, MODE_CHECK, UnrecoverableFileError, \ - UncoordinatedWriteError +from allmydata.mutable.common import ( + MODE_READ, + MODE_WRITE, + MODE_CHECK, + UnrecoverableFileError, + UncoordinatedWriteError, + derive_mutable_keys, +) from allmydata.mutable.servermap import ServerMap, ServermapUpdater from allmydata.mutable.retrieve import Retrieve from allmydata.mutable.checker import MutableChecker, MutableCheckAndRepairer @@ -132,13 +136,10 @@ class MutableFileNode(object): Deferred that fires (with the MutableFileNode instance you should use) when it completes. """ - (pubkey, privkey) = keypair - self._pubkey, self._privkey = pubkey, privkey - pubkey_s = rsa.der_string_from_verifying_key(self._pubkey) - privkey_s = rsa.der_string_from_signing_key(self._privkey) - self._writekey = hashutil.ssk_writekey_hash(privkey_s) - self._encprivkey = self._encrypt_privkey(self._writekey, privkey_s) - self._fingerprint = hashutil.ssk_pubkey_fingerprint_hash(pubkey_s) + self._pubkey, self._privkey = keypair + self._writekey, self._encprivkey, self._fingerprint = derive_mutable_keys( + keypair, + ) if version == MDMF_VERSION: self._uri = WriteableMDMFFileURI(self._writekey, self._fingerprint) self._protocol_version = version @@ -164,16 +165,6 @@ class MutableFileNode(object): (contents, type(contents)) return contents(self) - def _encrypt_privkey(self, writekey, privkey): - encryptor = aes.create_encryptor(writekey) - crypttext = aes.encrypt_data(encryptor, privkey) - return crypttext - - def _decrypt_privkey(self, enc_privkey): - decryptor = aes.create_decryptor(self._writekey) - privkey = aes.decrypt_data(decryptor, enc_privkey) - return privkey - def _populate_pubkey(self, pubkey): self._pubkey = pubkey def _populate_required_shares(self, required_shares): diff --git a/src/allmydata/mutable/retrieve.py b/src/allmydata/mutable/retrieve.py index efb2c0f85..64573a49a 100644 --- a/src/allmydata/mutable/retrieve.py +++ b/src/allmydata/mutable/retrieve.py @@ -24,7 +24,7 @@ from allmydata import hashtree, codec from allmydata.storage.server import si_b2a from allmydata.mutable.common import CorruptShareError, BadShareError, \ - UncoordinatedWriteError + UncoordinatedWriteError, decrypt_privkey from allmydata.mutable.layout import MDMFSlotReadProxy @implementer(IRetrieveStatus) @@ -923,9 +923,10 @@ class Retrieve(object): def _try_to_validate_privkey(self, enc_privkey, reader, server): - alleged_privkey_s = self._node._decrypt_privkey(enc_privkey) + node_writekey = self._node.get_writekey() + alleged_privkey_s = decrypt_privkey(node_writekey, enc_privkey) alleged_writekey = hashutil.ssk_writekey_hash(alleged_privkey_s) - if alleged_writekey != self._node.get_writekey(): + if alleged_writekey != node_writekey: self.log("invalid privkey from %s shnum %d" % (reader, reader.shnum), level=log.WEIRD, umid="YIw4tA") diff --git a/src/allmydata/mutable/servermap.py b/src/allmydata/mutable/servermap.py index cd220ce0f..99aa85d24 100644 --- a/src/allmydata/mutable/servermap.py +++ b/src/allmydata/mutable/servermap.py @@ -21,7 +21,7 @@ from allmydata.storage.server import si_b2a from allmydata.interfaces import IServermapUpdaterStatus from allmydata.mutable.common import MODE_CHECK, MODE_ANYTHING, MODE_WRITE, \ - MODE_READ, MODE_REPAIR, CorruptShareError + MODE_READ, MODE_REPAIR, CorruptShareError, decrypt_privkey from allmydata.mutable.layout import SIGNED_PREFIX_LENGTH, MDMFSlotReadProxy @implementer(IServermapUpdaterStatus) @@ -943,9 +943,10 @@ class ServermapUpdater(object): writekey stored in my node. If it is valid, then I set the privkey and encprivkey properties of the node. """ - alleged_privkey_s = self._node._decrypt_privkey(enc_privkey) + node_writekey = self._node.get_writekey() + alleged_privkey_s = decrypt_privkey(node_writekey, enc_privkey) alleged_writekey = hashutil.ssk_writekey_hash(alleged_privkey_s) - if alleged_writekey != self._node.get_writekey(): + if alleged_writekey != node_writekey: self.log("invalid privkey from %r shnum %d" % (server.get_name(), shnum), parent=lp, level=log.WEIRD, umid="aJVccw") From 3423bfb351b717281a9088404f2a090534179fc4 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 3 Jan 2023 11:31:29 -0500 Subject: [PATCH 1325/2309] Expose the pre-constructed keypair functionality to the HTTP API --- src/allmydata/client.py | 34 ++++++++++++++++++-- src/allmydata/test/common.py | 51 ++++++++++++++++++++++++------ src/allmydata/test/web/test_web.py | 51 ++++++++++++++++++++++++++---- src/allmydata/web/filenode.py | 21 ++++++++++-- 4 files changed, 138 insertions(+), 19 deletions(-) diff --git a/src/allmydata/client.py b/src/allmydata/client.py index 1a158a1aa..a8238e4ee 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -32,6 +32,7 @@ from allmydata.storage.server import StorageServer, FoolscapStorageServer from allmydata import storage_client from allmydata.immutable.upload import Uploader from allmydata.immutable.offloaded import Helper +from allmydata.mutable.filenode import MutableFileNode from allmydata.introducer.client import IntroducerClient from allmydata.util import ( hashutil, base32, pollmixin, log, idlib, @@ -1086,9 +1087,38 @@ class _Client(node.Node, pollmixin.PollMixin): def create_immutable_dirnode(self, children, convergence=None): return self.nodemaker.create_immutable_directory(children, convergence) - def create_mutable_file(self, contents=None, version=None): + def create_mutable_file( + self, + contents: bytes | None = None, + version: int | None = None, + *, + unique_keypair: tuple[rsa.PublicKey, rsa.PrivateKey] | None = None, + ) -> MutableFileNode: + """ + Create *and upload* a new mutable object. + + :param contents: If given, the initial contents for the new object. + + :param version: If given, the mutable file format for the new object + (otherwise a format will be chosen automatically). + + :param unique_keypair: **Warning** This valuely independently + determines the identity of the mutable object to create. There + cannot be two different mutable objects that share a keypair. + They will merge into one object (with undefined contents). + + It is not common to pass a non-None value for this parameter. If + None is given then a new random keypair will be generated. + + If non-None, the given public/private keypair will be used for the + new object. + + :return: A Deferred which will fire with a representation of the new + mutable object after it has been uploaded. + """ return self.nodemaker.create_mutable_file(contents, - version=version) + version=version, + keypair=unique_keypair) def upload(self, uploadable, reactor=None): uploader = self.getServiceNamed("uploader") diff --git a/src/allmydata/test/common.py b/src/allmydata/test/common.py index 5728b84ad..37d390da5 100644 --- a/src/allmydata/test/common.py +++ b/src/allmydata/test/common.py @@ -105,6 +105,7 @@ from allmydata.scripts.common import ( from ..crypto import ( ed25519, + rsa, ) from .eliotutil import ( EliotLoggedRunTest, @@ -622,15 +623,28 @@ class FakeMutableFileNode(object): # type: ignore # incomplete implementation MUTABLE_SIZELIMIT = 10000 - def __init__(self, storage_broker, secret_holder, - default_encoding_parameters, history, all_contents): + _public_key: rsa.PublicKey | None + _private_key: rsa.PrivateKey | None + + def __init__(self, + storage_broker, + secret_holder, + default_encoding_parameters, + history, + all_contents, + keypair: tuple[rsa.PublicKey, rsa.PrivateKey] | None + ): self.all_contents = all_contents self.file_types = {} # storage index => MDMF_VERSION or SDMF_VERSION - self.init_from_cap(make_mutable_file_cap()) + self.init_from_cap(make_mutable_file_cap(keypair)) self._k = default_encoding_parameters['k'] self._segsize = default_encoding_parameters['max_segment_size'] - def create(self, contents, key_generator=None, keysize=None, - version=SDMF_VERSION): + if keypair is None: + self._public_key = self._private_key = None + else: + self._public_key, self._private_key = keypair + + def create(self, contents, version=SDMF_VERSION): if version == MDMF_VERSION and \ isinstance(self.my_uri, (uri.ReadonlySSKFileURI, uri.WriteableSSKFileURI)): @@ -826,9 +840,28 @@ class FakeMutableFileNode(object): # type: ignore # incomplete implementation return defer.succeed(consumer) -def make_mutable_file_cap(): - return uri.WriteableSSKFileURI(writekey=os.urandom(16), - fingerprint=os.urandom(32)) +def make_mutable_file_cap( + keypair: tuple[rsa.PublicKey, rsa.PrivateKey] | None = None, +) -> uri.WriteableSSKFileURI: + """ + Create a local representation of a mutable object. + + :param keypair: If None, a random keypair will be generated for the new + object. Otherwise, this is the keypair for that object. + """ + if keypair is None: + writekey = os.urandom(16) + fingerprint = os.urandom(32) + else: + pubkey, privkey = keypair + pubkey_s = rsa.der_string_from_verifying_key(pubkey) + privkey_s = rsa.der_string_from_signing_key(privkey) + writekey = hashutil.ssk_writekey_hash(privkey_s) + fingerprint = hashutil.ssk_pubkey_fingerprint_hash(pubkey_s) + + return uri.WriteableSSKFileURI( + writekey=writekey, fingerprint=fingerprint, + ) def make_mdmf_mutable_file_cap(): return uri.WriteableMDMFFileURI(writekey=os.urandom(16), @@ -858,7 +891,7 @@ def create_mutable_filenode(contents, mdmf=False, all_contents=None): encoding_params['max_segment_size'] = 128*1024 filenode = FakeMutableFileNode(None, None, encoding_params, None, - all_contents) + all_contents, None) filenode.init_from_cap(cap) if mdmf: filenode.create(MutableData(contents), version=MDMF_VERSION) diff --git a/src/allmydata/test/web/test_web.py b/src/allmydata/test/web/test_web.py index c220b0a0b..bb1f27322 100644 --- a/src/allmydata/test/web/test_web.py +++ b/src/allmydata/test/web/test_web.py @@ -8,6 +8,7 @@ from six import ensure_binary import os.path, re, time import treq from urllib.parse import quote as urlquote, unquote as urlunquote +from base64 import urlsafe_b64encode from bs4 import BeautifulSoup @@ -32,6 +33,7 @@ from allmydata.util import fileutil, base32, hashutil, jsonbytes as json from allmydata.util.consumer import download_to_data from allmydata.util.encodingutil import to_bytes from ...util.connection_status import ConnectionStatus +from ...crypto.rsa import PublicKey, PrivateKey, create_signing_keypair, der_string_from_signing_key from ..common import ( EMPTY_CLIENT_CONFIG, FakeCHKFileNode, @@ -59,6 +61,7 @@ from allmydata.interfaces import ( MustBeReadonlyError, ) from allmydata.mutable import servermap, publish, retrieve +from allmydata.mutable.common import derive_mutable_keys from .. import common_util as testutil from ..common_util import TimezoneMixin from ..common_web import ( @@ -94,14 +97,19 @@ class FakeNodeMaker(NodeMaker): def _create_mutable(self, cap): return FakeMutableFileNode(None, None, self.encoding_params, None, - self.all_contents).init_from_cap(cap) - def create_mutable_file(self, contents=b"", keysize=None, - version=SDMF_VERSION, - keypair=None, + self.all_contents, None).init_from_cap(cap) + def create_mutable_file(self, + contents=None, + version=None, + keypair: tuple[PublicKey, PrivateKey] | None=None, ): - assert keypair is None, "FakeNodeMaker does not support externally supplied keypairs" + if contents is None: + contents = b"" + if version is None: + version = SDMF_VERSION + n = FakeMutableFileNode(None, None, self.encoding_params, None, - self.all_contents) + self.all_contents, keypair) return n.create(contents, version=version) class FakeUploader(service.Service): @@ -2865,6 +2873,37 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi "Unknown format: foo", method="post", data=body, headers=headers) + async def test_POST_upload_keypair(self) -> None: + """ + A *POST* creating a new mutable object may include a *private-key* + query argument giving a urlsafe-base64-encoded RSA private key to use + as the "signature key". The given signature key is used, rather than + a new one being generated. + """ + format = "sdmf" + priv, pub = create_signing_keypair(2048) + encoded_privkey = urlsafe_b64encode(der_string_from_signing_key(priv)).decode("ascii") + filename = "predetermined-sdmf" + actual_cap = uri.from_string(await self.POST( + self.public_url + + f"/foo?t=upload&format={format}&private-key={encoded_privkey}", + file=(filename, self.NEWFILE_CONTENTS * 100), + )) + # Ideally we would inspect the private ("signature") and public + # ("verification") keys but they are not made easily accessible here + # (ostensibly because we have a FakeMutableFileNode instead of a real + # one). + # + # So, instead, re-compute the writekey and fingerprint and compare + # those against the capability string. + expected_writekey, _, expected_fingerprint = derive_mutable_keys((pub, priv)) + self.assertEqual( + (expected_writekey, expected_fingerprint), + (actual_cap.writekey, actual_cap.fingerprint), + ) + + + def test_POST_upload_format(self): def _check_upload(ign, format, uri_prefix, fn=None): filename = format + ".txt" diff --git a/src/allmydata/web/filenode.py b/src/allmydata/web/filenode.py index 678078ba3..1b0db5045 100644 --- a/src/allmydata/web/filenode.py +++ b/src/allmydata/web/filenode.py @@ -3,8 +3,10 @@ Ported to Python 3. """ from __future__ import annotations +from base64 import urlsafe_b64decode from twisted.web import http, static +from twisted.web.iweb import IRequest from twisted.internet import defer from twisted.web.resource import ( Resource, @@ -45,6 +47,19 @@ from allmydata.web.check_results import ( ) from allmydata.web.info import MoreInfo from allmydata.util import jsonbytes as json +from allmydata.crypto.rsa import PrivateKey, PublicKey, create_signing_keypair_from_string + + +def get_keypair(request: IRequest) -> tuple[PublicKey, PrivateKey] | None: + """ + Load a keypair from a urlsafe-base64-encoded RSA private key in the + **private-key** argument of the given request, if there is one. + """ + privkey_der = get_arg(request, "private-key", None) + if privkey_der is None: + return None + privkey, pubkey = create_signing_keypair_from_string(urlsafe_b64decode(privkey_der)) + return pubkey, privkey class ReplaceMeMixin(object): @@ -54,7 +69,8 @@ class ReplaceMeMixin(object): mutable_type = get_mutable_type(file_format) if mutable_type is not None: data = MutableFileHandle(req.content) - d = client.create_mutable_file(data, version=mutable_type) + keypair = get_keypair(req) + d = client.create_mutable_file(data, version=mutable_type, unique_keypair=keypair) def _uploaded(newnode): d2 = self.parentnode.set_node(self.name, newnode, overwrite=replace) @@ -96,7 +112,8 @@ class ReplaceMeMixin(object): if file_format in ("SDMF", "MDMF"): mutable_type = get_mutable_type(file_format) uploadable = MutableFileHandle(contents.file) - d = client.create_mutable_file(uploadable, version=mutable_type) + keypair = get_keypair(req) + d = client.create_mutable_file(uploadable, version=mutable_type, unique_keypair=keypair) def _uploaded(newnode): d2 = self.parentnode.set_node(self.name, newnode, overwrite=replace) From ca00adf2b406f4392da8d832357b341306d5ad9b Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 3 Jan 2023 18:50:07 -0500 Subject: [PATCH 1326/2309] regenerated test vectors with a more convenient format It's more verbose but it's easier to load and interpret. --- integration/test_vectors.yaml | 3218 +++++++++++++++++++++++++++++---- 1 file changed, 2881 insertions(+), 337 deletions(-) diff --git a/integration/test_vectors.yaml b/integration/test_vectors.yaml index 11d5134df..46ac7e890 100644 --- a/integration/test_vectors.yaml +++ b/integration/test_vectors.yaml @@ -1,338 +1,2882 @@ -params: - convergence: - - !!binary | - YWFhYWFhYWFhYWFhYWFhYQ== - - !!binary | - ZOyIygCyaOW6GjVnihtTFg== - formats: - - chk - - sdmf - - mdmf - objects: - - - !!binary | - YQ== - - 1024 - - - !!binary | - Yw== - - 4096 - - - !!binary | - LCa0a2j/xo/5m0U8HTBBNBNCLXBkg7+g+YpeiGJm564= - - 131071 - - - !!binary | - /N4rLtula/QIYB+3If6bXDONEO5CnqBPrlURto+/j7k= - - 131073 - - - !!binary | - uqWglk0zIPvAxqkiFARTyFE+okq4/QV3A0gEqWckgJY= - - 2097151 - - - !!binary | - BTBX/ampNfLU+ox7xipBGiaSbgC0kcB8Gy7BkJB4oKI= - - 2097153 - - - !!binary | - w6uP8Tcg6K2QR905Rms8iXTlksL6OD1KOWBxTK7wxPI= - - 8388607 - - - !!binary | - yPi3JHKKbWaEEG5eZOlM6BHJll0Z3UTdBzz4bPQ7wjg= - - 8388609 - zfec: - - - 1 - - 1 - - - 1 - - 3 - - - 2 - - 3 - - - 3 - - 10 - - - 71 - - 255 - - - 101 - - max vector: - 0-0-0-0: URI:CHK:mri732rh3meyh4drikau3a24ba:6tsj5wvcp6szdhmrbu5bea57wduoza64y6nd2lm7aleqpsxjm5la:1:1:1024 - 0-0-0-1: URI:SSK:64ipvwidczoqhptiwxewux75a4:47pnj3eykp6jfx4db274qfkjtndo235p6l56ah54nq2ndasskgha - 0-0-0-2: URI:MDMF:2bncspmundtrdanr5ty5rswm5e:nt3azayg6eyebhjhuf67p2kahb3wf2pedmtitlubhjc34unjj46a - 0-0-1-0: URI:CHK:e5tpbcmalcq6nn2zhd3qvg4chy:emzsvv2xnkhhrj2oatds5hf2cney25awi56ybeq4ofiyehzyakua:1:1:4096 - 0-0-1-1: URI:SSK:rxaltidfitcuzkxpi5lupojd6y:l7bzwvlzlcqrpg7zecg3plk6cfvvgzoylyryxtbv56b3hwgz5lpa - 0-0-1-2: URI:MDMF:5mqmhjtvr52gqvqziy4kox7blm:tfvldj3lph5ww6thcewusaxjnzhajfeyuui5e264cqpgigxok5ra - 0-0-2-0: URI:CHK:ok3slnjd3e56za3iot74audhl4:2mjvrb455yldouoxwzbx4sbsysowipqje6ifa4pmbzqahj5j4mnq:1:1:131071 - 0-0-2-1: URI:SSK:a7xyeaedjdi7vej32o522r4sme:n7w6h57yuxcrlkgkhjbe6akf7lrnt6jd5gqa2u2nbxboechpbfxq - 0-0-2-2: URI:MDMF:qctbamqytlidaqc6hhj7kd5yqe:exdrlt2auxsl3cewufeunjxa4672zrsulx3wsdxgk3z4s2uvob7q - 0-0-3-0: URI:CHK:worle55uksa2uqqeebm4yxnihu:vta76jbmejt2pxx4prqa75xawpdtx42cmzzhappeetzjksym37wq:1:1:131073 - 0-0-3-1: URI:SSK:wvjziwdykuw7dcthuyvbozdc2y:lzbiyksn5phd6mw63r4ixllresqn6suluv6xpkzq2y77xplskmxq - 0-0-3-2: URI:MDMF:nk4lmqor4jif6jh4ptpevqvlsy:5ihzv32rcarelcingl2jgkqumokl4unf3vyzaxuht73vnnjkjnbq - 0-0-4-0: URI:CHK:6kccrgbtmmprqe4jcfi7vf6v74:wpivl2evi25yfl4tzbbj6vp6nzk4vxl6lbongkac7vl3escvopsa:1:1:2097151 - 0-0-4-1: URI:SSK:j4ikqxq54udpd6pffhzyh22gaq:davtelwhbu2off6h3xzgjneeaxu4ujtvstwxp7kdrlnlluq7r4pq - 0-0-4-2: URI:MDMF:5ya7auna2ojhb5qxrmy33ohtyi:olubk3gn2icihwa6zkyebqdqugdnhwczq5dziirkt7koexim2a4a - 0-0-5-0: URI:CHK:ofgig3loex6pev2eymmvohv7wq:yasbfedqnueaajdcavuba7kxbdqzn7e6zx3y6cbvucgx433vlzcq:1:1:2097153 - 0-0-5-1: URI:SSK:ze3nkp7unkx3omh2qka74x3zhq:4ltf23sdboqiwvkwhuvhk2ehbcqyvsenx6oxisc2z3mkqgidjteq - 0-0-5-2: URI:MDMF:pehv4ip7wvu5vigq425lokcxam:peuvelif4letrl7d3ou2k3pcfm7sv4d3kszjhjxpo4wwek67vnbq - 0-0-6-0: URI:CHK:scwepfpkvhuc3fc5jkpmrodsfi:wui2ixaxyac76mbxtr6diw2ajky75wehk57f2kqgytddnpo4rozq:1:1:8388607 - 0-0-6-1: URI:SSK:i6nerj3afbsbinhyi4yq32ttyi:hlgzk6pm2msqedm2ixhysvdewwlaiqn3xravycy2wxpikimo3gtq - 0-0-6-2: URI:MDMF:bpulrhen44p63tqelmtq64rafa:xhnlxbyabauxbafaecnynjznhliqcswjamym52kq4dj4jivuuv3a - 0-0-7-0: URI:CHK:5eeprb6lfxclwt7fieskd2euly:ffjgjbrxfl6d2ug2a63iaesxtrqa5pk3yaagfcldqvo7x4ok7ykq:1:1:8388609 - 0-0-7-1: URI:SSK:5tzpr45bx6pmqkhb6j4ofcgoyu:cjpggju6lfleijv7ucrmclvpqm5nrhdymqkwzjkopbyz42umyeeq - 0-0-7-2: URI:MDMF:xkqjvm2uc6ajoji4cvokmh337y:smvwzvyrxhde6j64l7s6dx3xiohgco4t744gnnuaizrcehovn3rq - 0-1-0-0: URI:CHK:nnv4vrtlxmzkurfzvonj22leua:ywcyijrfnwykraku56dq7v3o4ts3xsxqfgmk3kgzwhq2cpest4pa:1:1:1024 - 0-1-0-1: URI:SSK:sqylzhpnxjxlgytx6wtjpmgkle:jlol7tvkpo4esw6cfd4lzpyq2ra7vai6fn2fpt3qkglbimwunalq - 0-1-0-2: URI:MDMF:oewssqeva3bextinsbsvddke24:p2mquc646afacs4p73xwtcrtvapvzwmjffvek36w4gebgalufzwa - 0-1-1-0: URI:CHK:wlszole5m2emf6wbhp3lnlyfxq:irdttvny74gxdrcotzfjslfq7p24kyiigyrm5shhw7zyj3hsiuba:1:1:4096 - 0-1-1-1: URI:SSK:4rfhew7z7fig3ztoyfspnqkzhy:7bp5wg5agcrju66sh6zklithnf32nsso5kgyznl36vv3vrg74p2a - 0-1-1-2: URI:MDMF:mztzdkhjxeut4lzwynb6hgqzj4:r3ujkurraq52qjbp2gulbds47nlrjif3pq2whpwjw4dqn5r3mepa - 0-1-2-0: URI:CHK:4ewm23jvdtm2i5xf4gck26wg5y:7cujxxc34mkfkmwhbemqtuixuektknmhhxlmujcekibf5amfwwga:1:1:131071 - 0-1-2-1: URI:SSK:tcvykalplqpck5usnjljvhil2y:kgyz3qryyabm7ppxo7xn3q43h37yz5ygjl6mavthrunki6sjcdya - 0-1-2-2: URI:MDMF:tpke6qm6ra5dcv5ef34kjq334u:yexpy6cp3tl5bypjhs42r5hs6pxmcptk77wc43qfb2odd5wkm4ta - 0-1-3-0: URI:CHK:heczpdxphw5frp3ri5sh2hpqei:6wflx6lphy5mhtpfb2abznoa3yk27ynbqbzcecs362miu72vkowq:1:1:131073 - 0-1-3-1: URI:SSK:zmt4xsuswb24t7xb5ghc6x4zoq:khwlexwe3bfpkjl6k56niynndw6usr63ujpqsnyz6oplugnooqva - 0-1-3-2: URI:MDMF:wtxng6wxbjzasgmzjol2tyzp2y:cu3eni3wqffuqysghrb35cy5m3h6zkpnoasghejlnn3tutkv62ra - 0-1-4-0: URI:CHK:mz5siv27pqttsl2f4vcqmxju64:rvxhulxtufho6pdbwj7rifneb6pei5fl4fqptcce7jdkbyrjfaya:1:1:2097151 - 0-1-4-1: URI:SSK:dbjrnvd6ze3hfh6dpy5p776yjm:kopm44nxmgsapuphibtxy7avh5jzzsvq5uizg43lvfpk2bp5qgwq - 0-1-4-2: URI:MDMF:yo2yjrc7qlrl6xpkg46v5odx6q:2dh6tymdchl36aqrqayqqhhvxj6byaxshadc2johx6d2jxmemfuq - 0-1-5-0: URI:CHK:shbt5viqjzuewblgt6qeijry6a:je3omw53tmmluz6fvqupx4uh4jaejc3fcvfjt56rfzokzhgdczvq:1:1:2097153 - 0-1-5-1: URI:SSK:pjvcabyr7hlzvo3hgfwayomvyy:ioykumds24zhxwour2ktrbfr2g3iyephqz6s6lriu2bpjh5n234q - 0-1-5-2: URI:MDMF:cuy7igduaoj5c5o7sxud5dgc3i:wfuxxoehvu5xqyy6teljt4gporjazqqwewtcynowgrg7ylliu3ra - 0-1-6-0: URI:CHK:n2642ofjfr5qljptdwjbwjz6fq:uwtjrewrev42yvez7br2574xu5vleallwnyddjaxqquvouv3dyra:1:1:8388607 - 0-1-6-1: URI:SSK:vqzgr727yx6zei3ougcx73j5m4:osrlcedb3ws5lz6ofdcpf3erro3ydqvwedssjtvi5rzlb35mc2va - 0-1-6-2: URI:MDMF:mxyohlnkjvad4p3mk4ssqyqgcy:p32sx4fnl5thwjpszmalpdvvnrsaurfevrkkybiyy5sg66yrq7ka - 0-1-7-0: URI:CHK:nktoeeuydph4acj2fux4axyaj4:g3fvjwanenwsgdcn3oxnau5gtbdmzbnbevlrzrb5qe4yukuwhejq:1:1:8388609 - 0-1-7-1: URI:SSK:vmmmxar3clfixjsjswr5owjt6q:g5dsw44pkbg3lvbhrmpjki3oqqjvyyv4dkbejcm5svojxlgmmaya - 0-1-7-2: URI:MDMF:dedcqq5ryz7p6bup7vnqhlbp4y:dkiinwnqyizu7q3vrpordqetnwkoe74k4mplyjpw4dhwv4mgnj2a - 1-0-0-0: URI:CHK:ycvogzi6wllnq2bkx3t6zdtwju:um42l4yen7jiwdfgirvedtty3tt3xuhjiyxzqoourvughtxjar3q:1:3:1024 - 1-0-0-1: URI:SSK:3nlzd3o4goxnkms67hjezvnx3e:5cw77s7ywr6gmzoeu2dr62sl3mmzuk4uyn7tsqyb6ycc5uaglxtq - 1-0-0-2: URI:MDMF:ilkr2vh5k6ob3qe3cnrio222nq:4b4j6ysy6t6rfga63qauqdq6oavyhmqe6nusyccbbdzfm42fjfvq - 1-0-1-0: URI:CHK:3yjtuv5h2g45g2cncg6hka3euu:bqqhp6u73ldawial7rtrfomx2qkmboyvqve6ywo2jpvfbu7zptga:1:3:4096 - 1-0-1-1: URI:SSK:7gzsibil7wlnxvhf7jnnhugumi:s2tdz77gmlgge4726cjseflqiv6zqjtwolofk5rnnx7qa56f372a - 1-0-1-2: URI:MDMF:obvmtzzryj2gfxdp2o6y4iqyoq:vy7kr4euwkhvrpgjqjonsy4xfif6qy4mqxcw5r5tekdyt6o4rdka - 1-0-2-0: URI:CHK:annnqmu72p7iels5loqwlxt2zq:s7wv3jfi2hlpcvp4dnloc4eex6vre42kwiel46achaie5n5uodqa:1:3:131071 - 1-0-2-1: URI:SSK:sxos74s6bhwr3n25fxjabz3o7a:nugsx6go3a2pwne6p3hravnu6kcfjmzd5eboeewjaxyczczpllpa - 1-0-2-2: URI:MDMF:yxhafheawf6oogddsr3icqcrqe:dt4zatgywpz3tjicga6iiz7ce6ccqspkccjnwobihkmh35xnwaxa - 1-0-3-0: URI:CHK:rhkln7unkktot72mit5dmuqbdy:ilo6u6hugipdimyrzrvlam47xsmp3ur2lwnrtbecmvocb2664zxq:1:3:131073 - 1-0-3-1: URI:SSK:kikhu6vcfuhnmbkihqekq5ezeu:e6l2k5h3kwrpccg5koba3atxp2y23iyylwvrhvctlbsphtpywvjq - 1-0-3-2: URI:MDMF:hxdnms24wh2o54glbrhw4egolu:77kjhrpztiz7eapsipjb6toldabzca767lyvh47iut7insdhrp3a - 1-0-4-0: URI:CHK:7yjcwwt6454lbv3pni5mjofsxe:y5nwpzwmvpvr3gqxnykjixprpxw3w6qyqmszf7ijyxnm3wl6f5oa:1:3:2097151 - 1-0-4-1: URI:SSK:fanakrkirjczr7v2w5l6ovwt6u:uyrwhi4oqqehewjfomvris4gmbehrlidxb2drh2xold5entrheeq - 1-0-4-2: URI:MDMF:4dyxo6lvt5v5qjwpveolbnguve:h4ck2q6pd62pwb63dfewvfeyfck3gjqie66gj4pwwf2nk7an7l7a - 1-0-5-0: URI:CHK:r7wva6tisq6m5zszgr7seu77cm:oqlgdw3hi72qtahpsi3h3yryxpqdshagvt6xnobsppkwp7ic4cia:1:3:2097153 - 1-0-5-1: URI:SSK:agiakuvmrmzv6c34cfi4mdwdzy:qq5gymi6bvqvt7lykgdznmuoufety3a6rnyn4o22wk2xqirnbdrq - 1-0-5-2: URI:MDMF:pm6fmmvxu5zmypkbmpqt6qpawq:wrj4w27ldtgfw7djfhednxzti6rmqdyx74n5e45qcwe5euptcbrq - 1-0-6-0: URI:CHK:mz7pzh24u6flkfmfvvokdwmriu:5mzo57de2ywsdqkdvm7557gcikkpjqbmj65mk235t2xyoejtpbta:1:3:8388607 - 1-0-6-1: URI:SSK:q5jatla7dgcrykoeyy5hq2kne4:n2bgno4z5qy2f5uq3vrgyqq4lpa5o5bhuzgyy725qlksswkihrwa - 1-0-6-2: URI:MDMF:mijwsnxcbdcyvegro3dlfb2kvm:lsn2etmig274xnfwe6nu6wd5lu7mvdy55sqytwb2nzx6ko3xtqoa - 1-0-7-0: URI:CHK:54sq6lrqwqlpxicg747gdls6ri:2aaxyrdytn7r74my36ek434hlbhe6glrgr2ic5vvpc5bbdxmvo6a:1:3:8388609 - 1-0-7-1: URI:SSK:wusjn7evp3wutlijs3bomfmtka:wnig2pqxofw7socupyhexz4qiuwqcky5aq6tqoymdlk4mscm7tpa - 1-0-7-2: URI:MDMF:lpmgpoqsbjqekgh2cb2crlunda:de4l4ox64kx7lutli3h3iefxsolkqrzxdxqchnc74msobat4dhra - 1-1-0-0: URI:CHK:pbfcdvxhbikxd2hcc43oel3v3e:y5txpmiptmoz36ionmki3p6krmdbiqasw2v3wdq4ia5lmrhh33lq:1:3:1024 - 1-1-0-1: URI:SSK:wpto56y2mcqjz76yeq32dqg4za:7qwecvg265xwrflipssea3qvgdbce3xp52odm6f6dwxiunq3ipqq - 1-1-0-2: URI:MDMF:holxufnqnricjbf6pfxsajv6ua:iffvui7drjicjcuoul5wdxn5b56znrvjpaon4ntc5s6pml5pgqda - 1-1-1-0: URI:CHK:vkdc7iae4pgkdomhbbb554kv6a:g5xkatxv2entfssfh5eqgexb5j3qs6jvohofjqo5p5erlxlx2dga:1:3:4096 - 1-1-1-1: URI:SSK:j3okig3bvfg2tz3bf4yakmaqkq:m3fmgcxah23izgz77a3jl4bxzn6ziwga75drqkzhnuhazwhd4deq - 1-1-1-2: URI:MDMF:gnloxjxwuathfupvhmjfiml34u:fhqeulalg3xkaja7fcgukm2arguh2gipiizceftnkwnadqfv224q - 1-1-2-0: URI:CHK:zvla2jyu5fqlb2s63zftyfjnla:yqspaexhew55cvv7w2m6czezhzkqnyk5fea4gvnplc5za4xlvbva:1:3:131071 - 1-1-2-1: URI:SSK:m2a4klrarck66kxdaelqirusje:u2mpjlt7vrkhetfifxv77zhw6vxgifx4cwmp62i4qurhrrjtaefa - 1-1-2-2: URI:MDMF:ek7p7ocgap3c44njbqviux634m:6nfyyt5pper3wvqsp3rnubgploukfogejuo4po62qyi66fmutlaa - 1-1-3-0: URI:CHK:3idf2xqm5wnw2q3nbva5vmpaiq:7nmkh5q55omjloezibvyzvveq4gqeaivei5ypfm5vfuugwiz6hpa:1:3:131073 - 1-1-3-1: URI:SSK:ycfor2ol5iv6w6uukisf7xianq:dp32lujnfnpn64bhzfwkc7poewdkhluiz3bmsofka26fndxe5cyq - 1-1-3-2: URI:MDMF:46g6h67kbymmny5mxaqgoeq57e:r7dojkuirkw6svgesc3qnry5vkquam5ayuydoljdfsgirffkjcpa - 1-1-4-0: URI:CHK:o7xdokumtcukutzgmmpfme4nnm:nn3s5gau5dychy6wlodwriuit4wyavfze6bo4icywcbhdb4ccxla:1:3:2097151 - 1-1-4-1: URI:SSK:s3sfivcsgqqcv44rh62f7ybua4:zllpdgffhzci4ddbfyfx22yzimnlv4jmsbyf3lig7w3zlk3lowea - 1-1-4-2: URI:MDMF:ammiuctyg7v7caimclofugplqq:jvwrtrj6yx7xrtmpckcecbcsozbhb2jpprgs2nosii245yj766xq - 1-1-5-0: URI:CHK:vws2xrl2nch2hsptqb36yqqu3e:fhujsv5rkhyiqstrktjvfy5235c7gcwekyd3gux3bv2glsgstjga:1:3:2097153 - 1-1-5-1: URI:SSK:6jx3psk4ymb4m6xijdkmwyvn5a:pokhzlauyftwdfeh7fpx5vudee35tzh4eqg2nuzfv445565zge3q - 1-1-5-2: URI:MDMF:2kv3aehx33jpkmmdxuw56mlv54:papxsfb63bakhhcsmnvxqvfgtl3cwvxr4ecxgyemk7bewvdpzdva - 1-1-6-0: URI:CHK:5anac46dpfsjumutzsoovvtlpe:yuc3gqe7m6s6xp7k3uodcg2g6k2bac32gt5amu7gdx7k43qnudxa:1:3:8388607 - 1-1-6-1: URI:SSK:wawckuhibbpqpnuqpukfmeykjy:gz4qevawur75onpf6mif6vyqs52wr53624pbqirq7ioicw26ltdq - 1-1-6-2: URI:MDMF:g3ssdqnaayw2ibczy3stibpmdq:3pmbetun4gpewxspp2h6lkmnez6cn2yorxmfmeef4tpziq4yhiza - 1-1-7-0: URI:CHK:fb6g3fkfbtlwzheujzvc2dn7dq:644je55ri6q5halshuxfesjz2afhqtebetuzqqbfecp6yhqwbb5q:1:3:8388609 - 1-1-7-1: URI:SSK:mbxn22dpw3te2bmttgijzza4my:rdkopjhcyuya4mazhiazun5hppykjxpz6n3xmx4npn7x6lwfkw3q - 1-1-7-2: URI:MDMF:ennvchlb4z2h22fcxmfzqfwlt4:yq2hqxdlhenxnxl4pv4bz4cou2waygkaouk42iozctjrue7r27rq - 2-0-0-0: URI:CHK:jdm6ytmx2i3ya2bllsdzxurjtu:iqr6tyysaseegzfurhuywy3mbbwkbbsov5bt3fo6oazpyv7olvda:2:3:1024 - 2-0-0-1: URI:SSK:rfdsbtoh7ig76hmujuyooyvwri:iihu65yi7f2str7eump6iqv6reqatvrffd74tmaunbgzi733vkeq - 2-0-0-2: URI:MDMF:jm6yqftdmdoqxvyuqn5vnx2bsm:5gobkzionvrsjv24fgb4t3o36cgd2oyzwhizq2lwcj4hbrwl2u6q - 2-0-1-0: URI:CHK:7wz5dhkyob3gmkydzgbptk5qku:cl6ovq5a3km7rpfhyb3putg656sp57lhnp7aexbocsbbahdxnfia:2:3:4096 - 2-0-1-1: URI:SSK:utxhffimno7m4u76fscoxhzdym:avszduoxd57rjstwfi2fqatf47yibuy6hnc7z26pvsl53nirw47a - 2-0-1-2: URI:MDMF:fwb5mg3kz2nzmhdzfgg742khp4:fcytu2aox234opw2pipywkapnfl64afxzrhcmi7w7jveihrmoy4q - 2-0-2-0: URI:CHK:aagw6esdm2msphvgnr3rlzg43e:zqxtrlee6hh3vtfg6ihtssjqvr27tpvi6gkmngnhiguttjz7yyja:2:3:131071 - 2-0-2-1: URI:SSK:xgedi566hv3vocftdfywzvw4he:z5lmieqwlswkijomu4mks6qvtepl7e2hrbs7druekieymricw2ua - 2-0-2-2: URI:MDMF:esutc5ohuhcmj726xv66wihjhu:bcvsqlfbqepgwwnucfjeatlxredtblhf2isw6rgztgi4cgu7a7za - 2-0-3-0: URI:CHK:e6uhixvbm3g5amrzdzd3mnmsqm:4n5aafrb64sqpfnhcdpdfrk5gh2qznprjky6q6jybuuqy2q6z6mq:2:3:131073 - 2-0-3-1: URI:SSK:u7o4qtogmvnshwz3wbcslcxiiq:n5jxccr7wj443wjyjiof7grtkst3dxb7g23zhq2vcijzpzi2zpia - 2-0-3-2: URI:MDMF:nctwh42xkr5q5b5gwew4r22gne:zwlchalto2xquywhk7ux2r3dbhdq3btadrz36no2uftlgomjb63a - 2-0-4-0: URI:CHK:abfmzuiczjwt6e67f3uoyg5i7q:rk7ie3afnp3xgvnxu5e2vioq476xqwslqxgyx3i5xxevsjhkcvba:2:3:2097151 - 2-0-4-1: URI:SSK:7liwiw7qfs4f4hv5gpfih4a6gm:6u5homs7iry4ryebmj7ji4meedrxbn5vl5y5jd3rsfo6mp5egwra - 2-0-4-2: URI:MDMF:emh52u7gv3x6r6sc4lj54d6f5i:utoggiwnd5ptkgsw5c4qaftjzbh5w6shbvfc3pja4qfgsppdhieq - 2-0-5-0: URI:CHK:34bzejdzpqinmuzdmqenkidf74:pcecbl4ulygiadgdagonbgqmpb4lomjz4v4vqyssr56reyib4q2q:2:3:2097153 - 2-0-5-1: URI:SSK:kak4npmxsndcrbiry4nr6mwaku:qrilttfrkbyxaaxptpa3s4rydwv57mgarbxmtwz3ouli3b6d3mia - 2-0-5-2: URI:MDMF:htww4ycieablkeuu7nyw3q2phm:hvrth4e5sv6tjwjxsmjfzvsmnqbbwhgceyxz3cr5wuox6e5ajyfq - 2-0-6-0: URI:CHK:ej56vl65kawat62wsxuvbcd2wy:tv65cgghaddq2p7dyvimmff24l34wsx6mhzd3dyprfvrdo52qtma:2:3:8388607 - 2-0-6-1: URI:SSK:ocrry76nfkqgia643p32xfowxa:nnnc7xaknc55iibxhvskraifbx24a6h3q5dpmckkivmjx4eig4ua - 2-0-6-2: URI:MDMF:preyyqadzm55vijzfjqfd27xbm:jv4d74sxklayo2emq7ue6mapeqh7gldzyn7zz2eigosbti2luloq - 2-0-7-0: URI:CHK:ufshkmuewwql2xsyiecbjhh7x4:qoywwygjqrmifowob3mvwkhfskk6geomyw5e3qlgqzui3kymu6nq:2:3:8388609 - 2-0-7-1: URI:SSK:fevaztou3q56upx4uigz4byywe:sd2kfppinkyo5a6ji6y6ftdktuyam2ywg6mqzvrqsesgbjcyhijq - 2-0-7-2: URI:MDMF:d4oaonhqdbb7ct2mtcbguroxpy:bx4xmffvttca6j4sitb74phqppokcvkymx6yzqj2765uk23gssgq - 2-1-0-0: URI:CHK:ld3thziutpaqv25nbtoqebhtru:ri2obsvzl2etyuv3qnchn2wvw5mh5rjuvbjaqkqbvyprn7xwamoq:2:3:1024 - 2-1-0-1: URI:SSK:7lhegvunppwq5bmvay3nce5jt4:uuxdhl6ap4hpvpaeo3jgha727nl5o7ktbrxi6fai7dp55omhlspa - 2-1-0-2: URI:MDMF:ifs2eqqeokipwz4r5zwxbcacq4:7oqu6ex7yidy6bqotojqedik5r3vwkmepp27qez3a23nzofpjpqa - 2-1-1-0: URI:CHK:pz3mga5pvqd4bpsz7pbyna2qtq:w3ptt7xwwgywod3u2rpgx7iggvm2kpwl7d2ryr3u7mcaxgqotlaq:2:3:4096 - 2-1-1-1: URI:SSK:cezgyjtq5ugmh25mryojcdjutu:bu36zk7bsvghs5bm3zdkmllbrlq3vbvcoa7crmnz7ai2oiwjbyqq - 2-1-1-2: URI:MDMF:izdrqjjyk6nyfpoexeqnyhmxcy:54qffrqnpcvfq33qpu7dz3xbmd2dxbiritwtvy7eraujoknskymq - 2-1-2-0: URI:CHK:rsh7ysvsgbwh5g3xuj6z7zky3y:ykg5rhzpzqkvdy53qqyghfbcusibynfylggbhypo3iibsrrzvlkq:2:3:131071 - 2-1-2-1: URI:SSK:xnm6td5a2v65efzx4zyyr4gena:alzhu6j37of275k4u3uovxsi4u57zcarzzw7rozx5fskp4mnz64q - 2-1-2-2: URI:MDMF:76nwwbjf5eslzqf7gxmmu43tdm:7vwv5acs3yr54az5eaqwt5kv7sgrfdoimskxxxbhos5il3bcnbwq - 2-1-3-0: URI:CHK:7xqejfqfej3u3zp2qymqd2iqeq:po4t2tzkh4d34ku6gdhlocadfwdyweiyckyqz57zdi7ndt4ikj3q:2:3:131073 - 2-1-3-1: URI:SSK:uva7lh3b4j42mrcp5utf2lcyxi:3peveaddnwuup7vbouuy6urkyog2ovucim7yv6a66h7hjnrbei3q - 2-1-3-2: URI:MDMF:7lyl6pejhg2gimf5omvjmolckq:ufgtgmrd3k25tzj4zr7rbup3vh4fiwmwljkktymcuminqs3ino3a - 2-1-4-0: URI:CHK:kzhg5zp4hcs75eboack4kqpcdm:g2cwnhpfwxrv3c4lrmpjyj327a274oa5qt4isu4avjixbu7vblrq:2:3:2097151 - 2-1-4-1: URI:SSK:wzqmll7265ckpuaquuu4tipzhi:fyajlm7egfm7i6ijowmt4sgke37uspn7ay7v5mr2tpmnt3hh2m3a - 2-1-4-2: URI:MDMF:dloi4oc6aos4isw6rqhtxzd26e:fugp7o2rss7w26ud3d4o7daj6ors7sofec45q5j4gggvtonuucda - 2-1-5-0: URI:CHK:6sf35smixwvpapf2zw7sllxet4:ks5zd3hkd7ppwobhnymezckszexpychngnkipxfvmlcng5fgjbea:2:3:2097153 - 2-1-5-1: URI:SSK:7mq4vfi5j6jzpszjyboputuv3a:qartnhxp3ii4wyjk6h44vwrzlayj2qj4l3xmiiqj2ryhdnvdzora - 2-1-5-2: URI:MDMF:4j7vlr323wmo3vnmpeaemv4b44:hin5lmydp7b2odgu63uqpbxuron4vewnwle5g6yzgvpitq53tgyq - 2-1-6-0: URI:CHK:dnmjqjku5vyrkeaulae4bt6juu:d47airwaqedeji2omacwsp6yfqp5mpurj776gmzvq43c6dovtdca:2:3:8388607 - 2-1-6-1: URI:SSK:3erqoieayuam2jvu4echcj3azu:hk4ko4bj2nlhjh4kqxx7idxk3xb2vmksitkft43blilhyluyqt5q - 2-1-6-2: URI:MDMF:em5zwqit3eevv2orxfh2i3v4ku:6emaakoo52w5e2yzwufsje5zzkxiz7j6vjxwrbvkyvf24cthzwfa - 2-1-7-0: URI:CHK:bfjedwcwjehjzpwjsgwkfk7rja:jpwea2sgz4hfohqab642yj4cmrh64w6nfo7lv3mtacfzicurqkva:2:3:8388609 - 2-1-7-1: URI:SSK:7edtpkmasd6ehqcxhhfqaaodgu:trmtsw3g6dnvg2ojmmwt4mcvkcd3mzpdbwraqhrhzwboq63xccxq - 2-1-7-2: URI:MDMF:esyeaj2wof5udiw67sqxg2oja4:rggac3zexaunabzpvb36sqimouzchhcvaiigmsvmjrmgb62th5xq - 3-0-0-0: URI:CHK:dx7tvyr2fc4u7lxjc6kehq2svq:tiy4qh2g6lqejxcaym3rr7ymkdkinn4qised6kgxloj7sptsqu4a:3:10:1024 - 3-0-0-1: URI:SSK:6b2saxjadoy2ievtxh4ri7cspi:qmdtijkjdf4er4rooi3ka57oh33whelwecy3euksahu3vep5inaa - 3-0-0-2: URI:MDMF:c4yzz3zuapsmteiavjx56nfc7y:fia2oloiwm62klq3qh7tb5v5rbmwaol6tvpfd37f5ibapf4jyjmq - 3-0-1-0: URI:CHK:yo75evk4cte3b7rdw72zxvl5ye:ex6h7ff7nclucjtsqwgwu33qgmb67t4ezbrki4zbgurwn2ct6bbq:3:10:4096 - 3-0-1-1: URI:SSK:axmognjdoajpylni3kac6b6tlu:lpvdrbkq7qt4hvjkkxbmzdjfehnfp7y3e2qxylmb7v4w6fmyywka - 3-0-1-2: URI:MDMF:uvsqupyfamsvbitsenyexxkz2q:fb44nsshdrscvhuonh5fqmhnpbxugw4evyoxnunyronduv3skwsq - 3-0-2-0: URI:CHK:4gokef54smahrbfr4kq3jhc4zq:owpwwfp5gof2vhly5u6jdnbsfuwwwhqkazpsbeg3nldxv5pse2iq:3:10:131071 - 3-0-2-1: URI:SSK:2wxje7wkktur2ys4e73hholec4:enqfpbenh2sdcugbd22qb5tsqa2x43vzr73y666k3s4ktyl4oq4a - 3-0-2-2: URI:MDMF:2e2vrraewqlffijhfjjm7odqyq:wurnspwx7nqdxota5lzgmjdab4jdcfs6slb5qujljoegm4i3x2ha - 3-0-3-0: URI:CHK:7vfgl5cv4nlzqx35z4uthjv36y:nnueftbzxfz6u5yjxwwofaxzzft7xss5wzfh66rrcwv2zwrm63sa:3:10:131073 - 3-0-3-1: URI:SSK:uaxoqctux3abakjjyi3fojtcvm:y6gqrke2oi5w55pznh63ockqyq7txu5my7fw6mde6f6iw6uqkkjq - 3-0-3-2: URI:MDMF:ko74zoqcprygi5ipc775mavlsu:dbvql5htmnld455yxqbkml7csfku44cfzbuy5mfpxczbx7nisvva - 3-0-4-0: URI:CHK:cy7fmjsldeakhfd4psbmghmqyi:6a6uvyai4jkzz6hj5yjugm6uie5etvymcudgiwwjh47apz636zwa:3:10:2097151 - 3-0-4-1: URI:SSK:ix74qdf4oy6oigb5kcwkjl7lwq:vrbfidc5l3e2u76ztzeaezfem6w5budpmx6paph6vwlkukmmq4ra - 3-0-4-2: URI:MDMF:alpmmh4rfvcytolewejupt6sse:hmtxrp53i75v2mekiqbiw7hgxm2zm5sglcv4bq6v4zhbsr257wda - 3-0-5-0: URI:CHK:ptsofqwylmkvzmuvrw5n34j3ma:ky2fs7xrlke64w6kfmhzsuilzxbhfrwwzkxih4rykpbxrr3bxhiq:3:10:2097153 - 3-0-5-1: URI:SSK:pefhxppdqkfq77zsiwbdr7ye2m:whxta7t67iyvjsr6dia674hwibsr67vz5nwrxaeoedzal5qg6yga - 3-0-5-2: URI:MDMF:ybbbm5l3yr566ki4z4tpnqv4hm:temkyl3jg6xwk7o345pkovmx3mxglefpoof6fzqiiqxsjl4xw2ma - 3-0-6-0: URI:CHK:tyt4hfvn44igztqh5zeiusggcm:ex3gzzn7byvhajithwbo7c3p7qwqxmnactaxxsxsxowkafyjrf7q:3:10:8388607 - 3-0-6-1: URI:SSK:gitwhiiauwxo6xyur47gaevk2q:fsjra53xrguxomizdtftembdo47ypydbahq26jt67yvqylhuyzfq - 3-0-6-2: URI:MDMF:hp3uxpsp73lyy2rz7sznnt7qli:macxhhrmtqswkzcnyfna3dtanwi5xkvibtv7uitlma6sr7nkklga - 3-0-7-0: URI:CHK:xymheose4rdlspgydkzr4nqkre:z3pfrvpq5fdpkoybhdxppwbzrt6ejf26xh6emzlce2sgquljginq:3:10:8388609 - 3-0-7-1: URI:SSK:eunztljebzuw2ja6qafuqesuly:xoqie57c4eqr465nrwihhnhfjmn44qj2jgzn72typ2raph5vizsa - 3-0-7-2: URI:MDMF:ztkemleegq7saj4j3aebzohfju:lxz2564f3tjanxlfscvrjbdm3cqen2gp53w3owmnt7umov6fn2bq - 3-1-0-0: URI:CHK:x55owxzhsezfoayaxe7jpwnove:vawdgtqpxyntgy5i2po2twgelrynkfcjgwm7publnlbdp7hpqmfa:3:10:1024 - 3-1-0-1: URI:SSK:yffv3eigw2bil2zacthvbuecme:r56c6xlljx2ccaqf3dlesz6zi4wd56ekohiou6xu3z5rhquopjaq - 3-1-0-2: URI:MDMF:d5cou2vdhswmfh2tgw55q3ndpy:cpu4fzg4zijyhqdkw6ftghlfkczculab3nbqtzqf3nlezuuzkoiq - 3-1-1-0: URI:CHK:fe64krzyaeff3d4teunjbetkzy:27hrywwaffqiqcgfkmzwbot3iamotr3bey2l5kaladmdmxuaz5ka:3:10:4096 - 3-1-1-1: URI:SSK:hrcs4vs4ae2ji47fx2xe6jfvoy:ga7u6v4cundqk33s4jcnt54d4rihjwosrp5xfx6m3xvqbiardo7a - 3-1-1-2: URI:MDMF:frgfl4n5pppywn6ez72dxryomm:fu5gnu24mwhmldodkcde3b3nk6bu3f7amsc42rpbl6x7dafg4txq - 3-1-2-0: URI:CHK:gvajllsonkuscfemygbnqhq2re:uwyilm5a7so4blhsaielnf34u2qbaqmudd73opjkgodgg3okeaga:3:10:131071 - 3-1-2-1: URI:SSK:otczcexqybjxjobsnw2gpfh7ua:s2ijirfjs4wuzde4qszag6hmdygcmflwi5ywwvqblij5unccjrca - 3-1-2-2: URI:MDMF:7flmqsxaoo44d6xozhvchumi7e:cdy6in2tgiuwieuslgrvsahv4i2mtsapbuk3pztegzqd44phvj3q - 3-1-3-0: URI:CHK:zdmicwopo4p4h4wbfcbnwcrvyi:6qn75anpvs5gls27f4lybisis3udvjfjhatxiny7c72bcbtuztia:3:10:131073 - 3-1-3-1: URI:SSK:4dsb5diiy4c5ph5bilbeorb2y4:2i35prduaaaj7zdojoagjkzlrouk2wvgbqscmfevn6pq6hn2c3ca - 3-1-3-2: URI:MDMF:k5tkdlb3vxjmeq5ocn4k7nn7v4:3bjb4iilbnb2zs3pxyksyac2huau5yiizs43xbtglewlsq5zjswq - 3-1-4-0: URI:CHK:bv6qthmetlhdnwc5tfjqamp3yq:ehz4ttd4g7ktkxvbovt562wfedc6jgnt5c6af7wxgp7jbwfwhoaa:3:10:2097151 - 3-1-4-1: URI:SSK:u4h47atwwwxnildul6hstzjdpq:om4e6ogzgblvgxry3os2gm4qnauiypfjiax4paa4avvawnc7gnka - 3-1-4-2: URI:MDMF:jexmxy22l2s3f5s26q56oozbia:4wt3zuvm5afubpxugo5ieyzy3yhxurmra72tyou6n42aofjauoyq - 3-1-5-0: URI:CHK:n7ogyjbo5jigvxgel5ll6q4vbe:3yjb3zq5hcdavv7ruefawal6euyvjx3lx7quslvasjellv63cxya:3:10:2097153 - 3-1-5-1: URI:SSK:q3ehvqgpiydqljr5baecgfoade:e3qd3antznp3yrudd3vlvfhve57uebfowd6s6adjppiv3h5rexea - 3-1-5-2: URI:MDMF:mdq6e4jzzjcvnnzzdcw373up3y:luvwwba76brnwovmqwuzfdmiowp7suigojmfe7qbk3lkwncjzyhq - 3-1-6-0: URI:CHK:d2hbvcmbex7fm3qu22yj4qnkh4:2xwqxbawwgn773hht6etox3oypvqqjv2orktnthfo2e7vibko7ha:3:10:8388607 - 3-1-6-1: URI:SSK:nlk64f25lbnm42zbycv6ypchea:ziipysr3cwntri6txx634qx7xuaybt3af2fr7afu6bio44wyh2ca - 3-1-6-2: URI:MDMF:5rw4gxpvybgu75rzaes3vgwy7i:7na2hbjq5ejuxv7azwpxemym7ds4nynnzf2xoptpgwdpob5ytecq - 3-1-7-0: URI:CHK:ccxkyfl2qtqyhihduarpxbdcci:e64be3i2t25selbpc5y2zj443gkdo65chs2o4tpqb7axud5lnxta:3:10:8388609 - 3-1-7-1: URI:SSK:gg4bdwy462s4gbqm2omovufnri:zswcwabtlmd2qdjcvj7ltmkkuf7xuiw6vmut4ke4k5gdcl7oqjra - 3-1-7-2: URI:MDMF:7trroesakxfrt3gwoh3m3lifea:ksv6fa5dmxr3epewosbm6hjllfmmbmiznmnh6bgal3nhs4gss6wq - 4-0-0-0: URI:CHK:2wvrzaobtfkoiqeqwryjzvxatq:7zzaycbdhepzaqtdrku2sbb572h64ggwstu47osa2gn4ol3eglmq:71:255:1024 - 4-0-0-1: URI:SSK:hgef47gcwxn4einhqanzspdxsq:px63z6ezudouh7rcurhov7jscvi5p2ri4tjmx2p3sg44z5qgu7ta - 4-0-0-2: URI:MDMF:4y3lzdqx4dtimj4h4oh4pyhnte:x7xlolo4aemt3lmbha6ejjodxyh7wkhhgu6ua3kvdvpfz3t3647a - 4-0-1-0: URI:CHK:7wi5xn2uwerkjzqs5ndn4eimti:llrws4uig2fpi3rtla6lk7m4t6zudjhszdfjgqtihghfrvs2robq:71:255:4096 - 4-0-1-1: URI:SSK:5cseulzm26fheuvquntqqvyyri:j5tuxus63kag2tcfit5b3cgojgvg6hksydeumpnxrr5lcroyemlq - 4-0-1-2: URI:MDMF:tbhr6ffrepvshxvaxh3bvl6if4:cdlp3yeqvt7q4fw547j625j54eiluuebwyx6iikgqoyuscmlotea - 4-0-2-0: URI:CHK:6nvs23bhrpiwiz5prqdtvztujy:wlg7g522rpdoitpm4qwhmctrjhnh4zfloiq6uq4tsvaoawg4slpq:71:255:131071 - 4-0-2-1: URI:SSK:svlkrjk55ti5k3prcpazndiizu:mzewxyzfsfsaufod73awsi5ixcdcu4vnrlcb6y4pk4jzxp6agnwq - 4-0-2-2: URI:MDMF:mwcxdmqtjxrmtk73v6ib3syac4:u4yq4rkluijyy6uxv76enaynjrorwuik4ihljpqtglr4xinrjasa - 4-0-3-0: URI:CHK:62dtbeowncjj62jnwbggaufvbu:6r2sapg2cm6dvmylodyxabrj63a736uouzsqyacnimgo4svnktva:71:255:131073 - 4-0-3-1: URI:SSK:7ybfzwfadqryj65daupc5mtjim:mokqdwv37xdj42yg567jaauazuivcnvpmi4kxubmduwdgqj6qxaa - 4-0-3-2: URI:MDMF:nodj7mjiycqbf4zlb2l5sai6de:ozqrakvrodeb4uc4nndgtbnryacmpzg4e43x6adcvew5l2qvfa3a - 4-0-4-0: URI:CHK:hk3a5ync7y3dnwowqjqoa34eae:y4f5b2xqa3lslh2vti5fdbho2syqhbj2p6f3enlhxsjbfpt7zuta:71:255:2097151 - 4-0-4-1: URI:SSK:pwzgrbwxthdnu2gfjnbycmj5wa:jvhexrr3e4iejx4o3kri7rhhenpqt5h4dtdwev7r2noyrdahehtq - 4-0-4-2: URI:MDMF:sfras4yucfxpqakxx4cqbefvxq:3r7gltmdsev3vx2fvcj7mi34czdqpm5odx5m37yeqnpzek2ijakq - 4-0-5-0: URI:CHK:qa4hwhwtd3cpvut74biixlv6ye:uuj4ujg3mcf3lqpuvvwf5pszcvviyb4cm2jjicu7ugiypojjie4q:71:255:2097153 - 4-0-5-1: URI:SSK:xpwtwzrqhs4pjc4sinxzbulzn4:vamd7m2f7lrhn67ygcuwdsd4oqk5fqzhncr646ut72birecttw5q - 4-0-5-2: URI:MDMF:ud6fekwwtgoivsp6smbs3jweyq:rdnf6kuhxgiirwqthumyerqwuvezw4jtxtqglkseijzpedpugcjq - 4-0-6-0: URI:CHK:r7sv6u3ihzys754twjltuzzpza:tpyoxinubkl4kvg6hipls6ezun7t3yyqj6ix3vp6tvnzqirzzl4a:71:255:8388607 - 4-0-6-1: URI:SSK:fii277xjgusxszdo4jwhdiqwt4:vvx3em5ywpq7hcpbisuyuhpriifhnhttf3l3ntkdgkwtyhsj3s3q - 4-0-6-2: URI:MDMF:xufeawqo56oeitacojihy4hvhq:d4zi53idpc775d63lgvjojobfnmsbtewaty677qgw6aezzckacma - 4-0-7-0: URI:CHK:3esfsjn7csgnyqmq5afbgtinay:xsrhdomrbrtzftg6u4hgipm5tkaomumbdlxi5hpvpvawe3bjikua:71:255:8388609 - 4-0-7-1: URI:SSK:ih26p3vtz73m4vtafxffdkw42a:5rgjtx7z3fjrgwwzls7xmajoos6jkkeh7nnay7yhfdlfk4kyvi2a - 4-0-7-2: URI:MDMF:cj4oi3gmmlufchtmo7uniam6tu:xx5vppvu7py2h7rlknkjclqaal6oi2d2eeehga2cszmdx5egkrda - 4-1-0-0: URI:CHK:wxmzfzvgaqm3xfzgaqrlndgolm:6t3tqcphsutigxnnzyp2xx3afklcpeytnh543mpqptquvqqahwea:71:255:1024 - 4-1-0-1: URI:SSK:cbacdsnd5p6dnjjb7ds7ngs5ly:xrq3fhpz5m2yhhlsk7qon6ljcb7zmqqzzeeuf57jnmcpshfiwowa - 4-1-0-2: URI:MDMF:76zgyx5geis4pkrmbrixtmhjom:7r7ewpfiz3en25ffacqelntevvxx6oyldwh2b2l2yc25rxa7dqqa - 4-1-1-0: URI:CHK:r3q3q4f4zbsipxauq6knb5dfoq:lkae5mflammzxkoc7gllt6qqyg5gqdxeejmxtgcscblqh5tv5obq:71:255:4096 - 4-1-1-1: URI:SSK:icp52tuqpe5at2ugh36hsch2ki:ntip6j5ifu7ysm5m3rauy32rxdhfsubl7nsibpyhufatmxqnfxqa - 4-1-1-2: URI:MDMF:57cjkmrvqwi4a2oswkq7vzuhga:ez6div27j3azyfbsnycrgwxklylse3zexprqlt7tcd7hbihc2vxq - 4-1-2-0: URI:CHK:rdrzetasw4w7tuweqpev65d5vq:gzzb2v5fzrduk6kx5xx27mvo6dgnd65ym5lm7re53in7xuudhsxa:71:255:131071 - 4-1-2-1: URI:SSK:grmx2nfcvfnhh7iljncp2aff6e:sj2znocnowgarswtnmvfc3mkawlxahv2vm7nq365jt2uti2x5adq - 4-1-2-2: URI:MDMF:fnzfrawb63fofzivusnbmzrsem:s6ol6wea5nwt25bql33kwlwwkhgrh7ubfgqr2rz2foamz4yzbfda - 4-1-3-0: URI:CHK:e2bjefibvz4tgu6jgf66gw74bq:mugtu5bx7h5gcivevmh2gmwoc2kkhmobrzshkuj2dgrtm3siysfq:71:255:131073 - 4-1-3-1: URI:SSK:74zkfoe364z5wp43uanq4sjctm:uihfuthtjz5uaa22bn2ehbsqfsdga6xryztmce3hbl5vf2kzesuq - 4-1-3-2: URI:MDMF:posrxmsietgukgtamv5kjxi3qy:eo5w2yiwdmgo2xvozuitqxf5xfxleppdanh6t4w4ikzcift7hfca - 4-1-4-0: URI:CHK:gy7bci6yvllxzvhh45khcwp3u4:qsfjfmcl64zey4k4o5cfnh2i3mzboahf7bhtggceszrulsv537vq:71:255:2097151 - 4-1-4-1: URI:SSK:cmt4ks4vzuyeu5uh35wm3zirmu:5k3gawnpqcvokrmq72nddhuuho5sqhzqirkoi52uwyzdnidkbk2a - 4-1-4-2: URI:MDMF:2dhxtuv3mkrqc5dfdjulny57au:ptkxn6eemtivpczwoxoxvzivv7dyzn6q5z4unr3dmvdy25ymnxqq - 4-1-5-0: URI:CHK:pcta7uk5cpioxzv5nxxqsogw3q:gnu7fx56k3j72bbevirn4kdu27yfpvttzl3qk2zypwqt7tw4rijq:71:255:2097153 - 4-1-5-1: URI:SSK:aww7jpirjcubd2psmrnnx2vieu:m732exwqik55a7r6hf36vjyomi3w537am3e442xotme2iqdmsogq - 4-1-5-2: URI:MDMF:xmjottltglnlcccs247rmrylue:q2tgyaiuymz5z4m534a627uszdvi5mlrwytyvh7iefr4n4fw33gq - 4-1-6-0: URI:CHK:znxyqo2qglpkhjafnuesu4foge:trau477bj5u5wjfpy4oq5e3aziajgbpyfjdv4svyf27m7atpa34a:71:255:8388607 - 4-1-6-1: URI:SSK:qlbkffx3umudmmyf7pjaou42zy:whzwp6jz7stuw3ketnkvisr2n4t2qb7myaa3ex6q3stjfc5ixj3q - 4-1-6-2: URI:MDMF:2yozfuwqq4h6f45d56wqq7c7hy:4xokulr4nkgu5t72ndf6cdegitgaojyybv77k4btumlth3lypcma - 4-1-7-0: URI:CHK:mlzs2hbztak5fxjkrkuuvnpdpe:3myvisewm5uklimp2xucrwep75sm2rizfi2sq5drhlqjszkpdyeq:71:255:8388609 - 4-1-7-1: URI:SSK:jxz73uawqyauzohhk4q5xroyxe:d3w663ufccfoxxfdqw7dvezepyxzsotkz644l6a4izturxbhqmja - 4-1-7-2: URI:MDMF:gkxsktuwbhkqcwwxp52n3zj2na:lrwgbajbsspxpg6ov3otn52wk5rtln2v74n4xzozhr4ukg4mosga - 5-0-0-0: URI:CHK:mftnvh5l3brshwoupvu6kz35cy:sncfokby2tjykbr3zgi3i6lhgerg4kz7fsrslrlxdy7upwkzq6lq:101:256:1024 - 5-0-0-1: URI:SSK:un7ljbwxsvpcvyzxhnqyagjttq:ds52mgfwg3gw6fhhdoihrxwzaye4i4tpugddyx4r5reebbl2gpwa - 5-0-0-2: URI:MDMF:acn6rtbcds6uivf7a7o6z4tzri:jvvkm56qlueoz4pnbyj5fupxcw22cnuhnhyoeytt3tfp4ib5czda - 5-0-1-0: URI:CHK:sshu77h6opnto3jnngot3lwzl4:4cjhp6u3i2uzdzrjitje3inlhs6gnlgwc6wgi6vtedc2grgvcpza:101:256:4096 - 5-0-1-1: URI:SSK:n5nwguxxp3d6kh3i3ewrm77zxi:be3ivcy544d6s2ggzkw2ql4yeg5sf4feeimi7vzdglk5kilnkzga - 5-0-1-2: URI:MDMF:bvgdxytmeefqcx7x6pf3yki44a:p7sxdwfsl7qeklsrrijtubhput5pinm7x7jaeu2klt3vb4qytz6a - 5-0-2-0: URI:CHK:kl2jayrstlf5q3gax7o65ybivy:mnn5doti4wllssdg35ymojuiv4hd2ir7i6asurgms3ea67watm7a:101:256:131071 - 5-0-2-1: URI:SSK:7w3snvdpns7ea7evpbbvc43y6i:r7o7furnq3ma7esaqz65mqwwb4m3bbrndlyb5dbtb4atv2vvkksq - 5-0-2-2: URI:MDMF:iadp6yjiwyxgrzzedbiczb33iu:yv4yvkoizk5ox3ddfa2eyitemgkwqnd6jy67dxygkels6h74yxla - 5-0-3-0: URI:CHK:a6wuekh5mynr4c74ma3b2gswfa:b6lovwnbv3523n5kaxga3ruxa2dvxef6h4a2jnl6lasn4fwvvdba:101:256:131073 - 5-0-3-1: URI:SSK:6elghkg7p45xhyq474wqgipmi4:ud3wk2gts4n4j2eczhiuabhtf4wlolexnsfnmhgmw2gg4zpzncha - 5-0-3-2: URI:MDMF:gkeqwnqd57gudytjtim577en2a:5vr5lj7kmgfrl64sbsqnqgs5m66drexvh5dseejptni4j7bo3dwq - 5-0-4-0: URI:CHK:pmuo6vpcodol76sexq7s2ojej4:irp6rxnkgwbn3h4g5fzgjpqgsvhvycxobr3xrk3zr44r64wimgqa:101:256:2097151 - 5-0-4-1: URI:SSK:u24u3hw3qfmd2xki3hqsfpmwaq:le2qsy255qbusbxhdbpupofutipobez6da56ot6c7v6pndb7rw6q - 5-0-4-2: URI:MDMF:27obnopt2uprbfoltizn5nxhda:as4b7sendu2gu3qbttnnxk73revv3fw2nslc34flggnpidkg2gra - 5-0-5-0: URI:CHK:fh4boug3wkciqhkgpusjsjyzbe:sy5lwooxchqu5mrzwr3mgc7qtwfjgmpz2srdmwpaip4iuj3tn4vq:101:256:2097153 - 5-0-5-1: URI:SSK:p44gzfrmpjvwvv7rdioal3tcay:r5agbss2y72szn2rzqqninppyej2hip4aysp6uffv35e472nnbza - 5-0-5-2: URI:MDMF:3kdkkcykmxxm5uws4ro6b6xw4m:gomvjpqavm3vk6plkycpzxyhbx2t4g53vb62gm2tptpi3gcs5hea - 5-0-6-0: URI:CHK:asmjgajvwpwvylxdcf5rm2gyd4:imwtdi4eqq3wm4bvk6npho3ta2fewceqzl4f7wyyabg5i4s6h3mq:101:256:8388607 - 5-0-6-1: URI:SSK:sdtbn4trlyju5aimti6cfvhjxa:icvxmxt7vbab4mkchpw2q2pbz6pvftw3tyzu4rdjhysgw63ft47a - 5-0-6-2: URI:MDMF:a3zqfcnhd2vvyxk52sch2gjxii:nfaiz3lznarkiqpym6hzxjalcidvm3cb4pd4aizt4jote6ildzsq - 5-0-7-0: URI:CHK:yrznm2uuybfbda6jlj7tqs3yum:zy45lpc7ilfzgltxxqvy55wcokiay2fbbbv2wdvu6ubjluh4gf4a:101:256:8388609 - 5-0-7-1: URI:SSK:kl6hw2qywih3hb6snjo4ti3muy:dnwcw3wj26jx7t3z4zt37xhrmwui2gqtjjw537qjdsuropboe5vq - 5-0-7-2: URI:MDMF:sfs3c7wvdjxxvtb4h3lgn6qmve:6vwlktezoopvhd44f6qr35ffwnvxkbrexio5cdgtyol5h5luc3fq - 5-1-0-0: URI:CHK:ihyeqq5peg2pdyw45ct5mmj4ay:2rzhndv6o4knlvo4po6rvbnknlcitasseea3mirqjor5c7quh5ra:101:256:1024 - 5-1-0-1: URI:SSK:nsfj767mvco6cwxvfgcarl3ok4:rniviz4xd5aprg6bkozeqr5ubh5pm7qw3vimw35vlp335lff4bka - 5-1-0-2: URI:MDMF:jtjo4rpjsfsmj45anozydonuci:bk5beemlcpagv34l76wuijomr2s22stpdyszdhxrl3hbnfwxth7a - 5-1-1-0: URI:CHK:mvk5f2mghudk7xnknqddhvnl4m:c74myeugtix7e7l6uwybtrdtfselha2qrcc7hhbymqr6brtg4cea:101:256:4096 - 5-1-1-1: URI:SSK:a347ijcfid4ko4ndqmzukib3nm:f656xyo2ghvx2gp4gotwfbjtbztxv6qofn6egiyfsdpe2zitij5a - 5-1-1-2: URI:MDMF:3pmna4pbpcprn5bcrrprb2hlyi:h5ohpuh4xe5zwabi3xtgilcj4kykw7qhfrfsuy5dug6oessdrs5q - 5-1-2-0: URI:CHK:rvytwcc4sbmmkrdwh6b23ckv5u:klk5nqpbv37pr2qavcebcugfnikarjdefjlseo3ru5uv7guai22q:101:256:131071 - 5-1-2-1: URI:SSK:d7eaudatu6r47pp5zpimcjvcde:mit3ynp7gje26ejulvsxaessydzetkddscmgcktyzu4kdddtxceq - 5-1-2-2: URI:MDMF:hvsxlqkcu7m6srvudrdf667pp4:jh4cjypstjtkrsid3wwzaph52qbaxhx3aephrdwbxu6fd7unxkna - 5-1-3-0: URI:CHK:ocqx55dopakqnzyui6q6euz4gm:wklziyoctwrz5zjfg5wfejz4bnaagtg57fgjlspl4ba6ij6n2obq:101:256:131073 - 5-1-3-1: URI:SSK:6mxjz6yydgyeefq4mwvc56shti:wxw3d7e5jmtragswb42kgt4dffdjcpe2yzq2wvnx7rcwjuhjiy6q - 5-1-3-2: URI:MDMF:3mcv72otyu2ih3qgpt6tqizdai:j45sk3r4izog5vmae5f232q3ilnyoyoenavy6jlz4uytxjzl2jlq - 5-1-4-0: URI:CHK:lf6qub3bkdhfozpdis4bmorj7q:zte3u3ohbuom3iqjjx4odrjylcpw5myw4n4kwf4zcaif36vngf6q:101:256:2097151 - 5-1-4-1: URI:SSK:zbm3epfnv7pqdjtkn2d4igkora:yqei62xxffh5fpibml5uvptfw3e2rjf7po4l2nmh4nlocw6lczgq - 5-1-4-2: URI:MDMF:qgi3t2uwvvfjur7vi23h23xyce:k2mhiduzfdfkrsoo5jy4lhuxsjlqced6fz4tftu74hgqqjewy7tq - 5-1-5-0: URI:CHK:a2qoww5xx6so5uftwg4msdtrpy:h7hwsjco5i3jkd7icytrtsuzcjvjzlnkr2wwau2tjlj5wehjrokq:101:256:2097153 - 5-1-5-1: URI:SSK:vw3rhtwjip7bsfysysvwokkhqm:5ujsp6l2ywkqxmwebyznyszdtyttmzcymq5yz7lqjt54dnhezmaq - 5-1-5-2: URI:MDMF:qov2vzt2rv2woudodsblpban4a:upem3xr63i6kqeq3ku4dval7iwybsnnf2trisb7zus7igbuniyuq - 5-1-6-0: URI:CHK:tg5w4m6hi5ezeapuix3lrrbbtq:7kerk5plqpsd2upnole3ifxoegxdux7prazbvzeu2kyilc6idk7a:101:256:8388607 - 5-1-6-1: URI:SSK:xieqbrwa2s3vmxj4tg4lrf2qea:bphqgbbtytmbknp33wgtu4ezktpbkicoly5o3skiaadghcdxmlea - 5-1-6-2: URI:MDMF:qfrl5z67reetjmnufz4imgz5cq:aomtulzpd63u22be44vk3w2rxi3vp6cx4a7xnsjjohp7qkojyaza - 5-1-7-0: URI:CHK:ioridipksj6o7ph3bcf37vslmi:pzhkwm4ry67ohj6dcs7zwnsjrvfqzam6tuitrns5vuo3uoomau6q:101:256:8388609 - 5-1-7-1: URI:SSK:amruym46dleng2rjwhwxxgqyba:gkrjlcu2g2aycgdhp3alr754b5sm6vztoalqw3evhsfidgrv2saa - 5-1-7-2: URI:MDMF:h5gk2z5fz732jpvrjq6jmayk7m:dkbh4h5s2glge5mskgq6jzkzhhcq53xt5e4e7hv6em7woufi2q4a -version: '2022-12-26' +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:CHK:mri732rh3meyh4drikau3a24ba:6tsj5wvcp6szdhmrbu5bea57wduoza64y6nd2lm7aleqpsxjm5la:1:1:1024 + format: chk + sample: + length: 1024 + seed: YQ== + zfec: + required: 1 + segmentSize: 131072 + total: 1 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:SSK:y74wcrcls3al6zxz7kx7f673z4:syoy43s3ywv6knmfecsubpl5mkgqxttjivi42tnthlwkirdvpvdq + format: sdmf + sample: + length: 1024 + seed: YQ== + zfec: + required: 1 + segmentSize: 131072 + total: 1 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:MDMF:aqy46c5phihqbh7gvacwyjjhmm:txdtibqlgmsrozp5ufsixeplln3hrcgjlrgkyu7vvbo54lvjq3sq + format: mdmf + sample: + length: 1024 + seed: YQ== + zfec: + required: 1 + segmentSize: 131072 + total: 1 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:CHK:e5tpbcmalcq6nn2zhd3qvg4chy:emzsvv2xnkhhrj2oatds5hf2cney25awi56ybeq4ofiyehzyakua:1:1:4096 + format: chk + sample: + length: 4096 + seed: Yw== + zfec: + required: 1 + segmentSize: 131072 + total: 1 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:SSK:ievenb2az5hen2dd2lgr2rx4km:zix2o4lu25r6om3s2locfmmzyagii3eykrdxkjtbcjtf7djmpplq + format: sdmf + sample: + length: 4096 + seed: Yw== + zfec: + required: 1 + segmentSize: 131072 + total: 1 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:MDMF:roegsxmll43vrml5gu3pvv7j3y:o2hii67gi3kgqil6b5cy7bisahbw7s5y2qyfbi4oni2wiebig6jq + format: mdmf + sample: + length: 4096 + seed: Yw== + zfec: + required: 1 + segmentSize: 131072 + total: 1 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:CHK:ok3slnjd3e56za3iot74audhl4:2mjvrb455yldouoxwzbx4sbsysowipqje6ifa4pmbzqahj5j4mnq:1:1:131071 + format: chk + sample: + length: 131071 + seed: LCa0a2j/xo/5m0U8HTBBNBNCLXBkg7+g+YpeiGJm564= + zfec: + required: 1 + segmentSize: 131072 + total: 1 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:SSK:dp3kn36nz3gpto7lgd4sgadu4i:7xsljpakiof654td2lxz7mtmnev7fp4vqhdz6yp6zm4r37gd5myq + format: sdmf + sample: + length: 131071 + seed: LCa0a2j/xo/5m0U8HTBBNBNCLXBkg7+g+YpeiGJm564= + zfec: + required: 1 + segmentSize: 131072 + total: 1 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:MDMF:aflykwudyx2gqfk6csy5u4lvni:m3kmwyizqwavmgnnx2lit4nqdobg7ze3okh4exuhxp5z5ljskdoq + format: mdmf + sample: + length: 131071 + seed: LCa0a2j/xo/5m0U8HTBBNBNCLXBkg7+g+YpeiGJm564= + zfec: + required: 1 + segmentSize: 131072 + total: 1 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:CHK:worle55uksa2uqqeebm4yxnihu:vta76jbmejt2pxx4prqa75xawpdtx42cmzzhappeetzjksym37wq:1:1:131073 + format: chk + sample: + length: 131073 + seed: /N4rLtula/QIYB+3If6bXDONEO5CnqBPrlURto+/j7k= + zfec: + required: 1 + segmentSize: 131072 + total: 1 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:SSK:pd7xbhzt7v3ejras7g5acq5nry:lvvjteik6le7hm26lyd42i56pkkqbzu3fm7vi3kbtc2tzk2qv7eq + format: sdmf + sample: + length: 131073 + seed: /N4rLtula/QIYB+3If6bXDONEO5CnqBPrlURto+/j7k= + zfec: + required: 1 + segmentSize: 131072 + total: 1 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:MDMF:k4nqwrnxjiu3iw7r4gds2ddsii:bx665xasl5dsqippnj5ntgdofq6rtj6g377fygqcdcxvqjauhvhq + format: mdmf + sample: + length: 131073 + seed: /N4rLtula/QIYB+3If6bXDONEO5CnqBPrlURto+/j7k= + zfec: + required: 1 + segmentSize: 131072 + total: 1 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:CHK:6kccrgbtmmprqe4jcfi7vf6v74:wpivl2evi25yfl4tzbbj6vp6nzk4vxl6lbongkac7vl3escvopsa:1:1:2097151 + format: chk + sample: + length: 2097151 + seed: uqWglk0zIPvAxqkiFARTyFE+okq4/QV3A0gEqWckgJY= + zfec: + required: 1 + segmentSize: 131072 + total: 1 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:SSK:z6vgv24ns3ku35sv3z2upmjujm:l7qtsggtrzz33qxina3h47jx7e6wbtre67hilopouaraoxrdlm6a + format: sdmf + sample: + length: 2097151 + seed: uqWglk0zIPvAxqkiFARTyFE+okq4/QV3A0gEqWckgJY= + zfec: + required: 1 + segmentSize: 131072 + total: 1 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:MDMF:3zbhbl6c5s5tecu7mec6xpczbm:o3r52ullezeismr3bzbstwe5arbyx26vmfiblxz7sb7yjvjfc2xq + format: mdmf + sample: + length: 2097151 + seed: uqWglk0zIPvAxqkiFARTyFE+okq4/QV3A0gEqWckgJY= + zfec: + required: 1 + segmentSize: 131072 + total: 1 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:CHK:ofgig3loex6pev2eymmvohv7wq:yasbfedqnueaajdcavuba7kxbdqzn7e6zx3y6cbvucgx433vlzcq:1:1:2097153 + format: chk + sample: + length: 2097153 + seed: BTBX/ampNfLU+ox7xipBGiaSbgC0kcB8Gy7BkJB4oKI= + zfec: + required: 1 + segmentSize: 131072 + total: 1 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:SSK:lop5wrefixcec6gdbtokmvxmoq:7n4usur4eupz6ikgklgtwcpnooazlmzfmwnly4qidnf2tmahmp6q + format: sdmf + sample: + length: 2097153 + seed: BTBX/ampNfLU+ox7xipBGiaSbgC0kcB8Gy7BkJB4oKI= + zfec: + required: 1 + segmentSize: 131072 + total: 1 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:MDMF:dx4uss2swjscntutzmadrnb4ce:yxfietcglts2qu3p7y5ufhmoqndkkqqtb5fic4efawkxrnspoqna + format: mdmf + sample: + length: 2097153 + seed: BTBX/ampNfLU+ox7xipBGiaSbgC0kcB8Gy7BkJB4oKI= + zfec: + required: 1 + segmentSize: 131072 + total: 1 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:CHK:scwepfpkvhuc3fc5jkpmrodsfi:wui2ixaxyac76mbxtr6diw2ajky75wehk57f2kqgytddnpo4rozq:1:1:8388607 + format: chk + sample: + length: 8388607 + seed: w6uP8Tcg6K2QR905Rms8iXTlksL6OD1KOWBxTK7wxPI= + zfec: + required: 1 + segmentSize: 131072 + total: 1 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:SSK:ae222l7khyyye76gr4jkqhrtsu:s6rgutrzybfijp7og4f7ol3pumtpqhgcfu3m5gbpwlde6i6r52ha + format: sdmf + sample: + length: 8388607 + seed: w6uP8Tcg6K2QR905Rms8iXTlksL6OD1KOWBxTK7wxPI= + zfec: + required: 1 + segmentSize: 131072 + total: 1 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:MDMF:akoeczwc4fiogo5f4umlpzepxm:53putm3likdcbzu4g2cvwvqale7pp2silt5w3lnx5m2nml7qtrka + format: mdmf + sample: + length: 8388607 + seed: w6uP8Tcg6K2QR905Rms8iXTlksL6OD1KOWBxTK7wxPI= + zfec: + required: 1 + segmentSize: 131072 + total: 1 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:CHK:5eeprb6lfxclwt7fieskd2euly:ffjgjbrxfl6d2ug2a63iaesxtrqa5pk3yaagfcldqvo7x4ok7ykq:1:1:8388609 + format: chk + sample: + length: 8388609 + seed: yPi3JHKKbWaEEG5eZOlM6BHJll0Z3UTdBzz4bPQ7wjg= + zfec: + required: 1 + segmentSize: 131072 + total: 1 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:SSK:oefrpx2c7j2lslwsb4jvjcsqju:2m2nqlkgn5xoiip45mwc47zpcno74mayx2smd6lscjb7esme5aja + format: sdmf + sample: + length: 8388609 + seed: yPi3JHKKbWaEEG5eZOlM6BHJll0Z3UTdBzz4bPQ7wjg= + zfec: + required: 1 + segmentSize: 131072 + total: 1 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:MDMF:eqll4gbfnhbiuideqzgz2zgtum:e3t4vmtlcr4olgnk4wmocsbafhojaj4xakzolm6ds2srncgi67pq + format: mdmf + sample: + length: 8388609 + seed: yPi3JHKKbWaEEG5eZOlM6BHJll0Z3UTdBzz4bPQ7wjg= + zfec: + required: 1 + segmentSize: 131072 + total: 1 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:CHK:nnv4vrtlxmzkurfzvonj22leua:ywcyijrfnwykraku56dq7v3o4ts3xsxqfgmk3kgzwhq2cpest4pa:1:1:1024 + format: chk + sample: + length: 1024 + seed: YQ== + zfec: + required: 1 + segmentSize: 131072 + total: 1 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:SSK:bw66cmomyxpxzydovsfai4in4i:pxx76km232rfwl7zllsuxfjrpidq565r6p2ylf2h3e6frp65pq4q + format: sdmf + sample: + length: 1024 + seed: YQ== + zfec: + required: 1 + segmentSize: 131072 + total: 1 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:MDMF:tucs3y6kksigeaekahmz7jllum:5te4f5wh36j7m2x7rgrt2czofolh3z4zms3j6g6v7iw7aja4uywq + format: mdmf + sample: + length: 1024 + seed: YQ== + zfec: + required: 1 + segmentSize: 131072 + total: 1 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:CHK:wlszole5m2emf6wbhp3lnlyfxq:irdttvny74gxdrcotzfjslfq7p24kyiigyrm5shhw7zyj3hsiuba:1:1:4096 + format: chk + sample: + length: 4096 + seed: Yw== + zfec: + required: 1 + segmentSize: 131072 + total: 1 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:SSK:ntbuk6cv3mntzs645nb2yur5ea:ymrkers6rtknhsgg7oencxtaqfrxdtl4f4e345pguxlngpxmezka + format: sdmf + sample: + length: 4096 + seed: Yw== + zfec: + required: 1 + segmentSize: 131072 + total: 1 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:MDMF:tw4agh6qsiyp7eb6fc2q6wutfy:qu5ljqu3l3zetuddp3agsmq72nhpsw4kf3vzdfmbzwmyenefhfwa + format: mdmf + sample: + length: 4096 + seed: Yw== + zfec: + required: 1 + segmentSize: 131072 + total: 1 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:CHK:4ewm23jvdtm2i5xf4gck26wg5y:7cujxxc34mkfkmwhbemqtuixuektknmhhxlmujcekibf5amfwwga:1:1:131071 + format: chk + sample: + length: 131071 + seed: LCa0a2j/xo/5m0U8HTBBNBNCLXBkg7+g+YpeiGJm564= + zfec: + required: 1 + segmentSize: 131072 + total: 1 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:SSK:hgrowa5hbtubpvwl7kwuvqxzwq:oys24ziecxnvnnccdnawktcfwscpn6ukt3bn6pyispgqqoxzyunq + format: sdmf + sample: + length: 131071 + seed: LCa0a2j/xo/5m0U8HTBBNBNCLXBkg7+g+YpeiGJm564= + zfec: + required: 1 + segmentSize: 131072 + total: 1 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:MDMF:llsyycaa6sv3mozamcbrj25dqm:oaixtd5x4hkhpsdokd77qosigiiitpjygzn3j7fy5c4a4qffjifq + format: mdmf + sample: + length: 131071 + seed: LCa0a2j/xo/5m0U8HTBBNBNCLXBkg7+g+YpeiGJm564= + zfec: + required: 1 + segmentSize: 131072 + total: 1 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:CHK:heczpdxphw5frp3ri5sh2hpqei:6wflx6lphy5mhtpfb2abznoa3yk27ynbqbzcecs362miu72vkowq:1:1:131073 + format: chk + sample: + length: 131073 + seed: /N4rLtula/QIYB+3If6bXDONEO5CnqBPrlURto+/j7k= + zfec: + required: 1 + segmentSize: 131072 + total: 1 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:SSK:gkxj4zig7p6vdmq7qek2e6tcny:2ujwy5icd63hqdeibaaqag2xdsasbtut3unxgvhejmifvwwm7tja + format: sdmf + sample: + length: 131073 + seed: /N4rLtula/QIYB+3If6bXDONEO5CnqBPrlURto+/j7k= + zfec: + required: 1 + segmentSize: 131072 + total: 1 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:MDMF:iu3cu24qvryvfycl5hqbgs4p6e:a6vvnybvlkjlttah222hgykj6ekxdcfubqkifm6jz63aqigxnyuq + format: mdmf + sample: + length: 131073 + seed: /N4rLtula/QIYB+3If6bXDONEO5CnqBPrlURto+/j7k= + zfec: + required: 1 + segmentSize: 131072 + total: 1 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:CHK:mz5siv27pqttsl2f4vcqmxju64:rvxhulxtufho6pdbwj7rifneb6pei5fl4fqptcce7jdkbyrjfaya:1:1:2097151 + format: chk + sample: + length: 2097151 + seed: uqWglk0zIPvAxqkiFARTyFE+okq4/QV3A0gEqWckgJY= + zfec: + required: 1 + segmentSize: 131072 + total: 1 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:SSK:u4w73dcf4szq7g435f45rkkc64:jmy5yl24cxuvxoe6rviuigub5rupumwqaalyhum4auvgmdugukyq + format: sdmf + sample: + length: 2097151 + seed: uqWglk0zIPvAxqkiFARTyFE+okq4/QV3A0gEqWckgJY= + zfec: + required: 1 + segmentSize: 131072 + total: 1 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:MDMF:jg2rhh247nqaqjep4vw5uz7na4:mqfow763upbanik2i7xvgah4mxsofusfarjcxcsqj6mpd3iap7ua + format: mdmf + sample: + length: 2097151 + seed: uqWglk0zIPvAxqkiFARTyFE+okq4/QV3A0gEqWckgJY= + zfec: + required: 1 + segmentSize: 131072 + total: 1 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:CHK:shbt5viqjzuewblgt6qeijry6a:je3omw53tmmluz6fvqupx4uh4jaejc3fcvfjt56rfzokzhgdczvq:1:1:2097153 + format: chk + sample: + length: 2097153 + seed: BTBX/ampNfLU+ox7xipBGiaSbgC0kcB8Gy7BkJB4oKI= + zfec: + required: 1 + segmentSize: 131072 + total: 1 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:SSK:mbtm4xgwifkvsvgiku75lshela:uqxgrmldjvdt5wdjnxr2sxgg7okjc2p3xyhjuo66adesetnj4ipa + format: sdmf + sample: + length: 2097153 + seed: BTBX/ampNfLU+ox7xipBGiaSbgC0kcB8Gy7BkJB4oKI= + zfec: + required: 1 + segmentSize: 131072 + total: 1 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:MDMF:hytvw2gqv566qy24hdsevhikm4:m76axh6i3guh2wuwp242pea6a6jretjxu6otjwp2xcwxhzd6tw4a + format: mdmf + sample: + length: 2097153 + seed: BTBX/ampNfLU+ox7xipBGiaSbgC0kcB8Gy7BkJB4oKI= + zfec: + required: 1 + segmentSize: 131072 + total: 1 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:CHK:n2642ofjfr5qljptdwjbwjz6fq:uwtjrewrev42yvez7br2574xu5vleallwnyddjaxqquvouv3dyra:1:1:8388607 + format: chk + sample: + length: 8388607 + seed: w6uP8Tcg6K2QR905Rms8iXTlksL6OD1KOWBxTK7wxPI= + zfec: + required: 1 + segmentSize: 131072 + total: 1 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:SSK:z2nquzg3q2vu372xgwinybgrq4:cfdb4k2vovqljuynb24hbkdoqdtrjuf3paj3ei4kdv7jpjcmzq5a + format: sdmf + sample: + length: 8388607 + seed: w6uP8Tcg6K2QR905Rms8iXTlksL6OD1KOWBxTK7wxPI= + zfec: + required: 1 + segmentSize: 131072 + total: 1 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:MDMF:qolwd4bimeicby7cr4genjrjdu:c7pijeu66hqyzjvqbepagujq3gex6hzbq37fwgjj77voe2yhjjxq + format: mdmf + sample: + length: 8388607 + seed: w6uP8Tcg6K2QR905Rms8iXTlksL6OD1KOWBxTK7wxPI= + zfec: + required: 1 + segmentSize: 131072 + total: 1 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:CHK:nktoeeuydph4acj2fux4axyaj4:g3fvjwanenwsgdcn3oxnau5gtbdmzbnbevlrzrb5qe4yukuwhejq:1:1:8388609 + format: chk + sample: + length: 8388609 + seed: yPi3JHKKbWaEEG5eZOlM6BHJll0Z3UTdBzz4bPQ7wjg= + zfec: + required: 1 + segmentSize: 131072 + total: 1 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:SSK:xlosprcfmf7fghy6cp6d7j5gae:7di57dzhhx2chb22wddh5kspoqchcyaavkx3lyblld2q3ectkvka + format: sdmf + sample: + length: 8388609 + seed: yPi3JHKKbWaEEG5eZOlM6BHJll0Z3UTdBzz4bPQ7wjg= + zfec: + required: 1 + segmentSize: 131072 + total: 1 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:MDMF:pkul6c74muke6w3doyljnhspxq:3cc6azlsrdbvbg5jvtx75ft33hmcz2eprkfwfu3g3yms55mk4btq + format: mdmf + sample: + length: 8388609 + seed: yPi3JHKKbWaEEG5eZOlM6BHJll0Z3UTdBzz4bPQ7wjg= + zfec: + required: 1 + segmentSize: 131072 + total: 1 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:CHK:ycvogzi6wllnq2bkx3t6zdtwju:um42l4yen7jiwdfgirvedtty3tt3xuhjiyxzqoourvughtxjar3q:1:3:1024 + format: chk + sample: + length: 1024 + seed: YQ== + zfec: + required: 1 + segmentSize: 131072 + total: 3 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:SSK:eh55dfvzf46gcyd2u65bnmek6a:dx7nph6gpag4ijgzrro3rfezsjb7xair3whk7nvpiijfbiw6tgpq + format: sdmf + sample: + length: 1024 + seed: YQ== + zfec: + required: 1 + segmentSize: 131072 + total: 3 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:MDMF:jmlnqnpysa3ypddys45phkfjfe:2fkhnlpb6b7mqvc6cu3w7wsd6vme2swiymblvx7j5i3vhifph65a + format: mdmf + sample: + length: 1024 + seed: YQ== + zfec: + required: 1 + segmentSize: 131072 + total: 3 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:CHK:3yjtuv5h2g45g2cncg6hka3euu:bqqhp6u73ldawial7rtrfomx2qkmboyvqve6ywo2jpvfbu7zptga:1:3:4096 + format: chk + sample: + length: 4096 + seed: Yw== + zfec: + required: 1 + segmentSize: 131072 + total: 3 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:SSK:jli44aazmdb25uxpvyr65h5ea4:swm6gbbdwggi43etxxt5pnpwwzkcpb4tswe57xyl27tnf2pbkfaa + format: sdmf + sample: + length: 4096 + seed: Yw== + zfec: + required: 1 + segmentSize: 131072 + total: 3 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:MDMF:rqdgaoblt54p2jn43jye5ax7ri:md6p25qcasl2umo6udexfsp6hy67gi263a7vf3drnr2wx7e4fina + format: mdmf + sample: + length: 4096 + seed: Yw== + zfec: + required: 1 + segmentSize: 131072 + total: 3 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:CHK:annnqmu72p7iels5loqwlxt2zq:s7wv3jfi2hlpcvp4dnloc4eex6vre42kwiel46achaie5n5uodqa:1:3:131071 + format: chk + sample: + length: 131071 + seed: LCa0a2j/xo/5m0U8HTBBNBNCLXBkg7+g+YpeiGJm564= + zfec: + required: 1 + segmentSize: 131072 + total: 3 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:SSK:jvqmilw4sq73al6ahoj35yxoc4:7q2n6widrd24j26h3jfodnxkia2ggc4hq2dkpcavkf7w3jos4pwq + format: sdmf + sample: + length: 131071 + seed: LCa0a2j/xo/5m0U8HTBBNBNCLXBkg7+g+YpeiGJm564= + zfec: + required: 1 + segmentSize: 131072 + total: 3 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:MDMF:pca3f6mqonmw5om3vkkhsx6j2m:khnmpqz4w6bdrgmm7vbfe4a6p35zug5kinlmtd7btnhnp7ihlqua + format: mdmf + sample: + length: 131071 + seed: LCa0a2j/xo/5m0U8HTBBNBNCLXBkg7+g+YpeiGJm564= + zfec: + required: 1 + segmentSize: 131072 + total: 3 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:CHK:rhkln7unkktot72mit5dmuqbdy:ilo6u6hugipdimyrzrvlam47xsmp3ur2lwnrtbecmvocb2664zxq:1:3:131073 + format: chk + sample: + length: 131073 + seed: /N4rLtula/QIYB+3If6bXDONEO5CnqBPrlURto+/j7k= + zfec: + required: 1 + segmentSize: 131072 + total: 3 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:SSK:ugzqpyk57sfmw3xt3arpyh6qu4:octvdn5fzqhyzhedtleqky3uoti7jfzmlbfchqrhgt5zc5fs4qma + format: sdmf + sample: + length: 131073 + seed: /N4rLtula/QIYB+3If6bXDONEO5CnqBPrlURto+/j7k= + zfec: + required: 1 + segmentSize: 131072 + total: 3 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:MDMF:wuo2iiq4gdnvhe32ycvumfj7ce:lkbc7tsugufgw2xfiirjhqfujupx3347kus3j3ezcrvdkzknousa + format: mdmf + sample: + length: 131073 + seed: /N4rLtula/QIYB+3If6bXDONEO5CnqBPrlURto+/j7k= + zfec: + required: 1 + segmentSize: 131072 + total: 3 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:CHK:7yjcwwt6454lbv3pni5mjofsxe:y5nwpzwmvpvr3gqxnykjixprpxw3w6qyqmszf7ijyxnm3wl6f5oa:1:3:2097151 + format: chk + sample: + length: 2097151 + seed: uqWglk0zIPvAxqkiFARTyFE+okq4/QV3A0gEqWckgJY= + zfec: + required: 1 + segmentSize: 131072 + total: 3 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:SSK:cb6fgurdmns5w7jflfy23w73em:cbhxjn5oq7y6dpmlytlp7hubfdrrozf3krx2v5pnxwp4t4vrx2ma + format: sdmf + sample: + length: 2097151 + seed: uqWglk0zIPvAxqkiFARTyFE+okq4/QV3A0gEqWckgJY= + zfec: + required: 1 + segmentSize: 131072 + total: 3 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:MDMF:witiz6jbim5vox7yjisvwnyrhy:nfq4ku2xhgv4735ef3zxhi672cdvoeft7otpnm5nb5sv7yzkjrwa + format: mdmf + sample: + length: 2097151 + seed: uqWglk0zIPvAxqkiFARTyFE+okq4/QV3A0gEqWckgJY= + zfec: + required: 1 + segmentSize: 131072 + total: 3 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:CHK:r7wva6tisq6m5zszgr7seu77cm:oqlgdw3hi72qtahpsi3h3yryxpqdshagvt6xnobsppkwp7ic4cia:1:3:2097153 + format: chk + sample: + length: 2097153 + seed: BTBX/ampNfLU+ox7xipBGiaSbgC0kcB8Gy7BkJB4oKI= + zfec: + required: 1 + segmentSize: 131072 + total: 3 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:SSK:le27bab7put7a7rljkwgelvygi:dqy75zxapt4jjkkbswjhplaqyvtrfwdp5mrn72epcw7xcew55zrq + format: sdmf + sample: + length: 2097153 + seed: BTBX/ampNfLU+ox7xipBGiaSbgC0kcB8Gy7BkJB4oKI= + zfec: + required: 1 + segmentSize: 131072 + total: 3 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:MDMF:sb6ub2v5fqezulwyinr32lbf6q:fp6d2q463oluwsyaidq2dmpk2p6artvlyu7urtq76w2iqql6bpda + format: mdmf + sample: + length: 2097153 + seed: BTBX/ampNfLU+ox7xipBGiaSbgC0kcB8Gy7BkJB4oKI= + zfec: + required: 1 + segmentSize: 131072 + total: 3 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:CHK:mz7pzh24u6flkfmfvvokdwmriu:5mzo57de2ywsdqkdvm7557gcikkpjqbmj65mk235t2xyoejtpbta:1:3:8388607 + format: chk + sample: + length: 8388607 + seed: w6uP8Tcg6K2QR905Rms8iXTlksL6OD1KOWBxTK7wxPI= + zfec: + required: 1 + segmentSize: 131072 + total: 3 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:SSK:jp6u5obzgtzt7236v6to6c4ei4:ae7jynsdpceqfvbaa6skwvlippw6ss7ikrbm6h474tij6fwigh7q + format: sdmf + sample: + length: 8388607 + seed: w6uP8Tcg6K2QR905Rms8iXTlksL6OD1KOWBxTK7wxPI= + zfec: + required: 1 + segmentSize: 131072 + total: 3 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:MDMF:43wo2yychy5ougonjnbzzsvssy:t4uuvenzh2fimgufbgbcqdxc37tlhsl7lli6fj5qoykujxbn2rtq + format: mdmf + sample: + length: 8388607 + seed: w6uP8Tcg6K2QR905Rms8iXTlksL6OD1KOWBxTK7wxPI= + zfec: + required: 1 + segmentSize: 131072 + total: 3 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:CHK:54sq6lrqwqlpxicg747gdls6ri:2aaxyrdytn7r74my36ek434hlbhe6glrgr2ic5vvpc5bbdxmvo6a:1:3:8388609 + format: chk + sample: + length: 8388609 + seed: yPi3JHKKbWaEEG5eZOlM6BHJll0Z3UTdBzz4bPQ7wjg= + zfec: + required: 1 + segmentSize: 131072 + total: 3 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:SSK:5wrb7mqc2hzri7ekj4zxl4qad4:usym75mxkmnqkgqbtrui4e3q2362sa64w35u4bcumrkaibuljpiq + format: sdmf + sample: + length: 8388609 + seed: yPi3JHKKbWaEEG5eZOlM6BHJll0Z3UTdBzz4bPQ7wjg= + zfec: + required: 1 + segmentSize: 131072 + total: 3 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:MDMF:ldwwgiceepruadjywtjayvabse:zf7jofp4y6woanufvuvu2s4xaz6gdm64pfxqaufzmazjafycdvma + format: mdmf + sample: + length: 8388609 + seed: yPi3JHKKbWaEEG5eZOlM6BHJll0Z3UTdBzz4bPQ7wjg= + zfec: + required: 1 + segmentSize: 131072 + total: 3 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:CHK:pbfcdvxhbikxd2hcc43oel3v3e:y5txpmiptmoz36ionmki3p6krmdbiqasw2v3wdq4ia5lmrhh33lq:1:3:1024 + format: chk + sample: + length: 1024 + seed: YQ== + zfec: + required: 1 + segmentSize: 131072 + total: 3 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:SSK:vr7yiykd4yny5ygad6qsm7wklu:66ftrgc46osmagfzaxwzqpcvaemx674zh6jllvqhe4sqtbvv7hkq + format: sdmf + sample: + length: 1024 + seed: YQ== + zfec: + required: 1 + segmentSize: 131072 + total: 3 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:MDMF:aev7j2y474xff6wkczqmae474m:j7yujo66irpomwzonnf4e4uvvbdwsfnq2luhy2mope3u2z6mm7ha + format: mdmf + sample: + length: 1024 + seed: YQ== + zfec: + required: 1 + segmentSize: 131072 + total: 3 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:CHK:vkdc7iae4pgkdomhbbb554kv6a:g5xkatxv2entfssfh5eqgexb5j3qs6jvohofjqo5p5erlxlx2dga:1:3:4096 + format: chk + sample: + length: 4096 + seed: Yw== + zfec: + required: 1 + segmentSize: 131072 + total: 3 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:SSK:lzy6tvnsepowhab5gn7go3xcl4:ecgdtffw5rfujxbi5653zojr3lxj3lkf2cgwf5s4qz4ukag7cshq + format: sdmf + sample: + length: 4096 + seed: Yw== + zfec: + required: 1 + segmentSize: 131072 + total: 3 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:MDMF:lp2mhym5ltuihpfqmwyouskqxq:yxmxqzsn3px4qppjsn62psi3nwedbcxtg2gxlryiifcxtlf66ceq + format: mdmf + sample: + length: 4096 + seed: Yw== + zfec: + required: 1 + segmentSize: 131072 + total: 3 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:CHK:zvla2jyu5fqlb2s63zftyfjnla:yqspaexhew55cvv7w2m6czezhzkqnyk5fea4gvnplc5za4xlvbva:1:3:131071 + format: chk + sample: + length: 131071 + seed: LCa0a2j/xo/5m0U8HTBBNBNCLXBkg7+g+YpeiGJm564= + zfec: + required: 1 + segmentSize: 131072 + total: 3 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:SSK:waofhwlohtuqpcwuwl7fxozc5u:r5qrte3v2npvzirkiujhjfyi3poobwtyzhwtqbjiw5j6lgaxfsdq + format: sdmf + sample: + length: 131071 + seed: LCa0a2j/xo/5m0U8HTBBNBNCLXBkg7+g+YpeiGJm564= + zfec: + required: 1 + segmentSize: 131072 + total: 3 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:MDMF:lg5dfsjapjmq64z4klon5aembe:an3vjcuovsxhgwownhlle6di5utdgdwe34ji7xdcrugors5nlmfa + format: mdmf + sample: + length: 131071 + seed: LCa0a2j/xo/5m0U8HTBBNBNCLXBkg7+g+YpeiGJm564= + zfec: + required: 1 + segmentSize: 131072 + total: 3 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:CHK:3idf2xqm5wnw2q3nbva5vmpaiq:7nmkh5q55omjloezibvyzvveq4gqeaivei5ypfm5vfuugwiz6hpa:1:3:131073 + format: chk + sample: + length: 131073 + seed: /N4rLtula/QIYB+3If6bXDONEO5CnqBPrlURto+/j7k= + zfec: + required: 1 + segmentSize: 131072 + total: 3 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:SSK:u2wzqdemrrm2h5lntniddeajmq:b6cf7fvw2m23ulucwe63poe5ebhqiwppmcqoabkzsoc5m2m3sthq + format: sdmf + sample: + length: 131073 + seed: /N4rLtula/QIYB+3If6bXDONEO5CnqBPrlURto+/j7k= + zfec: + required: 1 + segmentSize: 131072 + total: 3 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:MDMF:v4bediwxuymp2immfnuhfx4buy:vix7mxu6wgi46bamw2y4etbtm3exc522i3xasrvy72tcin2v3uka + format: mdmf + sample: + length: 131073 + seed: /N4rLtula/QIYB+3If6bXDONEO5CnqBPrlURto+/j7k= + zfec: + required: 1 + segmentSize: 131072 + total: 3 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:CHK:o7xdokumtcukutzgmmpfme4nnm:nn3s5gau5dychy6wlodwriuit4wyavfze6bo4icywcbhdb4ccxla:1:3:2097151 + format: chk + sample: + length: 2097151 + seed: uqWglk0zIPvAxqkiFARTyFE+okq4/QV3A0gEqWckgJY= + zfec: + required: 1 + segmentSize: 131072 + total: 3 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:SSK:fsjb2ym6pgn3fupo66jrmmwxia:3ix37zaem6w2zsotsk5hjj4ygst7r27zecfpvehnugwyw4lq3ija + format: sdmf + sample: + length: 2097151 + seed: uqWglk0zIPvAxqkiFARTyFE+okq4/QV3A0gEqWckgJY= + zfec: + required: 1 + segmentSize: 131072 + total: 3 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:MDMF:otowqw6mrbgz4tiyslxpfi3k2u:xhqeu536q4q3qi6hhqyjd52zthrdchhtmpxmu2pt7n4ff74kl37a + format: mdmf + sample: + length: 2097151 + seed: uqWglk0zIPvAxqkiFARTyFE+okq4/QV3A0gEqWckgJY= + zfec: + required: 1 + segmentSize: 131072 + total: 3 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:CHK:vws2xrl2nch2hsptqb36yqqu3e:fhujsv5rkhyiqstrktjvfy5235c7gcwekyd3gux3bv2glsgstjga:1:3:2097153 + format: chk + sample: + length: 2097153 + seed: BTBX/ampNfLU+ox7xipBGiaSbgC0kcB8Gy7BkJB4oKI= + zfec: + required: 1 + segmentSize: 131072 + total: 3 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:SSK:fhkkaxbg7y7piwv7fh5ylpdxhm:rewuizpn6v4xqy6df3bddfhp373iw76lh4qx6stzrwzskpedegza + format: sdmf + sample: + length: 2097153 + seed: BTBX/ampNfLU+ox7xipBGiaSbgC0kcB8Gy7BkJB4oKI= + zfec: + required: 1 + segmentSize: 131072 + total: 3 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:MDMF:p3d434skzrqcrxiotnp3u2ij34:tc6mps24psn2zg27ryfuquovzgfgrkxqg3ckibisw4ijftnuuakq + format: mdmf + sample: + length: 2097153 + seed: BTBX/ampNfLU+ox7xipBGiaSbgC0kcB8Gy7BkJB4oKI= + zfec: + required: 1 + segmentSize: 131072 + total: 3 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:CHK:5anac46dpfsjumutzsoovvtlpe:yuc3gqe7m6s6xp7k3uodcg2g6k2bac32gt5amu7gdx7k43qnudxa:1:3:8388607 + format: chk + sample: + length: 8388607 + seed: w6uP8Tcg6K2QR905Rms8iXTlksL6OD1KOWBxTK7wxPI= + zfec: + required: 1 + segmentSize: 131072 + total: 3 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:SSK:o7gx4tbbuqac6khwlosk4l5ddy:sy2po455xvmrz2o5mvvjzmjrzlpue4fzqudprcgxfpa3hxnkx5sa + format: sdmf + sample: + length: 8388607 + seed: w6uP8Tcg6K2QR905Rms8iXTlksL6OD1KOWBxTK7wxPI= + zfec: + required: 1 + segmentSize: 131072 + total: 3 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:MDMF:xck4irhtemsujzgkdtdthdpare:rfrfsjzznueky3h33uptvgxdtiwbqaelfnwtt3jcve4ju63zxfna + format: mdmf + sample: + length: 8388607 + seed: w6uP8Tcg6K2QR905Rms8iXTlksL6OD1KOWBxTK7wxPI= + zfec: + required: 1 + segmentSize: 131072 + total: 3 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:CHK:fb6g3fkfbtlwzheujzvc2dn7dq:644je55ri6q5halshuxfesjz2afhqtebetuzqqbfecp6yhqwbb5q:1:3:8388609 + format: chk + sample: + length: 8388609 + seed: yPi3JHKKbWaEEG5eZOlM6BHJll0Z3UTdBzz4bPQ7wjg= + zfec: + required: 1 + segmentSize: 131072 + total: 3 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:SSK:d7uzl2zychjkmm7siybhbjys3m:cmoram5uvqusxc3x5oi5wy7lfsvffjnfyktkywnc6camhiitjmfa + format: sdmf + sample: + length: 8388609 + seed: yPi3JHKKbWaEEG5eZOlM6BHJll0Z3UTdBzz4bPQ7wjg= + zfec: + required: 1 + segmentSize: 131072 + total: 3 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:MDMF:lo2p6saqx5kvmfgrsnf6fhiwei:zhjuapt2wwkacd5oa2li3aahrs36q3dckmljdbrsnsnmd7fygkxq + format: mdmf + sample: + length: 8388609 + seed: yPi3JHKKbWaEEG5eZOlM6BHJll0Z3UTdBzz4bPQ7wjg= + zfec: + required: 1 + segmentSize: 131072 + total: 3 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:CHK:jdm6ytmx2i3ya2bllsdzxurjtu:iqr6tyysaseegzfurhuywy3mbbwkbbsov5bt3fo6oazpyv7olvda:2:3:1024 + format: chk + sample: + length: 1024 + seed: YQ== + zfec: + required: 2 + segmentSize: 131072 + total: 3 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:SSK:znxwd4qkfko7ss74cgtribl76m:yubnd3unz4ipv2mm3khl4jbpwzzoxa3mnenq6n6tqpfydujfde4q + format: sdmf + sample: + length: 1024 + seed: YQ== + zfec: + required: 2 + segmentSize: 131072 + total: 3 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:MDMF:4m7y4dobd6ret6ikj36afo3bki:bm4oup3oijsfshzisvt52xcif7ks7x7sq3u7k5aarc44kzqlhabq + format: mdmf + sample: + length: 1024 + seed: YQ== + zfec: + required: 2 + segmentSize: 131072 + total: 3 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:CHK:7wz5dhkyob3gmkydzgbptk5qku:cl6ovq5a3km7rpfhyb3putg656sp57lhnp7aexbocsbbahdxnfia:2:3:4096 + format: chk + sample: + length: 4096 + seed: Yw== + zfec: + required: 2 + segmentSize: 131072 + total: 3 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:SSK:36b6rvsrqya5oqehistbht2y3q:ynyxc67qeewensnisyp5voubt5qkf27fkmdspzpnkzgmmuejrnya + format: sdmf + sample: + length: 4096 + seed: Yw== + zfec: + required: 2 + segmentSize: 131072 + total: 3 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:MDMF:gyxpdtxuyzamxrehlyjrw2pf4e:wrdnxnzcivydr4szmfxseaqgawacudx74n6wmddgfrglkoi7467q + format: mdmf + sample: + length: 4096 + seed: Yw== + zfec: + required: 2 + segmentSize: 131072 + total: 3 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:CHK:aagw6esdm2msphvgnr3rlzg43e:zqxtrlee6hh3vtfg6ihtssjqvr27tpvi6gkmngnhiguttjz7yyja:2:3:131071 + format: chk + sample: + length: 131071 + seed: LCa0a2j/xo/5m0U8HTBBNBNCLXBkg7+g+YpeiGJm564= + zfec: + required: 2 + segmentSize: 131072 + total: 3 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:SSK:dicmwi2p2mzxp2nf7j7nxj2tma:iialvaqlims3gf4whxfmjysijvesovxy4q4ecjrm636j2cnadhwa + format: sdmf + sample: + length: 131071 + seed: LCa0a2j/xo/5m0U8HTBBNBNCLXBkg7+g+YpeiGJm564= + zfec: + required: 2 + segmentSize: 131072 + total: 3 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:MDMF:usko2l5rufksoav3faz6w3vmue:tkpane2dyaul4mosc6lc35os72t3s54ga4u7dedlxhu7fhi3ovda + format: mdmf + sample: + length: 131071 + seed: LCa0a2j/xo/5m0U8HTBBNBNCLXBkg7+g+YpeiGJm564= + zfec: + required: 2 + segmentSize: 131072 + total: 3 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:CHK:e6uhixvbm3g5amrzdzd3mnmsqm:4n5aafrb64sqpfnhcdpdfrk5gh2qznprjky6q6jybuuqy2q6z6mq:2:3:131073 + format: chk + sample: + length: 131073 + seed: /N4rLtula/QIYB+3If6bXDONEO5CnqBPrlURto+/j7k= + zfec: + required: 2 + segmentSize: 131072 + total: 3 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:SSK:tc346l5ky77pyqzunsobixc2am:4jk7vy4kie35wg4tdthqjczbc7pv62qbc5dbh2kf653q2k6optbq + format: sdmf + sample: + length: 131073 + seed: /N4rLtula/QIYB+3If6bXDONEO5CnqBPrlURto+/j7k= + zfec: + required: 2 + segmentSize: 131072 + total: 3 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:MDMF:xomeoo2rxocwsrm6lgui4krebe:jz7oodng7ym2fb7g5hbxqacetkb5aizdoqi5lxyavgrg7goyq45q + format: mdmf + sample: + length: 131073 + seed: /N4rLtula/QIYB+3If6bXDONEO5CnqBPrlURto+/j7k= + zfec: + required: 2 + segmentSize: 131072 + total: 3 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:CHK:abfmzuiczjwt6e67f3uoyg5i7q:rk7ie3afnp3xgvnxu5e2vioq476xqwslqxgyx3i5xxevsjhkcvba:2:3:2097151 + format: chk + sample: + length: 2097151 + seed: uqWglk0zIPvAxqkiFARTyFE+okq4/QV3A0gEqWckgJY= + zfec: + required: 2 + segmentSize: 131072 + total: 3 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:SSK:ixed54x47cxuzgsfwm7g4lhzhe:nrqzkdl6gb3jrnalhprhtqy2o4vj5u2cvpyf7nnagxximc36yeiq + format: sdmf + sample: + length: 2097151 + seed: uqWglk0zIPvAxqkiFARTyFE+okq4/QV3A0gEqWckgJY= + zfec: + required: 2 + segmentSize: 131072 + total: 3 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:MDMF:i4curydlvv2ki6a6iz4ttgtdhq:3augdg7vy7z3ubjer6nroe7foxwwmgl2hpg4o6zqake6zvpe43ga + format: mdmf + sample: + length: 2097151 + seed: uqWglk0zIPvAxqkiFARTyFE+okq4/QV3A0gEqWckgJY= + zfec: + required: 2 + segmentSize: 131072 + total: 3 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:CHK:34bzejdzpqinmuzdmqenkidf74:pcecbl4ulygiadgdagonbgqmpb4lomjz4v4vqyssr56reyib4q2q:2:3:2097153 + format: chk + sample: + length: 2097153 + seed: BTBX/ampNfLU+ox7xipBGiaSbgC0kcB8Gy7BkJB4oKI= + zfec: + required: 2 + segmentSize: 131072 + total: 3 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:SSK:e2bzrxjvgyhmc45bmtj4rlerce:wobzlw4rpwuufqmjo6tep5mhrbebs3wdqcy7zkkefbs2fxacok5q + format: sdmf + sample: + length: 2097153 + seed: BTBX/ampNfLU+ox7xipBGiaSbgC0kcB8Gy7BkJB4oKI= + zfec: + required: 2 + segmentSize: 131072 + total: 3 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:MDMF:fxnfxuggzofpoorlm2zn7dg6ty:tk2zuztd7j4gqmn56c6ffddqgzwenbudl7muvrjwve7gghgvceia + format: mdmf + sample: + length: 2097153 + seed: BTBX/ampNfLU+ox7xipBGiaSbgC0kcB8Gy7BkJB4oKI= + zfec: + required: 2 + segmentSize: 131072 + total: 3 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:CHK:ej56vl65kawat62wsxuvbcd2wy:tv65cgghaddq2p7dyvimmff24l34wsx6mhzd3dyprfvrdo52qtma:2:3:8388607 + format: chk + sample: + length: 8388607 + seed: w6uP8Tcg6K2QR905Rms8iXTlksL6OD1KOWBxTK7wxPI= + zfec: + required: 2 + segmentSize: 131072 + total: 3 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:SSK:5pyvbw7axzkw72rtlvwelgfcfy:d5acbwniecttboqvyn3ihe7h5ehpigy6u5tz73vf3aj25yeikxsa + format: sdmf + sample: + length: 8388607 + seed: w6uP8Tcg6K2QR905Rms8iXTlksL6OD1KOWBxTK7wxPI= + zfec: + required: 2 + segmentSize: 131072 + total: 3 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:MDMF:bcrt7yjrgcofkamredsw5x4bbm:hqiucmqok5qvns6yh2o5sibe7bg3ijodtp2ur3wkxhmt4bmwin7q + format: mdmf + sample: + length: 8388607 + seed: w6uP8Tcg6K2QR905Rms8iXTlksL6OD1KOWBxTK7wxPI= + zfec: + required: 2 + segmentSize: 131072 + total: 3 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:CHK:ufshkmuewwql2xsyiecbjhh7x4:qoywwygjqrmifowob3mvwkhfskk6geomyw5e3qlgqzui3kymu6nq:2:3:8388609 + format: chk + sample: + length: 8388609 + seed: yPi3JHKKbWaEEG5eZOlM6BHJll0Z3UTdBzz4bPQ7wjg= + zfec: + required: 2 + segmentSize: 131072 + total: 3 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:SSK:5gjangydj5e7ziu6jmzuo2wvam:wb5wy5civ5e6tecfohqjvbj5r7iggv765lm3jpeq4iecbxye2apa + format: sdmf + sample: + length: 8388609 + seed: yPi3JHKKbWaEEG5eZOlM6BHJll0Z3UTdBzz4bPQ7wjg= + zfec: + required: 2 + segmentSize: 131072 + total: 3 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:MDMF:pdbqgmt5wmcccwznv4ksqhukra:e2yek3bursdelejqx2maiqk7maevukhqzzjbw2wtzqyhbs2o4s6a + format: mdmf + sample: + length: 8388609 + seed: yPi3JHKKbWaEEG5eZOlM6BHJll0Z3UTdBzz4bPQ7wjg= + zfec: + required: 2 + segmentSize: 131072 + total: 3 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:CHK:ld3thziutpaqv25nbtoqebhtru:ri2obsvzl2etyuv3qnchn2wvw5mh5rjuvbjaqkqbvyprn7xwamoq:2:3:1024 + format: chk + sample: + length: 1024 + seed: YQ== + zfec: + required: 2 + segmentSize: 131072 + total: 3 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:SSK:zamaagvy42hzahq6e5ze275hya:c2yru2mkkzmmaovfzez7g63tczbz4vsfncfiamdjwtay2f4rbf6a + format: sdmf + sample: + length: 1024 + seed: YQ== + zfec: + required: 2 + segmentSize: 131072 + total: 3 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:MDMF:rdemgunqoy4rxxmpyhwqmolmwi:dwt4vxhvrcgk2zupwa3nq4mlzr54mkrenjrv36t5co6yeg4oefpa + format: mdmf + sample: + length: 1024 + seed: YQ== + zfec: + required: 2 + segmentSize: 131072 + total: 3 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:CHK:pz3mga5pvqd4bpsz7pbyna2qtq:w3ptt7xwwgywod3u2rpgx7iggvm2kpwl7d2ryr3u7mcaxgqotlaq:2:3:4096 + format: chk + sample: + length: 4096 + seed: Yw== + zfec: + required: 2 + segmentSize: 131072 + total: 3 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:SSK:n7hevwssawse5kret3d4yux7ge:i4y2ff2lv6rw2byrvxcl2k6hda26ol25ucv2wn7t6rw55k3ype2a + format: sdmf + sample: + length: 4096 + seed: Yw== + zfec: + required: 2 + segmentSize: 131072 + total: 3 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:MDMF:gwrzotpvkkffjfw74bipj6g3ue:ue2frkaek5t6jpstjehu6ah4euikq3jatkdgfkgspzjhtucguw5a + format: mdmf + sample: + length: 4096 + seed: Yw== + zfec: + required: 2 + segmentSize: 131072 + total: 3 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:CHK:rsh7ysvsgbwh5g3xuj6z7zky3y:ykg5rhzpzqkvdy53qqyghfbcusibynfylggbhypo3iibsrrzvlkq:2:3:131071 + format: chk + sample: + length: 131071 + seed: LCa0a2j/xo/5m0U8HTBBNBNCLXBkg7+g+YpeiGJm564= + zfec: + required: 2 + segmentSize: 131072 + total: 3 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:SSK:5ajxk7jw4g22ga57hayghtvnam:exlqadg54twmlyhkgicn2iaqt65luupeebh7mheabppsdlh5ye7a + format: sdmf + sample: + length: 131071 + seed: LCa0a2j/xo/5m0U8HTBBNBNCLXBkg7+g+YpeiGJm564= + zfec: + required: 2 + segmentSize: 131072 + total: 3 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:MDMF:c4esc6ob3alfyqqob76oveefeu:ramdhh6ufpbn5ygeo5gilwkxyx3oi4i7k7g3pbic3nf6qpvw7m3q + format: mdmf + sample: + length: 131071 + seed: LCa0a2j/xo/5m0U8HTBBNBNCLXBkg7+g+YpeiGJm564= + zfec: + required: 2 + segmentSize: 131072 + total: 3 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:CHK:7xqejfqfej3u3zp2qymqd2iqeq:po4t2tzkh4d34ku6gdhlocadfwdyweiyckyqz57zdi7ndt4ikj3q:2:3:131073 + format: chk + sample: + length: 131073 + seed: /N4rLtula/QIYB+3If6bXDONEO5CnqBPrlURto+/j7k= + zfec: + required: 2 + segmentSize: 131072 + total: 3 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:SSK:5bau524o4vbrnowrba2j3l44ca:skxdvkz5ufb4u7fegccpmmq3tx32tqse6udtb6v5cutb765i5toq + format: sdmf + sample: + length: 131073 + seed: /N4rLtula/QIYB+3If6bXDONEO5CnqBPrlURto+/j7k= + zfec: + required: 2 + segmentSize: 131072 + total: 3 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:MDMF:hyyofiigvckijkrxqlgzq55e7a:uth6x2l6ocak4xcozm5izzsd24ahymu35luxdmufb5oqo337g6mq + format: mdmf + sample: + length: 131073 + seed: /N4rLtula/QIYB+3If6bXDONEO5CnqBPrlURto+/j7k= + zfec: + required: 2 + segmentSize: 131072 + total: 3 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:CHK:kzhg5zp4hcs75eboack4kqpcdm:g2cwnhpfwxrv3c4lrmpjyj327a274oa5qt4isu4avjixbu7vblrq:2:3:2097151 + format: chk + sample: + length: 2097151 + seed: uqWglk0zIPvAxqkiFARTyFE+okq4/QV3A0gEqWckgJY= + zfec: + required: 2 + segmentSize: 131072 + total: 3 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:SSK:cibwsbzizcapwki2exx4rkr2qu:gkujzvhxmjhb2z4wzecyc225xyxe3tqqfvaf2s2oc6rzmdn5wp7a + format: sdmf + sample: + length: 2097151 + seed: uqWglk0zIPvAxqkiFARTyFE+okq4/QV3A0gEqWckgJY= + zfec: + required: 2 + segmentSize: 131072 + total: 3 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:MDMF:kupuwzss5lowgfl6adcvhgazfq:2jfedarh23rfnwzxhdkae6b2mh2dfzmh7uadpw4pcr622bls4mfq + format: mdmf + sample: + length: 2097151 + seed: uqWglk0zIPvAxqkiFARTyFE+okq4/QV3A0gEqWckgJY= + zfec: + required: 2 + segmentSize: 131072 + total: 3 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:CHK:6sf35smixwvpapf2zw7sllxet4:ks5zd3hkd7ppwobhnymezckszexpychngnkipxfvmlcng5fgjbea:2:3:2097153 + format: chk + sample: + length: 2097153 + seed: BTBX/ampNfLU+ox7xipBGiaSbgC0kcB8Gy7BkJB4oKI= + zfec: + required: 2 + segmentSize: 131072 + total: 3 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:SSK:w5nx47bm7jdf4l4ht47235krte:deoa4hwzahmy2bcws7hz3ry3ubpemllu5marnaocewhode2ucdpa + format: sdmf + sample: + length: 2097153 + seed: BTBX/ampNfLU+ox7xipBGiaSbgC0kcB8Gy7BkJB4oKI= + zfec: + required: 2 + segmentSize: 131072 + total: 3 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:MDMF:argkvyswpyjfrtmuyg3jvwyswa:6d373u2djhnw6ug3yutlghkpb22bjmphl76vviclk7rc5bsj7koa + format: mdmf + sample: + length: 2097153 + seed: BTBX/ampNfLU+ox7xipBGiaSbgC0kcB8Gy7BkJB4oKI= + zfec: + required: 2 + segmentSize: 131072 + total: 3 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:CHK:dnmjqjku5vyrkeaulae4bt6juu:d47airwaqedeji2omacwsp6yfqp5mpurj776gmzvq43c6dovtdca:2:3:8388607 + format: chk + sample: + length: 8388607 + seed: w6uP8Tcg6K2QR905Rms8iXTlksL6OD1KOWBxTK7wxPI= + zfec: + required: 2 + segmentSize: 131072 + total: 3 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:SSK:whwbzc5q4qtpfmmx5udqt4ajxa:7vpif7ukkmrmnifk3mech6g6n77x3345t7oqoqh46apzwbwomqha + format: sdmf + sample: + length: 8388607 + seed: w6uP8Tcg6K2QR905Rms8iXTlksL6OD1KOWBxTK7wxPI= + zfec: + required: 2 + segmentSize: 131072 + total: 3 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:MDMF:767fhe755a4stlnvnm6y66mi4e:cbq5fdp6zcjg2xjqanlwsgc6s3jbc4hf62drzhjcmlsfjolp4xya + format: mdmf + sample: + length: 8388607 + seed: w6uP8Tcg6K2QR905Rms8iXTlksL6OD1KOWBxTK7wxPI= + zfec: + required: 2 + segmentSize: 131072 + total: 3 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:CHK:bfjedwcwjehjzpwjsgwkfk7rja:jpwea2sgz4hfohqab642yj4cmrh64w6nfo7lv3mtacfzicurqkva:2:3:8388609 + format: chk + sample: + length: 8388609 + seed: yPi3JHKKbWaEEG5eZOlM6BHJll0Z3UTdBzz4bPQ7wjg= + zfec: + required: 2 + segmentSize: 131072 + total: 3 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:SSK:ri6woi5eti2ig6mbxvytuyza7y:lgdm3v2jghtjlpv32rjuzosqd4cyhr6jkohyxaw6nmkv7i4t3oua + format: sdmf + sample: + length: 8388609 + seed: yPi3JHKKbWaEEG5eZOlM6BHJll0Z3UTdBzz4bPQ7wjg= + zfec: + required: 2 + segmentSize: 131072 + total: 3 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:MDMF:zujw3a76gbitwmnaqd26kbsdoa:gajfmtn4iuexx6x36i2aftawkkr4xeraxlkspxg6uqhsw3iz3iwq + format: mdmf + sample: + length: 8388609 + seed: yPi3JHKKbWaEEG5eZOlM6BHJll0Z3UTdBzz4bPQ7wjg= + zfec: + required: 2 + segmentSize: 131072 + total: 3 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:CHK:dx7tvyr2fc4u7lxjc6kehq2svq:tiy4qh2g6lqejxcaym3rr7ymkdkinn4qised6kgxloj7sptsqu4a:3:10:1024 + format: chk + sample: + length: 1024 + seed: YQ== + zfec: + required: 3 + segmentSize: 131072 + total: 10 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:SSK:onr7swqc3kgyec6ocfcvyjmx6q:or2bm3iuf7crsietg6dkidtjbmfiu2eulvllgz5e5uelqt2ey5eq + format: sdmf + sample: + length: 1024 + seed: YQ== + zfec: + required: 3 + segmentSize: 131072 + total: 10 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:MDMF:3ceb4g2e4twofve5jkewepvnaa:oztau4y5nizlwn3psdukebxhhgitalp4p6b2tcvpm4ogiswwcuua + format: mdmf + sample: + length: 1024 + seed: YQ== + zfec: + required: 3 + segmentSize: 131072 + total: 10 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:CHK:yo75evk4cte3b7rdw72zxvl5ye:ex6h7ff7nclucjtsqwgwu33qgmb67t4ezbrki4zbgurwn2ct6bbq:3:10:4096 + format: chk + sample: + length: 4096 + seed: Yw== + zfec: + required: 3 + segmentSize: 131072 + total: 10 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:SSK:iugq34v2swtiwjbyy2sce7cjte:k3bloctsmve774usd36mg34pmxiu52w64zhaarvhwwlubnq66h4a + format: sdmf + sample: + length: 4096 + seed: Yw== + zfec: + required: 3 + segmentSize: 131072 + total: 10 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:MDMF:bnvrx3ubou6wyjqmz7ds2dqjlu:oaq44pgwyogdvtf5lb3dtotdfhrzedwex7svyjmnhibnelotvzxq + format: mdmf + sample: + length: 4096 + seed: Yw== + zfec: + required: 3 + segmentSize: 131072 + total: 10 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:CHK:4gokef54smahrbfr4kq3jhc4zq:owpwwfp5gof2vhly5u6jdnbsfuwwwhqkazpsbeg3nldxv5pse2iq:3:10:131071 + format: chk + sample: + length: 131071 + seed: LCa0a2j/xo/5m0U8HTBBNBNCLXBkg7+g+YpeiGJm564= + zfec: + required: 3 + segmentSize: 131072 + total: 10 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:SSK:y4onydf5hf23ndbobhxtucmfsq:3zpmw5rcysqk7jf7yfhqfiptn6pwelgqfoh5tcajwp42isa64mtq + format: sdmf + sample: + length: 131071 + seed: LCa0a2j/xo/5m0U8HTBBNBNCLXBkg7+g+YpeiGJm564= + zfec: + required: 3 + segmentSize: 131072 + total: 10 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:MDMF:xalxckir42vumn4twsj5sr47qm:mlkqxei4kgqy6jwvwfwwwj6q4bvlr3qutbn7d3wcgimvjaip42ja + format: mdmf + sample: + length: 131071 + seed: LCa0a2j/xo/5m0U8HTBBNBNCLXBkg7+g+YpeiGJm564= + zfec: + required: 3 + segmentSize: 131072 + total: 10 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:CHK:7vfgl5cv4nlzqx35z4uthjv36y:nnueftbzxfz6u5yjxwwofaxzzft7xss5wzfh66rrcwv2zwrm63sa:3:10:131073 + format: chk + sample: + length: 131073 + seed: /N4rLtula/QIYB+3If6bXDONEO5CnqBPrlURto+/j7k= + zfec: + required: 3 + segmentSize: 131072 + total: 10 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:SSK:bptkj6t37usy4usosftkkuhv7e:ckumolq5lngoe3nkyqj2x3bfd4omxjpe2z4ajfeqfxh6hbfzavvq + format: sdmf + sample: + length: 131073 + seed: /N4rLtula/QIYB+3If6bXDONEO5CnqBPrlURto+/j7k= + zfec: + required: 3 + segmentSize: 131072 + total: 10 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:MDMF:zuqe744q3gyn7wmkuj2wrvscbq:n2z2hcwdlmtiwbg6tutzlkdiwjccqazvxapj6kek22rnff7yu47q + format: mdmf + sample: + length: 131073 + seed: /N4rLtula/QIYB+3If6bXDONEO5CnqBPrlURto+/j7k= + zfec: + required: 3 + segmentSize: 131072 + total: 10 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:CHK:cy7fmjsldeakhfd4psbmghmqyi:6a6uvyai4jkzz6hj5yjugm6uie5etvymcudgiwwjh47apz636zwa:3:10:2097151 + format: chk + sample: + length: 2097151 + seed: uqWglk0zIPvAxqkiFARTyFE+okq4/QV3A0gEqWckgJY= + zfec: + required: 3 + segmentSize: 131072 + total: 10 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:SSK:yys5phbf5zmwlmie6kf52n46ga:lh3huulqjziteauw2l2vngnef2jenb3bwrgylcqh3pklpntpzh3q + format: sdmf + sample: + length: 2097151 + seed: uqWglk0zIPvAxqkiFARTyFE+okq4/QV3A0gEqWckgJY= + zfec: + required: 3 + segmentSize: 131072 + total: 10 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:MDMF:5rl3h6trfxgyavfavxmnnzqhxy:humgb27p2kmwnw5dm4shyjeprvzemaxokjgol3jo4x7kpgkg4peq + format: mdmf + sample: + length: 2097151 + seed: uqWglk0zIPvAxqkiFARTyFE+okq4/QV3A0gEqWckgJY= + zfec: + required: 3 + segmentSize: 131072 + total: 10 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:CHK:ptsofqwylmkvzmuvrw5n34j3ma:ky2fs7xrlke64w6kfmhzsuilzxbhfrwwzkxih4rykpbxrr3bxhiq:3:10:2097153 + format: chk + sample: + length: 2097153 + seed: BTBX/ampNfLU+ox7xipBGiaSbgC0kcB8Gy7BkJB4oKI= + zfec: + required: 3 + segmentSize: 131072 + total: 10 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:SSK:yjjkeiwrkl4zxvotwzgjodmcxm:ankmcys42fn27r733ngkw3ez5xfhh3g5swjcwwmggpjpwm4icy2q + format: sdmf + sample: + length: 2097153 + seed: BTBX/ampNfLU+ox7xipBGiaSbgC0kcB8Gy7BkJB4oKI= + zfec: + required: 3 + segmentSize: 131072 + total: 10 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:MDMF:6kbz7i5wmha5hiil4rdypemuqi:yd2wd25fpa6tlhovmzw7sw7hzdurseapr66uuft6dc57a2sxlbia + format: mdmf + sample: + length: 2097153 + seed: BTBX/ampNfLU+ox7xipBGiaSbgC0kcB8Gy7BkJB4oKI= + zfec: + required: 3 + segmentSize: 131072 + total: 10 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:CHK:tyt4hfvn44igztqh5zeiusggcm:ex3gzzn7byvhajithwbo7c3p7qwqxmnactaxxsxsxowkafyjrf7q:3:10:8388607 + format: chk + sample: + length: 8388607 + seed: w6uP8Tcg6K2QR905Rms8iXTlksL6OD1KOWBxTK7wxPI= + zfec: + required: 3 + segmentSize: 131072 + total: 10 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:SSK:zufl4t2imcsu7v72ajasbpgw6q:hm6kgc6zunfcmgf64nne3tcaonrgd4fvyuuavo3nq6g33rc3ttua + format: sdmf + sample: + length: 8388607 + seed: w6uP8Tcg6K2QR905Rms8iXTlksL6OD1KOWBxTK7wxPI= + zfec: + required: 3 + segmentSize: 131072 + total: 10 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:MDMF:ds6otzysc6jsfv7oz3yaw7ha5y:a24cpkrdpsx4rqyg2wq42i7lxpm3lsw677qe2wktss3r57cpmrwq + format: mdmf + sample: + length: 8388607 + seed: w6uP8Tcg6K2QR905Rms8iXTlksL6OD1KOWBxTK7wxPI= + zfec: + required: 3 + segmentSize: 131072 + total: 10 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:CHK:xymheose4rdlspgydkzr4nqkre:z3pfrvpq5fdpkoybhdxppwbzrt6ejf26xh6emzlce2sgquljginq:3:10:8388609 + format: chk + sample: + length: 8388609 + seed: yPi3JHKKbWaEEG5eZOlM6BHJll0Z3UTdBzz4bPQ7wjg= + zfec: + required: 3 + segmentSize: 131072 + total: 10 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:SSK:uuge2bhk3xgcravcofl5pjsvie:geu3qejgjdmz6gylzgua3xnpa5dbpie6ld67ai2jswwpz4d6ioea + format: sdmf + sample: + length: 8388609 + seed: yPi3JHKKbWaEEG5eZOlM6BHJll0Z3UTdBzz4bPQ7wjg= + zfec: + required: 3 + segmentSize: 131072 + total: 10 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:MDMF:hhpclg55lfzmzzdqgnj7os7tda:26l4i2iniero76up73gcih3ksdqlfff3ztbtc3qekzzfumochsyq + format: mdmf + sample: + length: 8388609 + seed: yPi3JHKKbWaEEG5eZOlM6BHJll0Z3UTdBzz4bPQ7wjg= + zfec: + required: 3 + segmentSize: 131072 + total: 10 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:CHK:x55owxzhsezfoayaxe7jpwnove:vawdgtqpxyntgy5i2po2twgelrynkfcjgwm7publnlbdp7hpqmfa:3:10:1024 + format: chk + sample: + length: 1024 + seed: YQ== + zfec: + required: 3 + segmentSize: 131072 + total: 10 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:SSK:r62womj66zogjch4dhh7e5egsu:sgg7qnu3vqcr2bjds44rc2ogvm7oy42og76icyq7bztjvtnhhiba + format: sdmf + sample: + length: 1024 + seed: YQ== + zfec: + required: 3 + segmentSize: 131072 + total: 10 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:MDMF:6zfoxkfxatfndxvk3facte3a6m:armxhzyb7dw5gi4re7hlf3dt7pr3qnreowpffieqkfuhafz6htzq + format: mdmf + sample: + length: 1024 + seed: YQ== + zfec: + required: 3 + segmentSize: 131072 + total: 10 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:CHK:fe64krzyaeff3d4teunjbetkzy:27hrywwaffqiqcgfkmzwbot3iamotr3bey2l5kaladmdmxuaz5ka:3:10:4096 + format: chk + sample: + length: 4096 + seed: Yw== + zfec: + required: 3 + segmentSize: 131072 + total: 10 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:SSK:ycx4fggszh3rafrrn4cxk63mdm:tnwwkuhngbsw2edbpcf6foknukx5eclklr7bmvpkqtiavcr7sq6q + format: sdmf + sample: + length: 4096 + seed: Yw== + zfec: + required: 3 + segmentSize: 131072 + total: 10 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:MDMF:zxko7c2kaa55krevcnwroqh75a:sjxf56lafuvvuhxhz66c6hf2sufqbo6as43bnanr2qf5s5d3gr6q + format: mdmf + sample: + length: 4096 + seed: Yw== + zfec: + required: 3 + segmentSize: 131072 + total: 10 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:CHK:gvajllsonkuscfemygbnqhq2re:uwyilm5a7so4blhsaielnf34u2qbaqmudd73opjkgodgg3okeaga:3:10:131071 + format: chk + sample: + length: 131071 + seed: LCa0a2j/xo/5m0U8HTBBNBNCLXBkg7+g+YpeiGJm564= + zfec: + required: 3 + segmentSize: 131072 + total: 10 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:SSK:yquetovseokj3fuoyjnf65msze:5qkv72qjjpj2udgwd54wvm3zx7j6w5iqxnvmip4l237sttefjgia + format: sdmf + sample: + length: 131071 + seed: LCa0a2j/xo/5m0U8HTBBNBNCLXBkg7+g+YpeiGJm564= + zfec: + required: 3 + segmentSize: 131072 + total: 10 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:MDMF:mmaxw53ou3sctw2f2wbcrvtokm:gw6y5hl6pk2nnthguuso26olkxctjse7d455xldew53ggkblhniq + format: mdmf + sample: + length: 131071 + seed: LCa0a2j/xo/5m0U8HTBBNBNCLXBkg7+g+YpeiGJm564= + zfec: + required: 3 + segmentSize: 131072 + total: 10 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:CHK:zdmicwopo4p4h4wbfcbnwcrvyi:6qn75anpvs5gls27f4lybisis3udvjfjhatxiny7c72bcbtuztia:3:10:131073 + format: chk + sample: + length: 131073 + seed: /N4rLtula/QIYB+3If6bXDONEO5CnqBPrlURto+/j7k= + zfec: + required: 3 + segmentSize: 131072 + total: 10 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:SSK:233n2j6rvhem4zrie57ab54dmu:ncmydfq73cyq66wegrdo4a36pmzjpvlbrielye44nb2cpc7atxwq + format: sdmf + sample: + length: 131073 + seed: /N4rLtula/QIYB+3If6bXDONEO5CnqBPrlURto+/j7k= + zfec: + required: 3 + segmentSize: 131072 + total: 10 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:MDMF:l3lh5fxgpco7fq4to7j5b55nva:ut3x2b2hnl4nae6oabrvfomkzkb7b26rohze6xqb522safyetwuq + format: mdmf + sample: + length: 131073 + seed: /N4rLtula/QIYB+3If6bXDONEO5CnqBPrlURto+/j7k= + zfec: + required: 3 + segmentSize: 131072 + total: 10 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:CHK:bv6qthmetlhdnwc5tfjqamp3yq:ehz4ttd4g7ktkxvbovt562wfedc6jgnt5c6af7wxgp7jbwfwhoaa:3:10:2097151 + format: chk + sample: + length: 2097151 + seed: uqWglk0zIPvAxqkiFARTyFE+okq4/QV3A0gEqWckgJY= + zfec: + required: 3 + segmentSize: 131072 + total: 10 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:SSK:cjjj6zrxophjoshbkndzr6f4pu:pul7pnc7ann3z3bkob4gpb7vkj3brwhvprw7o76467jukwmgkw7q + format: sdmf + sample: + length: 2097151 + seed: uqWglk0zIPvAxqkiFARTyFE+okq4/QV3A0gEqWckgJY= + zfec: + required: 3 + segmentSize: 131072 + total: 10 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:MDMF:j7dbexypbpityjrq6gdh4sqrqy:vzczmef6su6javwr22wfxkfegwnymh2g3de4gplbtveobfjjrdfa + format: mdmf + sample: + length: 2097151 + seed: uqWglk0zIPvAxqkiFARTyFE+okq4/QV3A0gEqWckgJY= + zfec: + required: 3 + segmentSize: 131072 + total: 10 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:CHK:n7ogyjbo5jigvxgel5ll6q4vbe:3yjb3zq5hcdavv7ruefawal6euyvjx3lx7quslvasjellv63cxya:3:10:2097153 + format: chk + sample: + length: 2097153 + seed: BTBX/ampNfLU+ox7xipBGiaSbgC0kcB8Gy7BkJB4oKI= + zfec: + required: 3 + segmentSize: 131072 + total: 10 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:SSK:3yt7vo7ddtlnasybhdylqzgc6e:2gqcvr6xkrxmnk3xcxbukja2po5wzipn6mfty3527vxja2zkzrnq + format: sdmf + sample: + length: 2097153 + seed: BTBX/ampNfLU+ox7xipBGiaSbgC0kcB8Gy7BkJB4oKI= + zfec: + required: 3 + segmentSize: 131072 + total: 10 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:MDMF:gpkovok4tnkbrfffaywmtio7fu:jt4bbegmwqnbhixwdb7xuo6ae5izwq6y7bmttrpbpx4ox6n4tqgq + format: mdmf + sample: + length: 2097153 + seed: BTBX/ampNfLU+ox7xipBGiaSbgC0kcB8Gy7BkJB4oKI= + zfec: + required: 3 + segmentSize: 131072 + total: 10 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:CHK:d2hbvcmbex7fm3qu22yj4qnkh4:2xwqxbawwgn773hht6etox3oypvqqjv2orktnthfo2e7vibko7ha:3:10:8388607 + format: chk + sample: + length: 8388607 + seed: w6uP8Tcg6K2QR905Rms8iXTlksL6OD1KOWBxTK7wxPI= + zfec: + required: 3 + segmentSize: 131072 + total: 10 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:SSK:sczqxshv5qg7dijhrfln6unia4:y5c3qv45mz3namk4yiezidym4wsajseyboxwf7qqin3cmov75wuq + format: sdmf + sample: + length: 8388607 + seed: w6uP8Tcg6K2QR905Rms8iXTlksL6OD1KOWBxTK7wxPI= + zfec: + required: 3 + segmentSize: 131072 + total: 10 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:MDMF:lmoglq3yl6j3wz4ook34unismu:jfleiqdmoyfyvc63etssojyvjfsbptg7q3dvhi25bxlwe2lfnxxa + format: mdmf + sample: + length: 8388607 + seed: w6uP8Tcg6K2QR905Rms8iXTlksL6OD1KOWBxTK7wxPI= + zfec: + required: 3 + segmentSize: 131072 + total: 10 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:CHK:ccxkyfl2qtqyhihduarpxbdcci:e64be3i2t25selbpc5y2zj443gkdo65chs2o4tpqb7axud5lnxta:3:10:8388609 + format: chk + sample: + length: 8388609 + seed: yPi3JHKKbWaEEG5eZOlM6BHJll0Z3UTdBzz4bPQ7wjg= + zfec: + required: 3 + segmentSize: 131072 + total: 10 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:SSK:sdh5twhpxdz6hu6ikhnywsmv2q:iphfqmziwxm2vq7wzemqld6rxp2quaz7y2ecnkpt4d4lojdt6dna + format: sdmf + sample: + length: 8388609 + seed: yPi3JHKKbWaEEG5eZOlM6BHJll0Z3UTdBzz4bPQ7wjg= + zfec: + required: 3 + segmentSize: 131072 + total: 10 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:MDMF:j3t75ilecnew72pxtyluc3a7s4:f4vhqdtxzg2ybfs3j5gizewu5bjncgut7kszftd7hk4cm6o6q5ta + format: mdmf + sample: + length: 8388609 + seed: yPi3JHKKbWaEEG5eZOlM6BHJll0Z3UTdBzz4bPQ7wjg= + zfec: + required: 3 + segmentSize: 131072 + total: 10 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:CHK:2wvrzaobtfkoiqeqwryjzvxatq:7zzaycbdhepzaqtdrku2sbb572h64ggwstu47osa2gn4ol3eglmq:71:255:1024 + format: chk + sample: + length: 1024 + seed: YQ== + zfec: + required: 71 + segmentSize: 131072 + total: 255 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:SSK:hcekwnkw6qfllqmr3zy6g5j62u:2xzhdbgle6pvey5hjcanm5jtwkhtseslmy5jyamdxw7wr45okyiq + format: sdmf + sample: + length: 1024 + seed: YQ== + zfec: + required: 71 + segmentSize: 131072 + total: 255 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:MDMF:lcolwznisx6sore75biky3refa:4r4rnx5sbml3gg6z7bex6ytg6apwtmko6tujcqga4pr4cufuxmoq + format: mdmf + sample: + length: 1024 + seed: YQ== + zfec: + required: 71 + segmentSize: 131072 + total: 255 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:CHK:7wi5xn2uwerkjzqs5ndn4eimti:llrws4uig2fpi3rtla6lk7m4t6zudjhszdfjgqtihghfrvs2robq:71:255:4096 + format: chk + sample: + length: 4096 + seed: Yw== + zfec: + required: 71 + segmentSize: 131072 + total: 255 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:SSK:slphep77cvx6qw4d5qvliwa4b4:epchbi3vvkzgktlgizmfenwz7iu4cb2thyyrru2t4sm644fthkna + format: sdmf + sample: + length: 4096 + seed: Yw== + zfec: + required: 71 + segmentSize: 131072 + total: 255 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:MDMF:2t6yhzbskevam2brq7nirnptq4:dftxzmiizqrh737bny3nj3f2l3u36w4d7yyut43dynx2jeqtmvqq + format: mdmf + sample: + length: 4096 + seed: Yw== + zfec: + required: 71 + segmentSize: 131072 + total: 255 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:CHK:6nvs23bhrpiwiz5prqdtvztujy:wlg7g522rpdoitpm4qwhmctrjhnh4zfloiq6uq4tsvaoawg4slpq:71:255:131071 + format: chk + sample: + length: 131071 + seed: LCa0a2j/xo/5m0U8HTBBNBNCLXBkg7+g+YpeiGJm564= + zfec: + required: 71 + segmentSize: 131072 + total: 255 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:SSK:5od2qv3xadhzghtg76uogaqjnu:fcab6vjvpe25dorw5brgqpa64p3zsxemfer7gzkp56aprbrsrwwa + format: sdmf + sample: + length: 131071 + seed: LCa0a2j/xo/5m0U8HTBBNBNCLXBkg7+g+YpeiGJm564= + zfec: + required: 71 + segmentSize: 131072 + total: 255 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:MDMF:6yga2mq25l6hxha4z3bt5ob2ny:h3cxanvsutketgqw6qkmeugugfxn5gzn4jzz3dykoi727ok7757q + format: mdmf + sample: + length: 131071 + seed: LCa0a2j/xo/5m0U8HTBBNBNCLXBkg7+g+YpeiGJm564= + zfec: + required: 71 + segmentSize: 131072 + total: 255 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:CHK:62dtbeowncjj62jnwbggaufvbu:6r2sapg2cm6dvmylodyxabrj63a736uouzsqyacnimgo4svnktva:71:255:131073 + format: chk + sample: + length: 131073 + seed: /N4rLtula/QIYB+3If6bXDONEO5CnqBPrlURto+/j7k= + zfec: + required: 71 + segmentSize: 131072 + total: 255 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:SSK:grht3kifmiwg2fqtokxrkcklz4:k4gz5cy7kqa7niyj5ighz6q7pkztbfimgdux4omuu2ljjdxglwia + format: sdmf + sample: + length: 131073 + seed: /N4rLtula/QIYB+3If6bXDONEO5CnqBPrlURto+/j7k= + zfec: + required: 71 + segmentSize: 131072 + total: 255 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:MDMF:2kzk7tyx23b3too7ejuybpudxm:5l2cwxkxg4tqlzlmctbi4ogbdyrbj762mvsaz4cq7lkwbhnjtlza + format: mdmf + sample: + length: 131073 + seed: /N4rLtula/QIYB+3If6bXDONEO5CnqBPrlURto+/j7k= + zfec: + required: 71 + segmentSize: 131072 + total: 255 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:CHK:hk3a5ync7y3dnwowqjqoa34eae:y4f5b2xqa3lslh2vti5fdbho2syqhbj2p6f3enlhxsjbfpt7zuta:71:255:2097151 + format: chk + sample: + length: 2097151 + seed: uqWglk0zIPvAxqkiFARTyFE+okq4/QV3A0gEqWckgJY= + zfec: + required: 71 + segmentSize: 131072 + total: 255 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:SSK:nn4ojnd7snzrb72qetwohd4cde:lqp6djtltpnppkfy4uhjx76h6zsglverablp3cng4xllzh7wghcq + format: sdmf + sample: + length: 2097151 + seed: uqWglk0zIPvAxqkiFARTyFE+okq4/QV3A0gEqWckgJY= + zfec: + required: 71 + segmentSize: 131072 + total: 255 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:MDMF:ogpjvm26q6pe5akks3idggipbm:q5sw3t7h6statgllhfsthowhc6op5jdpdfe4cx2pvwhscoewjada + format: mdmf + sample: + length: 2097151 + seed: uqWglk0zIPvAxqkiFARTyFE+okq4/QV3A0gEqWckgJY= + zfec: + required: 71 + segmentSize: 131072 + total: 255 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:CHK:qa4hwhwtd3cpvut74biixlv6ye:uuj4ujg3mcf3lqpuvvwf5pszcvviyb4cm2jjicu7ugiypojjie4q:71:255:2097153 + format: chk + sample: + length: 2097153 + seed: BTBX/ampNfLU+ox7xipBGiaSbgC0kcB8Gy7BkJB4oKI= + zfec: + required: 71 + segmentSize: 131072 + total: 255 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:SSK:icuadu2dbhfheefmqrm2mayg3q:wf5kszmayp6vd2t6hnpb4npqofomznorzlhtzvsbg7xtpxqahona + format: sdmf + sample: + length: 2097153 + seed: BTBX/ampNfLU+ox7xipBGiaSbgC0kcB8Gy7BkJB4oKI= + zfec: + required: 71 + segmentSize: 131072 + total: 255 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:MDMF:z72yysftl3jicwhnhcsm62jdje:w5fp2ag7jvyazv3sm23qlooopmxvgfzotj7wwomd2z2svy6hy76a + format: mdmf + sample: + length: 2097153 + seed: BTBX/ampNfLU+ox7xipBGiaSbgC0kcB8Gy7BkJB4oKI= + zfec: + required: 71 + segmentSize: 131072 + total: 255 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:CHK:r7sv6u3ihzys754twjltuzzpza:tpyoxinubkl4kvg6hipls6ezun7t3yyqj6ix3vp6tvnzqirzzl4a:71:255:8388607 + format: chk + sample: + length: 8388607 + seed: w6uP8Tcg6K2QR905Rms8iXTlksL6OD1KOWBxTK7wxPI= + zfec: + required: 71 + segmentSize: 131072 + total: 255 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:SSK:6gxdkh5q5bgygex4seqdde4sai:7idrfdnijmiagjt2reroab32tiw6izdywnnpjxu3m7be5226nygq + format: sdmf + sample: + length: 8388607 + seed: w6uP8Tcg6K2QR905Rms8iXTlksL6OD1KOWBxTK7wxPI= + zfec: + required: 71 + segmentSize: 131072 + total: 255 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:MDMF:ecfyk2zk7ojl37qyme67pylsn4:7w4ga3fulb5dyolujq56f5jsbifwxc5lk32z5c6siq3rs2dlbnsa + format: mdmf + sample: + length: 8388607 + seed: w6uP8Tcg6K2QR905Rms8iXTlksL6OD1KOWBxTK7wxPI= + zfec: + required: 71 + segmentSize: 131072 + total: 255 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:CHK:3esfsjn7csgnyqmq5afbgtinay:xsrhdomrbrtzftg6u4hgipm5tkaomumbdlxi5hpvpvawe3bjikua:71:255:8388609 + format: chk + sample: + length: 8388609 + seed: yPi3JHKKbWaEEG5eZOlM6BHJll0Z3UTdBzz4bPQ7wjg= + zfec: + required: 71 + segmentSize: 131072 + total: 255 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:SSK:kfjuhoyehy4t7ebwgewj4redpm:awgxuay5brzn7aupt7r2jjwk5mhoedmgrd2qaszyhdx4fpaet35q + format: sdmf + sample: + length: 8388609 + seed: yPi3JHKKbWaEEG5eZOlM6BHJll0Z3UTdBzz4bPQ7wjg= + zfec: + required: 71 + segmentSize: 131072 + total: 255 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:MDMF:au3qa6mx56pgi7ouxtscajmak4:qg3s3ewcl754onl5xyzotbbgfia7vojar3m25ac7nv4wgolvibya + format: mdmf + sample: + length: 8388609 + seed: yPi3JHKKbWaEEG5eZOlM6BHJll0Z3UTdBzz4bPQ7wjg= + zfec: + required: 71 + segmentSize: 131072 + total: 255 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:CHK:wxmzfzvgaqm3xfzgaqrlndgolm:6t3tqcphsutigxnnzyp2xx3afklcpeytnh543mpqptquvqqahwea:71:255:1024 + format: chk + sample: + length: 1024 + seed: YQ== + zfec: + required: 71 + segmentSize: 131072 + total: 255 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:SSK:hkyaycbxeyhwifvygulnlqx3r4:zwck3zce5brcuzp3zvpc6evfxifppj5nl4jbw7tb2anbzis3jqha + format: sdmf + sample: + length: 1024 + seed: YQ== + zfec: + required: 71 + segmentSize: 131072 + total: 255 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:MDMF:ssbx7xhxjr24tygmvigbkw2tgq:du7u4ayfueljiftdfo5snzlz3aqcf6xwt5nrw4zpopqlcfgteuda + format: mdmf + sample: + length: 1024 + seed: YQ== + zfec: + required: 71 + segmentSize: 131072 + total: 255 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:CHK:r3q3q4f4zbsipxauq6knb5dfoq:lkae5mflammzxkoc7gllt6qqyg5gqdxeejmxtgcscblqh5tv5obq:71:255:4096 + format: chk + sample: + length: 4096 + seed: Yw== + zfec: + required: 71 + segmentSize: 131072 + total: 255 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:SSK:2sip2pgg5xmeaqujp37qwqnhlm:jppsyllwrbfalet4lkhi3r5r74355bzquajkp2wzgu4us5ko6h4q + format: sdmf + sample: + length: 4096 + seed: Yw== + zfec: + required: 71 + segmentSize: 131072 + total: 255 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:MDMF:calpjzlexgja7qk5d72wyeybqm:3upvwrkbbi7p5453sbs5wuie7hwxv6moczcb55pqkmfowlwehbhq + format: mdmf + sample: + length: 4096 + seed: Yw== + zfec: + required: 71 + segmentSize: 131072 + total: 255 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:CHK:rdrzetasw4w7tuweqpev65d5vq:gzzb2v5fzrduk6kx5xx27mvo6dgnd65ym5lm7re53in7xuudhsxa:71:255:131071 + format: chk + sample: + length: 131071 + seed: LCa0a2j/xo/5m0U8HTBBNBNCLXBkg7+g+YpeiGJm564= + zfec: + required: 71 + segmentSize: 131072 + total: 255 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:SSK:m52mezcwt462pv7crh57xch5ka:eskrgchqwjldpyvgjit2cv56xd55gra3ejiyoms4wegfhw64bmea + format: sdmf + sample: + length: 131071 + seed: LCa0a2j/xo/5m0U8HTBBNBNCLXBkg7+g+YpeiGJm564= + zfec: + required: 71 + segmentSize: 131072 + total: 255 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:MDMF:jlknqnzgwcb426mudiclobjejy:tmj2no6czfzaqa4x6qolw7ysegnbq4p57jc665mrvrbrwxigtkoa + format: mdmf + sample: + length: 131071 + seed: LCa0a2j/xo/5m0U8HTBBNBNCLXBkg7+g+YpeiGJm564= + zfec: + required: 71 + segmentSize: 131072 + total: 255 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:CHK:e2bjefibvz4tgu6jgf66gw74bq:mugtu5bx7h5gcivevmh2gmwoc2kkhmobrzshkuj2dgrtm3siysfq:71:255:131073 + format: chk + sample: + length: 131073 + seed: /N4rLtula/QIYB+3If6bXDONEO5CnqBPrlURto+/j7k= + zfec: + required: 71 + segmentSize: 131072 + total: 255 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:SSK:r7zizt5ikfo4ubjxdmtjwap2ii:gmbwvgjnjk4qilg36okgw22hlhfico73e64wknay4aharfonevzq + format: sdmf + sample: + length: 131073 + seed: /N4rLtula/QIYB+3If6bXDONEO5CnqBPrlURto+/j7k= + zfec: + required: 71 + segmentSize: 131072 + total: 255 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:MDMF:3gbqsb33r6zm2lmh246hwycwru:6yuzbmsxuqaqqi3pg4u3q5eiihpfwrteztfub3xyoa7uzcixubxa + format: mdmf + sample: + length: 131073 + seed: /N4rLtula/QIYB+3If6bXDONEO5CnqBPrlURto+/j7k= + zfec: + required: 71 + segmentSize: 131072 + total: 255 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:CHK:gy7bci6yvllxzvhh45khcwp3u4:qsfjfmcl64zey4k4o5cfnh2i3mzboahf7bhtggceszrulsv537vq:71:255:2097151 + format: chk + sample: + length: 2097151 + seed: uqWglk0zIPvAxqkiFARTyFE+okq4/QV3A0gEqWckgJY= + zfec: + required: 71 + segmentSize: 131072 + total: 255 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:SSK:3cvm2bawfchcqvrbu2f4h47zra:rhqo4ecqmidnwhp3b5v62kipf55wqt6x3nwtmwnvcjt4ytwnop4a + format: sdmf + sample: + length: 2097151 + seed: uqWglk0zIPvAxqkiFARTyFE+okq4/QV3A0gEqWckgJY= + zfec: + required: 71 + segmentSize: 131072 + total: 255 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:MDMF:kkigaypv6feshol6sfug2xnsmy:dd2ll6ga2wgqhlcdl4nvx4m42rs6a3cqcnxglpy64skvlyq5yira + format: mdmf + sample: + length: 2097151 + seed: uqWglk0zIPvAxqkiFARTyFE+okq4/QV3A0gEqWckgJY= + zfec: + required: 71 + segmentSize: 131072 + total: 255 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:CHK:pcta7uk5cpioxzv5nxxqsogw3q:gnu7fx56k3j72bbevirn4kdu27yfpvttzl3qk2zypwqt7tw4rijq:71:255:2097153 + format: chk + sample: + length: 2097153 + seed: BTBX/ampNfLU+ox7xipBGiaSbgC0kcB8Gy7BkJB4oKI= + zfec: + required: 71 + segmentSize: 131072 + total: 255 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:SSK:hnc74in4qv6bpgmydlzsenhaf4:ysrykdwa6jn2gngdqh56x3jdhor2nsamyjd6b4aqtutzbtdjkdba + format: sdmf + sample: + length: 2097153 + seed: BTBX/ampNfLU+ox7xipBGiaSbgC0kcB8Gy7BkJB4oKI= + zfec: + required: 71 + segmentSize: 131072 + total: 255 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:MDMF:hilmjv7cb2w47njz3znintqueq:hl2jtn5uaujxqwfvexvwfvkfo5iliexcnwzyjt6etyjsc6advsja + format: mdmf + sample: + length: 2097153 + seed: BTBX/ampNfLU+ox7xipBGiaSbgC0kcB8Gy7BkJB4oKI= + zfec: + required: 71 + segmentSize: 131072 + total: 255 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:CHK:znxyqo2qglpkhjafnuesu4foge:trau477bj5u5wjfpy4oq5e3aziajgbpyfjdv4svyf27m7atpa34a:71:255:8388607 + format: chk + sample: + length: 8388607 + seed: w6uP8Tcg6K2QR905Rms8iXTlksL6OD1KOWBxTK7wxPI= + zfec: + required: 71 + segmentSize: 131072 + total: 255 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:SSK:xxc3ireoguv7dkgubcjicnxt2q:lp3rg6vawrypxldd4x7geajtzw6ns7yhe7zno4obuottc6bym6da + format: sdmf + sample: + length: 8388607 + seed: w6uP8Tcg6K2QR905Rms8iXTlksL6OD1KOWBxTK7wxPI= + zfec: + required: 71 + segmentSize: 131072 + total: 255 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:MDMF:ii23y7sh2zcjk5qy75ytened4m:k7of2k6pmqjthqaql6hw5t22s3v67n5jpvqb5nvq53rxbphuanqa + format: mdmf + sample: + length: 8388607 + seed: w6uP8Tcg6K2QR905Rms8iXTlksL6OD1KOWBxTK7wxPI= + zfec: + required: 71 + segmentSize: 131072 + total: 255 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:CHK:mlzs2hbztak5fxjkrkuuvnpdpe:3myvisewm5uklimp2xucrwep75sm2rizfi2sq5drhlqjszkpdyeq:71:255:8388609 + format: chk + sample: + length: 8388609 + seed: yPi3JHKKbWaEEG5eZOlM6BHJll0Z3UTdBzz4bPQ7wjg= + zfec: + required: 71 + segmentSize: 131072 + total: 255 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:SSK:z47oxnf52xbyyzz2jsx6injuxi:twr773m3memiqykwwhfapujfeyki3i3lnrwrxif5spatnb5lzx2q + format: sdmf + sample: + length: 8388609 + seed: yPi3JHKKbWaEEG5eZOlM6BHJll0Z3UTdBzz4bPQ7wjg= + zfec: + required: 71 + segmentSize: 131072 + total: 255 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:MDMF:43ot6bjdwvfzibci4i3xfdow4i:dy526wxcdosiqocz65xxfqyymwtrtfdvu4nxll4mthp46xojosma + format: mdmf + sample: + length: 8388609 + seed: yPi3JHKKbWaEEG5eZOlM6BHJll0Z3UTdBzz4bPQ7wjg= + zfec: + required: 71 + segmentSize: 131072 + total: 255 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:CHK:mftnvh5l3brshwoupvu6kz35cy:sncfokby2tjykbr3zgi3i6lhgerg4kz7fsrslrlxdy7upwkzq6lq:101:256:1024 + format: chk + sample: + length: 1024 + seed: YQ== + zfec: + required: 101 + segmentSize: 131072 + total: max +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:SSK:4rybfby6ul2szc3aap7mldlbke:23cpk6vnlgv6n4tadfgkooh3rqsut2xuchrkqjw3fcnleihoqd4q + format: sdmf + sample: + length: 1024 + seed: YQ== + zfec: + required: 101 + segmentSize: 131072 + total: max +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:MDMF:dbc3cfcgtkv33y4xrcfzqget2e:3amzzlftmmugjpcb4octrtzi4tzbdcrnenv2kpbufvkcboin4haq + format: mdmf + sample: + length: 1024 + seed: YQ== + zfec: + required: 101 + segmentSize: 131072 + total: max +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:CHK:sshu77h6opnto3jnngot3lwzl4:4cjhp6u3i2uzdzrjitje3inlhs6gnlgwc6wgi6vtedc2grgvcpza:101:256:4096 + format: chk + sample: + length: 4096 + seed: Yw== + zfec: + required: 101 + segmentSize: 131072 + total: max +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:SSK:4kafx6n7k3etff25jfb2mbq62a:ll2njj473qvuv5oif4ro26jgmyljjvbfanuke2tljhmel4rbceja + format: sdmf + sample: + length: 4096 + seed: Yw== + zfec: + required: 101 + segmentSize: 131072 + total: max +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:MDMF:2nbq6g7kipjvtjy5i2jmpjajma:zcqhmcgjll4zc6da73kmlkynxhgm7ppwbvby3fmoey75gfwys3ua + format: mdmf + sample: + length: 4096 + seed: Yw== + zfec: + required: 101 + segmentSize: 131072 + total: max +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:CHK:kl2jayrstlf5q3gax7o65ybivy:mnn5doti4wllssdg35ymojuiv4hd2ir7i6asurgms3ea67watm7a:101:256:131071 + format: chk + sample: + length: 131071 + seed: LCa0a2j/xo/5m0U8HTBBNBNCLXBkg7+g+YpeiGJm564= + zfec: + required: 101 + segmentSize: 131072 + total: max +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:SSK:jlmmbswo4xbiib3bc2hdlajacy:uqvabifbvq5di2r3dbvc7bqmnbv5aqldxxrmduuhjun35dotmw2a + format: sdmf + sample: + length: 131071 + seed: LCa0a2j/xo/5m0U8HTBBNBNCLXBkg7+g+YpeiGJm564= + zfec: + required: 101 + segmentSize: 131072 + total: max +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:MDMF:cuogxv55hdieeepwl7i6ce6taa:u755dpuna763w5uaw5lmqixmsxov66c4fvb46tduiszko3m7iq3a + format: mdmf + sample: + length: 131071 + seed: LCa0a2j/xo/5m0U8HTBBNBNCLXBkg7+g+YpeiGJm564= + zfec: + required: 101 + segmentSize: 131072 + total: max +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:CHK:a6wuekh5mynr4c74ma3b2gswfa:b6lovwnbv3523n5kaxga3ruxa2dvxef6h4a2jnl6lasn4fwvvdba:101:256:131073 + format: chk + sample: + length: 131073 + seed: /N4rLtula/QIYB+3If6bXDONEO5CnqBPrlURto+/j7k= + zfec: + required: 101 + segmentSize: 131072 + total: max +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:SSK:u2bbj2wg23qzervxlfkua5dj2y:p6fo6wlvh7gcdq63ufp47rdiixxtx2yaeg3llg5byxjhzofkacga + format: sdmf + sample: + length: 131073 + seed: /N4rLtula/QIYB+3If6bXDONEO5CnqBPrlURto+/j7k= + zfec: + required: 101 + segmentSize: 131072 + total: max +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:MDMF:fdai3c5ahhbypwczn2sigdpdlm:kw3fmle5qbml4yz77wwrzz3tbvo65r3xdw347qu2ha3gaa2lidgq + format: mdmf + sample: + length: 131073 + seed: /N4rLtula/QIYB+3If6bXDONEO5CnqBPrlURto+/j7k= + zfec: + required: 101 + segmentSize: 131072 + total: max +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:CHK:pmuo6vpcodol76sexq7s2ojej4:irp6rxnkgwbn3h4g5fzgjpqgsvhvycxobr3xrk3zr44r64wimgqa:101:256:2097151 + format: chk + sample: + length: 2097151 + seed: uqWglk0zIPvAxqkiFARTyFE+okq4/QV3A0gEqWckgJY= + zfec: + required: 101 + segmentSize: 131072 + total: max +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:SSK:otczmevjiescnpe6rmdi6pugxy:flwoyy4kwbbex65vopjvt7au6ywjo4rgzbzt5v65rulmn2aufzfq + format: sdmf + sample: + length: 2097151 + seed: uqWglk0zIPvAxqkiFARTyFE+okq4/QV3A0gEqWckgJY= + zfec: + required: 101 + segmentSize: 131072 + total: max +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:MDMF:br3wgiraxyrrzr6yxmcqwnwl6i:bd67wszbnw43kl76jjbfp4s5q6d23eswrag2fu3ztplwvfmtbara + format: mdmf + sample: + length: 2097151 + seed: uqWglk0zIPvAxqkiFARTyFE+okq4/QV3A0gEqWckgJY= + zfec: + required: 101 + segmentSize: 131072 + total: max +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:CHK:fh4boug3wkciqhkgpusjsjyzbe:sy5lwooxchqu5mrzwr3mgc7qtwfjgmpz2srdmwpaip4iuj3tn4vq:101:256:2097153 + format: chk + sample: + length: 2097153 + seed: BTBX/ampNfLU+ox7xipBGiaSbgC0kcB8Gy7BkJB4oKI= + zfec: + required: 101 + segmentSize: 131072 + total: max +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:SSK:m4vwawgyausecci7ndiz44thde:e27y5aqvmnb532bv55gwrz3r74m2ff34ls2epbrr27weqg6x3una + format: sdmf + sample: + length: 2097153 + seed: BTBX/ampNfLU+ox7xipBGiaSbgC0kcB8Gy7BkJB4oKI= + zfec: + required: 101 + segmentSize: 131072 + total: max +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:MDMF:yrdgo3x6zc7qtu6qhgschwujqy:lpkg3rluntszbvn64lu247pxjg6urbebktxjm2yfvkw22nuzwcuq + format: mdmf + sample: + length: 2097153 + seed: BTBX/ampNfLU+ox7xipBGiaSbgC0kcB8Gy7BkJB4oKI= + zfec: + required: 101 + segmentSize: 131072 + total: max +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:CHK:asmjgajvwpwvylxdcf5rm2gyd4:imwtdi4eqq3wm4bvk6npho3ta2fewceqzl4f7wyyabg5i4s6h3mq:101:256:8388607 + format: chk + sample: + length: 8388607 + seed: w6uP8Tcg6K2QR905Rms8iXTlksL6OD1KOWBxTK7wxPI= + zfec: + required: 101 + segmentSize: 131072 + total: max +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:SSK:utp6f43bkziptssu5waoexpumq:b7ok3lg5qe33gk5s3n6yv5gvz6xh65xpflyhfkyzpwpvec6pqroq + format: sdmf + sample: + length: 8388607 + seed: w6uP8Tcg6K2QR905Rms8iXTlksL6OD1KOWBxTK7wxPI= + zfec: + required: 101 + segmentSize: 131072 + total: max +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:MDMF:qelscz6cozb7oei42jed7jauuy:e67rqvxcxd2gvy6wgc7tjzpeieltl4252wutucvfvzinhybcp2lq + format: mdmf + sample: + length: 8388607 + seed: w6uP8Tcg6K2QR905Rms8iXTlksL6OD1KOWBxTK7wxPI= + zfec: + required: 101 + segmentSize: 131072 + total: max +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:CHK:yrznm2uuybfbda6jlj7tqs3yum:zy45lpc7ilfzgltxxqvy55wcokiay2fbbbv2wdvu6ubjluh4gf4a:101:256:8388609 + format: chk + sample: + length: 8388609 + seed: yPi3JHKKbWaEEG5eZOlM6BHJll0Z3UTdBzz4bPQ7wjg= + zfec: + required: 101 + segmentSize: 131072 + total: max +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:SSK:gvwazbxny7br5jaebq5lm6vkwu:frn54skmpr2v5i62v2cv4o3etglknbubanzgkcz7fjly76zs5bmq + format: sdmf + sample: + length: 8388609 + seed: yPi3JHKKbWaEEG5eZOlM6BHJll0Z3UTdBzz4bPQ7wjg= + zfec: + required: 101 + segmentSize: 131072 + total: max +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:MDMF:ltix4y33d3up5xceqoeru3fh7a:3oijenm7ex3u7bccxtc37j2ur4phqwp2cxpwonmlr36pt6v4siaa + format: mdmf + sample: + length: 8388609 + seed: yPi3JHKKbWaEEG5eZOlM6BHJll0Z3UTdBzz4bPQ7wjg= + zfec: + required: 101 + segmentSize: 131072 + total: max +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:CHK:ihyeqq5peg2pdyw45ct5mmj4ay:2rzhndv6o4knlvo4po6rvbnknlcitasseea3mirqjor5c7quh5ra:101:256:1024 + format: chk + sample: + length: 1024 + seed: YQ== + zfec: + required: 101 + segmentSize: 131072 + total: max +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:SSK:d46ph6imfcmvrnxbkz27rvphaa:fu7b3wvcrl6lq2gwhjr2lydcayqnojdb2a7nsble7wyah4u2u7ra + format: sdmf + sample: + length: 1024 + seed: YQ== + zfec: + required: 101 + segmentSize: 131072 + total: max +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:MDMF:dxm44lpq67irpvkxtq2kenkuna:4unsmxd4gauloblh66viurs7xcjoegkgxbgbnvvsxbvjejfkgfja + format: mdmf + sample: + length: 1024 + seed: YQ== + zfec: + required: 101 + segmentSize: 131072 + total: max +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:CHK:mvk5f2mghudk7xnknqddhvnl4m:c74myeugtix7e7l6uwybtrdtfselha2qrcc7hhbymqr6brtg4cea:101:256:4096 + format: chk + sample: + length: 4096 + seed: Yw== + zfec: + required: 101 + segmentSize: 131072 + total: max +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:SSK:kxgfid6gisa6fviwjvtvtitriq:y7p4sbiltsl2zz4iiu4l43e3tcyam7uaqdkyux62axocqrb2jjfa + format: sdmf + sample: + length: 4096 + seed: Yw== + zfec: + required: 101 + segmentSize: 131072 + total: max +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:MDMF:nwpcaxxhzcbflhetu3x4bcm7hi:4fwjthb4ywmuhe4v7d6or64hvywtpuhhkemz6n46jnkwjbotqqxq + format: mdmf + sample: + length: 4096 + seed: Yw== + zfec: + required: 101 + segmentSize: 131072 + total: max +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:CHK:rvytwcc4sbmmkrdwh6b23ckv5u:klk5nqpbv37pr2qavcebcugfnikarjdefjlseo3ru5uv7guai22q:101:256:131071 + format: chk + sample: + length: 131071 + seed: LCa0a2j/xo/5m0U8HTBBNBNCLXBkg7+g+YpeiGJm564= + zfec: + required: 101 + segmentSize: 131072 + total: max +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:SSK:hxtguhkn5d7vho5efeezt6txku:rw7t2vg4msj2h5qkkhw7vn6iotfv2xrnqu6sl4en4b34xogpteya + format: sdmf + sample: + length: 131071 + seed: LCa0a2j/xo/5m0U8HTBBNBNCLXBkg7+g+YpeiGJm564= + zfec: + required: 101 + segmentSize: 131072 + total: max +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:MDMF:wqgkje6573uydranrmfzhv5jye:o2fdspbx2eum66zdxaibssqzqor6fcy4gc2snvj4mthu5vah252q + format: mdmf + sample: + length: 131071 + seed: LCa0a2j/xo/5m0U8HTBBNBNCLXBkg7+g+YpeiGJm564= + zfec: + required: 101 + segmentSize: 131072 + total: max +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:CHK:ocqx55dopakqnzyui6q6euz4gm:wklziyoctwrz5zjfg5wfejz4bnaagtg57fgjlspl4ba6ij6n2obq:101:256:131073 + format: chk + sample: + length: 131073 + seed: /N4rLtula/QIYB+3If6bXDONEO5CnqBPrlURto+/j7k= + zfec: + required: 101 + segmentSize: 131072 + total: max +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:SSK:ina6rycsefbdo6uu7vpa5hmmkq:zlfppgohbsmbmq2nkigkt5oqiodwbrokfncngghmy4brz7hhwvua + format: sdmf + sample: + length: 131073 + seed: /N4rLtula/QIYB+3If6bXDONEO5CnqBPrlURto+/j7k= + zfec: + required: 101 + segmentSize: 131072 + total: max +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:MDMF:ursxqckxmvnkiby7euppiw4ve4:muw3hismhi5f6sunwgighiwiumxuhmgfba3etfyfu3ss5ztqnmeq + format: mdmf + sample: + length: 131073 + seed: /N4rLtula/QIYB+3If6bXDONEO5CnqBPrlURto+/j7k= + zfec: + required: 101 + segmentSize: 131072 + total: max +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:CHK:lf6qub3bkdhfozpdis4bmorj7q:zte3u3ohbuom3iqjjx4odrjylcpw5myw4n4kwf4zcaif36vngf6q:101:256:2097151 + format: chk + sample: + length: 2097151 + seed: uqWglk0zIPvAxqkiFARTyFE+okq4/QV3A0gEqWckgJY= + zfec: + required: 101 + segmentSize: 131072 + total: max +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:SSK:cnjidn5qfurjhp7xruq3qmhmwe:c5zsc3iw5l4msqhwig43cihejtbs7gmsw5yxzbbpux7ivf2by7nq + format: sdmf + sample: + length: 2097151 + seed: uqWglk0zIPvAxqkiFARTyFE+okq4/QV3A0gEqWckgJY= + zfec: + required: 101 + segmentSize: 131072 + total: max +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:MDMF:srixdotz4o72bxooz3w4hzn5pq:vgj66wd3f7dktqiexut3643w2aujb6rgkgmmgr4wvsfdm2x6ma5a + format: mdmf + sample: + length: 2097151 + seed: uqWglk0zIPvAxqkiFARTyFE+okq4/QV3A0gEqWckgJY= + zfec: + required: 101 + segmentSize: 131072 + total: max +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:CHK:a2qoww5xx6so5uftwg4msdtrpy:h7hwsjco5i3jkd7icytrtsuzcjvjzlnkr2wwau2tjlj5wehjrokq:101:256:2097153 + format: chk + sample: + length: 2097153 + seed: BTBX/ampNfLU+ox7xipBGiaSbgC0kcB8Gy7BkJB4oKI= + zfec: + required: 101 + segmentSize: 131072 + total: max +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:SSK:yivcybgondauziz7hbc5auarrq:px2j3rmujcek36kzbxreyqjsxeosqu6zkosdfyokapajycr4a7fa + format: sdmf + sample: + length: 2097153 + seed: BTBX/ampNfLU+ox7xipBGiaSbgC0kcB8Gy7BkJB4oKI= + zfec: + required: 101 + segmentSize: 131072 + total: max +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:MDMF:psnlh7cnhyyjvthsftpwsc4mg4:ykaikqm76z2hu5jjf2t4awbc6kow7667sfnbw3iapi6gyuy3rz6a + format: mdmf + sample: + length: 2097153 + seed: BTBX/ampNfLU+ox7xipBGiaSbgC0kcB8Gy7BkJB4oKI= + zfec: + required: 101 + segmentSize: 131072 + total: max +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:CHK:tg5w4m6hi5ezeapuix3lrrbbtq:7kerk5plqpsd2upnole3ifxoegxdux7prazbvzeu2kyilc6idk7a:101:256:8388607 + format: chk + sample: + length: 8388607 + seed: w6uP8Tcg6K2QR905Rms8iXTlksL6OD1KOWBxTK7wxPI= + zfec: + required: 101 + segmentSize: 131072 + total: max +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:SSK:znaz4yzpisflh6eb6vgimp2hqq:4uza6tbj44ayk3227r4rmo55hnryxlofxzra7skrlxutqp3qg3ra + format: sdmf + sample: + length: 8388607 + seed: w6uP8Tcg6K2QR905Rms8iXTlksL6OD1KOWBxTK7wxPI= + zfec: + required: 101 + segmentSize: 131072 + total: max +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:MDMF:3ukzh2xpaf5cbag3h757kllgke:xrtzgpxtdc4f6mljutdtycduavcf7mq3csotbmkxuwgghw5i32tq + format: mdmf + sample: + length: 8388607 + seed: w6uP8Tcg6K2QR905Rms8iXTlksL6OD1KOWBxTK7wxPI= + zfec: + required: 101 + segmentSize: 131072 + total: max +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:CHK:ioridipksj6o7ph3bcf37vslmi:pzhkwm4ry67ohj6dcs7zwnsjrvfqzam6tuitrns5vuo3uoomau6q:101:256:8388609 + format: chk + sample: + length: 8388609 + seed: yPi3JHKKbWaEEG5eZOlM6BHJll0Z3UTdBzz4bPQ7wjg= + zfec: + required: 101 + segmentSize: 131072 + total: max +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:SSK:ztsdfp76v62nww6kplddq6eiy4:2hz7dxqp2wfngpra7yidxsfnocmx4iqmffym264cqofhekrcocuq + format: sdmf + sample: + length: 8388609 + seed: yPi3JHKKbWaEEG5eZOlM6BHJll0Z3UTdBzz4bPQ7wjg= + zfec: + required: 101 + segmentSize: 131072 + total: max +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:MDMF:l4b5t44iqpfcmqk5vap6dwkery:pnq25jpqlmesas4nv55hstin2pmy3ybl7yinu2tk2ruwkhx575ia + format: mdmf + sample: + length: 8388609 + seed: yPi3JHKKbWaEEG5eZOlM6BHJll0Z3UTdBzz4bPQ7wjg= + zfec: + required: 101 + segmentSize: 131072 + total: max +version: '2023-01-03' From fb70ba186779ec04997f69756eb368d0047aeeea Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 3 Jan 2023 19:22:38 -0500 Subject: [PATCH 1327/2309] Generate and consumer the new structure properly --- integration/test_vectors.py | 219 ++-- integration/test_vectors.yaml | 1920 --------------------------------- integration/vectors.py | 132 ++- 3 files changed, 203 insertions(+), 2068 deletions(-) diff --git a/integration/test_vectors.py b/integration/test_vectors.py index 3d8b8ca85..4f5ef6e8a 100644 --- a/integration/test_vectors.py +++ b/integration/test_vectors.py @@ -4,28 +4,30 @@ Verify certain results against test vectors with well-known results. from __future__ import annotations -from time import sleep from typing import AsyncGenerator, Iterator from hashlib import sha256 -from itertools import product +from itertools import starmap, product from yaml import safe_dump -from attrs import frozen - from pytest import mark from pytest_twisted import ensureDeferred from . import vectors -from .util import reconfigure, upload, asyncfoldr, insert, TahoeProcess +from .util import reconfigure, upload, TahoeProcess def digest(bs: bytes) -> bytes: + """ + Digest bytes to bytes. + """ return sha256(bs).digest() def hexdigest(bs: bytes) -> str: + """ + Digest bytes to text. + """ return sha256(bs).hexdigest() - # Just a couple convergence secrets. The only thing we do with this value is # feed it into a tagged hash. It certainly makes a difference to the output # but the hash should destroy any structure in the input so it doesn't seem @@ -35,7 +37,6 @@ CONVERGENCE_SECRETS = [ digest(b"Hello world")[:16], ] - # Exercise at least a handful of different sizes, trying to cover: # # 1. Some cases smaller than one "segment" (128k). @@ -51,87 +52,66 @@ CONVERGENCE_SECRETS = [ SEGMENT_SIZE = 128 * 1024 OBJECT_DESCRIPTIONS = [ - (b"a", 1024), - (b"c", 4096), - (digest(b"foo"), SEGMENT_SIZE - 1), - (digest(b"bar"), SEGMENT_SIZE + 1), - (digest(b"baz"), SEGMENT_SIZE * 16 - 1), - (digest(b"quux"), SEGMENT_SIZE * 16 + 1), - (digest(b"foobar"), SEGMENT_SIZE * 64 - 1), - (digest(b"barbaz"), SEGMENT_SIZE * 64 + 1), + vectors.Sample(b"a", 1024), + vectors.Sample(b"c", 4096), + vectors.Sample(digest(b"foo"), SEGMENT_SIZE - 1), + vectors.Sample(digest(b"bar"), SEGMENT_SIZE + 1), + vectors.Sample(digest(b"baz"), SEGMENT_SIZE * 16 - 1), + vectors.Sample(digest(b"quux"), SEGMENT_SIZE * 16 + 1), + vectors.Sample(digest(b"foobar"), SEGMENT_SIZE * 64 - 1), + vectors.Sample(digest(b"barbaz"), SEGMENT_SIZE * 64 + 1), ] -# CHK have a max of 256 shares. SDMF / MDMF have a max of 255 shares! -# Represent max symbolically and resolve it when we know what format we're -# dealing with. -MAX_SHARES = "max" - -# SDMF and MDMF encode share counts (N and k) into the share itself as an -# unsigned byte. They could have encoded (share count - 1) to fit the full -# range supported by ZFEC into the unsigned byte - but they don't. So 256 is -# inaccessible to those formats and we set the upper bound at 255. -MAX_SHARES_MAP = { - "chk": 256, - "sdmf": 255, - "mdmf": 255, -} - ZFEC_PARAMS = [ - (1, 1), - (1, 3), - (2, 3), - (3, 10), - (71, 255), - (101, MAX_SHARES), + vectors.SeedParam(1, 1), + vectors.SeedParam(1, 3), + vectors.SeedParam(2, 3), + vectors.SeedParam(3, 10), + vectors.SeedParam(71, 255), + vectors.SeedParam(101, vectors.MAX_SHARES), ] FORMATS = [ "chk", - "sdmf", - "mdmf", + # "sdmf", + # "mdmf", ] -@mark.parametrize('convergence_idx', range(len(CONVERGENCE_SECRETS))) -def test_convergence(convergence_idx): +@mark.parametrize('convergence', CONVERGENCE_SECRETS) +def test_convergence(convergence): """ Convergence secrets are 16 bytes. """ - convergence = CONVERGENCE_SECRETS[convergence_idx] assert isinstance(convergence, bytes), "Convergence secret must be bytes" assert len(convergence) == 16, "Convergence secret must by 16 bytes" -@mark.parametrize('params_idx', range(len(ZFEC_PARAMS))) -@mark.parametrize('convergence_idx', range(len(CONVERGENCE_SECRETS))) -@mark.parametrize('data_idx', range(len(OBJECT_DESCRIPTIONS))) -@mark.parametrize('fmt_idx', range(len(FORMATS))) +@mark.parametrize('seed_params', ZFEC_PARAMS) +@mark.parametrize('convergence', CONVERGENCE_SECRETS) +@mark.parametrize('seed_data', OBJECT_DESCRIPTIONS) +@mark.parametrize('fmt', FORMATS) @ensureDeferred -async def test_capability(reactor, request, alice, params_idx, convergence_idx, data_idx, fmt_idx): +async def test_capability(reactor, request, alice, seed_params, convergence, seed_data, fmt): """ The capability that results from uploading certain well-known data with certain well-known parameters results in exactly the previously computed value. """ - case = load_case( - params_idx, - convergence_idx, - data_idx, - fmt_idx, - ) + case = vectors.Case(seed_params, convergence, seed_data, fmt) # rewrite alice's config to match params and convergence - await reconfigure(reactor, request, alice, (1,) + case.params, case.convergence) + await reconfigure(reactor, request, alice, (1, case.params.required, case.params.total), case.convergence) # upload data in the correct format actual = upload(alice, case.fmt, case.data) # compare the resulting cap to the expected result - expected = vectors.capabilities["vector"][case.key] + expected = vectors.capabilities[case] assert actual == expected @ensureDeferred -async def skiptest_generate(reactor, request, alice): +async def test_generate(reactor, request, alice): """ This is a helper for generating the test vectors. @@ -141,27 +121,34 @@ async def skiptest_generate(reactor, request, alice): to run against the results produced originally, not a possibly ever-changing set of outputs. """ - space = product( - range(len(ZFEC_PARAMS)), - range(len(CONVERGENCE_SECRETS)), - range(len(OBJECT_DESCRIPTIONS)), - range(len(FORMATS)), - ) - results = await asyncfoldr( - generate(reactor, request, alice, space), - insert, - {}, - ) + space = starmap(vectors.Case, product( + ZFEC_PARAMS, + CONVERGENCE_SECRETS, + OBJECT_DESCRIPTIONS, + FORMATS, + )) + results = generate(reactor, request, alice, space) with vectors.DATA_PATH.open("w") as f: f.write(safe_dump({ - "version": "2022-12-26", - "params": { - "zfec": ZFEC_PARAMS, - "convergence": CONVERGENCE_SECRETS, - "objects": OBJECT_DESCRIPTIONS, - "formats": FORMATS, - }, - "vector": results, + "version": "2023-01-03", + "vector": [ + { + "convergence": vectors.encode_bytes(case.convergence), + "format": case.fmt, + "sample": { + "seed": vectors.encode_bytes(case.seed_data.seed), + "length": case.seed_data.length, + }, + "zfec": { + "segmentSize": SEGMENT_SIZE, + "required": case.seed_params.required, + "total": case.seed_params.total, + }, + "expected": cap, + } + async for (case, cap) + in results + ], })) @@ -169,8 +156,8 @@ async def generate( reactor, request, alice: TahoeProcess, - space: Iterator[int, int, int, int], -) -> AsyncGenerator[tuple[str, str], None]: + cases: Iterator[vectors.Case], +) -> AsyncGenerator[[vectors.Case, str], None]: """ Generate all of the test vectors using the given node. @@ -184,79 +171,21 @@ async def generate( :param alice: The Tahoe-LAFS node to use to generate the test vectors. - :param space: An iterator of coordinates in the test vector space for - which to generate values. The elements of each tuple give indexes into - ZFEC_PARAMS, CONVERGENCE_SECRETS, OBJECT_DESCRIPTIONS, and FORMATS. + :param case: The inputs for which to generate a value. - :return: The yield values are two-tuples describing a test vector. The - first element is a string describing a case and the second element is - the capability for that case. + :return: The capability for the case. """ # Share placement doesn't affect the resulting capability. For maximum # reliability of this generator, be happy if we can put shares anywhere happy = 1 - node_key = (None, None) - for params_idx, secret_idx, data_idx, fmt_idx in space: - case = load_case(params_idx, secret_idx, data_idx, fmt_idx) - if node_key != (case.params, case.convergence): - await reconfigure(reactor, request, alice, (happy,) + case.params, case.convergence) - node_key = (case.params, case.convergence) + for case in cases: + await reconfigure( + reactor, + request, + alice, + (happy, case.params.required, case.params.total), + case.convergence + ) cap = upload(alice, case.fmt, case.data) - yield case.key, cap - - -def key(params: int, secret: int, data: int, fmt: int) -> str: - """ - Construct the key describing the case defined by the given parameters. - - The parameters are indexes into the test data for a certain case. - - :return: A distinct string for the given inputs. - """ - return f"{params}-{secret}-{data}-{fmt}" - - -def stretch(seed: bytes, size: int) -> bytes: - """ - Given a simple description of a byte string, return the byte string - itself. - """ - assert isinstance(seed, bytes) - assert isinstance(size, int) - assert size > 0 - assert len(seed) > 0 - - multiples = size // len(seed) + 1 - return (seed * multiples)[:size] - - -def load_case( - params_idx: int, - convergence_idx: int, - data_idx: int, - fmt_idx: int -) -> Case: - """ - :return: - """ - params = ZFEC_PARAMS[params_idx] - fmt = FORMATS[fmt_idx] - convergence = CONVERGENCE_SECRETS[convergence_idx] - data = stretch(*OBJECT_DESCRIPTIONS[data_idx]) - if params[1] == MAX_SHARES: - params = (params[0], MAX_SHARES_MAP[fmt]) - k = key(params_idx, convergence_idx, data_idx, fmt_idx) - return Case(k, fmt, params, convergence, data) - - -@frozen -class Case: - """ - Represent one case for which we want/have a test vector. - """ - key: str - fmt: str - params: tuple[int, int] - convergence: bytes - data: bytes + yield case, cap diff --git a/integration/test_vectors.yaml b/integration/test_vectors.yaml index 46ac7e890..ca16a1e92 100644 --- a/integration/test_vectors.yaml +++ b/integration/test_vectors.yaml @@ -9,26 +9,6 @@ vector: required: 1 segmentSize: 131072 total: 1 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:SSK:y74wcrcls3al6zxz7kx7f673z4:syoy43s3ywv6knmfecsubpl5mkgqxttjivi42tnthlwkirdvpvdq - format: sdmf - sample: - length: 1024 - seed: YQ== - zfec: - required: 1 - segmentSize: 131072 - total: 1 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:MDMF:aqy46c5phihqbh7gvacwyjjhmm:txdtibqlgmsrozp5ufsixeplln3hrcgjlrgkyu7vvbo54lvjq3sq - format: mdmf - sample: - length: 1024 - seed: YQ== - zfec: - required: 1 - segmentSize: 131072 - total: 1 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== expected: URI:CHK:e5tpbcmalcq6nn2zhd3qvg4chy:emzsvv2xnkhhrj2oatds5hf2cney25awi56ybeq4ofiyehzyakua:1:1:4096 format: chk @@ -39,26 +19,6 @@ vector: required: 1 segmentSize: 131072 total: 1 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:SSK:ievenb2az5hen2dd2lgr2rx4km:zix2o4lu25r6om3s2locfmmzyagii3eykrdxkjtbcjtf7djmpplq - format: sdmf - sample: - length: 4096 - seed: Yw== - zfec: - required: 1 - segmentSize: 131072 - total: 1 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:MDMF:roegsxmll43vrml5gu3pvv7j3y:o2hii67gi3kgqil6b5cy7bisahbw7s5y2qyfbi4oni2wiebig6jq - format: mdmf - sample: - length: 4096 - seed: Yw== - zfec: - required: 1 - segmentSize: 131072 - total: 1 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== expected: URI:CHK:ok3slnjd3e56za3iot74audhl4:2mjvrb455yldouoxwzbx4sbsysowipqje6ifa4pmbzqahj5j4mnq:1:1:131071 format: chk @@ -69,26 +29,6 @@ vector: required: 1 segmentSize: 131072 total: 1 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:SSK:dp3kn36nz3gpto7lgd4sgadu4i:7xsljpakiof654td2lxz7mtmnev7fp4vqhdz6yp6zm4r37gd5myq - format: sdmf - sample: - length: 131071 - seed: LCa0a2j/xo/5m0U8HTBBNBNCLXBkg7+g+YpeiGJm564= - zfec: - required: 1 - segmentSize: 131072 - total: 1 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:MDMF:aflykwudyx2gqfk6csy5u4lvni:m3kmwyizqwavmgnnx2lit4nqdobg7ze3okh4exuhxp5z5ljskdoq - format: mdmf - sample: - length: 131071 - seed: LCa0a2j/xo/5m0U8HTBBNBNCLXBkg7+g+YpeiGJm564= - zfec: - required: 1 - segmentSize: 131072 - total: 1 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== expected: URI:CHK:worle55uksa2uqqeebm4yxnihu:vta76jbmejt2pxx4prqa75xawpdtx42cmzzhappeetzjksym37wq:1:1:131073 format: chk @@ -99,26 +39,6 @@ vector: required: 1 segmentSize: 131072 total: 1 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:SSK:pd7xbhzt7v3ejras7g5acq5nry:lvvjteik6le7hm26lyd42i56pkkqbzu3fm7vi3kbtc2tzk2qv7eq - format: sdmf - sample: - length: 131073 - seed: /N4rLtula/QIYB+3If6bXDONEO5CnqBPrlURto+/j7k= - zfec: - required: 1 - segmentSize: 131072 - total: 1 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:MDMF:k4nqwrnxjiu3iw7r4gds2ddsii:bx665xasl5dsqippnj5ntgdofq6rtj6g377fygqcdcxvqjauhvhq - format: mdmf - sample: - length: 131073 - seed: /N4rLtula/QIYB+3If6bXDONEO5CnqBPrlURto+/j7k= - zfec: - required: 1 - segmentSize: 131072 - total: 1 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== expected: URI:CHK:6kccrgbtmmprqe4jcfi7vf6v74:wpivl2evi25yfl4tzbbj6vp6nzk4vxl6lbongkac7vl3escvopsa:1:1:2097151 format: chk @@ -129,26 +49,6 @@ vector: required: 1 segmentSize: 131072 total: 1 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:SSK:z6vgv24ns3ku35sv3z2upmjujm:l7qtsggtrzz33qxina3h47jx7e6wbtre67hilopouaraoxrdlm6a - format: sdmf - sample: - length: 2097151 - seed: uqWglk0zIPvAxqkiFARTyFE+okq4/QV3A0gEqWckgJY= - zfec: - required: 1 - segmentSize: 131072 - total: 1 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:MDMF:3zbhbl6c5s5tecu7mec6xpczbm:o3r52ullezeismr3bzbstwe5arbyx26vmfiblxz7sb7yjvjfc2xq - format: mdmf - sample: - length: 2097151 - seed: uqWglk0zIPvAxqkiFARTyFE+okq4/QV3A0gEqWckgJY= - zfec: - required: 1 - segmentSize: 131072 - total: 1 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== expected: URI:CHK:ofgig3loex6pev2eymmvohv7wq:yasbfedqnueaajdcavuba7kxbdqzn7e6zx3y6cbvucgx433vlzcq:1:1:2097153 format: chk @@ -159,26 +59,6 @@ vector: required: 1 segmentSize: 131072 total: 1 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:SSK:lop5wrefixcec6gdbtokmvxmoq:7n4usur4eupz6ikgklgtwcpnooazlmzfmwnly4qidnf2tmahmp6q - format: sdmf - sample: - length: 2097153 - seed: BTBX/ampNfLU+ox7xipBGiaSbgC0kcB8Gy7BkJB4oKI= - zfec: - required: 1 - segmentSize: 131072 - total: 1 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:MDMF:dx4uss2swjscntutzmadrnb4ce:yxfietcglts2qu3p7y5ufhmoqndkkqqtb5fic4efawkxrnspoqna - format: mdmf - sample: - length: 2097153 - seed: BTBX/ampNfLU+ox7xipBGiaSbgC0kcB8Gy7BkJB4oKI= - zfec: - required: 1 - segmentSize: 131072 - total: 1 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== expected: URI:CHK:scwepfpkvhuc3fc5jkpmrodsfi:wui2ixaxyac76mbxtr6diw2ajky75wehk57f2kqgytddnpo4rozq:1:1:8388607 format: chk @@ -189,26 +69,6 @@ vector: required: 1 segmentSize: 131072 total: 1 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:SSK:ae222l7khyyye76gr4jkqhrtsu:s6rgutrzybfijp7og4f7ol3pumtpqhgcfu3m5gbpwlde6i6r52ha - format: sdmf - sample: - length: 8388607 - seed: w6uP8Tcg6K2QR905Rms8iXTlksL6OD1KOWBxTK7wxPI= - zfec: - required: 1 - segmentSize: 131072 - total: 1 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:MDMF:akoeczwc4fiogo5f4umlpzepxm:53putm3likdcbzu4g2cvwvqale7pp2silt5w3lnx5m2nml7qtrka - format: mdmf - sample: - length: 8388607 - seed: w6uP8Tcg6K2QR905Rms8iXTlksL6OD1KOWBxTK7wxPI= - zfec: - required: 1 - segmentSize: 131072 - total: 1 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== expected: URI:CHK:5eeprb6lfxclwt7fieskd2euly:ffjgjbrxfl6d2ug2a63iaesxtrqa5pk3yaagfcldqvo7x4ok7ykq:1:1:8388609 format: chk @@ -219,26 +79,6 @@ vector: required: 1 segmentSize: 131072 total: 1 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:SSK:oefrpx2c7j2lslwsb4jvjcsqju:2m2nqlkgn5xoiip45mwc47zpcno74mayx2smd6lscjb7esme5aja - format: sdmf - sample: - length: 8388609 - seed: yPi3JHKKbWaEEG5eZOlM6BHJll0Z3UTdBzz4bPQ7wjg= - zfec: - required: 1 - segmentSize: 131072 - total: 1 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:MDMF:eqll4gbfnhbiuideqzgz2zgtum:e3t4vmtlcr4olgnk4wmocsbafhojaj4xakzolm6ds2srncgi67pq - format: mdmf - sample: - length: 8388609 - seed: yPi3JHKKbWaEEG5eZOlM6BHJll0Z3UTdBzz4bPQ7wjg= - zfec: - required: 1 - segmentSize: 131072 - total: 1 - convergence: ZOyIygCyaOW6GjVnihtTFg== expected: URI:CHK:nnv4vrtlxmzkurfzvonj22leua:ywcyijrfnwykraku56dq7v3o4ts3xsxqfgmk3kgzwhq2cpest4pa:1:1:1024 format: chk @@ -249,26 +89,6 @@ vector: required: 1 segmentSize: 131072 total: 1 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:SSK:bw66cmomyxpxzydovsfai4in4i:pxx76km232rfwl7zllsuxfjrpidq565r6p2ylf2h3e6frp65pq4q - format: sdmf - sample: - length: 1024 - seed: YQ== - zfec: - required: 1 - segmentSize: 131072 - total: 1 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:MDMF:tucs3y6kksigeaekahmz7jllum:5te4f5wh36j7m2x7rgrt2czofolh3z4zms3j6g6v7iw7aja4uywq - format: mdmf - sample: - length: 1024 - seed: YQ== - zfec: - required: 1 - segmentSize: 131072 - total: 1 - convergence: ZOyIygCyaOW6GjVnihtTFg== expected: URI:CHK:wlszole5m2emf6wbhp3lnlyfxq:irdttvny74gxdrcotzfjslfq7p24kyiigyrm5shhw7zyj3hsiuba:1:1:4096 format: chk @@ -279,26 +99,6 @@ vector: required: 1 segmentSize: 131072 total: 1 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:SSK:ntbuk6cv3mntzs645nb2yur5ea:ymrkers6rtknhsgg7oencxtaqfrxdtl4f4e345pguxlngpxmezka - format: sdmf - sample: - length: 4096 - seed: Yw== - zfec: - required: 1 - segmentSize: 131072 - total: 1 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:MDMF:tw4agh6qsiyp7eb6fc2q6wutfy:qu5ljqu3l3zetuddp3agsmq72nhpsw4kf3vzdfmbzwmyenefhfwa - format: mdmf - sample: - length: 4096 - seed: Yw== - zfec: - required: 1 - segmentSize: 131072 - total: 1 - convergence: ZOyIygCyaOW6GjVnihtTFg== expected: URI:CHK:4ewm23jvdtm2i5xf4gck26wg5y:7cujxxc34mkfkmwhbemqtuixuektknmhhxlmujcekibf5amfwwga:1:1:131071 format: chk @@ -309,26 +109,6 @@ vector: required: 1 segmentSize: 131072 total: 1 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:SSK:hgrowa5hbtubpvwl7kwuvqxzwq:oys24ziecxnvnnccdnawktcfwscpn6ukt3bn6pyispgqqoxzyunq - format: sdmf - sample: - length: 131071 - seed: LCa0a2j/xo/5m0U8HTBBNBNCLXBkg7+g+YpeiGJm564= - zfec: - required: 1 - segmentSize: 131072 - total: 1 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:MDMF:llsyycaa6sv3mozamcbrj25dqm:oaixtd5x4hkhpsdokd77qosigiiitpjygzn3j7fy5c4a4qffjifq - format: mdmf - sample: - length: 131071 - seed: LCa0a2j/xo/5m0U8HTBBNBNCLXBkg7+g+YpeiGJm564= - zfec: - required: 1 - segmentSize: 131072 - total: 1 - convergence: ZOyIygCyaOW6GjVnihtTFg== expected: URI:CHK:heczpdxphw5frp3ri5sh2hpqei:6wflx6lphy5mhtpfb2abznoa3yk27ynbqbzcecs362miu72vkowq:1:1:131073 format: chk @@ -339,26 +119,6 @@ vector: required: 1 segmentSize: 131072 total: 1 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:SSK:gkxj4zig7p6vdmq7qek2e6tcny:2ujwy5icd63hqdeibaaqag2xdsasbtut3unxgvhejmifvwwm7tja - format: sdmf - sample: - length: 131073 - seed: /N4rLtula/QIYB+3If6bXDONEO5CnqBPrlURto+/j7k= - zfec: - required: 1 - segmentSize: 131072 - total: 1 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:MDMF:iu3cu24qvryvfycl5hqbgs4p6e:a6vvnybvlkjlttah222hgykj6ekxdcfubqkifm6jz63aqigxnyuq - format: mdmf - sample: - length: 131073 - seed: /N4rLtula/QIYB+3If6bXDONEO5CnqBPrlURto+/j7k= - zfec: - required: 1 - segmentSize: 131072 - total: 1 - convergence: ZOyIygCyaOW6GjVnihtTFg== expected: URI:CHK:mz5siv27pqttsl2f4vcqmxju64:rvxhulxtufho6pdbwj7rifneb6pei5fl4fqptcce7jdkbyrjfaya:1:1:2097151 format: chk @@ -369,26 +129,6 @@ vector: required: 1 segmentSize: 131072 total: 1 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:SSK:u4w73dcf4szq7g435f45rkkc64:jmy5yl24cxuvxoe6rviuigub5rupumwqaalyhum4auvgmdugukyq - format: sdmf - sample: - length: 2097151 - seed: uqWglk0zIPvAxqkiFARTyFE+okq4/QV3A0gEqWckgJY= - zfec: - required: 1 - segmentSize: 131072 - total: 1 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:MDMF:jg2rhh247nqaqjep4vw5uz7na4:mqfow763upbanik2i7xvgah4mxsofusfarjcxcsqj6mpd3iap7ua - format: mdmf - sample: - length: 2097151 - seed: uqWglk0zIPvAxqkiFARTyFE+okq4/QV3A0gEqWckgJY= - zfec: - required: 1 - segmentSize: 131072 - total: 1 - convergence: ZOyIygCyaOW6GjVnihtTFg== expected: URI:CHK:shbt5viqjzuewblgt6qeijry6a:je3omw53tmmluz6fvqupx4uh4jaejc3fcvfjt56rfzokzhgdczvq:1:1:2097153 format: chk @@ -399,26 +139,6 @@ vector: required: 1 segmentSize: 131072 total: 1 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:SSK:mbtm4xgwifkvsvgiku75lshela:uqxgrmldjvdt5wdjnxr2sxgg7okjc2p3xyhjuo66adesetnj4ipa - format: sdmf - sample: - length: 2097153 - seed: BTBX/ampNfLU+ox7xipBGiaSbgC0kcB8Gy7BkJB4oKI= - zfec: - required: 1 - segmentSize: 131072 - total: 1 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:MDMF:hytvw2gqv566qy24hdsevhikm4:m76axh6i3guh2wuwp242pea6a6jretjxu6otjwp2xcwxhzd6tw4a - format: mdmf - sample: - length: 2097153 - seed: BTBX/ampNfLU+ox7xipBGiaSbgC0kcB8Gy7BkJB4oKI= - zfec: - required: 1 - segmentSize: 131072 - total: 1 - convergence: ZOyIygCyaOW6GjVnihtTFg== expected: URI:CHK:n2642ofjfr5qljptdwjbwjz6fq:uwtjrewrev42yvez7br2574xu5vleallwnyddjaxqquvouv3dyra:1:1:8388607 format: chk @@ -429,26 +149,6 @@ vector: required: 1 segmentSize: 131072 total: 1 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:SSK:z2nquzg3q2vu372xgwinybgrq4:cfdb4k2vovqljuynb24hbkdoqdtrjuf3paj3ei4kdv7jpjcmzq5a - format: sdmf - sample: - length: 8388607 - seed: w6uP8Tcg6K2QR905Rms8iXTlksL6OD1KOWBxTK7wxPI= - zfec: - required: 1 - segmentSize: 131072 - total: 1 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:MDMF:qolwd4bimeicby7cr4genjrjdu:c7pijeu66hqyzjvqbepagujq3gex6hzbq37fwgjj77voe2yhjjxq - format: mdmf - sample: - length: 8388607 - seed: w6uP8Tcg6K2QR905Rms8iXTlksL6OD1KOWBxTK7wxPI= - zfec: - required: 1 - segmentSize: 131072 - total: 1 - convergence: ZOyIygCyaOW6GjVnihtTFg== expected: URI:CHK:nktoeeuydph4acj2fux4axyaj4:g3fvjwanenwsgdcn3oxnau5gtbdmzbnbevlrzrb5qe4yukuwhejq:1:1:8388609 format: chk @@ -459,26 +159,6 @@ vector: required: 1 segmentSize: 131072 total: 1 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:SSK:xlosprcfmf7fghy6cp6d7j5gae:7di57dzhhx2chb22wddh5kspoqchcyaavkx3lyblld2q3ectkvka - format: sdmf - sample: - length: 8388609 - seed: yPi3JHKKbWaEEG5eZOlM6BHJll0Z3UTdBzz4bPQ7wjg= - zfec: - required: 1 - segmentSize: 131072 - total: 1 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:MDMF:pkul6c74muke6w3doyljnhspxq:3cc6azlsrdbvbg5jvtx75ft33hmcz2eprkfwfu3g3yms55mk4btq - format: mdmf - sample: - length: 8388609 - seed: yPi3JHKKbWaEEG5eZOlM6BHJll0Z3UTdBzz4bPQ7wjg= - zfec: - required: 1 - segmentSize: 131072 - total: 1 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== expected: URI:CHK:ycvogzi6wllnq2bkx3t6zdtwju:um42l4yen7jiwdfgirvedtty3tt3xuhjiyxzqoourvughtxjar3q:1:3:1024 format: chk @@ -489,26 +169,6 @@ vector: required: 1 segmentSize: 131072 total: 3 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:SSK:eh55dfvzf46gcyd2u65bnmek6a:dx7nph6gpag4ijgzrro3rfezsjb7xair3whk7nvpiijfbiw6tgpq - format: sdmf - sample: - length: 1024 - seed: YQ== - zfec: - required: 1 - segmentSize: 131072 - total: 3 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:MDMF:jmlnqnpysa3ypddys45phkfjfe:2fkhnlpb6b7mqvc6cu3w7wsd6vme2swiymblvx7j5i3vhifph65a - format: mdmf - sample: - length: 1024 - seed: YQ== - zfec: - required: 1 - segmentSize: 131072 - total: 3 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== expected: URI:CHK:3yjtuv5h2g45g2cncg6hka3euu:bqqhp6u73ldawial7rtrfomx2qkmboyvqve6ywo2jpvfbu7zptga:1:3:4096 format: chk @@ -519,26 +179,6 @@ vector: required: 1 segmentSize: 131072 total: 3 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:SSK:jli44aazmdb25uxpvyr65h5ea4:swm6gbbdwggi43etxxt5pnpwwzkcpb4tswe57xyl27tnf2pbkfaa - format: sdmf - sample: - length: 4096 - seed: Yw== - zfec: - required: 1 - segmentSize: 131072 - total: 3 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:MDMF:rqdgaoblt54p2jn43jye5ax7ri:md6p25qcasl2umo6udexfsp6hy67gi263a7vf3drnr2wx7e4fina - format: mdmf - sample: - length: 4096 - seed: Yw== - zfec: - required: 1 - segmentSize: 131072 - total: 3 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== expected: URI:CHK:annnqmu72p7iels5loqwlxt2zq:s7wv3jfi2hlpcvp4dnloc4eex6vre42kwiel46achaie5n5uodqa:1:3:131071 format: chk @@ -549,26 +189,6 @@ vector: required: 1 segmentSize: 131072 total: 3 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:SSK:jvqmilw4sq73al6ahoj35yxoc4:7q2n6widrd24j26h3jfodnxkia2ggc4hq2dkpcavkf7w3jos4pwq - format: sdmf - sample: - length: 131071 - seed: LCa0a2j/xo/5m0U8HTBBNBNCLXBkg7+g+YpeiGJm564= - zfec: - required: 1 - segmentSize: 131072 - total: 3 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:MDMF:pca3f6mqonmw5om3vkkhsx6j2m:khnmpqz4w6bdrgmm7vbfe4a6p35zug5kinlmtd7btnhnp7ihlqua - format: mdmf - sample: - length: 131071 - seed: LCa0a2j/xo/5m0U8HTBBNBNCLXBkg7+g+YpeiGJm564= - zfec: - required: 1 - segmentSize: 131072 - total: 3 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== expected: URI:CHK:rhkln7unkktot72mit5dmuqbdy:ilo6u6hugipdimyrzrvlam47xsmp3ur2lwnrtbecmvocb2664zxq:1:3:131073 format: chk @@ -579,26 +199,6 @@ vector: required: 1 segmentSize: 131072 total: 3 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:SSK:ugzqpyk57sfmw3xt3arpyh6qu4:octvdn5fzqhyzhedtleqky3uoti7jfzmlbfchqrhgt5zc5fs4qma - format: sdmf - sample: - length: 131073 - seed: /N4rLtula/QIYB+3If6bXDONEO5CnqBPrlURto+/j7k= - zfec: - required: 1 - segmentSize: 131072 - total: 3 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:MDMF:wuo2iiq4gdnvhe32ycvumfj7ce:lkbc7tsugufgw2xfiirjhqfujupx3347kus3j3ezcrvdkzknousa - format: mdmf - sample: - length: 131073 - seed: /N4rLtula/QIYB+3If6bXDONEO5CnqBPrlURto+/j7k= - zfec: - required: 1 - segmentSize: 131072 - total: 3 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== expected: URI:CHK:7yjcwwt6454lbv3pni5mjofsxe:y5nwpzwmvpvr3gqxnykjixprpxw3w6qyqmszf7ijyxnm3wl6f5oa:1:3:2097151 format: chk @@ -609,26 +209,6 @@ vector: required: 1 segmentSize: 131072 total: 3 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:SSK:cb6fgurdmns5w7jflfy23w73em:cbhxjn5oq7y6dpmlytlp7hubfdrrozf3krx2v5pnxwp4t4vrx2ma - format: sdmf - sample: - length: 2097151 - seed: uqWglk0zIPvAxqkiFARTyFE+okq4/QV3A0gEqWckgJY= - zfec: - required: 1 - segmentSize: 131072 - total: 3 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:MDMF:witiz6jbim5vox7yjisvwnyrhy:nfq4ku2xhgv4735ef3zxhi672cdvoeft7otpnm5nb5sv7yzkjrwa - format: mdmf - sample: - length: 2097151 - seed: uqWglk0zIPvAxqkiFARTyFE+okq4/QV3A0gEqWckgJY= - zfec: - required: 1 - segmentSize: 131072 - total: 3 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== expected: URI:CHK:r7wva6tisq6m5zszgr7seu77cm:oqlgdw3hi72qtahpsi3h3yryxpqdshagvt6xnobsppkwp7ic4cia:1:3:2097153 format: chk @@ -639,26 +219,6 @@ vector: required: 1 segmentSize: 131072 total: 3 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:SSK:le27bab7put7a7rljkwgelvygi:dqy75zxapt4jjkkbswjhplaqyvtrfwdp5mrn72epcw7xcew55zrq - format: sdmf - sample: - length: 2097153 - seed: BTBX/ampNfLU+ox7xipBGiaSbgC0kcB8Gy7BkJB4oKI= - zfec: - required: 1 - segmentSize: 131072 - total: 3 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:MDMF:sb6ub2v5fqezulwyinr32lbf6q:fp6d2q463oluwsyaidq2dmpk2p6artvlyu7urtq76w2iqql6bpda - format: mdmf - sample: - length: 2097153 - seed: BTBX/ampNfLU+ox7xipBGiaSbgC0kcB8Gy7BkJB4oKI= - zfec: - required: 1 - segmentSize: 131072 - total: 3 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== expected: URI:CHK:mz7pzh24u6flkfmfvvokdwmriu:5mzo57de2ywsdqkdvm7557gcikkpjqbmj65mk235t2xyoejtpbta:1:3:8388607 format: chk @@ -669,26 +229,6 @@ vector: required: 1 segmentSize: 131072 total: 3 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:SSK:jp6u5obzgtzt7236v6to6c4ei4:ae7jynsdpceqfvbaa6skwvlippw6ss7ikrbm6h474tij6fwigh7q - format: sdmf - sample: - length: 8388607 - seed: w6uP8Tcg6K2QR905Rms8iXTlksL6OD1KOWBxTK7wxPI= - zfec: - required: 1 - segmentSize: 131072 - total: 3 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:MDMF:43wo2yychy5ougonjnbzzsvssy:t4uuvenzh2fimgufbgbcqdxc37tlhsl7lli6fj5qoykujxbn2rtq - format: mdmf - sample: - length: 8388607 - seed: w6uP8Tcg6K2QR905Rms8iXTlksL6OD1KOWBxTK7wxPI= - zfec: - required: 1 - segmentSize: 131072 - total: 3 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== expected: URI:CHK:54sq6lrqwqlpxicg747gdls6ri:2aaxyrdytn7r74my36ek434hlbhe6glrgr2ic5vvpc5bbdxmvo6a:1:3:8388609 format: chk @@ -699,26 +239,6 @@ vector: required: 1 segmentSize: 131072 total: 3 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:SSK:5wrb7mqc2hzri7ekj4zxl4qad4:usym75mxkmnqkgqbtrui4e3q2362sa64w35u4bcumrkaibuljpiq - format: sdmf - sample: - length: 8388609 - seed: yPi3JHKKbWaEEG5eZOlM6BHJll0Z3UTdBzz4bPQ7wjg= - zfec: - required: 1 - segmentSize: 131072 - total: 3 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:MDMF:ldwwgiceepruadjywtjayvabse:zf7jofp4y6woanufvuvu2s4xaz6gdm64pfxqaufzmazjafycdvma - format: mdmf - sample: - length: 8388609 - seed: yPi3JHKKbWaEEG5eZOlM6BHJll0Z3UTdBzz4bPQ7wjg= - zfec: - required: 1 - segmentSize: 131072 - total: 3 - convergence: ZOyIygCyaOW6GjVnihtTFg== expected: URI:CHK:pbfcdvxhbikxd2hcc43oel3v3e:y5txpmiptmoz36ionmki3p6krmdbiqasw2v3wdq4ia5lmrhh33lq:1:3:1024 format: chk @@ -729,26 +249,6 @@ vector: required: 1 segmentSize: 131072 total: 3 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:SSK:vr7yiykd4yny5ygad6qsm7wklu:66ftrgc46osmagfzaxwzqpcvaemx674zh6jllvqhe4sqtbvv7hkq - format: sdmf - sample: - length: 1024 - seed: YQ== - zfec: - required: 1 - segmentSize: 131072 - total: 3 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:MDMF:aev7j2y474xff6wkczqmae474m:j7yujo66irpomwzonnf4e4uvvbdwsfnq2luhy2mope3u2z6mm7ha - format: mdmf - sample: - length: 1024 - seed: YQ== - zfec: - required: 1 - segmentSize: 131072 - total: 3 - convergence: ZOyIygCyaOW6GjVnihtTFg== expected: URI:CHK:vkdc7iae4pgkdomhbbb554kv6a:g5xkatxv2entfssfh5eqgexb5j3qs6jvohofjqo5p5erlxlx2dga:1:3:4096 format: chk @@ -759,26 +259,6 @@ vector: required: 1 segmentSize: 131072 total: 3 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:SSK:lzy6tvnsepowhab5gn7go3xcl4:ecgdtffw5rfujxbi5653zojr3lxj3lkf2cgwf5s4qz4ukag7cshq - format: sdmf - sample: - length: 4096 - seed: Yw== - zfec: - required: 1 - segmentSize: 131072 - total: 3 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:MDMF:lp2mhym5ltuihpfqmwyouskqxq:yxmxqzsn3px4qppjsn62psi3nwedbcxtg2gxlryiifcxtlf66ceq - format: mdmf - sample: - length: 4096 - seed: Yw== - zfec: - required: 1 - segmentSize: 131072 - total: 3 - convergence: ZOyIygCyaOW6GjVnihtTFg== expected: URI:CHK:zvla2jyu5fqlb2s63zftyfjnla:yqspaexhew55cvv7w2m6czezhzkqnyk5fea4gvnplc5za4xlvbva:1:3:131071 format: chk @@ -789,26 +269,6 @@ vector: required: 1 segmentSize: 131072 total: 3 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:SSK:waofhwlohtuqpcwuwl7fxozc5u:r5qrte3v2npvzirkiujhjfyi3poobwtyzhwtqbjiw5j6lgaxfsdq - format: sdmf - sample: - length: 131071 - seed: LCa0a2j/xo/5m0U8HTBBNBNCLXBkg7+g+YpeiGJm564= - zfec: - required: 1 - segmentSize: 131072 - total: 3 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:MDMF:lg5dfsjapjmq64z4klon5aembe:an3vjcuovsxhgwownhlle6di5utdgdwe34ji7xdcrugors5nlmfa - format: mdmf - sample: - length: 131071 - seed: LCa0a2j/xo/5m0U8HTBBNBNCLXBkg7+g+YpeiGJm564= - zfec: - required: 1 - segmentSize: 131072 - total: 3 - convergence: ZOyIygCyaOW6GjVnihtTFg== expected: URI:CHK:3idf2xqm5wnw2q3nbva5vmpaiq:7nmkh5q55omjloezibvyzvveq4gqeaivei5ypfm5vfuugwiz6hpa:1:3:131073 format: chk @@ -819,26 +279,6 @@ vector: required: 1 segmentSize: 131072 total: 3 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:SSK:u2wzqdemrrm2h5lntniddeajmq:b6cf7fvw2m23ulucwe63poe5ebhqiwppmcqoabkzsoc5m2m3sthq - format: sdmf - sample: - length: 131073 - seed: /N4rLtula/QIYB+3If6bXDONEO5CnqBPrlURto+/j7k= - zfec: - required: 1 - segmentSize: 131072 - total: 3 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:MDMF:v4bediwxuymp2immfnuhfx4buy:vix7mxu6wgi46bamw2y4etbtm3exc522i3xasrvy72tcin2v3uka - format: mdmf - sample: - length: 131073 - seed: /N4rLtula/QIYB+3If6bXDONEO5CnqBPrlURto+/j7k= - zfec: - required: 1 - segmentSize: 131072 - total: 3 - convergence: ZOyIygCyaOW6GjVnihtTFg== expected: URI:CHK:o7xdokumtcukutzgmmpfme4nnm:nn3s5gau5dychy6wlodwriuit4wyavfze6bo4icywcbhdb4ccxla:1:3:2097151 format: chk @@ -849,26 +289,6 @@ vector: required: 1 segmentSize: 131072 total: 3 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:SSK:fsjb2ym6pgn3fupo66jrmmwxia:3ix37zaem6w2zsotsk5hjj4ygst7r27zecfpvehnugwyw4lq3ija - format: sdmf - sample: - length: 2097151 - seed: uqWglk0zIPvAxqkiFARTyFE+okq4/QV3A0gEqWckgJY= - zfec: - required: 1 - segmentSize: 131072 - total: 3 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:MDMF:otowqw6mrbgz4tiyslxpfi3k2u:xhqeu536q4q3qi6hhqyjd52zthrdchhtmpxmu2pt7n4ff74kl37a - format: mdmf - sample: - length: 2097151 - seed: uqWglk0zIPvAxqkiFARTyFE+okq4/QV3A0gEqWckgJY= - zfec: - required: 1 - segmentSize: 131072 - total: 3 - convergence: ZOyIygCyaOW6GjVnihtTFg== expected: URI:CHK:vws2xrl2nch2hsptqb36yqqu3e:fhujsv5rkhyiqstrktjvfy5235c7gcwekyd3gux3bv2glsgstjga:1:3:2097153 format: chk @@ -879,26 +299,6 @@ vector: required: 1 segmentSize: 131072 total: 3 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:SSK:fhkkaxbg7y7piwv7fh5ylpdxhm:rewuizpn6v4xqy6df3bddfhp373iw76lh4qx6stzrwzskpedegza - format: sdmf - sample: - length: 2097153 - seed: BTBX/ampNfLU+ox7xipBGiaSbgC0kcB8Gy7BkJB4oKI= - zfec: - required: 1 - segmentSize: 131072 - total: 3 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:MDMF:p3d434skzrqcrxiotnp3u2ij34:tc6mps24psn2zg27ryfuquovzgfgrkxqg3ckibisw4ijftnuuakq - format: mdmf - sample: - length: 2097153 - seed: BTBX/ampNfLU+ox7xipBGiaSbgC0kcB8Gy7BkJB4oKI= - zfec: - required: 1 - segmentSize: 131072 - total: 3 - convergence: ZOyIygCyaOW6GjVnihtTFg== expected: URI:CHK:5anac46dpfsjumutzsoovvtlpe:yuc3gqe7m6s6xp7k3uodcg2g6k2bac32gt5amu7gdx7k43qnudxa:1:3:8388607 format: chk @@ -909,26 +309,6 @@ vector: required: 1 segmentSize: 131072 total: 3 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:SSK:o7gx4tbbuqac6khwlosk4l5ddy:sy2po455xvmrz2o5mvvjzmjrzlpue4fzqudprcgxfpa3hxnkx5sa - format: sdmf - sample: - length: 8388607 - seed: w6uP8Tcg6K2QR905Rms8iXTlksL6OD1KOWBxTK7wxPI= - zfec: - required: 1 - segmentSize: 131072 - total: 3 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:MDMF:xck4irhtemsujzgkdtdthdpare:rfrfsjzznueky3h33uptvgxdtiwbqaelfnwtt3jcve4ju63zxfna - format: mdmf - sample: - length: 8388607 - seed: w6uP8Tcg6K2QR905Rms8iXTlksL6OD1KOWBxTK7wxPI= - zfec: - required: 1 - segmentSize: 131072 - total: 3 - convergence: ZOyIygCyaOW6GjVnihtTFg== expected: URI:CHK:fb6g3fkfbtlwzheujzvc2dn7dq:644je55ri6q5halshuxfesjz2afhqtebetuzqqbfecp6yhqwbb5q:1:3:8388609 format: chk @@ -939,26 +319,6 @@ vector: required: 1 segmentSize: 131072 total: 3 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:SSK:d7uzl2zychjkmm7siybhbjys3m:cmoram5uvqusxc3x5oi5wy7lfsvffjnfyktkywnc6camhiitjmfa - format: sdmf - sample: - length: 8388609 - seed: yPi3JHKKbWaEEG5eZOlM6BHJll0Z3UTdBzz4bPQ7wjg= - zfec: - required: 1 - segmentSize: 131072 - total: 3 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:MDMF:lo2p6saqx5kvmfgrsnf6fhiwei:zhjuapt2wwkacd5oa2li3aahrs36q3dckmljdbrsnsnmd7fygkxq - format: mdmf - sample: - length: 8388609 - seed: yPi3JHKKbWaEEG5eZOlM6BHJll0Z3UTdBzz4bPQ7wjg= - zfec: - required: 1 - segmentSize: 131072 - total: 3 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== expected: URI:CHK:jdm6ytmx2i3ya2bllsdzxurjtu:iqr6tyysaseegzfurhuywy3mbbwkbbsov5bt3fo6oazpyv7olvda:2:3:1024 format: chk @@ -969,26 +329,6 @@ vector: required: 2 segmentSize: 131072 total: 3 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:SSK:znxwd4qkfko7ss74cgtribl76m:yubnd3unz4ipv2mm3khl4jbpwzzoxa3mnenq6n6tqpfydujfde4q - format: sdmf - sample: - length: 1024 - seed: YQ== - zfec: - required: 2 - segmentSize: 131072 - total: 3 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:MDMF:4m7y4dobd6ret6ikj36afo3bki:bm4oup3oijsfshzisvt52xcif7ks7x7sq3u7k5aarc44kzqlhabq - format: mdmf - sample: - length: 1024 - seed: YQ== - zfec: - required: 2 - segmentSize: 131072 - total: 3 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== expected: URI:CHK:7wz5dhkyob3gmkydzgbptk5qku:cl6ovq5a3km7rpfhyb3putg656sp57lhnp7aexbocsbbahdxnfia:2:3:4096 format: chk @@ -999,26 +339,6 @@ vector: required: 2 segmentSize: 131072 total: 3 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:SSK:36b6rvsrqya5oqehistbht2y3q:ynyxc67qeewensnisyp5voubt5qkf27fkmdspzpnkzgmmuejrnya - format: sdmf - sample: - length: 4096 - seed: Yw== - zfec: - required: 2 - segmentSize: 131072 - total: 3 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:MDMF:gyxpdtxuyzamxrehlyjrw2pf4e:wrdnxnzcivydr4szmfxseaqgawacudx74n6wmddgfrglkoi7467q - format: mdmf - sample: - length: 4096 - seed: Yw== - zfec: - required: 2 - segmentSize: 131072 - total: 3 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== expected: URI:CHK:aagw6esdm2msphvgnr3rlzg43e:zqxtrlee6hh3vtfg6ihtssjqvr27tpvi6gkmngnhiguttjz7yyja:2:3:131071 format: chk @@ -1029,26 +349,6 @@ vector: required: 2 segmentSize: 131072 total: 3 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:SSK:dicmwi2p2mzxp2nf7j7nxj2tma:iialvaqlims3gf4whxfmjysijvesovxy4q4ecjrm636j2cnadhwa - format: sdmf - sample: - length: 131071 - seed: LCa0a2j/xo/5m0U8HTBBNBNCLXBkg7+g+YpeiGJm564= - zfec: - required: 2 - segmentSize: 131072 - total: 3 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:MDMF:usko2l5rufksoav3faz6w3vmue:tkpane2dyaul4mosc6lc35os72t3s54ga4u7dedlxhu7fhi3ovda - format: mdmf - sample: - length: 131071 - seed: LCa0a2j/xo/5m0U8HTBBNBNCLXBkg7+g+YpeiGJm564= - zfec: - required: 2 - segmentSize: 131072 - total: 3 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== expected: URI:CHK:e6uhixvbm3g5amrzdzd3mnmsqm:4n5aafrb64sqpfnhcdpdfrk5gh2qznprjky6q6jybuuqy2q6z6mq:2:3:131073 format: chk @@ -1059,26 +359,6 @@ vector: required: 2 segmentSize: 131072 total: 3 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:SSK:tc346l5ky77pyqzunsobixc2am:4jk7vy4kie35wg4tdthqjczbc7pv62qbc5dbh2kf653q2k6optbq - format: sdmf - sample: - length: 131073 - seed: /N4rLtula/QIYB+3If6bXDONEO5CnqBPrlURto+/j7k= - zfec: - required: 2 - segmentSize: 131072 - total: 3 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:MDMF:xomeoo2rxocwsrm6lgui4krebe:jz7oodng7ym2fb7g5hbxqacetkb5aizdoqi5lxyavgrg7goyq45q - format: mdmf - sample: - length: 131073 - seed: /N4rLtula/QIYB+3If6bXDONEO5CnqBPrlURto+/j7k= - zfec: - required: 2 - segmentSize: 131072 - total: 3 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== expected: URI:CHK:abfmzuiczjwt6e67f3uoyg5i7q:rk7ie3afnp3xgvnxu5e2vioq476xqwslqxgyx3i5xxevsjhkcvba:2:3:2097151 format: chk @@ -1089,26 +369,6 @@ vector: required: 2 segmentSize: 131072 total: 3 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:SSK:ixed54x47cxuzgsfwm7g4lhzhe:nrqzkdl6gb3jrnalhprhtqy2o4vj5u2cvpyf7nnagxximc36yeiq - format: sdmf - sample: - length: 2097151 - seed: uqWglk0zIPvAxqkiFARTyFE+okq4/QV3A0gEqWckgJY= - zfec: - required: 2 - segmentSize: 131072 - total: 3 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:MDMF:i4curydlvv2ki6a6iz4ttgtdhq:3augdg7vy7z3ubjer6nroe7foxwwmgl2hpg4o6zqake6zvpe43ga - format: mdmf - sample: - length: 2097151 - seed: uqWglk0zIPvAxqkiFARTyFE+okq4/QV3A0gEqWckgJY= - zfec: - required: 2 - segmentSize: 131072 - total: 3 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== expected: URI:CHK:34bzejdzpqinmuzdmqenkidf74:pcecbl4ulygiadgdagonbgqmpb4lomjz4v4vqyssr56reyib4q2q:2:3:2097153 format: chk @@ -1119,26 +379,6 @@ vector: required: 2 segmentSize: 131072 total: 3 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:SSK:e2bzrxjvgyhmc45bmtj4rlerce:wobzlw4rpwuufqmjo6tep5mhrbebs3wdqcy7zkkefbs2fxacok5q - format: sdmf - sample: - length: 2097153 - seed: BTBX/ampNfLU+ox7xipBGiaSbgC0kcB8Gy7BkJB4oKI= - zfec: - required: 2 - segmentSize: 131072 - total: 3 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:MDMF:fxnfxuggzofpoorlm2zn7dg6ty:tk2zuztd7j4gqmn56c6ffddqgzwenbudl7muvrjwve7gghgvceia - format: mdmf - sample: - length: 2097153 - seed: BTBX/ampNfLU+ox7xipBGiaSbgC0kcB8Gy7BkJB4oKI= - zfec: - required: 2 - segmentSize: 131072 - total: 3 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== expected: URI:CHK:ej56vl65kawat62wsxuvbcd2wy:tv65cgghaddq2p7dyvimmff24l34wsx6mhzd3dyprfvrdo52qtma:2:3:8388607 format: chk @@ -1149,26 +389,6 @@ vector: required: 2 segmentSize: 131072 total: 3 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:SSK:5pyvbw7axzkw72rtlvwelgfcfy:d5acbwniecttboqvyn3ihe7h5ehpigy6u5tz73vf3aj25yeikxsa - format: sdmf - sample: - length: 8388607 - seed: w6uP8Tcg6K2QR905Rms8iXTlksL6OD1KOWBxTK7wxPI= - zfec: - required: 2 - segmentSize: 131072 - total: 3 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:MDMF:bcrt7yjrgcofkamredsw5x4bbm:hqiucmqok5qvns6yh2o5sibe7bg3ijodtp2ur3wkxhmt4bmwin7q - format: mdmf - sample: - length: 8388607 - seed: w6uP8Tcg6K2QR905Rms8iXTlksL6OD1KOWBxTK7wxPI= - zfec: - required: 2 - segmentSize: 131072 - total: 3 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== expected: URI:CHK:ufshkmuewwql2xsyiecbjhh7x4:qoywwygjqrmifowob3mvwkhfskk6geomyw5e3qlgqzui3kymu6nq:2:3:8388609 format: chk @@ -1179,26 +399,6 @@ vector: required: 2 segmentSize: 131072 total: 3 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:SSK:5gjangydj5e7ziu6jmzuo2wvam:wb5wy5civ5e6tecfohqjvbj5r7iggv765lm3jpeq4iecbxye2apa - format: sdmf - sample: - length: 8388609 - seed: yPi3JHKKbWaEEG5eZOlM6BHJll0Z3UTdBzz4bPQ7wjg= - zfec: - required: 2 - segmentSize: 131072 - total: 3 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:MDMF:pdbqgmt5wmcccwznv4ksqhukra:e2yek3bursdelejqx2maiqk7maevukhqzzjbw2wtzqyhbs2o4s6a - format: mdmf - sample: - length: 8388609 - seed: yPi3JHKKbWaEEG5eZOlM6BHJll0Z3UTdBzz4bPQ7wjg= - zfec: - required: 2 - segmentSize: 131072 - total: 3 - convergence: ZOyIygCyaOW6GjVnihtTFg== expected: URI:CHK:ld3thziutpaqv25nbtoqebhtru:ri2obsvzl2etyuv3qnchn2wvw5mh5rjuvbjaqkqbvyprn7xwamoq:2:3:1024 format: chk @@ -1209,26 +409,6 @@ vector: required: 2 segmentSize: 131072 total: 3 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:SSK:zamaagvy42hzahq6e5ze275hya:c2yru2mkkzmmaovfzez7g63tczbz4vsfncfiamdjwtay2f4rbf6a - format: sdmf - sample: - length: 1024 - seed: YQ== - zfec: - required: 2 - segmentSize: 131072 - total: 3 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:MDMF:rdemgunqoy4rxxmpyhwqmolmwi:dwt4vxhvrcgk2zupwa3nq4mlzr54mkrenjrv36t5co6yeg4oefpa - format: mdmf - sample: - length: 1024 - seed: YQ== - zfec: - required: 2 - segmentSize: 131072 - total: 3 - convergence: ZOyIygCyaOW6GjVnihtTFg== expected: URI:CHK:pz3mga5pvqd4bpsz7pbyna2qtq:w3ptt7xwwgywod3u2rpgx7iggvm2kpwl7d2ryr3u7mcaxgqotlaq:2:3:4096 format: chk @@ -1239,26 +419,6 @@ vector: required: 2 segmentSize: 131072 total: 3 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:SSK:n7hevwssawse5kret3d4yux7ge:i4y2ff2lv6rw2byrvxcl2k6hda26ol25ucv2wn7t6rw55k3ype2a - format: sdmf - sample: - length: 4096 - seed: Yw== - zfec: - required: 2 - segmentSize: 131072 - total: 3 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:MDMF:gwrzotpvkkffjfw74bipj6g3ue:ue2frkaek5t6jpstjehu6ah4euikq3jatkdgfkgspzjhtucguw5a - format: mdmf - sample: - length: 4096 - seed: Yw== - zfec: - required: 2 - segmentSize: 131072 - total: 3 - convergence: ZOyIygCyaOW6GjVnihtTFg== expected: URI:CHK:rsh7ysvsgbwh5g3xuj6z7zky3y:ykg5rhzpzqkvdy53qqyghfbcusibynfylggbhypo3iibsrrzvlkq:2:3:131071 format: chk @@ -1269,26 +429,6 @@ vector: required: 2 segmentSize: 131072 total: 3 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:SSK:5ajxk7jw4g22ga57hayghtvnam:exlqadg54twmlyhkgicn2iaqt65luupeebh7mheabppsdlh5ye7a - format: sdmf - sample: - length: 131071 - seed: LCa0a2j/xo/5m0U8HTBBNBNCLXBkg7+g+YpeiGJm564= - zfec: - required: 2 - segmentSize: 131072 - total: 3 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:MDMF:c4esc6ob3alfyqqob76oveefeu:ramdhh6ufpbn5ygeo5gilwkxyx3oi4i7k7g3pbic3nf6qpvw7m3q - format: mdmf - sample: - length: 131071 - seed: LCa0a2j/xo/5m0U8HTBBNBNCLXBkg7+g+YpeiGJm564= - zfec: - required: 2 - segmentSize: 131072 - total: 3 - convergence: ZOyIygCyaOW6GjVnihtTFg== expected: URI:CHK:7xqejfqfej3u3zp2qymqd2iqeq:po4t2tzkh4d34ku6gdhlocadfwdyweiyckyqz57zdi7ndt4ikj3q:2:3:131073 format: chk @@ -1299,26 +439,6 @@ vector: required: 2 segmentSize: 131072 total: 3 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:SSK:5bau524o4vbrnowrba2j3l44ca:skxdvkz5ufb4u7fegccpmmq3tx32tqse6udtb6v5cutb765i5toq - format: sdmf - sample: - length: 131073 - seed: /N4rLtula/QIYB+3If6bXDONEO5CnqBPrlURto+/j7k= - zfec: - required: 2 - segmentSize: 131072 - total: 3 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:MDMF:hyyofiigvckijkrxqlgzq55e7a:uth6x2l6ocak4xcozm5izzsd24ahymu35luxdmufb5oqo337g6mq - format: mdmf - sample: - length: 131073 - seed: /N4rLtula/QIYB+3If6bXDONEO5CnqBPrlURto+/j7k= - zfec: - required: 2 - segmentSize: 131072 - total: 3 - convergence: ZOyIygCyaOW6GjVnihtTFg== expected: URI:CHK:kzhg5zp4hcs75eboack4kqpcdm:g2cwnhpfwxrv3c4lrmpjyj327a274oa5qt4isu4avjixbu7vblrq:2:3:2097151 format: chk @@ -1329,26 +449,6 @@ vector: required: 2 segmentSize: 131072 total: 3 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:SSK:cibwsbzizcapwki2exx4rkr2qu:gkujzvhxmjhb2z4wzecyc225xyxe3tqqfvaf2s2oc6rzmdn5wp7a - format: sdmf - sample: - length: 2097151 - seed: uqWglk0zIPvAxqkiFARTyFE+okq4/QV3A0gEqWckgJY= - zfec: - required: 2 - segmentSize: 131072 - total: 3 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:MDMF:kupuwzss5lowgfl6adcvhgazfq:2jfedarh23rfnwzxhdkae6b2mh2dfzmh7uadpw4pcr622bls4mfq - format: mdmf - sample: - length: 2097151 - seed: uqWglk0zIPvAxqkiFARTyFE+okq4/QV3A0gEqWckgJY= - zfec: - required: 2 - segmentSize: 131072 - total: 3 - convergence: ZOyIygCyaOW6GjVnihtTFg== expected: URI:CHK:6sf35smixwvpapf2zw7sllxet4:ks5zd3hkd7ppwobhnymezckszexpychngnkipxfvmlcng5fgjbea:2:3:2097153 format: chk @@ -1359,26 +459,6 @@ vector: required: 2 segmentSize: 131072 total: 3 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:SSK:w5nx47bm7jdf4l4ht47235krte:deoa4hwzahmy2bcws7hz3ry3ubpemllu5marnaocewhode2ucdpa - format: sdmf - sample: - length: 2097153 - seed: BTBX/ampNfLU+ox7xipBGiaSbgC0kcB8Gy7BkJB4oKI= - zfec: - required: 2 - segmentSize: 131072 - total: 3 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:MDMF:argkvyswpyjfrtmuyg3jvwyswa:6d373u2djhnw6ug3yutlghkpb22bjmphl76vviclk7rc5bsj7koa - format: mdmf - sample: - length: 2097153 - seed: BTBX/ampNfLU+ox7xipBGiaSbgC0kcB8Gy7BkJB4oKI= - zfec: - required: 2 - segmentSize: 131072 - total: 3 - convergence: ZOyIygCyaOW6GjVnihtTFg== expected: URI:CHK:dnmjqjku5vyrkeaulae4bt6juu:d47airwaqedeji2omacwsp6yfqp5mpurj776gmzvq43c6dovtdca:2:3:8388607 format: chk @@ -1389,26 +469,6 @@ vector: required: 2 segmentSize: 131072 total: 3 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:SSK:whwbzc5q4qtpfmmx5udqt4ajxa:7vpif7ukkmrmnifk3mech6g6n77x3345t7oqoqh46apzwbwomqha - format: sdmf - sample: - length: 8388607 - seed: w6uP8Tcg6K2QR905Rms8iXTlksL6OD1KOWBxTK7wxPI= - zfec: - required: 2 - segmentSize: 131072 - total: 3 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:MDMF:767fhe755a4stlnvnm6y66mi4e:cbq5fdp6zcjg2xjqanlwsgc6s3jbc4hf62drzhjcmlsfjolp4xya - format: mdmf - sample: - length: 8388607 - seed: w6uP8Tcg6K2QR905Rms8iXTlksL6OD1KOWBxTK7wxPI= - zfec: - required: 2 - segmentSize: 131072 - total: 3 - convergence: ZOyIygCyaOW6GjVnihtTFg== expected: URI:CHK:bfjedwcwjehjzpwjsgwkfk7rja:jpwea2sgz4hfohqab642yj4cmrh64w6nfo7lv3mtacfzicurqkva:2:3:8388609 format: chk @@ -1419,26 +479,6 @@ vector: required: 2 segmentSize: 131072 total: 3 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:SSK:ri6woi5eti2ig6mbxvytuyza7y:lgdm3v2jghtjlpv32rjuzosqd4cyhr6jkohyxaw6nmkv7i4t3oua - format: sdmf - sample: - length: 8388609 - seed: yPi3JHKKbWaEEG5eZOlM6BHJll0Z3UTdBzz4bPQ7wjg= - zfec: - required: 2 - segmentSize: 131072 - total: 3 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:MDMF:zujw3a76gbitwmnaqd26kbsdoa:gajfmtn4iuexx6x36i2aftawkkr4xeraxlkspxg6uqhsw3iz3iwq - format: mdmf - sample: - length: 8388609 - seed: yPi3JHKKbWaEEG5eZOlM6BHJll0Z3UTdBzz4bPQ7wjg= - zfec: - required: 2 - segmentSize: 131072 - total: 3 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== expected: URI:CHK:dx7tvyr2fc4u7lxjc6kehq2svq:tiy4qh2g6lqejxcaym3rr7ymkdkinn4qised6kgxloj7sptsqu4a:3:10:1024 format: chk @@ -1449,26 +489,6 @@ vector: required: 3 segmentSize: 131072 total: 10 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:SSK:onr7swqc3kgyec6ocfcvyjmx6q:or2bm3iuf7crsietg6dkidtjbmfiu2eulvllgz5e5uelqt2ey5eq - format: sdmf - sample: - length: 1024 - seed: YQ== - zfec: - required: 3 - segmentSize: 131072 - total: 10 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:MDMF:3ceb4g2e4twofve5jkewepvnaa:oztau4y5nizlwn3psdukebxhhgitalp4p6b2tcvpm4ogiswwcuua - format: mdmf - sample: - length: 1024 - seed: YQ== - zfec: - required: 3 - segmentSize: 131072 - total: 10 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== expected: URI:CHK:yo75evk4cte3b7rdw72zxvl5ye:ex6h7ff7nclucjtsqwgwu33qgmb67t4ezbrki4zbgurwn2ct6bbq:3:10:4096 format: chk @@ -1479,26 +499,6 @@ vector: required: 3 segmentSize: 131072 total: 10 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:SSK:iugq34v2swtiwjbyy2sce7cjte:k3bloctsmve774usd36mg34pmxiu52w64zhaarvhwwlubnq66h4a - format: sdmf - sample: - length: 4096 - seed: Yw== - zfec: - required: 3 - segmentSize: 131072 - total: 10 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:MDMF:bnvrx3ubou6wyjqmz7ds2dqjlu:oaq44pgwyogdvtf5lb3dtotdfhrzedwex7svyjmnhibnelotvzxq - format: mdmf - sample: - length: 4096 - seed: Yw== - zfec: - required: 3 - segmentSize: 131072 - total: 10 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== expected: URI:CHK:4gokef54smahrbfr4kq3jhc4zq:owpwwfp5gof2vhly5u6jdnbsfuwwwhqkazpsbeg3nldxv5pse2iq:3:10:131071 format: chk @@ -1509,26 +509,6 @@ vector: required: 3 segmentSize: 131072 total: 10 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:SSK:y4onydf5hf23ndbobhxtucmfsq:3zpmw5rcysqk7jf7yfhqfiptn6pwelgqfoh5tcajwp42isa64mtq - format: sdmf - sample: - length: 131071 - seed: LCa0a2j/xo/5m0U8HTBBNBNCLXBkg7+g+YpeiGJm564= - zfec: - required: 3 - segmentSize: 131072 - total: 10 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:MDMF:xalxckir42vumn4twsj5sr47qm:mlkqxei4kgqy6jwvwfwwwj6q4bvlr3qutbn7d3wcgimvjaip42ja - format: mdmf - sample: - length: 131071 - seed: LCa0a2j/xo/5m0U8HTBBNBNCLXBkg7+g+YpeiGJm564= - zfec: - required: 3 - segmentSize: 131072 - total: 10 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== expected: URI:CHK:7vfgl5cv4nlzqx35z4uthjv36y:nnueftbzxfz6u5yjxwwofaxzzft7xss5wzfh66rrcwv2zwrm63sa:3:10:131073 format: chk @@ -1539,26 +519,6 @@ vector: required: 3 segmentSize: 131072 total: 10 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:SSK:bptkj6t37usy4usosftkkuhv7e:ckumolq5lngoe3nkyqj2x3bfd4omxjpe2z4ajfeqfxh6hbfzavvq - format: sdmf - sample: - length: 131073 - seed: /N4rLtula/QIYB+3If6bXDONEO5CnqBPrlURto+/j7k= - zfec: - required: 3 - segmentSize: 131072 - total: 10 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:MDMF:zuqe744q3gyn7wmkuj2wrvscbq:n2z2hcwdlmtiwbg6tutzlkdiwjccqazvxapj6kek22rnff7yu47q - format: mdmf - sample: - length: 131073 - seed: /N4rLtula/QIYB+3If6bXDONEO5CnqBPrlURto+/j7k= - zfec: - required: 3 - segmentSize: 131072 - total: 10 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== expected: URI:CHK:cy7fmjsldeakhfd4psbmghmqyi:6a6uvyai4jkzz6hj5yjugm6uie5etvymcudgiwwjh47apz636zwa:3:10:2097151 format: chk @@ -1569,26 +529,6 @@ vector: required: 3 segmentSize: 131072 total: 10 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:SSK:yys5phbf5zmwlmie6kf52n46ga:lh3huulqjziteauw2l2vngnef2jenb3bwrgylcqh3pklpntpzh3q - format: sdmf - sample: - length: 2097151 - seed: uqWglk0zIPvAxqkiFARTyFE+okq4/QV3A0gEqWckgJY= - zfec: - required: 3 - segmentSize: 131072 - total: 10 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:MDMF:5rl3h6trfxgyavfavxmnnzqhxy:humgb27p2kmwnw5dm4shyjeprvzemaxokjgol3jo4x7kpgkg4peq - format: mdmf - sample: - length: 2097151 - seed: uqWglk0zIPvAxqkiFARTyFE+okq4/QV3A0gEqWckgJY= - zfec: - required: 3 - segmentSize: 131072 - total: 10 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== expected: URI:CHK:ptsofqwylmkvzmuvrw5n34j3ma:ky2fs7xrlke64w6kfmhzsuilzxbhfrwwzkxih4rykpbxrr3bxhiq:3:10:2097153 format: chk @@ -1599,26 +539,6 @@ vector: required: 3 segmentSize: 131072 total: 10 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:SSK:yjjkeiwrkl4zxvotwzgjodmcxm:ankmcys42fn27r733ngkw3ez5xfhh3g5swjcwwmggpjpwm4icy2q - format: sdmf - sample: - length: 2097153 - seed: BTBX/ampNfLU+ox7xipBGiaSbgC0kcB8Gy7BkJB4oKI= - zfec: - required: 3 - segmentSize: 131072 - total: 10 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:MDMF:6kbz7i5wmha5hiil4rdypemuqi:yd2wd25fpa6tlhovmzw7sw7hzdurseapr66uuft6dc57a2sxlbia - format: mdmf - sample: - length: 2097153 - seed: BTBX/ampNfLU+ox7xipBGiaSbgC0kcB8Gy7BkJB4oKI= - zfec: - required: 3 - segmentSize: 131072 - total: 10 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== expected: URI:CHK:tyt4hfvn44igztqh5zeiusggcm:ex3gzzn7byvhajithwbo7c3p7qwqxmnactaxxsxsxowkafyjrf7q:3:10:8388607 format: chk @@ -1629,26 +549,6 @@ vector: required: 3 segmentSize: 131072 total: 10 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:SSK:zufl4t2imcsu7v72ajasbpgw6q:hm6kgc6zunfcmgf64nne3tcaonrgd4fvyuuavo3nq6g33rc3ttua - format: sdmf - sample: - length: 8388607 - seed: w6uP8Tcg6K2QR905Rms8iXTlksL6OD1KOWBxTK7wxPI= - zfec: - required: 3 - segmentSize: 131072 - total: 10 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:MDMF:ds6otzysc6jsfv7oz3yaw7ha5y:a24cpkrdpsx4rqyg2wq42i7lxpm3lsw677qe2wktss3r57cpmrwq - format: mdmf - sample: - length: 8388607 - seed: w6uP8Tcg6K2QR905Rms8iXTlksL6OD1KOWBxTK7wxPI= - zfec: - required: 3 - segmentSize: 131072 - total: 10 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== expected: URI:CHK:xymheose4rdlspgydkzr4nqkre:z3pfrvpq5fdpkoybhdxppwbzrt6ejf26xh6emzlce2sgquljginq:3:10:8388609 format: chk @@ -1659,26 +559,6 @@ vector: required: 3 segmentSize: 131072 total: 10 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:SSK:uuge2bhk3xgcravcofl5pjsvie:geu3qejgjdmz6gylzgua3xnpa5dbpie6ld67ai2jswwpz4d6ioea - format: sdmf - sample: - length: 8388609 - seed: yPi3JHKKbWaEEG5eZOlM6BHJll0Z3UTdBzz4bPQ7wjg= - zfec: - required: 3 - segmentSize: 131072 - total: 10 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:MDMF:hhpclg55lfzmzzdqgnj7os7tda:26l4i2iniero76up73gcih3ksdqlfff3ztbtc3qekzzfumochsyq - format: mdmf - sample: - length: 8388609 - seed: yPi3JHKKbWaEEG5eZOlM6BHJll0Z3UTdBzz4bPQ7wjg= - zfec: - required: 3 - segmentSize: 131072 - total: 10 - convergence: ZOyIygCyaOW6GjVnihtTFg== expected: URI:CHK:x55owxzhsezfoayaxe7jpwnove:vawdgtqpxyntgy5i2po2twgelrynkfcjgwm7publnlbdp7hpqmfa:3:10:1024 format: chk @@ -1689,26 +569,6 @@ vector: required: 3 segmentSize: 131072 total: 10 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:SSK:r62womj66zogjch4dhh7e5egsu:sgg7qnu3vqcr2bjds44rc2ogvm7oy42og76icyq7bztjvtnhhiba - format: sdmf - sample: - length: 1024 - seed: YQ== - zfec: - required: 3 - segmentSize: 131072 - total: 10 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:MDMF:6zfoxkfxatfndxvk3facte3a6m:armxhzyb7dw5gi4re7hlf3dt7pr3qnreowpffieqkfuhafz6htzq - format: mdmf - sample: - length: 1024 - seed: YQ== - zfec: - required: 3 - segmentSize: 131072 - total: 10 - convergence: ZOyIygCyaOW6GjVnihtTFg== expected: URI:CHK:fe64krzyaeff3d4teunjbetkzy:27hrywwaffqiqcgfkmzwbot3iamotr3bey2l5kaladmdmxuaz5ka:3:10:4096 format: chk @@ -1719,26 +579,6 @@ vector: required: 3 segmentSize: 131072 total: 10 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:SSK:ycx4fggszh3rafrrn4cxk63mdm:tnwwkuhngbsw2edbpcf6foknukx5eclklr7bmvpkqtiavcr7sq6q - format: sdmf - sample: - length: 4096 - seed: Yw== - zfec: - required: 3 - segmentSize: 131072 - total: 10 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:MDMF:zxko7c2kaa55krevcnwroqh75a:sjxf56lafuvvuhxhz66c6hf2sufqbo6as43bnanr2qf5s5d3gr6q - format: mdmf - sample: - length: 4096 - seed: Yw== - zfec: - required: 3 - segmentSize: 131072 - total: 10 - convergence: ZOyIygCyaOW6GjVnihtTFg== expected: URI:CHK:gvajllsonkuscfemygbnqhq2re:uwyilm5a7so4blhsaielnf34u2qbaqmudd73opjkgodgg3okeaga:3:10:131071 format: chk @@ -1749,26 +589,6 @@ vector: required: 3 segmentSize: 131072 total: 10 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:SSK:yquetovseokj3fuoyjnf65msze:5qkv72qjjpj2udgwd54wvm3zx7j6w5iqxnvmip4l237sttefjgia - format: sdmf - sample: - length: 131071 - seed: LCa0a2j/xo/5m0U8HTBBNBNCLXBkg7+g+YpeiGJm564= - zfec: - required: 3 - segmentSize: 131072 - total: 10 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:MDMF:mmaxw53ou3sctw2f2wbcrvtokm:gw6y5hl6pk2nnthguuso26olkxctjse7d455xldew53ggkblhniq - format: mdmf - sample: - length: 131071 - seed: LCa0a2j/xo/5m0U8HTBBNBNCLXBkg7+g+YpeiGJm564= - zfec: - required: 3 - segmentSize: 131072 - total: 10 - convergence: ZOyIygCyaOW6GjVnihtTFg== expected: URI:CHK:zdmicwopo4p4h4wbfcbnwcrvyi:6qn75anpvs5gls27f4lybisis3udvjfjhatxiny7c72bcbtuztia:3:10:131073 format: chk @@ -1779,26 +599,6 @@ vector: required: 3 segmentSize: 131072 total: 10 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:SSK:233n2j6rvhem4zrie57ab54dmu:ncmydfq73cyq66wegrdo4a36pmzjpvlbrielye44nb2cpc7atxwq - format: sdmf - sample: - length: 131073 - seed: /N4rLtula/QIYB+3If6bXDONEO5CnqBPrlURto+/j7k= - zfec: - required: 3 - segmentSize: 131072 - total: 10 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:MDMF:l3lh5fxgpco7fq4to7j5b55nva:ut3x2b2hnl4nae6oabrvfomkzkb7b26rohze6xqb522safyetwuq - format: mdmf - sample: - length: 131073 - seed: /N4rLtula/QIYB+3If6bXDONEO5CnqBPrlURto+/j7k= - zfec: - required: 3 - segmentSize: 131072 - total: 10 - convergence: ZOyIygCyaOW6GjVnihtTFg== expected: URI:CHK:bv6qthmetlhdnwc5tfjqamp3yq:ehz4ttd4g7ktkxvbovt562wfedc6jgnt5c6af7wxgp7jbwfwhoaa:3:10:2097151 format: chk @@ -1809,26 +609,6 @@ vector: required: 3 segmentSize: 131072 total: 10 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:SSK:cjjj6zrxophjoshbkndzr6f4pu:pul7pnc7ann3z3bkob4gpb7vkj3brwhvprw7o76467jukwmgkw7q - format: sdmf - sample: - length: 2097151 - seed: uqWglk0zIPvAxqkiFARTyFE+okq4/QV3A0gEqWckgJY= - zfec: - required: 3 - segmentSize: 131072 - total: 10 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:MDMF:j7dbexypbpityjrq6gdh4sqrqy:vzczmef6su6javwr22wfxkfegwnymh2g3de4gplbtveobfjjrdfa - format: mdmf - sample: - length: 2097151 - seed: uqWglk0zIPvAxqkiFARTyFE+okq4/QV3A0gEqWckgJY= - zfec: - required: 3 - segmentSize: 131072 - total: 10 - convergence: ZOyIygCyaOW6GjVnihtTFg== expected: URI:CHK:n7ogyjbo5jigvxgel5ll6q4vbe:3yjb3zq5hcdavv7ruefawal6euyvjx3lx7quslvasjellv63cxya:3:10:2097153 format: chk @@ -1839,26 +619,6 @@ vector: required: 3 segmentSize: 131072 total: 10 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:SSK:3yt7vo7ddtlnasybhdylqzgc6e:2gqcvr6xkrxmnk3xcxbukja2po5wzipn6mfty3527vxja2zkzrnq - format: sdmf - sample: - length: 2097153 - seed: BTBX/ampNfLU+ox7xipBGiaSbgC0kcB8Gy7BkJB4oKI= - zfec: - required: 3 - segmentSize: 131072 - total: 10 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:MDMF:gpkovok4tnkbrfffaywmtio7fu:jt4bbegmwqnbhixwdb7xuo6ae5izwq6y7bmttrpbpx4ox6n4tqgq - format: mdmf - sample: - length: 2097153 - seed: BTBX/ampNfLU+ox7xipBGiaSbgC0kcB8Gy7BkJB4oKI= - zfec: - required: 3 - segmentSize: 131072 - total: 10 - convergence: ZOyIygCyaOW6GjVnihtTFg== expected: URI:CHK:d2hbvcmbex7fm3qu22yj4qnkh4:2xwqxbawwgn773hht6etox3oypvqqjv2orktnthfo2e7vibko7ha:3:10:8388607 format: chk @@ -1869,26 +629,6 @@ vector: required: 3 segmentSize: 131072 total: 10 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:SSK:sczqxshv5qg7dijhrfln6unia4:y5c3qv45mz3namk4yiezidym4wsajseyboxwf7qqin3cmov75wuq - format: sdmf - sample: - length: 8388607 - seed: w6uP8Tcg6K2QR905Rms8iXTlksL6OD1KOWBxTK7wxPI= - zfec: - required: 3 - segmentSize: 131072 - total: 10 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:MDMF:lmoglq3yl6j3wz4ook34unismu:jfleiqdmoyfyvc63etssojyvjfsbptg7q3dvhi25bxlwe2lfnxxa - format: mdmf - sample: - length: 8388607 - seed: w6uP8Tcg6K2QR905Rms8iXTlksL6OD1KOWBxTK7wxPI= - zfec: - required: 3 - segmentSize: 131072 - total: 10 - convergence: ZOyIygCyaOW6GjVnihtTFg== expected: URI:CHK:ccxkyfl2qtqyhihduarpxbdcci:e64be3i2t25selbpc5y2zj443gkdo65chs2o4tpqb7axud5lnxta:3:10:8388609 format: chk @@ -1899,26 +639,6 @@ vector: required: 3 segmentSize: 131072 total: 10 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:SSK:sdh5twhpxdz6hu6ikhnywsmv2q:iphfqmziwxm2vq7wzemqld6rxp2quaz7y2ecnkpt4d4lojdt6dna - format: sdmf - sample: - length: 8388609 - seed: yPi3JHKKbWaEEG5eZOlM6BHJll0Z3UTdBzz4bPQ7wjg= - zfec: - required: 3 - segmentSize: 131072 - total: 10 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:MDMF:j3t75ilecnew72pxtyluc3a7s4:f4vhqdtxzg2ybfs3j5gizewu5bjncgut7kszftd7hk4cm6o6q5ta - format: mdmf - sample: - length: 8388609 - seed: yPi3JHKKbWaEEG5eZOlM6BHJll0Z3UTdBzz4bPQ7wjg= - zfec: - required: 3 - segmentSize: 131072 - total: 10 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== expected: URI:CHK:2wvrzaobtfkoiqeqwryjzvxatq:7zzaycbdhepzaqtdrku2sbb572h64ggwstu47osa2gn4ol3eglmq:71:255:1024 format: chk @@ -1929,26 +649,6 @@ vector: required: 71 segmentSize: 131072 total: 255 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:SSK:hcekwnkw6qfllqmr3zy6g5j62u:2xzhdbgle6pvey5hjcanm5jtwkhtseslmy5jyamdxw7wr45okyiq - format: sdmf - sample: - length: 1024 - seed: YQ== - zfec: - required: 71 - segmentSize: 131072 - total: 255 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:MDMF:lcolwznisx6sore75biky3refa:4r4rnx5sbml3gg6z7bex6ytg6apwtmko6tujcqga4pr4cufuxmoq - format: mdmf - sample: - length: 1024 - seed: YQ== - zfec: - required: 71 - segmentSize: 131072 - total: 255 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== expected: URI:CHK:7wi5xn2uwerkjzqs5ndn4eimti:llrws4uig2fpi3rtla6lk7m4t6zudjhszdfjgqtihghfrvs2robq:71:255:4096 format: chk @@ -1959,26 +659,6 @@ vector: required: 71 segmentSize: 131072 total: 255 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:SSK:slphep77cvx6qw4d5qvliwa4b4:epchbi3vvkzgktlgizmfenwz7iu4cb2thyyrru2t4sm644fthkna - format: sdmf - sample: - length: 4096 - seed: Yw== - zfec: - required: 71 - segmentSize: 131072 - total: 255 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:MDMF:2t6yhzbskevam2brq7nirnptq4:dftxzmiizqrh737bny3nj3f2l3u36w4d7yyut43dynx2jeqtmvqq - format: mdmf - sample: - length: 4096 - seed: Yw== - zfec: - required: 71 - segmentSize: 131072 - total: 255 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== expected: URI:CHK:6nvs23bhrpiwiz5prqdtvztujy:wlg7g522rpdoitpm4qwhmctrjhnh4zfloiq6uq4tsvaoawg4slpq:71:255:131071 format: chk @@ -1989,26 +669,6 @@ vector: required: 71 segmentSize: 131072 total: 255 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:SSK:5od2qv3xadhzghtg76uogaqjnu:fcab6vjvpe25dorw5brgqpa64p3zsxemfer7gzkp56aprbrsrwwa - format: sdmf - sample: - length: 131071 - seed: LCa0a2j/xo/5m0U8HTBBNBNCLXBkg7+g+YpeiGJm564= - zfec: - required: 71 - segmentSize: 131072 - total: 255 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:MDMF:6yga2mq25l6hxha4z3bt5ob2ny:h3cxanvsutketgqw6qkmeugugfxn5gzn4jzz3dykoi727ok7757q - format: mdmf - sample: - length: 131071 - seed: LCa0a2j/xo/5m0U8HTBBNBNCLXBkg7+g+YpeiGJm564= - zfec: - required: 71 - segmentSize: 131072 - total: 255 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== expected: URI:CHK:62dtbeowncjj62jnwbggaufvbu:6r2sapg2cm6dvmylodyxabrj63a736uouzsqyacnimgo4svnktva:71:255:131073 format: chk @@ -2019,26 +679,6 @@ vector: required: 71 segmentSize: 131072 total: 255 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:SSK:grht3kifmiwg2fqtokxrkcklz4:k4gz5cy7kqa7niyj5ighz6q7pkztbfimgdux4omuu2ljjdxglwia - format: sdmf - sample: - length: 131073 - seed: /N4rLtula/QIYB+3If6bXDONEO5CnqBPrlURto+/j7k= - zfec: - required: 71 - segmentSize: 131072 - total: 255 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:MDMF:2kzk7tyx23b3too7ejuybpudxm:5l2cwxkxg4tqlzlmctbi4ogbdyrbj762mvsaz4cq7lkwbhnjtlza - format: mdmf - sample: - length: 131073 - seed: /N4rLtula/QIYB+3If6bXDONEO5CnqBPrlURto+/j7k= - zfec: - required: 71 - segmentSize: 131072 - total: 255 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== expected: URI:CHK:hk3a5ync7y3dnwowqjqoa34eae:y4f5b2xqa3lslh2vti5fdbho2syqhbj2p6f3enlhxsjbfpt7zuta:71:255:2097151 format: chk @@ -2049,26 +689,6 @@ vector: required: 71 segmentSize: 131072 total: 255 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:SSK:nn4ojnd7snzrb72qetwohd4cde:lqp6djtltpnppkfy4uhjx76h6zsglverablp3cng4xllzh7wghcq - format: sdmf - sample: - length: 2097151 - seed: uqWglk0zIPvAxqkiFARTyFE+okq4/QV3A0gEqWckgJY= - zfec: - required: 71 - segmentSize: 131072 - total: 255 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:MDMF:ogpjvm26q6pe5akks3idggipbm:q5sw3t7h6statgllhfsthowhc6op5jdpdfe4cx2pvwhscoewjada - format: mdmf - sample: - length: 2097151 - seed: uqWglk0zIPvAxqkiFARTyFE+okq4/QV3A0gEqWckgJY= - zfec: - required: 71 - segmentSize: 131072 - total: 255 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== expected: URI:CHK:qa4hwhwtd3cpvut74biixlv6ye:uuj4ujg3mcf3lqpuvvwf5pszcvviyb4cm2jjicu7ugiypojjie4q:71:255:2097153 format: chk @@ -2079,26 +699,6 @@ vector: required: 71 segmentSize: 131072 total: 255 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:SSK:icuadu2dbhfheefmqrm2mayg3q:wf5kszmayp6vd2t6hnpb4npqofomznorzlhtzvsbg7xtpxqahona - format: sdmf - sample: - length: 2097153 - seed: BTBX/ampNfLU+ox7xipBGiaSbgC0kcB8Gy7BkJB4oKI= - zfec: - required: 71 - segmentSize: 131072 - total: 255 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:MDMF:z72yysftl3jicwhnhcsm62jdje:w5fp2ag7jvyazv3sm23qlooopmxvgfzotj7wwomd2z2svy6hy76a - format: mdmf - sample: - length: 2097153 - seed: BTBX/ampNfLU+ox7xipBGiaSbgC0kcB8Gy7BkJB4oKI= - zfec: - required: 71 - segmentSize: 131072 - total: 255 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== expected: URI:CHK:r7sv6u3ihzys754twjltuzzpza:tpyoxinubkl4kvg6hipls6ezun7t3yyqj6ix3vp6tvnzqirzzl4a:71:255:8388607 format: chk @@ -2109,26 +709,6 @@ vector: required: 71 segmentSize: 131072 total: 255 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:SSK:6gxdkh5q5bgygex4seqdde4sai:7idrfdnijmiagjt2reroab32tiw6izdywnnpjxu3m7be5226nygq - format: sdmf - sample: - length: 8388607 - seed: w6uP8Tcg6K2QR905Rms8iXTlksL6OD1KOWBxTK7wxPI= - zfec: - required: 71 - segmentSize: 131072 - total: 255 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:MDMF:ecfyk2zk7ojl37qyme67pylsn4:7w4ga3fulb5dyolujq56f5jsbifwxc5lk32z5c6siq3rs2dlbnsa - format: mdmf - sample: - length: 8388607 - seed: w6uP8Tcg6K2QR905Rms8iXTlksL6OD1KOWBxTK7wxPI= - zfec: - required: 71 - segmentSize: 131072 - total: 255 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== expected: URI:CHK:3esfsjn7csgnyqmq5afbgtinay:xsrhdomrbrtzftg6u4hgipm5tkaomumbdlxi5hpvpvawe3bjikua:71:255:8388609 format: chk @@ -2139,26 +719,6 @@ vector: required: 71 segmentSize: 131072 total: 255 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:SSK:kfjuhoyehy4t7ebwgewj4redpm:awgxuay5brzn7aupt7r2jjwk5mhoedmgrd2qaszyhdx4fpaet35q - format: sdmf - sample: - length: 8388609 - seed: yPi3JHKKbWaEEG5eZOlM6BHJll0Z3UTdBzz4bPQ7wjg= - zfec: - required: 71 - segmentSize: 131072 - total: 255 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:MDMF:au3qa6mx56pgi7ouxtscajmak4:qg3s3ewcl754onl5xyzotbbgfia7vojar3m25ac7nv4wgolvibya - format: mdmf - sample: - length: 8388609 - seed: yPi3JHKKbWaEEG5eZOlM6BHJll0Z3UTdBzz4bPQ7wjg= - zfec: - required: 71 - segmentSize: 131072 - total: 255 - convergence: ZOyIygCyaOW6GjVnihtTFg== expected: URI:CHK:wxmzfzvgaqm3xfzgaqrlndgolm:6t3tqcphsutigxnnzyp2xx3afklcpeytnh543mpqptquvqqahwea:71:255:1024 format: chk @@ -2169,26 +729,6 @@ vector: required: 71 segmentSize: 131072 total: 255 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:SSK:hkyaycbxeyhwifvygulnlqx3r4:zwck3zce5brcuzp3zvpc6evfxifppj5nl4jbw7tb2anbzis3jqha - format: sdmf - sample: - length: 1024 - seed: YQ== - zfec: - required: 71 - segmentSize: 131072 - total: 255 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:MDMF:ssbx7xhxjr24tygmvigbkw2tgq:du7u4ayfueljiftdfo5snzlz3aqcf6xwt5nrw4zpopqlcfgteuda - format: mdmf - sample: - length: 1024 - seed: YQ== - zfec: - required: 71 - segmentSize: 131072 - total: 255 - convergence: ZOyIygCyaOW6GjVnihtTFg== expected: URI:CHK:r3q3q4f4zbsipxauq6knb5dfoq:lkae5mflammzxkoc7gllt6qqyg5gqdxeejmxtgcscblqh5tv5obq:71:255:4096 format: chk @@ -2199,26 +739,6 @@ vector: required: 71 segmentSize: 131072 total: 255 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:SSK:2sip2pgg5xmeaqujp37qwqnhlm:jppsyllwrbfalet4lkhi3r5r74355bzquajkp2wzgu4us5ko6h4q - format: sdmf - sample: - length: 4096 - seed: Yw== - zfec: - required: 71 - segmentSize: 131072 - total: 255 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:MDMF:calpjzlexgja7qk5d72wyeybqm:3upvwrkbbi7p5453sbs5wuie7hwxv6moczcb55pqkmfowlwehbhq - format: mdmf - sample: - length: 4096 - seed: Yw== - zfec: - required: 71 - segmentSize: 131072 - total: 255 - convergence: ZOyIygCyaOW6GjVnihtTFg== expected: URI:CHK:rdrzetasw4w7tuweqpev65d5vq:gzzb2v5fzrduk6kx5xx27mvo6dgnd65ym5lm7re53in7xuudhsxa:71:255:131071 format: chk @@ -2229,26 +749,6 @@ vector: required: 71 segmentSize: 131072 total: 255 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:SSK:m52mezcwt462pv7crh57xch5ka:eskrgchqwjldpyvgjit2cv56xd55gra3ejiyoms4wegfhw64bmea - format: sdmf - sample: - length: 131071 - seed: LCa0a2j/xo/5m0U8HTBBNBNCLXBkg7+g+YpeiGJm564= - zfec: - required: 71 - segmentSize: 131072 - total: 255 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:MDMF:jlknqnzgwcb426mudiclobjejy:tmj2no6czfzaqa4x6qolw7ysegnbq4p57jc665mrvrbrwxigtkoa - format: mdmf - sample: - length: 131071 - seed: LCa0a2j/xo/5m0U8HTBBNBNCLXBkg7+g+YpeiGJm564= - zfec: - required: 71 - segmentSize: 131072 - total: 255 - convergence: ZOyIygCyaOW6GjVnihtTFg== expected: URI:CHK:e2bjefibvz4tgu6jgf66gw74bq:mugtu5bx7h5gcivevmh2gmwoc2kkhmobrzshkuj2dgrtm3siysfq:71:255:131073 format: chk @@ -2259,26 +759,6 @@ vector: required: 71 segmentSize: 131072 total: 255 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:SSK:r7zizt5ikfo4ubjxdmtjwap2ii:gmbwvgjnjk4qilg36okgw22hlhfico73e64wknay4aharfonevzq - format: sdmf - sample: - length: 131073 - seed: /N4rLtula/QIYB+3If6bXDONEO5CnqBPrlURto+/j7k= - zfec: - required: 71 - segmentSize: 131072 - total: 255 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:MDMF:3gbqsb33r6zm2lmh246hwycwru:6yuzbmsxuqaqqi3pg4u3q5eiihpfwrteztfub3xyoa7uzcixubxa - format: mdmf - sample: - length: 131073 - seed: /N4rLtula/QIYB+3If6bXDONEO5CnqBPrlURto+/j7k= - zfec: - required: 71 - segmentSize: 131072 - total: 255 - convergence: ZOyIygCyaOW6GjVnihtTFg== expected: URI:CHK:gy7bci6yvllxzvhh45khcwp3u4:qsfjfmcl64zey4k4o5cfnh2i3mzboahf7bhtggceszrulsv537vq:71:255:2097151 format: chk @@ -2289,26 +769,6 @@ vector: required: 71 segmentSize: 131072 total: 255 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:SSK:3cvm2bawfchcqvrbu2f4h47zra:rhqo4ecqmidnwhp3b5v62kipf55wqt6x3nwtmwnvcjt4ytwnop4a - format: sdmf - sample: - length: 2097151 - seed: uqWglk0zIPvAxqkiFARTyFE+okq4/QV3A0gEqWckgJY= - zfec: - required: 71 - segmentSize: 131072 - total: 255 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:MDMF:kkigaypv6feshol6sfug2xnsmy:dd2ll6ga2wgqhlcdl4nvx4m42rs6a3cqcnxglpy64skvlyq5yira - format: mdmf - sample: - length: 2097151 - seed: uqWglk0zIPvAxqkiFARTyFE+okq4/QV3A0gEqWckgJY= - zfec: - required: 71 - segmentSize: 131072 - total: 255 - convergence: ZOyIygCyaOW6GjVnihtTFg== expected: URI:CHK:pcta7uk5cpioxzv5nxxqsogw3q:gnu7fx56k3j72bbevirn4kdu27yfpvttzl3qk2zypwqt7tw4rijq:71:255:2097153 format: chk @@ -2319,26 +779,6 @@ vector: required: 71 segmentSize: 131072 total: 255 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:SSK:hnc74in4qv6bpgmydlzsenhaf4:ysrykdwa6jn2gngdqh56x3jdhor2nsamyjd6b4aqtutzbtdjkdba - format: sdmf - sample: - length: 2097153 - seed: BTBX/ampNfLU+ox7xipBGiaSbgC0kcB8Gy7BkJB4oKI= - zfec: - required: 71 - segmentSize: 131072 - total: 255 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:MDMF:hilmjv7cb2w47njz3znintqueq:hl2jtn5uaujxqwfvexvwfvkfo5iliexcnwzyjt6etyjsc6advsja - format: mdmf - sample: - length: 2097153 - seed: BTBX/ampNfLU+ox7xipBGiaSbgC0kcB8Gy7BkJB4oKI= - zfec: - required: 71 - segmentSize: 131072 - total: 255 - convergence: ZOyIygCyaOW6GjVnihtTFg== expected: URI:CHK:znxyqo2qglpkhjafnuesu4foge:trau477bj5u5wjfpy4oq5e3aziajgbpyfjdv4svyf27m7atpa34a:71:255:8388607 format: chk @@ -2349,26 +789,6 @@ vector: required: 71 segmentSize: 131072 total: 255 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:SSK:xxc3ireoguv7dkgubcjicnxt2q:lp3rg6vawrypxldd4x7geajtzw6ns7yhe7zno4obuottc6bym6da - format: sdmf - sample: - length: 8388607 - seed: w6uP8Tcg6K2QR905Rms8iXTlksL6OD1KOWBxTK7wxPI= - zfec: - required: 71 - segmentSize: 131072 - total: 255 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:MDMF:ii23y7sh2zcjk5qy75ytened4m:k7of2k6pmqjthqaql6hw5t22s3v67n5jpvqb5nvq53rxbphuanqa - format: mdmf - sample: - length: 8388607 - seed: w6uP8Tcg6K2QR905Rms8iXTlksL6OD1KOWBxTK7wxPI= - zfec: - required: 71 - segmentSize: 131072 - total: 255 - convergence: ZOyIygCyaOW6GjVnihtTFg== expected: URI:CHK:mlzs2hbztak5fxjkrkuuvnpdpe:3myvisewm5uklimp2xucrwep75sm2rizfi2sq5drhlqjszkpdyeq:71:255:8388609 format: chk @@ -2379,26 +799,6 @@ vector: required: 71 segmentSize: 131072 total: 255 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:SSK:z47oxnf52xbyyzz2jsx6injuxi:twr773m3memiqykwwhfapujfeyki3i3lnrwrxif5spatnb5lzx2q - format: sdmf - sample: - length: 8388609 - seed: yPi3JHKKbWaEEG5eZOlM6BHJll0Z3UTdBzz4bPQ7wjg= - zfec: - required: 71 - segmentSize: 131072 - total: 255 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:MDMF:43ot6bjdwvfzibci4i3xfdow4i:dy526wxcdosiqocz65xxfqyymwtrtfdvu4nxll4mthp46xojosma - format: mdmf - sample: - length: 8388609 - seed: yPi3JHKKbWaEEG5eZOlM6BHJll0Z3UTdBzz4bPQ7wjg= - zfec: - required: 71 - segmentSize: 131072 - total: 255 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== expected: URI:CHK:mftnvh5l3brshwoupvu6kz35cy:sncfokby2tjykbr3zgi3i6lhgerg4kz7fsrslrlxdy7upwkzq6lq:101:256:1024 format: chk @@ -2409,26 +809,6 @@ vector: required: 101 segmentSize: 131072 total: max -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:SSK:4rybfby6ul2szc3aap7mldlbke:23cpk6vnlgv6n4tadfgkooh3rqsut2xuchrkqjw3fcnleihoqd4q - format: sdmf - sample: - length: 1024 - seed: YQ== - zfec: - required: 101 - segmentSize: 131072 - total: max -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:MDMF:dbc3cfcgtkv33y4xrcfzqget2e:3amzzlftmmugjpcb4octrtzi4tzbdcrnenv2kpbufvkcboin4haq - format: mdmf - sample: - length: 1024 - seed: YQ== - zfec: - required: 101 - segmentSize: 131072 - total: max - convergence: YWFhYWFhYWFhYWFhYWFhYQ== expected: URI:CHK:sshu77h6opnto3jnngot3lwzl4:4cjhp6u3i2uzdzrjitje3inlhs6gnlgwc6wgi6vtedc2grgvcpza:101:256:4096 format: chk @@ -2439,26 +819,6 @@ vector: required: 101 segmentSize: 131072 total: max -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:SSK:4kafx6n7k3etff25jfb2mbq62a:ll2njj473qvuv5oif4ro26jgmyljjvbfanuke2tljhmel4rbceja - format: sdmf - sample: - length: 4096 - seed: Yw== - zfec: - required: 101 - segmentSize: 131072 - total: max -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:MDMF:2nbq6g7kipjvtjy5i2jmpjajma:zcqhmcgjll4zc6da73kmlkynxhgm7ppwbvby3fmoey75gfwys3ua - format: mdmf - sample: - length: 4096 - seed: Yw== - zfec: - required: 101 - segmentSize: 131072 - total: max - convergence: YWFhYWFhYWFhYWFhYWFhYQ== expected: URI:CHK:kl2jayrstlf5q3gax7o65ybivy:mnn5doti4wllssdg35ymojuiv4hd2ir7i6asurgms3ea67watm7a:101:256:131071 format: chk @@ -2469,26 +829,6 @@ vector: required: 101 segmentSize: 131072 total: max -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:SSK:jlmmbswo4xbiib3bc2hdlajacy:uqvabifbvq5di2r3dbvc7bqmnbv5aqldxxrmduuhjun35dotmw2a - format: sdmf - sample: - length: 131071 - seed: LCa0a2j/xo/5m0U8HTBBNBNCLXBkg7+g+YpeiGJm564= - zfec: - required: 101 - segmentSize: 131072 - total: max -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:MDMF:cuogxv55hdieeepwl7i6ce6taa:u755dpuna763w5uaw5lmqixmsxov66c4fvb46tduiszko3m7iq3a - format: mdmf - sample: - length: 131071 - seed: LCa0a2j/xo/5m0U8HTBBNBNCLXBkg7+g+YpeiGJm564= - zfec: - required: 101 - segmentSize: 131072 - total: max - convergence: YWFhYWFhYWFhYWFhYWFhYQ== expected: URI:CHK:a6wuekh5mynr4c74ma3b2gswfa:b6lovwnbv3523n5kaxga3ruxa2dvxef6h4a2jnl6lasn4fwvvdba:101:256:131073 format: chk @@ -2499,26 +839,6 @@ vector: required: 101 segmentSize: 131072 total: max -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:SSK:u2bbj2wg23qzervxlfkua5dj2y:p6fo6wlvh7gcdq63ufp47rdiixxtx2yaeg3llg5byxjhzofkacga - format: sdmf - sample: - length: 131073 - seed: /N4rLtula/QIYB+3If6bXDONEO5CnqBPrlURto+/j7k= - zfec: - required: 101 - segmentSize: 131072 - total: max -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:MDMF:fdai3c5ahhbypwczn2sigdpdlm:kw3fmle5qbml4yz77wwrzz3tbvo65r3xdw347qu2ha3gaa2lidgq - format: mdmf - sample: - length: 131073 - seed: /N4rLtula/QIYB+3If6bXDONEO5CnqBPrlURto+/j7k= - zfec: - required: 101 - segmentSize: 131072 - total: max - convergence: YWFhYWFhYWFhYWFhYWFhYQ== expected: URI:CHK:pmuo6vpcodol76sexq7s2ojej4:irp6rxnkgwbn3h4g5fzgjpqgsvhvycxobr3xrk3zr44r64wimgqa:101:256:2097151 format: chk @@ -2529,26 +849,6 @@ vector: required: 101 segmentSize: 131072 total: max -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:SSK:otczmevjiescnpe6rmdi6pugxy:flwoyy4kwbbex65vopjvt7au6ywjo4rgzbzt5v65rulmn2aufzfq - format: sdmf - sample: - length: 2097151 - seed: uqWglk0zIPvAxqkiFARTyFE+okq4/QV3A0gEqWckgJY= - zfec: - required: 101 - segmentSize: 131072 - total: max -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:MDMF:br3wgiraxyrrzr6yxmcqwnwl6i:bd67wszbnw43kl76jjbfp4s5q6d23eswrag2fu3ztplwvfmtbara - format: mdmf - sample: - length: 2097151 - seed: uqWglk0zIPvAxqkiFARTyFE+okq4/QV3A0gEqWckgJY= - zfec: - required: 101 - segmentSize: 131072 - total: max - convergence: YWFhYWFhYWFhYWFhYWFhYQ== expected: URI:CHK:fh4boug3wkciqhkgpusjsjyzbe:sy5lwooxchqu5mrzwr3mgc7qtwfjgmpz2srdmwpaip4iuj3tn4vq:101:256:2097153 format: chk @@ -2559,26 +859,6 @@ vector: required: 101 segmentSize: 131072 total: max -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:SSK:m4vwawgyausecci7ndiz44thde:e27y5aqvmnb532bv55gwrz3r74m2ff34ls2epbrr27weqg6x3una - format: sdmf - sample: - length: 2097153 - seed: BTBX/ampNfLU+ox7xipBGiaSbgC0kcB8Gy7BkJB4oKI= - zfec: - required: 101 - segmentSize: 131072 - total: max -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:MDMF:yrdgo3x6zc7qtu6qhgschwujqy:lpkg3rluntszbvn64lu247pxjg6urbebktxjm2yfvkw22nuzwcuq - format: mdmf - sample: - length: 2097153 - seed: BTBX/ampNfLU+ox7xipBGiaSbgC0kcB8Gy7BkJB4oKI= - zfec: - required: 101 - segmentSize: 131072 - total: max - convergence: YWFhYWFhYWFhYWFhYWFhYQ== expected: URI:CHK:asmjgajvwpwvylxdcf5rm2gyd4:imwtdi4eqq3wm4bvk6npho3ta2fewceqzl4f7wyyabg5i4s6h3mq:101:256:8388607 format: chk @@ -2589,26 +869,6 @@ vector: required: 101 segmentSize: 131072 total: max -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:SSK:utp6f43bkziptssu5waoexpumq:b7ok3lg5qe33gk5s3n6yv5gvz6xh65xpflyhfkyzpwpvec6pqroq - format: sdmf - sample: - length: 8388607 - seed: w6uP8Tcg6K2QR905Rms8iXTlksL6OD1KOWBxTK7wxPI= - zfec: - required: 101 - segmentSize: 131072 - total: max -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:MDMF:qelscz6cozb7oei42jed7jauuy:e67rqvxcxd2gvy6wgc7tjzpeieltl4252wutucvfvzinhybcp2lq - format: mdmf - sample: - length: 8388607 - seed: w6uP8Tcg6K2QR905Rms8iXTlksL6OD1KOWBxTK7wxPI= - zfec: - required: 101 - segmentSize: 131072 - total: max - convergence: YWFhYWFhYWFhYWFhYWFhYQ== expected: URI:CHK:yrznm2uuybfbda6jlj7tqs3yum:zy45lpc7ilfzgltxxqvy55wcokiay2fbbbv2wdvu6ubjluh4gf4a:101:256:8388609 format: chk @@ -2619,26 +879,6 @@ vector: required: 101 segmentSize: 131072 total: max -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:SSK:gvwazbxny7br5jaebq5lm6vkwu:frn54skmpr2v5i62v2cv4o3etglknbubanzgkcz7fjly76zs5bmq - format: sdmf - sample: - length: 8388609 - seed: yPi3JHKKbWaEEG5eZOlM6BHJll0Z3UTdBzz4bPQ7wjg= - zfec: - required: 101 - segmentSize: 131072 - total: max -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:MDMF:ltix4y33d3up5xceqoeru3fh7a:3oijenm7ex3u7bccxtc37j2ur4phqwp2cxpwonmlr36pt6v4siaa - format: mdmf - sample: - length: 8388609 - seed: yPi3JHKKbWaEEG5eZOlM6BHJll0Z3UTdBzz4bPQ7wjg= - zfec: - required: 101 - segmentSize: 131072 - total: max - convergence: ZOyIygCyaOW6GjVnihtTFg== expected: URI:CHK:ihyeqq5peg2pdyw45ct5mmj4ay:2rzhndv6o4knlvo4po6rvbnknlcitasseea3mirqjor5c7quh5ra:101:256:1024 format: chk @@ -2649,26 +889,6 @@ vector: required: 101 segmentSize: 131072 total: max -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:SSK:d46ph6imfcmvrnxbkz27rvphaa:fu7b3wvcrl6lq2gwhjr2lydcayqnojdb2a7nsble7wyah4u2u7ra - format: sdmf - sample: - length: 1024 - seed: YQ== - zfec: - required: 101 - segmentSize: 131072 - total: max -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:MDMF:dxm44lpq67irpvkxtq2kenkuna:4unsmxd4gauloblh66viurs7xcjoegkgxbgbnvvsxbvjejfkgfja - format: mdmf - sample: - length: 1024 - seed: YQ== - zfec: - required: 101 - segmentSize: 131072 - total: max - convergence: ZOyIygCyaOW6GjVnihtTFg== expected: URI:CHK:mvk5f2mghudk7xnknqddhvnl4m:c74myeugtix7e7l6uwybtrdtfselha2qrcc7hhbymqr6brtg4cea:101:256:4096 format: chk @@ -2679,26 +899,6 @@ vector: required: 101 segmentSize: 131072 total: max -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:SSK:kxgfid6gisa6fviwjvtvtitriq:y7p4sbiltsl2zz4iiu4l43e3tcyam7uaqdkyux62axocqrb2jjfa - format: sdmf - sample: - length: 4096 - seed: Yw== - zfec: - required: 101 - segmentSize: 131072 - total: max -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:MDMF:nwpcaxxhzcbflhetu3x4bcm7hi:4fwjthb4ywmuhe4v7d6or64hvywtpuhhkemz6n46jnkwjbotqqxq - format: mdmf - sample: - length: 4096 - seed: Yw== - zfec: - required: 101 - segmentSize: 131072 - total: max - convergence: ZOyIygCyaOW6GjVnihtTFg== expected: URI:CHK:rvytwcc4sbmmkrdwh6b23ckv5u:klk5nqpbv37pr2qavcebcugfnikarjdefjlseo3ru5uv7guai22q:101:256:131071 format: chk @@ -2709,26 +909,6 @@ vector: required: 101 segmentSize: 131072 total: max -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:SSK:hxtguhkn5d7vho5efeezt6txku:rw7t2vg4msj2h5qkkhw7vn6iotfv2xrnqu6sl4en4b34xogpteya - format: sdmf - sample: - length: 131071 - seed: LCa0a2j/xo/5m0U8HTBBNBNCLXBkg7+g+YpeiGJm564= - zfec: - required: 101 - segmentSize: 131072 - total: max -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:MDMF:wqgkje6573uydranrmfzhv5jye:o2fdspbx2eum66zdxaibssqzqor6fcy4gc2snvj4mthu5vah252q - format: mdmf - sample: - length: 131071 - seed: LCa0a2j/xo/5m0U8HTBBNBNCLXBkg7+g+YpeiGJm564= - zfec: - required: 101 - segmentSize: 131072 - total: max - convergence: ZOyIygCyaOW6GjVnihtTFg== expected: URI:CHK:ocqx55dopakqnzyui6q6euz4gm:wklziyoctwrz5zjfg5wfejz4bnaagtg57fgjlspl4ba6ij6n2obq:101:256:131073 format: chk @@ -2739,26 +919,6 @@ vector: required: 101 segmentSize: 131072 total: max -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:SSK:ina6rycsefbdo6uu7vpa5hmmkq:zlfppgohbsmbmq2nkigkt5oqiodwbrokfncngghmy4brz7hhwvua - format: sdmf - sample: - length: 131073 - seed: /N4rLtula/QIYB+3If6bXDONEO5CnqBPrlURto+/j7k= - zfec: - required: 101 - segmentSize: 131072 - total: max -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:MDMF:ursxqckxmvnkiby7euppiw4ve4:muw3hismhi5f6sunwgighiwiumxuhmgfba3etfyfu3ss5ztqnmeq - format: mdmf - sample: - length: 131073 - seed: /N4rLtula/QIYB+3If6bXDONEO5CnqBPrlURto+/j7k= - zfec: - required: 101 - segmentSize: 131072 - total: max - convergence: ZOyIygCyaOW6GjVnihtTFg== expected: URI:CHK:lf6qub3bkdhfozpdis4bmorj7q:zte3u3ohbuom3iqjjx4odrjylcpw5myw4n4kwf4zcaif36vngf6q:101:256:2097151 format: chk @@ -2769,26 +929,6 @@ vector: required: 101 segmentSize: 131072 total: max -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:SSK:cnjidn5qfurjhp7xruq3qmhmwe:c5zsc3iw5l4msqhwig43cihejtbs7gmsw5yxzbbpux7ivf2by7nq - format: sdmf - sample: - length: 2097151 - seed: uqWglk0zIPvAxqkiFARTyFE+okq4/QV3A0gEqWckgJY= - zfec: - required: 101 - segmentSize: 131072 - total: max -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:MDMF:srixdotz4o72bxooz3w4hzn5pq:vgj66wd3f7dktqiexut3643w2aujb6rgkgmmgr4wvsfdm2x6ma5a - format: mdmf - sample: - length: 2097151 - seed: uqWglk0zIPvAxqkiFARTyFE+okq4/QV3A0gEqWckgJY= - zfec: - required: 101 - segmentSize: 131072 - total: max - convergence: ZOyIygCyaOW6GjVnihtTFg== expected: URI:CHK:a2qoww5xx6so5uftwg4msdtrpy:h7hwsjco5i3jkd7icytrtsuzcjvjzlnkr2wwau2tjlj5wehjrokq:101:256:2097153 format: chk @@ -2799,26 +939,6 @@ vector: required: 101 segmentSize: 131072 total: max -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:SSK:yivcybgondauziz7hbc5auarrq:px2j3rmujcek36kzbxreyqjsxeosqu6zkosdfyokapajycr4a7fa - format: sdmf - sample: - length: 2097153 - seed: BTBX/ampNfLU+ox7xipBGiaSbgC0kcB8Gy7BkJB4oKI= - zfec: - required: 101 - segmentSize: 131072 - total: max -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:MDMF:psnlh7cnhyyjvthsftpwsc4mg4:ykaikqm76z2hu5jjf2t4awbc6kow7667sfnbw3iapi6gyuy3rz6a - format: mdmf - sample: - length: 2097153 - seed: BTBX/ampNfLU+ox7xipBGiaSbgC0kcB8Gy7BkJB4oKI= - zfec: - required: 101 - segmentSize: 131072 - total: max - convergence: ZOyIygCyaOW6GjVnihtTFg== expected: URI:CHK:tg5w4m6hi5ezeapuix3lrrbbtq:7kerk5plqpsd2upnole3ifxoegxdux7prazbvzeu2kyilc6idk7a:101:256:8388607 format: chk @@ -2829,26 +949,6 @@ vector: required: 101 segmentSize: 131072 total: max -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:SSK:znaz4yzpisflh6eb6vgimp2hqq:4uza6tbj44ayk3227r4rmo55hnryxlofxzra7skrlxutqp3qg3ra - format: sdmf - sample: - length: 8388607 - seed: w6uP8Tcg6K2QR905Rms8iXTlksL6OD1KOWBxTK7wxPI= - zfec: - required: 101 - segmentSize: 131072 - total: max -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:MDMF:3ukzh2xpaf5cbag3h757kllgke:xrtzgpxtdc4f6mljutdtycduavcf7mq3csotbmkxuwgghw5i32tq - format: mdmf - sample: - length: 8388607 - seed: w6uP8Tcg6K2QR905Rms8iXTlksL6OD1KOWBxTK7wxPI= - zfec: - required: 101 - segmentSize: 131072 - total: max - convergence: ZOyIygCyaOW6GjVnihtTFg== expected: URI:CHK:ioridipksj6o7ph3bcf37vslmi:pzhkwm4ry67ohj6dcs7zwnsjrvfqzam6tuitrns5vuo3uoomau6q:101:256:8388609 format: chk @@ -2859,24 +959,4 @@ vector: required: 101 segmentSize: 131072 total: max -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:SSK:ztsdfp76v62nww6kplddq6eiy4:2hz7dxqp2wfngpra7yidxsfnocmx4iqmffym264cqofhekrcocuq - format: sdmf - sample: - length: 8388609 - seed: yPi3JHKKbWaEEG5eZOlM6BHJll0Z3UTdBzz4bPQ7wjg= - zfec: - required: 101 - segmentSize: 131072 - total: max -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:MDMF:l4b5t44iqpfcmqk5vap6dwkery:pnq25jpqlmesas4nv55hstin2pmy3ybl7yinu2tk2ruwkhx575ia - format: mdmf - sample: - length: 8388609 - seed: yPi3JHKKbWaEEG5eZOlM6BHJll0Z3UTdBzz4bPQ7wjg= - zfec: - required: 101 - segmentSize: 131072 - total: max version: '2023-01-03' diff --git a/integration/vectors.py b/integration/vectors.py index 3224e5c6b..22c5b8830 100644 --- a/integration/vectors.py +++ b/integration/vectors.py @@ -1,18 +1,144 @@ """ A module that loads pre-generated test vectors. -:ivar CHK_PATH: The path of the file containing CHK test vectors. +:ivar DATA_PATH: The path of the file containing test vectors. -:ivar chk: The CHK test vectors. +:ivar capabilities: The CHK test vectors. """ +from __future__ import annotations + +from typing import TextIO +from attrs import frozen from yaml import safe_load from pathlib import Path +from base64 import b64encode, b64decode DATA_PATH: Path = Path(__file__).parent / "test_vectors.yaml" +@frozen +class Sample: + """ + Some instructions for building a long byte string. + + :ivar seed: Some bytes to repeat some times to produce the string. + :ivar length: The length of the desired byte string. + """ + seed: bytes + length: int + +@frozen +class Param: + """ + Some ZFEC parameters. + """ + required: int + total: int + +# CHK have a max of 256 shares. SDMF / MDMF have a max of 255 shares! +# Represent max symbolically and resolve it when we know what format we're +# dealing with. +MAX_SHARES = "max" + +# SDMF and MDMF encode share counts (N and k) into the share itself as an +# unsigned byte. They could have encoded (share count - 1) to fit the full +# range supported by ZFEC into the unsigned byte - but they don't. So 256 is +# inaccessible to those formats and we set the upper bound at 255. +MAX_SHARES_MAP = { + "chk": 256, + "sdmf": 255, + "mdmf": 255, +} + +@frozen +class SeedParam: + """ + Some ZFEC parameters, almost. + + :ivar required: The number of required shares. + + :ivar total: Either the number of total shares or the constant + ``MAX_SHARES`` to indicate that the total number of shares should be + the maximum number supported by the object format. + """ + required: int + total: int | str + + def realize(self, max_total: int) -> Param: + """ + Create a ``Param`` from this object's values, possibly + substituting the given real value for total if necessary. + + :param max_total: The value to use to replace ``MAX_SHARES`` if + necessary. + """ + if self.total == MAX_SHARES: + return Param(self.required, max_total) + return Param(self.required, self.total) + +@frozen +class Case: + """ + Represent one case for which we want/have a test vector. + """ + seed_params: Param + convergence: bytes + seed_data: Sample + fmt: str + + @property + def data(self): + return stretch(self.seed_data.seed, self.seed_data.length) + + @property + def params(self): + return self.seed_params.realize(MAX_SHARES_MAP[self.fmt]) + + +def encode_bytes(b: bytes) -> str: + """ + Base64 encode some bytes to text so they are representable in JSON. + """ + return b64encode(b).decode("ascii") + + +def decode_bytes(b: str) -> bytes: + """ + Base64 decode some text to bytes. + """ + return b64decode(b.encode("ascii")) + + +def stretch(seed: bytes, size: int) -> bytes: + """ + Given a simple description of a byte string, return the byte string + itself. + """ + assert isinstance(seed, bytes) + assert isinstance(size, int) + assert size > 0 + assert len(seed) > 0 + + multiples = size // len(seed) + 1 + return (seed * multiples)[:size] + + +def load_capabilities(f: TextIO) -> dict[Case, str]: + data = safe_load(f) + return { + Case( + seed_params=SeedParam(case["zfec"]["required"], case["zfec"]["total"]), + convergence=decode_bytes(case["convergence"]), + seed_data=Sample(decode_bytes(case["sample"]["seed"]), case["sample"]["length"]), + fmt=case["format"], + ): case["expected"] + for case + in data["vector"] + } + + try: with DATA_PATH.open() as f: - capabilities: dict[str, str] = safe_load(f) + capabilities: dict[Case, str] = load_capabilities(f) except FileNotFoundError: capabilities = {} From 746b3ca595f5d62650b12752a25804765aa175b5 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 6 Jan 2023 11:15:59 -0500 Subject: [PATCH 1328/2309] Document adding latency. --- benchmarks/upload_download.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/benchmarks/upload_download.py b/benchmarks/upload_download.py index bd1b08e7a..239c141ab 100644 --- a/benchmarks/upload_download.py +++ b/benchmarks/upload_download.py @@ -5,6 +5,14 @@ To run: $ pytest benchmarks/upload_download.py -s -v -Wignore +To add latency of e.g. 60ms on Linux: + +$ tc qdisc add dev lo root netem delay 30ms + +To reset: + +$ tc qdisc del dev lo root netem + TODO Parameterization (pytest?) - Foolscap vs not foolscap - Number of nodes From e236cc95a56ed01dafdc874515ee6ec2727eca83 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 6 Jan 2023 15:36:08 -0500 Subject: [PATCH 1329/2309] Move get_keypair to a shared location --- src/allmydata/web/common.py | 24 ++++++++++++++---------- src/allmydata/web/filenode.py | 17 +---------------- 2 files changed, 15 insertions(+), 26 deletions(-) diff --git a/src/allmydata/web/common.py b/src/allmydata/web/common.py index bf89044a3..2d5fe6297 100644 --- a/src/allmydata/web/common.py +++ b/src/allmydata/web/common.py @@ -1,15 +1,7 @@ """ Ported to Python 3. """ -from __future__ import division -from __future__ import absolute_import -from __future__ import print_function -from __future__ import unicode_literals - -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, max, min # noqa: F401 - from past.builtins import unicode as str # prevent leaking newbytes/newstr into code that can't handle it +from __future__ import annotations from six import ensure_str @@ -21,6 +13,7 @@ except ImportError: import time import json from functools import wraps +from base64 import urlsafe_b64decode from hyperlink import ( DecodedURL, @@ -94,7 +87,7 @@ from allmydata.util.encodingutil import ( to_bytes, ) from allmydata.util import abbreviate - +from allmydata.crypto.rsa import PrivateKey, PublicKey, create_signing_keypair_from_string class WebError(Exception): def __init__(self, text, code=http.BAD_REQUEST): @@ -833,3 +826,14 @@ def abbreviate_time(data): if s >= 0.001: return u"%.1fms" % (1000*s) return u"%.0fus" % (1000000*s) + +def get_keypair(request: IRequest) -> tuple[PublicKey, PrivateKey] | None: + """ + Load a keypair from a urlsafe-base64-encoded RSA private key in the + **private-key** argument of the given request, if there is one. + """ + privkey_der = get_arg(request, "private-key", None) + if privkey_der is None: + return None + privkey, pubkey = create_signing_keypair_from_string(urlsafe_b64decode(privkey_der)) + return pubkey, privkey diff --git a/src/allmydata/web/filenode.py b/src/allmydata/web/filenode.py index 1b0db5045..52ef48e1e 100644 --- a/src/allmydata/web/filenode.py +++ b/src/allmydata/web/filenode.py @@ -3,8 +3,6 @@ Ported to Python 3. """ from __future__ import annotations -from base64 import urlsafe_b64decode - from twisted.web import http, static from twisted.web.iweb import IRequest from twisted.internet import defer @@ -26,6 +24,7 @@ from allmydata.blacklist import ( ) from allmydata.web.common import ( + get_keypair, boolean_of_arg, exception_to_child, get_arg, @@ -47,20 +46,6 @@ from allmydata.web.check_results import ( ) from allmydata.web.info import MoreInfo from allmydata.util import jsonbytes as json -from allmydata.crypto.rsa import PrivateKey, PublicKey, create_signing_keypair_from_string - - -def get_keypair(request: IRequest) -> tuple[PublicKey, PrivateKey] | None: - """ - Load a keypair from a urlsafe-base64-encoded RSA private key in the - **private-key** argument of the given request, if there is one. - """ - privkey_der = get_arg(request, "private-key", None) - if privkey_der is None: - return None - privkey, pubkey = create_signing_keypair_from_string(urlsafe_b64decode(privkey_der)) - return pubkey, privkey - class ReplaceMeMixin(object): def replace_me_with_a_child(self, req, client, replace): From 3ff9c45e95be2a8144ff70e35a1e7e41ea67ded6 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 6 Jan 2023 15:40:48 -0500 Subject: [PATCH 1330/2309] expose the private-key feature in the `tahoe put` cli --- src/allmydata/scripts/cli.py | 15 ++++++- src/allmydata/scripts/tahoe_put.py | 41 +++++++++++------ src/allmydata/test/cli/test_put.py | 72 ++++++++++++++++++++++++++---- src/allmydata/web/filenode.py | 1 - src/allmydata/web/unlinked.py | 13 ++---- 5 files changed, 108 insertions(+), 34 deletions(-) diff --git a/src/allmydata/scripts/cli.py b/src/allmydata/scripts/cli.py index 55975b8c5..aa7644e18 100644 --- a/src/allmydata/scripts/cli.py +++ b/src/allmydata/scripts/cli.py @@ -180,10 +180,21 @@ class GetOptions(FileStoreOptions): class PutOptions(FileStoreOptions): optFlags = [ ("mutable", "m", "Create a mutable file instead of an immutable one (like --format=SDMF)"), - ] + ] + optParameters = [ ("format", None, None, "Create a file with the given format: SDMF and MDMF for mutable, CHK (default) for immutable. (case-insensitive)"), - ] + + ("private-key-path", None, None, + "***Warning*** " + "It is possible to use this option to spoil the normal security properties of mutable objects. " + "It is also possible to corrupt or destroy data with this option. " + "For mutables only, " + "this gives a file containing a PEM-encoded 2048 bit RSA private key to use as the signature key for the mutable. " + "The private key must be handled at least as strictly as the resulting capability string. " + "A single private key must not be used for more than one mutable." + ), + ] def parseArgs(self, arg1=None, arg2=None): # see Examples below diff --git a/src/allmydata/scripts/tahoe_put.py b/src/allmydata/scripts/tahoe_put.py index 1ea45e8ea..fd746a43d 100644 --- a/src/allmydata/scripts/tahoe_put.py +++ b/src/allmydata/scripts/tahoe_put.py @@ -1,23 +1,31 @@ """ -Ported to Python 3. +Implement the ``tahoe put`` command. """ -from __future__ import unicode_literals -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 +from __future__ import annotations from io import BytesIO from urllib.parse import quote as url_quote +from base64 import urlsafe_b64encode +from cryptography.hazmat.primitives.serialization import load_pem_private_key + +from twisted.python.filepath import FilePath + +from allmydata.crypto.rsa import der_string_from_signing_key from allmydata.scripts.common_http import do_http, format_http_success, format_http_error from allmydata.scripts.common import get_alias, DEFAULT_ALIAS, escape_path, \ UnknownAliasError from allmydata.util.encodingutil import quote_output +def load_private_key(path: str) -> str: + """ + Load a private key from a file and return it in a format appropriate + to include in the HTTP request. + """ + privkey = load_pem_private_key(FilePath(path).getContent(), password=None) + derbytes = der_string_from_signing_key(privkey) + return urlsafe_b64encode(derbytes).decode("ascii") + def put(options): """ @param verbosity: 0, 1, or 2, meaning quiet, verbose, or very verbose @@ -29,6 +37,10 @@ def put(options): from_file = options.from_file to_file = options.to_file mutable = options['mutable'] + if options["private-key-path"] is None: + private_key = None + else: + private_key = load_private_key(options["private-key-path"]) format = options['format'] if options['quiet']: verbosity = 0 @@ -79,6 +91,12 @@ def put(options): queryargs = [] if mutable: queryargs.append("mutable=true") + if private_key is not None: + queryargs.append(f"private-key={private_key}") + else: + if private_key is not None: + raise Exception("Can only supply a private key for mutables.") + if format: queryargs.append("format=%s" % format) if queryargs: @@ -92,10 +110,7 @@ def put(options): if verbosity > 0: print("waiting for file data on stdin..", file=stderr) # We're uploading arbitrary files, so this had better be bytes: - if PY2: - stdinb = stdin - else: - stdinb = stdin.buffer + stdinb = stdin.buffer data = stdinb.read() infileobj = BytesIO(data) diff --git a/src/allmydata/test/cli/test_put.py b/src/allmydata/test/cli/test_put.py index 03306ab71..6f33a14fd 100644 --- a/src/allmydata/test/cli/test_put.py +++ b/src/allmydata/test/cli/test_put.py @@ -1,19 +1,17 @@ """ -Ported to Python 3. +Tests for the ``tahoe put`` CLI tool. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals - -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 +from __future__ import annotations +from typing import Callable, Awaitable, TypeVar import os.path from twisted.trial import unittest from twisted.python import usage +from twisted.python.filepath import FilePath +from cryptography.hazmat.primitives.serialization import load_pem_private_key + +from allmydata.uri import from_string from allmydata.util import fileutil from allmydata.scripts.common import get_aliases from allmydata.scripts import cli @@ -22,6 +20,9 @@ from ..common_util import skip_if_cannot_represent_filename from allmydata.util.encodingutil import get_io_encoding from allmydata.util.fileutil import abspath_expanduser_unicode from .common import CLITestMixin +from allmydata.mutable.common import derive_mutable_keys + +T = TypeVar("T") class Put(GridTestMixin, CLITestMixin, unittest.TestCase): @@ -215,6 +216,59 @@ class Put(GridTestMixin, CLITestMixin, unittest.TestCase): return d + async def test_unlinked_mutable_specified_private_key(self) -> None: + """ + A new unlinked mutable can be created using a specified private + key. + """ + self.basedir = "cli/Put/unlinked-mutable-with-key" + await self._test_mutable_specified_key( + lambda do_cli, pempath, datapath: do_cli( + "put", "--mutable", "--private-key-path", pempath.path, + stdin=datapath.getContent(), + ), + ) + + async def test_linked_mutable_specified_private_key(self) -> None: + """ + A new linked mutable can be created using a specified private key. + """ + self.basedir = "cli/Put/linked-mutable-with-key" + await self._test_mutable_specified_key( + lambda do_cli, pempath, datapath: do_cli( + "put", "--mutable", "--private-key-path", pempath.path, datapath.path, + ), + ) + + async def _test_mutable_specified_key( + self, + run: Callable[[Callable[..., T], FilePath, FilePath], Awaitable[T]], + ) -> None: + """ + A helper for testing mutable creation. + + :param run: A function to do the creation. It is called with + ``self.do_cli`` and the path to a private key PEM file and a data + file. It returns whatever ``do_cli`` returns. + """ + self.set_up_grid(oneshare=True) + + pempath = FilePath(__file__).parent().sibling("data").child("openssl-rsa-2048.txt") + datapath = FilePath(self.basedir).child("data") + datapath.setContent(b"Hello world" * 1024) + + (rc, out, err) = await run(self.do_cli, pempath, datapath) + self.assertEqual(rc, 0, (out, err)) + cap = from_string(out.strip()) + # The capability is derived from the key we specified. + privkey = load_pem_private_key(pempath.getContent(), password=None) + pubkey = privkey.public_key() + writekey, _, fingerprint = derive_mutable_keys((pubkey, privkey)) + self.assertEqual( + (writekey, fingerprint), + (cap.writekey, cap.fingerprint), + ) + def test_mutable(self): # echo DATA1 | tahoe put --mutable - uploaded.txt # echo DATA2 | tahoe put - uploaded.txt # should modify-in-place diff --git a/src/allmydata/web/filenode.py b/src/allmydata/web/filenode.py index 52ef48e1e..680ca3331 100644 --- a/src/allmydata/web/filenode.py +++ b/src/allmydata/web/filenode.py @@ -4,7 +4,6 @@ Ported to Python 3. from __future__ import annotations from twisted.web import http, static -from twisted.web.iweb import IRequest from twisted.internet import defer from twisted.web.resource import ( Resource, diff --git a/src/allmydata/web/unlinked.py b/src/allmydata/web/unlinked.py index 425622496..2c7be6f30 100644 --- a/src/allmydata/web/unlinked.py +++ b/src/allmydata/web/unlinked.py @@ -1,14 +1,7 @@ """ Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals - -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 +from __future__ import annotations from urllib.parse import quote as urlquote @@ -25,6 +18,7 @@ from twisted.web.template import ( from allmydata.immutable.upload import FileHandle from allmydata.mutable.publish import MutableFileHandle from allmydata.web.common import ( + get_keypair, get_arg, boolean_of_arg, convert_children_json, @@ -48,7 +42,8 @@ def PUTUnlinkedSSK(req, client, version): # SDMF: files are small, and we can only upload data req.content.seek(0) data = MutableFileHandle(req.content) - d = client.create_mutable_file(data, version=version) + keypair = get_keypair(req) + d = client.create_mutable_file(data, version=version, unique_keypair=keypair) d.addCallback(lambda n: n.get_uri()) return d From 1d125b7be80f4ac3300fffd6e343d41df6debde5 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 6 Jan 2023 15:51:36 -0500 Subject: [PATCH 1331/2309] news fragment --- newsfragments/3962.feature | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/3962.feature diff --git a/newsfragments/3962.feature b/newsfragments/3962.feature new file mode 100644 index 000000000..86cf62781 --- /dev/null +++ b/newsfragments/3962.feature @@ -0,0 +1 @@ +Mutable objects can now be created with a pre-determined "signature key" using the ``tahoe put`` CLI or the HTTP API. This enables deterministic creation of mutable capabilities. This feature must be used with care to preserve the normal security and reliability properties. \ No newline at end of file From e829b891b300dba118a18490cfc74e356b36915c Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 6 Jan 2023 15:51:59 -0500 Subject: [PATCH 1332/2309] important data file ... --- src/allmydata/test/data/openssl-rsa-2048.txt | 28 ++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 src/allmydata/test/data/openssl-rsa-2048.txt diff --git a/src/allmydata/test/data/openssl-rsa-2048.txt b/src/allmydata/test/data/openssl-rsa-2048.txt new file mode 100644 index 000000000..8f989f42c --- /dev/null +++ b/src/allmydata/test/data/openssl-rsa-2048.txt @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDF1MeXulDWFO05 +YXCh8aqNc1dS1ddJRzsti4BOWuDOepUc0oCaSIcC5aR7XJ+vhX7a02mTIwvLcuEH +8sxx0BJU4jCDpRI6aAqaKJxwZx1e6AcVFJDl7vzymhvWhqHuKh0jTvwM2zONWTwV +V8m2PbDdxu0Prwdx+Mt2sDT6xHEhJj5fI/GUDUEdkhLJF6DQSulFRqqd0qP7qcI9 +fSHZbM7MywfzqFUe8J1+tk4fBh2v7gNzN1INpzh2mDtLPAtxr4ZPtEb/0D0U4PsP +CniOHP0U8sF3VY0+K5qoCQr92cLRJvT/vLpQGVNUTFdFrtbqDoFxUCyEH4FUqRDX +2mVrPo2xAgMBAAECggEAA0Ev1y5/1NTPbgytBeIIH3d+v9hwKDbHecVoMwnOVeFJ +BZpONrOToovhAc1NXH2wj4SvwYWfpJ1HR9piDAuLeKlnuUu4ffzfE0gQok4E+v4r +2yg9ZcYBs/NOetAYVwbq960tiv/adFRr71E0WqbfS3fBx8q2L3Ujkkhd98PudUhQ +izbrTvkT7q00OPCWGwgWepMlLEowUWwZehGI0MlbONg7SbRraZZmG586Iy0tpC3e +AM7wC1/ORzFqcRgTIxXizQ5RHL7S0OQPLhbEJbuwPonNjze3p0EP4wNBELZTaVOd +xeA22Py4Bh/d1q3aEgbwR7tLyA8YfEzshTaY6oV8AQKBgQD0uFo8pyWk0AWXfjzn +jV4yYyPWy8pJA6YfAJAST8m7B/JeYgGlfHxTlNZiB40DsJq08tOZv3HAubgMpFIa +reuDxPqo6/Quwdy4Syu+AFhY48KIuwuoegG/L+5qcQLE69r1w71ZV6wUvLmXYX2I +Y6nYz+OdpD1JrMIr6Js60XURsQKBgQDO8yWl7ufIDKMbQpbs0PgUQsH4FtzGcP4J +j/7/8GfhKYt6rPsrojPHUbAi1+25xBVOuhm0Zx2ku2t+xPIMJoS+15EcER1Z2iHZ +Zci9UGpJpUxGcUhG7ETF1HZv0xKHcEOl9eIIOcAP9Vd9DqnGk85gy6ti6MHe/5Tn +IMD36OQ8AQKBgQDwqE7NMM67KnslRNaeG47T3F0FQbm3XehCuqnz6BUJYcI+gQD/ +fdFB3K+LDcPmKgmqAtaGbxdtoPXXMM0xQXHHTrH15rxmMu1dK0dj/TDkkW7gSZko +YHtRSdCbSnGfuBXG9GxD7QzkA8g7j3sE4oXIGoDLqRVAW61DwubMy+jlsQKBgGNB ++Zepi1/Gt+BWQt8YpzPIhRIBnShMf3uEphCJdLlo3K4dE2btKBp8UpeTq0CDDJky +5ytAndYp0jf+K/2p59dEuyOUDdjPp5aGnA446JGkB35tzPW/Uoj0C049FVEChl+u +HBhH4peE285uXv2QXNbOOMh6zKmxOfDVI9iDyhwBAoGBAIXq2Ar0zDXXaL3ncEKo +pXt9BZ8OpJo2pvB1t2VPePOwEQ0wdT+H62fKNY47NiF9+LyS541/ps5Qhv6AmiKJ +Z7I0Vb6+sxQljYH/LNW+wc2T/pIAi/7sNcmnlBtZfoVwt99bk2CyoRALPLWHYCkh +c7Tty2bZzDZy6aCX+FGRt5N/ +-----END PRIVATE KEY----- From 2dc6466ef5c260e7cb5d46cfcbfc456399f29b71 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 6 Jan 2023 17:12:59 -0500 Subject: [PATCH 1333/2309] fix some errors reported by mypy --- src/allmydata/crypto/rsa.py | 6 +++--- src/allmydata/mutable/common.py | 2 +- src/allmydata/test/cli/test_put.py | 4 ++-- src/allmydata/test/common.py | 2 +- src/allmydata/test/web/test_web.py | 1 + src/allmydata/web/common.py | 19 ++++++++++++------- 6 files changed, 20 insertions(+), 14 deletions(-) diff --git a/src/allmydata/crypto/rsa.py b/src/allmydata/crypto/rsa.py index 5acc59ab2..f16b1b95f 100644 --- a/src/allmydata/crypto/rsa.py +++ b/src/allmydata/crypto/rsa.py @@ -13,7 +13,7 @@ on any of their methods. from __future__ import annotations -from typing import TypeVar +from typing_extensions import TypeAlias from cryptography.exceptions import InvalidSignature from cryptography.hazmat.backends import default_backend @@ -24,8 +24,8 @@ from cryptography.hazmat.primitives.serialization import load_der_private_key, l from allmydata.crypto.error import BadSignature -PublicKey = TypeVar("PublicKey", bound=rsa.RSAPublicKey) -PrivateKey = TypeVar("PrivateKey", bound=rsa.RSAPrivateKey) +PublicKey: TypeAlias = rsa.RSAPublicKey +PrivateKey: TypeAlias = rsa.RSAPrivateKey # This is the value that was used by `pycryptopp`, and we must continue to use it for # both backwards compatibility and interoperability. diff --git a/src/allmydata/mutable/common.py b/src/allmydata/mutable/common.py index 33a1c2731..a498ab02a 100644 --- a/src/allmydata/mutable/common.py +++ b/src/allmydata/mutable/common.py @@ -66,7 +66,7 @@ class UnknownVersionError(BadShareError): """The share we received was of a version we don't recognize.""" -def encrypt_privkey(writekey: bytes, privkey: rsa.PrivateKey) -> bytes: +def encrypt_privkey(writekey: bytes, privkey: bytes) -> bytes: """ For SSK, encrypt a private ("signature") key using the writekey. """ diff --git a/src/allmydata/test/cli/test_put.py b/src/allmydata/test/cli/test_put.py index 6f33a14fd..98407bb7e 100644 --- a/src/allmydata/test/cli/test_put.py +++ b/src/allmydata/test/cli/test_put.py @@ -3,7 +3,7 @@ Tests for the ``tahoe put`` CLI tool. """ from __future__ import annotations -from typing import Callable, Awaitable, TypeVar +from typing import Callable, Awaitable, TypeVar, Any import os.path from twisted.trial import unittest from twisted.python import usage @@ -242,7 +242,7 @@ class Put(GridTestMixin, CLITestMixin, unittest.TestCase): async def _test_mutable_specified_key( self, - run: Callable[[Callable[..., T], FilePath, FilePath], Awaitable[T]], + run: Callable[[Any, FilePath, FilePath], Awaitable[tuple[int, bytes, bytes]]], ) -> None: """ A helper for testing mutable creation. diff --git a/src/allmydata/test/common.py b/src/allmydata/test/common.py index 37d390da5..db2921e86 100644 --- a/src/allmydata/test/common.py +++ b/src/allmydata/test/common.py @@ -635,7 +635,7 @@ class FakeMutableFileNode(object): # type: ignore # incomplete implementation keypair: tuple[rsa.PublicKey, rsa.PrivateKey] | None ): self.all_contents = all_contents - self.file_types = {} # storage index => MDMF_VERSION or SDMF_VERSION + self.file_types: dict[bytes, int] = {} # storage index => MDMF_VERSION or SDMF_VERSION self.init_from_cap(make_mutable_file_cap(keypair)) self._k = default_encoding_parameters['k'] self._segsize = default_encoding_parameters['max_segment_size'] diff --git a/src/allmydata/test/web/test_web.py b/src/allmydata/test/web/test_web.py index bb1f27322..7793c023c 100644 --- a/src/allmydata/test/web/test_web.py +++ b/src/allmydata/test/web/test_web.py @@ -90,6 +90,7 @@ class FakeNodeMaker(NodeMaker): 'happy': 7, 'max_segment_size':128*1024 # 1024=KiB } + all_contents: dict[bytes, object] def _create_lit(self, cap): return FakeCHKFileNode(cap, self.all_contents) def _create_immutable(self, cap): diff --git a/src/allmydata/web/common.py b/src/allmydata/web/common.py index 2d5fe6297..25e9e51f3 100644 --- a/src/allmydata/web/common.py +++ b/src/allmydata/web/common.py @@ -6,7 +6,7 @@ from __future__ import annotations from six import ensure_str try: - from typing import Optional, Union, Tuple, Any + from typing import Optional, Union, Tuple, Any, TypeVar except ImportError: pass @@ -706,8 +706,9 @@ def url_for_string(req, url_string): ) return url +T = TypeVar("T") -def get_arg(req, argname, default=None, multiple=False): # type: (IRequest, Union[bytes,str], Any, bool) -> Union[bytes,Tuple[bytes],Any] +def get_arg(req: IRequest, argname: str | bytes, default: T = None, multiple: bool = False) -> Union[bytes, tuple[bytes, ...], T]: """Extract an argument from either the query args (req.args) or the form body fields (req.fields). If multiple=False, this returns a single value (or the default, which defaults to None), and the query args take @@ -719,13 +720,17 @@ def get_arg(req, argname, default=None, multiple=False): # type: (IRequest, Uni :return: Either bytes or tuple of bytes. """ if isinstance(argname, str): - argname = argname.encode("utf-8") + argname_bytes = argname.encode("utf-8") + else: + argname_bytes = argname + if isinstance(default, str): default = default.encode("utf-8") + results = [] - if argname in req.args: - results.extend(req.args[argname]) - argname_unicode = str(argname, "utf-8") + if argname_bytes in req.args: + results.extend(req.args[argname_bytes]) + argname_unicode = str(argname_bytes, "utf-8") if req.fields and argname_unicode in req.fields: value = req.fields[argname_unicode].value if isinstance(value, str): @@ -832,7 +837,7 @@ def get_keypair(request: IRequest) -> tuple[PublicKey, PrivateKey] | None: Load a keypair from a urlsafe-base64-encoded RSA private key in the **private-key** argument of the given request, if there is one. """ - privkey_der = get_arg(request, "private-key", None) + privkey_der = get_arg(request, "private-key", default=None, multiple=False) if privkey_der is None: return None privkey, pubkey = create_signing_keypair_from_string(urlsafe_b64decode(privkey_der)) From a806b2fabaf985a28d9872ea2db292a30c91810f Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 6 Jan 2023 18:11:47 -0500 Subject: [PATCH 1334/2309] Fix some more mypy errors --- src/allmydata/crypto/rsa.py | 22 +++++++++++++++------- src/allmydata/web/common.py | 16 +++++++++++----- 2 files changed, 26 insertions(+), 12 deletions(-) diff --git a/src/allmydata/crypto/rsa.py b/src/allmydata/crypto/rsa.py index 3554ec557..f1ba0d10a 100644 --- a/src/allmydata/crypto/rsa.py +++ b/src/allmydata/crypto/rsa.py @@ -14,6 +14,7 @@ on any of their methods. from __future__ import annotations from typing_extensions import TypeAlias +from typing import Callable from functools import partial @@ -70,22 +71,29 @@ def create_signing_keypair_from_string(private_key_der: bytes) -> tuple[PrivateK :returns: 2-tuple of (private_key, public_key) """ - load = partial( - load_der_private_key, + _load = partial( + load_der_public_key, private_key_der, password=None, backend=default_backend(), ) + def load_with_validation() -> PrivateKey: + return _load() + + def load_without_validation() -> PrivateKey: + return _load(unsafe_skip_rsa_key_validation=True) + + # Load it once without the potentially expensive OpenSSL validation + # checks. These have superlinear complexity. We *will* run them just + # below - but first we'll apply our own constant-time checks. + load: Callable[[], PrivateKey] = load_without_validation try: - # Load it once without the potentially expensive OpenSSL validation - # checks. These have superlinear complexity. We *will* run them just - # below - but first we'll apply our own constant-time checks. - unsafe_priv_key = load(unsafe_skip_rsa_key_validation=True) + unsafe_priv_key = load() except TypeError: # cryptography<39 does not support this parameter, so just load the # key with validation... - unsafe_priv_key = load() + unsafe_priv_key = load_without_validation() # But avoid *reloading* it since that will run the expensive # validation *again*. load = lambda: unsafe_priv_key diff --git a/src/allmydata/web/common.py b/src/allmydata/web/common.py index 25e9e51f3..8f81aec94 100644 --- a/src/allmydata/web/common.py +++ b/src/allmydata/web/common.py @@ -6,7 +6,7 @@ from __future__ import annotations from six import ensure_str try: - from typing import Optional, Union, Tuple, Any, TypeVar + from typing import Optional, Union, Tuple, Any, TypeVar, Literal, overload except ImportError: pass @@ -708,7 +708,13 @@ def url_for_string(req, url_string): T = TypeVar("T") -def get_arg(req: IRequest, argname: str | bytes, default: T = None, multiple: bool = False) -> Union[bytes, tuple[bytes, ...], T]: +@overload +def get_arg(req: IRequest, argname: str | bytes, default: T = None, *, multiple: Literal[False] = False) -> bytes: ... + +@overload +def get_arg(req: IRequest, argname: str | bytes, default: T = None, *, multiple: Literal[True]) -> tuple[bytes, ...]: ... + +def get_arg(req: IRequest, argname: str | bytes, default: T | None = None, *, multiple: bool = False) -> None | T | bytes | tuple[bytes, ...]: """Extract an argument from either the query args (req.args) or the form body fields (req.fields). If multiple=False, this returns a single value (or the default, which defaults to None), and the query args take @@ -724,9 +730,6 @@ def get_arg(req: IRequest, argname: str | bytes, default: T = None, multiple: bo else: argname_bytes = argname - if isinstance(default, str): - default = default.encode("utf-8") - results = [] if argname_bytes in req.args: results.extend(req.args[argname_bytes]) @@ -740,6 +743,9 @@ def get_arg(req: IRequest, argname: str | bytes, default: T = None, multiple: bo return tuple(results) if results: return results[0] + + if isinstance(default, str): + return default.encode("utf-8") return default From c9e23dea13a51114f47c8c64ed1caf34d57cfcdc Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 6 Jan 2023 20:59:48 -0500 Subject: [PATCH 1335/2309] we should always be able to get these and we always need overload now --- src/allmydata/web/common.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/allmydata/web/common.py b/src/allmydata/web/common.py index 8f81aec94..b55a49d4d 100644 --- a/src/allmydata/web/common.py +++ b/src/allmydata/web/common.py @@ -5,10 +5,7 @@ from __future__ import annotations from six import ensure_str -try: - from typing import Optional, Union, Tuple, Any, TypeVar, Literal, overload -except ImportError: - pass +from typing import Optional, Union, TypeVar, Literal, overload import time import json From 85234b07a0ce4f12be3676562a0ed7a24f650d27 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 6 Jan 2023 21:00:04 -0500 Subject: [PATCH 1336/2309] load the right kind of key! --- src/allmydata/crypto/rsa.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/crypto/rsa.py b/src/allmydata/crypto/rsa.py index f1ba0d10a..b8de52c4f 100644 --- a/src/allmydata/crypto/rsa.py +++ b/src/allmydata/crypto/rsa.py @@ -72,7 +72,7 @@ def create_signing_keypair_from_string(private_key_der: bytes) -> tuple[PrivateK :returns: 2-tuple of (private_key, public_key) """ _load = partial( - load_der_public_key, + load_der_private_key, private_key_der, password=None, backend=default_backend(), From 8c56ccad725ebb6c9ae745a60d6258ed0f8b5b2f Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 6 Jan 2023 21:00:10 -0500 Subject: [PATCH 1337/2309] fall back to *with* validation, not without --- src/allmydata/crypto/rsa.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/crypto/rsa.py b/src/allmydata/crypto/rsa.py index b8de52c4f..c21b522cd 100644 --- a/src/allmydata/crypto/rsa.py +++ b/src/allmydata/crypto/rsa.py @@ -93,7 +93,7 @@ def create_signing_keypair_from_string(private_key_der: bytes) -> tuple[PrivateK except TypeError: # cryptography<39 does not support this parameter, so just load the # key with validation... - unsafe_priv_key = load_without_validation() + unsafe_priv_key = load_with_validation() # But avoid *reloading* it since that will run the expensive # validation *again*. load = lambda: unsafe_priv_key From e893d06cb35fd7c32a8682b3f538086c663f30de Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 6 Jan 2023 21:00:21 -0500 Subject: [PATCH 1338/2309] RSAPrivateKey certainly does have this method I don't know why mypy fails to see it. --- src/allmydata/crypto/rsa.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/crypto/rsa.py b/src/allmydata/crypto/rsa.py index c21b522cd..3ad893dbf 100644 --- a/src/allmydata/crypto/rsa.py +++ b/src/allmydata/crypto/rsa.py @@ -123,7 +123,7 @@ def der_string_from_signing_key(private_key: PrivateKey) -> bytes: :returns: bytes representing `private_key` """ _validate_private_key(private_key) - return private_key.private_bytes( + return private_key.private_bytes( # type: ignore[attr-defined] encoding=Encoding.DER, format=PrivateFormat.PKCS8, encryption_algorithm=NoEncryption(), From 3ce5ee6f0304131a5cc4ebbbed2237842084f3d2 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Sat, 7 Jan 2023 07:17:40 -0500 Subject: [PATCH 1339/2309] get Literal from somewhere it is more likely to be --- src/allmydata/web/common.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/allmydata/web/common.py b/src/allmydata/web/common.py index b55a49d4d..470170e7d 100644 --- a/src/allmydata/web/common.py +++ b/src/allmydata/web/common.py @@ -5,7 +5,8 @@ from __future__ import annotations from six import ensure_str -from typing import Optional, Union, TypeVar, Literal, overload +from typing import Optional, Union, TypeVar, overload +from typing_extensions import Literal import time import json From 22227c70948e6c057287b50ca05224f14227bb09 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 9 Jan 2023 10:31:48 -0500 Subject: [PATCH 1340/2309] Support old pycddl too so nix can keep working. --- setup.py | 7 +++++-- src/allmydata/storage/http_server.py | 9 ++++++++- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index dd50e0fcf..e4045d76b 100644 --- a/setup.py +++ b/setup.py @@ -137,8 +137,11 @@ install_requires = [ "werkzeug != 2.2.0", "treq", "cbor2", - # Need 0.4 to be able to pass in mmap() - "pycddl >= 0.4", + # Ideally we want 0.4+ to be able to pass in mmap(), but it's not strictly + # necessary yet until we fix the workaround to + # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3963 in + # allmydata.storage.http_server. + "pycddl", # for pid-file support "psutil", diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 6d22c92df..aa2e532cb 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -11,6 +11,7 @@ import binascii from tempfile import TemporaryFile from os import SEEK_END, SEEK_SET import mmap +from importlib.metadata import version as get_package_version from cryptography.x509 import Certificate as CryptoCertificate from zope.interface import implementer @@ -59,6 +60,12 @@ from ..util.base32 import rfc3548_alphabet from allmydata.interfaces import BadWriteEnablerError +# Until we figure out Nix (https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3963), +# need to support old pycddl which can only take bytes: +from distutils.version import LooseVersion +PYCDDL_BYTES_ONLY = LooseVersion(get_package_version("pycddl")) < LooseVersion("0.4") + + class ClientSecretsException(Exception): """The client did not send the appropriate secrets.""" @@ -557,7 +564,7 @@ class HTTPServer(object): fd = request.content.fileno() except (ValueError, OSError): fd = -1 - if fd > 0: + if fd > 0 and not PYCDDL_BYTES_ONLY: # It's a file, so we can use mmap() to save memory. message = mmap.mmap(fd, 0, access=mmap.ACCESS_READ) else: From f6d9c5a1b237c4819635be52eba22d2563d02260 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 9 Jan 2023 10:46:09 -0500 Subject: [PATCH 1341/2309] Fix PyInstaller. --- src/allmydata/storage/http_server.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index aa2e532cb..306d54ea0 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -11,7 +11,7 @@ import binascii from tempfile import TemporaryFile from os import SEEK_END, SEEK_SET import mmap -from importlib.metadata import version as get_package_version +from importlib.metadata import version as get_package_version, PackageNotFoundError from cryptography.x509 import Certificate as CryptoCertificate from zope.interface import implementer @@ -63,7 +63,15 @@ from allmydata.interfaces import BadWriteEnablerError # Until we figure out Nix (https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3963), # need to support old pycddl which can only take bytes: from distutils.version import LooseVersion -PYCDDL_BYTES_ONLY = LooseVersion(get_package_version("pycddl")) < LooseVersion("0.4") + +try: + PYCDDL_BYTES_ONLY = LooseVersion(get_package_version("pycddl")) < LooseVersion( + "0.4" + ) +except PackageNotFoundError: + # This can happen when building PyInstaller distribution. We'll just assume + # you installed a modern pycddl, cause why wouldn't you? + PYCDDL_BYTES_ONLY = False class ClientSecretsException(Exception): From 825fd64dddc860e24fd85dcc891c728ef35779e6 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 9 Jan 2023 10:52:24 -0500 Subject: [PATCH 1342/2309] News file. --- newsfragments/3964.removed | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/3964.removed diff --git a/newsfragments/3964.removed b/newsfragments/3964.removed new file mode 100644 index 000000000..1c2c3e544 --- /dev/null +++ b/newsfragments/3964.removed @@ -0,0 +1 @@ +Python 3.7 is no longer supported. \ No newline at end of file From 1482d419181c76d760359b09867c6d667f0753c9 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 9 Jan 2023 11:01:45 -0500 Subject: [PATCH 1343/2309] Drop 3.7. --- .circleci/config.yml | 52 ++++------------------------------------ .github/workflows/ci.yml | 10 +++----- README.rst | 2 +- setup.py | 9 +++---- tox.ini | 6 +---- 5 files changed, 13 insertions(+), 66 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 4dcf2a2db..21f60368c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -167,12 +167,7 @@ jobs: command: | dist/Tahoe-LAFS/tahoe --version - debian-10: &DEBIAN - docker: - - <<: *DOCKERHUB_AUTH - image: "tahoelafsci/debian:10-py3.7" - user: "nobody" - + debian-11: &DEBIAN environment: &UTF_8_ENVIRONMENT # In general, the test suite is not allowed to fail while the job # succeeds. But you can set this to "yes" if you want it to be @@ -184,7 +179,7 @@ jobs: # filenames and argv). LANG: "en_US.UTF-8" # Select a tox environment to run for this job. - TAHOE_LAFS_TOX_ENVIRONMENT: "py37" + TAHOE_LAFS_TOX_ENVIRONMENT: "py39" # Additional arguments to pass to tox. TAHOE_LAFS_TOX_ARGS: "" # The path in which test artifacts will be placed. @@ -252,15 +247,11 @@ jobs: /tmp/venv/bin/codecov fi - debian-11: - <<: *DEBIAN docker: - <<: *DOCKERHUB_AUTH image: "tahoelafsci/debian:11-py3.9" user: "nobody" - environment: - <<: *UTF_8_ENVIRONMENT - TAHOE_LAFS_TOX_ENVIRONMENT: "py39" + # Restore later using PyPy3.8 # pypy27-buster: @@ -312,22 +303,6 @@ jobs: - run: *SETUP_VIRTUALENV - run: *RUN_TESTS - ubuntu-18-04: &UBUNTU_18_04 - <<: *DEBIAN - docker: - - <<: *DOCKERHUB_AUTH - image: "tahoelafsci/ubuntu:18.04-py3.7" - user: "nobody" - - environment: - <<: *UTF_8_ENVIRONMENT - # The default trial args include --rterrors which is incompatible with - # this reporter on Python 3. So drop that and just specify the - # reporter. - TAHOE_LAFS_TRIAL_ARGS: "--reporter=subunitv2-file" - TAHOE_LAFS_TOX_ENVIRONMENT: "py37" - - ubuntu-20-04: <<: *DEBIAN docker: @@ -445,7 +420,7 @@ jobs: typechecks: docker: - <<: *DOCKERHUB_AUTH - image: "tahoelafsci/ubuntu:18.04-py3.7" + image: "tahoelafsci/ubuntu:20.04-py3.9" steps: - "checkout" @@ -457,7 +432,7 @@ jobs: docs: docker: - <<: *DOCKERHUB_AUTH - image: "tahoelafsci/ubuntu:18.04-py3.7" + image: "tahoelafsci/ubuntu:20.04-py3.9" steps: - "checkout" @@ -508,15 +483,6 @@ jobs: docker push tahoelafsci/${DISTRO}:${TAG}-py${PYTHON_VERSION} - build-image-debian-10: - <<: *BUILD_IMAGE - - environment: - DISTRO: "debian" - TAG: "10" - PYTHON_VERSION: "3.7" - - build-image-debian-11: <<: *BUILD_IMAGE @@ -525,14 +491,6 @@ jobs: TAG: "11" PYTHON_VERSION: "3.9" - build-image-ubuntu-18-04: - <<: *BUILD_IMAGE - - environment: - DISTRO: "ubuntu" - TAG: "18.04" - PYTHON_VERSION: "3.7" - build-image-ubuntu-20-04: <<: *BUILD_IMAGE diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d7fa3244b..80b312008 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -48,21 +48,20 @@ jobs: - windows-latest - ubuntu-latest python-version: - - "3.7" - "3.8" - "3.9" - "3.10" include: - # On macOS don't bother with 3.7-3.8, just to get faster builds. + # On macOS don't bother with 3.8, just to get faster builds. - os: macos-latest python-version: "3.9" - os: macos-latest python-version: "3.10" # We only support PyPy on Linux at the moment. - - os: ubuntu-latest - python-version: "pypy-3.7" - os: ubuntu-latest python-version: "pypy-3.8" + - os: ubuntu-latest + python-version: "pypy-3.9" steps: # See https://github.com/actions/checkout. A fetch-depth of 0 @@ -162,9 +161,6 @@ jobs: force-foolscap: false # 22.04 has some issue with Tor at the moment: # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3943 - - os: ubuntu-20.04 - python-version: "3.7" - force-foolscap: true - os: ubuntu-20.04 python-version: "3.9" force-foolscap: false diff --git a/README.rst b/README.rst index 317378fae..bbf88610d 100644 --- a/README.rst +++ b/README.rst @@ -56,7 +56,7 @@ Once ``tahoe --version`` works, see `How to Run Tahoe-LAFS `__ 🐍 Python 2 ----------- -Python 3.7 or later is now required. +Python 3.8 or later is required. If you are still using Python 2.7, use Tahoe-LAFS version 1.17.1. diff --git a/setup.py b/setup.py index 1075e2129..edef7a4c3 100644 --- a/setup.py +++ b/setup.py @@ -223,7 +223,7 @@ def run_command(args, cwd=None): use_shell = sys.platform == "win32" try: p = subprocess.Popen(args, stdout=subprocess.PIPE, cwd=cwd, shell=use_shell) - except EnvironmentError as e: # if this gives a SyntaxError, note that Tahoe-LAFS requires Python 3.7+ + except EnvironmentError as e: # if this gives a SyntaxError, note that Tahoe-LAFS requires Python 3.8+ print("Warning: unable to run %r." % (" ".join(args),)) print(e) return None @@ -374,8 +374,8 @@ setup(name="tahoe-lafs", # also set in __init__.py package_dir = {'':'src'}, packages=find_packages('src') + ['allmydata.test.plugins'], classifiers=trove_classifiers, - # We support Python 3.7 or later. 3.11 is not supported yet. - python_requires=">=3.7, <3.11", + # We support Python 3.8 or later. 3.11 is not supported yet. + python_requires=">=3.8, <3.11", install_requires=install_requires, extras_require={ # Duplicate the Twisted pywin32 dependency here. See @@ -388,9 +388,6 @@ setup(name="tahoe-lafs", # also set in __init__.py ], "test": [ "flake8", - # On Python 3.7, importlib_metadata v5 breaks flake8. - # https://github.com/python/importlib_metadata/issues/407 - "importlib_metadata<5; python_version < '3.8'", # Pin a specific pyflakes so we don't have different folks # disagreeing on what is or is not a lint issue. We can bump # this version from time to time, but we will do it diff --git a/tox.ini b/tox.ini index 96eed4e40..3e2dacbb2 100644 --- a/tox.ini +++ b/tox.ini @@ -7,11 +7,9 @@ # the tox-gh-actions package. [gh-actions] python = - 3.7: py37-coverage 3.8: py38-coverage 3.9: py39-coverage 3.10: py310-coverage - pypy-3.7: pypy37 pypy-3.8: pypy38 pypy-3.9: pypy39 @@ -19,7 +17,7 @@ python = twisted = 1 [tox] -envlist = typechecks,codechecks,py{37,38,39,310}-{coverage},pypy27,pypy37,pypy38,pypy39,integration +envlist = typechecks,codechecks,py{38,39,310}-{coverage},pypy27,pypy38,pypy39,integration minversion = 2.4 [testenv] @@ -49,8 +47,6 @@ deps = # regressions in new releases of this package that cause us the kind of # suffering we're trying to avoid with the above pins. certifi - # VCS hooks support - py37,!coverage: pre-commit # We add usedevelop=False because testing against a true installation gives # more useful results. From c4153d54055e0c2e33343de812a77b14f7069439 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 9 Jan 2023 11:03:15 -0500 Subject: [PATCH 1344/2309] Additional changes. --- newsfragments/3964.removed | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/newsfragments/3964.removed b/newsfragments/3964.removed index 1c2c3e544..d022f94af 100644 --- a/newsfragments/3964.removed +++ b/newsfragments/3964.removed @@ -1 +1 @@ -Python 3.7 is no longer supported. \ No newline at end of file +Python 3.7 is no longer supported, and Debian 10 and Ubuntu 18.04 are no longer tested. \ No newline at end of file From 8c418832bb6ba62fefb1c740bea973cbaf5c07f9 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 9 Jan 2023 11:06:57 -0500 Subject: [PATCH 1345/2309] Remove references to missing jobs. --- .circleci/config.yml | 9 --------- 1 file changed, 9 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 21f60368c..834c5f9d2 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -15,16 +15,11 @@ workflows: ci: jobs: # Start with jobs testing various platforms. - - "debian-10": - {} - "debian-11": {} - "ubuntu-20-04": {} - - "ubuntu-18-04": - requires: - - "ubuntu-20-04" # Equivalent to RHEL 8; CentOS 8 is dead. - "oraclelinux-8": @@ -85,12 +80,8 @@ workflows: # Contexts are managed in the CircleCI web interface: # # https://app.circleci.com/settings/organization/github/tahoe-lafs/contexts - - "build-image-debian-10": &DOCKERHUB_CONTEXT - context: "dockerhub-auth" - "build-image-debian-11": <<: *DOCKERHUB_CONTEXT - - "build-image-ubuntu-18-04": - <<: *DOCKERHUB_CONTEXT - "build-image-ubuntu-20-04": <<: *DOCKERHUB_CONTEXT - "build-image-fedora-35": From 34f5da7246b725589d5dc1ec4febb1bce8c4029e Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 9 Jan 2023 11:08:31 -0500 Subject: [PATCH 1346/2309] And add back necessary anchor. --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 834c5f9d2..9080c43ed 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -80,7 +80,7 @@ workflows: # Contexts are managed in the CircleCI web interface: # # https://app.circleci.com/settings/organization/github/tahoe-lafs/contexts - - "build-image-debian-11": + - "build-image-debian-11": &DOCKERHUB_CONTEXT <<: *DOCKERHUB_CONTEXT - "build-image-ubuntu-20-04": <<: *DOCKERHUB_CONTEXT From 6bb57e248de0410faf1d7b6ab43efa99a3b1e5eb Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 9 Jan 2023 13:09:59 -0500 Subject: [PATCH 1347/2309] Try to switch Nix off 3.7. --- default.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/default.nix b/default.nix index 5f4db2c78..e4f2dd4d4 100644 --- a/default.nix +++ b/default.nix @@ -29,7 +29,7 @@ in , pypiData ? sources.pypi-deps-db # the pypi package database snapshot to use # for dependency resolution -, pythonVersion ? "python37" # a string choosing the python derivation from +, pythonVersion ? "python39" # a string choosing the python derivation from # nixpkgs to target , extras ? [ "tor" "i2p" ] # a list of strings identifying tahoe-lafs extras, From 7b2f19b0fa0ada0f1d5a05056927fc84fa3fa327 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 9 Jan 2023 13:16:04 -0500 Subject: [PATCH 1348/2309] Switch Nix off 3.7 some more. --- tests.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests.nix b/tests.nix index dd477c273..f8ed678f3 100644 --- a/tests.nix +++ b/tests.nix @@ -5,7 +5,7 @@ in { pkgsVersion ? "nixpkgs-21.11" , pkgs ? import sources.${pkgsVersion} { } , pypiData ? sources.pypi-deps-db -, pythonVersion ? "python37" +, pythonVersion ? "python39" , mach-nix ? import sources.mach-nix { inherit pkgs pypiData; python = pythonVersion; @@ -21,7 +21,7 @@ let inherit pkgs; lib = pkgs.lib; }; - tests_require = (mach-lib.extract "python37" ./. "extras_require" ).extras_require.test; + tests_require = (mach-lib.extract "python39" ./. "extras_require" ).extras_require.test; # Get the Tahoe-LAFS package itself. This does not include test # requirements and we don't ask for test requirements so that we can just From b05793e56b3bfcedd75c633fc9186adf02aad48a Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 9 Jan 2023 13:45:22 -0500 Subject: [PATCH 1349/2309] Meaningless tweak to rerun CI. --- README.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/README.rst b/README.rst index bbf88610d..56451701a 100644 --- a/README.rst +++ b/README.rst @@ -45,6 +45,7 @@ Tahoe-LAFS was first designed in 2007, following the "principle of least authori Please read more about Tahoe-LAFS architecture `here `__. + ✅ Installation --------------- From 046d9cf802e541d77f203ad1d29edbff6f628790 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 9 Jan 2023 14:25:47 -0500 Subject: [PATCH 1350/2309] Another meaningless tweak. --- README.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/README.rst b/README.rst index 56451701a..bbf88610d 100644 --- a/README.rst +++ b/README.rst @@ -45,7 +45,6 @@ Tahoe-LAFS was first designed in 2007, following the "principle of least authori Please read more about Tahoe-LAFS architecture `here `__. - ✅ Installation --------------- From ccb5956645e4763999b35e165095681fe8025b40 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 10 Jan 2023 11:04:22 -0500 Subject: [PATCH 1351/2309] 0 is also valid FD. --- src/allmydata/storage/http_server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 8e987b4d1..387353d24 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -572,7 +572,7 @@ class HTTPServer(object): fd = request.content.fileno() except (ValueError, OSError): fd = -1 - if fd > 0 and not PYCDDL_BYTES_ONLY: + if fd >= 0 and not PYCDDL_BYTES_ONLY: # It's a file, so we can use mmap() to save memory. message = mmap.mmap(fd, 0, access=mmap.ACCESS_READ) else: From 828fc588c506586e56c610bdfdc1a170eb73acdc Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 10 Jan 2023 11:10:31 -0500 Subject: [PATCH 1352/2309] Add minimal docstrings. --- src/allmydata/test/test_system.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/allmydata/test/test_system.py b/src/allmydata/test/test_system.py index 55bf0ed8d..10a64c1fe 100644 --- a/src/allmydata/test/test_system.py +++ b/src/allmydata/test/test_system.py @@ -521,9 +521,11 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase): def test_mutable_sdmf(self): + """SDMF mutables can be uploaded, downloaded, and many other things.""" return self._test_mutable(SDMF_VERSION) def test_mutable_mdmf(self): + """MDMF mutables can be uploaded, downloaded, and many other things.""" return self._test_mutable(MDMF_VERSION) def _test_mutable(self, mutable_version): From 98624f3d6a32882d5fb2283fa61690226e81299e Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 12 Jan 2023 09:53:07 -0500 Subject: [PATCH 1353/2309] Attempt to workaround for 3960. --- .github/workflows/ci.yml | 9 +++++++++ newsfragments/3960.minor | 0 2 files changed, 9 insertions(+) create mode 100644 newsfragments/3960.minor diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 80b312008..2a8f5246e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -86,8 +86,17 @@ jobs: run: python misc/build_helpers/show-tool-versions.py - name: Run tox for corresponding Python version + if: ${{ !contains(matrix.os, 'windows') }} run: python -m tox + # On Windows, a non-blocking pipe might respond (when emulating Unix-y + # API) with ENOSPC to indicate buffer full. Trial doesn't handle this + # well. So, we pipe the output through Get-Content (similar to piping + # through cat) to make buffer handling someone else's problem. + - name: Run tox for corresponding Python version + if: ${{ contains(matrix.os, 'windows') }} + run: python -m tox | Get-Content + - name: Upload eliot.log uses: actions/upload-artifact@v3 with: diff --git a/newsfragments/3960.minor b/newsfragments/3960.minor new file mode 100644 index 000000000..e69de29bb From e142f051b497124cb8828106242d14cb46643431 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 12 Jan 2023 10:01:09 -0500 Subject: [PATCH 1354/2309] Cats solve problems, right? --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2a8f5246e..1b8ab0d57 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -91,11 +91,11 @@ jobs: # On Windows, a non-blocking pipe might respond (when emulating Unix-y # API) with ENOSPC to indicate buffer full. Trial doesn't handle this - # well. So, we pipe the output through Get-Content (similar to piping - # through cat) to make buffer handling someone else's problem. + # well. So, we pipe the output through cat to make buffer handling someone + # else's problem. - name: Run tox for corresponding Python version if: ${{ contains(matrix.os, 'windows') }} - run: python -m tox | Get-Content + run: python -m tox | cat - name: Upload eliot.log uses: actions/upload-artifact@v3 From dd89ca6d4f2738f92534f0b33da2a365b9f30016 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 12 Jan 2023 10:36:39 -0500 Subject: [PATCH 1355/2309] Another approach. --- .github/workflows/ci.yml | 8 ++++--- misc/windows-enospc/passthrough.py | 38 ++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 3 deletions(-) create mode 100644 misc/windows-enospc/passthrough.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1b8ab0d57..1eb12bdd5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -91,11 +91,13 @@ jobs: # On Windows, a non-blocking pipe might respond (when emulating Unix-y # API) with ENOSPC to indicate buffer full. Trial doesn't handle this - # well. So, we pipe the output through cat to make buffer handling someone - # else's problem. + # well. So, we pipe the output through pipethrough that will hopefully be + # able to do the right thing by using Windows APIs. - name: Run tox for corresponding Python version if: ${{ contains(matrix.os, 'windows') }} - run: python -m tox | cat + run: | + pip install twisted + python -m tox | python misc/windows-enospc/passthrough.py - name: Upload eliot.log uses: actions/upload-artifact@v3 diff --git a/misc/windows-enospc/passthrough.py b/misc/windows-enospc/passthrough.py new file mode 100644 index 000000000..6be3d7dbe --- /dev/null +++ b/misc/windows-enospc/passthrough.py @@ -0,0 +1,38 @@ +""" +Writing to non-blocking pipe can result in ENOSPC when using Unix APIs on +Windows. So, this program passes through data from stdin to stdout, using +Windows APIs instead of Unix-y APIs. +""" + +import sys + +from twisted.internet.stdio import StandardIO +from twisted.internet import reactor +from twisted.internet.protocol import Protocol +from twisted.internet.interfaces import IHalfCloseableProtocol +from twisted.internet.error import ReactorNotRunning +from zope.interface import implementer + +@implementer(IHalfCloseableProtocol) +class Passthrough(Protocol): + def readConnectionLost(self): + self.transport.loseConnection() + + def writeConnectionLost(self): + try: + reactor.stop() + except ReactorNotRunning: + pass + + def dataReceived(self, data): + self.transport.write(data) + + def connectionLost(self, reason): + try: + reactor.stop() + except ReactorNotRunning: + pass + + +std = StandardIO(Passthrough()) +reactor.run() From e997bb9c398d191c28c7a82840a2df3bbbb657e9 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 12 Jan 2023 10:39:50 -0500 Subject: [PATCH 1356/2309] Also need pywin32. --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1eb12bdd5..0f8040e96 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -96,7 +96,7 @@ jobs: - name: Run tox for corresponding Python version if: ${{ contains(matrix.os, 'windows') }} run: | - pip install twisted + pip install twisted pywin32 python -m tox | python misc/windows-enospc/passthrough.py - name: Upload eliot.log From ee5ad549fed7150d2a6782ca4409b27c5821d546 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 12 Jan 2023 11:12:00 -0500 Subject: [PATCH 1357/2309] Clarify. --- .github/workflows/ci.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0f8040e96..011bfc1ea 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -91,8 +91,9 @@ jobs: # On Windows, a non-blocking pipe might respond (when emulating Unix-y # API) with ENOSPC to indicate buffer full. Trial doesn't handle this - # well. So, we pipe the output through pipethrough that will hopefully be - # able to do the right thing by using Windows APIs. + # well, so it breaks test runs. To attempt to solve this, we pipe the + # output through pipethrough that will hopefully be able to do the right + # thing by using Windows APIs. - name: Run tox for corresponding Python version if: ${{ contains(matrix.os, 'windows') }} run: | From a765d8a35b8d0bdbe6163585dcdd96f4d6f7eddf Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 12 Jan 2023 11:18:05 -0500 Subject: [PATCH 1358/2309] Unused. --- misc/windows-enospc/passthrough.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/misc/windows-enospc/passthrough.py b/misc/windows-enospc/passthrough.py index 6be3d7dbe..1d4cd48bb 100644 --- a/misc/windows-enospc/passthrough.py +++ b/misc/windows-enospc/passthrough.py @@ -4,8 +4,6 @@ Windows. So, this program passes through data from stdin to stdout, using Windows APIs instead of Unix-y APIs. """ -import sys - from twisted.internet.stdio import StandardIO from twisted.internet import reactor from twisted.internet.protocol import Protocol From bbd3e74a5f681c973f90b881805b69512391d464 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 12 Jan 2023 15:17:08 -0500 Subject: [PATCH 1359/2309] Always place an int in the parameters total field --- integration/test_vectors.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration/test_vectors.py b/integration/test_vectors.py index 4f5ef6e8a..aaff441b8 100644 --- a/integration/test_vectors.py +++ b/integration/test_vectors.py @@ -142,7 +142,7 @@ async def test_generate(reactor, request, alice): "zfec": { "segmentSize": SEGMENT_SIZE, "required": case.seed_params.required, - "total": case.seed_params.total, + "total": case.seed_params.realize(vectors.MAX_SHARES_MAP[case.fmt]).total, }, "expected": cap, } From fa55956d29feb31f6e7fd194d1d3889244a23133 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 12 Jan 2023 15:18:54 -0500 Subject: [PATCH 1360/2309] Always write an int to the test vectors file --- integration/test_vectors.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/integration/test_vectors.py b/integration/test_vectors.py index aaff441b8..201599d95 100644 --- a/integration/test_vectors.py +++ b/integration/test_vectors.py @@ -141,8 +141,8 @@ async def test_generate(reactor, request, alice): }, "zfec": { "segmentSize": SEGMENT_SIZE, - "required": case.seed_params.required, - "total": case.seed_params.realize(vectors.MAX_SHARES_MAP[case.fmt]).total, + "required": case.params.required, + "total": case.params.total, }, "expected": cap, } From dd51c7a3f1dc86b942a45174c1ff9077ea0a4bf6 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 12 Jan 2023 15:19:01 -0500 Subject: [PATCH 1361/2309] Handle an empty test vectors file --- integration/vectors.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/integration/vectors.py b/integration/vectors.py index 22c5b8830..3240f9c27 100644 --- a/integration/vectors.py +++ b/integration/vectors.py @@ -125,6 +125,8 @@ def stretch(seed: bytes, size: int) -> bytes: def load_capabilities(f: TextIO) -> dict[Case, str]: data = safe_load(f) + if data is None: + return {} return { Case( seed_params=SeedParam(case["zfec"]["required"], case["zfec"]["total"]), From 2490f0f58ac3b170cf768d7e40cabebbb699c4db Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 12 Jan 2023 15:33:37 -0500 Subject: [PATCH 1362/2309] some minor rationalization of the return type --- src/allmydata/web/common.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/allmydata/web/common.py b/src/allmydata/web/common.py index 470170e7d..c49354217 100644 --- a/src/allmydata/web/common.py +++ b/src/allmydata/web/common.py @@ -707,12 +707,12 @@ def url_for_string(req, url_string): T = TypeVar("T") @overload -def get_arg(req: IRequest, argname: str | bytes, default: T = None, *, multiple: Literal[False] = False) -> bytes: ... +def get_arg(req: IRequest, argname: str | bytes, default: T = None, *, multiple: Literal[False] = False) -> T | bytes: ... @overload -def get_arg(req: IRequest, argname: str | bytes, default: T = None, *, multiple: Literal[True]) -> tuple[bytes, ...]: ... +def get_arg(req: IRequest, argname: str | bytes, default: T = None, *, multiple: Literal[True]) -> T | tuple[bytes, ...]: ... -def get_arg(req: IRequest, argname: str | bytes, default: T | None = None, *, multiple: bool = False) -> None | T | bytes | tuple[bytes, ...]: +def get_arg(req: IRequest, argname: str | bytes, default: T = None, *, multiple: bool = False) -> None | T | bytes | tuple[bytes, ...]: """Extract an argument from either the query args (req.args) or the form body fields (req.fields). If multiple=False, this returns a single value (or the default, which defaults to None), and the query args take From 2d23e2e6401e5c696002e9eefadd30193a39201f Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 12 Jan 2023 15:37:07 -0500 Subject: [PATCH 1363/2309] some doc improvements --- src/allmydata/client.py | 16 +++++++++------- src/allmydata/scripts/cli.py | 1 + 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/src/allmydata/client.py b/src/allmydata/client.py index a8238e4ee..73672f30a 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -1102,16 +1102,18 @@ class _Client(node.Node, pollmixin.PollMixin): :param version: If given, the mutable file format for the new object (otherwise a format will be chosen automatically). - :param unique_keypair: **Warning** This valuely independently - determines the identity of the mutable object to create. There - cannot be two different mutable objects that share a keypair. - They will merge into one object (with undefined contents). + :param unique_keypair: **Warning** This value independently determines + the identity of the mutable object to create. There cannot be two + different mutable objects that share a keypair. They will merge + into one object (with undefined contents). - It is not common to pass a non-None value for this parameter. If - None is given then a new random keypair will be generated. + It is common to pass a None value (or not pass a valuye) for this + parameter. In these cases, a new random keypair will be + generated. If non-None, the given public/private keypair will be used for the - new object. + new object. The expected use-case is for implementing compliance + tests. :return: A Deferred which will fire with a representation of the new mutable object after it has been uploaded. diff --git a/src/allmydata/scripts/cli.py b/src/allmydata/scripts/cli.py index aa7644e18..579b37906 100644 --- a/src/allmydata/scripts/cli.py +++ b/src/allmydata/scripts/cli.py @@ -189,6 +189,7 @@ class PutOptions(FileStoreOptions): "***Warning*** " "It is possible to use this option to spoil the normal security properties of mutable objects. " "It is also possible to corrupt or destroy data with this option. " + "Most users will not need this option and can ignore it. " "For mutables only, " "this gives a file containing a PEM-encoded 2048 bit RSA private key to use as the signature key for the mutable. " "The private key must be handled at least as strictly as the resulting capability string. " From e6ef45d3371321e5a4667e43a3d53223bdc153c1 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 12 Jan 2023 15:37:12 -0500 Subject: [PATCH 1364/2309] test that we can also download the mutable --- src/allmydata/test/cli/test_put.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/allmydata/test/cli/test_put.py b/src/allmydata/test/cli/test_put.py index 98407bb7e..e8f83188a 100644 --- a/src/allmydata/test/cli/test_put.py +++ b/src/allmydata/test/cli/test_put.py @@ -268,6 +268,11 @@ class Put(GridTestMixin, CLITestMixin, unittest.TestCase): (writekey, fingerprint), (cap.writekey, cap.fingerprint), ) + # Also the capability we were given actually refers to the data we + # uploaded. + (rc, out, err) = await self.do_cli("get", out.strip()) + self.assertEqual(rc, 0, (out, err)) + self.assertEqual(out, datapath.getContent().decode("ascii")) def test_mutable(self): # echo DATA1 | tahoe put --mutable - uploaded.txt From 47ec418f7afcc745e7b0376e9485812351efab1a Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 12 Jan 2023 15:43:54 -0500 Subject: [PATCH 1365/2309] Test that we can also download the mutable data via the web interface --- src/allmydata/test/web/test_web.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/web/test_web.py b/src/allmydata/test/web/test_web.py index 7793c023c..4c828817a 100644 --- a/src/allmydata/test/web/test_web.py +++ b/src/allmydata/test/web/test_web.py @@ -2885,10 +2885,11 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi priv, pub = create_signing_keypair(2048) encoded_privkey = urlsafe_b64encode(der_string_from_signing_key(priv)).decode("ascii") filename = "predetermined-sdmf" + expected_content = self.NEWFILE_CONTENTS * 100 actual_cap = uri.from_string(await self.POST( self.public_url + f"/foo?t=upload&format={format}&private-key={encoded_privkey}", - file=(filename, self.NEWFILE_CONTENTS * 100), + file=(filename, expected_content), )) # Ideally we would inspect the private ("signature") and public # ("verification") keys but they are not made easily accessible here @@ -2903,7 +2904,10 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi (actual_cap.writekey, actual_cap.fingerprint), ) - + # And the capability we got can be used to download the data we + # uploaded. + downloaded_content = await self.GET(f"/uri/{actual_cap.to_string().decode('ascii')}") + self.assertEqual(expected_content, downloaded_content) def test_POST_upload_format(self): def _check_upload(ign, format, uri_prefix, fn=None): From c856f1aa298a10ea91707fcce91899931a8924c0 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 12 Jan 2023 16:16:55 -0500 Subject: [PATCH 1366/2309] Censor private key values in the HTTP log, too. --- src/allmydata/test/web/test_webish.py | 10 +++ src/allmydata/webish.py | 98 +++++++++++++++------------ 2 files changed, 64 insertions(+), 44 deletions(-) diff --git a/src/allmydata/test/web/test_webish.py b/src/allmydata/test/web/test_webish.py index 4a77d21ae..050f77d1c 100644 --- a/src/allmydata/test/web/test_webish.py +++ b/src/allmydata/test/web/test_webish.py @@ -202,6 +202,16 @@ class TahoeLAFSSiteTests(SyncTestCase): ), ) + def test_private_key_censoring(self): + """ + The log event for a request including a **private-key** query + argument has the private key value censored. + """ + self._test_censoring( + b"/uri?uri=URI:CHK:aaa:bbb&private-key=AAAAaaaabbbb==", + b"/uri?uri=[CENSORED]&private-key=[CENSORED]", + ) + def test_uri_censoring(self): """ The log event for a request for **/uri/** has the capability value diff --git a/src/allmydata/webish.py b/src/allmydata/webish.py index 519b3e1f0..1b2b8192a 100644 --- a/src/allmydata/webish.py +++ b/src/allmydata/webish.py @@ -1,18 +1,12 @@ """ -Ported to Python 3. +General web server-related utilities. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals - -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 +from __future__ import annotations from six import ensure_str import re, time, tempfile +from urllib.parse import parse_qsl, urlencode from cgi import ( FieldStorage, @@ -45,40 +39,37 @@ from .web.storage_plugins import ( ) -if PY2: - FileUploadFieldStorage = FieldStorage -else: - class FileUploadFieldStorage(FieldStorage): - """ - Do terrible things to ensure files are still bytes. +class FileUploadFieldStorage(FieldStorage): + """ + Do terrible things to ensure files are still bytes. - On Python 2, uploaded files were always bytes. On Python 3, there's a - heuristic: if the filename is set on a field, it's assumed to be a file - upload and therefore bytes. If no filename is set, it's Unicode. + On Python 2, uploaded files were always bytes. On Python 3, there's a + heuristic: if the filename is set on a field, it's assumed to be a file + upload and therefore bytes. If no filename is set, it's Unicode. - Unfortunately, we always want it to be bytes, and Tahoe-LAFS also - enables setting the filename not via the MIME filename, but via a - separate field called "name". + Unfortunately, we always want it to be bytes, and Tahoe-LAFS also + enables setting the filename not via the MIME filename, but via a + separate field called "name". - Thus we need to do this ridiculous workaround. Mypy doesn't like it - either, thus the ``# type: ignore`` below. + Thus we need to do this ridiculous workaround. Mypy doesn't like it + either, thus the ``# type: ignore`` below. - Source for idea: - https://mail.python.org/pipermail/python-dev/2017-February/147402.html - """ - @property # type: ignore - def filename(self): - if self.name == "file" and not self._mime_filename: - # We use the file field to upload files, see directory.py's - # _POST_upload. Lack of _mime_filename means we need to trick - # FieldStorage into thinking there is a filename so it'll - # return bytes. - return "unknown-filename" - return self._mime_filename + Source for idea: + https://mail.python.org/pipermail/python-dev/2017-February/147402.html + """ + @property # type: ignore + def filename(self): + if self.name == "file" and not self._mime_filename: + # We use the file field to upload files, see directory.py's + # _POST_upload. Lack of _mime_filename means we need to trick + # FieldStorage into thinking there is a filename so it'll + # return bytes. + return "unknown-filename" + return self._mime_filename - @filename.setter - def filename(self, value): - self._mime_filename = value + @filename.setter + def filename(self, value): + self._mime_filename = value class TahoeLAFSRequest(Request, object): @@ -180,12 +171,7 @@ def _logFormatter(logDateTime, request): queryargs = b"" else: path, queryargs = x - # there is a form handler which redirects POST /uri?uri=FOO into - # GET /uri/FOO so folks can paste in non-HTTP-prefixed uris. Make - # sure we censor these too. - if queryargs.startswith(b"uri="): - queryargs = b"uri=[CENSORED]" - queryargs = b"?" + queryargs + queryargs = b"?" + censor(queryargs) if path.startswith(b"/uri/"): path = b"/uri/[CENSORED]" elif path.startswith(b"/file/"): @@ -207,6 +193,30 @@ def _logFormatter(logDateTime, request): ) +def censor(queryargs: bytes) -> bytes: + """ + Replace potentially sensitive values in query arguments with a + constant string. + """ + args = parse_qsl(queryargs.decode("ascii"), keep_blank_values=True, encoding="utf8") + result = [] + for k, v in args: + if k == "uri": + # there is a form handler which redirects POST /uri?uri=FOO into + # GET /uri/FOO so folks can paste in non-HTTP-prefixed uris. Make + # sure we censor these. + v = "[CENSORED]" + elif k == "private-key": + # Likewise, sometimes a private key is supplied with mutable + # creation. + v = "[CENSORED]" + + result.append((k, v)) + + # Customize safe to try to leave our markers intact. + return urlencode(result, safe="[]").encode("ascii") + + class TahoeLAFSSite(Site, object): """ The HTTP protocol factory used by Tahoe-LAFS. From 1a807a0232996447c47dd095a13498488e16d541 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 12 Jan 2023 16:32:32 -0500 Subject: [PATCH 1367/2309] mollify the type checker --- src/allmydata/crypto/rsa.py | 9 +++++++-- src/allmydata/scripts/tahoe_put.py | 3 ++- src/allmydata/test/cli/test_put.py | 2 ++ 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/src/allmydata/crypto/rsa.py b/src/allmydata/crypto/rsa.py index 3ad893dbf..e579a3d2a 100644 --- a/src/allmydata/crypto/rsa.py +++ b/src/allmydata/crypto/rsa.py @@ -79,10 +79,14 @@ def create_signing_keypair_from_string(private_key_der: bytes) -> tuple[PrivateK ) def load_with_validation() -> PrivateKey: - return _load() + k = _load() + assert isinstance(k, PrivateKey) + return k def load_without_validation() -> PrivateKey: - return _load(unsafe_skip_rsa_key_validation=True) + k = _load(unsafe_skip_rsa_key_validation=True) + assert isinstance(k, PrivateKey) + return k # Load it once without the potentially expensive OpenSSL validation # checks. These have superlinear complexity. We *will* run them just @@ -159,6 +163,7 @@ def create_verifying_key_from_string(public_key_der: bytes) -> PublicKey: public_key_der, backend=default_backend(), ) + assert isinstance(pub_key, PublicKey) return pub_key diff --git a/src/allmydata/scripts/tahoe_put.py b/src/allmydata/scripts/tahoe_put.py index fd746a43d..c04b6b4bc 100644 --- a/src/allmydata/scripts/tahoe_put.py +++ b/src/allmydata/scripts/tahoe_put.py @@ -11,7 +11,7 @@ from cryptography.hazmat.primitives.serialization import load_pem_private_key from twisted.python.filepath import FilePath -from allmydata.crypto.rsa import der_string_from_signing_key +from allmydata.crypto.rsa import PrivateKey, der_string_from_signing_key from allmydata.scripts.common_http import do_http, format_http_success, format_http_error from allmydata.scripts.common import get_alias, DEFAULT_ALIAS, escape_path, \ UnknownAliasError @@ -23,6 +23,7 @@ def load_private_key(path: str) -> str: to include in the HTTP request. """ privkey = load_pem_private_key(FilePath(path).getContent(), password=None) + assert isinstance(privkey, PrivateKey) derbytes = der_string_from_signing_key(privkey) return urlsafe_b64encode(derbytes).decode("ascii") diff --git a/src/allmydata/test/cli/test_put.py b/src/allmydata/test/cli/test_put.py index e8f83188a..c5f32a553 100644 --- a/src/allmydata/test/cli/test_put.py +++ b/src/allmydata/test/cli/test_put.py @@ -11,6 +11,7 @@ from twisted.python.filepath import FilePath from cryptography.hazmat.primitives.serialization import load_pem_private_key +from allmydata.crypto.rsa import PrivateKey from allmydata.uri import from_string from allmydata.util import fileutil from allmydata.scripts.common import get_aliases @@ -262,6 +263,7 @@ class Put(GridTestMixin, CLITestMixin, unittest.TestCase): cap = from_string(out.strip()) # The capability is derived from the key we specified. privkey = load_pem_private_key(pempath.getContent(), password=None) + assert isinstance(privkey, PrivateKey) pubkey = privkey.public_key() writekey, _, fingerprint = derive_mutable_keys((pubkey, privkey)) self.assertEqual( From ed74fdc746078df6e9163b294afb5249c2a6ab28 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 12 Jan 2023 16:56:20 -0500 Subject: [PATCH 1368/2309] write the data file more safely --- integration/test_vectors.py | 43 ++++++++++++++++++------------------- 1 file changed, 21 insertions(+), 22 deletions(-) diff --git a/integration/test_vectors.py b/integration/test_vectors.py index 201599d95..e18aca116 100644 --- a/integration/test_vectors.py +++ b/integration/test_vectors.py @@ -128,28 +128,27 @@ async def test_generate(reactor, request, alice): FORMATS, )) results = generate(reactor, request, alice, space) - with vectors.DATA_PATH.open("w") as f: - f.write(safe_dump({ - "version": "2023-01-03", - "vector": [ - { - "convergence": vectors.encode_bytes(case.convergence), - "format": case.fmt, - "sample": { - "seed": vectors.encode_bytes(case.seed_data.seed), - "length": case.seed_data.length, - }, - "zfec": { - "segmentSize": SEGMENT_SIZE, - "required": case.params.required, - "total": case.params.total, - }, - "expected": cap, - } - async for (case, cap) - in results - ], - })) + vectors.DATA_PATH.setContent(safe_dump({ + "version": "2023-01-03", + "vector": [ + { + "convergence": vectors.encode_bytes(case.convergence), + "format": case.fmt, + "sample": { + "seed": vectors.encode_bytes(case.seed_data.seed), + "length": case.seed_data.length, + }, + "zfec": { + "segmentSize": SEGMENT_SIZE, + "required": case.params.required, + "total": case.params.total, + }, + "expected": cap, + } + async for (case, cap) + in results + ], + })) async def generate( From 312513587fbf1c0a1fbf35f96081ef086d83d062 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 12 Jan 2023 17:27:37 -0500 Subject: [PATCH 1369/2309] Switch to FilePath, regenerate w/o "max" --- integration/test_vectors.py | 4 ++-- integration/test_vectors.yaml | 34 +++++++++++++++++----------------- integration/vectors.py | 4 +++- 3 files changed, 22 insertions(+), 20 deletions(-) diff --git a/integration/test_vectors.py b/integration/test_vectors.py index e18aca116..aa5cde42b 100644 --- a/integration/test_vectors.py +++ b/integration/test_vectors.py @@ -129,7 +129,7 @@ async def test_generate(reactor, request, alice): )) results = generate(reactor, request, alice, space) vectors.DATA_PATH.setContent(safe_dump({ - "version": "2023-01-03", + "version": "2023-01-12", "vector": [ { "convergence": vectors.encode_bytes(case.convergence), @@ -148,7 +148,7 @@ async def test_generate(reactor, request, alice): async for (case, cap) in results ], - })) + }).encode("ascii")) async def generate( diff --git a/integration/test_vectors.yaml b/integration/test_vectors.yaml index ca16a1e92..39801b3fe 100644 --- a/integration/test_vectors.yaml +++ b/integration/test_vectors.yaml @@ -808,7 +808,7 @@ vector: zfec: required: 101 segmentSize: 131072 - total: max + total: 256 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== expected: URI:CHK:sshu77h6opnto3jnngot3lwzl4:4cjhp6u3i2uzdzrjitje3inlhs6gnlgwc6wgi6vtedc2grgvcpza:101:256:4096 format: chk @@ -818,7 +818,7 @@ vector: zfec: required: 101 segmentSize: 131072 - total: max + total: 256 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== expected: URI:CHK:kl2jayrstlf5q3gax7o65ybivy:mnn5doti4wllssdg35ymojuiv4hd2ir7i6asurgms3ea67watm7a:101:256:131071 format: chk @@ -828,7 +828,7 @@ vector: zfec: required: 101 segmentSize: 131072 - total: max + total: 256 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== expected: URI:CHK:a6wuekh5mynr4c74ma3b2gswfa:b6lovwnbv3523n5kaxga3ruxa2dvxef6h4a2jnl6lasn4fwvvdba:101:256:131073 format: chk @@ -838,7 +838,7 @@ vector: zfec: required: 101 segmentSize: 131072 - total: max + total: 256 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== expected: URI:CHK:pmuo6vpcodol76sexq7s2ojej4:irp6rxnkgwbn3h4g5fzgjpqgsvhvycxobr3xrk3zr44r64wimgqa:101:256:2097151 format: chk @@ -848,7 +848,7 @@ vector: zfec: required: 101 segmentSize: 131072 - total: max + total: 256 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== expected: URI:CHK:fh4boug3wkciqhkgpusjsjyzbe:sy5lwooxchqu5mrzwr3mgc7qtwfjgmpz2srdmwpaip4iuj3tn4vq:101:256:2097153 format: chk @@ -858,7 +858,7 @@ vector: zfec: required: 101 segmentSize: 131072 - total: max + total: 256 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== expected: URI:CHK:asmjgajvwpwvylxdcf5rm2gyd4:imwtdi4eqq3wm4bvk6npho3ta2fewceqzl4f7wyyabg5i4s6h3mq:101:256:8388607 format: chk @@ -868,7 +868,7 @@ vector: zfec: required: 101 segmentSize: 131072 - total: max + total: 256 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== expected: URI:CHK:yrznm2uuybfbda6jlj7tqs3yum:zy45lpc7ilfzgltxxqvy55wcokiay2fbbbv2wdvu6ubjluh4gf4a:101:256:8388609 format: chk @@ -878,7 +878,7 @@ vector: zfec: required: 101 segmentSize: 131072 - total: max + total: 256 - convergence: ZOyIygCyaOW6GjVnihtTFg== expected: URI:CHK:ihyeqq5peg2pdyw45ct5mmj4ay:2rzhndv6o4knlvo4po6rvbnknlcitasseea3mirqjor5c7quh5ra:101:256:1024 format: chk @@ -888,7 +888,7 @@ vector: zfec: required: 101 segmentSize: 131072 - total: max + total: 256 - convergence: ZOyIygCyaOW6GjVnihtTFg== expected: URI:CHK:mvk5f2mghudk7xnknqddhvnl4m:c74myeugtix7e7l6uwybtrdtfselha2qrcc7hhbymqr6brtg4cea:101:256:4096 format: chk @@ -898,7 +898,7 @@ vector: zfec: required: 101 segmentSize: 131072 - total: max + total: 256 - convergence: ZOyIygCyaOW6GjVnihtTFg== expected: URI:CHK:rvytwcc4sbmmkrdwh6b23ckv5u:klk5nqpbv37pr2qavcebcugfnikarjdefjlseo3ru5uv7guai22q:101:256:131071 format: chk @@ -908,7 +908,7 @@ vector: zfec: required: 101 segmentSize: 131072 - total: max + total: 256 - convergence: ZOyIygCyaOW6GjVnihtTFg== expected: URI:CHK:ocqx55dopakqnzyui6q6euz4gm:wklziyoctwrz5zjfg5wfejz4bnaagtg57fgjlspl4ba6ij6n2obq:101:256:131073 format: chk @@ -918,7 +918,7 @@ vector: zfec: required: 101 segmentSize: 131072 - total: max + total: 256 - convergence: ZOyIygCyaOW6GjVnihtTFg== expected: URI:CHK:lf6qub3bkdhfozpdis4bmorj7q:zte3u3ohbuom3iqjjx4odrjylcpw5myw4n4kwf4zcaif36vngf6q:101:256:2097151 format: chk @@ -928,7 +928,7 @@ vector: zfec: required: 101 segmentSize: 131072 - total: max + total: 256 - convergence: ZOyIygCyaOW6GjVnihtTFg== expected: URI:CHK:a2qoww5xx6so5uftwg4msdtrpy:h7hwsjco5i3jkd7icytrtsuzcjvjzlnkr2wwau2tjlj5wehjrokq:101:256:2097153 format: chk @@ -938,7 +938,7 @@ vector: zfec: required: 101 segmentSize: 131072 - total: max + total: 256 - convergence: ZOyIygCyaOW6GjVnihtTFg== expected: URI:CHK:tg5w4m6hi5ezeapuix3lrrbbtq:7kerk5plqpsd2upnole3ifxoegxdux7prazbvzeu2kyilc6idk7a:101:256:8388607 format: chk @@ -948,7 +948,7 @@ vector: zfec: required: 101 segmentSize: 131072 - total: max + total: 256 - convergence: ZOyIygCyaOW6GjVnihtTFg== expected: URI:CHK:ioridipksj6o7ph3bcf37vslmi:pzhkwm4ry67ohj6dcs7zwnsjrvfqzam6tuitrns5vuo3uoomau6q:101:256:8388609 format: chk @@ -958,5 +958,5 @@ vector: zfec: required: 101 segmentSize: 131072 - total: max -version: '2023-01-03' + total: 256 +version: '2023-01-12' diff --git a/integration/vectors.py b/integration/vectors.py index 3240f9c27..66c9fa88a 100644 --- a/integration/vectors.py +++ b/integration/vectors.py @@ -14,7 +14,9 @@ from yaml import safe_load from pathlib import Path from base64 import b64encode, b64decode -DATA_PATH: Path = Path(__file__).parent / "test_vectors.yaml" +from twisted.python.filepath import FilePath + +DATA_PATH: FilePath = FilePath(__file__).sibling("test_vectors.yaml") @frozen class Sample: From 0eee22cccf8af1a95b176c2a91b4de91539f9411 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 13 Jan 2023 09:53:38 -0500 Subject: [PATCH 1370/2309] Pin older charset_normalizer. --- newsfragments/3966.bugfix | 1 + setup.py | 7 +++++++ 2 files changed, 8 insertions(+) create mode 100644 newsfragments/3966.bugfix diff --git a/newsfragments/3966.bugfix b/newsfragments/3966.bugfix new file mode 100644 index 000000000..ead94c47c --- /dev/null +++ b/newsfragments/3966.bugfix @@ -0,0 +1 @@ +Fix incompatibility with newer versions of the transitive charset_normalizer dependency when using PyInstaller. \ No newline at end of file diff --git a/setup.py b/setup.py index 3681dc441..a9b42d522 100644 --- a/setup.py +++ b/setup.py @@ -148,6 +148,13 @@ install_requires = [ # for pid-file support "psutil", "filelock", + + # treq needs requests, requests needs charset_normalizer, + # charset_normalizer breaks PyInstaller + # (https://github.com/Ousret/charset_normalizer/issues/253). So work around + # this by using a lower version number. Once upstream issue is fixed, or + # requests drops charset_normalizer, this can go away. + "charset_normalizer < 3", ] setup_requires = [ From e64c6b02e6370290a9c6d9ca17257b2e736f693a Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 13 Jan 2023 10:29:22 -0500 Subject: [PATCH 1371/2309] Fix a typo --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 011bfc1ea..4d67f09bd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -92,7 +92,7 @@ jobs: # On Windows, a non-blocking pipe might respond (when emulating Unix-y # API) with ENOSPC to indicate buffer full. Trial doesn't handle this # well, so it breaks test runs. To attempt to solve this, we pipe the - # output through pipethrough that will hopefully be able to do the right + # output through passthrough.py that will hopefully be able to do the right # thing by using Windows APIs. - name: Run tox for corresponding Python version if: ${{ contains(matrix.os, 'windows') }} From 18278344344561745ab407c65dc37d7b3868fb84 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 13 Jan 2023 21:14:37 -0500 Subject: [PATCH 1372/2309] Re-generate vectors with a very small CHK --- integration/test_vectors.py | 3 + integration/test_vectors.yaml | 120 ++++++++++++++++++++++++++++++++++ 2 files changed, 123 insertions(+) diff --git a/integration/test_vectors.py b/integration/test_vectors.py index aa5cde42b..7a2646b62 100644 --- a/integration/test_vectors.py +++ b/integration/test_vectors.py @@ -41,6 +41,7 @@ CONVERGENCE_SECRETS = [ # # 1. Some cases smaller than one "segment" (128k). # This covers shrinking of some parameters to match data size. +# This includes one case of the smallest possible CHK. # # 2. Some cases right on the edges of integer segment multiples. # Because boundaries are tricky. @@ -52,6 +53,8 @@ CONVERGENCE_SECRETS = [ SEGMENT_SIZE = 128 * 1024 OBJECT_DESCRIPTIONS = [ + # The smallest possible. 55 bytes and smaller are LIT. + vectors.Sample(b"a", 56), vectors.Sample(b"a", 1024), vectors.Sample(b"c", 4096), vectors.Sample(digest(b"foo"), SEGMENT_SIZE - 1), diff --git a/integration/test_vectors.yaml b/integration/test_vectors.yaml index 39801b3fe..8c3bacbf8 100644 --- a/integration/test_vectors.yaml +++ b/integration/test_vectors.yaml @@ -1,4 +1,14 @@ vector: +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:CHK:yzxcoagbetwet65ltjpbqyli3m:6b7inuiha2xdtgqzd55i6aeggutnxzr6qfwpv2ep5xlln6pgef7a:1:1:56 + format: chk + sample: + length: 56 + seed: YQ== + zfec: + required: 1 + segmentSize: 131072 + total: 1 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== expected: URI:CHK:mri732rh3meyh4drikau3a24ba:6tsj5wvcp6szdhmrbu5bea57wduoza64y6nd2lm7aleqpsxjm5la:1:1:1024 format: chk @@ -79,6 +89,16 @@ vector: required: 1 segmentSize: 131072 total: 1 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:CHK:tkjwggbz6p4wvuipe3gtmgfmsu:cnbcggp4scaxcde6vtfzga7bsuja4qjfbtv23xhaofwhbw5exjrq:1:1:56 + format: chk + sample: + length: 56 + seed: YQ== + zfec: + required: 1 + segmentSize: 131072 + total: 1 - convergence: ZOyIygCyaOW6GjVnihtTFg== expected: URI:CHK:nnv4vrtlxmzkurfzvonj22leua:ywcyijrfnwykraku56dq7v3o4ts3xsxqfgmk3kgzwhq2cpest4pa:1:1:1024 format: chk @@ -159,6 +179,16 @@ vector: required: 1 segmentSize: 131072 total: 1 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:CHK:rl4bzmselnuezmapjlzssnqg2e:p7kvin2fnemochuxsmh6ot75qpbfhrscbxi5i74bhqdhzcy6i5eq:1:3:56 + format: chk + sample: + length: 56 + seed: YQ== + zfec: + required: 1 + segmentSize: 131072 + total: 3 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== expected: URI:CHK:ycvogzi6wllnq2bkx3t6zdtwju:um42l4yen7jiwdfgirvedtty3tt3xuhjiyxzqoourvughtxjar3q:1:3:1024 format: chk @@ -239,6 +269,16 @@ vector: required: 1 segmentSize: 131072 total: 3 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:CHK:vvh7fppprucnsblhp2nq7ixyze:ck2nmw5uynyyhbr3s7h5ciffgzw766bt3e5n3qx7r4njjzqzkn4a:1:3:56 + format: chk + sample: + length: 56 + seed: YQ== + zfec: + required: 1 + segmentSize: 131072 + total: 3 - convergence: ZOyIygCyaOW6GjVnihtTFg== expected: URI:CHK:pbfcdvxhbikxd2hcc43oel3v3e:y5txpmiptmoz36ionmki3p6krmdbiqasw2v3wdq4ia5lmrhh33lq:1:3:1024 format: chk @@ -319,6 +359,16 @@ vector: required: 1 segmentSize: 131072 total: 3 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:CHK:r3orpfw7tqaxbgxc62hpmhihye:meyeb5lt7i4iyewahb6lxzohn6jxrqgi6b73zv4gxzirykpnyd7a:2:3:56 + format: chk + sample: + length: 56 + seed: YQ== + zfec: + required: 2 + segmentSize: 131072 + total: 3 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== expected: URI:CHK:jdm6ytmx2i3ya2bllsdzxurjtu:iqr6tyysaseegzfurhuywy3mbbwkbbsov5bt3fo6oazpyv7olvda:2:3:1024 format: chk @@ -399,6 +449,16 @@ vector: required: 2 segmentSize: 131072 total: 3 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:CHK:sxrylthoxskesmlhrfuxnifkdu:kizgaeiazgpjgsffkotumbu2dtxziezw73ybwo5pfzleuckqaiwq:2:3:56 + format: chk + sample: + length: 56 + seed: YQ== + zfec: + required: 2 + segmentSize: 131072 + total: 3 - convergence: ZOyIygCyaOW6GjVnihtTFg== expected: URI:CHK:ld3thziutpaqv25nbtoqebhtru:ri2obsvzl2etyuv3qnchn2wvw5mh5rjuvbjaqkqbvyprn7xwamoq:2:3:1024 format: chk @@ -479,6 +539,16 @@ vector: required: 2 segmentSize: 131072 total: 3 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:CHK:hah7mxwfpqemm7icdh3hwsa5fa:6epvxt2uxh42obpnfn4wkrplqml7voh7aqpnqnapu7ffcyn2hk3q:3:10:56 + format: chk + sample: + length: 56 + seed: YQ== + zfec: + required: 3 + segmentSize: 131072 + total: 10 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== expected: URI:CHK:dx7tvyr2fc4u7lxjc6kehq2svq:tiy4qh2g6lqejxcaym3rr7ymkdkinn4qised6kgxloj7sptsqu4a:3:10:1024 format: chk @@ -559,6 +629,16 @@ vector: required: 3 segmentSize: 131072 total: 10 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:CHK:xt3owduddxqodfhp3c2fu6yzr4:bvk5d2igrtlo64kbpyypajyi6bjzrnvl2blcavxhguiupjthelra:3:10:56 + format: chk + sample: + length: 56 + seed: YQ== + zfec: + required: 3 + segmentSize: 131072 + total: 10 - convergence: ZOyIygCyaOW6GjVnihtTFg== expected: URI:CHK:x55owxzhsezfoayaxe7jpwnove:vawdgtqpxyntgy5i2po2twgelrynkfcjgwm7publnlbdp7hpqmfa:3:10:1024 format: chk @@ -639,6 +719,16 @@ vector: required: 3 segmentSize: 131072 total: 10 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:CHK:sghm3tydjjaadmiiuda3flhmne:5fqqykrndg5kydmhwetwqdzria4ap475j2qfmq2gmklzop6y6tla:71:255:56 + format: chk + sample: + length: 56 + seed: YQ== + zfec: + required: 71 + segmentSize: 131072 + total: 255 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== expected: URI:CHK:2wvrzaobtfkoiqeqwryjzvxatq:7zzaycbdhepzaqtdrku2sbb572h64ggwstu47osa2gn4ol3eglmq:71:255:1024 format: chk @@ -719,6 +809,16 @@ vector: required: 71 segmentSize: 131072 total: 255 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:CHK:hz3x6hgz6osbyo5he664ntvxiu:7hpheae4wou6b2davtrizoumqqh2k3vo25erhpgrq5w45txmjeka:71:255:56 + format: chk + sample: + length: 56 + seed: YQ== + zfec: + required: 71 + segmentSize: 131072 + total: 255 - convergence: ZOyIygCyaOW6GjVnihtTFg== expected: URI:CHK:wxmzfzvgaqm3xfzgaqrlndgolm:6t3tqcphsutigxnnzyp2xx3afklcpeytnh543mpqptquvqqahwea:71:255:1024 format: chk @@ -799,6 +899,16 @@ vector: required: 71 segmentSize: 131072 total: 255 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:CHK:ljiodj4tijkfzej5nqdk5h7cnu:3fb7y2osytliwli3oez3y7ece6rjwfrpa2zn7uwrfmn22cisk4aa:101:256:56 + format: chk + sample: + length: 56 + seed: YQ== + zfec: + required: 101 + segmentSize: 131072 + total: 256 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== expected: URI:CHK:mftnvh5l3brshwoupvu6kz35cy:sncfokby2tjykbr3zgi3i6lhgerg4kz7fsrslrlxdy7upwkzq6lq:101:256:1024 format: chk @@ -879,6 +989,16 @@ vector: required: 101 segmentSize: 131072 total: 256 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:CHK:td223psbzins2k6m4frmfw26xy:opgmb6zhnwsksydgjwzpfdoz7epm4ynzmvkjuw6s2jntioqk72ga:101:256:56 + format: chk + sample: + length: 56 + seed: YQ== + zfec: + required: 101 + segmentSize: 131072 + total: 256 - convergence: ZOyIygCyaOW6GjVnihtTFg== expected: URI:CHK:ihyeqq5peg2pdyw45ct5mmj4ay:2rzhndv6o4knlvo4po6rvbnknlcitasseea3mirqjor5c7quh5ra:101:256:1024 format: chk From 4eec8113ee5225b53355d46ab6d44d0f68b6d33c Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 16 Jan 2023 15:53:24 -0500 Subject: [PATCH 1373/2309] reproducible ssk vectors --- integration/test_vectors.py | 49 +- integration/test_vectors.yaml | 1082 -- integration/util.py | 106 +- integration/vectors.py | 42 +- integration/vectors/test_vectors.yaml | 16202 ++++++++++++++++++++++++ 5 files changed, 16362 insertions(+), 1119 deletions(-) delete mode 100644 integration/test_vectors.yaml create mode 100755 integration/vectors/test_vectors.yaml diff --git a/integration/test_vectors.py b/integration/test_vectors.py index 7a2646b62..4c21bb093 100644 --- a/integration/test_vectors.py +++ b/integration/test_vectors.py @@ -9,11 +9,13 @@ from hashlib import sha256 from itertools import starmap, product from yaml import safe_dump +from attrs import evolve + from pytest import mark from pytest_twisted import ensureDeferred from . import vectors -from .util import reconfigure, upload, TahoeProcess +from .util import CHK, SSK, reconfigure, upload, TahoeProcess def digest(bs: bytes) -> bytes: """ @@ -75,9 +77,11 @@ ZFEC_PARAMS = [ ] FORMATS = [ - "chk", - # "sdmf", - # "mdmf", + CHK(), + # These start out unaware of a key but various keys will be supplied + # during generation. + SSK(name="sdmf", key=None), + SSK(name="mdmf", key=None), ] @mark.parametrize('convergence', CONVERGENCE_SECRETS) @@ -89,18 +93,15 @@ def test_convergence(convergence): assert len(convergence) == 16, "Convergence secret must by 16 bytes" -@mark.parametrize('seed_params', ZFEC_PARAMS) -@mark.parametrize('convergence', CONVERGENCE_SECRETS) -@mark.parametrize('seed_data', OBJECT_DESCRIPTIONS) -@mark.parametrize('fmt', FORMATS) +@mark.parametrize('case_and_expected', vectors.capabilities.items()) @ensureDeferred -async def test_capability(reactor, request, alice, seed_params, convergence, seed_data, fmt): +async def test_capability(reactor, request, alice, case_and_expected): """ The capability that results from uploading certain well-known data with certain well-known parameters results in exactly the previously computed value. """ - case = vectors.Case(seed_params, convergence, seed_data, fmt) + case, expected = case_and_expected # rewrite alice's config to match params and convergence await reconfigure(reactor, request, alice, (1, case.params.required, case.params.total), case.convergence) @@ -109,7 +110,6 @@ async def test_capability(reactor, request, alice, seed_params, convergence, see actual = upload(alice, case.fmt, case.data) # compare the resulting cap to the expected result - expected = vectors.capabilities[case] assert actual == expected @@ -130,13 +130,27 @@ async def test_generate(reactor, request, alice): OBJECT_DESCRIPTIONS, FORMATS, )) - results = generate(reactor, request, alice, space) - vectors.DATA_PATH.setContent(safe_dump({ - "version": "2023-01-12", + iterresults = generate(reactor, request, alice, space) + + # Update the output file with results as they become available. + results = [] + async for result in iterresults: + results.append(result) + write_results(vectors.DATA_PATH, results) + +def write_results(path: FilePath, results: list[tuple[Case, str]]) -> None: + """ + Save the given results. + """ + path.setContent(safe_dump({ + "version": vectors.CURRENT_VERSION, "vector": [ { "convergence": vectors.encode_bytes(case.convergence), - "format": case.fmt, + "format": { + "kind": case.fmt.kind, + "params": case.fmt.to_json(), + }, "sample": { "seed": vectors.encode_bytes(case.seed_data.seed), "length": case.seed_data.length, @@ -148,12 +162,11 @@ async def test_generate(reactor, request, alice): }, "expected": cap, } - async for (case, cap) + for (case, cap) in results ], }).encode("ascii")) - async def generate( reactor, request, @@ -189,5 +202,7 @@ async def generate( case.convergence ) + # Give the format a chance to make an RSA key if it needs it. + case = evolve(case, fmt=case.fmt.customize()) cap = upload(alice, case.fmt, case.data) yield case, cap diff --git a/integration/test_vectors.yaml b/integration/test_vectors.yaml deleted file mode 100644 index 8c3bacbf8..000000000 --- a/integration/test_vectors.yaml +++ /dev/null @@ -1,1082 +0,0 @@ -vector: -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:CHK:yzxcoagbetwet65ltjpbqyli3m:6b7inuiha2xdtgqzd55i6aeggutnxzr6qfwpv2ep5xlln6pgef7a:1:1:56 - format: chk - sample: - length: 56 - seed: YQ== - zfec: - required: 1 - segmentSize: 131072 - total: 1 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:CHK:mri732rh3meyh4drikau3a24ba:6tsj5wvcp6szdhmrbu5bea57wduoza64y6nd2lm7aleqpsxjm5la:1:1:1024 - format: chk - sample: - length: 1024 - seed: YQ== - zfec: - required: 1 - segmentSize: 131072 - total: 1 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:CHK:e5tpbcmalcq6nn2zhd3qvg4chy:emzsvv2xnkhhrj2oatds5hf2cney25awi56ybeq4ofiyehzyakua:1:1:4096 - format: chk - sample: - length: 4096 - seed: Yw== - zfec: - required: 1 - segmentSize: 131072 - total: 1 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:CHK:ok3slnjd3e56za3iot74audhl4:2mjvrb455yldouoxwzbx4sbsysowipqje6ifa4pmbzqahj5j4mnq:1:1:131071 - format: chk - sample: - length: 131071 - seed: LCa0a2j/xo/5m0U8HTBBNBNCLXBkg7+g+YpeiGJm564= - zfec: - required: 1 - segmentSize: 131072 - total: 1 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:CHK:worle55uksa2uqqeebm4yxnihu:vta76jbmejt2pxx4prqa75xawpdtx42cmzzhappeetzjksym37wq:1:1:131073 - format: chk - sample: - length: 131073 - seed: /N4rLtula/QIYB+3If6bXDONEO5CnqBPrlURto+/j7k= - zfec: - required: 1 - segmentSize: 131072 - total: 1 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:CHK:6kccrgbtmmprqe4jcfi7vf6v74:wpivl2evi25yfl4tzbbj6vp6nzk4vxl6lbongkac7vl3escvopsa:1:1:2097151 - format: chk - sample: - length: 2097151 - seed: uqWglk0zIPvAxqkiFARTyFE+okq4/QV3A0gEqWckgJY= - zfec: - required: 1 - segmentSize: 131072 - total: 1 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:CHK:ofgig3loex6pev2eymmvohv7wq:yasbfedqnueaajdcavuba7kxbdqzn7e6zx3y6cbvucgx433vlzcq:1:1:2097153 - format: chk - sample: - length: 2097153 - seed: BTBX/ampNfLU+ox7xipBGiaSbgC0kcB8Gy7BkJB4oKI= - zfec: - required: 1 - segmentSize: 131072 - total: 1 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:CHK:scwepfpkvhuc3fc5jkpmrodsfi:wui2ixaxyac76mbxtr6diw2ajky75wehk57f2kqgytddnpo4rozq:1:1:8388607 - format: chk - sample: - length: 8388607 - seed: w6uP8Tcg6K2QR905Rms8iXTlksL6OD1KOWBxTK7wxPI= - zfec: - required: 1 - segmentSize: 131072 - total: 1 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:CHK:5eeprb6lfxclwt7fieskd2euly:ffjgjbrxfl6d2ug2a63iaesxtrqa5pk3yaagfcldqvo7x4ok7ykq:1:1:8388609 - format: chk - sample: - length: 8388609 - seed: yPi3JHKKbWaEEG5eZOlM6BHJll0Z3UTdBzz4bPQ7wjg= - zfec: - required: 1 - segmentSize: 131072 - total: 1 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:CHK:tkjwggbz6p4wvuipe3gtmgfmsu:cnbcggp4scaxcde6vtfzga7bsuja4qjfbtv23xhaofwhbw5exjrq:1:1:56 - format: chk - sample: - length: 56 - seed: YQ== - zfec: - required: 1 - segmentSize: 131072 - total: 1 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:CHK:nnv4vrtlxmzkurfzvonj22leua:ywcyijrfnwykraku56dq7v3o4ts3xsxqfgmk3kgzwhq2cpest4pa:1:1:1024 - format: chk - sample: - length: 1024 - seed: YQ== - zfec: - required: 1 - segmentSize: 131072 - total: 1 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:CHK:wlszole5m2emf6wbhp3lnlyfxq:irdttvny74gxdrcotzfjslfq7p24kyiigyrm5shhw7zyj3hsiuba:1:1:4096 - format: chk - sample: - length: 4096 - seed: Yw== - zfec: - required: 1 - segmentSize: 131072 - total: 1 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:CHK:4ewm23jvdtm2i5xf4gck26wg5y:7cujxxc34mkfkmwhbemqtuixuektknmhhxlmujcekibf5amfwwga:1:1:131071 - format: chk - sample: - length: 131071 - seed: LCa0a2j/xo/5m0U8HTBBNBNCLXBkg7+g+YpeiGJm564= - zfec: - required: 1 - segmentSize: 131072 - total: 1 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:CHK:heczpdxphw5frp3ri5sh2hpqei:6wflx6lphy5mhtpfb2abznoa3yk27ynbqbzcecs362miu72vkowq:1:1:131073 - format: chk - sample: - length: 131073 - seed: /N4rLtula/QIYB+3If6bXDONEO5CnqBPrlURto+/j7k= - zfec: - required: 1 - segmentSize: 131072 - total: 1 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:CHK:mz5siv27pqttsl2f4vcqmxju64:rvxhulxtufho6pdbwj7rifneb6pei5fl4fqptcce7jdkbyrjfaya:1:1:2097151 - format: chk - sample: - length: 2097151 - seed: uqWglk0zIPvAxqkiFARTyFE+okq4/QV3A0gEqWckgJY= - zfec: - required: 1 - segmentSize: 131072 - total: 1 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:CHK:shbt5viqjzuewblgt6qeijry6a:je3omw53tmmluz6fvqupx4uh4jaejc3fcvfjt56rfzokzhgdczvq:1:1:2097153 - format: chk - sample: - length: 2097153 - seed: BTBX/ampNfLU+ox7xipBGiaSbgC0kcB8Gy7BkJB4oKI= - zfec: - required: 1 - segmentSize: 131072 - total: 1 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:CHK:n2642ofjfr5qljptdwjbwjz6fq:uwtjrewrev42yvez7br2574xu5vleallwnyddjaxqquvouv3dyra:1:1:8388607 - format: chk - sample: - length: 8388607 - seed: w6uP8Tcg6K2QR905Rms8iXTlksL6OD1KOWBxTK7wxPI= - zfec: - required: 1 - segmentSize: 131072 - total: 1 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:CHK:nktoeeuydph4acj2fux4axyaj4:g3fvjwanenwsgdcn3oxnau5gtbdmzbnbevlrzrb5qe4yukuwhejq:1:1:8388609 - format: chk - sample: - length: 8388609 - seed: yPi3JHKKbWaEEG5eZOlM6BHJll0Z3UTdBzz4bPQ7wjg= - zfec: - required: 1 - segmentSize: 131072 - total: 1 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:CHK:rl4bzmselnuezmapjlzssnqg2e:p7kvin2fnemochuxsmh6ot75qpbfhrscbxi5i74bhqdhzcy6i5eq:1:3:56 - format: chk - sample: - length: 56 - seed: YQ== - zfec: - required: 1 - segmentSize: 131072 - total: 3 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:CHK:ycvogzi6wllnq2bkx3t6zdtwju:um42l4yen7jiwdfgirvedtty3tt3xuhjiyxzqoourvughtxjar3q:1:3:1024 - format: chk - sample: - length: 1024 - seed: YQ== - zfec: - required: 1 - segmentSize: 131072 - total: 3 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:CHK:3yjtuv5h2g45g2cncg6hka3euu:bqqhp6u73ldawial7rtrfomx2qkmboyvqve6ywo2jpvfbu7zptga:1:3:4096 - format: chk - sample: - length: 4096 - seed: Yw== - zfec: - required: 1 - segmentSize: 131072 - total: 3 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:CHK:annnqmu72p7iels5loqwlxt2zq:s7wv3jfi2hlpcvp4dnloc4eex6vre42kwiel46achaie5n5uodqa:1:3:131071 - format: chk - sample: - length: 131071 - seed: LCa0a2j/xo/5m0U8HTBBNBNCLXBkg7+g+YpeiGJm564= - zfec: - required: 1 - segmentSize: 131072 - total: 3 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:CHK:rhkln7unkktot72mit5dmuqbdy:ilo6u6hugipdimyrzrvlam47xsmp3ur2lwnrtbecmvocb2664zxq:1:3:131073 - format: chk - sample: - length: 131073 - seed: /N4rLtula/QIYB+3If6bXDONEO5CnqBPrlURto+/j7k= - zfec: - required: 1 - segmentSize: 131072 - total: 3 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:CHK:7yjcwwt6454lbv3pni5mjofsxe:y5nwpzwmvpvr3gqxnykjixprpxw3w6qyqmszf7ijyxnm3wl6f5oa:1:3:2097151 - format: chk - sample: - length: 2097151 - seed: uqWglk0zIPvAxqkiFARTyFE+okq4/QV3A0gEqWckgJY= - zfec: - required: 1 - segmentSize: 131072 - total: 3 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:CHK:r7wva6tisq6m5zszgr7seu77cm:oqlgdw3hi72qtahpsi3h3yryxpqdshagvt6xnobsppkwp7ic4cia:1:3:2097153 - format: chk - sample: - length: 2097153 - seed: BTBX/ampNfLU+ox7xipBGiaSbgC0kcB8Gy7BkJB4oKI= - zfec: - required: 1 - segmentSize: 131072 - total: 3 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:CHK:mz7pzh24u6flkfmfvvokdwmriu:5mzo57de2ywsdqkdvm7557gcikkpjqbmj65mk235t2xyoejtpbta:1:3:8388607 - format: chk - sample: - length: 8388607 - seed: w6uP8Tcg6K2QR905Rms8iXTlksL6OD1KOWBxTK7wxPI= - zfec: - required: 1 - segmentSize: 131072 - total: 3 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:CHK:54sq6lrqwqlpxicg747gdls6ri:2aaxyrdytn7r74my36ek434hlbhe6glrgr2ic5vvpc5bbdxmvo6a:1:3:8388609 - format: chk - sample: - length: 8388609 - seed: yPi3JHKKbWaEEG5eZOlM6BHJll0Z3UTdBzz4bPQ7wjg= - zfec: - required: 1 - segmentSize: 131072 - total: 3 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:CHK:vvh7fppprucnsblhp2nq7ixyze:ck2nmw5uynyyhbr3s7h5ciffgzw766bt3e5n3qx7r4njjzqzkn4a:1:3:56 - format: chk - sample: - length: 56 - seed: YQ== - zfec: - required: 1 - segmentSize: 131072 - total: 3 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:CHK:pbfcdvxhbikxd2hcc43oel3v3e:y5txpmiptmoz36ionmki3p6krmdbiqasw2v3wdq4ia5lmrhh33lq:1:3:1024 - format: chk - sample: - length: 1024 - seed: YQ== - zfec: - required: 1 - segmentSize: 131072 - total: 3 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:CHK:vkdc7iae4pgkdomhbbb554kv6a:g5xkatxv2entfssfh5eqgexb5j3qs6jvohofjqo5p5erlxlx2dga:1:3:4096 - format: chk - sample: - length: 4096 - seed: Yw== - zfec: - required: 1 - segmentSize: 131072 - total: 3 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:CHK:zvla2jyu5fqlb2s63zftyfjnla:yqspaexhew55cvv7w2m6czezhzkqnyk5fea4gvnplc5za4xlvbva:1:3:131071 - format: chk - sample: - length: 131071 - seed: LCa0a2j/xo/5m0U8HTBBNBNCLXBkg7+g+YpeiGJm564= - zfec: - required: 1 - segmentSize: 131072 - total: 3 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:CHK:3idf2xqm5wnw2q3nbva5vmpaiq:7nmkh5q55omjloezibvyzvveq4gqeaivei5ypfm5vfuugwiz6hpa:1:3:131073 - format: chk - sample: - length: 131073 - seed: /N4rLtula/QIYB+3If6bXDONEO5CnqBPrlURto+/j7k= - zfec: - required: 1 - segmentSize: 131072 - total: 3 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:CHK:o7xdokumtcukutzgmmpfme4nnm:nn3s5gau5dychy6wlodwriuit4wyavfze6bo4icywcbhdb4ccxla:1:3:2097151 - format: chk - sample: - length: 2097151 - seed: uqWglk0zIPvAxqkiFARTyFE+okq4/QV3A0gEqWckgJY= - zfec: - required: 1 - segmentSize: 131072 - total: 3 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:CHK:vws2xrl2nch2hsptqb36yqqu3e:fhujsv5rkhyiqstrktjvfy5235c7gcwekyd3gux3bv2glsgstjga:1:3:2097153 - format: chk - sample: - length: 2097153 - seed: BTBX/ampNfLU+ox7xipBGiaSbgC0kcB8Gy7BkJB4oKI= - zfec: - required: 1 - segmentSize: 131072 - total: 3 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:CHK:5anac46dpfsjumutzsoovvtlpe:yuc3gqe7m6s6xp7k3uodcg2g6k2bac32gt5amu7gdx7k43qnudxa:1:3:8388607 - format: chk - sample: - length: 8388607 - seed: w6uP8Tcg6K2QR905Rms8iXTlksL6OD1KOWBxTK7wxPI= - zfec: - required: 1 - segmentSize: 131072 - total: 3 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:CHK:fb6g3fkfbtlwzheujzvc2dn7dq:644je55ri6q5halshuxfesjz2afhqtebetuzqqbfecp6yhqwbb5q:1:3:8388609 - format: chk - sample: - length: 8388609 - seed: yPi3JHKKbWaEEG5eZOlM6BHJll0Z3UTdBzz4bPQ7wjg= - zfec: - required: 1 - segmentSize: 131072 - total: 3 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:CHK:r3orpfw7tqaxbgxc62hpmhihye:meyeb5lt7i4iyewahb6lxzohn6jxrqgi6b73zv4gxzirykpnyd7a:2:3:56 - format: chk - sample: - length: 56 - seed: YQ== - zfec: - required: 2 - segmentSize: 131072 - total: 3 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:CHK:jdm6ytmx2i3ya2bllsdzxurjtu:iqr6tyysaseegzfurhuywy3mbbwkbbsov5bt3fo6oazpyv7olvda:2:3:1024 - format: chk - sample: - length: 1024 - seed: YQ== - zfec: - required: 2 - segmentSize: 131072 - total: 3 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:CHK:7wz5dhkyob3gmkydzgbptk5qku:cl6ovq5a3km7rpfhyb3putg656sp57lhnp7aexbocsbbahdxnfia:2:3:4096 - format: chk - sample: - length: 4096 - seed: Yw== - zfec: - required: 2 - segmentSize: 131072 - total: 3 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:CHK:aagw6esdm2msphvgnr3rlzg43e:zqxtrlee6hh3vtfg6ihtssjqvr27tpvi6gkmngnhiguttjz7yyja:2:3:131071 - format: chk - sample: - length: 131071 - seed: LCa0a2j/xo/5m0U8HTBBNBNCLXBkg7+g+YpeiGJm564= - zfec: - required: 2 - segmentSize: 131072 - total: 3 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:CHK:e6uhixvbm3g5amrzdzd3mnmsqm:4n5aafrb64sqpfnhcdpdfrk5gh2qznprjky6q6jybuuqy2q6z6mq:2:3:131073 - format: chk - sample: - length: 131073 - seed: /N4rLtula/QIYB+3If6bXDONEO5CnqBPrlURto+/j7k= - zfec: - required: 2 - segmentSize: 131072 - total: 3 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:CHK:abfmzuiczjwt6e67f3uoyg5i7q:rk7ie3afnp3xgvnxu5e2vioq476xqwslqxgyx3i5xxevsjhkcvba:2:3:2097151 - format: chk - sample: - length: 2097151 - seed: uqWglk0zIPvAxqkiFARTyFE+okq4/QV3A0gEqWckgJY= - zfec: - required: 2 - segmentSize: 131072 - total: 3 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:CHK:34bzejdzpqinmuzdmqenkidf74:pcecbl4ulygiadgdagonbgqmpb4lomjz4v4vqyssr56reyib4q2q:2:3:2097153 - format: chk - sample: - length: 2097153 - seed: BTBX/ampNfLU+ox7xipBGiaSbgC0kcB8Gy7BkJB4oKI= - zfec: - required: 2 - segmentSize: 131072 - total: 3 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:CHK:ej56vl65kawat62wsxuvbcd2wy:tv65cgghaddq2p7dyvimmff24l34wsx6mhzd3dyprfvrdo52qtma:2:3:8388607 - format: chk - sample: - length: 8388607 - seed: w6uP8Tcg6K2QR905Rms8iXTlksL6OD1KOWBxTK7wxPI= - zfec: - required: 2 - segmentSize: 131072 - total: 3 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:CHK:ufshkmuewwql2xsyiecbjhh7x4:qoywwygjqrmifowob3mvwkhfskk6geomyw5e3qlgqzui3kymu6nq:2:3:8388609 - format: chk - sample: - length: 8388609 - seed: yPi3JHKKbWaEEG5eZOlM6BHJll0Z3UTdBzz4bPQ7wjg= - zfec: - required: 2 - segmentSize: 131072 - total: 3 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:CHK:sxrylthoxskesmlhrfuxnifkdu:kizgaeiazgpjgsffkotumbu2dtxziezw73ybwo5pfzleuckqaiwq:2:3:56 - format: chk - sample: - length: 56 - seed: YQ== - zfec: - required: 2 - segmentSize: 131072 - total: 3 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:CHK:ld3thziutpaqv25nbtoqebhtru:ri2obsvzl2etyuv3qnchn2wvw5mh5rjuvbjaqkqbvyprn7xwamoq:2:3:1024 - format: chk - sample: - length: 1024 - seed: YQ== - zfec: - required: 2 - segmentSize: 131072 - total: 3 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:CHK:pz3mga5pvqd4bpsz7pbyna2qtq:w3ptt7xwwgywod3u2rpgx7iggvm2kpwl7d2ryr3u7mcaxgqotlaq:2:3:4096 - format: chk - sample: - length: 4096 - seed: Yw== - zfec: - required: 2 - segmentSize: 131072 - total: 3 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:CHK:rsh7ysvsgbwh5g3xuj6z7zky3y:ykg5rhzpzqkvdy53qqyghfbcusibynfylggbhypo3iibsrrzvlkq:2:3:131071 - format: chk - sample: - length: 131071 - seed: LCa0a2j/xo/5m0U8HTBBNBNCLXBkg7+g+YpeiGJm564= - zfec: - required: 2 - segmentSize: 131072 - total: 3 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:CHK:7xqejfqfej3u3zp2qymqd2iqeq:po4t2tzkh4d34ku6gdhlocadfwdyweiyckyqz57zdi7ndt4ikj3q:2:3:131073 - format: chk - sample: - length: 131073 - seed: /N4rLtula/QIYB+3If6bXDONEO5CnqBPrlURto+/j7k= - zfec: - required: 2 - segmentSize: 131072 - total: 3 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:CHK:kzhg5zp4hcs75eboack4kqpcdm:g2cwnhpfwxrv3c4lrmpjyj327a274oa5qt4isu4avjixbu7vblrq:2:3:2097151 - format: chk - sample: - length: 2097151 - seed: uqWglk0zIPvAxqkiFARTyFE+okq4/QV3A0gEqWckgJY= - zfec: - required: 2 - segmentSize: 131072 - total: 3 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:CHK:6sf35smixwvpapf2zw7sllxet4:ks5zd3hkd7ppwobhnymezckszexpychngnkipxfvmlcng5fgjbea:2:3:2097153 - format: chk - sample: - length: 2097153 - seed: BTBX/ampNfLU+ox7xipBGiaSbgC0kcB8Gy7BkJB4oKI= - zfec: - required: 2 - segmentSize: 131072 - total: 3 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:CHK:dnmjqjku5vyrkeaulae4bt6juu:d47airwaqedeji2omacwsp6yfqp5mpurj776gmzvq43c6dovtdca:2:3:8388607 - format: chk - sample: - length: 8388607 - seed: w6uP8Tcg6K2QR905Rms8iXTlksL6OD1KOWBxTK7wxPI= - zfec: - required: 2 - segmentSize: 131072 - total: 3 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:CHK:bfjedwcwjehjzpwjsgwkfk7rja:jpwea2sgz4hfohqab642yj4cmrh64w6nfo7lv3mtacfzicurqkva:2:3:8388609 - format: chk - sample: - length: 8388609 - seed: yPi3JHKKbWaEEG5eZOlM6BHJll0Z3UTdBzz4bPQ7wjg= - zfec: - required: 2 - segmentSize: 131072 - total: 3 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:CHK:hah7mxwfpqemm7icdh3hwsa5fa:6epvxt2uxh42obpnfn4wkrplqml7voh7aqpnqnapu7ffcyn2hk3q:3:10:56 - format: chk - sample: - length: 56 - seed: YQ== - zfec: - required: 3 - segmentSize: 131072 - total: 10 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:CHK:dx7tvyr2fc4u7lxjc6kehq2svq:tiy4qh2g6lqejxcaym3rr7ymkdkinn4qised6kgxloj7sptsqu4a:3:10:1024 - format: chk - sample: - length: 1024 - seed: YQ== - zfec: - required: 3 - segmentSize: 131072 - total: 10 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:CHK:yo75evk4cte3b7rdw72zxvl5ye:ex6h7ff7nclucjtsqwgwu33qgmb67t4ezbrki4zbgurwn2ct6bbq:3:10:4096 - format: chk - sample: - length: 4096 - seed: Yw== - zfec: - required: 3 - segmentSize: 131072 - total: 10 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:CHK:4gokef54smahrbfr4kq3jhc4zq:owpwwfp5gof2vhly5u6jdnbsfuwwwhqkazpsbeg3nldxv5pse2iq:3:10:131071 - format: chk - sample: - length: 131071 - seed: LCa0a2j/xo/5m0U8HTBBNBNCLXBkg7+g+YpeiGJm564= - zfec: - required: 3 - segmentSize: 131072 - total: 10 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:CHK:7vfgl5cv4nlzqx35z4uthjv36y:nnueftbzxfz6u5yjxwwofaxzzft7xss5wzfh66rrcwv2zwrm63sa:3:10:131073 - format: chk - sample: - length: 131073 - seed: /N4rLtula/QIYB+3If6bXDONEO5CnqBPrlURto+/j7k= - zfec: - required: 3 - segmentSize: 131072 - total: 10 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:CHK:cy7fmjsldeakhfd4psbmghmqyi:6a6uvyai4jkzz6hj5yjugm6uie5etvymcudgiwwjh47apz636zwa:3:10:2097151 - format: chk - sample: - length: 2097151 - seed: uqWglk0zIPvAxqkiFARTyFE+okq4/QV3A0gEqWckgJY= - zfec: - required: 3 - segmentSize: 131072 - total: 10 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:CHK:ptsofqwylmkvzmuvrw5n34j3ma:ky2fs7xrlke64w6kfmhzsuilzxbhfrwwzkxih4rykpbxrr3bxhiq:3:10:2097153 - format: chk - sample: - length: 2097153 - seed: BTBX/ampNfLU+ox7xipBGiaSbgC0kcB8Gy7BkJB4oKI= - zfec: - required: 3 - segmentSize: 131072 - total: 10 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:CHK:tyt4hfvn44igztqh5zeiusggcm:ex3gzzn7byvhajithwbo7c3p7qwqxmnactaxxsxsxowkafyjrf7q:3:10:8388607 - format: chk - sample: - length: 8388607 - seed: w6uP8Tcg6K2QR905Rms8iXTlksL6OD1KOWBxTK7wxPI= - zfec: - required: 3 - segmentSize: 131072 - total: 10 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:CHK:xymheose4rdlspgydkzr4nqkre:z3pfrvpq5fdpkoybhdxppwbzrt6ejf26xh6emzlce2sgquljginq:3:10:8388609 - format: chk - sample: - length: 8388609 - seed: yPi3JHKKbWaEEG5eZOlM6BHJll0Z3UTdBzz4bPQ7wjg= - zfec: - required: 3 - segmentSize: 131072 - total: 10 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:CHK:xt3owduddxqodfhp3c2fu6yzr4:bvk5d2igrtlo64kbpyypajyi6bjzrnvl2blcavxhguiupjthelra:3:10:56 - format: chk - sample: - length: 56 - seed: YQ== - zfec: - required: 3 - segmentSize: 131072 - total: 10 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:CHK:x55owxzhsezfoayaxe7jpwnove:vawdgtqpxyntgy5i2po2twgelrynkfcjgwm7publnlbdp7hpqmfa:3:10:1024 - format: chk - sample: - length: 1024 - seed: YQ== - zfec: - required: 3 - segmentSize: 131072 - total: 10 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:CHK:fe64krzyaeff3d4teunjbetkzy:27hrywwaffqiqcgfkmzwbot3iamotr3bey2l5kaladmdmxuaz5ka:3:10:4096 - format: chk - sample: - length: 4096 - seed: Yw== - zfec: - required: 3 - segmentSize: 131072 - total: 10 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:CHK:gvajllsonkuscfemygbnqhq2re:uwyilm5a7so4blhsaielnf34u2qbaqmudd73opjkgodgg3okeaga:3:10:131071 - format: chk - sample: - length: 131071 - seed: LCa0a2j/xo/5m0U8HTBBNBNCLXBkg7+g+YpeiGJm564= - zfec: - required: 3 - segmentSize: 131072 - total: 10 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:CHK:zdmicwopo4p4h4wbfcbnwcrvyi:6qn75anpvs5gls27f4lybisis3udvjfjhatxiny7c72bcbtuztia:3:10:131073 - format: chk - sample: - length: 131073 - seed: /N4rLtula/QIYB+3If6bXDONEO5CnqBPrlURto+/j7k= - zfec: - required: 3 - segmentSize: 131072 - total: 10 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:CHK:bv6qthmetlhdnwc5tfjqamp3yq:ehz4ttd4g7ktkxvbovt562wfedc6jgnt5c6af7wxgp7jbwfwhoaa:3:10:2097151 - format: chk - sample: - length: 2097151 - seed: uqWglk0zIPvAxqkiFARTyFE+okq4/QV3A0gEqWckgJY= - zfec: - required: 3 - segmentSize: 131072 - total: 10 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:CHK:n7ogyjbo5jigvxgel5ll6q4vbe:3yjb3zq5hcdavv7ruefawal6euyvjx3lx7quslvasjellv63cxya:3:10:2097153 - format: chk - sample: - length: 2097153 - seed: BTBX/ampNfLU+ox7xipBGiaSbgC0kcB8Gy7BkJB4oKI= - zfec: - required: 3 - segmentSize: 131072 - total: 10 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:CHK:d2hbvcmbex7fm3qu22yj4qnkh4:2xwqxbawwgn773hht6etox3oypvqqjv2orktnthfo2e7vibko7ha:3:10:8388607 - format: chk - sample: - length: 8388607 - seed: w6uP8Tcg6K2QR905Rms8iXTlksL6OD1KOWBxTK7wxPI= - zfec: - required: 3 - segmentSize: 131072 - total: 10 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:CHK:ccxkyfl2qtqyhihduarpxbdcci:e64be3i2t25selbpc5y2zj443gkdo65chs2o4tpqb7axud5lnxta:3:10:8388609 - format: chk - sample: - length: 8388609 - seed: yPi3JHKKbWaEEG5eZOlM6BHJll0Z3UTdBzz4bPQ7wjg= - zfec: - required: 3 - segmentSize: 131072 - total: 10 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:CHK:sghm3tydjjaadmiiuda3flhmne:5fqqykrndg5kydmhwetwqdzria4ap475j2qfmq2gmklzop6y6tla:71:255:56 - format: chk - sample: - length: 56 - seed: YQ== - zfec: - required: 71 - segmentSize: 131072 - total: 255 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:CHK:2wvrzaobtfkoiqeqwryjzvxatq:7zzaycbdhepzaqtdrku2sbb572h64ggwstu47osa2gn4ol3eglmq:71:255:1024 - format: chk - sample: - length: 1024 - seed: YQ== - zfec: - required: 71 - segmentSize: 131072 - total: 255 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:CHK:7wi5xn2uwerkjzqs5ndn4eimti:llrws4uig2fpi3rtla6lk7m4t6zudjhszdfjgqtihghfrvs2robq:71:255:4096 - format: chk - sample: - length: 4096 - seed: Yw== - zfec: - required: 71 - segmentSize: 131072 - total: 255 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:CHK:6nvs23bhrpiwiz5prqdtvztujy:wlg7g522rpdoitpm4qwhmctrjhnh4zfloiq6uq4tsvaoawg4slpq:71:255:131071 - format: chk - sample: - length: 131071 - seed: LCa0a2j/xo/5m0U8HTBBNBNCLXBkg7+g+YpeiGJm564= - zfec: - required: 71 - segmentSize: 131072 - total: 255 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:CHK:62dtbeowncjj62jnwbggaufvbu:6r2sapg2cm6dvmylodyxabrj63a736uouzsqyacnimgo4svnktva:71:255:131073 - format: chk - sample: - length: 131073 - seed: /N4rLtula/QIYB+3If6bXDONEO5CnqBPrlURto+/j7k= - zfec: - required: 71 - segmentSize: 131072 - total: 255 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:CHK:hk3a5ync7y3dnwowqjqoa34eae:y4f5b2xqa3lslh2vti5fdbho2syqhbj2p6f3enlhxsjbfpt7zuta:71:255:2097151 - format: chk - sample: - length: 2097151 - seed: uqWglk0zIPvAxqkiFARTyFE+okq4/QV3A0gEqWckgJY= - zfec: - required: 71 - segmentSize: 131072 - total: 255 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:CHK:qa4hwhwtd3cpvut74biixlv6ye:uuj4ujg3mcf3lqpuvvwf5pszcvviyb4cm2jjicu7ugiypojjie4q:71:255:2097153 - format: chk - sample: - length: 2097153 - seed: BTBX/ampNfLU+ox7xipBGiaSbgC0kcB8Gy7BkJB4oKI= - zfec: - required: 71 - segmentSize: 131072 - total: 255 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:CHK:r7sv6u3ihzys754twjltuzzpza:tpyoxinubkl4kvg6hipls6ezun7t3yyqj6ix3vp6tvnzqirzzl4a:71:255:8388607 - format: chk - sample: - length: 8388607 - seed: w6uP8Tcg6K2QR905Rms8iXTlksL6OD1KOWBxTK7wxPI= - zfec: - required: 71 - segmentSize: 131072 - total: 255 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:CHK:3esfsjn7csgnyqmq5afbgtinay:xsrhdomrbrtzftg6u4hgipm5tkaomumbdlxi5hpvpvawe3bjikua:71:255:8388609 - format: chk - sample: - length: 8388609 - seed: yPi3JHKKbWaEEG5eZOlM6BHJll0Z3UTdBzz4bPQ7wjg= - zfec: - required: 71 - segmentSize: 131072 - total: 255 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:CHK:hz3x6hgz6osbyo5he664ntvxiu:7hpheae4wou6b2davtrizoumqqh2k3vo25erhpgrq5w45txmjeka:71:255:56 - format: chk - sample: - length: 56 - seed: YQ== - zfec: - required: 71 - segmentSize: 131072 - total: 255 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:CHK:wxmzfzvgaqm3xfzgaqrlndgolm:6t3tqcphsutigxnnzyp2xx3afklcpeytnh543mpqptquvqqahwea:71:255:1024 - format: chk - sample: - length: 1024 - seed: YQ== - zfec: - required: 71 - segmentSize: 131072 - total: 255 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:CHK:r3q3q4f4zbsipxauq6knb5dfoq:lkae5mflammzxkoc7gllt6qqyg5gqdxeejmxtgcscblqh5tv5obq:71:255:4096 - format: chk - sample: - length: 4096 - seed: Yw== - zfec: - required: 71 - segmentSize: 131072 - total: 255 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:CHK:rdrzetasw4w7tuweqpev65d5vq:gzzb2v5fzrduk6kx5xx27mvo6dgnd65ym5lm7re53in7xuudhsxa:71:255:131071 - format: chk - sample: - length: 131071 - seed: LCa0a2j/xo/5m0U8HTBBNBNCLXBkg7+g+YpeiGJm564= - zfec: - required: 71 - segmentSize: 131072 - total: 255 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:CHK:e2bjefibvz4tgu6jgf66gw74bq:mugtu5bx7h5gcivevmh2gmwoc2kkhmobrzshkuj2dgrtm3siysfq:71:255:131073 - format: chk - sample: - length: 131073 - seed: /N4rLtula/QIYB+3If6bXDONEO5CnqBPrlURto+/j7k= - zfec: - required: 71 - segmentSize: 131072 - total: 255 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:CHK:gy7bci6yvllxzvhh45khcwp3u4:qsfjfmcl64zey4k4o5cfnh2i3mzboahf7bhtggceszrulsv537vq:71:255:2097151 - format: chk - sample: - length: 2097151 - seed: uqWglk0zIPvAxqkiFARTyFE+okq4/QV3A0gEqWckgJY= - zfec: - required: 71 - segmentSize: 131072 - total: 255 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:CHK:pcta7uk5cpioxzv5nxxqsogw3q:gnu7fx56k3j72bbevirn4kdu27yfpvttzl3qk2zypwqt7tw4rijq:71:255:2097153 - format: chk - sample: - length: 2097153 - seed: BTBX/ampNfLU+ox7xipBGiaSbgC0kcB8Gy7BkJB4oKI= - zfec: - required: 71 - segmentSize: 131072 - total: 255 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:CHK:znxyqo2qglpkhjafnuesu4foge:trau477bj5u5wjfpy4oq5e3aziajgbpyfjdv4svyf27m7atpa34a:71:255:8388607 - format: chk - sample: - length: 8388607 - seed: w6uP8Tcg6K2QR905Rms8iXTlksL6OD1KOWBxTK7wxPI= - zfec: - required: 71 - segmentSize: 131072 - total: 255 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:CHK:mlzs2hbztak5fxjkrkuuvnpdpe:3myvisewm5uklimp2xucrwep75sm2rizfi2sq5drhlqjszkpdyeq:71:255:8388609 - format: chk - sample: - length: 8388609 - seed: yPi3JHKKbWaEEG5eZOlM6BHJll0Z3UTdBzz4bPQ7wjg= - zfec: - required: 71 - segmentSize: 131072 - total: 255 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:CHK:ljiodj4tijkfzej5nqdk5h7cnu:3fb7y2osytliwli3oez3y7ece6rjwfrpa2zn7uwrfmn22cisk4aa:101:256:56 - format: chk - sample: - length: 56 - seed: YQ== - zfec: - required: 101 - segmentSize: 131072 - total: 256 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:CHK:mftnvh5l3brshwoupvu6kz35cy:sncfokby2tjykbr3zgi3i6lhgerg4kz7fsrslrlxdy7upwkzq6lq:101:256:1024 - format: chk - sample: - length: 1024 - seed: YQ== - zfec: - required: 101 - segmentSize: 131072 - total: 256 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:CHK:sshu77h6opnto3jnngot3lwzl4:4cjhp6u3i2uzdzrjitje3inlhs6gnlgwc6wgi6vtedc2grgvcpza:101:256:4096 - format: chk - sample: - length: 4096 - seed: Yw== - zfec: - required: 101 - segmentSize: 131072 - total: 256 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:CHK:kl2jayrstlf5q3gax7o65ybivy:mnn5doti4wllssdg35ymojuiv4hd2ir7i6asurgms3ea67watm7a:101:256:131071 - format: chk - sample: - length: 131071 - seed: LCa0a2j/xo/5m0U8HTBBNBNCLXBkg7+g+YpeiGJm564= - zfec: - required: 101 - segmentSize: 131072 - total: 256 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:CHK:a6wuekh5mynr4c74ma3b2gswfa:b6lovwnbv3523n5kaxga3ruxa2dvxef6h4a2jnl6lasn4fwvvdba:101:256:131073 - format: chk - sample: - length: 131073 - seed: /N4rLtula/QIYB+3If6bXDONEO5CnqBPrlURto+/j7k= - zfec: - required: 101 - segmentSize: 131072 - total: 256 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:CHK:pmuo6vpcodol76sexq7s2ojej4:irp6rxnkgwbn3h4g5fzgjpqgsvhvycxobr3xrk3zr44r64wimgqa:101:256:2097151 - format: chk - sample: - length: 2097151 - seed: uqWglk0zIPvAxqkiFARTyFE+okq4/QV3A0gEqWckgJY= - zfec: - required: 101 - segmentSize: 131072 - total: 256 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:CHK:fh4boug3wkciqhkgpusjsjyzbe:sy5lwooxchqu5mrzwr3mgc7qtwfjgmpz2srdmwpaip4iuj3tn4vq:101:256:2097153 - format: chk - sample: - length: 2097153 - seed: BTBX/ampNfLU+ox7xipBGiaSbgC0kcB8Gy7BkJB4oKI= - zfec: - required: 101 - segmentSize: 131072 - total: 256 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:CHK:asmjgajvwpwvylxdcf5rm2gyd4:imwtdi4eqq3wm4bvk6npho3ta2fewceqzl4f7wyyabg5i4s6h3mq:101:256:8388607 - format: chk - sample: - length: 8388607 - seed: w6uP8Tcg6K2QR905Rms8iXTlksL6OD1KOWBxTK7wxPI= - zfec: - required: 101 - segmentSize: 131072 - total: 256 -- convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:CHK:yrznm2uuybfbda6jlj7tqs3yum:zy45lpc7ilfzgltxxqvy55wcokiay2fbbbv2wdvu6ubjluh4gf4a:101:256:8388609 - format: chk - sample: - length: 8388609 - seed: yPi3JHKKbWaEEG5eZOlM6BHJll0Z3UTdBzz4bPQ7wjg= - zfec: - required: 101 - segmentSize: 131072 - total: 256 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:CHK:td223psbzins2k6m4frmfw26xy:opgmb6zhnwsksydgjwzpfdoz7epm4ynzmvkjuw6s2jntioqk72ga:101:256:56 - format: chk - sample: - length: 56 - seed: YQ== - zfec: - required: 101 - segmentSize: 131072 - total: 256 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:CHK:ihyeqq5peg2pdyw45ct5mmj4ay:2rzhndv6o4knlvo4po6rvbnknlcitasseea3mirqjor5c7quh5ra:101:256:1024 - format: chk - sample: - length: 1024 - seed: YQ== - zfec: - required: 101 - segmentSize: 131072 - total: 256 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:CHK:mvk5f2mghudk7xnknqddhvnl4m:c74myeugtix7e7l6uwybtrdtfselha2qrcc7hhbymqr6brtg4cea:101:256:4096 - format: chk - sample: - length: 4096 - seed: Yw== - zfec: - required: 101 - segmentSize: 131072 - total: 256 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:CHK:rvytwcc4sbmmkrdwh6b23ckv5u:klk5nqpbv37pr2qavcebcugfnikarjdefjlseo3ru5uv7guai22q:101:256:131071 - format: chk - sample: - length: 131071 - seed: LCa0a2j/xo/5m0U8HTBBNBNCLXBkg7+g+YpeiGJm564= - zfec: - required: 101 - segmentSize: 131072 - total: 256 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:CHK:ocqx55dopakqnzyui6q6euz4gm:wklziyoctwrz5zjfg5wfejz4bnaagtg57fgjlspl4ba6ij6n2obq:101:256:131073 - format: chk - sample: - length: 131073 - seed: /N4rLtula/QIYB+3If6bXDONEO5CnqBPrlURto+/j7k= - zfec: - required: 101 - segmentSize: 131072 - total: 256 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:CHK:lf6qub3bkdhfozpdis4bmorj7q:zte3u3ohbuom3iqjjx4odrjylcpw5myw4n4kwf4zcaif36vngf6q:101:256:2097151 - format: chk - sample: - length: 2097151 - seed: uqWglk0zIPvAxqkiFARTyFE+okq4/QV3A0gEqWckgJY= - zfec: - required: 101 - segmentSize: 131072 - total: 256 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:CHK:a2qoww5xx6so5uftwg4msdtrpy:h7hwsjco5i3jkd7icytrtsuzcjvjzlnkr2wwau2tjlj5wehjrokq:101:256:2097153 - format: chk - sample: - length: 2097153 - seed: BTBX/ampNfLU+ox7xipBGiaSbgC0kcB8Gy7BkJB4oKI= - zfec: - required: 101 - segmentSize: 131072 - total: 256 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:CHK:tg5w4m6hi5ezeapuix3lrrbbtq:7kerk5plqpsd2upnole3ifxoegxdux7prazbvzeu2kyilc6idk7a:101:256:8388607 - format: chk - sample: - length: 8388607 - seed: w6uP8Tcg6K2QR905Rms8iXTlksL6OD1KOWBxTK7wxPI= - zfec: - required: 101 - segmentSize: 131072 - total: 256 -- convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:CHK:ioridipksj6o7ph3bcf37vslmi:pzhkwm4ry67ohj6dcs7zwnsjrvfqzam6tuitrns5vuo3uoomau6q:101:256:8388609 - format: chk - sample: - length: 8388609 - seed: yPi3JHKKbWaEEG5eZOlM6BHJll0Z3UTdBzz4bPQ7wjg= - zfec: - required: 101 - segmentSize: 131072 - total: 256 -version: '2023-01-12' diff --git a/integration/util.py b/integration/util.py index 39da60399..aaa9c98c2 100644 --- a/integration/util.py +++ b/integration/util.py @@ -2,7 +2,11 @@ General functionality useful for the implementation of integration tests. """ +from __future__ import annotations + +from contextlib import contextmanager from typing import TypeVar, Iterator, Awaitable, Callable +from typing_extensions import Literal from tempfile import NamedTemporaryFile import sys import time @@ -21,8 +25,17 @@ from twisted.internet.protocol import ProcessProtocol from twisted.internet.error import ProcessExitedAlready, ProcessDone from twisted.internet.threads import deferToThread +from attrs import frozen, evolve import requests +from cryptography.hazmat.primitives.asymmetric import rsa +from cryptography.hazmat.backends import default_backend +from cryptography.hazmat.primitives.serialization import ( + Encoding, + PrivateFormat, + NoEncryption, +) + from paramiko.rsakey import RSAKey from boltons.funcutils import wraps @@ -225,7 +238,7 @@ class TahoeProcess(object): def restart_async(self, reactor, request): d = self.kill_async() - d.addCallback(lambda ignored: _run_node(reactor, self.node_dir, request, None)) + d.addCallback(lambda ignored: _run_node(reactor, self.node_dir, request, None, finalize=False)) def got_new_process(proc): self._process_transport = proc.transport d.addCallback(got_new_process) @@ -603,8 +616,76 @@ def run_in_thread(f): return deferToThread(lambda: f(*args, **kwargs)) return test +@frozen +class CHK: + """ + Represent the CHK encoding sufficiently to run a ``tahoe put`` command + using it. + """ + kind = "chk" + max_shares = 256 -def upload(alice: TahoeProcess, fmt: str, data: bytes) -> str: + def customize(self) -> CHK: + # Nothing to do. + return self + + @classmethod + def load(cls, params: None) -> CHK: + assert params is None + return cls() + + def to_json(self) -> None: + return None + + @contextmanager + def to_argv(self) -> None: + yield [] + +@frozen +class SSK: + """ + Represent the SSK encodings (SDMF and MDMF) sufficiently to run a + ``tahoe put`` command using one of them. + """ + kind = "ssk" + + # SDMF and MDMF encode share counts (N and k) into the share itself as an + # unsigned byte. They could have encoded (share count - 1) to fit the + # full range supported by ZFEC into the unsigned byte - but they don't. + # So 256 is inaccessible to those formats and we set the upper bound at + # 255. + max_shares = 255 + + name: Literal["sdmf", "mdmf"] + key: None | bytes + + @classmethod + def load(cls, params: dict) -> SSK: + assert params.keys() == {"format", "mutable", "key"} + return cls(params["format"], params["key"].encode("ascii")) + + def customize(self) -> SSK: + """ + Return an SSK with a newly generated random RSA key. + """ + return evolve(self, key=generate_rsa_key()) + + def to_json(self) -> dict[str, str]: + return { + "format": self.name, + "mutable": None, + "key": self.key.decode("ascii"), + } + + @contextmanager + def to_argv(self) -> None: + with NamedTemporaryFile() as f: + f.write(self.key) + f.flush() + yield [f"--format={self.name}", "--mutable", f"--private-key-path={f.name}"] + + +def upload(alice: TahoeProcess, fmt: CHK | SSK, data: bytes) -> str: """ Upload the given data to the given node. @@ -616,11 +697,13 @@ def upload(alice: TahoeProcess, fmt: str, data: bytes) -> str: :return: The capability for the uploaded data. """ + with NamedTemporaryFile() as f: f.write(data) f.flush() - return cli(alice, "put", f"--format={fmt}", f.name).decode("utf-8").strip() - + with fmt.to_argv() as fmt_argv: + argv = [alice, "put"] + fmt_argv + [f.name] + return cli(*argv).decode("utf-8").strip() α = TypeVar("α") β = TypeVar("β") @@ -707,3 +790,18 @@ async def reconfigure(reactor, request, node: TahoeProcess, params: tuple[int, i print("Ready.") else: print("Config unchanged, not restarting.") + + +def generate_rsa_key() -> bytes: + """ + Generate a 2048 bit RSA key suitable for use with SSKs. + """ + return rsa.generate_private_key( + public_exponent=65537, + key_size=2048, + backend=default_backend() + ).private_bytes( + encoding=Encoding.PEM, + format=PrivateFormat.TraditionalOpenSSL, + encryption_algorithm=NoEncryption(), + ) diff --git a/integration/vectors.py b/integration/vectors.py index 66c9fa88a..bca8b0bf3 100644 --- a/integration/vectors.py +++ b/integration/vectors.py @@ -3,7 +3,7 @@ A module that loads pre-generated test vectors. :ivar DATA_PATH: The path of the file containing test vectors. -:ivar capabilities: The CHK test vectors. +:ivar capabilities: The capability test vectors. """ from __future__ import annotations @@ -11,12 +11,16 @@ from __future__ import annotations from typing import TextIO from attrs import frozen from yaml import safe_load -from pathlib import Path from base64 import b64encode, b64decode from twisted.python.filepath import FilePath -DATA_PATH: FilePath = FilePath(__file__).sibling("test_vectors.yaml") +from .util import CHK, SSK + +DATA_PATH: FilePath = FilePath(__file__).sibling("vectors").child("test_vectors.yaml") + +# The version of the persisted test vector data this code can interpret. +CURRENT_VERSION: str = "2023-01-16.2" @frozen class Sample: @@ -42,16 +46,6 @@ class Param: # dealing with. MAX_SHARES = "max" -# SDMF and MDMF encode share counts (N and k) into the share itself as an -# unsigned byte. They could have encoded (share count - 1) to fit the full -# range supported by ZFEC into the unsigned byte - but they don't. So 256 is -# inaccessible to those formats and we set the upper bound at 255. -MAX_SHARES_MAP = { - "chk": 256, - "sdmf": 255, - "mdmf": 255, -} - @frozen class SeedParam: """ @@ -86,7 +80,7 @@ class Case: seed_params: Param convergence: bytes seed_data: Sample - fmt: str + fmt: CHK | SSK @property def data(self): @@ -94,7 +88,7 @@ class Case: @property def params(self): - return self.seed_params.realize(MAX_SHARES_MAP[self.fmt]) + return self.seed_params.realize(self.fmt.max_shares) def encode_bytes(b: bytes) -> str: @@ -125,16 +119,32 @@ def stretch(seed: bytes, size: int) -> bytes: return (seed * multiples)[:size] +def load_format(serialized: dict) -> CHK | SSK: + if serialized["kind"] == "chk": + return CHK.load(serialized["params"]) + elif serialized["kind"] == "ssk": + return SSK.load(serialized["params"]) + else: + raise ValueError(f"Unrecognized format: {serialized}") + + def load_capabilities(f: TextIO) -> dict[Case, str]: data = safe_load(f) if data is None: return {} + if data["version"] != CURRENT_VERSION: + print( + f"Current version is {CURRENT_VERSION}; " + "cannot load version {data['version']} data." + ) + return {} + return { Case( seed_params=SeedParam(case["zfec"]["required"], case["zfec"]["total"]), convergence=decode_bytes(case["convergence"]), seed_data=Sample(decode_bytes(case["sample"]["seed"]), case["sample"]["length"]), - fmt=case["format"], + fmt=load_format(case["format"]), ): case["expected"] for case in data["vector"] diff --git a/integration/vectors/test_vectors.yaml b/integration/vectors/test_vectors.yaml new file mode 100755 index 000000000..6b28e2302 --- /dev/null +++ b/integration/vectors/test_vectors.yaml @@ -0,0 +1,16202 @@ +vector: +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:CHK:yzxcoagbetwet65ltjpbqyli3m:6b7inuiha2xdtgqzd55i6aeggutnxzr6qfwpv2ep5xlln6pgef7a:1:1:56 + format: + kind: chk + params: null + sample: + length: 56 + seed: YQ== + zfec: + required: 1 + segmentSize: 131072 + total: 1 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:SSK:k6wvqieksrwfbggxxfx634ymcu:nktp4feownhax6kladditlqli6c57lxtjpmx2l5tinv5xbmg24xa + format: + kind: ssk + params: + format: sdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEpAIBAAKCAQEApTEPAXxC8mAxFzIiXFh6buUBZkEELsv6rTnpYW8QbDSwViHs + + DA7zdugOEU6RRjJ/6kOFj1UFzg4sgHWrOi91wT7JQLPrNh61r0d+UmmWa8c3fw1+ + + 0n444frx3cdNBEXeqvU36CdxHO06AVD4K+///w4VuhDQZBoFjQ7xKjuYyYrePvaa + + 0Tl+Te5+gm6rnZLP+p/y2iOtg1rLzYVJCW1QCQdJx69m/NirhJ9LFVn/4mt8x0mn + + CwT4PAY3W+O46J++z3iPiY3YQXlzdoWh/uksFTs5d9I/XOoZOlaAdF6qOb8cUxwr + + xlYCGjudDGY5t3C0F0LSmZs5M9ErTKVWE4jGfwIDAQABAoIBAASpKbCtxCLh0brx + + 0EQxSuMLMlaMPlayOHdUNA72QLXSrgWLwDNm5a4x6NmE4rO0VWs8IByURNA0niwX + + guwsn7h3XWcDZrlaANQgH+ip5xZofIl0cGLfR73EJyyrGRDUtO4rt/QY5eHvyEfq + + rmGAb0ssyjJTNmVFXfTo6NXBTc3qnuwQYEVYBu4rp+O9ssv2OB/otQBUwN6oo01o + + yhT3PfuPMca9f+8VTxPFPPW2eou1Nr8lBuwGk3h5KkyNGTvgNHuKM/7ICBK/wAc1 + + dFo47jneQZeAZwJfCFSTBIx5ADE195h98zTTJvFJM5wxwuBC86qSKNnbQWmVrO/K + + FBZpt4ECgYEA4zWEG1DC4qAVaDPslyhHVraX0e/570EnOilnqxBraHKmp5vdxucq + + WbezO0791A5DztAobsi4hMXrPQG97Of5yoxnJCjm4Uvcqn4RQFGw3I0nAB827fTC + + 9KXEXhp5ofrYmF1Z2k6X3GiVMs0Wu6pHYPiCbXU3bOoA/zMD3pKFA/ECgYEAuh+/ + + BnAPaj0sEjYsZS6FdUvFMiPD+4IUsCsTXfGTF3rG5h6g0vPHPir4v3haoAW6F8oa + + Y49re3869yTqpagLctzMtrUGeZ7LEc/FzGaPfrgkKDzK2QCdR2wvNBet0DVjdWwn + + 53YoWvKf2M+ooSnOT/1FiOJfuHzJetZBbV36IW8CgYB22D9JqmzF7cZEwyQ1zLPD + + /65Z+ZRaOVIzcgTvzZ7g+1eAxF6086WLWDNAColqqit9uhPsHsGlcYEiYA7gJFbc + + Q6SPnXVm0y+RXm/XnONN+ec0gR9SSHzRSwPz1RVaTMOOrwWY0xNMDsg70lrZvq+n + + YVWXu4BKT/xFgIG9ohZBgQKBgQCi1g1xW28RGo3JLR4wM8BNO8o9sK7RByCEdFtQ + + UH7JBwCm6dr4VJFXYY8ZLPnUkM4b7BSkUCDP/iMfgGvOHLRPfL+Zhc0xcGznm2jJ + + CF24lvADSBSMQA5aI1s07xaBV4Q5gjNzPJvX3fddX2h//6xhrQs91Be8t2gqkPLS + + 9WpV/wKBgQCVaWipSEsgoDuIgZQSebAfRUE4eS69SGhwwxZ8esYYUKBcJA4Q3BGB + + s2hWwmTe4aL4kyNV86SfkMoMefa1mXV5LL4k5t0rXatdb+95/jX3ecQqELH+upaK + + JLRvVhJfaLJ5XN9WHjz7ebR6NRYcc5xAAM1qTRRjBgkX06vIVbh4GQ== + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 56 + seed: YQ== + zfec: + required: 1 + segmentSize: 131072 + total: 1 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:MDMF:wgmyhgqybo6uemxz3l2rqnxemy:zugfpwxmiz54a6ml6ocei3xdchqufogpghyer4az6vmzyvxadrxa + format: + kind: ssk + params: + format: mdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEogIBAAKCAQEA0BZ+gvZma5bZne1GBzhT7pNOql7TH3/kIqUAWiX8xfC+n0Dp + + 8/lPVaztFlLBYK8bT5Z0UCRxHnWtM3+tjb8DQeoeCoUqqS5WavW+p9iv7xS8yMjl + + 7KrrsAnDW2JpkbKLj+i+YQCgyOdmKVyrDSSUdBYNwXdsjTmgv4H+gXqkkdIbl0Fo + + z8185rMDxiuV6IlWA1dQCUEVB6tLhovloUJ5g8fDnLbv3nDh93C7aadZAn4hCkrh + + 4WmaSH5pB34F+IMsFXdEwpjvdTVWrPPRo7JRTvBRFSH/B6YpIfR7DWUiXqKGYlJP + + BXYSFslZ9M5HwLNtOnQam6E2cIjPrf7CXRFd6QIDAQABAoH/cD5loCLni3WoL0ma + + tUkutRyJhXa88Na6PGPcNt9Q6LXJlnZs7YfJuNzrHyJNiCZ3i1j2JLK98Ul+jfHi + + fqf9XKRq6ubdt610jAsD81fp7RmpRIfjlNGf2FJwW0zgEhGpkWjMq2pSDRE+Eqkt + + zu9EhTgfDlp4aMCU8S0fq9ZmAha5gDmfnbYKZbBEYtnHUWPkJ/Io19iJxa7HuXlM + + GOIBZz4U+epDxFpxPmJ3puZWR7eP/aYp62lViObg92Er/kliSPZ7LAACIBmZFrXD + + d9VsIScWnrfPdvrsy05VGJvlonUw/Hlo8vlSvcYDO72DlpgPmkuosAzHZFmimlyv + + RNcBAoGBAOIIuWG9XS7aH0kMaZC2a56TXKMevoa6ydOaiwumm/ojfzcYkbLmgTBZ + + UGzpPXe5w/W/GYrCeOxk3XPfSJPVEeUhKdqA/JyQ2xXXjHu7UCKkhUPCZh1USlSt + + aXP4fL1fEqUZT6tRBUP7utXuoi0Ga/kiMWjB1b0VgAoBH/uoUCLpAoGBAOuss3wd + + gbYmBmi2U+TOcUoSPf4E8BjTFGSuXfvtsGjy7OU1w6Jj41pnYTxQWoyUVJy3wG+B + + IwklM+DNMw4o4sgVcqlgG7grt3adpcuVIQY+5rXyO4ZcG2vi5UAO3Yk+9c/AvbPJ + + pm6p4p0BjXMI8w7KxBHSyfywc3HvkNLlRIMBAoGBAN+BiO4g9ZdykCUHZQt3lotD + + ZALYT8Whxhi7ZGqs4OdDWnP8k3W3gF9ysZhAOku9IQxLXtJa4n++bUw6qeWkdwF+ + + /YfWq/OVOU4ryfo/ikn3LN+HxrmRs75viyrlt1L6Q9GFacYZY3+J14Hbafnjs7iy + + GvFfWh6St/0sh5etIzChAoGBAKZCjdSvlESGCttwVTsDkNSqjeVYYnGA59AnWtJR + + 2rQPPKRvC3bSdR/f8q70GQ03z4FH+JAxUCAxiKm82ZnRqjtxNhTbYnLJFIKvsLkw + + mb2oPmZ5Xxjofcfcp9JLKmqaahuIY8wkJC/J1b7hy4It/BqhXTUdubV0Xd0xHsBJ + + Uc4BAoGAbN8Fx/7GKOaR34m2PRywftvLXbjxQ+xGtp1k2ewymndqiurpqd4ckYZj + + aVtmHWlgQ8St37dhBXFnGGtnTyiyQd4s97svLImvyQxAo95lU9UPApWrerzF9C9w + + /jAPW4j8xpD4AiXmSvC2nUR5jKPSd5B2qEypp3aIShBdDys7YYI= + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 56 + seed: YQ== + zfec: + required: 1 + segmentSize: 131072 + total: 1 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:CHK:mri732rh3meyh4drikau3a24ba:6tsj5wvcp6szdhmrbu5bea57wduoza64y6nd2lm7aleqpsxjm5la:1:1:1024 + format: + kind: chk + params: null + sample: + length: 1024 + seed: YQ== + zfec: + required: 1 + segmentSize: 131072 + total: 1 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:SSK:uprgind2aeoklhu5feejamwe4m:u7ettdqcxjw375tu5jxxqe7uzx5v2wilwsiwiolbo4ery5hy3nga + format: + kind: ssk + params: + format: sdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEogIBAAKCAQEAovxwmrC8O1S/mvxTON8SitECU+pMmsPaM91xnDtBCEAXS/dZ + + baWgQ3D6ytt8b1X+LZdXo5alRv9Qqub3nNBaRalenRar5lQ6TLfREq3GaXHVbZoY + + 1UQrBfUjaNniC5/SWpTyv3wfVWAgk/Z9Lx2979jqKL8MqM2irTdWNBhOsmui9SIp + + 4Mi/qhJf5zuAVe/qX6Qz2Jhn4jfstim1Rid8ez1PtXyEPRuQr44pBpNRiRejXJ+U + + 3tZqkM0FJVgxT0TLkrQZhjLlF6aHNWY8jHooxhenFfgZTYv22KlACI2YOEytTm1x + + 8HggPrD+AlDBZ3W/VbQfLvukZv+XwOR0e2Z/KQIDAQABAoIBAADlHopMM1p9+RXx + + Ou57YIyRt2lvmDex1AfquE39wzoZNhRWwJ7Do7WyI1QuZ3z2ySnF41dabala/DOn + + 46xwkonZteLbBqfT0Q7woWLXJvghqbGcQzdeB4rFl8yLjwfaB9KZ1ehf/4OR58OS + + erzILXf9+/FFvOa9CgUg7Irwc404I8OVBYFP/Su4HpWDiAXi31QHpFW4Iyf9yM0L + + 5BWfcpBIxkNQDDgX1Qfmzm3lOioQ3sg+ka7KPuve+2pnn2cS4Pu1zdXN/edtdmtJ + + 0Uw0sCf6WHi5X/dNA/FdKY6qL0Hdd/d6XiR3enHQV2ZWnENt0iGoxIYur4KbqNmW + + vofTy8UCgYEA3MYl0WOSyuv6HVTS4ZEsZL5UypthpeHLlM0qmU7kLKYsN3LI27J9 + + jcCr26+saEg3Gtg/sXAkh6RQhlHmLVWS8I9yNXI+WiKBiuQsR0dAwBKhM1OXtLke + + WDV1buSkzgbxawxLFaxV5G31isRGz8XO6v+pJUQcg9JGpFkL8+3gm9cCgYEAvP3b + + KclJW/tZSggY7+9nv27EzwSYnTRdjYYcUtugKqm/Pe2wJm9vixqXWMMvmM4Kd/Xw + + vgQg+gfXoOxQuBsDZHP48grqiPsAVQsIbgMiyWW9UvzYQxezC2584sMtygPiBb8Q + + rQ0A8uEaa3I3U6CRQtKZvc0opK7OymXFzu9mXP8CgYADQsn8NcRNSv7+v+n9eu90 + + 7XrDI1hl4tfm8sDWUtv77NhqWT+uPwyrs1TWgdnCEI7/zoHiVQ21EzA9S6hiswjg + + lL3THETff/L54jTlOKA0NhI7d9idyr4v/1oksSvd/yxBsITLZSg/n4Ao9I03NGzB + + +9S7wC3LpKd2dfo/OBxBMQKBgGVf/jmR4SnXz3NomIfLcWk8L5GkM4DP4AbUE0lW + + yblYyF6dqslTKRACuYBBYryiePcUE4i5ij7UChQl7r5yrwUpODYNKPVFPk5f1qu8 + + PuKtEjr7qb2DbuUI5TB15Y/hOVI/xOAug33ExXkxEQBotsKTWSh4bf64TfA/WzW/ + + MLddAoGAZgnPZRDa3tvxrc++GHRxtkN+MOhJaahfBVxKsiBuWj+tA+XEyS/AvhMG + + mKMMSko/pYfor1FpeSrqnTFwCzIvXPt/zB6jJuHEUiDqwFSqfHOaMdSjut2/HDxH + + fVBPYYeC88V2snKjt9ZPmZ8569XK3/Cp0gbZqJ/e4bx1cTd9PJk= + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 1024 + seed: YQ== + zfec: + required: 1 + segmentSize: 131072 + total: 1 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:MDMF:omlhm7gyylpggroasrqohbssba:7ht36qh7byquddx26sr5ywrzgpviqgtbxlgosnjz3njxi3fs4mkq + format: + kind: ssk + params: + format: mdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEowIBAAKCAQEAjxmo/c3d8aSEPugOJh/TBt7G/2FTD85dDzI1ZRE04V81uhmz + + ND8k5wv25nSDCdh0ewPloRX3+uxbo6HX0wj1rWYFaDjxuiYQF7/qpv8OgKShakMK + + kCiwnn+DhISN+nEO3QtRr3b4nwUh+pqAmL9QJvOq092MaqhGXFQd4i9bQEAxH2P+ + + OjrDXB35/7KDFkGwVZO6n3ZoGJMhRQQk2w9+7yk3rJUdGOQKzoZG9MilOQgNM9z4 + + 0BfSIuGaokwhzdWIOfkLOiYQw1slod2/mYgYO/17YXX6afKSVR8D8ReoD63IyC6W + + UsVsS/2cV5hrsgxFDVBasvT0CqzZcJb5scDrAwIDAQABAoIBAAada4vr984DSl+0 + + D29gujsPkkhc4d+RrQiWTBSTcovWgF/Nb1TDdHu/uFaX3TTXzi6fk/5ZyyBMy8Gp + + KhZlzCGLXUWfmEEAIG9QnlLA6JU2xwVn+vWGBAAXqec2z29byZGbQ9fmGoETVipE + + +RvWgCiEzAlGLQcDJ1l+Q/FgOgoiu840VzEVaq3YbeX43DSno3JsROZ62+UjUo6b + + PUXrmpD363OQTh66zSCspztHQftiCUf+XoVQXQVT3vbBbMwoXCwJgiakHQNJQ9YP + + vtIk21weUPN1jGZ7i/N7UEQNVdwLTmg5Fc7zc3Gf2JailWLDqkB8BLp7d2sJLJtI + + JPvM26ECgYEAuVXyGX3EnYRY2v2dLsTv52onik/DafatZCAMJX4wC5CP7YmMFfJT + + S+HGPkvmH6eDK9LM2NSf1WmSs8yrjGEpI1ci3i4KHbLuPvnsBqeBgMZCbUQ3g9dK + + uWt8BL8R0rkzE3M7Hz1MotKtlQZcHsf/qqV30glLQl2NTSPaEK89iOECgYEAxak8 + + tUCSZESZlPCG+FXlUUJQp+dGSnsHcFZ2S+iQ0oNbx1QcBoORI1uiwv5GgiJayMlT + + g639Y2DNFNXZEa2kdxrjgu5tL33qwckP8SYbQI0fevjoc5zXuTjLB7MB07M/utJw + + D7EwFOLKEwXCqFEC13z9p5cuwGEl1NlJVKuofGMCgYEAhFHuRXDbnTJOVhtXy3pj + + Za8Oh9smw1KQvLl8spADMV6Gw6q+TzTxb23EIdoCdHseVX1tLymu66kyShhIKjN8 + + MXUWudXY9xc2rdO1RZL2DMB/0I8xq3lcKkGpC6J20SHUa4CLp2QWgPE1aP5fasKT + + sHvurhBgoQM1zOtZ1yumHUECgYA0Dz/jCS/FYuAEf1k9HPp57XpqzpoP0dmCt/MO + + SSGjoF9S349GE+7tHhx/ORN/AOdiTMxHOVMsknlRTIWQh2hyyk0z1fJB+OsUwQ0G + + 2Z+B3+lzrQ0kLiIPMasfywDnLiXR4c0MBQIB7j2Exxae2D9kXBI+yq3Qk4WwSs5q + + k4+buwKBgCHyWPzrSfkw2f48jdtWOwCSI8by3XnB+AxYy7ICgxXRTS7yTY4UKmZd + + mbrWpyUstquLMP5x7ChvUQU1aH7RbZFGs6ZSPd982+iV+/WVGl0wOCLczTqFKro1 + + p5L1pZf3lIN0X9LVRIAhUk8RKjuINcuWZHPVtcTjJ49hK/L409Wy + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 1024 + seed: YQ== + zfec: + required: 1 + segmentSize: 131072 + total: 1 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:CHK:e5tpbcmalcq6nn2zhd3qvg4chy:emzsvv2xnkhhrj2oatds5hf2cney25awi56ybeq4ofiyehzyakua:1:1:4096 + format: + kind: chk + params: null + sample: + length: 4096 + seed: Yw== + zfec: + required: 1 + segmentSize: 131072 + total: 1 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:SSK:ekp7frkwxjgxjc4d4wwqtyf4bu:ty5ope7attfwtmqlc72n3uwzlf4mslshmoa5xw2eohqh5me6ajrq + format: + kind: ssk + params: + format: sdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEowIBAAKCAQEAza1esoFmMt4RWB36A8xGkpVKtPMGx+DcWkQ2QaAxld/CjSl0 + + 2KSyH7vIeijVh3wids2V1WkHvCw9C9ZyNu+TUrIAXqWEGWUijObXDZUaiyCkeNXf + + B3x5aKAZBc4sjBhxZvo2AggPEdC8JTyNAGRTpBSXVfNTKcu4np3W5nwrToSIJyza + + MHEZlYeUMa9uDwKcL7Ephjdle+bJ0jzIIiC2GZfamZYlMEmhUNQxbQRTqy2E3ovQ + + 7JrH2KL2edbk7/s/an6iOa6YDPUr14llk5YD5a7j+66bE40spF2vRTt4PHUCIJ6+ + + PY5fFHYI0l0tRnf0h/RPeOBME7JqlqSncInvkQIDAQABAoIBAAOoVEt3eX84s51h + + +ZXnFF9zkhK4EmgcKaD9jusxd4ZFNpUK7l/ypFQEA214M81jLcv37ZTgJ3X/QaXn + + 7iunzKJzJcZpGjl3IuNXcM42F0Tve3tGXt8nxZMwqyc/Nou2fNBGbW7RcL6p7Amb + + uzxtPE9J+jO6JjHDZJzl/MoIHd9JW/5xdfndaFjjcPcOtJ/3MIrsNhTZmCT6zkm2 + + kh9ABaUw1JlPYi5+TpNPlfAHLBuXqqJfZTreZjthCbevnoAG7AAfFQPiOITN9dyu + + rmbW5rNgeNNcR5vXfVHI8+M1/F4TF4W5HWGtt/HeibOSXrIRcKNQJButlhvb6yRW + + Pzs0KiECgYEA0aZWvQDYRUeqUiMt6KdUpKdTLlzu4HNn6Be9yr2jztd5ka3mM9yv + + 0if3qJ9JpH+TAHrKGS40VREB49dFO7wVDB21mzTloDu+qKQSk25h6WuVB4udILuf + + tSPLBhQkStE1+02/gMMLIlk9c6ByTuLQoq0erKKNwIf8m0Dkhko6AXECgYEA+yYy + + BK1dOjdePo5dkRB12w/h+2whkoosspRuB6Hg8LY6SGVCXJM8cEgkL2CFNVoVcxGx + + hdKpsnYTd4wMbHnPNmEgeh1EN/5bjqkfrgU7O8pc0BrfOUUcgdywHMQAk4+w9DiA + + 2IIAErc+VVUMITF5AjMvtsibG29eOGLbwCArwCECgYASpY3Pb7TMrKwcdB6QM9nW + + bz95vzBL7FfQj9QEpUtdiVK5v1LbSASnV4CykcBWDja/8yvog3CKJGIbprj0sCzb + + EAVoEZNe5hF2JGm2jTnOLhBqRGOsVqPE07MqDj6QHP2FJYwj4rUpz/AkSaABHjFa + + VrWEu0yKVE4GbQYmX5G7MQKBgEY0lybXj4gGkkHKaj1y7H8gIXu27muYVIZXF6rq + + hYbEaeZy5+oY/nwkrnjP8mzHkddoysct7GIGv8pbS93G7zW0UO/R3pAIem+Wt1Re + + AgDkwK0r1dqchyuGFXT1FXQqZrzeTqY3MO4Ka1JPQ+TDf6Atzti5myJAL4ZznBpI + + 4/IBAoGBALQ5gDwHaz8C7tttgPULp680JKbkMk4xZpw2tYinR+VeavNSlBbF88zt + + iX03spiC0knxlZcie718f73fTn7934PkFNKqOx0SuPv7A9xxcuZrGgMHERKICfD+ + + qmfXAT47ZsTzbiNto7efTdrLQ8xmSf0KNupXbpVrMs+b+qsQZ/Vv + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 4096 + seed: Yw== + zfec: + required: 1 + segmentSize: 131072 + total: 1 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:MDMF:egx7nc5bzqg3phfhootzyso27q:ompckuxqvlnwgdk77jkksfpxqu4gajv72bpzcgokcnwmx3nvpkla + format: + kind: ssk + params: + format: mdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEogIBAAKCAQEApZGQcaVR+M18Rp/NV+nqLDdEzOAaCPT0c4vhuDmUC+5fmIgf + + Zcsnqj1ZphJqicCTht1ukFOhtF1gtJd6554SwoM/VR3IsmZf8OTHkSELXo+R90X0 + + j3aqnlSDtrP1yacD39xVCExPtjRwE7VbV6RMUNjCB0PAm5mYeixMlXzBUd9bWS8U + + GcCvF7KoJinE93Q9IOMcRmMMmOb6pJh2zN6gNlubN2Te5OCiA76lCxa+cuPvjDHt + + gmdxJhLFdljtoJL9cvOuKFpxqTe+Hdd30Mkn/9LAG6j+ZH2b5GPcvivs3bq68FzY + + 9XXdS1Vpd8XuxJKR5oN/MIpIn2peXWQ3KwzbVwIDAQABAoIBABEpUmiFM0bvvabw + + X29yXoRwwh/eRrSQ91mWsTHQPgkyjxQXX/HEKftaWpV9KS/YFzKOdyxcjtFMMH7n + + iKTDXLxusDzZVnkvZVhpzkm7vBr0FLQluyC3sx2wMurYImzhc+RbSTEP/98p9kgE + + r1AZRpPGs+3e1vMJ66UWPGXuRXd/3YzKGFqQgw0BD3KpS5o8661e/lJ/i2O4Q2Si + + hRSYdm1ztZ9Q1F5WvjX/SKOXOGz9LHk8VoY9/KDsrYaEr8JVIiM0fnGf6FHqq33B + + /X49f//IXqeqI2ZFhb9BSYGRxwO0Xp3OaR5dIpq0c+qyPPfYuCCOF+8b5lToWty+ + + PPMO69kCgYEAwbQXI9WnPSW6Z8jmqHbXZet/cJscUlDBIxH7ANKVxOr4OHWe9w4s + + 9XI6Y5ZsAYN6QNBUVx3rxuDq4sAIiNTXCPRCs7/ChoIDPU/8R7oZLma46aDorUAE + + fJfaLpsh7DBKXfNq9iKVQFYiK3imBZkpZ5EEvVNELz8tNbiEvk5JLhMCgYEA2tEW + + FiczDFzEapU8Phkbiy9o4f5XRqbT6KXzgiVWzKzzpPhTPU3I8iVz4ETGBZ+SVAIx + + yWW1Gm00ThJfYdsJ6MytX6DYXQviG0MndF3gUSGywCFn504ecwuako5lN9TYBmEc + + 429qNJLzdEvY4rjbivzjK/FA62gNOyTjU+Ghdi0CgYBFj5XLwZsoQ1c4lBX5I4xg + + xnxihOFb0jI5lOhtuDIeoD75j4vBru6ISjgbsVYiCQQrKGVRT6ZvKjBPs6Sc9sou + + JgGaKWADC8d8CjBP1c3bMvpus+E67kVuNN4eZIl/Fyxtps+finXMv+HPeKkuU31h + + +tsX3kIbMXXb1+KbsONozwKBgD3FisukM0gJJDXGfWQ2aE0pjB1IVNEQJYBm4NBb + + xB4xsPJgW/dRbynUotqr748E1iU7HVzyhma4b2yeyShx0mFS7pqxaIMT6LezhH7Y + + RYwBzFlq1M86gWQO4YsAAdj6ECX04lfeSwged/Xbt5WBhBC/hU4RZDdQf3Oz3Sz/ + + 5DndAoGAGkThXjZdC745jhAJ8aX0NimCIfDuXR/h5HQL3F5/3CnPPduNxJVi/Q+E + + haKAhzzap6pFsi204fAcWCBMBHH9aCv/AlAN020G/3FDKY+CBKlrsvLAq3ca2xD1 + + rKylwatgnJuSva3IpqexUVLbugxw2N2cF/Z41DBfivqchp4LEo0= + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 4096 + seed: Yw== + zfec: + required: 1 + segmentSize: 131072 + total: 1 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:CHK:ok3slnjd3e56za3iot74audhl4:2mjvrb455yldouoxwzbx4sbsysowipqje6ifa4pmbzqahj5j4mnq:1:1:131071 + format: + kind: chk + params: null + sample: + length: 131071 + seed: LCa0a2j/xo/5m0U8HTBBNBNCLXBkg7+g+YpeiGJm564= + zfec: + required: 1 + segmentSize: 131072 + total: 1 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:SSK:gwtadrxpxdxeplb6rvdlyjoq5y:2uyxymbtbui3pnmyhovcs425lld2nzyrrcnw7k53cebmoxr44mha + format: + kind: ssk + params: + format: sdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEoAIBAAKCAQEA1ix1QJb6xHiPmHLiXiOf1FV0VlsIu8aasLnw9J+12mDn04u1 + + /ofusd01Sr+j5goC7cpxZhCF8jrGcxHcjQxgiX6AIsYkGTENQq02zu4gV8614efZ + + pWYJE4+ed+SaO6oKRRkS9rZIoJVUS3nrRHzfyUBuuqRRB16PPRIx7y37kFANGaTT + + XzwGCrWGp/dk5kt8E8FpZD+AqLiRCwZcQlLSrVn5pfQIgKLbU0aoZEnRytwz1pJW + + zDSHAY/rQ9weh+xX2LfSJDy9i3jxk3t7FbmUUmWyESpdV+KuPIQgtNWV3quKLphe + + v4uR1DZQnJxUlnU4GQjDmCIAnBkPIg6PqojFOQIDAQABAoH/BGRlAy4VdnICf93b + + RpL0dCZMfHjhdPhds8IcbufXkuLp3iy/Trj67CrdLOtBMTaDWN0N9kngdVc/Opzj + + KQFX+XneptfpZZrb6sIinZvEjghvMnLOw9WT2hLX7R2DDDYwf7pD6UtTsfdeAy48 + + 5OqqrDXmD23PO5d02IvG+mC3B/6Sq3aG7052H0+Axfm/DGz905ojcsWx+0qOMEDG + + HjR+nolrAePTXBm3kQpKt5x2B38itmG7crWKxWqARvHnPAXP59e9gqEZsFWo65WJ + + LtiD3rdQIpJ5vRi3g12JULXohYzt+o1ydAYXIiaaC46jkgjn+07qDKkMJ0TXn1Sl + + yx4lAoGBAOfnA9IW3cn0axoiO6+xPMX00+zpvFbHtX1jN622LGjDDvfqXqKDYVbV + + 4otQna1/phJyHgsLUIREo9brmRuUgulkZcNn3Q89R+GG6tmwPJdUZgwD3ZcmiEIw + + azE1r15ZPOeBwjtb0u4JxFU5POiHjrMimRqlC7uk8qr6H8mokXyNAoGBAOxt1FuZ + + oD+PH73DusA42/8mjarIAkISbvxDbv364/iTbcKgYJbD6HAntCjno3LHLhcy2ITP + + 8K5dcRcd8RKVf4pUY1LbHVWmslj0emOcBZ0jfNHiIs9qyk5ROxdzG1DEZNIDCoSi + + GAnXES7JVnIGbcEcveCmTn7PV11EEl5u7h5dAoGAJdVmpivc20no/0Z+fldoFtOu + + j4RCmdXTIjXBq7GA5UaNdpzh+5l7k/MpFpl8YAXnTjMX+61I4YthP3sIa5t7ECC1 + + CYA0bHwO8hhU0FcUS4wVafhnenVq3YGQu2KKzdW5PfvJeG2up+8n/M9txHH5Mfh/ + + Cf3LQD3U6VgNP5UkxzECgYAQTXZyJoK1P0I6DJAJByKsUlU8bHQzaB/9Bw1VOAKW + + NlxAKlzeqH2TljlHBMnxdSiJcvkZF1mKPGk65dakqGhV+oGqye8Y49iyZ5E04yJD + + 9pl5w3URBlUS12kSsd41UIV/MbR89sxfiVPm/P0X+beBtGCnZ/BLsDJe/P2jQ1Tq + + BQKBgGYpeqtu9yEEKRbJWnTP8VApwbkc+oiMbsBtNtmFFhUnSjq0GlIb3VGImEKj + + 4lP59fLmK3VcWYzlNOxmwze4hCGtsmEAWSXM6LeYRcE0mmUgwUtKihHfczMLs7bp + + frPMT0b4+wM7pGi0aaTCbDtDvHBY8I/llKC3/ama2Vyr6gsy + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 131071 + seed: LCa0a2j/xo/5m0U8HTBBNBNCLXBkg7+g+YpeiGJm564= + zfec: + required: 1 + segmentSize: 131072 + total: 1 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:MDMF:s24y72rli7tygwogwisvt6pkbq:kkqntx5umv5vpxgaehnvtzarai7ks3vgjzsbb3h222birkpml5pq + format: + kind: ssk + params: + format: mdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEowIBAAKCAQEAy0L2RkAg0jvEueOYXbLzednZzQU3fhJqnIvjARAxJWtPDOsW + + 6wruLE+s5IHW69mNyCcnJ6Oq6fQ0Iv6pneE7xO95HxRkv7s98DAB3ZBDI/8Ca3c5 + + lZx9RAJ+qfbKZ6deUE1do65vijVtUszY+sYFFMo19MXs/Yo4uvpsH7xvcic+0hoD + + 3WS2P/iJDSj6CiIn8HIuWjA7ES0cNcxmBtqfaIDxm3TmSu4U9FmWbhgtDEN/inYp + + WAQ81wlgqJAb7/qjAHB+QukoOj/4FMTHkzSmdD64lPzMLHKFOA52rFh2ExX82+fR + + AV1CqAzpzZsplgPX+93fyXf5fGjkqiEA89Y0LQIDAQABAoIBABibn4GzR1X0dvVz + + VLG5VclBguETmduJQr93nxC0o2KOmogrmP91OA8EyV9zya+Ni+D4voCJy8odrscy + + 0hmjWE9YF6+X0je1JUNEKKGokrxTpfkZOs8+XhsC+0876dbBOEWcDDNiDa5rl/Pv + + iXBY02ooLf6XjMDIQGSAp2DzOQHWf6knn8Dk8PUlspv6u0GScQ8EQPlBTDREMOhT + + eknqr5ALTLbCrj35E3s64AELAzuKKDTj5FZQfQRlS4Ir0EntBFU529FT0rwRbz4u + + orltCzEr6s4EmdSuKpjaiBPy0rNte+eHOJv+H42y+nmBf/mFTq7Nl2fpAetDNSuA + + N5TrVrECgYEA1l7HJkg1gAxwm718YWoO7xpneaSKtYagjqPjuvTbMtDScSaPQMQ1 + + JjraJ1RXd9iw/FR2+XKqNEGSU9/E0mL75cEu8uBSGGe88pufVf4Mc8v/tbFzquyz + + uYxEZ88cYhWLGTWCyG3RGb8JvUG1fkZ0wVCQ68a6RQD/XTZRtj2jlR0CgYEA8rvt + + Yu8kOu7wMRQaCDZy2g74wjp3KnyfNnpFL1hFqojDcQ5eqkW41EHc9pOgOVmRcJm3 + + nZdCa70NarWaH+FwdzUqv2ovGfUDkmt+c0AY7LD4aGObdh0NYGWRazMgtl96Pu+T + + e5xnpGcgvUvwuHt8qJfJljlN4zF/JiJFbZojPlECgYBiKy0P/ulhJlE7QN8AzUzh + + ejoAnrVWw7wrFipnp1HqR27Xmkzn3/Jm+3SDpkAYBgemxhdlzHjdTVnxRvwfTG0G + + nh0d5FQ0EO2aPGIPQzP4o2cKkaTilVsIkY+R6mqZEDyO4s5tcrzbCX0wSjMPDLzS + + +k4javJKP1ayHPn2duu+kQKBgQCgvXATDvf4CtiGN6CRhbUCz91Nibf2K7anNcrw + + 8kyYBJ8gE/r+WNNvw/nWU6ZLtBOK9FBSjKMQg44J9x6MNBbs6glX3rI4RzdJU+PV + + 4EFhJEQrpKKDUfPUvQ3SZnYoLwvd93q75bQAe8aDdHGBSU0gu/tjfqkkZVek4hcF + + 4IesMQKBgDBIwX5qcqtAuCB4QtSXzfAq4J6QDTMBkQvFRDCDDThum7F3ekfKIQan + + 9kIjeHnIUQ9OfbZ5etoMabNBJYni7uEQixD9s5E4vR26nEjXndgaqcG1MMIZoKEA + + GPsZBYwFh9lGrhsdHqW8L29s923b3RUrB7VOaUkm69cr+RRqpVdh + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 131071 + seed: LCa0a2j/xo/5m0U8HTBBNBNCLXBkg7+g+YpeiGJm564= + zfec: + required: 1 + segmentSize: 131072 + total: 1 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:CHK:worle55uksa2uqqeebm4yxnihu:vta76jbmejt2pxx4prqa75xawpdtx42cmzzhappeetzjksym37wq:1:1:131073 + format: + kind: chk + params: null + sample: + length: 131073 + seed: /N4rLtula/QIYB+3If6bXDONEO5CnqBPrlURto+/j7k= + zfec: + required: 1 + segmentSize: 131072 + total: 1 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:SSK:qcmjdosuiaagbmbgc7f3cmy7mu:zfrk6ez2pngtyrvj2mqgigkk7omanei54zeakwwrig56ysn5qngq + format: + kind: ssk + params: + format: sdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEpQIBAAKCAQEAmwWL2NS9gfKPBQbjci69pMPkDvtxHvv70JjC2RGvt1tcJxZu + + e3L4LvSFu8pZaGEffNFvM286giZuJLM9RuYeLGVk1ohDyRoF98+hZYGdv5Zsk2nn + + 88m9bBwi59P1U6b4Q9TcmpkALHc+nlKbJd3cI18ma8OE5lw8Zy1O/Il5K1U/5nf2 + + Z5k+AZ6RKsEGQ269t4izjruSfjdVqNSYKmBb33y3xZoORW+w8o+d2564gdr9wXEW + + vKcY128Hdj+M8G/Nn9tHliP2xleyjjL2OGow7HDk2qorQDaf6pCG35FjF+VkcGbi + + jiN5Zy6f4LZG6lrWx5LjNqEMujCdrB5MQEhxGQIDAQABAoIBAAwQWc5bWu6VjeH8 + + I9/fSZCckn9Ee8yHm1SDyNkL7m7gWBruHNW0UCJWUuB2+i98bG8CHOtlpunZPAXU + + 4YpGjvdRQjBMTSsloGyFLtgXhstRtVwg+CUxkFFNHL3KEcvLYQNlWbAZw0jSlE/N + + maeQHQkvJs4o2mszZxt37A9M0v5HTlUucI5cxdMbi3H/1KnRkCuydHZ1HVCduo99 + + Yb/Dbf+u6tJ/rlYJ8NNPxjTHCM4xsSr7VYJWDD2nAgwHz59jprML4tk6InR+5xFq + + wA77gui2SyDsexKtvci9RLSJYArj0AY0PPlcQmUksrNr5QzRZgYI/cL7GqBXWbh/ + + GRnNpJECgYEA2mlIGKG7oqvZ1x0RAxjtttOhbM0bbSTYlBEpNzK4lUnEixFYgFpQ + + Thf1UHwdVSM0U1Cu/tVZG4NVO2XDWzymJOb3KWUobBE4bLMfqYRI9GigITwkjTLw + + 8lPGT4al9qqooE1WsryGzL/R7oqjoX4JAyTXEYHPT0XO5h5jCGMoogkCgYEAtbN1 + + gyS8eZ3KxcjIAOzD+Y6Fnt2BQywN9wuXUF7nSljP7DuprEzmnCuH2TUA32VglPX8 + + 3aADNtXiyT7KbXyii+tWwXT6h16Jc1qZJtUd7DXz/xIFZRKbwgG6No3TxYJe0kdN + + pDWdECuAagdH7FTcxiSTDwB9DlPZr5SRLZhB2pECgYEA0ng2A5aHLEESkRrvc96n + + 5FCX9DLKxSiGlFjdMNXtzd7iSWkTscxWKosn2MFhutNL7yWHHQcW8U5j6fMsiFUv + + fcwcTYWvqEQH7afHUSGq8+uGs8AzMOXwDnTwW15TvBnEmYUtkNvfwprugEKVYGAF + + 60OrBLHkxm1s7ZBGuqRjWZECgYEAmVU1IJN6vcKr4FZ8eVNUWh+soRDZyV6+9jCA + + 46EC29mwtPDwUWef4EBX5rN05hB9/ZbMahZjP/4k4KEtYFGiNiNGVgEqfdwIcCEP + + RxbnpnMtUZ3akZ2vdXvRscHj6TQIYrkrSxy3S3L6bf9w8X33xPoOY8WMwu99r07X + + aLupLxECgYEA1Cg34GSuNWC9M/YIkH+l8oASMg15NY032PWQLmGwLvp2M0hLGYzP + + LzKpKs4tgJPcHFcgLnqMpdWorTJIw3TtNecTo8y5H3J4rhZIH5kMzayF5sHB0QFu + + xfy3oKGpIvekUkf8l7QWhq7tBaJav8Zd10MrIL6eFtbmZG5EED2yE9k= + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 131073 + seed: /N4rLtula/QIYB+3If6bXDONEO5CnqBPrlURto+/j7k= + zfec: + required: 1 + segmentSize: 131072 + total: 1 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:MDMF:dl35z7rympvclushgyujvlhvhe:6iutrwlkcpm52fyc5vatkw7z2if26xoiatt2ocyrl7awjshzewsa + format: + kind: ssk + params: + format: mdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEowIBAAKCAQEAuTKFAzPfEZKOCb/DE2xRMyp/OrDXicd1KUAENBHOlCOv2AMG + + 9KJF1Cpbw9xJ9w1QJUmkcrT8r+2QFP1ECncmvwUExOTq1wN1Fq7aAU5HxdJKZQGj + + vx1R7aEgvECFyiAAT6dWYeXNb59n541mtUP+zUp5X/DH2UD1s0MO9NNQE0ZD7Gmj + + od3n+AXYaqg5Cle+qNwjGlujlJQepis7CfZsf6h6L4FPNbToVFXzYrA3aYPBiCmU + + w5TOlxlZS7JhSg6dTY6poYBbVFBPoY8fZEnOTfFFA+Y/ope1Gwsw/Q1J5TItN29J + + 8mUgdfi5P9ILeoKlpGdaGLOsRraQkYSONjhYrQIDAQABAoIBAEaQyNzpCWCtOoDd + + eAuxFJmN4k+vLVl6zhorIc7jUBbjKDADK0XQhRHsF+4fxHElufmTP11TuAqi1ukg + + faoNL47ObzxEy3SlBRrhAgFIXhGy6JTnFIkQN3T3lb0VSsUy/1tadBA2W1piX1l7 + + 5/w+jdqUO35ChSuzVEt7TDoeQF8vGxCxrHFogu+AsPb8EGAbxJT89r07fwnFvQ4G + + wLpzedeyooz92cp5USYRTOZwYVjLj/B6Kk9omCZe+yjPru2HEhE1wTY5/ftqsZ2h + + YLKfXNI++xmwU1MRYVFAR14hHEuGgNwuogGwizLA8brh2Itw4hze8YzcX+N3UfkA + + nkOvX0MCgYEA6DboXuzoS2vSd3i9WNtzk6susX8CsJ8jnQ3QcLrtTNrGsBNR2KwJ + + IWrZp6eqr8LzM6bw5ipO9j1HUsGbXYm1djx/XCa+Fb7e7LtsQo+/sv7P9iLhZ96x + + WqI553w1653/gUfgzvIriSP4PhfupJxqwP4yGzm8AwmcDOqBb8thO0MCgYEAzCq8 + + VQLnSVBmvB8nGxZslc5F817kDnoFjqnPY+2cnJMQlf09379c9KVhBDO7eqETaNMb + + 4IDQM4ebtOhItQ6n4SRi0u5ZhJdwsFouItgYoE7pZvFtcM8Cs1O8dvhwDCPkqE9V + + QO1I0jVFIwTKddETDcbC+ykbv+WEIgWzcfHqRU8CgYEA05xFwUtOnHxTPUAv/Gtv + + NWBHmsRNZTqAL7zI+BG/8ctkSEwyx6puX5+JXPiz2JtlGOrGmFhxwH8zIb0Aogq9 + + 7FNRFF7R1essJrrc+wMYBDuks34xvn/3SsqOzd4pHN/MWLlxqeSRu9WlgKA6fpNz + + zQ9YBetk47e8FyEUdxX1MxUCgYBgaLHEJvnWedv5a3CI7v7Zgq0vbhic6WvkYTVo + + h5STrzJ+0TW9iVy4vbthQ5h9IMDMmBuq2Mj3/Eo/lAx5SvFldEwiNKEa5nQ1InB0 + + zbxbPsgib1Dxmx84VQtC1q/6W5ynCcdFQIdJlBQQpDuChPbNY5VBCrlq1VOeyThi + + Tw0EKwKBgAz8tUCZTLqbZ6wfiYdy65OJlW0j0npyYadU+SZdzvqhKSG5+WjgXHfQ + + wPiI7a1Ak4+r4X0QOhopFlXRATDYc5Rl8rJwe1v+5MV/V+MxavjCSiaxT8tULlq5 + + HNyDuTFkGx+B1jxvZYRCPmEk52JUGGemMZ2RKcQYQcmJZ80X7Ugf + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 131073 + seed: /N4rLtula/QIYB+3If6bXDONEO5CnqBPrlURto+/j7k= + zfec: + required: 1 + segmentSize: 131072 + total: 1 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:CHK:6kccrgbtmmprqe4jcfi7vf6v74:wpivl2evi25yfl4tzbbj6vp6nzk4vxl6lbongkac7vl3escvopsa:1:1:2097151 + format: + kind: chk + params: null + sample: + length: 2097151 + seed: uqWglk0zIPvAxqkiFARTyFE+okq4/QV3A0gEqWckgJY= + zfec: + required: 1 + segmentSize: 131072 + total: 1 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:SSK:qcis3fvjbbmzunzit5svs7imbu:jfqtuto6lbeqgt4fwh6gkmb4uwm6fkff5ave7quak7rxjsfs7aja + format: + kind: ssk + params: + format: sdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEpQIBAAKCAQEA2yEB38vpu/vDnChg+kmW2EocnNln41P69nIF9/6CECDpP3gl + + qmRWWHANi4e5nlfa8nS+vOu5t8fhd+kDfxA6vM1I9eZN6pSezksyK70XCeCrGW+N + + OYeVzg6xMAzX2IhRWr2fFYemvtPd9PFx1aWoU7i42jkde3or9Iu/DaIi4qpQQwfS + + A+5Oy8m1ZkDKqE3g9f6f7j9pAsTIKlkOzq48iuDdwlB2ZQlFf+clT7B32So3Al5M + + X9nEP1FrlqzSXkrakTL45OB8x43Uj66zEo0Bz0mkkawI1GWFGkvJ+Mb0kYLheAAA + + Rq9TxdQrJztnPCJhEZ+LgZCONXEvzETn7/U1iQIDAQABAoIBABTqmKgzvv1/nFNa + + k1nbY7evk1LxeaDRvQHV2SCr6DRWlTfGuymsPDBi5G1uKNdRdt0agbO8zdRvaDku + + 7R4tfr3Tho8E9b5aLgfDq71QZlOTYMxOa+zoNpzIUDzQQluGijrJdixIRNqwPzPd + + 8XS9maIBiZDi+h/K20k+JvV2hYfxy/lrBy8kWJjsASPkKiuN/i2kvqT32kxw9RU4 + + RHz8F8dzsHPbT0Kbk3VwJA9tSYdMLOniHlmETuQ358fd01xJMy6KvLzi/Tl9s3lO + + /J6zmYkuJ56XgErcxRZni31KsMuYUPxkvAfyjeoK7nmJNVtY0A3hPNspSQRFKPww + + 7FWSzlUCgYEA8quXOaHqxbGa5kR195F48DPt/Um+jP4C1WNskYS47YQGuhqrpYPJ + + 3LjaZWr+nfSPaV+xwGePNDTSC52v1Q3+DdXhYAmnvCj6U7FMMkcUJHFbQrefdta1 + + YUZd6EPiLvDNDVeYcLL4kCJnznsWI9e6gp6t/cI6+lq9tBwvwzm+nusCgYEA5yph + + VWmTlMgYG6YAblk3OI08lp99jCOB7GnBWuBFg3s0OmY3w+kjjT1AYzT/nr/Fxv1p + + kt+deClSILqR1JdAQhhy7JYZtvSj3D/Gq/C3dBFSkdZf71k3ZGWkt0M60M061OAE + + WKvnFg9X4R7oBQs9z6JECaxCKL//ndgU51WaKFsCgYEAs4uDew+yrYyHqAFVKtPG + + ICq71eB/DMBPhmRmipAhZxJ9C6r5/p8wdo+KfukX8/RjOzqjQFEe4iiGlDOaSc9t + + ff0WIEFkilHjTJLsZnKyk3gPZqCHapzXXF580oGPUt21ST7bOd8hCzt5hIsLSX+u + + rkALSaowitUicKU+LXqG7/sCgYEAtQTC45ehMcje2AfOHptOWsJ+x5RtQ+gqPW8z + + Mm6dALDh3TleQdO3O0rTuNwvr6iMv56BpbnmHcp9vZNbzxYCA8ARfqKr0FESX86x + + TMNbZVCLUBiHV26NqdjOe5Px4sBTaY9i1+0FMIkjT+5b0ldTN9zhWpHB3Rc8m+Yx + + uFWYOjECgYEA7uVh7iyGnHQzlEqrdGmVhLulCe9gYvKHUxX9RrcEDIIP3hC8A2HB + + YBqF50ODzPo2G52GOXL6CjENZ9PQXTG9o2nQtB1k7us6Dxsh9T5yxp9fEAV0R0om + + rMHlu5JSGjo/eEGPTQhKYXkJGBNZzBqMBgQay4vdmS5vnBVP8brpIao= + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 2097151 + seed: uqWglk0zIPvAxqkiFARTyFE+okq4/QV3A0gEqWckgJY= + zfec: + required: 1 + segmentSize: 131072 + total: 1 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:MDMF:5nwmwaa75p3mjf6yzi7ejaecg4:uxl3eoj3fagsubkfjrhx72s6k2kbunkimuee5gt7fqtukzn6anxa + format: + kind: ssk + params: + format: mdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEpAIBAAKCAQEAhXkJkX7At8/h4BgBfLWyC/RKG9CuTb61kYLzwE9AmTRUAjnU + + 7wnv4DvHj2ZGwCIrEWL+4KkzNcBWH91gQuIfl0cDFOhrLljGrrTzWoFMzhkWQm1p + + FUevha0DvD76uWjuEYcS1OMSIsk5hxdK8tbQSLqk9pVgfA8sa4Nj7lpYrdNSoyJR + + +DalDZyvf5/SbKOnu3ETiUhg8CbyG8AHy3GLyRE5YmnwdloPzXuDR1Xrr5VmFnn6 + + 7+3pvDD/STUNIGg1bb/IgQ5TlPcZp6rGk4sHqQHa5YJ82jA6Zf4xO45jUZZsG73s + + huVzGtnBBUFl7VXW7f6Cl95vk/V0tZEVTr/LQQIDAQABAoIBAAZJNYSnKg9eGH6z + + 0robWofKmQTnVpYtwaJZPv6THPE6MCysqZUabDQszJC52eIpmcqnVWaiQVmqNcQp + + amOr/53hx8jfy1By+OR4fC+KgGICd3Rob7cDWcZbaB4g/zDlOrUTnfTtvshpnq54 + + j9yQ9l0+gQ9l6JXfJxHnLbkngx6okEUbHKab9KCk1iclggsydveoNS/4vDpGeZNw + + G3BySsD+tUR+XW+oRe8xLU62rAT5HSTLjGtlUWMtJ4POoGeRPNiohNHeVcucCdQc + + jSnuQe2LalAHMa9bpII2qSAUbKVbEy+RT8DsTj5AKxhiN0F0HdxWGAFKlxNHcaxd + + x5pWA3kCgYEAvH5OoTBgSgEGiJoHB38kM5xNoOs7W/OWCurqpnJXSpADC+biu5vh + + 5pqlBXqBTrmDddlctI1JQXXcYPHhE0O40m7exA1Tzj+tgoLozgbejEHKNINrCs4o + + 23kKrnmya+JTZN7scMzjP3v52SRw/W22Sr293TCfSL7WRmRgpnZq/z0CgYEAtUZD + + JKHMtvthC2cuHRsoGik+KvtzJVVBaml3tSgF9XkhkHT2+fZ8gf2fFhcl6xSU5Nd+ + + oYBmTZQLKFwLQotj6rJ1kWvLeAaMR+FV/87zaLdaKmPKlQtL7uywN4JSWOjWFKmO + + JjxOKZ8W3hYz7UY88fZHqvhN1+yXvagIf8Ft/FUCgYEAtTvolFk0K9OCmbMnURDx + + GOKPTUr/vvCdco/e3/0OazW+iCIOHP7LnHNSecsJK0151cURutQh/Fu7ckb+9wvl + + WAecDvsVejiFtvfxqa63KjpTllxJfpEsfaGLIKkIYWyybElfIzIMycyFNUAxl6p2 + + XLTFKjiG3mYHFpWKzGMNi60CgYAGt/TzHaAVxBljr85Qu9nvpkmslCc/YfqLtB8A + + stwNrhClZwBkYVNaCglkazU3kkq2dJo36CdihrMnKsosDDiG0Vh0LFedOjjmzR4/ + + 3e69mdYYrhwrDAEjeNhLJmRg8ThGCca/+go3lrLlRlNkXu8RVLxxRMS11QoGuHyg + + J44pBQKBgQCptTkB7sPmteFe1Yuc3TBUiuV+QiVuW4gMGulaj20jGebegZcOPYDu + + xvrUThzeoJPJEiv5QQNQIIdnmJAcaTtPOh26ivMKDBlOlaP5cIjLhzC0fObJ6BRe + + mZxl3hyrfzE0i0A2Xr7Izw+xEViw903r3qYtN0/rjP/DbIUPbwIX/g== + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 2097151 + seed: uqWglk0zIPvAxqkiFARTyFE+okq4/QV3A0gEqWckgJY= + zfec: + required: 1 + segmentSize: 131072 + total: 1 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:CHK:ofgig3loex6pev2eymmvohv7wq:yasbfedqnueaajdcavuba7kxbdqzn7e6zx3y6cbvucgx433vlzcq:1:1:2097153 + format: + kind: chk + params: null + sample: + length: 2097153 + seed: BTBX/ampNfLU+ox7xipBGiaSbgC0kcB8Gy7BkJB4oKI= + zfec: + required: 1 + segmentSize: 131072 + total: 1 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:SSK:yxecfym6ggu2pxfyxitnxuzyxi:tphao7q3qc2rd3jc7fsflqgxljohmiqwcvkbfp5vzdxit56mcyda + format: + kind: ssk + params: + format: sdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEowIBAAKCAQEAsNY9rFTsXaFGu9ou853zP7BO/Hxej3pTn9am3vLHBDh6oXiK + + XA4xCLGVUl2pRwhvJyZrNTtyfBQKOEjYHucrgAhZKpCN6Yob3B3oy09DEAhU1bIl + + VTaRVFthaQqU2woNpyK6I9XqAWhh/xybcaNs2xexSS22lnSoL8VnmU21Yqtq/n9c + + UXAHIKae0D+rK1xiWeSLSjBwy/EHz81MrfKz7zoLCt3qEBsCQGC9e5S7CXYrIgjU + + ZnKH2ZmPpHfWA9R1BS39S/q3iMrJgTsIPz3R38pSfpCbxhGqiTN5IZLHAHxrpEyr + + ClRpAN4qpfFnsrz5RBbBn2mQNR5CSXvzv8aTOQIDAQABAoIBACO2XJWnP+Xnylxp + + I3bFFQktbsIsTr98lZNP2vrm/jyyuEdQS+br4ciu7mhILIXBJQt7xYZmV0hKFsdH + + pMfW6TDN3s4LC/HYV6iELM2UWAeOmy1d8Q6surxVyQ4Y2jeDJ/8zMvLGQmAe572I + + 1jakqbj0Z7QO5JMtg4LEQ9gQursua5YVkQTSGIXJQUbkeM5DujcktyGkTT71vlmQ + + NGJz1JE8VSTWAGFcoWSh3cEg6PwiJCXXqGGhP5tgGfvUT7c5vzZSnYuHAJpwoTaV + + IoF7ICL94mRLJEOA32YNwk6vMkZ0YQBv2li63AsGsqlvMCvI+7K+RFyINju/W6rs + + fkyws5UCgYEAySvEaYRtzxZjvuX9atInCqsHG1aZQaRArE9DhaO6a8zMkZND7AP6 + + BkfUpqpPUFg1NVg7FfjSIZDinmo8RpeJKoLG8b1Mwk1ewPlJ9caKcUF6fUAimYwz + + lyxgpJ484LY0eh3NtZrkrXmFwbLTJZ1nX1NOBYyHcJRqDsYQBE5uyZUCgYEA4Qie + + qInNOpmywQq8CUbCkNZR26lsgfMQRlD9LLHyeZ7Vli/uQzVXm1Yp6O2Q1ISedcqA + + 9xTARqDEL1AOAh8/6gYcwQTiWBAJnyP1op4dpYDx7HEGDquFJr/0XhTQSyNO5Jz1 + + wOogHAyFoigiKwchG0/Y6FomBjtrzML95IoxYhUCgYAgH4YIl2X1eIzK8ezKfu5P + + DMpgui5UxgaxvSJ6F4/wIM7VvB9Pc78b+6JgTrfFi6BLeWBN/OKJC5q0UyB24UG8 + + 8Q8VkPXN3Q7xX51IysBWn28Qywn7XODsFeEEyGPOOiodCd9MTYSQkuQh7w06Z20X + + UrUVu7/w4TIiU8xA88lLFQKBgF1IdHSLAx8anYX2TDJQOdFOdopnNgq8Vm+/nOON + + NGWEGSfz4IHNt+41jpP2/sWJ4CIV+tXxrS7Z79lpBxWMHLOHKx48RxOYOlTU1Ds/ + + 7iwwQpjv1UH5ie2hPsxNNncfQNH12s3If90At9ibaGaLwwaOV+0hiYel1C8CWbsG + + KihtAoGBAKNwLwOEH+N+bJr1Ce626JnAcvJpOu0rPZ7Y2Nyi1fwBGIiME6+93/0S + + YdRBLneQB8gfGmqCK6Jso9XUTn6yQyXx1ARSGwvQ9A8omJFfjLKFQ5Z5qeLID668 + + BEgYH+Ob4vwTfNcMJsamI0XglzBmwmG8mE1bUUA8xZAwh2rHTdIB + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 2097153 + seed: BTBX/ampNfLU+ox7xipBGiaSbgC0kcB8Gy7BkJB4oKI= + zfec: + required: 1 + segmentSize: 131072 + total: 1 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:MDMF:ca6fonfjmpenkjyz46fvnxuxla:z2olssb2426dgptfsifbimefukkhlbab53pz7jxv5cxkpumzxhcq + format: + kind: ssk + params: + format: mdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEpAIBAAKCAQEAt8xN84sxgxySP1tLtcuXgZ4WxfZHd+oO/MXx4v0mEGpiUJw1 + + Qdb7WJtIg2tBMS8Q5GgAEZ9c4qrE7MJ7bTJCZn5j5oQszrOQRLcLdIK3yOevWlqh + + wSiJ9qiqFxM/fGmlkSJGAU+V2MC3aSl1Ak9+0BgPBrOozj92NoCeOHOyLnURXmwI + + ZNcoQJc2hxMdoVQKiDPo1Y9UbvZ0CyW1/iy6X244CdnP0dUNa5JHu72TEU6+Yedt + + +a8gzE0tbkktg6T76y7GWf55fqS5ekzJgGszellICVbGQInyDBG7B6rIYZr0+DdC + + ZVPK/4q0yEAUbAg4fF+QB25HnNBfVOagQpQMgwIDAQABAoIBAC9XW+K1wSCMzOyt + + xtACKzmTLzl5SIpOCuM31yiI3POQe1dZDOyzA5Wcla5oA2g4P8kdMptXaXTm2IdF + + RsZnEixVNMUs2V+6Z5gTb8toWg9RAd0riAt5NiQG6Jy98/XHPoKmCdMPnUCxzuwy + + 5fUc5cSS1df7kajiNsAuG9LdlhEZ3WvE8sfbbFPIQGOBT2UHOohTtgxCYsE1/Qm8 + + ngPrlQrHkZpnHrRCAqbMvGjtr3KEyrDyLGAUmKVxTDz8Es7wXyzVARQnsdmYySxy + + dStQR9YkO/PS325zygR8Ynp6U0bZb7s8bO189hJPIBzFE+p9Xm2H5r+DQruu3GhA + + 1bTtFEkCgYEA41n2ArxU0HPqlCxHdKkI+YbVbebF02Rp8gQIbuzZ8l+agsFcUCXc + + c8c3ZVi7rFnDZ+PlCBLoG0ZN2oVGQu8B7jNc+dXzC5AdA9LLk+Yd9D8hwAXEOg4t + + r/tPQDvGN5f1BDO77aquSN7QC03D08pFw777PUEF04DVa2cA1VCgsEsCgYEAzvVf + + l3gc+XW44LAyVs5Z80uhZkkVRTRdJHvols6SPdY4DU3Bv8UyDkZM/Vc+xQI6+wx1 + + 1Rkel0Uqf0HYgmTZEjQ0IgQczxWrhra6lBRJHqZKldI6D6NcuokTepBADBKHryq1 + + o7oj4YOoZ29Ck1Xj5aPuhYroBXp1n+dMB/WSIakCgYEAoXB7IZwsOc1mEIuUvgFe + + DxowqhbJ+P7/wEwe1O25IcPDiv/VFlCcR1Z6PqwQsCUZfcc1FlOen+d/VyF2MAdZ + + /pRYfEvxhw9xmwpvZvlr4cmGpL0zhuoUhTdWIk2PxmBQKwi1dOHTWollf/FbkiO7 + + AHG4I9ntUi/U3KxKyi6zvBsCgYEArookJ1NmZECTPfN7UNhQ5i4nnWMPbEEAOK/D + + dcQbc8lBln64YypEz+McNSCqUG5UHbvheGnp8bukXpTCqx2wMHkUaoe7YC6vbTqY + + WiBNlmq6RmZ5Dw1APBU0903GpifOhL1pWP64Gg32Ld2YcTejrt01YSzIBy7DGqtv + + 5NqHdpkCgYA1i2pMoIwKh05MowZDPejnHLkxPHv5mlJiOZPM5lHlzUIHRXJH7MZl + + IvwJRbDncvup3DDa0ZHYKLop6WN/l1smS3P1pTbL3ngcHSR7BCQ6KvfXDmjcSSRx + + TxYXbqWoO+lWzU0LaRspV28AgMcOHi7+FQsiM2icFClBL4P8R1UISQ== + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 2097153 + seed: BTBX/ampNfLU+ox7xipBGiaSbgC0kcB8Gy7BkJB4oKI= + zfec: + required: 1 + segmentSize: 131072 + total: 1 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:CHK:scwepfpkvhuc3fc5jkpmrodsfi:wui2ixaxyac76mbxtr6diw2ajky75wehk57f2kqgytddnpo4rozq:1:1:8388607 + format: + kind: chk + params: null + sample: + length: 8388607 + seed: w6uP8Tcg6K2QR905Rms8iXTlksL6OD1KOWBxTK7wxPI= + zfec: + required: 1 + segmentSize: 131072 + total: 1 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:SSK:sovuio6fbomgdor3sypj6ywd2i:l4caqcoo4xflec3tbpt5jseaob22rqzeseb7fda6asms4z2ybwva + format: + kind: ssk + params: + format: sdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEpAIBAAKCAQEAwsvOEZ+3GvfC5j4n03btkN5wjxOcjfhmetfjy+lnIvdWZWpz + + 3TrCZZqwhOhJUZnTh1maowvEWJyWNpcziA/Z2GBrKEA9GhToew/52+TzSmIEzneU + + vM9kR5Xr5KHPsM72FCpemBgtKqS9gXODjZfdz2+wUu7Ki97vSp1uT/1RX5jyARDA + + GhqXkGU9O4JM5s9UGoP1JEmBiCWDhTzDhmpRtm67YW96pBA/pGzp55ZYnF36BM1i + + ggzu6gIuJYmPPP6LZ0uJ+LQknsXHVQxsk7X3zCoUBmcbbM2GOeoC2j5Jd3WQ50u9 + + DbwMyrrCbJNNJdicVxn7crWk6UWIQc/hL8RPfwIDAQABAoIBADaQaxURtXMW4p+m + + 2nYH8qypOkNBnZFA+se/MH5eTzcCrE81HeZivrBCP97CyELUwWVA6qlwMtwVZJg/ + + Cz66He3XuDxqnhLvt109UOJRA/sacLk60s1+lFre+lgtISWoG1LzuVKGNySiR7j6 + + l+dyGj4wTWY1oEPEuyed6Jf8X65Uhjnn/fCQsZcMl8Sj3CbaPPYemENE5fd6Hnna + + 9sZVfZRg3zZsN4oExcggu7kpEKGIe2atVrPrVmutFtqhj3VHDnHdWgfci6G6ETin + + K9TfDuxv072xUWcXhUv0Hn+fak9OGweX9fVa6AH4K3fmRxGuHHQAhXbQTUQO0AdF + + sNPQqX0CgYEA8bqlaep0XHNVER2Hkz2/xjmHQdhK1fhpU8wHocE6TejUoxuWHGYn + + 2JSS0HO/Trt0M01SKFwYiVSnHY1C7SHCjH6dvIxGQPYkkEvUopB8TFZQ9xktbUlU + + Mn2ocXMXy1GBi/IuWY1qNgusdlPi6kPod18joNgZ95GEe0J/9MJ5weMCgYEAzkvX + + SDSJCMuwqCgqJFpB9kEMV3SPLnlqHoeB622taWrD/8DDd7dt7JiZUZ5VifUXJBzj + + TWALBnSnaie3zgBXOku96uovvgtofk6f/P7wSu5t4OReh3vMGcd9fcysSpxJbeuu + + U/Rxlk6ZTe3lAlVMM7++DOkGidqkpmozE46l/rUCgYAXLowgfTCNkS3uR0OyNjDH + + BMtY4DJFFN6c/6sXsx0xTYve3I1nydA2cAEoZoFJPqblKJwhbLuZp/mi1uI9NYif + + yqC77UPrhO96uxr4QBz7gSegmtSFb4vYj75wqtX0VKu0zRPu2KX/6tyuOFtBliOc + + Fw6mpTLQUC9BVt5IjcH5ewKBgQDBE1g8uvaaJdGDwHuYpGTh7gV4AJ5VV8tLIXYl + + +vN3GzavsiD/dczKyBOOwQq74Ig1A1h1vXL0Ks/ZWaz8f3MkG2l3aJEgZBr7Q+kW + + 5x/McZSjC/mxAduHMR8xUxLZjaZn21HAP6Ljk1KGDiXs5ho4wLdF6/5znQ/GtNRy + + 9GpFlQKBgQCH3NXTfsc55BZu+tooYbsL3wZ9iRn5oux6+MfJcl3pTxtW5twBELgI + + Kg+SjF19Mqx8pgaNqnbmltlxcVQW0zA/mverUrc6HJ4SgiNzojqLEqmm5+JVPnse + + r2qBE2Y6FaBEDfQknOhBwquM/nAXk4X71hl1hQZBvZq2R7WKbc0UgA== + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 8388607 + seed: w6uP8Tcg6K2QR905Rms8iXTlksL6OD1KOWBxTK7wxPI= + zfec: + required: 1 + segmentSize: 131072 + total: 1 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:MDMF:jc3ui6y2uhid5oy2nt7kd2vhpy:5vww6u4wsitqgb4mkrvy3tfvks3cj6t3smfuryzdzjbegbamdl5a + format: + kind: ssk + params: + format: mdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEpAIBAAKCAQEAp7VAtzy4jNgUD9wLg0yAbKnIk/vTaDNGpnPbLjqXJP+BNErp + + h0aPmWCDUOqnyLwtgqDdrl+juwvx5cF/wRqRYQWB00eCG5egccx2VpVgu19K92da + + 69IGGsRP3//qwYc3CD/p63/MRcM+VlS52XdI0hSMpVqmj1ywDRXTvN5k9leaBB6c + + Of4MetXRX6xQCEB+KrEQ8uVrdoHXEi6ePZpJRAwsP0N+dAhaMBS/TZqdR/MYm12w + + aipJeqxTHFj3gKD2FIVizB7L2ViILd0pS8cf9h4B+9ZsrVkdMk1sysSci8LIEH/y + + lpCaLytW9HfGzKmyRSBHGzZCfrCR8ip2sM+y6QIDAQABAoIBACGEKkJUisdvGZdP + + 0Sc030eYKONWRRpCgSCb79ZN1E4LGB3EyOYFloY/EQ9XTh/iZ6//CT6jk3u6t+XE + + ZY1Ii3xZ1ufMFzb/dwu3IoFMSjA4K6nFCJkveJPZ3uKz6Q0zQi7OYyfy+vaIPgmP + + 1jKdUbrWa4NSWg41pmN/FLlessu+RFHj4ye3rYzPZ7VMSy7MXpft7F71k/wxJEPN + + bg/1DIA74LWlDtsLmfIhv608H//GE6EqA47kkIpfBfgaOuhcWn/Iw8k5fuj8/VkF + + C9sCWk0g4XL9/yvoi3JdI/Mfak6OANqI7mDKosJAuLtl2ZUwoK9vxUwLVuais8db + + /xe8/vkCgYEAzkvTQGFg9UEpgbif30HhU/TpdUJO03Vz6P+RY2JsIFjs/ICmG2md + + khnKoWY2tZiXE0630IPgm3Y3blsCAmyrRVoIFzSD6FdUfvYY+Bn1g8tHlbXxT2C7 + + Af1VAFKEJ3aNZd1wzcR1gsWsMZVFOLkfS7NlnLWsFo+pWPY473x7RjUCgYEA0B1X + + d2nP7wUdlAT8OkyXVORfGeOVgpOlEWwXNSQpbjtW/ZmAhJHAouWT6le0LLjx9TBk + + Td2WauOPV5B2M3JSQveBiQtM5g8UQxywF4Ks4yhhjD4WZ9G1vWioR7f5NG+cda+P + + TollPzWX6SOwXwJsmwxZndeWGHd3DhmCwPUZAGUCgYEArv6hN8ajAciB1hlv/Gmd + + I6PoeeCCj1vdtDM++EhgIlxsw5C51x0TXgDk416aYBcNaIJo6MdFu3pfcQxgOwBF + + lPHXVR/mGSwjcAOAkM0sd9zzX2rURRpv6DMmbLySgAtPzK44Z0QUzpayB+lwq7pV + + cti+BF4TmZvJ8r4C9BvrUlUCgYEAnARTLQ9jNdIE8ZGnMWF31cl6ziLCU+ixx9Tb + + tRgOAzhzJ50rLrdBzh0D/ZuQVDK2GVUU7Rbgi/Na449GPZ1HtDJuprmVBadqTkG0 + + dXuedpEwR/3HuD8L2xoZheKS7U964PMjIQJ5p6Ba6Qm7UA62MqpYiK81M9RjqWtQ + + ja1w980CgYB1iRfNAUsBOAA3wzMZ/u3JiwuQaAWqnwiy79UsCK4LQ9zP5uJ9hiJF + + dTsSF1VuupNWNdVKifvQypOLrU0AFrHqVJiypKXA3F/1fqQE2RWXS1B+c3QZJ9Ga + + ze+uCK+a0b4noZE+n22hQbc4PrItHRoQUpB/aFJ2C3A1IP2vPpHa/w== + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 8388607 + seed: w6uP8Tcg6K2QR905Rms8iXTlksL6OD1KOWBxTK7wxPI= + zfec: + required: 1 + segmentSize: 131072 + total: 1 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:CHK:5eeprb6lfxclwt7fieskd2euly:ffjgjbrxfl6d2ug2a63iaesxtrqa5pk3yaagfcldqvo7x4ok7ykq:1:1:8388609 + format: + kind: chk + params: null + sample: + length: 8388609 + seed: yPi3JHKKbWaEEG5eZOlM6BHJll0Z3UTdBzz4bPQ7wjg= + zfec: + required: 1 + segmentSize: 131072 + total: 1 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:SSK:eklrgikjshlanhotc7plpaonca:4vumrwqbsgy43agvgfoqtorya3bs7wmjedjcwjhlhurloybghhuq + format: + kind: ssk + params: + format: sdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEpAIBAAKCAQEAtvnHGO5pobH7ONT5LdgIKMJhbMpq/BsmQ9WQCkfW3CZ28tDn + + LloR1x4rfWnkXNtGlgRUQwPzlwjZ3bWd8RHNEFgeiXaZvYmrL4oJhIyAp9USqa7l + + b8dOU/ZwTp8nxEw9RvHbalX/APx1Uyv3mbWwmzmorQ61grERI1+lMg4uSd6ySpML + + 0HJxhUIC/dIg6Of/hR1Edm5p26bUCgjNLQCxXrQB6NGfRf9IHbLlR3g+qXwEafVx + + TwNHl8yaAMNml9INDzdpNcBTLT+Q+ILbiNid58pcO3jP6qfREp0S9bq+Emcltgf5 + + soegODgMqNJdqKs14XdtSz0f7qJxIWnI4DGK3wIDAQABAoIBAFcI4cUApttEg0ip + + uWsujtcAewIaGKCZs25h1/Wj7VZjr4HZj5WzPzgxgCNUKs1meiFipsgHyacGjUdS + + G/Iu8vl6yO+/K+sF4Jko0lUr1gi/J/Txne45Ag+bMhmbx/kuAJnN8n4WsMkBzTcG + + O2zwiTSUzSCgVgN4ATxvwu7X4vm8dzD5eoUZGerFLfAFvrbiULCoWooPDEVeOse+ + + SrZdm1nwmaOTTYhDJK185nDubn6CEV1HDs1fDrVGAya0K7T/WtqCWW39b+FxOyJz + + 8AIiDohn+WU1062p8XfHTs5g1mXP+EvRBk75ePjSuTO+HPR/eOoxqgfqGEJ/UyIM + + nK1epy0CgYEAxrViiFNlvOIjvYq838rb8cb2pwmYtoTEVJygBoaJiI1dJyHtGMC7 + + sIN5K8Q4tddqX6JB0WPE7HE3Q3/lLZI5SoU/IhrXQTCny6q1u2l4SwUoxbiYW3Cc + + MLvVaDrPxPwm15yN9NCNz+yl11DcQT9A2uLj3l0O/OQb41DXACPh9aUCgYEA67so + + IhrIWRLC0MQmwrg/5nK1wizN5XZwlAgiB3FE2a5xPUvMPtlRTHb4ZNOmWs33ak6c + + WMlImIzF4EvuIVyFdWZQuPzfJJRzPRFVwCSddAvKSvh/NUr7GS+8wrMivkQ5P/86 + + Zh0w5ZMOTixU/OaptmkDSDpG0i8aA3ZdkmqyPzMCgYEAkhjgpizzG2oFLyHndn9X + + MS/BP9T9dAyvsSorOkEGs+CEAfaetVlXZhN0Lqqpq4EDk+bfj41URyeCo11Qai4d + + c13+qhuj8ilM5aDQ10dXi4jyjlUHqAtmuyoPYQAErOdbw6E2ei4wZhSvZlzsZAiW + + rZiuQ1qWX3dzzbEtMswvIYUCgYEA4Y4vFJLz6ObeqctGG0MZQXO5HpaoXEs75Sjz + + BpQHARK9H52LTQe7lqKvgipSHsi9WGbnirzuTalFHR0KObnBqVfBHYA4M1Qn/+K6 + + XiOq1QMDCUFE1sVsBel7gADP2aaF8QpR4qtDwic3pO0eVO6QrQ1GKrI4WZzgEzgK + + yLJ246kCgYAx3v6a8bivaZVTML+tYGXt8A0oCL68kgTH+ZqezJiS9MiKT1Z/mLJM + + 8TS39Nr+HWZ40dqmh8RZSfGxEManEPqWH4gxFpQV/nhnktRJFQmTBkRWmqmdCZQG + + LeAspfI6SmfEn8RdLTKyNYu5Mnf8C6W+/gTADugS1/LcROBc3YhdWg== + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 8388609 + seed: yPi3JHKKbWaEEG5eZOlM6BHJll0Z3UTdBzz4bPQ7wjg= + zfec: + required: 1 + segmentSize: 131072 + total: 1 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:MDMF:lfisprnkod7r2ueddiduvj7pzm:rb2c64xi6s2n5uy6yvtc2xd6xl5tmprgkjykjuew5rz6yxqpkidq + format: + kind: ssk + params: + format: mdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEpAIBAAKCAQEAxOk1j3k4247ayUyyJKmm0CljRFhnKIChD7N2AGkIW6Sqqr7B + + UOboZuDbg5+iv84uhvACB2JF23mGH+Rf/OMIettMByRWvcUdx76cB5k4KTDqw6Dt + + iECh8f9Oa60oGDTsSmMTmyeDRVlsM3s3Cu4D+UDbdog+Tz9ael+y02Q2LtFVzfvA + + K2OEI8Tghu7PW8ebVSxbPAtnpxQi+yh8mfQpBdyQdlSjcvdeap9WQgtv+zdZb41t + + LU8uXUyLRuPQAShlowcfcGfxx8TX9EIuCqKapWoY20wCfscKXsZYhkMZbN6Yo7+e + + 9vfmeeh7wV/YSndl2XrnJyPxbghcVYOdusrvDQIDAQABAoIBAFlzZZjpJRqcYShb + + 9nswNG7Qtl8IV8hu8nuq9zqFfD4BZmRNZo1FcCK4GBBJlwnR9JHo+sr26iwjHvpi + + 6PX8/s+synNeHydzIa2pGcFb6cbQiX1YID+quMaxx6KjlRi2Bfde3bu4beo1jrEu + + Uplc+aIjw+6rQr8GVShNS/O6zOBj/TufeiJ71ZTjPUux+n+cbBY7wR//vj3UMO3n + + rkHVb4xj267g60deSJfNsNjD/dZSlVAcM+EatqMI9dgXUtX522HQmfrGPgEReRDP + + V0jhHL+SH8lOAQRsvXrrPehfc02t8aEhATJoxBXwPFQsgT1zgkRCctJhcnVLOzR/ + + XmAstBMCgYEA4jXYrxwoGvJI43tkTwqv/Z/lfLasezbIUrzgrc37b8c7U7mz9EKT + + iEg2c7sxYCJtCH2uKt1SkJNgeH/NZI1UqZjDFH/F6AvHKLpIg9FaLLm/85lBVsA0 + + eXXU6atLEY8tQD2Et562/NcagZgan+RGfYGArPANaYlPSJE5L2S9DlcCgYEA3tea + + ZiHg/3tQvk3mbfzTDTUwp8Y8uD0lh3ze09607nSoNdR2dnXIMH16OjfN3i1j+vnz + + WJJ59rUEf9YPnUjNGyZ3Pe6dWE/i/hjGps3IATm67WjBYnUM4pw+cy34Y+65k3bM + + g3Cj4YIgVIXUYO4daGql4GJ75kWpA72qKsmlxzsCgYEA1REE3MM3n1Hwh5v0umKF + + q+2MuXBSe+f4vb28HtkyaHGPFuiGcJ642Zey+kUqV7N1YZcHksZOe3DlX/p42qoo + + QWpq7QcAwPU/DMSRgt+RASmgfHEw0uZNRs5O0h2OoqZqZ+TJ+i4bi4GMLN64zTu1 + + jYeKTNn6uBomPGLVKyfGzxcCgYEAie2BD34guYEmNOQaoDFAoIgvmWjF5HNUa0wK + + z7Ck5IMoKklbGW9FfV3s7WPk9IO7wng6+rOO8fiQ1F82Qu/wo8FnRNoQYbzwjr3f + + Fxd/l+KXpKKWL86rLwfuT3RArfnwuylo5GIvzUCxqh87mNNJOHvqN7w9XAX52urm + + DJ3LEkkCgYB826c+IHEeXBLuPe1+TLdBJ5GTwjsuCQ2ofrXVvJG4ZLwRw6DE1i/L + + WaqO2bTHqOWYVou7lHvEtKFJ58swPS08tXMWZYtsqHRtdxQ9x4F6udELjT8VOlaX + + SPuf/Dgbo7a/QDD5+Ki2hLn7YPXxXLJcOb0n/STn9KM224JviRfGzw== + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 8388609 + seed: yPi3JHKKbWaEEG5eZOlM6BHJll0Z3UTdBzz4bPQ7wjg= + zfec: + required: 1 + segmentSize: 131072 + total: 1 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:CHK:tkjwggbz6p4wvuipe3gtmgfmsu:cnbcggp4scaxcde6vtfzga7bsuja4qjfbtv23xhaofwhbw5exjrq:1:1:56 + format: + kind: chk + params: null + sample: + length: 56 + seed: YQ== + zfec: + required: 1 + segmentSize: 131072 + total: 1 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:SSK:2ipcnc5v4xte3a6nbhfiehe5gm:nui7axbsxlgvfocho4i4nvauimfl3uowlgmerf3fifd3cqgn3aha + format: + kind: ssk + params: + format: sdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEowIBAAKCAQEA0FX28PA6wQ6SPAn0F2jEjuxV/DFMiW0AFizdltf58v+KJdbx + + QTAckR5k8ufUiaP3McGxddMeefWeQfg/WwD0XnZmKjE7pPc7fXH2vYYyjlOmARfv + + 1CufCGlpDqskmFvD+3W3mXnsbYTDzfFWHA5r2B65FiudlyItzgK/gz6mJywdQbn2 + + cQTRZq4M45Teh6iKe2hDr0+CTErCEWtB1r7izZE366shnyx4zMH7w5dYgQJdCgTI + + aWhN/hb6y/tgPNUZurYBOIihxHgLfhTJENf3jIja+W/knwUw8C3O6B9DziS2YHrT + + 3fYTF0XONBLs7y+o0QXbaRWyI8znhrl5ext0BwIDAQABAoIBAAR6pKl/cLPv3UL/ + + L8lFDlzIRfz7DlsyBbt0UXtJv2zzA4RWv68YGrUgAymZxF8FMG5YbLlMxa33kuR2 + + Mt6BAb/6Ka4kitS8IAJNbfGbLgETWVFSs2xLV8r1gTW4hjvkVS1V1ZGuJmAgZ5lI + + 5AIMaVMnLfGFFIlISdXRB08KDMZwyW2fURRYqZgKxyeU8W6EfXDnOCRAxeGi3v4y + + PFn9Yg+y7Up8sUJe2YvErAuLRKTQ0VcgvI/r/IejD57SSO7TQYNauaTx73DJcGbL + + c2u2S1Tf1wwlEldDu7JBZsmA5jFIe69ir4CQDIGxMFdCMdbflJSQXLMcMR0H8pMR + + EiV+idECgYEA61xskFPjGkuhj9m9My+FB999YcRHuXh8xuhEBCS4+OvbpfV44QJ6 + + 9gkqdK/Z8QXGKvRgZRYZdwrEJxZZdB3xao8uIpJmaLScVEC4Mg7Q+3PyFsoeccmW + + vvq3mGZ1bXbuKiyQDHPflbzbYddTBoQE/DSZvlqFzo+qo2c6rNhJmpkCgYEA4prb + + K6vN96iy4UP3GV09f8/xO8S1ovYwV6DYSdPdkb9fRQgvwTyPXlBcgzjt4Y+uoabM + + O/YAcxcCSry8T9kWIeL+VM6LiFQx++AulVfWGjnOjyJlhx8vCvFyY8Osb6c3GTeq + + TLqoX5ps9gBarnItIq7Ii56pSOuyatH6fa4kR58CgYBRqXDVpvWOQx2cfs0BvIQo + + 1id3y5WjSaXpkd8/nMo9PACrFX/KeoTVZxq+/+DbmshGUSI9EKznO+oRMdT50AXa + + ljFIt4km3Tu8k/QVEkT6aiFePOTRUEOooe8fxrUJtREvuuSEHZQ/LRblXMOm6Bme + + tFV/0YLJx9lJ9uBJ5oWrSQKBgHPrDZTgdSNsi90KPHwgI1afk+KkNNphH8ejwyC5 + + HY3yHJUeo/cwuJJhf4Gs/Js3Ofj9b1p49C/rpEOBGr+p6FV7XekaI2ygzVTwkEPb + + Q+30hkLYMKGXhSQO8RoxvaL8IgZnYFmR3pHRWE3bTogQZiBo0rQBfM2NrJ5SPdZO + + 38Y3AoGBANIWVHTeBV3/pxZvzTWj3s8xjcultFOrli8p0W5XxFU9uMgv0IK9okhn + + TYoEJMUvHsIsy4bkpUELcdAD1jmMUtMgb4T2cS5VbBqatigMjMKeqeaNB0G3dMqb + + HTg8gd98E/5UJ7gomC5sE7upU5HZa9L/TPODa2cvXZqwiTCh8XA8 + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 56 + seed: YQ== + zfec: + required: 1 + segmentSize: 131072 + total: 1 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:MDMF:6clvxlbhtt7czgom6ofessp7nm:3lg5q6y46dnw5c37kvdmtyg5bgqreaexhcaah36elawdh22tbm4q + format: + kind: ssk + params: + format: mdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEowIBAAKCAQEAwdqgG0qOUObb8fOyQ1HoCtvuKs/D4TieYsPuaCOFXuSwqWH0 + + 873/kE8jUNjdklCRxP8+V+sU50nA7pZaR6oq2/n9Gu5y0Utbg5U/OPWTp+Osk9gL + + G6oIzJmaij8Y5y1Hb2C0WTQ0bu9vMzgY2LtAcApCm5eFt9EKa7VbU0NgJBgj7V0e + + 8zW9dNtW1SLa8VtAgKE+9Ts2CrYMzhYWjfKg4MW7v7kaE9SbRWCFdmkc3lJ4bl1J + + BmK/AYkJsK8dvWZWwvT2rr+vOzAkz6ryRzqwObuebnRcfWhvyAD1PC0llPo0TVIh + + hwGS6m1KYV2xr4gx6ZmHwJ0BzwjdCsmx3x04TwIDAQABAoIBAA0AtdH1Nfpd022O + + 8itk0FxmGyGtzo6juXzLddYJx+jkP9tLGEVli0GQcibhA2x/t5j2O/LLigGBJ3R5 + + cYD/SSFFl+ey2QQj4PWnBAobvGjVZl6DIV+cj9S72t6G0/J+gLbX6DdX8pc17tU8 + + xohHEYMfLqksFTD0T+KHB1ZC6mB3ss01jr+9wEOvByW81xWs8ybA8L4EJULHl54b + + ToH9b/0k1xqzj3e8WTn0BG5quXH8VTHH9BFou5P4tPBBo23uYf7pmaD5YAeVlHaM + + LU5XKr/fUiuGpu8pUJ7VxwT7yTi3R7UsOrKoVPrAdq+FjE1HrqOOi1XMmrYUiaWR + + gAow7EkCgYEAyCtBvim/eqv0J5637JoE5VwEg5OQVYe9NbEA7WIUFu+MUD8oq3fB + + o/fq2z/F4LFuhni8ow1QNeZ4ReJnGz33VkM/p0owCVCI+7F9ySM+fHz9NcTwdBmC + + CpWrfhSsfTNcNkQfLQLUHEwI72bWpfQC+1BgvzI2fEZfeqiXt3A4ZBcCgYEA9+x1 + + Zt9pQCgDwlGkz4EVs2TObM/Z0ouNjVcXF+qMj6aHbKKoW3S/DkzGlbfZPtMS/Uqa + + U60HFkyCQTgmZwzLj5FWJ+GSCpj28/PamiQOfqBoZ0AH7OuTVw3R6C5hQyq5RklP + + GjOfqwDGBtoEp2PdRV1rj1ez7awo1pZmsW3OmIkCgYAFrxzFzp+uVxWuzlYAtPrw + + nGVQay9NDna0AJu7Ie7aG+FLIhAAlnz8L/0OTshKsh8mWGVa5/TgIvRFX8F3x5Gv + + dGdpU7T7frr1Erw0qviKRm5WSYpecZ78t/VPtjyTrZKvw81y1MK7LvmN+sibm8s9 + + 4bFtnHppmwH5FLKCNgCT7wKBgQCaFmCxW1FzCmurrkqcnUH7iT+y6UwcS5firKox + + txk9fubUYhP5I4pLPPR/wRBIt68pteBM+VFaTpr2JgvYKF+sD0xY5R17cK6r2HeZ + + LafEk7XP1kAWxCODC5fWklzo/fjA8nczdbpa8dQiFgamcq7nmbRsFrpBkaqgFEIn + + LHQm4QKBgCEUp089t+7uWuya+DBmdM3z7THKgsE7FrM6Xn/NTHDH/3FEmEzRoiY1 + + C7UV/3xKqpSLwDORmJ/0N1pxCyFqN6lW3zMsWM11eBaLxbWsoXm8WKEsvHOTXg+F + + hWkdjQMGG/nLYpdPycaLs/qDqMybfLJXG1ppCa/aLJMMS10r+oIA + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 56 + seed: YQ== + zfec: + required: 1 + segmentSize: 131072 + total: 1 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:CHK:nnv4vrtlxmzkurfzvonj22leua:ywcyijrfnwykraku56dq7v3o4ts3xsxqfgmk3kgzwhq2cpest4pa:1:1:1024 + format: + kind: chk + params: null + sample: + length: 1024 + seed: YQ== + zfec: + required: 1 + segmentSize: 131072 + total: 1 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:SSK:5jgelx7ioj3fl52fobc67zccg4:e5adtxw5e33jwoqi4xqj4xaysygxsb7hgcls5tjfx7grnsgxxv6a + format: + kind: ssk + params: + format: sdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEowIBAAKCAQEAizP4H3C26DWZsfCkJLixuU8B6rZLjWAGwoqGdvJTZu3sj2Df + + gqStygSVKrJnHcwRK9rOmo+unH4X4WgMb0klH1whL4MzvDOwhX2TuGiC6/W5PY4O + + WjQNv+2zTMRnwYxKgdMOvVfA+7bNoX9TnNjjNOyB+wkvZwxx/Ck3RCJlty8XJk+f + + uLFDpRa4Cu/4adal3yEdxh9OGhYjlPwgS/HI3Nl5IN0S03XZxNAtwI9s2yYHPXB3 + + bKqZHXn1Om9BosgXR01lZbVPUlQbDkjnZTNIIOF8SHQSNTm+Bo7qVjNrhClLMIg/ + + aWM/BBnqfgwR45Ip9nueoRM1cmWG51NEotO+lQIDAQABAoIBACqkH0gmQ2lLbgrT + + f7yd9RciTCCFegxTE48JVxpdrc20aUgccSs4XeIp2DXNk4fNqJ7p9mrjQ6Y9e/w4 + + 3sJCQkRieOnwg2sN3G9v4c3V+fDlAzsHZn8cPfAClO+ZpHzmCDbPm87FcGDLBR+I + + /OhpieP+5OwsyqAC8HHBgGP3M3hS2VlxE6AW6F9axpAr37v/ruJghlYz0TRCGGLq + + 5GtkvvZ/ZF6p7NSX7fAfdx6zSezsKQWH0ikB3QVFt2audF3CzRJiBJ4d6U9K4Dud + + 9Jq5lBCosN+EmPgWfEQzBXo49elyAOSRmYxzfSWRqdCs8XNlG4qTtj8oYrbHTDeh + + AqMMlpsCgYEAvWyzp1W8aSgNNs5lU9WLPr/K3PcUnFeg5Vm+fch8UG5ZcZCpTcKH + + xmzADJi2eLehJtovBhChN2R7THgpqOQgZZFlQcsbcYcNYXprGdFW1X8zhofuWXzs + + vD0UkG7txNrxZINIQFzLwbrb3flQmLpsrtpphniXIGuQGuOMxqIIAy8CgYEAvCCg + + STFm/U93agdd+w7CErkzuYpdWmbqOG9+jN0Tq/8/k7L8TJyfkX26E6zBQlSZkS73 + + +HR2aOgVmPxynDD+E9IwdHLxP9wRbz9BYsjptL2Vbm02/eI3fXVmt6osHRudzAFc + + ggkhD8W6fUqoNB/Mbdhplk6+tlXtTMLIWB3XeXsCgYBpZMHQqPNbzt0LUWsvafE/ + + yJamuxLMqjTrZzOF6LbCSaOafFK24TWKQZfZal6cbA9N/reLOFV67H1t3q3POp6L + + 5IniQY/TasEXK3XLt54Iy+1vPNJxGADf+1wlwJKqpOcKdcENjpQQBleu+bjOQWuX + + Hg74sr/jWfWkAFejbSPoIQKBgDRksb8wrwolM5Cn9JiTB6HHSoyF6HHg76JACvKY + + L35bXA16b6G2jQosBcKs/jXG8e3pMs5TQRb+a+VriU/OpTRH+Y605FNwqrpc14z3 + + f38Cvbc/W21hryqVo8HK9vY0VsIWLvlYKYkG/GUgga/im0CMYPunep21WJ1kMf+4 + + b+Y5AoGBAJWjZ3WiFf6b6AWlY6Yg40Pya53TklFWehWXWvZj6JExxZ4wDDJxFoRY + + U5DF6w4JrTHy5rSeyK4qRjTLmNgU0uyVzsVskXoPJ8ed2iXA1FB2P0co80vPncQd + + XMlWTDK1zVRoimOiVhbX1wPXkKgRTCTxL/SBhz2TZB9OYPBZmIjA + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 1024 + seed: YQ== + zfec: + required: 1 + segmentSize: 131072 + total: 1 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:MDMF:cxqvidbgpxb5bl7kcwcl4bjld4:drfez6t6wd5dwbg5le2fp4ksugnjex3nksqksss5t3pb6pngmula + format: + kind: ssk + params: + format: mdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEpAIBAAKCAQEAnoqIdreTcHst+7JUmKdbjqtjcclyrOWbcPrSiUb6gLRQxua7 + + 55auJN9/DObSrvlgUFRUFIOLpjwPFs1X+hkrtUg4UZQ5j74bq3R2bjCUxbxgVVgY + + 9wd+KixeTvnVrIwVAhKEgCKLMcj7yBW8RYVAU52l+yIu7Lk8CyrNAM2/F0GGo3NS + + UaQVyCE7aMgZigkyW9FgRdHdSdfvwOO2HFJfzSZXWR7QIAysaFkNsD2+SoJnrHb3 + + 0be+8TDU8+Vh6r4u4D7kugvKUhwR7crj+yuGaWLBc0XXOLUXGdA304/l+8dO5Bxv + + fmAS1F6SqgsYPnHncvIML78GoaqUTBoKXbG0KQIDAQABAoIBAAWN3k7vapCdcMyB + + WxNFLgZVujGw2viUQNb1+I/TNGCI1flz46fEPf9Vc5y4I5p6QCiBncz2rfr8c2hg + + eFlZuKU8vZlbt00HfgNqkfbACF7pillEZqJ/3mixPiVnkq0SnsP1nLNQUn6jjIY+ + + KbF9+WAR37AzFUBZPcbj4IOg0aB0RB1BDFLAhvRchHVStC1v1deC8XWWMNmxYVye + + /zT6CDRotz7Ygp6lxXZYep029wS0vIZlDBTURGuLqDPXeDe2iTn4CDT04S0PVRzQ + + 0jA+kZKSejHTL0lFXg3c8sqTOH4uJrb7d8q/+H8nNX2WtM56O5gsbYWTRCBKVoYG + + FhXwcpECgYEA15kr2gAx4YpR35sokgC2I/lPe/WP6SVmaKU2FYTFTFaL6dAZtodz + + ebXJQNZr2dfn8FXtviaNWulaqqNNVTgEz8lTtwV576BFOptVGwM16t5eaMIfw146 + + fK0sxGpUqMRSdMr81pzTwwHcdDs2j7B5fE2HfvOh4KVRX8Ck4tdcH7ECgYEAvEAt + + FQ8IJ09/9m9KL2mLoU2x2qpXqCPXdyd1D2Jokm1HUdgCCG5BIOGvue4SVQZ654ek + + 1DLU9CgDr3z03fH6FLRQdir5MOLx4kih2uTt4iTo4Nsa2Z9pwO+axORUvewW5GNC + + kYL9Yy1Q9rg2bav7mPqDlcAUWsUqGmsCEurmMfkCgYAgtNXYLmtiwa8F8u3GqGD5 + + OBr8vRXl0oykl1uLDCc6G28CO1WLQSUdc5xiP6UA2SYQaZi1XffXsMrWVAupP+RK + + +Um/3A7RcUjPST0x6dzGEpHT5o8W/jZ1L3g5G8BYEeBIY3rTu9rMHH4rC8iNJ8Jm + + PwStF5yZDbs4gWsCFpWdIQKBgQCEmxxmqikPL+Qu3uQ+E7YlEQrIwpduvJipuaSv + + Cp4pD0te7q836xp7pB7Z9Ub6l875y0YjqA70Uj+OXZJLyYllDkNjig/xDNxgjtNc + + 00hytZdJ1W3LgIzJOL8oFMNQ6b6ScQ1SXRhKxYAz2z2T8cMQVt9cHGr6Kcrnwxs5 + + 4jf3WQKBgQCpJhKfp0xkfSXRxewWn0mKl8JiYTCQaE2hlDJXnJMkTGNGSB+OAr/R + + Q4/O5dsO1Vr0jSx3w1oW/7EofSQJvuByVBpJm44p574B9Q4CD+nH/KKEoX+ZJSUC + + A3qs9c2jolDrPpS8IjJERrmZx5zTfucCVNOR9HGhUJ8iCKvEdEp7Rw== + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 1024 + seed: YQ== + zfec: + required: 1 + segmentSize: 131072 + total: 1 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:CHK:wlszole5m2emf6wbhp3lnlyfxq:irdttvny74gxdrcotzfjslfq7p24kyiigyrm5shhw7zyj3hsiuba:1:1:4096 + format: + kind: chk + params: null + sample: + length: 4096 + seed: Yw== + zfec: + required: 1 + segmentSize: 131072 + total: 1 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:SSK:ui5nb2ljhekqtnzsjjxd6kvjha:jhqo47cpqszio322x7ya7a32ecn7y3einrce6mszdfnopyjlfita + format: + kind: ssk + params: + format: sdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEowIBAAKCAQEAoMgeFJ8lHJqb1Ygnb/QcCmHbv6DQGIzEMVEwigLJfKzCF2t3 + + UgYj5hkxcc+OlUUjfOEH5DyoN1mTrieK7Cn6+XULkdhyc6zD0uMEN9oc2vcEEVnG + + fQl46IhMyF+7/aJZ11cWH/bCSnSm57jF2MG/bq+sa6LjK2z8ydthTWi1PC1fFC+p + + egDOEzWtr/3P2sM9E7UUVRHkeVrDqcOMgxJq9gYtGI2GCYnx29rCdh390GAiU2wr + + R3YIKqRmJfRb3MrkMYyjsP3tsS2Z6B2+gGAdr9X5ydpV0ml7mKrPEvtEE2OeGFiv + + oYh3Txl5Hochh0LOyoHfTbkmzAHz2T0LHBNDRQIDAQABAoIBAB3WveRoW7AZDno8 + + 1Erd9DVGD40bGHux5jhb38UBOukRO80yY9jcbF2oB8neMhFIXUtwDPGiAzsQfAyq + + aIknSl1xCDZnQ+htZANXn+EIsOm/Rak9rs1mTGLlZtCaGc66ym8hSakhd9HvH8mp + + /Efbvz4YyshIGN5mkeyZcw+1csspugnDc7CIwvroBrqdJHs82hXM6yscWsLsthdA + + HvuY2WemSxDiCaow1OP4pMWcrnC16G0chhKqAefTufiEdcJchM34dFjlEOW3qsFq + + ghnD9P2uG5pI/vYP9cLhLEqTNTTMTq8OK7yjl3GufoDleADxzCIPvkaAEnSstKEu + + Fp5TUZECgYEA1tXAQM9iq/ZQOwy3etyzRRP12aGcEPpsBCzT+aucU/sRPntU/iEa + + JwZ+mwty5LzHdHU8ssYYA4CtvwTgGxpRFxdSVjqwMRXXMUSZA1bt1oUD5Q6yM72R + + ysf0NeMQgbDbbVQhdF7cKawOMRzwzzUkJAcXECTKwYs+rvCGCRWuj5UCgYEAv5bn + + pVOkx2CSACvXXvIBZZY5MRzzoyHpgGHOUX0Z3lR5dfLAfm59m8j6VDDNsGRYMN07 + + 7AMZcVp1J6HBuUSLuE6wes1q2FYu8yctijmgMEOpCXT5z+YWG0oQ8AEiBRSIlz2s + + 8NmFgUmyeJ0BWMog4By1zQtyRZoxGIgx/VafuPECgYAyvBYD+DXwMGIwH8ew3zAC + + 7zzPIYhOxiT+M2v3+VwYxSEEZXHj9gNMFg+OI/0FIcPkr88e1QNUyG2/v7IBFIzz + + 7BEIxiFX5jWEsBOGo1/VmmIaFQdmiq1Ee0Yj97StPAwF3Klt5v0NZlGPrar89CrN + + y1LaACZV4MFz5N9yg8lOpQKBgBbYoODnG4Qm8OISWElbJG1/v2wq3qa6WYTUpOy0 + + tUv82MsG2ot5E4NrMOavNyfsn1OcXhPjvrn0pnnGYTp9gQfGYmcSbcZEaK7YIicU + + fhSjTNny2ANBlatFZsWn7O2cKDmYwjGqTrA/IIgfeNSkrczrv4Ym8kZ4f5hETWm/ + + VaaRAoGBAMs7UVWOCybQ5mQVzcu99wWBNfi/pIRt9ypnGbLwgvH+wrAsf6w8N9tX + + iDFg5j1nHfBSr7bc8QWDvzp+rqNJcwoduDpDQAtFfYZbBnzKsp1nTEExV/XZP05Z + + wMhBGlQ3wpWMCYkZi4TaP/qGvIh+3yt9xiCdXa6Wt9uzU15FiaDp + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 4096 + seed: Yw== + zfec: + required: 1 + segmentSize: 131072 + total: 1 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:MDMF:miepn2lr6abe477ynqttkf6rqi:kd52ue5gkfi7chsmtk3tciijbib4fmgimneyybfazljrtkczbawq + format: + kind: ssk + params: + format: mdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEowIBAAKCAQEAsAxxyLdBwIlwo8uyK2iZ48xOXOezQkGOWqzvVh4YJrtxEhJp + + naYXF1mzk3SJgTMXC3NsWnA6tAFRGBrM9vbdYA3MmX/iq1eDBafkswPv9Ggsgijw + + QLTRP0sPD7YSmLsO+ZpmuB5wFq8ckOg4GC7Q3uRBKpqQ0/kSDW1hoghN9XTjaJzs + + t/SiLlYND/5hRIElLS5Ruq+GFmpz41TyRQouP2A59Rz9ILTolA1jPyfEv6AAtPLY + + C1taYo2oVS/G94LKMqYCNjWWYt9T4lDBci9OFj+whClceDapFvAym4yJeU1WclIq + + rYHwQYGMH0u5RDJlu0jtdXJWqDQZEhZLIs+KvQIDAQABAoIBACVsO8rGM961CKH0 + + 9LaYCXB8V2MV6MvqihN54/WLN6iSG07TZaqaqhlvWsY7VViGzv0C5/NQnJXzmrS0 + + S8Iqx3O58zZlEj7If0RWRH4OVfV/KIjxoWKr3TgmYTj+g/T9/IiwGupEJCEaT8j4 + + 6CWx2/opjLW8/hDlRwJeME0klUfaGwz7CkveSUP4p7lDZ/MoOm3J+7iC6wIJLFAJ + + 3D3GlUYtUw+5+Gx3zDFXT33SlaTtIUubAzloVUvAG04o3bYwTCNpXFZPv6JAH6gp + + 97NjrvCPsPx0d2xZA0ObTXXc2MqNkWo9BFOzHecdKHOIL7Y1G2UCH5+y+dtEsm3d + + mu+87FECgYEAzuI7Ur664wi2tx3I3i7vyhdOJrzswKiIC24yuPW0eS/XsiRcWyK2 + + rt85bBRcISV7gEt4g/tixdK+EHJSfpoE/IQM6yqTyedUXmF7disfH2dDaQTeNDh/ + + WlypZlYYtiPtUJs+t5hulWteH1oLiUeJhUwlG9+aQzuBec5qyPn7rDECgYEA2dgl + + 4jq+VyqfQ2NOWQsi3A+1IZsX2yoL1uN4CypWibF4N61N/wkL9oyEA8n8JclPAXZG + + j9TmhIJCq/PSfmc9XLZGSudf7AZh1wRQwDOqONA5o+vG4n9Hlr//2zjA7HEFC0dE + + Wx7mCbBJknl3PMFuxk+DusQB1+jamGsLXZp4wE0CgYEAwCA8s2VJLZpkBL52UlAI + + hAcMntEIlQpt/R+Dn10fEwQpLdiypDgiq1fGfeaSgH3MqaJs8zS7z7ccpy1kCwqB + + 4vfG/4X05aYdJeElxOHa71D4u0i4CosFSiePceg23r+Sni7uGZZH7B9fs4HuALkc + + r1u9gpsvKYzTewkFBkuRO6ECgYBO705S2hxMM2qAHYSvKSTZfmuQoMUVKfgeRlAi + + I5Y10HOSIR7o8Zs/HA1d3huaiYYyLmxFA8z/aL/F1NSJ7tjCNl3kGFCeknVzVuH3 + + swDUE0c/iViIi7wh+LI5+ieVxSIhwxIWvmx2SEVwaMj239RG0VsXGpzcYkiLAAaf + + RTDJ5QKBgHNl9ki2heneOnzApk94KGozB5S/GTnQUUzDSyMp3UGyrrLkwLuBMEd2 + + 26yItmtWuWB+Pjd8n4+NUYB3+s5zeK3QPO9vXe3FX+L4J0UYCW5Lr2Q0FvK3svdO + + 24eoBuTkWBv08xz4K34aB6d1jVsz4AMW3vwQa57wky6lbhE57UBJ + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 4096 + seed: Yw== + zfec: + required: 1 + segmentSize: 131072 + total: 1 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:CHK:4ewm23jvdtm2i5xf4gck26wg5y:7cujxxc34mkfkmwhbemqtuixuektknmhhxlmujcekibf5amfwwga:1:1:131071 + format: + kind: chk + params: null + sample: + length: 131071 + seed: LCa0a2j/xo/5m0U8HTBBNBNCLXBkg7+g+YpeiGJm564= + zfec: + required: 1 + segmentSize: 131072 + total: 1 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:SSK:hwubqcv3edgfopyatdczdw3sai:dinqtiuqgfnr2sax42d273glnd7szfjomomd4rxk344uwvhwaira + format: + kind: ssk + params: + format: sdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEpAIBAAKCAQEAv6WWmbzM1iOYxPnAwlmMvTCfP3DkF09Cppb/5h/+RUWw9LrG + + b6/iDSyJZWwyJyzgic8WS93xQLxl9HLrkmnMzTOaqA1yyeMLgt4tAR4UknHOsDqu + + /006hbW7D7SUrTGrqV3mtFdHvTjrI51TVpeJfeql1A1SVde7OV3ZmqTnHEFyY1fU + + KywYJ8f8Gr3xl5qJKBWaZI2V4LySKz4iAO8hBwHiuY+gdQ+nAR4gke/YrJOIZHZo + + rq0vYSvD0QMNNLWZoOLJs0DT/4gU1rSomjAv1297CmYkHtjEZwGW4m7rgo4JahIc + + fppqNvlPlUvnThmSJun+3rZBY7c17dD/qyfVrwIDAQABAoIBABvmGqzpv7YCu5gd + + NZL1X1ghTmV5ZTMBflXrEHirOqRR92dBE2cp5xH85EmH/SsXzN4y7+9+cUL3yi3S + + Vvna/g33T7HcN1Qtgbz84/dQLjV9bNXZzSTsVLMnWAJ6ytQFsZQ3z8B8HjztHsnx + + +rJV4BWdBaP/hndprt80ituI2v4RRnk2hJ7RCwDp8XZzf+98MAHeB19PsmKftPxQ + + yaXnv2GuYxayWYWt0wlBwD3PsvJaJ/OUHbO6/sRTN/3IaPt6Go+D1Gsyirv4vyrO + + MUx4ygsfxqWg1trgyjMknZe6o4r3PPZsDC8R9fB3xc7FROA8rCP66Ok6Re7cnPyi + + lAZbWSkCgYEA4iQavryiR7yEgTP7kbpVDNp0TgPyqLJxajhXEz8TkOM9Z/+0tmh9 + + 1YqemuVsyNfccem218Gz+OX20asedXYb7TArB4InLn78vKGW59vxK137vuhj2JWR + + FCj04DiQMPzKWmxPt1/ga8p3rZPcEZ76ZMGHXSG03T4kplryrhIVHnkCgYEA2POH + + k5dp+0ybvK9uQZIAau1rYmzJFlxgcfCud8a+e1iYK8EaCG/J5LMvxkynGuJ9vNMA + + CckXroX+etk72LJgJIcxgGWIajNtRkSrcQ7nWo6TEYt2glY2htPWFrPz+RgeCDhS + + ceqLwqfC5YsFWBeYPNImwcXvlU0YoseKMW1Ma2cCgYB9Q16FNNv3PJdxMigxirM9 + + 0WwHIuyxQVbNbbPd91yRLy5+gwfI2oyJUqWUS208u0Vi3ADp9mQIhOl5Ln5Ktke1 + + 1K6hFBk8Ch9ZJXD/sbcfPIoML5HPENox/pXV9b75Q62a9NAbVUJsstQkE/kc0aEF + + WqXukpMq0hdfBpXSkjWckQKBgQCZo/2DnFtFyH8SJPrkHM2G7BR8Y6YU297BUj18 + + PZdwKtG5Sstw5hoIiI1w1aAR/gwlyRfh1jObOPF7dpRXZhuIQuXflAgDjd/5P3Ba + + ZL+a9hVY+3c13nBHE4YuFcrVwSqjj59zZTMM61muzcE/HZaGnB0uZUrCZRLpVH6d + + elYASQKBgQDVbzggm0ylJamRWy+apZ48Z5OBpAbMS/YsEptxboWnku2Iu9r+4lr0 + + QZX+sXaoaDUc8kHXlQDcz2UYj/5AANeoLWlu6PoqoRseSMWaDB7RQkF5nIa+TZGE + + TgdNQuxRa1w5RLwu/brRqnQ0SGdDqa6ulIJldkyy0aTq9XeOjlxhqA== + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 131071 + seed: LCa0a2j/xo/5m0U8HTBBNBNCLXBkg7+g+YpeiGJm564= + zfec: + required: 1 + segmentSize: 131072 + total: 1 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:MDMF:lmh64vic5llzathy7v55tgjjge:ai4qfl654ats5tpjdmy24jfsfxirbhkt4sgb57gcomkfrqq2cfjq + format: + kind: ssk + params: + format: mdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEowIBAAKCAQEAvliwv3nQaoYx3rPj5XY8bFgTtUOTyUaH07dYtIP0gCZejJJp + + /ZVO5JdeOOIzXuBkCB/1uN2aFbDdoGNbQbEsd22DjCDwMxan0MuRHVOeddHY65rL + + RwjFX2UKyCRtQliAlEzgniEYXOc1TJmjtAK73ftbO5+MeUKwFY4Ba5CbGx1XX913 + + 6LynlAM234/g0v+B/aBaRhzKVn8mS3POYKi5pcAxkpg8Ywj6HwND6QuZvHmji0cY + + tbE5UUHi3NuDwJG9aqxt/kxwyhPwMGG+Q5WJIdmKP4dGMjLDlIw4btSL4uPe6dq4 + + KK6kJ4OCgcm0VJqpSFOiOK05fPaZ7sP7JkrN1wIDAQABAoIBABgsZU3752cP4dd5 + + mxCyIlxUFzSm/2bJaUiO+Vn7hBqeRNWvZnyI8LsBKjspJwL+llWd0XQH2KC2lH7g + + /17pZE9KfjFWoYqrbuaKY8SIsRAfdV/+iaBc0cwapfLjBWkumi27Ua9jXpe12UQA + + IxUiX7+CQ4Tf71QbDwe9wBpsA/a+XBvKB+BlLWXRSQQAIBmUQpmxzPBWLQKryHcT + + xlEdyI4PQzLMOaDfQ40YRAPrDnYoUrK3L3HtXbKKp0MeF7fx9aJWzHD20Ssenj3j + + eOFAOkDnG3wqvnqi8lg+O63YQBCOF4AUXAMWJYEmwp+e8LjTc3SLZGLM9L1Mi4iC + + ZaJyHBECgYEA4fM3i9jvhn7YsV00goS9qj99lPzthtp+Ty+jh2rCEaZt1IaZCeQi + + l//MXE0lqLf+UyA6ELZVnGJxlq2jJn08LTwpRHXWSBGrsO2Qkq9HyrDstMyu3Fji + + jwJFKfHpMrTG2b4kF+STa7ySCOOFIVNsAjOcnJdlWnzwD6tyZBbveH8CgYEA16lM + + hLtmOAdF7qG50tOdCPzmnncCvvPIJhmPlKix6ogzeW9NR3BiqfZAETgV+VisYLgC + + 6CjLfe5266RGVWNFZbns+hx+FzCtmHmLavr4lwFS0NtyXm0IE/4g6zybMgfwDNMC + + 32YW1aTqCdy4ZhrcJpLT97DNl0OnaQrhMgY6vqkCgYEAhfg5tReZXbuULAXBfqnJ + + 80nV4iLdixm9zqHGaiJokyKE+IAd+Xlk8Y7f0tKDQ7hkeVEgXIxf0mukQd0OYWHb + + 7k4/gbIErZKcpDkXgYGgJZQlpUW/YDLrkjOcYrRmuoPpa22L5QbIShby14Zfh1T5 + + M4z6jPZPSAnQJNpY5vOaZW0CgYBqitgjptU8DtPMrZc5AZROEWr5lIAFyDf0IqKd + + Za3n2PvdHVCHX41OvDowh43LjrQyYBYHjcfiYgHcLl8U5iMtu2nIsnTUjhblAf8P + + jgdrypqYViGtZp4cCmtG670cPXGpVEHSDgRv7bY1wxZSUyi54cXYUz9uYFz/dwGE + + DjHNaQKBgEhlpj4spuSaUC04wjMxgzsJOqnZvBm59INCnEZKXknqlVHmOpHB89C5 + + YWUE+uDTOszzR/UxblvGZuT/4b9L3bm2Ie0M44WUd87dkEhFLFMsfuWxm5//TIh3 + + qL+sDc4sZr0nhD3nwomLvuBSj0sxqUzm9L2uV2hOMzM7h/3RPK/y + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 131071 + seed: LCa0a2j/xo/5m0U8HTBBNBNCLXBkg7+g+YpeiGJm564= + zfec: + required: 1 + segmentSize: 131072 + total: 1 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:CHK:heczpdxphw5frp3ri5sh2hpqei:6wflx6lphy5mhtpfb2abznoa3yk27ynbqbzcecs362miu72vkowq:1:1:131073 + format: + kind: chk + params: null + sample: + length: 131073 + seed: /N4rLtula/QIYB+3If6bXDONEO5CnqBPrlURto+/j7k= + zfec: + required: 1 + segmentSize: 131072 + total: 1 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:SSK:seuvwjkc4tkea52zq6j7qr2uli:dv7fdqnxuilecqhi5gpyabpg7ijkmmm7e2sl45p3er3d6sd7ytma + format: + kind: ssk + params: + format: sdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEogIBAAKCAQEAzP0CgFCh4OB8OgcEgvXxu4ijaX/cSKfyZYG/tnJR69EPKyH2 + + CuE/JEcWvCAQvnUO5P530EmY/RRyBOR7gkVfArkx5nco62Z270p4eA2FrLWGu2iR + + CJOrcol5CCesormsrqKV3t6ka0ofiQq9AydK346FdUJLDSUyFOqUVQPbWjgWJ6Po + + D8jv21Kwup5qHjhx495BfuJx/foBAd8LmaRF53lGSdsTUS/y+yL3OOvqaV/gAmba + + hAoAZpTbWiYIn8BvGdlT9TIJZqewT6ZHOAjnA+j9T1BRGztZfP+T+whdSkFWzGe3 + + 7xziwijyuwgpYtsUogOlIc/2/fztqs224r64zQIDAQABAoIBADWgAJ4Buf9iqozh + + lhgOcAEPwzQPq1hkeyB722PGr1Ch/bZaaYu6FjMO1886EjdI1y8ntL9L6ZZXWWaX + + QQo4zJyhRwET7iP6x6Vc1XwOiYg/arIvLjXQr7rEZOGxw1NEgHyk8tD9bITWvL40 + + jXK8PjWSiq48u/aB4wKexVQiMKl+Z7tG8zPUq2n6a0GsBy3ZPnallwAnIxGFZEL8 + + 4+5VG+BSGTWJsd+5ahGsA41seb8+sMUZtKpbQsNXBVqVM0OgceZPUs3M82D9ImcN + + sricXvLSh0z9yesPzWVwWozyy25TxROE1+Fy0P2a7qmoN2YX7l3INX8FalutAbcY + + 7epzdR8CgYEA7US5BLeC0oZVXurIVrjIVO4+qvpl4TeIbPSs0tuIWNqy5fHoCyDJ + + WRIRr1eGwBKBheS3lJOVnBfPgFVef/r4TVIZmj+iXL1g/9zMpP9nll2tw7F2gxGA + + ZwZEx7g8CV9dqCgYBmjZZVKwLwDDtkU118QAzys2cspHJ/rtc3Lb75cCgYEA3Svk + + /YI/1uscIrFg3NCECPtyL3Bg9AC1YyipQKVOfWAQwG+O4Kh2wrfQOgJ4C1KN3mxW + + vOxXjnlQrL1KyZeyw5YFuXftDk8vLjtvs+GLwKK+vlDtgkLGOKmbPU5MS8cf3d38 + + qoZGjzSgUpg/MVvJgB/zAPqJGcBH70z48C21pzsCgYABlQS60FJx/u1QzbX6Rg8n + + 6dLHJxZI0yr4twTz/vzAwuyQdfV7JYPSMTmm9qlyXG06rFTBC97ihJIgo/EWX2EK + + evKqwaPehHDCJAHFU+Kn8QX4mRVWOGanyTXqMwNLeLRSK7pFSKuybkO4fIPRklKS + + lr7+oqYhS9H/pT+yFmD7DwKBgHMCI0ZMF6RLh8rmj+bjKvV8w0i12ESppajVeQWb + + sC/z52IZ4KMkFvV0Hfw8Um4Y1JrnnUcKYxE8Nl5M5Hnlv1iDR6DFIukA9hjFYXWZ + + gFGAj01pycelr2vBjm8XqwbwmbqGd5+4yTIofIHWl220PBi7BGLq5KYWXZGrZfuG + + 2WIHAoGAITXgWLFrWR2e7bktlbW3HJRLEIPClA2GTrc9PLmBpgGXRw9Gb/UKXYsa + + 0/YUxFVbKAA2B19zUq82grmS0270oiZh3H+S2BMCNUAWsWau6m0CTuN5VBE/PHVQ + + WZiLCks4FQkFwChLM1/9rueao/AivakRd5US01AbLEi2PUBkhdE= + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 131073 + seed: /N4rLtula/QIYB+3If6bXDONEO5CnqBPrlURto+/j7k= + zfec: + required: 1 + segmentSize: 131072 + total: 1 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:MDMF:pxwqxxop3zz7inebttwm64lqwa:2ntxvchwrq7mgea5te4l57gv5jbtmtx7wjsjps52y6cgrajswstq + format: + kind: ssk + params: + format: mdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEowIBAAKCAQEAwjdBh78eL6MVW+dm9E0gy2M+TNjXh4AVX+5UJv+rOpom33vS + + ugdkU2mr2jYHhGlHiU97iVi9BHB2mtZBpieQVAG86sJiksN7Y3s+SPe6hvebXtdy + + c0+ESruPwmkYq1nrN2PRPtxJRrEGv+s3/P13Iwl1qvgOOB6BF9QFE3eGPfPfbezg + + O72rhNSSXuJt5EuX7/jeoBev/S5+ola8kKxI+SK9Cx0jEJPxmNf6lQXd/fihk7rs + + GbHpQQbm6Y23GDicQPqhrhDpWA7XyBluV/VSAYY22f2dVLH8XYt499fBwmWGBNFj + + hjIZO0PmDJ4HQ7Et0HOXGbeYv2mAgcCrGVJ9VQIDAQABAoIBAAKb/HDm8/Be6AwO + + jVcN7DlfUXh111t2MJNT3+SQPcwxQwFwp/Gg5MusGUd6v1obkf75xuae/xcerbFB + + 3KrvUCSYy2F6EBn5r2A0SS9wyJxEml1JVrvO3y+j2ngZsl+m+x6I5EhMbF2bRkRw + + 1BU9kIqzd1W/NG2zlzdrPVA4JGETrjdNrZbegc5EFaWFeg2VveOdKwrtGSupuNVf + + yiggTXxsTXcfcy9dqdxe/vTNZw5hQFnZrjycwQ9J/dUSotlVo5W85SqpsRVK4m26 + + +9mA2yvpDYxs0+fb8C80KAq/BLP2Y+sAnU7f0cKSsLGioN+Q6g/71wf9zrCrFPix + + rrDSe+ECgYEA1obViMRRvlDpFuBLPpoMr00tvXmTJmwCvqCksoL1nRFIEuNsWQSw + + 63gETyZuRFz+xXacUq4eOSoLcrrW2CPeMOFD7rlvSKbG1MpU+OI62UZ8A1YJC0ap + + YegJjc/1rn3r2HEjSphZWt3SWXz0gbVM8yVvOt8c2xyCcxxMRRlfrHECgYEA58M3 + + EF2pqKc7USkTdMKHKogHRJgDINLDz2NZf68WlZkfv7EXekpw107zjmqQFPesdG8Z + + oZ8KjVVU97my9/eODMXFQ5V0Mztk4Fv+cGAnTenuAtJ9/weJiR5Uou8rpfQOuBMp + + bYnwZnI/BwPdtS6B/jkS5KMHR+Ri3lxlmp+EISUCgYA48Yl0yEe6cNeuTtMqRtHf + + JmlhxgedR0ZjO1j8WW7AxnmPKfb0mh4sIqtiJx1V4ClwWM+d0sILAnIPfjDRJpQv + + /Vt+3pH/guV8TkjH16UvT1pTuF6mM5d6eZEvp2fbbWlRBpcLke0GBaN0RYrRc0J9 + + uA4SXm7Wanbl/zjvjpCqwQKBgQDjSueozELEXWXmHcOwActv4cJG+lIvEaTpskSm + + 3X7nrimd5L7itzjdX9eq90Vg2tmtwvu/Luu5WlOfM+aaG5WbXyYsNtmkGP7Arlfl + + u9cwKVi8OdVJlQnEiRN2S9thwO3ihyBdBifXQPohFiCMPRVNzomB44UTc5+m9bTL + + pN9/ZQKBgDUfZQquX2TLgnTTRm79FSh72Ji9gV8wKXyAcvzI/N8nhgE7xPMm/ih/ + + pPMAjyEV7vc7ISOBGZ7Jg4/p5p11jcWmLt7CSQY9LOLCMIedFl0+Ole93LivyeFn + + sy74rH+TP8PuEjQuzAK+unwzY5udBT/lTdYXbKXzPR3ceoarKBn0 + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 131073 + seed: /N4rLtula/QIYB+3If6bXDONEO5CnqBPrlURto+/j7k= + zfec: + required: 1 + segmentSize: 131072 + total: 1 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:CHK:mz5siv27pqttsl2f4vcqmxju64:rvxhulxtufho6pdbwj7rifneb6pei5fl4fqptcce7jdkbyrjfaya:1:1:2097151 + format: + kind: chk + params: null + sample: + length: 2097151 + seed: uqWglk0zIPvAxqkiFARTyFE+okq4/QV3A0gEqWckgJY= + zfec: + required: 1 + segmentSize: 131072 + total: 1 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:SSK:qje4blwekvqst5si67jnjomuhe:yjjgmegtlwedgwnrsfuu76yesdqpdz645u7gbtnfywsiybvkul4q + format: + kind: ssk + params: + format: sdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEogIBAAKCAQEAsUqS5wnsfJt9NEGBruv1IF4ekvr0XtyoP4ckfT/tkSnU7txD + + rLJzVKbBENAeKXcLkoStGXyc3WtNcUHcA3kJi9H9pn7UBqaZ05q16mMh4Z/s/l5Z + + xir5F0XIGBgjOf1H0BDAqlzQhkvyd3TlJ0+mhHaP2KoYZ74TsZ97gMrEn2Q9ozyG + + nijnYKlOLpvnBwW2fg9kpZi+3MuLOFxaXdTr+mH372DaVghBnU15jr2DE+WI7VrS + + k0n7+4nps+cWhgP2njydCCqsL+0sBsI4DKHZbY6fhMkqXSQyyTjs276BpHgpgLwF + + omAIiS2zE0AzS4Wof9xGCBjsRO3sI8uECOKa5wIDAQABAoIBADEIK9qj3viTVCw4 + + lbIX5eI+xXvm1eDKa+mt6YSOQpisFgy9dCX18HmP6MNKm5ziJJwv/2OWGBgQjglt + + qnh3aBF4UQtT9jWkq9Re7ELXic5JmZS76V4qElvCW9V2D4ABMXQ0veQf6TfLF1K8 + + TIfzulzWIXBNkpRWeEHelpyG95wQ+jD76++GPn35EAfusRShdwRDOk/DXNp/cqoj + + 93GXHTlCZGwiD6yae7MLNKcbsXkQISuRuhR195fHw1PbGmajhfw733LfgrGiNWWD + + P9+hgsvwjlxE94FsArxLZICO6VMUBVqpEPRktqeeSlG0WTZJdDB5L4sqGhWJ43hr + + zh/0FqkCgYEA2bMoJCgqJhcuhmMIk7dVVuc9vFbxK8kqaf0NMuc/D4Tf2hx/WPMk + + zyK3Jq/UL6hUj81UrcyQ8OAvm4QQ7PlbAOu23rW2J/mwytmYkIctetYkrNeXA2DB + + Hvylcfl9HeMqUfYWmPb0akY7MpTpyqNyZJP9a32OJL4SJKrJPbWZN9UCgYEA0Ht7 + + t8rryTxg8oVTcRgznq99ikPswozElEAWnwCHKRr9+b5H81SXtZM/0VHZSWjcLk6F + + qUzmxs5bZ9PapnBQXUXo6ds0o5uXkIng6Nx7aVHG4WtYNLWE2I7k6JYw7C4nZ3Fb + + qNiYFI+D1xazBZ+MiuVss8BYkIhQTZ6Poq+9gcsCgYBLojvS/AVQwIMQe32yXGKQ + + 07wWIBqf/L74ncslIUQ+bwqaq4Xu8GKceFIrZbERcakXYN4Hl+fPWAQSQrriqetd + + EYeyLm1/y/cJMroXlG9PmvCZADneGZJe4qXUSDqY1KCSYy4MrNfTyFyuwR/MoCaR + + HP1RiAiHaWXCSXerMdlulQKBgFGZc1P9jYoHIt7phj5GvbWHdHiQm3OOS0bHStNT + + DpPtJ6j/bAP2gSalip3wDj7oVv2c6D3ahp0bmbUqu3LXlOzc9wvJK3I57Pm6rZgW + + 7ArN4izKqgx/W46zZy8N0fovGmcnfDu7AtNRVMXz8X/q8cRPhdtZFpEDeYLX49pG + + NMM/AoGAQQqeB1BWvwZU02Gaj6W0SrybX57+reP9ivile8Wjk3n13OEj+2hjDZ+r + + EPEl+Zdg0R5Y1k9lyPeJxqYXMNA7GeOrPq/DmuaKd+em6kEUSaVwc67gvi8sOMgS + + 6u3USnDhiv/Kclzua98XFIFRgrkBcI/YwkCjJ2NvOJAluBT5s8I= + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 2097151 + seed: uqWglk0zIPvAxqkiFARTyFE+okq4/QV3A0gEqWckgJY= + zfec: + required: 1 + segmentSize: 131072 + total: 1 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:MDMF:h7lcy4sijoe2nuvvf7ulplahce:yxbndqsxxqxnzd7qd7mtclmhhd22a4jmoluosbitgzg7zuxrlrrq + format: + kind: ssk + params: + format: mdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEogIBAAKCAQEAzmdmHTwMgVhsonzBdU+1WNs7ZlmaYUdxR9YZVL30INgqIEnB + + Bn2DTiNWI+lTMs0wxRvFAmFP+kSJ+XCvlucfMRp65PWOop92xIYwu8Q4Pyv6xkdF + + wck/OMTBRUNjavAcuAsHgbR+LfT5WEufuA5NpKoSdtVcnxYNWaX6XhHOcCBk5dm6 + + K6QhsbmijeCwNoF1n/6wuiJZ1KN4IajckNqmjaDta8bpgerrnfVnmCLLiHIVFiOw + + IorlRCyPkgMzAIMhitcOm3Z0dEzFoE/ckDVOTYjfoh/PMSe+3+/BlIO/zB7JJJr8 + + 23TyCqBN9wYAsU2e09Vn/faXuUmDFgY5DXVU9wIDAQABAoIBACaACf+dAlYkKMtc + + Sve3YQPMjPVn9FB984brTDlO31k7CQyRxVQRGGt8UuaK8K5ysMyrg+GQRktP+o6R + + MueKf/p4ToEjvrHd3dkFkNSNYtKBwRq4E650e/r6VHS3f7VkSW8Y+5L5mGm5HsOW + + A5pg7KGw6ZXJ8adpBR96QsvGNYQbaPaYqToHBec5Ey7gon1C62PoF9i+yw17J0G7 + + dG+C4GJB40+Wp8bK1FmDTWdoa3j7+Ws3/jfceeMCIzw/q51PTQOyPRHRfLr+57mX + + RihBgAths+5wEHdPQ8HKG7y4HJJQiMY40Lusl+kV3Dg6eoU8rDLd9/2kBPcT1rLY + + HBf6y8kCgYEA3ZhGsyOX0b+HdBas9FZhzday4L3ypMliOmh3TuZ6dsOVfn9YFRP+ + + Sj4svjlVu/qrJFm0DcPS+5zILuUc5vByXLS6r+CAl+/1zJ8PHD3vScK7JiYJ0D6f + + dp6GZ35otQuL1tcY/Tn3dWlLJxU04UoZruQ3Wk63kFuk4SMVLL7tWHkCgYEA7nNU + + T6x51orxtFXjTNh2ATXbhL7Yern583o/0lAJregsglDhOdGqrrTt0voODix2fCaB + + wq+UhCvjh5ZQ7IRwgcK5X66VLwbq/lBL/d4zn/EDnx3JRJ9hC/68h/u+ToVJSGCt + + jCYbL3eh4BZa5wN0SBFUbQokjnJ+mNM6yNomnO8CgYAh5mvad/V/5xcn0VhAQP7R + + aKkQ7L40K4LVgKnP7j6J8L3sDjtBbj+WyBA8QbU1/tEzzG1ZNb4PNBsD4ZUcV2iH + + ejadNXE2zUUDOsoq/eafmCTdXzBdJVdr5DCXoKUQHWYVRe7Svo127tbKcdoXJSjs + + soktTaGTehGtR5qzr7nLsQKBgBNqIHs8N89YEMX2GEOxfCotEGqGf2m+qrNASOH+ + + 0krulHEn1K64e4UuBg8ffPV6eUsyd246jYUVbbkkbAJV5jMqf51iwZLKpWd/cjCB + + XwKuxPS3oCOONoCbhQ4tWRlbkNPryzWWBLCgtPVh3JTimx0jDBS0trVCbTxUNn0U + + BgDRAoGAEjnEFmrPDEI5Ut2gtdgByd8GmbBeISlADCd96zKWm4XoOcNFkNOsD9a8 + + qXvSVKgeMoKc4p+aTOEn3g003ryCE3n7jhRDsPsu2lhkDHfWbf0D38BMFfIbgeoH + + Q2C2l5PnwhrNK/+7oYhF6xJaTVAUcY4JADQqeVfmJx6IFVhQSec= + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 2097151 + seed: uqWglk0zIPvAxqkiFARTyFE+okq4/QV3A0gEqWckgJY= + zfec: + required: 1 + segmentSize: 131072 + total: 1 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:CHK:shbt5viqjzuewblgt6qeijry6a:je3omw53tmmluz6fvqupx4uh4jaejc3fcvfjt56rfzokzhgdczvq:1:1:2097153 + format: + kind: chk + params: null + sample: + length: 2097153 + seed: BTBX/ampNfLU+ox7xipBGiaSbgC0kcB8Gy7BkJB4oKI= + zfec: + required: 1 + segmentSize: 131072 + total: 1 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:SSK:6pah3govdilsqlc47l2zrispta:7cyrr4k7elzjyvn5j3bnk2q5cbns4yxckiz2nxsjbe3xe2ehnzza + format: + kind: ssk + params: + format: sdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEoQIBAAKCAQEAwe/MhxnQX9dLfyb33PwsTTdoVU/ccfI3kCzjSQBgcizQBKtU + + j+H9RtfuGmaL+foD0NUtNqLn9ztRegtBc9srWEBC2To2m9Q6+u09SRShSB5uHmDI + + CqAaQAwE6LOEmcUS4WVtXfVMYMiPcEtWEfT8+n40CWd7+rBSg10436gtaMPrDyK5 + + /5tzpqugLM2/N6cLlTzsADA3VpvpKe2NAlNCCCx3y2/UFPMPMnFkKmxgzDd59vW/ + + h3TRKlOjf3i6OYCcscncyAu4hP3DIty3Y9EK+lzGRwiekt6ztopFr3gog3jE+/OL + + sL5FFzJKsq4EdxWejKp8q/R/Xqr5ZQQh+M/k4QIDAQABAoH/FNSZ4QRrpaK+JJ68 + + aehmAWIrTVnzi0B+VEf358rYqYMxXdUekyH2tjO/zdq0Wmwd9S+PA0uDweaRvyG3 + + 478keLVoL5YQWEZk8/ThU1XjKpgUDJ5+rUoPuICeLfvXLbYtOuJmQK2CM5z1rLM6 + + D0FrGHOStxd+CZFxb/5+xDflTUs+mYwbvDsHq1RxoKkg/uwtL99GyBopIoTdcDJK + + D/P0KT75JONOcGSkveuepcStk44xh6Aj11i7l2BHH5R6mwluq0fmMPIDe1T8U6lt + + P8oJW4yPDVsnc6xqep97YC1+TKZruLHIZou5n1AL4oA2VgW13GUmy0LhZNcCtPkX + + xHLxAoGBAPJjxaHiP8DUQIefTxM/lyLfoCSHJQEjmq1nStnAGYH+zACkmWQDrpNJ + + krvkqoMmy0hzdlgPrT8VOu/rHv/R+TLdy/3mI2O7Ezz0pPds980Ol1i/oPAFqf93 + + fD8gxOFyOim/2dyS3unEhyyycaifMQ4JU44DlQDxSu9s7DtLNUGRAoGBAMzTiiZ4 + + Io9ffFuy9B+xNAQsgA2AUxkrptJd12q4Fky7Zs6SD4HjWmGMcsYen2KuP9Au7PZy + + /+8Yp6ZBvqkTkEJ6CENkehyNutYOCGPw9XL5TK0Qfw8vvhhg8L25TGU7voJ+1JEO + + Avvhg9WzGS4QSP3iszl703AD3B7sot4YQMZRAoGBAL29yGlu2IU0IceIt7fToZXV + + BGFTwW3g1yZCo19NdypBsKQYNVMLZs85WrnmyGueJKd0awGIVA/7qIVCwqNzVOWy + + pgr86lsZiHfA8poVHO3SLDt21p7NcEPg3svz9OqeJlWkLwDxn7nS9BXTIhHje90H + + A/c5apywRf6if1HzD59hAoGAIk6YSCM9HqiOqslJjHlgzgYqGJjS0ld2ZKvlJfHZ + + glatPJJIWKgc/lPI8Zg1eBDZjWQeupS+e2y0v+spJSaqtge8lJUiwt+WWL4W965n + + Xi+VgTNPJNsJSwoJqK19t0MPgMn/jqA7LbczHrsVz5pYr3WmMU2lN5Dd8KwQB4Um + + bFECgYA/TCLA/F7YHK3hw6cP8IrL/L93+30E/poQZjzlWPFUBysoO2YauvxD6M4u + + 1N8ZdMnWEKoEc4oPLiGLsQQDz2ICXy8lr2kFzcKnwnAXE0AYokGYZIQlbPRjA4xY + + GBHm3ATXgNJH0IzSHSX96YM15rVvD+uvcjlTbwWdVUYpBDENQQ== + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 2097153 + seed: BTBX/ampNfLU+ox7xipBGiaSbgC0kcB8Gy7BkJB4oKI= + zfec: + required: 1 + segmentSize: 131072 + total: 1 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:MDMF:4hgxxkkiytc23y5asf23wbumgm:t6yzndqbn4bhjksnlxp26v6f4mxbet4wpviz73qpp54fvwawdwha + format: + kind: ssk + params: + format: mdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEpAIBAAKCAQEAwnocUCzqdON41li8FEhElOcjhTUDbUVm8eMWNncqLvADlobP + + ZfCqXIbSmrYeaLlUY7tLeC2ApABStBPRgCC6PbzuhSTS3Js65WCPgJIFzxfgCkNc + + zYIozajsqbEgcHxeajYFUu1y1BeAIcN9Tfnm5ZuhjqD5naZEXgpfbD4x5Z8xFoxs + + 4LgcXAbMtEmwdj7cLfie7YStUAlrPJwFFAQUjWyN3Qvx8foTKSqI3H4jH3heVct3 + + boFcyXOSd/DrcKLAwfxWFrHW7sZnatJcg+UarRmXfBBOi6SorjabVUd7IdStt4z5 + + hLtloKW5BkAKI8yPa1CiVcK0Wg57oumVPQjgjwIDAQABAoIBAAHJyDWkvd0oSTBv + + 0P7povrph58HG7UgAtrtUANAwHooH2E2V1ikOgog/FkZgGBpzhDJzwnl6MSgZMzz + + 3s9WvbM3hForZR9oSOKhnNWEiVLN4skatOh39aRsP5sQ85p29dlJnG4kuK/3ycup + + POZx7fpsZWyfmDFSaAG/zePQ9s9Ev4XPUTAJZ6YAUxutPXv1HddvPLP3419k92lR + + zoAPiforCfdJrvRcCqspb7GRYbFKEKgLBrahdQpr+d1rzlOHofl190iGxKcgvNUl + + tcFN6bev960/aan71ykvyRjDWwpXZBBudtXQsySarYxnzeXMaoqfdq//TM8a+ydx + + 1/FckokCgYEA2Vi11HfFV03/pLv5qeDf6Jg5Xb+uFLJrb3hm5Pdx3iNiGXJ+8NNy + + dE2RW8wUSz82RibU2cAlRVcpPEdQ+PMTDVhWBG9Yuw2ZHAvY1ribQHWwOgC38Si8 + + X14BZR0eWC5fFKLAJZREacVGplcNF+LOYWbpsR/iZ9Lw3MHGnrp8ivcCgYEA5RAz + + 9kMryCmtGHCfxqKdjlCiMMiTKjVMW/v0cNwZpH7lsr/OEFLa02DB/xBhqT0W53Lb + + S4e5JfccEy7mbsW0EXJQHmtPErgoYOueWKOpbVmB2ZQuvx1miZdFBabGcyhs8/Mi + + Dke//9ui4nSpCPAe3UL3V8X9OL4+DmDjyJz/mSkCgYEAomufngJPL7nzE9kBbsjE + + qt2u6PcIESFwFeIlCnA74KQSeC/O2ws4md8phC8S71Ryq6PzJjJn59SF1Sz6Pr/v + + eeaMiU3oQgicZZAY4AUex+Hq6r2EuCwX8TCf3D8RYRZuKU6iRrLxGRW6gS3GdBYi + + 4jj05E+OcsX5Bw+r7Qwxa+sCgYB0x+H19yDnF3hMMX8DwfwZhjpqLJf6uNmJO9bP + + gyb/mkJ48xiXceZmRboh07Q2mBKJRSFQTI20MVt63DpW1yyKiIEYQRU7MfBEGVvN + + TQMf4LY2uzlp7g9MrnZd/zzFkSKa7KW8KhBU3SEZ2ugiymix3WZEtYf32eXBZtw6 + + dvBIoQKBgQDYH8P/gLrZku++MWbQZ6HtxIXz6iJPy9JTaa1q2Fi+9hIqetsb95uB + + UBVTlGbaFKsTkywQj3cO4c9uptHq7VK5DkM7O//6cDtVvFZloH9W/QpoQ/t2XMll + + el02/IKmp7dAx9kAdloS+OgYGB1QLp0LsCxjCrXKAWJXLpkp9EaCCw== + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 2097153 + seed: BTBX/ampNfLU+ox7xipBGiaSbgC0kcB8Gy7BkJB4oKI= + zfec: + required: 1 + segmentSize: 131072 + total: 1 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:CHK:n2642ofjfr5qljptdwjbwjz6fq:uwtjrewrev42yvez7br2574xu5vleallwnyddjaxqquvouv3dyra:1:1:8388607 + format: + kind: chk + params: null + sample: + length: 8388607 + seed: w6uP8Tcg6K2QR905Rms8iXTlksL6OD1KOWBxTK7wxPI= + zfec: + required: 1 + segmentSize: 131072 + total: 1 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:SSK:jr3l7i3eegfx7gp2umfxhohsiy:qz6pu2l4dckgar2wouwl43turrkpokddxy4dqqmtdqe73rksf2ea + format: + kind: ssk + params: + format: sdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEpAIBAAKCAQEAsDyWgH/oavnJ+/I+kZuiO6uqcHm0Eetz59LfgATNImyx/U+b + + CPm4sICXC96mHgjY63lwzbBMkjjJFd61G3Hs/jzLktPivgYU2bGltqxXEkeyk9h7 + + YzevjA0yZzg5mck8zECNAafmjHvGZK51l5z0DVpzfcxr2SmBIWMCFOonetTfkui7 + + 5F3uEsgri1jJnNmNF1zabwlRxNrOWk7DxV7tv/Nb8rk+JNBwdJHsCd+u80tI3s2/ + + l8WWEXCClcTaW0Ta2/6tYouRG/f79G4b7a6RKu0zuv2ntF55YXbnxd5CyuYn8DPI + + JfIpfBGRdf+b3+gDxXzx30s+Y7kbJGdN6AJRxwIDAQABAoIBAAGeyihgosm+K3ZY + + Dji81q8sswooZEl8PglSPO/qjFNSSRhmk1+OC7n/dZjA/L/90pi5+d1YXj6s91RQ + + jmLDF42zxMyLbRjjcU6wtfSNlANbEftpZ1eX4paF7ImWwaBYCQAtnaIGrRdNrECf + + 5Bsc6Woz5XNW7UNg+U6oRdeCUyICAsHl4GwM7TgHnAJ35ksoPcN4YVjTxIFtjeWI + + 1LVus032XEVHyiMzpP6y8xcwDT/qE0zG1L8hxph8QbdfH5Skjx2nZDfLYiKaJQrT + + O3Nl6XO54fOGMowTD/yGkNR9sOynHdlGQ6uMlWqPWyqPr4Gn3ihqu4/PFXRSzMGT + + F4xV8QECgYEAzwQ8IvZGKN72m4NtLhPvmUvpBE8LPBDHzxah8ozQeZO5SD5InsMy + + zkrkOn+qHymxyuFMhmhUMTJhsqm48Id1TSWN/UwCO+Fg9c8vhjlSq902o0D3U3jG + + 3V5tg56UxCl6foJ+xletlkTKoZIaDvb+bxF9R6zlY0fpvTlE0nC6OkcCgYEA2e/m + + uCbgxZ7vaYJZGgYMxpsH0hzrcuScC0XqS7+EnlQoVUnLqxThgUbEwE1d1sf2P7ir + + VrFCBhAn/n3MBwSpfgbZ3IvFhSvGfHlA0R6GEnG12gg+rqQHDQdgMIti4rrlSb2J + + HlOe9pnLg7tlglfFWH/veofZNc8Zdhovee9DbIECgYBJw4CKFKa7OXc1wobMvF3L + + ibjlyCSAqpoHuFDMVFCUgYarr0XBDFy2FQltrr+3iuvHFrBl1Bbr0L/vIXq8egfa + + DV+iucqx+4TJEaIleZdzlcc6NJPsMkTp7BOpqn/nxb/YBDeYBPXdbXWmTKDsZCYU + + /W5ec8Tos18eBaH4OiKhUQKBgQCjpYOuxerEGfsWU/2KD/7p5yGxQWv/AvC1elNb + + e70ekn0Sxe38Uhqe00AMUkvjapVa9dUarNGx8dHGRDm/D14iNwzCkeXIgL1zXC0y + + meP814vA464Fvz9YJjCxYwjmzYY8n+jlb88Oxx9NlJq9jCCwuqhdbsLIp/ErgLAj + + tGkBgQKBgQCmJWwxwTB9mU/KINfcBGkgJv+3FAg3/G3Eq++6H/4h0mrQAVj6Q/bB + + KKFzmem61c712kxMo88uMfR2UFSiifxVX/aEKpmOSuG24JoBE9WXlhGFPSP5r7mK + + cfUEQMpuvj5DD3MNaLBlqVxk0VxabV9s+8MrLM5DZ0udhgA9plmP2w== + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 8388607 + seed: w6uP8Tcg6K2QR905Rms8iXTlksL6OD1KOWBxTK7wxPI= + zfec: + required: 1 + segmentSize: 131072 + total: 1 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:MDMF:irkh3ffvh2eaitliu6lzh677xq:thprgu7uakbkjzsjahw3ub4j6upj5ejplibiuxh6pomwtfsvz7na + format: + kind: ssk + params: + format: mdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEowIBAAKCAQEAmFXg9LZxuJXGCRRquQXn2G5E0PU2td44P9DUpPyl79Qe3iRi + + dqjU8JpV3qsM4OyY1kPLZT9DDjYBGGfCOrEo+QO3/ZXJqRGz7WvcvcM4zXoB2RO1 + + zEOgp9GXD2QZ3s22ZXhG3Ii1hFMn1AyPq614lA5fhvehHj+2cDOcdxQxVFgVIS24 + + hAs/i5cugaJLfIe9S2MsGHogT1lyC4OjPm3rioqaJ1PMMPWYd1z5+/HmIekVlBmX + + A8hngf5NMwiHJP8foS+ag+byuWe1SY9jc3KOytrnrIw1vu/KpstBVZV/FxdKn1Xs + + M89pj/JrxBFh3Gr5e4bnFDBJecgRIHYRMK3M1wIDAQABAoIBACG30IUZ5O4AaMcV + + t9GgVwL21VCTFjsHJtgpNwgVy/zbrMFquEifchKXdq5EmiMm+2VhuCF+8S6yEWf/ + + f2RSVklX41/DydEcVAEXQNLX5TjF6qbL0A+YYHUE1TTY6UkBq3+mMbkaoWLarRQo + + e5x6VxgeXlKXeRgi7hTDt7w7wfdy+DN4yyeJiH166EtQyEIvg1u009iSwgiRa7G6 + + XT7hy1zZZDknfgnfomZFoXSlrt8duX9OBgIYisDGny5uhEVcvSiMhft91wJj+Y5j + + tioxT/+sHY6OlPgNMqU888P7P3DyL36I5TmbQ93yjhKpdAP3e71MJhoI51UdgVnv + + KAxEBQECgYEAy8CTIaREsDo7lWlj6Jb5mH71piFblifW1FEPRqOVjASlTcs3aemW + + h5EMIzXpwRdHLPrd+3wQy/2NATfwGwJqgxoxnnuTpb5DAQF6CK5845b521+x18uS + + sfpbij3NXYcEdF64IQGRIkf17rsCVvcCpNRfC1UZXnHqfqh04IorZ5cCgYEAv2YF + + mFC+wk1hSN0RrJO+ZqyWeWkdJwU/aI18mo/ro+oxYIEJBVIteH6Lt5E1lCx4deeQ + + AwPgv1G7zo5134Tcf+o/1kQMhh7qd5oNlGi0r5jku38TjfERBZIl7e0OWDJ50naK + + mNsN3KqqbOtHC4GipPjnEOYJzJr/qNstPUvnbMECgYAQwquXtdqMoI2sMbotNNYd + + TDxKyS2ugWJznqNiDSzNEsjCSHgrdzKRvkXAU7wBzTdmpNBD0qXTEe1ab06J+j3m + + wO3Z+pJfrPH4EDYIpsnRMucku492j+FmUJDdI05UZjnglLYSyP02U7MQS0PbAYCv + + LGURGpP2p+pBNvw+SD9fywKBgQCm25xZE1uaLLd5PDDiUNMW07NDGR4vHGYREffl + + Dz8Q4WQ2i4d/ugqmFzxaxh79lF9X+o4T8teGMw0VoCCmwj8wzNjmRODeNCmYJxdb + + oISU6SfPRZOYlOaQAr9KUvXEcgy+LFXbuGy3SZnV5q9DGrreM5fNpZ45X48ueBVS + + cM/KgQKBgDGJMoBvSYINh4vj7re/HTEFXhCgQoFxiewkdHYJq2AOF3c2h0mnR76H + + Ub8BegJE4MiLZTjsqQJnJkBBtt1YL8y9SZH497KSjKOEvMbgEZHsuuHdV+KRC6O8 + + OJBycdnr64J/u3fNVQ9Lv6QcJMmAU9drhub/ao0z5r9LNPgHDP/H + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 8388607 + seed: w6uP8Tcg6K2QR905Rms8iXTlksL6OD1KOWBxTK7wxPI= + zfec: + required: 1 + segmentSize: 131072 + total: 1 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:CHK:nktoeeuydph4acj2fux4axyaj4:g3fvjwanenwsgdcn3oxnau5gtbdmzbnbevlrzrb5qe4yukuwhejq:1:1:8388609 + format: + kind: chk + params: null + sample: + length: 8388609 + seed: yPi3JHKKbWaEEG5eZOlM6BHJll0Z3UTdBzz4bPQ7wjg= + zfec: + required: 1 + segmentSize: 131072 + total: 1 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:SSK:thrs37wytqak3wamf3fyjs52j4:n3irbt2hs7g2qthkrpsllqhoxspmca2nah326tnwa4wlbg64kkxa + format: + kind: ssk + params: + format: sdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEpAIBAAKCAQEA1VrS+13yFy5YBzglh25Kqrqs8v2CMoyTtmlr9khQ2FNU1Xta + + gskWuszB406PFKmxU8q4yOTCavWERh73Bo+1mumRjgNONfQRBgHSAQfDkiKGxvOu + + t+ctuH+NvAcJFh1uW5c0vL8OxdbosTwbVkIWz9qrj6HTY6yAWTzukcd3CB+rv86o + + Xb55r1ZCRcbRfCnjVSLuhta3bt5yeGbabWmTgQqPZmwuMa2oWPWO2wzi0M6kCB5U + + K8Bc2aZ3Xa/bMqpohmPwXVx8Pflt2vzcd/0g9EULOyxNDwRIY/eMTboCGQvKmfbu + + WxJotkJqLqP6WeuZauK8HCssXielksz9ne3CCwIDAQABAoIBAELmCzDJaN8O54g/ + + +TiJgz0ccp1wkxIRlUGFtdYQH9Vs77VOy/clYYyqJoOFPwUOHm21K5LGdBXArTyl + + efSjPCD6aur6K1xsjqfxCy3Khu68B8G7aAX/JY1r5X/XPuihythKRb2HNPUg6W6l + + d7bo2ylKmi/b4KIo1Ufl/LJWNoMjhJGKyD0Ib88PNziILrUP8cd5QWqaBEGH1df+ + + 5FTfQAzU4OfrY64SK/loN4uG2PeOLCdX6/rfWf9m67hhJevZabCHGg3rIIpNzdbv + + THjCxvrjiuozvtXMrZ6Wh60DYRWdUwcTTFQbWRPPiIijQki3AkqyUExG3y0Pm8FH + + gM4GBOkCgYEA6V8A14+T0vhX3y28haL5z+7Gl0V4Zvk50YEGYusVNYPfhTFeKweg + + eniVQaDFBa99rjXUoSVEHT02QGGqmGw7mgEosqJuQs3e1/OTF1hp1UKwscwYYQ0H + + 9gIKOhkpLORg2A2MpQcREg+AgbgluEHaIhlnLaA5ttUIzv5pMx8F3WMCgYEA6gr0 + + E3gLTz/KqlSPoNnCS0etSUkpW7iCPzzRyNE/i4bx+WxISdGY5fNowTy3sMzwN5kX + + 8gOfRmW7M9j8Ho3ocYY2Qcwh5kb4ayCcb85N5zUlLdddm991eWP8pz3ArO4Tw9VR + + 6GoaYE91ZQggZxTRwD/duTggoYGJURC9VgjL3TkCgYEA5zhm2Cz8dMn0Hj7ti6an + + Rtq4TsbY/YWvQKFK15U95VDslMYOHCopWU7B601ECFcQ+huBucv3idTNPMrHwM9z + + 2imNzjfbcTsSsPo3YakK6u5xrSeffADyQ09QHLIzNrRsM4RxNk0jH7bWRzBRxxcP + + 7jsnHHCk3j6CxLwTNUBmiisCgYA443S0jsdg+gaPJILM/GFn3wJV//yXmN+/806i + + 24nwplqG4DUqDFJ4ApSB8/pKdWYmfYX+g7bha7T3Q1T1MFVB0ve5Qp8y1ClqEME1 + + xBXXj2l8HQ9Z5hUt7onpNO9ymWQgg+em8LN8mZPVfQYzSDI74spITUZRO6VfGQyM + + rxKusQKBgQDBIgHEWMyfNawAjnSk0PJOSyBydLRQ4W6XkSMLmHqF05WUii4i9Meu + + OLvi9VObTZyi89IUcqZBs48/3B7FQ31IsJoHPugnzov2C7JHg7+EzFDPupDQdMcG + + PnxRNGkN6ZvIfZnXO1ZzwGuuwOH/n7LID2uL9ZnHtFuoouaOaoiJ3w== + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 8388609 + seed: yPi3JHKKbWaEEG5eZOlM6BHJll0Z3UTdBzz4bPQ7wjg= + zfec: + required: 1 + segmentSize: 131072 + total: 1 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:MDMF:y44dzsyjkmisidhfgleytdtume:pllr3w4u56xxl24zfdm4b57ppxzv4pq7g7pnx5cfun3gx4iyf32a + format: + kind: ssk + params: + format: mdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEowIBAAKCAQEAoTdi7OsNEmLySMALvN1FoBGSBQEf3WFpXZVHOHxWycEGkHfj + + s5Pkm9ZXew3EwB3+KpKo67IeojMvgdhKUKd6kzI5opMJipp4b3uqleN0eB4a7U8p + + 6QmwwncabdF21cEyJ/FQmxi5tNioj+fO4vO2yzNRUpS5oJqNZEIBjftJPdlhBR6u + + Un1uYb5fCQ8vgnNNvZXVdYF+R285k6htnaM4/kn9YjONcaHYQv0ykO3rsNeLdKUR + + IlepsqGtMBMwsN+bWdz5KUVEn77DvaMu+Z3ziveee7+FrMj79gSAUVPifga6toF+ + + IrD7Z6Ryyy+7yYciWdNi7hJForQHQb76o0TsKwIDAQABAoIBAAQpoMbFl5alcmRk + + J7QZMdkyYEb7zUhTNxFrr3qVufdHTMrseuHMoZxPz6jFkBJ2fnUNSLTz6kGS48Oi + + WOI16NQlzwIpHMJ06ZjV+cTD/40IbfaO/YyJq67hvLogLJrT/F5hUeJR+Z2Mnebq + + poIKppTJlEHMcEc60Qgegi7EfGgkiw1uphbOmzKhbipYGVRvAGofjKxiU62Mh5pz + + 6MJOjaAjN47wV9JaoXKmO4xs1ewUAaAXaV/DdynBEM1ZBS4pstdCCxD34tz+OIUA + + wj5dKbmj+7v4V9J+y1T9Y7qNCmBmCwdAyyBzCvVZV01SACdOkC9yNPkF+bAU6L9o + + RPhmSFECgYEAvOG5mxK4nKRSKqZzxby/t+jzUTeMnMAUMWGp7/VBMsqJ2MrZYcxs + + mXFKhmHTIn921rp3Vvp/JzR39o1nVdtg74cmH0OLU4wuBzMwFoG/A+h/VY86v7MJ + + 0dMHrZWHTew3Zw/fkF+GDkJy0lzA3qoJqi1VEBN4DTwLZD0gQ8d/WTECgYEA2oD8 + + GuvALdyxs193OssSijAeEmJ3ezP3SWbkQuA8UGD1ULe+AS4T96W/WcKTLoREiT4+ + + wIIjDdgEXNXiD6e/0vzDj30LSHZELyfyKcWYX4o1G575JKd69ga9Roa7n0UE/mHM + + qOqAXbfLYqhHdTgXdDBERH9VtOAAB6aLCl4dxBsCgYAzrl+murygv6Vr3heXZ0nd + + /HN3KYfj6/qaeGqTKbwpNZn6I6bPR6v/YCxQELxAmDfgES1OM0RPad/ZKl+38krX + + v1cC/uxEc/q0JaFmxyGI5DjTJFmi0k5Bh0h2io93FsciAAnf6wM3K59XR+HOCyCR + + 282GlI0oseE8EC2f3hpOQQKBgQCHSgnOmV26h8U3LMrkCkyGZ1iXRYR5MinQtvZq + + OfDeS8pYmgv5KxCN64BZEVKUIK1W1MWB6JHPxoqc+Ikp7FGnT32+YEwWJ7P8Bp24 + + I3I+5ZIQchQND+3gWzfibRXKfa+j2eYgSGIGpQA3K75i48IR3LjIOJdWkMMz+Xhp + + iPChNQKBgEI9dKOoGJzpC4CEN9NjbS8RxjzznQuW2pc8hAVgGSJlhFhTk0XwjZwz + + S/PFv0JawKa6xotQ959iVDFxHksBVw2NPfH5kbsEvmUaId6mOJjhLQDtXMzyIln6 + + LMXJeJlCDZLMMBJBXh+rc7p8osbH/DX1vvfLTn24YwFhU/tNQdfq + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 8388609 + seed: yPi3JHKKbWaEEG5eZOlM6BHJll0Z3UTdBzz4bPQ7wjg= + zfec: + required: 1 + segmentSize: 131072 + total: 1 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:CHK:rl4bzmselnuezmapjlzssnqg2e:p7kvin2fnemochuxsmh6ot75qpbfhrscbxi5i74bhqdhzcy6i5eq:1:3:56 + format: + kind: chk + params: null + sample: + length: 56 + seed: YQ== + zfec: + required: 1 + segmentSize: 131072 + total: 3 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:SSK:o46hswymw5hlh7xpdzftdc6ztq:aeqveubp7ukzywobdfhgrzt45ogurmp2wtzjt2rz5nfnoxrhyqrq + format: + kind: ssk + params: + format: sdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEowIBAAKCAQEA3JZ7Z+cKR4dntNRXVn0C7Iahr9sZ15MqHzMbTjHX+Xh74A88 + + sAkLXEdu1cVT1mmIm+ViahPuqi9+XGMaPCw66blfiUIH+2tMsgVmR6J2VyOsGtXb + + pYVq5BYk3nblqGbCEfFhHb/DDNN8PoFwEow8QAjlFa7CWWsnyJQjqvvfEMWGcOI2 + + v2OREt5sYuFW9agQ+9LaPKQGK8DXlmXsNiiLmLcqCirdmjSl7tB7eb1Jcj7Ck2O9 + + J3VS+wlD20lG3AadiwLFm9wM/ZptJltXlCZvYZcnGGXVO9FNIyL+tTg4p3l3e4kd + + X4h6BwsfLr7LsDBxIE3voef09OpH8bRINoVUxQIDAQABAoIBADgcs3GfzPabFB9k + + sH7YuAiwyqpwQqea0Ok01+pRNY5JPsGlPpvNAS3NIf2Q/52YJN77P8iaH2j9QdiA + + gSjzW10fAZVpzZwAFHdodjcctZu/AEWnRwNY5/LzSxeoCQ2Ibi+gRkMKB7TYi09f + + H8IoGB9148hbNycF4g3c2SHihkC+dWSAAPxNxBVdyXPlINxb3O7tMB8T6wb9PWjr + + itA3JlzDtKQLx3TiZrUonJ1y34f9nWff5fYUkJrIAMu8CaCU5t7NTs76NnH5SQ9v + + URAxZQcT01OKT7/vz2+9QsDVtuoiLxZdmiqJo727z5w8JTXU4CrynSj0D/6xllD9 + + GNEpGNUCgYEA6XZliEDQMrvYEFJm28wVo5GgxFStzX56bKgF9J3D8TxFEUNzALQ1 + + n+t/yjpae9q4tAVjiVfl+3hvgYnyigOMsvEjCBbGoac2FTFCIxrrfMbeX3G1O1Eu + + 2CkJfwvSx+pVvxMadiHk5BKIpYY3f+2xdjFJtXPIma1dIIZv2M8hCXcCgYEA8eHp + + TXSBhNXeEXhCwgXbr7mtLbgco/iAAw+He3YG3GyJqXrOmPAaRXSTgoSyClej8uv1 + + o9VLhOmdX8wwfuPU4OKgxXRcU6vGh4TusSA0V1kPz8Dki9DfvK5aYrbTWOC/pFBX + + 6xDs4C4pukKlKBKfUemtDK/cqo0LMijM2Hc5oqMCgYBCIlTuvRV9WbMCJKWYm/6B + + QG6fTzGQ5cQ+ZXaSbeKkwqL6GfZI+8O5Epg3rEIXlcT+0gv5SxoOG3bS5kX7jLfd + + tOtsji8ked6bMEIA+c49oYQ621YwgHXZq/5RrALAuQQjRYEYd8+EQC/PW+764VWF + + Gr87lJn91ptr7ElgzIQaTwKBgQDa7BFw3SXsyHT5ctNZMFwpq/AmFSE2909FdeS1 + + xZloH4RpNJGQsp/UhTKNSvSpj7D/yLjG0+JKJfceIX0zG5otAHFqxWpbAHnrZlFz + + VyaIeD9rVbaFJUObTmLYPYkEREavvVgVlXgPXzi9MFyy7Efup4TMms8qPgYIHA1r + + Tl2H6QKBgFiAa5p7YO3HPTG1k5lLKDyJ/Fof2WsA7j9VswKrnXOJXLlT4OmY8WjY + + Eb8D4FMNdzyAzUcPg7Eku4C15wK3ST/aDY91wtgOKOic7hqcpVuFx/HXia4DqHjs + + /Prv+5FhteFz9uTo9H8pDFLuQA8ut7UF3YRgF4Z5ENkmu0MKuDE0 + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 56 + seed: YQ== + zfec: + required: 1 + segmentSize: 131072 + total: 3 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:MDMF:3xmus35ftdvmmunmebcz7dwusi:eplir4flebhq37sxzvwp5mkqsdlvi55kvlgmnorhkobuadyvq43a + format: + kind: ssk + params: + format: mdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEogIBAAKCAQEA67z6lT/5WpiAfDaKORKOUhWJlJb55p+K6v8H7QR6gggmrQTN + + MSrBq9X6qXH4pZZIx9PmAjIdIoTQlxUsKM2p3g+s5LT5ADfBbc/1bZiqFWjvPZ4y + + GggvXoLbJgrjnHSLc1/y7k7gqOzE/pmGzjhznJ6mO6MlJ0xzP7phMYr2EaKiKPDo + + 9HrUsxnjh66G1lfxrJIxy0wavsnjoVfbtT2kb0oj7MkEzx7kI0h6g3pyKP19yBFs + + S9W1Vm4UL1OZWIf8uPPrvjh5Wsd9sUMAmxxeDi9/W8GCt3HSo/Ge8QrykdHFaqkH + + YGkqG1jlPCgxeayzL5EfotG9qejcHEW0zrW76QIDAQABAoIBAAFyVJlTiKyxJeIH + + 6vuPAmznhpjGVRHrZbdWdE5/NLSVPI4wQDAZN+ddi0m8kiQ1/WFYirSgaRmxS0m6 + + DrN7ZkZKE1YInu83kwog+LvA8D5BuWzH3+fVUrEXnXpTc8fIgU3mcf+wtTk5fBCn + + PDK0xE+FkQs++jc5BWCyke5zZgTVVySEX/o1d9byHGLur/F6RShyHKvU0IoZ8x5l + + 7OUkHrPsYCuHOdePE4+YGSklhJ1I6/mW0AsH8BaiczjXMsMQNO4VRO99gub+yUuQ + + qBzn6xhsKrEnVmyttIOGS0cxsmaqMCZ+JYntsJo7/iKtQcAZQz0EaHwZFynPJIaL + + 39al/rkCgYEA+EVMnwff53RGe2KwLNpo2lDzld4KrWB44S6/dUwIJjKVi4NnSK7H + + JcyrEMJnsyWA8AAgXZTOGxMSQUEDY69Ih3+OY4cwU/uTZjCibIS35fLhxlSW5kyT + + GyKP/n/AYZUgFWZwIgZNluoRsv4j97wSNly/221idngY56whnf9feq0CgYEA8xPL + + 3Va7XCqX/vrT/m2HWmBvHIvfRK69uEe0wIv/YSPvoRD3no66+CfwbU/CX95U0O4p + + 8dnyM7tuKb3GHl3WHOCTq5cvXL9qOd1h3vLU7QPnDxOqnUGEhbVGzhHTC9LWTQYM + + h4TSxrIDYVhlWJlq9L3S9nB641HemDrXI70Wya0CgYB/en3cTpvOaarjIgpaDY+3 + + QcfBVTDgU1/eKDXQ0ciBbInTCBbZgDzrkMrpoRjEKOaq1TXJN2YZCtLdxLcr0U4J + + nRqMylarWMsXtrM/y2nt3afGQZr2B62lSjrrr8clk//UXTQIlHn0mp2Z7dqkEuK7 + + HSa6UdE0CXioRH9CdGUfRQKBgCRyzHflAHUigeYe8FjPTaN0oFSUeKcQ2KvgPK8+ + + js2fGNh69dZVqp15R6jsc8XyTZ+ChtGYD6RIL42cwi9dfLSZzCrHobdzkFca5gkL + + OnhLxILTPRsVbuypsPNHYvD77VxhUtGjTgOzP6SCH7g4UPxf1llTpmmdphYHhKj8 + + OoWFAoGAAbvvCI2VggZqX/FoEpotLrvcSOiKn6havN7Ey7v19Hya1zXSw+ED+GEi + + 612CtkTSB2CoWdpXsdD8urKA+CB94My/WRvNK+X2mPtLSRJkr0ZrFBRKxEnbJZhI + + m7+WuWG3wkeHfe0e68sWkV0DjKAJDDVuCHJNQd3DRqdybG2ZRQs= + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 56 + seed: YQ== + zfec: + required: 1 + segmentSize: 131072 + total: 3 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:CHK:ycvogzi6wllnq2bkx3t6zdtwju:um42l4yen7jiwdfgirvedtty3tt3xuhjiyxzqoourvughtxjar3q:1:3:1024 + format: + kind: chk + params: null + sample: + length: 1024 + seed: YQ== + zfec: + required: 1 + segmentSize: 131072 + total: 3 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:SSK:ff2y7e2fpz2ohrx5xyqzgtbaoi:nmq3dhqyhvdjgdufzhu4lu3k75dtarphkg3d6pm23qpaaz67t2gq + format: + kind: ssk + params: + format: sdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEpAIBAAKCAQEAxBHmofSaG2NHf7287D61VUYfWhJyps4VJohY2PG7BK1z7wly + + 4gRqVBfOOMi1VvHXySLKaY3RJXe4U0abAin/cRgSO/HxoYcsH5kYq1/iGGtFuBZ5 + + 8XbHeMxxASJ+UruHydwqAP/rZHj/goQgN0hZI3P5oxLF+xDkys2wBZu2W5qocRAo + + F8YhcQar9xOUyrwxtfsoR52a72vY/EWFcg8DtYm7sX7ZG8BH+QT3IpHIfUvaOTaZ + + 6tGDa1p2EBHheHUUb78McOrhlTfZOqPpSFFwksuOZL1Mv69mlPwQjvQ6AMA3CVsu + + SBIRIImWdHWeeNlsyEzIiLCQ9TUw+EtSjKM/5wIDAQABAoIBADxDQD7A/miyj/Q8 + + LgfykitefR5jEygfqTKJr70mNxQN99cdcVj0gHXOR0z+q3XIqUkhz1K4CvNYI6g8 + + yEHXBLMO8fPIvjqmYDJqDMIHm2dj+S7Ggb5sgoynUYhGwMrO5sJtT9+0yPW9ltLX + + p0s2imcyKyUrDPzIyXln1NU0cc0fZxqcCRKy7L6H/uOtgBRZ4PNyDUNtSxmbjmfE + + iqm58C0lkeMOPa1Mj6cqH8PD0aU8y+N7Vdy2PrxOXHv/+6Y9eRx1S8GvL4wFmtgL + + WHUwwfxqNXmL4JdsQn8TSw78p3I47CNSk4QXXdvDDBrLc7ejVZU73JgYh5vsQt5P + + ZfXEMfkCgYEA6KUtfM/DpaehB13qx+23ArYEyHQ8Igqma2pNoSj1szW88BxLVzBb + + CEJ+tb9wcfQ5IJO74mqE9huvNUmeU2vUVDwusUEe+rnOUeqkL/sSkX51GAqVMkWD + + 3Qq+N5f7PqhWPjE0+alm7Qd+SMJoj5FBWs7wntX/A+17Q84ZtF1NT6sCgYEA18DD + + XtoZ5p0lrC1UhdI7EQPQL+S7/h/sgiek+M6wiqe1uk2b6pkGTLFHRRZFTLXnxEfI + + 6WWIQPkhmmU4eX7uqn6+9TV6a610PnseW/JFWc8vVRHwXAOvSzkO77kldVEDNtLa + + aQyXXcHsnNZXWWrKKrLlGx8GTJORBLUjrCzxxLUCgYEAyO9BZnecJ8usjUxUp/Ft + + C+5iGzApb817B3N9MSDLdcmIMmp9uASP24ZzIk8Cs6mYXca7lEckJ9ypa4D2Ol77 + + uPVx7q6sLymkRaQ/wyE7XGa4g9dAHXdk+Nl6iVG/MtL6CiU9+BSUTU0XiYg//yAa + + LnBl6woxhBbtTBcKpHmheJkCgYEAjr9yRDKnimaVA1smnjffbr2II/gBzfyPPfo+ + + 84PFWKfn2/D3ZPuEKH/uuK4ogb2lL7+TFaFgyiRLcFziRbiO7m1XqOOOMOodjC1n + + g8xCyE4FchKhZi/l7i49TKzCNOG5768IZRK4n4bsJ0TFnFrEkgW1AgG/6DCGdYfn + + p0ZBXDUCgYAVJ0l923qS5EzG8+AP7a36yooMTdrL0NxvIskIJVoO9jBqhQGOxk/z + + cbWB5FO2e7k4Sn4NjNs7KiXCkZ1OJCVvBSL/YXFKGtPcOq9CW0ZJIa1pdlgIYqTS + + YmWzzF22o5WkCTt4I6tCvmq3b1poI37GdHmS+wG41rPKbc7mY+RahA== + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 1024 + seed: YQ== + zfec: + required: 1 + segmentSize: 131072 + total: 3 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:MDMF:kgahmjtyuv57zk7tcfzq2n2sn4:26tksbkcuhgw72qmklz2ihmycmpybl5u24snnay5kln5wumrllvq + format: + kind: ssk + params: + format: mdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEpAIBAAKCAQEA2elUOcI4tjDbw4ZqGXI8mBbGAhn84A/Qm2KH9LbV+I13CmFT + + Ck9RjjyHYQa1F9XFJ1CKKE/glUOv+uXQxOq6+0bjgOAaooKS1zxeLmy4Vc6EcHiY + + m/puBZ37mlGOUROZ5PktMWDQ6WB5kycHwVrzIXtHB/JuHCA8S9jWAyhxqlsYpm/O + + C0RfBjrUyZXR5KxBS6IEjlC3KYHeqh8Jlre+weXOpFhl/z0SKzyEOHA4H96xFmKj + + qVvVOF25uh1ocS7A6+M1b19UH5TRQ0dj9hAFz8B04LaHCBxToDgSIDqKGmFd5vsl + + 50ohULKmi1Dl3Tf9EZ8YCjL1KzD0VRBTun6lYwIDAQABAoIBAAHpvv3DrYg1p3TV + + e+dm247l+W15uVxAyZON72Sf0EwVZLbvQfGWukARWmYNc87Wi/cNwFDBkjIa/Re9 + + J3ux8UmVBqr/6BmrSDjiO29weZ8+p/jbYHecYLouU2I4X8pESSTN+KU9+WwfqTPc + + 4Mkx2W2Vm2itYWVgPsmOP9orY17uRYRjHwwC8Kj3cWLmZcQI49o/SYp4rjgNHFpZ + + u/4ggH+ewO+5UKUJuQPCxFL45UAzl+nPeVhjpb3PxVzd+CXxTWVwWyJ6PqCHe4h+ + + kz6mGHMglt/yQdZDYWnnlVHMtfTFbvf4YXqser4oChkhmLASr7tr1aL6aMidpuV3 + + Abq8bNkCgYEA7PYc2+3ehahZSf4cidzIsL2MIU5EvO59Nu0o7mvFzeZY7bTn+04/ + + csnrqXucj0p0vwRj92oEaZQqrGDQlk0DEmOufk9TVRwAj2clWF3d5M6bR5HTxb2/ + + DmmdPfnw/Y+PCJ0ZDG8l7405Rh4PwzctZZUv/ZetSxdtDn4TJx3F/HsCgYEA62tk + + U+k36X1cyu+seJ+DQNl4+zm4Fb7cLSySKNT6f3BNey5RdN0A54gPMBd3G531/MKH + + EsWW/SfE8m6a1nBvu+6Op/GzFqopKFl+nR4hbnvmWBH3KOu/12iREUEAh0ZJR6pX + + 757eoUm4gjK4vwBzv4O+8clskU7oUGILW5KT6jkCgYEAqUhtd4Somq2ZFC4wbyDG + + UtUm3chPfPWXiHzG6AUgK6cq0q6Rp8vPsg6kh9CiGQ/k9W2KiP85Jb/O+JS1jxp3 + + XlTOHLhI3R2DHO9gE5ADbGlZLzjzpGmYqxAyYEtFqa88TLgGZAangEpQp1Hkit7J + + VK/OuAj6qRGUPG0++4venC8CgYEA1QwqDloXtGE0EZ9W+Q56LLziZJB2jI9eGC+m + + 0fbz/1J1fA2Nv/GlOOMDw6TosICCNc0hihZwrwdHj5IS5A96vpuEVG5CgTda6d4b + + 3DqBTMgpy/fuMgUvZtSFvBSUUteDx6xbykl+9n2N0Z3vXUMefOnQamW7r8C2MtCX + + sLZ0z9kCgYBuvHidGdCncYfx7hPjMaPH/6J2YjS7yhiJ1QK+1HW4mUrqa7sHgUUK + + F4MTGf3o+q1HUKrv/CrrUm3a5CoEd+uLJD/3fjCkdCnnkAf7P3XFBJUv7sJIDFij + + JIrFWM8k47vH2jPPyzr6FMJSUUeVuO4Zz0tEOc0VufR21dnkpnZAHw== + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 1024 + seed: YQ== + zfec: + required: 1 + segmentSize: 131072 + total: 3 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:CHK:3yjtuv5h2g45g2cncg6hka3euu:bqqhp6u73ldawial7rtrfomx2qkmboyvqve6ywo2jpvfbu7zptga:1:3:4096 + format: + kind: chk + params: null + sample: + length: 4096 + seed: Yw== + zfec: + required: 1 + segmentSize: 131072 + total: 3 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:SSK:ym5fhzrwsby3hvapqocblodfmi:j3seystjreodtepfokv4qnp2fwjb6lwg7ti2cdi3f777uym5hycq + format: + kind: ssk + params: + format: sdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEowIBAAKCAQEArLuTmZEerhhv9SgwmhcPcMoOFxFldMayyd5daEWGH9o57hW/ + + F4Al5eTLwnur6RaTYKKuPu+etjSBAdO1HBMKEmhuDXeYOgWEWAe9LluQU7ffmRRP + + Xgsx4HaSc7nZZVPhWMERAeZMOUwyB55kMby2Og0CcxCiOOhUyC2KgeN1jPRN0El1 + + Hsuq9LUgg8Fe/m5tqZ9TDO3UiJiu0IhLPUrO9Os0cLLntbtOo8S0vt0AzsyZsJcO + + 4CKzf1pNnfF0ca/F5P2O7YHf6VR1P5HqUA+4tdizOu1sYqHIRA2IeO6x6P6DbFRS + + zRS4b4UOHQ5G7rriqzUztWypGuJmbI/Q6oLovQIDAQABAoIBAAFUtA57TE/EYojY + + 4MaWXGXx7JnEmZh35A52a+S4hzZgg+pzDCUGDzF+NoxLgyRsqkFUzfMwSfPQVwxS + + TjaTgy2aoXrX+81T19q/+1Ce3690pUcK6l47x4xAJaDS785rAi66oa/daPA9FDQz + + 7HfOPJIy1PZZf7Wto2LKdQl+Q0+mpydV2UnldRBTUP0h6maT6Ln8VxSVnwnaTyn7 + + PdeZQNDC6I26lD3upg/aun4NS+AxigOgXkk1qi4hJHTCeQED3/qQsnlEMf7n+RzM + + bnzHrxBd1v3WOK9uf3hpKep+GJH/VOq2YizFOE3nBsr2zxTlIlUIjMnz4xoBPzqb + + A/qlxXECgYEA84GSzuPL34cbXy6cWbndxyipPlAqNfMsnawavxzgRqAjM1201O0A + + TQv+Ohw2YAacnewx3oq+CB4ilPIUydz6saMNJBIE/nRyei2PksjvTa7xEtC8PHMl + + RHN0kVPDP0snjjIsq/H5u9ReItaJNdMuglFtUGTPt+KKWjGI08i5FM0CgYEAtZhm + + 45pdRtyDQ2yqkWAW0htUF6pVpMgyip6iAgethd1tBNUS0t+Zn97o2p3kFd/UnyJF + + jAHEMJeHeg7i2DchHq5YBkp00ETprWpNiMZwEV4AkONAqhDNtPZTMPaSda/pm4uT + + asBW5jyimXQYs6tE+ssmmyUTWHIKAZupIQuBo7ECgYEA69wKzjiZRcbA/X3RVZuR + + tJGu9LuDV0RWZ9bHBWw71EzSK7PNLxzs2LQQKEshY/ujgdfBKhRrIsPFrU2aUzim + + 3p7XYKPPkIRMSgmNcpkMKcuUmCv01/yUEWxfcVCX4tux0arJ2DaGNafrEoWI28jU + + 2Md0QZWUGUHlzp0CMljO5NUCgYBTtVboYAXTXl7bu8G8lbCvVY2kAw7LkMVLhOhl + + Syi/5lwUuCufLRdhzJ1F+TZkpvMaD/BDI6VOSOtYZnhG9tK7k95buAK05q9ZEwF+ + + pQqP1ucn4rmyK2DHpCyhC2hj+50R6Hsh4FuuchD577xbRf3cJb08ExEh2h+mshx6 + + cRVnYQKBgH4AkaL2yHqbeoK8G7q2uaztH0qSL3wavov6fu0AiQR/izujdZU8RXgw + + 034LgVHK1oi7pD8ANLWU1u1QmqluELg4npGerVJr9XqELLeDlkgElOy0eKzSm9j0 + + oslhbAEFxzYGJqE4py8GPkI2Axxf1BjEdJjHIerOPk9VcmikLnmV + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 4096 + seed: Yw== + zfec: + required: 1 + segmentSize: 131072 + total: 3 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:MDMF:i2mcgpkadj7etqz4mbsa6kucsq:tg2gfrsc5w3fb6hr3e3aprf3mt26azhywaafu33nqfxutq374m3a + format: + kind: ssk + params: + format: mdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEpAIBAAKCAQEAqrBi/bpPWjUv1YLoTEr/CKKvYMzIu71EUVrrNwJ/FfNJlJmj + + T1XB0KYTbzUGUqWW7lFMAm0UfOYaz/IQjLbFpWDJh15g7HavTvsiT0p5yeCuM5Hm + + SRC2VY0Bmr5euc7dG8UWIsjqa0bg3LVc42dgAtX32S9tIEOF5Ftgk45CXMyGevgt + + Xjno4mSYHJJKjO2vYnttbMl96KSE2N8KSTvASHfSQE9WP50m5hMsAhYj9u5s/E8U + + zO/fKkddFEW6qPbk8CSndeZ6nFCert0FyxTIJxiv5qsHQxui3GEsSY1U/HJo9HkO + + CbtzsNfR0mMTzF6i/hPkNDfemBSNs8p873auZwIDAQABAoIBABWBov9oNa5ejDfh + + Riayvl6WrPVL6DDrgIulooRsXpnj7Q35q7+HxSNmgYVeD31jWtiNSr/1gYLZNWCl + + Fdu8/btALjRNunWg4KbZcrG95wl+M0TRKcxj/C1cVmrqeKH9xBNHKmpYmVzJ8fQt + + L9aBRHInBpMJbD0H9PtYXhtJbegmMiFUIKKr4oyj4WiIeYn35k8rQTRI7E/kVGVG + + vx1db0Af5FTZ0zA7YuhpE9XA92g77vnWBb+OfbW6FVJ1BtTGYSEM0G/E9CbBSOi3 + + LD1zIpJL7Oqqt0vm8wSzfzxAZ0R8+xAJU92RCS5IwHRsgApB72w4DwuDy2bLW+HF + + W5qQ5LECgYEA4zgWVUqP7ChgLSeVs3CjQOodfKYIrn7a07W6qPGe4fPhTIz2dOa8 + + +nCT6/xrcY+jVHjpG2p+uvypJuTp1RZ9YgreUnmf+ORgjFfaJyn3vJVMVHotdyJP + + MM5hKknaB7rBykKCQhwYvehOlIzGD5NIKslkXZAX1sdU1X+ZjQjCArECgYEAwE86 + + gt+sPleCnES+RhaWmFoXZi4gM2KjMaNE+3s3mLKk3LNTq7I8qgYbXoCgmcgWPwJx + + 8DiQ8PS4VFVTZKvnxfiLV9XGMP9+HkaT2jkm11k8KN85p2wsMrJ7NlKce89E1ubH + + kWJQq7qOZM9Hl13Xb8iwuhJD3x9UyatnAjnOmJcCgYAZ5E5HMdPsqT0saBJa/D7e + + Ks9pYNIkcDgnX9IBZmcggFXwDzAWaiSmtSVmAsGLkz6dZZnKkfwW+qubzwIGUiW/ + + glWLOGjOR9fopiopxFKCntCv36xGoxY7DYls9DVwJAvpLGMDfYgkO9CYhOIc7D+R + + AJn7P2w4AUbdfUjWFWVmQQKBgQCEMLHerlOe0taUBmjokrRX6220LjayO7ZD86AC + + YdN4oivTDW2RU0aB9QqxLie3LaOlElAxuSBgkUd3qONXCxeZrNxTtz2yBp2xv//3 + + /Fsnok5JJhBidmf3PVqWn7izHmmKcz5xQCyFrwocX6MteDMTwtdAQDfpUocczTZU + + gFnz5wKBgQC4GpegOkRD22AOC2xc34DaMx3djUjmGWj39+aZNf4zWE6odu4oVYrJ + + QIQT8R9TnujWTtTUiNjsH23JpOSca3TRIqrMBKu9kyQX+t7YY6sKxJdYxOxbfOIn + + AxKOgRDY0Pq8Cvarm2239hPgcjC5l9eynj6jRfU1Z/D1yU6id6ouYw== + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 4096 + seed: Yw== + zfec: + required: 1 + segmentSize: 131072 + total: 3 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:CHK:annnqmu72p7iels5loqwlxt2zq:s7wv3jfi2hlpcvp4dnloc4eex6vre42kwiel46achaie5n5uodqa:1:3:131071 + format: + kind: chk + params: null + sample: + length: 131071 + seed: LCa0a2j/xo/5m0U8HTBBNBNCLXBkg7+g+YpeiGJm564= + zfec: + required: 1 + segmentSize: 131072 + total: 3 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:SSK:7utfrlscj2ouzmjl6ktz27z3si:vvxvzep77g4qvrqnaa6g7vcsvc54n25cc6fjuahrymuyi6lzv4pa + format: + kind: ssk + params: + format: sdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEowIBAAKCAQEAuBpUR6IyfX4Vi7ggWVjSMQMNw7PdlEgxamEeCuoQBNDEJB50 + + IlxZId1c12tu8fESJ6ix82aNSvXLaPWwIJI1UODmj8TRlvlafxrqeSlixi+1onwL + + Wdh99r6fpM7mf0gKfz2BeDh8ZM/9o9MNhwQkxLv9aP+tBk9xXZrfBQ+i6ZMCskr/ + + ZoL37k6H7psMLVIaCOuik2mS0k15FogSSsB444jSr7A/gospWdj6hpiXLLu+93+n + + Y6Qv4VzTDTN6HIb9ApYbUy4VaUEc6hXCP0yoGWCegaqPclNHuktTphuhrmfKJ3T9 + + OK6bvcp01UaJdh7ZAs7JaosNY+v3XnhwVADiFwIDAQABAoIBAACuL6jXQmheeEhY + + IIJ64g7YvkHHsVQGnJC23wfElPncJs6R8D5e1pN+iU4XXw6HJFjSLSWeyg1SzM5x + + JN9tX3qkr9xQ4WdsOcnlwe0W2pFjFsDWDYdCHzfqdAOX/ZfzjEgZG09Ry0R4+SzL + + L0BB2gU4+hKSoQyT/ih7ekNjvVVUFc0RR/OgQcUoEPmW2AFh8qV7R7kOmz4ErG+l + + /g0vhJQ9hOYzu80NXyam2QnFcyAhgS6yhwQxYsy6dahpxS8eNKMDknKmRRXu14yO + + NU2ZGwH12pLzITkucOAN1S3zWNqhrh13ADx4EjTQ5ip5zpIaQLUFpW/x5OEQsUQe + + RbYC8ZUCgYEAzR/sKkR9pQnMGrMAzFj5zgssWqwEwzOtwaTL0/iKoMkMuAo8AfX1 + + ac6V7EvDdE5tbJ2qWTKeHA3X9oeio15cHWam4pdMVRGTNhu+XOilHydZbJWnwYLk + + 8eFV2vCfB8H+y7fwVyFBBwoFZgHhWLeHX/OTCHIhV8xhf2nJUjI0TEsCgYEA5cOo + + He6KOjjbrC5ug4jmfyzbhqkQskWN8r6DQrz1/52qPAHeiF0RV/EHcOzvhGWaxJYV + + BdymNTmKCi3Xn/oTRs3EJ6ojG2Z6IbliV0Zea6jesaOMPzmLHTt8NThKJr4FTsJJ + + s7ZR6m6BgooLOlo4ZpMzarxFgdid0oWq2BNeCeUCgYEAzK+/JUplKmwFXNskv8VF + + uSKTJwOiWPtXtvTwZFwOUXVuGLQ1vyslsmhwWHQd3RBpxsnp88o71fjGeX5Nf8Io + + HzqQ62lYxUadZI/4vJN2Ogk1BdKsrMAmH2vhFXGo77/Ytoac8QUA87o/OtRDfxjc + + oJXZMcNZnFgZLmBsgXYRk9MCgYBNBiRLtHXeQsVRmVcu/SvYIl+NawvP14VYhQlX + + zCTjhiVVbIL/T8PKqWCHOMaqqa0SjgWKK4gEe7+M3gVU+e6QY9aIPX77ZoU23QDc + + pRhuGvRctKkFYPMD37cp2C7zgewhlPxEJLCdWGJOMpzE+Q3DRUGNXIQonUd7FZhK + + S2PRCQKBgBStiNUaXxe2p8waFGS7s7aXsJHnffWeRXFCpC3KyUmDxU8StujRkT4l + + i5UPPhhu+uyN2sGSVv6I2T3qYPMMMVAgAoc1TAK2louq3YKo0al5ZJVhL7A+gFUu + + 6VqV5cEc46hydfQS8oAlC+wN9R41CEMZVM5o5eSQUxQJX7EYd5CK + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 131071 + seed: LCa0a2j/xo/5m0U8HTBBNBNCLXBkg7+g+YpeiGJm564= + zfec: + required: 1 + segmentSize: 131072 + total: 3 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:MDMF:l3zgxf6kuww2j6ncsc2xocsdrm:xxuxnhpkg7jqnhf6jujj3q2qeqagnoipyhvpnohvwyk4dden7hlq + format: + kind: ssk + params: + format: mdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEpAIBAAKCAQEAs9bfSRdiJEUd/MltLPZn8tsG07oAfolJq1sdJhPCmDhHUBt+ + + m2+meR1HGVeaywpcqzdqOoTtv0x/nTl7hDtZqW2N3ropb1XTnaPvg++raFzJOfdm + + +dIBvruf66X+217t/p/aJ5if2tgZNgoMHw/mTjLfPM1xDAbO4cj4YHWiNhtTIZeO + + vtbrWUWFRy+2fxG9DdKbGhmNw0I97Fk6Cq4JayhYulHIOBx6lQqfy5hCUkYKOOiB + + sBqSbXlifnQ0Ez4tRghu5NakvQVJ9cWfm2p/N0nqsZ876Yytku/7tjHbV8E85Q3j + + 1X7l0aS/qIpf6Gd0GGyPznWWY0kPqsoqV3sfMQIDAQABAoIBAArmc++t8Wah4tSq + + z8l5IOlNIb+NB1EkGJlAg0aGzZVk2duu4vBgZs5x+hh84Ra76Mx+5hsoafGdnSGG + + NaiY4VEd4Qq2LWNAaDxmjpKoYPMJJrAzAOSU+EKbhDCoBce9m/7CKRqby1qcHQET + + wFLUp6CnQDUi/Z5dPkZcpENSdfPB/7Df4NocNjpMBMvvKuP/MK8ecuGNWpy/Qn+X + + LaTkJ2NOEBslz9cpZXNY9o26Z8ty7duSITwzoFpc2USFS1v4xHvpk53+PgPxcyU6 + + WHQsx8Yh1oME+KG+A37TXxTMgDrlCsPtWIiJEiM3AzK4Wb49ehPwu3Cw0S/iYKUG + + svx3ZIECgYEAzOKipuw141V7k83cLJdDdlNqUJx79wE9ADwTey0z3VwNh9xxi/7v + + b6Nb35F0/IN7DVywqElkrIvM5+pJKv4mISFt3TPVT0oEumdN+HpQEn3psVyg3pNp + + fD9Bhinbc2zLvGZ3n4fAFDfXysjjCyqs+orR8ksKen5UH2uVtMwGUyECgYEA4LSh + + 8pHvpq7ZCLGr6C5WgXvmuFc44TQyXs7TOVvGO+In8+VwCJpYz6tT+8eur7hJVScb + + Q2lUXrA2tZ7pIUPf1hetoJ9fM1ACeOyTrPXgWFUOhyk8MRysat9UsLtczOnD6mpP + + quUf+kv8t6qHrHrh8bx4IK8tcOMZiuTYEs0WWhECgYB3GqDXTKWe/EiUia2etmhf + + VuqM5gsicjPV+RaSGpr16ddrzXism4zxZxO3icVqLbzQ7bs8eT3vGG4Lu6TBO3FK + + /TXyy3kLWMoa2ob3FZOKzGuX0XMrMKK3ucYLijWqiep+IUsVEENW/YeSuOlTyoE4 + + PI8DvR/gSaP5h/9FVP2wQQKBgQCJOebY84Suf4MtiwuX3IyZwOfy1dl3tt+4BIj8 + + M27JbWDG0uxrZI8uK8w7LAQjbeDi7uH4dh+/P8/5dJWc6g2NeqJfQFTsSkVoQdoh + + u3qJl1Aq/OS0fXVSQxc+Yv3WakBqLQiALjMsMTGhnLQEgnrvnRCjrTeMBDS6HO1T + + 9glbcQKBgQCsrNn9xBLI/RlHE1tD/5bLSHJRsnuc+5ek9JKh7AdioFHYPTVJ9nfT + + uDf/8f6acF0fNMcC1ZbqcTd9LMO0dH/yCP55QKtPPNVmPdx5ilEJ5m6yt+qwTRbo + + np9zZOmcaT5orVWs26R/aN+zwYoAMeANGp/P1RBCoTpo7s0NctfiNQ== + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 131071 + seed: LCa0a2j/xo/5m0U8HTBBNBNCLXBkg7+g+YpeiGJm564= + zfec: + required: 1 + segmentSize: 131072 + total: 3 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:CHK:rhkln7unkktot72mit5dmuqbdy:ilo6u6hugipdimyrzrvlam47xsmp3ur2lwnrtbecmvocb2664zxq:1:3:131073 + format: + kind: chk + params: null + sample: + length: 131073 + seed: /N4rLtula/QIYB+3If6bXDONEO5CnqBPrlURto+/j7k= + zfec: + required: 1 + segmentSize: 131072 + total: 3 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:SSK:tq5socaiwunkijgdq7j5ggkhj4:6s76nfeflnfuzjsh4jf7inwtjmil43xbskvjd5ykfuhp6cwqma5a + format: + kind: ssk + params: + format: sdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEpQIBAAKCAQEAvy7SCw1WfwAdhbT7BIR0VN8omAinI3vlgeuSGEtuPVVjNNCa + + l33uacBw1wtmul+h3aJjWThEs7TxRyEHhyIJCq/9hE7uwuHyUWcR8esOYFJqEa1U + + fDeSAvTX+hWZSC7Bge3dSQubZXCC6Xa2VRkdBCauWxszzHvITWIXb1QbkyYPEg0X + + bDoSoMbw9tfFys2GrVTM1RZX62qagKDwhlOCQlMDGvPqDRlZ+qBddvOKpQq80C5e + + /SDhlcFoYaz7Z+mV5fZ+kjSUp7n36J/Fha8uYck/FJm1PhY90ND6vRuiKEdZYXfm + + OGJzxK9Wk/hLtZptgDqa6BG898oRqo2kJdXY9QIDAQABAoIBAAL2nHFKQLH843Mj + + KmwBCxns9CNYQlmstjZnne5GRYSLVi1rLKa7or+UHpNQpr7rEsFRUfxhGjx4XmsI + + cfggL0SdClMGvcg8CogwHg8b8ZWuldJr9iPR4RYAXYJ7LUODYWBYuofNS/7IYaXN + + qtUThVrL+iGs3MPRS5jQQ1u6QjIekfJc0yEGCvg8huj2grpzZV/XMBaOyFVjr8aA + + m68odNuEcQwRvv705NhDyOAMJnkcZJbK/H7n+OCY4M6EDuvNpn/n8N9s3yMQ7r0k + + uo4uql56cIc2LxYFUAGQ5HIaoEbceBk5JZCCJVCh/IWGqDJSTgGHn3x5ip+R+jC8 + + hcaFuesCgYEA9L5cJRCPk7Zz9p7lwYrnMJlSjXnIo9855WRwjK+9G290zxQczh1Y + + twjeSdziaVXR3t0GGHYBF6Gr6wV/dxLHZGPlccfYXnIQTmku6SiBz6B6YBrWLTmz + + y4mFR0D51WVBdFLskG9r/hoUQ6UYKOV3YxxSXu9BTeiHOgnm+m7l0Q8CgYEAx/nU + + m9Mh4tvf4cJ56Mo2q/R8habcAyWAnp+MJr+qCiZip7kkKxtnWlmLebRUkZYIAbbN + + JTat+UygnCw2oWeFyIAJBWjzcwano8NThY7LGOax0JU2aNocu5gzz/aaN3EyqZVp + + Cgk/qyoF9EPYOielRii6KD1nulaJCbL4GHFhrbsCgYEAprs0fQ+uMHxAvgd8EIE3 + + hNU+9yC7PmBpycvGHSHwG8uvcQ+LnCND99Wz0fAH0qjjhAdhCrMBhX7fZwnkz1Lc + + wZiIjB4QWi8syq4/hhnRbYgvNl+x/zdrNEMop+UtDmKf18ZSYQd3M7HCkl7beajx + + z3RQ7VnjTFcYIML0NzHroKMCgYEAxiE9CPZyyHXYp7ErX/2ZlV0yUqkzqtppSMAC + + +BFFw7CsZkkFEMCh8d5uVjLY5zWi0S/wqUI3tJy7NICJz/jlj/Vq+rU1H24kghhw + + lA8aIp3O5z4vHkub1DHEg/NscCnzbBngbFUlg8yrAYyGm3fURGLtrhjIwNIkDDwJ + + mw4bHSkCgYEAiKmrzboctKbD6gr2t6vZ1tXCevxDp4IOIO6u6qoMt48Gggw5oZYw + + NU2uRB3KatsKLC74uDs6zPTc+jap23T8185V0w097Xn0gA5FXIDINklp4dyq6Erp + + yjE/73oP2P55us18FAG3WxsR626Mes2mDVbB9t5e5/sVtNSAJMzQ2fM= + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 131073 + seed: /N4rLtula/QIYB+3If6bXDONEO5CnqBPrlURto+/j7k= + zfec: + required: 1 + segmentSize: 131072 + total: 3 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:MDMF:omvs34f76c7mxp6moabdea6gte:5yjko35bt3ks3eeosni22fyshwe75p63ilzso4m3x4f3w5ay6qoq + format: + kind: ssk + params: + format: mdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEowIBAAKCAQEA9ext47GMBnIkW0mbRLDHZlltA89uH+UMkyGyIKEeyA+/XUNP + + djLlthN1f/3/A4yqgf5SGzxAuZ2QtqDaYCqmHQ5ad/RnFp8QV2hAvEjp/tJc05ZW + + NsAZg9f9IUSqOPlzwGDHDQGtVofowkP1Wuo/pnkEI57uqdbQoPKtznQ80KcFBLDg + + KGc5gu159mY8q3WtzEhCUr4l9yueJgeKq3Er9tb1jXKW/WqZHfaejmt/HE952MU2 + + TyaOGWxHfudEh/W5U6+TKL6Q7En6042IV2x6xz4SD+k7F9eO9kSMSw/+Tazk9rbF + + g8NahhzvjzqJhNEz8/AaXxsaQ5mJAXRHtBzFJQIDAQABAoIBAAMbnvmi2GmcCqwu + + qzBvekuBImntNqte/F5Uk+smHQ35M6QVUx5SyeCX0vZa9lyUnzbi4ownGZ2jK5CV + + 9PQEgbJzkwTq3lym26HHJ89ZWGpdCgUUys0iZged0okoqcIwXroV+J4D5WCaJDyI + + iN1tGsTBc6IKJNoROfQ2wkFa2XC0YqSZ5y25kowk1zwd2S43XXWpejTLFGeHkeSx + + WfPPIVZjPnokg/LFjZm8WWUxGE7L6ZDSseV6PUDfnBKkRVSVHHkulrEOowe920bj + + 25pm4PytfpM4W7s1o9+Z9b0qpFmDIX9MFxdcPXRZH5oibNiEK2Varxo7OS/SXBgW + + BupkfoECgYEA+ggnF4eR+42vOx3hsRsg09nKOkfmzu+8gM/oHWJ4TL2jvq3+2duV + + zXuEDc7hc4J1Wq9NZ9MV/t2gxuOVp6W6vzVGHMiGmTAixu2S9IcYeRe9x2K4p3nx + + ULjg2FRDRytvUPofpE/Y6hHldcZjBtsF20kOokuyBYhHG0K4AsGGruUCgYEA+8ss + + H/gK+IACIjywcocrEdKXyd9bOELhW6vPS9/IiUe2ZLcbjshejk3NiCSNrBA6V+PU + + 3jnV/2jHeEldeE/fd/kDaJOCPGv6lbN+UHmk0R69GCVVai1zTBWmjAq2CIgfiAwM + + fs+gYFf4+OAbpo58i3mOUPCpG948JgrRJI8eGUECgYEAzkS/i0fKhQ5kC48hS+yn + + bl5z2RTMMtfQWSwrv2InAJhKZ9o/LxdaRESrsoCDublce023u/mGYdYQ90N1iPLO + + V0Pp7YD4mZP+fMItxBFXfT66z6x/zZpqHEAJLi6Fukb49IMEa5d7yc6t0DW0KEm0 + + US26JuXvnWTJ1JF8ILnrFIkCgYAL7usecMEEWfy/5qRuKR3PcG2lMaK/HdxUXeYr + + MGXuq6lnSI5TzAc/M0zEYQcd2n8JX1DdX1xXCH47oy583zw2EWUp9aO8fVmY8rLP + + 2ZQIHS7VEB/mMlU+i+AizvclnF3yMq/86pYtOr4f/W8SC7q3WYF3MJCzM2siWmzj + + EK1agQKBgAsKaCJYbtgpbPJ39EF93XbsN1m/y7yDdyPUvQbim1u36mAWJg0QwJY+ + + BmtEHqZ+U015m/5dkHNKNgYPHPE5BedaPIGWkPDU9CpQ0g61J2itSfJhqMSNImJr + + RuH95SDlk+Z8/MW7iJ8S37SgJcja1dlTqiQpd9O2ubyYh+BNSZyZ + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 131073 + seed: /N4rLtula/QIYB+3If6bXDONEO5CnqBPrlURto+/j7k= + zfec: + required: 1 + segmentSize: 131072 + total: 3 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:CHK:7yjcwwt6454lbv3pni5mjofsxe:y5nwpzwmvpvr3gqxnykjixprpxw3w6qyqmszf7ijyxnm3wl6f5oa:1:3:2097151 + format: + kind: chk + params: null + sample: + length: 2097151 + seed: uqWglk0zIPvAxqkiFARTyFE+okq4/QV3A0gEqWckgJY= + zfec: + required: 1 + segmentSize: 131072 + total: 3 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:SSK:c2hklyqlte3ar7norf5nrvowaa:2pd6rxd6hnnkyifcxce4pf25g5fzcrszxr4r2vcu3vhest7m5sza + format: + kind: ssk + params: + format: sdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEowIBAAKCAQEAoOFuH+ZLysnPs7kK7pVWeUJ0C71ftfcmncL5FgGQV1MMsupz + + gWAXlrsUT5toD8s7h8z63S+R6e6+I4EYxc4DoH0icTkC6wIBXa5ODpJaMQkFa+5M + + kDu4YvO6QUfGyC2J4YE0wySXNTMSnYOEmC7Cfouo0uVzvxR95bP0VWgXJLxMZ1u3 + + N5nNP94syH56Yy0Aza5VjlhwmB/HdqXBiV/wQfjniNjWgkz3yxl8U/FIkDHEtrD0 + + svGzoL2AaNPKlfkDM+SnaB6tYfozZw7xP64sLy3wOVkWjfc9JEWxo+Gco4nMK6pu + + uqcq+vX2m3fAAetw2brletd0zpNhqw+9WtjiZQIDAQABAoIBAC9FDU5iJDLZSSXN + + YODpEBdg5yfr5ItaqwX/m6BTpU2DIWAQcw+4ZDXtkfIx/0lktYEZQTxsFbteYo+c + + BuNXvMkS+2O5FJpoZG5aIKU3azitJeKoieZ3JZ4tbrRvmoCGoNSZWh9cSPFgqD+P + + vQ3Z71uvPVN6B6BFLRio30mY4/Pux72huREGa2EvfOhhQm9/euIW1JqvMcg8MIti + + WZavUcLxdpPTsiNKVIEEJUtoCK9xbbXuEmKKuakJfAP3epcPgBxIJBBo6mrPxIkc + + VVSgNRNGACOrILLxEvV4Oy1fynh/VJD8fZyA2vppcaQERn6R89z12zGvanXGwIu3 + + CVRcP6ECgYEAtvCNZ+RSXocG1Piuds6SiATFFbvQYoayuxwlrkqaIvD8M9V6Yj7x + + 0aXY7RMHF70hjGue+ZvR0B9hBqLGpwddLbxx/UcIv4sZGVg2YF+v5Ar8Ge/pdW9L + + i8gbbdbapWzErQJtX6BYyQMh5KkeRARWfove/JY4nvXgUzYcSeSogsMCgYEA4SGY + + TE+QOmZgdjtBxyjx+LX090Rjg+sqI8HyV55gA1C7RcOcfK3CbIJOMhddbT3samVK + + PTxjQ0hP9v00MN78hdyW7XIQJfIvEdtbnigivUzILgI9ejgQ+NnWBZ0706/i1Y7m + + t1hwUToHmV9JSObQyvd5hyhVtp1xWblzOXPGY7cCgYA382KMP9yhZJLGWDijxZIz + + X6IXf5XATIolh/pOUCrMPQAlqkj/+1hiUmMCPyuQKxwzokbA+NM24CIAsZAoTaxF + + 7LjAShV238gRZFVdLGbTTDjGhgXVEPD+E3mwImJE7ftJHtDsylHdSMP493B2RQ1f + + LtBIWHmAxJqTWJ1WTETtmQKBgHCKM7DKASZAcS4JNzuQy0zx4JAO3tReJUWUuUl1 + + gTeHDuaz/zEQR2WoyeAeb/ShBOK22aK84j4LEvY74vAfOArOl6AA6fOeGkuJ5UWt + + eJg6nsLpGcRT7KAJfQR3ciXDAdiRw+GZUyQ3pv7TdDX+NBeSGG0pC5frInOg0enB + + Z0YHAoGBAJswQoQ6cOsi3/nCIjRG4uqPaLCg/g+qJsShbfW3u+9j3sFwffaz6kYj + + ggqC2HYlerbj+7O54rlK8A72yHfw01R1cD3BbrkPG8qKNhn9llmhRC1xlHV7j1tI + + j4R4oV0et4AAO1wbH83sESajyIYdpe+9JjqL5al//PXctNwo59OM + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 2097151 + seed: uqWglk0zIPvAxqkiFARTyFE+okq4/QV3A0gEqWckgJY= + zfec: + required: 1 + segmentSize: 131072 + total: 3 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:MDMF:vssudzh5hmbquhrlqnzyrl7xgm:fkkgisy5ozu5kfyuzfd3ookqkqj2citrlbbt4kgdsmwa5znlrhxa + format: + kind: ssk + params: + format: mdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEpAIBAAKCAQEAyKkGhkde2GQ68zuzyMAmrhiOyFo9uu7/JdhVy/XLqgWaCCj1 + + d6ciBMJj//02/GSm8byqB/I/g0+hqJVRTDTICMkbFoIRR8el0Qi6AzOlBrvS4ztX + + BFxgotuwuvXP+8F+Am39G1ei1bcQqSyDRQhSBa+CLc6d53kOBEy7TUIm/yFYIlFq + + uR7jEvUyqgbPz/2vIGPTvb4cG/04RxzGriYvzaiL46SZETkRJ2L+WEpBdf3w1/6V + + fKVRJBRVmt4ifoBV7op77qMl4bAP01ZQiG152zIF8COSWxtPCgyV0TTEmOj0g/5b + + TsglYWnTCMAClcjuKoNh24ROmrfX2VrHqMsEMQIDAQABAoIBABVhANt0pjPK9gbt + + QPnuExDwd+H7z2DnztJy6q046nKady9QYdrWOUcliO8AxQd+F9VgowMGueKdLN2f + + zxId+4QIHTU4NWwe5tlPIzZtHbOKdm0UaPCDgR5I5tr8jqTFmE3c9x8fJq+7efB0 + + WCYWPVryuJ11ypgbazVlEX2pQytiayLrYfx/FW4xDlBjffq3cijqlMHV7LdVxWqa + + NFUI1oEGvJEky4kns8kTENwV4Cy33vTNSsNS9dRvUgIfY2fdjFT++GRz1HYtfWVk + + F+vSKr8Xd75fbKLiu3dK+5Z8ba9KapVqIucwKn0YQqHuZC9p0HtNbN7a7476cdvF + + jMj2fyECgYEA0N8aO5QHktxURBL78hwx08WgrFuNaLafc/i2wGTwS7LqhhyeStLX + + sfTm3x7U6IcIkpb9l69B7/nUA5zdghvS1sxn6r3seh35GgktlaapomAbGYyClPVF + + N0SHR4GGgcYqjK2/cYDXgJneY+h/Ph0M737wjGfxHt13cdN/pkrnQwkCgYEA9e+f + + fnz8/e5r00ct1EEBxkKOTDJD0ZQYBj/bcMPhA7s54qiZbZL4wYR4DgYfFoPKZIRY + + SKoACQfkdodwFMy+54Lzfu9VyrnoukDRzwgJvMfV+xQeafbTRnYJew9ICb/EVNCg + + 1ClNSOkZwHlvYVwxMTS7zGda21gPspX2rjnSOekCgYAodO5J1/RXl+GiheLTFG76 + + S+9BM0KCo8zi06viPCrnHrKaY3StnYU17O/DC9/FYlJgwmpANSwaZVORl5K4HteJ + + z3HZYAwr4x5a0qhHsk5tKxxUqIiqfY94kwd47De3b0DSmtzYCVK0kBkpVOFAkLPu + + t7G0IHXtuovmOkchWKTOsQKBgQDc1a6CBfmmitCHhwK/9R+Cx4C/KuN67WAlPHHv + + b/Q9RYFU5c/fdHmqSykCbry7mtvCJpSfqwcdFNkxFayvAKrrd8rt0DtZLlar6Eh9 + + fto/ibG7IvWscNaGDre0qKQnHOtOvYes+ulK7wUQr/ozknUZmiCICsaq7wgpdD9t + + cr4zAQKBgQCLZCdKGTbqbgzTFzG98anWEta8soj3EA14qrkapMgwm/zrCx/Hmgp7 + + k1BxY3CJmB8B4i3ctSLcD412NHPSaZutxGjgVJTw03qfKxscJGivMhTUjrPYTOTD + + jhF611YHyP2NSdUlhHXd5Z/3Ova6PdE26BKiGUqldqzx75/VzHlJ+A== + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 2097151 + seed: uqWglk0zIPvAxqkiFARTyFE+okq4/QV3A0gEqWckgJY= + zfec: + required: 1 + segmentSize: 131072 + total: 3 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:CHK:r7wva6tisq6m5zszgr7seu77cm:oqlgdw3hi72qtahpsi3h3yryxpqdshagvt6xnobsppkwp7ic4cia:1:3:2097153 + format: + kind: chk + params: null + sample: + length: 2097153 + seed: BTBX/ampNfLU+ox7xipBGiaSbgC0kcB8Gy7BkJB4oKI= + zfec: + required: 1 + segmentSize: 131072 + total: 3 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:SSK:xpybgom7doy5akhpaomuipjhge:vwvxgsbm43zib2z4ibgn4o7pht3wh3dazpo7kgtn4fm6pauzr4rq + format: + kind: ssk + params: + format: sdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEowIBAAKCAQEAtct5n8YHDPlic0Drl8bCk1oB8UvQ3S+LgdeQGwFzjew1agnu + + y8vYFrvqioJxWlxm1Vt/ZwrcPORJ0f2RqMLKxxqTc7sMZOmypCVobwxMpCwXK9DG + + kAqxLaGOLqC+JnoZ6GFR1/BSegR9IlSpRkz6QJbIquobSC1QKVwXpPy+mhVp3Efn + + pHFocfbh3HVZy9fvaaFUGCWw6VdTv5qJM5plZLhMojdV0u+A2UDW37rcKiBqIvgj + + Ulhz5Bye/+DkvhOss4yMPvJyO6yz9hhJYXApYxqFa3IXXBc2m2kSIrMO+nsuXC5r + + x47fXtePM9voaxxUUfhej208vk1ErYr3P0IqyQIDAQABAoIBAA47vbxmzaFESsMd + + kXyHAy+0f7uvzQzuRqjOMvIMV2rklCuG1s+VuIfSI2tACoYxvxrkEKnliagKUyXJ + + 6bw58RSs5d/NJKunePU9WQt9vefqLE/BxzQapDPfhuFrdCvQykO9j+H9ZsW3IYF9 + + Oaofh4XkUFaCYQuyAYlVdKP2Jmmrkr9TGMAaeHUtMEn+ttVHW+SaFrjU6ptmDZfQ + + 0/I1ON2yh4LYn5MRk5TZT5k04eZd6eGHRTuLDUmK4ZhNbo6IUPsUVSEJy88K2eSF + + ytdaWx9Ux9Xtt0K3Dmfqyujc75NvtkYWRKlDyNsL4xCa7YTurIs8xzxyVvbARHhv + + dVgqrlECgYEA88FhwHy5kTo9LJHrkkOJdWy2sNMrgOF6fvOC6Dm/MTCBQNaP/cMh + + 8lj6gb1dtlvkoxyICXXqJGJCjdCkZrB2p4Y1A+Mvb6mz6CI6a9e/wHq+2ZHOScYH + + 1xkGNDoEytaKdENoJyf9aFM4PxaRpwG8eYvOcnpT8go0x5hN3UB27tkCgYEAvu1M + + tr7ihrEJJ/BqRNxIgYLVLkJIUI6JeBLy07H8OnzzIIG58e4n8LQ0bJHnWStFnMhH + + I7SKdTh3IaiFfA2VD9WMW4GQijPLItt43dxzDPRP3RUtOexIJ/meANTc3jMUnqQ4 + + KheBC3AKv7qBTGam5SmE7pjSQCplUrVptzKMhXECgYAxcAtPavyIA/PcUkwhAimi + + 80WqX2n3XcPmc6UdTHkGlPviFqJlqWn9KSbFoY6cKc8ZdfPxV0UB1BwDf0mYujmW + + iJXAEBfS4exnLGoE7WEqvLpwji30sIFukti7Rvkp2pGCOxmot2eh/R7vTLiF0shT + + LpPUjBLyiDdkM/O26Bg3IQKBgHN5KB2aw3y9FBGQyWUOadfSnkaVFhGKs7/oje7V + + RfzF13IAo8qbxJJDGzXS5L48eqTBSK1iox8UYJD90IXf3Rivim1Jpna/rotNfAOL + + MhZSqP7IsQrISjfLM/HCzDajZEQyhDmI76ZQRGADV/IyX5xYCSsZSIhAW/my+NYw + + /2YxAoGBAPGyQ2qwKcbAgGGgSXqRRN73vgYfEijr1axCi7TRsv3C88NlUsnH80nu + + nwesTL65aTLb3lgm/h3mxp0byKkNr465zzmiDgzZWOQk+VLDmZmCrgq13RtpbUaN + + OLqlO3RYMxwsqIwNHACz9DNHhBWtul5DfKon2UqlZiYBoZ5U8HZY + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 2097153 + seed: BTBX/ampNfLU+ox7xipBGiaSbgC0kcB8Gy7BkJB4oKI= + zfec: + required: 1 + segmentSize: 131072 + total: 3 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:MDMF:c5xr2pp33zxmelaszjzkwmjc3e:4zbsei7gh5ea735gyq742rrppnjte4jf7cvnolweu2s4tl42piqa + format: + kind: ssk + params: + format: mdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEowIBAAKCAQEAswxWC+ueq16cftv8gT15QlxXK8Qxmkjg8o7cujkbm1uMxpgn + + Ft52fe4hlcnv9l3z2HRDm2E+LoudhOVTPHCyLddoajUQn8vER6P+5uSlS5bNgCqt + + 3k9qRYRq1Fw/1x8fFyEbfQdbuIh1OKIA8G/5HlSPpG9ItAcGdji7Z7Of+jXA8wdR + + 40hswOHIYMaBtyeDH0XN9XLRAPYFJHgJXdOrEd6pO6+2EBLEcvQoCZ5aDvNiAuVG + + JHuxpzkIhXSJV0JiQJLJ+apJdptW1+nr2Kt6Q6+nPZYwMfsuCwuQc/xcwEJVLDwa + + 1rZXd17+jNnCYb5M/exaMemazvhGHZ6g52LyywIDAQABAoIBAEELOjcaYYnf1PpA + + 8HoC2wpAgWpk26Aw2YdEXutH07+cgoeivpCQQHt/BrRjp8jYWL1Jf0XzDaPbFF4y + + 8QoD5rbAii4LGP70B1n/OZqndWUAY6cr2f3o27JlaGm9GXQM2j6MyG+jPK7M48iv + + EahHBTj/fy89PiwoYTCRa4NAvd0nDmBoLaxZuQtdbmBQkCUtfkCUt12VBRt35uFB + + wj06udcPfyBLmWC9qFKo4VFB0KBvEgHPXesy0Q01x964ZUMzz3K/HOt3YiKdAvpI + + sxXNSvqCKseDA6rLQhAbR2CPFXUmb2xfl6v82FwOL6T73kLjQiXiy2g495M/g82I + + hep+OAECgYEA2N4xlve/msvdBIRT4WatyDE5sBIt4h0Zwu2SMnYBi9eYi0sgbiC1 + + EXx+UfhdeKjMOke4AQFRjDrdflnd2jVnVjVI2rsdI1tjX91h4LdMjAJZd0I+WYCR + + y3hj+8v2+TSg1L7KcerbtoJcg9irOVvqbJ8DI4S/F6tgccqEZNTnQ38CgYEA01sh + + RxYfILK2YXoM8aEEXCemWpggFkhdnBguZ268Mk4hZ7+ViuYsDaBA+epWiaIG9PgQ + + tr6C9fBpw8GbdV+Fg18W12N+jQ1231SOb/azOS6dWzuJ9HritYwlQ38taQFaOggf + + CSQQQmjlc0WbO0XPsXoKZ2sN9I5bBbuSdKRUxrUCgYBYKt1WVxrawA73CyVe+fOk + + 8/5UCtAEoXgbu6I4SamPRPOLjdt9amay2T4x7RtzNozxFL9GCVcx/6yU9cwwLo34 + + imk4I+JQwZLBIqvsRBkmwr3EsnXOxWqAok1jzSR3ZGIOnBKKBcWViaI7KBdUln3T + + 80G/avSVluL64C67H6N12QKBgE8Bd7UM7eHZLBfP+dqw5+JS5/phd00dC/D3kREU + + 8cCUOCSCFzJuy/Tj/KXvFR4ptRQJTqYhHO82STLlwmjjphLvjqhBBuNPLypYf04X + + F/O+GxApd24uKWTX2G4csirYWJPsyT0vf+xzLaIjWN2VQQgEqLLz76mFNT01Wo/D + + hfUpAoGBAM96jRnTD8ohGUoQ4sZueqPR6zUlEzteKVCbeTtE1gHZAz41aunWeOLA + + QCPU/+g/F528QPN1noc3Mlm9IobfkA1nPL1nKNqjfT1ymif38FXYpy7qRWxHz6Te + + YPuPFms7f35CqG7w4Qqt9kBWstI2kXOSJQXH/Fm92PkU5eYGEz4U + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 2097153 + seed: BTBX/ampNfLU+ox7xipBGiaSbgC0kcB8Gy7BkJB4oKI= + zfec: + required: 1 + segmentSize: 131072 + total: 3 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:CHK:mz7pzh24u6flkfmfvvokdwmriu:5mzo57de2ywsdqkdvm7557gcikkpjqbmj65mk235t2xyoejtpbta:1:3:8388607 + format: + kind: chk + params: null + sample: + length: 8388607 + seed: w6uP8Tcg6K2QR905Rms8iXTlksL6OD1KOWBxTK7wxPI= + zfec: + required: 1 + segmentSize: 131072 + total: 3 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:SSK:uhmjy5p7og2gg47j7aqgyevfcm:wxz4blxknv2cbocdqh3tpvh4s64rfy7npvw4q3lptmgtxb2kzpbq + format: + kind: ssk + params: + format: sdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEowIBAAKCAQEAzOwnD57Qk5sgnHKTlvoqACw5UfQNychMYSse/i6zOVKkTbAb + + hIx58SmbcP2iJes2V0DVmGbkGrvRBUqPwNBMOQ2B8BWl5W//lEK4hdxtjant0VUa + + KA6spt51UbcXQX0wiVE1+4DUNLZ9mgFONIMXgqlvVFe135hAb6ULW//afG4TsCkP + + /lsADBhu+GAhGsYytnY4A/RDw4E9k3nN9NO9sCvq0y8tleGLSDRW6UkQd4uXMTwL + + HHci/ll7v0pTFNjT3j7VgvXHU7ZYBTMAo+mLbmtUbtF87FUf3qi3a/Zy8sirkw0/ + + PJpMaoyeRokP1dcr/OtdAW3DQaA2zSAg9ObGowIDAQABAoIBAFHA/SxsLcZVo0MH + + Kv6Wu17qRcv+U+nmsSIq8+hwdSwvXkFoOvI8oQGnmc4QQjpihoF06kIs+l/4AkHc + + J1HDSEWSr/46hL7uWcaqf7dX45Ua8DgNfavxfsvsAF4jb3G/IjgGYEUAdqi5DY79 + + alfk3OJR+oppm7OiqEJiVA/WGTJ+eO6NZt6MjAyS9TORTf9S+EC4aX/YbsMQZSMP + + K5Ixn6cnwqdJSX1CPkfVhAQ7KJK5vLFX0DYUicILEo6Llbh++6iWInWimON+toPL + + nMtH2qAj9ZQTVYr8Br4Xc1upAkNr9epxj0eip4GYyFl8nY0YxFVHzp7HdXQCPHDy + + nvaoGoECgYEA2KX4+QrMSzDn6Iy2gZG13bHNPNXnje4p5hvqclmP170PTxSath0z + + XA5m318t9pXE45oTRdti6+B1d14pQuzeHfgMq7F/vMqw6Oa2xSDcZjtzzHsPEPLA + + cYtXdXJSvgwv5X+j+dFIhPRXKW1Md2Vr3C5ag/QKNduBBU8f6URm7MMCgYEA8iTv + + HwAOg7uS46+lxQyMvi2tt8tk+0b8XDEUgvkmn8dKzhSWEJmZ18o0brDUB2W4D0Ox + + wN5qcB2Cr92h34hMSgEzuD4sA8dHA+aYlPmTsk889Abc5p20Yy/VEmYVOetfZ2kq + + 3Wzd+kGCiHerApkMqAFcbUjwIbiHA7d28jfmoKECgYEAvM+JOKJsgWtR8Z4QwMNY + + mKmIkOhrMYrLATx7CsV7Uy311ZnDa8vvIt96UFoHGMxWF3YELfGROLkaJrntg+Ij + + gkLX6Bp9lO+hVpkb2JlW+9H8jc0ByGeHyG0D/9tuuSqt43lmUyZN6XF5NSWIatX9 + + Nps/T5iz/VQcEaBv00BF4zkCgYAqvbo3jpsBRaq35dks3vo413dCafR5Jh6FZ2Rn + + efMHYPYjSh7y7ynonRiEMVI7vAixKRHHKXtALvVSdZyNCFHu/idS7iZ2xEYUui9U + + nHklkDcCG/QCAPRGTbsedEZq4tEEP9wBGaZU9htEW1skKj/Bp/vYjndUfG3YihnE + + x3k+AQKBgFoLVbsfhKHT4DnW74LWb4QhsdVns8/6DzM31yItJzwHN6sn3M4JsGQq + + 0Ee4oRrLejIvZfSwxmZ/s1R9YzdXkog3d1pEO4UDonKYxEzMYTB6mqJ3YJuj4Qdl + + lGBfFVHS3rukb74jqQC9VO74C2VrsCRA8jX5irtO+NnXKoXO0KQK + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 8388607 + seed: w6uP8Tcg6K2QR905Rms8iXTlksL6OD1KOWBxTK7wxPI= + zfec: + required: 1 + segmentSize: 131072 + total: 3 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:MDMF:nrmxas6u3e43bjtlxsbroc7elu:a2pnhh2akkelw2ywnuwhc4grrahfohcck6adx3fwxikndwemm2mq + format: + kind: ssk + params: + format: mdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEowIBAAKCAQEAsjzS01v9YJCM1/d+W20MYjPQJi/bTRA0bwLaq8W0M0XRsrRE + + n2ifkiAuHaORaKmAh8iSh6fXkN+PjDSF9Tbuq5K6/KK6IwNL5lp/QE0UDaLYbeNR + + lba/jfB8LCBzCp5C9nv9NMZP4xgEhrrUQFEQAt+sMajDWanasNmjj73K3SdQqgkt + + nwSxXdOIJ6HdPhMR0OOj5G3VaObgF8cTAo6F5RVhOiyJhrRT/21WdJI+3F6CwiAb + + wh3HPqBFILwtxm8uvQYf5TSClpjfH8RepQQp7JddxCStZp+U23Bx3Cpp3vM485AU + + ngYfc6gKVDvwFSD2mllCcKigMmXgneWti1CyyQIDAQABAoIBAEbRIdb2isKuTDeW + + zy6WMkBmY8J4a0LAOIUO9kEfiUyB5iKBu242zIfrn0cJcUHLbxUEHSwnBOA74zYK + + vFrEm6mx3/d21EwLCEIbHMo4lcohNKrckdLRTGSh80Q5FFxYqzRx6RXp4V3SciHx + + 41k2nAz4P9tvOUbL1OdFYdY/y3V4enGaFADsfzGXP0pOf1hlwx0/BrpgvcInt37s + + gT19cv5Lu8tQR7AtYFaM37OacMcdNHgjARfgzzm3YcrTVNie4Fcz3WMJi0toYzGR + + LA7AWVmuWyKLFkRf8Q/eYjydWDP4Qo3frfYuKZkHTugmKsOTKJ5UzxoUX7Yy2kot + + WnZKm00CgYEAyeK0Ua4FW+qFjzdJke7v1jrWaHTNflh8KLfO8xqSovXaWHUVknlJ + + Z233iZalKSlynHol3nUDCXIdfwS72wL2rvJ+VMYZsfcYIGH7JlP09quU+bFZ/zlj + + nOl0LmsfbpHc2go9FIkGBEmlYD50tpwV98ucJTYNxLQicapBsAzNYI8CgYEA4gNn + + 9KNLfJkkLiabROZsGR39OhXNTr9uTuZp6WaimkrwzmZ9vhg2QI8Ckt3iqOOn62Fb + + h19GG1HjNwfnlzhZZJtE03CZE5DJ1e8gUfKnelWwC3vMfcCibVqRSXBRygrtY2cu + + /9PqT+NX2qrZ4qd6OdmluxfJh3Vj0rXxMyXNsycCgYB0iN5Zf8AsLJXn85wOFwRu + + fwwgw7uSwPT6dA+LmL0oQA5HnV5UbJqIj5uh2kmAFyLHXGLbpGOaYjrQhSUC6RUI + + K4Xs3WUbq2xL1QMqPrBaavTVpSA0CSaM/t1HpiJAqwX2/o3/epD0jKZfhe3NMxAj + + N27ss+UCtJBlWEgOnXU31QKBgDAS8WXD5iaWnG+EnrpFGPEuw9I7GPSLG3eE4zpW + + LngLQLVmb5CjrcaFpNKAh9nMsscKamGdDlh5To9CCyzLO5h+vmELLkRPI99xgbps + + ltsaptuKdbC57NK91PF+BqenM19Vb1XTSZ+8h89nT/k6DnGHrgzhvmglvBnxwWBT + + xjE5AoGBAKPPRpspLyJE3pwZoskhlrTu1bZmIWoYqEOZJKIqJvgH9bzG125egzHm + + 54miHH0lerkxnPOZcJIYgimVjwwx4THC1wRJ7p1q7Xa1N3GSsk0zF4Bp0h21aS21 + + tunj+RR7TE+8q5u2ZQsnpXg6msfAAxUmYhE4xLKeR01yZXlgFhaZ + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 8388607 + seed: w6uP8Tcg6K2QR905Rms8iXTlksL6OD1KOWBxTK7wxPI= + zfec: + required: 1 + segmentSize: 131072 + total: 3 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:CHK:54sq6lrqwqlpxicg747gdls6ri:2aaxyrdytn7r74my36ek434hlbhe6glrgr2ic5vvpc5bbdxmvo6a:1:3:8388609 + format: + kind: chk + params: null + sample: + length: 8388609 + seed: yPi3JHKKbWaEEG5eZOlM6BHJll0Z3UTdBzz4bPQ7wjg= + zfec: + required: 1 + segmentSize: 131072 + total: 3 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:SSK:m6aebevnsdiwaxdtu575uesj5m:ulkkl3gmdo7ww3gko7obbbweo3k75xvyriwe52cqe3g5nbeotxaq + format: + kind: ssk + params: + format: sdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEowIBAAKCAQEAr0tGN3hBcT0GxrLDM7v8GwM4wuVo0tOv3p5uIpdH9HtHg5i9 + + rz1sy3MTGHrEDtDt/fABSxSwu1R7U2CeILzg/K01Pcn8AekmMHno8DlDds51ZKx6 + + rIpf8kd7N8sKpxMeWpVSM2P/wr3c0TH1KeMwT6xSNrTNVaj5z9ejSMhRd0nlDNCx + + +j5uXsL3ZTGTbiq2lREW3eF3pVouy/7wK/MG3A3XI9mOHpN9UsgJI2j8Wcbn5xVj + + nBYxOxZQ0IMw4R4tRUw7EaIXtDI/lV3/yp6Pq7UTKllkqUPWZ0mnMd63gRAE0kJF + + pIUd3QqxpujOF5jXmsuq4QL5Ei3eTWEfSjiXTwIDAQABAoIBAA9IVG6A7nutOApa + + Csis3/WJmgjr2pq0LUYPBp0UwVzgOUOQ18tlDeIi6IxJemRW9MN0iY2pLSCijz9H + + /Mvv4PxGgY4fLk21rPMBqJGg/G++34nlEQDlQ34qReTUWoZnFU4NF1CMVv+nG+RP + + zbGsMUm7mqNTBKqDg9wJ+cizpjPsY/iXH7NF3J9y1I/MzWcop+RsR+Ep2A55MZS/ + + OeaVfgItESWSqWYW48mtQM+D7p88CCbGl5vFLb/zRktQ7PpCWGnNKJbLc0476hSW + + hS3dsQTGH2Cdu7TrggIxknR3SCS3xeoyKkIbSxBOTBISiWdffNdplF+W1GZrhZ/m + + u9a/h+ECgYEAxWuGk54Kth/9JNA6U6OM08OPbddPa/iAFzNmP8N3/yF3SOPfzeCm + + xlUheBYQ0uLE7BgkhapuIYCQ2e6BvPE9NI2IN3sU6HFOtQNOd7NfAsGwIhVaQmhp + + s1ktQB1dOIudvdlz3zX/E4EOXHpyUsYQbB6V0F4k4gC50L445NHIAr8CgYEA408C + + NW6L/WzILo2OhCaGF6PIaJONy1I6+Qbif5v1bOm/0kehgGf38455GQPulKVDCbTc + + azKLH8NRIuRe18io+uvwDfwjJFfRWfMgOXDkxBuY1fT4m5IPCiwDux4Lz6Msab3q + + Bj0MnJcoHrvyD+Fn+LbuQAQ5mNd6tkKowVs133ECgYEAhgc7BUr9gKn1BaIshw35 + + FOemn27Wp7m81IN7vnxpIhfJUP4LukzzTKENKObqIxH7mUHGwcx0GmCbdqk7AVhS + + MjSILwprpmcOhUuqYQ+wyEFQ38LZVU5nvHAljWqiGDqJLBPOW9LfypEKe/RRWyrG + + iXC2SxEvPxQ5EqOiIo7dmCcCgYAmiZOfSXG0cofx1JAP+ZQMV/k3OaT1jqhu5erq + + pZ9TasHZvck0wuu3wDTpt8/wJaCa+a3RAs2xgeS0nLEztlJn0C5vwIqYs8bLkDur + + YWd3lBIyXAj2HyormFC9nZd1CX4TI16U1i7YMYxcwZKFfLqq4SC9e7nkHswwMFb6 + + CSO2EQKBgG2YRfM+7nZ0s9i0v8OgAEwL6q/cQISuEIaGdWQXxhWW4Ob5u+3KsbgU + + nLNbAaycF/abhB5uf6IkDufzeL8yIeu6d/+nq40t77bM7P60tncjhoMiJtOVOKxJ + + 283ZwYz8hzRvDiH+FGyq/+cb7b2FI4/RW6TzDBMCNxX3RrgqC88Z + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 8388609 + seed: yPi3JHKKbWaEEG5eZOlM6BHJll0Z3UTdBzz4bPQ7wjg= + zfec: + required: 1 + segmentSize: 131072 + total: 3 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:MDMF:3yhl2op3nrsrbevgpgb6f642me:twhn55j2i6br3mpinhj4ipktkk5bi2enitakg77i4h3qkihnusaa + format: + kind: ssk + params: + format: mdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEpAIBAAKCAQEAxotYVkVtYuRGxjNB4HJsn6tz8UMc2aCck4Sv5fBK5tpsfX8L + + 0MyGHo9sGTX3uIgUYvrXjkVZQxA4NaCIoCaB2rUOhPbSYuySwc1ZwBleZEFKfHf+ + + +fm3dV5eKWg7m4tzXqpUIBPD+0f27Ch+xvy2fkkIJPv7vctCDCsltl2G5lOjhNF9 + + xmj2dITiZDVCMnbIzX/E2R/IRUSA0sWWswJ1MbvbMPKZfn0DMGxuQt5hfVEcA6hA + + dock7p3UFgwz5SLdhz/DRz6S/Wv5KAyVMuJaEr0PnhNfuRLh+Gf572NK9dFgg3F/ + + 3MnU+L/VpkjrmSnydAWZetqA0cHgR0h0myK7yQIDAQABAoIBABQp+f1eBPVYjXsK + + 4N+Cg3dmXmqz6BHOouRFAyfBQxwltgO7U8f/g4YGRyJa1acpM9fXFa4Ca0W5N5oz + + rPiF3KWJgplcp2KKV//Egx9UPD7GlkHknnHFC52pCTtflYEvNRb5ybwtwbdLGF/k + + 33ZGW2LhOCloaYldo54wgh6eqkkJKvyZPjfIbQoFwc+17Bd6r/HTz1cBj5UoL0qV + + QyVRX9K8NqPUAJIDiGRffJWgTPUOnuuqxSK45CgzMhqoAvQr1Np2DFo+Hc9eSZgu + + WtZGnvUY8pRsOrQUxgN6MH8BGUNvdjpK6+Mnhgo+TzdpGviAU3DpBqC8vlTfm88s + + tQn8P6ECgYEA4oLKTu/U85TVwSSeSBh5bIXPcLosrsQQIENlPNOjjQMTAnmGqibO + + YHssf+8W0xIzJxA7aqH6mOjEQUDfv7YrNtzO388aqew3/rNdy/azjq3GARKOCvFI + + 0GCU5wHtnG4vaVbpDmfWWlNAy9MLGdX4P9wH8y9BJjnsdxsLc4XlI+ECgYEA4GR6 + + UTLLp1rUMJnT4UN0qK4QxqFPx9iDZSQldhsGYXHbRUPXhI0qL0487/ZeZHnazDt4 + + 19Sib8dHB7/O55KlUlXshyEjzD2FsV58t5XjVQrce3EDZCTOb3NYSMC76wA0BQoY + + CjihmszeP5ny/aYMLNPKkftfKGer7BjZDn+3lOkCgYEAwORkaGhwzqXWik4mxHqj + + HLmu9+5zkrjAitkZ43zPcIxHqfnXphq58Quzz5bJtyFukjuOfbZG8+R1DKS0Zkw5 + + 7NSJD6sMp9vTq4EPxVvneP+e+NbWQ5dKTLmS1E6eDHMAyRIMEgp3TiBLs8ebUnsW + + lztHQd7h+i2lo6BSViSWB8ECgYEAo1WAE4q94tuiiJ3wNJA9YmsRmwPgZr+bJQvi + + mM2jH1sZGJoBTmLSygxRHvpeSxTHxtGjbLdCZcrQUTu1B6se24ff25yrygceQbVd + + YuSfzU9SniftKAACo+153bstDinfs6tdRFNkjqGBRRpyXV94jUi8svYelfKgmgKc + + PImKv8ECgYBJ+vxGJiBWnxjV1iIXHfp7E/TfXRIEj88avL8w1MELyM9eUPI/drJL + + 5v4zHPDOmWcy5I5ivmkbnNASHBszag9wXb8/6srJwHzRpyUvjy8g3XiqxhrC8LKm + + 74w15W4Yual2jdWa8DbyD411kkFXlFxUNAfk3vef2gl21t11UoceGw== + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 8388609 + seed: yPi3JHKKbWaEEG5eZOlM6BHJll0Z3UTdBzz4bPQ7wjg= + zfec: + required: 1 + segmentSize: 131072 + total: 3 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:CHK:vvh7fppprucnsblhp2nq7ixyze:ck2nmw5uynyyhbr3s7h5ciffgzw766bt3e5n3qx7r4njjzqzkn4a:1:3:56 + format: + kind: chk + params: null + sample: + length: 56 + seed: YQ== + zfec: + required: 1 + segmentSize: 131072 + total: 3 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:SSK:vauvfigv7juzb5ja74dqsaxzg4:2wuk3rewcugoewhacjaa3zornswzz6wrg6rv4mdrfsajbf6mckqa + format: + kind: ssk + params: + format: sdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEowIBAAKCAQEAmh7Awenbu4CQjcBW0LVpPTnI1YznrSBeitZc/88XgF1+C9jH + + Gk6apQtQE5tqFs+8B0ss8j6avgAXJMUL5MmBYMsb8qtF2w+30CNcyBPOywNFoVfh + + 8f9E/G45mfC3gxtTI7LXAf8+oEz6H66PpHhPmicjb++CNlr41Nb2YA4JIUieb1R5 + + jUkm1K10tcgvmOKSrU2N84qQq1zas241WiPvpXVpYAzr/VHTc+DMvdqk4imDn+dm + + ntkCJyV/BAk2ZVqmk4hlSWwrbKilK+2rH9rlMqGGN1NDLmhOUa+w2Q1ZhiTlvzh1 + + xqaeCe69ZU2UsBoPyHYTogcf0CtcPrhV7hk4JwIDAQABAoIBAAmifIhi10K8gczq + + xkKb5K1YLG71NRKEoIRrbDrttllm/tc8wQ2q9k31DBd9sr8kU2vdTj0Cnufb15aL + + 3vd5hWYIrIGaJW7RZ7tSSp2TZ20XkkXI2a4oOCbTuTQfcUl37tWfe4N7cm3RAh3y + + 6rXsc4V+ht+biHdfbojXu2U722RCLgUt55U+aE4tUQtZ15A2TbcI6Q/LxeqGgHGX + + ZJ6c63EwrYmAlmVUB0VuaXk77HC7qxdXADe2LT10PYGQpGSa4CHJxCf8jCffsTFa + + pOBd7MEO/tX+HwfrQmoGwJGFcJkQutWytz8v6ceLeqDXhnsXKifQuGV+mAM07dW+ + + 3Dfp7UECgYEAuBIXIMsXn0c2kdecPbZGBDgOCA/LH5EqwKidV47dmvc2hJ48Drrv + + bqtSXGB/XH36i0m663ax+KOjtgoRTBHd4Xly4ZnRAG7Ik6DKxH9rAzrUylwGSg4i + + UTpINmk24m+G//sSNKF7idTh6hv6e6oVZ+kIGd1YF6CxsKp+4QhhHOECgYEA1lh+ + + 2HfWVHdOUdyN3SMKqL4w9UF8Oejef8L39/mKz4TPkBQ5uFYKYLF083rTFhzRsPqA + + vPijsn7aZBNj9D7pwn+gy98M8Zq4svBPMCuqXGBqM7QAHY1ahno+o3jlxDm2ddRM + + b6PQmmVVq/alzrDPpz/J8KHRXe1OvZhOFC9hLgcCgYEAnY6Wj3Jn8OWC90lIKqa3 + + verBT/M82fNnVeu+anEWjQvodZIANFeclO0+nWXX/rKy38Enp189LWfcvPhXH/b3 + + JoXPaP5BoQ4yz/LFPXcXgXc9J02n8IGyrDaoEzLyUNZIBxrA1Z4X4b3/9mUmfe3z + + TrNwRLtrKSZakq8N1c9XWOECgYBiEbFPl1zP3ppN6AxcTikVVZeOzvxofnw2llzf + + 7yOsmMZi1G4oQe2Tmf25XMvxhRQH1kVKsLQs+c8wFJMZ8CMB42UNgiso67Jv5HVG + + w+O5Sj+tEkEvRDpT5uB76NevdPxfYtfqCFhsG8sb18i7DbikfBIH7/Gb+PSa2HF4 + + 2MisxwKBgAeQ1tkpSHFsm0VilBRknflbe1HRBAaebq3sYVeUR5zLqA6MZGgTXGU/ + + WB5sBZANJGRj3mLry6QNH8e4+L3+2yObkXH1JsVG8KdYrOV3pKmTTwrfnU9sRqgA + + zVP6zTIlJDnL7sQUOpOmD6pvzgQlQBN+zkKwnmZS5SZTX8cTEFqi + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 56 + seed: YQ== + zfec: + required: 1 + segmentSize: 131072 + total: 3 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:MDMF:ez7wfw7xb5vsy3shceefdlui24:4higxbbkczkgpcisvsj3u25ybbw3smyek42pib4h3fjkcolnba3q + format: + kind: ssk + params: + format: mdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEowIBAAKCAQEA76JNQfiGWQy7HAqPN7YjkBenSybWjE2v8uFuz/Qufb/EMysc + + FODyFVmToK0++qOzYa4NytdsXGfb1HO4wEovJhUivW12lzIeB5niC+I4hCF61Rs+ + + XqyxGrgVQ2er2JNYRireqKhJ6VSttaUqyA1EA6/sp/xQQjRSCdqMK8E006379Qy4 + + GSbiJBbJVwVuChWA7G8xpDACMMNC8BjS++33mJCgm7PW2uxmKwSz5OEuPHXrcMSF + + q7hPahaSXao/SaxUvQAWBw40SsGwiH64auoomNKchDzEAT04bwPx735OO0DckYXp + + hpR6y9s3FwrTl5r85BzPRvyUVov7CLfMnsjfUQIDAQABAoIBABL16xoAsZSjMrzd + + wwY79aVlkbmbCZfRX84eczemEPWnMj2AODkYsV7qFwm8G4MWZ8+fR30YvXy0RQsS + + 2vfwBroDKxwE6MC+2OxuCxo4nJMr2P26qZ0xGdRM43XRYqIAypfGtZZvtmVta083 + + keKBVjPafCWwi6MpY6Je9f9SSr1C3D4duSMLfAwCdCntxWbWF20gYuoJWvNb3P5Q + + eMj1uL/2z8b9uRRGhlbmWvbJNXWyEdSJFDmlBCS3fHwqPVos1yBsdKK4LemEospH + + HREWCNowBktA1C1AWjSvvGu8Qjdn7xaEyM4P7EsVN2Au47e5bEp8ghoyGmcuU9E/ + + Y+O8DBcCgYEA+OAvhOECk0nW33/cf/qOuUw/wiBsxBfLL40d7PVT71OYQCrR2dGI + + nRRJK+fKe4CsGqNp/kF3Zef5o8lm6Yp5/ZesazU4vVhnw3vFvnnt75WYKYzdApHy + + 32hWK1SAgvOUtNcBnd3I/UxdVm8o/dhY1W3BDfhN40Ngn12byaMVcEcCgYEA9n5k + + CZam1WgPeXWDKLrPgt6SUY1HCEAtQLkH7UlMUOfoXpMeqKQ0PO9/7PCp+g0MHbzY + + KKQs7asbao985c7i5EbxUopO4TonwJ9TrhOvrhQxRHji1Q4CLFiChhUSFjzgyECb + + lNsKatxzta9/Gl0VFHwSBNuk0Cm4KCxlJOr316cCgYBfvf8J43YWK4XaHVo6ca2O + + Y2Lzz32IQo8MEAG/MvHDVClyJgbtAMrJgxBTL6yZrnqHFO6lvZGtRnynIcfReFBN + + 2ped9q+JSAVDEs6T5FxAmxAai/JKFtOUVpMvwCZgOkyu9TfN/5BewY32vnTKkvw5 + + vytRsIBmOXlmVaClBXQt6QKBgQCysggl52CFP45QWD/AjEWZs29RzeDb+2KTFFDJ + + 1iSMVsNfpLpKOdhhAKO2Cva+/yx0do4iUHr9xdj3VJSQKX7VTRTv6LKslzNwclEA + + 1ua6hYr9/8E6AZDTw0rEl4voMTQoGKZxsKYJuE3uPg8f9rEsi5Goke8Wtdf6z8x3 + + ihwo6wKBgG0bYUcYJGUEilGGqHicZ7g4Wvd+inwTJMwYy1+vFKQbTRF3zy0TgGqg + + Jnv4bW/vAd3N75/xWWeiayYINrvtIplDcqzCRkX1cY+Wf7j1dJDzPh3cfbxuYm0V + + I/kfDbWrMeggSX2KMIOXCHHMnrDet/MvLrThZN39sB3fLTPi9gEr + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 56 + seed: YQ== + zfec: + required: 1 + segmentSize: 131072 + total: 3 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:CHK:pbfcdvxhbikxd2hcc43oel3v3e:y5txpmiptmoz36ionmki3p6krmdbiqasw2v3wdq4ia5lmrhh33lq:1:3:1024 + format: + kind: chk + params: null + sample: + length: 1024 + seed: YQ== + zfec: + required: 1 + segmentSize: 131072 + total: 3 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:SSK:ynlgfwgubnngeblvcgsqorw2xm:3jkcozzh54l7dqrawb7jugq4ycfbslmd2urnsbboef64j2pg4oiq + format: + kind: ssk + params: + format: sdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEowIBAAKCAQEAzaUt1gNzX67V0Rg1IPWw6l93oU+UN3I9DVEM2odXsYA9dSmJ + + v+DwcK7BQHewr95kxWl5VgZ8dcQyz6Cg498eWd7gZ5lvKdypT9vMv5fHxPkAXLwa + + ek+lJa4T4Nw+QGeBsOUw8RQ50NvFS4CA8J9VrJ4ZJO9mEbZp97AsUA1y6JIDBwtR + + pJH40rrT1P/zSwH9naxSSFV/g+8arJSrWEUECzIRgEu9TAljHsziDZSlYJqNFIhx + + 22LgYzIxL8GW4ujW3S7mEDq58Gc7iLTviH3VsEPdAtZHTY6kYUfT6bnObREbguCm + + +fsIeKwpgfay6Ap7RTXcW+s91067Ol5HkYvbYQIDAQABAoIBADVAOSLKiPU1dkur + + S5Kp3HKMXxOH4lcLP3Dz1HLACj69+OweYfusWUaskgFKHRglbA3Mlq1mh5MNR6UJ + + MLBhJeBavNxG2IjMCZHS1m2kdYf1fJkG4opalmav8ZjQH1SZGPXAG5DJzoDdb/Tx + + pTHp6IsG83bjgRhEFqObXJYsLV24fkU49aBph9rhs3tJa/KnH8OOjDYvdhHtlDcz + + 4e2j/iluQ8dQwrDFSKI+SURkkFpHNHghpim2KyVjjvCIs5bdZooEbVvw3FbDCo2B + + yzhMuO1xbRPBDv31aW0Uss4+h+hRnF1qL1EjCx5Gw5O7EO5PIMPiIw6RitLo68ug + + HaQ6DVMCgYEA0kqruSASilijTxyQsIh+PhZsmCg6N1lfCXSxBHzzme+zBZgUpxtB + + ZecBR9Fhl7vNb3SE9dBRzuBH9qDWAxp//aXEQL/q2iSAzdLIqR96b4b++qvw3W0V + + mKL0KosgwSnkrjOS8hjq1JyfRdvsgMCbzGuaRKfa1VAqnHzgLHMERvMCgYEA+lf2 + + 1S0hcL+gFcNOCFOy+wVVC6++S0pf8aYRkYx+F3kR0/SjGF6vbLX1vgWIr+hvGD/t + + i+g+2JJ6xpsk/Spoz+7SDigN6xnQ4ebxqC6meNdrigWGji27a+7eUxtYBo8zoJdC + + HDi9SqQA+WyC0hg5zSnFXSXZSCQggvyjxtWNkVsCgYAuHOepEapfIe61s1rbCyM7 + + tCkd+HxDlNptNWR3ynqUf+ZuzJmCx0xA7zXtrLFM14bF8PQS/xphVfcR0tT7Gz2D + + vmzZkfwK18RS3ezYgSmU+TJCf5+yvm/k557JEXceRHR76p1Hb0VXV/zpEb+7wACq + + A9JxSamH6ytc41k5BgOjFwKBgGZG/+IyMQJWV7nsc/n08B+cGxXONCmgdjhMx8q2 + + ImHGpeD5hpSTQopggMikjCaKCLFYlN1fAiYLGjv/8Im6BN5GzOzZsm4FuxBAASTc + + AklGgXn/LezyhCrhiVVcy4bKhKYshebvy24uOPOuQHhDS4IleavHpdDSabH6M5Mt + + dkwXAoGBAM2XgNw5f+/hR85uAqypyi2/JcGVGBhVY5i04s/WMpkiPziLwueZDx4s + + 1+yIw1MiAFimmNRBo2wOiG61P8hBzQypp+iHWYmU93YBYkL5w6HZk8Q0fIkVXqPX + + 2bexcajdG9uVDrCRJrFYD4OjV3xLUqD8z9+IXLLc97jBNi6lrdOo + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 1024 + seed: YQ== + zfec: + required: 1 + segmentSize: 131072 + total: 3 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:MDMF:eb5wv5te7y6k2lgjrnn6jtqjyu:mt6j6a7paoezixhnuuiaixmllwsvlgqp23zcwbkezizchadbttrq + format: + kind: ssk + params: + format: mdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEpAIBAAKCAQEAsrjGRwDgqlBtkw08/NIN9iaKIWT94GDYRl4ldWPbXjBl2NDg + + I1AUVAZreZuoyptUhmCGunlnQpdp30toXO4yEN6J/weLf43xVOtyEPI02cwbJV1m + + h5XLZ6jvOw0DFgtTtCuFWQIc+O+hWjF0irHZdV3ogyb1Er7YDSJNfo4dntIdxP0W + + wpda49rcqGdzmaJpoOLwTQl9+c4UxHRP69NTwbsIRAsQsXOqVxV20Ghl89/uVCY4 + + RmM7IbaCAgNRpCfG1vgPwfNJmKY0O5TdASqQkUg/qiCDFEf5RqAsR60mNw/1fFbb + + alHbDIuvZ8wUnmLpjLHDN5MOXKWds0cfPsfN7wIDAQABAoIBAE7XCQzAe9tWAIhq + + whkrVqJcDPo/UWlef3nHRVH8O4TY58zWE9IwHM+WR2oNe0/pZseipDx1mtI69ibd + + XowEPczIRurcerLJvIjAFoEYP61Gh0Eb60Nrlp/DW8laa56ZX5Lu0fPaZUqBd1XQ + + 1D7sxueqBgx5LopW6vscQ0BNVA6/mCeXDMfHZm04GqfFvCInHrKdpRTIs3f9fedY + + XUDgYVIEhZ1KdBqD5o10s2yroPs8xSvDCZpnmfwDnbAPGO+KxTsQvsLYk/j2YnQ5 + + u6Cf5iPmB/HjV2DJU8bs4JOPHf6hVP/QB5a+96fEIyjXIXoGg7IeX/uVOagPB/FS + + IGW2ykECgYEA3sKWMQbbuQgso03SufeM6JshGLZGHxOt71twbiZjiW6jHfAFrZBT + + UgaUVM4oc8XsS/PZtY/bfPIIXY4tebt59rejW5SycE24GOkEKSVqicgf/g8mIto0 + + yfXL7y8Jt7dQJaRVr3+N3t9K4WEu9LDq8k2SnPEGuUkAK3fRpVeLbK8CgYEAzWPt + + m70fHq0KnObhErJlnaBOzmOosWqK1MG1Qh1wqAfIpR2Mp+R7whHqZ9JzU0d8rr8G + + 4rbNjTaPQ3X5xtS6HU2jSu9wRv7xY2IBreZ6Q+CrgnbDaTxQPAZUz98Wrqsb4BLT + + 2BgP+E89edsLhmUyLIDWefENWKfyXCiEgoimgsECgYEAmMLamo54ecB4VBkXbL6t + + 3AoePUMqfT9SpXWQeYlL80BzDiG+0xLJgNPQPwQNy68sZ723TAJ2Y43bXMUWvIdr + + kVzH4xLq94bku/h4CPuGvywFfIXJAlefoew0yTb5tAo7JUU4GZ0gnnmEcWDjAZyd + + 0kKOS6Aim0fLnQOTOo75pzMCgYEAh0cC7+mvfofgjpkusx7W+OvmG9/d8wTGbf0r + + wmEbm0CNMdt1kftWW+tq5XjiRn62K25cPaTDW/gMghVJL2FbOAOzwp5T6B7wpFGf + + 44cDDoQC0sogSMbV3cMZx1QbX24JzRr5dsHaeuTOC91vCNTMKC2vld9jt/neEj8J + + j+QrL8ECgYAz7vY+kTuX9y6PoWLmiPg3oBTBzF2kvenGJ0XTDmwzNHi4dr4kd9wT + + 6UWKBxPwZE2GqT9c5/eS9wr29QTDmi+7/gpX/DkmXvSuqK9csXoIxNkt64rrNCNR + + YUJzBtHwB3UihT/y8O1zc+4DZbMOuVHhGQ/L6AbXmVw7DmLsoJz9Sg== + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 1024 + seed: YQ== + zfec: + required: 1 + segmentSize: 131072 + total: 3 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:CHK:vkdc7iae4pgkdomhbbb554kv6a:g5xkatxv2entfssfh5eqgexb5j3qs6jvohofjqo5p5erlxlx2dga:1:3:4096 + format: + kind: chk + params: null + sample: + length: 4096 + seed: Yw== + zfec: + required: 1 + segmentSize: 131072 + total: 3 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:SSK:uelnsh3gbs4pgkpzjfl5qp4t5q:a2os42cflsgyls6axuhesdc3r4gqiqz4fbbtwoqq35xghhpfzmia + format: + kind: ssk + params: + format: sdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEogIBAAKCAQEA2HVfU9IdqaqWU99YRYrBc7wZTye4X7IaHhiLzVtn1iCYFyvx + + fGirEuBLGzV+3f6qrfRPpLLCHi63UTK4UJj1kgxf0KOQL9guikyqCJ8+J1pUOfld + + ZzfF5chLIjgkv60liYerUebiE8Mr2GOgTqguLL+g0VhhAv/PgJ3MYZygRl/c+9gg + + mt+CbXNqHMjjYtkR138eB00c+QmauOzbFZiLVkTbLiYl7SZFHnmRO5P3A7OrDKv6 + + rfSOm50CW6T8Fe771fKrCOTqVdRw/I0/++ahSo+j9tY9JKPT4qSz01gZRYUfInjm + + At5XSR0aD+78RTFe+kAXXnx7L0OdokEwW4Og0wIDAQABAoIBAAMVpXaSDJX7MG0J + + XaccZ+Y7fg8E+oT+8zCCyNp/fN3FSa5HNwfWL9HnfxBQ1UuIvfcu/3J07mU6E4+W + + v1nUflwYWIB3z9xI3KMaQR7FD52XCao6MWKj+uo9QlthS8zELbR81XHh+fF/ya77 + + RHaL8EMOcGH29lur8nSVkOU3M1nb5mdclFd+CnmkNBCxQviK4jmotQIUT++dGWC4 + + 9oBg/Dwoy9dvqjnmKPkWqVSwqDx+TRtZWyHxkUgkkaKz07MlamBqv50iTCqQxvux + + tLnRupSkr/Z9myPBI6v64sZ/AsEznPNbY2sRymDf3S44cYmB1UNr2rz0jjmTMUBk + + ChLrQSkCgYEA/TQvIGHLTA4uDEBLJCYmLe7QlOHfy/71gWMWFc6m/rsRy4FSDJKf + + 9M82myyNzPwUfUpFV2rx/9vYv8TeGG47rEOeS0gwDKVbfl3zCsHmuWGiJOCQhDts + + NWRwojA/v+8PxZOLAbiAkbrNW5tXefBSyZTJ5A0o+i7iqdUhbbBUO7UCgYEA2tlO + + 0rMuI0Vki/3E4qbfIvTcdEwQfHtm7zH1+8HwNbJQ/VIBxQuSqLs21b29RPOdC6pZ + + 5XdMiD/rcdysqcYlRljuHr6c1foTZiriwjPrT/J/iLtiQcu163yuEWhkolgTuPvo + + 62qnoq92h2J6hOlp+PSeM+SlSnEAQvxXhyWgD2cCgYBkh2d+j9VLaQXXT1+GBq95 + + 5StjMRrNv3hx2olWNyoOUO+LwNh2rXBcnjir+1CBZkQsSmSlhIx4bSztVphnUrzW + + dDJQ6WRKYQyma16nkrysNZtO0OoP1hfsSuh9PHLTHXNBmobCNCK3uVb3XAGrJEN6 + + TVyq8p6mVh8gFsKi7jNDUQKBgBdIHvaTUUk3TKcH7DYggoR5gCpvHSHhDuZLblvG + + GgPcYHlSjBWmUYfZws+iS8xWDlL7YGzk8CNeiXGnhEbbaYO+WjazGIQ7Am1QCqeW + + VmY+6gplxOIzBbtznCEF9g6/R/nZ8sF4qzTHbdihRV92ZWuyulHS9TKiKuD1b2pV + + Ol3pAoGABithsvuEddWMd823bBdZ2/NLVVI+5S0/c18IuSAwieEnhzPl+38m4M2c + + 9+c1LoYJ+ksD1JEA4YWnVIni65tjF02ZE3D4QQlai8NmXhpkFXghdtH20DAw6wRS + + vAq+5yRl/vZkgfsg+AMhI2br9I4VXnIu28b9PL+tGhkB4Ob4JGI= + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 4096 + seed: Yw== + zfec: + required: 1 + segmentSize: 131072 + total: 3 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:MDMF:r7b7a2hc7ihwnb2kil5ugrkgmm:to2sfzyq53oc3z2om2aanz3o3wy6ouh33yobmy4myyvfyrbcsdeq + format: + kind: ssk + params: + format: mdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEpAIBAAKCAQEAyOxN6WrKut9PPH3jDgMCNos5axRnF0Niw1fXsX6k1R/B+zwV + + YtU8+LSfs88/XeuTxJ7ttbzyJhTVYs+GGwwVVpYKWI64vtjsTOwmDdGf9r8az0eE + + aQ9NupA0FFtA4BywDXNvXZyRki0EgggQZ1iS316X6AkP06AkeojMsN4D/eUK9xFS + + LNNSFUlwIDAPH4JaZnCDYOQKrROPI9/779YebUDecZhMQiFadIDrbrKn6XvEFx2o + + w1mI27HF8ZFmk+PVKqMwfG46gNw5+JHBlvhICYe1NzOBFPmSUHYI6FKD0jJeYfIJ + + bANJ+m2aA/PsYebaFg5RhfyQ2bDjwzkEHqFm3wIDAQABAoIBAFQDErTV9Xzb4NrP + + XIBAW82Iu3J9rnl4sLQzZ7oM1UlUJR0yy1JvDTaE9/4MW1efKENfnM+P+MRZk7vk + + QBPRIp74z8ylqLQMKgoj9+lxTGy1DbW8Fq6DOqIWp+AXI/JRrH+DU/6Vd/ziG+9v + + BcTgsVD60ZOxLk/zty0RRF10B8FCJSE7YUcj95c3JqP9+Vcxf7OmEvFJ34dEZFWX + + dadZqCf3mcKe2oGjJqpFYBzMZQIzLWD/4K8fchUAZcvnNwv/wGLicYcZvJlY2GTo + + 3kbF/REWWGjlEMoZtyXDBUxLdpOnFcRo9dPKt+WVdUlhJ6OYDS5Q5lTC8gLprD0r + + WR+srhECgYEA1QEenkUkeWQWzk6B1ZmxryJtY/N0KecYN26BGHKzW0DREDfA85VA + + vXCY3yUl+ZFkwtY5fwCKNG2dSMqLSpmnBaNUPBbk/O42ILoGG2zOp9yh2Aak+/Ub + + 1vHZ76y8ShZQM+YisrObMZt1jQjIUflKEBiGzjawp5oTCK/GHoAA6icCgYEA8Xrj + + 0O0GPAPbzJXp7TS8rZJgFtkdeTv0MLlq7aBvvHEE2d+XYVcS7UAIDN1pIIRZySci + + AqoeHmHRnG+mF5zdnvjVopLtNmgepDM9OvCo6/6kQA3kr0PPQj7axhXsA8OMDnGL + + qauv335pM+bPVOWX28iT8O604WSmxEOhLLjCKIkCgYEAnZ5medfQVcOq3J9blCRX + + R7HCIORWYWuQj/RFs0GtVylviwC214jqj0Ry2y0yHKtqVIMRqNlNa95xNRwsVte8 + + sH9cJdsLN99OTolZW5H4ml65pJHGJGwMXdI54xF/g5NfZgg2ROaDQQI4ylRlZ8OA + + +sgreQ0fS+bHjvYDNS6jfqECgYA7KvtLI+iVJ/ThShJJVtSsSuNUddps7C3HCoeS + + te7q415m7Awxg55Vl4zhahbqKsO9L+N7d6dtllY/2HN/8aWz4BCohwusexKW9R8Z + + pAIf4QLp1v2jnB/agYAlbRWpTm6w001/Q1wSjOzGFNXUXXU6Gwl0zWhwmbLrAA8r + + 4BFi0QKBgQCIKDRNImt0/X0MtQH0aiSkubyA/RwOe1nq5zJRI6w1A57hjSj60Okw + + RRKqgc4e3XhcbDPkdJiCZp+PywzNmty1nKsrHxQ2XcxqCLhHXn1D7cBXyzldbxfM + + tr153WIvd2s4LDVic1/aAddikl76evm2ei9poh3vVqUDzkCTDwKpoQ== + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 4096 + seed: Yw== + zfec: + required: 1 + segmentSize: 131072 + total: 3 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:CHK:zvla2jyu5fqlb2s63zftyfjnla:yqspaexhew55cvv7w2m6czezhzkqnyk5fea4gvnplc5za4xlvbva:1:3:131071 + format: + kind: chk + params: null + sample: + length: 131071 + seed: LCa0a2j/xo/5m0U8HTBBNBNCLXBkg7+g+YpeiGJm564= + zfec: + required: 1 + segmentSize: 131072 + total: 3 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:SSK:ugxvnpmo2gpclarr3zdqfkfx3m:vkjlbeaf5bez7c7nycx2xujehytbyc7e4ehyajrv76gfbyhhjkia + format: + kind: ssk + params: + format: sdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEogIBAAKCAQEAmbutuUcf00SZfCrQf5efeqlRZ1lt/mfZGtJSLYIEf6BGq2+p + + u5kF5g6mOTcbFgnv2wb2k1V4djK8bvWRlAcz+l4Y7T5GKO2t30S0v+RvRwZt5Zk2 + + MwGJjDyUGCsB47C5mlfZzLysgyDqqSzIb/9AzrrOPR7nHYuMgLe/MM3FKH9IqFhy + + Et93dQ35qXKczuCVP1+OE4Ws5OjtDGCf2hkJwrKRzW25EIix9jXLC/85YyJY/ORS + + 1+KE4U6ZkXRTmJR7Z7jiNwavgF3McQ+WbmY3D/noJOvLvcUdc/OUyXrUKBnfQ9EX + + fAenLRqN8Q9Vye2vckqQoKQ4FkvDtJbg6QAeAQIDAQABAoIBAAH2EYduz2dEBhVh + + S7BT6SH3QnUClpMxC7B6dXE8SPlw8T6vVlbsVbr/nLTpUyCVeEhswsyHV+th09Vw + + M9+V2MRAs1XPnTmFRVXRj1OlTepBAs/4NFWzfKE/OvDYIJQuab4XXES1ja/IxbfP + + 5JUPQeFuhKfdBuyHtIgyVFChfKJXiWUn1Lumb07GLbtmdLe5AAlkZ6G0XyWySXX7 + + ai0x7rpm7wurLx3TclSZPN7eKesqUKiIEgvgBtTOJ+3f4q4XSUBkYcUmer9p3kP1 + + HMh+BHDrshs9fw+1JiVxG5T4nHZWO0YKcPzJWq2jxR3KZ5KKFGQCQMg/EvCPUXQS + + WCQ0xrkCgYEAyBbFh19y/eh4nmOn+PMIPI1JPrP/nPFxmxzfefLr6tfAjsNcSjFB + + +Ms9ng3kTeC71pGBldFOQCyvu1fopFe5EVNK0z1m7VcCrnxIi444mC8aXBqTGiVn + + oFFxaOvjLfpK/SAXMi3CT4a/cYuPLShHpoaoDZvezVHxLNkMRD8gsz8CgYEAxLDf + + sI5YCTeeWFxZplaOqKv/O4gxm8QlwAqE9sCawxVjY3xYgR/pXCJWj9RbbGwXwVeL + + gc87BfdHqafPuOk2FJdyGwHTUJtIcUCWanHva406xjSTPjPuphbuvI4ydiochdFP + + TLK5kI0W7N7GFOo1BzQ6RNhtcJ932m2rPFQnHr8CgYAHt9kmv6fP44fDlFSGZdmL + + fGe243qYszeOpC56pcQz6t6ioyaMNho1XqGh1ydXWbPlMvesr8Y084RT1bBDpp6c + + 7HmWbGfr/886q9CgkXvdYvPBWcUS3R6CMKIPSgoZW+5IlVRPuzQjnS8FUjzToRoi + + ck9JNxoBEYgcEsNGXqkEQQKBgCG2c1DOxRYnW1On2JHjKiaM/H1WtbIOJ65H30xv + + 7NbdNqDZsk3Hi3cIR6/1ZQoraNLxz26bd3FpVfYlVjxKdMOIxb0NTgv14a/Pszhh + + ePkFRvqsDkTOH+yF57uX39xTEXp6Ss5Jn/a/yBsnf+obzqUCda5RLkjsfF2LCJuZ + + jO7/AoGAbj3i20/BEAphG7+M73ii1sTJdL96G/A8QDNba53+A+rmczO6YgMrd/FS + + 3Gab0l6ZEBliFDGWF6VtG+YsCByRpZsEHbYCux+E6k4K3SOximh8WcWgB7KU5lQm + + VitHBiC8NJHt7PnVgWclDzGIZiM86v5c4il5qD4AjdIu6S29JaU= + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 131071 + seed: LCa0a2j/xo/5m0U8HTBBNBNCLXBkg7+g+YpeiGJm564= + zfec: + required: 1 + segmentSize: 131072 + total: 3 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:MDMF:t6ljkehb35cjzldhi7hzsnquou:5qg7zj773gjqejptkptbjkcugmwywuvmo27uq33sloafub7fck2q + format: + kind: ssk + params: + format: mdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEowIBAAKCAQEAr2+8XTlh6He0sHac+kTS6tN7tzZRndDu9Tru0q6iru0Y5az8 + + /3QKQ5b5PBylSvc7ouIJK/OLu0avfnCO3qSp5hJF11PdZcQSM4zy5P7m1WjloBTu + + i39L0wRa6TA4/Uxkxx50RbemP8x+JsledWzQu60TjXOPj2uuMveVLmb0t+xCpx9U + + QppR7AxeNe7ot3cWGqgzs/QwOKcHCOA4vLmkA1wh+uCL/G6ZrxNmpl6vQgMML7GL + + qV0rR02lL3qmdDwW/VYl3QlA2pPvDkaQe47wg9ei3gP3vgx7A96R7Y/Na9AQV9ZP + + Az9l6q1eB5ow5EpQM5MFc8GVNwwUqTpvVWSOnQIDAQABAoIBAAsjfAErQIUi/Izr + + qwHU1tNkBAnY4Au2FUXqrPkhb2DN2vPSLOoHMxOhhUeExhXhZp7r3Qs2VlvYnBHa + + EagfKk5aQKbwQzFP5pvxSgayDHPmShYE3jRrK6RFNYRytFuYuxlNXLKEe4C3ehb6 + + WA36j7IqxgAII0hG3PONdqJQlR8MPO8aCTnpS/2XKP7/xYejTXjHvLzGP19Iod1m + + lRDRwGnAyz322g8tNz119TOmklxmDdZAcRsgUvA6xlGxy84TDrEHbSdl6gfEgh0D + + VwzTnO6pOXw+kDIs/k5MPN9RdlE+7mh4zb1g2aP5bGjaAD9wLhhe3NHgcfaPPuOO + + vmbnBAECgYEA85L7VPF3JmCU5vS6mmTure0IqrwL/shodReaSYWvJA4E/WBdkeu4 + + XH/AjiZVEbvhXQt8wGuj5Jeo0SpPrWnHL1CLbzhhBO5h9Do+5M02c4GG2DSOFt4P + + kvMkZLx5POqwlSxUGC2wp0KgsMFkQFy0Yg/6f0iKTB8WN96IrLXGc50CgYEAuGLk + + i9cakBTW+dwS3bV5JG+Ajokakm24vzjMNkAI4su814d+uvCz4B6TtmDrpw/biFza + + NYThUvTv6jOMtATtkTqYvrat7e4hf6/y/qqNSluzKLPqn42I193gEIyNopw9V2Zf + + r7ePLJwr6EIvuDVdHG64AzNxnqw4OAsh2963FwECgYBd8MY2UJqflohXOvPtMBhN + + xCmfj78gmLKQ1nWO/Zw6z51lC5GLAdqs8iiVqnsMx+V3OUL4A4vGUiet5B+uxiko + + OmxMjPX+LOJii0ROgkcJ7V7QbBSRBTwEdPoIUBiCQhGwttQILzb+i1fmU/ASUq7P + + U1JNXPDZwvOSwKT9122ekQKBgFsbEg4+pLNYeLhQk0nVJxxns7+54tVDPavOZqjP + + jxRw0sgz/Nxlnps4wIe27/lGDpUcO/2BwMv6lqjD+9vfK3s81sg0/0+2//pVd915 + + bAK3uJh9/YoEpv7ydIn4yOr2BCExRkpOioHiUJecTHPaej1YP7flLVjXg5e9eGdp + + blIBAoGBAK8DxEooYqNeB8cGouSkj/6d2kezseiXgcmhpgdrn0h6jlrAWo93wKmE + + ZTA5+yQm6EoOKXYk29esopYPd1mD/GzzQAy2Cl9eDPwQpykdeShwrArroiZ3lLHz + + EB+CyNV12Y+MAncOQQt0j5o63oR+ihlhxCY4sRPfdsK3okqdLdBo + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 131071 + seed: LCa0a2j/xo/5m0U8HTBBNBNCLXBkg7+g+YpeiGJm564= + zfec: + required: 1 + segmentSize: 131072 + total: 3 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:CHK:3idf2xqm5wnw2q3nbva5vmpaiq:7nmkh5q55omjloezibvyzvveq4gqeaivei5ypfm5vfuugwiz6hpa:1:3:131073 + format: + kind: chk + params: null + sample: + length: 131073 + seed: /N4rLtula/QIYB+3If6bXDONEO5CnqBPrlURto+/j7k= + zfec: + required: 1 + segmentSize: 131072 + total: 3 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:SSK:b7ro2vtzpv3ssm5xxgobj4dc6i:t3eqqhurxqmiqbp5rgd7cqpciingkx4jjo2mmyfwqjhgaitqwubq + format: + kind: ssk + params: + format: sdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEogIBAAKCAQEAn1J0hRBe0PNPWhJFeZBcam7MuqqQ6omOCa9dSULWdDIb7h0m + + WP+Wjib8Ih/YtuKX02XgluK2Phj41G0d3IGKiEfZ/WyZ7ygNoKOYNkv7P9r/acZz + + AGuR4genl6Ogo5ZMkMtxBHPdGZ8CusO+COuWnGhAm7kfC4FU+fPj+9tHlPt97WaU + + d9bqgV4qfqHVZmKeKTpfTM9r2wq9j2Hrrpqe4cH1uXH1bp4rqHLFakW6hUxOx8RM + + KdW27vhFPcWfISjAb20cBo24DrQWliL7ZRJ3eyiKsftPiCR9hjyAtS52dsW1Akf3 + + jA6laqgHbvBBmoZWAoeIPAoY4t1TZ1pwsMNxXQIDAQABAoIBAA1Z55HPEWMJQLkX + + luLdCiGRL27lJEfDRzfgjjy5cSdDm7uUjcYfhQpckfx6FrscugRpIS0Dyqnhhdin + + XD1CTc2l18q48x1ridjQXM0QCPoM7CJ9Et4SJaN/aLf4alnLGpd3tPzeiMTA4oWs + + KZytwW1R/zgNh2B7cheQLKbKdXEykDidTYl+lDB6vn/p+bU/1jTziqsuHwqNP8+q + + pfU4SHK5ViHsjBok3TTkgqqAEAB1xRqnkI7Sh318g7Y/mLY/6lfSLPTxUY1r3FEw + + 5veLvPtgzCdhZOIbNJ4r9CqJ6dbUn4sFRAwHzmyghmJ2coL3AcjS64f6VEYxjfZV + + FzR64TECgYEAwxt3O2I50lcjXnUTXNzvAl6y15E4NFA7TVeqO5LZSi2+pHalRofn + + W5TIPOlwH6hxrmomb1gj5bJxpiQA7/8fvyMEwvIoaF78fWE378UdWIGeEgPrNZlz + + RBD5ZKeQ6GpPn/w6LSj6y8jH19N7DfYSoHdVpV4agfOdS5+0dk5Gen8CgYEA0QvZ + + zjulCIv7E+JhFk/zuiVv3k56lgrj1omkaFSE16O1VxNQyLFX8EBVt7jUXp0w4Zq9 + + +B3PWwZEj67GyJdI7krkTY+I/nOdCoVotRb5kj5yEWBtIQp4OrLzklWtA0r+pDx1 + + 3/SEii7iGLiBdlOlpMVV2uacyu47jFpuV+kyTiMCgYAWZLagqDt+uuWiV8mrJOiB + + 2yCnwVE0H+lOjTtKryYlb26sLbn2iG6zgjYhV6G44Hp7zE8xBGrKWFrW+NbqtNuN + + 8pT/Uw/0OsK8GUZ0TKl7mRTteGmsszoZm+Ej/l+RbXJKKIb82/E9JoRZbzp2dcHZ + + jRjVbCGavL1XCrOJyJ4qPQKBgFBQiEbW3YoSFc3G7NwgrZg35+n2JtzcpDp5uWOo + + DT24FOS2dBQXJp0UappidZ1AMVaMGC5qbY8gMlktogvRK+D5fwtZeR2hl5VCOj9Q + + 62PHgBWzAVpvZk/PDwuKxST9vCWnYPZBQGbCqnUq9fpbGsnaUyj97wF8U/6Rg9Fc + + s8oXAoGAEd+/YeU9jAdC1F0gOMyJF+Zx4P5rtFpZnkpUuEd4826cqRtereEP1ILb + + VSMAL7HIgHao6Qxm3rLaP6w+m0Asy16u4nD2YZ/KAwI8UCMbugcqYVkGBZ+VDhB/ + + 07H+p2YX3nxrQCZ+SO2SeIdvUNLGkAvl1p+Uaa1OCa8zGLMWt2Y= + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 131073 + seed: /N4rLtula/QIYB+3If6bXDONEO5CnqBPrlURto+/j7k= + zfec: + required: 1 + segmentSize: 131072 + total: 3 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:MDMF:5zirha4u2skivahf3cpsl5vxgi:3kgb7qi5y4qsx3mts7jltclahdupgxieleycnpnxkg75mvrvbv6q + format: + kind: ssk + params: + format: mdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEogIBAAKCAQEAtCN2kwnkjeag9FiUvRprIk4rqgVzuzF4saeUdM4jH0LaBYzz + + 1+tK832B0D9E+6M8PDA4RSjlM6V4sqboe4W5AxZ8bihGGHjWtpjZQJB/QtMSfImG + + ac5cOVoQ8JbWfDKDpexdDRcgVgqHeLZG6+d6zS1XziLw0GsOxvCvQkA0XdABibNh + + BZzvStjGNOM0W1NNpAGe/tNLo68ZW4ELEjgbuFVqcVljEPR2A1jwzoLl1+oM+0BK + + DT3DygKkkSO553HnjNFsLp8A/9yHrfB+k+MNb2Jqlyup9FVHA4oQ39I6YTWvO1NH + + S9Y0q5bFfjkEdLtiDtTKMyfJwZdb80IgtaC5uQIDAQABAoIBAAJk4kOMAxybrxHw + + R3HH8xqOnWfyEJqxSqBZ0NBImRDmS419VRROjT11Mo9498q8XaWTInxQ0dMA5PzC + + 2R4jJdVTrC2unVff60Kb/28rPHW/5mP/U+j+FB2zA7ye1JTr+vHulUICR6y9ERXa + + nlCuT+SAMMWNk1PByH1+X2XrAocoodXK8BnXzmuzYAS2t4gN2xERoZigsoWJ+4u/ + + BsB9gjFmLysw6E40KznwpJECbjRhR7MSzglHSs5fdfWvH+XfQO08n0KyVbtgZqbu + + 3xu3h1m8HniJExZcrfsGj0le8JukdVFS64nfKBn0xbzrtfg8Toq/MEGaeY8oVywp + + zZhfcjECgYEAzSCvTGyCG8TFQecRw49rl3xGdvODlJ8/vkqkeOwwPHlvKy1DtM/Q + + kcXwDCL2xhnvK6uF1TCH07CZKkdkAV7+SUyd04nuvAuRBdCsvkbN7D8oV2jNlp9p + + xW0Yl5Jl2V1fouuO9s5QQYXeVsAk6X9xMIcXoHDJscdNXbqaDT6ini0CgYEA4NA/ + + MEpc4TBQJVfjl13d+hJdM7mESCRS0i5g+uykCLb/8dsid422bcKXehTFFKEIq7cJ + + yAP/r4lQsOnoZ2nXBUVUqZjBxz5SjYXRZ/QiCsX8StUmou2HxK/a5ccJoTYTfPHw + + Je8QZEWAFz75r2ao5kWv76AO+5NywA6iRMpzzT0CgYBDFl9+xTZAUriY9zOuG+f6 + + YWDCYp40K2kzmUH1cnnMLYMYQfOU3Sq/olcCASVoYO8B/1UEBp1FtMpDM5oXgLP1 + + 0SMFHmWABuBlYHw+tvV+QKG3BMXIb1auhSG34N+CmbE/nX7iZVOGOnwfLzRjUZT5 + + ZBVsGbc9d4tsDi14C3Yv+QKBgHsQI/boTg2LJ+Q5R0GdxZxVnyVoYUwobhnV/4p8 + + LZMDsfmP7j8pmPpechMG+ZdAS4HMEZOm9Lj/XudpM6ogWu7ss9qe3zyVFhWYcjgI + + gPYKyP+hzKOViSOW7CmqGdBgzKwxuDbbtcpd7S9MbtugQ8bB0PxITstSPJd7q0Ii + + 3N81AoGAdwEu+rWuqM+jIANV78lKGxhTrhq0byZJxNZI79GDsYdle5o4xGw+dGQI + + Z2K081N8TC8SZZCXrXEUMQPCfy2mYgxZ9bUcdJOTozdT6Kg3ohRoTTSi2ySh82vo + + HpZKFLMxIXKOGjG4pR64ihn7hURl48CNeFkho9wMfg9k5UZsaoY= + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 131073 + seed: /N4rLtula/QIYB+3If6bXDONEO5CnqBPrlURto+/j7k= + zfec: + required: 1 + segmentSize: 131072 + total: 3 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:CHK:o7xdokumtcukutzgmmpfme4nnm:nn3s5gau5dychy6wlodwriuit4wyavfze6bo4icywcbhdb4ccxla:1:3:2097151 + format: + kind: chk + params: null + sample: + length: 2097151 + seed: uqWglk0zIPvAxqkiFARTyFE+okq4/QV3A0gEqWckgJY= + zfec: + required: 1 + segmentSize: 131072 + total: 3 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:SSK:77tz5f47honr2le442nicoq3gi:jnocx3f5a74f4dzcfegxml5dtoegc3iskb2zsnbchz4vzodafgpa + format: + kind: ssk + params: + format: sdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEogIBAAKCAQEA0PWN1WMZgHQHTKfMolkQBZGcckQOispsmQB2SYLH9draEKYN + + gfBH0cPhxGmnfHazNjM0BZMfmdpbHhZafy4ynGDo5qyhwcwM+GV+KL0HrKU6jish + + jBH15qGOdxKnDInYq7hMFaTyehGsL8lfeoHCY8sGFSyWcrso0YkvTfewv/azVX4t + + mtRk3OvORylCvHUGffuTRcofERYXlbUFaX/Dozq/BOTrMZjzukTGxx2CzwP1/SXz + + cm6tcH5/f+AgP46frbuasOe7p1O9RiaI0Pi5zYY1QXJPMAyM0X22V6Z3OWPM5/IH + + kcJ0Eq8h82cPOZYAJ+kqbzP5QjY3J4sVFB3R3wIDAQABAoIBABRot2atHDOIoaHi + + DcGZk1AH7dDXRthVdw/mlKcPZ/piWsQfg9g6ILmjOSzW6O3mJhDYJW+Z9A8x3Y5t + + vn8Hgxf0+yp0mAP2qxmjyBOwisxZAwQZwFgO9QaGpwSIRNqbqBb1lDDVAH3dtgSg + + 1XuAqvzWOozc4wDnuM/mZ0FlPNUy1RZUtdZVGmfzzgo3jc3/p4OAxadKDiNKK1du + + 0r7kbKR/EWyU305bdT85eui8yeYZosAZzCuGORi9MgccXTL3oXmj/RDiEqsWSXpm + + yaNCyXsIKQ97x9WIVpSM2RoSlB6zWr08KROPyvzc2wUWWtWdfQ7cDUg4IrUr6/8o + + o8ZIGeECgYEA4k38I4UV8p0xwmiLnjJido/0EWvKWMA9Va22c4PGetMhOiCoT6uS + + 7Xly7gK3t7rWRZj4nZjCFJmMhT5ml5T4J8r4eGXCcHl3X7FIAAIUdsaVacs3OLbs + + MslUrYXocFluF7lecagZhSJR4jItl4cbthYucmWLO1u6Z44gOoEs2pECgYEA7GDm + + 79jqcyqkQAVEGIoDn8LTG/HtUgpJExwFCrCa8qEomZUTx0R7s63GCxIT3FIRQmtG + + KR/Y9xs+LMutbu/nW6OsbeMJUORQZ6HliCBXiL5TiSBe3pbf9Q1XpB5DW7r8ka8U + + l821oR/2jL662bkajq9CpujY+iibreGe42puvW8CgYBc1u1njQOSApcVUFpmzfjC + + 9w+Dzhq3CjafXaKKBTd5z//Dnv4toQ+nyLkzl33TLB0XdEgaLz7/wHZ7ezwPV5fu + + i0Af9G8uQUaNxWbqSfAnQhSt0CaZZ8HCnAHXJiZTYPzfUrbCHdpKWegJydgWX+Eo + + dDUdzTavZVQ1g4MJPVEvYQKBgGiPnNgv/dWf4TQooCyysFO1XKkZ5T7LKfP4Cwrl + + gEUfoNP/K9aTppyem+I9xudIrjXROiHq4pC8Tk6GcluGZ7MTvayGJ5LOy/prlRsY + + I2BrwIwB87VGzB6cHk6MzIMBPcQ7zEIyTsvNVcSAgirZRLQlNriae5B88hCCo0Q5 + + ym6lAoGAIlS1fLRkigI2PHmPTpXhOCzJ2v3Okw0JS++Yq4Pha9XxyCgqobXNduLJ + + lpugHyL3hEgPLUewuaomGke6uEN0GoJ98WDt57AiJ2UthNI2icgSucLdo7qW9A7S + + oXWH+FpHHkYjAHy+FjOuwWBt84VNrHixTW8WRNSq0c6aFVDajGM= + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 2097151 + seed: uqWglk0zIPvAxqkiFARTyFE+okq4/QV3A0gEqWckgJY= + zfec: + required: 1 + segmentSize: 131072 + total: 3 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:MDMF:6ybmjtp6o7x7h6wbcut3ao7ih4:kdnpi3rc47su4eypnjq6bgx7ixc6qle53slv2d7hpzhdygf3emra + format: + kind: ssk + params: + format: mdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEowIBAAKCAQEAw9r+RsfnlBqGlQp5KcdVAy0j2X44Ut9erFEoFPq28VnT/IAU + + A5DQDYg5+xpYijOeF3F5dkb/IvqWDIEmOqEG/dvku+1I5PtnqOclqohI1o3a/751 + + kbgB5mFET6wv5OCwcnG1I5Qo1XCdB/n5HYjs6otvxUkW6P+votKKu16P8YnMO95B + + jT0MxZUhPaqc58u2kXI5+TJH49QIVmQ/aMWsXGuuhzHBPqb+3G/2lGKfBSC3Z5Hv + + N2IQOOP4z69D+w+re2ALaMrHwQMzPFskuI2pZ1MHsRoFi31FmvGPrViFnd7/0VvX + + I1MCh1qJGkgTmMZgvJNST3kBna5BcCdL9TcuoQIDAQABAoIBABHQEo1MdB7vtKrM + + e42VsAEsc1S+GpBK+XjRnsQds1LLGTEfUvKqEooQiDlywXe8TxYRv3rG5UCAqvHz + + Mw9lAtZG0AxZfeY5iUl+0FmssHc3CqJ054t7wUx7LzPR1L9LwjB+b/uO55HV/qox + + jXsmr2l7igxW4+sICijUXkLBTHUqqu17sYEpHkMFKBfL57CXXkKg6nPwTt0CWFLo + + Ps2Jj6rbkqzzU9KSMUDhbkrck83yiBNHfeH/TkALj59E1e+8n5b2oCDW5YS9nNlj + + 6hwOMSU/Hxq0NH5j41Krko61aqmojY8b3QRBT78YVdJinKPmzPwefxjq+hUqmUNk + + 3XbWvxkCgYEA6yvyLeAJ2cyuDXk3LmaDuT9oOHcKDkp4uT+IfpbEk9lEapPzBTpv + + wavS5FLsSllr8yiv/cY6yf2TX8DolO8+1/rdXPPjvm6XZv6nIggYQRdI0TtbiREq + + 2JKbf6H4a7PR2ShbSTibpjS40VnYYR7oYfqmAgeBwKwqZ1pS+npiThUCgYEA1TOh + + IXK4FoMpCuv8Zd9RA+nQlDrZklckJchYCi/wxCREjjVuEw7NWtJlZsSBqYmnfoZ/ + + dAI6e8vIhuW9YFwV6vLao1ASgeAFCZgZsXkl9sHNKqKG7DTmOigPDrjvYtFTILRo + + mAw+4zAeS89zMkr9i98MKdhWTPwNJInUigOSzV0CgYA7PKqYG6LflcsR4cKgkXoE + + o5AhCPsjdmbKYtKC8H87rrKpFfNVEc8svZc1pB2Y7MVgTpNmHRSZ5KHGsNTlDw6J + + YMt5qoVnZnwEmYiH7foOC0twSL9Z21UrkGJS1/23Q2hMhvnXi8bJKuaS9UqnzB1E + + 8Nn4EOQCIFveBMZ6CXHRsQKBgDHOQ+AaeqLXtSjWBDqQNs7hOlbGgLlNHiatbNPE + + a0yG5HUMSlCtbo+/Au1FDr1aaQSHyxKAysTM0GWjGeB+4qfmX+ky9X/do4+gNrBd + + Ct9gWtuQ6FAZ84a2gP4Befrtx6umOaD7i11rikhPiCvBlQWt75t+7HpDj5ZvlHVB + + bHQJAoGBAJyLQXh3+DPK9QiZABRhkMUsaa4beMczIfRPuOilQacgjRoteMuBH1IN + + QXkTul1GOoIi3VEuHTSXY7zrqx0fDM9rm7HzmQl1ArOR3t0n2vZ/ocgxU4B+bc5o + + jsfcdXkvRK2MCKZiUL35wGhEPdsUv91uT3TT0fpqCHNF5jCDxVBZ + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 2097151 + seed: uqWglk0zIPvAxqkiFARTyFE+okq4/QV3A0gEqWckgJY= + zfec: + required: 1 + segmentSize: 131072 + total: 3 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:CHK:vws2xrl2nch2hsptqb36yqqu3e:fhujsv5rkhyiqstrktjvfy5235c7gcwekyd3gux3bv2glsgstjga:1:3:2097153 + format: + kind: chk + params: null + sample: + length: 2097153 + seed: BTBX/ampNfLU+ox7xipBGiaSbgC0kcB8Gy7BkJB4oKI= + zfec: + required: 1 + segmentSize: 131072 + total: 3 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:SSK:s4d2dnoiozjyzljmuemn7jmvpu:q76u6oplidnchjhsa6ds67xrk3aknnuiwlnm6yaymkfxzsvyhtea + format: + kind: ssk + params: + format: sdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEoQIBAAKCAQEAySMnWzmnU4vBL/wHXVsmXG/Y3WEWQKdhwGZ6xxmN8m/E8UKi + + PwzeEWeCdiZ3QySEAYQWY6dENVZz9TAbZ7nlwuxp8uV34JleUFPr+b9jE2rAqG29 + + jrsqU6D4+/CsXGsP+qXhlfzINNuRU1vetPJc1VmKVEXMV1caRnuScw38/evRnDxZ + + Ehw0b86T/4LgZ6Nx3BVQQCDbGC0asf0Xw63ghCVnaLLxFBpM1ZGeLalk/xWy5e0w + + 3ICnQDzP66ErZojRAgEVcYhDi+M1sHGB4yyIPLMxewHIr51IyvweI6QjeKWftxiv + + dmWCFZwJZFoZIGskK+XGwIhuUV6q2fRLcHp/VQIDAQABAoH/XimimH82qAz7okyw + + TL52syzQUleGOjLM7arOTApskH+zXjeN6/aOi5dWfEc4ONmhbzNIQEfIh9qUT8C5 + + ruVBXboZ0DBXauW3aG9lV37F081kZF5BrYDqOJmnmaBgRyH3Ko8ot6JxXdFtCRWF + + BsSPDnlgrgPBIljDzYFCuSYJn/xmbOvs2TD52pyJCeFLH+q1idC5rEmop/EOvwYv + + fJj4dDrCgDWXNU+orDcVDXYp4OpzyKDepaxdSoD99MLX8FtUw/11ssWmXSR9CN61 + + tD6F30Rq5q6K/iPNh1QE94CNDMiLEdBI9OgyPfywva2NIehV/YUeb8NxWYPztphe + + T6olAoGBAOVCMRUD1VJrhC06AFAczTeb2/YVF65lFZhf+YTjRnkWeUjoC5U+nM1N + + FP1qG4uR/hT28V1o+ZN9p5jEpcU7r1vW7ifMNUDC20GIi8xlpKv3lYfWK6J5VL5M + + +UgmrDcSongRDAAbe/0r1drH6RlIC589VXRvSvGcycft18Hv886XAoGBAOCZPl01 + + Yf2frZM15apNjy/1LyUE0YL5EzOQ0eEDPO/D6oN40qkiNWp3WA/prrj7lWObork6 + + 9d37gFnVwfXK8pP5+J7TPTsAsAt3yHrZrCQg7YiIJNEDTs2cirHjEEFv6d+pTOoq + + eUcJvyQwNLIfkJ5UTPcUIRH8Y8K3jGg80IrzAoGACOnN5rdDb/TmKqv6nyK/h83z + + e1nOleUwNcBlfxknAEYzaPY8nQzWI9U/X6rkb0S50C7Zq3wNWAKmpXXfzA9J/hQZ + + Jkr2NxJcW+vnI4dAI794fNOC1spI1S1A8+EtCOcckfZ3tPlclLdDlUH4ehcm/IXx + + 8JjzHPmvjqpcnRmrLPkCgYEAty7FmqgLgBxYKZTv+HLBsk+7X+oKJ1SWwJwBUhCe + + BsA36XsF9kScZHVqMbBafS1UrqUllwXrul2CVcLuK1aXevGKQZ/wdMseynur2+bl + + a6IfmhfQT1jvUOu4g1W60GRCz9T5kpOJztK4Pv/COvVbsob3Lx4PyuebRhkGP446 + + WNkCgYALjJBQ6eMDg2CmVwtXbP9ryHpgKZXmEc+BkBKbfmk5tO4bdiydTX1xXK+t + + yxVjNlVGDuowmFmLr6Mo9pE//+GPjp3udcwxUizdq35V7MiaYMeP+PvE2ZDldESr + + qB3KGfeN5jj8lE3Rv4B2vHthqJlbxeFfATgRt60udOLT2MYoQA== + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 2097153 + seed: BTBX/ampNfLU+ox7xipBGiaSbgC0kcB8Gy7BkJB4oKI= + zfec: + required: 1 + segmentSize: 131072 + total: 3 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:MDMF:54bul55haopklyyzawbnetxp4m:ww5wq3cqszi3qz26ytkcnqtc5msnkiu34at72xaq25cfdw53rl5a + format: + kind: ssk + params: + format: mdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEpAIBAAKCAQEAvGBGNDSciyc1taYbXJ9nykEXJztT7lLadtqtA1HtdWBprQmC + + 3c6tPSaal2HiMtbKrsXxn0kvgZrELpiQ5sRtuuP1484fsp56bmMr+rRlcnXOAz3A + + 9d9A7aqiImS2nkwzg95aFwgXFAl0U1ca9ZJqHBGgIt8atWfJ0Ws//G+tKjG/CwGO + + zacycXbhARoneAlzAYY3DWlY7BYQfmC2yvG7lJcTrS2gzDLPzn+ffHS2YXWkj1Z5 + + dBerjFCzgkjDeZnPyxSUk6BOLUhz0O5k6kMpd+N0YeZUoEjuHxJnFVkTaG75x8I8 + + gUACgtwLV/7OYKH2Jsa/LU2Gc6HuQtVbIVRUlwIDAQABAoIBAAQHRQkRxPU+2emw + + Cy7AY+5R1QZsQN+8Wtqm04NaKatUa/4c0XUf2dmQbUbme3ld2YjCR+gBnyf04NQk + + RBkDQ1t3S4dbHZdqxzB8I3EhkXfQqB6un+VCKHfMqdn1cGhTNdjQlPmgHSo7B4dO + + ug9dWbKO9+82+TG+7fRyCDTESJ4517h3mNTMeMLWZG6KolroqXn4to/iKf+hjj7c + + xvuLrgswUGM15BFaiNLh99DAN851s4HCY4AgpWZvpJ2/D2iKCB8B77kO+pBjxDqo + + lxMU83akFBQEtBrnXdIh24uzQnl9Ls9+3TxMAaIalw+LedvZYH+xMzhKJOlQ6Elc + + bb3+NMkCgYEA4KnIMHU6jool2W2CrGHnCzF0f5uraZLNwzsDQQnLclDm41/IakDZ + + /vB39xzV6QdEBK/eGzZRhpf6pMBeSlHowrClAGH6qK9hJAE8DHNUrzDGFCt5BkAh + + rSxqKiLQcl/dP98JC9YQra5R6IvOFuyO/TPe+ihrS0IWZMa0vaNkwH8CgYEA1qbC + + lYc4wOPoxaXYhvObtrK3qIotYUaMuyYilHlVfOPohpr5wd7N9fZ0B/Twn77uIx9U + + VhwjNI6V+sa5ZoFjqi9XD2ZmPZTbyEpLpbLuYMecAvI+KC4EmSxv2sGdscr7zDvk + + zdHGCjFh1DqC1y1XUMve7TTbb5/n2oma41/OX+kCgYBVbhpy2tEWjM/Ru0PaeywZ + + ZIfxUme/MJTP7WvSWoAji0IRKkYSqXB78kMcE7n/78Rcp+ekn2Ym8TndVk1Eo5sI + + FZXY7Gkdpfshbtq/vUdxivF3kARobRChQmdoeG6dX3jJpe1Rs+gJs2TwMeF/dBr3 + + i7b5l08dghbz4V+vUSepzwKBgQDMwBFIdM4MINo+/m3GfMWBxoQt/nA/I/7F3iCK + + JBsJoJSDIX0wEwm/nzEbDeghWQzq782QvhJO5dvmdH0RbEbXZYTUKcdI4p+rNENo + + cX+1TXJh1RS5WvwD6EFiF+IGYCtDq7YbJgiUXHqG6LE59AQgC/g/qHXQymVtLmlS + + jmbbUQKBgQDA+CPfuyqG0QO3lfdsR/ya6fP/ybPSn0ovu1crEO0hruGppgipLAAI + + VCSyGh++1DLR5+3ynGx8iSvhYPyYyrbdflDA8DXxijr4JNXboii/nlwmYsv2korx + + Vd+mmSHOiFyKYShreE8YCCqbYtPOE2TFfoeHqTNkDBXsRlSHQaHgHA== + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 2097153 + seed: BTBX/ampNfLU+ox7xipBGiaSbgC0kcB8Gy7BkJB4oKI= + zfec: + required: 1 + segmentSize: 131072 + total: 3 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:CHK:5anac46dpfsjumutzsoovvtlpe:yuc3gqe7m6s6xp7k3uodcg2g6k2bac32gt5amu7gdx7k43qnudxa:1:3:8388607 + format: + kind: chk + params: null + sample: + length: 8388607 + seed: w6uP8Tcg6K2QR905Rms8iXTlksL6OD1KOWBxTK7wxPI= + zfec: + required: 1 + segmentSize: 131072 + total: 3 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:SSK:l2zvtygxn7tvpomu4vmyjqlzpm:bp32x7vsjk7gikid6dqgak7hjpmfyq46taj2qg2tuzdervxfesxa + format: + kind: ssk + params: + format: sdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEpAIBAAKCAQEA8ObaMnm+1qCV58BZl1ULTwe2tHBdVYTsb4WPiCoJerK1SvT9 + + Rs3oZA34D0ViSYkaAdz2lMaH1XNOZ++8NTY4t2U++fkIbOe0xkjHQ1dspExAytwH + + KmM/u4U4/h7IAQYg077x229FiaQ2UPdxwFOjG2aFFmXGI68qz6gqr4E2XrUjx09p + + wfbY/D6aJ4JusrY2512rNbfYAx+OvkQLIkZ3XlbUiYFMwcS3NnPJbS9oHzVGrR2w + + AijATGTxpAVXNZc6dt60rlbhFGesxRwgeKHYUeqmvmwkvq25wMDNIqAzOcXwX85i + + K9m8WyPp6kh1lZ3UVdmLJa4ZnTti26xOv1RfVQIDAQABAoIBABIE0nFQFewr2sqY + + 4pqlK9FffFUGypRo+t5kmRXQPyFEWLcgmAlBwY4qVVGfGPjzHlThWDhMmUBn/Ydc + + sTExuxBMrGc6L10l/6mNLApncaLgaUBDMO4EunGmR1sKpl8dPDtaXvDQ49ylwcJQ + + n9uI5fxYsL+6IRXuNj+ODpNOEOkIclvP9d5V1aQmClbfsOxY0Mwvh3cfZRItBbfo + + e8nRyPJRQXzUwvQ35e/e5TM5ootgDSqaaDRp4m05t5Kf/w+7Br5+eiOSKW+bgJxO + + OPE9tAajgFkil1zK4rDLN3mN6ZWikfSeIKPzSGNHCff4VBgLhv+tZO5mgnqBn7N9 + + 57KjeEkCgYEA+nYG6wnPLa7mScwDus/Lf1B4OOH36k9gXYhEoxi3A7loqCBL/fYm + + 7E0+4Qdkymmff+4G6HikvdLN47a7M/ClHncTnT3zwwTEC42VphPOLI2/uPDDkOHW + + TvYXKOk6TIUzUip3hCBiwsnEIOoDxzgb6/VccYvJJb0n7yB5PdV4OfMCgYEA9jq0 + + uAufrh+CYXBh/YUIUgO2kWsZ2PCHvH6ur0EmBkhhLhPcpLRHuFTZJbOqWpn62lkr + + uytnc6NByKkbrNcqzKT3B9+2TynOcROoC3+9kQKMHiEQDwm9pUx+zdu9rolS0nlv + + PgcU/FvSnKmqg4W9SJ1Bo0XysZpMyGEvxB3QS5cCgYEAxBExST3cmf6Y+JxlLxEM + + VRZBhwYedaa94XqTgLoQSzIR48uksaLIxaOS3cZT+MDGw/cqIUKQdKlZ1DFwSzDP + + khHVoPqmoLxSXFjyFZjbhbVRqQ2RixHAGwA7ESPDJ7P+gQwNk7lmluYsSzfmzUX3 + + Vbg2Lg0n4gs5/9CEGQvLmlECgYBUzitoKDi7FAcn4DkfxC31cWnz89tXKKDXfxpT + + KjEagNtXr2eTIrSA/Fg97/+AbQBFK+kv8ecToOsLXZM2mHUZPsgGYjq8UT3VHFwI + + edqkkygHSIPragNzZ0FVTZWrA4kPDNwPlQjZUhbb9mPQIMPsupzcyz6nhOllKnP1 + + K/+NyQKBgQCD65f7J7czMm+3RJaFLEaNU6bYaXpB77Th9Df94xevGcNVhcL/A+ya + + JRK/D/+Cq7iudYAaU4YdyM5XRjpbdGGl/PdqcEEgfnNAtYyQ2Ygp9u1hUsMCwGRh + + oFQ1FT15a5FDE2a4uUPayq4I5dupJATHboLDFCosuUxva7+GAUVf8A== + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 8388607 + seed: w6uP8Tcg6K2QR905Rms8iXTlksL6OD1KOWBxTK7wxPI= + zfec: + required: 1 + segmentSize: 131072 + total: 3 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:MDMF:b6plxmkzdirhgwthfddux7ijjy:bdnh5zgxw2uex7plu6bzomx4udy7zeebsoquvdxnh2lpqetov34a + format: + kind: ssk + params: + format: mdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEowIBAAKCAQEA1Sg41bKZHsJPHX4rV4lMju4vmgToay4eoUhydPQyJN4/K6fZ + + 8rtcRsN2izSvodzuSm025GA4vxICN2FFMZPF1bAWaOJVszozTmDEJoO1v76cjptm + + Wqw2PZKSZK7eh14AzSLc9Bz6yKKS8uBTGOch4XTP2MMkQh08knpQAs1IzohqkNWv + + F96p8MY2q7JyYIH0NN1k05K+UiIdxL6Yp/C7EeaJV2yKVmgNy54mug6B8+m42QWc + + jfyvq3nQu3FsaAjxaDbaYLrOOVV6meWpk3U9kUJoQnQ4EAMzUa0OU4e5GcVCl6v5 + + yfR4Wq9GqgtSLXSie3z75EcF5AqjxxlxxFCyTQIDAQABAoIBAAcxyFlOIeTr4ge4 + + znWx6KeaWnj0WXPkppwC+foAlACyj6dFjxGmSUMKLeIc8SCheFmCviuPI9svHGwK + + GG/H8RF5VAhOO15FRJ4MnhI+t8+0+0vE6vt8fIgvfklvrYscHSLPXm3O3JgRBKy1 + + 7ZgVlQsrCijizUJ+AiFfh3vQufheioR9OHF9ZeS6WP8iEnoSklnsOppWjX9CdysS + + 6t+3mryY/LTyxhwTTfco/dhdUG8tmL1RIhuI3xTrhHdTDuEAFP37+9QoxFMMUGIE + + 1nysscCIG4lyHRX3JtTrRIWjXj4z74do5a6E1qpjImQGFzHNwCBqyJ7n61Fw+hFO + + +lU2vYECgYEA1S8IU2F4HqQDOSDDf3/ZavZp8AcHdcTSDgmcBQ8x5Wd4VP8MCn91 + + wMN/aixfHMAJiy4Q0qfsgSpCaovUrw/uCSY0uCXLhc4PC0KWUgjhA/RPtel+LN4K + + 56tnaWvk/IMAGzCXwgXWpFVjAkiiV++CcantUS//6kPeAtirsoatvfUCgYEA//fS + + V7IFZnt9WfUNUNXjvfXvdeQFDYzaqMhRkecoGTrlfUOa8AByS9IT+QpPefo8xhjq + + q7shAdsv1PXNF5vQJSAZ/wfxiqOtu+rX2PyN7RZ7t0KMqm8Mkngy9dAJ+9QWbdf8 + + BOlSdw+8qz+Pfbnm22OCla1FdjPQkABcqP960/kCgYAegJE/ZOXL9ImlheOS/Zb9 + + L+6uckMF/bhUW9mf+7GW8jwMZUWyxtPxVceISHr/YRa8fEXZ7j7vqD1Cg2lV9wCG + + /Jl0c6vwJDCQ2uEpMa4IY8935sWv48FJroOoWNC1tISyXzyHfVBdyP3WmM/ppxJR + + 8w9Km4SRX06Ht7qxW4XGdQKBgQDmR74kxzO0j0SmuZ/RKZxKOgfEt+8T0bSmRBGe + + gafBiwsLNtcdNEmfjNALLQtzYX1ret8kwKVhViAiJ0DsDHGl9MtudWcIo1iZxx2J + + SS0mLyP+KxECBAX7f8fY/eD9fkDvcXB5uq9GDhJevkAJjEX0+gFxRwFG5jasVqcG + + I1INgQKBgEuTunjdt19PXq0PPe3F3Ic8smXw3ggcEd7fRFe4YsaGxr2kfiz+Jrgr + + E+t4x+0wvKflh/M3R9KGC5k/8t/jyphFB47GOk9carUOglAy+LJu/3szpBWPdPZ3 + + HGLxO5E9bjBSSdWKRB5NtfVZDiCxU19iPeCYaCy0Ji4k/teDN0a3 + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 8388607 + seed: w6uP8Tcg6K2QR905Rms8iXTlksL6OD1KOWBxTK7wxPI= + zfec: + required: 1 + segmentSize: 131072 + total: 3 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:CHK:fb6g3fkfbtlwzheujzvc2dn7dq:644je55ri6q5halshuxfesjz2afhqtebetuzqqbfecp6yhqwbb5q:1:3:8388609 + format: + kind: chk + params: null + sample: + length: 8388609 + seed: yPi3JHKKbWaEEG5eZOlM6BHJll0Z3UTdBzz4bPQ7wjg= + zfec: + required: 1 + segmentSize: 131072 + total: 3 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:SSK:ekyljbpy76pxtpbkuch2m52nla:f3hirwkvbj74mltkneiipttxccixja37xjx4vqg3oavtfwexvsgq + format: + kind: ssk + params: + format: sdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEogIBAAKCAQEAiXTj5P86e7L/b9Y1RFaIq7xNt/yjLJh+aqDXonoNLNv9poMN + + gqhI//0CjKAP5KSMDxKNS0PEneoqCBKCgGKm+Psu22v7w3pyJS4vZjT9Pq2LMqWa + + AeXC+i+swUp4B+bQemek907lY0mQo7v6QE2yOCa2URxtYYvOP3F7Q3Zu32qrv6DS + + B3Z6F/4ZRqiOTZoTDV8++zrNrSp0ddOBcN+nT4LlDaAEE1ke/TT581+74KE8wUsR + + h2yKUkKjf0Z/x9BwGaOvfdN9Im0FTNJ/W69KViOox5wKRK6pScWdmIwrtiaXZw70 + + nBRsrnJzzqvQZuBKOCBhqNn4vysQ4NTtMrmnoQIDAQABAoIBAAKSGWer+k0Gm6Oy + + JDbjThUK0C1JCp2H5egFRWMi2IzmjweGhex+oeGKZXokeDJKKDBpTr6B8EhqcQog + + 8SiI2nSSRghmgZk1+LKHUEL0v2kPrShvqRdbU6/YJRP3BOhTlxc4SavdDDSKKKwB + + 6lE6OBi4E0rQYZ9PEmFktEMeZj5uukNzv6aFOAZonK21JecbnabJHVHUYsAjSfkA + + MNQ0fJfYR5z1Vkekd1ewmg/AmSA8V8ziBx5Bc1EPXMgJIiVgZQ1FpXQq2SMFB3XL + + Re0RX65gja3IQknA2H3bDYTnbBzCivlFxjk5//JMsXktwylJ/mJyCVQ1cwq6CTmS + + udeOyj0CgYEAtxEPS6cR1UWbFAiplP1W0ngRaZJKLKfRa8TLHDIlt+wIdqA78kyM + + 2JcVdyvS1KqC0h0aptJftJR6OsZ2urtJfqkJ0OG2iqCghnGBqHIhjDus42xc0gPl + + yONlW2KJ9vEN49Z6PctM+uXl8YbQJWbDMHgSTF10uyhr6ddxiPsrFO8CgYEAwDgR + + v0VoOEoSumbC1Q6AOMV2MUDCjquqm5pfHESl20SpihZN4UEbcDWJ4CNiHG/yRMoZ + + cPvOeS0+utzBHHUOVofq94D3kP6G180HhDMmr1Nvcuxr+NMr95On5BdRSxSbcAAZ + + K7Xtj6iHQ9tEqBDKY2NertRRjgj65fO+/zS1rG8CgYBMFmwcDoGL+hU9m2gYg79d + + VQgvr9zieJHDUBT3UCR7MEBIRcsEpyp3Lzx9vpovR/t9pxkXsyKSJJA0854PeJ5Q + + ZaOtzNKZBbASkQTJ5T3qUjdGgxiFNZeBCnprJCahm4khZFiEbIY/VeRfoZ/Lm82O + + zKkWUlWdIGzR0Xjf7Tz3wwKBgBxJI/Ntl0SRQehELu+DTsML67SbvwWXpWd4c/6I + + 6480r24ukg9PsWX1uvBMxKdCofgVdWD27Q9P5SdCTPiPESkSnzUEuWmQyu7+sNh3 + + Xn32XTQgLlNTX+jyxYX/GGtgAO+eVBXmk6rMNft6TMQelGnDua8od0fbcnBcSgLs + + Er/pAoGATVJwz9ZT/hdfo7kGhk0PA5z37eTmzcEVn6AI6SzMLu1L+bt88x7nWMKT + + K2kIMpK5N7shLOSITl+i9eZxM4FiEPGGnW888gLL6oYgoYCyiQqlkShusIe7siUr + + NrHKOluBZrazxXNjUl4Eh/o2heK16ZtxsFSelQZMMj9p96X+7lc= + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 8388609 + seed: yPi3JHKKbWaEEG5eZOlM6BHJll0Z3UTdBzz4bPQ7wjg= + zfec: + required: 1 + segmentSize: 131072 + total: 3 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:MDMF:osegaioo22obqkky5yolffimiy:qlyeat5tp7tktzlxqntqqxxwjkn6ayi7zrin7ekmumfsfezqcjza + format: + kind: ssk + params: + format: mdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEowIBAAKCAQEAoVz4o/FWAv8hDRiYcj7Avu/oWS0xUFJLLm126YeKzo+KpEFY + + gJVWkIlV/blL2JQQMMq5gEJvg6LqSkaZIrD6mr3lhIRviNXhs4EWsAkZC/mch9qW + + QlCelLb+nJ47yL2gShZnSaWB6xM8YDyvB0UydzoEhyzUGEq9x94oNIDEDsDmeq8V + + oL81berDsOJX+c6HuD3IXQcd5mkj5VIM+9gmlWnv3oI4Aw26ip0nzveoDlUycdKN + + kzibaMqX/CqdBxthrCOcQicGM1/FAd0cPQaEXqu4Dhz0lrdAsSYCO7LGrlODDStm + + Ar0annnzaDxQ4B4blF5t0yPO38imcP+Gzla3twIDAQABAoIBAA0Eeg6Hvp+ZQ2pS + + DKJSy6vboMvo6GyJZwVE0W3/gEQvskiT+PbOlWAtpCFG3IaJU1EMWbCuK17cOrhp + + P5tb2au4HBb3tCO+1WlsxY7H/RxJM8aF7M9Gv7RRmvK5lSsZmR/A9O4tCvES6TD+ + + VERq4apapNje1fFrvimsk2PA65AhAOteCtYkUeA7HReOttNr94q1id0QbUCzv3Dk + + 1SaXwPG2Nn6pQEDnUenhBeYFmsBjGj86mQkuU0eE5gU2L3YcAife0Xbadw7R31Mb + + MWta+wfM+19+C5vGCF21N6mpVNXPb82ZtxJ5ZhpSRgy541TiR3Bb08JwlaKXTSDx + + LSinc9kCgYEAzZWqzsEoTqF4//9UJs+NBlUUVhyrlLNm8fSiC3TUzFbbzIxxapAC + + xE5gQdpxpYM/cfl55qsdSD6ngv14XWShXJ8D0lAcdm7IG/dO2Oquepj5ejJkLkm5 + + bQQfSVuXWE0n+FkjQSTJRL2oCa8kktg0z08dVq7EJRiLpKO1S2ZE3+UCgYEAyO8j + + cytcNwFFltLUbo5NkF1xwk9amcC8XZWkuMdqzSwl1HMNas8cc1g6ZCMg0l160s40 + + IUbAasDD3GLFlZEq75tWPvIX1UlEtXAd/5pf3RcwB7N2xbp87KAFcf5Sabltgmvo + + ioqMn4PuUWRDRQnqWPlqJTg1oWSbx6MNaSwYZ2sCgYAMQ++q4i9LcarMaylUH3Hk + + fNL3yEIcXw+3Q8cfM9s2TcBTVdW2a90eZSatByFcpJX2cNHrBy56DvLjh8fUmppd + + 8kbCF3F7R2S89mZH3siGG/ZWagc8E73yWRqcv9ApvoCx+m92BYHUjhQmb8KY2Dle + + XPP9JfQh2nMKYZIBa5qUWQKBgQCFqSXwt5g48ryizo4HGNwZuz8wHW9MNbxXmHKh + + g+3Um5hykTIMqcboJ3l4ITH1Hb/VONvOguz+VkozcPS0QIPKLY+agZo/A+UTuIgL + + /lnkjUci6EuKzjnJgcz9fkq+D138UuG2PuG6Pp2qQMLKywS7uPXV2mU6fd1uWFVU + + b8OwDwKBgGThhPlcWEAud/Vxg4quwujhlOvLEasu8V6GvdAlIoRXqjJa3sS5R7X6 + + 2aLJb+4dFq9KrBqX8my+PxR4HqvxNTALhC/JhZ1Pm9ME7j9jIM9ttXDDuEL+SG3e + + QqhyjNyRBb0aWtwXSGHEFOzcDWTJPjSChhfVAXTFAXNhsc/qpeZH + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 8388609 + seed: yPi3JHKKbWaEEG5eZOlM6BHJll0Z3UTdBzz4bPQ7wjg= + zfec: + required: 1 + segmentSize: 131072 + total: 3 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:CHK:r3orpfw7tqaxbgxc62hpmhihye:meyeb5lt7i4iyewahb6lxzohn6jxrqgi6b73zv4gxzirykpnyd7a:2:3:56 + format: + kind: chk + params: null + sample: + length: 56 + seed: YQ== + zfec: + required: 2 + segmentSize: 131072 + total: 3 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:SSK:i22zmzlu6qlcbwjcxfughllk6e:emj4f7kmqnwlx66efhdzuy4llfuayx4iasv3vhjtl6y5x5tis3ba + format: + kind: ssk + params: + format: sdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEowIBAAKCAQEArG9yGPwyUNWwPrsuF30gOZiPK0j3vSM8slD/gs3lJmk0Yh/0 + + IZ50MDYofU795rgGYQcQxEqWCVPkf9g3/ssCv7shHIdfAkq1v7nawrqDXAd4CijA + + 5pVmxttULzFEYTG+0ATOBr5PRCZieeU/K6v1Y+5gUi8iC29MuFEa0aSJ5Z1fU4NO + + cnIoNOOb0GjvxZR+PbGP37zReRLGPzpOD21XH6rtPoGgPZef0qQRqrPKYwdLmNEe + + UUf8YSBi2LmzXpLhKPW5V4lYQ09sVa69WOW5kQtj+xGINTngGC41mLC0v9Rksl7b + + 3bqgNWXYXFqzEE6E7AazjbbN22KNqOVhJkHEFQIDAQABAoIBABj7q6ENlx+pmje4 + + iHDMRvk5TpLzpzsuygo+3Io/2RHD39xQq18clVJv4lVndrdxFbGEo8wAz3SqBDr2 + + oYRHtw3+54j53wzWtLcCzzxz5/jTNzPnnC08W6/3kp6kyXa4jaAXdh85fwQNeKqX + + CZxC9ZKFNMzrecgE1+2DiLpzl6wd8ETSrXqNeYLCbiecDkSG8H5o1h2ZSJu471p+ + + t+l/EJZU0Fgk01RufaeanWLCd6gT9lDWnKEzGiugFzX3NG3mh0ussBH+WBE9S8hL + + C8RaN/q2UMfASiobyO6nZYc2BH4FvPEhJtWVRfCMTT3V6nsNEhmdNrklvruJRc5X + + STpnoakCgYEA2v+SZBSmihk9cEEF9hYNPvmcChy2aTrCsz9jYyT2/tEJAKYnD6Ai + + AEnemI/3aDjBi0Y0fnsgH9j5AnjDK6sEO6l5gAaTkHurVUrdM40efLmwnMzz1uRd + + CFIn+wZ2dQ+kXEwa74JlfOMfUQ41PX04xFdKksnf6dGG5CHFaAqQET0CgYEAyZHd + + ZrQgEh09yyWj97+9W2Pjk6zklEimrxPelytwRQ9WfWBDjfWQzCG2h8EaDDZ0oCTA + + 3FCXlzkM5gnaMnLHNjfl3QY9irKiqs/wzfLz6NpyeFhGLTeaCh4p+VYmmjHZgL/h + + cyJsDB6h2cErqJJnH25XRZcsQkCaF0hKR48he7kCgYBCWBZzN0ZUo9zW+vvhV0Dg + + CSJadeRU8LY3M0barEIfZBhEGBHRTAPA7p/+u+6JplgL51LT1l0fCM43D3qg6gg4 + + QtlKDbP6m1yGVE265k+MHX0Bo51jRn9gm/L8uzJ7uCdkxrGKSYiRUwUTuygp3pup + + 73/qBDpPTWh+CDUTlc+bSQKBgCPzkUKlM+cnMgNOtl0U5MgtG8UWHDrabmhhqdza + + kY6vuqRoDASA3Q+bn7u81FGDUO/TPlbNRQxiz4skDLfcwu1HsQbn+wgG7n560h9Z + + iuloNOyEChg8h4vwb1oaZI4x//I3xxVK+Wx79jAphQju+9eeTZCK8wjqDtHCQgVb + + YQR5AoGBANaXcjIHh5jqKDibUkICqiRdoYWrRsp/SiLiUx+07qNrADEGwH5CUVGr + + v7tgSKncoEb4cs2nI7aukJWGbsgTs2vXWusa+ZgrWX5PpEfd1SalaCjEnv26VHkD + + Rbl5L8TlLtqJSErOyTpIeK4lK7L1OPCuZ2PA8u+gWn8C3m0JGr8Z + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 56 + seed: YQ== + zfec: + required: 2 + segmentSize: 131072 + total: 3 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:MDMF:upqygomnuz244nfqnblom524zm:3c4jgtn76exlzgdzsf7qlqao73lynmwbotm6qcjrsxfcx2aipfgq + format: + kind: ssk + params: + format: mdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEpAIBAAKCAQEA4is1+B0+BAGOK+Hf/2sc/nXx3f+IlgE6wPnKZ73Ikm7DLeSX + + u55huWfH14DErWaUM0eER2f/ZE8NdnqXf1225u4dEsk6yLJuznnxXIzkZwtRGkUX + + 0txwKX1Ff1ipzThs8bf70m7BFbhNWmTUTZIBYEnhTpTMm0FcIWv0hLe6X8wcJXf8 + + rnvzuqXezIhpI6Mk/6MOLMYKx0k9+XUU7atdEI4OuuuXUKPxoxNuOBhyQ6wUmIQr + + NSnLelLPDHkYRGoVcx7p+KLOBIfXWs/tZcXL2c2jVH+e8Hn44tVv70y8QCsAYuYN + + OZS5Xd77pdD9vSK3Zjo08ddkP0HUBkT7rZh93wIDAQABAoIBADzgpbPF50H7yzF7 + + qKgfRFwoEjUPycuaxB5afkVjW8AyqT3KqJ77WFGoIi4bPpVwJZcR+oSf9SoibzzD + + bdD+QDOzx0advMF16gaQ40tmrzofXTLFg06iQFyimBjZnEcdl5GO1O1FG5sFk2iv + + +Edy9ATfjhJxUgu+UZa7cMNikvuiuyyg3QBFw6EdGnOgPTFzvaeR5GsAetQeVxPG + + 21Y6KATO4T8s18kJhi2W4EB5AVq5ZNZ1cwxe/uSm8KVJlrowGvVIC9tbRk8lKUb4 + + pNXsiIzewLKwj9ClCbwtO9WrYU3/kED4Q0goCm5wBhDU37xZ9XK09fG+moxr/xRD + + wz2G2DkCgYEA8dZ8SvrSniR3LMeSpg2sr8NGqgNQjZYez2IfEhUh7mgpkJ+lTl3R + + vq1gXOo6KnntYS1vX+D2XT+cg+mqHnoAi6Hyfurxe6Et+Sid6/L8/zofgiC/uq5m + + p6oKIFMdNGv7+NtKlWpQJ7vez+w0KVJpD/k4sTGfSXl49u9LJtX4iYcCgYEA72nS + + m87ykiPWqDLaBIK5bQoKJs52abreRRXAYgPEtATI+SsC9s4DlnXGXvXdCO7ZQcxZ + + 9WjDi8fwSWBwFMdybMP9PqXi0BQFc9IctGrZIgzmnBHAVYz/AIvTTLk+MgVSziQr + + oI2tCF0RrBur5kkEpAcPXT142IvjWvpjsmLunukCgYEAwEAVEPoyYvt0Lgn9X7px + + NEyVqWP3LodPuOc08ggQsFjn6guvuwvESMPFXjfpw4ioF9+psVvCHkEKaKdh0NaG + + BnrYruKQ1Ao+5NrQKBlD3JXVJHpqULqB6vm3ERlhlyHc7mlN8lfQnrWwHDSXBt53 + + nPYvhlV/XkaNzihO4vGooZMCgYEA7A9QfRZZMOUrYx74vpfSkwPiLI9ITEnXnRCs + + ZzhF/CX3r07MlmNdQD6SQNF1hrhS+UCvtnz8yldywjbXbHWXikzY56uS7w2+rouO + + iAoOXDeSLnKGTRQ/3t7/kdfYzmNXWTBq39yxrtxtb2C9Zsu6Sq03Zf0VqZaMrwjR + + wnMvyvkCgYBtOO31JrwBiPAUfQozPJykvOxZwfGVK7zPPpAF9JphP3bTisqTip/7 + + 7cqCMwbYmyTVCkfrcyHq9bZKtE84ZI+T8iy3j3km4nILniG+y/TWXC+1OAXuR4L6 + + SearomoSCffhCN3P/NiTaBRtW8dRyfwE2DIhy1pZoRRAx7WPGeI85w== + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 56 + seed: YQ== + zfec: + required: 2 + segmentSize: 131072 + total: 3 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:CHK:jdm6ytmx2i3ya2bllsdzxurjtu:iqr6tyysaseegzfurhuywy3mbbwkbbsov5bt3fo6oazpyv7olvda:2:3:1024 + format: + kind: chk + params: null + sample: + length: 1024 + seed: YQ== + zfec: + required: 2 + segmentSize: 131072 + total: 3 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:SSK:7nz6fgbygb663pqzi5dh4bvgcq:yrckbpnw46pkk5theuzl3d2inlxkrysgqpzpcqyt5477s4nq7xqa + format: + kind: ssk + params: + format: sdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEpAIBAAKCAQEAsqOsKOGJndmNm0TQIaF26G5Q9lgXUUWbFijpk/+/cpGVCCx+ + + v6mUaPZAlfTH6okXKf4gx8462H9rk3tmnXfk7OCpRwXWhaDzfItSH42RhPZ3ri/A + + xMpGytIVqLRfz/rKpP6N6tXe67I1CgOzGe9yZnsJ6cdi/R1RnJyOq7tF3mfqYS5b + + dUkvI4agEjozS8bPBYdMTdw3fv7GwHYDcP/MkiWD+x1IlU0ngmPfgt1yzF68bq/6 + + 8wggmTVwB3H7eIHRU49VgotudWftpwHsEwho8MGEQbJwdy2YqQiX9MXR1P4Pbvff + + ELKvm0Idz5DZ1oP+BK+qHEq+gOPf7PQrGO6OPQIDAQABAoIBACLqHHbtBeGlKKkl + + POyly0DIduh+9Se8TAB7xJNZlAiHbbJoR+mb8lbFcoAclIpBexaJBc0ngJbZ6KOt + + pbO3QDYP/uXTvUbm21AHRujF2aA8L84KpUmRI172yqbrgiJ7KOowmnpAjM5SSU2I + + xZOXGivvdlOL1cwU0+OhMb1c394E66nbcNycMdt5i0QbIC9NhFwNG8TQL+5dwxvV + + mwEBvxSkAYE57eERfRpr8/oQVFn4OGcfHU5KAEuSENi+xEwheMLpb7lCxR9Xq8jD + + aUms6POYvhYpEk60kwHem4/jHQV32S2P2dSzcZufaeZU69hFbmFPuwMS2YqLzHQy + + tRac2CECgYEAyUEWEGRNje/9saCqHXkHtSwTdi3uUr3fGSBhdKJMJPov0rfgojJ5 + + gxxfV3QvSO6I+YIEUI0Ij7nHBuBey6aRm/3AxWw4Qq9rPW8rbLtjkWbf3TeeC/Kt + + qFMfPXOeeQOUuFaoA2m1Xd8EXLbPr5iaCNP7ae80MiZNtlaW8P9kEOkCgYEA4zu7 + + vnG2upHww3sWmtdUSGfDWBhLpxtOdLGDhebFFKYwIOEoPf/ec36jptC83WoQofDb + + dqWRRZMEjozUxRPBtAS4TkePdf+7CDAE+9VFiVLOaZCFn6kTj44FEL8YvTIgR1Z8 + + 1qouidrCY/y84m2x3Tt0TyZGj+xlsk+wzW4F3jUCgYB2iB39o4XF7i5GGvF2kF0I + + yJ/hv+WY1/l5LAgaEKi2MqBOBDyKax4EKYbB1E0xMER+Z6Qw6Q+8ztc45pcObNlf + + vZF29WkhZX3M3hf+X1OiRKve963fLZw4AlTo9ZrFfWVvOKKV+AF4+yvvi0BBFKjM + + QEXYO6lLTCIDHXajFFgUWQKBgQCeG1HmkPizmBgN6/cuheT+/DPPeBgrjbRpPZpl + + 8MvwMjIKrp9xhDcj5Vm5GERRSxuHki8hvtH1tvXUuejRt41v1FjpHqGTWPyqFb9h + + +mMHybYVfZl8HgieOhMMM+riuZ38BRGXy5HWGYBoUdKbOfgoFtY2vEscmT+pcgly + + 5rrugQKBgQCUZd9VqLDMPsx5OOIzjYNMUb7xxXfp/XiZkhiXDbAAjVN6tpFGPMes + + Jrl5+XW7wGqsyJxTj2G8BAVBw4DIPnf9CVxc5Gs7DtzNitCn625Hn7Ss4WKGfk2c + + XGOTIuMlJsgvnbE2GsVtIohUIKyBr9lpPKaY+HQCejnYtkdu5o4RiQ== + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 1024 + seed: YQ== + zfec: + required: 2 + segmentSize: 131072 + total: 3 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:MDMF:gvrje64nislvfxxr7ot5x6du4a:g3yh2jhto3csbxtatoh3zgopu7heazuhlgrcngasn4l6sajcpgta + format: + kind: ssk + params: + format: mdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEowIBAAKCAQEAubXy7c1CAjoBJS7qJpAtDEmNV7rwyHYbDfIsEQqvuDckkrbW + + osGY+w6/CXVsXB7A3N9rABZu+pd4s+oIZqhENkZ0S6tuKVs8pgj0Wi2CNfviQ0EZ + + 6K/2BrsvW7dzN/NqCqX9jTBouOi4/wbGrRQf9+Gd5TFMaiNHNT8rRAfWz2Bl73ZQ + + UkDek9YIvS1anEqtjeX0BuxrBviy/NpDdlSYhDGTm+9SKPaX63MW6BUfNQYcZKmV + + nAXvw8vRuELq81lgwgXEtnNdAte7qz6n7wpcHi570/+sB2z2N9wPUT5UASavp8Ux + + ABKADrowXBKk1X2xn+ltkSAHTsGXb0+MZHg8kwIDAQABAoIBAEKaq3A3M+vZgsyj + + qU3AWq+z93HV9YJnvWdAiiZoh0IR4NePpKYFugicrs5FI2Jckz4EEPuckBvm1F7u + + Win7Qk+W8CGtb5p8guFnh7+J38/dsTX+tLyb0yhx3NfPkQ6picgc5TVMfdqHeMXa + + V8n+VELSU48+IZJVabYCnFFPYG7KHWI8ATeue3+Bw27kre+WCkWSfWW/Lj8biHBy + + FkDLxBGr5rVgZSodoAyFtZzunN0k3/U47ZF5bsUAQ0bpJolnXY7qGXWh88cZfxEv + + j9FTxKP7SbWDhdb/yh8BVVqbJhRgzPZuK7d4t4zlNsCpIc69vxBw8xwkBNKTiKwn + + 7Cf1xOUCgYEAvGa8iBLEyY3Z7nsSmtRBmBt8H9nNizK6n5JaOrHd6wj0Zj1V+dS/ + + WIfce+RyTPAX9cMwlCv39/C4UlmuHSTiH+r815GRPMg0fAyxz/z6IBQWB2pdTlTr + + URvDKE1wdXozOLZgyxRJoGuTL6XRVBYG2FDdCdOslrZtMe2Sw0Zjnc8CgYEA/FgT + + D4Un2FX8VPprr5wqJupsoNYhDBmaJv/NC81QEC5fUAX6myBpm7cFIwKANzsgE5op + + BqO1J+aEZs09cYszZ6eDbiKpkpWJEaOFT9+YjD9ViElRCPuTiHIK991eMN9N79Yf + + r/MAQyFWuPAn4OulKWn43EwjtXHfjKjnANQ3Cf0CgYAN6ehyhDBUUk2N9zjghlxx + + x1XbZFJxvUVbE4vmWcxx1y91fYIj+TpIZ4A5Bh4K4JBkbg3gY37kqLp0GntpW5f3 + + k3so0G9RddeqcaWQHra6N8GIuqo5ZrwaOVqoV0++3U97GLz9QnpNhqRQGIblFtta + + jl5Eo4VTfBWEYm88TK5+sQKBgQCvmv8QsuJKm3PxEx/zYmK3GDYmKz1uNTbgYu0n + + hGZuDEdJ4g9G+uRjd5b8iRX+2Yd1/LcGJtC/hpynCbbzmCJaxOkisL5/As8TVk9E + + iV6YYs67/AGHlcNSlcJqQUP2EMAk4kbE4/9PuBiotH+b94DFdDi53caP00H1mei/ + + 2+69ZQKBgETEqefImLCG+Drm+T+VsEvNbWUNQcgxQ6O69z9UiJOg0FR219m2XTVX + + mBlJESLuplJ3482qxJfrxZnVT+3ajlofD2zDg+Lxcjzf51eUusBFs8sZseVTUbti + + oiDTLpGCKDpyRRiKpwy/1kuLntP/hbzd3rSQBnxmP+QkQ3CukfWH + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 1024 + seed: YQ== + zfec: + required: 2 + segmentSize: 131072 + total: 3 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:CHK:7wz5dhkyob3gmkydzgbptk5qku:cl6ovq5a3km7rpfhyb3putg656sp57lhnp7aexbocsbbahdxnfia:2:3:4096 + format: + kind: chk + params: null + sample: + length: 4096 + seed: Yw== + zfec: + required: 2 + segmentSize: 131072 + total: 3 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:SSK:v37qrnptbkbw6ljh2n2ffwej5a:sfuywdbrrrzyfr2f4zigib5qmmviexrvukt5et4jucjwykkjtgrq + format: + kind: ssk + params: + format: sdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEogIBAAKCAQEA5dJPx3tIzQBSP8THjUZXZs+1DZ+gxWfrgC2bhQy4OzTbkeQx + + V6FasL6vDDNczZujTKklEIiC79jprpfRyjJ+CA3VkekebkrX7nbqNG8j/C8kiQ/H + + NFL7RSUdcGXwLo9xGw5A0FJsgO/dFvtZE/U9VsPoKhTyFzN0K/ieC5zl5D4EDYpx + + jRFd4fNn+jP99AfyZx2DodGFS5kIAHKzDPdqPzvr5055VQT3yyzYrXlXRWPz6JMP + + pXIyGa2iRlCFCgU2jha0tpopuiQWi+EgMKrT9eRZbEvA59JZth6NDsbN0gkP6+2Z + + gzqzbJtyn1vNQILeeh/na6oAe7w0T+pn57tVGwIDAQABAoIBACCAlFjVZi/b5kXv + + eftQYeb/5A6nrzCL6GHp0U9JQ7rX2F+zIolOoAlUBmSW1P6dDsS2PTAv5jiueCoB + + faF3b5yK/FPU4MFfY1dtyOSefTvanPOnYBhVzgRy4c12FTg4gBn/84mixoabpaxs + + 4qWwbrrZHPnqmWxPkhPv5sYkq9yR5gsx4G0g0qTsXHOCJkP7Jdfjf3bol5NEeXpW + + 9trCXtVd66D3VKGOHD+RkVLaYnG/8K2qhJSlqAn/6he8MdoQ1yKF900RPCHQeHb4 + + 6KeFSiSOzCe6L4dSKZ58kemu7itZ4IrIEK29x3e9D3BJfk51SYlGq4hbtAzl8dwO + + HM7w+FkCgYEA6B77+CBqYwC6HDd00NyMj8nC1Al9COaEWeN5kK8k9jm7zynsQJhL + + ngvxq5q/ZlG1f/BkgespLw6mlrGGr1OGueXJVnGeX+3p+RqcnzPJOGukiFc00+j2 + + VK+mqZDbNdmAXmtda6eF9WSxrDULkXCF8JduWr97nttbJPo9UjIR29kCgYEA/XbE + + 13/ehGUnFW9YFs/vAls/xhu4B8kGIZXlpIWIt8gbZPvx8R7d17R97H+u5g1J7i/b + + 9jKJvDhQAIdl/Ar3Q/vrmMTEvDiQuZidzBb4Z1r+C+1SDdYH+q1jHvAX+9arhpNe + + qCOlBT/tOQRFhzQqK/JnZNfXsnf8AOAYwMZ5pBMCgYA/nk5c4TWHUOmxVhm0LN5x + + glDdoIQebl+T616kIvy0Z3pr+wd/ZL5E4O0ppU4UEwz1tcM2QGeXOCK8ZoeNgg0I + + 4kveX2GS1TgtR/fpQl5CEm6T16Lo+Y6aA1JgYw1Rov0l47NFEDMM4L45fohfIkHz + + gO2D/bs5/NDsP5GS95ohcQKBgGda2wGVHsOWC83tzVngCHJJi0PZYb2q91kSqsXf + + vdRTQPh41DuifovLCd46YrNkj9UUpvlJumiJ/fV5QNj6D8IlI/jzo9WsqzdDSHVE + + mJ5suFNcvqztretGcLjY5q7G5sLFrT+a6VuuqakqWL+9QcUR3597dHVN//DLcMyL + + ImcJAoGALfrYfEd+RWcE7TEwfWguVwHskdBuoGgUkYwq9Vq5ECUQe1W7xuF0N0XA + + vvSN5NqFm+psN/nJc99riGjvtrkXyNJMhIhI/iExn21Me3sGttXCU3qVPxKFO8TQ + + 8zDQ0bRW6mxlPdUL88+X4UsIsiCU5zEc2ioY8gCq355EZw+6rok= + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 4096 + seed: Yw== + zfec: + required: 2 + segmentSize: 131072 + total: 3 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:MDMF:v2ryyypaudkr73uthfwax7ilca:b7eyzlxwml5gsvt4altyx3tirirv7edqne4krburi2fxl6zge4ca + format: + kind: ssk + params: + format: mdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEpAIBAAKCAQEA5bwGbQ9bJxRdy3oA40Wg3uEWT9EqtYIzrqBVGqu6dANN6ALH + + /+OH6LZvCYn2O4ztQ+Ddas+B+SbZxTfH2BH1A1N3aB1xc+VpY9ameBWLIwZil5OI + + +QODjbZmcIYx39r1gGt9OkCltv9+GMjsg5BxcwiqRwMSfS4//LEEHtb8R5GZ2mJC + + cEfkoYFsr3RUzPLU09Fz/9NNYp6q15kO43u+9+r/T3dGu+3FDB2uZgfK5LPFAbNy + + QrVc4btfVfUVjG6yWLpn++DWjcWDHNW7xCBeTS49ebyWXjiXB1AxD7uT4zxxhF7e + + a3eirX+8SKahoo+8z/EICeZdjD/tugdj23SUywIDAQABAoIBACkTIJ0AOV533DtE + + sYLxEI249dnIfpfcUyw+O2kc2iXi71tzn9mnD0Yy1BCDC7TjAgr4We4+crEe2qHR + + 0tfVghaZpkhFt2Ku2lSA7NuckndtLVSHit5m2+8K9S7aN3GcPve6gDXZmCdrb9qz + + leICAd685mDy8ivSiJs/9Qokiw+qaHdN3lxYw6sCU+lmrkDVMM3maDkwcmnU7stJ + + mMrDYNCfrf0MLYUWl4TJIogyhfcxL2ai4eqOOj9ArwL52AtZIBNerhr/xRwSYR8k + + KJ+5CnsWTlI7wbB+bhcmOUcMGcBJVKvn+pcGF/2PRtejpg6U70lOlyQPuFXSyAR+ + + 0Q3uOIECgYEA+AagJ6FyYjbZa9MeGJ9OeZlxkllOw7VFwSOw+XwOtIsUK5dRk3zH + + 8RnIBg2nJA5k9YtMjOvCfx0kgb+Lo4TgJv+vkZ8QOEM+GqgjJqTF9yuhJI2As9XE + + 9/sOervr99Bisq8ME26+bxturhQ1/ySepAI3LAkjj5ErnM/kaSujxksCgYEA7R7a + + LV+uaahBfvBFVSDGZFu+AmivY8a6C3xAhW5Tei0IfsgU3XYiyevU4Wo8nxDKbvGE + + rE7De6aHSEyJhHFiGM/Bx4CSQ0FgAGUdNEkI7bAiOHhO+z+ITyY0cPznVGejul8G + + pJnN26yP31Uf0OFkfPsVnucVAjqAJhC2WgCzW4ECgYEA6XKrCcI+/ExuBrwWhsxj + + O7b+m+Ytaa3UMv8aEyj+WlbRrFnn8W8wbjF7AJ+XIyvdQPRVIArD7YsLkogsscNe + + i7Z7lQ/nX12DNent8/CBWK0bJmF9s0bQ9yu5rDH23zCnxVFXh27kFYX3figNztGz + + 8+EV9v+/FeFo8FcIwSjPJNMCgYEAwnDzPhPg5NlJY/tJD2aB6Rfl9vm7IRl7xCFO + + k1wF7gDxn20Y1wWhv2y4s2O3dESDi3hXcChiWooTEzFX7xg+9dOftqXyyl3YiFpi + + GVbukGJHnYDiW1scvrK4fBKW63rVIuX7f4xz53hYvi2CmnnJOkd20kfxzVMFdLFt + + pt4+NYECgYBqUe8ZkWThpIjfjv3QihGOw7YClupxMMdDpGAZIfPQyDAsZuO4PB4o + + qPHQY9v0Uem+RQRViimIpDHaXcO0eDKY//EjEGt8RhSZK/ijsuBPAXXGSGSMfoa7 + + xA37opft89tUJW7e10uzICWi69TbsTVFjaQAMy+XK6c0/FWAoIV3Mw== + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 4096 + seed: Yw== + zfec: + required: 2 + segmentSize: 131072 + total: 3 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:CHK:aagw6esdm2msphvgnr3rlzg43e:zqxtrlee6hh3vtfg6ihtssjqvr27tpvi6gkmngnhiguttjz7yyja:2:3:131071 + format: + kind: chk + params: null + sample: + length: 131071 + seed: LCa0a2j/xo/5m0U8HTBBNBNCLXBkg7+g+YpeiGJm564= + zfec: + required: 2 + segmentSize: 131072 + total: 3 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:SSK:7wrmdyazliwoctoaz5tlxewemm:4bprnjqsci2ewyha5hnq6n2yxstw63dzlys52yizqcuy2qhbp7za + format: + kind: ssk + params: + format: sdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEowIBAAKCAQEA2QfEx1iF21VtPUDgUhUHJAKiourwkTEApPN74s9pcLrkypzY + + Csn4ngvT2mLDw7LalYiIkCmUBLO3p96e4LZjgnIYfDGFQLPqXT6VR13Ufpy0fHFN + + k1+aNaWTN3m0Tr8yMCMuP0aQkmjcVzsA29wV1Vg7dcZem6o4XasmSj+nazsLBFT1 + + jahae1u0XC2kisxemcnvMklulfaSlKtPKq7HV+///EnGuSZ8q38HZ0yYciax2p+r + + UJ16OrHMHr+WuCcSM0zaBjLpcmzJCTch4gm8B2V7EXIPtvNFa4E1ch122W9/F1TG + + Q3KSWuxextAsdfCz9CjhvuQJtdpYQHjXzLvi/QIDAQABAoIBAEb9eItwVFCbT+Ey + + YG3Y/P31crdxvADyE3DhSAu3ppi+OWphBXX5/L3Nxp1vovNXhJJXF7x4LTeghZl+ + + g1+jqUcZBRNSq3CvqSCZAQFYGtLTdWIjOanUIsAbid0ijS0Y81S1nUILVezeKfzK + + iwxfoDCp7MEogvfOJSPWgO7WhW/YRRwDG3Zcfgo6fKU9lUshJu97uLj6yIJkh0Fs + + F90MIWxs4TeEhK/QU46ZsX6lXCNOs47myvtdvhgKcyETnPHAMnZ8K6fbzcJl1u9t + + /maxtoqMTqAyIy2otR48LrASvbEPkrLO5wg+Jolg7SzWKQXGvgplv1Ao4m6Xw1zr + + FmtlbM0CgYEA7YVl5i1GFlDniM/BVX6KLcVqXSe5ejPLvTNcbDSp4Y0Kg2dfYynA + + 5fNDCWuw0t1cJgTFt6wfHwwbTynqlUo9B1cDwDHmizGtHOEiQh3O+2mpDnUGSwWe + + WAfu4XJaoqy9eRvaDfXOhMCzLjUWL6I96WFmpCSUlBPA9zewQxL69TMCgYEA6epE + + AnMfEGOHtWv8Z/llmRA+FS355d8zBJloSKIda7ogVDmcRiexTu/26yuRtRHHG8nh + + nq97xW6pdj0boVDnzoWB9HZnsiqvS8spc75B1Mh0Su28lkzy7BaEtxcphUa/5x6n + + OsR/QCiN4CSVuLXxG0+u6PidH311ds7uZasgZw8CgYEAmmrif42phjKdBH4E9D8r + + OGyjJOMBm6f26g9tI8/tLf0S+7EF+6MWjKjlSUehEsXk9baekDWvmfC2BHZ80wgL + + uyzf/GC0wIPQRvk6238jpKHhzctZBwclFZg6voko0Z+6IvVvgynuVLIvC3hp7xfs + + ZkDziP1bNxXMmyyyRDkfvK8CgYAysYV5rm9OAvP3OmbiNadyC5YYyvT8f2m0FncG + + PrP3k8fL2QxoG9QOUm0FvFSAlFC9UfwmgstlFz18lXO2ey0xkbd/PmXss9l3qJjc + + L6Bet+6UCn+zZwvCZILwlwF1k8alFPyS/ODDC8bri6Iy/KM7EwLKFI8gsvTRAbmi + + qPqFuwKBgAHoXx4HW9do1WyGsCaJ8rsAxKyOx6wjx7MJO4NoSVcyudQXE2rTkaUA + + roqaJRZPt/Hbrj71YLsow7/LvpR0jvXFWPLgaiDDwQxehP/pOVVzW8VqujrsSAlL + + fRGCTZKGaWj5NPVjyk1NHrBqIH6tVuRWkKdLCM93+HoCz8AmnTGK + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 131071 + seed: LCa0a2j/xo/5m0U8HTBBNBNCLXBkg7+g+YpeiGJm564= + zfec: + required: 2 + segmentSize: 131072 + total: 3 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:MDMF:3xhseftp3rjgmnhahuvptvg2yi:7dsywoo3awdp4uzwhgepzokecufh5kaodtpefizfv3nzjtdkcsda + format: + kind: ssk + params: + format: mdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEpAIBAAKCAQEAyJfHAmbSrehM7PZPfuHaio2hTr6kvHXblMiojGhyZ+c+kpca + + x5l5yzMEgNUMuX1hwawqD4zUfh4C+ReWuXk6sQKonRymRfUFfHy1Nrr4tAYo8Lm4 + + 2SE9fSTMXoy69KZgHcwMuowMGt7x/uubaVXlFEdm5U7imJ4HbzHyQ7QmbB8MRuQE + + m/CRTDW2MTRNPXNAfjYY5Ll+YSUgaz2H6AwWJFrJ7/c/Lfn7a22jD7UDCqN3/CoF + + TzTjmkgum3DfsXf97q0qyV8ZswJTXRGj6qhu3gBJKZ8VNi8BsqjVZ118+1oxGUJJ + + 54xlaSVF+yUJBvTIbkLm5goDKxkBBUptD+pSywIDAQABAoIBABtD5h0BdOmGhc4/ + + vJZ2hIoIrkBR0Xp8WphineZX3BUbbXnnaBBxYAiqqpYIX25nCH6WtDDg55Es6yKI + + fkg/niapQdn5JvCjWVeOa+NAjsWJgM8Xr3Rz/DOiaNUBM/hFFRN3xNMmbg7I9wO7 + + aqhqsHSNMANDMbGk6UXH+DITrpVLdTTjhi8iLbpXTz+qh0tzIP71OBqfI6IvO3Fp + + Yj/CaTjNBwovuClKMKQzmLCS+arNqX+ya02EmO3IlqFA0hzd+7QFHuNORqu9et6M + + 0eDsNA7jEpFpGFZCiiC5ke2uPq0hsidVb2PitADYGFzJE/X7Ey+AihLKNABaVNDN + + wn2vsuECgYEA+LJrb8zDD1fZN3Q34UKpZTqXiUJj7a0CnqTlWu5MP+kctZhRcO2i + + W41PhJH3h/UmMfYWqTp6qOuPIsUPttUQPtYhfIyePuM14h1tgUzd9XpwJC/CAx0W + + 1gze/ouTTFbN8qygL0dXQci/SMNYFwyQzzrFva2YrHUFo/kHLSl4/2cCgYEAznu8 + + M0C9oj3JT0rJaZ+tGwDEoy5A7INP+0Lt+BB9tfPKyUz+Iw6642QlVeS/Xt6PqET1 + + CndcIKJ+hfq4bVMIqgB/LqCdKZyGed6D0jbZKPK0UBHIjxH/54Bv/B/zrX/d/vDX + + P7e2dt2M7GjYEv8hnLR5UoIBLfHIubmJnvtqhv0CgYEAzdmw//s2wa5vR21VC4lH + + +VhEMgLX/9Uiw/mtNlTknEnxz4Xic1ze9HTFCvBfORP7p4MQQsb63HMOKTN/zFAT + + gE9xrEwgd+FNqnm9ODdNyXCs/ebh6f3b9xT0RzF0nM2E7odgl4Gvge4OFsZKVdm6 + + yyzUnCnio2zBXHY2MHWRh6UCgYBGji1e7g5ec/Jn78wnFXLXOUn34IQ6zRv1ZYdf + + LnNmSynN40sru4rMzJmdYg6qYi6Adx+sNeD7HctSCLwgTzE0tfq/eg55+4xP9GLi + + 3+8QeO54Nbtsd+ATwOWDJ3/il0DKLo2+rg3hTA8tcR30T82yeFDEirvQcT//hpCq + + DIr4GQKBgQC7/Hwv2GqWVHp83LqlGFUp7XiNEUbWzJeDhRtC5UrM+ZAOzelwmxmw + + 1yQNH1n5LVNrTXlXQl6RZ9OAGMkCsKaRdr6BGnkLaj2KyfhfeQh3ieqnbZ4AL40U + + dBIMMBC20tiqJ/RgTG13RnM3ZZnIq3o0ED/7y0qR5Kub2fgEvVTg1A== + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 131071 + seed: LCa0a2j/xo/5m0U8HTBBNBNCLXBkg7+g+YpeiGJm564= + zfec: + required: 2 + segmentSize: 131072 + total: 3 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:CHK:e6uhixvbm3g5amrzdzd3mnmsqm:4n5aafrb64sqpfnhcdpdfrk5gh2qznprjky6q6jybuuqy2q6z6mq:2:3:131073 + format: + kind: chk + params: null + sample: + length: 131073 + seed: /N4rLtula/QIYB+3If6bXDONEO5CnqBPrlURto+/j7k= + zfec: + required: 2 + segmentSize: 131072 + total: 3 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:SSK:xryj47onuklcp2ogizxiopkdtm:2p2w2seg2xpefulijcksejccsrbqsa4wvcvbowjxwwkbhvuz5mga + format: + kind: ssk + params: + format: sdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEogIBAAKCAQEAw4YAcqbljcbU1/s5lRi7p+A78jT64FIpGLyq54HyAM6PWbJK + + 6UbDZEbEIknszxXZru0go/ehynHZ8RYxEDGz/tuLBwhevgqSwWA6ONOdrS3N8F9M + + mvo4+TOUeD3uUIhUf0E/u0yskenSeTTuDGTrDgf4U2jmMpXjVmIfTBcPhveL0Z/f + + Yr5qxzY1wCWYv1+e+MUppClc8kLJ+MBEL+mbtDSqWl/DKgfGTR4Q1aeTKbdfVVus + + ALQO7HMSt7MuC4s8dkqjZtn8YhPq8tYJirzUdC8JHC4m/oLUHpGMt3/vlaBQgJyS + + KCGaNlFQohfa2qR528DfTP/A3xR06Tboq4pbBwIDAQABAoIBAAaWTzZEfHSjHUy+ + + 6Fs5fSmPpi6DJJFztXIGHlMGyu4PUJrmN1MoefDdjubBr9OCg0zfmrayUNiVruw3 + + vWSz62a0vu9BgZOewySIhwtVyKZ9Lw/ElQTCwukMGOMOKoKWfTiPdbGQpBW6F2yK + + YQlLJWIi2HXS7PlST9l7QPA/Y4BQkErTIH/MA/362q0PNRc2XDfzN6U4F/SuM2Dj + + geGHropqC7q5Cg1CLtsUU5XyNrK2V8JCUjnhdHKAAHEhUPE4K8K03jqHcmeAAWri + + 8rXSMXqhKxcIJMZcDu7pSEL8yGRnNCIIlnlRKCevbgJE31I0mfA8zfnwvj3vkx2U + + guMZaokCgYEA8oMxmelU2iymtUAVj81nlqBtJJ2f8HsThaX934x3i7GtlzpRDOz6 + + BLlRTpSR9+w97RCVui4y9Kn+B1KRFHvBfATcs9v3wVsbIu4YLmz1CKXU5XedRFxI + + BzGF2CkXk5LetyyUKY0mIUyar1+pQ3hF970kmcOQZdjrsLee4+DwJLkCgYEAzmXL + + fuJw4rf+122dra4psLQsCk71aknuWvJ6yCpQ/CWsjMmWt7HXhY2iMiKY7FHVr0Cq + + GDGItWzVQMwyFWzGob1s3dkNXsDCsaMOazKLwmbg1aEo7d8ttME58pbRHSNooZdD + + mzoNlGTSSQsKTepQ+cLcYaRRNp8FmqAzHdbmHb8CgYBQJoQSNkfRA8jlRpTZvi1q + + XwMzgtUFiefd2AqcA7TO+p5AyQlYmEnZndX9fqTvp6if3UdfDT3SFwzaJrPEbVJ5 + + RrIaz6yGvzGszbw4O9KQVR6T6ICVw1oa5ocx9gLQx03MhHNDeF8Nyl+lbpxmrC2T + + v3OFTlk/D/51nXpqHkHAIQKBgHma5lPC7MnXqJGa5v0OkUeoUA5eyR+voXz6Qrcu + + n3qAY/KrT165rIbmlPq/AaSy7piMG+uXO7nQ/rBn3tZauYlQBxWKreL25X8t1+/2 + + 3vtSDAQyKOBFzzMhaZfxnhFx7FLQwadyg8+7u14H4DFZ7g3J7nilDKiG9xFMc/GP + + zRMXAoGAcYUuBGp2DFGsR1/1OSk1xu7lGn9IgNGiZlrB/bjuhs0t3/ZJVi1l7O5w + + 21kFvpJbKHL8VJGk6kJdHT43zmcHrFPBkVK84mLXHXXupRn6QUJhG6il8H8XmaVs + + zjyWVUaMjc+MXydts+MWNivvZLqdQ7L4suUwJIecdhFxMGfgfGo= + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 131073 + seed: /N4rLtula/QIYB+3If6bXDONEO5CnqBPrlURto+/j7k= + zfec: + required: 2 + segmentSize: 131072 + total: 3 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:MDMF:6fbvodb4cfvfyztxliweqitaau:ub5jwuj34npt3jixo22rqzoa2vpc7bahk7sdwcb3k3nukjez4y4a + format: + kind: ssk + params: + format: mdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEowIBAAKCAQEAzbgi2MRkM7QQKadt//8U6R9839B/oFyGFcR6BBn566tGvoA8 + + 62J4O8N1Mzs+1TE9ObQ3meejr7SL0Ksr7QI2rcfpes+gUDY4eK6Y8ypxdDmG9tnr + + CsD0RpZP3N6rFiqCDSW+oCuf0s9UoQyfKtdRCW8NttcQHAplNBFTOUMfiqJdknLf + + KMWsVcH6t0H/a4jkteXpHd/HS1TIzlcBXVWJmPyhGIFGDMaXhF8aZZRN2vA8vaed + + TuoegnNA4ZJ3fbhlbRjpbKsugrmLitxJwKHuTKuHWSbYOF6SrF57S6xYC1g3q3sD + + mEhgDK2YdTMp5npn+wMxLsZFIaQHNCXaBNQp0wIDAQABAoIBAElZORt+0pdYwVaI + + uwDGq2b/ch+/EHJV6v0B49tog6KSnBO6V345ytLMOxJ8MkgDWWgkqJp4a/Vu81cA + + YRYNbv+BQu8l7mwLGRF2d1RkMrWU+Vk0k8huyeoNGAaRYgDyQRJ8/b3QMBkTEYKm + + pG26/crWTNZ/UeAdmL3622iUVT+xRp0cQphDRuF7qBaD6E/+CYbBAZhvFEPhyUEC + + jiVz3VgXUNQaheo+5mETcgebO9GInGM0Xx13j4JuSJIkOkrRwcuP7BKAoODvRZzy + + uFtuEt7hhQ9Ok8hGpkmJhgf+tUWZfm55IsHMOaYU9n7BVocG+kAsQrmqWHI9LaPi + + zGnW5bkCgYEA1y9/8wj6Mq5vzm3Z3M3zhOBoYpsZ512zKlxJ7Qkj5YB71VgdNOEq + + Ke97PLotZ3+HS625AYJzxFn2vXKtw/TtwZWODz1eFy8OqNlUBYcy2Iv/rLhpaZ6S + + d2jEchazdRvsHyy02smtM4mPpjl9Ta5L6OCn8ZOSjDjRCcFS9GIjm28CgYEA9Lz+ + + mrJAP+Xv+4Kr/whln8MlEOAhHc5XbQyBaloyjXNwQPjaMLxsqmhQSBS3J/Nvtozf + + Y0IsTas42fpZhC8CfCsMmozETlb8RNIRHB06vFgjsr2JCpEMyxPsxw0biUKWxHoW + + /MnAxMfu+SCedLe7fgcUdFCHGrQyjqdUTTEnNd0CgYAliTuggWhjftoyACeIxMQ9 + + 4YdT7ApQuZ+PPBoJJxcD6a2wQXaWytA4EHZG2ZMiArTvFpa3FLJtBeRsl5yIGil/ + + Iz9smR/Ym3TLL13guPy9mW44CtMYgXi7K6NY42zaeMxvg7TolVWJL+3G6TBaZJUJ + + QmcJVNyzaRXNxdNanePoMQKBgHx7HK0meROHKtcskbs4VCg8o7+/oFh+uW0X7UNg + + +VGFI9WIPcKINGVAhYUENFy3r1yUrpLX95zRuCr28U0QdB0f8Fszui12hP2kM1uv + + ak6eLhod2XRsbqtkSQy9mAHqwrDQwJx3KfttDhndA3uucEkb5MV8qBtnCEgSyAgz + + NhRpAoGBAJ7HJVTQGZcT3DF/IJTg4JaINvsOtUCWsyegm889udpzlqoiZLAb2Ot8 + + I/Qis/9wawNFKbwNRYtWyuBivlpx+jnkaTLc8mqqoOafe9CS2b4Uy7AIbf3m/F9h + + 6X0XjnZrqd5uZiuaa2URuEw1srj2pMDTehP/OraDy/bJjrgzZ33p + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 131073 + seed: /N4rLtula/QIYB+3If6bXDONEO5CnqBPrlURto+/j7k= + zfec: + required: 2 + segmentSize: 131072 + total: 3 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:CHK:abfmzuiczjwt6e67f3uoyg5i7q:rk7ie3afnp3xgvnxu5e2vioq476xqwslqxgyx3i5xxevsjhkcvba:2:3:2097151 + format: + kind: chk + params: null + sample: + length: 2097151 + seed: uqWglk0zIPvAxqkiFARTyFE+okq4/QV3A0gEqWckgJY= + zfec: + required: 2 + segmentSize: 131072 + total: 3 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:SSK:rgmmkcpc2g4srtr73wk2z5g2zi:mp3g4z5srhmtfxj5rgveq4gxbxnlcstsfa6kxi3h4exbz2f6ka2a + format: + kind: ssk + params: + format: sdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEowIBAAKCAQEA1gC9/WdwsSuvx6Sl7Muz09YTew8YjtmY5JqPea9mlo0354Se + + go83JHmyzpImu3GVB6Z8h0rQzgPUgc9S8/79eQC24lrO3YjgPud+syhk0rW+FyiI + + URGtTswtq9adXVYNq2YsxeErdMSEfVzUHHTQ0wuiXbBihWmKg+c+fDgxgJms8mAg + + FKgL8LZsic55ANUGgWJTr6/M4uab1rgAlvFLl5Ke2XPvTjMR0YGjseLU7koMLByN + + UyKalXDLrL3fmQjE6H07ZirzANuje/AzC3ki1u/32xEDrMpWNukirfcHdyj2cyjU + + jrlbclQPC017pYhzHbkuPD60wSJW83FYP0UxUwIDAQABAoH/DDHqAs3rBnEB8SPw + + 0GOpm5vLedv4fvmqkMqLdCKbh/hGUa+PwHREHZDh6lBDEwzgZms+1yvrQ4O0/UYa + + 7IEl9SQfFpRE8KncY/CbaTAYJiQdjgwNdDCmFProMDZmdlwTOQv/3zfJpH6Aze75 + + xdEdr7RSSbMCF4YTIYjER2wNCpKfGZ/PJmo4mtFIm4MpZj8TcuQjIher7kuJfHEF + + h54QgbNdQ6bmXuHbGb8k5vAqlW2O4eHxnIsasY+EeMR6uUTbEadPYEZcgG3aKq0V + + smHLlkes9i65J1DgjoTNtr/tcBLyOqF/LBbEEQzAg48DD3gyv95RVGdLdtE8ZMds + + 9yFBAoGBAPh07zbXEDiW83RWYT80qwuV7mKgmRkoaH801YoIchEUfz57hVCG2OVO + + hhEQZn8D0fcHVA6mekD1Ba69IObkgHq1QiyypUgMz6fP2AamviIC+NGRnptd+w09 + + 4UPiIcMudDKrUZkw5rne5YrKBaVsucQxlXwtUXA6XdQ81c+/5vjBAoGBANyABiDl + + vw/E0hSvJ7Rs1KL1wHfG6pXyNEswIQWwA+1VHw2ET69cmCJd7qRlQqDyvkTBp2XR + + UtsqUvcQtVL/G0MmT8NIXSE2vvFEvSPvoWxrJhB7M9jaBl9BYIdeZxVzYMiHo6Qu + + XNypZ0m8GLPunOlyJivMLcJpHcHgY5PynHsTAoGBALX556+yC4Z3QW9nSSjjKZh9 + + wzFn0Vq01vy8tN652toZui0IiZd2fOxO/DEJYxkKskGNk4p7crWbAQOAMNYMbPHz + + Srm0SwyfnYSa3e3ZOQ9uP9I3JwVC63tCZHi06uerYZ4vDr/2KjffQx7JYyNLpDBH + + 5OYjxy89ALZPrIbSVpjBAoGBAKlQ1WPlhzUAmaCwbvioqQ8JTmWrJO9HMMibiH/p + + jNptho7GjrnFjDy3jExIRUV5oIkDextABTOt6E83UUUOB00k2hLGOl0KwMxbUDGM + + DJRIIs59DG7z2/jBJvJLlzRtiF/zZ8DmqP/4RQvll8Jy86J+uLjg7DJgrSz2tQAi + + R+5pAoGBAKzHFNTmBSYYZlWVeiV7J21iGle6Vc8DSZHMzObd2aYGx8arnTdH1Y2a + + qglMSdUsUA4jHP880xFoMimcXnQOLPeKsAtF6LCOJw1HeLUY8P19fvm5mZnOy2S3 + + WnCEuBI5ZlmF6l1fnRbfdgC1bA2Vma/94XkWG3vqmiHAouh0vZfC + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 2097151 + seed: uqWglk0zIPvAxqkiFARTyFE+okq4/QV3A0gEqWckgJY= + zfec: + required: 2 + segmentSize: 131072 + total: 3 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:MDMF:6f5qr4yrplexsp2cdfrozual7m:qzniyjbnxrtvqgv4sn5yq2mksiedndju7nw6of3ea3ku752clhuq + format: + kind: ssk + params: + format: mdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEowIBAAKCAQEArNBRHeEXTwth42HXbxwTZ8SuuF3H60KUQRFx3zXaJbWAl6jX + + kCHuNMA6ioK7L2cGMHUfTDzNiExP6a0WLVteH5voQcgnJzmlFLa+WZCsyi+srvr0 + + nYm6obhgj9ihr79XVGmDNFxKHPiNmXPV8eqU3SLuSsU/Y/dSA7NLUna070nwGgG+ + + ctK5wxKUnbQZL6bVBo1qhkOCuBHTJBHCoL4P5xv5cR6lducyi2IVtUB7lZmUFwPe + + IptvuAZp2l7kYQ+/ScpaGk/NIV+69QsJ9dsjZJVE8mk7n7aTvy9Teojdcq1JhQCQ + + 1U0PKaqsEklzqkLHJ3xWlMGLfncFhnwwubCYQwIDAQABAoIBABe3VwqEs5AzfbGY + + 4dnrvnYFNf0zUZZlwrbTUA9T8qYuLIGjuEGdhnVS1DXiDxJITz8jM7JgvcwwvN7S + + 1DJRUa+A0/UDJOxrKs6W7bSY+D2fIVG6OwvLtQMwrH/ROQ9HcRKykEEFUV58deJT + + VU8n5FocyxsTyslLTcQYPQQKKnaUMy6haRNdr1Uc/NuBc6Zd4VyqJ2ltRYKFJV9l + + MXzHtBbPLuwO5nSIuDS+5pnVhpGhY3m8bYXcsYdpyG/952nvf1UQNZr9RwtpzWLC + + mkAh/GAArW8CNc0BWeqsnLsLfP0H6V4L+agzZuRoCKi8g0q1p6Gye8dIdXcbuiJO + + zk7sr+ECgYEAw4e8zi8f2Io1aLEa5ZhZnmw4gHKiP9qO4tWBg2iAaqAgDqdhSj/e + + u9W9peL3MS56bLKEDGYUlICLIrO8JVHDpnUdS34HJ5gr6gjPpUu5ieo5AJvOYfHo + + i6XJmSuQYYUkNd3Ps0pyss9aR8S2XUa4XAh5tPBuMntENWhhTlBdpdkCgYEA4kIZ + + dG0tQKpsg6Jp+e3mlPjYElX8t0legjHpxIsJE9CKPew+DaPJbLSkoE1bx8edKI5X + + 4LMvQ2PKS/BugSC/c5z1KhJwUtvHdipW6sNmaNze7hpi7NQkBZf81BspNmfJ57Tt + + 0GMlExfpNcWZk4lBmWvNIpZ8UFCkWL2zfNySkXsCgYBhylxiXm019oGZt6H1HEoO + + Ep/7ldmRx/RYfGHG4BgBu83spkfhQ6pZFSBBfA8XOOCfxnSGYvN+BgAQPgYmQAtz + + D/Wz0PcxFUk5RmjbidDkqhESPdptX/hnB2aZRZFzRIyEqEf9qolM5qmHZVmzsu/3 + + j4GXPfxPIRlPAMJR0Z3UmQKBgAmoAogahLzmyRzRGK7G/XlMKYSW0ONNqU/rK2vs + + 9yU2WEAOThOs8tLF3uTMiGc9WLK7aHq5iwHYR3D4QO8X47PedgQmp06R/LBJXE5G + + qp89FfKZg7FR2Hu4ody3kAm3YkGWUjP7l0B6W8Sku0o1qGwQ0r9wJrwSxQDYj8l7 + + bHHzAoGBAL4OvQ9MbyUvV+pSoqR2EGCKjpRfKD0aiQK0OYOAuHZH6jPNkb57fmop + + 45WYpbuXK+TQIFbtNApszkTBeevrZOT1sDCldMG7l2nRY8pOZsqUWqfiR8wtVVmo + + 72jkBD/xAS9wztCtm/b45WYKuv8s7SLhTD/T0aav/Yv7hyUxbDYq + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 2097151 + seed: uqWglk0zIPvAxqkiFARTyFE+okq4/QV3A0gEqWckgJY= + zfec: + required: 2 + segmentSize: 131072 + total: 3 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:CHK:34bzejdzpqinmuzdmqenkidf74:pcecbl4ulygiadgdagonbgqmpb4lomjz4v4vqyssr56reyib4q2q:2:3:2097153 + format: + kind: chk + params: null + sample: + length: 2097153 + seed: BTBX/ampNfLU+ox7xipBGiaSbgC0kcB8Gy7BkJB4oKI= + zfec: + required: 2 + segmentSize: 131072 + total: 3 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:SSK:whwvpucrncmbbxnhld6palajce:muvfanna6fyof4h3zzswt44gjs2opt64kbm6jipjsex6qj27y44a + format: + kind: ssk + params: + format: sdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEpAIBAAKCAQEA2LyHGvyH5SaY7lu7XkEwcE9nT0K9x2fDp/yA5TPlJvV44ru8 + + JykYGquQwKkMYtA8wVxIC8D4jHrWbwbhM/UUK8b33U9G3hgYsi1ksVDXO0AZTKNL + + GyCNjuF0bPkeWMd13jwOChyAHaRQdXGGRbEobALDljE0+n9ihEw27Tbw6vv7pyog + + u6XEwURFjDCTTELMIIA2jfaUThQqjF9jMNZAQLJw/w40BEIlyJ/y7u9XGzbsj1UA + + RC6nTYeM0hN53OTlNcZry3zIO58zgPufkSUV1LdqYj26mUx+ILdtG10VIMfDz0fg + + WGbjKVBVRCw8Eb7NhGaSgjGiFfv4GFU1/yeocQIDAQABAoIBAAIIR4nhs7+wIazn + + hDg6qy18bcsHmfpjvykPuYuHVcG3JCMX0Kh7LqA5EGEMHMMpnHjm3dVADJya83CP + + AL+GhRdG1PtuE0DWepcjd5SgPfputrtaHc8jRPc3EZWgLZA2eO+Z7NBHCuOsRHsL + + ctoWXTs2Y0GLlH7OVmedmYIWhYJl5ikfW+0nCFGpFtgAigsIJ5RzSrQWTKNVfJFM + + oOgvchPC9Bjs/Vc5kDutFcidkM2gdDlKIjmG9Ux5BnDfPtkpMokIlj3vlhKNkE9a + + kNO3US62Eg1OGs4cTHcKHAkjwavxghfL5wglgKaK8xhvtshr/ym2uLdO1j3WUoMJ + + qZm1jtkCgYEA991xSuwkr0xe0OacN55uFlRRc8aLeh8mV4r8POqkFjXZVsigZ+1m + + FEMfJFVj5J58Dvl3A7TgIGM4qP1ldnE6Fb+GBJzTRb7MTjgPbUwXYmBPmhsz2pXA + + NHIXXeUYOMGrcWGLy8Roy7ppe3Wudea8DYMskkN9kHBCwGzhfZ5VM2kCgYEA39mL + + IYR4H0CexpYca0VUA0DbyCh19fkR7u92Zj5F6VF2GqmBSEtRa4XAoObYvNEm45f5 + + gkx5oLgmahibdB/iaQc4JdczqzgD/I92LfOxWfMcdfxNku2VLYtTFYNRJDdSmZHt + + GVS2KVOtxm524PZ+Pse8YdB2tVJh/c6vCpU1k8kCgYARyXta1A1h4woe1Z26RA1E + + XvKla0cREXEv8RJe0LvLuDuLhcQ1EQ01QQfYFKShgFoIvRA0XOOEj3o+bki8si1n + + 6CGW7SYgKCwDJPS+dCptbdnohjE3a22qldFldI5DbGqALW7ZxZN7ozn0mSJW5aLz + + GUm2iU9WcSfpJScdW6JjmQKBgQDNWAuIcLOcv7OnKlbhlJRv85Rp9avYO2ZXEDZF + + roSFduPnq2zcO7Nx9h1xvLI/64FIMMaC39KHO8aJdw9LpGAWxrecBuDwBQ+rJJNd + + rfoYMKsAFLW4vdcmE3Pg/Th3B4TvOW0N2qbMHGYB7J2C2ruOrb1C4W+z/+HCaVIr + + XBrs+QKBgQDtHOWbd+IvyAc+j1G8yL0vpoxsaD0K4bUzywCeI3FwXOoPU3eyufs0 + + pqiJKaAVfsIXTerk5SCUMwBnp61feQephL2X8Td0oi4dCYLsRz3c6a38F3jv44Rc + + UQ0gnsCGH8EKXbAsDUD2qi8XsN8I8UD6hi00JRjuliDoWVcR6a63Ug== + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 2097153 + seed: BTBX/ampNfLU+ox7xipBGiaSbgC0kcB8Gy7BkJB4oKI= + zfec: + required: 2 + segmentSize: 131072 + total: 3 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:MDMF:wij2d6hdexhefscaqhtvj36ojq:6njb7giycq77cs3cpzj42bprznakf2larkyfpp6yejteu6x26nyq + format: + kind: ssk + params: + format: mdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEowIBAAKCAQEAwSHlr0hpeAdvXcYr3LMj8Tiht4sQm0SiVnK+26BSHx8GdRkJ + + pgpMaAT74Oukc89X0gHCGN76yYmf9FertkpYAtlgj9yf/BwIWMnPHqFkhDqSxi8q + + d1PR6xOrRqLwbdtfB6gR9uazjaQ2nnsxz8zQbJ6TZmQR9ipJP3FmLSZLZ3jL/8cI + + 71c6/AaxlVxiKMyATykoyWkFxw4JxuWD9ff/E0fxJUd2dApvFIQPAK2RZUo/LIGH + + fwm3oHqIFl5UQDrJArZwWMA0bCTukXIIM+dxBO2n4bBA3MAX92K0baJvUaGC9L/v + + n5jzdva1Us7zx9dRiaDFTHhDdH3A2Zltd2GRcQIDAQABAoIBAAGKcouDuTEvpH/K + + gQ7fGEE1pPnQFIaQ7VdhmG7rwZSisIr6aMNRYNc3MSFT+fKhVROC0h8ft3TIX1Ks + + M3gmPHtkgPQRiVkiCgmXeGqSSgPcIdqDoEaV+uunDepSakGtyCKz+zWK6m7twDs4 + + PgSOPzxa8RpmP8aE2KNL9XSUjLyUpIqoYVamR13QKHQwImBVpfjcHzOHFrCwfDKB + + iVhVqDHPff8zmlge0nSQSt+ikJ0l5XIL+WTDWbHewahbLHP/hdN7QCbvAkENUreD + + GeMV5U4NOYMJUv+UzdL2PHGAYr2WiNtZzoaBbKO52l16wCT9FLlOT3bPxPBrkx5H + + gRYTZEECgYEAxcd8nHpF7WHEf7m5oe8OCiAD4WzQ89yNIw4UIW5PjcC/hivjy5Ua + + KeAOFPlIeLsbsKMzyxU80VjfGwdHsl9Tj/1ZckqWW+l5H+rwLULfy1ydeRLDQGNl + + G5IBiF6+Ps1G8Qwnz3COcyPmHNNAuqXmJs081F3o5wi86dOHCM7XvkECgYEA+fw6 + + Y/S6y7TJwAPX7L7JRua1C09az9BTMpjWWqBh/pVjBo7jjFLOKfGzPZS7QoZqCmhV + + m/tvTPi7EHcvSdDW0y2JJ/M3fmkxhohSTzdwAqEl891mGslSTPtWTdZ8MJyJb/I+ + + z2q6ViS8+APzjf7F1/6VpfCZF4QdhcRptqQKZzECgYB0vp1azH5MckKIVnwyDydN + + aLqBrTbmS9Dv2VaeqTvCY/1p2Kx9NoUcJMqLLN7PjTr6GEvxW5bryDbiAHkc3FI6 + + E4ViBo8csAM0iPy+6tOpegDmP+ILNuCu1o+bDLnl3kw66z7wnvMnGhCyAS0bP+RM + + ESgP/2MERU8mAxuZYmdNQQKBgQCdbqxzMLfG/EcmZwU/8nMd9MNFqScewyryLXCp + + SGIOi5P+mFRTlf6CSdZAzP8ViUMU5NotTq6sgeSFHRop2ZzBB+ddwn1LXgIzoHx9 + + qQMglM4rA15/NhRfqNWUVaSGlL61QpEt3SAWijJ72zkyTqXYPluOUrSHK8vP539P + + 54UpsQKBgH+D9NjJvplum1FL1YI4CORhVYa/vYqE5b2VKxUUyruGVDzgxKqkITj7 + + blLBlIjS4Cas+s2CPauBtYJiENZEhCDJACKjU72YGYnRU2GbGhHb49DQcBhvJk49 + + JI/tLgL+DSThPLdeIkAIRnpx9zJmCkVKOd4xAO/ta3k29MYOr2IM + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 2097153 + seed: BTBX/ampNfLU+ox7xipBGiaSbgC0kcB8Gy7BkJB4oKI= + zfec: + required: 2 + segmentSize: 131072 + total: 3 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:CHK:ej56vl65kawat62wsxuvbcd2wy:tv65cgghaddq2p7dyvimmff24l34wsx6mhzd3dyprfvrdo52qtma:2:3:8388607 + format: + kind: chk + params: null + sample: + length: 8388607 + seed: w6uP8Tcg6K2QR905Rms8iXTlksL6OD1KOWBxTK7wxPI= + zfec: + required: 2 + segmentSize: 131072 + total: 3 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:SSK:v3zxebrrlgskiey4ueqrqz5hku:nxccancsocy5iugopecpdzgqntekq3mvvo46s7nm7msm32fcqsga + format: + kind: ssk + params: + format: sdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEpAIBAAKCAQEAvWKXMowTtKLHOE9bzZz45Km1vFUmP7E8y8rrImnQRIrhxz+v + + eFGcEIPQMyplPeLtfz9Q6DUh+XD+t2kgXe87ME06/a3ZcVL+omlZxZnilHkobQYQ + + l/lrLl1vtKK8721KE9RY+r6lVp+DKyQwtsuzsGYB2p/ZUzkozqveQ8yT2JOjU42R + + vLxDTd4gkW5KulWjnHp9sS1Ic+v7FfU/M5mNpbyg+b0Fo7G4rghfeV5zg7UMoxDc + + 2T8wXkopUBAW6EJj45SuGcOvACEt7uLSqEutpHNF6kWF9nG+XRlHg2xpgn7SyyhZ + + O9sfi6pb+Iu9J+vH/XyWmVgHFzp3CZJHMFxZCQIDAQABAoIBAAP0kO1WlRvG8Yu4 + + xpVRA7a836WPDrUyVa947bfCh33C+8uuRhMoey6yHhFPf51PBcBMWXt8DplX1Y4N + + lUY49p6/4i1Fqf6uqdBJDH2uxNduf1xljceqxyUJAQoAAxuqB+vJmdEk1a2tN69Z + + OmY682YJ/1xqTb7p+PL2DnaSiXzysTI31WbycKFlIrpQPXDSkJ+/oOonxWlXeISs + + 0YIw66xFDgF3MNGfYaeqmNFnCt54nfTLU2+U4koRH5FNRsjkeOXnhTmO4dt4x3hu + + FpBX6E65q5Cumw57MSflw9JvzRmR7E9+KTnyMHF1PIcPIEcBCZ45ybxZcr3Y9nqF + + vrRvvaECgYEAyrgeuqOeJQZMUZUzxikadlTFRVs5sK4LihXk/Q6LuswX4QDzFmLL + + 9KvAYl0E4ILSptZ8rW9gCP3oWDNWylJUn5RNEX111JsGTLHA5QMrPzp8Ear9wCd5 + + 8lX2O/2nSM6oBgryAAWskuQHcfmFTf8tub3m40j7NugvRSRjQC/TVBECgYEA7ylK + + iJXxuv8Xd3a9lGgVRCIFPT0uiszFrN61d35r+RAG+9F5sQG8nQzxEYtJNHTMzx1e + + 4oOTViTIMRLXHVQXm49Ogf2WY/v3ro+OUIEU5o6gyaqmyXxwu/kf0wxFDP6lxeSk + + YDSPo8XC/iPvacH0yVMZtJjxfJJpua7nEjCozXkCgYEAq4PWdAElN5w5jDkZogp6 + + 2i1k7wZ9LBBFsSJPKRBahsRRW8zq30Dd4XhDgLXE/5OQWRpWSINYFKOHJsDhKLM5 + + 5/6YqjilLimvzcoDM4BX4dpAyM4Mfbyov7GdcSpuk/pNTTeLgxtJ5MpLxlHgSJqj + + fGjA5gKEkfMms3BTDSapvZECgYB8x21Uv+7EIq2KrdARqxBVYO6c2dv7nQURwYyq + + ULJi2wLZxZwZRw+yXPs1rRc/oCTvdqJ3yjBIBJ7SQ8MqUSKUDfvnBHi/p8m9MLcO + + t5pBBG9NaJTmkN98o2kAQumP8xhonHdKnoHG77phwDv8UK63j3zc5eMwnG8+6ssy + + iWK4+QKBgQChb4a2GAVwcn3yTMx0lfTftXuwlaDwk7eva1Jou5lvR9tByAusppdR + + HZiCooBEEGDYtmM+Q0huLJ9bl3UK4tTFO3DyvDSkubYvBMhw5Or0iSmtqkxu7Qto + + 9T1WIZPPFJ2PoIlbVZOGPUqUw73Ar6VK5eAFI3QVNN1AoOLUKzFJvA== + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 8388607 + seed: w6uP8Tcg6K2QR905Rms8iXTlksL6OD1KOWBxTK7wxPI= + zfec: + required: 2 + segmentSize: 131072 + total: 3 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:MDMF:nwr3zieggdkguobo6suezt3b7q:sbswvh7qr7y4vgl5efbgn5uevhnimhquaajhutbawnw7gadufclq + format: + kind: ssk + params: + format: mdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEowIBAAKCAQEAsiFWSx7bRsQ/ImDm1eE7vRa/IiUg0mgnI5Nvn12+/sg3S8eS + + tHnURTFwb5ALFwE08/jOYmiH/ZZqD/xWDRCwZswe39MgH8p3jV9ZKFyBDm420hg/ + + IgRZ313kC0V0PPwv7n6bvJpGLke8820VHlNCzTdrbmSaTZj3AuOCNz/+h/fBZgtD + + XY8uI/0hynSNoaXC37BiTeem50Ml0Ehjui03891EZysLx48WJYTp1tK7X7of7luB + + hb3JV+C7asEKqxq59DZMnIDo+vIWr6NPeE6pqxMe0bfGj/b21foHhYRUuIg/Hd14 + + 9RNSlzyEPqdf14GbwcGIEW5TH3FzNAtLHWImVQIDAQABAoIBAAwGF4zVDaCafRd9 + + On+r5ywllavAnVVOjfvHBzUW7x5EFgVxuHuhsJwmEOya6MC6BmDEhevbGfjaVxjy + + o71Yh8u8kgXyOpwivsymZ76DdfurIVyvoc1SRV3AOPUw0D6QmEytM6Z4tG2Jzp2Q + + 2qjUHnF7QO9v74F25o+Fm2PO2EfFqus/3X6k+3vFvpvgs/XEvJFJRiV5luQ7+r6X + + PyNHjpJvW8H1KPFK6z0qElAodu0RB/FlGXRjUfuyCwA7/va0YxDkiSPTtx05XGPN + + LMeZYDe3iBd0SMlw51uqMpx5rYWRRYje4vUAhkTFHP7CH0kWHn7X8eDIPZppwxix + + c7k2dSECgYEAz3Orb7naad8pzvTS1lokmpRjTwXK2DM+Z+Fi4mHbjxO3XZLyrcfF + + ONEXzDIQMyTzkSZGJK01/pZ/GhPoWZ6F3ebWEyOqFpjcmEBQIc52YFjn8CChklW8 + + 4KuzOO5NHWvjGAniZbAfDLE+yz3DsWi422DBf6mAjTS/S18E/NmvrnUCgYEA29EG + + azLy5YdvOxJvOEHsV1ODY8OBaCAVLZaWlkcbJAlmBjvazE0Uygo77ZWNl1if0EeU + + 9YVIpkDaD0ZwzfF+JQYfMRp2YsGoJJj8t3ppbwDKMJZ5Vbw768z84JP8uYGFVT/o + + gld86oUcKT+rwzIgx9iiW7mPrxt5I5/tNazDXGECgYAbVia1Jkx9vwaHWwOdc2t3 + + Yy5i72R3cOk8Txr0seh4xiRXlFGeTMDUZ9k28zHnS2s9KUn880Y2Mia1jQIFLTp+ + + rzhudTiomaQX/AGTMt2ufIizv9kKq3mkMXwAeIZ45gqa1FKdC8RLq9+WcKEk86PI + + ZMuawv9JnDXI/NBvcVARMQKBgQDB2Ikyi9GL/G1oyI7wK6KCOBGMLuK1smU6uKu/ + + hqE2nFsucCY5OFh2+6NxlwswRmVYxWdlRM6WXmZuRg5AbxBxEf77zHxOBr2C2K80 + + Fm1YCHhFdM03gDHPdgwi+B5McR3l2d/u4bw1DIGTFqUgE9q4oiA7h15ga3fepLAJ + + P3tgwQKBgFdt9NBBXmDeny2eDxXBz1OY112usnR2XV+nTBwfu+XqnnurY3QYRrrS + + /XeOEP2jlyMpy4c5uBJ+JhlIP0098quZ2wfC5TqsMINWBoqWHX1JaP7cStnYrp6Q + + ebmiqnqsF8HVebpYPv2wplJgtA1IT+EKNkbHqsanQkODJqKdtuda + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 8388607 + seed: w6uP8Tcg6K2QR905Rms8iXTlksL6OD1KOWBxTK7wxPI= + zfec: + required: 2 + segmentSize: 131072 + total: 3 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:CHK:ufshkmuewwql2xsyiecbjhh7x4:qoywwygjqrmifowob3mvwkhfskk6geomyw5e3qlgqzui3kymu6nq:2:3:8388609 + format: + kind: chk + params: null + sample: + length: 8388609 + seed: yPi3JHKKbWaEEG5eZOlM6BHJll0Z3UTdBzz4bPQ7wjg= + zfec: + required: 2 + segmentSize: 131072 + total: 3 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:SSK:wtihnlmjxihghrwapnvtsjgkjq:gmz36hpd5vi34dubmh3mk2jahhtitk7zbbusnlknklah7t3toxjq + format: + kind: ssk + params: + format: sdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEogIBAAKCAQEA60U7XHhqto/vkbXa2WGs7b2ZtVgmubgoW35dfluhIYB5HbH0 + + Cf/IAgsj9qMkxuaNrol3Fs0O9vW+zQ4WgbpNV710buUwg/OZssv7qPtQLjJGRdVw + + zB8Pw4633SGo60J9wO5C7vl14zfT5AZV/BLXTLUbOOA8SScIgcMiSCzJ7YLQCNjq + + djzP+9uErB6eT0yhKYkueqJOccV+CxP4DQbguruMpOVgfZSqMmvThdO6kLR4Zv6T + + I/EJzUjpdBrywAug4xhc2S3RlywiyD1A0DcnHRWHO1kcB7UplEDNA4M1nzei3nps + + ABDEvrkuP67+vsbXZZL15IoMlJoStkh3c2gQFQIDAQABAoIBAADNhxPj0ufM0dyB + + oVxlGix8twbNwTRlR1LHAUEmh++2QClQx7QNCCh0L+scOO5Ygd1D2Z+qVcrboVf4 + + 6UE7CNd92O9ZD/2ZrnKMbaZXIa9u/diO659CN455Zrk4k1gSLWpR975YTZUK9D94 + + 6b4fYyMEZPTJaQYij8OhRnpQLH54EUrDwUfFQGAArRR5oCMHh4TOFz/My1J+HifR + + d1ue27YWUDeDWd1U4vh2qIy+V5tQFnisgMw2QQjFTxkre5nqn/vikmmvf0vxlccy + + eHn2wLJmBbzK8WaO0gaoLXRDBNiZBfY17YAxpUMOujwg08VbrNY4gNpLgOjmsvFu + + Tr8qeTUCgYEA/mWKRmRzkQ8y/XCvnoF1jCNeztkNBQi3ST/7CF+f20FSbvtlErz8 + + Lv62DdyvPnDMvu28nk+MiQfel4maDG1nRMBnjeXuFbetpeJZc2tAh06zVPCQPACr + + RI1Wujd/+QI49PfmBKe2nV84GwxxEC5aCvviawjbIIigTLy/IU2LZAcCgYEA7MDV + + EX8Fv4bEgGt7TXF0vkkM9h+iGtYTjdUZmsdkNvzW40CXlusYzJ/INZwgAkQPn7F3 + + 8y/dAnW3woJ+vCcozQOa/i7sAhxMZNV84OgFTGQE6IKw74aUYCdT/bCAhrmIaOfC + + YnGkino17EMyU1wskF3mKuOEbO8qQgC2gkt5/AMCgYASbDQJSPj9hkZBCEoPhnyG + + u4EAJcPFm436ZgG9537iF+bqVpZJNxpkJNn2QwcF1JFfOkQwir44pjM+ch6Py9Rw + + rCZTplUJiZWvr6aeryOrKM3f1tP7JGlCu6GONrqzw69wPguQRrz4xI6BlvMRIuou + + ZXNOIQQNZReGtxx4Qu9XPQKBgCxiIg97npo/K4tfmufzww0BKNrjJ0Kcq2HFd11a + + z+C3GZnUvBZg0G9b7O6P7DhAhiVL4c7HREl3xBFE4XloZe+5I09PgJMMtw2YMCcB + + mCyv+3OTPJRKyHoWJVrDwfR/x6DTAc/uugfzzTQTjNWvy/Lsh3+201aQp31kINLg + + T2f9AoGAXkW10qtVSGKG21BovxbMbVrl9nmP/jYWDV/hY2JVwAXho5htZQucbva3 + + A/25Q7WpaZECWeJLlafL6gQRnZBKWaMC2oIm+4+w6I6JsSRi+XLNL8NdPx5oSUGQ + + k230y22YV4bWpNVg49VB8xgjsMhoKhvaP4lR1pn0Dke+06y1U8o= + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 8388609 + seed: yPi3JHKKbWaEEG5eZOlM6BHJll0Z3UTdBzz4bPQ7wjg= + zfec: + required: 2 + segmentSize: 131072 + total: 3 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:MDMF:hsemzrh76z2ha6lrv2glaydane:nbbwvf2ivfojz5sb4cnlkx7z66bp2ou3teknd7uobuvbi4dcvlna + format: + kind: ssk + params: + format: mdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEpAIBAAKCAQEAx70raDxNkfMamQm7E9lJRh9UagQv80WtuuKhvq2sidJ1glsO + + 3IdCAe9dmzZR609K5OLJnHvR2EGoXu4VNLtSXTFlSDgO+tyRupAuFY8Dp4P/EQfR + + nPqJOqIGid8xxjhMAvIKK7GW0Cd3iVXfJo4remn7zxBWHexpFy6N6zE+cARV4EeD + + NkSv/bmJ55ZAFZyViCBuVxEhRJYKqmPeAmCSgilc7fX8LO0E2aZ4baRIrXZxvwvV + + kW2CoDVl95eGw6vR43pF+DODH8HiOuP+uq5OJRap5KUgZiTtQv6cXLqoW54KPUQW + + 5Et5lPZTBbBWRsBoQJw0Sq2RR90slptjqthj3wIDAQABAoIBAEL46HGSarYJy/zN + + ePdeT4XeImlLzyIkVmzH6dDsHeK2eSVEz/ZcueK5NmtBKvWaCDQ34L8B+2omFcUC + + 0oR0XNkXo2y0Mz2lMI3cIz+iTOjhxugYdY3LqbDJvCSFfISIwt/n9UYSTU2tNhUM + + AH9Gg0iP+dlDkoSFDPWza+2U/OkEyqZmyd3buZtoCVIkYqe7DbdjRRBP7r9FuxhW + + jQR5J0CbK84A9IRvCjf+uMKCnzn7zZV2bGzn75YX92+ul7pc4gMLAL/amwJRI9SL + + 3EGixH8RhVsOD2itM/X8gCKQeFI2J5ll8pQxRi95AkZ4z1FAakcUWFxq1FJ7gLk6 + + t+3Wy4UCgYEAy5tqqU1SvtDJUVWu3vtpeHUZFlSJ5wVHCVSfUWkqNaZ44xKvSMQD + + UnQcWsk+KRP13O8wVtUhBNxCq9zwW2xTvRXBFwomx1bxt309syxBakSiAVym91Wd + + gDrQf4xAGRKqtvB5TJsvTIyO3BL7stCpGq1LRZ6e7s8beeWYYRu6PFUCgYEA+yLw + + Yomx2E0x9t/PVhqACE2lKMZH7uBH6Xgvg586Ps3nEvr48YuxYS9czPhQdS8jfgQQ + + 8ioWuK2vRKAIMMDJJQnfGCrvAq+5MpE+cFo9T4AKi+CjO85zHN30pHD9SMEqbnSH + + gDVRkxI2CYVmVh36+d2KteS2zbNi+fNznaK102MCgYEAwA5jBz4bzkhtjd4v7L/k + + Vi7GskyeJB/TSRbcjVOQ8DiOkUsfspjKtW03DeAEVYUxhuzMgSvbUJVgAnOO+f3t + + 409w6wW1XJVDvpxRpgAZ2F7THkvCZ04IGlvgLmAiWkREafndwYgkjqWLYEY7zAmN + + ac+LUCl2q7cKqOoM2ZTpEF0CgYBB6fSv2DYOcIxpoGp5zfDGvSJZJlmg78rQE1Rd + + NoCCFWbNy4NlWmXO/TBdN9teNmYZYBXWiYd3J1b2Kw6bRS5GA2ZDoJkk2lxAUnDR + + 6k1nPVMHTYlqXBBIhlT8iA9idhid7wXVd6kWcdQvAY1PkwTZafVLMmFsceXLdsNk + + n10bwwKBgQCPgfYyoEada42vJjjgm8uHpnovj552f0cZS4ROg8mqKaBoheHKf/Pw + + 46Yc81TGJTsEczPy2MFwtTuKOooMaC0jjLKsgaM/tyf+zmJBzut5Cfi5BXteyH00 + + eRCU+N1i36oKLxlPisAbyOwZffqYfFomVQv/x5vSfx8njkXgA+Ic0w== + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 8388609 + seed: yPi3JHKKbWaEEG5eZOlM6BHJll0Z3UTdBzz4bPQ7wjg= + zfec: + required: 2 + segmentSize: 131072 + total: 3 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:CHK:sxrylthoxskesmlhrfuxnifkdu:kizgaeiazgpjgsffkotumbu2dtxziezw73ybwo5pfzleuckqaiwq:2:3:56 + format: + kind: chk + params: null + sample: + length: 56 + seed: YQ== + zfec: + required: 2 + segmentSize: 131072 + total: 3 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:SSK:gndzkdmlzvbzhjwkupooylrs7a:32kgw3vndmzeufhfruyu62dsun5ghndbnsurgdmzty2x7m4h3ida + format: + kind: ssk + params: + format: sdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEowIBAAKCAQEA3RN6UtTgAInFX601J7cDvb4/trLFwkAX39f2+4lUMS1ZnXwS + + vdvoeR3Fj2L8pNufV3R/qki7ZM3Rf510YVpFJvXt9XzGNYLluthu/r56aNz0P+Ga + + 3MQ9dOq5QGKTxtF/ixSzrZCxqkwCiZXaoLfSb69S9PLDeGBsLDIusqmarRzpOPJ7 + + uUTDwyX8c0SVhaDrcoCacfFMo9wY7mrSJuhImuJLDsru5XoDcQo+DKMBQUyWvQ4F + + HliF292ZQmHaq2OcRkgMwWH3ZhWHar79XjehYLvGOBQEEoXtgm41cYQI1wKvgRvr + + x9lCw7zBrmJVa3HrxbdtXDW+egVvH4RyIRI4/QIDAQABAoIBADGnrlHsfmOgjjRv + + MwE4mh6EHMtsW/7FZpdgapkUv1RMW1SECbGbMxwBE96g3R4qNh/uir40l+KMWAHR + + 29IB9IZLtqbs35glTnQpKMUPA2+KMVIn2iC78xHPpsxPV+HQLFWQ0MqrNTyK1gcR + + IYn3v8xWFMvvuvfOsH08yEBY1+UJrl4IgKgukzSV/FD0arZcnM1t26jkbxM9MbdL + + rfP4LKGbL4PmgQsgTAgn8dx7uutAgh4AJnk6hZVN9UE7k5F00vvXSO1F5i9/xWBD + + 6eo+zlAOuWKGmOpGp0G5u49RSblr7guXcWnwPdQmMfYwe42id+Ew8bcKnnMg+10U + + /tTlcAECgYEA+PfNetLAxhT4Dq7cM0sn0C6Z+/REuf7YmASQfZ29irkYo79zb64h + + f0eGEeukeJUCtfUJ8BqY3848DU2P/0ITb/dLHfoL803MH24uj1YHk3totsN/NHuq + + WZreeISQ6K9Zw1sBCmCWmybPfO+y8mSCaF+q50GZRXy/V7qbKq69EZUCgYEA41H/ + + vUVqeJ3hZC+GguAwbOY4RQ1BTcfcdfu087UGjq+Qly/LKt/y05kHnZUSKWLoQRXj + + udw34MTxgI/L7htBEdy49EO9MM0Nfs2noG3hKFmwZE2gBUdIWeFsKvWhcBuYp0mg + + f253Y6QXdTmiRzvW7i/psNE2z9iNoXH7tyKs/8kCgYBVGk+Qxm3Cx/QrALagifYo + + AWX9a/f6JBThkd3aMotR2geEIbNR35HvsgEwKv5jgXwVupcVDeJnzlVUrsikFnAS + + e9OfgZOILXWy4LTlpiCc1zhqENVwmT0XuAqH47is8ROb5YWriGyyyEdwi3b9yEGT + + b/A5cID18bhuQok7w9M5KQKBgQDRHR2lf7XyP0qYXx/eRV5Gz4H0A72PT8v+vQ45 + + Is5ldBwO+GhtiJZZEO1wiTGr4NDHDtvunibJHmMLYTy4TVoOlH2QNsBTpE5F1+nc + + Kzh1ZgxeOQp70Jc+F6Dp5AwelURYn+KFV5l8j/cEX4BpByMw+eKARfWmPhAL9E8a + + qUt8AQKBgHfLj3vvSqU0Ymyn4rq6cYtr9R7GutQwDkhXxLxoY7xC4YwUXP63V7yk + + makdfVSYpqW7ego5oi+Cgu9QYnNN4wFP6+C/oQP6tUhDlX+9ASrNuiolacWr8Ci5 + + xpbfhXJqw0UZ6/p+2j0sBTn1uvZ/JrwTs4FxT3Y7U3lF8KZVYKC1 + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 56 + seed: YQ== + zfec: + required: 2 + segmentSize: 131072 + total: 3 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:MDMF:vlk7vnwekb3ppcge6ohmnjp4ja:36supzzq277xjc4t4uhuu64aqljx2ws6zptqdafwnc736vq4dppq + format: + kind: ssk + params: + format: mdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEoQIBAAKCAQEAsoIhuv0XgpVHhsgGW5OHmBGfBSTQcBaZa+GOKITgaz2t1rXm + + Kdjne7sYzw5p04zphNmzqlam0ho63dsDf/OFUPv2kiG9yzXuGs+7R0GSLjFr0z+V + + 8R9zGdyv/G+4h0g/i3ZJamY+py/r3SK18ik425F0W+ASkcvwntJsFGZegifVtMAF + + r2LArlNIje6cczPK1OTio2CxK1lMw4RbZ4aAFXSEiefU9r2CNiXgR9HvsJT2l08H + + rL31WDZccU57gRjN6iG6s6Nmnvc3Gs9WDZRmIl/LAqv1bSHlViBCXUX5Om/fCOIm + + CoU2cT/v+hG7MWyuPfUPvDTfcAOSuZaDwGUC2wIDAQABAoH/K1kZLSSeEO0vRrZM + + KkYpJ0R1g/TsqPYpS4lP0Ycdou0sycxiQsc+xKJ/48gcQBh38fWWgPGJ5nt4JWff + + Rwhb13lYPHmfx+PQw7IDncoj4BPK8KxVkmLmEIxcMBub3pOCMDEJTaKGlGg5XqSP + + NRR4Oi7tkrdXIGXl3gDl6LjzOlJegK71hHViYzbaK5uTszTn0EKRzYk2fUmX6udx + + yvV0LoNRlj6wSUVvXew75Y17Son6bh53/zPCU9E2QyN3V2BMM5csr2aszAVZb5Yy + + eSRbohbqjQqnmgmhF0o/3eKLc8qquK5//f6SPvfi8w/62dNny/lbOQevjTgPxkoD + + 7wFJAoGBAPbtss191JLHCH5IlLnIskhebYSGuHKTCQTQ5YUR9AZPHk8XOzE0ZLp5 + + YhKor0mOnxOWsZksYGDrUE6kJwoZAx8eHEkxipkjDXagLqExFH4ZonAycsncfjD+ + + X/0yS3wiGYQoHqzjBXMEpnKPX20O9wkmKT6S16har7VIX3Z97LI5AoGBALkQ9VsO + + NKzSS9yXMF+u1B7cu8uqStba4tBeYlJvXUyVvodlX1KlAATwI3lCrGphBJAVfZc0 + + Kmx/HXv5AiCAwWDsDYDYQz6ZTgoYD1f13qDlRFubjL3ZSNgwRsLOdQT4wO5iarY1 + + P6x4ylwgGNvf/+noP9y0eFvPBzdgq+f3k42zAoGBAJdqXJkrjr1OdQPTB/gAfGpq + + FOgOIG6JgR9F5Wg7ARMZUvGWwkJC6X17T0s3yvzlCuDdKBxQHO1xfjYq7JGBkuty + + 8E9lpKKQ3wGd6doIGZPVrkj0dnUX0v3CDiRZwfXlhxYF8AF92GqWMGbRSee7JHqk + + vufS7ZEbwuD79yXWw9zpAoGABiuLkpqZpP1p7BPaWAZTKig/1p1520n27+2Fp6vw + + 12HStV7q262Gn6OF+z/+0Zkkds1Qn57snytpxz1ZFc5VJC8akCYlr8uar3l34X3g + + C0s5iThZa+b3p8WMRmhtvFmyzP/ZAPQriEuKq6GiUopYVOsaXfhiXuU7H1yIvrYh + + ZEMCgYBZR/dtsf1MBQSa/avlpRuWXT+kWspp+470z8KND6NL0tXbL7GmGfHoe02N + + Le3krngmE4hHaZovTSBqY3vPVuamukZahuf5lIqmO334LAkG53ggxWYSC/DDK7j0 + + ML+GW3P0/8+jrx2j2DDcOLK+7G/rUtdAU16hO5JyznlwQLa2nA== + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 56 + seed: YQ== + zfec: + required: 2 + segmentSize: 131072 + total: 3 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:CHK:ld3thziutpaqv25nbtoqebhtru:ri2obsvzl2etyuv3qnchn2wvw5mh5rjuvbjaqkqbvyprn7xwamoq:2:3:1024 + format: + kind: chk + params: null + sample: + length: 1024 + seed: YQ== + zfec: + required: 2 + segmentSize: 131072 + total: 3 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:SSK:f7zhz7uiaqui7hqxinamtovtm4:gjz7wb6tw3glspfgdld3sqt5msdyxbh4w5xmiqembwosixnylpaq + format: + kind: ssk + params: + format: sdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEpAIBAAKCAQEArwKijDMVsmEEfQt+ahwOoN/RR/zh9EmYnwvBc57S1XqYYD+8 + + XSuAV2LCiraBlc+cJsf8aYpkb2syLXXQZr2s3LavbRN65Te+ilBx72bqArgynHhM + + avS/ecDupQg5Vw79mW5evsBEb0rkbavrX8gmkP0d2rS9Sy0b2zku9Qp/OiLNODcq + + NzT+MBH1i8v/HVtfW0S6az2KAH+zX0UFrzM8zPXsPCIJTDyoAh5/FJGnmhIrdep0 + + mu6PJp01jhU8pLJjj6orOaqvY26/PcWEe8MbruDSIUg4Zyk/tl/dAtplKtfzZT/6 + + 7D5kLV4vFsULiBaRvANlMuSU/wg67+vDdrzPwwIDAQABAoIBAAv+frxkDeMdQgz9 + + 2iqUhK4i2Ll5y9SNrK+NwzLU2jc2QTYreBHclt2mT5XpHyVwxo9j2lkzWmHGc3hp + + ICDCdBPmU0yC7sPB18Wr8LsLDxOjoxhVKEuWPX8vKUvXLfLY/KlkxoqFK8uC0vfv + + NeDpGzeJmV+xTl3WGBgkqaKylviZZ/QzVLnJTT96xouxlTYZqpbw0cVIkceW1JwL + + Ww5SCFTgq23u3b89lAaXUncdN+NIi0FkfrGmLylypohBHhhgDLVOZkrcnkWwWFBv + + 0RlEzzarNYsKFZX2yKQ4YJfzwtAlt18jOeLJjrOhjdJn0hZ+WzZ6x1G37FDAllFu + + TSOZ+qUCgYEA3MqZBsK76J8lxsnvostR4W4kNyK3crOhsfIlQvqeSG1/DjpoeoS0 + + vGLeVwT3MhlziHkPu7TvA7182ILc+NL+/TcEmPrYwEsKq2eI4U2C5W/fcLFvpIxn + + UBejLIyNDtpXjOd7nu2mOJ88lQdKJRiC6l6W5FXqYq9ipxGmrkmfY/UCgYEAyusb + + nt/TDSSgt8AO3W+KLeWjG75CYCH/4PIDdMCo+u7A6q6/38PzOs8i6f7OsjtuBDLu + + CgZZx+erYRFICHUom8KgLfHv0i42w9rKA+1sKEE0RDZFsLhz/LqixDick2Q4f4Vm + + JIk942MG/LH/Y5UVbFNr40vg+T9hPBev4+iDSdcCgYEAqKZotW1SM6I9LNdbILLF + + 3LhRGXx/PDJSNKaOJ9dfyFs7Thb3b36mv6+Vvkqgt7gRNBGlHvBaEjVPg+KR/87L + + z4eTD3es0VWA1OTE/bRDZBZMSrx+VuaYk+k6TvEdXlcRwSOgnglRira3g+6JiERs + + 27Fc+RVXcAIgDRXCiCbchXECgYAwEn7app/zTygcIA3le9U6hlqb6fkDmUprWipj + + cHkX6ZQehQPD2UI4PnZBBTKmmtm3ePFXwqVmbIX3Wwa7qjXSoMsd12E/Y99pit2t + + DIRBDSF6v3jHIwunZffFkLvXVzjjTREjurfEtOMk3m5ogxsuLJ00nfdQVSmN+Pac + + gasIxQKBgQDIJdsrB0MiZyL2QDipKRs4sKpfa90TQZekuBH0eT0Y5gEp23yUbnqE + + 11rmDavNEU7Ce3CIIHcnRfAFcv/cS/BLwyaVzAODT8ma6rjoMKbxM98VnOnxTpjQ + + E5h7QBI3rG/fzB0ysxrZqkat+bR3vP7Lmq1Fzgam6GO6qVzMEduOow== + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 1024 + seed: YQ== + zfec: + required: 2 + segmentSize: 131072 + total: 3 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:MDMF:pprkcpehlvoizup2rw4gbjzqgm:bye7dr2xg25mogjntkaxnnmm6hriqcs5hy53byf3bc4zavyzhs6a + format: + kind: ssk + params: + format: mdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEogIBAAKCAQEAuPFtYiEqdnlCHV3RT3LRqJ0qXM+QJGnIc5mSteC6g4nX59v5 + + afurdhb1QZupk1KxPcje0WFqLdREZpHYHvRX8M6cp7mupo7HhoqRPlN0s42t3Ost + + LJnZKac3OFARCDuOBHYoCGezFqHmJiQRJl5q6EYmiFBY/JoHHna1gsaApEmZvZjX + + XzcaGKjQVWwT7DrYbEgdxNr2YJJjWEpOWuiJpNHoEHITXpK4E+4k0PU/LPURuQ/l + + kfox23kh19Wf01U7/CDxoZ9Y9mfijvaUtS8F/BontkbpWN7JUwiuW1wygc/rQK8t + + +7en/dQ9Ap6VX9eceu816/N+zuakypjP5hUyTwIDAQABAoIBACwZqc8iAHmivZC6 + + G9y5kOQHoh/igMkmDl3+a10CWwdducW4jxdmI0M0A0SjRULzj38fpH5CH+sQuETL + + F0F+W2/5HKLkJJDj8BEVfr/hb60XJjPNQoblosKLdJ/xe7Y+WUWYFUC31Z0aewJy + + TEKddhmwDKUpn6aQZg0uGmc2RVunJ0r/OwhgTzjtMkds9w+iYu9DbB6RDLRQBWXJ + + eRYGg2PhJ8J73Bk0oQMAAEOdgLWdOpbkaf8TbQHu2QMFE3uv00YMJivAaJ0/dGmn + + dujUzeloBFPDfhEEIL9Ind97Ix8lbCw51rJgKv78K6yJuV+zuQZjkLRpKvPpMmr0 + + 37VOzKECgYEA6E4NkBH01BYLS9OI9pV+DL1DnMeAyJyHWV8E71Thn+OMZOqpVjmO + + RJkqcy11wQCRO8JSKg/FfPBi05F1N0xAtwKYLK16Gk+NUWqthRDqM0tVuvxqObSg + + U/1BpZoLK88chF0r4XAFoEMX/KIeJg8otV3sM/+Y3fQ/j9i3q96J3qECgYEAy86p + + gobkIuJ580SgABR99rcQLXHeyBa9pl6wsiSHCpwNYy1TZ+csiGNlbEUSRdxJOqZm + + C3VGKOJoSYr44MElp65HixaNoyQKBWxVLJD8BSqQYqvkRgGQxTyX5OtVpvgEkxmB + + dy7Sy4iW14yRvClTS12T1XCk2K45Ujmd7vWCGu8CgYBmA8DY/8mwSW30gpSnFMch + + +Qt0EfhwIK0fhia4o2HhwR+qQZLTlrrvTQPjSJdphkJBJ/jFF9/2GeqMVlhPTGEu + + /SiulhAE9eJtWpeQ0/jFRdQEJUzQwo2V1KW7f4ZgWrd/ORtICNWvp0clXlw3Anky + + DGjp/Ni4v8YZ+WXPSA7rgQKBgCv1A9xyKYxYmoLcf0HlKZHnw+Z5U9qGBRt3+tZB + + SJsCM2T7pqyXUKSOA5cJgrpsm6K5tvKrtZkl0+ZgwfL/1ZZH4YhfMedI45xt1CUL + + lD+tAX02o8Jxnf7cZcpq84tSnPH5I1JIWBCsAhS1bc1OgHeV1EfJxtQxJ43TfXvH + + mesjAoGAa+822jANAvu642etlFbbtVMoeFFFaqM0+QM9yHazpeO1cuKlTnMnA3XA + + ztLTX1YhnsUOOHNMyJ2+o27mTy6GhKBTa+rx1DSYBxYvXKHMUUNQnomeIR9W0obi + + MWsw616EtbP+KuDJF/fuWX55vyf+pON9GGkAnpnOVH97rC9zXTE= + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 1024 + seed: YQ== + zfec: + required: 2 + segmentSize: 131072 + total: 3 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:CHK:pz3mga5pvqd4bpsz7pbyna2qtq:w3ptt7xwwgywod3u2rpgx7iggvm2kpwl7d2ryr3u7mcaxgqotlaq:2:3:4096 + format: + kind: chk + params: null + sample: + length: 4096 + seed: Yw== + zfec: + required: 2 + segmentSize: 131072 + total: 3 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:SSK:urwhxkkv2pwxp6kkbikg7g3gmm:4tfyngd4lxccrddmiw76s4jsbhmyljitk4u4ue27lsdtvdydcr6q + format: + kind: ssk + params: + format: sdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEpAIBAAKCAQEAoJGKHKNnOybDXGm1fzftRDzJEBez+fkeIFKmL6GIi42O6zlm + + 2gGLRc1u9cvKZTEaFfkEqQeG2E7+MNdSVjCPTWpIVZ5Gzae8Ziy2VfFdXw8wStsG + + mhXLH5hZJTJGNec8Fksjj8TgFxuPvgR21xTXXOLt48oytNLLRP75dgp1njBN58Kh + + KY6nK7m9b25dwRhH/9KCwKOqcwF5kN1pGNnB0X9CcLVnKyv7geSyaCTuk3WfMQ9k + + B7RkOmlbgMTgyTJ7yyJSmFA+oVZO08CvkWr1WwmraUoLEQawJ4qx1MYuMKgFfxSZ + + 0qVg7RJx6GHIwoXSq83l3lmxLa7Od1xBcjbUSwIDAQABAoIBABbE/LZZ76IiOZLp + + xJyRRDqoegSnr9RzYLPJtJpNiEzt2oX9wlmI3YSdAK6nYwCdiWrzQJdto1AaR46K + + gjkJstCSEUbe5oB0WFGO5p5iV1DLGRiMXa/NBlxpIL8XFYDAVTN+HUFedD6ioGwc + + OvP+Fxorbfue6TjeKYgTtjFog0xWqHtqC3LtA2FHUngGBDNdjQxcW024rStIvwir + + rlp5dQ/Il++E29XYybDVcQPLk9NlCMYrj/aC1N0lAdiY4eeDLeeM4KacEODNiIxO + + xzgJpLFN966Na0lk1Yy/7uluLVZgtYqGPOe2N3p5qehF9diTnNT6nU5Mqki87bBg + + yTCerr0CgYEA3tPgSJcAGQ5vSza2h+R55rRR0sb3sg2i+Itn7CXXyJpDJAruqf6R + + DZICmSYHkNhJvW1yvNXEsD6L4Ory2pbNcX3lFk12pN2m38MWOR1jyiYzgW6X1WLV + + hmBCmovYOQZ2x4jWNCiiIMvwo7dxFaIUumDPB+iH6kXuo7A+G0R6d70CgYEAuHjs + + Mo9EIX3x0s10MI+Jo+Pgcm9gfzU1AsCL3mAqG3Lv0tzrRObzjt86WMz45Efvnr0d + + lEFMiXeQivHcCYThGLc+I5V3bTLIAlTDw1p0iMu62jSDP3pt9YVnzk4emMYIh/VJ + + JF4NBi5LkZBZlA29NRcEeoVkBGCMYeOg8ziNGKcCgYEAj6ovet3QdFc4LlgyS19l + + sPcloi4iWSwtnO3UrQ6hF3dOPpjF09iLkSJIhpFcY2jv8i/0wAdbbv6ElRkmRwTf + + pIK1BzIegqFeC/ruAxkN07HZl2PEhRHZ9W9uwdHUMMAYKQHyiWKBVX/nwMZvJLGB + + h8EO+lxT9RntiKADCvWVuEkCgYB2akEMj4SnjyYtMG92QJ2VE9FfA/nIjooR0zG1 + + tLsy1Yv3KpLnru0HeGoG2MSoHTlHB5S2N1h/Ib4qQukBP0gTSoVb6DU6Zo+XV3w2 + + qZkGuuid63mYxOlS4qjo+KKRZQXS6HRkIO9xWURvE189N7iOHNFmKLw0Rxm2OJ13 + + o4SHHwKBgQCY2Cv/ga46sW2RY0CzDnlmWP5758x3Ugy40+DW/nItTi1Fza1/oL2E + + wVjEvfr+yMTiRI7jGjnYBDDeSrPHq2m16NxdGLQnLY0xWEdyQ88DYlibAjie3zXP + + eEm8719/NRLnYMRPFrKASpdQzijZNPiX766G26/xZur64kds25WXLw== + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 4096 + seed: Yw== + zfec: + required: 2 + segmentSize: 131072 + total: 3 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:MDMF:3zv76egtkwfc3hezy6dm3hgh2m:t2n4zvvk2wm6p5tdz6m4oqij5wowsxnlpwhwz76wynb2wwpzmdxa + format: + kind: ssk + params: + format: mdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEpAIBAAKCAQEAksSSDT+xAhppAtpeEU5FGgowdf6KfkR02XCuGjHam/jEdx3v + + iBdMDGRswFIpLeLyM+a8gNs4dcLn/G0e9AGHikTDucBCEDeW4uPq0B8+INNX4h9R + + cUitHOA7U3MvZRFDfLOzNpfYdKrKbhHE1T5NLPQiqVhYuSD06T0y5mZr0O2ZRn9a + + bwCnxLrTu0fOEjQNukWvEkpcy2vq84bRqBr1zCMzMkeFh11HOhPb22BPdnSlRvWl + + PNeBrnxYVcs8QNFqM5ffVUMyqr5gphtjAmLiICsB6BkJbTkEeXISwyCOek5m9R9q + + X9GZh/yUrTKbhgNSiHVXvz/Cpjw1P0YifEJo7wIDAQABAoIBADUxGN3EX5qrh7OJ + + AN60x0aQus+I2Ri6Jr9Hn1HPD7PHjSy+pLll+CHlo6RwIoyG29EDpv3sdaH4aauK + + wNUeWMk78tO3YjoOa5j/kXKsYA/1iLxjLVkpRdRZUCcGb/7pKtRfLGx0y/Y8j/Ek + + b3n5gm7wbD+DzWQLFbgSfggSxrCJX5mjcPBJOmp5la2Lp6CsFN7SargMBr6Y04hf + + 9Mzvq8COvwcYT7gLmLZFUU4zv3rHZwdOTsWJsyDZjABmYGiJmry94xbh/ar25hLR + + O6OMaFWrFWXID+4T0LowwZo7gX1LDam8Ine451njORL4rNGB1E2C6n00HaqXsnu0 + + P0Oq6EECgYEAx/DklSZgwVh9c+UsqVAQ3BmqtKYyvaMFDoUZUejgKdi4NQC11FhK + + rCBhS/ITmLePDzZgZm+613wlOaQayOi0Jnu/LFRjLv2UmWFfcUAlbRxofhVfni2S + + 5tbL0NSnroGBsvIp9+amecqIIpCQfhyeDc3ghLAzscx3cXC4/xQsHoMCgYEAu+sV + + suCzcaKaqtxi05Ma8VvkbLqn+WgKO7I/qHp18GvYrLwjgp5wUtMDyj9MDI4FCeDB + + 2BENOTceVP6TYl6lSow10sTO5JUOsVlZaBfcPYzR1G79eX7lrQxCHDgjjxW41veO + + y4Bf/U502E6f4BUrQ1s5xYUwgf2qlY3Pf8D2ACUCgYEAs5Li6iZ+7gg5HJcvhp3X + + lqciz38ZwYKh7wmR1SRP+KWhxFDv/liSMIggeuJfwWDThzkyWa5t5E2m7V87g0il + + TI8GA52DO1gbV6rB2uhe9OF35A30RA/wiY1Pny7vr2a3g23GTdWFnYtOu6SVcf7n + + 4cQPq3zJ4R2gBW3VaZvHiFsCgYEAqZNmrVjgFXdqoyzlcY+aDJuj8goucn5UXbJo + + h5yauS5ZBOdyE/jt24/YJ7Ye5mVyXouX4Wbhy/PVR1XDok1OU4tbNqurF9L6w0eh + + yrFdaZ2d7FmMGwtML3CUZ+qxC/nKJxKWpUVfWbJm9ptc4lW4CLxV0cxzDZrfSL4D + + tYFnfJECgYABXvwJksbkZxSE6Jjqi0nW4l3JeH6+Ac38zDBj1fG06ILNrh7NKaRe + + pp1JF44m5sY3LeVnsbAdsipwAHr7AzYAbXAkgmJzVbUciB9bKbeHCAVCyBqqKID7 + + EL+sAqigrYLqYZ4T2ev/KAH590IZqeYAXbewOrMi2moS4EPitIsQAg== + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 4096 + seed: Yw== + zfec: + required: 2 + segmentSize: 131072 + total: 3 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:CHK:rsh7ysvsgbwh5g3xuj6z7zky3y:ykg5rhzpzqkvdy53qqyghfbcusibynfylggbhypo3iibsrrzvlkq:2:3:131071 + format: + kind: chk + params: null + sample: + length: 131071 + seed: LCa0a2j/xo/5m0U8HTBBNBNCLXBkg7+g+YpeiGJm564= + zfec: + required: 2 + segmentSize: 131072 + total: 3 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:SSK:mwxshkf34ewmdca4gaknxkwgca:rfahufbykveekuhceto2ynwhckuswliwgiwpo3vs4dibfyvl6xga + format: + kind: ssk + params: + format: sdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEoQIBAAKCAQEAmx/LUVxlqGwQnbjQ2PRSXsJIK2wojqBM6z4H9ilFpVUafxiA + + tCy+MegxTsRbUYPAmfxYDxXi82GNXxT87OsEjbtk8UWOqlNrqTiskZqvoHSWt0/q + + peaKTaYi9PWGLfBXuKnJMCHyz1Cke4EeLEH8P/Sg2IyYbkuZ3y6Bp/mxRwfTtxnB + + DuyQLHGAf8xU1hR9Vcn5G1Ku1f4PgYIknhQbJ63BZpBC4T2VTnrUXqygNe1xzIxF + + 9SWMGCyQdM0k03DMD7Nhgjw9vWL8OgMUqULu7aE/cR3foFECqK1qpiPZ26PKleVW + + mDvfemDIUMSOYP+mYVv3PGXmArCNunhKQ4ujGQIDAQABAoIBADIN6VJAiUD2Vco+ + + 540KEUYoVJdGWDPlf8xsgK8qlCGMO3eFVYpN4bVC4h4zd+/unohRh6yeeFPmR3LF + + 1/MuxpJhRGoh8q39KwE4m16EVmVlGXjfHa0Yncn+cMswKnLKWdPpXVTdr3a748dC + + W5UWWandasVVYJ4+YNFGNWoZRN3RJM+1hcTTuPOr3a01g4U8cTLcjOcitCBhW7xA + + ebpCsdqD1xx3yMBynU4a+0Iyvsijj5orPetZHoqU20MYrrQMLVXIqt8IBYsa5U16 + + 6ZOTf/NyQWKLR7RURSbTChT7WD8PCoxbamdMi9z+2y2ftfr9JfqB7LDEScOnZUoC + + KRvnCs0CgYEA1do6lBtMfy3XrxawKLIzHI7Y+qeQ1LsSyYeBeZECLNEeZrybwu3O + + w7MsCovzWIRKTjZtz6kO55iU3Wx368UPTdCCrjRws+diO9/Z8w3VXJP/KV+T4AWN + + 9Dn0eAefc12eEcIKkDfrcVk+8/rgciHtltcfbHcqEGUyB173dnfA/OsCgYEAubJ5 + + NARl2kJ99WA1xGi8/rqlHjivnhiTpo+UGqtMi6yAlvV1trn/tAYNfFHuR3ip2tJ0 + + s8mHEeOzwyhwH86Pkzbl1KJ2wUdX9+BsO5FqZ9wR6K/AwwUipkkk2OgHuHrDLnde + + AQUWAYW1wmnIOTgSZmo3zmvI4lCYvx3IYgaNDwsCgYAOSf2eBdDvsoV12oM8xONr + + ZhQTc3zW6gUQWDCLiefmTLbGUJXryW4GX4Ny1PUWlghM/5AIzxgC24we22+L3mfu + + YB9LOo/JRY2nyIZMmkEGZZEoF43O6zAYAINYPdImqDu2nguMpV/i+/6b2MiEd8Xj + + TU55NeEmpUxZd7v7O3c2rwKBgQCq113W02z5Pk8v3pHY3xtxpzmd8jzv0GCWzmVN + + m+dSYSP0vmLL95cegqsJgz8bFhH+tbyUY4YWmUya8asmOB2zLMCJveZPr1lpPVmV + + /BTO9JKtZnSLd0AHiCeUPvRLbvX+2+bqPUmfoOo1sKh6q/GRs4sgJ92rCMdenQHr + + 3WcNPQJ/VLI+fiVBRc5VkAUsywrhmG2tE2XKTygARtlfVJXImPRuyO8d1opah743 + + lPsxHv46zm4S4sa9Ik85Ktlkmg7t5+fiEQ3DyQwgsPfRGETgGyATniQv+4AUL+JX + + MkSQKCxm8NVx+/igyLPDYnY4P1fU1GHYhf+UbdW64Foyb936Cg== + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 131071 + seed: LCa0a2j/xo/5m0U8HTBBNBNCLXBkg7+g+YpeiGJm564= + zfec: + required: 2 + segmentSize: 131072 + total: 3 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:MDMF:uug7cdmfgrcrze5rwnoonojcoi:ihoyicjkxebylo4db75inmmqmvredrvxbbnbamrmuzfpjd7cc3zq + format: + kind: ssk + params: + format: mdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEogIBAAKCAQEAtljs4KYJOc/c8zXWR2/St2f7524LPSXMjJQmzrMlE/oDswfB + + fksJITRrzIbMeHSscru+XgjSKdybFgPdLf8xotGTgFLRb0jfnNpclcqu025rgEh2 + + pnITYRZYTkrB1wlKsajFNwyRbdIUWL8/wDQC0XdPMPlxcCWyV9qDBYP2FzlTSDmw + + fYb8QrUr5DV2XErzpsmIMHqEgpkQF3Q1xp+BwwxXKttbUaHjMDD/sn/gCL5MAlnv + + LmQo1TwTKZGmzKnpqmJOXTDwDGAYaZc1MeeMmQ5GSxaWB7xWeHXIFu/QPKgGngY4 + + OpzqcQeLFMU2jnlnJr8rrdocJjwMCz8jto78RwIDAQABAoIBAAsEDJdhCx34zpEg + + zPz20UTZiCXJ3O2eQCEJUt4vg2DAEgN5nKOxC8WmQZGTCSeJFjaTh04YpqhMa/q5 + + /rLTKl/63OLRM1bPkQwMQwXyPyUwaVfQHjQy9bRqoai6f8L/tt6ht/xPNY/sCOT4 + + Sh/Qjzpb71GhmCFIHbc3UQKpygRKt4mPvRjvC/AIm7+704WGDy1Z0Mo7KUD8nvtm + + 81DO4SNsdVhF5OTsALZZOSsuUbnk+hivdP1l9sF7nPprGjhpGolAWQag6iywoLMV + + /orCZ5+Mqr8T6Qs4LA2EEWW1OA35CwqH4RCAZ2xQF8I+V+NCtsh3vMZnerTFRtV3 + + /1LhnhECgYEA7r7Dh1pb4FQXErgxrJxM9oCBupOMGUQKHPPwb4AKzy3z3PcAhDCu + + uF+CppX1HWlOtwFZ1tJzH5cUp58yVuQtVR/9DxDJkXZ93+KeggDmnU0PPye6WwlL + + vf0Mprco6BLSPxiZWIWhH6MsaaSvrfrEDGLfVcyGZHFy4uMu2qsA6ncCgYEAw4ay + + Fx5PQGqvvjdY1Peow8I9tQSCVzCvpfGiROrTjPaFdI/qU46vRrnAn81KhsbgS8FO + + hvr3F+nNeDvAUJjWOV5WUH0hBXHeubyP6Pjh1H+LOV+7W+4I13stMjNDBAT+e2PU + + KYNsdNtbrlT5YxmJk60BQoK+xtpScury/zjSILECgYAmTw3o5iLf+B5Lrqqp29qt + + oykt2wcb9sL4qlvmSFFztRfwWOIIVBd1Fj5MpLtUINW0n87enZ5Db2atDupw7uQn + + SJ6+kB8H7E9+YUq16ZcXnonXxHQur2sr7TLefX1e38ZEwZm5jpewD+rMeNSHwjk7 + + E5JqngriiyG4LmQSSmY3OQKBgCzOLv1ROsP+LqueL0MORaQmXNGgaOXmCDo0twSn + + 8zZ4P3jIid//8HZ6loOIHa3o4Pk7IO2ZkQnvz9/fgWB2xZB757emFO0UfP9/EFNI + + xSdW2uaY42xbjbcjSOYaDR9crZxE8hdZQH8+zTGT01o8PeSTXpiJMYKMARzIbkrC + + EJThAoGABIoKLYir150APbI1PdpIPjPAiBfrqdSF04JVYbsO/SxTrbqwcQNP1/Lx + + sllFJhrX4ATtNJgQhjdkvWqOBohExNlg9RTUSEYNF44VND8dEwnZ6OIfDlINaS3A + + 3mK4MFcooU8DcbI8y+EwfjyZJjw4I3WobNrw2RFDfd3ZX3C1qtQ= + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 131071 + seed: LCa0a2j/xo/5m0U8HTBBNBNCLXBkg7+g+YpeiGJm564= + zfec: + required: 2 + segmentSize: 131072 + total: 3 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:CHK:7xqejfqfej3u3zp2qymqd2iqeq:po4t2tzkh4d34ku6gdhlocadfwdyweiyckyqz57zdi7ndt4ikj3q:2:3:131073 + format: + kind: chk + params: null + sample: + length: 131073 + seed: /N4rLtula/QIYB+3If6bXDONEO5CnqBPrlURto+/j7k= + zfec: + required: 2 + segmentSize: 131072 + total: 3 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:SSK:5njnw2umlefih5k2pqevitaq7i:etegoak5g2anq2wt7zvwofeozb6p6nuyvefyqf5t6yydrbxqy34q + format: + kind: ssk + params: + format: sdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEowIBAAKCAQEA2vzC/BWtWIvTZQaDPqY1JxY5N+KycdH/RPC8LJ5rgTX05RdX + + ddq6lIvkA2tDMUabvu3UTq+mB+PRSH1zBZ0NMJ258oXMCgLYQMg8G9Om1ERDi++i + + ELHDI3zXzhK5Inoszg2U9yg2I45bex/w2oF05aN9jyLO/9hlXzNp7Ruvhy+BmK2+ + + K0aQSJWrPfPEKeObVcFC/08PohZJS4UOPTI+tox1PMrpJbA6F16YtQXXKq2KT+zC + + lBJvk/geeAFXS0oZY5OqY1TYkSlRd38NgPN6BXBCHOX9euinUnRGZmnWacicRd7e + + hhZF84aRGxAyaPlQRn7aqWLokjaziAhM3OGDPQIDAQABAoIBAAvRknBqdxWNTlZo + + eJLcA4hdga8LdBgCfmVpHK7HygOKNvJaRSUeLe2wcxjgJBs3tVYjnc61Wh+Y4wWn + + h5qo9DpIeO2m3PE5YBR2+g+CZ8GTAZY+059VCLQUm80KY6WBtINWZlDEgc9/cl59 + + xdD1JarzHOapuURDmIz/yFq8oMeJ5jrTu3W9WjweWkguik8a3315X1lJvVIESZsM + + gCGjLvCGyUG4jJp1DgC8rBxSVQ6XEq4llsWQ8W4825wvUA/1cMt4aJpwDs//rfET + + a/ISkqVQLx/4edmNeLrPXSJzfTkY4/7RybIf26FzKQVCzKYPqkffl6DyJykFUxHQ + + iCPJXiECgYEA+lgPt/FmAqM0HX9KRlSziOEoBk6MkxDNjaRfqQxh7bPBysZEr5Ea + + K2Fn5nYsKD22TXXZS8Ae13c+x9zJN4cOltR9DrdLVjTJtWPiZ9xHhqyEQ+g5MFNz + + Q9Gp5/TmMES7wrnAZ8GeGBA943oKrzwSwC/zVDIaJ7jYS8sjG839PqkCgYEA3+9X + + AEHHqD82rVXMtDbP6VHeAjvdAVNt9mY3L+kjEyYhwIBhuRwHlbXXqEIAvWgkq90G + + Jw50EJunU8fPOxsMoIeeYiSSEWAHdTo8PQWMRP9nJe4QMGHFkBE/6hLMjpRb4SsA + + +ApYo+niqfrIbqFVa7cpDiKeOUXmeANGeHjQ4HUCgYEA9x2+RmCvxaK8avGfq9Uo + + c9Ft5Ovcr79CaLL9Cq4CbNWoUjVsz7F4F6JLIZ8872wbbFMMcE3xI9e9zSQQLBPR + + Pun5mHEumKX7BmbWspcqs7HPzgiJiz6U5Tktcp64KqVugkVBvCnPmQlPTiDGMzwl + + djjfBRl/3/4C5K5ctbGcbiECgYAqJrMJqVgbo0p3dh8CDQ81q+NOKFaBWWLpbnQU + + 4J1pjVPtGD1Myqni1EeztDjPbjr43rG5yE6wkZv9eS7YwU6vKNf3QUr9WkYNGtkb + + 419z3V9dFGKXuM+nPpf5R3CZpfNlfuK/zbLBp9SyijIQIO4jSGbB8mI2BaJMFNG+ + + +37VwQKBgAkDhkwYoPe4SSLDuY0IGSGIKpcwoWhjTf8XqYqn+WnPzY7PPLBvY1D5 + + xmQBviFK4VeTqzTbKnCRMa0y7ghXuNfQHakojNa+Z38QeGviqaQXEzH/FeV4oBgg + + DegYr/wrCofy9kqDQswSPlNb66eGLe+JIangrozz1vBf7WOML2m/ + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 131073 + seed: /N4rLtula/QIYB+3If6bXDONEO5CnqBPrlURto+/j7k= + zfec: + required: 2 + segmentSize: 131072 + total: 3 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:MDMF:d2flo3xpwvfdgqmaibnsktn6om:7z5zabf3g3qktv2k3go7eli3nwmb2ocu6dweelykbgie4aenjcoa + format: + kind: ssk + params: + format: mdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEowIBAAKCAQEAsYUiF6mG9Q85KsOd8TlnbDuYrK12pWx5GRU3HGDyNLkQs0Q0 + + wfXHNn+EBzkiPI/S1jJIgdvytBKua3DR0vpUBnAZ5Cl3hqz6yp1emGLZEDuhxaGx + + t6b3KKBGsCgKVueOFU7pv4IgkZlNqmt50MENAizbR2C8iaoaJWbdPE3YACv7/OXB + + H1fj9trQTEa0ueL6JjR4CdO4pLy2LOFDvpCDQMl7EhwsWqtOeWKcmiTEV8Kouxn3 + + AhWD8M/pfmrFyT3uGneX3Q0KQQsDwvEAJ+aR16AYpryreqGR+OnhdEz1BKesizsy + + qR1UJ41JqIwf+dL6ueZmMQaMbIUVOQhEceLjpwIDAQABAoIBAEikVBketC0ft6L6 + + PW1yshGmKYmvyfdTdhJ/jfe87CALAvx4kqY0LvrsH1jdVlc1+27PUMBjAuQRKPKq + + ThJpgWzI/q9REKo5qr6yuvzcpjpwTHiU/CZM2qLzQzneiKybQJcTna9SToWGGDP+ + + mvCDrxEOzgRdX4lt5BkeCLYenJ8ksTytV8BavWwu/5CtzrAXA192tKdeW7iabA0s + + jOXxopJr7jBYq7Cx5xsbOxBGwpxLYP5BhVRfAGwxp18hKyXcElwRmmCrZ+cCI4+X + + TZSe/HSTOKTBG9laXRcJBjEOdLrFq6Sa74ilTdFm+PC6WuCIzcqWuIZE3hW+jVbb + + 4YOo4nECgYEA60+jL+VYDA65Ig5DSJiG4WDkgQNrq1vW914/QY92cvk9PFeRRoGm + + d3ef+MRaiStC9OTr9K4Zr4oYI8kPtlMgHjxvvyg990OBO0xifFeHVuJqt69uUgkX + + /xykZtYO94TF4qPWZ4psMmuqnsOk3z/83mGpTwHqOatr7bzYPCBb0HMCgYEAwSC9 + + awIJfJ7pOqBSgY1YUl2f9TXqecYA8dfPaGVf0c+g27KXnM8iINzkIAfyB147kjg/ + + p1t7yyFgUUyTVwa0ezHJPZ9Gi1cYxiy4lOu5lLFcH4oHHNeZb0Ke5H/rHYrupwo2 + + HO5/IFVf8Zfp1X+qyeTEdXff7teBetNn3zzUFv0CgYA1NhkM56v1bg7naJpGfFdj + + 9+k0U3Wxll8SKTnctXhvn3T9hD/R1dezBFYkhyKCCkpl3q6M8iHU1EGJNhpbfIiy + + za/nZk488AL1SdyriY+NUj4Xs5Aa9Pt8MRnsN1PDHT8ydSIy39Z/wGEg7dUGtw2T + + rDoBJ8mzqNQLOr0bO6YHiQKBgDrRtd75Z9pEq9PnMDm0ysmLKkSMfzVHUNJXYBvz + + hBNqoRtIcVSY4VQQ8omu4c/Mq2gFKZ3XBwT+zU71e4ptyFoc96WE9P9LL4hr5mu0 + + v3jB68TPTQtDvr9cEviU3Q7KWZUWTxTQrncyiV4TXmxfzaxfuFXuhI1BpXW7HU+o + + PxAhAoGBAI+2VYkif8AfGx7km9UfpG+21Hn3I8igw63doyWVnOeCfVXfFu1bXVpi + + gj4yJ/wR+FR8qTpTn32UIHFTvcxsuZc8Omtdnr7xBswWoTvY93fzzP1vQ1FsoSEf + + +/zZiVJT15jFe3DBSnPIpGJ13INc06mJI4+BPyaoamt14GamfYCb + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 131073 + seed: /N4rLtula/QIYB+3If6bXDONEO5CnqBPrlURto+/j7k= + zfec: + required: 2 + segmentSize: 131072 + total: 3 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:CHK:kzhg5zp4hcs75eboack4kqpcdm:g2cwnhpfwxrv3c4lrmpjyj327a274oa5qt4isu4avjixbu7vblrq:2:3:2097151 + format: + kind: chk + params: null + sample: + length: 2097151 + seed: uqWglk0zIPvAxqkiFARTyFE+okq4/QV3A0gEqWckgJY= + zfec: + required: 2 + segmentSize: 131072 + total: 3 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:SSK:n3oliuimgv2di77yhho2sqq4wm:omp42y6qt5ve7nxoipcwo3fe7qbllkmzr7kzayaoyygx7pf5m3tq + format: + kind: ssk + params: + format: sdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEogIBAAKCAQEArjAAPTxIkfQW28UHNQ4YJKyBCcOHmR9S0eZd8zStGJzK+3bu + + fkjxkLp2Gmc1vUGd55lwpBPmKLA7g9m2yUAAF2+hhPk/jrOc/3TvcDo+E1fv5pVY + + MeDD3Ls5EN4kfQGhbipUgfNBs0JFO6yONn7xaYnFCWYk9qiRguw+Y4egFNFqoP/o + + +pfFRztn0ROtFvWezu6zNiltHakI+j5yidF8i+kBlctZxmzFBra3diV6tYbtTlBf + + utEcVw1h6wAUITGIec6N9bmY0ObT2nYJAC5e92/TV8m66+bO9eHPDEYMypE83XDT + + rFhPJ/MIVthxGMeOAzcmKGdWIuF2f0y4taEX5wIDAQABAoIBABAAZ904PwBteHYa + + /QUCNPSVhksj076c0opmy8WuVqJ2sOz16YXfZJWjk3rsdVLcBsoCXgcsrs2ZFvaP + + VwvY3clJX4CsNwsAdBFBqEdail5Tiz3XBWGboNKTvnPOvHJhZneM2vOPKb9yfJK1 + + UOEvuzSzS88Hu6iPJsLsufSBvpJ63bDKoS8TQGUsWwYQvuAAJpMFRe0uKwrCrqCb + + GYyPXazKd0Kqv9WIU5ljLZdm8QdfA7YU/2icBN5hoaAxvjTnd7Ojyqs8qqtYbFvC + + 2yyrgLy0pcQR51tTxQRv/nP3Xgtq4rJv7Zgqlgo/g1TzYCslQcNGTzdCHmEfru7L + + LhHX2RkCgYEA1GYXozyWDWt0SIvOeaR3Ebi2BL5gIf/zbK45/ZEO/C1H6Hp9R8ba + + X0V3fFYbXNa/f6+mSU3IQNrxlfFf/LtD6/bSqE3a7M+kyG8C+A+cF4MsSTgyu8jQ + + wmEeqHry5KOjoYIBJdfRN91XxHe8+tuEEvYABU7PKU4LdUH2je9EuU8CgYEA0fHX + + ipNcSj3XvVjh6oM5tLAtM/c9eUE5/AAV+BnS4imDm3i5WgnwtE3VCOH6aHB/en/p + + nW1AEoL/Pk0fJkY+CTv5OfQC53MFH8IkNocMCwW8APuRJy6waIvKMU5XDA34mAFp + + xGevwQBXNMyaBY6Gbzq1zMhGJUCVHWPFaKnq4ekCgYAZ8K4KXafl063L/mclLBTu + + sSRpx+ZtwJi2OUET2td9rPoPRoZucbbR0+YX5VxKJmAU9BrW8Qz3/sVqjqQudaCB + + /Q8VRwzpxyJU6FnwedeSd469EoP/szLryni4Euv/SIz/eKUzPfxrWjkR4Z3O9WhX + + +HtgKpPac5GqrHe0NfiquQKBgEWSlVEQ4Gah89qFl+g1MGxWbcRozHBgUyzVgnJD + + bIUSKNDewt25qZC2skBNUsRFc5lOxkYrLC52RsuIlygB4xEAVOkFmejFTw9lMMb5 + + Hd6ROepBc6q+aCtdF9YbFfGit5z36urxSWb2C/AtVWU+BALcO97vB3/U1RV2OLck + + h/fxAoGAK10WdXnNYJ0v2+znHY+g2cV+U9/1ZO1hBIwgJKEdz/v05IfSEWEwhgq/ + + ClmJe3LTaJxw/J0iwRqIZTtNUPJRn9xLYEouv9wEdzw9jczPZ8u6pg5Il8OJu+sc + + +JXvMICkAIGi1jeVVQF5lAeHfmmRZYQJCUAmU4N32StfQiN6/uU= + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 2097151 + seed: uqWglk0zIPvAxqkiFARTyFE+okq4/QV3A0gEqWckgJY= + zfec: + required: 2 + segmentSize: 131072 + total: 3 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:MDMF:4yx6wxx2gw3pw5i2dexq6u4mtm:rsvgkzh2bxrmeuppl273ps4qgqfnxokff5fhpw5ekuqpsk7fzdrq + format: + kind: ssk + params: + format: mdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEowIBAAKCAQEAlCwlHOJnRd6uaZhsH0MGDhZxoa4G4+PyQHE+/YA1B/LbhbBu + + FmI010z+XAbGzzhXBIdNaJ1TtBx7HXveH2NzudXwlNNQbUlTAQdFaSkWCAURXox9 + + be1RrzNPHmadiTkkRh0WUPmkqjxiKKo6GdVHbrFjUPADuzQYG+1fFwG7trAPueaX + + zFGM4hdD6PMx58W33rntyx4MF30QiJ1FdQh/hcTxTlkyK6duVLTSV0++BxoSpg5p + + P1NrrojIR5QVXquBYF+KkKUDrs95xgwyyjfTtpYkjzrNmGaUOv+UYVe+cqgxLKuG + + +6t8eZAPDPS6PhlDDuxCJAJnnC6HIbpLupJ8uwIDAQABAoIBAEe713vUYAsDc4zL + + rgy0dgn786dCiTNq960bJlOz7fibKovejm1nvg09ySbkYPuRWw9mMaOkBxH7d98e + + SLsJes1NNdvXMei2xuiIjKIMsg3P5kjP2ymM6y7WuEcPhtUYROdszZEGSyHfeeYW + + A2reRmbgmiRlDmljHwjmMlMBE8+tUCMvCZ/dDbA12esaiTqgQgJDnBq1xNBlyOHY + + CItjYwI9+6SetN36csBYjHwfRPw/H80B1Alx9yHer4CUhypn9QfQiFG/ROpbawRC + + 4YRz1Z3dQiOofBxUf5xix+f69tF28Cl4p41mqFyNkx4iy6PScedV0ltl6yGXJStg + + k8y9OAECgYEAzF7o+qce1cCE2UQeZz7opq7x26XxzX15sQU77XH0f87PiRG4iqvW + + ctq+Esv0LDhbfWCQY/BB6xw26STCt9IRcHXik+uVk5L9TKVxPIPaugrIp964ssWW + + KScnersk/2ObG6TepuEY4WBsJAPiGgR5ew0ApLRsENBrgqYJrQC89PsCgYEAuZrF + + e3WzcfUBOr22CBF8JGbcWjHwN9a7hSXoCZ9Rl1vO0vq0VGMpiYaGM2+i0oWDW/vD + + uZEeJiwoKYCCjbYMlHfOCRZYtiF/o8U4rB0jcQ5o65OiQHsm8jw7eHLsZeGggAhP + + YnF4OTduJteUrVbx10qKMoiQvDSzFYDvwVh8i0ECgYADHxv94BmXeDZPPzwbpZlW + + Gmv1R+aWlekK7CKLMOdkIFuJI20nKRLAdFjc3qKfHkk/c/8gl6XaGnc4Pmh++EVt + + 608HpVyGgYM+7XP6UaVAnDOOZNd7W4s8m619sWgSQoo29OC1udBweNGOB0Un0pOs + + bnlpCpxv8U8DEtgo/U7liwKBgH2f2iaUJd7t2+UsXrbbTtE8pcyOnG7O8qFOZN2O + + biUqSLTYZ5HuhEDHQrIxz1z6bUym/XTuWh+wJ4bfqn3MSHt9E4FnFKhByCjK5m7o + + UgLFpBI/HMTUFipCxmXiM0tKCd5ewYx6DMt9TxsPM1yXypzToPJPKNeaO9RELwMI + + p1OBAoGBALrcFbLv0zzWjcrLZj34eDB+LMVot2rISlJtr6X961ZhoOWOfZI+B0+M + + LLmBSpPooKYVqpZIv2kWama4YCrcry8vgG7MkB5n+jkiGmNwFrNwJaV+OiLARVU6 + + LcEtObxXUb+S+WkSEhylyQ38zXDR+LgGWjPzKdU3duCjL+/G//O/ + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 2097151 + seed: uqWglk0zIPvAxqkiFARTyFE+okq4/QV3A0gEqWckgJY= + zfec: + required: 2 + segmentSize: 131072 + total: 3 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:CHK:6sf35smixwvpapf2zw7sllxet4:ks5zd3hkd7ppwobhnymezckszexpychngnkipxfvmlcng5fgjbea:2:3:2097153 + format: + kind: chk + params: null + sample: + length: 2097153 + seed: BTBX/ampNfLU+ox7xipBGiaSbgC0kcB8Gy7BkJB4oKI= + zfec: + required: 2 + segmentSize: 131072 + total: 3 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:SSK:7awpweikdkxbw7x7i6emo5d5vm:iij24bgytahmjjidcerztczj7pludytjfgd3quyfwzvindfion3q + format: + kind: ssk + params: + format: sdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEowIBAAKCAQEAvrLo/Z177W8famI+g8Avpb3jmP3yEdB+gNhoddoUFPxsV5Qu + + C1JLzjMMHlZFGEunom7EijiTzpKLyknkWiXGdXWfQ+8Q/o0aIoGZHVp5j6XhHbWW + + mRZcLXbtc3ZT0a6W1q6niBKiXYBn0Q7nsHwf9EpPWn0pMa6Uy0nKnD7d1kqz08sE + + JXlQGJUUXlEqVZW8G8D6jS1alFpRm+lS3LYsUoHREPgRMzmJp2T8m9JUm/fwN2LJ + + 3rdmkbj2XjEbbwz/412b0WlSfk04F9OVNBcZzKAere2wIGqTgcW+TK1n80yab+ik + + vrbybiuGOWQXFNOim4LNuoj/kbAZhybTKjdedwIDAQABAoIBAAs1OrDFOiZltJDG + + sdC06yTKBbMIbqf4LnP/xBjBHRMKrvEKxxYCDtO33lhEgihMy5wMkV9j6eDNoVeq + + gut/DHkubsWwXe0hrLQm4WSydf71i3osIdJhJQ1eNjoZOneFbZCA9vaHzQrC/dcL + + EdHAMD4yxXn9BzHIH/vYT22VdRy5PYmVQ0b8O9YEQhC6AtQpms6T5jKPNAuRSRlD + + xjqvRES1k3afAZjBIddGrx8ie6C2GnJl/C7tKsVPQVrGVh0wEIZVzuBNLtWLwFp1 + + zxZ08RGwtT181e1Urx7dWrWjbJi2KHnbPgHIaRvTdvnBaiUX3p7MJfnquyNFt3mU + + WdfoKhECgYEA9buO2fFcjBMPswktFUnThHJjKOfsUuHFWbnfmUORTS36OrFHOhso + + mmG6dSk4rdxnOfJ/mu3a7EElicM2gnhluN/D816DD+80BLCZmce/4zLab1ZU+dW4 + + UWXZBk1/ElO15dsGCSQe54nHMGH7Pfg5nDfpd1MVEw7Bwakf/YNtZ38CgYEAxqqx + + DhLOq8JqDwbKOlwVOyU4BEk3f8HcDVcgbBzF7THHf96rVf//iPxjvysjtamwv+ml + + S3WJbuQzVB5efBIXLer5RSWL6k01YzctOYxEbDGgiXxHXG/STZITnUC3pL6aY2Pe + + MMU/kzDdiT39A0Sw2plh6Ns8jpmNHLj9+vJGxQkCgYA/y198qTJzkwdCXaF8o1vs + + SJ4BoqQxqDdJ4f1wlqAEP2l1D00EgsR5v+FeRUNXr56E5rXGDPYG26rZJvrhyEvw + + QPdoGSNBYcJJbWeTCs6AN1WKDgmlipx9VUmQX1Ib+euBLulUOjJjvdsebnGBVw3t + + xn4v4jvYZL5cfoG1mQcwFQKBgQCkEqKJWfT/m1+WK2hmzFfocfOSbpl8VLGU/ujT + + AOxh2aPGsjJUo0j6bF9AqbMjPBKyXJdb+6VWRPczOKWV2Cb2kEHv3nNwPPWjjBU4 + + muSDanUINvCEogFQeRzj2WgRkizVeswtASphOJEt4FkOEvPwhY58Dlwz9RK6rvlr + + AB58aQKBgAd3SjZ60ek1q3DoQtCZKDKQq2q7chXMZx2OrOEQOR+DB5XEks5IwCvp + + pIgb1CCgYdcxtesJBO5eDIsoIbYOSxBPMnsTFfsOqAXDaGNu9pIkcEmmGDjehwlh + + W8kSZXq8nnR3zMzq71IOZzYIt3LUfLql79jycoOZ0CRMZKMxbhry + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 2097153 + seed: BTBX/ampNfLU+ox7xipBGiaSbgC0kcB8Gy7BkJB4oKI= + zfec: + required: 2 + segmentSize: 131072 + total: 3 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:MDMF:rbwwosqh5a53jcddvkl4jlchey:xc7yci2psgn6hygl6k2qc3wjaglpqxqy4gk6ij4vo47z4acmqlwq + format: + kind: ssk + params: + format: mdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEpAIBAAKCAQEAv11NTvTJICddHij3GyfbhtkFH/cFZkPuqF7XDcjha6myDbHX + + IbH2+jS1aubjf3hHKvKx0fl3CvdJMJUsz94Qrn6XsQdy8hu/lVpGTUCdyaw8bIwX + + c5o0sSd8d+yRgdG1Y4vz2optCHm1sKKMzxDnNODM2KX22Hlml7fbdp8q18ByLzh6 + + ZgjowVxL8coyDBldpvkw4r9m6mXJHDIi3MLOqvtdcAUIKZ4q0dCmtoVrZpHJl/YZ + + bHhMa+eBqpSxjpS7uNZ8szszEAK+1aG/fKQPi4z/SXdUy5v1beYEInDFpvXmjDEb + + xDnvKhJhBXmaQsSOzpUSVA/u0OcGxwLHiRSfTQIDAQABAoIBABxMRIiat5wu3fz+ + + A06LdhHKiVC5A14kSQgyYBxMepMkaK1QQVcc/T/yJ+qrQnSA2YtPEL8TZAhl2Xea + + 86G4faCEHVPjHVsSgeHo09EML1kZhGTr0XL5mHWi+Gu2eqznESrrkO+d/TId52F7 + + pBVhs0L1RC18W1SXHTXtzQENV/VH4RWwi/Aa7wFsnBD5lv9bxqPp3D5Z98xzB0jz + + +OpkpxLm5bJ7yCkjO1cubX946cfRn/Hj+oBwkJFb4OsRg07u9JySTSixQRtO15Ph + + R6cibGldFdchNkr9pc4Y+MJcCxFHHv5N6i/LpKTCKPpFUV0LtwACJ/Z+StM7CrCX + + zgNToAECgYEA8WWVHTTBJahw3zmJrw0ygNUsT3AdrGK70xtzhhUz4+HgoDBVckyH + + 8d2yI/7Fnu0AdifSf/QTkAPGaXOXL+5l7WZ8X7FXRvPnyMaFo+WsN4gL0hW49WfE + + LJHRvklv1kCygQ7g/MEuDidSeFl327vUVg8v7ZxegRCdOTnFg0KLv00CgYEAyvDj + + Yqty/P9Yp0LS8nvptXmhtqRweQqSJdDLIwpPAxmDpWe+47FRq9toLB3vAojq6fyU + + W3SMW/Bm7hi4Rj0BUt/WDkSE8ZvzlBxcaLZXWs84vVpV3jZjzkgXf9Plv7RKN7ur + + XFz2gqZ41N5fnyYkm/IJRwOk88AKdeGmhEv8YAECgYEAlBYJH92ZD40BkS8u86BY + + 9wfPIvxYd8QqDRuuBvdC2e1ba2m7QV8Jlqq1+bb1bMVfnxxW2f/VcGegdFhgyxqo + + lLZmXh3guLov2s9OdHkU6QwglESXLpT1l5Hs5ZsPbJRL7Tg/dU7c/fnJceMQ0E+t + + tw2iDVX785lJmi2CqT9Nk5ECgYA5PL+lMJ355Trn0d0VLwW3fVqy3KYsPWMC72Sb + + uWiXgzayDBS2u2hBhFxZNQgYOu2mmOpu8Ow1chRVyvsONF6PNTp2Q7ULP+TvPSCD + + GAqDPjbOkQ/u4IA9ye92yhjefMcB+RhXsJCGQNWLlDx78pIYuacMNGbtqJhKrx37 + + 6kKAAQKBgQDXLFKt8Kg9dU6/IbAP3nUg8Wqqzxglym2FZ9TJWMTJdmND2AQhEkYc + + mHhUBmsDsooidZ9wjdDAQAH+tt188V7hcpphOvaG50/ln4VB8N7C5LosM0fo5Jh1 + + 5Ry0vnhr4HldVHPmfDA3N6ClcM4n9/ex/Js1BmMiHzF+JGGXHjRWTQ== + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 2097153 + seed: BTBX/ampNfLU+ox7xipBGiaSbgC0kcB8Gy7BkJB4oKI= + zfec: + required: 2 + segmentSize: 131072 + total: 3 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:CHK:dnmjqjku5vyrkeaulae4bt6juu:d47airwaqedeji2omacwsp6yfqp5mpurj776gmzvq43c6dovtdca:2:3:8388607 + format: + kind: chk + params: null + sample: + length: 8388607 + seed: w6uP8Tcg6K2QR905Rms8iXTlksL6OD1KOWBxTK7wxPI= + zfec: + required: 2 + segmentSize: 131072 + total: 3 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:SSK:v7lmtefejcs67fv4qoppjrdg3q:3xvveqfckxsnp4sxb474uewywjhpjv6wg2fooxcx6tnrb5gxrgma + format: + kind: ssk + params: + format: sdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEowIBAAKCAQEAkl1jyJ78S8yDqSPYJzglfkVx22MRTC5Pb1639llFcu8fA+Ft + + ttmoGV3T1R+8ZujVF8b32qu15AOoSfdttHZNqCgk5wP2vIOrcP1UtiDN/L6IF5mW + + 5GM85i5DCLZJnoHgRzA7zAsIFrJgVSwSKZM9RAFGQt1fQn3lUMH4W6u1MYDh6r4P + + F+++MF629Gm4CV3mxjTtQ+fRHoZ5tAviJ+ASJ1J0ygFVI8YC52TqnH3w4fHqpu2q + + RoihiRwYjGyp9oGLNfWebzH0Wbja5Zm8wCPW+IxOhEsVnuDgw/ocs1xF7wsmlAEB + + IpIFpEc6xvrdTWEsggoNoePhn+fmwPr04sn0uQIDAQABAoIBAACp2CNOK3kIM4IC + + 1vDrtpc5S8p4b7SQoFFo/bgOgolwMZ4nhzoAmiVAbvx0sAO5+ggpM/Fqc/xnrmkb + + bDth9+aTct2o7UwJmsGzANin7iA/YbWLZC+jRYV4KO3FicLkY/anyfWaCKDaR/+m + + REcYZWjFJiMrmQDfp+88R5GRGFT0LaIg3jCSSr5kwbgQdZFK+bwbYvr9YAhF2wp4 + + wMZVctrU0BrpKRVQYphJ6DNUtd0MzBsVWRveowfhMoQnOK1jubXKangOQ8+a16p8 + + ov7LqnPFQ/EsHtWgpw1ssMzrhh2KMbGJaVlXtSFDv1A8zjuY1y5h+HHTj5r2Kn3C + + 3OPA6WkCgYEAziSu1aXxK/WAr8ZtCnRgfbWh1bmhOY+kwyZwFSez/fXik6qQUiU2 + + MesFBm+2DKl78MipS3ca9l6Z+tfaW1JxEP+ghbaXq74a0eLeHO9nzBzY1GhTYPLv + + JbYmzQG3q6uOu/ii67gnkEeoPeio3p639fABvi5Mo9AknOUBFOq0q80CgYEAtcOI + + 4yNoHBKMrkQtYEzP9dPXCHFo27FytVkI9/Obgy2477Ps3KsvMdgOdzSNP5ZgLNBl + + UgETJeOWRai7saVhFnto4Kuo3RBAbO+DgMoSMCcZ12qEaR3j+Oj95ZqWuPw/ld1S + + 2zapkv4uVu8VVDO8yyb71zp27Ftwij7FjDfG+J0CgYEAgzpm8isJNGq82SkAET+0 + + jVIrC9t3/ySqRnEZuN3lfy4gZtCVvzVhIrXyJP7IbZcXB1k2LIxN5bijXUQ8BRae + + U6vnjDeIphQHDsXVj6X39cAHaHBhY75C70bdvHPzcJ1t58uIK3a3+Okk+QQ7PDzd + + 7voyodbngwDlzdsarS4chaECgYAGowoniQ5vH/pFDrY9cvCRAFg0tbdndjZDCuo5 + + 64o9IvlCv2YhtJp3jnUQwzl5HeuLF1zrvqBNXN8K0htwZCKEaKMuuPXkhIhlseUy + + Wa6KVZMq+3e0QuQlHZTPwnJIdOV5emhhGsDcXi2g/P/hYDY/kL/XXwoinUAhvCMI + + eKzqPQKBgEyP2NxMeNwaPoEjltcyYyYvriDJMAqPyZ+ntLhHJLMl4Tz6yR2VROWJ + + hQ8mnSGJJfKgMf/yxXDZfoSiMK89pekmCGGQsZN32Pp/zW13n9pMuYRhTDbGZDZb + + qEVa41y+2bdvz2P7q2yZAudDhc05o3FSX7QKk5EeuMtIkOihso92 + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 8388607 + seed: w6uP8Tcg6K2QR905Rms8iXTlksL6OD1KOWBxTK7wxPI= + zfec: + required: 2 + segmentSize: 131072 + total: 3 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:MDMF:oto6ivpouf7ay5wjujhvb3w3za:aezel35hpiz3bzi4z5an4lmx4c6aytncahbkymrvyddvw2tgefrq + format: + kind: ssk + params: + format: mdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEowIBAAKCAQEAvKWc+29gXtr/g2Ar3x556n+9SMJK6KLTYs2VMJJgc8xugZZK + + eaAYGH8xQlpPjxc4DZO4lIp+2fv9/t/u64Fonmvz7nHFME6j0DhQZL9EMZxEwzGW + + hgtnTmc1HeeQf04l3BnHGJQZDGfLqNVKa9Ff+lfESfuUbFGqmuQWXH3ljZD0j1oG + + 6uSM/oIMiuwPr+HW6Dfz378u2+b37xTN/BGN7NfawDeEzjm2AtS0JsUV4mEtNtJ9 + + jPcgCTcdAys2hWTV/tSJ8h3NUYeglC4sz3P5owwJWsyO1oWD0PeL3DCn+9qJ71d5 + + ssGQBduWir1cXwIBlegGJrjSCG//5TsY9W+s6QIDAQABAoIBAAFP1oZiGSW3uKip + + ecygqeDhWAfiQAKbpUQt4VB36B9OB+OzT5vGavx6n/VR6vU4CF4BzboMt4KdD8Be + + vsrY+MkHP6hEFsa1+Uoophh5QwhkSY8g8GbIvARtz88ALf9QpA9Ch6GqX/032JD5 + + QL38tAHp69XG39qb+8d9eBFXF8pS+Wnpz7QcuopJWKX0OPdSwM7hOV+rg8igkmbW + + IZBv2NM8dczjhHDewJiExAggvoJlEhAuX2jOyoGCEFRgcR3Oen85S+lmEX3RN6CX + + GJJsaenWdUpP/5pCIHe5iThPZ7osU2AwLp0Y2dyc8ytDbwaoclX57VnmMksDyfWw + + 72xZON0CgYEAxoolFefkRReFf4icko0KKtHPa/9O+s1feSuAzAT1KIr3yzsFRRu6 + + ziVF6jUk7Zr4lbk6r7/bxgeLRd4+2d2TxzNMaKbBCgaRxwhmx4peXk9Ycoy7AcMW + + lNHx+zTtW3CBYsUPYBehIwCO3WfXnvm83tWwguJHX/cBpowASif/Eb0CgYEA8z6D + + 78pyXPU7x638ZoBie0jNxv0B/lVi08Y6VH+oHW/B/RGWDF0lwNCTZQkz82Zl8nyl + + X+KewZ1d2oxEIhkWrhEo2AWUtv/ax0IRuzNmBekwW4hlA0/4JSTvaJPQRH71+8Pg + + k1ct7hgVGIHozp3gu7hYeEVmPmmS3+PAJmkXvJ0CgYB3QzTT2+C7wE1pNt8XCbI5 + + 1p8K+Oqwrf3UA9XyuGesWw5O/r1Drkyg2LMO5a2xLY52IjamrFGQu6dl6QNITFoh + + JyeXFdSP+TJIpTtYUj4t2OwAo5kSjeZar2L0y+5pJ0QR2N5LkuYw6Hzpcx+LV+mk + + 0iid9t95Ph+3tBHYef424QKBgQDqcbC8p8V+bycFGF6TdN52sP8U8brAJhAwyXhj + + BP9GD/dLMW4L0KOYqe/GjA40ZNeR1i2Ws1gMiN5yzIrGyqOfdg6F1ys1DnkRYE6y + + vaFxxQXE0zt469TiCC1wADfWLQBtfqevm3E7cJ60llGLA4Qdqloq4cjgEuVrQZpr + + 6xLjyQKBgCV/I3PWI7ShyxxqcW4naKUbZuOgQUZJ6UoF+KRwWmSi+gcdjLxcfqvV + + 84EgdaIPgh4NLKDqaj2MHFkILKhDXfgcVZ5evquwVH2hwwOyf86uI9a2yrWcJsy/ + + EyhYqdxLYiF/mVCusAkcMdPKOb2VhLapMw8ILcREMswrOnpLEUAi + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 8388607 + seed: w6uP8Tcg6K2QR905Rms8iXTlksL6OD1KOWBxTK7wxPI= + zfec: + required: 2 + segmentSize: 131072 + total: 3 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:CHK:bfjedwcwjehjzpwjsgwkfk7rja:jpwea2sgz4hfohqab642yj4cmrh64w6nfo7lv3mtacfzicurqkva:2:3:8388609 + format: + kind: chk + params: null + sample: + length: 8388609 + seed: yPi3JHKKbWaEEG5eZOlM6BHJll0Z3UTdBzz4bPQ7wjg= + zfec: + required: 2 + segmentSize: 131072 + total: 3 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:SSK:ey7ogxmnmp6rl7p565cp7r4x7y:opwwoooqzrp64hmta7syc3olj3nalfmjilouuitoimkxqch4kmza + format: + kind: ssk + params: + format: sdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEogIBAAKCAQEAtEAuvlzk8pp9nXSP7bybiM5SFZA+DzDz+yDy7iDHUb97RseR + + DA0jVF2aNdB3Ju+EdoBsviU0lMjWu1qdqMFPOmi49Et7Owyj4F4y96xHA2t6tijm + + 2Y40/tZGEDB45LQsrPJ/qFEfb6oqXXclarbPe/ZWKzwc/oAtF9sdbqOrnt/3YYoB + + uakVX/2ZwFzlZrQIL8YJzZXXkm2LOqJ5k96mZzlOAoPaJcfYePCbLxASZ83IlDRc + + EBojklzDsKXBxw3KlnKXvK09K9JofYAYYFtZacCDiZkDeUopJOT3FgI3IoPLjitH + + reUQqpekexPp1SHgg3FH7+OVjrkhnaaF+DP1WwIDAQABAoIBAD36C5h8zGP2Ztaq + + 64os3bXOaz7q18vVYy6oB5+FOcOL+VE+8UqZgdpSTOHQCggjNwKf6cP/evLlk5/b + + 6nXJ8fn9ZArroTWOhRJykUfDvq8YV5smuSl40hQFjRWn9Ql+QhY9U1OGgS6d7e8x + + NnZY4UKYUsyO4NFJNTgMqTQPpsT7XJnL8lntHRs1bKUa0mciOR7YRBsH2sXgDIJy + + hxqUHjRQju/CSI055YGMxp88lXxp4cALqWH26ThIM1FryrWRZuhbX42jB0TjrAK3 + + 1VfE+DPsp4gnDd+p/rSUzkpTkFWct/PjRaMM3F6IOqNJr43zb9+rLs5N7oEPwJw5 + + CWUrQzECgYEAv3BP4tWXRCfGVpMTeR/6bWtitLlOI3OUuD8jpGBLpwcI0im+CMl8 + + Uyjvo15YqncuBPLYfckC12D7yzmZmru9a+tY1GmEV7cIgUTFaT8NlKbr77MX60yl + + qq6rBVWwkRNt0awktChn+beFAewxF9VTCfKPgSlp7qBb4Nn2our3oi0CgYEA8Qn2 + + 12lQG2xmhjLQalKfsxEUQG8BojJPSck5bLGpwV1XL7/ygcYOpGyVeNOFKUNzOo3T + + RlhbFZSKcPuRV8QJRQqj4uVXa0j+g+zpCOIocB1GGVDfxgGcLH/04ng/ttBXMOJU + + coD78uXXnVxIPP9jQVIF5ddNocMZo2uUAyeKEqcCgYA//3blWQwxn65hgNeQtY0N + + iUm9KvmhRmFgWtM6f2qrEuHzCDtcSqdCUbwS/FZd3mvHAbw4CLvnbqeeX8om/T4s + + 1seicwfoHus789afAZIzsL3NKy0C32O+tJe9t9DIHxumbYrzo1JnG9/eLayX0Bvr + + hmhNAKBGQtuURql5+1z/nQKBgF8bYmV+rVgUvqNm+2tobJEYRRhjdI6OMVDY8Cqe + + M3ATp2o037gq8O4Z1iSVuW4dqiLJgTq5dD8gnDuWV7P8qveuChpmCcdQRvTBDvYt + + Xm1Wb6lfitwzGG9KkdKmReWZcT3doBqKIF+oJxp1Jh/DWWOVvLQC7yPLupsLwJw6 + + BrXzAoGAcdWwqRl9uFjz90yM9K/JrjoNrhzuPoL/Nr28QkEpKmtgrE5+8FZaraIH + + pMB7jzeLs6YUUx7ClHA33L8zxiChp7AsDwIPT18rf2oEkqaXQgDn7r2NV0n6xq/l + + 0LvtFkI6TFDveAh3C8ANm3gJXtjiQrwWnU0UGuj61U7BdAKjJKY= + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 8388609 + seed: yPi3JHKKbWaEEG5eZOlM6BHJll0Z3UTdBzz4bPQ7wjg= + zfec: + required: 2 + segmentSize: 131072 + total: 3 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:MDMF:rxfm5lemmemurohy4gqpuephha:tdo5tzgxfnklgaecvgyttogn3lyfgabplddabhdp4ilawedyja5a + format: + kind: ssk + params: + format: mdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEpAIBAAKCAQEAoec4LfFaZ5vWVohEmozJfp4zPpG//BmHK2FBJFqafNZW3vo0 + + 4dsnXlgYN4PiVaXWqtK0Q6S6API/QAmmuACVMNxIc488T2gzDX8/8eY/FOvnrNYy + + xZ4ob4MvMfUThcJ8azqa8HOZLHn/kgq5CsGajlnpVimVpiuD6k6l3ualtw9745Pp + + bvij3ZlifcYzDqc1UyK3D06XA0ShyjFD8uGd1rggkAF4E9AFfljl38pUJrcqU4ui + + EZzdjU2qY1hrHFR1BY+agL4PT/go2f8fhCbZpTyJ2EDJIJGVerQ4kkp8J+uNm+ZW + + hbnH4y9zwt9/3eYbdIdb4D6Ujr8jjJ8pOLk0NQIDAQABAoIBAAaG7euVCqbmECbB + + hdNQ1rS6LjWyplgRZv+9z+SZZ7E3UOqnXdqdihsSSroLB2TNjUR3JYy0+ny14Qy0 + + 9uaXIdMFwA2ZZv4kJMUoB3Nta8cG2ODnN8YvLjeLEd8eBfEppvg86IJfuEg9parr + + fL9Gw9mpg6/7q2YdSstEHwkuW9yJfLX+shRekV7ouu7zDy+CTQltndurNQUu4Crq + + GJTs1bp5oXZtBZQoDAQykd4WJbb0XfJtx7ALiYEBngX9V+NA8rUpZcbMKXhOeZHg + + spf2eWUVWL7VfArN8LqNKbTqQbNtstZlVhoRAjShxATqjtVv6FSwdStnCvr/VJcB + + n6dmPRcCgYEA2qu5hoauNn95ZyMqIenhEndDqqMaR30dPvQceTLxD5sYPMliRRc3 + + r+LLtFMF4LGM4isMxuZlRcLea+josMcnlMH8zLwHbGuavQ7Oq4D06Ks32IGaJRO8 + + 2bS0uZQE8tUYOxMpkQactG4NjjaDadhYS55+qWKiX4nnH5+7U2NYSCsCgYEAvYqo + + KOG8d0gysogLFUi2dtU4Igf46CZ2jVgQ8hsB52R890tT2Gdlteyc/Y1FEO/kIadJ + + XqtHGccW9lnJbulq5BwR3++ueyjQsk6bnoNvgtBXPhE0gO138iZ+hEQu5pgEPLHw + + TJf5p58vxElrNz8klu5sRxvB0o2AB0O7yKJu5R8CgYEAnh8bEtoE08et5BSbfNaA + + ODghqBw0/ojMQx+GD2X0xpIiHqKI+ujlDbx0DLsUPvxkoY77uEAV7zIQX/uVd28r + + gfgcc3dr7syIojk43O9tKWnWAisFadYx80MmhCMyyN2qnd0na4Vaf2YtSy7ELB+T + + CWtcr+NxAqDXjhiU/qGRzu0CgYAIVQ0ZZvsC/2CDKqnaEK08whjKnjEZ+37grcto + + 6TkHNAquUFhqPflhqvonx0sO+Iy90f3OtJbWkkL3J3FMd+RkDLvYbU/tSBkMjZoX + + uM1xIbmEF/uH42iPc5PCOsEZD/u3s1bN9yxZaw0NgvC8qADyxZ5q7dRybhf/ucGK + + i2F2nwKBgQDRktqOutgkLAJedrDQPZu/PcKROJPFhranOrwsgigyDEQafCDQU+66 + + ilfG463YuwWv+a9hrguhmiuo5N+KNnKB+DxVAp8ZstGjXoy/dCK4GBvPg0T91pZB + + hnYS9HmebrDkcwjzaXfPGxJz9+F9xTRFPM/wv0rA5OobWz0v2dqObQ== + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 8388609 + seed: yPi3JHKKbWaEEG5eZOlM6BHJll0Z3UTdBzz4bPQ7wjg= + zfec: + required: 2 + segmentSize: 131072 + total: 3 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:CHK:hah7mxwfpqemm7icdh3hwsa5fa:6epvxt2uxh42obpnfn4wkrplqml7voh7aqpnqnapu7ffcyn2hk3q:3:10:56 + format: + kind: chk + params: null + sample: + length: 56 + seed: YQ== + zfec: + required: 3 + segmentSize: 131072 + total: 10 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:SSK:xck4f6aog76yfi23xfnygmfxcy:qcla75enyyitmshc3bvu76dct2lcszvrqyyp32pwg7qb2j72qceq + format: + kind: ssk + params: + format: sdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEowIBAAKCAQEAvWvFaTZQXoS8Cs4I2JB+1kWVoBSj96SeANqONT7BzkdbvIYr + + bgVhPLYNBc8o1UHiVDAO5Nnhbnp1MxOsxx55xJ5bTXxdX2JPAH4l/zHDxapwEu30 + + AnfQ7IDcGxLw1NeabcVzmf48NgWswkXhiae52u9MCaE1gRluQ55pv3HYW5YgnWjz + + T/vP6nMh46L0vuzsSKIxwVsYakx+zSyTaAaR0wT/UdjESvM2znvlRqSFmv2K2rIz + + tNTukqxFzXCK+c5/X88BR4aFtW4bFzkIVVDbaxcbicMlRQvelYhMZRP9yUpOgl48 + + IvicsROo69Yjs8n3CQn1r+sx+nmO+4hPfHMB7QIDAQABAoIBAEmUAFXTHDrq0mRc + + DgGZdztiQjGxctOyJRHt04mJPB0ViOPdNieBfXjounxEMPdNpU3QcSiiHbgdZ0MU + + 5GtgQiqG3K5nnZl8hXWKitXrDcHNZ8VimwMOaaBEmbsGi3gR29HB/hqWL+tIHwhD + + vs0K0t3hUCb5cOAZ8cPgV/Felo1UQeu7dwVYrWHxYLj9YoMy7cXICSDpRrwTXcjO + + SR9qNv+TP38xBO6+cFhGYpZ73fDP/fbvKMwSq+EKxeJjpWujWctByAohJrHxum5p + + l+bCrTFPrEicG+r3nq5JQyzFjHRav9TcHbxaRKfkUpsaJx/kATbxxfGOrUd2kvVA + + 1857dr0CgYEAwHe76JE6dwSR8KUdCwz31e0kpau2Cn851sJGVTjXIR219DBMT0Lg + + c3Qq/kn7IzGG9uTRQ9KRJpCdCDBi0EhVo2cEKsb/YAYEQHx4yDPZx0E8BAdwx3F7 + + lwH8+7f5fy4Cy6r1rhIX0QjRzeBkwxXwru5fGa/KZRaa9dKrR607SwcCgYEA+/KT + + n1jb7a7QgKJPNlj0r3SaRL8gnj2sWztW2kd82TliUghfdFwzEpu+hBX37c3GMedJ + + MS0eQx5ccVIZgchScLoJOTG5z91n+s2N9Ni+PyyDEwBFeBzMG3iN0xWUcYjVkqIx + + Ea0pOuWj7bHTHwJ6xc6NZ+3HiIlyJfU083ZoqmsCgYEArZKNjRSD9FfTwYE2awPb + + 8jp2RU5Q0rCgGbSEt1CWepAPytNPzl9Siexm5YMUkE2XGMuMiay5KF1csMjqJEpH + + qSA7WtSx9AgZB4r5ZhuUuCR1mnCXXdZTDgFGBECLKg31iXV5MO2yOtrIUvGeDW2Y + + 7Dme3Exzq6yyPSUrQG3SvjsCgYAiH1x2/GXs7vw2L8ViqvGYwcYTAX+9bsTlJkhB + + D+WM1gTG73NeIw6Xupg283K8tl3dbGGxU1cB6B7FCkWCGktwEQImyOFNkcL/aM+N + + Fb3OeIzYCfVeqyfJoK40pHuSVOH4FhdnOXiYDXoCO09Ip+FQ4QStyrp3d4YKNgeR + + 4buTOQKBgAnWPN4EpoCe5gBJ/dgbk0mPdGv3g9CNTXD4hJak1SjPybJoEh6ao5tC + + g+taaKrwfzU0X6BDkRnU25PNRs7/vtoSqf7XZOwUSjKfpZ5Vv9+58l4CfhJxiMpZ + + qiEpJQsyLoDqGKzUjFwiwzjSHlu8otcjb9od7GO422TmuJEknJj1 + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 56 + seed: YQ== + zfec: + required: 3 + segmentSize: 131072 + total: 10 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:MDMF:brrsipa5gkkna6tojwmc2hcrti:rr3nz27m53mcrg5nqhknc23fhxg2nz6t3etlc5gpv6vca6fleddq + format: + kind: ssk + params: + format: mdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEowIBAAKCAQEAvt+w/o6xb3qswBmVFYjKQGV0W+i0kAlL5cits8Bs9HA9tU7b + + Bdwx8NryMtkQR+mYuKsrfkOZZnK8c23uh4wgEOjIF2Tnmmu4SRpcUTqFQPNplHVV + + K6l2aqbRM9KYP/FJY6Al3ZHmTrVIvdXP9vAWBwLWimUphZ0+wz4Si7y6SrwARtRb + + BPqoMfSnznsWZBCkKEIe/ed0/zoQcEO8+GznupMm59Kjw5035pz7leo17/kf343q + + aKMTPIWZGwxPiN9MmDKV6Ujlw6aYYAA+nq9iTUDwmGJ2DjgnYx7JWid7tK0J2Wqr + + BBHJUrFBTSfsO/pmczmNUJa2bngUKq5WjuCaowIDAQABAoIBABPZMQ+XiQ39pL8p + + Kd6eZeHCaxIvpa8guFrBvoZlqS7WCSS0eYQnfK3+JpdxCQdhXDc/3Xr4zpffsIcU + + VGyV/rOjcUM1g/wD3ZsEebscqcSySzVb6iprKdw3UqPf72Me1THd8nIS/O8MXO8Y + + r9KO7st12Rd1I5c4XdFxv/319y1UJzdbLEiBy1Ongc8XEdkPABXMMpSAYZGKi/4t + + l5LZ8/fSasgcxmJBR/TEwN9BtqkL+EspDWjva6p+PFmJRbNvK6QuwQ8T/1iPaJSY + + Er94JMvb7Kb3GwKqnjirgEvruzY00EaXh0X/5gD9x1OLHxjdjAf9EZO0jWEyRm3u + + NEFlt7kCgYEAxiD6XZ1GGWVumCEsAyWhRWCGktqy3F5iMclqNBn4xZBaSUwp5pD0 + + NaRHx/E+EPj6X2OHVDXT6MZp6w7gHm5z22At7qnezCa9PpyN2aFLPrqIyWFRzj2p + + PGKWI9zeB4ZTAMKht1Z6uGRMnos14Ej1fw1zMbCQ+C20IFtG0arV3V0CgYEA9qA4 + + lw3/xH+2DQZqbJqyyuXF+d+0PJlTeTOBdLfHwhWMH3AtK/k8tAxwaWWxXnJx6TL8 + + 6Up7JlUhZUG3PtzbV9mHhJUPB0xYddLmFFGwdZz8IAtjEoFsqeR71puGamEP+nYX + + uqLD4U3QeHxcGpmN+sK0dMOXQ4Cs069gOYOw1/8CgYBvch1iixTjNCsBZ6daHdCZ + + NbJ86IezbWPOnX0f0XwdpRUkJbNr/h1gDwhRb2F6KpKrFVEKDT0lsnXhwnxOodKJ + + k5BCr0qjiyboESe5QwEQR9ypahSZ7hVD4jCR+6rokKYfx1svxXVCQyjWBXhIsMFm + + tioVyTvCXfL0QGOVjILAAQKBgQC2m2lrRx1C1EDqof54zY5mtv0Ah8e/OtPYoO9Q + + iacpqKSovnlj3tY4hiFRmM9cnCaFwZAL+G74sf3ZKHBS5lquUE2MOIX5JGk3TGG5 + + V8btPsBbxbKkiBn6LUgYXe2HpLic/YWSVmPs1Z3vKD1WIK5EppfRAOVmQMc2sdrw + + mvZ85wKBgDDbaHWS8omeJTojnyx7OKHo4SjRjK068sdKXsGsXlMQBRWBXwMlAvda + + vbcX62A8vx046Jo3nnRgMwghh+bPwltz0r/OkVpcZ20CgkomNt99/Um/EcpxwJHU + + sqPw/aQdIv5MC12w+Q0U/zSHAo91JtaYVjP/15uymD0bHi35Ux2+ + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 56 + seed: YQ== + zfec: + required: 3 + segmentSize: 131072 + total: 10 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:CHK:dx7tvyr2fc4u7lxjc6kehq2svq:tiy4qh2g6lqejxcaym3rr7ymkdkinn4qised6kgxloj7sptsqu4a:3:10:1024 + format: + kind: chk + params: null + sample: + length: 1024 + seed: YQ== + zfec: + required: 3 + segmentSize: 131072 + total: 10 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:SSK:oyn4tomwx2lbijba3xhl7rgaiq:hqgcwcw62dwdm7ayfobetecvgxau2r6gzl4u35l7vgtb6oa5zrxq + format: + kind: ssk + params: + format: sdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEogIBAAKCAQEArG5elAgFslXu9Sxyjf34WCmIruPJRsFDEoMy3dKGIQYFfKv2 + + /gvQGALQDG1PBrHMNCVKoFcphjuP6Xsy7kw7siQ0sH5TwbtTcKPyYVfKq9SXkTO1 + + db8Lg1Ezi2rA7oV1sntfyS1SBqlMrOhby6mHwrjf0pnjwSkwvLLMjrfxYatgEAD7 + + HfrAKt2z2+uhnyJx8q/oQCncoGH6zw0HiCeJ3Qf0IsV5nhfTWAhLrRqnwkCqB/yl + + PHR98xyZ6RcFm0m88r97GH3JmWuj427xwydGYWYISUSR1Ae1tbExifYFUP25hTIA + + Kz4Y6mkL/1h3JEFwYopFPdJYLUdS4baAm50WpwIDAQABAoIBAAmEkBjotnPJFYcX + + /HzE+4vWQxKwRSBwM4UWk9y1rayt+eiPT3NncIWaxiQhdn6+mrB4LH3cQdEEgaWY + + JNANijADmprxZisn7WumyQ7Be1Dvy6v6qDYHJRoLBebYriycVkpTUA65PzFZ7/8N + + Vl/QDEvdy5EC6JT1cpi/39Wy5pKG/WTvWHDwddur/nocgayi8eCy1zyI0QCwBpPk + + gzL1/LAvjfE7NRYCaKkFBUdHvXmE2wvEGu5XBheULRNnzIQ7dDVIHHZArUVULVYo + + oXn/7XCg7v2aztXN4DiE7F1Cbu0ievqUyvA8O98xX86JGBHIIodzIQ6jnt0+fkNQ + + Prx3mVkCgYEA4jWllsf+sW70FZmgeE6Ah5kddKlWg1C9szbqecHBOoLucfndbpC4 + + Xn2ZSVGXYdoj45fewama9thdSzk7Cj0Lw2Zfxc5rb4i6qgfwllw5rcCDHD+HE3u1 + + PT1bey4yDM84XYIqiQWSKok9tBhLViF5W3BNuZrEASeDcnMNfjQqrF8CgYEAwyOn + + neuOzrdAnaZlE8qpc3yD+/dZIacJfkSEaxDMT2kVyLrRTfJ7U6IXGGpfYEjT4WRA + + qDQzsJok/R9BbpwzZjrc6c7kR+Ct9OpSVJBVanLuoCM7CePKB6OrYlhP71Z303Zc + + PjaJkO/RgSqI5KfCt4cVe0VP+ecP82laIsznOrkCgYBxKyubgpSuCfc88y2v4n40 + + 2Go/GhS4/2TYSuoFXeSgtC48gSfBj89dHnLYlmQoxSxdSXZc5tArHFWYM5qQ5beD + + 2yyg1kMzenEAbZZ0ctE8VuqA8FtQaPxkFdU1jAfoFqd5SIylHk9gzmY7Okg+X+LJ + + 1yZba80RUsZVNLAUal7K+wKBgAn3QTE0fYebHkau38yh9gN64Xa1zCyGzlpPf3/E + + TNrlYAJvYA8eCiRcS9eoXxSYw5FoQFEW0Wj8hlUTCpFuksVuzid2tHvjQp8WdHvz + + HxmfowY0pmg75O588lzEa9iqTtZS3iUjPeVUChwRowoiczRSRsuT36DApzTkNYE/ + + e7OpAoGAGU+zqOQ/I6qe2f3awSezBFQK1B43YSFnYyWFUVcoHN1Gdir/7VO5tROb + + 0szku935s5fWqRydLXT7W7tRpfR7DB4IIVMr92BaKWEHPZdNBWKYddpzOzEp9qWo + + QoubFJCdM83ubtISMH/Ovf/i0rVgSPTGg4WG5ow6Z2RJ88ruJEo= + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 1024 + seed: YQ== + zfec: + required: 3 + segmentSize: 131072 + total: 10 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:MDMF:zjjxvnxrs4logkm6fzgyg7cxy4:ogp3chqbwuxtt22jfoasxh6rqylndggift47yum7b4q65qts2pca + format: + kind: ssk + params: + format: mdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEowIBAAKCAQEA646A2Y0zoJFYAnAslkocvo/RcuvT8xyjTrjL/3GjWQS3/udV + + xniRm0WMMIq+bWlhskXLyM7/KNDNCV+fsnKsUgyaXZrmSQOiUFzBLWFhAT17yCwT + + T9Xw3mA3P3uBSn2PrnkC7XW6wgV5jT+di9td0i7C43k3QvV/X/hXZjVFWpbSBKbo + + f8NmBjG1gKX7ddLoSZRxg8wc+UJJOMrXF/z9wv0hPMNG/Y2pfcmNiBNlh/SJFu2C + + +pl32nrq5cCnJ1r+S7TUd7ccaX06iaJWVsnCPAupy703Qpw4rByXj+4ZTVuN9DyA + + hrUbkzaVK5Udt67wRW5QkXewq7bC28+e68EI8wIDAQABAoIBAA9W9TLK/GDepjBp + + IADc1wNdN+1icyfX+8CNeJIW0zrhpkkeGoVgEPy0v5I3rs/5jAhWfZStpOCCZx/O + + +jDeQTTfUjwknA4xg46f2QqnVpsEkdIxde593FThnNIbAl/t+P81LTN2i7tv14Bo + + c040x7wqGbyY27hAvCiX8mUbCfxXSa4+PPfUzrPeroaDawCkXW2Y6A5Kny3peX6m + + m3dYPGq2cBRxyDr87e1enOBJeBXCa+AbQiuExINS6kTqw01pczAe4F2ZmlRsa8xn + + 4hjMu8wdi1WkxFLMswjhs6qaXiS3A5vduzdojjwNyOVTSl+/Zfdi8ETFFKTb79lN + + 2kRPhmECgYEA/tjAxk7fJwCQD6u8TM0Ae2A2qDvHqNJObR5W/77LHs6OIZw3GXdZ + + nG6mXFPw7OgnbSzBB3TZBuYgwBbWcYnp+5Fp3zsVcK30kOBmDVqedEPOasnaMHg2 + + voS/JmpZa0KvsF/6hmIfVK93mNWQvAlzPA3/F5waOd9ScC9Cr9ZYCFMCgYEA7J9m + + +cSTHcfLl7QvZvm3wi43quC0ejLPYqYl73wVSHWEWutTTYoNfJSol0qwEfVlvT0s + + S8TWlVGuVq+zlMnaqnAHfLswVgY/lFPJt10nHrERdgqCJR5Kxj+Do36nhRkRZfqh + + GyztLKe9xN/jY3zhYDZhZZ+tdRUd/VzmdIcAaOECgYEA8hKaKsTIm5ehQAF1P86K + + 4qalxG/kW6xI6sWjBhMJhh3WTH7Cp+ICsOE6DQF/HMn4iW+1e4u2iyMVgOEwmXDT + + XS7nTjAlUX8rjGJbDdxCH1Y5QJ60Ls5B0f7uQ2NJxOT3VaYVpoiWEi8Kf5Z9gN/J + + IgZ5hMe28bn76Kw7wCLuRBkCgYA0NyDEMSq9wZ8dxPdI5AY25XgHTzrEVH4LKNrq + + NBmGOdiRL6jcTYCYYz2o1SRxchOXZO5ncfJgVPwByRf512lXfw1H6w7JjOtu0eaL + + fhTp4u0VfVAm3L5nbRChfYt+BYAfXuU6V/mmhwWLclR9WctqLdXkVQ4z7gsGI//+ + + 6uOeQQKBgGmGvJ1yr5UCw5/rzSoRwGlSFGv9BF6UpglnXEGYtTUKaY3HISfYbd2Y + + seHdl2dQT1ws9Nv1sID7bP/Ea0L5Vdzk1w+GMoBYctAyWdyiJs6usl9dpsMk8Qpl + + 78pgDYDloeGjA90ff4tRlw9UTkx9vCWBXa5c1OlIZxb/6Ln/Uf/d + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 1024 + seed: YQ== + zfec: + required: 3 + segmentSize: 131072 + total: 10 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:CHK:yo75evk4cte3b7rdw72zxvl5ye:ex6h7ff7nclucjtsqwgwu33qgmb67t4ezbrki4zbgurwn2ct6bbq:3:10:4096 + format: + kind: chk + params: null + sample: + length: 4096 + seed: Yw== + zfec: + required: 3 + segmentSize: 131072 + total: 10 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:SSK:qy3zzvtmugghvguwl3phedeuza:a6zdrpyep66nwgxthmzgcy4u6uoq56qmkcmrai7yaxjuq2znywfq + format: + kind: ssk + params: + format: sdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEowIBAAKCAQEA1RbLEX5lSZVslOz6qEIa6TZ2zoidqibxCZ5UPldKiTNeh9G3 + + Ecv36wClMPmBF813yJJudyNjhYM56M5ohbaceo2E80mZr+bP6STUtKlfJh/sfe77 + + BdhKeYh7XAlseN+8D/x5ArxssiQAKNLAa0ked80aaRlF7CNF4rEwqKSlnyFXB3q7 + + QDQGZQrzSlpLvh1g76NcCQK1AYMmpu4XD5CUUx+HGKo4bFJGp5OhCGMCCNj+uocq + + mDAktiy0uIhI6K/vxgv0FVT1tTmeo03NwoWNDGsVNYBdW7LhDEoYr9/221H6wRp2 + + 4lEaFgANEs3oioM+9iCQZSGVprOpmeqxzJCBrQIDAQABAoIBAD03yyc/dMHrF8LB + + QlHMjAasCv3S6djUTzNANVujoFpCU8oZScrnGlZ9XPfw9lFsShlpWCsKE7FrvdtQ + + UV7404Ox3Jw4bNrIKLsGRcWRUzCUw1B6s8s+FEdOGoKagntHa7P8CJfsoh2bkiAo + + S/eGjiZE2m2PQTNR/uXdmekZRCuu1ioJX8zQXPTnF9GW41ss0zGh4ntfRevbXWUu + + VvtfJl3RCD/wj7lDY84SteLp5ih/IZJbxBLjKM6tancxkU88T1vIun/Opn4nRDje + + vt+V3EVIBPEuaIuAIbxU0Fmrs9nOW8k+D7gK+rDM1gPxKR7sqt17D/Kv3OB5Pcs5 + + A7v0KFECgYEA6FZwpBPed8JYWekHatWpJ2gmiLvU6EHUGNv3LaaOd2NLU4SMXd30 + + e+WqZKizF3UhuKD/T87YhZ8I45pipoTo4flHPVz42xdeLEF5pabQDKk3KGNEFVth + + KM4EpLlLntQjAmFlNrzfSF8l5ENMk037cmGPkoCPlBTdYB1A3WQt43MCgYEA6sp/ + + tG5Ns4wSa1wC6qtt4jRelanwfOucb7yONuN1f7DNbew2cPdKhkOk+whmGTVkJGnz + + dB862tY50/3zDCll8AyCRqzsQLsl3tQAUo2H+0J1NreKTaGtLpVPc3CflM/zVx8g + + G62lxq07a2yWGCiZuPjGiyqtiCxSbAQ2U2Nv/l8CgYAEZV5MPHQBIBQ730TcqJ5C + + uJ3CCIvGuTgiIEdU/cnESISsV92wCPsPPRE0RlzdHMI+lA1AnVFLde7dH5auP+WI + + IQdQCepLeu21OKfsknNtSeZZRUeMf+Yet4cu9rKPlsPyz5TyrDAtVl+JKhzQzLDt + + QRtOUlBlJN/raaJIjhSwMQKBgD1LUh1zclt+JMzcP3KuAEi+bTbbH4otJDDTY7kW + + lnUYXfjlYq0JEe6NOEPExIquMo+DDWhyQrYgmQYr1MiHAjKxwUzcFe0sLk3GwLLM + + egRxLBJ1xehQXdq8Zfp4G2EJDLjgykwPgCimzs1TkreJ2d+9Km/oW1ciYv4J93i7 + + i+A3AoGBAIL1rB9kv6P1z83WtT0KSM2+SfML/QTHF2UpxMm9LYpci4kYxX7rWiii + + yJbGoLajm6PFv8+gX8gUnq13FWrilxTmO6XrxECyblueP0M7fm5C5sl4etqMtxJq + + c+DTtNGadm2iME0cE2acZCmg3x8UfYfzTvDM1IqznNO25XvmoHdc + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 4096 + seed: Yw== + zfec: + required: 3 + segmentSize: 131072 + total: 10 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:MDMF:s4yxatb23wpmyoehz3c33zqqbm:dntfov2hjyd5o6hekmkkuyjgre4n3spcxgu6oq5vcukdula3j4wa + format: + kind: ssk + params: + format: mdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEogIBAAKCAQEA2fHpyFtfnt28GF+kqmI8E3eCoYsvVkhZPfDlqR6pWt9To1Im + + +G7NTv9XTsidpsIq5fG3G3PA6gQ9jRrKbgD5x5te/HXr6XaWzamyWlCyDzIAHvGv + + LasAfZONnyavcivpN6Ul42OITzKmN4VYOneEv15JVC2lhTc9cYdeNxgT3C6wol8M + + /Av9ILns0C2IGg2+2jcvxuEJ2LWA0BCXAWFTSuVoUbNDjt6u+kBtGQcO5NN6gi1M + + wcR65baYaFNLY/q73tp4JvPp/sCWlByOxRfK3cuZH/zo09mlN0IEkbvV7T7vNluU + + xu8izxAeme+Ywbk8fqps1jQFZNLtivSiMj0HdQIDAQABAoIBACKqdA/ehM5CoPqp + + f99qt6PgXwKt864Vh5MEIFYu89YUmVDsLhmsRMjGHDos9nKCBjZupObfbwsUo0EF + + 8WE06NtN2c/DO+51t+qSODWzCddukblaTOMGhOdJ0ygXjFGB2DB10BnOiO279ddK + + mkGBJY1rhF4OJ/qRUkXJYnwrPsFxz9cqzzT6faKWwnHQvH6p+1al/rXmPgqk8t+D + + QP0UFm/YllMUXvd831I+arCDdX67wTxwsaCKzUOTOIngyy9zWMlanaI4EqgoBkrp + + E2wwDgnx89IzZ5NTncQth+b5QPLEGjUWoh+5fGgRkVnVYws9/lGiQwU8Q3dBob+4 + + 6Mym3wMCgYEA+E9HM47jweg8hrItW92x6PC77ZmDNtuU7GonE4khayiNiRWwleea + + 6jix+XKqOihr4CGIB0fO4qLW0gkMRyezzKidkMWDGWC5jOyIqp5tEYvBHiI14L36 + + F8JGJbWWWhpUexlpj1z0W/OJouRq1PnMBuCTC8nEQ7K7fiXzt88F+ycCgYEA4LHj + + gJRhT20VhT42cGZbrhRPqasmuZYC3zMYArbu0Ji9AdU+10qYAROe01r3Bjdbo8lo + + tG/3QuU7zvktHTi9ag9XBZuYA1Y9M8xbZRJXYj+7oSF2neTI3uSwMjsyEEUvN67B + + khAeNhKW/QgPvDB0PepF+YbzdkfanLuXoi9B+gMCgYBNSuAu/FuJEHFGvE/CONAY + + YlcdLpvZh6BjtudS/WyZnpXwBgBhqSZfoiZEL50tXUe3DLj7Cy8q/OVBm+9mdsVQ + + /2uMlO6qB6G6bCZeddIdlBMY/i0nN/uRSbfsJQoYIfoKF270YUrvFG/TdKaMhPUt + + btpW4Qdmy0vxiH7EyHxkIQKBgBd33PgoB0Xhcdb52XvB5R94dZ2WB7Roi6I+Vuqp + + qqXU3iDb4fVgkCHEp9kRbi2TCJpBxhLaguvUv3ttoR2lOHtkYMVwK99lWX0Ygg87 + + bC8R0woQUbBKHgTRw+lrL15tq3HYadVUo6MoK+b/uY0BTpLM7kQSqUkYVif6m+rP + + nsd7AoGAXiBveJk2fbLBwgXiv4PFjI9tWI7IQ1ZobMsPW1k29Qj42U4F+T1K8BtK + + lOd7UKFJr6uGwf4zUOtNXH5lur3bWh4v0iJYXro5vxomKVo2ZWNIJ8VHiPIpqBDg + + KyL8fAZQmRj9DHqUlYnfshwmAEG5zoJqzqWRL2RPRYvq1qAclI8= + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 4096 + seed: Yw== + zfec: + required: 3 + segmentSize: 131072 + total: 10 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:CHK:4gokef54smahrbfr4kq3jhc4zq:owpwwfp5gof2vhly5u6jdnbsfuwwwhqkazpsbeg3nldxv5pse2iq:3:10:131071 + format: + kind: chk + params: null + sample: + length: 131071 + seed: LCa0a2j/xo/5m0U8HTBBNBNCLXBkg7+g+YpeiGJm564= + zfec: + required: 3 + segmentSize: 131072 + total: 10 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:SSK:seadx3svf77hkyujx4ebons66a:mobkn3t3tbxgwhjria6yxho2is2c74d6dis6xbwxxogxybehcyna + format: + kind: ssk + params: + format: sdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEpAIBAAKCAQEAo56YIQmQb06Q6InhqE55cyTRXhi0BfUYeScWrgTXDB7K5/PJ + + RhRxTE5K1KJJ/cHOf1nxauXWnWB5N74JUSEIPu22mHgV4uJc8ktJ71XBpci6v9U+ + + yoPM3X5/XIor+zGoFeFaNFAIaNlnItPWtrv2FiL00smW1O3JB0EvLwN1MKKwJNEE + + sBnIpmmUqhJn7IL0DuGZTDOpIGyq7+jwIOEzoQ7yN4hwqR8Vj1xVKorsEe/WRdJt + + WrtmAIOTppBDenye0rXtx5xjBbJd3wfnUTM4RDbRLOvhhqSWdwDis6oQmJBN/6kU + + j70Tj6AOECQyghdAZy1WJP1X3ay5wubPgm6t2QIDAQABAoIBADMnbsWJcXQzOn/R + + N9FAc50JqkGCdKoWJigejesrDTa3W9Wn9Mnps0BZi/CtqndhA9fx/VXf9LiwREWm + + rtAEBUlzVW6WwLT183w3CK8AfzH/L0+xclerXD31ggkjE7wNmtD4axTG3tI1Ahcz + + 5sGrwzTJigRqzTLWAs83VHKc4KMrAebPS/SvRATS8f1OA2rHHkeTlHg/jRbr4Gt+ + + OQTsHomH2VfM9VBgB2CdWxluioA2B24OU35h370+KJl+6NmBxeaxiBn/Vx2hjXBc + + 4QunZ4Vk0pI6ioSwym76C8rVdwO9wyi+IdmeEK+lNbggY43P0UGE0du/v4Y5wFQh + + UTWjey8CgYEA0c4t1QCOyjsacTwk6VgsLROte54twxWqcer13mBlQJlOpzvcUZ1l + + LB5ck1hY3oqWTl8aTwLuKgkvsK0lhiqN631XsePwoEOiwzxLl4wFtGSTRvHwfBbX + + EGWAAx5mWt6r2reZMjZeKymp9jWbMC8mbR+XdyyTvzCc7eG3O8auFP8CgYEAx6Ue + + CD98ZE1S0GPJ47KZeHpNeOu85zq0oyP27g3b6vahFxfBbCZNqLA6s7yMLKRyx6bg + + ev27ja1ANdCvjAlATSM2AVKIYcwRE9Y2x+uatliw3Y9SnxadhpIwDrgNJ+OtAJWq + + iUmMQgqkeGEyRaWBCFab2ED9b9hk9/3tTB3/hScCgYBmPADLXWU3GEvPR854wlVs + + db1AkpicCm+u6R58CR7ttobEKQA36OmG8RiNWCyd7IxHjkIkpDnn0+ggQI8bbJsR + + WFemQHtdrPegCT6Qj1OsTqIRnQ1hekO8IqmZW3Pm7cByaKrG9AU5JSlD52VCuocP + + /6fwE5G/RXIC3M1L3ImxgwKBgQCt+Ie8Hj5yVSMmLt7eCWNNJh5ekeZSBMkmJI/o + + D7GlBXeI3Q2TBanEppTwzQvFVyQiMJwK8RI/ukpq2sguml0rGtTTwCzSM/Zpt9CS + + 1A9EePLejycrNJTekINKQD5OlUrLaKBr8+hCIG4D7IbXRAq1zmsNvkxa61HI/MCN + + BNMGHwKBgQCtrCi6yB3pVK7Ie0n1KlTJTu3PDlnJVFvUs+tqXnf3HiueQVkPKVMt + + zG+heN753a+dWnOUl+R/oKzy3DtHIcdmGtC9O2/fs74hJiojBLn631im6zGYYNJD + + MocXDdKBilZtVQQ6J4fGCMGB8lK2PZA8pwKBctMeOq84CwlOkICElw== + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 131071 + seed: LCa0a2j/xo/5m0U8HTBBNBNCLXBkg7+g+YpeiGJm564= + zfec: + required: 3 + segmentSize: 131072 + total: 10 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:MDMF:rqdwy25h7srkxzovybpxvke44i:ccx4lh7qe6ce3j35zhdwdu2d6nehnxv4j2eymk5bbiwlq2webngq + format: + kind: ssk + params: + format: mdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEpAIBAAKCAQEAobqIH06fWJed7sJbSsOvf+tg0vaNlWqdItvMgOOnpUvRCPj2 + + f9lV9U0nK6fU1IeVwHv3dLfUOYGPzVaSyiX5GZZo6IQWxBQ/8ejjjrCU/a4Y+TZn + + BI5OB4iIWKnEPsyrEdtImiEewQA5BAgajzOU+SCJ4B0uwKkpW7LRhIv1lqDKHg6L + + O0TuAk5TPaJDpSnpJq1d2oxle2XWvQPyy+sQxdvS8FL6o/yy53MDjP04nyB/pq7X + + UlTXo7z/e0Px+2YCM2sI2G984ocoWHdZ0FpFHmnJk23MIU23918JcrsCtIRM52Ss + + RY5HR9ObfL4YwU6fiVZ1E3kJMYAcCJ39XbmYjwIDAQABAoIBACk7eb3lmRmImiLH + + mWfPyRwnYemXI1SvOD2tZQ+NOu4ZDMOpWYsR2WjvUSe/o7LFmIfY8ydmQKyinAuB + + YW45TS9ZWgjBuF4oPX9K3U1BNtMQQlyzIoOWVk10YTKdoaNTIeAtFG77N7CEAoVF + + HaRZxcbYJV6mggdreVhgGCufVS8gClTplSk0mKSI4ZhyUMtOxbEo61Uv8R8cM3Ps + + hdI2nI3nmbjCtk4ajHIGa2iBn44Wpg8bfzbiBA9A3lDbExZkStgDY4xuPBY/aXFz + + wCg2WuuOMywUhPnU6CpH8Yj6gV333wYCo5rfBf5Ot5KaRjVwzAlz/tzdu+cjx60F + + u1znuFkCgYEAwXEKT6DmC7aiGJ20jBjtZNXPfPbE/S1I9Gt5IE0BW4MvW33sIcd0 + + e9skcNNRdMgR8mgm3e9XU4oa/UkhuwtLdGH6vcXEE4r9039X6yrpUdZEZuPtgr8P + + 6qWVfSN2Td2XL3h6UTM8ivjJCb2CwVlds1a7g6DVD7bvz4UGCnKWhZcCgYEA1gf9 + + UGPSbweVXgUm1vR/nCJfLVXAbnDLvX3Erig3jX490fQPsnnk7MiOoDmFZh9jAHdL + + 7oiAZrOdjn3kh+uhtZiH5x4x16o8CAU1Ii1P/dza52tg7C9iIxoXgcUFf+9Gsz3Q + + qOWtkTwdr04VpUovw6kjzj+qDZUi1kH6/yxzk8kCgYEAqSxWCyu45HeVrZeGhZtr + + SetfaXda8dv/2JqBNQmDbWf+K7KlpykLKyKM7QsySsKKR4hkrWWa5pl6XxbtI+qN + + 07u4kOz7POgqciQFXMqLgKG18pHVbqnvnpOvd+Bin1hy1vYzav43LYbEMvuE9dlV + + A/mPRl+K1hJ0CfXZQZvTHgMCgYEA0nwK97sjoQNhNpR0bOMIeEEpPsldNH+DLnh4 + + KxnsAB+NpmOR6GCN7PsToKjQ8uydDUFFEHF3bQjpQs+2JqFpZ9B4nqcIN2L4JJ8S + + cOkFCNDhCsOEDuJObdzkDz/2N6nV6sI46VDuz6zCOLve962sqYw2ZUgg8bigCPvc + + XoSVqVkCgYBrCzwmUcWlU9j8FqHxcYMxSfgBg7ntxyNsioidk4k7DUDMp6bDgsTe + + 0nBri4PvTmwvyNdPGYC8E43z6ysRIZqtT8O7USWuYFTAiU7FcWxpNqV9JdNf7Pe3 + + GwRgN/Iq5SwgHzHAuLwTAeGGzpPWcHNBeKJ8VMp0HqSH6OCZOQyxtw== + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 131071 + seed: LCa0a2j/xo/5m0U8HTBBNBNCLXBkg7+g+YpeiGJm564= + zfec: + required: 3 + segmentSize: 131072 + total: 10 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:CHK:7vfgl5cv4nlzqx35z4uthjv36y:nnueftbzxfz6u5yjxwwofaxzzft7xss5wzfh66rrcwv2zwrm63sa:3:10:131073 + format: + kind: chk + params: null + sample: + length: 131073 + seed: /N4rLtula/QIYB+3If6bXDONEO5CnqBPrlURto+/j7k= + zfec: + required: 3 + segmentSize: 131072 + total: 10 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:SSK:ben62gmw6buwgx3o4ndfvqe6da:qt2ru42whduyofgesstswdxtlskmufvkg3lg6lzxdt5bxtqhv4za + format: + kind: ssk + params: + format: sdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEoQIBAAKCAQEAvKXoxfyr1z40UlLYMQ64W0KVziNAowZ2wFjQLz9wn5PvJZs2 + + xOnCyvhn/8brO/rSksANWIWKXSH0X7a7TNrw0bMa88yTG0LETjMyhzGGXWzWmFUA + + YK2NqSKijlokMQwdOCm1MfN94zXerBUW4ipeKm8bw21Ailfv6V5iv1YQamxTv33B + + BQVQN2UzDijlxAXRRMq/DG9rVGhuelYtRogyfQJWzlo3cPTHEAAPjmFPG6rWJGR+ + + oSaUuLRqQ49F14dyvRO3VVwQa3u/cno3x3ojDpkson6WcVLCLJHXhZwebd2FXWvp + + ih8mIFJFh85yUYU8owRzUIUsxwPzg5ZmLl+YbQIDAQABAoIBAFAj5yQct9+jpFSI + + ryEAEN9sBPniTfYzq8UAtcgsmiqgjMqcCoNSjxbsujmVhp8fac8/2SuO532zC/6R + + QTZgGEftX3jMon3FOmHCLCf0qRENSIjEK3nmoLSGayowLwnLDKqsRTZoK0WXv/W4 + + q9T+jKxYMSIvSmi6/MdV+nswE58xlOCwkHXz6teCAJFX1nnaPqeTDIwpgZ4DIjh2 + + uml8hRHql+e8k6BM8XJqp6wvNBzPZuECuaiV5vO4Jf5TXn6hoFoAr9mwhlybcX0Y + + FhUzntVTM8jEXnr1dRNP6fuou1PKFhXYj3ISQfeMarZBBs7kg9BpMVofugXlGnHj + + LY2Slb0CgYEA0YVMksfK86JQt9eYrSOTKxwvYdLHXoafgNIOGQ+hm0zhnn5VwAh+ + + KmKarXvsu/Uh4tZ6w1AA/oxGwqJT26iQW22r/kGf1mD4ct6xg3mwyHa5KfFRUCHJ + + kH3rGMj0cTSd0N7R/6i+6I8GpchjOCP/aVivRsflGxW7NNwKd7ySBXcCgYEA5n9A + + kmDBFvjD5R5p0wFzAMS9778rHAH6eCEwItndmhSXd35UHrkEIGJdAWyWK74caiWY + + pLZvQKmxHyb2Pe11WUduxIo11xwtrHoYzQuOokKZgG9bbCU4hCVQWG+9pyshb5/D + + DLb2OJE1iPl89sdYGdhG58d7aDGsfWD1oki32jsCf3HC7tDDqm6eszUe2scnicDe + + jNuQlq+8aN6JLx5sXlL0a4yjC+w4sEhTQajwoJltf/iqe/2QcvnDMKh9ewrJe5go + + 9DoZZ6/+9udoAvpgGJy/2cnsPTpFHixWMlBCzHarGwVN9rfZ585d2j4pj7Xr1cJd + + ZM4Ju1v5cKxCzWw19xMCgYAezWadqRxkq33SQow1zH3H3oLbZRqntYP5RcYfAiph + + Cttq9pDbQjJQ+ZQgOpie49r8PGX3rQGVDJhE53oEsJT8B1XAIhAr3PIlmHN0A1Ve + + TbQhu7/l5dt1nV7tUpFvo43mUt1H97NTv+P9mAmhGOanHYXsN3ZAaFL6tlhdBYa8 + + PQKBgQCf0ZLxESRimOWkd/SXcUt+EcRTfS/leJG3xN0k/k7fvjyvU+YubDEhOSDo + + qP0KRIlvinOdhyCTTdXCZuQ81ao17nC2m4rpb6BihvuBNSTdjMh/ZtxP0wWzT4JI + + juCcjfvh8EkQHSgSFsZEYDjYKa80vOU1UiseUyhqcALkggOsGw== + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 131073 + seed: /N4rLtula/QIYB+3If6bXDONEO5CnqBPrlURto+/j7k= + zfec: + required: 3 + segmentSize: 131072 + total: 10 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:MDMF:57zjw7jihhbipyoi7ueb5n64oq:5nrlpvno5obhpucb36o5zbg3eldl4u7kjvnve2vcoowlsn4bo6tq + format: + kind: ssk + params: + format: mdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEpQIBAAKCAQEAmLgOSaGtpZP6I8vuyivxm+6QhUOn7IRcs3qUrpXZ2YC8vTHJ + + L0Ks6cfHxm31bRIheM6f71oOc9BJnUouhvyEIgQmsX9CdkdUOIQJ3QzXaQLWAmgQ + + NLj8Puvgvj2HDvkHx8NwtQSgU/B+tS11raLvbPTErUU264QRnFNXrIaitMWs4dtD + + cOU+Bbulv0PKMaSO5zFKGOv9/kaIt2YEFeANbrgLhyo0uSz/xcvMgPp1OdcV+jtU + + 0rL6JzwAlLHInGyzPwQNlm9bsIYDOBsDnSFcBA4wylsZnQd+/BAzfeMi08UjNNoN + + BOF3MjDpFc708Jk734OQCz+nVTNJIHbw1gzhLwIDAQABAoIBAEJ+POZNSVRi+hHU + + 7JrFCFTqyazkWLxvowcYM51SLIB5f3PmteBoaO3+6JoabTX4o288k8E8ljdRtIOR + + 9XEbiBJheVFmBdOG2gIjZ0ICIdYcgH6avZefBWEGBZv/IQthXURacXu3UHFLsHeF + + HAwmeZWYevuwO6nOnnZQiUdadYQ8MtHeBTzLbragX8WGtpzlZuBwZiKrUHN5yPSJ + + eb+oNN4Mw5afqQ/mlCqS5NagRLL0NyoniQQvDOK5M8/bz0TcCS4hRhbCbgl0V0yP + + mbLsfYde1vVe1Fpe3XwtDlZttgb3gRezz4HQZQF2/Xjn8ubERy6Iqgj6mMCwkat7 + + r9lIqw0CgYEA0B/R/ukUMtzjFRCbTQgLss3cAKLZ+iPMITpStcn5OMJKgxRFwS9y + + vsj86qDfs5yT/JW+4/lyXdzINSzLz5VaPr+b1vb5SNJsS8uHnyUSQo/UTblX9ExH + + KgaOG9oFJYNND1p66YXqtcJ2S9mqOzxz5NLBZu7Feir6nR1Dp0D6FE0CgYEAu9l8 + + InQeL4+vw2UlIUrYgHbp1Dlp8vY4frso0336H7trEIfttVXLUTYXWwzSjfHkVezP + + 64PrH8DvVmy3CWY2j5cldnXBsCnCTvbXp63x9u/h/47wg6U+a4NjnHlD3MSth6AH + + vEkc3lpocsqPLNRraJ4vVdejE2sZ36p+tpCmeWsCgYEAmotWa1x2ZEKD2UuIls3n + + qfGVcV98T3Ovi+j8LAN7rfsQS3+NQKPUJ/mlXTDyjDQz67bilfTQSQS+IkZOXanA + + 5qFvvlOMztd6FVpgLfvgME8PTlvYBQ9zNLDDa8kcUzvJyCHe7XNE041APJi4AN6m + + DH+3n5CkUVCC4pItf5APY20CgYEAtB85eWvwWdikR27f6Il1CdF8KyQWZIMV7ucV + + oZ3lTbIPWl2MYFlwyGFeic7EwpjUQlP9lq36sYr1s+AwrlGVNaBPqsQFQh74k3D7 + + nmwbXJXuFXeBRioXrU3iIPLiUHlCj46yfCd7B/aWuqNiIDFbAIjViLFpTEBhIefA + + 8tvG1RsCgYEAxtXdD3dbMN1GumMwUNOQj1nwyKqgdVmu772TAJ1y55ZN06912Zqt + + qW1v9Hvy4qzU3MpH8p4sXyBbUzxKFMt5VYAlE+/eSC6qzTEojpp1e6ACzpeOqTEt + + cChVzJG9ezlgp2ov2vCmvp/IH59dpdLDZ3r0o1iYRAYFu9r9iNVct/Y= + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 131073 + seed: /N4rLtula/QIYB+3If6bXDONEO5CnqBPrlURto+/j7k= + zfec: + required: 3 + segmentSize: 131072 + total: 10 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:CHK:cy7fmjsldeakhfd4psbmghmqyi:6a6uvyai4jkzz6hj5yjugm6uie5etvymcudgiwwjh47apz636zwa:3:10:2097151 + format: + kind: chk + params: null + sample: + length: 2097151 + seed: uqWglk0zIPvAxqkiFARTyFE+okq4/QV3A0gEqWckgJY= + zfec: + required: 3 + segmentSize: 131072 + total: 10 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:SSK:q6z6djixuqozvwnooe22vx2bzi:nts6mykkgnjqovy3v5ptbckgevicuo4wjztqddblcataz6llmima + format: + kind: ssk + params: + format: sdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEpAIBAAKCAQEA3L0cVo1jjawoeQwER817huQSRZUAA1G3saXarMyK17tG8E4h + + w9i1ieiCaxF0+cbKOLSabELsU6S+AhoBZRwdnk+oSU+YqXIfCeWw1cig9hIRT7cO + + P3yd78DF22yT5H1Le/hEjW14cWWfb84OLHd+MuIUy6ZqmiAf3N/K++/2NuYos6Re + + 6EPMELlw74Jq1GaOV9HqdZnak7lBlD8Fhc3jsOS717/1pwzbVUpa9Xk0MUihLhC7 + + kQGcHoBWQ0tQWAbjhsAcSZC2kUENVdH0MYpg2OuR5FGAmTqFrdXslYA8U55NIU1/ + + 7FeFH5Oygr6lGmcnNKVEl3u3/bNgeNwh/nAyNwIDAQABAoIBAADmXSBgiNiHLE5m + + BU8c9X/KG+WgYwogbKfIPc543P4JJj5iNdKxu6JkH21RT7vu8Aca0WBXRisHH7vW + + yvepuC3biZuW5qVA263eJleQxFCMfU6Tt/amoDwzJHtYWM1UfMN0x87DH1ETBtCI + + CK3CD50kfSI2u6DaOFcIVdsAvO0MsYs/Q7b6pRii7A/d4gRyTrmWtCOaMrzxlbAG + + 25l7nrnSuuuBFrg9wTF7LfbYuDqqQY/a+2JatcU+WMEPVyw7V7WOUqQJJR/QUv1d + + S7zP0vsar8iVzQiuknnFvS8K45Ir92gmPkovsQ0H7VIwuVx7hzA/MpkQcDsgpnNF + + X/iIOqUCgYEA6wiv1VMygji5eB55Yz9WmUEQB/WkfgYrTnypcXWpG/c4GaFckVre + + +wyjVRFW11ZpsqDJ+yveAU+rASUW61HzNNMZ43QlGBXSAyWkInkgfoRDJf9HSkyY + + FVd1BMGt+XGJRxXz0mU0/2Qear0PyLwLStfWSGw7ENCJah7XZ7dIMnsCgYEA8G35 + + NIxCPo4Ke1WjuxboL/VJ5B9+22ECFfWvK0rc3+T+zmJ1EExdDO8FxxqTW7AVz59w + + rtM9/tMxUn4YEHkL6eNefNeKxF7lkPRWSRK+uVs0bTsygJOZgRheZHGJK1lXtAHI + + ul8RXenK3KkNrGuPcJ7fvh0qAxnLWFLW+zSZYHUCgYAORRG/5vQ7GcyQ8XC3SOIu + + HdgmU5CwIhnBAyqae+VPkFv0mmpvXNAK+AJ2qL3YByQVt1NsD4bEF50vTZwtn2Uf + + wO1idOvHoZOFo2Rqv2XsqIUXKn+ekDXvnca6CjRQ38bQ7RFHpeNo2iBKpL3vlxMs + + cRxOe1u+spqVOdgkMOmOPwKBgQCbKP2ocdPWduhAy/XMKW5SdOPouoKtpR8peNJB + + CCEexLPEETom+IEcdayu33G1vB93TBf2WxEpQLYV3JY/Gz8bA8bYnmlJbUyNjYGZ + + yuUWzcs5qvhejeKEs2tHOxYgyZmV64jU7cFRcC2g1eCjIw8AySbvk/am5aCbMWrX + + 1wwceQKBgQCkuYGUkO+fIrGac9buxGSrTKJZoSv3ktfBK27UCW148vv7ZOmlrNRz + + r9lVjRkPI67Ays37ChIH5WYh23xg2edkA6YcftpxKYsRXgavkab+kgSPJEqP1Ptt + + IEALNn8JYEzLqUJFj79ljw7ln4XkNWTxzMnJ4GUPQxJIJA8EH11jZg== + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 2097151 + seed: uqWglk0zIPvAxqkiFARTyFE+okq4/QV3A0gEqWckgJY= + zfec: + required: 3 + segmentSize: 131072 + total: 10 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:MDMF:j3vcaycwlx5amtj27qsn2y7km4:p7cg7f66hemvv57z3jft3zu76755wpjimxo2ttfq4rwocd3gdkzq + format: + kind: ssk + params: + format: mdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEowIBAAKCAQEApnFcrtpSauyUmwqA/oO81nSe97hvTppQVo1V7r3yjryYFNJL + + Tkt5BcwTHCr4NNanfsxLuBb7GSbmMxcLjg58tvHmLa0lcLD8gt1fqCdrhX8asLMo + + d/XCbAlkrsRIRBI+LtDkqqSJlNfTUDNTbumHJjiZt7wYe5sOVoeWjSwp4uKgBr2c + + trJCpl6xY1G7x7+L2OgyfeKqG6/Zuc0SQS/Q7HJw49+WsdAQ4WDbapb491dYpfwl + + JtliOGv+YjdS5gU6YuqMNyx3Imi53COb5nWkbCBWpk7CuUJja/CnZvwvnC5YPxRc + + LhxW9qfgrsifUH2ZKXAPQQAFXD2ddWjwPPpNrwIDAQABAoIBAAN+ndOOAez8yqH2 + + tn6hhXV7PVs2JCAiXU1z6jn5Av68NvU49RvPudrFTiFpRYzWdO3UnEJhOSRuDKdF + + 9Jgm9bdhnNOYrxCOpr6Yp0mAimFjKcxL9q2OG2bpS5PfyySivWt+N07d5YWagnVM + + npPVk2DaD2AsMtdligeHEUIlizuYPdXsKLr1u0bkg3IXOTsPBqhNbyIMBXAwgR8G + + 5alaQt3jIj9Gr5FOYKUEj0sJSxVIF5nrJYvYskxa0wRdgGSqnfjsGyidAa4Qj+Yv + + I783RV0LwG1Cxmdyelh6Xs0+Lq/4tHGVcmFiGSuYGB5iqBMLikcc04xK3wd47E7c + + ywtUrAECgYEAxAhTcXHAqRfIMHEigFVZhF6uhIV6LLvrzHXIdLx25fo4zDk+zZaa + + MdM6HRexQxx3iXRLEh6oOfuA3Uv9jJtZ/751o92DetWPaDaT4yRCpblU5gcYBx/S + + nzxJoBqHPTjUF/7YWj7Zj881Ufly6tcQS2ellvFlivR+YYnrdKfH/vECgYEA2VvP + + oQO91bhNI3AzFFMYUIhZCliQxYVbnzaeiKKBECC8mE+p7lEslAFGr4PnjHIY8op7 + + l683dLGL/YREy9eBEE87lxmhHfYbL6W9gIGjQlxC+tonKtBXnm1JZSNX78E7qX/h + + 97vSrMru49QTA8kgaA3dUuS8F7yX1UMZaDX+Vp8CgYAJ2ZU/xQR2OqCvdm/SXPeD + + hDJmrEJITyT5AA4Td4jN43XJJTM3p1KWIFPyNEeO5LZI7NP81BeF2lJOTEwwLXon + + NI26rx21JVfwV5W0uxSyOQ7ABCk76mht4dydM9gJxno5vm9mkXPjGvlF5i/VBvtl + + no6eeACvK7vR8Nko4mlVMQKBgEkRSw/2oQdKaGwEWLd5Y5AW9c+7jBdKSE2SX+LQ + + thBE4QFWrmpV0WWDtE5mSh11cZt/ICMSnNLWqJe1sibQMCvaZs7Zp8bZp7PxxG2B + + pu808rM/SLFkzj+Mv4KHShVn4PWO7tiHxD+gDIR8E1RPdVxlZMRr7isQk/32C4Fz + + vSdDAoGBAL23mwBwTlgJ7/oHv8M241NY+Jg3NA8UrOlXz2p/FW3ZZ0eVyLUlSXTk + + rErPe+e+kPs9nMJH2Y2AbCPB1rEgEb6/x/us6/96DY4c4VWonZiTs5kj7Mi3qHr9 + + zXr+WAt8L1p90N0R7NeOIcnQlodHBZlAZ2CQAll3qZ9V9Q1i8Ug4 + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 2097151 + seed: uqWglk0zIPvAxqkiFARTyFE+okq4/QV3A0gEqWckgJY= + zfec: + required: 3 + segmentSize: 131072 + total: 10 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:CHK:ptsofqwylmkvzmuvrw5n34j3ma:ky2fs7xrlke64w6kfmhzsuilzxbhfrwwzkxih4rykpbxrr3bxhiq:3:10:2097153 + format: + kind: chk + params: null + sample: + length: 2097153 + seed: BTBX/ampNfLU+ox7xipBGiaSbgC0kcB8Gy7BkJB4oKI= + zfec: + required: 3 + segmentSize: 131072 + total: 10 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:SSK:63x4hc4tkqbgq5gaol7tlcmjxi:da4llxjmaowkwuhov22qb4ko6v2p2wrhxffdfybioaw75jb35z2a + format: + kind: ssk + params: + format: sdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEowIBAAKCAQEA2uybTLwxSCOAFQZnsCPI8a3RWilM0zXpUNnRUffFzFV2nV+i + + hObdCkLwsmqygH1Ssd0OQM/qEAzgdLscK0h8+jEfCYTi+YUMjmYImKeLrb8fwFD9 + + GL7WGkWQ8cnH+BkavLAOI22xrQIg8JigdwQWnz5qxvMSIKoypndihBphlmJMNO81 + + JMp5ai4QyKyZ/2K3DSlsMufitd1cxKW7NmWtbxztss11BwpQk3HikRNKocVRhjN8 + + lqRRFYsnuc5+tV8laJy6YcZqWIv/OejsebMvTQdwVKOiGFKjk0YdT1xOgp9Ml1gz + + JQUZdCYHDYsiPZvF2bxjbmKRNx3rruTMx2wkiQIDAQABAoIBACFNg/Rg3nhUWiwY + + nNZZIzzMjb/S74pjtZnkgKig8fh6+b/H6A+ilPZ2J2pku8G7DsTa1Uu7tSX653wq + + aIcXEFf49/k5O1PszvOshts+BYwJOnnFeDL2+NfnRDzbzq0pmH0ipQvzqGcin0Mq + + XKKuPwi7dH/OQzAv4+OZ3qUs5DJ9enoWq5czHaP4HI5NDX/D1TFjEBhAwI+YHP8T + + tCJH8CRgDgZPXw84EIJqsoUUVxaKzU3azlra+jtlk7kJ1mfRMaW+6f11bynUG8SX + + ub22LU8WrNNYRzN5STrBDDteb4S1iPLoEXca3LCddMNcTLfH87f6rnHeWUXR1o9s + + f+ZYz8cCgYEA9Zv1DUnqL3o9czlDUI1k2XzisID1EEBnSmSRxD/gJiZDfJxr0o2R + + 1enzG15LA/aK4v6KPpfVh8g0BhukFEzp5EpNHwOSHlcpz8j3k1OQ4rF2uUeml2p1 + + OMr/n2k4stu/I0fnibR2QBAuzZdBq1coCC5HFG2kyYemMWDO/BD7uLMCgYEA5C+k + + E82178C5TcEVTFoDLpX7sFMIVR0zSwEC/65gYwGCjIPtggB/M3TjTCqbfR9s61Xa + + 7KEjNuM3fhhn4M6KJQPZDpBu2mkd8VU/tbvXNx72L2REMfXklsf1yQKcryacbkVc + + ZGLgOqKG9kRAS5d2E9+m6IGwuA5LfLItghBb89MCgYEAhk/OF4FHRsVjW2KCNEfO + + Ub0gvoMXANcnZSBQMnD35ATivP9RW2g9yyxP3LSY80bctruZ4BbqF4HdKUXuWYei + + FEyplf8+5camv9FXykJVphKEKVhMetsl1XP1jDhfYDgZc3K75KtCS1BON/GyYL+d + + zbN4/WvkRK0grjoRlvi2n08CgYAGTja3gWC8rlOwjVxcTsR1vhlFZxX83CC1uuJt + + VFE/iyQjY+XlSMQ7FMjPKwI+8+ZbnnS9Qzqo4qB+8Ie2U57HpRKTb3RQvsTgDV4E + + VJt+33EoIBouU0As1nu5QUQ5JtT9yxbhg0X0+NbH6Vzpedb+d5iyJhtPCr4VRQsy + + 4+bWlwKBgFWNAe+pO9XXPhvE4vBf7IsafB5NuHCtR0VI2FZUowTXYaT3j3ZBoNGc + + e6xXv0fAvcU84hHYq2ib0FG3aUsSq6KT913o3yG1EiJmRJmqZvrLiWJA5Qlv2X7T + + kOEiH+ip1sdBTvPNU80974Q4WvNRG6Ejt+DrVLOaXm3otSB3ZB5w + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 2097153 + seed: BTBX/ampNfLU+ox7xipBGiaSbgC0kcB8Gy7BkJB4oKI= + zfec: + required: 3 + segmentSize: 131072 + total: 10 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:MDMF:uhpq5jscjr2qolkmoqkp3fgeny:svwlfa4b2yxv7glkhga5ogkhffhhvsbqzpkkcdpxrnf5s5i2buvq + format: + kind: ssk + params: + format: mdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEoQIBAAKCAQEAh3FSORVvmWRXYrmV1q2+M56QzBqfEihkcBPBbmIXQSYF4uGK + + Gm+qQi7jeFidT3WzSZjy+4zeVM0HGGaG36vQosK20qwn0vy918Wnp6KqJckLiw7o + + +S7aUJEZAU/Fkj48d+JrJYK4bUyqVLudzWzxmTDFh356RKFGXgvKewT8nNtRO5ak + + uaiKCQVKcvNYfXuq0cacwZvUWRHFjk81Xax1ZhqKNoXzpv82TksOmkX/wFPptbtU + + dHLPgu38ErkYwYBcOMo91uU3cZ8X24PeWIH8s4JKhHipiHZM15qACiFK6V54TXKX + + VWgCYziX5ukwpHnYUy6bwB8YQBF+v5qbXwoPEwIDAQABAoH/GjBPNbpvWbmNLAm2 + + b0wo+tIuLUj4eQpWYVVwkWdmF7LCcJwrl/D/esyWLy7zO+oGQLTSRtF2K+94777j + + VVxjexUrRJEFIka8bnxJbqCFRckZ8klvwr7Md8eWjipeiWh/SK7/CMG952Rriva8 + + DHyEOpqzlv9dpOeKM6UUAbV1It79UIWBBUY1iVez4P8ng5PmpTKgcdTwM8NnXcJH + + amscOThUY6QPJFhHmSDQN10GiPQJMPrqRDXrqSNRT3cB/+gEzY1fwXiVzroevJx8 + + 1RtOc3Jq/g6Bt/ogX/N8vvR1uIn5AhkYJWUHVdnjT0uhywGJWaO3WNIHm3hNaq9+ + + XtyhAoGBALborYvkIcyDsKickGZCHwGQDSa/4Q6XSmmDL4NVzpp911uWp+FLt98C + + wPe3NLqFWesH0u6t2m2lqXFO6/gGssU7eaq8xrP7yzBQeKLlDKCfzPrEbLBhgcdF + + CmD/xrNFLDioEa+0NYFnXGgdmRKttSr1IqReFOdjIXsplGLAcH9zAoGBAL2Q6gkZ + + 59CZwABeZ/ItO3wkxPu8IDnx3hWOKUmxgzdDpSt5O238GTNjSeaPZ/BsN7BbE86F + + TGzfjQ7pINDAaztm9714AuRIQOKPBOZooJnW7ErkYU+lvwULsnNa0HpJsopkXcns + + PoeNSjlMPlE5SXAM+YWs1xyMxxtx4nuP1QnhAoGAcv6YVZIJGd5Vi7xbIJ9DhST+ + + z7TlFtpRQ0Lh9U1WRlUFt6RhScjkAgZmMZdyRC4gmR5jJAITiMoVXJKE0nvLmyrI + + VGq49mFAntCI98jPhpDRO3uQ5dd300N5wgAs+Xps0fYAoJnI5eGI/EBXg6HIfAiA + + ThyEQfFWFGvQycE5OTcCgYEAnU2WV20OxzP+do/gc78DIJYme7p1h3/kSUDJlCRg + + fUh91CBqp27Nvq3CkjcoCgLTB13chsBoVeP/2oKrv24czZM5OxlOVP58EUSazVO/ + + CUmmlNMEySIB6/7z2vNeEkv7gwmcJkYK8VLWZ8uT3rTJ8thhaoKtkjxjsKuFRAFr + + yCECgYACtlo9j0uqunY12qwz2cKbZ0j+fKafX52WvJUdgyOq15YBl4WAFJlBeCNq + + KfelZTKQcyCFxJaNzDPgK99OpLiF/kzFAt5RQIuwabdkpphOV3RPe4vWDNG2S8ug + + sULT7yvROK5TPDBFobyIpmX1b8M5sXSVH3e+f5hWgKfdVF3MwA== + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 2097153 + seed: BTBX/ampNfLU+ox7xipBGiaSbgC0kcB8Gy7BkJB4oKI= + zfec: + required: 3 + segmentSize: 131072 + total: 10 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:CHK:tyt4hfvn44igztqh5zeiusggcm:ex3gzzn7byvhajithwbo7c3p7qwqxmnactaxxsxsxowkafyjrf7q:3:10:8388607 + format: + kind: chk + params: null + sample: + length: 8388607 + seed: w6uP8Tcg6K2QR905Rms8iXTlksL6OD1KOWBxTK7wxPI= + zfec: + required: 3 + segmentSize: 131072 + total: 10 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:SSK:ridp3gpk34rfwdilzxclgeeb7i:vo7rrapooixj2qlepe2myiwzjus2zxge63pw4dbvzllfgvftxeoq + format: + kind: ssk + params: + format: sdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEpAIBAAKCAQEA0jmBeNgAnc47WkR5CqDDp03oGUREmm+fQ/xNjzKEWVLDAI+K + + vZx0Z63uFqv/QxEOomWsrjbe23W6GVpN/60l9R2jkpkrh+F4QSbiDICK5xH1tMnF + + fYIvlBTya58oITWQrdSg4lmFIJVcTA5YysMM/tzKtaeStLCC6xbj+AGXCZlSAlAa + + 53w5p454XyJtxORWSuj0eZnOOWNgbXh5VBIR7xWQO0NrJXHz9ZDyM8at7cxNSjnJ + + syMRVT/ig71DoZL1RGRYTQo+tivForH57luGmta/1VwWA51uswN1WPmgs34BAvtO + + fNsPi+T6BYdmRPyjAgYaE5sfsRqrMCCa8pXFIQIDAQABAoIBACJbfqMC3lrsGnwp + + Zlo2ewjWrTZP7MxhWMyyKUoxeJA7+/MVV+30P3wLC4yOBf/D2Q9GgbR5SEPiwjb5 + + hiI2qKueRp5OGooFxJ2/8+q/frQUFdg7BQ6lnmQGZmDwwoy8OziFtdjjG47/NMtD + + VFSoZcZ+IZe9sDbdWuV+uLmkJ4yhmRGHxGvR1EHYC9rQlSsDx6mtFShOVnNg0XzI + + wDE+wxVclqZLqXNkmJjc5yIfjK0SiBmp2iXoBJCC3Q9bAaA5yiqQ523v5EFdkhDD + + sUiVaqS6WLCzrJ+5C2DgEyPaEm87p3aN0558JTHP22vj2VoABmoWzx/JoClmSH7z + + MGkSt0cCgYEA9Hjw4ucz7bB6vLj64GYoV+Di8h3MjXyDRQXttUQ86O7WfKNNZsZz + + 3KCe38SLwEJOBtgjer+Wlme4znJ7heALRAU5UmkWUELtzs3ybSceAz0rAQi4h7fu + + AI9NSNMcYQsPWh81kCtfAuOhCTnYLuOB5HnD9TjoL31fGPYKjgGNxrcCgYEA3CMn + + trfkY3nDwWFxyxF4tUk3qN0kcA6CzPV0677hhWY/d3Z9+u3Iv41hQnylcLA/o22P + + HNtJBSy6SeQrfmml4MLvz/IUIzGP6yKr4NLa+szGa7rsahJ1S+g8Kaoc8i/5DuDP + + S7SUdMFaLtMqvjjZxNPT/rOyztwKhts8BEZuOucCgYANmNBk/lDwDlm1N2CUrHnf + + 8V0N9ERVNjCi3SKMa2Ar4GTDh92dMrps8e4EKg1PwyBN1yWaBR/d+6TWrp1aI8zc + + mqHGiJu6GQ7a6q9qDLvpmDRVGWQSAFPXaiD1RPCWISRYcdXrz823/msNdU8lxHeL + + +o3AjMq1IXbxj3Wk0kdNXwKBgQCsEEplVh0M58rrZxgDqndX84+uzJNDhwQT4bNu + + 5LbvhvkKjjJwJNXpaz9fMYA6sXg8bFEVNA1CHzDIurCIUVmXcabyOXwl+gJMvr/r + + rcP9jnt1Dxjk2+KU30PPKSkQ4BBi4bMFsHLtQ4gS23koT9VfNFcaWSjk6TbNK6Ug + + jlBwdwKBgQCmc59up+9CcHRFvk2qYgp3R96APHeL6u+BSI3LHYyOYT6T+29rpFD+ + + EX5g0nuc0ouI3YHVvl/RNXXz/8mtysHWUYoSj555qZqo/Kt2QC4VfYFW/fpX5b3z + + nLgpUtzY5bLRxCXoB3LsMy8LHl9ehyRmLeIjOiXTHQ3vSrV5kMYJEg== + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 8388607 + seed: w6uP8Tcg6K2QR905Rms8iXTlksL6OD1KOWBxTK7wxPI= + zfec: + required: 3 + segmentSize: 131072 + total: 10 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:MDMF:is64t3zpeyh4dmxepkn3x7m5ye:nwfs2s4gjq4ca6vsmznwkkjm2cztpdcuunfiaqbhol2xvlad5t2a + format: + kind: ssk + params: + format: mdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEogIBAAKCAQEA3FWV6LYIi/XT4PKt9oQ2dKE0FpR0aPm3oSJJE/VHOzHC0KXe + + D/rSSos7hQAidFBxO1yuZFyG03D63Yx9n3FapREPYHq0MYA+YLst9ikuvKj6A41I + + nqu2Jm1wWSLC4sG7uWdbpIK77HD+OLaDAl0PntXS44IVuuorY6WuIl3wJzKZeh5v + + 0sIxGSawM7GquYFfJx/9DM+BWtvnDTuZEG+V02XvZ6/33U1CCpWmYH8Fn3W3Dmne + + jw52BEEVPLXX1eo18TxUmPWhinmgSfF+dLsFo5n6wfwoA3a1QTrsA5PIqAGtMHgo + + DuXDj4m5UBst5OX16DE+pPmyiXifMIg/gd/13wIDAQABAoIBADH491cgci/MQZHz + + eKFAu1kYdsfoQ77LZGqXbBuqtc0nLBhGhmb5bFib25P+w9G9rPDZxHPeyHWMWlmF + + U7il7PkjNWmcauIPRBaMXZBHJuKDMLE9igryxw1QJPsSd0EWz4ztdEuLmzO1LPOP + + 8YbHtJNBy+Ltzh/mnJCtMyF4TM+WrpzffEQp8BhJqfdmaYGlMrtj02etg+l3XvSf + + /+qUiDwK2MEp2KbrQoDCkAGz+xUUVcCsBKhdBH9Lu3vWI9SoXx6ljeylXcQaOO10 + + gJnukBV2K+HYIO8UkQqdm24j+OMA5ekWjXeSaakW73hfh+sMk+slEFwlFnnuKxXI + + J1YhUYECgYEA7luK61NwuAEBsIdRKqomu7wuk5zr/9EH5wUhAWCP1L8gMID/Q2Yo + + ktbSY6xJy2jqa9MwqY9Iu8vyU/GGfGgi7enbp6vPSQeO0sYCqPMFjKBmJqwUp/Eo + + QGSP3XwJs3zK+Skxl/nvFSJndvKkuAfVfI6ynPz0I/qB58FjIW/z6WsCgYEA7KSI + + nPW0U9rNDfwCPckbdWNjysn7jl6lSNmNJj9d6iDlNx3CLAwoGr27zp4YEqa1i1oO + + ZxOVjr6A/3Do0XXV0j3b6YNGruAyOa4jgBvDoSz7CfVRi4Vl4J89jaNhuqnOHEut + + PFenBZUUm++HillETEyp+P1aYsKwovlsQ00E/l0CgYBsMXlZYEKmAy71JjcdmqaC + + SOULdAtbz1I69wUITwB6nVbLLYKw4UpBfOl6/NVyU2k1EGPiU3u8YtLYb6WQCuTw + + AVsHPOGWUKvv2JmUftth/dzgaPPnV3vh3sO+0XLF2jt35c7xIS349ejpATLrpgKt + + y0ggImHfgvI8dHe+0cZxiQKBgC9grZ1HMAhN2RoAp245Uk8JTBRwpfWWC19vduv+ + + ac4TMfD7+0EYWfsom249hrJNQDGbISEP8bR3fZomv+YXwmxqSBoTV1ZxunyD2cWv + + SVZ+i/AtdlsJpSD4oLk3ybw2fPZ7TD61idH7S/oAVdGkF6FzA+C+0JbPRdALQdqj + + k+ldAoGAaCrRerbYUDouYIepA4oK0Vw0XrS4Jv2fuzO5SORU5kGp4ZVC/7bL5JNy + + obw/HZD3NMmUcb8ShBBcNriX/IPM/S/B8y6PF2yLFet7lUNclyN5LtT3kvi52221 + + oBNL0mNYOo+J9w3K66Uw8M+df1t+2Gt3rYtprjWDgOZnlxleBks= + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 8388607 + seed: w6uP8Tcg6K2QR905Rms8iXTlksL6OD1KOWBxTK7wxPI= + zfec: + required: 3 + segmentSize: 131072 + total: 10 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:CHK:xymheose4rdlspgydkzr4nqkre:z3pfrvpq5fdpkoybhdxppwbzrt6ejf26xh6emzlce2sgquljginq:3:10:8388609 + format: + kind: chk + params: null + sample: + length: 8388609 + seed: yPi3JHKKbWaEEG5eZOlM6BHJll0Z3UTdBzz4bPQ7wjg= + zfec: + required: 3 + segmentSize: 131072 + total: 10 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:SSK:mzceoqnz4uohtzae3qcgxkxl74:4mbfxhucdkbwcvnum5rjy7sncthwxppglkygqub6lbydf2n6wrya + format: + kind: ssk + params: + format: sdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEpAIBAAKCAQEA1mIefLi0nhjyHfmZ49dm37vw3r6V/Q32a6MSqvsi7hz8rR4E + + ngbxyY5ZwUmLhfSWvVszukjUcHz3FB40hyDRYMOEFA8JSb5ElVePtPmXxA51G/tU + + dPlmLnuZCGfiKlOWaQ+8Xu17HRdJ3HSEQjaJVpMMVTh2r+xe+0IEyCqdHUlNp+qY + + Ya+DumJ7TOZOdF1DmFdpOpnm01YK9bDM7iLlCfb27RZWxoN7gOKVpR1lFHXCO9Lu + + QQo2G2uu7nE/w0zEMk5gOdjuKpga9j7+Zv+E1NXTwF9sadv16VmpEeAqeeICNh7a + + OUKj/jeU5d1+zvYBCG7x1ZgLGXykiMJ74DKYLQIDAQABAoIBAAoTrZwuMGs3//Vx + + vI6NmuvMUTufGLy+0cTocuGvkUpA+Y2Hmi71Y5sWQljIBLNktksrRMiuULIC5bg/ + + 3Tc2zzCtsAEjXcvmEiI07e/TRZN1HIMWsrcW2/s2WxCelW5o5GqGz1Nk9UL+S739 + + ihP0rUrw+YTt9QI66ZIE3eWsvxrXt5+FKvyclQVVikfRb5WMVceYYpYoxIqUsWhn + + BMkZO9mK3I9x7V8u3OsAAWYdquhNOBf2RtFXFXASWSKr5FSjWo/f5QP+Bf6n8kX3 + + SHsRek5WweDkSjFXRwAE+urB34TRbMfcU+sVIUOFZiw/ZvSqRiJ3BnfaTNSKfJhj + + gGnpTP8CgYEA1pndhW+qbLbQ8F2/R9cNxXCIIYKnj6/8zpUhdrJJpPoDdADWkDJJ + + NJ2q7jHgRUrVKw3cHniFx7cguJSEx17b654NOH3Bg8EZIpBBFjTJLuZEINNMvp9O + + NVGEE0EaPVn+46cR16ZD3GXTS7QOULQeaP1IsRG1110QN5ikq4ItOncCgYEA/71/ + + 7BzlQBLwBZClWeKqwS8mkK38gP5D1nFk+uS2jxIYJsb6cO6KoSY0gSvDqqfrGfgf + + t4HjOqWhP3sJw5kyNgKJTE9Dq1es2Wh68MSk1QMhyDrNXRZDbfayz81M3PySqsWk + + BvxzDdi95ZU8fFdShT4JdpWyhu+MHj/YmvBjx3sCgYEAqXcd+KaCtZD0lCvjxm5r + + 4JOJ3LSZb51xDQ21PE90WoRYP739sicTqior9ieKzA1ZIsOyJJnWQy04+KnH5Mzi + + 7ECGfirIqyvMln/F9iw/Bvstp6JUw1932iECJFZPy00LPGkNbPdONXhvkCOi/lYO + + gagqRDIRH/3MtaqjtxB4eOcCgYAhF0QWKScAw3KLRcwfdVTi6lbzIZAqoLvmY4XN + + cQquOIkne1eshTEq6OaiUCdhTZj+Izz3YbclP4k9zY3V4Vy94FYjqZ337cBP4VUH + + EmrBpUYZwoIQKXFQKTu557aqYYQY1LoErWW1xPXNXyIUdLgYxY4z6erPyu82esxs + + P+6pQQKBgQCDnH8yqABNGRB0fZdgV62M2JF8uLnJnW2fAyWzj99lUxUUZ59FO47m + + SGCkZ0n5sgA+H8WqEXiTRQYSVAWF3hnY5Tn/cnv2ju0oT0+98pk9sSN/FrR3NtnZ + + JnE13OS+wLqu8AeOv60XMbKv2jMWlQUO4vY3FohZsTNfZ7OM1RHxjg== + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 8388609 + seed: yPi3JHKKbWaEEG5eZOlM6BHJll0Z3UTdBzz4bPQ7wjg= + zfec: + required: 3 + segmentSize: 131072 + total: 10 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:MDMF:c2d6zphb6e5qchmyes5tb7fud4:ns3clwx6gdzeguxgu4tffjeadjt6fiqmxmqmg25kxqvoa4v7qbba + format: + kind: ssk + params: + format: mdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEpAIBAAKCAQEAxlbX6B6G4qcs09eVFgA0AvaJ2DdABWBPTKLRUvVdXg07Y+FA + + UHe1s/LBiG1s90Ghn46B7eknPFzA5cFV4iud/bqpEw2Kwwys4rRWwXdUF5FALYPV + + OIOBSLo49dtTYl6AM3bAacg4KHuUXSLECAEVmqAwVuxUT3mJHfs2e+f0ProurJsK + + hfQYbxwpMpGFVfwjDjehK5isWH9jgmXggSusD5HasNrLrTvqYORNjjgUwZvP+LBl + + QMNoV76bZNfHMBzT42edEVE+i2ptjVKbpefWTwxaH4vpRJH09mmuEVfydYygeCxE + + UpYZSweufsq1EV1qbAyRNL7PTrHZWxoPy22MbwIDAQABAoIBAArd5dA/OyDk8iHp + + mfuagQvRtsWEm9RCx0vjUDiAeAaeQmBB20FTzCR/viNiHdb1BUyikeZw+htAPuz3 + + qRbbHYaYbDxxs/+x/8qGASQL2Szw/BHiVLoKPnjRHZJrqC4BQ0eBwwYPGCb9Q/Vp + + Lh6WjJ8sFd0iIqmeEFyqD5cKia/I1xxkQZC/B4xjCT36A8YKRCQKMcJ++Ik79c39 + + KPakYqcvjeGiM6rOu8NvrKgMqHQ6VagIizXW8WgmgqTbvYOdCV/MYfvTXVVJ598r + + C80LhftXOdjMJsbXm3PxD1PcfNmSyMFw+sMdQsBb1teUUZ+uhx+Wn1cNTAQk82bT + + FKYeqV0CgYEA7i2zKGSmBJENnAtChBfkhHoE4Cx7qUugWhbuRHNKZV2XiJuNwHb9 + + ui2QVgCrCpxwNc1ZK6TiytW7yg/E5rjxa44TUh0yVdEYSyQZX5i/BfWWOWDOoVdA + + gKniPiU9eT6DeJGKRN1p9Rtkx14nQISB8I2ocCtMwapeCgpp46sqqWMCgYEA1S4G + + DSnsydcgx47IVOOv3zUovKfEbNv4VjOJ4Pt7eVTJh0qwJOnFhBTh2FwK3lIsN2uk + + VbxvJT9XcEBy+kq96cKimQHQKMTBdSwml9LGI6dHmfsl4yMnhqAfMtPtFDiJpGUZ + + 2/l3Ybg7TbhX6RqNlBU08Zzz7VPzkKX8s99KBIUCgYBXwLqrfTm1oQPUpEljhbIK + + JTK6rWj6XQS9bIlo6tlUM4FrMXSunqio+bSeGyzpge3NxNS/wcZVWR4ROnIfV7CL + + IhN4Q42SFLHQrYIzuIFY3rz0cvhudUksnmre3rWhgCjMOUMqUDGDvw4IbmYj3S5K + + xMZ0XV+wUubG6ENPQHc9ZQKBgQDTLRFPjv2DALn3FWk8NoStL0LYh7TcRZefBNUL + + 6vNowOYWQJV3K6C+89S5+IvHqk0k5VvYlp7fnfynNSDw8oNpAqcBvTsQd8BQq1jb + + wy8GeJpEXfctJ1DrWsktF6TeCBfJo2FXeKubQN52YiurveMME2nsApfcvPIlk1he + + cs4m5QKBgQDdteelQs0L/lkYXldLqcb9KhIPLzxskJpjdlt+W7AsF6v1ayOKE9Y+ + + 8x/Li2b0rowY74UKz3xDGWN0ZQucPs3aDsxDG5rYl8nosDL50q+TNJ8D81Gu3/II + + qVDgw7XuL15mYWWBQS9m4Jge0Ww/MLxQGIPplbToFUSo3StI4QJsXg== + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 8388609 + seed: yPi3JHKKbWaEEG5eZOlM6BHJll0Z3UTdBzz4bPQ7wjg= + zfec: + required: 3 + segmentSize: 131072 + total: 10 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:CHK:xt3owduddxqodfhp3c2fu6yzr4:bvk5d2igrtlo64kbpyypajyi6bjzrnvl2blcavxhguiupjthelra:3:10:56 + format: + kind: chk + params: null + sample: + length: 56 + seed: YQ== + zfec: + required: 3 + segmentSize: 131072 + total: 10 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:SSK:exwdnakezxzb7ca7xxirywsstm:hleh3a6wfqbtfgshmhhhtpcuyxpgpr5yz5gktvtv3f6oghhnh3pq + format: + kind: ssk + params: + format: sdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEowIBAAKCAQEAqyZss4AeZHIr9oEVMvbdTn70HiaW1CFY1TTKDkMdurYflEf8 + + jTaQzzKsaSdMeZIX1/5BK/pmCgLWgQ2sxe1WvvWr+yzcWzlOgIMAZbYnFr0UGeBY + + 9sy38st5pIQWsisWdu7CjvEOv8sja8HJMwGc8heQj0gxYl1czS+eLqOsQ08mKlEq + + ayPF8rHPYvSHrV0kUrnA1AxeLv+bcF9mxL35c0E24+RNLLAdV5CEhProvmYkO56j + + FPD4swnT8bYYRdYq4lyda8HzK6QjWAW1a7Kl3llcPhi85kTeBnkSyg5LwTlBVpZ1 + + aDH3MUeVyWcr3wKIB/8ESV0CI74EfQr9p9mQ6QIDAQABAoIBAAWj+Sxjxiylih8g + + QzJ4KhRAkRtOxoUEedTh/eBBNEE/aBRHGJLPV++qMf4vsECn8M76p0tzn8oH7KmZ + + aaamwyd9OX8oS/VaMsT/vEM94PUqA1f0eb4dglj3PYGvZD+YNa/y/7i+B1BJGbhf + + DTpX/OQKJIaiWJVPMiQlg/sxw/c+XlgSO4g1fpebSJoKQjpXufkDcC5fGisTu1H2 + + IPMz5EYbYkg15tcPdHQZNG4QoEzD3jKpHCXKfJiVX/+LOwvk1+vxuXE+Lw82ktGh + + k5IQTPSSfXLgR2v2iVuoEH4mneamSuyjxi5jay3PCgiKhYyVkU97xIhFA5FIcHPU + + vV28A2ECgYEAxdNzI1uBGkFX2SJnKeG9ApwNb2ix+onQqWYjSNvJigy2Xktetszn + + R1SjEPhILr6T+hVmW/gtOJz35fm4xPVlwEdFnkVY7J3X+EyX+evwpNw1RpsbUKp5 + + VgLsDYjAy9LPFOPV2f/CxM8Thfcsz/lryUuHD3AU7UyMB6td5hFr7EkCgYEA3XrJ + + Rf17q3VkbT4u9XORCqWSUrdPH+bMjUzWW8VEj6O99V8W3ScyuNF4CKb1rga8G8ZT + + cV0UEI5+yRKOeovRjhW7QbS28yhWwAlSiVxsQ21iHYE+Ep1RFg731rGMT+HT/kLQ + + oRPxjUuVZ5g0CWuVL+RqzFplCSU+UOF5+/00P6ECgYEAw2GGivo59QtqTxrqVvQL + + sEMeBdWaSn7IpjYpTTE9yOmrSFAaOGMBXXLbJsyAxiIVll6CXP0s9IgbUnikI2rW + + 1uPNf3awT+nJPwOu6fg8EScoOxbAEJh+BBQYvXk+KVCIk/I96PPwkl6OwrYP/Uwz + + R3kf6IBjOsdqWbzHnY3BUHkCgYAs4s+b0a2YqCf8Q9f8grlocPngranpizr1gBcJ + + bkdg3QyIiAb4NxN+hWVQS5YK+O5yqpUKqpSAboCfe5VInMGRjDHxNRDG4uwB62HA + + 2OxQFgEGfcT4vM1MLShpaH5JSjlOlHf3zTTtL95NqnkRV65akG5ckA1d9yBT//5a + + 5YwLQQKBgFOdazTFzSvJO0lGDQnF8zvDtLGR+ynGuP4Mi5J8xoUQach2L6edFP0u + + agjkx2clRLGfWqmyziEYha8aa/rVZfr5bPCz/i1IWDCNZo5ccg35RfJtP3sYbrtG + + 8ZiTTs7d+1WlngDUWDHa/aNGLZM6gr08XcMtPOjDjQsUhkXmFEsZ + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 56 + seed: YQ== + zfec: + required: 3 + segmentSize: 131072 + total: 10 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:MDMF:n74rwqiah3chzxknpktu7sf36q:xdwwuwhs2owcnk64t5e4exp4vho22y6vyr6mb2xg6htidi5oufya + format: + kind: ssk + params: + format: mdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEowIBAAKCAQEAmaf8JzMPBSQYNxft3tCLOeMX+r5fdFvmeYvk+n/xdgueZIHB + + UNm61tjPaRPAyopGlBILzfrNjLDIEIJFi9SXF5esnkH/NEdfhqzYZV5QZQ9s+IiB + + Vu8RUofty92T6XE/pNJgouQu20YLEWDCSeCBqxMcW/DJBFcNP1KHqK6tAWlz8917 + + Ty5Wu+q+NGN8ecuQIoja+WdTeID8mqUybsObh1QOqIiGriyNhMq18cOngoJLcjoY + + +EP12LhP7yIqvm2ScSYZt8KkUvrL4rruuQ83ilHs7DktbQyqPkeDWpsliGWDtY3y + + AK45cxJFw4GupZiAPor3cS0kXB0EYq/cB9chjQIDAQABAoIBAAbe6pkMIO9Pal/I + + S6QrehZID9HwAk+vLlKgDUigQPlQ0q/W14CYg9DImBmwPu4vmbFUTz/SJ6/TVdbb + + JGX4xxrQdvEKrXE2gdCBWRnLSlgChRJl9DahcQpaNqlnio4lOL3ThGu0PV4jtyn7 + + RCTOi63NKHb6ANsXU2nR9Gv4B5dpAIAErMMSJf3WrVD6DfwD/0/2JV8D4KFKlhWk + + OXEwSodaPG9HLxqZ9p6pdQqv2yn4uQBfdUUJVXgTZ1DwhNWvbNmG0S3VAcWbhGje + + kULDQF7WuXbte4lSRh83L3kL7PgGK2b/VfyX4IP7urSbMum/bZ1V+3GKkcM9cxP6 + + q9yxPfcCgYEAxSt2Dl8DkxY4+t/Dt61+MQFVWJip63T+KQiietrRzFlGDa2vrlKh + + wdSxDaQTWWAc80XLMHtgTxbI3SRNr873qYoV7lahbf8utKn0w58DgAQQzI7vfeso + + n+QNHweozsyKdCepnPUmtu/EtKpCMX2tj1QQ08zn7k8rHlkfr3yRu3sCgYEAx4DM + + sQ4ja7k0N6IT03fJK/Eoj4LUwvs8pVarDAQiZ4/T7hDZxdAdRjH+Vy4H2Z1ywY/D + + BUXCBLfQI+wr0DjGMBpTxBEM+t7ujTUbauMkCMI3fZrV5Zk7UK0EChBYgGhUiYis + + YaFyL2eiEXv4SomCbim3Fw9pp/sbBBgseEA45JcCgYATUVfGvq5l+dZpVgUh+OCV + + QpEvFf4H8LV6JbttmATYJaMEchD1Xmk0yXbzZDD1H8KWXy8yN9ROy2ewqv7li7ye + + IsZVTK2STl8wGjq989Vu9HcE47g5ORII4FocwS5b3JRwHvayRx6c6870+H11xd98 + + XHstlTTgF2edGJRPKEBLAwKBgAgvh7aIDvn/il3x/4BAvPdZmMFyq8ooRs+945zF + + mqfHJfnxpQ3RwTG9IWNwVxAdvrSkcmsH9rL828Rtj0qm2bLlkaRM0syEUyNmF27m + + TPczCNXVgYs/I0jnIHBNRWRXY4iVHAWRez7osKSpAoIEbF6axZFjp4El83DSkRiK + + AguXAoGBALZyw248cG249/ra4PwNzto5bal1mIzEMfG2z8s3Ra+ogKKzNQN+EbtG + + SgATLvwyPiZrHseQ7QtH74STSHkhJNunD7Upjag6lxiap5WEKoX5wC29sYyRvLTo + + uLsRkgovxnv8O6qdnvAU92Pz3465gFYP242QS6wWD2+Bve9nYuIZ + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 56 + seed: YQ== + zfec: + required: 3 + segmentSize: 131072 + total: 10 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:CHK:x55owxzhsezfoayaxe7jpwnove:vawdgtqpxyntgy5i2po2twgelrynkfcjgwm7publnlbdp7hpqmfa:3:10:1024 + format: + kind: chk + params: null + sample: + length: 1024 + seed: YQ== + zfec: + required: 3 + segmentSize: 131072 + total: 10 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:SSK:3vrqukqlq3dgfuuldeucfetaoi:7ruowcqpadtqzzqj2mdj4a3qguqs2aznccmahwj5z5aqepgmjmta + format: + kind: ssk + params: + format: sdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEowIBAAKCAQEAqm3D02icCWXSTGKLQrscIevvZJ12qzlGc0P9y511jEaKw5Xd + + RTcZ4isNCRK3jnRu7TZkYfp7j0wdzMH9M6/BFMZOJgUUCl5Jn7vn9TD0KzBIU5Vt + + uoNQ/ZIxuUd104FC9ywTg8qNjnT1OmhR3BQse2rS7trn1biaujRt62k4SRXZKsRm + + oU/gkkuKm4TddUA+NSksO95tVQiiur/BJjJk0vV1OOgOtp6PKXw/RWAEhwbdbH0t + + rbj5xqH6agLX7Iz9sIH/5zw1qJr4Z6CBUp/9U68Yatt3WzhklroHvpZ0SK3TTlte + + 9BufbOgIKzLTIpfsne6sbIi+0LjbGYKKWo5zDwIDAQABAoIBABJCZ1gnnYweNh1L + + 84qnPFjgC8p2Wmf233brAm6FxLnONwDEdiv7vtCt9xwRPsxK6jWM/c1HhmRwbcLp + + x8R5YJDmvCmzopWHy5CLE8t/vrE/34fg+xwgBJXeS2iD4PpTn4aW2NJmaaspGbrz + + wU14ddmVNNs1ZeBOgnlPs7UklUyzFyJXAGENaB2MHqyJlSxIo+KkIxEjYNBBTfro + + 3VKr5FEePr+4jU8oHSXcspCIZG1UUqUaIjGh1nFPxHeEgerGOgCHDAmh56Bnc57T + + 7pY7B+A4HgMo1oCCGTnhRVnLkX948ahl37kHSq/1jZnnN6ib43LLbgGTwbtElg9o + + 35NJz60CgYEA4yul7Njf7kBPRty19oOxLmhusJs4l8w0rvVlYcPdgXm7qhmdg++Q + + 3crpuN8M1ld8PD4jV1gunMQ4ganCyIAkn6VWR8qmKG4WtUNxY0AH9wO2z0+y9lKt + + gp1h0UIDR+qqZMKxsIyYOeOstqi1UjptT3bM3N6gcnePgZdrj0f6t0MCgYEAwA6u + + qjL6xywTxd0GE2knEr4gKSVvYjoEO/Pq2lefd1+fQ9KNIlmfjlUwj6bqFmf0Ej6a + + CBLINstjFMTPAHupJU2wcopeHQpsfWSydzpwklgQaczUn4mM882vO63/pc/qMndV + + 5J+Dstt+YiMHHhJZK6XemuNugy6xYteI72+k2kUCgYA+RLyai2f2OpKAbgdCpx5u + + BhoxNprwoPzf6Ev93F5fGyshmRvgCk6/PNuL3Tf7mMdpC+9MBdPhDLggcpP9uYJQ + + cFWSIC4jbumyjeYKuoZ0YwQ9Fy+K7Wa6IsGpRlr3348NR4DFUAR5+bph5ySsgW9t + + FLda43s/ZR5k+0h0YdqLWQKBgCnWyPww8OrU0lXnaXxvCuENZCoyiopGg0egQohg + + UFAMF8EJrE9QYO775gFVZmeNK+GRm7KojM9LDYGnwkSjq9yBiS/artf6vlmuxyYs + + J/vpjWHPCn7a00cFhugkZq3zllx6HM8aZPFg2a59XP6TFrPohY8OfrO7R092DtHC + + a5JVAoGBALrst8gw9BjZs780Pi3rB30pYa6OMFsUCNvFK+DeN5dinJw7CbTIFaBh + + hyHGBSDpO8qZgMdqC/6T9+Wy3DN1M5jypztzLcdxSjeVIzouWzsNcgndOXmMPIjF + + jBoFl9nELSboDm7orelbBtme/U8jiuqCKpKuKr+GWBAT+f6Ij2D4 + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 1024 + seed: YQ== + zfec: + required: 3 + segmentSize: 131072 + total: 10 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:MDMF:nndoruti53vxwabaf74wzqhw44:5pglds7tvdgweggqsmku2onqhlrtslkv2y7da2efvzeux7js74xq + format: + kind: ssk + params: + format: mdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEowIBAAKCAQEAzrBrxgaiiBC+cMqgaoea2MZkQ1Vxqo+O5FJFu7H5WkJ666XB + + jvC/cA1pp/4tmyC+Rswx3Bbp30Q5ZdM4SfhhghY/kTYpQwrV2MvBNS+hFcMZgXpS + + 9FW3Eo3Mh4kFbEox0K/W4QHYEW3l5640xH6UcFHmMAQE0zlRo0bFqUNbTEk+Rn8D + + Ldh3Qg5XKy/6jlHSRspYIqNpTHxcSsn9U4dKswu98UUYalmC6T98RGRr3W9N+lUV + + jOADONPT/VsghXWu3vyf1S1NdpeNA4r7GaJKvgkiLUMHNy30EQHRAXR31CGPW+NW + + 570wvEWXiyxmZQhY0lKZnEhj1/CAS7mcfVkkAwIDAQABAoIBAATbWGz60u7nldII + + sORP82+MmeaLJ3SekvkChej3MajRTxoidv3o4la7ufPcoS24A0Ceo71MPIqmi8K8 + + x+HVGFV6OFwtLaMJqiTCBPQ+/kYIo5zLRw++w+KHunqk2Z/Fzo3c0+vNo0oljvV2 + + vn6visVo50PRlFtySVzQ2Ow1TPvp4dZ7u2YM9zO019tcDB/mFCFhMFMxjBx0WfWv + + +WRY7b0DtATaqwE2H3F9rOFFCr4Zl01GB2ZmVFQfoAM7GKhOaT48QXObn2YEqHvo + + aFL90BocL1rWGg7MOEE4oWN4xORRkzjJ7TfibQ8WUpTWhG5Q3FGqOv4mVxGT7a6b + + NOBW7OECgYEA0qDdy1ALRyIOYiNnH5311FgA0ZApBFw5UWe4RfzRlgRQt3787mOk + + 1e1BdBr8ja/pkS2gTXAoc2wcgzSHuygyyLn/AQ0inZ2Sg8KdAU7mE/nwvKoN9HIX + + xwGxxJASxZzMVxZnIzdGMqGPVx0hCalOQwftq6ZzvXFnnlBOWIcLEDcCgYEA+zZT + + G6SOedIx5l49+DR7PTEPubVojn8lYlHKuJ142ZiMTtTpGhZ6DXbRwIYtWf+WILqI + + HnuD6H5C7eY7JPcEIzYHpXQU03tBIRjioh7ToynIXF26k54kTBJ9fst0QP8ReQrH + + q73SmMhc330nvlcbZH9/auelPLCCWz5uAt+67JUCgYBITYJ1hW+ppm4rkB2ZQ98c + + Wm1FgughoAro/+LI26WSir6ujsACkWAHM5+RXKYveSCDfpcVnhe0r3sGKyUgwQbV + + 0stPsBOe6XVfF5JP3aarWtQh33pU3El/PfypDg/zmASpLH6RHytQvBb5f31U1LKR + + 3gnfL49xi5lXRhfu2cSZdwKBgQC1hZ+wDcxWAqjECb1FqMaUhOsUCh2vOfjNfsS5 + + ejBlK3HXVMnLbAptyDnwoAQNUD4vEBpjzGSYjwPV29NI9qUqvFPyHlseJaX+QHkj + + JJtQ/1QkSiYTnOYlggbkpCcxAB6kFEILu3J9q+pQI6OgSlkk2Ww8133yyKipPgdI + + VFpBsQKBgF8DwwJQQHINYbAau6m0D0+3B4piATNnlFOW/8ghWWBahSuXBTrKy0Jv + + X1lU7/MRrcYZZrnAO6FPk3/FpA2ONwpaZhq5uL3p21FacaWttTc+hAwltuWR80LI + + oda1pqGc2MFqyFCmyz+H19dceWJ3k7i9WXj9NwXikFmymlWPZbe6 + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 1024 + seed: YQ== + zfec: + required: 3 + segmentSize: 131072 + total: 10 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:CHK:fe64krzyaeff3d4teunjbetkzy:27hrywwaffqiqcgfkmzwbot3iamotr3bey2l5kaladmdmxuaz5ka:3:10:4096 + format: + kind: chk + params: null + sample: + length: 4096 + seed: Yw== + zfec: + required: 3 + segmentSize: 131072 + total: 10 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:SSK:itdsr3oaiy6ymh7mxq7zhznjge:p4hxxunt4fm3xhqgtsk3j2qfplpxwvwi4xakcps52n4pdvy63gvq + format: + kind: ssk + params: + format: sdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEogIBAAKCAQEAyFwxvZiVH3Ayl6wRlOIUHVILY/pyHLqioAlzaRKdMx9/13KN + + ULOZ/npHjmdej4OCQ8mwbfLQLp1+r2aesrnpcJ4rAKFKf5fPrenDmF8WoV7UCsoC + + 94tRJh+Am+9PbX0QiGSMNwWH1FLGSyFRfs2PNsuc/QrTbghgdvDw3ZvKsBt8PR8d + + 9AMwkWBgGs28EUXo1Xf/Mk0fu19obpLvwObUJ7ZSHsy/ePzmwrcPlfL+61mZ3MPI + + BLhaRsiSshvwDpzfBBEn2pu9HhdSUmLBhYL4hvf914JnCGaoMcqfNfCZIvSe0foC + + MICzdbq1IvBfrOqt8lZCgcgPU+jJqGoQEa2laQIDAQABAoIBADmnge6vWfXueLh5 + + XKPNfIFFax6tYinPMN3BanLpVs/vt/9cqLp4vA7ky/N33leIvbLY9kplLS/ExUAe + + 1PrUEY8FDJXFU+UsX6gJVO3jKuVrnrOuFrV54vOH7B+y+NWmP9wnpstsbX4VBZd3 + + 8nX9G8FmTPnppBaNFYkUYxM58dTDyY5XWpXch6VOmRs0sBUD//6jicFmlRGb5owF + + /yS5Zz5Bh9jdEKzb16jAlAG2Qb/bTBN8S4eoy3EK/FzeYZAXmUT7PlhJFCX7Fz20 + + r4xofFOPJyJpi0EUQQpAudh3dkm7oJBoqvg2Hdo44JHC/C7asoB1kXNDsVQ9i7hc + + +BUQk3MCgYEA6hBN9eJBMgGCTz1fJVZ7XAcfL2kG/NxGfTpfcaOA2SkR3R+GiZlN + + aiNGGTWFpF7L0GZQNtyIkNPOG8un5FVvSC5hreunWbFvzPyrSNwv837TFsXZ9RMs + + Qlywmjd5b/k0FShXOHYAoCJTzgs3gUBZLBqUbVjL1a6YdYFGgsyECWcCgYEA2yNE + + tL+pqVd7NW5p1yqXjKejUzfURXiTAF426UVqisGZhWkXQvRARLT+qoDXFNdLxFWa + + Kqy/LQrvehla/NiDZbMftwa9GyGRq1HgQpXDOASg6ZKYepw0YDEnyHKnldqNGwxN + + EJ4e/OATBSprsK+hIjvGcwYDwgaUK1I5AC/gCK8CgYAr6Ukm8v52KjBPO11JPPNB + + rZhdJaAI+i5DOhtDz3/RvdG7ITn1QIx0eA+jlRXwY1RrUXaFBFSejw3gyxFBVgHd + + kc4Dee1Yd2BZHaHotl5MmSNy50Vfo+wuuwLqu7ONnTv3KC1My16MrEP6qMIN/ot8 + + KbRk2z7KZMn3aXxX95RhywKBgDcT7jjf02zUqAsN7Vw/QEgB+nL4HUo4u/njtDl9 + + UQH/Fu8JMueJLH4YX8nLCEQcuNZoDY+cS5CupvIxXUUfxibRlq8R6oXfMhW1RoB1 + + 09NIlokeZ2SpziA/OpiO+MAZZk3eEaCTnYZBa4Zo2xhVjFJmY8KVSGyD6snYqKr7 + + XXvRAoGAId99u56Llk/kW8pz/Q/aySCfyo63d3GrTc476h525pUrQVfcF+DCiFXh + + 7OKY0rfmWiBhlSlH36fGL1ZlMLRdEGy0iGJA7gchEJadwK7sL68XX5hdlnz/I9eR + + NGoe4uHWgIEoBA7ieUCTE6GCG+3HUQbtUBDLrtNeqTwUB0tlU4Q= + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 4096 + seed: Yw== + zfec: + required: 3 + segmentSize: 131072 + total: 10 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:MDMF:7ch7z4jgu3qdu42esntv6bza2y:or7w27w6p36fgpykao3cm3mclap2wd4aehb5xybudejtpt4nvw3q + format: + kind: ssk + params: + format: mdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEowIBAAKCAQEAiMBOTLdfKpmY4ssFaNnMW13QvLdN7iprMGElW6uHnXrFDKu8 + + BH9HO+o8QGDZYhueAE2mvNM/y8FW1Qk+M5EP3hxeuAWkHJC7zINOltv94cnSAw72 + + BH5zif+7PxEIgn2ylla8xlUJ80BQdzEVP60yMMyJNDTgP0zdllHQcjCgnKJGQqXB + + sL8AUDj5s7Dl0i5SIBkY//aSl7+CeR8+sA5k33VGI+SnLlKBEAeD8371wwtF0VEn + + zDIQSlSyX+IPQ0wmFXkOXaXYtVvMuiDmMzYA1DOKkPnayCSYCvSe18XdMSP/Z/Cr + + rT+2H0fMmp0p8KatNHXoWXJTirLqtKGiLOrLlQIDAQABAoIBAAjUxGmzYNevLh6d + + RkboY8hVtWRugP+bqSrpZyB1og8jLcT89SokLw05OfVdW8R4bJpv6U/h44mMvYcJ + + 7wSk/k2Vbu9628eFeD9DjohzAgD6B9AvP+d35A26IFU5DB/jLqyDQvMa7EbTdS7R + + UmI3lNluaADhVkb4N4oc0/V/2utqfiuv6ILMTe7JvPqfJQnxvjJZASzpoGdLf+CJ + + VqnKOBSJSjVniiB9LDQBIWdhdsCfHeVpK4IL+oyLlW79c7f8GkgjN16iUxhm2Vk2 + + 2Qq8I3o7LIX36EqbNTAsPT8K4s0u4+yKYYZWKzL+6vc6dWV7HjibsRUMasUnco7l + + 4ddRS9kCgYEAtzt7Ox4hMPdZh25c+gFo9ACAywM3yrK3rwyqzNnPPCIDJES/CfJL + + lBQ1sprHOIZq9ffP+us/8eBdkWdWzGqKFN6JVEHE+98xKfI4Ua5wf7RQF7jhePo3 + + 5IDpPDE0etijV198gxD6HJp9c2LMERXogj3T9tjqAnB7pMYyC5RKHIkCgYEAvw9E + + SBODIsy+Arl1B1Enl24kNjh1c0q7VBgqwiB49pSA3IaP/tR73hPrQivEoVTjyP15 + + zPojyekttI6TeY/V6AH+l2m6zUcKVdYgXWD/QewtYTQKg7y/AADRIBEQY1MP6mCY + + sHu/dy6OzOwuZGR1NQk85uMVHQh600kvvnIWq60CgYAsYantndSoSaFT3nWC0Mid + + IWoQwkzHOhanvce5KqC5jft403X6cMfBrEt9YWQT2usZfNbRjh3E9nVzfLZXeQ7N + + E0HsOKn/4AXGhTcDAd+Z7xDfTha++MyE+nyD6d8uSj72MNi13mzWdM0iH7ISCV5x + + /YvT5KJ5yMkKFj+U8mwpEQKBgHNCUn7oxoOH4FjkaKUxYCEKYO4UwUX8H2Zr7d+O + + l2qpy9M9mkCxDsi6W4JfxQ9Oltv5jjEJ9e0orlnuaSk8jF6aVWwibH7KDIIb2wp6 + + KYMrZ3TsYCt5AgCOfZpKsQg6Y6+Q9owBG1Ba1erp0FLgB5UnLYZcF7CcHPy5egP5 + + 75NBAoGBAJxOkzJ/ybNiMgg/j7RuDLmizK70XVHsakjQZrqB3JbGj4nxPtY7phg8 + + ESAtpj+7n9LVd5GSpKsdZQYZW5SYUddMhHAbVjHhUKqNRwIxTy+m0ZNYf5oRMscO + + c7bVOFekrtmcMIy/tdyfeqOxNFgwmDU0TGhx5G3/GcjrHLAlaG9M + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 4096 + seed: Yw== + zfec: + required: 3 + segmentSize: 131072 + total: 10 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:CHK:gvajllsonkuscfemygbnqhq2re:uwyilm5a7so4blhsaielnf34u2qbaqmudd73opjkgodgg3okeaga:3:10:131071 + format: + kind: chk + params: null + sample: + length: 131071 + seed: LCa0a2j/xo/5m0U8HTBBNBNCLXBkg7+g+YpeiGJm564= + zfec: + required: 3 + segmentSize: 131072 + total: 10 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:SSK:saislx24hefommttekhesn4rvi:fl4mgktxt7j55najjipvgj4hlh25sietywncuqja4gcv2fycmpqq + format: + kind: ssk + params: + format: sdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEogIBAAKCAQEAo3w7C1ayXwHGmcYs9KeJp5DzuGucgPSHaydNDMutbRWm5Hd2 + + Oat0ZPsrgeVurJUT0TgDPS4SeLDVSIb+7Rg647yqEp2K5M1avn4Qwf57zaqPfubc + + 9ijDN6tGxtDiV1q0q9ZIIMaoM2GpKpxRDOcunzW/ttzno537aaUI2qbCnMvmkqan + + t0xfBsq/+JIDvEu5sLAKFBivnY38L12tU+1tlwFQ26DJbXmApeOyrnaRlgH51yCp + + q3sb94VJhdkB5TKjxVVURulo159NhBXfMU0lK9v3f6+2+YPh6NkkfgNYW4HbIYfT + + mQgVlDKqRD3duzVHjfcTzbofihrml3y7uV80jwIDAQABAoIBAAR++ndhsf7E42ZV + + OeVHOj+Tz/ARcuNlANhkjfeSc7mH7+gL6ldvAgDI8OBd+UDhH1i7S/OBxzDvLrO+ + + QAKF47thno0WghGwHWpsYx+X/7r0kYUG6lVmt5UErn9HPVeeVKZTz/X+y0oRHyrf + + KPzMMCRha7KmuUQZqHwLoDELMazy2er+o9nH+AXh3R4ZChezV+N8fEVvMPD7HNXW + + rqLFUtr+ScEc0b0ExhISzOBeHupOyxZPVsm983i262yfX/aEzytohqy0dtfLW+7+ + + gT3JRB4iSKVG2nicOSqyLlshjkVMMwAUs/ZzJM5+bpDQphtTzVjvR0dsJVovrnae + + PDz7CeECgYEAtTibsN5jvqsEDl7HSk6qCsTQsiCAF9dGXRZti9X2KLCiyoEgAEof + + QzbLYBYFA8OD7B2bl3Bc9oASboE7OHChQOoesg2kH2fg9zfqvPuAPFXpT+y5KuUc + + 1LF8ptUW6JyocSF6wDFbLkiUkeK1An11J/Z4ZK+2cmnVym8PpuBnn9ECgYEA5vIW + + HM5saxFpX+8ujNFgBn0HkyU+ZIWM0fQ3+2iqesgwfjNQUxSB/3rOzSaHh5s7zsmC + + dAlMqDyKvt1lIBD79YDfgVUvYezNkuXKm+U9dvWEmztZcQbPbxeK7g/2CbxSDokE + + mDx7Dx4VinMqCI4D07vEiVSLQozggIlKoklsBl8CgYAHCCZKa6a1LE+g+x6CjKDe + + gBqU/tvZkPni/M7NYUUG+Sun7fC+8iFaa1Li7JfPOJPy4oc6DhsdWYTdktgobX5k + + VXFReWQH7/DzxtCt+phUPwUpm8bnmjJPMn/ivVwBNKr4kNMBiCjAmAJj0scxTIry + + PQcY6RSMRf0MuNiDoiuDMQKBgD9yoPqXB5g+t1mA56QOXbhKn0sgv0x0mGSSGNM8 + + RSHoX9I8HMRGbRSYU7pu7GsoDb1ZBTsF1wadY2zefErb/6zKFB1/Hr5jhXLnKMu9 + + pi5Jc34GRyNTQKf/qs6OmgTAtTaDFD0S2Kgllrtruk+RXKHOA0fLb1sAQyltDpEZ + + ZNE3AoGAEYP6aG6CjX8p6OOBfVoPJR3is7V0TdlIilaFAYdsjRghCGt7jGnLMgX6 + + 05y3mUtlc+k+rCUiJAvUr9SHQvdsw2cXH2rBa8tskzy1RrSRO+n6mOHd/2UfDjll + + CzcGMyyU6rytmuv+xD++59PzIjXtmkh7m64CFrgknbzQ/9r0oTM= + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 131071 + seed: LCa0a2j/xo/5m0U8HTBBNBNCLXBkg7+g+YpeiGJm564= + zfec: + required: 3 + segmentSize: 131072 + total: 10 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:MDMF:e7vrv6maesiv4ax35aos7sq23e:xphbdz3aavk6tulpxjegzvfrrz3jjdv6gni246yugsucul5zgjya + format: + kind: ssk + params: + format: mdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEowIBAAKCAQEAsXNKD8HGXeQN/FhCv5e5iAFetHF8kHPNMg58SsA89Nug13l2 + + FKw+K8vECd2VjThRDmYCar4UbyIedtiHvJz8/HM3rCwGRw028oJEjaFyWtdyBGfm + + +8QB3rmPIWSj2IdMloyRRPIeU3Ae+ZsJr5dgfsv72lQSA6wgEBF/05nAV5iFDFTL + + 3wCQKAgSVWPQJIGh18QBVvppkCvRNjdCf5QqCkRzB3I1UUecU0kaHyylPutRQCjM + + O/HUf3vb91VCcqrwghf1B4JQXB3kG6HTQJYseVKM07G73ExyyPNDvFMXQthfmeMe + + 5EfGq/rVGG+Iqw4tTDHgiC0PlmQ17JCDyMmpsQIDAQABAoIBAAFC6LkX01cBTxmO + + 36MXYyeLXHLU3sSormdIynNY069uefxGyhBxA2m0k33ki1q0kTbEgieudsGt5ONA + + VucqoxhQ4e79WrT8YV+2uLcRxCjZ22mapY3/DWpvEtxoYpXbANquBeiUaMN8B8ar + + Znr0Z6iBFFGftjzR0fdxRhGGeCiQkJSHjMe02FwIDT2QSzSmGSk9IFefjXoPXwi7 + + h9Ll+wuXGa8GUGHGaVrfcWTogTw0mBOMMqoP/5FqRiiMrLn4svNXlIGfrkXwQf5J + + d9NfUEP2ilhqvGuYQSsXTu51BW39PhO9bBpXHMf7P+ZN1KVytul43y4dmcLmOnrL + + eFsFQwECgYEA7Uz2+Ep123Ny/tCy4L961jfoDJBdHHmaH8pte/FEPu4Ns7Kde2eW + + /qZQjniCWFPZkUbyLjYUuR6lmHUWxl+F7kypoz+5EBcWrHRTzlBqtSx9aIhRZoVQ + + b70q+eIx8Zu21jy159+qN8yS8W47gSDrWfmKYrvXjAq34Ey9mIK7EwECgYEAv274 + + 6NYtCzWcJpu32dKmkniyOkS7T1a9yWfn7EKzAHfwfHjcVWRQV7q0pYPNhHQwuTic + + /89wknS33KNLyPZ2EpGkqyw8S+SdYQyz1m4y1PIbWKM11ZlKJvb1fHr2D1WACrgF + + VaW3CHBFDFW2cZsKti/X+9SJgwq5DTMi8Qx/hrECgYBpZHxvzApKPB0/xQsdPI3e + + 5JegNOHVysBEDFDR8lbgKDRXsiW1cE2krdMrY6RofF0t47eeBJDxowXjD2XdFwHR + + 06SoB5424jpEv6mVASxTaP4N1jVo9h7Ccd7LesW5y/HJds9Hu5PLEoXUyqOM90Tw + + Ah+POGREI2KFMTAnszBJAQKBgQCh0Ib8IZZfpEhC5luo9wOwSe+1i0Wdkd/I8Fi9 + + f7/ZRIj2Xh842xuCnKJ4SgodzS0mU7F6Fnm8goasLSgxTguONKgxvKmXKT7Suy8E + + sY+sKp5s9UDbNcDVYOku+K0nVwlthhGUTQiDTItBGu6l5v1N9PEnwIcgSp8Thkch + + 5IOjcQKBgCyoihvhDkNSRQcUoEtZ4Fn6tqqUB1dI+Dx03FXeh79QjcQw9ftS+wEq + + EK6VVdQYg8ZR7lZboDA86JwqTgyn+r3qtf4M0r52AxPhZqdw4rqTtVEQuEM23Aw4 + + z8qYs9yyfUvBhDKhiguAQexiIBiZKneKNK0a2GXgGYF/asPJVlmg + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 131071 + seed: LCa0a2j/xo/5m0U8HTBBNBNCLXBkg7+g+YpeiGJm564= + zfec: + required: 3 + segmentSize: 131072 + total: 10 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:CHK:zdmicwopo4p4h4wbfcbnwcrvyi:6qn75anpvs5gls27f4lybisis3udvjfjhatxiny7c72bcbtuztia:3:10:131073 + format: + kind: chk + params: null + sample: + length: 131073 + seed: /N4rLtula/QIYB+3If6bXDONEO5CnqBPrlURto+/j7k= + zfec: + required: 3 + segmentSize: 131072 + total: 10 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:SSK:delnikz2uniwffxdizmeesqtea:nmpnpk6ve44qcxxiskfv6sflsj7szj7f6whrxlw2i4lfcwjhze6a + format: + kind: ssk + params: + format: sdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEogIBAAKCAQEAlzf8n46p2/xzv5XGAf+jp9qmwEic1wMRtOOF46tHAysQTbQM + + gddD9yeXYiMWGFSRzmsrXM6cWRWLJDa1NWTh4yrhtFeEv/a1V84wkSdfYHKeqEe+ + + ZgtcAifVrFlJ3kUeXLvUyHGSCeD0fcVK1Z+cIXm3uUZwLZvx9mH13Ihb5XT7QkkX + + rApCEGAf0+EE9eIJ+S+XYK7/9UOOzPcPl1jiCESwXkvYOEIzqvPfWTm9kaM8mtaW + + D7GkhTlHv7628i20nJVQu0OUeZG5GsCgNehOJ6JghoKiMysEwSiGnVF+IeZmbRwa + + TmW9bLf5HmzLUwVu1fjhC+NY5jA2TN5rqKb9sQIDAQABAoIBAAdxyMionvVe6Gpf + + QtoclheYgLLQ6FK/+o5NkKz4hMSTyUfQBDkcqkKHwxDAYjE0saBoP4BgRgtvL3RL + + ErD0HFhBB09H6zfBmyQdWR/+QJhokCW/8XBH4SHUiPUFCMlN3Qyq7RLon2xH1DWA + + YdqiMs8llT1Eadee+KKc6DwRBPgm7qgNzhk7jqv0vKPJ7XuI1H21+v10bKX3V9/m + + 6hZ5ck3AuwaJrK0uw44YX1NjY8cf+QPZVhSNQaU1ow5K/AJwaQ4JGo3Sbxb00ZL/ + + C3hhaes2TJTvd1yhRsXCDxBwwepoxEDH4SgKFwjZB7KoVscbsIP/vL0NY8FFAUDB + + njV0n4ECgYEAvT7E4EebI3YtF48I1KwgJL0i8gcU43dZ/Uy5t0GOjb3O99Cu9Nnx + + 1q0JhVBTylHSX3Vs15kjPIyyyfIFnMMjHhpxFAEdDI16mwvP72Yf4TyCxPjrZrTY + + GDqnpUSKxuVmoxrtRlyjDLqlFVlgLoN977v0oPnxFkhheEdxWBxeesECgYEAzI9Y + + 13hMQSqYtOThSrknfG9g7NiNcACWBQyHu5YCIIqbvsfHBhLj7Y/CWQWyh9z6/tKl + + qm/nd2BlirFnO51E7XS/zsyUX2erSKRmGH45cy/My/IJ+5oz/Jmjo1P4WSN4hQpi + + VBWvSMRlyGkDz219bfe/skF7+HmLsW8dg9EW7vECgYBF3lrJgyZf3U2gmQplmnbz + + mXDBcqPfpzzuK9mVMvrykdVL4Rv3AlArNg+BzLpiw/qri6r3nm5H+Jo5vMUdr13T + + y2dcP1z+OW2+uIm4lTfH7JNLLauba8EskNs8RSYHcMKIDXT0uVbpaC9yxmCgS6O/ + + UuFqXV0JIQf7ZEUQhsjLAQKBgDhtOlaFipNXSrRrhnH0TR4YIyZyPeGtZ7SQ1kg6 + + gu+zDG898HqOb20ygKvJ1IuBu4LbXHN9Vt4pKxltAksBgOf3kolbCXqfwDHTl44e + + E37gqp9/bp2G1dxSDT+ahCEilbYtPR5wtN9fvavgu/pV+4mAE9L6GVZbQNt7CSs5 + + XBghAoGATM7bx5CEEku+WvoYZW0Pl0yqbJvVM9JJqRBNz8ZrycMvSGvHAgTaYBW6 + + CjiI8208ZiTI64czcHpFS053FNlg9i5NZT/Zm3G0sgFNBJTKZqeeVDa2VgmgM9YW + + c1dRfTY/tg++kjzuo0HNU/C9CQNZFOe03o59o/nf4hySp5Gk7Iw= + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 131073 + seed: /N4rLtula/QIYB+3If6bXDONEO5CnqBPrlURto+/j7k= + zfec: + required: 3 + segmentSize: 131072 + total: 10 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:MDMF:o6dunuyiagettqdylpolevia4i:rnn3djc76kylmrcbcvcxuuvsjfx52ql455je2dkn6egie6g3bjea + format: + kind: ssk + params: + format: mdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEpAIBAAKCAQEAsI8MtaBEjERTpqfLs2sUwwXUQvWP+Jr5mbd3qG0miqToAZXC + + cvsqY9B5UoKq+JdIUr0vrQup+5PpmCibGT7/dE4kkJZ1it8U7/z3ntp+Uy6OFNFh + + ft9bFYRYu7GqGiYqtT9avLix6vvEBpNvgxJapdkb4uA2PaYuLtgWssRcD3z7c9Hs + + 3AHoA1xwBbfqzMespx7XSbuMSHG4zP31FKSienQh/4OYi4SogsOuTnGtl4UEFfNR + + Tfd41CUiotiN1OqR0AuNHjInbmk4DYHVo7MiuzKlsbT7KcLdXgPY+HYhG0eriqST + + L6flnt82s28l1zN7Ed4d3+5e/7QUk2kzYp6sbQIDAQABAoIBAAI/L8g36+dlDzN1 + + uy/jUvZQYq0fdt+RCVAdd5ZbHTxycMlkYH8aFyYCByk3pHlZY4A6DBtFpLog3b4j + + 9iVSGeoe/HQilghYYmnTbEtHOIhSdVhqebUlnoEdmAt7bVC735tC3SK9rvXwkkQL + + KEYgu6qUorg2ZjpOnRPXiCJqQUmpJhZrXbkRvAK7d9G2DTCAwUYMCRxKdTqXLoX0 + + 18FagC5Bv4XlNwtr9jozi/jjl4v6fygk7BgGeYg1Ls/ciPlXd5UhMHQBRlbch5Sz + + Gd32jHO/Lg5dMoJHeLRO3Ido4f0TxL91YuyaQ3Z74qDoHSypUEUrHKLL3GzpMaIE + + dkr8a0ECgYEAyHj2hkfimNnv4l2H0pOJFALX2rdiKKZIQHFaFReI/1DQXSY2VqIa + + qXavlVXXTjOUO668yacQ8VgoaA5RhmfjaTDv8g3TyfCNLH2sbxUkNfLKyZCzJdtV + + e56eUn2TVemRIH80wlsljIEm+7bNwRm5DAM2kOWkbe5VfO+5W3D0RikCgYEA4XZr + + oCcDWhGkP/FIX+F4/HZyOAVIWOAKbszLQO0RWAiCr3ixCTyYwtWCjECQapKVk2kH + + zLuZUAXls4GyXC7jRK2a0BuAo295EfACCsH/wbfxb5tneWF25wiVxGbOuCzgV7at + + FtGwXwaXWg2Xi8rVnstQb4nKfM1XrCP29jHoVKUCgYEAiUy3Yu5W1mLk9X8jZ+hd + + yNPNrGFOnBKOh3xauvlcfaiGnFVwf9MUOZ4s0TVyeX+/9UROzjla1ECRo/qygUAj + + s0at/3TS6YqT1bXY5FdxbnVzx6sP10yp9jmDq3GP+BY4rC4TH023oMxPu7POpYMN + + hpmoxIJTJGtIJ4Izy9nHo0kCgYEA4WWv5uXZtfuZBsvCnQgeGdaYDWVKlH82LtrR + + /9CA3E91xtKTujY4Sd+FqY0KU2DD5CDGSWjqtlOO9cwdcYb2cbxU3uP/0GQq10Hn + + 6LVVaGbqGbd01KYZZpLwlu5ojztd9JKNrBhpiDZgrQiVjo1yzlNX0IoiQm5OzasO + + w8XVDHkCgYBfZBsi/1j3tlinYmF+by7h1x4QPwV17tYhz1ExGGWGl2TlXtnsGUsf + + o+39jbguFTOYUpxX8tcHyP91BhPyNXKcxxL5R33W/Px6kYCsxYbxN9v8GMeXde9Y + + gsDPWrg7yQ3M+bi4X1VqKfW08rlQazXW0RskZJVeOyFzixLcrrgqSQ== + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 131073 + seed: /N4rLtula/QIYB+3If6bXDONEO5CnqBPrlURto+/j7k= + zfec: + required: 3 + segmentSize: 131072 + total: 10 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:CHK:bv6qthmetlhdnwc5tfjqamp3yq:ehz4ttd4g7ktkxvbovt562wfedc6jgnt5c6af7wxgp7jbwfwhoaa:3:10:2097151 + format: + kind: chk + params: null + sample: + length: 2097151 + seed: uqWglk0zIPvAxqkiFARTyFE+okq4/QV3A0gEqWckgJY= + zfec: + required: 3 + segmentSize: 131072 + total: 10 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:SSK:z6kjddndo6ngbjqvakwxg6hrc4:47tmzwejcrygejl2zxhtxpjsoc5asj55gxrozdawxeyu7cvmbjga + format: + kind: ssk + params: + format: sdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEpAIBAAKCAQEApaaORk03LSPd5fCIpaxIReD1spllgA1fX3zvs88oIepVNl6T + + i98c8MDcWafHDON3UE9q4VDerCoowCXqfV77g62XY15V5G4L0yMOGGsV21r5BAwU + + DkK/bOPllzS54Jzxq/1T1eXxevyYSwYd1IXJbociW8aKoRV9/zlPxMTeV73qyjIA + + ygKqjcjZ4Y6WTPwiTUGUPD8oXW26uKoIyYOAneBACWLxTsnfzg6CjPQJwj4a1nBE + + 3kJtbrnGkm4xHKtS23qTANPNCRDZcVfny2ut/+G3cFtoCrm44++nBQB3bzow40gF + + YLiOlDrXFtCAJRrNX41TSNEeeWmmQqPSBWj6fwIDAQABAoIBAFKTN/owlLBCYGO2 + + 549a1f5LmX8p+5B9Wg0yLRV/z1w0wbykIb4Ifxc+tLlWqyGwJHKa3EcsdovxSjYa + + 0I0ls5BdEQneZUfFWcyq/WRLwW4DJ/4N/VNsj5s68eDRzlT7N3fKhSesBBgQYeSI + + TId8F8EqyQRh8QpCuffn/G00zDeFLfPb/RxyM1164bQ2pix1isVCF0uiLswER8Lw + + lq6c3HyEMO9jFFruVEghg8GbuoSCdxfNcrqJxbEUaOfEhEeUMB9FZA25ehSSt79u + + GZN9dz6XF0QGX96jOTunFkLUOY/duYty7A4/zbexajraqhG59zgZbo4bSmtnkGJA + + Ru3GajUCgYEA0ft0PGzCAjreUMMtOaP8dNN24PXap0cgzM1znYJl93pSJtKeNNsM + + oqvJmK+OZnMMaaV95kFBktqkB/yguvV62zqGhIfTv6P/04sAyAActwJ96pcKwsvc + + yPB9W6zjChmjJXvBaXUkU28fBx/Sj7VdMCx9R9CpQa4FPXcfJyhp+w0CgYEAyfP7 + + ivY1vUyyQXEIQlGC7ANQzl3KjcGmomSnK8zdX5DO0R0g+fMK1WuJtAa4d8w78uD1 + + vCQAvQekd3Gz1lcEVkRcx2kVvno8t+ICGFlmOWtlyKnc2Vjuz/cUc00H7QGn4Yn2 + + LMOy0sDIV+MMzgd7UEm/MvJ1SVKEbNsDRO1y+LsCgYEAyIOsLX9VjDeWz9xxNVeo + + 3g6IuK1NDOvZIHkYbFJ2+GmwRS5esO50FGqi6dDK1H4MXl4P6W5rJcbvWEkfWyjL + + Fsm+ZpQl2hzLUMCuEE47HW+dugRd3EI8JQ2xR3fCnoR4zHRu7ztTYvD72hvDQEPa + + JwR05b0Vw4hfrKAx+XyYJ4kCgYEAlXc1nEtMyqWQ6E43xp19QB/UFmfkGbZRFa9Y + + 6lndHXWXG71rQpJWWk4UxGCU9lT5qXBFbtFWmpClcKF+cAxG8XH3GL71kNv3REDJ + + PCwuNCEAW9sb0OC5HsHHKO7CBu9KyOnKgKb2GnUD0cgBGhr/cRSjpZk8pN+lkssl + + SEZU6TECgYBHAQ0oBwiB0NgRaa9t/EaNJn0SPTO7L4PNYDZ+CL4UOpKIhkkC0CXT + + yngdS3hZcqQQ2udtZT6OPc6zIisra8ByvNG1wY8VkPcie3mhn/4VcctvT+B+XFVP + + 16vGbTKDQNZ+8oQo/PrPzoZWbQO0TmeFzqBnq7JqX644DW/wremZWw== + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 2097151 + seed: uqWglk0zIPvAxqkiFARTyFE+okq4/QV3A0gEqWckgJY= + zfec: + required: 3 + segmentSize: 131072 + total: 10 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:MDMF:rtv4atjpdmdz65zmupx3n2574a:zeo7cvdplvkgafvke5hdatv3gpwjgj4esk5utqgclfmvxm2kr5ca + format: + kind: ssk + params: + format: mdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEowIBAAKCAQEA2HKyf4IlgsjChJH25B47I96YP0wU7iXFzzw7HCkPMUCGRpz0 + + HsXdjfiILjXPIFqSNDy/UFZMChO9flw51+ekB8FjOJUtTcFNuVcozoTZ8g+awt+/ + + NGcwTsdvjcR6rUJt9ZnRt3/RhV6WXclfM9MxdaZ8SJ28yRhxJpBy/tGLgyLgbGET + + M/SJ6csQ3NLBZ1aJIJqgV3BhhcloqAwr9LTy9EKLGkAxFvdTYX64s9sy9LBKnC0l + + 75o0JKNi/3+t2pk3otg2PZNWh1mdHatXXbqsWNfC3IpfA1CUfG/kxsoIZZ20SAsC + + wqw5HDHPK3P5asvqJxwZBhaeXesStJ9/2eb/rQIDAQABAoIBADy/eDSMOO3rXZiq + + hNICYBPRogZF2qv6IvnmTCq7pV1r4CPKYkOOwf9aDRJ3HLJWaSlLEWDBT6cWYj0o + + Mj3T7/gTQT88sxHbGm7VtQi9RZQH6CYgeQACpA7AL6Fozwt4lPb03GS1dX8KjIY3 + + AcbAU+XSu5f/2V/RQdSSfwvgkNjVOShfaKrRjyrfe3DvSLrUK4DjuqAZ3OwCYqUP + + V33+ohcLoMqipB9LjWLV3WyfTgi0M4jqxe1TglffjJVq9divTmfs7KcyYpyyah5O + + ZAtLq2+FOOhYrBE43tcQ7QR4rnxw3ehVyZroSuICpMyhLF5fXYpsCo8z+kVpIsZv + + Mwd71fsCgYEA+laMTyiMEWyZHUbVnIRst1BQdT8c7fVGadahPhIYzElziGir9CrA + + uLGzMlyuDAOTH6aW3pTFJYeNuPtI5OSsYyjL6SrnJDGoyBGlhu8aHvh5/u+Q2Fvs + + eDgCtcEVtGjxlQQq/+BLJgYa05RmjpMeO6rlNR/7Rk2V/9Hig7K91DcCgYEA3Vft + + NIa+8zm7kWON0amQ2CqWEdmnUFN0z5MERs8vioMeWFysaO6jubmuI+1DtDkSzz0C + + fG/AeFYI8ZcMQUeGJFsMU+s8Tkt17xkrCGvnOXSYBRwHvMhEIg7cCgB/DsOqm3A4 + + zwX0djLmSx7UVv6uJiegeqXY2e8QDxlaR4CUITsCgYByXloJzBd52mh1ZKgwsptM + + gIfRmPzphfYeYm0WA4SKyD/dIRz2FxYnCyA4MPlfCb8MZbplhAgxpiVMTpk14XcU + + ck3+f5hMA9f9V3qNE+2WGqT5oI9HGXAGWGh8ivMUkiFUmCvg7KLIg198LD9Sgcn9 + + Lo064RqWOtn9nvDihCWPrwKBgQCJkGsJOTGWAuyTKJdslgFCh/0q7OXyo1u24n1G + + 8N9wK5uBeV9h++bfuAoFpCFu8gXBrP5NjjrFz1rRo3nnXGd/UuLviQS6+GU8i5zW + + KBHWAKO2kTwx1RmbPTb+NF7DM1JmNrHn4KCVkX7VczyvMKvVZM11THvgvpZxe+VD + + CSOHHwKBgC8+QcZRLAm6JDG6v8YNPi1uOw57vJ1pSmq2rJtpJll3KRUiTCF/iF8j + + PjT+86nF/IORZV4M32Iyo/j+X5DDdrRpLsQEjhcPZAk5SRm9kLV9ZVupMFq1QuXi + + IIFc0UBhrOeJWd6uXSZSM9ZVUUj63GUA4+S2QUFcJNenffN2q/Ua + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 2097151 + seed: uqWglk0zIPvAxqkiFARTyFE+okq4/QV3A0gEqWckgJY= + zfec: + required: 3 + segmentSize: 131072 + total: 10 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:CHK:n7ogyjbo5jigvxgel5ll6q4vbe:3yjb3zq5hcdavv7ruefawal6euyvjx3lx7quslvasjellv63cxya:3:10:2097153 + format: + kind: chk + params: null + sample: + length: 2097153 + seed: BTBX/ampNfLU+ox7xipBGiaSbgC0kcB8Gy7BkJB4oKI= + zfec: + required: 3 + segmentSize: 131072 + total: 10 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:SSK:ctdk4rtj7uhew6wz32rj4pmm3m:cxyc66od6x5lkqyw35jsdk2nyszomusd2i4d4jxiyo5lor2gtntq + format: + kind: ssk + params: + format: sdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEpAIBAAKCAQEAu3XMP92QeEeRzRdTYGqful6psWVg3hWcSnRy045akcnup2m9 + + 5pJhcgjBpMuvPoCzhoyvaIjMk46gTbzwi9rsWPUpNgtNl3D4oASIXszo9bVD+PLD + + CERWJC+sBXvb50C1Y29fpGEh0P4747+goKmmYKhYsesGaqfl2k6VVfXygyhrtYwK + + knlrpkSVbeGDgnfp2WDI94tIE9TjEhl/m3On4a7XA3ggkikfBHqhE5rLsKIPOEz9 + + DvubZlxZkSTdrwn5mOWE/JqyhSrmePogYNjzrjy+8qKG3+KoQgvNVdzjAWcMXpRt + + zYZyPS+sv0pvqB3omWZlkIJurqI7STlkN2FU9QIDAQABAoIBAAvvs0K5y/IstHb3 + + rkJsZ6FJV8rI5sMdYydGhO09mjzAO+cDD6l31qaZMiNZKN50+XluydiBJW2b3k80 + + 4ag2F2iOq8IaNCWZdutRfpFywL6sfRiD9LE5ELcbJfvvaBAwiZw8Qj3IRYv2NEAL + + OqIgS0zKS2OA0JbH/BXLfSzNDVUWiOZKeyRjhlHp+tRUDQFFPa5pLTBP0I21xAsX + + GLF13DAjTOFcCIaGEtuAM5OZ93qKhDmGyBpjlPTz5kRJoMIChk1gOPLXKgq1UeYt + + m1XYlAd0qs1F39Mv+yqLr6nbKt34ufywddNGt52wivjScAtpiiqFVldola8rF1Pd + + lN84qgECgYEA119K7ntn2XQ9/NKwRFvaekfOcFFzMl+XEcLmrd9oeP1ffivb6ZIn + + gLoCgv5BpR7huTDTgTvcPgiasgt7NXFo8mahmNEghUBbcDGv87fMwi+sICjptV04 + + g3or1oQ5saLoo/W9kXoQ95dJF2rGKQAAHQrXdwxizWuyb2Xr6eE9BHUCgYEA3tKU + + suNvY1xl4iRe/ayUyfQqJIkTrRNe5g2r7lls7hwbBGN8cbxtQiyW+F4CxP8xuB65 + + fbdKWPpSewjEZs5UJDgvC3PrvKTbct+0lF6zXvND3BO52vQ6t13/2gOVh42CpLAY + + xt7tmkcr7QibCziNWXK8xCQe06RuM6C/Oxme/oECgYEAzYKYtcf76HwLSlyg5hnf + + +B7c1kBidAbS2JfqFq+/uPPNU0/2oIJeP28/Rk/nw/Ab4+K7b+320xrSwmJCR1TY + + l7VnLbMgHQa0OfKvuxf/wqxKysU/fVhevNavThsOEnspEotDQLYBysAJdtbkD+t4 + + MD5QK8Ed5naF5daTrrDG0KECgYBLEieHHZkpoLeyuQ5H6R037UtFg+ldJmmSmIiU + + hQxuLIntsJb8ur8UzHEQvJuyQ0g9ABz+fgJOeAfR6+I/wMQYb9VpxmRl6iUFTtlI + + I5/LHap/OyYi3qXpoYHRseNvB/47/hha6ECk+dWSxpN19FerCz0N2B2KsJtwSXgk + + MT2gAQKBgQDORaQBkm5324m4AsYoSpn1EEr0gzMFve0X59B68kNt6e2CAVfKA6A6 + + om/LcctyDIRUjOnjogmX/iyURSCJsR1FyuuQnECtWuoos4F47utK6doZd6fgyqC6 + + cTRwVuM3ndmK+eIKrIoDb69S/AsaiVHm/FqnGJKg8vCywKsc19AI3Q== + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 2097153 + seed: BTBX/ampNfLU+ox7xipBGiaSbgC0kcB8Gy7BkJB4oKI= + zfec: + required: 3 + segmentSize: 131072 + total: 10 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:MDMF:dq6npcjrjc75bymi66mlncgga4:27qt6q3aekswegvwpzlbsx4joc42dp42d2rutejhmkrmg5v5luqq + format: + kind: ssk + params: + format: mdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEowIBAAKCAQEAihmQMc0MBheS9Zi4NwC4zU4cbWxSoMU0L34nvR6fItQmfURC + + Ez3mvVk4ByhfXFEXUEmAn+juNbxDjB1D3YtgI6APtoFxAyDXMlMKO3wwE5UCZ+OE + + YvESyp/Tek44InOtBCyMy1VUPZcaE8hWIWG5JhysUbr2Piiww2GYOmdW4zZbLWJB + + hXSToyHgDvn89nbjh2xG0x1QwhYTqZH3m0HxQFuFIdScZ6Qro1tDUNKzpmUjmXPO + + cDUJl/K2r7WGNRvCypMUBatIvXM8Dt19ezyvOGdi3SvPFyLe6gVHZoUq+W5FiE5A + + B6R1bUB0T0q5Wja/8TD++YksCG5v/zQcEmA3ZQIDAQABAoIBAC+5VdNgAN+6Fdc5 + + x696WGLas4g8/vEANWCUQDdi9aublRGFHTB5G9wjkPEoSowkmeHtBL4+SNPZE57A + + HkvZdofZMJTpdpyWJMgHWmnkKNkbjZFJVt66YLwVL4f8r/l38DqZCq7Z9hqytRhR + + CzLOCqXZEtPLwH0KostiVrEYNTafeD3jgn0C4kHfpG4cALcEQHcQVLsjmizXLXSx + + 18SC12aUnAJXG8HD9Ge2CDFfKZx2RM3RRJvq5MZhe+WbQyidnWdwEObEVhRkSpgQ + + m52mO0thkxMnC9TtjtNc4R9ZP6Q/+bep74GJjX2nbPdlGojszVcHUli5kyXwwbi9 + + AFlheqECgYEAvLp7FL5XsIH75LnegRnjTugT7nu92kdXyVqEZ9ONS9/XcgH/pMkv + + nBZ4vBEUYWqeYlsZD80tZFYE0unn8niXlpdNsIlFZV2BnxME4ZnGdtFTsjFg1AJq + + JmosiUN2vZJ2mWPv9I7cb1C8Sg1JYnhOESHn5dhG2py6Nmvwxpic82MCgYEAu1M2 + + BfqBXD+Z1qTzKp+pppR8HIeRI0f/jYsDp5TD+XVb/HBeE6Q+9T1fI2yDJeIS4WO5 + + CHUZLj2bwYLOSre3A+co5nAhj4pT1C+Iz3GsMd+m7GPhDnpYpPTohE5YIxlilKKQ + + T4B0qSk2AZQuK2i+YecyNFkZT4b4DJ2tNkvlOJcCgYEAt+J+sAxxxkIwG4DagjGm + + H6jSWshoiDiBGWg/oCYpAuebtLKr0nRQFiZzBtMhZ3WJ0s1uEs5YTu3dD1/mpoLH + + OGw9vydQ3V4JQOQ4GlRJYlW81d90t72OjdVfhXKdTEJbmkMcds2HjFI+02w0t2P+ + + tISzvWhISRLyALqVQ/tI2X8CgYAymcSjEsr0xz1gDMiev+hM1hk8f6ZF+IHgkyeW + + kgnqDbieVSAkgB59kmlroTk/93SQK6bk0PTPV9cGC7Z72mp2hG+455s5Me15CKoV + + FyijhD2L52L4zTW6wWk5rAwE1yuY6NzAjPt2YmpzPLrIARBEU/Zsy5CZueSxS7pp + + S1EM2wKBgETqNKqw2XnGKu+PCRgjnkhLLWNxMV99w9XM7Il/d5OHxiQ8fSLH+Wnx + + DyO4Bt8cRgEQef/h1dwuI5MSpL/F5AmWAk070Q9u90atzmCPzukbbiJt0D67Fzug + + B4e53WwD9VpcHTC4tN93fagbSYci1WF6+2YIiO7rI2zFARuE/jL+ + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 2097153 + seed: BTBX/ampNfLU+ox7xipBGiaSbgC0kcB8Gy7BkJB4oKI= + zfec: + required: 3 + segmentSize: 131072 + total: 10 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:CHK:d2hbvcmbex7fm3qu22yj4qnkh4:2xwqxbawwgn773hht6etox3oypvqqjv2orktnthfo2e7vibko7ha:3:10:8388607 + format: + kind: chk + params: null + sample: + length: 8388607 + seed: w6uP8Tcg6K2QR905Rms8iXTlksL6OD1KOWBxTK7wxPI= + zfec: + required: 3 + segmentSize: 131072 + total: 10 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:SSK:5v6f6emfnb5opnhyodvjzoe46q:4dtbhyqrxhribcthqybarbueominfvxydbofslw2jazx5fqsddwa + format: + kind: ssk + params: + format: sdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEowIBAAKCAQEAhAOLr3FMSznpE2hrSqzEuImNDNa3iNyqhsBFGT50ZnvAbsis + + tZ0dNfyWVKsa1H+e+5ctyYRbpJ8Lpz/clzy3aBxxtEv4dlJeR9CWUpY0r4wnYMTd + + 4L505REujkMbEw0y79hRdUMB6dakA1gKU1TlwObDLYx2ItTVoO9uNYvOLguYCAvx + + qKuYTdaGE67026Y4uQXlUlFNJWIUcr35ZsMPgiP59VeyV/GHtVqRbUwEx7Mkfjoh + + Nz7V66jVcZnyMZqxZcD5mb/F42qRtcJRLpy9f8lTlJZsWZ+WQcUiskFUUz2jX8Ll + + XRAgKZ6QX/wdUHB3uPdYmZMDLpammgrgexD/NwIDAQABAoIBAAPKLxFk5duaMjG6 + + o3hFiD3P9q4LeY6Pq/ZdJxHeQOzMP3r7fp6VsxTHFCQigVoTJXCnJ2S+/8gbDlrJ + + ZfJfINjEfRMmKmAJmOew4Kl54tG2hIIUd+/CwYvt6vGhXyUBEcxG4BfHFt/5DhGD + + ToPHG41muOwVOZlh+/wE9s7Jg6xYrrQ7u+l0S4MAsWV+IPkKkaNJE+baJ3QWgJOk + + ryN9A7esYAses5cSSw4JMgintd96u7tB59sey87OJzrpRDAxiQ+2dOSJKkbJJdqS + + NBHwtLYPLdrRRMYGrvwZrs9LFTheqs2DSrl8qThn2mCZkUzXkqEyu3weieC3Kgam + + gD7XUNECgYEAuWvsqr1pHoTof6ozJliKACw5JLz5w+3Os/Drx3/W5G6ke6iYodPx + + yfLFDLkx39Fu2XXrN47JKuuUyEXuEnvMgu7GX3hxtrUljyTRoMo6S5BcZ8s3ocCI + + WliphC8pcTcMDrthUTs78u13UZEx1HqDVhb5b46kK3n3LMHUX/YF6QkCgYEAtkNm + + a9mD7fJ8Dcn3dQmN4c/ym77V2RV0pNIzyTbiC0BVbqOQ1y4L6nRrU9kQ5semEho8 + + sPfcw3cd3uz1LJuwyHz6I4GbSlKtpEEWUfDWjyrnCnv1Hxxxov+htrVYNH/WsLq8 + + 7iGIC4/9LNboSaaXc+Bz3umC4s0qAIYJiCW+9j8CgYAUF+v3vLrtgb2oSAtu9l1O + + E3zFzGzMnLKvsUX7wpDJBGxyshyIPO2Q0uwjqtYKySlYC31H8gM+0XS4F0vrWNsa + + vUFmCylXgV3mmzjUUdXrZmN9I/qNXs3n7H/CQVIeYLa/yfKL2P1wH+e0QSXDPtuI + + ssipHC4SQA9XHFIlbAXL8QKBgQCuNZQDB+AbIofCYkYdXul3adyZUvlxyhk4pRYM + + gGHkoTRHUR3THtcS3P3tIfAOtcudR+i0ueUQC53IgzMA1TtPFm28XFhC3O3NrsyX + + u5xJMZeuJLcxam2Pf8lhKspJO9vIBmUpM3Gmo5U5M5zJMOtYPbRi301UXQeFgpg/ + + wtxY3wKBgEzQlxcbwea/3VeYbkzm0l3zmcEzcR443J1SKfIDAPW+5i/W0VoFIbM7 + + t4C07jgAM7kg6RfcanG7Qe0g4beY3+mLopNM8+tYTEWIazg14fTCDMrMbm0CIyxy + + rYnVQaYTNsadlstOnJVQqQ6KLcWHadx2SnFll+RPp9h8UHZZeofX + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 8388607 + seed: w6uP8Tcg6K2QR905Rms8iXTlksL6OD1KOWBxTK7wxPI= + zfec: + required: 3 + segmentSize: 131072 + total: 10 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:MDMF:aqh7dflybkehfvjv2dtwm777qe:glkivk52anvjmc6b7ayncuq6fpxqsgel46uxyqqaajp7hf367ica + format: + kind: ssk + params: + format: mdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEowIBAAKCAQEAwMil8D68T4BxfJHUaEsFhPutYwqYV0jTnZHf/+TPS3SgkIbS + + OjzNzsoCZAyGDTA8ICfWOyY7W84hTGq6pSyXdBfokkpTiMC1kZFJKwqowUv5o6VF + + 6qH1yiOHR2KZSGxEhS6D0bIaX1PcUH9bipGT9AR/qiexc9RUhHBVb57tpiEziM3R + + UNpT5roSh7WxWuyloKDkzxA2/1FVeoxoa3Xc/vlIaTsbx19cJCMnamnnU8cpYfZm + + 0WMOHEG51v8GHKnQ/4si3KvDQvrPtG1IxEpM0guzZmEXG5z0eFo3C1zKfVFxPAbC + + eK/J0xKUbbq2P4+FgkK9vCR5UYUgxPLZ9GGFCQIDAQABAoIBACAbit+HY0+OYdhQ + + ZWL1U7cBP7BmHFc1LuFoYTk6P3getXs8qRi/9bsCFAHbwBvEM89bMyfoxywUGaGj + + iPBni9XvAXIT5PO6vMLAwsHjZZXD9JDXvtxEGy6OWkJ+Xm8ccREJXTT4h8HmsqPJ + + glKCynRyp1yMfdZ/v4/LMb+EZaosRNbLBzhgW70NGI/eY0S2DNjpNfx1McfYpKkK + + IsBqq5R0YdvwmF+yaU5BuKOYe8Vtu5S0m1/LRrTNHzNB6Hv1odwhzZU09WaVpZdz + + N77vj5s72OnsZsM2EeCEZjQ3x+gD9PnJ29pBlFtO7YLMvWjASOav3S74Vy3+aPG9 + + 65kmD70CgYEA+9pLETK94JzNG2kLkdG+ecyIy+DuLInfAKNjfIqz3JNZoizIcj5r + + 9M2haKX62bW6DBbmvp5vzvqoK3hyM4FqwU9PEr7p6AjF8ffoH/qta/BaV7rf68iQ + + b18J9rHhjIp52/Ke3P1tGtiNhP2u4kzh3mMtS9+b3Atb4vpaXlYug18CgYEAw/VY + + RmfQOOb6LdC0apARf1NUnGu/h1OjtqykgVZMl1f+WV60F2jw8OE2gd+XMqE6qWuc + + YQHAgQL5FYaPXJzWpgKo5WIE35+80xwHdwbABfGVfccCOGW44Ypm6hhHne1PaN7z + + wsRVOKHtb0bm7hBs1Y+ZEkpbNDdHB4PumQ0S+JcCgYAAk5FUar9Qgktd4rGqFcbP + + 1I4DmXIyG+asw7L4mACtYpDz9BJJYKcymj3iVW7rjKTuXicNDKPI333/C3mHcKZj + + 5uCRdGpoo4yAb0bSu+olsxkh1kWo7n6WIquNKv8PKUn6HOYML3BOfWxlf4ck8XQa + + 5DM4VzyuFkCRlm0ahiv5FQKBgHvjr43DsJdpIJ66pnYA468WJhZG4O5T6NtjRxYm + + U4ITtdEW2NE8HaiNGoL9s3/lA0t9p36FNwnZsVT0n8qztdl7MQDk+aPQP/dQbz/H + + WrvnQtYkbbjuRvcBI5O5Cf5EvMHWw1JOAnstlQmXUAUPCV/zy5kOvZ7Dm/qaZM0K + + wQW9AoGBAOzDDUkowmzcn1Fs16x0ZhRgzh8knWwKkoybULqpshDf7jj3gZOBBa4c + + 4VDUcbhdxBtqPv7243lYbkpZ+0FDYRIVu1iIVjvcNIxDMpKK8ZcQ2LfEDWT/3k7J + + ZUGmgegwepIT23xmS4nh25sT+2m9q0ugPpQAD7pCd5+Aci23dvQg + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 8388607 + seed: w6uP8Tcg6K2QR905Rms8iXTlksL6OD1KOWBxTK7wxPI= + zfec: + required: 3 + segmentSize: 131072 + total: 10 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:CHK:ccxkyfl2qtqyhihduarpxbdcci:e64be3i2t25selbpc5y2zj443gkdo65chs2o4tpqb7axud5lnxta:3:10:8388609 + format: + kind: chk + params: null + sample: + length: 8388609 + seed: yPi3JHKKbWaEEG5eZOlM6BHJll0Z3UTdBzz4bPQ7wjg= + zfec: + required: 3 + segmentSize: 131072 + total: 10 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:SSK:rdtcycjlgs4jrn2xfgrbbumely:rt64kcear75nsev3r6344qvxzqo3hcooigugckkreyqld7toy57q + format: + kind: ssk + params: + format: sdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEpAIBAAKCAQEAppvR9FvL+QCAN4iaEysV+pxecdHZSJwM1F9cQz4E38XnUhBs + + Le1eR4SNNUN/xVy+hPyiNBBU1eud0YTnG23ACoxN9qyf76+A1+uyXAKYcoUFbs4W + + 30MLp1bUyByx80rHrOsQUH8Hohk1qtP6RRW6e+tK3htm/OtvQwiRhSqscXW8ycFR + + G36Xid5wYLx610JnwPOYOGDYqhCYLxK0tSOZkQp/tHXt42ADNGMVLTe+kFLX8UKR + + znc5no2LRHdhDuG3mUrb+0WUWZHpK4S3uB/WZwbMLOCoy4fQs5Z0nXsbV1NrU77j + + LasfAaVjORirFVdKMPUVmJARKGFjQYvigVaw0QIDAQABAoIBABGaLNhwSmCIWQOE + + /yI/TxcnJiNIVHiDZCeb25ePGdy6f/H/oi5IAcn0iyaxdvJXFhnexxRRFWV0ezwD + + mpcfRUbYA/Sn0E32cNpfIHzwGUMgIq7OP0RfRP/tAJYT0gkuQWJXg2W9xgSuPSlL + + NAnQfd9RwJsusfbOuPaQFS/Ijmd+KPWfIMmSLwGfkNbOA1xHYbLNHYQfrYAosV9x + + W6P8BUXzL9vHwkHNmRi4EJt43QeQj8eXoHIsUhZ2ZwANVg3Lw84CAGqjMbxyFJQ0 + + iZ5nFNUSMRLFfNxMlft3DAqN+T85ZDFjjscaNC9IVcVo/ou2l5LWNCyzpDnyuTrP + + bAPfeLECgYEAtleGb/OO+CDTnyIqcvsr9NkE/O/lcnZZS0SB5YhDI8/SRzWOW20B + + rD+YTp1MQid069A9pKsIax6FW0eBWthohEe4N0CS2N3Jw0sWVIbmpWabYx86NBAz + + uto7OHSNoHZvoVg+ANy2Lwj3Prl1cKCKLxPpARrvT89kxBPBv7dQZYcCgYEA6elH + + glTu3KT9tj2T34xrtYY4PqhFgbdCHSRwKL9zHypadvGutN0CsAyPRjWm2Ie3pIs6 + + KnJg5Swo0T80NWwXPLfYzWAz+cJVMs5+vGjz28+A7FhjeYvNRZ231VDuhcc1AL9S + + JOq6QDiysDEBSfVstQP84j+FNpETk0p02JLhTOcCgYAxg2raE4ULE77jQ1/LgTDa + + d+PG202u2zw8GAo9zdaNbu1msMBLSzpdD5fIISaIADbbodxbTqYmkE8eDjit9n3L + + Db6UIlC92tvi0AzsPwV6fHZNYDlp0cx6PLBAEEY1AHQnl9KeYVCHTSP2QF4Hi1B6 + + oClxR2MchPCT3dmKubh3GQKBgQC1s6X94zYtpekEEO92nyDoQJweaB6eNhoggzax + + II8v7XmangEls+0rjoYZdwHlf/+yzQhhArqsK1KFwQAwY4flfbbnSsz1PfVq4ydl + + +m08GgO/FKYpO+U4J90u0pCG0QkmTHhl/wSxcJm17ktfBUvtjWx63/b+PVIkf5km + + x2pGjwKBgQC0nYYt/ntilVFsdCfV3LihC57LLwdejM261vsI+YW2klTzyxEFzrmq + + BnA1KZD1zGWhth5ZMgsYyTPCjWlyisE4+UukFE3UYeWHqJPLJvpWNjjznmrzW5wk + + oUHNGL3VfIz8nDIyrQdbv0z8IFhNbKtnfjqlM+c1+Uab+rVQTcH/3w== + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 8388609 + seed: yPi3JHKKbWaEEG5eZOlM6BHJll0Z3UTdBzz4bPQ7wjg= + zfec: + required: 3 + segmentSize: 131072 + total: 10 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:MDMF:ncobf2b52ul7stjknlrgtksjnq:qp5dwo33smgcad246gki2csqzcsts6e4r67y6mrqtinnf2q4lydq + format: + kind: ssk + params: + format: mdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEpQIBAAKCAQEA2Ihx4l/I4u2ki8Ncsn0/JhHldSllOmU9BMtgDdozGcGzQ7kg + + bX3zgf2n2HL/Hn+DouPiGoF+3lL9xBnVasEatMeuHuGrJUeFSgVFzSnlkN2j6vm0 + + n0cV7pk2G9yjaFGZHNppWYwiidUq/tq3b6PilOM+PVPAFOilXafAj2V84A46wkgX + + pNtZFR08r+Sytt2ZfMDjn1VeJggIKbssKPARLCrzhgEIVqBoXEL3isqf9JplbA4H + + H+oXyrkNaOOWsGuq91kXDRDNAR8xeYwHKs0MOUOWGRkgisdv2EHG3c1dZXVldNok + + der96cIwyRNPAJo2+DAdBdfYBqbE3innNHmQAQIDAQABAoIBABk32uZIeDb7Alzy + + BcvBNnzn51dmg3mc7NGwIKG8YJMxXKrDLCRUX5XWGu7khb3hj/fanoIDxD4ptZBZ + + 3zctrOnnuj8H0qI8MCksxV0IR3TrEKTUgYAYtqnb13zAjO92AWUQ3ZAmTVIhgikT + + A/t76hmjqvitgQNuYkQEE4h1LhP8nUPFG5IJCxCdqRBK2pUw551+MvS/9GJs75KI + + o/26+hUU/pI299VOwn3qETJaFiwka4XJpP/sLaGAQsdB00naexzzU9M/t6GjFPCP + + zG2Na9ppr7QU1XdOj2BsEDqFdUxtQyzHNE+UKKNa7OBQK4f9xiQqwaI2Ayiu8tpC + + QQRVknkCgYEA/j5RUAfktQg/yp9YDIWRgC+RxnZlVVEuAwrpmEXMIGxnrJ6cA6fx + + 6IjEYOjfpDPVN1NAXelSulSatHyLj/Dnol5dkW1q1Br3e4MeW/7IKNnFVbF/UHZG + + IRBFtKFPTAIRVWULr4BDKCq+ttwr/Eyfe+S8apBCwgFF6iv3kN7zKcMCgYEA2gdt + + sK2Z4Hp+ibkf3FMfcPCCCD5q3WVAzyOB0v9IFJzSmu/ibyET+dJ97ND8yWmz6DFF + + U+ybbV7BhYC/3tLo3gVTDs4x5OiRhX9+NEh3Uwtpm//JnINdgj5Pc8u9xxb3w5yH + + HTy+qEKBVUfOh+/WkfU2o3fO0YJHIim39nT3PusCgYEAgcL6q1csAr2wGVGUleeC + + KKOeymVZON9TFZh3OxG8qnvJuk/FnxQTorRTTobsxhjyZOdnvca9Q3606xN6A8BX + + 6QYyyWvID3OoBnEYiKmULU1gq2kJat7C0lNE0HlYSJnxkN0exrc3D4QpjJj5Fi9h + + YtGO3PC+MdiGf4trMpSoFRMCgYEAmYnxrSIT4wlgYwyDa1z+H0K/z55lE1Rit3yB + + yF0OHbXyejnEdA4PSzb4hvUFj7FoiHNqJxfQvMyl66YneHt+khudyida66D8Gc8W + + ySrfHRREYx9Wk2nPSBEpUpqAItwBzzdDz0sf2M481hmjUAeOS2sr9yI/+zqLbXuD + + mYP1OdECgYEAiHu1wNmw4FNfN5ZZDJoaH2sHpMsjzzitXXVM2tZcRaYlu2sz71uH + + 07AtMk5Sdh8E4z2GFRoN3+q4wJamXg24Yy4UUZgwcupIqZ6hk6I5LNvmTUrj5oG3 + + 01D4Ati93EPwD5XBEuhLh+c1YqzEoz0opo/5kOzwR//vA0RHp99GBC4= + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 8388609 + seed: yPi3JHKKbWaEEG5eZOlM6BHJll0Z3UTdBzz4bPQ7wjg= + zfec: + required: 3 + segmentSize: 131072 + total: 10 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:CHK:sghm3tydjjaadmiiuda3flhmne:5fqqykrndg5kydmhwetwqdzria4ap475j2qfmq2gmklzop6y6tla:71:255:56 + format: + kind: chk + params: null + sample: + length: 56 + seed: YQ== + zfec: + required: 71 + segmentSize: 131072 + total: 255 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:SSK:rqaf2txzrv6oh4znpy63d4s2jq:ojyyneb4p75ywfya4xmlxilckvy35mh4his32bmnzuiybdz6o6ba + format: + kind: ssk + params: + format: sdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEoAIBAAKCAQEAyOV62tMTsblQDvezJSsNTcZ8hPHsiSB24NanhEPDXCYB4vjG + + i6kiHsfvbyANObMHM9UMV/zDdWMbK3O316n/ok7wqc2IXZsoZLsyFP00e6IPtR76 + + kwCFU4koMnLGEUOua9MQ5AJCDG5aYwDXzqL9KdPxCHliBMtRtwoW+dH/je+HRSpG + + kWbji8vB2mQkTekJ+6ayTiKLyd3kd1sR/d/61kKnITMVTgcG2wQQb7o5rKV2XFeh + + IUX689Dh3NlDj68Ps6OIAQvcYjiL//bbjyTOHN0XVJzTQGTnPDF4rYnxOplCGoFZ + + 67ocrsmeZyEO7e+JFaftiFkCI33ixnuz57N4gQIDAQABAoIBABvxCEfZi3KGHD5L + + XFV4lJvzKzbQcSjXZOwRCZvp/YjaWRvTpHB2AI1jGzMLtFszQnaxP0tuQqntNAZf + + TLwdubTo9tcfLkQarvZnr5Jfb0ZyJsN3FhHjxNIhZapdRUgkb4+GpoxCoM4a0SPW + + UQMSr1R7Km9BIWLE+/i8hK/lcdOwiboUmtAdQsS/gJ8TamhO1UL17hf9r5rSUTsQ + + u7BHTYfC+PgR19grdU0ARLdpYmAY9fLktFNJrejBRncLr71292KSoEpYQNGOjcA3 + + 3NnDiFsuYIPDLceeybQRV+N/1noU4q7i/KwnwmX0IuGSuSUULnvga2Ug+UepvgvH + + HcDTjMMCgYEA3uCtXIsqTYIHGmvKYdMIL5kCGnCMW/glh257KGSmpNlE087eYTON + + fgY0zzkhimxj5DKF8Snr0x3lapSp6H5CFeqrm5UkzNLVgGRM+BGqiHlrFSTHOanZ + + JOZsX3B4LxT8vK+4gRQnCpEhJ57noEYgj74HYPPgjfpuGsVtjJA2Cs8CgYEA5sCI + + N3oP4yehCCQHcRG1AqV4e+65iKlSbpFWF4ucwMVAf2NiP2QSj7tkArDcpaRJIkbN + + od4kGbJXF1DP7I3Fa3ezB7AvfND6fEJLNRB3/5o0GB4L4ewO8KWMhI0iLfCMbitJ + + 8DgKCtEudTB/HoQCdPKbnze8wlmcWIZLrYUN268CgYADT/eDnpXcXQhZ/iwd1BMV + + EgMT/YQ4gbGdF6lA6m4HmSsKstJfQ3Lg4pq6UbEL65x4cb/H28Wjd5hHQzpbODUn + + OjuerlLDsIZ3yAXU0f5k1NkgkVFcrAeMItiNepBusrMm4r2tPW1vHMUPX681lJU7 + + TamyaS13LregMjr0kdgbxQJ/ZsFV515ztLPxAa8JoVBBSuxkusuT00eTbalKrTF+ + + nFk6X3/iQFhP67GG16vqldiSuLDO4UYKzWadYcSa0rxPLYwgLUxH2U02Ph9HXln3 + + FduVVygKIpD8Fi2iZWRz1AFKh8S/KDnMPwTnq0ftU6l1bp6arkwjwmglN0aWbK7T + + fQKBgEfZQ5/jSejcKXJnZNp1otPJi8eB8vTtXShrcpxFaFF+ua+ddjWbZ+5KbfUX + + fCRMDxiRVwFROTDdH6jx8JPzHkL7RoKifUMhrMP94Y0m9sVUdvoEdU1W+eENAYif + + aFP+pZQChMarh2EzqWJbQ4RP6wVq4jqr2QUGVOIpXON0fjz2 + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 56 + seed: YQ== + zfec: + required: 71 + segmentSize: 131072 + total: 255 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:MDMF:uk26bsxso2rrhxsgampqgm43wu:e6ndy2c7jfgkd4dcypcrisdojwfvptx57x7rs3svwkpbf4owcldq + format: + kind: ssk + params: + format: mdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEowIBAAKCAQEAnwlavQCw5YG48BGsuphgKXVttgWHX14DwIQMplv1x5SAq074 + + CqCwL5mj0RzltwDBaUeKRy/lOWMtgJZMUQQekacTRgl4qbwEqx7L39lKy6H6LZpK + + 18tt3XArOFy4oqqzDIEmjyE87wJD3mCJjQ/NfMIC8LzcDKo5zb3tDRaalQUtuf00 + + csyNhQrBNkF79L1TFGB/iSN/uGlRUk2x1PsRowOvQgFqsazswtARwrl7KyOE7w5y + + 58l+ooVYb2KjwL8KewaUdn/tfJHMzWgQcYjT7aqxaeqJVoj3XMCif8fK+UACY8hL + + MHVj/op0SbGvT9dqbXWoYrsmNv15ivXofa0zfwIDAQABAoIBABkj6WwnR8+ACjQp + + Fx0IKWtkXMuBZDz3J7CvLzC9KMU7/HsYKK3FaRSdPQA5iTa8r9ZssLdAIwRHYVIK + + cFX+SLbNqoZPyPtL9ZD3dVMVjnVSTbIXye0DA4MV0D9AqQ9N3LAFWosVvgQqX4Av + + 0o6yCNHH+Z8Eu+RkpG6Zr3d6M0WLJDXkcjljCxXWHpobxCJzWCqmQnWppXXDrlKr + + az10kdD0y5YxU2NspahEymW80Jjbrw6heyKLYoVIpzHtMmRtxhd0jydqdstFT0Jc + + ZiSfM4k0Sr0FLPcxJ2EbB36QmisZW94eOh/er4WxffubJnjRSRCKshWANwCOpr9F + + r204R4ECgYEAz3BAlPM+8K+vBsESIdmcxjw2ECYPySSCaQIQiX179mT/c1RnT0xL + + OgKk0zGNhPyVbQCa5viK+gc3QOFxu/py7PHtTO88GVHS1JOJGl5l5JCyzVCzhE3f + + Ml1AB7wA3x8FS4fuicsVFFEJixGccjIOthGBdhg6x5uTrLMbsWVZNBkCgYEAxERi + + FMcunTUB3tNr4vJq0IBu563lxgL4y7CqVJnsPuLrNrbrg3UyB6IPgLqBzCH+xm2/ + + BJ6gPwFtzvDfNG+CnRwcFNK9csPZZY2ZKqIgjxTRcWjiKhcj+SFxw3K5dfESzsqZ + + szW26V4CgB+Kzxs4WWEyzBV3KjOv/rXwY03oV1cCgYEAoqT37hG+4sZM7HXLOsE9 + + ++xP01+UdvhqS90zjCnYXTuZUxr1maZPQV+7TmAG/yNwIbQcwEZV6W5o8zUQkPvw + + yjlx/yWAsLWIIea/0+355DlUCEljR7Qq8XlN8AKHiGnxI+SjsmSJ1ZEoc2LOkHcR + + M84L/MVIqSMhqYIRj4jQZVkCgYAOpbfYKyFMdDdGhOrJTiQwmVUtjynVxEUDFpUv + + qSkbbF33gGFFN0rbjPmxNroXHPZhorEdzCTTbuzeA9X0mNnblcx2tV+UIA+qZ43l + + w6HAa+JRn205jO6PWjKeToKOzcYEjtQ3rquO8QgovbHjUPm8medrmbKCAMeCr9tX + + 3emYEwKBgCrSOr62QCLvyYbry3CpVzF4/pnL2aDIZzw02hqbzwxR3KfDOon618O4 + + au1LFRWwjecTDDH9ZWB//tMp14KHBtDob02QGKABcG9x/IbmcNupO3Y510aQtz3W + + ZBpV+hujw6tWBj8PfwOX54nYe6KQ4sLpiAcQhVR2Ph6rDkTwQkBk + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 56 + seed: YQ== + zfec: + required: 71 + segmentSize: 131072 + total: 255 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:CHK:2wvrzaobtfkoiqeqwryjzvxatq:7zzaycbdhepzaqtdrku2sbb572h64ggwstu47osa2gn4ol3eglmq:71:255:1024 + format: + kind: chk + params: null + sample: + length: 1024 + seed: YQ== + zfec: + required: 71 + segmentSize: 131072 + total: 255 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:SSK:s5llvgygyune34aqpulnw37mo4:nap3zaqvcvwif7tjgxnsftnz3qxtv6bwlegkpzt2kmdo4htwqskq + format: + kind: ssk + params: + format: sdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEowIBAAKCAQEA4yNOr0yItq8mSCiYSrl9srFX+CWr2DFp9vf998X4qByc8Y4E + + vSQgWhLAKtvbtvsmoGLUrfSdqgm/Zm+ro9r6kiKnu8QX2aOp/byybxOP03n1Hg9J + + aVdR9TmerZNR4JEqvwICZBjnbjtOcBs+0EU4SFhRZI2hA+Gl7xEeRYtHBunapQSn + + 0JQcyi/pkCZ3hAsoMVw0A/Kh5B5a49zJ1Az8OAzuXBbhQUFieVcMyyTiBMeQJTJ2 + + /7uaw8psaSQfcxnFVjFFh7RwybbrQF3wiBs8FkndLOvMHgQRJqNXjftSdmnuOq6U + + EATRXcwnTdoI/WUWVeYub9kDfvBcZXSNP4auYwIDAQABAoIBAA9wUSMPM7FSWtEt + + /5pmQ0n7GryhauEC2TrSBNXSA6KbPa7mW7uzeFN6N/+k+nYHwJdhDSdWm8TfOfHi + + uaryJNsvNqSuCBgt/+b6uLIq65+Q1Mo0EpklBOTpnR4boZJ/EewlXQSkWk64aNt4 + + rp6F2Sgc5xaYaqjF4XtMRMXhv3RghHfPyrq939MWKRSwr419IDlyUDR3ZQCh1Qm1 + + mjvBUR0eQ/eUeIibu5DKfCi/NL3pI29ECFdCKjbS4aMCyuXUBIXAlngTe9SjLfcT + + Z7AoLpYC1hoV+965R+jn18tf8q7WSfzqiLolh00iZTK9mEhTHq2IaFiQwKeh80Tm + + OR8KkwECgYEA+SkEPYYggcnkGnro2fdGX8EU2GfAfCiVGrtfQ5d/ZGU+N6qErfZ/ + + evKGoGjLQ5A9CurNsY4XKr0t8p4aw3Jt1X+0YbNBVFeAFgmWB0wFx8C7x2cY7i73 + + ptkOnrbP6iHxGcaJFdzzslLpFES9g/KNarW9eva3K/sB2MBFcxlg54ECgYEA6V+H + + N5WTDeURfdei3H21Fmb1/aCEfv3LqEt2pJUajFy41g6z4ckO7Oj6sE9x433M6VC4 + + 2+uhesL6AiBjKkqAMc1YEBO8pFRjKgAGxbuUbdRCvLNk8YBd26HPTMONlW6oBZDu + + ZqfZ/pqEyJsYAFcdYTlAKrARDGA7WwMVIAe05+MCgYB10oaVzWpr1ZvPRdX81Kjr + + uPNxjkaAr/QqavaWkPqF8DZmvnT1ir4n1q4BBu0v6vJiyjwwvV+JL2Kd+1Punpr/ + + vd7/4HOBPcttIGVY2ANXvXVOyxsH7x/fP39hYFObhSdtJ+xFcXGwHvLnScZQsg9b + + qcuLbUWbP5xU8j8lOZgQgQKBgQDQ/OQ5KbBcFBO67x2AeO5vFlsZ+uJMWvlDR/kC + + YCg7JFm+D8KU4pmEHQtKUoq535FeKxSwlO2x4uNCfkBvwfHVJ3/CPfqD6rI3DXkD + + H/1G8XumQryV7I+gvOHIa6Lh/AtpmKV1tsDoWPWqNAGlZF4CD+PflnZd79uXoEYN + + vfkKYwKBgEX62oKSPcVOnPnbRe1MjANJ6jvTy0SeUQ+WNhaNJtYUJ4HcQrxVOB3v + + WuH5X8m6gVvLiGDfPDSOPBovEZGAFqWGofN5MCgbjkbjW3/Et4d8OKgV0/88+/r7 + + 69J+hC4lXWC1vWa65Fi4FpmSfrQniw43acPJbAQ0Q3XzXpS0XESX + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 1024 + seed: YQ== + zfec: + required: 71 + segmentSize: 131072 + total: 255 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:MDMF:fsywl3mmaxoprbn7uogusakox4:6qfednkpeowa6ikzu47rwoztwxbyhc4ofccxrtbdtdsyhpt2veja + format: + kind: ssk + params: + format: mdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEogIBAAKCAQEAqgojWWXz9+4FGPFmfCSbDcKXDLQrVfkbafr1MZPV0NpncKgE + + bJaEYKqSvPo6pjLm2OEm6dJ6R6ekJuT5rUp7LYY/DSg78m4VJ0tujVEbrrDG5BSk + + dNGNaBTgM4/40VUwzWJwbLr4gJqdCrgI6zYk1qptGXozqRwTY5yZJkWM9A87OkrL + + Sbzj5HoBjn3J+LbjzmzFefVC7Cb7XZ6lNokvoNxxTum5CzDZXkc7LHIYrw5mNdZ5 + + 2OScTrA29/SXYBQvTrRArfVXEOHUEmRtgZwYRcPComO9YUzLAXrLTRQE6at2oZcM + + mNGLRmmr9RuT9yAalF8nDciGv/dlu/P1Gp5N6QIDAQABAoIBAAJu7TUTDRP4wcI3 + + q1jU8qwWfNaUQHeCCyPWRDDlFb75br5q64J85lP2aCDdfHF5UoDhh47g8aa5wSyP + + 6i74g4oLlpzmI1kTieioGKLbxlRJ/tQ8o5YK+djp5yoNu7v0Wf1I1P67vG3yTuBP + + pGVuft+Pv+QIseZDLgraCGRtQ97DnNl5egTnTtVS1hNu3GqjDSfTK3qR1LWanqYy + + ypWbWgUKygcFQv20mRTNotYmjRWIBaWdzSrvIA8pAUw5U6R+FZ2xfhvpL9KuUvIF + + 48WA00WpCO88az7RKJL+lUuC44qibduLBc4YEY96TCjKo1prN7nOGR2RR3qd+/vC + + gJOZPyECgYEAwoSBJayvhAGIU5+KIggwSXvm5VqlZSMoTtRC8WZghZKxmgX47FCR + + nj8VbVbVWG+lg8fVGXW6SCys4//NfbxvMP3IIN1tngb4klY6TIs3XIB7PfE0HtZt + + TCoI5mEwCWX3VSRPTqaLBS2y8Ad/4bNMyPgnQX1eQY3wDSBfI4CNQvkCgYEA38j7 + + zlFkhat8ZndJoFoJEYFX1saw+rI8OgXcKg/b5jtsJ/T+R/DLe26YYxFhnglOcKy/ + + LFm//z6rnky5ZCBF2WdYFMksxn20IXGwsDBiv2Cxn/SieWxQRfR1g45yC9TWNm7w + + +wuErTofXXHDyFwLnoaGV1hgfEfcvHW5mr/TLnECgYADdG8GyEZlxdEyCwddC2Aw + + Le8v66g8X597pvF5cCQOu0hEQA7nw5aShPRQeNZZN8Js0MPMK/cfCQwZEJYJwasH + + 57oCO2yS/fS0RKvMaDyXfAC0XPBcC9rtG2IFFXzQ7eqyrG5sKzEU6nbfJIL080ZN + + 23p3A08FQwwcb5LBAqt/oQKBgBEeGYzFkw/adzCLTVlzqZ/qKeLm3eC/Q3YYvqeF + + AQgSYYqI5e5wz8/IPOXPDY1+Hr7lp9Xno5UNoSkBq2iqQ02G5yjn3oHsWZv5S0+e + + 097ZsZyPpOHu1BEVyuteOQEIrb4KLGq3jdWGTaHjMtufls/wcFQ8EV1QTeUoiCL7 + + K1cxAoGAen+lV70vCiC98q/oBwt5SAvuuX0C7fWwjA94qRjvIopt6aJlBj3cKb2z + + FizF02dt7qrswg/uEiQb26BdUz+xctulB8kC5ZfLD6EssUItZjnvYT7IqvHHzJdw + + P/GIEwZ36zKwM4G4oCb/JgIykAHelSgd62FxJU0ACScXPPx1CZU= + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 1024 + seed: YQ== + zfec: + required: 71 + segmentSize: 131072 + total: 255 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:CHK:7wi5xn2uwerkjzqs5ndn4eimti:llrws4uig2fpi3rtla6lk7m4t6zudjhszdfjgqtihghfrvs2robq:71:255:4096 + format: + kind: chk + params: null + sample: + length: 4096 + seed: Yw== + zfec: + required: 71 + segmentSize: 131072 + total: 255 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:SSK:6dul24kri32nxlqpfkq6wpfcsu:ykq5hxiipdpthhz6ceqybs2itiyjoeoigloze3uvplzxnkgviwxa + format: + kind: ssk + params: + format: sdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEowIBAAKCAQEAslPNDO8333eMl0cGe55QqUzk8Pjr1shYREPtsWzTQ+viJVrr + + 4sbf8sZfYfZM5i4wogHLRAEdAi+U/kX2iX2Uvowg43FMOE3OKWLUYHGLQSCChhap + + hNkkvat15rC2O7YoPff1xHR1QfSNZDDeYUcfDNBZeg3KUJNSU2jTQYtlivA8YAj2 + + u60f1T4hCn7Xn0TKBUlNmzluewyM2Eh9H7NKqqeRtwGC0IAOORvLA2ymbtZDy/lw + + RyGjBVIf3owjFFzB/+UeG79rf2mPPqRwRJdhS1vkJVyO7zKNa19dWIbyXsEmpSTf + + VpQVxQ8qDjsMHyVVq92jkhP05AenMtDBtCn5nwIDAQABAoIBAEmAYfmNVi29BE1M + + IJ9uSxflEk5Cg06liEAm8X9aeB+8R5uXBLgVubPC0Qi7MMoFStVTwPjYLqE6hIJj + + yvCzus3pSxsEFWL1qt6DFj9kPX4MDNCA4cFYkRy+YdvChXJKK/8Sx5GAYN5dErQz + + sk1NN76b1+2HZpbciifIAp82+hUQLOWZCwjJ6TJ46Prt2KEfqbkL6/+Hq2dnBUJw + + 4GKT5ZFZSrcCK5Zr6NHrb2fjnCH5O8r14tF75tY3d4yY3EX/r4cC0dR1IGziZk+j + + oKpK3xv0zlrqd7mSe2I5vZLiugQisTLKcZUQtfiLT/lEtjj9HE0YeVZAWP78qB/4 + + mMJoMk0CgYEA02zrxFd7hHG3VdxvnvC9wzKYswroSricXKELa/xP6ChGG0z+nprC + + tImBbXhJ+nTQkHWZBZDkHUmbWUDcq+BxcQeBSRp5uzkkaxlXPy6boMKxFYaHJP1j + + 4mcHY7Q3UzP4LWN0NBft9fTJ5pX3kwWkLkQMy/ST5z9vGjbfaIrwfksCgYEA1+yB + + HNK3VqD60A17zMfUGb8s1qwzo1vCZo2VQy617wOtk8VzvmepB3bceEQNZnYcoydH + + TMZkVmiENBvkieD2O9DG9FkLc/PjKkbq2FVpyqNYlevvrG0OEMLWOK/f/fLn+FZl + + bnVj/O94oRVwfUSEk8Adsod7sQUoFIsVK2VnjX0CgYEAtLxv77A5TsdHSobehKiY + + D724+5VfbkDSqfyhnvZZ+MQ06jGvmDYELAFAOyyRUSF7CYL+BNwPpVm/C1V/Tw7W + + 6yDXTH7tgTcgAs3u33wgXhUQ/K276csTD/+zOXBduyq6BVL3i3DJY3CXCB87PNud + + tk9GATRbG1wGxgoSgXQEknECgYADj8IdcJhXlHYuolpNaWplNlMOA28inavaNzGk + + FwwnMh9V1abwGBOgrOQ8E5tI+l/EjSxO5uLWzgiIN4GQiKZnHC178FARDI/NrbfH + + 87i3//PBHVApvu7BdgVEkBoYvT34SayIouUQUf7iYVEmr8+kBEI5JKT0qYoctKKX + + wadwnQKBgB+4kUNjyfhbdVY31Rugb2ap3qNDDp6weMAFkv/LTRP7FpBw6qgDHXcC + + 4YjudF+rdaeK9rW53Fs/3DLDP/fp9OcmUwa1zS+Cprz1upq1gPTqe3Q1EYJz4cWT + + 0pbLs8DGeWMCWN+gsYFw1JhFOpwZGXbwhv9POi7mxhW7WDxLS5H4 + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 4096 + seed: Yw== + zfec: + required: 71 + segmentSize: 131072 + total: 255 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:MDMF:muvykqatscebumrd66c2doacmu:evblp5vn4guj7dcnevua5qxeounppfqrorxe7oicure6d3bl3bba + format: + kind: ssk + params: + format: mdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEpQIBAAKCAQEAwbNQmkA45byGaXBRjf9jElUoMF4hqAaDy3QrD5IyKu9AooUq + + /Sxnj0l7NRYrhVH5K1Y2J0cmqINQRiOIQAjxzcb1JYbn3MRJJ3KtI02YI11BFd6C + + G2ayuykXFsseQ1M05fBK3TtSDLMdblT6EX5FAx6OgieA/e5zOVR7ZsSwRTH/j1sU + + cvG/41u2LFbcT/SbuuBf8eBeZzjmVaACGmQ701o1dQJraJjpgQ9QbA4NUlJcXRou + + SunYrqtqRdqT24+V+SMPWDHVDTGkAcdlMUi/vz8UqGpNmONo8IQBqr1R6A41liNg + + jf3zQgTWVYsvResAPWbkmHWKzKVMC/gGdhH2VQIDAQABAoIBACi5/psIIM5xBqPZ + + uVQNW/PJUuNkj1gIUqKvALTL7N9pIaJqNIE52mZesViWmjz0YNrzS/yTMbYhsfml + + U+r+1nSJPhcPV+XroWP5cRzonjHlVB94gsunGrJOb+vbdjf6oTctgFgmtlg0Ot5t + + YIzYC0OeI6GLE8yQW8q0kCOp/FP30mlLTYzDYwkia/q4FcjxNRR81x9Jk4C2/l1C + + XLlthe5KW5TlPZ+xeZDksq8dP16KCTmYdXAZIi7I5WLQTsw+MxXnCr6ybZqCqUEt + + wzJvKOWVNhwehdVJwpfHZFX/IEGJPWgSvYda9jvEExf7txiJVkifFOulLyVZPJcO + + j1+/61ECgYEA+4Srm8nxYVX2gbKzSAAjFScwqj3C9UzS6kFaPGfmDguXPhZdhGO9 + + 6j8tQspirKYN2Ix8/NxmKe+MrTwjyAIdnQ3f96dpR86kIwx56d+N4ceK0Yfg5fSW + + HtQgecMh5TMcjjFe/NqEU/17e0eZfxSQbVoVSDKcZos5c7uOgJQxnEkCgYEAxSbm + + 5LN/1sWynmWXfntUO6pc2THxMrNMTHaAUBdQFrDghHoFybtBbuM9ecnpTwvKMxlT + + gGXWRG8CRfTPUU5LKlg/22/GTrX6hDP+sX/Q7pTWbJZ4LwJhfID9ngr4v4almGUl + + MJ5CQdCEu3cxn8PO1xu7/7j+WT2FogEUeQLWka0CgYEA6GP/z3S6Iy4zEkkTnz4J + + LD1GmLVyEgYGhs0VW+S/ylBpUMOHapBh5DK1VhX7L/xJpMDBpzzY5HxiZZnAkcdq + + pzcvrfovq1pBi+S2LCITTP56w/ihErd3kUp8KyThh40/IB573nLke1olIpXYPHO6 + + sl7edRPWMGUJE2bDVwgWAokCgYEAwJdpFN74skD8ZVnO7SLjPUoGW7I68hFPJp7Y + + Z+TuOsxc920QPGot2HoqMs/4l1xoERTbimFxN/bNXLNy1vVJ3jrJXr7JFVkWOZFl + + a9X1rys8cGVpUFreCrcjigEj0E1jdQTRmLXw+cQN9efRVUX9yAry0zPPXDQKWCD/ + + 89q+6x0CgYEAkmAFLhtlILXl51kd1uxZtrYCLvq6rO5Kq2lHsc7cnVjX8zGlLTX+ + + p3CAChirnYbBXlhn9sCf9OzPGMQrlW6q6NEQ5EJ7mc0SnZI46jQ/dJgziYvNnVqB + + LOkZ0bTT8fboKAOtaev/ubjKJ95aVdJTXjiuGwld8IbBUUlmt2IJKs4= + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 4096 + seed: Yw== + zfec: + required: 71 + segmentSize: 131072 + total: 255 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:CHK:6nvs23bhrpiwiz5prqdtvztujy:wlg7g522rpdoitpm4qwhmctrjhnh4zfloiq6uq4tsvaoawg4slpq:71:255:131071 + format: + kind: chk + params: null + sample: + length: 131071 + seed: LCa0a2j/xo/5m0U8HTBBNBNCLXBkg7+g+YpeiGJm564= + zfec: + required: 71 + segmentSize: 131072 + total: 255 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:SSK:pvkyxg2paj3uake66gidgapuhu:kk6x3uwm4tkx7jglftdiycsqk3hbarvi6gunbzrpdyucyn3iea2q + format: + kind: ssk + params: + format: sdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEpAIBAAKCAQEArWejWyCuXYhCalNiitA9iZ5EEFw+JV5x4HoaJWasXjxKdPNG + + qIWeqMenRSweYgvdXMEVzll+meFCaV0evEotu+5HBb/zrevcJJ6ISHKjwMoAh3Ri + + eCrWB+81cncxRghY5W6w6U+lCkFGv8F1wPafGm5RIhTp2T1IFDRDQ6mjzVZFjt7r + + 2CCDFi9aBt1V3xNefH8V+B/KFaeS26ECwlM2mcWWjID7jP2oHaG6i/cN5l4fwLjq + + BfZGJ7BY1YjZDiOIChwp4oLXQME+43jPW7mSJ/h0G+T1YVaPnc8L/7iZt+DbtLOL + + wAzmrBjIyMGRj/FeljNvt/tt7Cp+kMIhH/AslwIDAQABAoIBABLwfOEZIrJIjah7 + + TwoGUJJVXO3EhW0jcaCo9W4cVrs8Lo2zfIYvgfLBS728Yd0nmpfk5vLQx4kbF1vW + + teKu32vlTJCONJlMZ5EAV7ZB/yyxY3ln9tFVLGdVcyr7ZcBWbQ8yFdSFxGroUkfj + + Y28eAKasYeQtEJWPoe3C/43GW4OzuD+c4NZuPNTX3xs146e9NgyJcCjWrqvSj8Fq + + rw1lMQ2LFC2r7RYsIG8DjkFOy2NVrmu67wavFJc5dPYZrpC0E4dEg9QZv1Z3II+E + + aXUwTUQSIgA66Vkgv6rfFbvuWssq05HLb5wL3v2xaV0pRfqZsUigi30goB4Dzdbp + + mHd3tSECgYEAvVuUW+Uv151mV0ee49E4gYyuauYxMWCfr8jOI6iCcaLIbyAglMNX + + 3AiBAC/wfWRj9FPjNpAt33UWWEc4c9scBIdQLZ5dtrL7WH6QzY5SC2Qh8Rk/6ec0 + + opmEP8XKkOi++zbZAoMN9/1XAw0Bsjxg/qpd7kCJjxeZfKpAc/oTIj8CgYEA6m7D + + Nuu/swPh+da9vCu5Caao8fHbcccVWXQoeWJ6xroKwRxndKkhYGQ9FE+vawWxuFtn + + qdFDhwR/9Y5vIPKPMdlYNpx/aoGrCkdCPa+V7pxgM4eDAy2qTU657mFpCYqfKxLP + + r2GjDYbmtvpis6CKTIaOVLOYmCMtzYQajf67L6kCgYBb6BQ1GiNDcrkWicOb7ZOQ + + hXiul/WucqhvCHbNJd/SSeEg1qYZrkp5mIMMVThTlCNTllfExuwM9maXCFJlIScT + + J54J1kDECVEnXZ7otjgqITq8K7Yy0C5i8UIYNecguGbhxXhE2Nkx0XunFmwJV3b4 + + hDY9CoP5uMmdkYDhCbK5vwKBgQCqcgerCbKiFEObY58ljeCF/M8+warcSXPjSwoW + + XxyHGkKxbtZlQobKz3Z9KwaOWGCDeqmfFU/0fbgPMBTqLpEgHDb/1b7qEAbsfSzT + + LrNi/f0P2gnOKMh2VrPmdppo3omlRpMLn4BuWFOsW9WhZirHQtl/Cej7TDCECPVJ + + ohg0qQKBgQCU3ZdxbrhFetjRoQMuSKCROwV063g+DJ3zS9h1L/r4MEke5qCLfzvs + + OpYTB+LUMVFMy3G0woaAOMbqK6MXnnD45zeJQ7wEfJvZ/OLQ4sHN9DCQoRdcjQyH + + /hb6ob/JXdp7s5C01+DI7+otJUTGyZGWQZiRCudkHHTa6xmTAr5ixw== + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 131071 + seed: LCa0a2j/xo/5m0U8HTBBNBNCLXBkg7+g+YpeiGJm564= + zfec: + required: 71 + segmentSize: 131072 + total: 255 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:MDMF:beerzs25tktl4kl6ervv6zm2jy:4uu4lyc7p4p42hzzifnojvgbhsdjq6haseowvljxpfyjulf4w6ca + format: + kind: ssk + params: + format: mdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEpAIBAAKCAQEA2ZNtrBKh511xQFY9fhekwnZ6f0G/s2atWfiW5UfXMoeh9HDD + + klb1qk1Cnabz7wgYsDMUw02skm4b3chhzi1jTUFH20X1RqEuAqLTsqTl1s/QQDlX + + kO9b8f2qDzsVWMQFHHfrysRGb+Y7KDK3tNM7ljD5shJo8072QP7G8ORC+Wscf/WN + + Rrb/V3TjxORL8KckOB0iIwmflKtStvAGAYsSHIgU+3wMOffSKC+1gOeSMX8YUz/A + + 1bBJi1vwnSz0wwC6YNGRwmAYoM8ta3Sx/8Lv1yC18DPLrHNPzX3CpdFaomGfwSMc + + yyswXSCEs/YrXrM4Et+8G0NajBYGdlGBsmnepwIDAQABAoIBAF1fPJi04lBlNH30 + + xK0BPo7Jw6YrNDasYMaUvUUmQH8J4AIEBpodwY3VXDpF9LdnFRlAwq9R/TZWFJVo + + MjkGF3CHDGxYqHsoHpO5BvrKc2xtgKSfNyoW3rGKN9oTdATFEqB2AnXhJ41ME6Ub + + puTuJcs9t1qpNer8vweDjyLAAtIAT7vcGSayxGminW77qS1R8R1rrWPeOWUGfqJm + + MNfTkI1H2bUZ8QFoGz5GEytQxqD00LwZHXYWUe/PPZWuuDOhmx0QGd3JIoqyURRV + + +U1vuqeE8S+/9tmyP1jfp7L4oSSXEZerOYgXB5KX19JmXJ89cz6hkfe1VlZqBNjf + + GQO+a/ECgYEA+TXgaa/83Z4BKTL/pvQXczppRbxXrY5u6qenqi0GFCwjFTuFaLjr + + R3LwDxL69FU0H11WX6m+aPzse9kfyP/rBi2iQGo2Cl9JGPHFPwz/R4TTB4f5ZmOM + + o4t2Cwrb9EHQm35sp49P8y2XSI8aroyxqtRNMto/BszJHeuD3jqpqQ8CgYEA34Dq + + d7tIR7ajpGsy4GpHRSdY3iTRFOuFnmajZZ1pMw1KWQsIXBgQHPMvtUWFH/hZ2S0p + + 2sjcKww8TqDStVtuMF3KVp2YlLvU45l871qOf+xvXww0CW2grShvRu6rMpMQTOq7 + + eLCWWOxwl2jlEm/re8ked+ATUW2vyI7YXKwRAOkCgYEArUm0YWk5eNT806wdrvb+ + + M2bDevVLNmjbYZnw8VlbZ72FK6d2zen/2G/o02KMVEfG9aROgjijKZftzPSesIKb + + 53Dl6MqyByZYytqbIIumGxIWN59qYbMJQVOhYm5Loh39s5IGdcEmg98I2jCACi3V + + AQedIqY1u0G8+2wgBvBdtysCgYEAwUAce65JjwhSciXmdbgvK5Ib+ufmiKokfJPO + + kFwMzAGf2WH6tnZv6Dg1dg1IUB5Swb+VQwENrYME2g+gYQNPQS63dzEI7wGBz9G0 + + /thUAjQTECHjFIvftBkULkbLbA1QuND1jCNTvEukBqbB+rEe8YcyewAac/vdVBJ+ + + 7ZIxmZECgYAczaW0DADRxSmrfOykf/rL9EEKd4aOrXaiCWkREmMwoaHsDBYShq1I + + cYWGZFONdHFfI5RxA9mFdpOV8B/TnUtrc7x+x6xDpvktXq9bNAX+tniMKPRgbbAe + + j9bGVS3+s2mcO5CIXy2EvDjxuOA7VPYXvBJslkoXrbcSYw03mpjW1g== + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 131071 + seed: LCa0a2j/xo/5m0U8HTBBNBNCLXBkg7+g+YpeiGJm564= + zfec: + required: 71 + segmentSize: 131072 + total: 255 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:CHK:62dtbeowncjj62jnwbggaufvbu:6r2sapg2cm6dvmylodyxabrj63a736uouzsqyacnimgo4svnktva:71:255:131073 + format: + kind: chk + params: null + sample: + length: 131073 + seed: /N4rLtula/QIYB+3If6bXDONEO5CnqBPrlURto+/j7k= + zfec: + required: 71 + segmentSize: 131072 + total: 255 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:SSK:3nkwaseraq27epfjp5z72teznq:jqyndcqvfaur22h6yjzqfbe4cjiqj4jofutjx22hwv6ap6rqzx5a + format: + kind: ssk + params: + format: sdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEowIBAAKCAQEAodch9NkFF2Wgkcv29QJtIOgoY7hxLuij/0LY+NRojsrc4LSa + + 2NDkM7JGwZ807zMTpf6p1FhSEmTGFoOd/xPq2ZuhpF9VejQBlUdEfesb8yCSEsYD + + CohXvhbYrpQq7EIPfj1EelttDCSZCBEG4OLO6FVEqX4d88K7DnUJ+17D/9+HG30T + + qSN3886dtcO7B/MboPNwMBpVKWbTpNkPlRDq8utlxAmtJKSQ4BESCglelx3WzJX0 + + msQkNO/2EzXabml1IAVRKANknqd7kL4RqQmC2Rl5atSmw3DhOAyleBN2iHf99bZl + + TujyACe5BxBTKtycVykolbQGQ1u0GFdgRKFHiwIDAQABAoIBAEeulWwxCWfFDBs3 + + l5EKu5I1MdqFUaBgy26eyaJg1lTUtoNSizlYQJNDNcLBxPzjhyLhSpBydBuQhgpn + + zn2x8TXkEHLRBPek/ESFtej9zznfJcPp72PlYtOfo+ajWuWdFuantWJqh0C3Hw7r + + F7xYySMvzUMzSIn0qMxs+3haj36Pak1cmJi3NhcU8crs+Gy2MMsJEIv1OBxbqqJz + + 5wcGW2nHLE5rP8hkEM9LIjGSNWY+eZO7WxwfwRwg9y/Dy5l8T6RvFVHS0Est2tMQ + + uPfD9zbKR5gZiSuuK/0IVXmWEZYXGY8/bdgnp5CDIihBhp/lWmEyOuhe7ljLaK8y + + CsqPO+kCgYEA4DIyYQCFyPZaS/6CHOh7LyrF9F9vOcoFSNh0npWG0gmUIO8zsII3 + + VPe8W5Pf5J7brBiC9/a000lejsy71XTR2Q/kP8tdUjOtT2vObUHee6dp3LUOHEHi + + aGvBDCJ4Reoj39oJ5m0H/1QrE1YAN4xTUMDNxrjuC1CDopibCbbRgNkCgYEAuMx0 + + LfG5fybpybc7YR+Hd68uldhIDH5uWiepg4PKA+Kg8sWpc9D0LxpwFVQBixM3LNue + + wn3SJNac4z6+CfSSCgX4yufs1PqUT3UIKMr5zuYTDefmAXHTDZXY1Hati6zwvcSs + + GjSsb7RznNaPhlCfgionJ2c9seSAb7NZI7yXzQMCgYBjiAVzqQ677BqkWEYdXVyq + + 0Pt3BRNU/YohD++eI9Xp01TO1kMFXpn//8fAhELGtXviyDMEsKMQlicDkILnPeiX + + zAVSCQ/SGZ0cgEjxmmeST/2gfUTZaKqCHyxiHb91koAAtkTk5ozBXvWMrQaFoqeu + + VxpD2f/cSA9YlRVnV6Fk6QKBgQCEICIcy0xOHftfbrN00H8h0k2jczyoOikaKmtn + + jW19c6aRjUOHe+lqWCO1DBgCYJ29Y9TRx/XcwtjvHOfw5D0aD4T/Re0tpW8ulEVe + + LSmIhTUwZxIrDD/S4cViuuuABwklFR3bqrdzMnjKtRlu9evlu8+8u3L/4pj1xCxC + + gc+jEwKBgB2Z2Nv8mCK+ASTIujUAvTybOr7X5WK14sao+6kELQoEuHhWOf2tJWnn + + OyukYwkK66uYXiKPfxBlvgInhlU+QVsCEMjuitetNPsmq5CwSTuxK4oE8Ym1L/I7 + + T0voyKT0M4i5IvCiIuE+Larc1wqILLdkyo/uVQQYvgxQhiJAZFoW + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 131073 + seed: /N4rLtula/QIYB+3If6bXDONEO5CnqBPrlURto+/j7k= + zfec: + required: 71 + segmentSize: 131072 + total: 255 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:MDMF:5e5m656gdenkbp2zq3n426riai:umofnc6r4xbnizpkrazduchscmzuv2id4ejgkkjqsr3etgsxhi7q + format: + kind: ssk + params: + format: mdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEowIBAAKCAQEA3YXz6bwSWmLFCn+Ekx7RXV0ksXMdbJKfMUa4wyjhOpMYsUb5 + + XXMGK1ajhBsZiBX/akLBd3L+pmtHqK4W7x6NXME4Gl+OMs3SSiwFn43Dm8bzaL+U + + n9rOCZXxuPOpQv5mNW8c2/DBQ4PUqSoLAiRWXz2vlpcwqeBq6323ug+U54/ZaMp0 + + D4sGN4w8OHkZpOY/dQfEK9srzhoxI9R1XQBbXtOfUvL9Kpu63CjG0RCdr1d9QNrI + + qlocK9meDvClqRCh8bvUfQPEXdbbdqlLK9dHCtF9kaR0q6YvQcV98uN5EiqViKPK + + Ip7TY53Lr1G8tiVsLuKAzFQ0eODqBIpvaj0n+wIDAQABAoIBAE9nehhgwUVj3RRX + + xCpGJC7uub3fsP7fia+MlaLi7uTjoDi/Y5hDKEV1n1Q1sI+uruikeBu8fRojH0MP + + 8AmTboF+gwE1GlAMpeHPaM6Z7rFSfaKg9YHdWPhnpocw1A2/Cd0CcJpH8MamJR7k + + AqEobEtkXaHBnQBvgHPcEvTfK/VaWEZEb7cF440CeNTjzZ+67Nd4LEEsEsQoZT1N + + XoUZuceQ6bljNIVOgQMjCPiM1iJwELMDXKBjVa55TV3CUQjsunjwQUfr4juOPB7J + + KLeCp0MWaqF8AxvKDJ98KDQu8digdtgNp1fX2ERGGqcvD5HW8vkH5CMevu/Ciazo + + RVlG9uECgYEA/FRimmOq8Yfs8LDEvRpICz87M0L1i0D/o4ewMdnPHGyEliQYssDi + + iNpmddLf4KlHCAlaCBDhgPX7HWK+1oV/Qy7Oz8hDPHBtmgixHXj2PEM34EMDpTkQ + + MhNRAbM6r+sf6KRWD0xeMTz7IE5qpsueruIu0rkvf4P03gpuxb/x2RsCgYEA4L7a + + IhdwQ/5Rl3ttTcZhMXRHs1kEtSlRhuqrFFDC4HCQ1Qjw3EsDilt3nMjWiYpm7bMJ + + eLo1MtzpnUteJenfkRQ/FH6k1q3jv2e2EYVxb+8Du4bzXSEBSvIxpKMCwA7GYA7y + + IEbh1DJYpKQEtptuf/fnbNpO9ia8LhIShx6SuqECgYEAv1DT+i87eyoOMmg0oxR8 + + L1rf7fwE5HKB4WGN7B4y9GArHxN7Tn0ExbKiIQ+kA1kVrDg69Qank/ntTdiCzXAm + + j6+7yrsSj47G6xVQBQKj4AkvInBtISbk6rLOprVX9+4UIXYIckz61eZgmZwbLSAR + + zpNb4RXbt5k7XecXGgRwwKUCgYBS/gc9OZyKbzqgDsMhSlWP1pm3n/K+F2D0ymmc + + meosyUSidqfDIaxQBlDYQ839gm9Z7ZhczZ5hhvR50mAU7hVR1MEqh03FvPbyMpEo + + TTfDluaw9DegN1Tr4R315wBX/dzBkiNVSfeQzXqwaaUX7bPTa685IjCwc0NgW+od + + nWufoQKBgBKSqG4fgLA92xcDwvWdk8/rxwJvnWh37u/lX54LGqQbtzUBdFeGCXLN + + E4ddAUDjpFC8PFwNGOwoey5dCqYFvG6HubVv9V86lPC8FT5MuB72i3gslMTE7+2h + + 1BbPPB/Is++rzW0kIpXh6yyBdQU+dAZWv9Gg3WbzdTUwhMKNtpMr + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 131073 + seed: /N4rLtula/QIYB+3If6bXDONEO5CnqBPrlURto+/j7k= + zfec: + required: 71 + segmentSize: 131072 + total: 255 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:CHK:hk3a5ync7y3dnwowqjqoa34eae:y4f5b2xqa3lslh2vti5fdbho2syqhbj2p6f3enlhxsjbfpt7zuta:71:255:2097151 + format: + kind: chk + params: null + sample: + length: 2097151 + seed: uqWglk0zIPvAxqkiFARTyFE+okq4/QV3A0gEqWckgJY= + zfec: + required: 71 + segmentSize: 131072 + total: 255 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:SSK:6hzvdbfv7d3s57vij5kqamxfha:fygqhkuaj6e4cg27ocjx37rej5yo6cvn2x72dup2jbamj2rv7opq + format: + kind: ssk + params: + format: sdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEogIBAAKCAQEAnq8zfx/wNk/vmIqwnjyBx7AAcEMhCjhs+IcJTmX+uH7rNDbS + + ZA1wrioSBatNjZl+T7SG6OrSLfzuzMaLbhzL5gBVN2z1jWLn2jLyLEwYfJG+BrM5 + + n0nYtNd890a0ATM0dWNKkkJ7zl5bidwFZXo/YlvDBHbTkiTr01WiqLj1PzSDlixa + + 1oDTjAiQaJrua0Qtarpqgdgr7YAVow3Vo9ObyOJzotFuHGc4bLq0PL2SVVt+TvmO + + axvWjAbtwcwyeFJ5S7pjSIslbY4Izu5IrRWf2xFLXaTievlXxfxfkyxCtawDeWV/ + + kE5VeH96v2FYsK1J0gNmQlI/9b0uR/OZdekNKQIDAQABAoIBAAWNjsMByDDINrYP + + YLS1KuJk8itKThZ0FmwhjtPP+BEvg4NC3WaLOxdL73BJMeAuzYIldM2ZfIORoAfT + + YZzI/OGkq34hMYX6yIgerEgStfCnP8DMk98r43ZAnW+qaUTvoo3a6kPAAf/h/br7 + + mIuEZBqD94kadPIjhdoNFoBAsou9URBFuEKx80BDXAde5AfeV1skUy8Dgmcgai0D + + em3Kw4e1H2W4KclFZ653Hnu5tST6N+vm7oVv4RIHrtgxllxedB9lVnV24aErqvrs + + wuyluWCLJriax/Mi2fRbskk8p6ypY3Tb+0ivezE2z/cJu53/605T6M4AWKq+jNAC + + RLce05kCgYEAx56qM5G/V0Kgzvgf5ClbxPoURb5ACXMLpiKBYjtPH/I1EX6fB4Wh + + eQCnSwDc+xlDjUOm6dFxm2mf70omxXFuYYcISlPhdIfn5opRZwbOwJ9GVp8p13Kb + + g3WHIP6acDUAapsiAjGky5JSC/RtNl86l3sgA1+VzICCrzz061waaZcCgYEAy4C8 + + xkem1rskv286Naez+qQipZohMKBvuNxlFWGD31sYXnDoJdLQbRdoo3WpKWwCTl3m + + Kh1Wl7mUSPYG0lGoZUmbesDJql2PwfJHfwnk/+JY/cUhdSeeOiLeL7CpsboHt8T2 + + LWxMViXE43MPlubwwZz00/au3HP1j6xAC8wHlz8CgYAFsiNNIWWCSeZowW+3hO6X + + akNV0h3lpyC39tgWQ3b4hGK7Qw+qmUeIOlqLq1Si3Y+t4jZLCaziMFtd6pG8pIXv + + xniYFliiiJY3X87+z5TqriDFq/j3qs+BKsNWT618cia25AJOabg4Ds7EhI7xNDpp + + xBufvQR7N1eDRIwAgzpFtQKBgHZhkW8Wx2squpnSLl6ADCbFzJHhM2WCLvuu2e6y + + J3CLIYXu0F0QYcbUUz6jd6BtAHpuDTJ6lqD0h9pZpGY8smUZiKTD+YxtmO8N7aFt + + NBXWqkYVovzv6w+OsQm1D0IgIdU5cqvB0DZdCkf16x+xgGRg1dtoKRh9LGBDp441 + + RkUpAoGAExnRJWFKcoi1gBGSqV6aigqdCRZp0CeuXVeEo2kmLg4MxUqm3kX7bdTp + + ek1ILGYhhgJH+f7R3lDKwnngc/+vlWejdOHkEuQWPfsG19seukueSMgaWjkDqfgf + + M8RVPErerYnUM7mIpyiQXndCUxwYmEhnliFF7DeEhI9doJeMrFc= + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 2097151 + seed: uqWglk0zIPvAxqkiFARTyFE+okq4/QV3A0gEqWckgJY= + zfec: + required: 71 + segmentSize: 131072 + total: 255 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:MDMF:swtqq2rygd7pm2rpwqnw4ge3hq:bdl4ypfus77ukcuz5izjgg6k5l7lu6ncjpxg5oapvgsdlyz37ymq + format: + kind: ssk + params: + format: mdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEpAIBAAKCAQEA2PpVfdw4iK4vvihOf+4suynwzX8Nqaxpx7G943fvHi55fkKY + + dOsUbNXK7dK0vk7DYpMw4ydPLogEV6gOkU8Z6Aw2BAp4uol/C4TooP29UAnPlMka + + AmhjIlloh01N5ZHfNdcoueBeuovEmYsaTQn/zqKzYKRm7RESZEGhaxGOCXwCjhwu + + pVZzvNnTZs4OQmkWWrMoOCOulceuzvXnHLvOvYqMs3Tu6CHXsDQjGqtfgsDL2vR9 + + 8lmUDatAYWkIi0kbs/2bEb+Ckl/NsI/XOq3K/LYGthZS3taCpLBVBmsMMc9cDnNa + + K/i3dMT4fBPb2l63uDmUbE811RBaTEZp8PCb1wIDAQABAoIBAA1wX+Lbmigjux2e + + NdJ9qDPwJ+k1ITMCVKDuP4fQZ1/pWqlj1Y6JpZUMyYoLvsR6G2bxHUTh1pPYD9t1 + + XhrzmR6tbEZkIaXSxgAq0L/GnOGmtz6MwCQoXq+DddJ04B9udrKegM1rz2I4LZ/2 + + +io4YCmRG+bJ9/1fTSd8MH5xsr2f3USkb5f3gw91oezyPD3qoerksFKqwQ86bswG + + 9gsehcLZIIYOnSLpgIE1faQznV6P/aQ4PXdve68qKLpwgARfQwl0mYNRKnaxLLM+ + + VYzljc95kERweCcbF+l5BKx8eNmWiO4opXLmrd4ODwPX5jo5fQR25N+kUH8nt/oq + + CWsv6OECgYEA+5t9nX28UTCe36kPgbycXWkEwU46OF7b9DNUWak21H/ACiCdh0Kr + + Pv04iL2MFXPND9pIRfo4q2ysAHgI6lMyz0VyuHCXujlOFKhd5gCupIXCw1m6QPFA + + rEjWZ7GYSkryMcfjEnQV9pOIZBDj/16Kh6EberGUjzhdq/he1YBuoKECgYEA3MQS + + 0P7KfsZVoFHGFYbDnqS43jbcCQPQ3wrQCgD237AQs4tL1erm5zxoCQuB+W/aYQk5 + + zZhwF6rrQ52o06EWOF+OMsL1Z+KcJBKWpwjW/ATGcEvCEJahuXA3QddMyspdZoYp + + dqMjnI82RMVmPd9gkg68OFNBcATW+m+nPFIRUXcCgYEA2BJzMMHe7DZ96YNNDtRD + + 0DA05jDg7LIB4FgIUytvK9Q9vjS+M398go6Bc2ScHXwiGUASmw3Ehuq/V3O97EXg + + t4FjgKMomcNGm5TvdmsVj7JTTOIMgmLscEfo4InyR7LPBRMsnRdWGTgfhBfBRPgS + + rWEcsSQ5eTklsF6OSnmOB6ECgYAoH36n+1bEObnAPHx61xZgk+GBiYjuHoJstyNe + + XhSATRiL+SocQ+gZaLIjyrKhqgGPl0SpKCZfNtIxZMsVQ3atYjiO4z4E1nu4VqSI + + 0SN5hEioiixIJYhZEpsIXV/4j1TwWDva8wV649BiKVpOrnV3tjPhLMh82nRT6c0E + + OoopOwKBgQC37/FRGYrrg4HQYAd6bERCK2NqnP51qlvygFL59jPyCGQbp9BH/7wq + + uApjV4XdIhno+PTx+S7BaLM+rYXkZqWMFxOYHs0qlFeOPj/M36ebR5WOQB915Ff0 + + tgo9VkcVUQU3GyJrk2QzHFZ+C+oMkCHcJ4oDOOWXaJ9Holp2KuEMxg== + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 2097151 + seed: uqWglk0zIPvAxqkiFARTyFE+okq4/QV3A0gEqWckgJY= + zfec: + required: 71 + segmentSize: 131072 + total: 255 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:CHK:qa4hwhwtd3cpvut74biixlv6ye:uuj4ujg3mcf3lqpuvvwf5pszcvviyb4cm2jjicu7ugiypojjie4q:71:255:2097153 + format: + kind: chk + params: null + sample: + length: 2097153 + seed: BTBX/ampNfLU+ox7xipBGiaSbgC0kcB8Gy7BkJB4oKI= + zfec: + required: 71 + segmentSize: 131072 + total: 255 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:SSK:qdvgmo6nnk2jmb6btg4kbvumpu:a4xn3lgnhxsv4uj6g43zves5h4j2z6dl72bkxrphat7tmm64vwxa + format: + kind: ssk + params: + format: sdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEpAIBAAKCAQEAyFl7uvuYjGO6H/LbFpV3AKldXqGDmypEPa0s4/tVdKmV72WL + + bUFmjx8n9JQoS9fC+TgCMHaaLssCuizHmIpyKdJ+setRz4VMWz405DA8RFKnzrKc + + UuM2I3Sr8TSMVez+D/w5OLxdHVcL5IyAJi1ibnxYkniq6RXVNfyzaezxLHBSUAmb + + aSn2MJwe3Mgx0VxLd2Eg1RDk2DOFbzrT1cZa/gdfloC+b6DJXtqbkyKsiwTPKKBW + + NoYXzlC7KGv3sV0VCXSkawrJhRpg++eMNVN1vaVqShnI7xgrRxoXF/TUFJygmDtG + + 8n52G3VoEw+88FuoH7uEm8Hn/K81eXZUlWmNTQIDAQABAoIBAADN3DuX8QR8sLY9 + + HbW6ocIlVJt48jbILQ6UUSJ9qrbp1yhbgmQSpOaCnV3wtzQk6IKRS3uACrrWtvDc + + RFVeFlfauJHqVjLcn8lssc/s1ov/HGNn3JM4DFYr0nvxZ6Rt68RoBYgMLMXdsVlM + + zuKvnX41CI+c2b1PH850WSfSRq/mA1IwF/awMMc6ZDGO3lrlQ89nOzir+unY01PK + + P2rfZsoZUaJCXCMvtyIryFu1W8yoifhQPMa0pnUO7dVqykySQD7xxUHx8hEnOtFn + + mzkCYyYzPz3Tx/7X3pQblYmWFgMirsQxPuRmKDagmt6vb2FwGF4NlHVbK49TTWGw + + 4ARheH8CgYEAykyIT2T4X3YowYs2Z7skp/DuGimNRmctPIfafxhA738WjYWHS/zw + + ODwoJAg6K0obNkFdy5G97SwjcAR/8YohxZ+f8E+5iX0xumBy8yyHNCSnWQqekyiG + + btt009dVUbbwCWrdo7x+x7yyHsdU2yhYxEBFjJ2n90ufuWTU7xUU7esCgYEA/Yh5 + + 8Fkn46W3H5zjlG+A9ehwNNCndNmW8odxBxNXH3OwAggiEZy5uO1Ae9egIBep8qve + + by8BhCSy0VQQZcu+sgExtuesA0kilcyLNi9D/AQaWO2e3FcAiZ1BA0v12kVJawR6 + + nngseOudEXOZun1c1jEsetHT3KgU8hFIuEkLy6cCgYEAuJsAZNsyIAL2jC/atOw9 + + NhgRX8R9TDrJOAyNIh/i2eqyjPDGF1y5ZcfXpZHwayKUFH2v9x2HINB/gjBJBQTV + + br8Mt8I5ALNDVt2+6BPBSZ8NK58aOBXqH22afdpp3EjBYQapPUq8ss6KCLZDxD5c + + SrKQBRK1fWEAX7EY8xfc4oMCgYAHsssxFyP076U92n+2lCQwU1yE3gkXrTu+JYqz + + Ek1E8ThY93JBYqbpDJs2p3d/QfixG7LnYWAEaTDc1lahIKyrrwmZajN47hGUxt87 + + R/gigOVj6eM3AZVMmG/O79GJTS1LiJlIkpGXImBklUQHu6LEBj45hIGQY7IvH4M7 + + xUwMZQKBgQCoQN3iNLM3Zvfb5TA/nqbMQG54zimhqO4ahTpihFBpxxXXhefeLXkN + + hHVsNIud+c5ET9pgNAUbWhfKfO6j10YUvGlh5aFjOliGBGQA8sg/QtS04mxzRpK+ + + rPNAU46OgbFDCeN/eaCOs3nPYujL1DQkZ5XZ5YUU+k76hskwhKTbBg== + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 2097153 + seed: BTBX/ampNfLU+ox7xipBGiaSbgC0kcB8Gy7BkJB4oKI= + zfec: + required: 71 + segmentSize: 131072 + total: 255 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:MDMF:dl43xvmgtpakgi73zcvbgc2ngu:cgtbisxge4chvp2yeop5vye6e4e7wnrs32m6khaq4uplipbczvvq + format: + kind: ssk + params: + format: mdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEoAIBAAKCAQEAnY/8dSHOzMGDE+CG0JqXyoZnSRKczgax+YHNHqy7BAUor+V1 + + Ca+YKZdlEK1yirkbkGHm6Pd3gzytM5D9ah6L68eLg0d/HCWS5z34j6SvkHR4Pbrb + + OjmzKgCtyLkGN/MUoiwjQRJyU9IxsyYJ0bpdppiGh+i+4JamgyeBkFRE5ztsT0YQ + + RBPxewMI4sezFbiUCx1X2k8n4GCIAXG/NrZc+SfRlwp03KJQbDCJpEghU42Xo/cH + + xYDYXvL1EVEnw2KWHEoHQE2ZqlfVasYF31s4fk2+8vUkKeE7nAODSHjHY8PiFWGh + + lYLzUjCbjHmfUO6xwpnDuJSNMFS+OpbOIUowUQIDAQABAoH/bSs2YHHsNzJc/4ix + + 4Bc81LYLGjYrLxS0e4vT80z6xu5MIpN5ZByl8StUexmyIyveTUuIEiJkTCneV7w9 + + 2SkRCWxY3bzL9VSTVGU7s0sH2a7ZIOw2uUEBQjj2L/0CsgFaaoLqaku9qxYYGWhh + + pU7bVHKZw9Efb7zx4i2dN8Mreoo7CRt4hNLEtdRr2v2WJ9/Muyc9c4w/KuW5rA6y + + TyO4sQUhcQPNV3FQIE5Roza97xsepJPMGd0mdwZMDnl+dg+nuRY8z+4gN0vrIcMe + + Lbo39qj2uH1OndfTYaFKSj+oa+q5EdY3fWhxSyKFSkKrwKp1majGqCrKwa2tEdCx + + 0wwBAoGBANIe2vaCRd/ydkEWE/+rZIXEal/LMyo06O5XcDNL6LyEquaTrbYQujPI + + i4vV8LnE/Nl0UcjakUwiwclspoj2Uc2TJ7Wt3z8v90HuPEZjm1Ppb3KEweBFlhoh + + FJ2lwagDZtbG9aL1IslmTmkDIGEbwzOVRxbrahHsLdmhgViW86ZRAoGBAL/3SEXG + + bpQZ0VOHjVuGh1soo37ajkO340dMJXUEejvZtaSHZFTsY0nSjz3AlUc7E0gFJT3w + + S7wcMBIWp0zOykGKaI35Yf6ekizBvBUVI86oKrdnl7Z6iaKwlBWPLNL/jnKzLyTx + + YSnPoMv+BzacmWqrcBRw4PLOkW5u4XscqWoBAoGAE15GvrxJZpg58EvxsfqBfJcb + + WxMm9zgDVJz4ubHAlUgBXNm2BHdMQqO0wUIKO4V97Sl8tG/5PrRheoiqXSufZLyw + + x11sm613NDuakL5zvethm4PDP0IK0QPFm7aAwFT38MpMMCY6e6gTiDiCjpD5kFKt + + R96RW2+S1mG9w4W+ldECgYA0USt0QLlASaz/69B9ojNfh7rPRrdBA2vAsaL/ukGp + + 8BKODYwtjOMeanE5bjQA3rvJhAV7VPL/CFudgmkECNOceyE1mEK5xvOlmQMuZ72D + + g9dodqYlSE4cda1WFtgrhRSIdAckNVi6sWhsUAYdPx6csK5yE7Vq1xtRkoyHJe+S + + AQKBgE8joc9ob5a+jxj4E9Pse6tjqum9E66i7IRZBrI4gddtYqKUW5f25xfcYUHo + + SpVkuyVGIWLe9t7Z9jYoMis9/ykseiqzMgeuKCSZRG+h76kOvCrRIzLRJwKv4MPM + + Jg90SP/U7CyTvbvpN6oUbhDXxqSRdyRVuQqgBr3aT1/AJ4ba + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 2097153 + seed: BTBX/ampNfLU+ox7xipBGiaSbgC0kcB8Gy7BkJB4oKI= + zfec: + required: 71 + segmentSize: 131072 + total: 255 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:CHK:r7sv6u3ihzys754twjltuzzpza:tpyoxinubkl4kvg6hipls6ezun7t3yyqj6ix3vp6tvnzqirzzl4a:71:255:8388607 + format: + kind: chk + params: null + sample: + length: 8388607 + seed: w6uP8Tcg6K2QR905Rms8iXTlksL6OD1KOWBxTK7wxPI= + zfec: + required: 71 + segmentSize: 131072 + total: 255 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:SSK:wnyhczogkxbycoq42bhpklu32u:svvkovpiwd3oiabf46fcksxoawo57m6nh3ikefmqyoovgozio6hq + format: + kind: ssk + params: + format: sdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEogIBAAKCAQEAtVfoulOwIeSHFEK1T2CUkNg2LeYKROzkIJleLBSecaIYnRh8 + + vYcqjiNzLy8p+dhY7x9vzy3cgxE17S85kbe7VZamHbiXWT848Q7lcyhApzOiFlGs + + 4KivxU/oXCyDUmKx7TmfovB4EMm4AFtWq38YiztRx9pbJ8K0p+KClFmzdH9alK4z + + Br+0hrVQXZUDMpS1y6z4+Orz+RP2V+b4CFthawLD0SengqZBycd7TCZqFaPRpJLz + + rRnA5FcX/LcdAIeJTSskQuzRjRMHOTumj+mTyW/5oUKQPMZfzLfknzN2ipYKXis3 + + VxAyxbUSuYmh66mc1w+3wBN+oS7Mhj16yfbpqwIDAQABAoIBAAJ8bHrKVSOXJHBC + + ABSJ1NwDCQP/n4+HOaFmHGic9fjiKY5xVX8dcF5a6vJwNDJyM7l40mmUBrU0Y90W + + /cJiT4rtDnNE9+VO4FoS/KZHRzZKs5V8FEUWZ17qqU0KMBHNu661iYYMUeO5B1SC + + z5oHKZOkZadIaCjXKO7cRE9XcbDOeRSXa8mqCl/CZQEwdr2Zd+/YBaUhLdl7FZ19 + + zdz3ug2wVQKRRwyJf8wa1OIuMrSALwsTUYrwGmxU4mDo5BjkiiUnNjXbBRM0PRT/ + + sUo/5O8tPkaHLbiLN+FvLIGHE7DX1JvYRxafA2CNh2txyB7/krGlGZ7dKgc0S1wb + + 7c5A0ZECgYEA9Ptq9f12lsL2tLcO1P8fgFpEExt/wGrRQcP+efRNCIg5fgT9fwZK + + LWCBtNCXJFAOLsQNaCEMixrA0gJMfDOiGl3h5AcEbavvoJYsc1U4i1Hi5B8uQJhQ + + r/pmFR9aee4t0wAOwDOIMQjkdMBw3XnQXZzBamz6F1sU0iTbud70qLUCgYEAvX/K + + wqW7tpx1IT8FmVG6bBfzhdx8x0gXBPSy6xWQ37uLhvM2zf3x3HyIS6vhdkPoFM9C + + UFVx0QyWEi2o9slLps564nh4bgKTlaWwI4rlcNQi/YMO26g+ms22ocXe+LU873dQ + + Hw8M2ege0wKyD4JV4j6bYYiiPE06XQsnVYo0pN8CgYA6ZmD2KSkHAY0cQXNItVTG + + HT6TK4AF17Dws49LdUCT4x2JfBkOGeq+7H2fJAaTwn3PCi+D/jTmSEdlCOVAynI/ + + RNgfqsiUeGNUbdhE2jDzjV7AMOquvWCmwtNo/6Nq46uK3D2n9eDmh48mgeWl9m8E + + keTNwRLRVIYfHmg+4/aA7QKBgBYVDzaxg3dbMhcGtgtQx82S2PDvaab7UptkPHlC + + kRhRTYgTTX6hqg6MgIF47RQQA7pxEIQ2AMZglhhWM8tWV7d/djhv23DOYg5dOXJa + + 3DPStKUgIZodN/ZoJHEjksEetZQeLjsAUPoPn4/tT3yZLpLnwsmR1335beSryRCh + + w1K3AoGALMPx248bmEuDpoFE6eN2UN+KnPklcgnL1mES2S6pTrvMVjj9UqXz8tUw + + MrkH8MzLAk+bKpCCS2lk1W74elMokpwOO8fpE7CgvXnw+/+Sx00F2LQWd3CDTCLH + + oGN92BX+ro+XMH81GV2ZtXZ+i14S/w5BWtRbWSaeH1ziSmQHn6E= + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 8388607 + seed: w6uP8Tcg6K2QR905Rms8iXTlksL6OD1KOWBxTK7wxPI= + zfec: + required: 71 + segmentSize: 131072 + total: 255 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:MDMF:w67agpynf3ekrgn77oodf4e7pe:tp2kckjjzwurldnfquf3qbnrydtcxyhs5ctpxgd5ryg65orobgga + format: + kind: ssk + params: + format: mdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEowIBAAKCAQEA5RF9f33SaUdiHVu0Of8gxy2RpbFERzN1iFkaTTL3cJapemQ4 + + ljCNiG8tgKApsFmKi6F+ZJVjqkZKmOEzvYwkKRN1C3/qsOs7p45egY8s70E8deOA + + X7ryuuRJAG5xaSZ+4VHpWM1WZnIBqqKyPZIuROwcjDwpsDbLU3+XQkyNhOD34i5y + + 5revRI2vPjg63tmDHZ7CpmIay94ayBlIgXQlmmdF2KWBHYslnkSza6eJDe8DC+en + + IJcCfrZ0R5wEgYKH+YxW6ZyENTs2xApvdg368JuD9o76Bd8zKsgjpfSt9mWyjTQS + + WTgxodnddDiwu7md3LCqxvxeJjbFUXEfDzBwrwIDAQABAoIBAAOtEJFZ0LqPO1CS + + 9sigJlkDC117vIB+AzHKPZpsxnXj60uReA7IhT3TqLOOV1QKOSeSmblE8nceBDc6 + + afemlOxrmzzb82HW5XR1ou8tcaPh9Gd9eN/CMQA8eKAbiS4OFgplhyzZ0PUVzoJD + + H10HswXVIUsKnFjhF1etFQEOUOPAfAtEedGTN2ReGTQrXcLBwd74D0Ha1TFEcTWX + + KibVD0lYiE4poPPFiAF0KsHnHYidHJ0WjrrydGmBZb93+S2IETff+Vrgx5dWdnkD + + g/7ICYw05+CgkWG7FMo4yTRrIFaU80Nrrct+EMDb07PVONIDmTRu4a6sLVYIcDsD + + nW7l48ECgYEA8uRPli1whnxBOxNV87Ko/K6Ji0pAsn2TgSttzaDqD6HV+wrOSVgH + + HBFcLBJ417w0rq0oowe39dSKSO0xt0O70bMe0XvBxQKk5iLS8NMszR//MB+qz29w + + XDt71aC68p+wEpQqVC6iyPd3S9ZCZ+/1Lu1Qh/GFae0ocLz7FyAGEsECgYEA8W4z + + FpbCX5xFT1V2sbPHOBT08yv3qiMwQ2jlCqpsfv+aV70MRFFi01BrXUn8l7gL5dcZ + + EWORFZKKXI39zLunghrPgWiFUT4DqnQwT9e7dwWhnxlo4HpGJPtvxBCxgiCPxY6Z + + xpcELoyu1gRZFcxT9w+zdmQ7Yk0m0tqS40+1D28CgYEAxOXt2nkVef/qRUCEcdyH + + /uZiW8cisU75L0IMbiAe/fMcari0x2ITyV4NUTDcQ06vilaW1aphJ2hXfYzCu6St + + 8e15cyoWx2VAVcsvIsidzd89WD6jkirtc+dImMIGKr7m1fjEY5+2mKF7VL/o7ybn + + pFX+7WUN2PPGz7Vy+qkcI0ECgYBYTIaQz2idkUjkIAy+J1NIVpnTyhPVfPMs5FNI + + mFYACLnJNxIidmWfhX0O7H1ee+iWEhpP+stYSXUjLqdRVpyIAAg+exyvPvAWSlJV + + EUC14jBfQOrTlsTKx87ztWtGfWQ3y9TABgF4iOl0yrhOOaHH7U0kkroJVNBLM7ef + + PUqqLwKBgHRcKRmJ32sGssOvJqfbgnR0+qm0TyXDbt0blFHrBwp61a/Gv4H9AviI + + uRAiVpWasnQTKxrfnnnZ6sl2VQDKdh6z/6jDDbW0ps+NxhomXtUIgVl6iNuQfwFs + + PK9qwLv+lGhsJyRln5kiO3g4yJjl3rV6iU3CeaNO8CmA80nNg+n0 + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 8388607 + seed: w6uP8Tcg6K2QR905Rms8iXTlksL6OD1KOWBxTK7wxPI= + zfec: + required: 71 + segmentSize: 131072 + total: 255 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:CHK:3esfsjn7csgnyqmq5afbgtinay:xsrhdomrbrtzftg6u4hgipm5tkaomumbdlxi5hpvpvawe3bjikua:71:255:8388609 + format: + kind: chk + params: null + sample: + length: 8388609 + seed: yPi3JHKKbWaEEG5eZOlM6BHJll0Z3UTdBzz4bPQ7wjg= + zfec: + required: 71 + segmentSize: 131072 + total: 255 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:SSK:rcl5ruuw3bwcb6v4h7zhalhvxm:an63dzmzwznxhpwrw4umbnwcptpo6xglkpf37wxj6qfqwkj7io3a + format: + kind: ssk + params: + format: sdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEpAIBAAKCAQEAuvOrIeIsSg11qCGYkmxbAO9lx3SjF61sjGRhQpHALaopUQAi + + LNzBFdjJIoVYGNkI/xFrTfo3YwkWWvvsquvWrgnjXiLEc2hsAIq7t0tj6sbpsCG/ + + SEyq6v4cP7MXS+koR3bY5+GSOCVLxhRMW31AjkC452AmIwAI11GNbbdxiLqZcXgv + + 72NCIEiiBNUGbtrvHN2D3L5lH9i0MNJzhw+XBIY2wV3GJ4TN+ox7+JlZDuQR8iPq + + vgXzRvKH0t/0u/bL+eySBUQyYP4cYIZNWhsETuaMU5HfXxZzAQrcFA3Qf1T49TCq + + kgMovvMOhALfc74PZU/KuUeYQRpqZrJZL4ygzwIDAQABAoIBABojGAFoOA3wXMsx + + Sz3pZT23upCMp4KXbe4g2JIwg2A/AoC/FogUIvCVeuVXJC5xJXdXrZtcyKKRci0t + + PHTW/RAe9MlD7hg+eJ8Ixl7FbQY1YhWMWkxW36xz82oadT7ZLZb0MDYXDNTJrhLW + + 4qgF1mEs/kzm3d4V0qab0byNoZNeWbAkShqGPqKY6ctzUJxQ6QGRGM0dvhyJRez+ + + wcqjuzdQJ96hTkotokysmhyrojR8FHvA2xVtsMdYU0GPUIg/8j6P3lXPdNpROtbz + + M7k7YEFPFF/ORFCniUmNVITG1CFWkVcP2kK3UeRghtx/7XDYalWmiDVXx+y9KKOM + + 7Ie5UiECgYEAxHqj2zowAtrPJP5d0rzzIE9Lv+WtHpn75vUUbwp8Y//uxJw7VULR + + eNeZOfCvPnOSTUz+dEZUSofi2ryykJfF/XfEkASbNrrPNndCtkOAj+RACaANM5cw + + fhK50ahEPapsCew2SdMm52U9aHfgnsyOIdaxw8gBtS0Y+yhXJEzhji8CgYEA85Yr + + 9S+t5jBvNaDjBUTegj+FCuKAn7amH81uHXtYUeBbZdRa7IOjoYQLXLS69W3lH9wb + + 140PdjrYrGfCkveLDY4Dfj8jwrpPx3AIBG7no6huKhtW2JlI2heUo7XxwPvd3VFI + + A5sRKBZSE+8/8MBIBWp+oGRtilOGWTFuqh4PD2ECgYEAusrlsOx27J/dw4vY4xsk + + AZmhqITQu4EljYN+s7rCW8fb1iu59OsbfslqMT1zPepeMwN5/k1GobzinZY8JV9F + + qh4NT+YxMi0UBvIHCITQWvxjLUNuiZe5UIK5CmvwxLebEyvwyOrn16HWadVeRVqv + + 3dfhFQK3LOn9D/pgLnCxF50CgYEA52vtJ2y9Es1BWvoXtZHQtH4UsFqxSQwGmIBb + + 9baSGnfFXeF64OnQNEt3YAR0+2gFH1fHO+rQncsav/F0cpysh5w8xVzHZOINmbVe + + aJfViy8iOu7ue6pmBI4SsdbScD7acsIeYQ6aJjPOlxHe9aQ4yKx80XWYfKsOIP9N + + 3GHifQECgYA3DLtshPp1N356rWpJJuxzrTpuL0ROct0zWR+5rdBEIy7LTz+Iotwn + + gORrsE/WG58z7VS0nu00dfwKsSaP6dK0+DnrAhDUixWQmAutQ46nj+uyrg4cDAX4 + + ZqringGYaZeXDbm0/JcmyRSPixLcUYrGkBPl6/jeiJbTmJGzau/xkQ== + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 8388609 + seed: yPi3JHKKbWaEEG5eZOlM6BHJll0Z3UTdBzz4bPQ7wjg= + zfec: + required: 71 + segmentSize: 131072 + total: 255 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:MDMF:yls2bowrrs2wvyjdndkllchjjq:fcl6llxuj3bmzfwadvjdswvi7ul7y2ikwvp2svf5gblo6g7zvgca + format: + kind: ssk + params: + format: mdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEpAIBAAKCAQEAqMnCxUIeKVVqT6KEaMDpJykvC9R4eGTbsXJI8dO1X/ESsyAT + + 2JK/Msh1B00wJvuMPs2zApZ57heb5k3mRg+IVqiYcBW1wnnsj1vijT2pgPa1ZhVB + + ix0/eN9126jlEZGqGjI4NnuxufDNO6eiUKCD3qbcyF6Q+TZmkfn03DWEaygmU+DD + + nMcTuxqL5tcYDkl3UbiJV9CjDmKUu6hJRh2xryuFcj6WpHJguXWu1R087Oj4SZqP + + VmTo52bbuKg3QNW1eeHfDWaYH550oFRLDl5ojgq4SXRWuFGA+cLnGu6bukf+bkHq + + 68a62txhBtfDdfoLYiQ6270dNS20PbY/uRxzowIDAQABAoIBAFDqeH8MVV1HX3HR + + 3VxCrwNhEPbA4wgEgfWtbh7QeXEHJwnMZPc8UoDL7J6VeHIXwYISJrEk5ksn8ksU + + KUKJC7lPldSV887JmIiZaiB/4RS8MPZBVmyUlushZWTqsPYdOMjaLmygG/Gh6SGi + + GYRBjzZcFBfSjfmLBN0SUTqIRXUAwI11SEYmEJZl/5L/CNfmVbxTuGq7A1Usu6AK + + RZEGrDaPqPip/obB/WKMt1e4NibVST4q+CpFewuOiaWL1gZPQL/lL62HdOAmU65i + + TuT6645wvjyTtmYH6u+nj4IpR7j8/YbK71ShjCB47uevWu+75lVGVhlSDuSIaKb1 + + TxV7hgECgYEA4EfVci3rSCQ1O4GbmaChfvQeg/6pe5ZbwQ1BfYz6LuQKSyiXrDge + + d+47+QeDlxDM+lpjWBjzTS1eD4Wq4Qy3EwW9HzChDcA41tmKZ5r9KLctbbgqfq6h + + xgFg9GxdqpopHnGe3p6IrbSbJyKkd5JyNwN0dTowhsh0nkSBE5++fSMCgYEAwKjM + + 8AzqHSH/ObxACrdjQFM5q6pg+Scp11/ODYzPglwWVEceQsJ/tn+w2pz1hypJp19Z + + c6gr9o3oOkfWPsSognyzHkL1AjQFvvuWyfPg4NpVRq2g16YYvIWaY6JrP9GMxgt2 + + fiKzn6dg9dUoHK+BdQbDGshRJyXrvd0xzG7N14ECgYEArpD6385B7XrRPCnbNK5E + + RQ45mj9jJ2CWtiJdMR3DtS+lm25S76cWf/6cC27/y1s2UD5+SJnS9eUz6xz9LgG5 + + lULIOzicgpl1JDVadt254jEBWP8ZhFTkcbus/VJDbYBkNN/26gu3Eo0anlFmdfM0 + + lwFHad8K2j24F1/2n5GcsMUCgYEAiu1ukzAM5qMsY9rfJ5skxC7/qE29jg1yu6+H + + a/f9b1iudWmvZZ7R761Wv95to2GYKUy1uZQs16dvLg+9bBfuF+KKW6kW+ta+ygCs + + tMbbg+mNkuED2l4Y+mExeuWVhzi51dpQQRcPBnLxlXR3b3AT32rX6IlJE/zhaVGH + + Zo8EeoECgYB2OFnIATVM4BUAj8snqsnFKkL9Rq7gwu2eHY0y3uOKySM0TmydfFKr + + etrv2vkeWPtMwpXtEt4A38/pAnuRoF7f8M09nx/w/W9wk+c23mNf+VBMoM/lX/wp + + ZaRAXyXqOVPVEE4MKrozNaOCSlmjxMZxQSLB8VLwOHB0hcMkI9LW3g== + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 8388609 + seed: yPi3JHKKbWaEEG5eZOlM6BHJll0Z3UTdBzz4bPQ7wjg= + zfec: + required: 71 + segmentSize: 131072 + total: 255 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:CHK:hz3x6hgz6osbyo5he664ntvxiu:7hpheae4wou6b2davtrizoumqqh2k3vo25erhpgrq5w45txmjeka:71:255:56 + format: + kind: chk + params: null + sample: + length: 56 + seed: YQ== + zfec: + required: 71 + segmentSize: 131072 + total: 255 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:SSK:jofr7syf5rhfw7vic7izxstipy:iav4tic7zeutmsedt55yib7njlrmiazmoxnqgvhekm4tx2eqd45a + format: + kind: ssk + params: + format: sdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEpAIBAAKCAQEApsC64mN+RSE8c67iLI4Ld+IJjvceNF5gBI3EZ6VgvZJ1vvJi + + OjAEc/9LPDWa2Cdj7yvH91aXaba9uzXtScmskZ9/A+qLE1GEEYSP3TheL6T4p4Bi + + tTO3v2YWu+RwSzc4WRqWKrmUmrpiUVWNL9O0ch+oRYZYJqcvtZGBoE0a4Ka1DjO3 + + 3eCtCnl+vj1zybQv0bU5IizyNJQ17qaadoWkTBmE22t7/A04CUwK/lMaQEyJLqHJ + + QS2b9N9wjLkxUFMMLgWTjtQuA4w9vDSl73iVAK/GnzMvXOv34lY5kI0XW1ExoJ/l + + NNU2anyuS88U5r5hXTcfrJR5ZsIYxoiTx2gv6QIDAQABAoIBAAN7J0OZ3F1U1Opc + + 0qGnuvdPF5A9miqxdCtwKrMXtZnrhGv+qxyIG1WxFQjeHRwJUXmhFjj0fK9zJkmR + + 1gDp9gFpvRjvtOTLuTg05lxxxGyV8u9rO5RJDrtPBic4vPvi/JkGmC8u5dpnjO0h + + /jMrBhuyS7zc0bsH1zQBBD3ckjSxmZohdnBjS2ATMtgg2mTjtp53FU6GanEaaGjc + + BQUlonz/gqw/p6O/dY/B/2jEwKvENRq5hLDAdnqn83tY85LFSuXaTZVGYFCdqgYK + + YZVuTkpk833eKNUWVIEovqBQkcn1R9JRx9SsvZZK5aHg14+4lusmMbMaXnOTHv1P + + YdhVKDECgYEAy4cNQZAK7fBXHoCMuHbXEc5KwMmfbm4Spt20mZJR/DxgA2CagKLs + + 0S7BP7X23R7Jx3vY8AvEI8BvZyYyszzy5tsnbFXiNQ1LMkY9rWQ4sbCeYaj2LkaS + + WeSTXaByjbRTOTzZsq6XhQvGrmxtRAmrI2us/I1DbvIB6b4e5QbbOtkCgYEA0b6G + + XsFWFUtJGdjYC6spvX9d4eIc1Flr383ahPcSWQC7akaGn6HNnVWbiICJEJHrwPBw + + r25QVEeJY4MGxrqvAfRn9CvsvWPQs5IkyphSWQTZKd3cgf0JEb+vZG3PWAmjAJSv + + sRgh51jd0qHZQT05qyDt7CXUVx1UdlZlBb6z05ECgYEAmHqOgN70gTx9WFnAk3Zd + + PHbL5FFpg2ctzBvvcNqBV7K3z+/w8IyfVTxtBVlDIHgvfacYaQa3pH1IOQQSGdyA + + slnf2DcjqNFT089x59Rc8Sq8Dbhy70pp3LT1fsB08hr1+rzO8CIDXGbtK8IJvl5r + + +7ZwvCjtK1JeAoswRC910UkCgYEAlNqBZFgTrtMaUySo10cnPVxaFYgya6X2wAPJ + + NJpgRBgX6imZO0tKsIFj+3E1VTQqO2iooGhKzDVk1OHVek5dC6cX65sMzbA8GmT6 + + hWmq75BYSrUw3HPm7ti6Mi0YfOOB8lSTh7yXuyc/bk/87qbz+XZKRFDorNac7csM + + sRIRb3ECgYBxu7Etp/NBRo8VtRRLsqndCL5jLxON66cRq8MK6kzJFhzgdfp36ERo + + NV8HKvg0f5c+44GPpi67zyYiMWPwTyE9+Lq24DRdSMCec61G2E6L8M/q3UtlK/3o + + I8hpC26neWSUf3hNur2yDbufdetBZefKxGhS+yF8XRnrtv4Y1CRQhA== + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 56 + seed: YQ== + zfec: + required: 71 + segmentSize: 131072 + total: 255 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:MDMF:gsqslvviaskirtqbx4wegueyyi:2bcvqw7x3szvnmemjpthphinab3aqolfgjkxigw5ghfhuzfbdeoa + format: + kind: ssk + params: + format: mdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEogIBAAKCAQEAstun1fppsTbRKmHPeu4GlX6VoCxCE9liDqmewpjDK7hAKJ7n + + /EPUeHP9mFJZ2v5cY5E6KcbCrLJYCkGPUS86BfcumyINoQBtSGo2SV5iCD2yUv+f + + L4AerAcLWPdU81sRd1TU69/SWgQklqFh4ie9eGS2AWf2yOWJqMyl4iTcKuu8wq/4 + + /fA+iEaryk0oDvMRx8lQsQf4EgaSv/V2ZZVSbSlo1iA9TawUbZxsVtBPZ9q5ht1R + + 9g/KW563S3pjAfCITWTYYH1rnBO9+Tw/+X48o1EmIrW1ObLWqYV6d9q2v66ZQZ8h + + wlxeYhP8Q0V5siazvnFPDzxPWvKMeAtbDTU6+QIDAQABAoIBABZvR/+mn/BLJHRw + + /WH5jljdHm6PbqBnwY1+SDw3hi+rNl0CBa5WYcXUIsii98No6XTRyB5qYIvh+Poc + + XBo+VsRdy3pJDLWXxJ1zOSj9zkUjXAVeK/z80JwabBl2OLEnyKqTuPt3QT7qSx6b + + 0pfYDUOXOl81x7ZOWHSUavBRWE6HunX8/3y7qKWV9h5DGyGJ2G/IJXnI/uuwamiX + + YLKkeMKYczQHb4ke/e96QM7r2GcHqPisT4Vz7oD2kWuCsRTD5Y2s0wzxKKL+hJja + + vWl4JgzFZu5Fyxx4izoZuHHc76SKNass2zr8RidfDx7s2ozWZi4Q+wIDhE6MQw69 + + wlwa4SsCgYEA54jnAzM5IMm37nyEF5l8hMtNqIO1EpMg/J3T8rfGyG/wYFvubRmB + + zZ1NnVQIuqkjlsgEcGTKEKBvGJyH+bw3ekWnvNCSs9jhu0uAUzqhHP9hkETmgqr4 + + E/VsQGRsMQPCstDN6JgT1rhOO0xde3epwjhk+rW0SNbTbQAz6O7mAGcCgYEAxcHU + + IAs8e+y4wj+X6RdeipGeK2EFFSb5M+GUtsQlSYIc/w7AZLN0sBbKhont+/PLXLiM + + PEZgzfYCp6pRo7oBqs+IJO1xvt+jcOyRRJrwohxgs4IDKTTxfGGTJSFrBM7wJP4H + + Qcvclq87MG/AkYnCvzchVQz0B8J/BTcyfDpkTZ8CgYBSjKcApxSpMgJYDyDxYRHa + + LroDaOH4O2i5aHQWx5sh/3cOg/hgAYYcDweLHlj2ZDOCINIkWGsKvoidl8GLMqX8 + + /DSvxxVm9d6VbnfUNMUYl5zrWQVudRJ52zi7RJKmbxbNtlCTqxT3q0KJNdLmoGVw + + D7dBA+PBTIaZCEd5tyNd5QKBgEwOSLvuNlve5gvnE1CVKUoXyQIb9S12aL9YUa6c + + 704/GVHK5ZmVHxqeGVP29i1BTQQjAeQomRB9PfYn3fAfGIcN++lf3LAxKJXElfYR + + tNxUF6jSJs8RSpKwoDvWh2c5A0jm3fmjIvpc+GGfiSswFVMfK9We/reBSQLDgMog + + VvU3AoGATSkjR5caom0oPfYqZ4FGs6hKQkX0DXrnVPiExh0aJEy3J0oyPHsvyjJC + + 5U9a3yARLxS/mIgCX1B5Ti1Ti4YsnQq25fCS7qVVDpKV/cDI9O0y+UOHjqdTK3VM + + pzSD8utm8mb/la90IwST3gv2DZvblsG7ib1k3WwcQakfHa5xH58= + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 56 + seed: YQ== + zfec: + required: 71 + segmentSize: 131072 + total: 255 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:CHK:wxmzfzvgaqm3xfzgaqrlndgolm:6t3tqcphsutigxnnzyp2xx3afklcpeytnh543mpqptquvqqahwea:71:255:1024 + format: + kind: chk + params: null + sample: + length: 1024 + seed: YQ== + zfec: + required: 71 + segmentSize: 131072 + total: 255 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:SSK:k3rnsfs2yazedhiqpsdjyxdk2a:yufwbecazt46va7jsxrg6zdtfbznh2fmw46v24bltyjclbdy55ia + format: + kind: ssk + params: + format: sdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEpQIBAAKCAQEAsmqX0nYFM4DEpUyVDY/0/nftAicRAx0DR0TqlV+8+WbsDy7J + + jFRe3YpGPsG/SuhAykR87JshLKyJSvyAjIB4rS3RDjciqz+DhysNq7+0aGanMCYT + + lvUMZMr/O4kUQElw9GPwL42/p8YbeOYcwp6B/ZREXpNCmI/aSQLoNRxFBskh0QTD + + kKNVfycUkBRgw7XpJN2Zvz3IWq7Ou0ZHP6tNRXYRaOwdFM2/0mAkucPiMr751LeR + + L14e4jcC2zz6JVcz3o0UeGs8oRy8kcrpY1kJ348ERM+hLeESkYjOYYPexOc7paVI + + k1AFKubkC49Vl0YpTl9wSdKq06wyF+YaQpsd7QIDAQABAoIBABh6mv9/fVmyZGiY + + kAfHFVJXom9N7F3cvG8qE7RwaQVf+2ne3bzQ854aQ2aHXydYI9GMoYY2B5BxULn2 + + 2G1OkCAUne8BHhLYWUOxosPKfuZnFS+8PapTz0JB/tBMj7h9Sw/g6Vqg7GeIvQqy + + qcYDCOtBjTrbogK8E/M6AKGO0iKqub77ZK/iBQ7UZFqSZS+t7atPMb2oKWLGWtO+ + + pDbdpDxwC9ZC+h+tQDlqMjP7ZLvYqIezwI/YqEM7oOPNL+KvCYO5X6BNJdKoNuMW + + Po+dsY1PPixmaEeUIq9tv5tzqn2K0jbCtXbDkDpVXlNW7MCYTRCFGdejYYmizqmR + + 2YwKeFkCgYEA2ckoTYRPbmtT5HE/7l/AJpsdKZfqQ6Zng952y9BQuJgdme4S/vNn + + BMDxBg94Qy3Q3g463HLVSI7YJSo0R/Px6pVdxPcEjxjlj3PyCuDYCFbpGvvOZEbA + + N73QLpbtisVfyg4BFwA1Whn9SXD+KyJl6JDm2VQBg8/NZ2baOO1Tp+kCgYEA0bj6 + + xTPf4wn2GhQBBGnr20ILoJuLt8+GanRz3Xkb7a5QCeqdUY7hl0++fgeDJ2hd1mks + + i/pjPR8mzdI6eoXBjbxhOxZNJLN7ijTpYo9k0o6rF+ijCbo5wHubU7SLEdV0eZyR + + qPlG7QkF/665ZGEaB4FUVhyb3zRuY4DseU/eh2UCgYEAzmhRM06f/bXpF8yh2+mR + + 8sT2WbJqS92NpDSXAMoZhypce8Rg6pOD4sR+atEEmR72I073SHHpZNBFWMvsKvmw + + ITWZXpEDGCBviYtJLjg1Z4n/ehyHWxCXIv1aLp3K2sf/5j9plwQSjKevIAjgS79Z + + OJcEw5tTqDqtoT+guW1s6OkCgYEArIF+bEVeLG9eKlc3+vxxT9nEnKg1Rc2CoAAH + + 6i2bRmcyWOXN328qqn6ijyH4xKp5PUsnpEAh7v23umbpSSzKZ56DT8npTH4B6U3a + + hwKyCOvnWfQ2X2L57BUAT9ra5aFxfDLIMXhR2dmpQIXk4udoNLIxv98qa2/COUCr + + wqFqxm0CgYEA0Tv3TBFqDVP0eMKURY1EDwkjv751wtsCLcU9xqaWKIAa7OKzHDN5 + + 1VxTylsyU/7A64C5a9CWyGkMYdkYlgYqPnAo0Em3fzAty1BQZJzvzv7LtP6rB0qt + + 8NEwbiFpUorowTfRyWfxRGhYfD+Z5r+NcLNTTYn4qbFsP8h49iBKSWQ= + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 1024 + seed: YQ== + zfec: + required: 71 + segmentSize: 131072 + total: 255 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:MDMF:6h7aerepvayokutkd2gg7vuase:ypguygm6yi2xwivz2l3d6orux7ik4dfpv6b6ta3tm5ydpntjb2pa + format: + kind: ssk + params: + format: mdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEogIBAAKCAQEAowwAgweNcKjekhbkOYjboZg7YQKqOiJyKujfAs7FrLB1cRJk + + IdM2jvBv7E3PP6uw4VZVSOMxvx3BhNiU+oiR4ECzuDw7X0K25F6aEvWsbm6NQnlm + + Vf6lF1MgRBZf3AD7xmoqDjhsxsBrYvhByIWGGZdx0G8Zw56OhRCOMkSa44eHhgyz + + 9M1nobyVvS1lRRD9Em9c/16LkNz2zZQF/ebpo5jK7fuf7uYyc8JHmjB3J0OIMfKt + + Ubdt9FGJWvuhMM/gXP4d40jBkwopPwE50TCLFFH4hEIi7nhdvjdgQxg01ZYA/yXZ + + /GODUQbjGLAitDwIaqkWzLEOHeGweXYIPl6eHQIDAQABAoIBABFF8TvwbeSUj1fM + + wwrxW8tH1GqXnK8h/RRcrVufykNcQmTjPOZ0eOA1yrWvHJizOL72pXxeTWPg5CKN + + y2KrW0D1udR31RZne/a/qvT9P/JHFgIH2HadzqGk9dMgx7EIDaRclO4CvktkETxf + + 7qAuvSEy8STS0FjwEEs6kMX0jLAz/TSCGHNj0aiDU/cydz6TsDdQSV9AVUygZ70M + + LQ8vF5e8+nogwMDVwUAFch1VVwWvddyR/hoo3/hCDnw47BZYOwwO75YwcVDhVCxf + + oAxURnrlBXcAk2oOmJhRimw9fcT+WdClBHcq8e1ACHuK4JcGEqXnLKuSMBKd2nB4 + + SmYyzSUCgYEAyExi4Jh8zm3r1TJ9LA17v5oSK+JwZMSo6n62RTzUcI7/ri1MTM8p + + Zk/lHK4q/51wsj4Vw20AWCfQPc3XnceBxAYY2ehnEfOWX7BqQpDUYQn1o19+qVeK + + SpjLE4EMpTPxDvxkjkCflRVyLRomrX39kpXzE+6SZH+z7s5nM1MrdwcCgYEA0GOc + + vNnaWWb0+rS0x4DZddjNQaYHru3qN7B3+7Le+LMWnZQPyQ4FNdKSuABItS/Wwsc3 + + e4fo975YafXN6BRgwNmkpIFjazbJZsB8CLGB2qZFwP4L/HXt8KygX0aQqtyZAYit + + 3EN5jpSXMPCo8P5lVcpR2LuThaNq0F46Mpd09LsCgYA5N2DTaZvVWB8TEs4g5GUi + + MX/ZW2Dh2C+sdK/ajWreEGtHNRdjpZXc7Ru0mqgbxrynngaXga6kgBMDZKagIpqW + + BWvZ64Jt5VhiU0G3bCnO4opxtdi3xRLzBjyUgLu9AV5t+nk7DYjIjIzGB39e9euW + + kRET84WMAdLDd+CRD8QNxwKBgFLnnl6/qQ+yVzo6lEVerKUmyJoajKn6exkGuuVH + + B1AYJ6IvWoxZaJc+HCLZ8hMrYhyBl0AtFPEjKBeXtABlwwxWShssYrovxLZ9U5s9 + + y3SKe+vI5kndPPloJDFjaIChXLnwf4LG0WB5GyFcTUn7W6Ni52b7UTh0iDU3l52z + + BdOrAoGAQH/BnYcsQGkK2szMfGE/kH87CXKnxyCCu31Cbv8/YhgKsf7P8sl7fdmF + + rub+WqTFnWLVrD26nm5ZPzS1y3xzHdIJSrgDQiUX5st3aPvi4wsHmJjkXdg01kHb + + iX9WyeGf1gdpw33l7i/LxA56xz/TruHUWbAAqIRgwVPrmjSJhH8= + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 1024 + seed: YQ== + zfec: + required: 71 + segmentSize: 131072 + total: 255 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:CHK:r3q3q4f4zbsipxauq6knb5dfoq:lkae5mflammzxkoc7gllt6qqyg5gqdxeejmxtgcscblqh5tv5obq:71:255:4096 + format: + kind: chk + params: null + sample: + length: 4096 + seed: Yw== + zfec: + required: 71 + segmentSize: 131072 + total: 255 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:SSK:wijpjyc3v4fjksxk4bvumczuqq:bqcwlgncua2dccyqskz7dv5cs3kajaf24i527ksuyq3u3oosvjna + format: + kind: ssk + params: + format: sdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEpQIBAAKCAQEAtFKkGTyimfLhWurNlDMzBBTS1LI3DCmNUDZuf8qb2lu9WPXH + + 3FkqkaqjTXi2wNEPZkVQtLnrgrh58aIlFOpCpUvYmn4QI7+n0tqIMj/EsZ59S4/n + + T/ia21MBxJnV9X4Y8Q6KMrev5hqmXUPnaSaS3Fl3rqQ9Kuwg+KVjTHN5Lv5Mro1g + + 6WcDsHwKlkLZxP9NROb1oXAP0ehUMhJWvhAUiuIikHVinYe6lP2te/3hSH5kzzxS + + sccG9JwpIjrKSiSg9yJoP9TxZYUbEK21E12I0cEvTytWAFOc45XWZgzE1p4+zwar + + U/RKYNilJUWdNxiltoSlxME0VZPUdctCfWasYwIDAQABAoIBADnZfV2/Tyb2fYeu + + Zm+SEV7Q02Z7Gh/jwLsoC0EiHefqoI9Gomy1imubA7LZ9D9dkoQr3p1sO/r+9dgo + + PZ89HE5tS7sckE73sH57r0/3l0GoZ+fy7bGBPyT3t0x8UeDlKFlFYd6tgVff2tl2 + + 7GmWf20Dotq7RAheIqHCZV3kec8yKTqyjUeaEvNZBWczYqMXdOisj8N9j2FUrWRq + + PVu/z4yLc8oo2YoEgzSnOhTsmHpUumfxLcj32ZSr5Y4MZcmwCMVlGe0WoZHPoWU2 + + OumF8CqcFzIJXjHhLld4JZ/w8DdjjPaOP/wiAnD9j8LZuQ+oaL3HJxLEFOo4n0Ev + + +zyB6rkCgYEA8zNi3ZMNIyEK6+OEOvDKP4mKq2TRn2CBJGOHNkvM5i6wY897szKa + + 1jNXoE9pHF1onQDg/g1y5lejpVMdCjB4p7JLozMh8D2JQuKM9Zmws8C0641XWcVn + + TFBdIu7gp+zchiXWiBou/SvKIorTWMkriM5KR88lFVLwx4WC+X02LBUCgYEAvdAb + + vmzryZRAcYEeLfOoGr8UpCTDIIVrcIVh6e9CTQC0OAgDr3SwmypNtK1h8Rf5tLS7 + + O2MbOOwYvfAW33oL/Od/2om0CA35MyL90ETbh+zSPH+rSlLq2dbUKa5Y1VnTyUSD + + sdzU3iAy/hAPePOysA01Lo8vAP4paztR74BW/JcCgYEA3HmoVkEats8czI688IYc + + hA9X5FuI4hil2uxTxvhe8ApBpKqTdPgagLeY59815h4UWclTL13X3VR0Kcu4VuVs + + bHLpuTEAwn+28SjbK0hCdiLsoWLIXrzkEb4FQUcX6YSEwySIYWiDUscg/8GlKidt + + zR9fHcx/zN4dJHQ4MZ++vaUCgYEAivXPSeLd3+6sGyym1odaG3KmfuD3BVkH5hGQ + + ND5YMJ2CUr7zS8FHBeG7j7mbSXD++1+Q7xJIPK0EFBGv/R2Rpy4n+Or1JSxtsxU7 + + 8fxnJ6Sl6WqiEUCQ9LgFDRq5qEAh/2gsbcs5AAFcs4k4epkWyTJyK8rhY32u/vUn + + sAoqJLMCgYEAg6/I9L+HoMqN0P7hpek0uNVpEYaQ8OB4AXsnC8m+EJzzcAtmcw0Q + + ligos8cK+vsDpcRCV5uMBcDU7hfwIq8cf3f4BEfFVsuPWtZbhbPVAThVCJrL/jdE + + hIf4TMzMCuMyh1UQq5PmlEqobKpbLowktuvs9k9iWN8rpEhRim3vm7Y= + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 4096 + seed: Yw== + zfec: + required: 71 + segmentSize: 131072 + total: 255 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:MDMF:xhdmt6meav75nzygmnmk74rqfq:efuyehuxybs6jf7qn4qhi5up2mziuaiddv2rzg5v5mmtgzmiuwga + format: + kind: ssk + params: + format: mdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEowIBAAKCAQEAsRUOmJdDBY2BE6PCQ11VpPYLOYBTeM7p2FWHaFfbBrhTCFmZ + + 8bIhttRLPbtbYxi7GNsjVGr3w0c+rufJnrsW3bOg8eOj+0sTpzrMVzwawSGf5t3L + + KgNAOjgSBaaGWX8IPWc9kUCbCJVAgDUL2dLFQOH/9YPlaxqtDgbjx7FXuASc1d4U + + 2x9AdYOcFP31QaN+b9sF+x8y/V/H2LWG/q04OK+t+VyUnRWhY7IWRC5JEWvw5XMC + + LkDl6LmZaVvuKV+7u4/D5i9amhY6KQwajc8++0iO4piHixnLzS6zLGSMzxoKEUwy + + AlB4/DDZlVSivN42mP4qBNUtqYBC6brVVEKkdwIDAQABAoIBAAcEHMO8KOijnNVo + + kjE74nSaijYcbBihdUk5lX4NDqpRshOznewIl1nFR+t80xGDELt0YafEZGd1TC0Y + + rx4kSLMLnR9x87S1WpfJ0AACa22/5YD7kRAe3tpi3FWJYOtTPPVWpzoF3RHWMAVx + + LjRsK53D41gYtYWsoeZBxmz9pspHCGZbMqekiV7CSln8jHuXGEGrf4Z7r6BVwaGJ + + FIO9oESJhXp3oGYu+omq/WCHYPUVMKdWqtizVMBbvcXuALG6yeEAeCGk5mqJOfA/ + + AzyDU17FI2D12fMDhqUaTYhzu9I5stIio4/6RmJrwLaxK8efCAWohfHiJeMTIdyZ + + 8NvAEgkCgYEAx1PsUSXOowwG0l7wjmtYvPcl6xcnIsWMmYeTAv+JaMfWBvh1LPKi + + 56mnBEm7viu7SS9vKjnoFGizn6pxDTqnYeDnzFTLHtZayqmzbtedhh+Pw9Bucxgz + + LeBiTDqd1tzBSn5d4LwtIAECQW6m9FnF318azVVb/2vQNgDtZhy0Oe0CgYEA4239 + + arJgtB4tltS0P9c4kjDmC7qokaVDFZ7udHTsAd0VebteGe+kIIhDH9QUk83B6isC + + n13hYXkItBMixu5WWeoKlXWQ8rmm6989gsO5fiDczbQWHUaverSlfqt0dZdByrsm + + YtdWo1kvMTzpf7IGh/DRejj1XjuJWZ6LPCNfO3MCgYBiBFfJ06CYNtrH6h26uvjI + + +3Ou9hStmZ05Bhz5tXT5jIMnrFfagXowFxHlHujubAzNwUCV8CG6n33svuCW08fp + + brItnWjAwkGlNOviTq7MfIqyjLUzbawFHDjaKVzigm2eVyOM1pwOB9D3IhWBRP/z + + ho3keNwjbv3VAIG829KYRQKBgQDcLighnAAzYOQSGmtHQz3pip2szVFVcAG6dNu2 + + s1upkjiwWc2InpDvTfxuXAxv68vIwUsQrvr8OwlKDRymKyg+dG86s09ZLpOD1+Td + + LE/w5C/glnCydzR8P1fZgnSFQ6LWesl297NRAY7GxInqrpfUFDk5cttaF5mpwexa + + lIQmMQKBgDAP5J0OntdDvvqwfnXUOqL/ODsPaNMqZjBQUW8q/dZC3xxyMW/CKtuA + + Qji8dYENM5OJheY9muqI2zTCfdvEM7628D9NtIn7F0WDhVADdYYHggs83GbcC1uL + + U2vuh66vlkvFOdiD4aZwnawXBDE55HYjuYUybjyx7dzaLy6fDfJ7 + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 4096 + seed: Yw== + zfec: + required: 71 + segmentSize: 131072 + total: 255 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:CHK:rdrzetasw4w7tuweqpev65d5vq:gzzb2v5fzrduk6kx5xx27mvo6dgnd65ym5lm7re53in7xuudhsxa:71:255:131071 + format: + kind: chk + params: null + sample: + length: 131071 + seed: LCa0a2j/xo/5m0U8HTBBNBNCLXBkg7+g+YpeiGJm564= + zfec: + required: 71 + segmentSize: 131072 + total: 255 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:SSK:4h2egxe2omtmnmvmmqmhatse6i:54kptz4n2j6ig4q2ugywxtiddcfo3oaswyhzce7wq62ey4gxrm2q + format: + kind: ssk + params: + format: sdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEowIBAAKCAQEAu5KyHrCLGfxh+ne2HLxb2YpPxSdaasMWYwyGEGe56UCMCXZ6 + + xpp9YaUkZNeTEyjNivUYtPPyCFl/sKMiaDKFe7PcEnxRzLxlt64lCCQ851AV89j9 + + Sd6Qe3LNK2WodP7PBv3QdWnBEjjme6WZQ1IZfwyYW1ycJ/LsELa4Q/jZebM6r9hT + + aJpQlSGR/Ir4WERrybfooGSQCqNtXns4Dh96o19L00vvyb7U54zodnbAe4rVUzrl + + V8/hhbzw1Rc2ttAjSeR57jUGqvlDhDFfLmYW2rRqb+Y5sjz8vqqMAGQ2DSxf+iLV + + Kh9O+3oys4QiYCBzEDfwxPMkBILR/jWXuLm3nwIDAQABAoIBAESZ/V0uEmHZpXf2 + + intuBGXGqTAhGXeMjEaDkRC07xC5E75uP68dV5f7zxi2o0rRlIMq6vNbePzGxuWy + + dGYJfDpm6Kk2ILCxgr4wCck1f7TV3IGHrfNzXAJaVWF216qaettCvxgCKqPgfaNh + + SHGPuFV4JMzdTRtrRB1ExpXNkLRqT2gxThxZPrf5sw2zicg8+fjN9JliPEWa9kQq + + 2GiaNFDoi90kCedgakbGcNoVbKGdRMFS8jzJ5u4w8++TKBBwW1HByC8LutSyG/1u + + ng3oWBoh3i5RbeUcZCTKV8FN0GqZEKTwso83GIR6X5RjhmwcGAuQ495U0uQ7m1od + + p4vE/SkCgYEA/9dQQKOTpe/+sOwRoKeUATa60lJUdhs1bqHJv3IDYfjoxLy1Nwn4 + + OPsAlo3vSpTbEk4xeBq4Xc2VefCnVq9Xvmt/ro2UnHF8rmyPZlKlXUUokUO+3fL/ + + gqg2JPYSuMJUhyXnbo8Mcdu3HR0/J2n6rCy/a+yuqkXDmFbqkxDX2aMCgYEAu7CG + + ja8gllxKgNx0LcAahkKhLWysxBkc7oXTlsx23tc0dfXKQ95Oz16pRJsEoMAvQCKM + + 1UekhV+zJyAAPyzQvBWL6uhzc5Y3DlIJpGRG5I6oX1wc+YYTQ9b2/1N77QbmS4mC + + A8JOhFpFUj5upZX26M16+0G4DqFRJWgC4BIYAdUCgYBNR2iCXea3dOrl3ijk8jmO + + tE1yQlQo6McXB9+86F+FNH57DtVeLrC/5XGkCHODf7s8qEnhEZEnJHZGQx8I3CYQ + + 6r/CphmBt/YFad1W9xfkOIOsfV4mBMSRXuYb/Ahjrq+Bsz1Y8/S6X7fMH414Blcl + + ss6Pdwq7fB884OQyUCAjKQKBgD999eaUKbfzvJhe+0ZGyDJG8/ND4iXsQOdHik5n + + GIdF0c9duHDBEXQBF83Hiwc+PD278lxsAfHEb/x6TNsSNAKMX2q1++hMFo/XnL9p + + 1LmYsMihhoO6oWW/oIq7GR8TyHAhMkRHRPxs9SpfSFrnokEa0dGRZ8w7MhIvX1mh + + hVGJAoGBAJsL+KD+DQDZgeiKhal2Kvuv7xXjwTY71BfEHQRq0aE/ZKd+r74bfOKL + + YPYl4tuoHhz+lHrVHX2S6pz748cUSceLaJcrswi2EMkUDYNCB5RfEAZXTLuUOIfV + + nLVBNj/I7/uu5ZH9ml0hECVcE/VUMERq0P+8OcBa6lAh5G2x8uXZ + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 131071 + seed: LCa0a2j/xo/5m0U8HTBBNBNCLXBkg7+g+YpeiGJm564= + zfec: + required: 71 + segmentSize: 131072 + total: 255 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:MDMF:pi4occv6d6dflmwj4j3ta37wym:wjq4hq5z7uyszt5yzk7hbvunse3fbgfes2jinzcnbkw3aec2npaa + format: + kind: ssk + params: + format: mdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEowIBAAKCAQEAshiWbCIuj7q3vh+IegmyYGUjLcGxZb14yXMVtP5E2CVmZ6rv + + hAng9qgwPQ0rLDH3jsqgln1zISIH9IoTkMJ3/pUTZTmF4eJbjBn7go7KHUaHLl+8 + + Bgf/rp1wvOW0r0Uo86sppTkRqUp0Ww4pkGJFk71QMlFCrq2KRaLzWIAViQ9m14t6 + + Kg+D1IX7Fj8BvH7SNzt98mIYpsAfg6A2ioarhIa6yEyLcFf7B8lx3MZmEzRWjjBJ + + 0fVx78yVobYbJ9l0iXhlNs2O0iqhwItpimvcXzTQVsp6bxs2MRS8awqD81V9a3CJ + + S0dysc0cQFJRLp28/BRhIYgnHa2fkhvPcQ9Y3wIDAQABAoIBAB+prmiYJSYJgKxN + + B+MGgU+Q+5Ghe4wGhQhvrP7KK+wvrgalRcL4TKYdncHk6vWHBqe8z5Mhx4uu9LId + + sD/Oyy2YTGP1N5/Cshr07Zm3ECjnRpZQj+mUl3jwZcA3qIl2psK3fgZxYHn0Ej60 + + BGC2j/8lq7Heb5gFo20g/NmRoAKHTgQj+Jzz8En+0OnlZM5WfhhCjqNlJZJD6XTK + + 7A3BRYmheLZ7zJk6ynxnSlfBQY2Qa5dQ8LDj7lBFI7BaaDobRXuj57Wnua7TvqTu + + UtFWjdwvLp4wfwXzEXP2SaY2RYjNKCFVdFCgTPNijeJemwhqL7/q+TIpHQw33xxf + + dQIzaFUCgYEA2ojvmrgJ8LuPXdVQcsfNPE10ZCQVFUWdwgm9KGB94vyOtFKf6kaA + + UGR7I3IjC98mfu13/7ovJsuXGuqGujmnvJkiGcf9ADaAYBZJex73utNgCsk/wdpj + + 8ce2FnuSJYPVf44Uk5epSBVhm8PsJR8qmx97EJCGccHInYnltMw4j10CgYEA0KDe + + wl4uMcbMhN8m/NGe6U+n51Pb3HKwlM6AZWMIMdTQTellxRjZmGW5k4xzLypPcg9K + + VVLw4XGJ0UtK1IWTzAXLbXccPb247E1fw1daRC3Ns4RmNfkXHqReYdAkA+PEveJV + + eZVT4++U5rALDDv7IbpDfemZgSAn8pd/GIJbUWsCgYAkD0BqKUAKpwhLFW3G4s0s + + zCMOex23ettDL1Q1G2bqU35ApvmYMLXvjgT7nlPGG7ZAb3LDkbdCEYoHePduNyFE + + b4g+9M78gAHC2Sqa1EtQWpyYawjINf8T4D5di1pcMlrCR3GBwR6/tDd8+mE25uOi + + 4RjvbMmib7VouV5b7O7QSQKBgQDB8/yD2Ei7z+SM1mSJf1tr7bjbrzNj41/UR5JI + + L2QL55vsAsKxFKQeMNvwlw7yVzRahmqFnkEAZaxJXeToZHJ9pxly39vqbjm/vUL8 + + +HWbkrV8Yecf4D2lKAvyhS0mTJa4LPVnvfKqoi3ctObgbdbPiTl7kjM6PynENyFa + + KL46lwKBgCLm9MbqjcKN4rxyL3/GXMOM/fzzByV4/R44Lqqokz24zSHqXNgGFkY6 + + I2CsFRYVhLYry55w3JvJZaAOhcVUWECZq0CW2u7OKU6p3yvKnhABZNj2HMosAO2N + + Doxbw1b5rKoQ21WTphDyXJtkQ+D4t73SNLHKJcAXoynFz3iSJgU6 + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 131071 + seed: LCa0a2j/xo/5m0U8HTBBNBNCLXBkg7+g+YpeiGJm564= + zfec: + required: 71 + segmentSize: 131072 + total: 255 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:CHK:e2bjefibvz4tgu6jgf66gw74bq:mugtu5bx7h5gcivevmh2gmwoc2kkhmobrzshkuj2dgrtm3siysfq:71:255:131073 + format: + kind: chk + params: null + sample: + length: 131073 + seed: /N4rLtula/QIYB+3If6bXDONEO5CnqBPrlURto+/j7k= + zfec: + required: 71 + segmentSize: 131072 + total: 255 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:SSK:fftsqsnu3m2c7qh6go33cn6dlu:gejcmq4xor775rddgx22wyxyefnxii3hlwc4lfqaq2bsawmlyl2q + format: + kind: ssk + params: + format: sdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEogIBAAKCAQEAsBZiSzZEuupoJeNzcwVKFI+PeUtyJSg9NywDAvxz8Dl/pWL5 + + CZVluzfbzmcKFNBUQDYmDmHwdWJ3uBpP8iyUpWhpHrpTVB5btBDcpLlUP3XT0iyd + + LHBTUv6IglONpIAdBxWsA3Ck1ExqU64LbaqE6/NWXdoFoUD/8CJI/l8nishcp6Rt + + CrnXscgLC9eKIjb2INyWuUK0Sx0Jt9lqnTuw+VO4hHodfvZG4p3asSlk09uGA944 + + 2YAin5USfNhZbfw8kkGsKuCsZZStlgkLM4yP3Uzl9FD81yt2AUrDK41dAI/Sqn56 + + tXY4pRgd3DUD2wOBazJUcnqq+s2e6Xos3bgiZwIDAQABAoIBACFuVoIWDw13tIdA + + /CnBvtNRgDlUoBq63YhshDPcbzyUBg6F0GdH5HUbgVFaEblq5hv8y9PeN1Np+vXK + + lRQS77PJs2+Ai5KEYv+4VdO2Ve7odWtJShvmRYOTzKIFr/Yj1p8CN9K9X6XoziUN + + /aB4B91uKR8PZhM77nuOXtJgiXbZC9JsGiJBAFh66Fu092+CTfRrxQMGzo0mZGAO + + pcpFtb16lezmsMkk6ojc+/lPUNQw6ZEGDeZU/Ye5i7bbWD3h/mGUfszRw//ePz4+ + + W7fAKWftY3I7XGAeURKgpgCL8eq32GUGu3q8+EQilK24YA3GcNtv05mUyl3kdYDD + + 9tc/Qd0CgYEA1p/xGn77EOgFELshE6ZADWdfb4gzjNUwZNa87093N2N6jKdFqo5f + + B2DdL0znQrkJu8X7FCTHpLQLa4WwJxK8mbhWuX2NrW0hEf4Ts62eCt2YnSXnM5FZ + + ewgtMCVfK1Fe38e7pbB1eNeBxGzLZ+HNDou6i4YmimhZQxnqgBbUheMCgYEA0giQ + + 8Olwekgdl4N0E2hU0qIPXkDqaqCg03N6kpiOajEOhfs16o3MKZGLE256XGHeerDy + + 0ANEQh2Nl/AMGZm3+Meb3y0Uwuuws6/KwM/JcNgaLtQN/YB8IPR98Xpy52LNbUl3 + + kefZhgmglCtsKkkC9TyDUPGrhOPpwnvihDrwOK0CgYA7LhkWXEMwcznKVj8Vovbw + + ezuWjnDgeXyBobCxMDFIRZbqJ8mO8PkFGNGElGkEPe+QJlRIRqgCI18uw8tByunU + + XT8UoKkrU/cVdgDKv6nfhyDo3CW3U3Hf+e4z42otkJ9fhzyXwGNz8cCnf/RKbbnU + + M/U5OcFw0rsiRIgjz6fETwKBgG3lY4laa6rf3vPvKSYaef94Ilhr2PwPrfbVvnXD + + 8whnQLUj+1MN/JxndgCl1spNvy4tNe9XNjjt469zP7GJd/Ro5QsCOJHA5sUuHwWB + + 82HjANgtxmA1AT3xD5DxQ/wD+37KaRDj4jI7CVVd3wvFBhIxJniM6vTul9pWHfCY + + MidFAoGAM4LVlcuT9++F4xNUt2m2Dt1IMAZkPxUu7jgCDkdzI5o8e46XxBLpcxnT + + 9hoAgbEud+dAP93Ef67IVYznUw3YIWE/TUGOHQ34XxU+1UWeyXOYVmcUVx/nAAO0 + + vhurhFwi6VFFC7D/zfAIl+GL4pMQduZW+qTKpgT8Ct+K9XRxbhs= + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 131073 + seed: /N4rLtula/QIYB+3If6bXDONEO5CnqBPrlURto+/j7k= + zfec: + required: 71 + segmentSize: 131072 + total: 255 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:MDMF:zmzg6e3cbwab2iyqnn3icxpxvy:kmw57dgygnrprsa6xg34y7euvjkvmedmabizz6ijsudlzw2qlhha + format: + kind: ssk + params: + format: mdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEowIBAAKCAQEAwyDXoB/+OawD+8wWjumK7pY01ABDOe0++G7yHQUWfJIGhOlW + + GvW6miz1J1Qwic6FGm/Nuoqa8PoVKY3KqhsR55+AjkH/Ikc+8CqeQKGljmgcveM8 + + PL2oSAILvFZF03mhSKKZY4oFLBFrWB7KB+agOwkLQpOwPaYbncbjGGhI2i5dIdFW + + It5roL/xSvi1WTxNlJfjsdGPApdVtURP9EiR2fLOjpCFZFu9wO5J7a28jeRRN8ux + + 2N9FcPG3Nx1rmd8BkNip0O/AFzjTOTpNniqUBOjmqhz1PIHu+vhkizvmob9w2IjS + + JjORGEl8qgD6aTpTH7N4fv9IKGM4vwYpi8JzewIDAQABAoIBAEBVCKJDSgbrnpia + + relKOEL3BM3MlF15yaQQuAQ3VDWX00xovbm/wFjqb50a1bHpg9q2d8aDwhem6+k6 + + VVIGAL4zySedvKcphCecdXZrlPDBhJBaZdbE1MGA4yuh6f2SAUm4SggWTiQ8Tf7M + + j+FQ+Qzdq3e0x4tbw4keNGssnrBHu4euq71xjGMeKY9+lltfWXsc057dl/sESHlI + + 89vbdwUvrdvqiDpfUTT4nfc6shlIWDGYEFQAoO1TZcxfAldLJmTAduVxdnWiXYMX + + P0ozBfX90zD543PmCkqdHOnIMSEGqkZLS/lACcmcU9gUqOlroELgL16IH4si3LoH + + QvFgBiECgYEA49+yfcdSxQdG0z1Dpob2dWeO3LMeiUE9AAWUGA3q2DRnfb3qsBr2 + + E/QK96Mgp0impMTJ/dxtCRi/nYSA/A2rKPE+lMjcowSvFcDkkouJ4ePgeM3xLXp+ + + fvjjlfEGi+DkeLz8o3LlIfjypXTMiOc3utmQ0GGzcuOrKdqYj7ECo+sCgYEA2zZ1 + + rJn+Mwm29LtTht2skYVPqA95Os2u+6J5B1NE6rmqaoq3V8IWG96dXBRZt9NRKHB9 + + 6B3YppkYY4N4WaAQ2dI98b/PM7BqNfiFeUaEr2ppAxB9qy7X6uNRrQa/LvjsLkb2 + + unoOgHLIjW/w+xwWUE2eN3qO1Suq33pUTkrL2rECgYEAzsdZIwXSt/Pocxtu3hgu + + YU89tkvb89T9U528Sy+l4dd76gCCjJeKoYScxyaCJQqqHW5tlS2Gy/BnQLrSiOam + + YJq5nS2/+TXw4x6My+ZPkmnEchr/NbOoQfP8IT38IMZMzLtBzdge0HslRLr+N2UJ + + j0aKQG3H9wNdeLdiJVINAU0CgYAhXJpwGEedkN7tRA0kO1xmETncQ+6ZSnBVD5cH + + zF5ysqsC5/WbP4iJ2UltmBNHbLuvQd+HkfNE94vEqV+JlFi8LckLn7tzDGg9qoL1 + + wAu1fqZYtwvJH6nwr4Pgp2Q1S+D18greunC2j8GB9QVh0hZ7RjTMELToMGsi88Uc + + 3TlFIQKBgC8x9Gnshu0Y98p7AE8QSYYptliF6+RE79yPL7VqL8AO7G/oThBWwIXh + + w6d2JkweOUISA+KFh7syr1eZVOwDdYZ5FFuUoorwgmzO9MkYyqsO9Ug74lGpxKIM + + uM56Ry9B7WVwQPUvl+lvfs0BOeaS0BevKbKOLIA/2coprAThIl0L + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 131073 + seed: /N4rLtula/QIYB+3If6bXDONEO5CnqBPrlURto+/j7k= + zfec: + required: 71 + segmentSize: 131072 + total: 255 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:CHK:gy7bci6yvllxzvhh45khcwp3u4:qsfjfmcl64zey4k4o5cfnh2i3mzboahf7bhtggceszrulsv537vq:71:255:2097151 + format: + kind: chk + params: null + sample: + length: 2097151 + seed: uqWglk0zIPvAxqkiFARTyFE+okq4/QV3A0gEqWckgJY= + zfec: + required: 71 + segmentSize: 131072 + total: 255 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:SSK:o7snf35itqnyoxqkpoapqi3xia:un3axj3ncz5bzws5mdjvn5epx2kb73arykyspud556qyaognvkxa + format: + kind: ssk + params: + format: sdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEowIBAAKCAQEAu/5ks+NdjN33/76IH92yDttv2M/Aqcxs7E1/bs4Hi65FJUbD + + eT1e3VwpHgDju9WsQ4+73nzPrH+i4X3CJa7laO0gCCIEmuq6ujeL2c7T4FbPaaJL + + vjsbbxzThYmD69Sxyf2MEojGYL5mCZrqV8U5IPQ+dDn0XOtvolfM3Lqi3i9LWRkR + + dpFP5fqCxacw4ZgQFVFRShkWEWzNNK5bTbzMlNjqasBuCa9P5HtppZM8Xl/EkQ1Q + + vJBNDBVeTqfNYw1+xomrzLxmErHIHj762GTZiyhcEc7zZ9sLQ8Q8Mme/j4szCZOk + + VgMykqNWI6y1cC379daCDybxOMJE1gzhyyxbHwIDAQABAoIBADyVnN2OQglSRYih + + Xhwq5aW9GTv9pAD0tQuoZA+RDUR6KqV7Oya43PgoqcWWEs5na4cwbKKkhYb5cUQL + + M8TSKvOYK7EDSYmlaPz3RrYwXf7X6ysHVzKcuNgjqZVI/n9DgfJvKDOW6Zum8Jpa + + 1vfnQuR4YiIxxSsm4simVAq0iSVh+WKehUFniv0rRZNONGBq3cnKYYiRWwGK/9m4 + + QHEyTG+IJ08Wj9ZN8TVGbfceL9S6U2pT8fwUjYBX3di30aNrQL12kLCdX0S7EsOa + + ueGWEXid64tpAZJ5pcrOEdVo1KReUM1ghb8C7eOXJUgsW9+/9X/obdjvduOyeenr + + JzhcG6ECgYEAxn/3oRuNY2qG6OcFsj+8TbclbRXI10iTXz1/38GngbgQrqd+s18w + + iMZKcC+DIjwAWoL3qt9hDZtX0bJK27aDF70nKrJLX/QLustMwuzjG/d1JqIIesYF + + Qj3VANBqf/+htY417dn8RNZYgQmYuKZvUs/DftWRVa7EOsdEbphJ938CgYEA8nNT + + /y1jd97MLYUPU4K19SigDe/z6cggyWc1Pn6meB4ynVuD7CUuCkiNYah77jJrPhi8 + + By6+7aM1HrDvsytn1nYyWiHgftMgBNfgYGmEPJuqMFX1BAqNGVARWDmKK+4OSYd+ + + w92WFYQiudBVKT2AarDzXJzClZqX9JxtZVtEbGECgYBMfWdI17smAhi2ir9xLoo7 + + UEXFwU1BWCAh5SrvaEpJ/EnBY525NQcYzYBFtqlLed+RAUK6v5VAjwnKLnAWNkBR + + 13vOQiI1eW9Dra+ItYvWbQbhujKWTNQd8IGx7J39cN45ffFeFE/Xntk/8Bi/nrLr + + MFBfAaEdaVkIZV0DWP+3tQKBgQCKcMwv040Or4vLGkWMHAEmghISo0eV4I7IMkS2 + + 8L7BrAyeydjkiL5nZNJGR1yswOF3zcvgFhMzwpPceJAGsOxUC53o1ZtJD+kimtom + + c1ns+b4OZ6bGrfev0oZ06DY7q21BEzuRQAApPRBPJeTa7aFcSrpL0b9SibnnFUNq + + Mtk5QQKBgAamclnCCxuHZKIwv5FARZYz7kvo5FbnVDYUKnDbMyorG45O12AY+TTP + + IKapXaZVgmvHxo9jU4vEOEdSZWBoTUTfaqU63CTPt0qTSYT3yDrIS0D4XjmFE02z + + 8miML5FPzrbAmqBAuInPf8Kuygh3To6XIYlLelBtUHT4AvbPy42S + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 2097151 + seed: uqWglk0zIPvAxqkiFARTyFE+okq4/QV3A0gEqWckgJY= + zfec: + required: 71 + segmentSize: 131072 + total: 255 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:MDMF:frektgvj5mvd3vvsiznx64lto4:ll45oloqbsk3aygknsm63trp2d5edvjjhj73isjecykyva5ypxyq + format: + kind: ssk + params: + format: mdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEowIBAAKCAQEAukmiKqDEglepQN2zig3FeA9AIEou03zyxqlSZtrLwJkQdv9+ + + 8Gu/7sBox8OP7eUnwIexMq67fBOEVg10hovmRaGPjJ55zFmaCg/Bc4TGHuoaPtkJ + + EROeR46qqeJ7K9vgmEsBPmsGkA7X3Ldf7WnmxJ8J48i7cRSa5EGWOW1avSL+FWgv + + 7BD6g1V+r2eZ7lpaYDB4uOCm4ENF84mfcaQl0lclJBRacY2iXLe+KXvlk1lWUVJH + + hjAmyvB4eQxnD006QFnobMWaq0e2V8Vux4coCuNpSpipfMho0CqfYYKtF2hlIqo4 + + yUZM2UmgzlFZKsir+V54Wi8MF5vccowQSbIeUQIDAQABAoIBACrptXKucDY6bWHk + + 8Gv3+ipLERGfJSRQ3zhGXxYUhuVKHVHcT2ig2ajtJ/YEpc4+gKbIW1h6ifPuJwkP + + tm0cIyKdMg1JoHMjnOl+cajjyCPs97jMlFsbstV3FvdllcwnrZhHhvTTAMMEuFM+ + + 5tkxERjwLf4MCqnk/j1gonN+Lm3tEXptzSaSdnNfdkuWNDw8Bv5Yo+Jfu7Pb5aJD + + GYbOD8o/zAfuPMxOETw6UJn2Uy5uRms1OUbxSrQroarndkwhvcjm89o8VDjMM+sV + + qEj7aAWIIOAGnMlVeHILMBgfFJljAaTLeg7fuUPN2Z8UR9B2MVz12vKjsOane+WQ + + 8Rq4SXECgYEAxBLyWw630e3VXvpPRxh+/SRdKPrY34aVm6alr86TAR498KOYMwe8 + + bmvznO/azm6kmEIQXjuIt/5JP7/dqc+2SjQ0eAhbmQW7lSUjHuKBp6wh5DtJFR/Y + + UVzW5spHxdOSr8/NwPpYItVMXZxV3HQHs+SOPagiwRpwelCqxUF/d60CgYEA8zj9 + + Qd/7sOvQKqTTJFlJRpKidFXWBon+5d4dX4EGNoUicSC5Y+x5JyZf97WrybAaFH+N + + iAX8RlUIY1PHBtlFNNBtmGQBCObxgOX2LNnAFFI0iEMW3IzJQ4euga2+/Hy7dVTP + + KMQH2gU5p/jSa0lUf39VaUSdFGYCFWsAXSowpbUCgYAJNjaqroNWWo0mvC3TUkRN + + ElNKJJbh0Ynf2TF5lAP2DnysfJMe+qMQsQOuANrPzgTvnlL0iml+83RviU0ZuEeB + + Lvi0FvhutQU+GZOP1OZwgTbKaTqiwm9AS1NRXnmGwszmc6XgBiLz5/+BemHSTKU7 + + /2XrYaXYWqykInwTbmNVtQKBgDZALe01opRR5Pq+DQJ8j+WX63h7dOO8gAiRxId6 + + 5gHfLFGDdRaetl9PJfTApvKzvv13fgArJZwid16AX1JdwBwJqYhmNfzgVlnj8UcL + + wtZFh8YlAMJs/K99YiU2tfTndYC0TAjRwNaWd8fJrlWT468Und5/GXJlVm2kkk41 + + jOhJAoGBAJCGNXwTLn1tT4u4CSrm7aUXJpYzHYDt2VCZG0Ul0m11WP9UhrnZF4CL + + SYZqJinP8HoEmFxj4wwAmEAZPKJgB2zBJw+EEJg1Gh3aZiW35b4n7cjTuCpXXydp + + nFQK2C9j1mvjkjN91aAwDHs/4TKZJrE9pxDzFSfGVYB8hLVgTDNB + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 2097151 + seed: uqWglk0zIPvAxqkiFARTyFE+okq4/QV3A0gEqWckgJY= + zfec: + required: 71 + segmentSize: 131072 + total: 255 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:CHK:pcta7uk5cpioxzv5nxxqsogw3q:gnu7fx56k3j72bbevirn4kdu27yfpvttzl3qk2zypwqt7tw4rijq:71:255:2097153 + format: + kind: chk + params: null + sample: + length: 2097153 + seed: BTBX/ampNfLU+ox7xipBGiaSbgC0kcB8Gy7BkJB4oKI= + zfec: + required: 71 + segmentSize: 131072 + total: 255 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:SSK:mrqbm56ktfuwkaqvyjc2sxi23i:tuf7v3b6olgzfszpzaio2hvbwlvcvy52r6juxaxw3pfsmwkuytha + format: + kind: ssk + params: + format: sdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEowIBAAKCAQEAoZ1/YVjqmWSf6ka+6Kqz5zF9YcQ6hiF01IsTbMhCPVyGvn6v + + Xi/foccPEfNPM2RT+UNR2bARpcj/mnoD2r1/Jf1TQv7VcoDEsCNye4prd5c2hCTP + + DXhSAPAB9dHqvPs4EEon7srIpNq3TtlbWoDO7DKSDzBErGbxhu10guHoZBsYScaL + + +0R/KdiUQn4fHBVKTgJwOEYBaTwMpd7CxOnR2gnQKmEAYw/fXgYzNXAqni4AvOdu + + 72R8eMrOqr4MegG9DX4Pl4l45Hq3D1LCyo5SFS3QcD9zs3QPBzkqBAMzPIZ3VH6z + + eToUunHePOfN6AHWSv7D4rSNJQPxGfOpzhsnFwIDAQABAoIBAEpV5wEfpMhpQCTB + + 5Y2e9qCgYstVNpX7TYF1drnSYqVWqaN1IbRw0KvYo1XeU8+PlhBQppU6JuPaT7b9 + + 6Ef0YUdX/bQTAppoIA/kPgQU5tla8/hT9eh8Lzu/KSeoJhBGfMMBWNy86QzqjOX3 + + k81M8eAyYnwZ93xU3ULydWS+A+YG1KGluEjait2cmkgyGdu7l1tHhY9XVXUeJ3hX + + 6OUfSUNYDo5Z5ZLialAws89GvJxA9FiIQvLx3l31+X72wpfn5GdOvnO0TiCipB1b + + I4VOENKDdYhKHUJIIbBE0sE2i9y24GsSEebeRx7e9xmPRVcC2HJjY2feVGYho27w + + 8i1l+gECgYEA3Qmvf1mJhb+G9/UC+9y3PGgzpks+1PF/xIjIBw30PyDiyjvK+2aS + + bWXTne2ChRRETpMPnTSLOy1oE1Pn6g6Ud20VCB/M/c3qGFsiqV7CSKdj8vBViL1y + + +b8xbS97jwvIv+xsVU8rSy7TI4FkAPcfZUttfzr7AQ52yFUjjkR8nQ0CgYEAuy2o + + fmBY+PA78DyGzlcZrgsMeL8gaCZP4KcmNpOPika6tbUsIn/0gDcEWX5TunBjTfBC + + Hd0vh0ZSOhMHlPqOdVPh6hi7YNI1z57/LLZh3JVLJ9zcAsVQIdjS/XVVhq0vVl7y + + U1qQ17OEdCNGHrAit793iZZROOQWlvak40v+87MCgYAR4PeuEFr8U4qiQdI09xxn + + KXKMD+gMJ2CTUBEF6Q4JkSpm+0Em5pwPdz4PtydohkQkKucHazmb1sdlUNMgbn95 + + zXv3BUN6gA5gW/bIxl5mrAt8mg4BGnnTU7C2yTFwV56sT35PxDCXSzlO1Od24IZM + + ljZMJUQqSLY47BINLuL5fQKBgCMnDAHP7mWyGE+hzl9qFDSPdqQmoNtudonmWlLd + + m5OIfQArKkLAbRa3PmXgR7E38i5s9L3PEGIDXuXxNPdRpvd57W+dfXNNhzWa0ql/ + + Bxn6H8c4v0j17Xqt0dIv+wPz+nPqGPB2jcU0vadiCIUy5xJDLxvz0wUwMN3hLE5T + + s2npAoGBAL3K0N5zwUQ6WbhwUUOsySK/yRKaO/25gB32Ty0IptCRGcYFy0AlUAPg + + qk8LSRMSMiT5+pQR/Y1CfoL3SP1Uj4rIazdbyJX3kCuA0IsxbruDTJ5EPPHZxroO + + zuLxkMtaP3moi6ZN6bjlB/SoTzNgTXqewSjD9URejrUpI2qGIxbk + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 2097153 + seed: BTBX/ampNfLU+ox7xipBGiaSbgC0kcB8Gy7BkJB4oKI= + zfec: + required: 71 + segmentSize: 131072 + total: 255 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:MDMF:6vmdxm2fxaycmkip2duwcjjvkm:76m7xsaovohn77xmdijxemu4mjjyf7hnwh3kjtkzd2bmtwfq64wq + format: + kind: ssk + params: + format: mdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEowIBAAKCAQEAsZ62I1ouovh2L6+f6dWQdENE3E4h+ZL2VGOZ7MEJ+nQ8jlJ4 + + c27iOwYxHA3OivMfjOdqbOi6ROlmnymqNnJlBTRO77k8uSX4J+mC1+tqeTP31boz + + AOnJdSIb3uG+0ela8e3+vNPP9X/SBkDt6vMUaTpinKZvCwOSI6KdubtuvZSDZEug + + WZuYuA4E89pNw8GR2EQkoEyFpBaR2a8dv0xJRYMDzPBLeluFqhH8iTUWir/KcOS3 + + bESyRJWvPQHZk4WuKYF9VX0KV1w4GPOWmziEI0XoXS4Lf73yUfcT8YgUf/6T0Efy + + QhpAPJTKQhTeqOH4Enjv4FEWWvw9KFBe78/VrwIDAQABAoIBAD9f4R3xBfXZEBZI + + pBajQDTzcYTnjeDGKoUGuruKTvyhb5/aVibdt/OWbHxVgs36HFZClasBSMDgxGBi + + 1dwyac/3D3kiT0PCg+39t9VBpo5TWAjWtG8Ne3eDMY2PX233RJ9QqxUFwEwYjL/1 + + d85eZ/h5wAijq7gy0HhNg9hqw4L5e1boleRd058yKIhgOc+QWA7j1IyWe3/Z/ZMu + + +yADc8OJEIaIntls7mJNb6CXuQikxFWPtz5O7H42qFpsU5yPSDmjaw7MyM/7S26L + + l4I/YXYRsNXYqj+l2KPXSce8619P8TzTsGRRBiSKdRrbwQTKyHRjkp1l+DpV5dDZ + + QP8OC90CgYEA5vFnJBcPdApn+xL6BCTwqsu92FF+p7pGW2n2rYZTE9hN3vQs903/ + + wjLyrp2Ax33LM3eUkfl6XxOTfOHnZdNVpumMT8ARc7uL/+dsgydqNj9vnvepgiN4 + + fRhQPXftsfGnIgdLPuoowDppljvQ/bnsQ8lztBHwlstS7XXgUz62DuUCgYEAxOQ5 + + 1PRujR72L4LsiUYSG1mTu1echK3MDwh1l7eVwUk5SBTZUaTQpTKDid79cHcMUzPt + + cp1FlBgxi6xg5V6C/mkIW0tQ0zD89uhQOJKstr50LaB3dVEgFYng7nNa6GyeXq2X + + P+GsmZUAMsR5sFoG35Daz3zwrNabmPDxozZzdQMCgYEAmE0TTAW5Nzm1oSq+nwUN + + glWi+YmlEVATHi4fdAhluWyoziQRk3Zo+NVInkdYqjcXTvXJkQsJ3LG4Tl9cjxZ0 + + IgNbeSydVcmVZkpkkYnozaXAIwIJU724tCbYo/D3XKaVJifRQ8iA32SmRWFlTi7S + + 1VGBcHt0Qr4MDnXyXnO49NECgYAOSEfxrLGARyiwlZy28IBLv5m500clUL4msQRm + + twiD9t3S3sBM7dm8wgdMrwJPcDNSrcehssrjTUX6zcxRlyOFdPUIOlRonXscJgn7 + + sJgawYIH9UX1GqdrKI9KfM+xYH+0en8oQSSWF3rmM95n7n/lI8rblkKXJxIua/v7 + + TO3fJQKBgDPDQ3U7S1LTZsPlRwaq9abgnTSbu/Qt828RaCI1Cct5iflekF1TEjwu + + F3tZnslI5Su5H1BukxVyvS35SqKJ7vzMlHAhYp1T+c7l3GTAun6AvA3tEX/5heL5 + + JmityktNTsG3zrnPPiYmRxk9Tm1Ec5QPMwwYG+T8ZS+S8oYPpsRu + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 2097153 + seed: BTBX/ampNfLU+ox7xipBGiaSbgC0kcB8Gy7BkJB4oKI= + zfec: + required: 71 + segmentSize: 131072 + total: 255 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:CHK:znxyqo2qglpkhjafnuesu4foge:trau477bj5u5wjfpy4oq5e3aziajgbpyfjdv4svyf27m7atpa34a:71:255:8388607 + format: + kind: chk + params: null + sample: + length: 8388607 + seed: w6uP8Tcg6K2QR905Rms8iXTlksL6OD1KOWBxTK7wxPI= + zfec: + required: 71 + segmentSize: 131072 + total: 255 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:SSK:zf4xzfaaxdqql24hihzcosu3ry:6iyxlie44txks67hs3mir7k5oxruafv4jmnarmn4dzoqtu6zegza + format: + kind: ssk + params: + format: sdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEogIBAAKCAQEAwEUK2HGFp/yqwhrgRen19FcZa7QOshoBUlurIEf79PbKBxsR + + D4IQJAwsAJY4A6Vy5QM4U4pqtJpopirj6dTYnY2v9cfdAINRL4D1yqwKR8NJQK7q + + KKmS3kPY1cnoYlUz3qmykxtGJACDTRqssA0qvkzKPbR86beaFg4bT8esQw1W7us1 + + 1F1c1hFrI+9d6UO6a0SgwlZCy+zPMFLIeZJvE7bStDQsZWDKfLOmdFt4/+ks3t+P + + Gj3ZZVl/oxxmAP9XdI0zIoQsZ/ze0PkQ8Nbn+sJWJIcWx6JQp6a7KyCHt0W/8pkt + + m6YuZofNbWuWRU2bYyXdUyFO4dxAJUZOMI5ulQIDAQABAoH/PnUOB9QmD0a5auMP + + 5Zgkzh+dP4GUi84M9DF/s8mga6qydpdvZchx9FTavzHo+kE7D3wgb6OAL18eh3y4 + + ns6NrXlwckzExzezCaNRP/SM8/ATzZ39ZcqxGClKZQFJ5V5AnmqJzBKFgzp/RHSv + + G6CWxH39PJC2RWgHuvo7hqgC+RflJ3psa8x/4C5qAdMfSg09C0z9PJB0rxfKo92s + + kBIuTwhVjQrTL0tSntV/I0fFMRmu/ddikaEG5dtLpJEOd5ZPvqSXi5q4a1qMlZ1Y + + pEXUwNSMLHdIpGHs/BdO7nKAbxnx1NQYzkkSDcrfPMT5b3F4eIju2aa3jsz4hOe+ + + 8gC5AoGBAOM0DPiIY3T/pFdGlu3tTjAatx/li6iWGSmMnRR1adQvFA0k1XBUDcTe + + peK6sc/dgGUnkbhulLEeAp0oT/jeWXGpQeOzMgJReDDjuBIMrpQkYPt2D+o/bKU2 + + e06l1/E8ByLOr5y9zDO6VBHMjLk6wvdAXDYwbwDJ4xnVwH93zCqvAoGBANijhK9f + + sYlfqnovz5zqagddHSFuw9ju2yQCu6pXu8SOR3vKSzQvmKkkfRhEqKbiWtvDMmLk + + aqMSQTxdI7QG8/9FTUSLtq3j2mdHRBUKAVP8l7r31YiJ3V1y4WnU8FkGpLA9fb3p + + FahoSLBSUqifKrRV4ke0GsfrzSx17megqfv7AoGBAImY2koh/2m58MNSYtGRKAsG + + AuV0VRIyZOa+29qqCP+Ry2jyZ7jxjq0t0fTv8APdN4cLYbr6bV6euCKJaXVk43Js + + eRT3T1AMGugw4Sc9OvVI2tsvcxAAfUHJLwBAe/kCy6eO2NfqMiMZsxRcdtUu+yhW + + eAHxbyhhHAJna39HBATNAoGACnjdER0vF9ToCMAG6S7rsS9vGQ6hqPri3PrE15cm + + HHpEOletCvjCCGsbIPEwteB7Q+RLqzwfa4KWZLSb5Tfw04YmFgoq6nz5McTgJaQ2 + + LDkpnIAecls3uCy1eMgyVhtcGqjeSy/ZPCrOWLeiB0Sqa807AvxRzxg28s9AlwHN + + NcECgYEAsjgNzgVFMXftH2jzUZ1wsh+sWvcQXQg+vP4jktu4g2gkVY+OUVrM3PeM + + TRAEALeoYwBsKQilrSl02BqbuHC/KlhWRK5V1pD6/3DF2l+ZbqtcD+s+INC7qtEJ + + 8oY2X93CoQ6CoaSl/xt+MxGgz9/swmPwdil7RtLfSO/b799G2Qo= + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 8388607 + seed: w6uP8Tcg6K2QR905Rms8iXTlksL6OD1KOWBxTK7wxPI= + zfec: + required: 71 + segmentSize: 131072 + total: 255 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:MDMF:52vqpwav2qgidi7kfi4scwkdte:7neinigvwuvqtmkztw6rjcx442xtjq2oolli7gtewun7qatfseza + format: + kind: ssk + params: + format: mdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEowIBAAKCAQEAnpjFdXwFGIs7CTQpJqStcYsUAgW1WKwcB2p6qWdyEOMiYz8n + + nDtoy5TRDhIgY201ZbQ+cRkfbVXCP1be95R64h/jUO2edVZHD3SORUK3EobsALq7 + + jUtQ5wH4NVVZtW/r1cMIFD+iwPBqzRMAEku7MJf2ntWysmVu6R7/pCrLdemw5/V8 + + WUQytYOmDbqFH1CZlqpxGov6pxz3LBPBHPdwaM8xQtgHotc4NCCyoz4j9DaXaSNf + + JoeTgYBM1SrNYwQuSvJLV8VXlTe6xjEkVz7dRp8krDrJuDXEcs0BITdbrb1PV8fh + + 4eDDVDbggcjznSDw0IFMnuhqkfRJNLNSudiSuQIDAQABAoIBAAJPyT8FZACf1Og1 + + L61dxJ5tT8kYwrQsbAsqoOeTt6yp1uA59S6YihY/kM2C86BnYNoe5rMY0eWy1I4+ + + Sqkyq5jcrKBLGl5s98Owp/s39fmp6Eo5bo7obGE1nOPQHurfWwFmYpmC9PEZgAEF + + uCBMJMoYSPK3PC/P/S4eMs02h3ksPTCtCTWhouSpHRT2/fUxYi5+W8kXVI8gHgWE + + UBi8XZFxEdAIITAmXR9zCG1s63Q31ssEtakuiwp/Qbnk32yrDdkjAPsEMNoT7kwo + + BXAN1mGVOMlBLxcx64lPo/glbzgiJLQkbTvJnAKZbf0Uys9xwtZsRa9ZtJe6g6A4 + + 0NRIh1UCgYEA2hRWHC4N3JOlHEDQ41nrcK+Ubls5Yifruq5TcuFWplc4041KaXqK + + ttH8k+ziqGUjN3H3ouT/UH4GvY8IbBjGbESIXUOT9OHRG8PdS3zUCN+D0Euzbib6 + + cxt3jM+kcOjkspvAGTRABxqYhxsqsUEZ18OVIJnnIH+IEG3LgJCd94UCgYEAuiya + + P1yhmAvCEOorx6R5fmze3Qp2qt2/GVklxwR/rYMcw2Kf+772vd2k20Ge2R5OZbLU + + /eBUi+WUs3rEsr+x3IFT/Xuovp4FCcfi/zkUN+qYt+ArE2AvqkC1QOvUV+QJf2E2 + + g9BO2chGFs1ASjOs+gxKRoSZUsymJa+sPvCFAqUCgYASawtGwAD9sx6Lv1GlEfAX + + iUyw8VVsW9DF6Hk1x6BI1i7/dvxk4iua+yso1yXhcQFDaoWupUaG5s3s7oqYjpMb + + i8I0lkOFuBiwDp+/A2DpCu+YBPy3feVDGXvEUbkirBi8mPjlaAtMTku5hWrao5Pq + + LCOJKFZj4UF9mbhJOG2O/QKBgHkyoBevjeMVhHjOeUG2aQFMjqkHLsl9IfK2fklZ + + PGUQfaEUi2Gvp6FisPereGWPvSmnidDcQS3xfyR4P6S99mO+LZdO8UNmS5FadwP/ + + fJIKPvE1FdW/QEhtZ5Gj9NBiu2wZNQwKh8pu/nHJnJixm2IMri3KFKY6Y88U1eUD + + XxOhAoGBAM8YFY6k9NdX4Uxoqq+NwUfQRK8ScrZ5xF2Ay9phslaeZ6ixh42mJLGD + + 9Mve186qIEi52yNXP03bg1dyOJkzrRsCEXQcUXeFN79fzFcZRB8YujaRv5divmG+ + + L4Jg0GEDiLjvU74WMikbeejKaddrsV6Pv8A/IwwFvboi71e5ocpn + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 8388607 + seed: w6uP8Tcg6K2QR905Rms8iXTlksL6OD1KOWBxTK7wxPI= + zfec: + required: 71 + segmentSize: 131072 + total: 255 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:CHK:mlzs2hbztak5fxjkrkuuvnpdpe:3myvisewm5uklimp2xucrwep75sm2rizfi2sq5drhlqjszkpdyeq:71:255:8388609 + format: + kind: chk + params: null + sample: + length: 8388609 + seed: yPi3JHKKbWaEEG5eZOlM6BHJll0Z3UTdBzz4bPQ7wjg= + zfec: + required: 71 + segmentSize: 131072 + total: 255 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:SSK:rout3byiktrg4ntuwwa5zdyu4i:wxf4dojlr33qrry7ltrde2idxz7v4labogocg52mj2vrusjwizaq + format: + kind: ssk + params: + format: sdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEowIBAAKCAQEAsbsTzJshdeA57Vow4v4l8sxmZcyIHCo4Zp6VXqDXrBHMSDYN + + +NL6trC2T4CU3wQNpmxTrdK1ackBWf6b9tfygEACzC1rNqUmlcjdpiC9zx+Qp900 + + YBUc7BAQhhJRFIVcw4K9HJTL3UnrV4cyIuyKzLEQf5sBIa89FYREZF+XWsvcB7xK + + P5m9epQ937PGlHQIcoAlzFnq501JWWOjL9nALhI1bsDvuV/D8PzJeTAeo5RIK6BC + + xb/DtTuNwuMXnvAh/tj2GFPJMtQ6bCkRgugPtpIv+MKeiJ2W51GlgXHGsAUfnYTJ + + NU4y0SbCVo4959ACsXxTyAGFXd1rCf2JGr4IlwIDAQABAoIBABqx2iXzsQFm9dxo + + HQqjGKkQuVqV72Wnk8QaEp5dczdljvTTpaKXcc/J2AA19GZQ0goKoEDt8pCaf8j4 + + HI3lXoeT1be8JKvW/2Yk/uGqbkfzWkNUTr8VvLvaJ9kzXBlEdKvq8aOe18X7ic0i + + Qc4MEEbxW2SHMBLSos6eCLW4w99/mgPQlR+42Yib32/aMUfDMo3H+gDcAuf022GD + + CKOYDqQbgIFR+GDP+cmL+86h10OnyAsk/riNI3W/yCAW5OzRq6kZxBiogp7JJjcU + + BDOzMfTdHf5QGHD+nA26FZcmD57BAl/Gf6SHu/WZIjcTXu2wtF+G+Smeqk5KISJ/ + + EyubuxUCgYEA3ZCL3F4P/T3ct2TnYvbBx9CJcF9nH06uCwtGHh6XFFPDYFIk3wNf + + 52YZ0ayPXZgi+b7Z7b26nMEHzCwY+PP5NuNtly5HK8jJ5db/MOaSQfbemvAMh4lz + + j8hUehiOPcv95Tlqb03Z/i206Tsaweb6Yfg7MuEVKDVLXuowgWuwx5MCgYEAzVqA + + O7tIT4Yccb8JlIX6feE2Qy5ZYtPpQ3iPm86mwM9k73zMFQH7lKvr5GUkZ31aJFWg + + o3RKfI8UpF+denePJqSiYirHJ+t7tTtXpHa+pZsA8ZhEhgtY3Xzy3M97hpS3bBiU + + 9p8hstrWVL02ZNEBDtF74uzHg48a1UxVG3u+FW0CgYBUAUoF82P8kEfvAML6MrSm + + Hdr+UC25IQu8BDpBkTeW7WtWSc7Q/2aNRZjkdpik09nu9v2JtjXa2RUrxExzl40V + + 0oTqnRE++JIUIr/+um0ZtZARDpKxkNvP2BSvdj/4Di/liS9hpBLS3GGLTG2ItxqX + + qpZHZC+xXwOEqSZa64nLIQKBgFSnr19waHHoHofBsmhZBxenpR/y1oSISYw4AjO/ + + 8DxiAwE7WEJ8y8LRUPCZxXUoVuXNquhXQ3Gv5lmQ1TGsYgYTLqH7cpiBWkEvEoVJ + + MnTAvpXaKL19pgfAv7nJiunDGw5j39z/YvwBfQP38JmFE8ORFlpJNEKG1xABZMBs + + tcLNAoGBAM66r4drwUkNCV4OVpfjTvUFgIveuDwv7OHqcVCdzZPENMw47FrRSUhU + + rHS72Y47J/yVN0s4jGLfJJRjxCWGJwQAAEW4hrWfqK8XNnZi3tte1zMJo8Daq88m + + DC2aS9NUQRyYhMPDTVQ5uo6svKPbWSoz6mwERrEQSpmuGtG0/Um3 + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 8388609 + seed: yPi3JHKKbWaEEG5eZOlM6BHJll0Z3UTdBzz4bPQ7wjg= + zfec: + required: 71 + segmentSize: 131072 + total: 255 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:MDMF:tocfx6ufmjum3z2phuylf5t63u:cyaf3kny6cmws6oflgrpl3lc3d27m4cbpizlwqcrjkzvcpcd32fa + format: + kind: ssk + params: + format: mdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEogIBAAKCAQEAlTYNp0fY74mV8IsKNY4J8PWFpAL34P06ClTUAJ5/jA4UsX5E + + o5mfu5XHibCE2FwT9eTvlQMFH1Sq+OR08d//S/V4CGC48S8Tb3OtLTWTANQtNlfs + + VEzdvnyAZwlD8BOHTF6rothwaabsdiZkzijKxFvjDMcBsvTdXbzdbO504dgqC/Sw + + R/hYo09e0qNnM9/VcHQwPK8HvJhPCJGlt3XOYfQX67pJyvBIM3ly7BWCqI0t7WbD + + ngr09KOr0F+AwP0i9HQ07nEW1cwgkHEdKjMkTuDdrfvfyjiVL33OXn8l8kRbm+uR + + 3RHQfoYKUMaR/rn4ibJv1coFuOtO+izCiGOfswIDAQABAoIBAEawPPerChMxU1+J + + /2QvznXhW1bAMT7duMl8NpO0gyiO4y7TayE2fn4YD1gj0EvQE4TC2N33eE3Hhtgz + + I1QTkpchy6PsbrGUY9jBLKHmZ6ZU3raIIOYvJD5CLXKi6RSrq8V6dEXJ1De5ZPz0 + + Y31nxegQwBglj6CAcP8foqcgsS7syIuseLSuRWEy6GkvxK7qaLc3L2FoyTqftL6+ + + Rvaoc0oqU8XpTyx7Vs2NvB8RWNhb2T4NZvqP8IOAZkRQ4ueO4zU6vv4KHF8+zcml + + M0cTC50aCrX3zgMeUUVWm6k3vFK8uO3mdt7k3YNfNGXnwrejLaPnwBzh81kpD+85 + + 0VeV+0ECgYEAxfDodtZGqyff4dhPoYFeKvDELANVw+ImuSAkaVHQGx+ZP6hsxXA1 + + fwSnF7jitopiq3uAaYPU1yiGlgVCDbWyNhZAn8eRbzX9Js5ynMZGqyXdqhmVDxVa + + g0XPQ56qMYBlllk2WprCYbdRxvnNmGarCaei3vZHGviqk4rOjPXoEd8CgYEAwPoX + + Vf8PU8cXjAypzgT3izgeg9s4EHDDpeIziaoOGkVXwvSAdsjM1R1exr66z/7cdn10 + + U9YUx7AjVkQH87upS2yQr8tPC5yunNAmw5IceMNiQTvb/2I/MkUsMSyChaxvqOAI + + fKxY1DnwlmkDD+Tr/tp/JCD+ufqovu28y21Z9K0CgYArl4eKjFwZ23k5wqqe1d/I + + MyfwzXc44XhHsuVx8FuVbZsRYuU5giG17G9kEQqUyts6CsPX+PmJvNoO9e97F3W8 + + 5Z+r0Iad6FTtE/A3yI7NqFQt3t5t6PT7Dge8S5gNuMoml1UaFRUT8gxndqIpmwq1 + + 4J5E3hYAwZzHS317m7hVHwKBgFjoYCv8sSEWDuE1TF5gp3P6zQRO0YuxiFI63yfD + + s2+jFwX5A962MLjXKT1DzmnZr9Tfg+LENRqzKfSqr0c55IudXyO+9ZISA9i3hcSA + + 4qE402HepEMLDraoa+3T5eaURXV2kjJubRaKAzAo/YIrJBdsrzsEAJfKxkgA3ASV + + QuaZAoGARhyMG2qeoUzs335JXPJ3lGEtQFK2tSeJxGL/zT2lqLy9orHz4e1dI2h/ + + rj2PNI6LJsKANOdGeGkNgyXzP5RnTPMQ10tK1eZRNYeGTB3u2/E7psDFJXuHxB84 + + NYxoNA1n9G4jL/60pPerHUbNEXXr+v/lqdPFcPPa+MlPaFbrfUA= + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 8388609 + seed: yPi3JHKKbWaEEG5eZOlM6BHJll0Z3UTdBzz4bPQ7wjg= + zfec: + required: 71 + segmentSize: 131072 + total: 255 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:CHK:ljiodj4tijkfzej5nqdk5h7cnu:3fb7y2osytliwli3oez3y7ece6rjwfrpa2zn7uwrfmn22cisk4aa:101:256:56 + format: + kind: chk + params: null + sample: + length: 56 + seed: YQ== + zfec: + required: 101 + segmentSize: 131072 + total: 256 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:SSK:fn4mrtlcnpcps4rnzolo3ogiau:qko67s2m6qeg3tdbljexw654dek2rx7s6hog5ahkqiyq5cejxmdq + format: + kind: ssk + params: + format: sdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEpAIBAAKCAQEAxQnq3zmS38I+dYnHOhTmzY2yQ/IB37CvcBvEHvjnPngQtBAs + + OwR+0cc1GYMNkOpry/b5W2HtFDf4JOKWF57z+aNal6CrV2QHfO7dcLOj35e1V4hA + + Y5hTQhal8gl2qEPIyLg5sJrXebmUgqnkXHoQmnz7A2TJlSuXUyadI6VzFRLWg5Oy + + IV0Hef23zjXUm5n9IN51r/tyP2egkalDjlVd0rD1h50zyGOo5EXppktGyRkUy3CC + + PsqH8ZCbR0W6N4PCsWGrsu9rm0DTjwxfWpbnYJqBJSQWRmEZ4YnJRVm+MKXfBV9w + + 7V3xekGMzA+lwYF5Gc3i7bheiCrtST1RvlHtdQIDAQABAoIBAAb+o6JaAl9EH4B/ + + rB1hOZJJee8Ui8F7nba+nZc14cujaoBh5JgRwEjFKBroPpaK49nBQjfewZJKrFnu + + 20IqZ+HQTTp9vydiiyuBtUW3ctVQpuTdFuASO75oXGq7sEUn5txNQesFjCmrj1yW + + GF+6C5XYYvbLYKaVfhE7GS/3Qx8X1KLfUx/qhM4DvW+SEciW2zeuBVb0ilpuUmkX + + i7f9444eIZ2SvE/jdPL54Iym91wW5icXs9YNIgoKQoPB0OCUbAsiDIrIhvXemHOZ + + ltOPlccfbxxz1/B1fcP6t8aYvVHeCDv7aPgWSbPQ7J+HdQCimo2TshgpuY58OZbh + + 7rR/ECECgYEA1RsFum5dDpfqhU6IyQpR3CWVx9YVJRU/NRPD3LYYDRvoYv80TeQz + + 6ISoq/JOhmSw5Ff9LTcPsX6dUVJkPqPYyBAg93D2bunFCEvw48IpgfXzFAr5v0ye + + Xg4dTfyGRhyCUyvv7hZsWyr1Wd+YB2OoTHjvMYD3rwGmpbywVpwC+WUCgYEA7LL/ + + orl57GPpxiIzB90i9S7dZC8w7WGKwUZVHBIbXSW8HNOGrO8M8VjduDm8KKVM0Yue + + lZf9ut2nIzjKvtIUp9JDi7pKOOUHj2F5E7RU0hweoAGv7543ybGW32IT55bEvRbR + + 8bOmEfq4e0cg6Aah+GVsPHi2JyrFMV1lzGBW6tECgYAVR217ACoqmuDADud5q54g + + 7V/XZHkYCtcU5bRZBZXBOVgrCnCelnrYbOaqxLcylDtVkbOmIClg/9OVmzSHTLUI + + xROFobH5wT37ZhnXpDugzn5HMhFeGLh3i9FBSEXgGlipFWoPzA1lzRRStRDpK/pS + + KIE54DbbMr8BLaYt/8YMQQKBgQCHE79/BYnmtT37rBijLDd+5DfDrIqnbTraAWEg + + m9Sx2472hGAe4GzqbmRZddlC+NJV4u+lPw+1TDjNiONq8kiHXR7e5njk7w7ZbC7E + + Z+zf2tw/Q7c7b3c2yvnmkPn697dekV9OJ89mA0a0U2sb/m0AbCDQgbKxt17BRSOK + + 9o+jgQKBgQCKs0ujjCu3DvNODuYt3OUtziFMlF0om6cNCR4KgVKs8h/3mY8m0J9z + + tna58i1IvDq95yBFwttxSIXPh7y7RqDlvhu/2yRFtWEZw90nOmuIjzn61rFLuly6 + + FCHWEBOCD4jqKMLb6qmH7fQh5+n8X+/kyIn9D4kKFAFKBd22JIkD6Q== + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 56 + seed: YQ== + zfec: + required: 101 + segmentSize: 131072 + total: 255 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:MDMF:xssuhuqvupvlwchl3sprk2f4o4:jziwrrjb34qffqrmdxihkgnvs5wve3drgw723zynz66om2livyuq + format: + kind: ssk + params: + format: mdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEogIBAAKCAQEAqRRBpklSNPInOpu675erg5aYjdN5XZXES4D1sSKtE65W9rcf + + DBYX9OATQHR2+CqtqAEriQc6/xYjyklDC5935ZKa/H9hY/AS0Q+oIIYNXofR47aU + + 6h3jjNYg+Ynp6B2EfSkcCck71TpkQAht1xg4gpt5F+wWbQW3y6GVcw4VG1sr5cNx + + st2eIg7yJiHFvnpwRZOQiSZDRSJ7Ar5EP8eomyqb6y7UAgWs0E9217Jx8IYSbZZ3 + + fpuNpgR8B0GJztw2bcdtrDZL5Umfgnpv6Edh/omJTSo0zMbEcBnZEv0W+q79+q+S + + 1PukFCXy1RdaqHGgvKyRb/pfI2xp2eWqwfVSrQIDAQABAoIBAAKSlX8iDXdTGegJ + + N4/EBy7iNeGujoxZ0Nn0lTMZRniOsLG3TpMgq5j6ZytAumYDQY+AA7llhpnGrKER + + eqIsznu2k9d0rrbWZSA7ieuDbqT6EbunwF325MqfSsgmjOG5v+rRmyTGTuaV1Ab/ + + ZIB0f2OervnUtJtQY3VcjjQTykbCFtmSF4NdiFXR9EOdEwH4UFg26f6idaoEhho9 + + qzWt02fVN/hyU9IrnMMbHYc45SPEAW0/rVtFlxSYWz0l5/Ma+jnQzT4cQzQS1/M1 + + JOUORANFt/K+q7Iyddbu5xVxClzedi2pwekJFlQXRcVPXZXGJ7X3KVvfbx3GKUHc + + Hj2agaECgYEA4tMDj6xBF2RQoAJvQgpHCxYkarFPUDmM4+qjEUbQ09tyL9RXVzf3 + + 1MlK90EJP89MakYOb/T6+MMqSG5/xvkbrJO7kBMfyDmA7IaVzcMDdmUTLuWtqWsg + + mpXhnBO/utGEc6jzHisxMwvFFP7lcHq03WQykNUKSLJ2WNvHYdOZ5/0CgYEAvtPH + + 3NwGlr0n22K4/LJdKTtvGBiA/1TJ+XstGN5BY33LjlzaElVjetJ2c2+HmZeq6Omt + + GXqUYdXcbSUxPcL8p/ElDYAOQuOyGeAvZn2fub3mqFSWJZSGymCN5TGGv6p5NxCW + + LzqV7tOCZ62Y2LpntGG1b+NCf/0FNItDyLlzXHECgYAGLYsmSaHIOlI72XUgTllb + + AvJg+Y1YeQjOWGCyosQjURHOHbF3Ta3xXL4u99WBqGrDZj8Ua46+YcpwCJpwV+6a + + B7gPF4ZBFNffGVdRMGOSwPQBzf2p5KIRs81eS+dn9jbuU4azpqeDZWmrxbmIE7+D + + XCxIZ5UNH9c7WlkW4AWMHQKBgHcuR74dwRO2EcWIE+bm8x5EW28eJrrRVs+06YaF + + kSs1LsO8JAqdP+M+vPH9rx/zRK/w+cZW84NjESctumJLfIbbKfwThVSrZtmYVaJa + + RT65Zuys35Wa/NA6m4SQeQsNymTkvBfFLE0b1m8wUazSRuC2wZ2evzK2cODPNceQ + + Y4dRAoGAKO7vgYPdViIDpIUh3GW31Hq06VunMGgyEEq5ju2tdMrQuFQwKdo/ZrKq + + IePAVFWwA4YpW0I5vTR/p0OtD7iiCz6M/uVQ/D8I4LQMEzRamh6+BQUGHNWmR+Fq + + TCwwpPlW6D8IL6YiB8B0gNMTqJOdGw09DuAVOhbW0KMepvwkvq0= + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 56 + seed: YQ== + zfec: + required: 101 + segmentSize: 131072 + total: 255 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:CHK:mftnvh5l3brshwoupvu6kz35cy:sncfokby2tjykbr3zgi3i6lhgerg4kz7fsrslrlxdy7upwkzq6lq:101:256:1024 + format: + kind: chk + params: null + sample: + length: 1024 + seed: YQ== + zfec: + required: 101 + segmentSize: 131072 + total: 256 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:SSK:xyd3y6wxlwoca4jjp4q2hvmqt4:6p5fnb3dahfki334ykjopeodpcwat7j7hkbgob4e4oajyervreza + format: + kind: ssk + params: + format: sdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEogIBAAKCAQEAr6x6+RfkAriq3hA67zbuejAfXnMBbAMGf+K9AHsuYlJzw8Kb + + eoqtUFFvGMG+qB4zAI5zI0LwLzKda1li2tTzRzIlQ2ZEUKAoV9JD9TCMUr71s4mn + + jM9hO4sS740VEPqwArB3pg+/dPsFAKSNW5C/o5reU3/WMOmhUXQ94td1TpGvixjJ + + hFtPQdfhnTUKdzLU3Au0WFwB26LpvPRDKmGjJ+pvjnmWtY4J0IyPdgRVXXe3VYhX + + gM5N4e6IBY0hM7/HJNYiiZS3jNTXWZtPsij6vyb7vbjWTt8n1Or/Hj52JxBLcsrq + + Mq6zMgzKI7S19U5ZCvqiWEJtNXQPilXrzhtScQIDAQABAoIBAB7JAlTYEbJDY9gd + + 7oIAtZpuh/TAgSmJPzCWjqoArDB5RAW0exQmrLgUSTyEqVFjV2s7y2QMxTP0Mf9/ + + rtHr+wUJPdv5ljOt2VwIgjW7z/9pLPwNPbIwnli1prgZmG09DS6vd4w/mrzYh3gl + + HguDin3UdC0cTDAOpSE11mmD+e/uT21SxDFCXY/w2zfhzf+Bvk4iuZPOGveFwz8j + + oPrHG5ZuEtS64b0j8JnFseRpkWc+nVCqbP6DJHDy089/C0Auu/+ijk72jh1Kp1Ce + + j932ieDY5dElfJWXJwOshM8TG0+VoZ5szEWBegGpw3syGIlGZOc8cm+kh31WBuIE + + iSuwQzUCgYEAuomJoOhyaicT1VS7WHY6u4/N5w9LRDulrkes3PLv1dbq94a3gX6v + + K1n/wvHtTrJrus5QxsXExJqXmDPI6WniKA15zAhnk+70txb1FwSEK7Mzg8hOKo+T + + U1SzPNfq/6TLlJ261/+G1wGFY73iCP1e8CbmuznTo3HObabFZCuJ58UCgYEA8RdV + + KJjpD35+uKCceJfqHxAF7Rq1WAn/UlGzWWP5KeSt+CBq8dgxjjfoRQKT70ArXU0p + + bf4LeJHpbHiuMe6oCWtk9y9KNxuGt3vSSMhWs3nZDP8+3XQEItsHPROACm9eDzfY + + PBlcMqD0r85909n5C968qOjs5K7VO/nhDL+bvr0CgYBE+tVtLmgY/yhjbDj3Zokj + + kPMYbdxseA41m4W+EwxDrH0pWaUEev917YsZ4PLbdjlGVEMkrj+sYGqMuyGhxyj9 + + nLYckEMVPnk6N4AcqeviaRs0sV7OeFeHqju51TKupJcv9wAAHhsT6RkVoEM1BdUU + + w53xQFoWB+DJRbGa8ErH7QKBgFsjze6d96UC1dbn6J7yFvCNNyBOM3XHubyd5CYL + + 1BqRN28Qmj041GsGGYlVEyWj5YDM9bd+DUoUJuD5sihwJxgAgFetienRPxlH9tPK + + 4HPSwUnXiCVhgVrH4DGnmITZWv53xwfZMnB1RmrbrdeTlEF3f2x/OWat7TBSI1CV + + csQlAoGAHsFeiygmDgSaCvxlWBqMhG2Tcx/kIdzV9iMUsFLzsOrkLKjCrFZhdzjg + + fliN/lhVltkttVgaWeoE9oKheEwo0CbTRog2S1xhQsdG/vaAUNcTTSpfGUQiSj6A + + 7YUrhzG5vWMqW4P7pBYtf7C0PpgQExEBdvMkQOCbuTKitpCudAc= + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 1024 + seed: YQ== + zfec: + required: 101 + segmentSize: 131072 + total: 255 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:MDMF:24djav5cdiafx3bkazhgzjt44q:4d24tlgaq6dnutv666bsnu64tzzx2u7kfroa5pmxq6vf5rr4pi4a + format: + kind: ssk + params: + format: mdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEpAIBAAKCAQEAt9YOBJoP70y45U6H3ghYRMJbiG2bRIW8FoYH58oaqA7N6En8 + + xtvQX0Goq9wH4WHzNa4twtZvLeI2PU1wCJ0Fc5rkbUlSxYBzzHQaZa/fPIQJfomh + + VdGUWPO65BOSEublp2ljpar0iBBMU/W5GuSsXVQTxCNayiGRkiYTYhSPOyDkRiAU + + me0vLylHHft+5wXJSiusndp22ZEgR7qKCd1dG8U1F/BFXGXPv1EKkm20n5lRFFzL + + ISFvli+fXiCwnjYwoo+f2h5eecojRZZsDYepPnUthc+tM513l5Hj7yFlPfpGfLOZ + + As2dwnVH/djIdSPIwIujGF4HwCZPwfiH2oIzLQIDAQABAoIBACnR9WbpGEJnKPGr + + Q3/IVLIxp6p5yrZUGQVjsLUzVgyQr5lOCYXAeB7PXDxaZeWJB4+Y49qctvaQbSfV + + b6zZ8aVKoXfWFBEPZ2hlqiKjV2yYGePSEeRotK9mpMehRxvrMGe88xj7Mr0oPgDk + + l5stVaO2jneSVmNArzG43TR0+l9n+FObmDfUcoUZvE7CWsOFmwXO8Max6Wm9ZheB + + 8T/bfaahbsXfYVRUjgTFDhKBguePPyPr+7kxInJloZs/3oiIShRzRPlYZK4YiuWd + + TJjg/xPxaxnAsmvr+Pz8Knib6tqiErwFdjv5SnLy+44Tv+uZbuRkAjnKE4Nads0Q + + M5uk+WkCgYEA/7tVqv7yCL+5/izvuCEDPKGG2Ln7G1Viwlt723HccumMQFxEt3Qk + + QAbM3vESp6GyzL7yNDGkaDegzf4QlUiKwlgRroUM6rC2SBRtqykbR4m+eqrO1bDE + + Phn+k9AL/BGgGyz8qPWby8tN7jPKa9i6InljOry6U+wiStaYaIsN+PUCgYEAuAdq + + budrdeBt38o9ov/jMbe5m4uITEu64HWWZowS+FoIRNgRjB4qtZCMmqqUVLOUy845 + + 2eVFDGz8DUqo1s1Vfvop6seQLu8I6OVsLkyhB3FRK1nVKxtRKpqYpk+7OXQuJz1+ + + H2XRkrPxXECXe1h5LlLRmNb928let/8ili4cTlkCgYEAh0EnkCcDEAmHb52ItBQR + + yDGORnYnD0/byfvkyC2ycLyBR1EFrxmoSozOMmPCgBKPpKahJ2XSFKTHUeu8DZiu + + exdlUq5gJIiOABV943b8TJvXuL06Y974C/hnovn4PLt9uKHUh/BPFDxU3VVbDCs2 + + VyFokBpdWiGcCYTyWuig3TkCgYA1Hc7Wm+0kZNbR1SndNkZ5PzJPdwKsIt+ZkdcL + + WjrPfA0O8d5+tuZU6ZfrvHh7yimUeb2w6r/3Si2mGHqLJVEcCVC390nighPsROvo + + oS2JXGe1P4SLoKLYzS5qMnEzsBjyMomIvnazBUUQ/4O5klvHxxfAKa20FndEXFu7 + + RSveYQKBgQD4mdczo8e6Nz+PBv7jlsd/mRjnJzs1Q1tSdhjL65IKJ673MmB4oLTq + + mP6VE8W/bcy/6Nim5OrpfKEXzXt1GATO5UTgk21BafqZHXMQytcWcdD7LA0km8tj + + giBvfNSc+ivMsbwG9dhlZjAHHJWVbNxfdDR356vgTsVLSZsRbCJLUg== + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 1024 + seed: YQ== + zfec: + required: 101 + segmentSize: 131072 + total: 255 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:CHK:sshu77h6opnto3jnngot3lwzl4:4cjhp6u3i2uzdzrjitje3inlhs6gnlgwc6wgi6vtedc2grgvcpza:101:256:4096 + format: + kind: chk + params: null + sample: + length: 4096 + seed: Yw== + zfec: + required: 101 + segmentSize: 131072 + total: 256 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:SSK:bx2cqrsph5eo2vxvobyrow7rdu:t5ayg4kz22cax43uwt5auiu4dnmt4dllohkyerr3dxoxc4nahriq + format: + kind: ssk + params: + format: sdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEowIBAAKCAQEAt3knxNRwWNFiiJNOD1d1vq2pxaqVn7lxWYU0X8BUdUcONvW5 + + i019ISlqUPU+Z2KOBh8Zbpr7k0zMktAqeei/CmTHUcM4zmxZGFOFTgvTL/kZu66Y + + 6DvO20rBKtPzpFpyySeqUtstHKwtz73DZworSXljDdj6nUSySQB6JTs9wk28IsbV + + //gWrcZNhjxJf5rsYy7cFkcKW5xK9Axm9LIKTilf0wNeRpMiFO5u2rzKTlQX+UCT + + JojhpI7PnSMuRnMmA8D14BaJEIXw2Tb90diIqDuM1/DE1i2nLLchGGrfxkPJqDUv + + sVpAM1HD7ayStU1Dq6GvY30EICLseWoJunTiDQIDAQABAoIBADXASOZhVoiuzy8z + + +KqF9Pjn59UBJNSmf466d52VuyigqIlxc+pbyUzt1TfioWWoefNRKSI+RXXiCgz4 + + 73jHtzBUVhCeIQZYt8FotqUm0bg8Qk252RIwc2nLfMwPTFHaLcbA2CVuEMlVqBY0 + + ggqt8ACWj25/IuzwM0sv2JkPwggqPV+XAf3RtdsaJYG6i+N/pGOypaRodxcR1iWj + + 1WJq2vfzJeS20redrsEnP9K7SfGh4AV9Juky4H9W1YbON4NHLCyorRvU8lFPf+6Q + + MVss0UCIxJpVzk/Y1BuYBOOmgCeJyoNeANMjb6RSTaADD3UGaqHKjDzoqblgasrR + + o/kddAECgYEAuQG/UOdy1hgpoT6upA0/7wb5zpGJc4hmZUwvz/GONUzaNzjMGwOr + + SQIBtXVNd25ntGqWpsuUlk0HsJJHVwB/2ACQyJS+YG3qcR8Dz1Z0A4l0gWy+w+RI + + 7XERE6p1U/ui9gTr2QyKWZ6Y7eIAl7spiwApmdDeJNtr9UfbpVJlcGMCgYEA/eDB + + 88BdT1jP3WazAumrGQyKQvibFtTMVY5XiUaM07AN6EjELU8PcWjRZ451xJ9W0fc8 + + Gsb72FNgcWuphERC4qeagAHuKCAF74oywYfKU+AjItgR4231UBL0IEK2xnv9LDWF + + BBGOKC8mQAqYkvnZDyx3eHXBV/zmRR+vDF6nls8CgYEAkVjuFYHAlrMlAaldS0Wd + + lQzF9aQheMMQr0TLy3LbZsSaLAhTUmXvi8wny4f89HeowfV7pk8KzYp3ICHMKm4a + + AnlvRiaV6uxv46+aLqqdOqoi/guRVBVltiW+ZNTmmLR5sw7qu/s+NmqDe2CzVoGU + + gb/+7vlJjWtVxb5OsfOp/kECgYAN4jy5F8wCitjTQsqHXj/9HrJw9yeEGB8UjrQ6 + + zaDl8rrP+SrBT5GIojLRdvj5x7z3vo2K6VbcfbLIgRrEIPeHbaMFXRWpHBc3AlfE + + PajS7W7+eNKBnYHM3zx6hyt3r1ApGsQrdMpRaEKvPeUaJI+6RLRD4iywoyP0o8bu + + 5j5EAQKBgGa98kvhw21u84umuGP8H1ASAg2C3rGO7uPiq8VKgIoYbSwkAIeYcR/q + + Ewybhf8Yc8Y+PtLMQ1HlaJhUp2bw3QxV3aqYZuZ6rZNhiDLIcmSeWYxPWhwG46eE + + UWbY02wv+ckMPp7j1+JLrXPdfUdeGoIEwxsVTO2vYnZHH+GtWkyB + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 4096 + seed: Yw== + zfec: + required: 101 + segmentSize: 131072 + total: 255 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:MDMF:oc54ehxm4xmvhrkeukpq3mft4a:blz57s35sxrodbg6ylpxtlc5icgssdxbdzb2r5m3guuwpvpux52a + format: + kind: ssk + params: + format: mdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEpAIBAAKCAQEA24Hnx8PM86/KeJ0h8Q3H9g+OP7JdSYhZHYkbXTZfKAUihhs8 + + 1PY+wjUtaqyJW4FO8jxqQi8QoliLLT/4Tj8Is3D7friw5qlYoxU83umEut1y7GXC + + j2kgDi2SkBCcLcn39Z+Eb//d9854TDeV9xDyiQ7GK28n6h0bcEzEWCYL6XdNE6W8 + + 4rUPKfVNFqtstbrXkLNr2BZ9anB6Hv5kvxyCSyEtu/uOhFePnBRBFX3MYHuxL3lU + + y9CUdbtIIek9ybFgmIjnB9SBJ3M1Lcxa/dBHFy352X4fbpMjPoA5Lj9UBOXy0WKG + + jCJgulkSJsdqth9KY6pzYJEmh8xwF3rwz7Hg0wIDAQABAoIBACMZ8uaO+Qc+5THE + + btkNSxyeADFPZHuNwjJm6mlNeIn9yDeJw4CKoB6OQmT8kjp/wxAZeSR8QjyzzA3A + + XQSmL84CEzWAc0lvay0pCELdNMxs/SOwYhxswyOBRh6jiVYJJg2xJIyEbgpiifom + + KWUI4L/qDOaFL+zQGsMqg3tVGjKLBWtvlKN3d7gzood4SE56Vr0hUGeZ9sEXZrJ6 + + kcIDND5O94FNtIK53y/+lb7mMtKUVoraFLF1lK7UlT3Lh9T6iTax6OHjbHtQs/6L + + KSLXumtKNaLH6feU0NaEzAn07Xmr4B/6KW/5dOZzXrpstVegtECTO9QrLZEIFLyO + + x8JjqgECgYEA4J1R0iZYbcQtZHrS21pXWv6B9HNfMAwMNyZH8KAnH+ngg6FWEeDJ + + D+B8PcKLCBUqMqSkDRVty85px8drmsyvW+bZfyqrSAGKP3s+D5GuGk6kYDHSAo62 + + I2D5/4BtC7a9jMHY2beH6RpzesN4R6ttpQhGvKAjuseiTnYqIUi00cMCgYEA+i3m + + euGWM5Rnf/u6eoXs70ulNxPzJA98sbqJy1yXCgNfZhdkdygXPwDTq9ZemLJTOcY5 + + vK61feU8S+xBT1YPvewKF0IWgLPag/tFzTrd2EnK2Qtlc0MqkidBYAMAOFCqTxo6 + + 4ujbOkPY9T5Kc9b0lIUGyCicR37BouJ//y5hM7ECgYAjFA6iLkDjK58XMSNbBHne + + CR9MiPQVsdv6hOz5RFm33zOj+v9RHXTpGNruXkKOSZfkftfr/yu9h4f3nkpMy6ib + + Rqsy8/v569umXF3t2oeBLkT3jPBKW/VQAyYn4+ujx69Em0V9gu8j1XCxfHN9ZeVi + + v68kaDIMSn8rl8Kungc3NwKBgQCBDsym91iUoyoBS8qXCh+AEnXYQ+JZ5+Nbi+8p + + iUohUDwWXlrlXTkgtzx6mMuT2eo1E50VSMs3dtn0EJxgYPUd9HYAKYeSPTWsgCMy + + C/wFZ4vNC6P6IdwEKVwAO4wRgQtaYx2dkKIHHJj/anLd7zWcqEMnXkvAVhNuA4ok + + Cbj7AQKBgQC7q4JbGdNjqxuy5ftE9u31bkhgES5+MTrW59ov6tJZK4qru4RTeqwr + + QmvudO0s4evjufev7eL+kb/TxGcM+o8y0mp5VJKRQyO6+FNdpr4loJUg2XPtHyjq + + 6YYfMIpJWNPnt0f4zip0jOd5J8Iob6mJe8C0DLHsAJvfmnZFyf09YA== + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 4096 + seed: Yw== + zfec: + required: 101 + segmentSize: 131072 + total: 255 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:CHK:kl2jayrstlf5q3gax7o65ybivy:mnn5doti4wllssdg35ymojuiv4hd2ir7i6asurgms3ea67watm7a:101:256:131071 + format: + kind: chk + params: null + sample: + length: 131071 + seed: LCa0a2j/xo/5m0U8HTBBNBNCLXBkg7+g+YpeiGJm564= + zfec: + required: 101 + segmentSize: 131072 + total: 256 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:SSK:avkuy26pwv2j7cjc56sapn3fim:buvkvpbbxr5qmqiurhuidhipro6jimma4mjzliv23zfpozxuilka + format: + kind: ssk + params: + format: sdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEowIBAAKCAQEAuWyYTFaC3zq/JrBkVOANkU2EAkKOxOtATyRTx8qJCvFt07gN + + ZbKwcj/VXeawmris8SjQFofHftC2eCoaLdIi5D7sHlErjELitbPm1OruM2N3I3tO + + rLDyZwccYDqLhHVWhL7EfQBJcvD4DmAYNEUa9dfv4SHEX+wke5jrxLfb6gob2XR3 + + o5b0fBqoGCm8cUDrFodZh4g2G4hMYP1nSPyIUFV6ElFdnWqMTZB0kfMYelT1EVF0 + + 440xKjWZkTEXmNtIeoAYbnLTkwvt8vA74FOq8NDUgyyDx61ZjfaQkH7mQ7Oqhnyc + + LPX2ivaTZA92b6QnxxlNE3sV/QCOwgzTgMW/0QIDAQABAoIBAAG4Gqjhh/TZIvbR + + PZrmWWXam8HYG2ICwt3A+thgPblI4AFtpE0oNRfYFOq6FfLXSb4yKEy/LUe1GG4A + + iO3aFAn89dw5mS9jmt2/qWEZvQPjtRHyhZoXCWZQY/BV9p9vpZHVQXXdu/CZgJlE + + hZDtf5ieLAqQsTUI99UgB7aTFFJFCe8B2DHravbY/n+2irdBw5FzFujGH65tlDQb + + YSEKryMu/5JY51hABvY0+Oh4JvxqzRu+8wYwD7li0JgUWhgbMt414+xGBXH9ioE9 + + Tk7iZGb4tqkc990ZboT8dSEK4XQ7Kmcc+8+DUV3BS9NrxrhiISLxx35bPThDMYMT + + asRhyAECgYEA18pr3/otUnSsRKh6EMYi2Qn06si/vu5h5W0mtf/PdRdaCDoPETXA + + 9jFR+vO7P0/sumfd48+mOeqanwT11AbytBr9Xq06Ry/B+hpXzMwiG70FNn8Wia22 + + sDNKRE5GCDbxsQgp9CcW8HUJsAM5UO8cQ9syKNKRCec5JmxbqCWSO9ECgYEA2/mj + + 3cBulGHs88D7e+8q/oavqv9wzHhnEiNBBvNpkDKNetzZ49rJkcxu4KdNuQLkt1m/ + + FBnRMzPKzlrAAnEvol4IVK0S9ZsQ3d9PlFZpyghncnuvL7zVl1M0VVEfOLC0A82u + + zEHapqCt66kMb7Dwk/+9dTGYpi3vhvojnzJQRAECgYEAmuSfnjvzwFYjOX09cUDn + + zqbI+KZ0jFaMSqSYvtcKUOAcLf+OxSmygoVQdTPyWjXClOLtcRKiHLx7lF15H2KF + + YCZnbEgnpuVu9VlnYIe+i+6YCVAcG2Nn2P5X9sPAnTDjN9HGW4ybeKpp87+8qo2X + + 2lVCoe7TUSp56UyqVf3yA6ECgYBSWFgsSb3bU/EUqlg546UPlLGr7GV4VVYYJxRP + + ms0YiqQFqyjxr9Qm/QVAmcBxkpC1xiXOS3/RkADKUJRyFZbETDkIIaXoRP0CYXbz + + y4lcdNrsszo4P5MhS6dajLyIRzWL+vIFSl2kZJ/WiPi70tusO17bwQ4onyd8OqUd + + EgOUAQKBgEKtBZ806szsVIa0eFC6PHxt9b+N7YgeGoK8x9bCHNS11nXTibLImecX + + bhKCf/xmibabYO7o+V0O2RyKZxldRWds5Xj68fCFHER5Ax/9sF70/To2UouE1AHB + + rL4c5j+3y3pguBLfsjsWFCJxgAh5aETMnhPaGDSyy67D0PwFlezH + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 131071 + seed: LCa0a2j/xo/5m0U8HTBBNBNCLXBkg7+g+YpeiGJm564= + zfec: + required: 101 + segmentSize: 131072 + total: 255 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:MDMF:6p3cogoniw7gnwoyg2574ephkm:ag54qqj5o2bhyiy2zc2gu3vk2flf3kutphtstpphjgzzceyscpcq + format: + kind: ssk + params: + format: mdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEowIBAAKCAQEAvjficy3D4RnknUUEgM1Z76aRdQRHkHjLDa0mTJVFGVCUcI/l + + oapqe4YlcJvrjHHq4OCEDoTaSYG5ypsc2YIerM8rNwI+nYC1AHwg5Y7ou/l11TdW + + 32xS1wtbFaSLXfmTWsnt6ssWQjMjyxZ2ZcN2wiClfM3eDCp7lb29lVEd7NF5gKWA + + qHfy3ESMLJLkLSkqyWoUA8olXPJ67WVKP+jTuqzTHEZ42ItsDJ0eLq4jXqHJYQJU + + yAbCExmW+rFKw4Ae5j8Vh618p7joa1j3BuecNdbthSbRN79XknubWC/7MkMOjQGy + + TdemtMi5Opb/uUjE8pRzCenLGNx9IF3mOGoB4wIDAQABAoIBAE1TxpjwF9sgfZF5 + + hzUdRdxoqGUbkkQm9tTeeN1VKTv7R/ziYoVwE82XYQ0ANadogAVfABAu7dZICFFW + + 8Uly3il+JqE8Jlw9AFfsHit0BySzarV8w7IcBSkqkqKfu5A+byrPQArc+HV8+KYM + + waDo7xRH1T6BKi1j782VzsYura2hXsLKx6XNTBmjqf+fetZrZjDubdGQEACmadDw + + 29uZ9cs41bC1KPCpB20QzL/x54kczYchkpUm21Yq9LKSqtWoXQQjF+OkYkoL4ShG + + I+GEOuHU5i6qk94sre1TRjZLN0H2u9hhNr7i1r2hPIBWx1gFNItm64tEEjYxUJzM + + 83wzAwECgYEA6V+wuHs6WCMIBe8Y9UH0OSpVsK6b9TBgrNQHN/Lbs6zgGtzTm7mm + + r6D6doreYovS3z9yCJnmQZ+J03p+cnfiAghdH8BwevjkHKiS+bOqVmO3CBb+KC2S + + f4JjHcLEtXp6Oh86QiYwA9HmNFfEHuZFxje3JiF+JWNF7Ldmyz/gLsECgYEA0KkU + + gZp8ECeHjdaIHI/cyTjzTpjKKfDRRuOW8sxymdNaNiDxIL/wUtFfQocrtxvVIo4d + + Q0GUqyS6OgV19YXCVPB8Am+LgwiddNZc8ZYdu4dySF4azJXdKoa8E6PlQHq14gY5 + + NzsbcjjsTLdv2iD5Er/AO+rnPeldiYdpEGPYfaMCgYBiN9GqcsJlYaj4xl4cqntc + + q8KQr4wXrxqg4kN/eoiYoANZiuLMQWAzvm5rAZsCopJHPu6BTDQqHjjldkJNbsMB + + 0/9NY7JzLtjibtgcm07vONxJXVPuGO/1Fi0c02Hydu+GEqp0OJowoWBfWyjBUGzB + + NaWxOJtcpOFC9RUgKWvygQKBgBMLu+Fwlm5rDUZ3FIl24DJFzn+YFqvpXVDZKUgU + + PUmpLwzNyPSyUF9e2REbgXP/SF8VFbqz27wbaBwvr2qvwOM76DXYtKVLPgQSJP2w + + NBqP7HCKlmuiKkPddIFebmiKStvMsaBG9uRgKcF+5OjGJbX+Zq+Ra3YNPQp2n7Jt + + Sq99AoGBAMS9ZDgnYPu4H8MFXRuwtk9Ti7IjzSA8zBh1MpQFhdXMzg64s/8Moq5c + + eQHnT/j6bqVL3OevmJGqhwPPIuizWvskG3eEH1LC4+PPk2L+Ap9m5upIlzXLJsWw + + Irq0H0KV01DDGhJVIB0E0DG2q4bFnweXqQJoq1NxGxPrChcbo0Bb + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 131071 + seed: LCa0a2j/xo/5m0U8HTBBNBNCLXBkg7+g+YpeiGJm564= + zfec: + required: 101 + segmentSize: 131072 + total: 255 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:CHK:a6wuekh5mynr4c74ma3b2gswfa:b6lovwnbv3523n5kaxga3ruxa2dvxef6h4a2jnl6lasn4fwvvdba:101:256:131073 + format: + kind: chk + params: null + sample: + length: 131073 + seed: /N4rLtula/QIYB+3If6bXDONEO5CnqBPrlURto+/j7k= + zfec: + required: 101 + segmentSize: 131072 + total: 256 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:SSK:cgswz2wpgsmmxpsk5y6w7fcypm:orvbm4ilcz4cedxaxsdvlztgevn6vy7edrl7xhmeoubdovgohknq + format: + kind: ssk + params: + format: sdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEowIBAAKCAQEAtdjLVkFxYw9RpV2VkySp0nQabpd9//h3KNSIWi0S2WqJlfCp + + 1Ek42VQlqj0cZ6FoAgPiYCv3FIjem9T0LGYmp+UOv3Gu93GXgh6eIC/b6LZlYlRU + + L0PzDGi/rFhtI62YLluaLVeabJEI9SQThKRuDbtZSlpNG/WcWxoG1TL0hA83DPlP + + Ud3aeah6d2q97BFv0KlHTdq7MX8pjUUuxU5+JZlu9XE2TGOv9Wa+IUF18mTvRoLd + + rjq1R7nw5z0+Y90R1sLQ08NgOSjb+ytnl8LKXLtSI7v0t1Al8jJCsEq1HMnP8cLW + + 5Dj7Iy3QQkM7RudqQSaFzwHcNqweQknj6C18XQIDAQABAoIBABOXJ84YE3NnkBUv + + mtCU+j1I8saynV65Js/dmFRBV4QtXRCBMoVDNNOnpBMjWERgeB2sy3OEPlnA+7Jw + + Dl9wyyilSEx0STQxJvBcBLmqTkJTfA1MJnGdHA8HLeG5o7BcoEYblOWI8thQBlYF + + LUZ0lxf1n1ifl9UI8G+Zbd5ZX0w7+dnezSrOcXcNqLwmwtZn/mz/WOC857q1Gej9 + + xx0+hHIN/nKPkltvbrm7T/wfo4gs3wu3KCz7ewAUAtdtjWi0P5ibLNaRVcANuVjf + + /CmHvJ/C+Pl5PdPUqDt5Dg/aQJvfHXicCJE7Jv91spVv/dsJ7R00jpiyqP5j4a0p + + zqmzpuUCgYEA9Tk4BvlRQv4hpOEjCthRwsMaqETWcSN01gXA/eJNB7Xw4rOa1DZH + + 8m6QKKJsX0jsaL93ZrDI/6BcC4DYoptJOQ5vIJyKIKCYhld1jEZKNiQMsKmslGlF + + M9OsHc/GsmBg8VgZWfsXdthuYFmm9MvFhm9MRDQ7E/JriZgI+iS1NbcCgYEAvdaV + + bqAXDf6x4Z6pfX4T++qZg/iNDZOyY1vrmPkjEuWUorezzqUxkw/DOEtSdczPAc+L + + DUEE4pg1y/FAFvIbPSDX3DHsba3UobzA1GkOF9K8mZrB/R89Nd7OgnDBGjDlNozY + + KKjXilAc8oIiTUKjBsrwZLrw4MODc6r2EadxPosCgYAfi/fgNcy1cJoFaw0mBQQn + + qQ/R2+E2dtg9/EmCn81HE6nkkDR33m/NCVo0UAjfDTOUmiUTKeUBtbCBrlawPIfj + + 9i5npJvEbMSSa8fsftJnOqYDSCCyiwRjEXUP9L3cDrgJ9Ep2n+251UgFzyLCVUCY + + 9dJ657k97K7W6Z8mBvjk6wKBgGw6URkvldU5tkntva042sXNOtY9NpVd9d6lggzF + + RJS6ZGHcH1uZXEj+PIr0jj9wkzfyDdFxlwpkQo9Rq/so7hSMi+QSZjslVksbJEg0 + + 2H8GetWLoDrhu3Dh5JQDGmQHKjZOV9HeaHuHLumm/U1Ux0LRIfobhcZuUJv6BK2N + + 64b/AoGBAIl7UTiqbxoQmB8iOtHAI5ZNJrLmRek9tPnnOzaICgrCxq7Z0/B8ATaA + + a4xPwTtapnohHaZHvqgr6o34huCx4ZEFvYpsuo6vTVD3qUz6Rcnmy5PORW5WId3A + + 4jv+MQ/KAzgrgm1o0eWTauO6XL71xdVoLSEYeNpXa9yeRiC8e2vx + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 131073 + seed: /N4rLtula/QIYB+3If6bXDONEO5CnqBPrlURto+/j7k= + zfec: + required: 101 + segmentSize: 131072 + total: 255 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:MDMF:jw3kbrabv5oefmhvjlnii53v6u:b2432ea4hpbslu5dy6yy4ibnrvvol27p226qgm7p2udhx2bxyiwa + format: + kind: ssk + params: + format: mdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEoQIBAAKCAQEA0riPyzItZ74hqew/pA5c/1cgr2LIU6nzdT4jRLzgFsLqm5ZP + + BHVnB5qj8NhU8VkMI3U9ERQe3q6OKYka8BCCB7rpuva5X26i7Cs59Nmpn3VL2UcK + + DCSdpn7RrHqaxZRx7ERbc2T9hEWi7fAAaUhAF+2+cYocBedEcgp3/waJtmsFyp/b + + HAYZxLzjCDO6JDsclkh/Xvj3BoBemlKqET65QZl8RPGUn3/jRlRkR3uZBqzSX+iI + + BGIXiKSAnirI+ec+UbbiSjdyJATWG+Trn+6oiZuF1E1Vb4AjSns+MHPY0sA2fsJk + + L57IpqeXJaRC9Kc4ofBfORC/wFjT3AAXobnluQIDAQABAoIBABJkBLq7e55/B9Gc + + epZvIXswh7v+31R78/FS1cGpSVZ7Nv4SwX02YOJXQwEhZFJ3DtnmYMjFjIcrTWF/ + + I5B5pFuX2s/UOiwDzCjYAfQmbgkqc874Bf620GKIVXTb83eUg9fWxHN/CCg76qMh + + C+wkX+GmwHUI1HbIbx8T3lKt56V5q46pshFGW3fEw06Z7waW+qFAYSOvbaTlcayL + + JenivLYnlL0uQ8rq3dJLlaKj2yXv2WXv1GL2fxcdI00/twBcsU7CXf5GC0q8MFyQ + + ent/R5yyDl4wpN/WWnI6fe+OUI1YydooHz0/9xgdJpYEL01rmz1wP1ptt8SHNB+Z + + ZBPvBR0CgYEA1+jN4N4r+Ke6SWnNARI1LwnB3e228wlEht/b/PynImhz9zv57VAb + + NmGw3GTpQS0vd/FnI6dY96dvtBQzDzMUqGIaQphs5f0phGLQ9bmLJCJEzefenCup + + s+fsMJYW60zJUncZ6Nea24iJtN5eew3vF565NEKYGK5E1p0xloepDRUCgYEA+dkg + + Qjwh3UF6GTGs3Ot8q44ovCMetbSce0WQ09D37L38/E/8V9MKGTWhk2Zkp9FMmQAo + + VUdBXA0DacarPFGbIpZvu3Ln2Wx9FCAu75Q/XvFoPPEv+37LiQQZ+7tFTWiLL1Bg + + 6JRZnvIWSeJEDfx/ahZ+xwjRjdPmglb2tfIPRxUCfwKHt+HquJkxXf1+P+jDTdw/ + + QQZYwswWT7dE6E8OpubAUpuTGFqvlaINgwUSKamZ3fSJ36uLSn+cdrKlifOjpZpT + + i/s7zgrj7Jigj9JRWlASFrxS+0jZOiPhk+L930bin3lX6/XOkQIBl5uG/5RxlVux + + gHocTav8XtIlBW++Jz0CgYEA3cJLMJ7gy9qG/f+qV7eoQzj9jOd7JXp2fa+kOW07 + + 8OQ8vNJdvrHxP6jrjcIPSyipXQ/XvMFvEL34LpWIfRRNpuhxqaX2hXQWnJtoLXue + + t617gMPue8Hx894xFc8FVwyYpVkpeqXZ2gszn0Z2cxePG+F1i0GXhdPzv/JiLeH4 + + j3UCgYA/7sJn1Mf8FdIm4ufxQajMASUgYQ1+vtC5LlD+95lM6nNUfBziX1g51YN3 + + ZJEfm+tpL+vbwCIsCl+GSIsyZKf27nuatJkbdyssLdyxYmy/McT+LF1pbfgrtuye + + xAUZtFLdwZoG/wJd9DLImkMlsJHGP5HEKq+fuh2fl5Rdra9cVQ== + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 131073 + seed: /N4rLtula/QIYB+3If6bXDONEO5CnqBPrlURto+/j7k= + zfec: + required: 101 + segmentSize: 131072 + total: 255 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:CHK:pmuo6vpcodol76sexq7s2ojej4:irp6rxnkgwbn3h4g5fzgjpqgsvhvycxobr3xrk3zr44r64wimgqa:101:256:2097151 + format: + kind: chk + params: null + sample: + length: 2097151 + seed: uqWglk0zIPvAxqkiFARTyFE+okq4/QV3A0gEqWckgJY= + zfec: + required: 101 + segmentSize: 131072 + total: 256 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:SSK:vl335scvu2r23lujmtoojgpxxm:llq67pxqynexgqco2d4yfhtikmyxoqzd77jlykvr2oq4yhkiknlq + format: + kind: ssk + params: + format: sdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEpAIBAAKCAQEAwupALUN9JNh1i0NmSU2d4jqgDdhlaSUHLjXypU9Q4zUVXWKy + + EvREz44AWplN5HI33XUsEWlEsSOgyBshqt6Ja09L0z9aoyWgW35HYbZO1BqEtbUW + + woGv7OY/FJIouFxCi6KbqoHTu8jywECV3iS9lHya5PaLcapEzpjJnVmNA9g1DU3e + + 1mJCJP4IPb2o26Reuqq1F1G8vkgQFnF/0absKzUc3zQU/4p8F2jwpGr54Zggg/Dh + + AtwkPBFVOZJNYb1tUFpYT/+axpYn3syMce5wwzIkfaziwoLXEbO1mVLkl91waXhC + + L8H9U+1f8s1qR596VmeU8nQDUOgKXYw1wggX1wIDAQABAoIBABb9bkhod3BLH8In + + Vv86am7un0ZCyeNW/LvUnSQmcNH7xuNW6s4VhbA9fYkyH9/cIP67/VCoa/PA0gwI + + NzZiPS8tETJ/fH9Vxs5D3MOHr1CROCn+jAqxJUD5/2K7wpXMPAUgTuATpBe4IfnP + + JF4pUzsaX2K2OchUXv1HRDCNCXb0alvDjTpH3HQfwQ1a2F6y2d/RONEFGsTjaepC + + PKP+R0vuL+FNuOVEpV5SXhdyuUfzzb/rnSq3BCPWsrvSkPVTeyHfz0DjA0k5k9vs + + mprc7eQYZCoB0bJQUtSgt0b1w6jG3M7gksj4WbR9Esj4ychDvwUSYBgQaD01MSXC + + H8cBK0ECgYEAxGtgUKYIhxpU23ntk/EPr0Jq9IJ8RiJx+/yHXcAwd+RwRO47MXEv + + Ut/2ipB3xRSPJgigfopW0bNFOFubPWEttoDbcZLRXq+2btRv3eOWxjmu3V2re1m4 + + PigcCBF+pPB+0K3+KJC3b1KKJ3NZq0K07cuoVXStHbe0xz9b3Fxe4HECgYEA/goN + + lJ6HvW02WZHO9v42WOC7dYdTDX5KB7neP0U6d+mXvlM2aj8FbmaMjE1IbVf2C4Nq + + cgZ1EP/LbnOuLXsEJA3bud/hEJCWCHkomnAnjeO3VI7Ab9EcjxP/icYwOqsPwFQr + + jlfd48CmV1yxUfX2bcgZ4r9EZyrv45MvTgiRoMcCgYEAwhsO0oRR9xl5dG947fOS + + uXLceYedwj12ATyclXRBMaopnKUFECY3SyIS5PpBshxQHpj16jdR9ue/ZyN4NV8c + + qreKpSEfEQB6O+pqJpArxvboLNDNjxep8Jr4oKyRR/R2jkjz2yiwbi3WY4glvA4u + + +LSDRPgJwE10NFcTs6ABDrECgYEAyhXpM77gsFm+kIYjI+yaAx3TQe1Crks2TOsY + + 1zAVEOrr4WWEtgQoJ+jACaQ453K/sez6snZcjgdOJzEy788aPiwgDL8B5RF/qIHp + + QOHTNVZeso9Umh65H0CDWXAlUaZew1qxw2w2gUTxjjGYhWCqhi5WGUCaA4/ugRTG + + 3saGQmUCgYAGup3rh1KO5kxEYpTIi13Fg72SPaApsG8AMc1jUncfuX6BfpymuDqC + + o5Z2CRc8aruyyUsZ9WTD73h7A+R6WpUOd0Yz0kzV4SFhpmLvqpEFf/QoDncIFDL0 + + 8NPDU8gzMEl+uTHt22+U9kI7Gvv0mF5bXgjESZXM8BVqdVtsi3QK3w== + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 2097151 + seed: uqWglk0zIPvAxqkiFARTyFE+okq4/QV3A0gEqWckgJY= + zfec: + required: 101 + segmentSize: 131072 + total: 255 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:MDMF:ctfka7yb44bh6lhv5wnpigqyne:hruni3sq537s3lkckc76w7xaxj432tyvlzd7jgpzncqvpzbqlukq + format: + kind: ssk + params: + format: mdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEowIBAAKCAQEA05yQCFDrBVQeigp0LeWhVUJHfMzJyukKpIKHsnOzqTuME/Wo + + jAc4OBPGR5TUaNPolhsb6KJnYWnaOh+S28IF0tWVtNU8tonKfytImkdC42ogGeDO + + G4QM1opk5eouiXGFz89EEobdkrw5hSMXOTFjQaVkcZjOeXP3fBKQ2G3zTixXmT+m + + OLXM7RxFmNnx0ar0I0qMfCE9zlbnBV2PLQrJ4/dA9ictoPFgibKhLEm985qZutkR + + Hwlfc2R3nPPD3hSZ7mn9lZINJn7w/FkOClX+m2XUj8b3FZ+YxodA5N01egU6QXQI + + H9vznaU9rD+lRw0TiiiF6A0KeKlQ+Jn4xynl2QIDAQABAoIBACSi9hLig5YkFrd6 + + kNvDZne+5maWhBdj2opZ6Ql926ygmSN5hCleNJ8M2WbaPx45Fgsq/V19BJ4KeBRZ + + FBGFGYIDpYwt4PmPiKYUxdikHtIFtTIVyEleRuS8CDUAIvd71pmAfn4gqGr3uJOy + + 3Bn0UYVzj5zVQmYnrEDoa/h0rMurNBHTeiXaSecPdz7rgVWyyHg6dxsjf6iOthNF + + 2Wc97iorM+uRqwewXy4rKjW5zL+MnHr0NUDZW/vlUfOdE2VzUV4niz5mYfYL2PzH + + fBI7vzxlu1WyIybp2gKMOUS1UbS9me/duJla2wtGLYC1gMUh6OnHFOMk7OSUrsX6 + + uZfWxNsCgYEA7Re5mTG2MnC0uv04DMUrUlKADoaA3O3kV3GP0Ql5TLXY4XWr/vmK + + sW4Ygo8othOxGvwcGaDTtOYCykxhYjap8H2ZCM+JFvnit1jrL7iU4VJ9shQRrWZT + + de01aC81n6HoZPPc5TEh+nGdcM76dZdmwIaW4GCkBIWNGoWptf6Pd7sCgYEA5Hyj + + TZqLUwYus2cUbNKOfyYWky09UCsV6w4xbdaxJUa5u68mE3hLN7syr82JHwm5AjZn + + ItseFNYRzLMzeV22hP4LkxeyBaF/RBq0NXuZh68dDHbehfsJvOHSAMI7IbGMFSG8 + + zfZUtLonqbFUoWVK4R3akxoZ+fJOu3t3jWrWrXsCgYEAvr4oliPVVe0wqYMQtc1m + + lftDhOwW/ibxXpxBPMZnbRybmH9n2WD/gNF3LIpqEVn0USZkoQWvbMjjk8cxTad2 + + vsD8/oag3vg4upLx21mfhUstTrgwpJU/Lg+huOjKNlw2sAk1PLpzgJ4pMNmDzFj6 + + 1IczGN8G9ZBQPfcs2vsqhwMCgYAeay1+hmWoDvmmrsF8X2fTK6nzvCEejC3l1kTk + + X6HD2a+eegnyq6Av8j8kQpPPywaTcdS3Qj61/W3vN6hRrxU+jWfTFGOB9mcwFPIK + + 8MKW2sxePXEQm0RHnjTMHw+qQ63nnk85iGLskJ/5Kn+e4RJf+A6CaQYuTYEH2r8m + + 16NvAwKBgCRgNzvhRnzbZe5qYjkACHhej4tS/2yIHE2kKhRUedQuvt9O6jjYyKj2 + + 22Vo7FkwJO7AvGp5cxf0jREGQL/TPQJpPuKHohiTzHNCmEaNX8y4NrGy5upXFYdi + + AXYQYyHDC2U/RGCBtNeiuC5bXKb2eTx7a5e7Wje/aeQwGhBqIl7I + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 2097151 + seed: uqWglk0zIPvAxqkiFARTyFE+okq4/QV3A0gEqWckgJY= + zfec: + required: 101 + segmentSize: 131072 + total: 255 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:CHK:fh4boug3wkciqhkgpusjsjyzbe:sy5lwooxchqu5mrzwr3mgc7qtwfjgmpz2srdmwpaip4iuj3tn4vq:101:256:2097153 + format: + kind: chk + params: null + sample: + length: 2097153 + seed: BTBX/ampNfLU+ox7xipBGiaSbgC0kcB8Gy7BkJB4oKI= + zfec: + required: 101 + segmentSize: 131072 + total: 256 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:SSK:7fbakosfeg63ckbiuhgz2gbujy:zgs7fojm4gu37syyzdzcdfigwf74qkfz5z5oz3lxaqy7dyc7tz2a + format: + kind: ssk + params: + format: sdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEowIBAAKCAQEAld5wn4Q8j2GiySAr3mW/Xl+94oWndB4kV5VCNzB0bamvqWmy + + yv9KzLsEj3IDYQlKk2iGG6+QGjCG78rz0kfaE2YwzRwVYA5cchSzy6+ASqNRiVkw + + v5YDszsTchZrIv7Ni2PQzafnGPaKzhqWqLPF4DJlSXK3rU0idXjqm86pL9u2cavQ + + sEkm43UsZuDenTS0MZvXfIgqTqrtaqxp4qbhGBSTTBUyBQx8VXC/zH+NfqPWXGyZ + + AQ9jW/FqDFbjG5348Q16Au78WSSzLMKtgviguJonQULt3okwzgo994jt46gcCqRP + + SJE+TeIkdz4oq9MQ683Z7AqQhyN/5L4n/l5ShQIDAQABAoIBABAkdHcKFEfRWWpW + + d8MtrG4q29YRVVcRhBKW9hnhszi4pT4XL3XkB5eDsVsOCcUi7hBwmrlSsK/ReEdN + + 0fNdX+TlBe6hzr+Y7GYxSqhuz9+6NacYn0KTkvR0MYUBWyrazSLtbmkoY6DxtUO7 + + 42xqaK7cXsKJg7U78LE8g/CiUuDfnEnqWe2LxEgtfaMEEJgvyp0FutW6dJebtlxW + + heQqpZr+0aKGgXrUMwIhoIJy6Wv4DiBjnuAwWavRpMPCvFAaJMSGHf/TTQR5a3+X + + a/yZCGQ71TV0vkz9CNnxwX1XauJWkKSzs+zCbugpI4MDR9jL6FWuj48KCc/9Lk0e + + Lvtlp+ECgYEAzV6CHuRp4ISmqhlhZLRouaOeeGMKpDgh+4RPBILCQ5UfpSEivsyE + + eI6ng7yvz0TPRg6Nbmkh8VttnlNK9PXfv0FDYprlzYB+h2ZG98tI7qdK9C+LSbnz + + vUa6IJZ4ez2g3xqrWr2eLjFsXhyND+ZOu4E/g0iMM3S6AJH/0Kts1O0CgYEAutEi + + z0i/CnYY+CXoFK64qAVBX/1WauwzpiOMpFfb2OQ+DjbWTsL4J4XGWKw130SxWJ7A + + 6T/dtgfK9t1LUfTprSs+A5t8cPeIG5Bp9HTiUq3TAm0lIubGV+US/cB7muBKgv4/ + + ppdET3InhnLQ6CwxtyrXQpki3tRvJPAn1achGPkCgYBTchRDAyJ2JNAni3qEVb27 + + uFzao7ueMGS2cvM8bPkMRtp92THp/uXQqn4sTA3PlTD3UVBsTXGKRVEMJOHvGLya + + VKVRuoincI946rjpVINE3VraTzs0cMc14Dgep6U6xjbIkGiRzTwpntFeiBFVJYpW + + K9UnveGhwssVEj20hwMInQKBgH0/bVkPapV26/KiZ2BGa6KqM1RJosB4r3/5YXdl + + OA3HqBsbhL61VG4a8AnPGycfBM9nT+qRWPGLc/XiE3dU/b2Nujvs6JdMPUJNpduw + + 6XOI+mksB7PIiL2w5PSfMb96FDqSftYPoEqrO/iVzZ1607H71OnfhVNjlUhsgihp + + rnTxAoGBAJXyhoFN1dtbc5lulsertBsbslrksINDM5qjiqrLsMl0L+0HPieysfOl + + qLJU5sV6jOeAYu9z2PtgUpdppg9/il/5xkCgYjFsG443TJrKZ8vO++dTLz/i7sq0 + + bDljZVkezRSHQQgIdRuJ/wQlDWUk1tQ5dPGLRAWyF4OfnjOBMht1 + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 2097153 + seed: BTBX/ampNfLU+ox7xipBGiaSbgC0kcB8Gy7BkJB4oKI= + zfec: + required: 101 + segmentSize: 131072 + total: 255 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:MDMF:66alqxuv5jdgccbh4r7petivoe:qkkfvjzii5pg7urohmys26hwgxxz7kxjrk5n6yrgepmxhl5wxxaq + format: + kind: ssk + params: + format: mdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEowIBAAKCAQEApNQTfBcsEhSdsh9LXtHm+mZu9aA0TaKl42P9Hy493i03qHGr + + agkcSqVEF9TF96F90RJ0/dWNpFkGfZTVIR3bKjDQb7uM4rWOXQaPqPj0xaTi646j + + httNayNarHQvLP4ZZ9oTvCjavrW42aDh4VLiTXxS5SIZ9F7tRVA1KEz7CckN59Qq + + 5kai1Y9ZHyuasGW48rTfYRQYefak3jeeuGIL96Mq7qJqttuOO6KcWu8QMA42GxOT + + wDjatLT1eCkr9yQfQjv9ziIKPS7t6qXdi8yP05vFMFRN1EgiSxh22ruxYc0Q7/u6 + + NOnDpMxjgUkVAg+Xj0TU3T2RQe4LDFHmVPaF+QIDAQABAoIBAEMd0ClRTjK2jlf/ + + gjNECWeg2kHOUD3kouPqzSErNSoJA4blckUlHI4QqZ+ClnH1IkRF3bmWgayQS6JL + + PlXT0HBnnBhDKGUQRL4Ac/L8HL92GqiMVm4NUoLzzHI4hRUvCq1NEYgmopvRZ0nG + + xvN3Sor+uspujl8BYGA+/sZAQmCDA7AOr0G9ZMVNDSDt0cUiw6z4APbo3t9MYG+I + + 8OHj/nhd4cN9qvV7/CUZK521BlcK3R9XbRamDNeSkD3yi0Zw+dL7+0Mg4NWZq/4c + + cKD64OMRgBHsqNu5ln2Lkk3moKvu1/Bv0HORLwzp9FfP/LT5h+eDnYV7GIP0vT1c + + /6wo7jkCgYEA4zrRNr4k/6rcOv3uUZUMfNf2cX3oolERjIVmvA1NIbZUh/KcE/Rm + + Hzw5oGt5/TKEQ8zXBSJYq3fMbnHEEcskk8MtUx51jGaDB9N3x6uUDhN+aimjHysE + + kaJGRC591mzFY6gy4fjE4+91itX6Pr+k0xIEiDU1Oplm4gIj2gUHdgsCgYEAubKm + + D69KiYcLRg5oo7HDLC69JPiyucAvNqlsNkX2bMZNQQa8bgVyjymRiWLRL6dc0bnE + + 8mcge2QhYKrGGGqehK+F8b2ouz+nJQD6Qbi9kEtEhcwl+swovai+ZjscivIeTGqv + + uLfXG1GP2vlG7WN92WlIdni2uI4KBGSyk18HCosCgYAK9pmFhKMQWtQJXJsVAJX7 + + qAfR7fs9aZ/pIb6VMCcai0uEy6XQKKiMtUEqhkT6fGd5RfbR3phcnYkVgxOssBpx + + rqcPLZcKUR/dTsymq5aXH0WoJZ4jMNYlmKi/PWcA43qallDuKiyFutX2/t/2CxUO + + wf3J/Jc23pPiL6w/JqL3hQKBgA3V/L+AbQpQMIvYuP0xWnxpQxiFGzPx2NK2zuRA + + VDsIj2r/6Hw+FaoLC9fzr+hgDO9nawAwpN/stvvv3XCmSQdT2KQJYJALDxYXu424 + + CQ++O+3IJzBHk+WFtCID132WyqEg9dTKhdF4Q0Kqfhlj51WSnZ6OIfcgRijLo+6N + + DwY/AoGBANXP4ot40pSCFd8xZfi6pLzABjxvNRnYUjWrx+ZxCAQkTlBH2vow4T3Y + + ILMmoCr9vTqxP2Z5DqrNbvB7FlHbhOJarwEOnJUqsfT6+/637RoO+YdU8ZMQjd0o + + 9AECN7nyZnANwUd4FhIGKa0VU9oEBhNsMWbE3aoQFrlkBr8Y2AY9 + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 2097153 + seed: BTBX/ampNfLU+ox7xipBGiaSbgC0kcB8Gy7BkJB4oKI= + zfec: + required: 101 + segmentSize: 131072 + total: 255 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:CHK:asmjgajvwpwvylxdcf5rm2gyd4:imwtdi4eqq3wm4bvk6npho3ta2fewceqzl4f7wyyabg5i4s6h3mq:101:256:8388607 + format: + kind: chk + params: null + sample: + length: 8388607 + seed: w6uP8Tcg6K2QR905Rms8iXTlksL6OD1KOWBxTK7wxPI= + zfec: + required: 101 + segmentSize: 131072 + total: 256 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:SSK:4vpgtviu6is27ulopk2utss46a:egw2djhxc4opssnuzrystenfu6ieyww4zds5dhcmwhey3nsqcz3a + format: + kind: ssk + params: + format: sdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEogIBAAKCAQEAqwfCwchGfDSxmoZpPVtXwHIqlxTVVI6MjsT+LWHorlGlkB46 + + 0cELOP1/SMstSw3NP9KXo3o5R8CN2B4vm7PBbGNuOulmoo4vb3ZBeIUPNAiHM2Na + + Vwhe1yqCfadAtmktpGQPLMMgZpftRspnKLOnPIe+TUgHZR/E7sh8uaHGyK0TZVuh + + WS/mnXPdJWs14ATz8Gfzqf1t7AfK8En0dYhDqK25YsxZNObaPgA0CoRV+ZZ/H3lc + + ZxcP0pYfJOO1wszLmdxjpXqyff5oOHEidQvzf3qpUf/B6OAna1DR9oYsGyWUkXLR + + myGFmKsZ2LN7jY+OjpZFz03PhbYdA1lZbPQAywIDAQABAoIBABsEJ9HqgU/UwbVS + + GW+qwnV7fe94SAE1CfjyHzXWAHurbrFm77hQe9BYeBxsk86w8xoWJFlGJk7c7iBp + + wZrLzfhWyy5yeKxMNNaw4cNYPsA5FKTxGG/PuljXYIjZZvx+0snj+w2sE63WaTxv + + Hn/XGR9o+zjunCnE10aM1D7n0iRtXaVYjFbjvwpY7NsNeznnwb4jnTSFbAnlXfZs + + zylUtrhQh9hgcPEyvFeXn8TAjPa0ygC6nh3pYrSklJFr0Gy1kaCR58di3GFxBeH4 + + M94PbbCcz0ZbU9QvAt+y4yhT5YO4v2aotAAGWgA/xAmm9UvTIxU91iKbaKCfcwys + + rFnLezkCgYEAxIQR6HAUonoYxohY2DJp5c+D7lND9PIr6RIUX/nkHWgsVP0i8ZzP + + 8uimRjV8oIYEWpRRSg6cWkjHGnx577oMMTtajLIcz7aCVsbFWlBmHuyGdy/bMo6K + + gXJpzc+vhvOJ5mKK1aVC3mgK4DF2Gs8ExrP2456jZheLsLTMdjuyQF0CgYEA3szT + + BAaOab+I+Lmuyv6I9EgCzs7/bK2PjnR1W3zwPQr8fOF/AOV5PqJZYEaPXo59ddKE + + Oy8z8eq63ggUQ/TWcDpdh31z3gHiR8oqLr6PbqyX7XDsADO+xubQrFnm5VOzgfMx + + JgGIcreb+Re9M7PCBuUG8kX+PIWuSwCe2R/eU0cCgYAMxVRwmZANuwePJ182tZgC + + MkktnMWmznIiFGW0kwXLD3EKGOVDdGBjNdFQcLtnpy3zQP5DZM2uZFpkE0DNXnba + + YDQTPqP2r7KqtwIuS1lHmzFl33tMPs0remb71AphJ8SHb1H8bl/5GiPSzAQT2+5A + + h4N86VtPECqo0icTa++6lQKBgE3eNk3s4K8y4vNTKjUGOuVtmZWgIQNhsY+vQikE + + hI5BHbejtBijGvn6EdSlNIxuroiUV+S7faMqT780AakylBPLQk8NWIaaD/TZQl7t + + +QFMTxkMY186to2btAjYrustkspzLZVD6eV2KIwpcNX2GHUCbKgWMGIEssLB58Ko + + 8bIXAoGAEvhd4Khh96bm2noW1DuxzQ5E104h+63qhL+1E45dHuv4x/JeofhxM6nz + + CxdSOWA4v7gPh54BRRYF/PGiPYqSDqHabcq2pTD/QewpUVEmEaFD238TJT0hclYL + + nDLDFCIU+RB6qFYHw9RI8HaokJ9e/E3rW6HCBsDCKhvO3XFLEfI= + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 8388607 + seed: w6uP8Tcg6K2QR905Rms8iXTlksL6OD1KOWBxTK7wxPI= + zfec: + required: 101 + segmentSize: 131072 + total: 255 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:MDMF:ulidi3bj255xykqakea4fg5w2q:js7l227a44aarx3irvmy5jjpxnz4isszwu6q4hxoudolw2deqdzq + format: + kind: ssk + params: + format: mdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEowIBAAKCAQEAr4OybIo4VeHz0z3325Le6DBTa2GcEqamyxhUtOErMlObQakg + + TFZuxf5jI9fs+ShHGwFiMZkV6BjVZSL2AmXaY4SSepMvjgybLe0XozfknblqI/yq + + hAft2bILttUb/H5+QZ4JIKFwHKV1DEq5GTO/oacJ1NdartrwuYEqzjjZNyTFITEP + + 6nKQPoORqLb9vvszUsEJjWImxleILngOrvDCsDn02SRAYwOj2lVC1QmrVzYDeMSn + + yxUinuw39fwwQ/MHJOeUtRh38ecCQd2svdNzgcIGknsQGUkz8ojdGGmq3Hwohvdd + + +Rt1aqkaW0ci+TllQsFBxtlSCnQ/M9h34ztUgwIDAQABAoIBABgou52fJQQFVyej + + pwNtYwt443Kre+1BTUI1ditztxt1ULCoFA8N8q+ERadAaJkfRzJbbWXAWbiZ+n2y + + Y0SPOpFqRTNkIS6fY5jdwtwvrGNdi1OqytnjsYS+skgXa4PE8aIcm8sHDcSTrdnk + + SzhB3EXnFT5b6lqZPnt6YScDwjqJtdCOnDI7W8iuYkgaHIRU79C4bmRDAZTE3NNJ + + 8Z5wutXoQJIwJh8uNFluS0En1ZsB1Htnq1Yq0TRGqXvtSYOie8r/WxMJ6TNCKT84 + + a75tXl2fQbCZyDElRRFgp2b2PF/Wg4bxbdPmgZv/TSDAvXzFroLub3WKV8YNKIiY + + Baj0d8kCgYEA0T06jOCKWYrX/zHaOEYf7k1t3jUi9MRjKOgorgaW42h+MJseVL00 + + XwYfNOQoWJJkrg7ZtfpLYfqTzmNYYlsEyIqzCYHbLd6eWjPFy4yys5mmy/IhGVRO + + zw2c18/Yfh/T97XN2GbEcZgoXL2CdBcrV62QxM0mIVb+AdPCDhHzC68CgYEA1r0L + + WTazWDckS05pice07SvpitieYWCVWFFTzqQHFcbWSWFj7cRHUnX6KC2UJC0frAu5 + + 4s7h5E+OFRo8zqZiPhs1XVutxhycKMSTIJfamsKEE+TmW19QbkwkNz1z/g17Oq4A + + uGqECWYD9UOzFPHp3N5LEUfb9fmr6i1qqT5WFW0CgYBFQXrrvjaMxMQRl7KfBbbz + + 7XT8I6JaWdZoZ89vKocu5hs+g1lauvVmrmQN4abpCiuA4TF2Zk4lNAdQPNm4VGAU + + 8LOp5e1iFVlcid5iLUPI5oaq4o3KEHm1VtAfLpB9zpMeXnKvufQzlSVm7OMNAc46 + + yxwrx6tjRaP1ft2wQoiryQKBgQCNVTMo7qWvg+txXRR9SGG+T86QQe5L7QOecziT + + osW/AXV8KotYrHy8u0WAOC9ud/yGgdlAfCWU3P+IyBIJeNzkP4gp//Mplx74fhjP + + tOJ+RVQku90Zemw3jAmyCdJT/Y+DmY6D0idBAFHOlVZCjM39PpltsDwHcuJBaM7w + + wURrKQKBgB7OW4hFkhHRiHpFM3St9M9Y0t0gPbop6Ejls6dVkbn+wbzGL8h3dyig + + vq9/ZecoyZGJyVgcSeWfxeS6Ph2AWjTquvf4RHBAxe6vBtRa8In1qxZR/6ZSDWFB + + rHLHBFDuCzacIIzMafKVSUxNphPMhDPLTLjkQzKx97GG+Ue+EX4N + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 8388607 + seed: w6uP8Tcg6K2QR905Rms8iXTlksL6OD1KOWBxTK7wxPI= + zfec: + required: 101 + segmentSize: 131072 + total: 255 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:CHK:yrznm2uuybfbda6jlj7tqs3yum:zy45lpc7ilfzgltxxqvy55wcokiay2fbbbv2wdvu6ubjluh4gf4a:101:256:8388609 + format: + kind: chk + params: null + sample: + length: 8388609 + seed: yPi3JHKKbWaEEG5eZOlM6BHJll0Z3UTdBzz4bPQ7wjg= + zfec: + required: 101 + segmentSize: 131072 + total: 256 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:SSK:lcniehz5whmmv2jrqfrk66end4:bduw4q4vdi2vp4aytu77gz5uljlvacu325esisujxhj3rv664b5q + format: + kind: ssk + params: + format: sdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEpQIBAAKCAQEA9dLuspnfLBRf7mbfi85V/CuXdufMNH+QrTCJYuYOBmJiwsu7 + + bkkgTn/uGw5DEN+0M68HrAgklC9bUyDM3QBW+DlwyQ8c/eX79waws41BXbhEf2ha + + z6cHQO6ZSwunNpkKXxCkAtrvwchGAUJGVQzS420t2aOygQCo3OJDKg1cg+l7+J45 + + 1BKpL64qb857jv+368KI4VLyfK3gpNJLF+tgWfjrZTsD+NS4wrPPId0mPkcSxOMh + + ea3F/fyQL0uEVlsts5E0xa127+b0PpJhgM2y+wDBg8Mkpxjne3yaFzrJMn1mvMG9 + + JeyViBL35HDDNVUVYa95gFUUlHvb7hfFJoDWZQIDAQABAoIBAAIxkB/pJw43u5mZ + + SfR7JsqMpE5e31crIm3TZSUAgujM8q8Hlu5QMUj5UOlxE8Z+rlzjtK8vIZmFpe9A + + PWgfdXisT6NACNIa1TDbw2zPeoQelOu3OdvdDrQmRDB4EPhgwWqqHisaLCMtAJxf + + 5sDzW1g5eLwWuTuDd4ntbAwxYcj20PDeBi9z9Y2ObDi77sgXVqYo8/fsG1wm5aoH + + RoOnN5cA0EhV3xlXbqWEhiso0jLRiNYS92qhJCFmM6mRLaKa8f0PtNfc00ARV/hA + + PdOhavmxBfzT9FGtgTcD2SARMjJzNpeK4QmI8J9ICIMtWhOXoF8gpbN5VnAlce/+ + + x94VNiUCgYEA+4zuseq3EIxpDj3jyzaz9z6N19sDaxPDckNp3ApxxGgJm68ouBpY + + T6CLEUnrm9sHxJLrhzoTmBweTWTIh/P63ZpI5QP7o5/Yv4HDTCquehsgK7ruZOe3 + + KcfPScwnWbQON8wrqWuoEPCALqZuMx3OhWC0m0d3GSQ2M9P4CSpl148CgYEA+iwR + + roCsiUpCdiRmBqY9vTwp7jQ9nfrFQ4OobNDJQGZvw2i490BtX5rIN6KA7HsAlK8W + + vv8rS+36xE4QP5zmKVXxo/O+EpfxiY7+AdppiEwAFXi1rzwOdSrLiTymH3ZV5TAD + + D14xRJ/h1ErsU23kGJuTFCF9mQ2NcxoCL+MemMsCgYEA4niYse+qTyjKsHrB6kPe + + tLtJwsu4gR+y991/og55LKWp+NMy6sU1OsNEURVnHNOOY8kOaZm86FZwZadV7yiW + + dAqilCUI2eBgqNHv/VPz75UaWqSaWphPTDtLAZre1qEHp+6WZJq7Hj0Yemd2kWjF + + dUmCcMZfkHAMqI6vIbldJTsCgYEA4bMciKjCAGKkr12LRnh4vt8mnSc4Z+y0R4Li + + UrnSt20za8JxDXBsvJIyDC9pzO/zyDBmfw5LC4e6c5xSAHIXHDfTd60RUEkQup/s + + /dME3thiQvzSPTQbfw2K71duMHhcahb0y8qY/GaaISMvLt23qZPCD6lfXNPjR3Kx + + gm6PTh0CgYEAx3uCHVPRKnQiFS37mq0ILMNX/uzp/1u1RPg2X12G9udV6IJRC0MN + + kBYgKNYfkaNyY31Uq1WwKbaAdE94MujHIBXm0dIFHjhmUdKNY4JhGRACSXl26leG + + ypPnzLYGZj1SnzY+d4DlW2FmgmX67hhU/EYzxi9BEDLlr6CyKqcViTE= + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 8388609 + seed: yPi3JHKKbWaEEG5eZOlM6BHJll0Z3UTdBzz4bPQ7wjg= + zfec: + required: 101 + segmentSize: 131072 + total: 255 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:MDMF:w5b5dztozqfftxg2q523vnagtu:27vxh6qxrkchh6f3i3aetesnifpsyjsx7frmglalgq5gnsms5qdq + format: + kind: ssk + params: + format: mdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEoQIBAAKCAQEAuKXnDH1YYffYnNa6yrvT6KSnpWMIACmTxFfdUhRzaGIH5G3D + + EouUzJRyNw+8DWTwukve7fMj1Tu837qYG4qlP+4hrT6Jt66B95NZqEUcMEicQV5K + + j/yRkjwhVbEeLAQh5yPOYqr0rB27lMFNSB96fNafH4S+N1EQ9/aqSumAgoMVApQS + + xhaMylwrOroezo9eJ4jy6z6i54CuI0Ay/ojmNyQrpJSWRXn2YYRs+eKeKi2IIav0 + + dCM6SaD7wubreznrNKUETgP7jFFWAK/c+v/OAh5ktx89UZB9HHtM+sxW4H0n893M + + MLzrwgKcshiPO9plim0oHtoYh9OeYPVEGLUimwIDAQABAoH/SCCGIFrWK26lLp6y + + WH8GR9oJopEMjwOutQOdcHKMojmo25IEoTnk5gUWmGuNCa5kWmFIs6pGVQUAwmQh + + BqEh69cZUJMdOKLyIcNtQk28tR+n6eDrP1NpibXzT4XgQ1FZ7PAPrnsZGCKFI5Ze + + mc2yCxHLFoTDyNfhyPqRirGrgU8IGRtgXPj9LmoKSxOfZnU5+4OHwbpNpYI8E9Q1 + + LH0+G8bsvMw73LPSMb3HCWK8uilsLv6hmPMI02Y1MbG3OTg/dGbKH2LShegPsrwu + + D7OqVYVjQf2JmzxnBDPs/3yNDb5mm3ybJ5CdAEEs2HdAy1usyHoq5n4yKYp9VtKZ + + YeShAoGBAPGTBc8l/mlXV+ciFY5hTtS4NTyOce+jZMhlbCuN41myCou5jyiqu0UK + + SCR3OTkXYL7Puv1apxFPsYmMceTEE5M9KzBX1Ks0c2flHxCIoWX1LGrFcMnzz9lS + + DOP3rr23ShA/8A6Ho7ag1LWq/OTREOqO/ImFIxAU6XlLPjg+WGiXAoGBAMOspBbm + + 4XoYYms+vmEgPu61HKO/+qBjT2bNBn/AyFKrcyO4iA/BTt0gCSfMe+TI0YXgO63n + + W9KDEKPkybyvopOuIimo73RqeydwkRxT85YJ6HLUjbHWpwRsuMxWikXbpSELQayo + + qSlw4KbFd5K2BbgPh8knYk5zlo5aCR/O3LKdAoGBAMXZnxlgSbSm93RywurgoXqw + + /9D/7SrSTJmgD27Af6KXofF74VbyNfw+hoVvK+upTPAaHFCh7VDNT1+TKjitqkae + + A4BNfv1VMOu3iLC25lEl8uHjoROV3vZjL/GJipEQy9TxOL/9sUTDBlNfnk4dOFiT + + ERvkcaobJnjT+jqAPVIzAoGAbbsTCgTPzTh/eMTm3nDG2faW6P1v/yGyFWREkL/7 + + luCu4QlKxAsTvs2IVNlHYTV8yibFUPC9fYAihMZ4m2ejNE4iuloSbqaICcYGmmw5 + + 3ZoQ0NSB4YkOgFy4BV9Ci4pxP+agHcM3mhXC5cM3Gv8Yle+fph5/8p6/f3TeSQgo + + m8ECgYACL5kmkEe0X1SEa+B82EWLqm5lO0xRjkY0jcHi4O2dejKiYi7EdjJ0ubUD + + Yp/DKgifrFTrnEiMyh3ZUzz8xrVoDamjEgdZUBC7ddVjKfKojodJLSOVcHdfY6OC + + DOQGE1X6dZnP7YEQTUzQYcrrWeXIwxHxHXgI51lLI2Qye81CAg== + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 8388609 + seed: yPi3JHKKbWaEEG5eZOlM6BHJll0Z3UTdBzz4bPQ7wjg= + zfec: + required: 101 + segmentSize: 131072 + total: 255 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:CHK:td223psbzins2k6m4frmfw26xy:opgmb6zhnwsksydgjwzpfdoz7epm4ynzmvkjuw6s2jntioqk72ga:101:256:56 + format: + kind: chk + params: null + sample: + length: 56 + seed: YQ== + zfec: + required: 101 + segmentSize: 131072 + total: 256 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:SSK:hjxb3zuhd4rvidmlcdb3h7jns4:jsvjjjob6nua665fucn6afl2qs2gbna4fvyfh4z7pqumetn2hgjq + format: + kind: ssk + params: + format: sdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEowIBAAKCAQEAlUw6OvHhmBZC8sDriUJEApeHB+1vi3HhBYHVGwaiwLGzsfuP + + Wa67PruHDGpwdzbbBM9JGaoz4fz7u7x8DK7PO/9tUEZ4y4Z6iEGZKHd4OPio+Wlg + + RLobgVW9CHiFkijUJwfwSvPDVKhxpVTggFb16+fDoZ+/9UP/rZrkjhfGaO/YKmDu + + 3Mfb3mf58KSYYWoOujJfcjntIpxTB6F6fogmVOTpyBvAQMlDzaW6wpKAK/kw0c4b + + vErw2ZC/6tJmB3SCK6fNY5dhIQDMd+UWqP4A32E7O9LZla0bgC+nzHfHNxvfx6C2 + + X4alu/ax5slhwbyHLzipDENpnkLb8OHqZ8ZzCwIDAQABAoIBADdJVCLx4ZWdYMte + + b5qTpHXFQSbJYU4lLKwKaS0p5ukupRmay3ntf796WEdbvywWb0K3tB1B7xaXxWy/ + + HrzfmzRfoU5h2meb9BIzIJFgtG98fa5mvFSXCop5gpf5cZUvc2jEwtIutL3L5tHP + + vZcpHMZwO/zFGKOtu6fBPTP1T+8ZmYnHj0cwtSEBcBn/W8e3UjUu/GoHiPjfzyr0 + + xGb1WOjyvwp98pOWZ0+0lauOlPuIKmCnFNJgiByyHUR5zI/NFSLmzV2eKikvy9y8 + + WqMvpBNd4qnKLK0d6QQQvWog/T9vTbh7n3z5oNB3Ex74AbnZWBqqzRfPtgVnqjPe + + gU0U1YECgYEAwfDS57s7qCX6gu676D0ITnMozYdl5DVSircTctHAGmoeaQtKfNgf + + v4n3RwuiXaMqLzVgdJ8FVoiedfDOMGZo7PY0mfgkKPpwcCqbRLqzn+dzJ41ToYXM + + CF4QuBKcz6w+t6J95WflpKm+8qSWEIeY/p+FE44rO1tRS8DqQVuQgsECgYEAxRJb + + FAuDGx0rc53nqBp5zWT4x/eidQtHm+s0w+za82OMTqlgdSZNWyU6rdBbate9sn/r + + 7wSonaWolmIeRjcnOyAUW2eH8FVLPyalrADlIdNRGEClsdybjHDIefQ/SgbYKm+z + + A4JUPtTOUeGjg4CiHXvp48w1K2mFxdzcACPTxMsCgYEAmozRZX2dgtgRFDovYFkS + + v4Gh6HeXyQ59IrHWO8/O6L3cUhV/XJHWawsFFYa98yTNvyUoIod+94CT1qT5izRx + + NTTWokROfKFm7NvnNBQchLcq20ASf0tiVuCvLiEW+Z/nsus4rJHpPRlQY4ipVa7Z + + Sz/QCs9mwDx7QoUPqNnRBYECgYALAXdsqyfrP7nJfywMy026FsV+BWphNvwMzRnp + + RzUDGrAfRH5KjJUNXgrk4hn44YuKiHJYqt3vz+yWWWxvZ20ddDEu2Z1R4rGNGU9v + + R62EMhT5UcLvJ+7X7QSFKwrNy4wO8qYAsCqcR64uDHfhRDHJi74IJsNhZUc/QZJX + + v6h3+QKBgGeexmjqVZZr7vibx/TooEizGqrx/Kz7vYFN2wgFrZo+Yi6Nrj0/dEv3 + + 1p7umm1+MDh/acoQ42zW2wwhXn0yxTPWpWB5pZyiz0A7kJhwTXx7XWC6n9rnjuJE + + ELMWbLucVThn6GS0MChS9/pBEXdbHf2odboaFo0Pd0Z7hNWk2NNV + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 56 + seed: YQ== + zfec: + required: 101 + segmentSize: 131072 + total: 255 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:MDMF:peebvtagia7nkbfophcnzbo2k4:tkxiwgqxp5fzpmb4oc4dbgqldwlcqocr37m4qwtzfuesfhpnxf3q + format: + kind: ssk + params: + format: mdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEogIBAAKCAQEAnoh69ybyw3zQDi1FamZlbGjgvziqR5QUdo8xVXarkbk5LmNQ + + T0nXF3VUh5O3PHl4k9siXMTp2UmWxWaLiiHLD/r/t9dfXkiR51dx6YJcC7OkPYLM + + 3Fvh8JeTWwOITYIbHCqWa5YMYQqhauI2njE9fABTUBNPaA4PZpa8nhKW2QGHCTht + + 7vYQylqpRxM6MQEjBHw/8ElnRJymIluQ8to/eYgwZqBtC6EZoU3rhluFi9MfEYE2 + + Ua5xD4XkLEqASQBg9kRBSOrEhNgF++cehF1TiDAL4J0ydWSUAQIUOCkqS5HeWbQi + + 8V0wuCqUGth/HP8VGdh7YJMTkW7wR5lFAEXkfQIDAQABAoIBAAtwLHMVn9Fj+Xz0 + + XjxJjArQ3Fpfo8WLVRiixzvz1ngqpYoHx10ZJkg+gm1Pxpo5522/k1CfMoIncZXn + + iqzaOFT8VqP0iaB7Wu+WmxTuf2amvPRlMhO6G2io/wxDinuRJhSXrAeyKU19H11f + + WfJ6+gUu3tP5uLJ4xTqxKIW9MJ4ShYlB8YWJDZR9gm+XrvenzWi9e472PpdTr+rA + + jTmeTrPnPWvx6GUR484fHi4pCFn8Mxs7yA3Gts1x0FKD/dn6+TwbTfSNO4dyqTjq + + xz0o4gGCohQdy1w9pXBtUfQCzUh+vIbDxreJz1owexLvXpYWpfL3XqkfUcWfxZxE + + FBgZBBECgYEA04hmJ6YdsKmjZO1N/7uRP9nmz8v70Jxb6BJ2PtOdraxuIem0eDw8 + + pp7I6rfwCbd8VHm8K5OJM7ap8IuyClLbONWdjWcb9Tm5cjIggdiMcFNNR+KQCzaN + + 2UMMo15pKYYEzi32nIDbHY36zY75VvY1GnBwqYZMGEkjFQvsrzLcmakCgYEAv9vp + + pdepdQX+OJ+2avoJseyEsVRBKRn8S+lghGGxAcA4ONC2VumigA6R30BtPhvvpm5b + + 3CAR4WHQIn6bGADf0bBj5AwUsyh1rdNAEPCRIQNQNFGmbJ+q3dIVsAOIOR15FZyh + + 6jO8F3EeaNW/nofLd0kL6RsqNkVQ9buD2kCrQLUCgYAhRjJziDDhaj3WkXGUiae2 + + eItTIo4w6XeXkNfi2BzUhewpD38g7rDHsPB/44ExthgrnZ6Y6DNL3C7tNLxD3Xa4 + + gPmwlYiTUYo3SWVNp4en36KnbR8ldGZpx59ET4SRUJCO8jH5ulc9Vekezp+wKzh9 + + OTSvpf1wUIjhNaf8gy6qSQKBgG/4gFvxiUxquvuA+o1kb9QPHUIA0iaSq9QB1/pq + + qUtES4udA02l/NiPqEKK7zaYRzzym1nUvZqz4yy+hvVzTSyrrSCijFIjAsr3xyQb + + whdqP5zJKj+qz6W1vkKDTTVIJiqex0BQAS4NLpowZSy4Q97SGslouTIDMkS3szPl + + Gg0hAoGANhX3+SQxBZAwqkgoPwheBe75VIeQYQQ9mruxNJ6pya+PI+GRWxzrVF1z + + 6L1hBwHq2CHNiZ6R5ImXIIezNU0umqreojMn7rqhc04NvZqdrSkwPA5lqFnfD/ou + + 4BknZX2gObawZA54qHCgvKTFHlk0F5bavfqeqvuFu3miJCTQ7uA= + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 56 + seed: YQ== + zfec: + required: 101 + segmentSize: 131072 + total: 255 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:CHK:ihyeqq5peg2pdyw45ct5mmj4ay:2rzhndv6o4knlvo4po6rvbnknlcitasseea3mirqjor5c7quh5ra:101:256:1024 + format: + kind: chk + params: null + sample: + length: 1024 + seed: YQ== + zfec: + required: 101 + segmentSize: 131072 + total: 256 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:SSK:lhvxjaof6loeiw4crze6xczaze:uib2dujnn3ockx42o5oj6kpdaar777nfyvht5z74zwil2okawuhq + format: + kind: ssk + params: + format: sdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEpAIBAAKCAQEAub+SzMzecLZrHoxX9QwQIDv9chKraN/m05DP+qOX3ls/5Ev0 + + 3ASAhk8uVPonUQ05U26aOzgeAkvjMIq2wvf7b7il5ZXuE9gcif+vTpy/uhyPwhty + + hvAxX2mdJxLfiNkYsXOuU4/bSQEmvQN5eKHdoz4Pk4qNjXO97JM9XszzWC0XSM/h + + PayiaMgFeFRh4DRtO1f5V9IpCtsMZhyP+w8RuhFh2FAzO9qzLgKQQi5DiHQBYWc+ + + KVcIoTLp9gNpijppFIdhwmjgcZ/OBGKogHqi6rfGvaDq2RcK9QlKQ7h9BFfXGSRc + + HZCG9xRETu9H3p/rSe+RwWeUvmHes+KmmfAUhQIDAQABAoIBABp3W3lm74Lr2xN9 + + N8MottuA8LniQx4sWP1oMtopmSgLpGzpDbiTw6Rff+CHzDZWRgbHSZ6KfmwhV/vA + + qA3bu63Nh4XQ+R1Gu7pF/jqbRw5Dp5AmzQjBDKflqoi4vbUICeau7vXlF3+tdFGW + + PyabGbN60klZgpXXGgatbB8n4Lx9YeeD4/I36dqdqGDrGA9+8Vb8UPVWo4ytTYtC + + V9gzBduz3L+oT/FH5MyrWkb8sPhICYyZ21/9gOH8/dlLZA8mYxIvBBXumE+H7Wyw + + 1HDI6hFVJgneqQRisBvJOahKdvqbsJvqxXk58Y6dYPkISYsvqXNazMZztNoYH1jp + + bYkZFcECgYEAyDbMhjH2abUEY00Xsfmk0bro+GSmZvks2ZZh8W7UnaTZmykmbVpY + + nrc5bgqFulfAbI5Uic0dn15hvV5+HiaUov2ipQqCwGPhJKpVbdMhV4rNTVG+ktbL + + 17hI6rZNQ/1UWp8A2TsrAFA54aLdyuFpO3Wj3h+KG9hahMqk9hXlN70CgYEA7YDw + + dEOyYaU/aWeCZ1hAFNeDPMdRdOM3YFwKley0crE3UznacgCn6QX/K3ltbj3/XFUX + + M4O7f4kqxTrhGOBRcIoWn/YqteZe2Qlj4k1cET+Wfg3U/u/S//VhpIqegjIXaTAU + + q0s/O70yrDACSNxIYR7pf4J9Q+I4ACBg4MBkmGkCgYEAlDDbWUiJ1twBD270Zusc + + r4/k+FWnRPiR1cuVWxppjPWDi3D93FrO1UtQ2r43FSH2b2M593U2w8scFQpn1vE/ + + exS42efZt4U2E+lvqgZn22AFbYFfyVfrMRRaBEBDGFvdn+WovyEoRucasIPYHl6R + + gU0lqTc3Bj0xYrCLQQobyxECgYEAuufo0yZfYDbCY3nhBuFNdNlxX0hgU0No1f05 + + G2lvTH8oUefKgEMB4QEmIZlqxAIoTwprus+lo6VXsmU2tfP6Qz14tqPsUsAbzmN0 + + ZqiIls5a6ZKLF6G2hFYgZHPub/lpsQ70hSUvexzWnukdMyegEkZYbU9Mszp45aiV + + dOoTgFkCgYBJrgooAWpL7k0AT8M8ApxAu9tvKwfP+HXj/V1fM2pmtrfPNGjSK3A4 + + pyG2m+TbWO469WJ5oJTikZk+qiFgv4nIm2BDQfEtnZ9Hh3ypHEtlFlhn7rWsRHvv + + 7LtGzb4ohtIzgQCK+l8gcZ9h+VhfeLp0tp0jBUD272eeL0j6gwBVow== + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 1024 + seed: YQ== + zfec: + required: 101 + segmentSize: 131072 + total: 255 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:MDMF:i2cep5mifmrncv4xwen22ppu7m:yfgy5mvr2mmz7t4dflek6ixp6yckvcj3t5zktvmazwfvvekfevka + format: + kind: ssk + params: + format: mdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEowIBAAKCAQEAteX517k4p0pP+zpTv2q3u8n2VrjajlE7eITVGQiSh/y5Fvs3 + + KWpAPATaxkafcCjDraQv+FoH2QEdw6cVRNw5J/xlnofkNGIOgSVGMM3udfYV3ywH + + /+usPlfOpJ5PQJ0csbP15wOKhdeLCklvmA7PvzoyFV/80sWUnriM1FZqkfVWF6VZ + + eL4iBCz1z+c2rbAky5JRSXGhpX0Bw+G9f8Vch+JeJGE7p+IhH9E1bmqSj4PtZwe0 + + o4hcZVaZs1Doiah98Thz1/3hFvRIH2D8YO7nbvdlsQMSE90rReVnSjCsUO8mwumY + + /k+AGMmofUpfRvdrrBGgZhqWsPEuCx9pkW3qpwIDAQABAoIBAA/sqdJWf1y590WV + + xiYwaBRzKnNOLKgf+XZkHqnZ48Yu/F9EMACasjPu8t4/6Y5uqy7k/GQgMaawX9Q0 + + qPqF0yUqhhT6baKeYQmyYzI7nSBLon+OwcaIceRlWIx15ZdRCeWOzTzjxPlRtPT6 + + F+B2j3580Eypwh8LuCarHn0qcZsEeyyfRaJy9Ydpszqx6q1BwTOppF1OiQcjQvX8 + + 5p3uTTFPHS4aT/U5971WNGBhy1PyRJYoVpOzYJWqGfxtX6JmrWgZf9+DYQuEXUmp + + XHfz2ds0cE8vtcFfjwaylpnEXA3TiSO7055BnQQlEcsDqmgCErh7y8dQ4ZDR7Qrh + + zgiHtY0CgYEA4EsQ2NPYrlFFa2B0+jpfsq5g1+zDCYS2HeYtt35nz1KKMMiCdPJt + + Ene4cwo0dcKawlPNOQspRqIJnctgxVMYQw2OWIm3mn5QCL9YRXj9P0mkMMSGpsVo + + 3LDkXhLLbig+VhF9eSJdu7bNm6MTViI8aYclYVJ7c2oVq7Q3sIkWjzUCgYEAz5yv + + LE4Kr/CaGT4GQ7cusYsS6LXbZhpLVh+ce351K/T33j18NfEMhF6ctbGM4va8kgOM + + XrmjARpBG9RVmQn5set0Yw1Xzu716JctGGJL7dZ4dnaOzf/YKw3ZSipYpWBIGLfN + + 6buhTrB3ERwYrRwNlmMwArPXI7x+wL9R3+FsQesCgYEAvs081KdKseezHUgd2uwj + + krYi7iycMhGydzbjdzBSER0PL7ayu9erD8XGpB5vSCo3Ss7NSxSClXKsqY5kkRhC + + EHCMwibNiOChJv/XkKn/DYKQ6WeVgHN45Bya+KgWZGxZsxAH5C9m+5Pjzt1oSqKv + + L7pnAyaOnD0HmFyj70p/ZW0CgYAroYUa7YfHc+wes+9DGeNBQrYFm/pw2cPNZLVR + + KsFbLI9O8GMDPxZfVzbd5GN0a2Az23ULjz3XhHn8bEJU+Ei2gIIkMvCqN4QMjoDW + + qAnHARSt6LqYRlVarv1kXcPldXeRYkdvAJSk4ecT/HCfKM8eNNgpKTxkcT++KDb/ + + svM6YwKBgD9PdZuHZUWelB+ELEpvpqjCYWaO6NF6C5tWLPD7uTSuP12mYz6qUkG9 + + zuf3yvGkeiEYmajzrL/29MHEjphWHoFchlUAkv9Bmg+HjLNpC5iPyIB9kR03tC7Q + + kKedJkJQtfWNINQrcDvEEfpz1+xxfyKZ02DGBITjGAd5ssW3yEcD + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 1024 + seed: YQ== + zfec: + required: 101 + segmentSize: 131072 + total: 255 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:CHK:mvk5f2mghudk7xnknqddhvnl4m:c74myeugtix7e7l6uwybtrdtfselha2qrcc7hhbymqr6brtg4cea:101:256:4096 + format: + kind: chk + params: null + sample: + length: 4096 + seed: Yw== + zfec: + required: 101 + segmentSize: 131072 + total: 256 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:SSK:sghr4ibrl7tsy7ijarale5whtq:23bvdxndpvbygxjdrjjxbm2x5dncjnicywugvtnwopehqpl5qwma + format: + kind: ssk + params: + format: sdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEowIBAAKCAQEAszz2tUQEFK1L+sKzY01w/0UqsfpX/QD9GZiIg6BzCMkSq5Ui + + ONrE6SUa0aLswwEJAqxVLi8nO7zjBy2XPMKyrw4KovNodoO+7LhpQobpUz6yFyBu + + qu8LLQJzXZWVjxHngTmsLT6a3CWxXVJeMC6AoSXxyzvLeb9D5Z8+98N3H7fk0BPr + + DKLQ13/njPUGdKeLZ8PEuIp02cUs7/0f/cw4n3ZA4FGCpkS5q/TIo/k/imsiOY2p + + le50s1S1+3DAhoL3G4O+TgI5+0jsgyTH83f3tfyxpbM8tll3VZQkAmQsjJdnI5YL + + 0Bb8BhHYWJYLReOJzCQxpOK+UvoiOYBGyn/g/wIDAQABAoIBAEDGxo6KD0N9wdjV + + Zsl7olvPHngF9qisI8yNUMDpSsmhCYtTMXQEtGdiDog27oQnKp95sqsnRXGUeSQN + + +Ptvje4wD+4GM/mo8WZR21C8uzRnkytCgFxsWcihexoWRl/XY6hTNIOBfawUPz5v + + 1zRoifozYWhGqunMEvi4jaQzUyj1zJOfEdVmPsXdTz1q25HBZl+UmffdS4MVJW4T + + E4WXgFrqO9G2gnoOE8sbo7RY7NL685EKM8n5kBJN64S+F45lNzO/B5OmWETXTAZl + + DavcXRM61jPkOt3ZAT7OxJMPLF8uGEZtUGAFt28KLaTfUvnb18hwQzvWsfVKcugn + + XnQB0okCgYEAx1oBhxaWO2B2qldDfv/w+yhT4XzWZ4eDjJ7MT9yQw+jVepemN29T + + igHySEZZgHhdtdNdle3yg4TQoUyhs8Uhc+vSuMvOP4vaMTrYB+e7zOtgRleE2A7Q + + W5b+4y/LSa2VdSPvkF40JJjIDBJs2d7ql8wBUe8uDeo/S2YnhKRr47UCgYEA5ivJ + + 5IuNR3Vp9qeU9lsIS9G4T0Ut99n7lNjPhNzg/TW1VdJa/ADtCY/hgUjaoR8A4/PQ + + f7JCXrd+FX9EKvXALYo4ofRoJB8sUIps7/B5vRba9qQdev6s4jOutscqHaIePLOY + + W4uWeW2CNgjz2sJT/Ura50ZQWxCrGY0IxMryymMCgYEAjqxkG4KW0rgfNZpuvB4B + + Ij+iiOcHq+DYzXN5Vk7NbOjeoHaYh2Qtrb/m3sM6my+KIe+8Mumxf1820bo+oKKd + + ZpGIpql2WxSEfGdY5Y98YRS0OqO4d8liZaqTkZVLMNgC92tYsUI6n1aZFcq6DNP1 + + od5ns3QyydK0qgnajpv+e2ECgYAlZYfv8hyKN7F3udKiFDhM2U4w0vSdCHWvwWo2 + + FA0aFtfXkc3mk9/vZckl0Eh1VSw33S1LEhNmgDmkFIFm2XbX71U0OxmQhOAWVedN + + NK0S49u/pvqDOU2tkugYGlPlbKmtAyEF/q/8GqbFUL8OE/TBeqAGY446vYKPLDL4 + + hmDcMQKBgCQPtno05prufVeyuCGn6zXDNZkrN90lOL9oRh2g8xu+Zv8wLQZmyY9o + + qpGKQvLx6QpKCL2zmL4p25zXPobKK/KusqOyVhW9sI/ry3b+4I/IiM9XjMsY9FeL + + hkoE2IISx/cKvrxjFmccnffDZD1u8drriVKra2TRM/81P+tlhInT + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 4096 + seed: Yw== + zfec: + required: 101 + segmentSize: 131072 + total: 255 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:MDMF:cce6edzhxmwz5gilmwmfg3vu3m:3qjixjwr2n752zvevufhatb3a2ugcuey5w7u7fvykmi5lbsjqgcq + format: + kind: ssk + params: + format: mdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEowIBAAKCAQEA7EAXUuSzMzjd8py8jXmJJz66vnbGPrwSU22sl3gfRg87Ap4T + + k/PwAMIwbCpbZ1T4Nwi8RsMg8+JvZ/FVxJp9zi3C4rk8lKlJ0BGECKjyjQ57L+yE + + SNfUMB7G769llAR7UoYZYuJ3JgEF21UuRL9RH2JV+MPuEA/FhNUfaG424TmRZ75O + + yTOBztsbzxIgnrTf9JBpyn4iVL1L/mR8/8eT8UlLHgP40hpnhS+frA9vH1J4v1AD + + VVOW2DtYmyS2eAuQJXMAy6XATrd9hWaGy0X6D8q0fXwGb6VuU+zeTBP5T6YwawGL + + amAvqHFmiIYwdghLJp7UbFwmXgh9+oMvmk4/BwIDAQABAoIBAEhp/LwzzZnNvHo5 + + ALJ8pkWZPLRUw79G9ncMDvL+ptdao8PRoD6hbtdMrnr5ILszmEGGM++cr+URawR5 + + PMjeceFYtXu5O5B1s2JLfCULZA1IewndfU62mRuG04N0R7ZvCT3qTK26rLrBZYIt + + QdlgqwTrp15w7++MZgapLM9duQSfgcvw6mXpPWxJVdEl01icjgEU/aX/DMpu+uJK + + L3UfJOlMX0RNBxLWZIF09Ur94T6zc3HzzDB35kNNG/cxibI3qDh2KSBgaJudiZZP + + m+P/uX8+0xIt19Wi7ooXvNlgmueZraJWYLYlpKUOpxJlBd2CBAKooE0O/VWhidpB + + OEr+X8ECgYEA7N+yemfO3iJfThXruqfRqg7dCcmOWdwevS3AVxlYaXqyGXShD3C6 + + +BQPnX678+uIlp+aMcONAEdtY7SJVnQjlu4HKYiL64pVjYmrpG2Bk2igHxvNGR0v + + LUZgPG4pllGuOLx06PQUyipjTZGKYjPr6pqtOXMk2Y/jo4dadSXcIXUCgYEA/1OB + + tOR0zqB4G0bQ597UCZQZ9E4N0r/V2Yd6OUecnDxCDq8Keg0XXTHLZ56NvlR0hcn6 + + jrylKgxtuU5s37Zn5DaGOd4mQbOvqPnhMJ511A9bSWo8TthkDbUT1VaOeVEaQig6 + + bESqRxOTZbiJg80skWx70FLXIRx0/mPt3rIkswsCgYEAjOqXdxKCksvH/uAzmJt1 + + s8Gb5dKuiO7WqpypLCe73SRNB6/GkTTzRdpJX9yhW/7nBxRz2t8G5v+XKBWjDneR + + JJz+TcsZ0ko9kzIvlmY/C77WYyta3HHsOvb/EXRH8VEuYDpdIqjyJUMKSH8o4Dsb + + Qjo6i07gwT1Eo2hGfCLFznECgYAIfDv5SQZgv5B+R5I1woAFeXiLV/S5pkpzGj+D + + m8+mmZIQbtzIRZsbK8Z4wRow0xm0QIwlJjvO8+7Jk8Omg6dcPDulvK5EzLXvxa4o + + MXv0+jWscO1kKWjZ08S++Etv2LQosrGOW5HVHt9tJ/7Z9H2gr5xFxhsELK/urF+B + + YSY7FQKBgDDzEpSvjNAFVEpgMyPR0bKHBNSAQ8lcNZ7Q4UaVuiGluwvs0G4yLeak + + s6dufCJu4O6fKuxwIq7u4VLFMdG8XP4iNszJ0KJaxkfVUNdkdGun40gv29znm9qD + + cFKxkFQa+ko9Rtt0LbU032IEaCvOpJbk6G7bJZlF1w7lPSjJ6gl9 + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 4096 + seed: Yw== + zfec: + required: 101 + segmentSize: 131072 + total: 255 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:CHK:rvytwcc4sbmmkrdwh6b23ckv5u:klk5nqpbv37pr2qavcebcugfnikarjdefjlseo3ru5uv7guai22q:101:256:131071 + format: + kind: chk + params: null + sample: + length: 131071 + seed: LCa0a2j/xo/5m0U8HTBBNBNCLXBkg7+g+YpeiGJm564= + zfec: + required: 101 + segmentSize: 131072 + total: 256 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:SSK:nur52u6a4vs42cm2bezo5gvbdi:odirpm7xdlhvhneedjqnve3daaq2vunkdj3to5frmsytxxhcuqua + format: + kind: ssk + params: + format: sdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEpAIBAAKCAQEA1iFUfknFQ8V+AOW83RO72R3QhTSFqJmldpQydQHnzIT728GU + + dsE84C2tiHnk1eYpSjf5SXTPfi2wzXDpsW1EePR6nsfThVG5hkxryhEhBeCj5mak + + +mgaCiCarHxOzz4fedoC0KWns8BfTwCZv2vd3W1QiL2i7l1EwqpmsyvoWV4HHLKp + + nNT0uwVgPxEsrvCYKlPItnDyh+Xdr4Pbtg0uaoAWyFFkVKAF7YVbC2SEUGE7Af/k + + 0N3Q4Efmzj/vz+aB+8Pa4uflZ5ekf16o8US2K3TxOToIqawEMn6bfMJ5z2fi0pqa + + 4thFPlkFBtewMhezmzQT4qF7RVSMaC6P0Q0h9QIDAQABAoIBABDjGwdEqR6BpkDK + + 3Xyv8DocvFOtAzd7Oo3h/SK2LkI2YKiBmUROVA83+v4O4umtl6cHSA0vfaetUcq5 + + 82wvOl2xpjP8fWV/vvpk74FFnY2ZnENw5+TprdgLnzcoLIzykMfq1hr/XXzzGHEi + + En4Cs0Ihu161WfKjf2c8yhGqTk4x06jtEwicL/UZo+mB9+mtXq5vzI+/5pJisDiG + + zgYAw/p7IEk8BG1Gv+99+BVQuNKSz+M8V/Bt+DVVb9Iobz6ehUCWN12rQO7S8tgt + + VcJrXK0DxM0HrQvnSmmhbTqtdb7kUJvB4EeSS9qlv71tLlpqNFTvQdh+pdL4w4zm + + HbjOCacCgYEA+py0erJ+hyzEi8H3YZiau+3AWnncIkOr1q3LSD9gYWMxQjVpAqzJ + + pqeHPRmemjG0NbzzOMv9jDdq5Bmev0Jggk7k2pWhcZ8dz7VrAuKMa+dpGmktaqnA + + QcHFBTDHyuUjcm5XRTkcf3EzeXp/By3SqXhTRoscihVZWTN4C3w6hwcCgYEA2rvW + + 25htboHdnoR8nHi235bryy3GAajJPY3kVYKSpkiCk6uu0ZSpBCZF286ETdw5PoZk + + 2ldyzFvo9u/0dK4G+7t1oHDOwYrqO6h8G/P3nEuA3WKvt+Z+s6jG6vbs6uEmkmpu + + LUxKM8M1663IlgRQ/TOqJ98o5enKrvmwUdBW9CMCgYEAoJWNKBn77Y4IGy2c4JKy + + g70ixlbTcbk/AP64BYFmtsCirbQfp7EkPX+XrtUdxdwXh1+d0kUUIKbZ/XNVP2S/ + + BoCbMF005+N3bMLo4R5dsD7GIEBI89H1+ay6HEtXmnEdN5Pwo9CmrBrTSwHtJ6J7 + + HFCXu9oj3W80o23RfDqMHj0CgYA8maAMVO20mRw6Z8BSZYtc5OZM81CRcx7WA/LH + + 0hYpJZuvp/gWLpapBKWEIXI8VBA0B233pBS1E522lIJotTJQGf6bxcUyj/cXMjW4 + + VN48GhsIuueuDpj503/Q5zp6VIioNf5yZFmGf8X3lr0k+uspS2AQDd653A0Ab0Lv + + V8ZPewKBgQCjv8GtGEA+dbyhaXM4Rr+GVprQftbs65u8cj51kvILVZnUGL4NlLgt + + f4991qQ1OBYyktmxsl2MkcZ3bVc64ehIgjp52yN2tyye8jUoZioCXq814Ru6OTd+ + + hQPw1PP69yKBxosPoMuLTgxIuo3N6uv4bBiGgKut2hPIncCzzc9png== + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 131071 + seed: LCa0a2j/xo/5m0U8HTBBNBNCLXBkg7+g+YpeiGJm564= + zfec: + required: 101 + segmentSize: 131072 + total: 255 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:MDMF:k5beixrivinifsorf3gsa2nwve:ylto5ecf56zgcbozhrhvfiftldxm7jz6tlectqkorp6cmqlef6sq + format: + kind: ssk + params: + format: mdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEowIBAAKCAQEArCBOrgZyPxL1uQMs9Fu0Oynkc2Pt/KO145f2D8q/1RlwaHTZ + + LgSWzRXdMjlNWOz2Yf+tp1ARRwSr0tLH8J+50eSKKqtyrZCJyurl0A+xOpOXWdS2 + + sPEApM0EqU3wHtQHVbQNZdnQl3Tfvr7lMPudnlP4JVJ9BALU+fk4Hd/V1g26dlU5 + + Fefny8pBn0ET6Gxwxzsv9/EEzP5Zdh9RsNNPETs2YvjXKiXtwHSrLluKrub/5XC4 + + ZcWc+3bvcPIgk0DJC2+e++0H/vB4mtGrFW9JfkD4ZsbZFPN0hU6g25OKq5MRjNix + + 3Mnr6APKdW4d3KNwdWkXZbwlrIBZJ3gSSx9g+wIDAQABAoIBAFEQ/j10B8axFU3H + + sxp7Pk1HE7NM8z8zk2zXmyog4WxqCMkJj2fe/W8lxwHqfwVMxVWuZ4kParO5/XrT + + jxtC/u9d5bzm9qHMGzmYnBf77AqcjIHgbxKyzwzPCkz6ygaa8cFphY3coiNTBjX+ + + Dk+dkcSJ46sgSITlGI2K1OUtELc5BMZkfHHgaSx7WMV49IZakzMzeJcRzu7notFh + + zIxH+bMjSNy8w0Pb4lQZuATUJh9gsQKDI+XSWQTcYoW9/P4mEn41u/g5ZQuyLTNj + + /0obuVSzvcaVizeI5v0CN4FeAS+dOhA+rj3m18guuqDoW4m7KDLM1QL5C3S7N2Qp + + DPQzbGECgYEA7Xbvydy0cOiVDXBWkqmXzA/eyuD6DH74hFIJFxIoI75shplvjLgs + + uj5Q9RWukwxadlGiuRJla5urMlt9qhO6kx7pXYYWBdMWiY+2sx9dEf3z7NL64S7N + + IXtSsaKaS+EWZHBfcfzhQMVg/A/9Eg5FIwUnIy+81lnEKcNuCjYqPDMCgYEAuY/E + + KR7JPmpHOXOPCiJKWrhEhtdp1zTFeV5fERVM6tZbd7Qf4Ad0FRHfZtGP7kRKSEtU + + yWQC5VvWGDMgVwNnwAPpURYd5PN5yINwrBqsDpeQlj+3+06irdMvPII6wzs7KC5E + + JdfwDX4dFcKXNgV75VEp2nfd+zmIWN2TFmuCgBkCgYAoJZuUvUOkcy3//6YjVZjc + + XzKDilW8FxtdA6GVzPQMVv1yJC6/08N8GV0GkovZQJVqu5KPR5TuBHuFIAK25m78 + + wJUjwq+mfHGrACkbT4okqJK8z06rE4aKypbIgX0kpwFqKbV5SA+tK7Gh6/IVQ2Rc + + 71oWkNOUScjoZqoL/+xUEQKBgA0FiLd2AJtPq/XdJSGJ7HvXSH/J6BSBEIaG19cE + + DqTALCUHT+FRxJSh73JwrFAFHM1b8/Q5/3YG7sw98jwI8iPoYlwdWDWz3Ez05Fg/ + + eul/O1c/23JYP1RBaKQvY15F7s3QCVo6gA8CVZosUJ4q3lnmSzCYjsxNakMKMYM2 + + Qi8pAoGBAOVd3g8BqGeJ3J/CrChPAau9uxAG3RBpehsqK3EHInoLb8bIo4Mm+ezr + + 4EMIKjl/JgYOKM8h7QnazNQ0pYyg7k8bzSw6TXEssDK0G/ALGDouqFSDYdOyRUAA + + qe/ZiZGQqcMWp3UKxnQtGIsJEnXZ5nZqsJJUSVWNgIAAtZI0W3lr + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 131071 + seed: LCa0a2j/xo/5m0U8HTBBNBNCLXBkg7+g+YpeiGJm564= + zfec: + required: 101 + segmentSize: 131072 + total: 255 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:CHK:ocqx55dopakqnzyui6q6euz4gm:wklziyoctwrz5zjfg5wfejz4bnaagtg57fgjlspl4ba6ij6n2obq:101:256:131073 + format: + kind: chk + params: null + sample: + length: 131073 + seed: /N4rLtula/QIYB+3If6bXDONEO5CnqBPrlURto+/j7k= + zfec: + required: 101 + segmentSize: 131072 + total: 256 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:SSK:r7gqup4re736yv5q6zeupiemri:wg76o4jefkm4rcq3hkcsf6ecfdro5qpg3rwotqhdpc3cicppdxlq + format: + kind: ssk + params: + format: sdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEpAIBAAKCAQEAsHdxSt7u3z9iynZlqa9Mbx+Et6o/RxZxFWwF/u/LNuZJ+bgf + + X5kaU7KFqThr3YXvHGkTtsQ4AKVdKr7JqajQiU4fIyC0DVkzj7EvGqNpGcAMNcv2 + + 8iPXuUhvWujV43ThvI/AbGbLfxDeZAN3158Xrgh1FwEl1hReGNvwgKpMSoy0UxAv + + CMUO18BotQPXtThkKDC56USLoGISzaTKI3tDjQI9O9hYVv4n/XIP8qkjHkTtLLwZ + + TJTLdSVnXSzlDfwXJcKsPW2HIrWLOs15kow4681Lf4dv6JHeyLfruva4VUdxq2yV + + nWCCE+p2+2+hEzvnu3MGAgk5bvM8OG0+kMT2xQIDAQABAoIBAAZqsCGGleg4yWpu + + yoy/HYKnQlUdDp7StlXO1IX8FBHg5weQANYMqWwp4ESHvcZeSH7/jtJfyWCJTX1l + + 1svsTKRIMAd/44Qzvj/8nlJJPGO4ZnPuANgLt8cFQbABf5uShGMt//4rxme5Bjkx + + S2zkBwFSoz1XqEJOrHWaHnad8AHAPhCu4yIQasv2YoshkcKx4WvgLt3n6034t5PH + + VIuqQ9SpzFUYR0a6OiU5kRURyKohBK5fX0hg5hwS1Rl2OTRb7ZhRcYrpCXSEoLPs + + tq3L0K+yGBmcawzwVYoM0/JaxxDwgzWRGBOjEXs8ujcx7C9wftJnjUDj2nxO2IEd + + 5Ahyi1kCgYEA4rUVdaGKtmz+76qh4Ymn4T3Yji35+a2VGFoUb6GNCD1XYNz9jy0T + + LmbWUt1bqFEeaC9DTf0lwrLc4UITmeH3UjIB8uTaM/w861NynqbQgrKUb59JyWao + + t2nk90tJc3YhDUfifvYEldRs4WakWZCQa6h3zB9z0eJdhA+arlqGTO0CgYEAx0SE + + sbEcuq/dDaIEaDVA2BfITeIBOMWCoHdsVOwIMWHYEZ6W/04DDTLFWapr3Uzt7XHX + + ixIcaucL35pjY6fdNNyCY2wwkCJfCLU94BpdYvgHzA27nUjM7jcndkGajMuk4rqJ + + veoriuOhM0v2t+Gz4WK2XkyQTByj9I25AdkQbjkCgYEAt+ANxrm+SxX+dB8ea1J+ + + EodZ9H+/501txzGQr7YFMHCoRU0Ybx8tFo6cONuHMu6QTgo/eargDJmL4zv3r/EB + + 6u3afMo3XMCyHGAzcBB2v/rdv+cfLrYQE6tU5Wpv6bEfP6lVQIqDz45avTrGBErn + + iBo9CBdelhYWqT0KxW1wzkUCgYEAlEchMpRnm2eH50AbZWvTH7m6vHGjlRor1Lpo + + 61xj0FNNk/bdx4bGcIjKH6nX7+nx1lFzIbJNYSMiS7Y3pQ1hZpd7kv4LuQVKkFFF + + hMA5o46LRsUlSanFjLGP9Mhmd8SFoo1KN/7LfeNarbAmG7igwONSbyMr8OcS/cSD + + 2aMrPckCgYBM/1kCd5r245+iSPzO3k+003ECaLhQOEntBrHvis22OqBbEWZFujMg + + nD5QrFohNIWiH8LH0JgAQkP/DvOIEYJ6f4LdIvDWqxZMEGbce9O0jzSm6mAjLV9z + + 5x8FCU01dNT4dBxkDhjYOgdjsXIOc2P6ofoXk4Se471uOeKdSTWAeQ== + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 131073 + seed: /N4rLtula/QIYB+3If6bXDONEO5CnqBPrlURto+/j7k= + zfec: + required: 101 + segmentSize: 131072 + total: 255 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:MDMF:qig5fstexhz5jhoq5g4nysljvi:uyagmadjjoys6diivtrztzcfqicwiwswchfwfcmppu3fmshdpsza + format: + kind: ssk + params: + format: mdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEogIBAAKCAQEAwGlYMMfNp7uC+5rxGzNItvC0Y3vLkOGGpZjuR3WzjZCG1AS9 + + IJJVqEKCFet0AHLcUJT6sfozrnus5YCWwBul0pc4ieQC+xzP3OIev3e/93scTaG+ + + 6kp//CGUHOxGkwMjZsc4tPKitKeTwoXijVaXvEVViwRzm7K8OOBeOmiOuhqp6Bu7 + + iyMmfCT7Po/IlbLfXmpk7PIAuyxRPRBuCQWgmP7Zm5UmsWudJ87P4P243ZrMOVTv + + uJn2qROPhUOTFg3Lxv/i3HMO0fOSPMJkYBpJZmOvsF7cnFEAmfSH9blLrGB5lfwC + + yv+VlKLhxAiGgGxhQo82ltgDuN71nPYOrpmfEQIDAQABAoIBAAKx8kDdqmWbI85E + + wqcodUNo7l5QQ4VqhhV5K/yotNMEVBuLNPLfhdv69+g/cYY+yNOUOjygHIiPdcUL + + Se4T/Q/zKEEBmZT9N6alOeKUvNkMmqFpcduFhcX19CWH3dDmy+B4/o3Fsi4XKx1M + + vSPqbREfd5kmxVYU5cJjFykyVRR2r275abi1AglbHzscScBm2fFhDD3x2tAkHj7O + + kCIN776u2Mwz5r91oZKFoTaX6OzwXqEXWP4K6RAsjIqNPXiizP3kHCxyZ7YVGimI + + g0wySSidet4ebvEr9qkONjkaTh7BfLIWbgiDST+LVPbWQ6YJm2KPMbdK2aBqwNw1 + + MiQ9uXkCgYEA+wCUX8p2vEGer7tlwosrvsES6foF/10tLBAD87OJXw1iXIrpwhQR + + XJxSLYlF2jWZo5XNnoyxfsetTRzgTzbbWpMklV8bmQ7edGfCdhKWnWA2B1dsQHfe + + JTBhYkIQbnbYR10xlzfYUHNCsFEjF2GMvSezfuLSRLJwl88ryBcY69kCgYEAxD4d + + BIdrVZhQkswwbWirDEayrGN0qmwT/rD2C5Lv21UUY54vSyoQiyu92mWgrRDSKAPo + + Owne4ydqHhkb36+Ss7dqUAEoXqddlWdTQ5K2+f2CLq6FJgfSE61SOzEm/+0LImjj + + tVUOeZlhjqoVpgfQ6snCpYuUxMzOp35PJVGXYfkCgYBzh6fDk3glXHrC3hmPeulO + + qqWfBlK+YE/LaS+4exmuo4VznQjNKNl47AazKOz67BLkha4X3SBRf2zYAoOIUnKS + + dQmwqw8T2xEvORb7q8ChfUhBBs8vuTyJl4QrascPYSpZZp7NwImTNgorB52ERIU4 + + B08KBzLLJerHJTc8qMzyuQKBgCZuC3yxkEFo3I6C0hD66FQ1HBRKPbSKCbhcqzJF + + Chenp6CCf7x2dlrqq/ky4a5ClwUjDr1RB4bwVwWh4SWC2nW7O2SDdYZjvB3f6BxZ + + hN+b13yQzJ5P9cHItUvGKl7/6qhIZh9Ckt0ZPlOT1z12VmFENYv5s55+hRGj2Jf0 + + THLxAoGAKJjuqags6RfYV10+jm1n5mVyQBJKXj9jkWdMh4oV7a/Kx8LS/oNON+9g + + jdL1CsvLH8qG4fKyysqzT5jEeSgnstpSCbkA5JderOyKzZmVcSaJCamy0GlER/tf + + Lm8Svjy3AMykDrX6Id/y4hkgPfPVAvGc/1fPxpn2TEsGLj9wzxQ= + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 131073 + seed: /N4rLtula/QIYB+3If6bXDONEO5CnqBPrlURto+/j7k= + zfec: + required: 101 + segmentSize: 131072 + total: 255 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:CHK:lf6qub3bkdhfozpdis4bmorj7q:zte3u3ohbuom3iqjjx4odrjylcpw5myw4n4kwf4zcaif36vngf6q:101:256:2097151 + format: + kind: chk + params: null + sample: + length: 2097151 + seed: uqWglk0zIPvAxqkiFARTyFE+okq4/QV3A0gEqWckgJY= + zfec: + required: 101 + segmentSize: 131072 + total: 256 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:SSK:fqqrn6iq2w2o4fvbhce43gkqda:jupxaorkcxvuotuo4dghhecoi7drygxsmqu57bgpuadq6eshmcsa + format: + kind: ssk + params: + format: sdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEogIBAAKCAQEAntK4Mo12RhDyHAp4wqsbNWIYTUl5n6g1D5dz3CoTHzCL0bN+ + + I4HY9ORmvKKpfugBEJqSbslhHhvwe6ueUijA/oWUcZAAzJ7JskBqiSjaZ4fcTm9+ + + ov1FN+6QI4EJyNQrsWGdXSZ7JDpgSkEgFdEhU40aUL/J5NDt+dSCag5UUXcsZvrj + + ImrCd3uYGEvT6SGo4pbT1F5hC16pWU+GENc4B5sN8bfV7gSL/+2WainijX3yznhb + + qdB1jq6cUy4NXelqBzav2+dRzAVc/QeAzy6ixEUdrs8Fdy8tsSvBFh1KVTgcvnjP + + t4h81NorBMU6nGE3pMpSH5dCj4zYO7w5FuBN6wIDAQABAoIBAAmlyTSg7TdSXjKf + + 3DgPBscWX12Kgg3VvOtmis4r7B9v5n5lhdsITzKJEUiSJPOlijMFALIkH3chwVjx + + 0tswyJBtctf3JGVJm/zs9svAJIw3b4WANlQWSHceUbkmZH7DjKfowxNefAp2VVUN + + YbKRkpGtt5SCIfR5UItlgkqI9kxAdV+ml/SW4HJhfx+2qFxfHP613tUaYYHev7qQ + + eH0rSmCtrwWUYC1aCM+zCau1p3WWO5aYOVccaHvDtEJsS8YV+LJXc4YIrCvgPHBC + + AQZLB/Z/FwG+JRJdajWhz8Wtr1+MtQ1Gi9vQb7tXeS/TE/hVXWY94nwbbXCUinzy + + k7OIOoECgYEAttcm1vSIggzZpS0jV4+PiEDrN5GBUIwPDyZNn8tca4sy1lvkG/dV + + il19L/DsmLz/eJKTo/Uaw4X8/oZflE1MAw3XniNmgeVS92i+EOgOtkaJnwjrGBhF + + Nfv3Bpvn+GrEzVHMuxgRfq1qohaSMGxHre3XEorssgTaDbHo432aICsCgYEA3l9o + + jd8e0zWVkd4/Lkf8JMInPCBRWo+gyBfbke0CcrexWgk60t4KgKeoUn2n1ORBcLpU + + fBK9kRk1T/t04fyh2nJSBh1STlrOaeuV0FeG7LblhKOhQSDDedvqn1+OcarpVh0G + + CQZVpb+Rth438LJL+HlTydaFS+1olApNuu/l6UECgYArANN7vyvUGp2eAc3MLFG/ + + 5DTubuSRQz/PelzLdpMYIDcmv5oZEcUms/JbsjiTe/BCNYdQCrfuwLbOTmBwivWT + + yk+qO/1CE+O9mP8LDulW6aQ4qWpR0nOEzOw+u7CFducuu0yBvJlwx+zKjrB3fyAk + + wknRbKda/1Uh33Q8/S+g3QKBgGUk/lhxYQLuf36emRxC78QEb3YguQA5DgeVGnDw + + XcmyFb//LLtW9W35VE1ZDCqAO+e7SMw6dfD2h1I+7LYRg8jpcLeJRLORCAwTdMwT + + 07H9qr2+84y9C0x1I+2juBWpiIJ7pxAZyoEedndgnU8kuftlrB/FLFIRxRx450wc + + 6/VBAoGAYpZTjnJhuxY/RSFJPaRqY38VMrRx/+vYQw1r7bI1NSJJtnPnRdncHbj8 + + eVO+4mL7J7EKP+L9VTIOs5DR9lOMZiPgiATK3KF+rNOfJp+f4Dm17wJrxBapI6aq + + 50Tt/b6IwVXIg0O8JAhUfFAzEf9H8XXBjVE1thSyWROSin4CimI= + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 2097151 + seed: uqWglk0zIPvAxqkiFARTyFE+okq4/QV3A0gEqWckgJY= + zfec: + required: 101 + segmentSize: 131072 + total: 255 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:MDMF:yopwhqkdoqdvko3rkwvvvrxjwi:5ymwi5mcfinhtle4m74nco6yw3qfjjzxfy22ix7vk55ybfzkha5a + format: + kind: ssk + params: + format: mdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEpAIBAAKCAQEA45joc7rbhjHPGrvTZjJm94Qjec7SGCQpWsc5NIUz/dzbuA5E + + NLOOQ1qpVym7Nxz4UvDbX5Hc7jWaaVQYClq9WVYxNla90VGBC5eyZQL1p7cORmBB + + +eSrbPvHpYUMC4WPR3e5CeeH+dKSmkS2QTdPh2Jmk4wS1CF1Anf4fBd1XSPIql86 + + X0D/NagRVaFVNrutjBJ9rMVr9utrTpLH76FMpfZHl+pDSoJMzHUrbjLHK7nwcH2K + + EZY+UHJ8PEG+zDy6rsRFPt2pG+vpGIP8TyR/X0uTqasPMxMLkUsAAIQvT7RlZjtz + + KF8IbnNxPJVyP8Kx5NSeH1sfpq2N7iHA2W38AwIDAQABAoIBAD7rmOfVsrbIsl7L + + qkfIi1rGNyCHouF1rdEg3pm8cYEvO7cIiqafNSc9uy8TpGQ6KBSV7a/gHVnli8iu + + rO6/4zT9dSF2nYdupuRTjcgLJ2q0Wsft+I9jPlkkyi7iN1BAHjo9yPQKBDd2lXz8 + + nf4tklj1RTJpORNYJIcIL7PusE2M/3je3eu8nKdyaPvV/ysNqQeOPU9EpzoO+737 + + gE1rMtsHk4kE7Bb8DnNBzPEmCDaajdtTuHpY1Y1pavrM4FLp7GSJk7Qa6o4YMKB1 + + 5/TF8pUbCh4V0BJQgbb+vAbLVjM8o/CPlipviaLymq04GxDwkSq73+3JnltMGM2W + + KIJsHUECgYEA+K3nkONzJeTTVcDiXtEeAVz2cePvwr6dB96GubCUU5JL2fvNiOgp + + lmHRyov/a8v2UmkDAsSAshd5RCDn3zkFNNzz6fLqxninYnx6/ZNgMk0q475POxjh + + o9EYwILnBs5t86tHYDElrwvPR6/zVFthi+mCtFweLICorB8pugLy1TMCgYEA6kwg + + E1y9XkY/PExVAS1XFHi08byK2hkpXygqmY0U0zwKwXmEOc5hHRBKPwQDWoRvB/JD + + Dvj3XkxCxbXnEZGLzSrjJUUEOiRm0oEnMr/AsWeBCKMoT0+CiXiUxfuaNIkI2dH2 + + jPXX8E+QGoSYuw4RJSTRaMOj3iOVFljGfKuhnfECgYEAvqle3Mh2dXw+yAWtybKd + + RcBHt0RihDZu4SSsuNv4rSaCf2u+xxPxJrpzBc9Wkwh7H+4hf9K3NVQoBqMQBCaM + + pl4tqJY1iNviwfDcv2RqIcbmdlxoFNBb16SuTJNQm/hTdrpAbDDiSpZMYxM1Bd1W + + KdZr/uqNu+Mc73KpJFO0aN8CgYAikv88vDe5nLYiKMV2egFapQFWltMKoiHnx96Z + + cCc9kKOpr0vi1+Ce0FOUfvwbtGVKD+bzY6vlP22vDUu+3PJ7YTPJwSiBh/OgZqyp + + IYDG7RYudx0wrvP9Y0zY9mroC7zBn+k5HeIytRr3vs9m8wl2qLs6MXySAEA03v7T + + UOR1kQKBgQCw2euWQNd/s7v/GlldPB1RJcd+so2cjgoLRMh3ar9Uh2Y3ySKwLSRc + + NmH5g3dl1K+VYJU1maB4ECWB/Gmh2bnfFqfUHoPUEfssB/nL5sEWeE8l8H/osT/7 + + MfTZYIW4CBmWU8dTQR3kcPm3+leOfiZf6HbavnH0j+xbMskw+/otjA== + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 2097151 + seed: uqWglk0zIPvAxqkiFARTyFE+okq4/QV3A0gEqWckgJY= + zfec: + required: 101 + segmentSize: 131072 + total: 255 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:CHK:a2qoww5xx6so5uftwg4msdtrpy:h7hwsjco5i3jkd7icytrtsuzcjvjzlnkr2wwau2tjlj5wehjrokq:101:256:2097153 + format: + kind: chk + params: null + sample: + length: 2097153 + seed: BTBX/ampNfLU+ox7xipBGiaSbgC0kcB8Gy7BkJB4oKI= + zfec: + required: 101 + segmentSize: 131072 + total: 256 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:SSK:ybpdoqejf3ioc2vfgaq4xo7qci:3lwqfnsmzeb34lpdq4fkj5yyahmxt3b2yvhsxsfqavtrffllzjca + format: + kind: ssk + params: + format: sdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEowIBAAKCAQEAlthpYlADDHI3YZuTCHjeaIW6ZfYUeQyi3XPo5bSiASDWLFb0 + + F0bSgprF1Jxp0etSgj8eOozOWKak4Mx5Cl5lJKon+IiT6oU4JeZQ4ulqTUxvxikv + + ta5qD+jCTfua66Ntua08E2FEzqf6zEKwDDsLKps8KNdJA03BNrxDzKg+lMgSA1kg + + tAWRfZPaiKUP8T4yLI6LtfzJwm4p5KPkBSKJ2y+m28sbsPtaUXN2B1zhHYK/y0lp + + qZo5UrYr7nOVrAxoSMOUsPTA3+BPBQ9MfujmV4GOHDDAUsSYo6DZnnK+hC3SiUjq + + vQTengWFVhtScOVY/onivy1aa6pf2e5lOloq4QIDAQABAoIBABc1BEGT0w7pzjYT + + cUjZukiVClsAGPY783KOj+4xyXAoVSBNNbsUXPQAS8oB+7sdW9Gd/vs9s8v/wgNe + + ceDJKlZnDJniqUm+D9KvibzfRvN4JwTITAbE5pxmpZN0eh52mOhQC/C6A5P8/qDI + + 7Gd1Tf/SvNarsxfxsSAdUXD9vA23iFLnVBYRzLMajDApRbdUvn+VTLNee0emNaaj + + dpFfe45Q4ChjOeyZIurGUKm/Qf8yGtfym+asoQ5wQE+E0Q3c+a/T7k9MsHpPEkFk + + bt8H2TAEpptbyL7HMWAQNsBKiKiOq+uQTrxd+2oVrclOkKfXQBLthWunQKQGEFI9 + + DQabWuECgYEAwEDC3QMhPnrtJqYcm+U/owQnmozf2sHopxctYtEQRu8E5RzvP04j + + fbOA3g8jx7hDc/dMMcoFFCl4XmxN2Vf8dJFeY8Jn4IgUB0G7MrA0US69+giiR38u + + fo/Z6xnYmeRwWjbA0G9zRxh2Q5DNErgfognWtPd5JB8oBUOE6/MS0ccCgYEAyNzM + + Y075Pjw4oQ+wuaJgc2xrO/W12ady9cgEn/R57lhADjjqj/MMjNObo2kXsTvXhHJa + + SQnRepjbY3wxxd/tXXz0+vSABIYe+FS6iOTyH2uT8phWPRyTdRgjPxORC9UcNrmt + + ayltx8uKc/ejCTDAXjRWeqpV/YuCclhS4NhMHhcCgYEAtLxtNO9bUmyoA/yHyrtD + + DxK51J55WCORf3vXjB10yuqrVGTWOlJQJT0aeigLgBenOh8Tf38nNSQjZ8kzio8f + + 48pBzVEW7Mug4I2X2fgyxttFeAij3skewZakzFN5AHv0b6snqwwLeJvzmmNHl0CH + + ZIMRWQGJ3j54FjK6hEL4v0MCgYB7V7rasMA1C13q6WuoUqHPvyAKbdQBl+XsL6tH + + XiURy3dqRGEljCaEw9yq9/noh8rMNjxi5XiRRBRUfwtBa0SjR/xXrpR+Mq4J6F2z + + eoOD79el/Q3X8RhLq0rZjPZgwbjVkid/yqzzJ3YZyFOBbUJzlsgUA01SqLyt4rbi + + A3CaeQKBgAYBnzLghXOOzISsAIxU9ei8mT7YBC4Hj2p+9HlA35HdATtY+Ulz2WK3 + + cghLqGrfdZccG8iYSnno19sMwGmT6UyV5Gv+nG5prJvBjiOAeUwseHFIDVxzHD9s + + PMMImnOXcb/EFYndL7VhlJxxYCjSHj0ZivOvFov18OmMu9wKECuD + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 2097153 + seed: BTBX/ampNfLU+ox7xipBGiaSbgC0kcB8Gy7BkJB4oKI= + zfec: + required: 101 + segmentSize: 131072 + total: 255 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:MDMF:heohri2cx7hxspitv4rolwkmie:5sjg4nzwgaaixlznaespichb5spo6plws26d43ooagu7vjss7quq + format: + kind: ssk + params: + format: mdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEowIBAAKCAQEAmc2Qc56mQzopYjf7p5IuN3YoTLceZr9jwKRjVo2q3EfpoLXy + + +vxV/QO1Yxf8u+HajZXbueR8aWQLpsji8tomuCU84cTS1HC2YXqAFeoAyrYXamc9 + + h4MLx+lPUCBsTkHYg5OfdyD2QPWe8hqOFjq0GD7bIKn3hJBLndnOh3tlI7QyM09e + + eU/Mbo8dsJ3uon/a3ubLxeHG/GPj1uMX5V700XZIZEcAjJgqXwP+Hd+8geDXvn78 + + DYXXX7KbhOvuUHfp+iCyIsmiOz0qmS39PWs/redpJneEdSE7rabfrdxXXLCguudx + + WjPe2CVOAOJokNtwu3ccYuaF+Fq5N/yIj3gZnQIDAQABAoIBAAoFC9OUctVvXRHS + + ftkIW21ui6qPxXHBJzD+JKCXYxmtr6kyIU27kaiFjNQTVHoy+Qd/S0y9d2NwSpgH + + f01968bUWjaFGY0QeLk5/00uLPYHzde3ORlybpqL9whLzHJ+tKnBvMJIifJqbfvs + + wfPtyBzKP4FNwVvIAL0cWumVnt2n/j7OF3ER1XZuKZjq8EPj7ekcGA5CP7eBBkqA + + sFXjUETxWwUADj/jEuiBGW7zKlcDdVJ1y4xyVWRao/p7gEY3vAnVh6TsG+mCd2di + + N4r1uLkhLV79a9Y4w9KRGybMQEh8pS9JGfV8PeRxQywLSL+Ja1E1U/+1TjVW+79G + + MOb6FusCgYEAyxXF6FAGpZhLPLPVZ3w6JrtKZ3uO1oRUbuhQPBlOY2EjzUbflP93 + + KwAImwahjlW/DeLg2iX1zTKHVDTYBKyIW4Mod6a6xQQy4I1kTinkLyczNzkhnUU8 + + iTcUadPXH61kUcuReMfaoz9nDSixDu0z8/7M1aa8U6qGR4Ae+bs4oV8CgYEAweCR + + NiyRKXWaEej+ioR88F3kVYTbWadqGU+mgVez+uxfUI1BVJcIbRftv/dWkRiRLKWt + + LpJWGRC3S6655aPQuLuHrzrdvJbsfoUrg2oneh//gUY9Lg0rmO4jEjvKklSNeYpv + + h4L07T9kqnnZGemOytuBhVx66GyjTTeN65PSOoMCgYA63hvZBGF43NVqSiKg9bSR + + h5bAumMkMYWcBIFFenxreDv9g/7JXOf5MfBMp7Zq4NYZu1s8QOaoTW5G7W50pGJ+ + + TF2NmWnoNBhfWPzrX19Cf9Vru4bP5MLwb2PebUadaxB6WUzYuu3Yhkdj3Bi+3+lA + + X+qWP9e1VOfJkAzqjOeUdwKBgEkt24HIRq6Qfiwedt2P7pzHw+TntefcQjb1kpKl + + qQCgccW026DzNTIAYzQfRuSTklB45Kp8f9UMMzN06yQbti/UUP26SXHiwbdryqXa + + zrXRGB8ShQs522fpEwHR4b9j/NaQg1JyAsL+N6AFSAX423YEbpoI8zeBsg32VzJB + + ZIvDAoGBAIfor/nuKBi66QYvXeyPbRWicxCTAV9Bsah2G2xTCyO5c0Ac3Fj4xkXE + + mSrzrHGZZaGRlGlFqZYDy7qfaXh8pkY1gwpc1uMhqYvNTptPaCpC23WX4YgfY1fr + + KuRpZE59kXCogaZYzJIvN7YtSVPOmwsW/h+FBmWkL7eB3+7XE19z + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 2097153 + seed: BTBX/ampNfLU+ox7xipBGiaSbgC0kcB8Gy7BkJB4oKI= + zfec: + required: 101 + segmentSize: 131072 + total: 255 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:CHK:tg5w4m6hi5ezeapuix3lrrbbtq:7kerk5plqpsd2upnole3ifxoegxdux7prazbvzeu2kyilc6idk7a:101:256:8388607 + format: + kind: chk + params: null + sample: + length: 8388607 + seed: w6uP8Tcg6K2QR905Rms8iXTlksL6OD1KOWBxTK7wxPI= + zfec: + required: 101 + segmentSize: 131072 + total: 256 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:SSK:xyg3ycxyqwlggew4yuakicf7pq:diw2ygma3sejm6gfrilmr74seleyrsvujixkwgr5qrq4stdfsqra + format: + kind: ssk + params: + format: sdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEpQIBAAKCAQEAqwHXEgjR1fSRtrW9og7WwbSP4FyK+Ce9qM15D7eNVlVw3iuJ + + XOR2CoXrv3N1BQDz6dHggdsb2KAQxtLp6uaZJtBneVKDc1wVOINfmDDdU6n//LyL + + cTEO6LejOipcOUypo1w1wMS8UVVhVjuRSHQwN2HR0j649uPyfS72HP7+QkzlrDe4 + + qYB1T8q7w0X+EvdvqnrzStHfU009uKsWXLKNh20pUKxKirYY7epi5Yw2dcNuX+XI + + 3o/fLO6pJVQRwu6lQHIywJdua/3PlyRv+DJq+y7yNk1W+cId4OMCBzm004ugjDGw + + VOPe4BkcYpa4hOMNU/+0haKf9xvL8XD8l8khEQIDAQABAoIBAAamHQq9V0xIdLBs + + D04ZYdn492Wfr4MPx4DkUU5OQpNuYcOvoWkAVIMa+yQv3OdHhtRK5d3iqyngalqS + + qYfptJ9016sYzVXbSRNwDzPThY161QJKgejyIXxkpHbu3fRX4dohTBE9TP2kElNo + + IEFXDCdhGSeBqw8lZICfxm+w7emds8Q5v339j9hiLDIyNSIjTZEidLADPOdZwQ1Q + + cGk2BItVLsf5tamKlLpik2gfrVc5DtSkbAncY4eI6nvWfZd8Bn/K0LeqxUQ4XJk4 + + KSpNfb3kUwsLxBvb3kjAQUif1tSPlEBOnqN3YbSd2VKTCTHe2SLljOHfPPJWoxtZ + + A6z8FAECgYEA1D6+956RMQ3YLPCwaZJ/+IHFeg6JfN1krFxLew1nGOScUP4nELkV + + b8Wm8rUjOZX0BGyC26IAf03WGUwmN2HbNxxD6HtHnC24GS2a9tdN7NTh1GloStUL + + RIBdP3FwhtvwKCM3+CNzAA10D6KJVeMf2dLr6PgXSbxxS+T6IKdp6gECgYEAzkLC + + djJZfQBULYJiaOcMguhB9JxmB85kAAek0VU0TDNtKKEJGUr6icpI2/TgJOHhRC41 + + gq0tXkv7QvqZZX1Owc9n+B3OA2WuBu15G9l1QoWv5MLhPUI9sqXgt06uZbktQLz6 + + YMQ4QMo5qGE6ALl+R+MDUNdpkscCNmiaxfy6lxECgYEA0SYnvxEVmFZBMT/ZR59i + + 1brjo3yRxWbRXvvwMYkqkCAvXaylSFhqpGMMOd1/oa5/8KARb2c7wDcuhG1Ct46J + + m8wRqxVYorF22fDT5OyT0I6TH2Ljr+IyoUUxHmSl827mQFc8PxyHpYScWw/a77TJ + + 3Td423EmWbYFmzk/tk/jEgECgYEAoV2JVX8+k5TWRmRjKT7ZgvDB6OUSzbiic4OH + + Zl4KdDMni0mxHKCUMYiYR7zkPvaYjga4xmtFuygmgtgbelL2cpoY9PwcWHwMEk9n + + GGqgWlLMsWPlY0+XhVRQ4hgkSGD/Dk7KczoP6GBNi3XFMxvrt8HarjxY1APtrzNX + + It39/IECgYEAkrucWapwRmbr3v5IHcisK4Dp9ygNkEKph3ZqG7TLaqHZMiQ8l4Yz + + SCNFbNGEOzixLefEev8XAUVCaSvWe02PKBUru0a1jxiXtvxUFZT1QW9ndWDS9FWn + + Z6go15bSALqAfdS+/uKpvVTejkFh1Nsx1nOX/iJjEg4pog4IouvZKH8= + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 8388607 + seed: w6uP8Tcg6K2QR905Rms8iXTlksL6OD1KOWBxTK7wxPI= + zfec: + required: 101 + segmentSize: 131072 + total: 255 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:MDMF:wkhxfy2hw3xjgktv2wk5q7caya:xn722v2lrbpll6lumqphdircizjuafl67euotd7tjaixxn5lj7oq + format: + kind: ssk + params: + format: mdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEowIBAAKCAQEAoJgqinR7T31kTqPuSts1f1Ku7JuzlceOe+fIIzPMGRjn1k8v + + y5gfPC3l+2H+C7AlmFAHOIjflnqyV1hP0jIssrrchpMqCkn9GAcCHeTNJcGgaxb/ + + H0Nq9bT3PmWewXUNabcQBtBWX3u8BZ7BtOiQKPnE8V7nBQdpWOeH+KBStHJ0MwEU + + ModphtbjN8FfwgmXzxEUEoEqelgwQfcwfx5u51Fr2HQdaLuuuM3XZ/QUOYsBONbn + + GgI9N9IHJGIy35R2Ws/9+0qSc93XAbY1/C1TYnCAzNVxFpGjb0EQoG9wTWQMz95u + + oNPG25pblbyGIlsxhcPY+m61nTlbkqrsdc0L2wIDAQABAoIBABybjrR0VIUP/r7d + + h/TwwMJqHbwLbn3HezPKUcYnk3uDCsWL/KUld6b2PCpARguZ+NB9rROemknJmJxj + + oHB+vKSoEeGtNId5r6rIkNF3cS4BJIz/HzpX/aVAc+y7GIE400daM3IrSb+foJpV + + sgcCiK+r8q/WqoukStlqAThCgwketAMVwe+0UMfYCbpxTU4zi9hr0hqauoJwSe3n + + ZMS19x1cw1s7sSt+naA9i86ICZ3kPaqJJW9kOfxFd9tP3rigs5K0ye4mgbiFUGns + + +pKHbPxtWna++F+QUR9pH9UyD8W0dlu+klVyzUJTIXTrWRe4wDmwfaU6wAKZWidm + + 56JfijkCgYEA10AMC1JO84Ebtfh4vXLna9JYHT/B9qjyiz/lLDwTyKv6JEp1E/ng + + 8ASMUjO3411yOAlM1bl3kJnuCRYsHACWJPtsmRRUgQdR9TCyuYWAgLLPhV3BDy2S + + vej414UwhoA4LVrNeJmnVke5LJy4xNXBamBlFkcaEwlVY+4SJp+nkZkCgYEAvv9D + + jxDfoWdZJTue/hbY4xDe7qJ8mqsPZKUQ6p1Y9WKbHs/xrHyNpJoR6c8B3qcZghxN + + L1+83tfbz3aKViUONhtkY44Q4Madjr9XhprsHkO8J0yIha9ls9oyF516sKlvMNxn + + dHF6bDBy7DNQnOhvuI3aXLJsQeOO9Rf+BnEQmZMCgYEAkCcSadqbiTQjzMA0jBuR + + pIHgBOaYDYqjtGH8Jp2tWizifr9mnRQxckx5dOux8RC515FS5acpzato4Kj6rV4v + + L2E6H2KgHTE70ArnBpvDrW0S0WwySOnqZkjJrfxCvTDNboJrLKMqj/vEpX3nt9q3 + + h3g6+qpvaeRMTXo4qakuXbECgYA76nn4FHQC/xfBDV4IGYS6Xp2AwOpT3tu6V+nh + + n7C7cc51sQgAcyY//7Ek5rKQdV0UKuqvtNncEl07TNWCxqcZpCgu7u8uhEAC+tVr + + PYhayibpMSIWxfoinI1gSR+m8dAWxN2TctHTxLMYk9RzFJuPirh4oeRCGy/KhVdE + + EA4EDwKBgEUx6Qm4bfLbIvKHmL05IWXuSdwsAVG0bNPEZJf9ht+ykNcMOlvF/UWx + + Z7fk0jwFutfud7KkjBRvuSuDTrexFNsaIW5RqWU9iZf8Ep0eoRNAXcZk6sHCq+ds + + nDpIWHZq2Ir3SfIPd0GxWzasl75XnMN4rt1M37G+lC0AB6PVw9vD + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 8388607 + seed: w6uP8Tcg6K2QR905Rms8iXTlksL6OD1KOWBxTK7wxPI= + zfec: + required: 101 + segmentSize: 131072 + total: 255 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:CHK:ioridipksj6o7ph3bcf37vslmi:pzhkwm4ry67ohj6dcs7zwnsjrvfqzam6tuitrns5vuo3uoomau6q:101:256:8388609 + format: + kind: chk + params: null + sample: + length: 8388609 + seed: yPi3JHKKbWaEEG5eZOlM6BHJll0Z3UTdBzz4bPQ7wjg= + zfec: + required: 101 + segmentSize: 131072 + total: 256 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:SSK:66qkzmv6xs4vajgz322wfjf24y:7xee2z66ue32dln2z2zmffcza66rrxqdiptul7gm2uaf4p6x37ra + format: + kind: ssk + params: + format: sdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEowIBAAKCAQEA1Mh1wFJg8dsZJAxPH+vCXEjFTKZfZjLz3+hN20jAdmr/xtgx + + kSup1k8EvFXZ9gc7Vfu7JVb719r6b6Jeeibc1x9rKY+wf2ok4J+aMKANE/vUPU45 + + CN6BLdLQaeshbruToHcZqykwu9zhgjXmGWzQRiuKx8/3IOnx8IOefRoC5Hsge/yw + + ky+azSi//fzyThu+3wFoRXbOvCCcw+Rd+j7g37xUbbaXfIUs1UiRuovGLdoYsv9A + + A0MTzuJWnMjqr+4W7rYJ9xU4vkibkRxntDO0qlvwxIzLamUINnPSyCKWdytNuL1D + + GD4PPoPWL/PJx3FVI5YAbyeevkPiuYx5m6dpjwIDAQABAoIBACYeXT3dZCGfswLK + + s7gPt6tpm/LN0URRN3AywRPaFiSAqZ6ZJ1QO4ueSE1Kb/KZ/CCmwpYecbBRw1bF/ + + AHbYlHJzXfK9m4xP2xhkby5r1bvxPsXWyA/nMHQhkpWO+lfIgbta6r3HbMQS31FA + + z55ZaHxRm5SNFIQQdPe11IQrzz3X3yo23qLoGoNFTt5puu3d4ihP/JGe5zJPyVvp + + MVMb8RVXwcfBGnkzPff2mBN41C+/oQ7iESzCSZeNRPDft/74Ma3xKAlMN1/rBR76 + + kJEwDCqBpPmF/zDJ+ZXF6Up40CTBXCYa6Sf19Sas5IlFwEoIOF2BxjpW1swUW/wc + + zvBT1wECgYEA9t4szXjoU0z0BYOSngwB64K3c5Q3L8nPr1tjZv5allpEizhKz+qn + + NOrkb8hVBFrqmpTho5i+GI0oVn+ol7k0xaUjmC4h4crNx0ZtyYruWwxL+rt3OWdv + + zudC4KfkdhLfd0+XO76HXpJFpfV0j88JLP3ULwBGF37MUauBfpwbUo8CgYEA3KeA + + 6cTXfM43fp53sCRMN1TfQZ4ave/+iLK59SKGMCGa6TI33j1/1Uj1HL6HDNHyieRa + + ExqP1Sf/9iVVEqg45QBgb3xNCoWpcnxtBFKIuvQCpk1mRsG5nLOPHp8OuGGPRWuI + + 7gx3qSWx71VoqDWlRzFSUFW/2QSXK964TmNR+QECgYEAjv8/IH4qxSXMK+183kPC + + UPNU5IQ0O2BBByh+ucgYHQOItMQUwb8Av+xYClAWvwES6BvZX/Q4GOybMw+bTtef + + M+Vmat4+DhZ1gDrRmW76ho7m7APvGbdK0qSu3ociFSr1ep0F0zuYGjXMVkeKD0sz + + 23XklJ0p/K4cGCqqRfaS9Q0CgYAL887Z2t3JVupOo4rcMbsnLCPDzCqqqztgcD3+ + + d1ZJeSiJBT1dfntUNFWCrxdlrGG08nemnUO5SidlT/RhxFcAoJqYr2UE8uSQ3QiS + + uV3KsrkKBRtLLec+A8P25qrHdhFqsz6Blo9MzEvtKPU4V1+SkathyqNPwB3oNHJL + + XLnuAQKBgGsjwA8kCUwoygz/4kXk/vjtt42a+4g6pjRjNeSOZ+mnS5ijXdZj0faI + + 90C1LN6JIQ6Bg9n77dSCnCn2LuDf/sh3wbvxgyStq89UbMVrTs/e70Xkacvt7yn6 + + HIB77Wk165vZHdPGfej0NADL20eYMbjGAFqfSjxmqGUhVBvlYdV2 + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 8388609 + seed: yPi3JHKKbWaEEG5eZOlM6BHJll0Z3UTdBzz4bPQ7wjg= + zfec: + required: 101 + segmentSize: 131072 + total: 255 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:MDMF:r34bsxt6bqcmhegwkufjcnkhz4:d6oj6l4dw65yazscqi2dgtlvetptoryzsj6ix2kd76uby3a4d4va + format: + kind: ssk + params: + format: mdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEogIBAAKCAQEAsid0CP/N0kwJXdAvA/ipizC0or3iReAfMWLBLJco9ehKX47b + + Phfc9nlrcCfBFeeRUi2G53uaw8xZeLuBKt7VkBeaKHAmGQ9DWKKiKWDPXdm3E1e1 + + dcnmq+xLcgSmVV1rvetbER+MBMorzCeTk054UBPpFIepcyZCCBdVJ92MQjSG4YeR + + 162YckojPEtM2T8w3hIrUzHBC/mEmBDczzIXw1libCi6vchtA+g/DOzqgKjg57a2 + + dBSPMlD98Rew0NcGWe2ZZCgD6vvtuHkgTn1tD3m6wPRb2KLwsMqC+by/aqluyjN7 + + ppfT0fTxkjpgA7b2mTppjKATU36R3fT9lUap/wIDAQABAoIBABM7wIKQNgT0LIwH + + mRZ8AWHY5PIvlH7SuXYNdql9EEZWyQtpUttwnBcH+MS8+1LaveJPWUx4bbb3GC8O + + nzKk/WLMXtpAG/zXRwAKCX50yIPAqMvadj68iL4a3pz1SjfWbswBiBgLKISuhu1I + + KzMwujsloHjqRIL9Uwz95+BL7Nk2FbGEFYxRlmliiLfA6U2KEkHZbo4DpmVGyizU + + EGx3cTm/M4p7gC9sSa7l/LWhjGMfw09jMwx7Fv9NX+esqWmrQzHCjsEtYjuf4U14 + + HoyfrneIGDhO3IVHOIMG5En9BCT/YLW9wnioGWlGbBKjnJcQYMNPTZrlQhoeVJW3 + + Wmesy9ECgYEA4Exzq30VU9MMs6hwIXfHfekPj7evCeflZLyjlRJP0lHR+iMgc0rg + + FZ3HB/k5Dt4xySKIfJ7Htt28E29ep1/AnzWPlKCCZABM3USi/YEvocWgY61X3Slg + + bjYdRqviMOXk9QSEONRLgYn+QPSJeYQL0GqPw6eQ7bIVR1EPQqiwl70CgYEAy1Vn + + x4952UVai+8SibopC/GSc/jUKIzX0Rcw4h6UigzCMVkn7SESILic08JOye3pUN9C + + 7i0Sooy5B8CJ1xzUz08xD9h1bWsaFNFgOuV9P+U+GqNV4sJTNZJDzT3cvzgaW7mI + + 2iTSHsGjRxhJhR7VohS1qZot6fPkBPVks1xxFmsCgYAcHZk2OtSskDz8XDXKDDCm + + eMtpkXXQgRABI6BBtGzrCTSP7U1JBm62ZvOm7TeYxINrGfgP3vtb0cmcig5MXrVP + + f7BCyifuDxeTeOIRctscpSAovnbQEzqyNfhPfoY46OhdSjakxP+9+iUz0TNWVxYA + + BwuEVAHXucXvDZsjGPAh5QKBgFcwSg3yYedeq9LxMtvH7a3naks8WY0Bx9EqxpVP + + U5ZWnjaW6l3uHl3Vi7npyesgjzlUYtjKjwEQoo7GatTI0iAK7xjCUqgWktp2ZXMb + + 0LdDT3wQqdVQSmngTB6H9k4wemz2g842l7sEgUUNDwl8DVMw2izdpe553D6cExAu + + BXf9AoGAK3aklO/J503M7YwUz5dTLReQgX+JCZWtaiZ0UG0/87fsEM0vZ31+llP6 + + bZD47qNvhl69jx8jRc0N6Ol4q6vkfXN2qC+Ocbdhv1XjVK5O7451i/qPavbL7r0J + + 1KlxBElQFKsG4qyWQHFh3BS2BER/pFkqHvyoZ+AinoUKakljXEg= + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 8388609 + seed: yPi3JHKKbWaEEG5eZOlM6BHJll0Z3UTdBzz4bPQ7wjg= + zfec: + required: 101 + segmentSize: 131072 + total: 255 +version: 2023-01-16.2 From c28f10057b6d1bfebc8fabd595b49484c91024ab Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 16 Jan 2023 16:01:11 -0500 Subject: [PATCH 1374/2309] Move some more pieces into the subdirectory --- integration/test_vectors.py | 4 ++-- integration/vectors/__init__.py | 12 ++++++++++++ integration/{ => vectors}/vectors.py | 2 +- 3 files changed, 15 insertions(+), 3 deletions(-) create mode 100644 integration/vectors/__init__.py rename integration/{ => vectors}/vectors.py (99%) diff --git a/integration/test_vectors.py b/integration/test_vectors.py index 4c21bb093..fea723b80 100644 --- a/integration/test_vectors.py +++ b/integration/test_vectors.py @@ -14,7 +14,7 @@ from attrs import evolve from pytest import mark from pytest_twisted import ensureDeferred -from . import vectors +from .vectors import vectors from .util import CHK, SSK, reconfigure, upload, TahoeProcess def digest(bs: bytes) -> bytes: @@ -114,7 +114,7 @@ async def test_capability(reactor, request, alice, case_and_expected): @ensureDeferred -async def test_generate(reactor, request, alice): +async def skiptest_generate(reactor, request, alice): """ This is a helper for generating the test vectors. diff --git a/integration/vectors/__init__.py b/integration/vectors/__init__.py new file mode 100644 index 000000000..a7fe714bc --- /dev/null +++ b/integration/vectors/__init__.py @@ -0,0 +1,12 @@ +from .vectors import ( + DATA_PATH, + CURRENT_VERSION, + MAX_SHARES, + + Case, + Sample, + SeedParam, + encode_bytes, + + capabilities, +) diff --git a/integration/vectors.py b/integration/vectors/vectors.py similarity index 99% rename from integration/vectors.py rename to integration/vectors/vectors.py index bca8b0bf3..cccc25b8a 100644 --- a/integration/vectors.py +++ b/integration/vectors/vectors.py @@ -20,7 +20,7 @@ from .util import CHK, SSK DATA_PATH: FilePath = FilePath(__file__).sibling("vectors").child("test_vectors.yaml") # The version of the persisted test vector data this code can interpret. -CURRENT_VERSION: str = "2023-01-16.2" +CURRENT_VERSION: str = "2023-01-16" @frozen class Sample: From 8cc4e5905dcf7503218325477ab79886611cd5ed Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 17 Jan 2023 08:40:59 -0500 Subject: [PATCH 1375/2309] news fragment --- newsfragments/3961.other | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/3961.other diff --git a/newsfragments/3961.other b/newsfragments/3961.other new file mode 100644 index 000000000..1b8085b30 --- /dev/null +++ b/newsfragments/3961.other @@ -0,0 +1 @@ +The integration test suite now includes a set of capability test vectors (``integration/vectors/test_vectors.yaml``) which can be used to verify compatibility between Tahoe-LAFS and other implementations. From d14ba09dbb50388c1b8ee2b03bc118b3b5c3e8ff Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 17 Jan 2023 08:41:10 -0500 Subject: [PATCH 1376/2309] Some flake fixes --- integration/test_vectors.py | 6 ++++-- integration/vectors/__init__.py | 13 +++++++++++++ integration/vectors/vectors.py | 2 +- 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/integration/test_vectors.py b/integration/test_vectors.py index fea723b80..cd5e808c6 100644 --- a/integration/test_vectors.py +++ b/integration/test_vectors.py @@ -14,7 +14,9 @@ from attrs import evolve from pytest import mark from pytest_twisted import ensureDeferred -from .vectors import vectors +from twisted.python.filepath import FilePath + +from . import vectors from .util import CHK, SSK, reconfigure, upload, TahoeProcess def digest(bs: bytes) -> bytes: @@ -138,7 +140,7 @@ async def skiptest_generate(reactor, request, alice): results.append(result) write_results(vectors.DATA_PATH, results) -def write_results(path: FilePath, results: list[tuple[Case, str]]) -> None: +def write_results(path: FilePath, results: list[tuple[vectors.Case, str]]) -> None: """ Save the given results. """ diff --git a/integration/vectors/__init__.py b/integration/vectors/__init__.py index a7fe714bc..03f716cb4 100644 --- a/integration/vectors/__init__.py +++ b/integration/vectors/__init__.py @@ -1,3 +1,16 @@ +__all__ = [ + "DATA_PATH", + "CURRENT_VERSION", + "MAX_SHARES", + + "Case", + "Sample", + "SeedParam", + "encode_bytes", + + "capabilities", +] + from .vectors import ( DATA_PATH, CURRENT_VERSION, diff --git a/integration/vectors/vectors.py b/integration/vectors/vectors.py index cccc25b8a..06e0de0f0 100644 --- a/integration/vectors/vectors.py +++ b/integration/vectors/vectors.py @@ -15,7 +15,7 @@ from base64 import b64encode, b64decode from twisted.python.filepath import FilePath -from .util import CHK, SSK +from ..util import CHK, SSK DATA_PATH: FilePath = FilePath(__file__).sibling("vectors").child("test_vectors.yaml") From 3ab7fc3853f76a13cfd76aa3f3c55ea8721d36d8 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 17 Jan 2023 08:45:38 -0500 Subject: [PATCH 1377/2309] Be able to load the data --- integration/vectors/vectors.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/integration/vectors/vectors.py b/integration/vectors/vectors.py index 06e0de0f0..9a92b0d25 100644 --- a/integration/vectors/vectors.py +++ b/integration/vectors/vectors.py @@ -17,10 +17,10 @@ from twisted.python.filepath import FilePath from ..util import CHK, SSK -DATA_PATH: FilePath = FilePath(__file__).sibling("vectors").child("test_vectors.yaml") +DATA_PATH: FilePath = FilePath(__file__).sibling("test_vectors.yaml") # The version of the persisted test vector data this code can interpret. -CURRENT_VERSION: str = "2023-01-16" +CURRENT_VERSION: str = "2023-01-16.2" @frozen class Sample: @@ -135,7 +135,7 @@ def load_capabilities(f: TextIO) -> dict[Case, str]: if data["version"] != CURRENT_VERSION: print( f"Current version is {CURRENT_VERSION}; " - "cannot load version {data['version']} data." + f"cannot load version {data['version']} data." ) return {} From 5424aa973736507098d18ecbdba7698432ce9bad Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 17 Jan 2023 09:27:17 -0500 Subject: [PATCH 1378/2309] Only run the very slow new integration test in one CI job --- .circleci/config.yml | 15 +++++++++++++++ integration/conftest.py | 16 ++++++++++++++++ integration/test_vectors.py | 1 + 3 files changed, 32 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 43c309133..78c60daa9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -94,6 +94,8 @@ workflows: {} - "integration": + # Run even the slow integration tests here. + tox-args: "--runslow" requires: # If the unit test suite doesn't pass, don't bother running the # integration tests. @@ -294,6 +296,14 @@ jobs: integration: <<: *DEBIAN + + parameters: + tox-args: + description: >- + Additional arguments to pass to the tox command. + type: string" + default: "" + docker: - <<: *DOCKERHUB_AUTH image: "tahoelafsci/debian:11-py3.9" @@ -306,6 +316,11 @@ jobs: # Disable artifact collection because py.test can't produce any. ARTIFACTS_OUTPUT_PATH: "" + # Pass on anything we got in our parameters. Passing an arguments here + # disables the default "integration" argument defined in the tox env, so + # also stick "integration" in here. + TAHOE_LAFS_TOX_ARGS: "<< parameters.tox-args >> integration" + steps: - "checkout" # DRY, YAML-style. See the debian-9 steps. diff --git a/integration/conftest.py b/integration/conftest.py index 47c8155b3..dc0107eea 100644 --- a/integration/conftest.py +++ b/integration/conftest.py @@ -62,6 +62,22 @@ def pytest_addoption(parser): help=("If set, force Foolscap only for the storage protocol. " + "Otherwise HTTP will be used.") ) + parser.addoption( + "--runslow", action="store_true", default=False, + dest="runslow", + help="If set, run tests marked as slow.", + ) + +def pytest_collection_modifyitems(session, config, items): + if not config.option.runslow: + # The --runslow option was not given; keep only collected items not + # marked as slow. + items[:] = [ + item + for item + in items + if item.get_closest_marker("slow") is None + ] @pytest.fixture(autouse=True, scope='session') diff --git a/integration/test_vectors.py b/integration/test_vectors.py index cd5e808c6..27888e095 100644 --- a/integration/test_vectors.py +++ b/integration/test_vectors.py @@ -95,6 +95,7 @@ def test_convergence(convergence): assert len(convergence) == 16, "Convergence secret must by 16 bytes" +@mark.slow @mark.parametrize('case_and_expected', vectors.capabilities.items()) @ensureDeferred async def test_capability(reactor, request, alice, case_and_expected): From fe552bf146b9f6395b4708d2e7f49bf0fdb0f098 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 17 Jan 2023 09:44:27 -0500 Subject: [PATCH 1379/2309] Fix CircleCI config typo --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 78c60daa9..ebed577ff 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -301,7 +301,7 @@ jobs: tox-args: description: >- Additional arguments to pass to the tox command. - type: string" + type: "string" default: "" docker: From f2989c0a4fd77ffbf19e1c35d075262d36d647b9 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 17 Jan 2023 09:46:22 -0500 Subject: [PATCH 1380/2309] Correct the ProcessExitedAlready exception handling It's always okay to get ProcessExitedAlready from signalProcess. It just means we haven't handled the SIGCHLD yet. --- integration/util.py | 46 +++++++++++++++++++++++++++++++-------------- 1 file changed, 32 insertions(+), 14 deletions(-) diff --git a/integration/util.py b/integration/util.py index aaa9c98c2..356233e50 100644 --- a/integration/util.py +++ b/integration/util.py @@ -24,6 +24,7 @@ from twisted.internet.defer import Deferred, succeed from twisted.internet.protocol import ProcessProtocol from twisted.internet.error import ProcessExitedAlready, ProcessDone from twisted.internet.threads import deferToThread +from twisted.internet.interfaces import IProcessTransport from attrs import frozen, evolve import requests @@ -150,20 +151,40 @@ class _MagicTextProtocol(ProcessProtocol): sys.stdout.write(data) -def _cleanup_tahoe_process_async(tahoe_transport, allow_missing): - if tahoe_transport.pid is None: +def _cleanup_process_async(transport: IProcessTransport, allow_missing: bool) -> None: + """ + If the given process transport seems to still be associated with a + running process, send a SIGTERM to that process. + + :param transport: The transport to use. + + :param allow_missing: If ``True`` then it is not an error for the + transport to have no associated process. Otherwise, an exception will + be raised in that case. + + :raise: ``ValueError`` if ``allow_missing`` is ``False`` and the transport + has no process. + """ + if transport.pid is None: if allow_missing: print("Process already cleaned up and that's okay.") return else: raise ValueError("Process is not running") - print("signaling {} with TERM".format(tahoe_transport.pid)) - tahoe_transport.signalProcess('TERM') - + print("signaling {} with TERM".format(transport.pid)) + try: + transport.signalProcess('TERM') + except ProcessExitedAlready: + # The transport object thought it still had a process but the real OS + # process has already exited. That's fine. We accomplished what we + # wanted to. We don't care about ``allow_missing`` here because + # there's no way we could have known the real OS process already + # exited. + pass def _cleanup_tahoe_process(tahoe_transport, exited, allow_missing=False): """ - Terminate the given process with a kill signal (SIGKILL on POSIX, + Terminate the given process with a kill signal (SIGTERM on POSIX, TerminateProcess on Windows). :param tahoe_transport: The `IProcessTransport` representing the process. @@ -172,13 +193,10 @@ def _cleanup_tahoe_process(tahoe_transport, exited, allow_missing=False): :return: After the process has exited. """ from twisted.internet import reactor - try: - _cleanup_tahoe_process_async(tahoe_transport, allow_missing=allow_missing) - except ProcessExitedAlready: - # XXX is this wait logic right? - print("signaled, blocking on exit") - block_with_timeout(exited, reactor) - print("exited, goodbye") + _cleanup_process_async(tahoe_transport, allow_missing=allow_missing) + print("signaled, blocking on exit") + block_with_timeout(exited, reactor) + print("exited, goodbye") def _tahoe_runner_optional_coverage(proto, reactor, request, other_args): @@ -233,7 +251,7 @@ class TahoeProcess(object): Kill the process, return a Deferred that fires when it's done. """ print(f"TahoeProcess.kill_async({self.transport.pid} / {self.node_dir})") - _cleanup_tahoe_process_async(self.transport, allow_missing=False) + _cleanup_process_async(self.transport, allow_missing=False) return self.transport.exited def restart_async(self, reactor, request): From da6b198abc37863db42df8d0ae35625c44c88497 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 17 Jan 2023 10:00:06 -0500 Subject: [PATCH 1381/2309] True that the tox default is disabled but it's accounted for already --- .circleci/config.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ebed577ff..f04e05ec0 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -316,10 +316,8 @@ jobs: # Disable artifact collection because py.test can't produce any. ARTIFACTS_OUTPUT_PATH: "" - # Pass on anything we got in our parameters. Passing an arguments here - # disables the default "integration" argument defined in the tox env, so - # also stick "integration" in here. - TAHOE_LAFS_TOX_ARGS: "<< parameters.tox-args >> integration" + # Pass on anything we got in our parameters. + TAHOE_LAFS_TOX_ARGS: "<< parameters.tox-args >>" steps: - "checkout" From eb630c391f6ec9b49c499ed247a605ac9351f16f Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 17 Jan 2023 10:03:54 -0500 Subject: [PATCH 1382/2309] "Parametrize" in a way that gives us better test names. The old way just put sequence numbers into the name. This way puts expected capability strings in. --- integration/test_vectors.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/integration/test_vectors.py b/integration/test_vectors.py index 27888e095..aff662c67 100644 --- a/integration/test_vectors.py +++ b/integration/test_vectors.py @@ -96,16 +96,14 @@ def test_convergence(convergence): @mark.slow -@mark.parametrize('case_and_expected', vectors.capabilities.items()) +@mark.parametrize('case,expected', vectors.capabilities.items()) @ensureDeferred -async def test_capability(reactor, request, alice, case_and_expected): +async def test_capability(reactor, request, alice, case, expected): """ The capability that results from uploading certain well-known data with certain well-known parameters results in exactly the previously computed value. """ - case, expected = case_and_expected - # rewrite alice's config to match params and convergence await reconfigure(reactor, request, alice, (1, case.params.required, case.params.total), case.convergence) From 1d32326659d10f29f2fa8e8e9747ed49a29ff6f8 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 17 Jan 2023 10:06:14 -0500 Subject: [PATCH 1383/2309] Simpler, more correct process lifecycle handling. The previous version included a bogus hack where we just passed `allow_missing=True` when finalization was requested of `_run_node`. This was clearly wrong since if the caller asked for finalization, it's a programming error for it to already have been done. Fortunately we have a perfectly good finalizer already, `TahoeProcess.kill`, which we can use instead of trying to craft a finalizer out of the various pieces that make up that value. Also, nothing seems to use the `_protocol` attribute set by `got_proto` so let's just drop that. --- integration/util.py | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/integration/util.py b/integration/util.py index 356233e50..604007f11 100644 --- a/integration/util.py +++ b/integration/util.py @@ -254,10 +254,18 @@ class TahoeProcess(object): _cleanup_process_async(self.transport, allow_missing=False) return self.transport.exited - def restart_async(self, reactor, request): + def restart_async(self, reactor: IReactorProcess, request: Any) -> Deferred: + """ + Stop and then re-start the associated process. + + :return: A Deferred that fires after the new process is ready to + handle requests. + """ d = self.kill_async() d.addCallback(lambda ignored: _run_node(reactor, self.node_dir, request, None, finalize=False)) def got_new_process(proc): + # Grab the new transport since the one we had before is no longer + # valid after the stop/start cycle. self._process_transport = proc.transport d.addCallback(got_new_process) return d @@ -290,19 +298,17 @@ def _run_node(reactor, node_dir, request, magic_text, finalize=True): ) transport.exited = protocol.exited + tahoe_process = TahoeProcess( + transport, + node_dir, + ) + if finalize: - request.addfinalizer(partial(_cleanup_tahoe_process, transport, protocol.exited, allow_missing=True)) + request.addfinalizer(tahoe_process.kill) - # XXX abusing the Deferred; should use .when_magic_seen() pattern - - def got_proto(proto): - transport._protocol = proto - return TahoeProcess( - transport, - node_dir, - ) - protocol.magic_seen.addCallback(got_proto) - return protocol.magic_seen + d = protocol.magic_seen + d.addCallback(lambda ignored: tahoe_process) + return d def _create_node(reactor, request, temp_dir, introducer_furl, flog_gatherer, name, web_port, From 290bb5297f800095f7c6432e94b922e7d9023e82 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 17 Jan 2023 10:19:37 -0500 Subject: [PATCH 1384/2309] lint --- integration/util.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/integration/util.py b/integration/util.py index 604007f11..a39fa26bc 100644 --- a/integration/util.py +++ b/integration/util.py @@ -5,7 +5,7 @@ General functionality useful for the implementation of integration tests. from __future__ import annotations from contextlib import contextmanager -from typing import TypeVar, Iterator, Awaitable, Callable +from typing import TypeVar, Iterator, Awaitable, Callable, Any from typing_extensions import Literal from tempfile import NamedTemporaryFile import sys @@ -14,7 +14,6 @@ import json from os import mkdir, environ from os.path import exists, join from io import StringIO, BytesIO -from functools import partial from subprocess import check_output from twisted.python.filepath import ( @@ -24,7 +23,7 @@ from twisted.internet.defer import Deferred, succeed from twisted.internet.protocol import ProcessProtocol from twisted.internet.error import ProcessExitedAlready, ProcessDone from twisted.internet.threads import deferToThread -from twisted.internet.interfaces import IProcessTransport +from twisted.internet.interfaces import IProcessTransport, IReactorProcess from attrs import frozen, evolve import requests From e4745779ab9c54581fdd29ee257dc03886c8cb7c Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 17 Jan 2023 10:27:20 -0500 Subject: [PATCH 1385/2309] See if this helps tox/pytest on CI --- .circleci/config.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index f04e05ec0..dae69048a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -94,8 +94,9 @@ workflows: {} - "integration": - # Run even the slow integration tests here. - tox-args: "--runslow" + # Run even the slow integration tests here. We need the `--` to + # sneak past tox and get to pytest. + tox-args: "-- --runslow" requires: # If the unit test suite doesn't pass, don't bother running the # integration tests. From 6bf36bebd081d4c9cf7f2910bd55850313b209b2 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 17 Jan 2023 10:39:58 -0500 Subject: [PATCH 1386/2309] maybe we need `integration` here after all --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index dae69048a..daef552d3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -96,11 +96,11 @@ workflows: - "integration": # Run even the slow integration tests here. We need the `--` to # sneak past tox and get to pytest. - tox-args: "-- --runslow" requires: # If the unit test suite doesn't pass, don't bother running the # integration tests. - "debian-11" + tox-args: "-- --runslow integration" - "typechecks": {} From f6555381a94fac2688fa0d5b1943c527d3cee825 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 17 Jan 2023 10:40:08 -0500 Subject: [PATCH 1387/2309] start right away please so I don't have to wait forever --- .circleci/config.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index daef552d3..57198eae6 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -96,11 +96,11 @@ workflows: - "integration": # Run even the slow integration tests here. We need the `--` to # sneak past tox and get to pytest. - requires: - # If the unit test suite doesn't pass, don't bother running the - # integration tests. - - "debian-11" tox-args: "-- --runslow integration" + # requires: + # # If the unit test suite doesn't pass, don't bother running the + # # integration tests. + # - "debian-11" - "typechecks": {} From e53f68f4d7a4e09df06a3130701aff556c979c81 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 17 Jan 2023 10:59:00 -0500 Subject: [PATCH 1388/2309] Move parameter definitions to their own module, away from test implementation --- integration/test_vectors.py | 83 +++------------------------- integration/vectors/parameters.py | 91 +++++++++++++++++++++++++++++++ 2 files changed, 99 insertions(+), 75 deletions(-) create mode 100644 integration/vectors/parameters.py diff --git a/integration/test_vectors.py b/integration/test_vectors.py index aff662c67..656033dc4 100644 --- a/integration/test_vectors.py +++ b/integration/test_vectors.py @@ -5,7 +5,6 @@ Verify certain results against test vectors with well-known results. from __future__ import annotations from typing import AsyncGenerator, Iterator -from hashlib import sha256 from itertools import starmap, product from yaml import safe_dump @@ -17,76 +16,10 @@ from pytest_twisted import ensureDeferred from twisted.python.filepath import FilePath from . import vectors -from .util import CHK, SSK, reconfigure, upload, TahoeProcess +from .vectors import parameters +from .util import reconfigure, upload, TahoeProcess -def digest(bs: bytes) -> bytes: - """ - Digest bytes to bytes. - """ - return sha256(bs).digest() - - -def hexdigest(bs: bytes) -> str: - """ - Digest bytes to text. - """ - return sha256(bs).hexdigest() - -# Just a couple convergence secrets. The only thing we do with this value is -# feed it into a tagged hash. It certainly makes a difference to the output -# but the hash should destroy any structure in the input so it doesn't seem -# like there's a reason to test a lot of different values. -CONVERGENCE_SECRETS = [ - b"aaaaaaaaaaaaaaaa", - digest(b"Hello world")[:16], -] - -# Exercise at least a handful of different sizes, trying to cover: -# -# 1. Some cases smaller than one "segment" (128k). -# This covers shrinking of some parameters to match data size. -# This includes one case of the smallest possible CHK. -# -# 2. Some cases right on the edges of integer segment multiples. -# Because boundaries are tricky. -# -# 4. Some cases that involve quite a few segments. -# This exercises merkle tree construction more thoroughly. -# -# See ``stretch`` for construction of the actual test data. - -SEGMENT_SIZE = 128 * 1024 -OBJECT_DESCRIPTIONS = [ - # The smallest possible. 55 bytes and smaller are LIT. - vectors.Sample(b"a", 56), - vectors.Sample(b"a", 1024), - vectors.Sample(b"c", 4096), - vectors.Sample(digest(b"foo"), SEGMENT_SIZE - 1), - vectors.Sample(digest(b"bar"), SEGMENT_SIZE + 1), - vectors.Sample(digest(b"baz"), SEGMENT_SIZE * 16 - 1), - vectors.Sample(digest(b"quux"), SEGMENT_SIZE * 16 + 1), - vectors.Sample(digest(b"foobar"), SEGMENT_SIZE * 64 - 1), - vectors.Sample(digest(b"barbaz"), SEGMENT_SIZE * 64 + 1), -] - -ZFEC_PARAMS = [ - vectors.SeedParam(1, 1), - vectors.SeedParam(1, 3), - vectors.SeedParam(2, 3), - vectors.SeedParam(3, 10), - vectors.SeedParam(71, 255), - vectors.SeedParam(101, vectors.MAX_SHARES), -] - -FORMATS = [ - CHK(), - # These start out unaware of a key but various keys will be supplied - # during generation. - SSK(name="sdmf", key=None), - SSK(name="mdmf", key=None), -] - -@mark.parametrize('convergence', CONVERGENCE_SECRETS) +@mark.parametrize('convergence', parameters.CONVERGENCE_SECRETS) def test_convergence(convergence): """ Convergence secrets are 16 bytes. @@ -126,10 +59,10 @@ async def skiptest_generate(reactor, request, alice): ever-changing set of outputs. """ space = starmap(vectors.Case, product( - ZFEC_PARAMS, - CONVERGENCE_SECRETS, - OBJECT_DESCRIPTIONS, - FORMATS, + parameters.ZFEC_PARAMS, + parameters.CONVERGENCE_SECRETS, + parameters.OBJECT_DESCRIPTIONS, + parameters.FORMATS, )) iterresults = generate(reactor, request, alice, space) @@ -157,7 +90,7 @@ def write_results(path: FilePath, results: list[tuple[vectors.Case, str]]) -> No "length": case.seed_data.length, }, "zfec": { - "segmentSize": SEGMENT_SIZE, + "segmentSize": parameters.SEGMENT_SIZE, "required": case.params.required, "total": case.params.total, }, diff --git a/integration/vectors/parameters.py b/integration/vectors/parameters.py new file mode 100644 index 000000000..b21c8e666 --- /dev/null +++ b/integration/vectors/parameters.py @@ -0,0 +1,91 @@ +""" +Define input parameters for test vector generation. + +:ivar CONVERGENCE_SECRETS: Convergence secrets. + +:ivar SEGMENT_SIZE: The single segment size that the Python implementation + currently supports without a lot of refactoring. + +:ivar OBJECT_DESCRIPTIONS: Small objects with instructions which can be + expanded into a possibly large byte string. These are intended to be used + as plaintext inputs. + +:ivar ZFEC_PARAMS: Input parameters to ZFEC. + +:ivar FORMATS: Encoding/encryption formats. +""" + +from __future__ import annotations + +from hashlib import sha256 + +from . import MAX_SHARES, Sample, SeedParam +from ..util import CHK, SSK + +def digest(bs: bytes) -> bytes: + """ + Digest bytes to bytes. + """ + return sha256(bs).digest() + + +def hexdigest(bs: bytes) -> str: + """ + Digest bytes to text. + """ + return sha256(bs).hexdigest() + +# Just a couple convergence secrets. The only thing we do with this value is +# feed it into a tagged hash. It certainly makes a difference to the output +# but the hash should destroy any structure in the input so it doesn't seem +# like there's a reason to test a lot of different values. +CONVERGENCE_SECRETS: list[bytes] = [ + b"aaaaaaaaaaaaaaaa", + digest(b"Hello world")[:16], +] + +SEGMENT_SIZE: int = 128 * 1024 + +# Exercise at least a handful of different sizes, trying to cover: +# +# 1. Some cases smaller than one "segment" (128k). +# This covers shrinking of some parameters to match data size. +# This includes one case of the smallest possible CHK. +# +# 2. Some cases right on the edges of integer segment multiples. +# Because boundaries are tricky. +# +# 4. Some cases that involve quite a few segments. +# This exercises merkle tree construction more thoroughly. +# +# See ``stretch`` for construction of the actual test data. +OBJECT_DESCRIPTIONS: list[Sample] = [ + # The smallest possible. 55 bytes and smaller are LIT. + Sample(b"a", 56), + Sample(b"a", 1024), + Sample(b"c", 4096), + Sample(digest(b"foo"), SEGMENT_SIZE - 1), + Sample(digest(b"bar"), SEGMENT_SIZE + 1), + Sample(digest(b"baz"), SEGMENT_SIZE * 16 - 1), + Sample(digest(b"quux"), SEGMENT_SIZE * 16 + 1), + Sample(digest(b"foobar"), SEGMENT_SIZE * 64 - 1), + Sample(digest(b"barbaz"), SEGMENT_SIZE * 64 + 1), +] + +ZFEC_PARAMS: list[SeedParam] = [ + SeedParam(1, 1), + SeedParam(1, 3), + SeedParam(2, 3), + SeedParam(3, 10), + SeedParam(71, 255), + SeedParam(101, MAX_SHARES), +] + +FORMATS: list[CHK | SSK] = [ + CHK(), + + # These start out unaware of a key but various keys will be supplied + # during generation. + SSK(name="sdmf", key=None), + SSK(name="mdmf", key=None), +] From 4e2c685a1296032a080fdb7f311ed2e32c3a011a Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 18 Jan 2023 13:28:24 -0500 Subject: [PATCH 1389/2309] Fix test_directory_deep_check by having it re-assert its preferred config Previously the changes test_vectors.py made to Alice's configuration invalidated test_directory_deep_check's assumptions. --- integration/test_web.py | 24 ++++++++++++------------ integration/util.py | 14 ++++++++------ 2 files changed, 20 insertions(+), 18 deletions(-) diff --git a/integration/test_web.py b/integration/test_web.py index 22f08da82..95a09a5f5 100644 --- a/integration/test_web.py +++ b/integration/test_web.py @@ -7,18 +7,9 @@ Most of the tests have cursory asserts and encode 'what the WebAPI did at the time of testing' -- not necessarily a cohesive idea of what the WebAPI *should* do in every situation. It's not clear the latter exists anywhere, however. - -Ported to Python 3. """ -from __future__ import unicode_literals -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 +from __future__ import annotations import time from urllib.parse import unquote as url_unquote, quote as url_quote @@ -32,6 +23,7 @@ import requests import html5lib from bs4 import BeautifulSoup +from pytest_twisted import ensureDeferred def test_index(alice): """ @@ -252,10 +244,18 @@ def test_status(alice): assert found_download, "Failed to find the file we downloaded in the status-page" -def test_directory_deep_check(alice): +@ensureDeferred +async def test_directory_deep_check(reactor, request, alice): """ use deep-check and confirm the result pages work """ + # Make sure the node is configured compatibly with expectations of this + # test. + happy = 3 + required = 2 + total = 4 + + await util.reconfigure(reactor, request, alice, (happy, required, total), convergence=None) # create a directory resp = requests.post( @@ -313,7 +313,7 @@ def test_directory_deep_check(alice): ) def check_repair_data(checkdata): - assert checkdata["healthy"] is True + assert checkdata["healthy"] assert checkdata["count-happiness"] == 4 assert checkdata["count-good-share-hosts"] == 4 assert checkdata["count-shares-good"] == 4 diff --git a/integration/util.py b/integration/util.py index a39fa26bc..25aebd931 100644 --- a/integration/util.py +++ b/integration/util.py @@ -766,7 +766,7 @@ def insert(item: tuple[α, β], d: dict[α, β]) -> dict[α, β]: return d -async def reconfigure(reactor, request, node: TahoeProcess, params: tuple[int, int, int], convergence: bytes) -> None: +async def reconfigure(reactor, request, node: TahoeProcess, params: tuple[int, int, int], convergence: None | bytes) -> None: """ Reconfigure a Tahoe-LAFS node with different ZFEC parameters and convergence secret. @@ -780,7 +780,8 @@ async def reconfigure(reactor, request, node: TahoeProcess, params: tuple[int, i :param node: The Tahoe-LAFS node to reconfigure. :param params: The ``happy``, ``needed``, and ``total`` ZFEC encoding parameters. - :param convergence: The convergence secret. + :param convergence: If given, the convergence secret. If not given, the + existing convergence secret will be left alone. :return: ``None`` after the node configuration has been rewritten, the node has been restarted, and the node is ready to provide service. @@ -799,10 +800,11 @@ async def reconfigure(reactor, request, node: TahoeProcess, params: tuple[int, i config.set_config("client", "shares.needed", str(needed)) config.set_config("client", "shares.total", str(total)) - cur_convergence = config.get_private_config("convergence").encode("ascii") - if base32.a2b(cur_convergence) != convergence: - changed = True - config.write_private_config("convergence", base32.b2a(convergence)) + if convergence is not None: + cur_convergence = config.get_private_config("convergence").encode("ascii") + if base32.a2b(cur_convergence) != convergence: + changed = True + config.write_private_config("convergence", base32.b2a(convergence)) if changed: # restart the node From 69b25d932c77dd5efe1d32a598aa2f084137b808 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 18 Jan 2023 13:29:14 -0500 Subject: [PATCH 1390/2309] Re-enable the Debian-11 / integration gate --- .circleci/config.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 57198eae6..152d56810 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -97,10 +97,10 @@ workflows: # Run even the slow integration tests here. We need the `--` to # sneak past tox and get to pytest. tox-args: "-- --runslow integration" - # requires: - # # If the unit test suite doesn't pass, don't bother running the - # # integration tests. - # - "debian-11" + requires: + # If the unit test suite doesn't pass, don't bother running the + # integration tests. + - "debian-11" - "typechecks": {} From 280a77b53dba5ad25a685a23e8046d7767e005f7 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 18 Jan 2023 13:30:12 -0500 Subject: [PATCH 1391/2309] Convince pytest that slow is a legit mark --- pytest.ini | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 pytest.ini diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 000000000..30a7e19e0 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,3 @@ +[pytest] +markers = + slow: marks tests as slow (deselect with '--runslow') From 129c6ec11a53f0ee489f3c0a481ef933010efd8b Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 18 Jan 2023 13:52:11 -0500 Subject: [PATCH 1392/2309] Factor more infrastructure code out of the test module Test vector saving implementation can go near loading implementation. Also we can separate out some simple types from the more complex logic. Initially this was to resolve a circular dependency but that ended up being resolved mostly by treatming SEGMENT_SIZE more like a parameter than a global. Still, smaller modules are okay... --- integration/test_vectors.py | 52 ++++------------ integration/vectors/__init__.py | 5 +- integration/vectors/model.py | 58 ++++++++++++++++++ integration/vectors/parameters.py | 3 +- integration/vectors/vectors.py | 99 +++++++++++++++---------------- 5 files changed, 125 insertions(+), 92 deletions(-) create mode 100644 integration/vectors/model.py diff --git a/integration/test_vectors.py b/integration/test_vectors.py index 656033dc4..4608b9287 100644 --- a/integration/test_vectors.py +++ b/integration/test_vectors.py @@ -4,17 +4,15 @@ Verify certain results against test vectors with well-known results. from __future__ import annotations +from functools import partial from typing import AsyncGenerator, Iterator from itertools import starmap, product -from yaml import safe_dump from attrs import evolve from pytest import mark from pytest_twisted import ensureDeferred -from twisted.python.filepath import FilePath - from . import vectors from .vectors import parameters from .util import reconfigure, upload, TahoeProcess @@ -58,48 +56,24 @@ async def skiptest_generate(reactor, request, alice): to run against the results produced originally, not a possibly ever-changing set of outputs. """ - space = starmap(vectors.Case, product( - parameters.ZFEC_PARAMS, - parameters.CONVERGENCE_SECRETS, - parameters.OBJECT_DESCRIPTIONS, - parameters.FORMATS, - )) + space = starmap( + # segmentSize could be a parameter someday but it's not easy to vary + # using the Python implementation so it isn't one for now. + partial(vectors.Case, segmentSize=parameters.SEGMENT_SIZE), + product( + parameters.ZFEC_PARAMS, + parameters.CONVERGENCE_SECRETS, + parameters.OBJECT_DESCRIPTIONS, + parameters.FORMATS, + ), + ) iterresults = generate(reactor, request, alice, space) # Update the output file with results as they become available. results = [] async for result in iterresults: results.append(result) - write_results(vectors.DATA_PATH, results) - -def write_results(path: FilePath, results: list[tuple[vectors.Case, str]]) -> None: - """ - Save the given results. - """ - path.setContent(safe_dump({ - "version": vectors.CURRENT_VERSION, - "vector": [ - { - "convergence": vectors.encode_bytes(case.convergence), - "format": { - "kind": case.fmt.kind, - "params": case.fmt.to_json(), - }, - "sample": { - "seed": vectors.encode_bytes(case.seed_data.seed), - "length": case.seed_data.length, - }, - "zfec": { - "segmentSize": parameters.SEGMENT_SIZE, - "required": case.params.required, - "total": case.params.total, - }, - "expected": cap, - } - for (case, cap) - in results - ], - }).encode("ascii")) + vectors.save_capabilities(results) async def generate( reactor, diff --git a/integration/vectors/__init__.py b/integration/vectors/__init__.py index 03f716cb4..af97eadbf 100644 --- a/integration/vectors/__init__.py +++ b/integration/vectors/__init__.py @@ -14,7 +14,6 @@ __all__ = [ from .vectors import ( DATA_PATH, CURRENT_VERSION, - MAX_SHARES, Case, Sample, @@ -23,3 +22,7 @@ from .vectors import ( capabilities, ) + +from .parameters import ( + MAX_SHARES, +) diff --git a/integration/vectors/model.py b/integration/vectors/model.py new file mode 100644 index 000000000..8d9c1d006 --- /dev/null +++ b/integration/vectors/model.py @@ -0,0 +1,58 @@ +""" +Simple data type definitions useful in the definition/verification of test +vectors. +""" + +from __future__ import annotations + +from attrs import frozen + +# CHK have a max of 256 shares. SDMF / MDMF have a max of 255 shares! +# Represent max symbolically and resolve it when we know what format we're +# dealing with. +MAX_SHARES = "max" + +@frozen +class Sample: + """ + Some instructions for building a long byte string. + + :ivar seed: Some bytes to repeat some times to produce the string. + :ivar length: The length of the desired byte string. + """ + seed: bytes + length: int + +@frozen +class Param: + """ + Some ZFEC parameters. + """ + required: int + total: int + +@frozen +class SeedParam: + """ + Some ZFEC parameters, almost. + + :ivar required: The number of required shares. + + :ivar total: Either the number of total shares or the constant + ``MAX_SHARES`` to indicate that the total number of shares should be + the maximum number supported by the object format. + """ + required: int + total: int | str + + def realize(self, max_total: int) -> Param: + """ + Create a ``Param`` from this object's values, possibly + substituting the given real value for total if necessary. + + :param max_total: The value to use to replace ``MAX_SHARES`` if + necessary. + """ + if self.total == MAX_SHARES: + return Param(self.required, max_total) + return Param(self.required, self.total) diff --git a/integration/vectors/parameters.py b/integration/vectors/parameters.py index b21c8e666..0557fce19 100644 --- a/integration/vectors/parameters.py +++ b/integration/vectors/parameters.py @@ -19,7 +19,8 @@ from __future__ import annotations from hashlib import sha256 -from . import MAX_SHARES, Sample, SeedParam +from .model import MAX_SHARES +from .vectors import Sample, SeedParam from ..util import CHK, SSK def digest(bs: bytes) -> bytes: diff --git a/integration/vectors/vectors.py b/integration/vectors/vectors.py index 9a92b0d25..ecb335ce4 100644 --- a/integration/vectors/vectors.py +++ b/integration/vectors/vectors.py @@ -10,11 +10,12 @@ from __future__ import annotations from typing import TextIO from attrs import frozen -from yaml import safe_load +from yaml import safe_load, safe_dump from base64 import b64encode, b64decode from twisted.python.filepath import FilePath +from .model import Param, Sample, SeedParam from ..util import CHK, SSK DATA_PATH: FilePath = FilePath(__file__).sibling("test_vectors.yaml") @@ -22,62 +23,13 @@ DATA_PATH: FilePath = FilePath(__file__).sibling("test_vectors.yaml") # The version of the persisted test vector data this code can interpret. CURRENT_VERSION: str = "2023-01-16.2" -@frozen -class Sample: - """ - Some instructions for building a long byte string. - - :ivar seed: Some bytes to repeat some times to produce the string. - :ivar length: The length of the desired byte string. - """ - seed: bytes - length: int - -@frozen -class Param: - """ - Some ZFEC parameters. - """ - required: int - total: int - -# CHK have a max of 256 shares. SDMF / MDMF have a max of 255 shares! -# Represent max symbolically and resolve it when we know what format we're -# dealing with. -MAX_SHARES = "max" - -@frozen -class SeedParam: - """ - Some ZFEC parameters, almost. - - :ivar required: The number of required shares. - - :ivar total: Either the number of total shares or the constant - ``MAX_SHARES`` to indicate that the total number of shares should be - the maximum number supported by the object format. - """ - required: int - total: int | str - - def realize(self, max_total: int) -> Param: - """ - Create a ``Param`` from this object's values, possibly - substituting the given real value for total if necessary. - - :param max_total: The value to use to replace ``MAX_SHARES`` if - necessary. - """ - if self.total == MAX_SHARES: - return Param(self.required, max_total) - return Param(self.required, self.total) - @frozen class Case: """ Represent one case for which we want/have a test vector. """ seed_params: Param + segment_size: int convergence: bytes seed_data: Sample fmt: CHK | SSK @@ -119,7 +71,45 @@ def stretch(seed: bytes, size: int) -> bytes: return (seed * multiples)[:size] +def save_capabilities(results: list[tuple[Case, str]], path: FilePath = DATA_PATH) -> None: + """ + Save some test vector cases and their expected values. + + This is logically the inverse of ``load_capabilities``. + """ + path.setContent(safe_dump({ + "version": CURRENT_VERSION, + "vector": [ + { + "convergence": encode_bytes(case.convergence), + "format": { + "kind": case.fmt.kind, + "params": case.fmt.to_json(), + }, + "sample": { + "seed": encode_bytes(case.seed_data.seed), + "length": case.seed_data.length, + }, + "zfec": { + "segmentSize": case.segment_size, + "required": case.params.required, + "total": case.params.total, + }, + "expected": cap, + } + for (case, cap) + in results + ], + }).encode("ascii")) + + def load_format(serialized: dict) -> CHK | SSK: + """ + Load an encrypted object format from a simple description of it. + + :param serialized: A ``dict`` describing either CHK or SSK, possibly with + some parameters. + """ if serialized["kind"] == "chk": return CHK.load(serialized["params"]) elif serialized["kind"] == "ssk": @@ -129,6 +119,12 @@ def load_format(serialized: dict) -> CHK | SSK: def load_capabilities(f: TextIO) -> dict[Case, str]: + """ + Load some test vector cases and their expected results from the given + file. + + This is logically the inverse of ``save_capabilities``. + """ data = safe_load(f) if data is None: return {} @@ -142,6 +138,7 @@ def load_capabilities(f: TextIO) -> dict[Case, str]: return { Case( seed_params=SeedParam(case["zfec"]["required"], case["zfec"]["total"]), + segment_size=case["zfec"]["segmentSize"], convergence=decode_bytes(case["convergence"]), seed_data=Sample(decode_bytes(case["sample"]["seed"]), case["sample"]["length"]), fmt=load_format(case["format"]), From 9581eeebe5927ef2fb5754bb6820d2368860cfc8 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 20 Jan 2023 15:14:18 -0500 Subject: [PATCH 1393/2309] explain the repeated save_capabilities calls --- integration/test_vectors.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/integration/test_vectors.py b/integration/test_vectors.py index 4608b9287..c5b765c71 100644 --- a/integration/test_vectors.py +++ b/integration/test_vectors.py @@ -69,10 +69,13 @@ async def skiptest_generate(reactor, request, alice): ) iterresults = generate(reactor, request, alice, space) - # Update the output file with results as they become available. results = [] async for result in iterresults: + # Accumulate the new result results.append(result) + # Then rewrite the whole output file with the new accumulator value. + # This means that if we fail partway through, we will still have + # recorded partial results -- instead of losing them all. vectors.save_capabilities(results) async def generate( From 4664bcb32179a32992ab21c011493c49e04b0184 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 20 Jan 2023 15:15:44 -0500 Subject: [PATCH 1394/2309] These didn't end up being used --- integration/util.py | 39 +-------------------------------------- 1 file changed, 1 insertion(+), 38 deletions(-) diff --git a/integration/util.py b/integration/util.py index 25aebd931..c7ed31a09 100644 --- a/integration/util.py +++ b/integration/util.py @@ -5,7 +5,7 @@ General functionality useful for the implementation of integration tests. from __future__ import annotations from contextlib import contextmanager -from typing import TypeVar, Iterator, Awaitable, Callable, Any +from typing import Any from typing_extensions import Literal from tempfile import NamedTemporaryFile import sys @@ -728,43 +728,6 @@ def upload(alice: TahoeProcess, fmt: CHK | SSK, data: bytes) -> str: argv = [alice, "put"] + fmt_argv + [f.name] return cli(*argv).decode("utf-8").strip() -α = TypeVar("α") -β = TypeVar("β") - -async def asyncfoldr( - i: Iterator[Awaitable[α]], - f: Callable[[α, β], β], - initial: β, -) -> β: - """ - Right fold over an async iterator. - - :param i: The async iterator. - :param f: The function to fold. - :param initial: The starting value. - - :return: The result of the fold. - """ - result = initial - async for a in i: - result = f(a, result) - return result - - -def insert(item: tuple[α, β], d: dict[α, β]) -> dict[α, β]: - """ - In-place add an item to a dictionary. - - If the key is already present, replace the value. - - :param item: A tuple of the key and value. - :param d: The dictionary to modify. - - :return: The dictionary. - """ - d[item[0]] = item[1] - return d - async def reconfigure(reactor, request, node: TahoeProcess, params: tuple[int, int, int], convergence: None | bytes) -> None: """ From c46ab2d88be73a8decbaaea33a7ada5bd1a6ecbb Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 20 Jan 2023 15:20:02 -0500 Subject: [PATCH 1395/2309] Hit a multiple of SEGMENT_SIZE on the nose --- integration/vectors/parameters.py | 1 + 1 file changed, 1 insertion(+) diff --git a/integration/vectors/parameters.py b/integration/vectors/parameters.py index 0557fce19..e1fafcec4 100644 --- a/integration/vectors/parameters.py +++ b/integration/vectors/parameters.py @@ -69,6 +69,7 @@ OBJECT_DESCRIPTIONS: list[Sample] = [ Sample(digest(b"bar"), SEGMENT_SIZE + 1), Sample(digest(b"baz"), SEGMENT_SIZE * 16 - 1), Sample(digest(b"quux"), SEGMENT_SIZE * 16 + 1), + Sample(digest(b"bazquux"), SEGMENT_SIZE * 32), Sample(digest(b"foobar"), SEGMENT_SIZE * 64 - 1), Sample(digest(b"barbaz"), SEGMENT_SIZE * 64 + 1), ] From a9875b19c307fa7d282d744b1f89bf0b1c10b438 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 20 Jan 2023 15:20:42 -0500 Subject: [PATCH 1396/2309] clearer language in the `slow` mark documentation --- pytest.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytest.ini b/pytest.ini index 30a7e19e0..9ff725e7b 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,3 +1,3 @@ [pytest] markers = - slow: marks tests as slow (deselect with '--runslow') + slow: marks tests as slow (not run by default; run them with '--runslow') From 781f4486ac57c7c6765c8162675e657951db1703 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 20 Jan 2023 16:26:23 -0500 Subject: [PATCH 1397/2309] Get the segment size parameter right --- integration/test_vectors.py | 4 ++-- integration/vectors/vectors.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/integration/test_vectors.py b/integration/test_vectors.py index c5b765c71..3e0790786 100644 --- a/integration/test_vectors.py +++ b/integration/test_vectors.py @@ -57,9 +57,9 @@ async def skiptest_generate(reactor, request, alice): ever-changing set of outputs. """ space = starmap( - # segmentSize could be a parameter someday but it's not easy to vary + # segment_size could be a parameter someday but it's not easy to vary # using the Python implementation so it isn't one for now. - partial(vectors.Case, segmentSize=parameters.SEGMENT_SIZE), + partial(vectors.Case, segment_size=parameters.SEGMENT_SIZE), product( parameters.ZFEC_PARAMS, parameters.CONVERGENCE_SECRETS, diff --git a/integration/vectors/vectors.py b/integration/vectors/vectors.py index ecb335ce4..a1bf9c206 100644 --- a/integration/vectors/vectors.py +++ b/integration/vectors/vectors.py @@ -29,10 +29,10 @@ class Case: Represent one case for which we want/have a test vector. """ seed_params: Param - segment_size: int convergence: bytes seed_data: Sample fmt: CHK | SSK + segment_size: int @property def data(self): From ed7bb1b41ffb1aa0c9f437bf566c166bd22e74db Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 20 Jan 2023 16:27:05 -0500 Subject: [PATCH 1398/2309] expose the persistence api --- integration/vectors/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/integration/vectors/__init__.py b/integration/vectors/__init__.py index af97eadbf..9fadfbb8d 100644 --- a/integration/vectors/__init__.py +++ b/integration/vectors/__init__.py @@ -19,6 +19,7 @@ from .vectors import ( Sample, SeedParam, encode_bytes, + save_capabilities, capabilities, ) From f4e3e08e38c1a16c6a6b243fd16dafd1e0e9daa8 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 20 Jan 2023 16:27:13 -0500 Subject: [PATCH 1399/2309] re-generate with a case using an exact segment size multiple all of the mutables totally change because we don't try to re-use existing rsa keys (yet...?) --- integration/vectors/test_vectors.yaml | 13032 ++++++++++++++---------- 1 file changed, 7416 insertions(+), 5616 deletions(-) diff --git a/integration/vectors/test_vectors.yaml b/integration/vectors/test_vectors.yaml index 6b28e2302..718f94f0d 100755 --- a/integration/vectors/test_vectors.yaml +++ b/integration/vectors/test_vectors.yaml @@ -12,62 +12,62 @@ vector: segmentSize: 131072 total: 1 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:SSK:k6wvqieksrwfbggxxfx634ymcu:nktp4feownhax6kladditlqli6c57lxtjpmx2l5tinv5xbmg24xa + expected: URI:SSK:fs6ul2fju2fvb2cfx7gt6ngycm:hncpinwszbggrurbvuaaexnftk3j5wfr7473pj2g734mo2isxlbq format: kind: ssk params: format: sdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEpAIBAAKCAQEApTEPAXxC8mAxFzIiXFh6buUBZkEELsv6rTnpYW8QbDSwViHs + MIIEpAIBAAKCAQEA1I0X8E9USJxPRJmD6l3cjlyGYi9hXxJFb5km5/J7elPbYPP3 - DA7zdugOEU6RRjJ/6kOFj1UFzg4sgHWrOi91wT7JQLPrNh61r0d+UmmWa8c3fw1+ + DhdHmJcELYP5HxGBmfLavCBvFDO6nVA3TDwCPrI/7KpiY7uHzZkLgsLA8M45NaJE - 0n444frx3cdNBEXeqvU36CdxHO06AVD4K+///w4VuhDQZBoFjQ7xKjuYyYrePvaa + eUgACBESZcNioUqYLNHvYKLOqTDV+JwyQ9oWUNONd2jg3LQ+e4oyVwvxEZ41P5cM - 0Tl+Te5+gm6rnZLP+p/y2iOtg1rLzYVJCW1QCQdJx69m/NirhJ9LFVn/4mt8x0mn + u9wJI8OO/G7FItCL2Ts1OgjmNWEz6KN7MjU/2UsNfa2eK6mlZ3Wi1oprhmfCrWHu - CwT4PAY3W+O46J++z3iPiY3YQXlzdoWh/uksFTs5d9I/XOoZOlaAdF6qOb8cUxwr + +hjevqW46Qp/ddCCkBQCHKcV5ZsbBVxq6vqrYClUYa6Y5jzevMK8euuT+tA289sx - xlYCGjudDGY5t3C0F0LSmZs5M9ErTKVWE4jGfwIDAQABAoIBAASpKbCtxCLh0brx + jXpbY3eXaggWdeDIoDquOumCCkVxhoj3dvUKsQIDAQABAoIBAAIAsFSN0sv6WQ7a - 0EQxSuMLMlaMPlayOHdUNA72QLXSrgWLwDNm5a4x6NmE4rO0VWs8IByURNA0niwX + 6XDIYJ8gxQ1gx+iW6fuStFikIsC00JDZy56g3oZUCfCJ2UuPJSr3rFLwdUt570yz - guwsn7h3XWcDZrlaANQgH+ip5xZofIl0cGLfR73EJyyrGRDUtO4rt/QY5eHvyEfq + KEo6GIVRtaN7uYCaED4CLqcVQa8jKkvUkxOXd5Sb4JH/5MqDQurNMZW2Av96G9ID - rmGAb0ssyjJTNmVFXfTo6NXBTc3qnuwQYEVYBu4rp+O9ssv2OB/otQBUwN6oo01o + Wr/j6qjpTWBuJww9UIdmdnH2hVd2oz12+6Y/6nlrE2iGPDkQMPnkKXRb7xeaXJOq - yhT3PfuPMca9f+8VTxPFPPW2eou1Nr8lBuwGk3h5KkyNGTvgNHuKM/7ICBK/wAc1 + l6003hA4JRtzzS1uBb7cRuvyW/oOouBBxoP49a8UUoetgOMNDvVX4/16lRY3K6Vj - dFo47jneQZeAZwJfCFSTBIx5ADE195h98zTTJvFJM5wxwuBC86qSKNnbQWmVrO/K + VfserJz2R7QYKcfCJAe54VImGGhvq0Q76kfKsbX0xZ5fGFgS8LyAaZYyR6M3V88+ - FBZpt4ECgYEA4zWEG1DC4qAVaDPslyhHVraX0e/570EnOilnqxBraHKmp5vdxucq + qmUT2WkCgYEA1mI7uL+NEn3zRjjkpqqO1tmKfZVDayQ5bpOtJG44qpmv+eihBuu7 - WbezO0791A5DztAobsi4hMXrPQG97Of5yoxnJCjm4Uvcqn4RQFGw3I0nAB827fTC + S7V2waf46SwZAdUyXYxj+u0Dfnwre53tx7jdrntKNP9o1i8b3pZW13wv/IWq2bcA - 9KXEXhp5ofrYmF1Z2k6X3GiVMs0Wu6pHYPiCbXU3bOoA/zMD3pKFA/ECgYEAuh+/ + UFAhSlFjw9qj9nVFYHnqhygKGq+EbzkILp2eQUstjoWM4xCo1bRMMN0CgYEA/c/K - BnAPaj0sEjYsZS6FdUvFMiPD+4IUsCsTXfGTF3rG5h6g0vPHPir4v3haoAW6F8oa + YwVm6nyK6jMAK5zGWstliTPYkkSU79BvdbwXayIVp8CeDYPWpZxtqVQtoNvQwA9C - Y49re3869yTqpagLctzMtrUGeZ7LEc/FzGaPfrgkKDzK2QCdR2wvNBet0DVjdWwn + 8K2PuHrHFH3a16siXPrto0hoC3oXyyKKmqbeZLpafg1ngQfieVYS0A0qt6cVx+Sz - 53YoWvKf2M+ooSnOT/1FiOJfuHzJetZBbV36IW8CgYB22D9JqmzF7cZEwyQ1zLPD + 3gy7W7xeHfBSBbDPJR/G4gI88+9GVJdCVAfK2eUCgYB5ZaD56gZBfW7fyeG4ewZt - /65Z+ZRaOVIzcgTvzZ7g+1eAxF6086WLWDNAColqqit9uhPsHsGlcYEiYA7gJFbc + pTwmBvrpVdbrxdYatguCl4qt0kw09hHWOkioOqzZpO34OrjNfm0zLzl2S2v4ESMP - Q6SPnXVm0y+RXm/XnONN+ec0gR9SSHzRSwPz1RVaTMOOrwWY0xNMDsg70lrZvq+n + oKBvaENKJYNBHeYDMlC0rw8hSLPJmzYjRGzFf7cltc55Bkkl64Ohy0uFdvRgYwQ+ - YVWXu4BKT/xFgIG9ohZBgQKBgQCi1g1xW28RGo3JLR4wM8BNO8o9sK7RByCEdFtQ + GWT/Bkoi1X9FKS7h7LnkRQKBgQDF/jZvEGO8P/NNxwM3AlFpuok2go9LatyURxDr - UH7JBwCm6dr4VJFXYY8ZLPnUkM4b7BSkUCDP/iMfgGvOHLRPfL+Zhc0xcGznm2jJ + 0xKhrDEgb43cFSB4iJKzKMt/VHp/mGgrv/kBfCWYwqTY4NMpnUWLvowLh+7Ps95T - CF24lvADSBSMQA5aI1s07xaBV4Q5gjNzPJvX3fddX2h//6xhrQs91Be8t2gqkPLS + ziBmi0jUVDiN20y8Qnzid6L/KQRArxPxABWX9lWlHTee4NJ2r1dCL2TFFb7Tdjtz - 9WpV/wKBgQCVaWipSEsgoDuIgZQSebAfRUE4eS69SGhwwxZ8esYYUKBcJA4Q3BGB + ubBwUQKBgQCnEaEgoZ2Rqp3P7TzQjSzPlHlHBMXW28sEdXprdlwicU18fjDj5+og - s2hWwmTe4aL4kyNV86SfkMoMefa1mXV5LL4k5t0rXatdb+95/jX3ecQqELH+upaK + iozu9orcAD8AOGBNueKErSWiXhp+MuY4AvJuJZPV3gkMKKYc83HKWN05Xh8rkcZ3 - JLRvVhJfaLJ5XN9WHjz7ebR6NRYcc5xAAM1qTRRjBgkX06vIVbh4GQ== + KIHCyp3EBdtI+YDWvLISSqvFqCYtBR5v7eU9Ri2gOVljtmgJ8lARAA== -----END RSA PRIVATE KEY----- @@ -81,62 +81,62 @@ vector: segmentSize: 131072 total: 1 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:MDMF:wgmyhgqybo6uemxz3l2rqnxemy:zugfpwxmiz54a6ml6ocei3xdchqufogpghyer4az6vmzyvxadrxa + expected: URI:MDMF:tn2ekpkmearz7k3bivm3ikuz3i:wcqgst36kymoirczlkok5pqdekt5lgsyfw3oh7ecoro3rj5aiyya format: kind: ssk params: format: mdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEogIBAAKCAQEA0BZ+gvZma5bZne1GBzhT7pNOql7TH3/kIqUAWiX8xfC+n0Dp + MIIEogIBAAKCAQEAxMTk4iBj9yi+NBdRamEGlVB+/Vptm6uWdSZIF5CmliQHr0Ev - 8/lPVaztFlLBYK8bT5Z0UCRxHnWtM3+tjb8DQeoeCoUqqS5WavW+p9iv7xS8yMjl + KlYaiMxlFR+1RiV3LA7tzUbeJtHUsiaefAxtZFf4gtCKNRj1vC8lhbcH7NKxY9vr - 7KrrsAnDW2JpkbKLj+i+YQCgyOdmKVyrDSSUdBYNwXdsjTmgv4H+gXqkkdIbl0Fo + 4QMhNJ4NxYpC8qqcnNrlwULMw+eYLb2NUhuI2NOPKDjs1dgwypIEej0/m/Z7r+NB - z8185rMDxiuV6IlWA1dQCUEVB6tLhovloUJ5g8fDnLbv3nDh93C7aadZAn4hCkrh + xO6dKx4NYOnrQBoLNfgN7KlHCruhG4LcZxaMkuUD38brtxkd25/PlQ9tm77ODz9G - 4WmaSH5pB34F+IMsFXdEwpjvdTVWrPPRo7JRTvBRFSH/B6YpIfR7DWUiXqKGYlJP + WrU6/G5LSzTgWqrCQ6ww3UXfk9hfNqXob3loaTw2U+2M5gy73UduhHqSkyskNHgt - BXYSFslZ9M5HwLNtOnQam6E2cIjPrf7CXRFd6QIDAQABAoH/cD5loCLni3WoL0ma + WJRUyU9YC4K40IDtiJAlJJ3cShfVNDWCAIjKuQIDAQABAoIBAADRMS16j/UMpMQ8 - tUkutRyJhXa88Na6PGPcNt9Q6LXJlnZs7YfJuNzrHyJNiCZ3i1j2JLK98Ul+jfHi + 8o1xYLXJs0qkZP8iq8nJwmk3+bvMONdH+y+pqDY4Ob/oNU2uGybCMHJL9eE4ZDHn - fqf9XKRq6ubdt610jAsD81fp7RmpRIfjlNGf2FJwW0zgEhGpkWjMq2pSDRE+Eqkt + NNJJZOzn6/Qdye0lhjkQAw+2mQr+kauwqUlHxOFd5KsU2L6plGPsXsw6KvUx/DD4 - zu9EhTgfDlp4aMCU8S0fq9ZmAha5gDmfnbYKZbBEYtnHUWPkJ/Io19iJxa7HuXlM + cA1OvaHqOFZ6Qgrd+SSQ4wGKST4sZSwX5ZtMX2+o0eghkK1W5fMgbxSvMOWBOwqk - GOIBZz4U+epDxFpxPmJ3puZWR7eP/aYp62lViObg92Er/kliSPZ7LAACIBmZFrXD + uUnYeyFECxvntRhv0clYwvqfNetf4SMFcPVp+pCY7gaZH2rr3tVk3lSHWFI0pz4s - d9VsIScWnrfPdvrsy05VGJvlonUw/Hlo8vlSvcYDO72DlpgPmkuosAzHZFmimlyv + lLVtINuK2jeg8cfhZhnlWHSVW8C/F3xryNZCnacq4UgeoYwje8swez4/gCi0PyYY - RNcBAoGBAOIIuWG9XS7aH0kMaZC2a56TXKMevoa6ydOaiwumm/ojfzcYkbLmgTBZ + MbufpP8CgYEA8qLO9guRyFd9S4jgmVmu1TQLN8nX8sDOfkmiT7+V3oDJm/3HNRXV - UGzpPXe5w/W/GYrCeOxk3XPfSJPVEeUhKdqA/JyQ2xXXjHu7UCKkhUPCZh1USlSt + nX9UD/n2bpydEvjMCF7Mh4FaQvSFqSLTSBDoAY+4KD4KTecz3JD5yugBWbT7hWpp - aXP4fL1fEqUZT6tRBUP7utXuoi0Ga/kiMWjB1b0VgAoBH/uoUCLpAoGBAOuss3wd + 0jgUOW18w+2HFyLsEYFyW1xpIa1iZT9w/R031nGeH+q1CJcGfYRNXN8CgYEAz5tb - gbYmBmi2U+TOcUoSPf4E8BjTFGSuXfvtsGjy7OU1w6Jj41pnYTxQWoyUVJy3wG+B + wAShRkugicR3jO8geYzuDV0JOAsZnTdef0OLrTYoOL6V5TX1YNnSGsv8vTs0sF08 - IwklM+DNMw4o4sgVcqlgG7grt3adpcuVIQY+5rXyO4ZcG2vi5UAO3Yk+9c/AvbPJ + PGVQqEdZrYtgCJ3wehs30xBoE5+CZdAo8k810x18TYK7Zx5ZF8VjKT4LcW3SPD5I - pm6p4p0BjXMI8w7KxBHSyfywc3HvkNLlRIMBAoGBAN+BiO4g9ZdykCUHZQt3lotD + 7T0TUeOKZRWgF0V4uRMj2qH+4fX4fjkoSDkYM2cCgYBYaf0yaSrZLxBIGvuExcpQ - ZALYT8Whxhi7ZGqs4OdDWnP8k3W3gF9ysZhAOku9IQxLXtJa4n++bUw6qeWkdwF+ + hGNmE9Xt7lYQbLKJjs2Ew7czcXlKncc2WfR+0d37lnQiOqjWj/zFj9wdM88Uv8zv - /YfWq/OVOU4ryfo/ikn3LN+HxrmRs75viyrlt1L6Q9GFacYZY3+J14Hbafnjs7iy + oMF5+C3p9Bl7I7mhMO7lAj+jubBBgHJJGQg9mOjy2DX2t1IAWwQZyIXCsNR/Amwg - GvFfWh6St/0sh5etIzChAoGBAKZCjdSvlESGCttwVTsDkNSqjeVYYnGA59AnWtJR + v6neKY6uIK+RDr9ds30hTwKBgC8Svv8PDbJuu3wBfEoMfoSRG/kTu19lxO0M/PRG - 2rQPPKRvC3bSdR/f8q70GQ03z4FH+JAxUCAxiKm82ZnRqjtxNhTbYnLJFIKvsLkw + UIl52izjqgFK9tR7D1TcI/aUUiIbQek/38YIR6E+FQxfI4PMYCAPfEnWxS5owKAQ - mb2oPmZ5Xxjofcfcp9JLKmqaahuIY8wkJC/J1b7hy4It/BqhXTUdubV0Xd0xHsBJ + rdesu96nYe7DxtfI/e8ADoAtspnOVaLVUmgi++JnwOEF85WjbWHJkY2SxEF6nFOj - Uc4BAoGAbN8Fx/7GKOaR34m2PRywftvLXbjxQ+xGtp1k2ewymndqiurpqd4ckYZj + /oTJAoGAbhFazz1ZFuMJ8TwKo847lkC3TqWEvMzvoFgOLej6eqiZ93LtZG2+CG84 - aVtmHWlgQ8St37dhBXFnGGtnTyiyQd4s97svLImvyQxAo95lU9UPApWrerzF9C9w + eBDDyAVnNaHUa/HrX62B3Si6WN/vxsI9x191kaRUkiBBaqTg6TkruQwdFHMlFFHH - /jAPW4j8xpD4AiXmSvC2nUR5jKPSd5B2qEypp3aIShBdDys7YYI= + Vg7Pqd88N5H5gMl6+i4c/RgXw/vlzFkTgc3TwtUjwAWGQs8eAVw= -----END RSA PRIVATE KEY----- @@ -162,62 +162,62 @@ vector: segmentSize: 131072 total: 1 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:SSK:uprgind2aeoklhu5feejamwe4m:u7ettdqcxjw375tu5jxxqe7uzx5v2wilwsiwiolbo4ery5hy3nga + expected: URI:SSK:kwn44kjzh5s6lqyaeh3d74ziye:fammlnqo37yrfilvn4xwralire36de7ogpusp2uprtirwlpdbtca format: kind: ssk params: format: sdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEogIBAAKCAQEAovxwmrC8O1S/mvxTON8SitECU+pMmsPaM91xnDtBCEAXS/dZ + MIIEowIBAAKCAQEAvXGjmoIhKgrA08l9upgUZdyMOgOrH2rayM6/7tuaNjWzXfBL - baWgQ3D6ytt8b1X+LZdXo5alRv9Qqub3nNBaRalenRar5lQ6TLfREq3GaXHVbZoY + 048vgjD+PyWUCat9+Y/ZXO7RybLSSJ0z3tK+7177gaBBPffwp1ltlVR0mFEnpzRE - 1UQrBfUjaNniC5/SWpTyv3wfVWAgk/Z9Lx2979jqKL8MqM2irTdWNBhOsmui9SIp + 0wOzhn8+DBcrOV6819f2TcL1PLjtX0RnbHoVpQaMhacwE9laXSGvvDEliPRSTZGF - 4Mi/qhJf5zuAVe/qX6Qz2Jhn4jfstim1Rid8ez1PtXyEPRuQr44pBpNRiRejXJ+U + a2p4DmkoBaTILKT9ZYjUdYvWT0VjtpK0aj1kkBtAvJFfnfKw+SXEnuRb49QWlfZy - 3tZqkM0FJVgxT0TLkrQZhjLlF6aHNWY8jHooxhenFfgZTYv22KlACI2YOEytTm1x + wuQdAZKqpHv4lm8DNDXviU8mXqU1N67D2Nn+TIZhZIMIILlinFKvbQDgI74sLoMI - 8HggPrD+AlDBZ3W/VbQfLvukZv+XwOR0e2Z/KQIDAQABAoIBAADlHopMM1p9+RXx + PxnuSZELZkRF4SaV+oNIjOV1FUaPlhs7mVy98wIDAQABAoIBAAIVmYrDDBUizImR - Ou57YIyRt2lvmDex1AfquE39wzoZNhRWwJ7Do7WyI1QuZ3z2ySnF41dabala/DOn + 2dFeEhLE1zf7k0X3OGWlhtxSs3aXYjTDd+0wb4HX+RQqU1+68LKCZjWx3NX4rKOi - 46xwkonZteLbBqfT0Q7woWLXJvghqbGcQzdeB4rFl8yLjwfaB9KZ1ehf/4OR58OS + DBVw7bThJZTPsOMnWiTOdgJbYOn1WWZ+8wmte1z1Kvw3YLxfKqFdVnnP8LY0ohCX - erzILXf9+/FFvOa9CgUg7Irwc404I8OVBYFP/Su4HpWDiAXi31QHpFW4Iyf9yM0L + C1CT7NZFrxjlDnxdHYxWAceaTUpYms5+vw/XwNUtypjjqeU7yd5lZ8Oi3kRlxgDE - 5BWfcpBIxkNQDDgX1Qfmzm3lOioQ3sg+ka7KPuve+2pnn2cS4Pu1zdXN/edtdmtJ + tVJq96pzahruy7Xo+QOrqDd55zTzfm1lXA5ZQNMtSKOumSoJ9+/NTGkz9WYDOzUy - 0Uw0sCf6WHi5X/dNA/FdKY6qL0Hdd/d6XiR3enHQV2ZWnENt0iGoxIYur4KbqNmW + jBJDzKYP2hE6moUdDOUqK0sWPFmcRvMcVgxlg9DRuNFWzqM12jTzEaFqnBpRUoJ7 - vofTy8UCgYEA3MYl0WOSyuv6HVTS4ZEsZL5UypthpeHLlM0qmU7kLKYsN3LI27J9 + KuNBBlECgYEAxUo+GfuPB4QuWy3TlvbheSOP8JN/wJm6YgsisncroYdIibR9r0BW - jcCr26+saEg3Gtg/sXAkh6RQhlHmLVWS8I9yNXI+WiKBiuQsR0dAwBKhM1OXtLke + dsaQmJEPKjsn86qIIT10AtNXlieRg8GTtW+1E3GPDpkJYnqcQZ5D/R2AJPNK7Acd - WDV1buSkzgbxawxLFaxV5G31isRGz8XO6v+pJUQcg9JGpFkL8+3gm9cCgYEAvP3b + JeOAjZOWVoD79D8sC6w98+akURaO7TrTwXEQpSIrl+k0aR/RQCThuycCgYEA9dGs - KclJW/tZSggY7+9nv27EzwSYnTRdjYYcUtugKqm/Pe2wJm9vixqXWMMvmM4Kd/Xw + AWA4RVU++eFyXiu3fc08V7Asgh2h991FUWckP3/kRM8P9wLBJkBvolbQ6OA+kpKy - vgQg+gfXoOxQuBsDZHP48grqiPsAVQsIbgMiyWW9UvzYQxezC2584sMtygPiBb8Q + IatQ02fCw77ksathgv5X7dRwBq+ziBX4VSch/zZGXiQTxJdjDEDFvCOzmXmuTVgw - rQ0A8uEaa3I3U6CRQtKZvc0opK7OymXFzu9mXP8CgYADQsn8NcRNSv7+v+n9eu90 + ZXSXaQkTlOci+aknV4CHZ/uZl0IRa2gX5u6x1lUCgYBE+3+ZUCcjpqkawnxARdRy - 7XrDI1hl4tfm8sDWUtv77NhqWT+uPwyrs1TWgdnCEI7/zoHiVQ21EzA9S6hiswjg + qeeTY8+AhX/w9hnMsvRzhzzqwUxM8b8JysYWQmo+Bu8iONdeYAFnV4RBgVZU7mN6 - lL3THETff/L54jTlOKA0NhI7d9idyr4v/1oksSvd/yxBsITLZSg/n4Ao9I03NGzB + RjPXN5agsQvh/iMSoob9QspioRrqSlZ7v/9cAWXIm1L9hPUeo7wJwvRjUfLpqe8O - +9S7wC3LpKd2dfo/OBxBMQKBgGVf/jmR4SnXz3NomIfLcWk8L5GkM4DP4AbUE0lW + rTz3sGnztNvZggGFXx/6cwKBgQDg2CT1qTYvLNcKpwz+WAxhVF2yc2FyrnodBtbF - yblYyF6dqslTKRACuYBBYryiePcUE4i5ij7UChQl7r5yrwUpODYNKPVFPk5f1qu8 + q4r7ThbUXXVj4bAcNeomWjSCHcL+PJIUu+eVRx5d/3idjn4F3HE/CAZkB0g23Kml - PuKtEjr7qb2DbuUI5TB15Y/hOVI/xOAug33ExXkxEQBotsKTWSh4bf64TfA/WzW/ + 8cJl9xYMPAGc2z/s0D5NZXZ8llE5S8YQtsFbgMLZe0WBiRiEL/sqwHbvZK4cST07 - MLddAoGAZgnPZRDa3tvxrc++GHRxtkN+MOhJaahfBVxKsiBuWj+tA+XEyS/AvhMG + rO8bdQKBgEvcv+EJfffxdmzgRaZQaxxLdvrFH63ArPa/CnTMsltQUHifZD8H5F/Z - mKMMSko/pYfor1FpeSrqnTFwCzIvXPt/zB6jJuHEUiDqwFSqfHOaMdSjut2/HDxH + MaMXAN+tbbwcE3uId0UcwsSflJOCoHkI7fly08FYTUCzyWLcTrrntP256SU7bybf - fVBPYYeC88V2snKjt9ZPmZ8569XK3/Cp0gbZqJ/e4bx1cTd9PJk= + 1tD2fzeoHns+FePq5qSkmXyw9bKC4WaP7PYEHr8RZ9+z4tmmo1GU -----END RSA PRIVATE KEY----- @@ -231,62 +231,62 @@ vector: segmentSize: 131072 total: 1 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:MDMF:omlhm7gyylpggroasrqohbssba:7ht36qh7byquddx26sr5ywrzgpviqgtbxlgosnjz3njxi3fs4mkq + expected: URI:MDMF:uuln4c3nhnggvyge4trhl3d6pa:gzzewapuozilqyr7jo26dxv4hmip5nalhwsztkxxu3dcrew3k6wq format: kind: ssk params: format: mdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEowIBAAKCAQEAjxmo/c3d8aSEPugOJh/TBt7G/2FTD85dDzI1ZRE04V81uhmz + MIIEogIBAAKCAQEApavd9AgytDZu/VS2b253N8/2MWMUJJsAV+dXcMl1lXlD0yN9 - ND8k5wv25nSDCdh0ewPloRX3+uxbo6HX0wj1rWYFaDjxuiYQF7/qpv8OgKShakMK + Ljxay3m+ZIc3jJPqj5nDhxFz1Yd9HI0pJP224jJjlVHif9SpXzH9FVAsGIizYDMd - kCiwnn+DhISN+nEO3QtRr3b4nwUh+pqAmL9QJvOq092MaqhGXFQd4i9bQEAxH2P+ + ozHIZVctpX2ZbcQUbsskiGGnWwrDCrBDruVq+XnPbbGUpTMSFsXAafuKlvBG/zu2 - OjrDXB35/7KDFkGwVZO6n3ZoGJMhRQQk2w9+7yk3rJUdGOQKzoZG9MilOQgNM9z4 + e9TyEYPeenCz2+9A/Y4wIcK9z+dbKoof58R3XyQO20v4oD1cb2ohQreE778iEYJj - 0BfSIuGaokwhzdWIOfkLOiYQw1slod2/mYgYO/17YXX6afKSVR8D8ReoD63IyC6W + gQCjO/EiYz2ATiEOlPrdD+R4NjFxqzfh8SZWrIDyKho7bMavvBM0N7Fa20n5630r - UsVsS/2cV5hrsgxFDVBasvT0CqzZcJb5scDrAwIDAQABAoIBAAada4vr984DSl+0 + Czu6jZNVgqK5qS46otsQN9XCUB9F37IVZbIwYQIDAQABAoIBABzHki9QFETPlvDN - D29gujsPkkhc4d+RrQiWTBSTcovWgF/Nb1TDdHu/uFaX3TTXzi6fk/5ZyyBMy8Gp + nEKXyUCKER3LtSZVwdXDY6J9cL52WiSty9NyGyCxRbSPc8QpNuxavQdz7fAoUQDa - KhZlzCGLXUWfmEEAIG9QnlLA6JU2xwVn+vWGBAAXqec2z29byZGbQ9fmGoETVipE + ec0KARDiyX9ZQfRMZF7b33fqHTrm1mhOAOZGYeZO4mhW/QX/M8B6bB5//lNXt4Ge - +RvWgCiEzAlGLQcDJ1l+Q/FgOgoiu840VzEVaq3YbeX43DSno3JsROZ62+UjUo6b + FKfnhTGQ6kqHOIgJKumHUVFn58+n5khCgM6TKTOc6A4wqSfvF8Mt0eAlj26rPwUx - PUXrmpD363OQTh66zSCspztHQftiCUf+XoVQXQVT3vbBbMwoXCwJgiakHQNJQ9YP + sil+5uj+KMf/Z2Hb0/KBONLmRY2/dIwjNa41sxQ/DnW0Dae4PSKU0IojV3qz6mVF - vtIk21weUPN1jGZ7i/N7UEQNVdwLTmg5Fc7zc3Gf2JailWLDqkB8BLp7d2sJLJtI + upfeZEVcod3Kn5jbHvkL5VpP2tFkGz+v9RPEQ8Ipt53GzaWMvMzKJ+nKKhp77IYL - JPvM26ECgYEAuVXyGX3EnYRY2v2dLsTv52onik/DafatZCAMJX4wC5CP7YmMFfJT + TBGp9SUCgYEAtf+ri1c29XMhUENSTfSKfihxBdlABVfrVCeVb+5TnB9NyGxLdC1W - S+HGPkvmH6eDK9LM2NSf1WmSs8yrjGEpI1ci3i4KHbLuPvnsBqeBgMZCbUQ3g9dK + bFX9kn1pURR3d/0MJvgr5oLt7pFO3FmJ6cAeBYMadoxu0mKZZldnh9m69+hZ73ih - uWt8BL8R0rkzE3M7Hz1MotKtlQZcHsf/qqV30glLQl2NTSPaEK89iOECgYEAxak8 + im9VXLzKvwzVO7lpC83UMgJXyGs1EVQQ/fpCDlq8wEGhH/zIlrq/IpUCgYEA6Qis - tUCSZESZlPCG+FXlUUJQp+dGSnsHcFZ2S+iQ0oNbx1QcBoORI1uiwv5GgiJayMlT + xm5d9lmhhM7rGWXjzcWDI5Jla1SGe2bNDwTyXMFoG+W7ENXtO5bdgRRaiByrb0mh - g639Y2DNFNXZEa2kdxrjgu5tL33qwckP8SYbQI0fevjoc5zXuTjLB7MB07M/utJw + X+9r5PI241MkseQIHL8B1yEgsxExj7aebfujspeAeSA1IznaZOSFWOxG/L0xpYQZ - D7EwFOLKEwXCqFEC13z9p5cuwGEl1NlJVKuofGMCgYEAhFHuRXDbnTJOVhtXy3pj + G6dlk5YTv/d7nKpZ0AV7z1rvOER+KLhpWsIWT50CgYB8fTG90h5JrkKvQB1gLVE+ - Za8Oh9smw1KQvLl8spADMV6Gw6q+TzTxb23EIdoCdHseVX1tLymu66kyShhIKjN8 + EjOdKIleHlFd8uWI9qBCPjdaTJkgEpL276rPNPGBAFrnvDM/xKlit1RAxywGFUj8 - MXUWudXY9xc2rdO1RZL2DMB/0I8xq3lcKkGpC6J20SHUa4CLp2QWgPE1aP5fasKT + lujyJEdJp612QvNiyY6Loib9w/UglIcKxjTBhnG4VPLWM3DjHhtzSZ75/DsEa1hk - sHvurhBgoQM1zOtZ1yumHUECgYA0Dz/jCS/FYuAEf1k9HPp57XpqzpoP0dmCt/MO + IXZlkzNg3oNz+djLIVn0MQKBgGUfTZJCHGYlmPB/tgdsboFBn1mVUotTv5PXFU7a - SSGjoF9S349GE+7tHhx/ORN/AOdiTMxHOVMsknlRTIWQh2hyyk0z1fJB+OsUwQ0G + L63Og7XvZ9CzdGGyYuZ7hZmhD0eYpP1zcNeFwAm/b6H+OQL4Y/0NtBngcShS78b8 - 2Z+B3+lzrQ0kLiIPMasfywDnLiXR4c0MBQIB7j2Exxae2D9kXBI+yq3Qk4WwSs5q + NpnuImLtdgGWPk4f2DmxNlDAbMXNX/PfHTYBHwrjgvGF/rlFV+ewJzS8jB5xf85R - k4+buwKBgCHyWPzrSfkw2f48jdtWOwCSI8by3XnB+AxYy7ICgxXRTS7yTY4UKmZd + gMsFAoGAB1V5eES3Ud9t38YeEcc4zi7amA7oiLZgAlMStROTxBripUuw5aFvzila - mbrWpyUstquLMP5x7ChvUQU1aH7RbZFGs6ZSPd982+iV+/WVGl0wOCLczTqFKro1 + S09EunEasURmykYqdzUoMAsIBlwiFS1Ky1pliR/PDgkX2YVM+S15bisIrPu5DmGF - p5L1pZf3lIN0X9LVRIAhUk8RKjuINcuWZHPVtcTjJ49hK/L409Wy + bbYgYCgKvkxL0p7hRTeu0+czw29He/T0pNfvr6/+4nAL47oCju8= -----END RSA PRIVATE KEY----- @@ -312,62 +312,62 @@ vector: segmentSize: 131072 total: 1 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:SSK:ekp7frkwxjgxjc4d4wwqtyf4bu:ty5ope7attfwtmqlc72n3uwzlf4mslshmoa5xw2eohqh5me6ajrq + expected: URI:SSK:s2ort62uw74mbnvhqxgtclusli:xm4pmnye4mht7hoh5vcpt2zknqdvogujdo2ygmj2fb3chllp2ecq format: kind: ssk params: format: sdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEowIBAAKCAQEAza1esoFmMt4RWB36A8xGkpVKtPMGx+DcWkQ2QaAxld/CjSl0 + MIIEowIBAAKCAQEAughxLigyfdYXPUaFIYRop4H33/VxoWOha93f29U3uFfjeij4 - 2KSyH7vIeijVh3wids2V1WkHvCw9C9ZyNu+TUrIAXqWEGWUijObXDZUaiyCkeNXf + GPTARKNbLaqvANxHkscAPvKSTCJdECqzdg05AowyX0dbDPrBDirDVb2FF3UJ43oL - B3x5aKAZBc4sjBhxZvo2AggPEdC8JTyNAGRTpBSXVfNTKcu4np3W5nwrToSIJyza + cFIrDSpAL/+Yr14mVfvPtxomdw/P3rRocY9PtiLbSLGF8u7eliBiUsITbpQUy8bw - MHEZlYeUMa9uDwKcL7Ephjdle+bJ0jzIIiC2GZfamZYlMEmhUNQxbQRTqy2E3ovQ + 2s25/uBlkWXaflolma67NyVS83tugO13PN1W0pBqRRhmoJtWm6Eg6CRJzP/PeKmO - 7JrH2KL2edbk7/s/an6iOa6YDPUr14llk5YD5a7j+66bE40spF2vRTt4PHUCIJ6+ + OhZ+ybKTGWl3Z8Wu6w5TmEDrp4zHRZ3SYyGXrRppGhHhVQY8awBVCQzDf8hoUt3b - PY5fFHYI0l0tRnf0h/RPeOBME7JqlqSncInvkQIDAQABAoIBAAOoVEt3eX84s51h + y4ITWSIS3DqiI3n3O7JaPFKMTeFknv7hmgWLhwIDAQABAoIBAC3RtXqeWO1IZTno - +ZXnFF9zkhK4EmgcKaD9jusxd4ZFNpUK7l/ypFQEA214M81jLcv37ZTgJ3X/QaXn + 3BJox9P1/WSyUbXj0Za9M400A0DKmNL2M0EGSzK8n1DsmKTYHGLI08UtPp402oII - 7iunzKJzJcZpGjl3IuNXcM42F0Tve3tGXt8nxZMwqyc/Nou2fNBGbW7RcL6p7Amb + WmIpmJCJkkCIVSMpwZHpM2ozPwwLfg+CeD3GZycbcGrCA8uxnzSPTFY3QSETCmo6 - uzxtPE9J+jO6JjHDZJzl/MoIHd9JW/5xdfndaFjjcPcOtJ/3MIrsNhTZmCT6zkm2 + Of06RHJCPIs3ehve/ICyYjUq6tlRgKtLSWT/YcV/rF0HcLz2JOa2gmGHPlazLuYR - kh9ABaUw1JlPYi5+TpNPlfAHLBuXqqJfZTreZjthCbevnoAG7AAfFQPiOITN9dyu + 4xKm2xWzLAHTKI1JJi2C6eNla4yIKEuXtTK4qfSFCnL/L9UZuzR/jL6Eq4iRNcKW - rmbW5rNgeNNcR5vXfVHI8+M1/F4TF4W5HWGtt/HeibOSXrIRcKNQJButlhvb6yRW + 07nRRWppySr3G7nesoFgNxDvqDB0QpE5oTuLbSSiZxe+55tvWHfgECeqedpyN7Vt - Pzs0KiECgYEA0aZWvQDYRUeqUiMt6KdUpKdTLlzu4HNn6Be9yr2jztd5ka3mM9yv + b/CxNi0CgYEA7t4Ks+xEX+q8KIhEVsK3lncV1o1Rb/GLAmSqCqir5Aux9j6+WING - 0if3qJ9JpH+TAHrKGS40VREB49dFO7wVDB21mzTloDu+qKQSk25h6WuVB4udILuf + emwlv9rGQBgRXxCMDaYD3eBXXFgKCIKcu0OgA4racaEZ9zySSpqChyRJKNOick8c - tSPLBhQkStE1+02/gMMLIlk9c6ByTuLQoq0erKKNwIf8m0Dkhko6AXECgYEA+yYy + WXsnsTv6yQtN5r9iT/QuTpDXawc0v6O3fwAj7U3itjZdeYuakfb84nUCgYEAx2BI - BK1dOjdePo5dkRB12w/h+2whkoosspRuB6Hg8LY6SGVCXJM8cEgkL2CFNVoVcxGx + cWxKvcRYGhn0ClwjLw0ZzLj2vZpwKbu+kJiCFVyxHWgqAo/HtYBDGDAKLi9Xbu2C - hdKpsnYTd4wMbHnPNmEgeh1EN/5bjqkfrgU7O8pc0BrfOUUcgdywHMQAk4+w9DiA + 78Gdv1Xmgq7q9kLr91qH6hbIXr4fzC06R6kTjPmXwhtfKJcgagt8fNztKnf15wwF - 2IIAErc+VVUMITF5AjMvtsibG29eOGLbwCArwCECgYASpY3Pb7TMrKwcdB6QM9nW + fK0rZQ1SBtp4iYM5yr1L5LdkwAImry/jtOAdfosCgYBa7+F3Of2V3pGfhLEvrpWi - bz95vzBL7FfQj9QEpUtdiVK5v1LbSASnV4CykcBWDja/8yvog3CKJGIbprj0sCzb + DNgdhFN6oKRhVt19jVVTTjiEHMLug5uzQ7TjY3CSOhdy76PCFm/mamAX4dSABOmS - EAVoEZNe5hF2JGm2jTnOLhBqRGOsVqPE07MqDj6QHP2FJYwj4rUpz/AkSaABHjFa + SV6DursLA2AVRdQT3trOhDvt9RQlHIUYc9BaoxEItOsFa9sLwVRXyMCaGHY2dyeG - VrWEu0yKVE4GbQYmX5G7MQKBgEY0lybXj4gGkkHKaj1y7H8gIXu27muYVIZXF6rq + jBEhaMNkxzGy8jj4VOVVeQKBgGLsReVpCsiPU/tXZzuArcBZqrRmDZ7TstUyHwJV - hYbEaeZy5+oY/nwkrnjP8mzHkddoysct7GIGv8pbS93G7zW0UO/R3pAIem+Wt1Re + eS2qOQLTPQzaVAedJS0qINL7kFEsrWvSUDewIlgy+8fGOpgXJhqixTYk9Vf0FNeb - AgDkwK0r1dqchyuGFXT1FXQqZrzeTqY3MO4Ka1JPQ+TDf6Atzti5myJAL4ZznBpI + b2TiLkcUF6nnGiEjo9e8MjyBGtRRpaNPtJlF+64E1gu8vX73X2GTEP0n1BPWGhAT - 4/IBAoGBALQ5gDwHaz8C7tttgPULp680JKbkMk4xZpw2tYinR+VeavNSlBbF88zt + pu6zAoGBALYVsjIE+zj8c91ybrbNGLZbTg2KKr8fO0/07C2CvWH6HHN+lpyRTF9A - iX03spiC0knxlZcie718f73fTn7934PkFNKqOx0SuPv7A9xxcuZrGgMHERKICfD+ + QTGHviaFWoSLjgjmty9o0BMlvUtB1NNb2t2D1pNX15fWVYh3ALeFO9T7iR762Pdn - qmfXAT47ZsTzbiNto7efTdrLQ8xmSf0KNupXbpVrMs+b+qsQZ/Vv + KjfScNMgtEqigVRUEclyOlV+bH5S3LsMK4D7ripJSRvcQfsJ3LO1 -----END RSA PRIVATE KEY----- @@ -381,62 +381,62 @@ vector: segmentSize: 131072 total: 1 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:MDMF:egx7nc5bzqg3phfhootzyso27q:ompckuxqvlnwgdk77jkksfpxqu4gajv72bpzcgokcnwmx3nvpkla + expected: URI:MDMF:ftamrjwrnnk62rlturhbjwd7uu:b6qgy35eipo6kkzqem45uzxfl7prx25w7nevtqv76akv52y6fg5a format: kind: ssk params: format: mdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEogIBAAKCAQEApZGQcaVR+M18Rp/NV+nqLDdEzOAaCPT0c4vhuDmUC+5fmIgf + MIIEowIBAAKCAQEAmQmtP2klQmq9icuQMvprLk5aXgk2Nv5lPWs+6Qf9R9Tft3YB - Zcsnqj1ZphJqicCTht1ukFOhtF1gtJd6554SwoM/VR3IsmZf8OTHkSELXo+R90X0 + CboH0scSgfb1+dYfuH8FHhuKD341yVBj2uGqhYPZDiGodMoAnGp4ZElvFc4qLDvP - j3aqnlSDtrP1yacD39xVCExPtjRwE7VbV6RMUNjCB0PAm5mYeixMlXzBUd9bWS8U + Fv0jixXMKYykT0cxwvWlQjCu7bQwUABdh32ZugvGcg6Kl2ubE2HUPRfPNSOEQGx9 - GcCvF7KoJinE93Q9IOMcRmMMmOb6pJh2zN6gNlubN2Te5OCiA76lCxa+cuPvjDHt + DGHOogph8DNL0dm9xha1FggJLqFN8BnGk0nIvxA58oXG3xV8hsBJF5UV0BZcurFw - gmdxJhLFdljtoJL9cvOuKFpxqTe+Hdd30Mkn/9LAG6j+ZH2b5GPcvivs3bq68FzY + 98tohmKmAUObYCtUl67nQixFQHl2qUuxQo05EMwRUcjRsnH/ywUjzu1sgDs0Ee0k - 9XXdS1Vpd8XuxJKR5oN/MIpIn2peXWQ3KwzbVwIDAQABAoIBABEpUmiFM0bvvabw + OlMKPtT6go6Z+1iw7MEeQ0IS+rrRZmvwFhUCJQIDAQABAoIBAAnheBAUsn/RSHTc - X29yXoRwwh/eRrSQ91mWsTHQPgkyjxQXX/HEKftaWpV9KS/YFzKOdyxcjtFMMH7n + ccOjgMa37wRh7+ApGKbt2l1NU2sPMXU05z2WMenH9J1I2/ofew4nFVDWUlmhunNJ - iKTDXLxusDzZVnkvZVhpzkm7vBr0FLQluyC3sx2wMurYImzhc+RbSTEP/98p9kgE + qh4jB9F1q86fxF1AqugZGmDPOkmGysOZsPXjricivHsfm37gMomgi+T0I7cxNRVV - r1AZRpPGs+3e1vMJ66UWPGXuRXd/3YzKGFqQgw0BD3KpS5o8661e/lJ/i2O4Q2Si + O19YxVN1GIDws2iyB+HTz9I0oVTli+BIQAaEdOOO7cM1AJhiUilqXcmCXBxoAfoo - hRSYdm1ztZ9Q1F5WvjX/SKOXOGz9LHk8VoY9/KDsrYaEr8JVIiM0fnGf6FHqq33B + yEfchETAU25JgsirfqK+7QIbpcVB+00U9PKp1AM2ievPtXa6pmnuKZcSMjyEB9cj - /X49f//IXqeqI2ZFhb9BSYGRxwO0Xp3OaR5dIpq0c+qyPPfYuCCOF+8b5lToWty+ + sNMuc/A+DnkmqpHHLOXhQCDl/ETLtcnGxFhp6rTeN0TYf4cEEEi+fFcvRV4dPMqs - PPMO69kCgYEAwbQXI9WnPSW6Z8jmqHbXZet/cJscUlDBIxH7ANKVxOr4OHWe9w4s + ttF2PsECgYEA0brJPSqQUUyyKWbdLJ7LpvWx0w3j8SRb2mj5Wjzsv2rNCIrCnwZ3 - 9XI6Y5ZsAYN6QNBUVx3rxuDq4sAIiNTXCPRCs7/ChoIDPU/8R7oZLma46aDorUAE + E5ABYrvDGG2w8Mvs2Rx0v5jvvms4vB/QmxC/bjqOXaM7DY7+6oezVLhGgsg9sMN4 - fJfaLpsh7DBKXfNq9iKVQFYiK3imBZkpZ5EEvVNELz8tNbiEvk5JLhMCgYEA2tEW + 1neTcCmBKdQTTjzvd3QzFm/j6MkRITL7E6gLIspqRHY4+kCmAQo1R6kCgYEAus0F - FiczDFzEapU8Phkbiy9o4f5XRqbT6KXzgiVWzKzzpPhTPU3I8iVz4ETGBZ+SVAIx + cbqVoSYqM1M+GEI+8OAsTV4+xxVvx19FYgNjcRJ7aFfvTAPMtxOHxzLXcTWPS3rp - yWW1Gm00ThJfYdsJ6MytX6DYXQviG0MndF3gUSGywCFn504ecwuako5lN9TYBmEc + vTBQz5XPXLw4iKC0mbVyLxqBjS7AtU22vGRWiXi1Y9THM/G7P4bHxiaGcoBvX+nu - 429qNJLzdEvY4rjbivzjK/FA62gNOyTjU+Ghdi0CgYBFj5XLwZsoQ1c4lBX5I4xg + VzHCrDVVksT/2V8osreyaFp+tz3i1EoF5NTLRB0CgYACe7u2RbK/w7C4XMdxp8+x - xnxihOFb0jI5lOhtuDIeoD75j4vBru6ISjgbsVYiCQQrKGVRT6ZvKjBPs6Sc9sou + gmdAoIF6cXvE8klBkEcdXR5gY4Q6bdErIiFiEecVevcFYuTDDVs0iZMNJifd0mKd - JgGaKWADC8d8CjBP1c3bMvpus+E67kVuNN4eZIl/Fyxtps+finXMv+HPeKkuU31h + 82zQ2VCmOzCP0ImkLUcqUaREGCri1O2xXGkaguNMo343BvGu8GlKcri4IOlbA0eF - +tsX3kIbMXXb1+KbsONozwKBgD3FisukM0gJJDXGfWQ2aE0pjB1IVNEQJYBm4NBb + zA+Vsd/gP6YdEHbmvEA1QQKBgQCkzJ+S+ENtylfMtBHCIR+aLounLhBAXx8gS2LQ - xB4xsPJgW/dRbynUotqr748E1iU7HVzyhma4b2yeyShx0mFS7pqxaIMT6LezhH7Y + 16BxbxbEtV9+NpPyqB5PlQEQ9WmX87YmnGuO1+H7NGrDztPGD5fPCplkzuxgh1FB - RYwBzFlq1M86gWQO4YsAAdj6ECX04lfeSwged/Xbt5WBhBC/hU4RZDdQf3Oz3Sz/ + 31uhKIcOrfeYUhkaMHQq0m4msjyP57fH4TSX7O2z8Gyvfw5OrVWOTtggHU7ybuPI - 5DndAoGAGkThXjZdC745jhAJ8aX0NimCIfDuXR/h5HQL3F5/3CnPPduNxJVi/Q+E + QBub2QKBgDst8HtCy1kYdVFuDyG/CjReI6w3/NIeril7T3MixYGAC3zxfwFvq9aE - haKAhzzap6pFsi204fAcWCBMBHH9aCv/AlAN020G/3FDKY+CBKlrsvLAq3ca2xD1 + BGMEWmoIg2Vq3NGLE1qHrwCrVvfiHQOJDC1XBiBzH2sijIHO9xnWwWLSsOspMXMj - rKylwatgnJuSva3IpqexUVLbugxw2N2cF/Z41DBfivqchp4LEo0= + fX32WOapE8++CmasDj9ABJlSHbQkUn+iHT4ncBhce52sp3xE0kvf -----END RSA PRIVATE KEY----- @@ -462,62 +462,62 @@ vector: segmentSize: 131072 total: 1 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:SSK:gwtadrxpxdxeplb6rvdlyjoq5y:2uyxymbtbui3pnmyhovcs425lld2nzyrrcnw7k53cebmoxr44mha + expected: URI:SSK:gayk76ptouc275r3kxcb73gpxu:lsdwqdqlno2gkdldk6s4wlvbz6d7qviuotfg77t6fquhkx67go7a format: kind: ssk params: format: sdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEoAIBAAKCAQEA1ix1QJb6xHiPmHLiXiOf1FV0VlsIu8aasLnw9J+12mDn04u1 + MIIEpAIBAAKCAQEA2d5GQ9jkZXW0mqMtvm4qVD5nq5PR8Ni6DPhNk9+qpvtUTz6S - /ofusd01Sr+j5goC7cpxZhCF8jrGcxHcjQxgiX6AIsYkGTENQq02zu4gV8614efZ + 7QBFOkYz4Sbuzm7HYk40JWPDpTaBBUi2tB6sp/HUUQetqUeh4BSZz1VwB4pkcOrS - pWYJE4+ed+SaO6oKRRkS9rZIoJVUS3nrRHzfyUBuuqRRB16PPRIx7y37kFANGaTT + vr4pLeZyos65U/lB2RMUwcDKEqtbDGMKZjBY2ElYUpQWsPsVB94KH0/nTVFRJFVF - XzwGCrWGp/dk5kt8E8FpZD+AqLiRCwZcQlLSrVn5pfQIgKLbU0aoZEnRytwz1pJW + DrsuTdqwRSGoBXfKuL4MWWJQY7CUJ17ds6ixizAnRVAldHCWQvrqWIOaekLi2LAC - zDSHAY/rQ9weh+xX2LfSJDy9i3jxk3t7FbmUUmWyESpdV+KuPIQgtNWV3quKLphe + OFP6w+g+fQKzNj64mH9Z4sOHLHevrU19hITv2HyjvhGVV+YACYWHXFEchiLr8p08 - v4uR1DZQnJxUlnU4GQjDmCIAnBkPIg6PqojFOQIDAQABAoH/BGRlAy4VdnICf93b + CAWktc8xobZrAzeqVBV405TJP6FooROpwidqCwIDAQABAoIBAGBGZXV45GDjdUY6 - RpL0dCZMfHjhdPhds8IcbufXkuLp3iy/Trj67CrdLOtBMTaDWN0N9kngdVc/Opzj + Yq36n7DvjL9YyJauJvUxSpLUbWRxvq1wANxDWNQqDqXpnvDRKkGKPmfhYKTi43vS - KQFX+XneptfpZZrb6sIinZvEjghvMnLOw9WT2hLX7R2DDDYwf7pD6UtTsfdeAy48 + yI3q2jAy4LhX9MlP0rwjTl69Kg7i/ISbeDp67NaQNcs1H9d4V71Vvb/q8kDywP7F - 5OqqrDXmD23PO5d02IvG+mC3B/6Sq3aG7052H0+Axfm/DGz905ojcsWx+0qOMEDG + y2Qh7DjTnHiNYUOfCImb+IwisptKyUeER4b4kqgREhwBkNm3O7EcdDn0OK0zBwTr - HjR+nolrAePTXBm3kQpKt5x2B38itmG7crWKxWqARvHnPAXP59e9gqEZsFWo65WJ + Je8dnJ0tLRwszDUe+cSiAUY5fBM/UMq+Z5IJikk55FkLkvr+oam8Eq0s0oR4z4hm - LtiD3rdQIpJ5vRi3g12JULXohYzt+o1ydAYXIiaaC46jkgjn+07qDKkMJ0TXn1Sl + NHldRoFu20npfPM9SG7acViLWZIgD3YYXbF9LkBlk9X9ajNtSwE3QMtNHa8zko99 - yx4lAoGBAOfnA9IW3cn0axoiO6+xPMX00+zpvFbHtX1jN622LGjDDvfqXqKDYVbV + M7oKqEECgYEA+h94ts4x03c7cpetfvA+KM6O2NbWUmKo9ZQkO7q0orTo9FEB75nn - 4otQna1/phJyHgsLUIREo9brmRuUgulkZcNn3Q89R+GG6tmwPJdUZgwD3ZcmiEIw + NXZW//f6OZu3TZr+3yGA2F52rbtO97Rd8yIncR1ai6nd9THR9d9AGEmBZBM6tjvf - azE1r15ZPOeBwjtb0u4JxFU5POiHjrMimRqlC7uk8qr6H8mokXyNAoGBAOxt1FuZ + dVypg702tUGLU8NvvmFkkyUh9PD1paES33dr7LgbTH2KXxcaDts3Y0ECgYEA3vzJ - oD+PH73DusA42/8mjarIAkISbvxDbv364/iTbcKgYJbD6HAntCjno3LHLhcy2ITP + ORflPds+Px9Fb3vO6ZjAoLToGrhiR/gna8vm273Otl/WAXiDLHah+wCdlUgmGEm1 - 8K5dcRcd8RKVf4pUY1LbHVWmslj0emOcBZ0jfNHiIs9qyk5ROxdzG1DEZNIDCoSi + KkfckVuU2SGzfBY6YbQwEMAlZt1k0Z5DrYSydX6JWkJMqklJa7jZ+rGO/PsBatOo - GAnXES7JVnIGbcEcveCmTn7PV11EEl5u7h5dAoGAJdVmpivc20no/0Z+fldoFtOu + fLy+G+ysFfsqzPgbqEN6ZqvIQeNLNNgABU+11ksCgYBB+9VznFfGqpizNVJev3AW - j4RCmdXTIjXBq7GA5UaNdpzh+5l7k/MpFpl8YAXnTjMX+61I4YthP3sIa5t7ECC1 + gc9rYtmtaDucdZVNcIbAuasO8OPq2pYFI4/1/Ow1EGA+B9qe8I62Bc2XLWe6rwlt - CYA0bHwO8hhU0FcUS4wVafhnenVq3YGQu2KKzdW5PfvJeG2up+8n/M9txHH5Mfh/ + 35+6Fn1RhOF6EseJ7nhRL+sDhJMjig38PxK1H1B0ZrMjyNYMylKnAs+/d2XGaQS6 - Cf3LQD3U6VgNP5UkxzECgYAQTXZyJoK1P0I6DJAJByKsUlU8bHQzaB/9Bw1VOAKW + kR2WmEcTWbcMOOL18lzdQQKBgQCD/sr2Wui+Juu/3bjydy1SJbPQ7YV/W6oBxClB - NlxAKlzeqH2TljlHBMnxdSiJcvkZF1mKPGk65dakqGhV+oGqye8Y49iyZ5E04yJD + rB9p7/9PAYfisv2i8k6MEB834M98DRWKg6NTAA0qQsLGLzo737ecEsGRFHi7hJ23 - 9pl5w3URBlUS12kSsd41UIV/MbR89sxfiVPm/P0X+beBtGCnZ/BLsDJe/P2jQ1Tq + NxeUaWTQ4vIS0vL9Kx0NQtHLeqGqJMRVojw+t2heUqFRV4S3o8nuwLz4E53PGBVb - BQKBgGYpeqtu9yEEKRbJWnTP8VApwbkc+oiMbsBtNtmFFhUnSjq0GlIb3VGImEKj + D4Yp5QKBgQDxVnCHap5zYzzShVi00ygZcT+rBtldcrsj8Kjm4KE6qafB5M6R3gM3 - 4lP59fLmK3VcWYzlNOxmwze4hCGtsmEAWSXM6LeYRcE0mmUgwUtKihHfczMLs7bp + bFbG9DSE4hnuP0aJMrLEfmuU7X/nxuzU1JRXIzgFXOS2hoT33LbZBkYZrk/m9AcP - frPMT0b4+wM7pGi0aaTCbDtDvHBY8I/llKC3/ama2Vyr6gsy + QYt8oUu1Ad3WLADavwwU2ZeQiHERomK8b5FyfhG0x4cu5zdWiel//g== -----END RSA PRIVATE KEY----- @@ -531,62 +531,62 @@ vector: segmentSize: 131072 total: 1 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:MDMF:s24y72rli7tygwogwisvt6pkbq:kkqntx5umv5vpxgaehnvtzarai7ks3vgjzsbb3h222birkpml5pq + expected: URI:MDMF:ko4vxyhbyhfzkmucaou6leumvq:ysb7buuikg7nktiisrnm7je7nmdcqr5qxnfkzizhc565o4j2x5nq format: kind: ssk params: format: mdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEowIBAAKCAQEAy0L2RkAg0jvEueOYXbLzednZzQU3fhJqnIvjARAxJWtPDOsW + MIIEowIBAAKCAQEAneiyaICnfRuLwEd0VV/xNw+mB0KBEIJvVhmfYK43vq6VLy2w - 6wruLE+s5IHW69mNyCcnJ6Oq6fQ0Iv6pneE7xO95HxRkv7s98DAB3ZBDI/8Ca3c5 + qPOLPCx+bGv28IRU/0CKkFEN680Ww1Bdfv3DLUHisid3ISUMweEqQW3tKabOHgqZ - lZx9RAJ+qfbKZ6deUE1do65vijVtUszY+sYFFMo19MXs/Yo4uvpsH7xvcic+0hoD + ieOQIEFJ0dLFe6QnoMeiE+9bGdgtXi3bWzhV++1W+2uAdcjFGLpjHLozVstEGP7e - 3WS2P/iJDSj6CiIn8HIuWjA7ES0cNcxmBtqfaIDxm3TmSu4U9FmWbhgtDEN/inYp + +So1FQt02jTRp2GGgfOw6o4CKcwGuXmoud2dsl9j1dm2N9Ov7gJdz29I802wjmGI - WAQ81wlgqJAb7/qjAHB+QukoOj/4FMTHkzSmdD64lPzMLHKFOA52rFh2ExX82+fR + DZDSiWtgN90TU5480UENMqgMJ7jgLGQ3yJdQZDeDopQZ72RWpoXhuJWW7hy1lXwa - AV1CqAzpzZsplgPX+93fyXf5fGjkqiEA89Y0LQIDAQABAoIBABibn4GzR1X0dvVz + vga1vaxpy2XLfVS3fVsQycWAKs87g50Z35HoMQIDAQABAoIBAEiAzmHTKJP18N3R - VLG5VclBguETmduJQr93nxC0o2KOmogrmP91OA8EyV9zya+Ni+D4voCJy8odrscy + MSX+DlgUtEvDClWVPh+PVjFi7K99o3vtA58fa7+uQkHv9IsMh6ZHcRfAT58EJL57 - 0hmjWE9YF6+X0je1JUNEKKGokrxTpfkZOs8+XhsC+0876dbBOEWcDDNiDa5rl/Pv + COLFCzedd8QLANTUGR5wDyHJokosj5kVjtfUB5n3wDg6CXiyr4tP/igfD3o3WuVS - iXBY02ooLf6XjMDIQGSAp2DzOQHWf6knn8Dk8PUlspv6u0GScQ8EQPlBTDREMOhT + MtSYckpg+D4qZuoty+mFsvo+uFHR++gzkeS51+wvk+Psh5kVkTJ9eWD7WdNJbHP3 - eknqr5ALTLbCrj35E3s64AELAzuKKDTj5FZQfQRlS4Ir0EntBFU529FT0rwRbz4u + QKtrSWN5zqJYkSI6Jn7Ur1olKzFpT6QkIwHIAmyY9brwBkILdvPBtSTVRLX3sJaT - orltCzEr6s4EmdSuKpjaiBPy0rNte+eHOJv+H42y+nmBf/mFTq7Nl2fpAetDNSuA + XaX5zd3jlmGgvzBovGJc64jlSJWl5lhb0vA09wq6tVUTqlch0yBWsnIHxrZbT2bm - N5TrVrECgYEA1l7HJkg1gAxwm718YWoO7xpneaSKtYagjqPjuvTbMtDScSaPQMQ1 + /J3pRk0CgYEA0LlCapyHjXUD5GJ6wzXiY2EqzTCNhDaJHTpWrLCAa6jkZ2h4du9l - JjraJ1RXd9iw/FR2+XKqNEGSU9/E0mL75cEu8uBSGGe88pufVf4Mc8v/tbFzquyz + /u6uiVZSj7lIxy1UD/Skuiz5xWe0j+Oo2P5XnbzAA3mr5lEicqbo0+oP59mxfWJO - uYxEZ88cYhWLGTWCyG3RGb8JvUG1fkZ0wVCQ68a6RQD/XTZRtj2jlR0CgYEA8rvt + KQ7iiIikcf7s3V8+H07Tvgwc6XAQ9rSAWFxU/0zmSDskTRSFWcxT878CgYEAwaz4 - Yu8kOu7wMRQaCDZy2g74wjp3KnyfNnpFL1hFqojDcQ5eqkW41EHc9pOgOVmRcJm3 + 0IwB7NELsJiqEyjotGkeVxw7bN7F6/XjIU+TuMouyH768KqnatYBPvAPXSUNZtX0 - nZdCa70NarWaH+FwdzUqv2ovGfUDkmt+c0AY7LD4aGObdh0NYGWRazMgtl96Pu+T + 2Nqp/WvOGrL6wh7GCX0zj/Ro6OkoVwmDQlNPhsyl+h9yvTXO/SZOhUzlBoU/PkZE - e5xnpGcgvUvwuHt8qJfJljlN4zF/JiJFbZojPlECgYBiKy0P/ulhJlE7QN8AzUzh + 1ATL2H/Eo43iE/9EkREZX6ydQwdlHuQaHGiBYA8CgYEAryY3EKWnrlG6YWUuZS+L - ejoAnrVWw7wrFipnp1HqR27Xmkzn3/Jm+3SDpkAYBgemxhdlzHjdTVnxRvwfTG0G + eR+pviP3LTJiXw98ek9mhHFmsUvegtejvIjoF8FDaO3vn4xvFTCTJtPlCP1cbL57 - nh0d5FQ0EO2aPGIPQzP4o2cKkaTilVsIkY+R6mqZEDyO4s5tcrzbCX0wSjMPDLzS + CxRry6b/bisk0BHXmWRszp+El2d7ZJ8gvZ2LBU28yRhGBgINbFJGpx0dCdsLsSqI - +k4javJKP1ayHPn2duu+kQKBgQCgvXATDvf4CtiGN6CRhbUCz91Nibf2K7anNcrw + 5R0eClqqh4Rxkukced1XuZsCgYBGy4eYE9WQT0nKn8NrhYSqjdI9XWCLh1Mp0ZPY - 8kyYBJ8gE/r+WNNvw/nWU6ZLtBOK9FBSjKMQg44J9x6MNBbs6glX3rI4RzdJU+PV + 1VHWNnGrcF8iIf9YmimSbAXxsl2XvZXmvudsbz/DmrD2zHDvfwieEmVW3gOU7TFB - 4EFhJEQrpKKDUfPUvQ3SZnYoLwvd93q75bQAe8aDdHGBSU0gu/tjfqkkZVek4hcF + yVpEmAID0AMNDuI+vwXszBLbs1FO3jjCl4478VhbwL1nOeRCctGnm11Q5x8bj53L - 4IesMQKBgDBIwX5qcqtAuCB4QtSXzfAq4J6QDTMBkQvFRDCDDThum7F3ekfKIQan + zJeeGQKBgF82eley8ZEdCutxK1pAuP1FrLf2Xf3k43+sf4XviiONEZqn0GMUUAR5 - 9kIjeHnIUQ9OfbZ5etoMabNBJYni7uEQixD9s5E4vR26nEjXndgaqcG1MMIZoKEA + 1L6QOK6DolBs/8eQh0qWAM4Jzp1ab7k/ju0I4rXZtWPH1KChdGSh3GlF17KgWxTZ - GPsZBYwFh9lGrhsdHqW8L29s923b3RUrB7VOaUkm69cr+RRqpVdh + Fp7IgXxx2lqs9WCffy7BHqa26oASZlG83r9bbVgXVtgDyi1QhEQb -----END RSA PRIVATE KEY----- @@ -612,62 +612,62 @@ vector: segmentSize: 131072 total: 1 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:SSK:qcmjdosuiaagbmbgc7f3cmy7mu:zfrk6ez2pngtyrvj2mqgigkk7omanei54zeakwwrig56ysn5qngq + expected: URI:SSK:p2jakp7rmlimr7kdcyv7pmqk24:scsdwkdc7jmwgvz5cpttafschwome2dv444rkq4ra72wwth65tra format: kind: ssk params: format: sdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEpQIBAAKCAQEAmwWL2NS9gfKPBQbjci69pMPkDvtxHvv70JjC2RGvt1tcJxZu + MIIEpAIBAAKCAQEA6MkCkLlqIfmB8wLFgZ7c4ValETX48Zf4zJWOAFisLzvSOUJO - e3L4LvSFu8pZaGEffNFvM286giZuJLM9RuYeLGVk1ohDyRoF98+hZYGdv5Zsk2nn + lPHTSZYLA6zqMqZWYrvrIncrm1VkqXkQ00lKJih1GPXGXfTHZub8ZU6dUHySe59I - 88m9bBwi59P1U6b4Q9TcmpkALHc+nlKbJd3cI18ma8OE5lw8Zy1O/Il5K1U/5nf2 + r1QBOdwZSLczBVAx6B2LegL14IC4XW6cgq7aJrm1NVCAgTVaE7TgeKBgZtCOfj1D - Z5k+AZ6RKsEGQ269t4izjruSfjdVqNSYKmBb33y3xZoORW+w8o+d2564gdr9wXEW + 28IvBHxp0et6hJXMtNle5nbpg9VcHWWPcc/rbZeK+tTZkH8WAZXkyCJsDYjidr/A - vKcY128Hdj+M8G/Nn9tHliP2xleyjjL2OGow7HDk2qorQDaf6pCG35FjF+VkcGbi + hSGdgZVyoxdCB3lRBAdZKaojxbDMCKbT852KzlkAbvQTk9XaANbxaVknsNCzSfRi - jiN5Zy6f4LZG6lrWx5LjNqEMujCdrB5MQEhxGQIDAQABAoIBAAwQWc5bWu6VjeH8 + 0hIyFgRGV8D6pYzvPFTFAzLvBHNO4M2oxXDzewIDAQABAoIBAER4o72IHFTD/FpL - I9/fSZCckn9Ee8yHm1SDyNkL7m7gWBruHNW0UCJWUuB2+i98bG8CHOtlpunZPAXU + jWbUIKpTfxgx5PjDN9aNgwhNDNCT6wEWpOCgxQJXFQROv1CIps5B8ibgIL5+/q3u - 4YpGjvdRQjBMTSsloGyFLtgXhstRtVwg+CUxkFFNHL3KEcvLYQNlWbAZw0jSlE/N + w2kynJHewprF6ERItMJq3QC7gABls/yS3KFdt0KaoAFIicRdU2CwGA+agVI46oHp - maeQHQkvJs4o2mszZxt37A9M0v5HTlUucI5cxdMbi3H/1KnRkCuydHZ1HVCduo99 + ADZcUiSj6U42UKYw2D8FrCUvH0v+Kt6I8W4iIeuy6P2TFmp5/M/AdzVxz9tZPeHY - Yb/Dbf+u6tJ/rlYJ8NNPxjTHCM4xsSr7VYJWDD2nAgwHz59jprML4tk6InR+5xFq + s74aKp9yBnQJfoJCgLt7RF8gBeGLPyJVPesf5gt5GKHHPryujWOEnOZkesHl6uyW - wA77gui2SyDsexKtvci9RLSJYArj0AY0PPlcQmUksrNr5QzRZgYI/cL7GqBXWbh/ + iHI+x9k7yrypYwQVIH1wxklUFMzqmKF2qyFcgLF9TIAsDZT+0qJhc80nWeeIQ4Yr - GRnNpJECgYEA2mlIGKG7oqvZ1x0RAxjtttOhbM0bbSTYlBEpNzK4lUnEixFYgFpQ + Z3lL6AkCgYEA8oqywuSnDHkVJHxhH33XF4+VsjtHnPjh2K1yPTBBiEzOdD2WyVgL - Thf1UHwdVSM0U1Cu/tVZG4NVO2XDWzymJOb3KWUobBE4bLMfqYRI9GigITwkjTLw + ZmXXQmh4/fbc+ioHUK3GKyoqW7XK7wSLwnBDLZ5qGybKquvDFQy+3+jOYfz5Ux/v - 8lPGT4al9qqooE1WsryGzL/R7oqjoX4JAyTXEYHPT0XO5h5jCGMoogkCgYEAtbN1 + m/efDP6gAyLRc08hfEnnPf9u3scgW1J1klN/jk1YYW48GzRqnYvpQ5cCgYEA9bO4 - gyS8eZ3KxcjIAOzD+Y6Fnt2BQywN9wuXUF7nSljP7DuprEzmnCuH2TUA32VglPX8 + MAx6vMt7ZO8tNZGs/Pe5KfDaDNlyzMTKAzBpr0thHtbY3KhBtpE9eWZBvTeNIORE - 3aADNtXiyT7KbXyii+tWwXT6h16Jc1qZJtUd7DXz/xIFZRKbwgG6No3TxYJe0kdN + hmmZjiiGRcEQWDfbxiPeTLsUdbK+YmrkBi3PFrsMzcvDsmVsFag0MNQ95uiaDGmW - pDWdECuAagdH7FTcxiSTDwB9DlPZr5SRLZhB2pECgYEA0ng2A5aHLEESkRrvc96n + xhMzprSQQIPpOlDQo17x19e+EQjVbuWSZF1T+70CgYAPS6nsWokiYzxvGZzyZHg9 - 5FCX9DLKxSiGlFjdMNXtzd7iSWkTscxWKosn2MFhutNL7yWHHQcW8U5j6fMsiFUv + FyQEonJottVfWcjbjQCE/PsEH6IzvmaxpXiGypnewkO7Tw05DEx0CmuzbuqGWk4K - fcwcTYWvqEQH7afHUSGq8+uGs8AzMOXwDnTwW15TvBnEmYUtkNvfwprugEKVYGAF + DBRgex4L6k1brBSYbj5XVpI1YcPDdz8gIeSY6DHlILv+vp3I/cwkf8hZKkujFrct - 60OrBLHkxm1s7ZBGuqRjWZECgYEAmVU1IJN6vcKr4FZ8eVNUWh+soRDZyV6+9jCA + bDCeI8iQFGib+plCqEEkrwKBgQCQYjwqgCrgJvMsLSD0CdcOEMTO0KpSQrYjfsD3 - 46EC29mwtPDwUWef4EBX5rN05hB9/ZbMahZjP/4k4KEtYFGiNiNGVgEqfdwIcCEP + fsucJz+7T6XAhV+YWWE2pdCb0LkuSvW5xvRlhYriEsH4FVsg5JMNpCIxyAf47bdx - RxbnpnMtUZ3akZ2vdXvRscHj6TQIYrkrSxy3S3L6bf9w8X33xPoOY8WMwu99r07X + qhm30dOEW+l0PAV39JA24YI+3xEnmiTv4PjJTfI2901m53azimez3yPh/r7tnBqa - aLupLxECgYEA1Cg34GSuNWC9M/YIkH+l8oASMg15NY032PWQLmGwLvp2M0hLGYzP + v/1KQQKBgQDUdKGfCOGGLV8Mzk41foCzT31r72bO6niNyqlGrx0gn9GHMN5/+4oF - LzKpKs4tgJPcHFcgLnqMpdWorTJIw3TtNecTo8y5H3J4rhZIH5kMzayF5sHB0QFu + BJExrdUPi5w6FsbmYXqVUPfmKQER6QBitALbH1ASMCg+scoYZq5ic4J1gU7RWB1r - xfy3oKGpIvekUkf8l7QWhq7tBaJav8Zd10MrIL6eFtbmZG5EED2yE9k= + F1J+puZLYmg2Yfcy6l8mHjEez3jNCzkZ72xFRmo++UB5VGkYXcYqJA== -----END RSA PRIVATE KEY----- @@ -681,62 +681,62 @@ vector: segmentSize: 131072 total: 1 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:MDMF:dl35z7rympvclushgyujvlhvhe:6iutrwlkcpm52fyc5vatkw7z2if26xoiatt2ocyrl7awjshzewsa + expected: URI:MDMF:rn2dx65oniewqh2lzqfcosqr4y:op6fupeh5cchglxtz2qmxukfvpkltcciv4emt65j56irkloppijq format: kind: ssk params: format: mdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEowIBAAKCAQEAuTKFAzPfEZKOCb/DE2xRMyp/OrDXicd1KUAENBHOlCOv2AMG + MIIEpAIBAAKCAQEArrgsDZFLXR+LmZPnzNSaqgxy6pv8euDq8iTLTUmF2HWiBc97 - 9KJF1Cpbw9xJ9w1QJUmkcrT8r+2QFP1ECncmvwUExOTq1wN1Fq7aAU5HxdJKZQGj + MH2Tl1V41pSVY5a2bOoRN38LPs2Z0bP6dKDgaKNayV8ykWvhzlJpRMLP+K7XYATB - vx1R7aEgvECFyiAAT6dWYeXNb59n541mtUP+zUp5X/DH2UD1s0MO9NNQE0ZD7Gmj + qyVE7n34TytxWdHM+gHmlf4laTbQ56Ts8yZ1w4eINMaa/0OAFgbXBJZTElrtYlAA - od3n+AXYaqg5Cle+qNwjGlujlJQepis7CfZsf6h6L4FPNbToVFXzYrA3aYPBiCmU + 3ljAU6HDDPVoOsXtOsNXTsZbMssadi/5ESEn/EzQw3muAvDxLV6adEXk4/1HiY2j - w5TOlxlZS7JhSg6dTY6poYBbVFBPoY8fZEnOTfFFA+Y/ope1Gwsw/Q1J5TItN29J + 9EFLXUbKFWpfEdw13Goj385ok7/iLNR3ExCNbc6RefNeZYIY4uJGkWAKV68HyJUW - 8mUgdfi5P9ILeoKlpGdaGLOsRraQkYSONjhYrQIDAQABAoIBAEaQyNzpCWCtOoDd + HCag1gn0uoiEfc/1ichkvnJUytj0iUmK8q186wIDAQABAoIBAERJOv56gOf+gkMd - eAuxFJmN4k+vLVl6zhorIc7jUBbjKDADK0XQhRHsF+4fxHElufmTP11TuAqi1ukg + 6YTGu0Qm5WsIWB92dJz7AHJOf/9fyllXBCCpk9ubzkSfgduQdfAPc3crMivzk2DA - faoNL47ObzxEy3SlBRrhAgFIXhGy6JTnFIkQN3T3lb0VSsUy/1tadBA2W1piX1l7 + ZOzcS5jZ74uwp0Tq0zAeSYJUWZAqVYbzlPXc9Rn5JelbY0vlkaVMxOBumhSLFg7T - 5/w+jdqUO35ChSuzVEt7TDoeQF8vGxCxrHFogu+AsPb8EGAbxJT89r07fwnFvQ4G + CMDnkTNAb12hGccWjKMPPU5PqlwLgrUkwqEYrnNAXUXg6YTNaulW3BYi/gKcqyJ8 - wLpzedeyooz92cp5USYRTOZwYVjLj/B6Kk9omCZe+yjPru2HEhE1wTY5/ftqsZ2h + zinOFLL3gcQ5wR3qX41YLe1fwGPplC9TFuB9xyoJSMN0PJkhEGm4GAFdL6mjJGwW - YLKfXNI++xmwU1MRYVFAR14hHEuGgNwuogGwizLA8brh2Itw4hze8YzcX+N3UfkA + 1HHmEfSeHkvkAlzrDRVR/7e0xSL/nekJPY8wORLQT+fRfF+bhSCVeB5BKOFBIkIC - nkOvX0MCgYEA6DboXuzoS2vSd3i9WNtzk6susX8CsJ8jnQ3QcLrtTNrGsBNR2KwJ + PVdGNTECgYEAt8vEqMi6f+rm+atxOEScOpPyFoU6RDYIgmM/hGjiXp/UCX743fIj - IWrZp6eqr8LzM6bw5ipO9j1HUsGbXYm1djx/XCa+Fb7e7LtsQo+/sv7P9iLhZ96x + pujXFMbx9jv4LnnEjsd2wcx4SsncA/x9Uezk70+Y+RzpqzqiG7YNDDIQU6QJHh4p - WqI553w1653/gUfgzvIriSP4PhfupJxqwP4yGzm8AwmcDOqBb8thO0MCgYEAzCq8 + Bs4v6XOIdhzeWHaNav5ydb361Sm6Ppk0wxfT49gnxC2PBPd97/10++cCgYEA81uU - VQLnSVBmvB8nGxZslc5F817kDnoFjqnPY+2cnJMQlf09379c9KVhBDO7eqETaNMb + 9gocgdBignouVx99rL9OaQ4orsSvG3NY30C4LNYabDM3qMpYdZY6NjR6J1lDZyFD - 4IDQM4ebtOhItQ6n4SRi0u5ZhJdwsFouItgYoE7pZvFtcM8Cs1O8dvhwDCPkqE9V + X7LKlK5yKOUfQYs1qFIFelm/IT0ey1pEa/JtRLg23c7bGMqAh27WLHxyC86BwkiV - QO1I0jVFIwTKddETDcbC+ykbv+WEIgWzcfHqRU8CgYEA05xFwUtOnHxTPUAv/Gtv + Z6srKR/rwWVV6rtersvMzFdREetn7J6ksW4y9l0CgYEArlEfwukmIR5cVJ2qszA3 - NWBHmsRNZTqAL7zI+BG/8ctkSEwyx6puX5+JXPiz2JtlGOrGmFhxwH8zIb0Aogq9 + cENTDtLq8TjCF1AkNOP5Fr/Frf/z7ySxdaNOIpGRePVlrEanCrfZRXM0/9G1zz4Q - 7FNRFF7R1essJrrc+wMYBDuks34xvn/3SsqOzd4pHN/MWLlxqeSRu9WlgKA6fpNz + abwhYWt/7Xzjjhf9GgUvGMr+uyVP1HXMeXzi6io+Wa+FnidKdxi+3DcECFocHzTV - zQ9YBetk47e8FyEUdxX1MxUCgYBgaLHEJvnWedv5a3CI7v7Zgq0vbhic6WvkYTVo + Wtdliqo/BQwkohRNKGmeIy8CgYAoNCoQ16zL+Ww82AiN1iMCBfzjODaaYN347/5v - h5STrzJ+0TW9iVy4vbthQ5h9IMDMmBuq2Mj3/Eo/lAx5SvFldEwiNKEa5nQ1InB0 + q5aBucFVvMRmOz+P9YiaaeMAWyvaftFNnxD+rS8o+GlIf8IWk65Z/zenOxy6Rahm - zbxbPsgib1Dxmx84VQtC1q/6W5ynCcdFQIdJlBQQpDuChPbNY5VBCrlq1VOeyThi + GP/aSYCu9jyWBOZk+Xeik9CXiL9BJJKiNNIFkkN1iNM/20KSKBMmcwpupnBd0/ur - Tw0EKwKBgAz8tUCZTLqbZ6wfiYdy65OJlW0j0npyYadU+SZdzvqhKSG5+WjgXHfQ + YGRE9QKBgQC2bvTY+NYnwB5fVkyswl/Luy2B20Z2RCBWXazFfVQCFGQ+s5cMEUsU - wPiI7a1Ak4+r4X0QOhopFlXRATDYc5Rl8rJwe1v+5MV/V+MxavjCSiaxT8tULlq5 + rYVRM93RDxod6fMgXRPKYs6phgByLVi1bMRqJvFTx9/9/dtkjRlX0S8WETLHEyhY - HNyDuTFkGx+B1jxvZYRCPmEk52JUGGemMZ2RKcQYQcmJZ80X7Ugf + VHf/xJdGHph+8wBa8qt7vVoyX44ylAVWgXvmf2PTL6qvLKu6OLbUmg== -----END RSA PRIVATE KEY----- @@ -762,62 +762,62 @@ vector: segmentSize: 131072 total: 1 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:SSK:qcis3fvjbbmzunzit5svs7imbu:jfqtuto6lbeqgt4fwh6gkmb4uwm6fkff5ave7quak7rxjsfs7aja + expected: URI:SSK:k2opn2p3sjpp66iipfj5x7ylka:dcrrkpbqjs4vv4r6hrfnftza47zkhsrfbmkavwx5gadbphm4gyoq format: kind: ssk params: format: sdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEpQIBAAKCAQEA2yEB38vpu/vDnChg+kmW2EocnNln41P69nIF9/6CECDpP3gl + MIIEpAIBAAKCAQEAwoi+YH41KOcNqUaRWqZGXqPyG8Zs6a8fjTu0NzfiiPPXiBhe - qmRWWHANi4e5nlfa8nS+vOu5t8fhd+kDfxA6vM1I9eZN6pSezksyK70XCeCrGW+N + fCd5fHKHYXTr6W/YCqE4c1E9i2JDNtFULRrnLyjRwhnvaeWizeNN6A86aupuNN1c - OYeVzg6xMAzX2IhRWr2fFYemvtPd9PFx1aWoU7i42jkde3or9Iu/DaIi4qpQQwfS + Kt0XxDPDEgR8kxCh//kIgOALQs/QxRK8WMr+NfsV5xWLDXzPrfftKWvTmRaGRuPd - A+5Oy8m1ZkDKqE3g9f6f7j9pAsTIKlkOzq48iuDdwlB2ZQlFf+clT7B32So3Al5M + iuvVzFfpfFyQXHRZoSr5uxTIEiEjJWTEvWv7Qxam0mQp0FeGGhcPu8g2zDbrG5uq - X9nEP1FrlqzSXkrakTL45OB8x43Uj66zEo0Bz0mkkawI1GWFGkvJ+Mb0kYLheAAA + 3VEn4X6AtFwQOFkvQV9E8oFtFm9mtI3G8Zx0dVQw3OMM4/2vIrloQ2PwzymEPvx8 - Rq9TxdQrJztnPCJhEZ+LgZCONXEvzETn7/U1iQIDAQABAoIBABTqmKgzvv1/nFNa + Vnec7pY5AtWTssnihLLn9ZJCpcVr78gtlTAvJQIDAQABAoIBAEUSlLZiWbHKSJmA - k1nbY7evk1LxeaDRvQHV2SCr6DRWlTfGuymsPDBi5G1uKNdRdt0agbO8zdRvaDku + SwAq5fWYtNCT5fwbiFJ3jofEuhTyr+bM8E+ZHJPJfrRl9ZYPdmBf2lFn2ThyjXcb - 7R4tfr3Tho8E9b5aLgfDq71QZlOTYMxOa+zoNpzIUDzQQluGijrJdixIRNqwPzPd + YI4bbVbbYY9P8ihhtyrvuKvbLFzmHHd82csGcffigTMWkL0PPNZMsG5CHv18GJ7B - 8XS9maIBiZDi+h/K20k+JvV2hYfxy/lrBy8kWJjsASPkKiuN/i2kvqT32kxw9RU4 + Bkto2FaGbsJ8bcE+PeeBjp4UgS9rp1QLnFu5CFVEv9gju+MUDNWxWptBfLxlKAaZ - RHz8F8dzsHPbT0Kbk3VwJA9tSYdMLOniHlmETuQ358fd01xJMy6KvLzi/Tl9s3lO + Wu1yGuDa3nWY/JhP7vd/Gtum5aDqsPmI+8JCma5EjXgdJkG94bBBFPgKdkOClh+L - /J6zmYkuJ56XgErcxRZni31KsMuYUPxkvAfyjeoK7nmJNVtY0A3hPNspSQRFKPww + 5PSnN53RwuQwbKdFq1Py3CNz3WcrB2jq+W8EaM4NQX2fSp1SRoVECuk7NqxuM5FD - 7FWSzlUCgYEA8quXOaHqxbGa5kR195F48DPt/Um+jP4C1WNskYS47YQGuhqrpYPJ + q7oH9MkCgYEA078uAEnM2YovbquX602S4vnoZTtfLhqFrisrJwZfaEj9HK6mRJAi - 3LjaZWr+nfSPaV+xwGePNDTSC52v1Q3+DdXhYAmnvCj6U7FMMkcUJHFbQrefdta1 + xsWMK9a3ZmI9v4PnWzYiNjWr9SJ3RP1AmaCuIjncjI1iN+sJfGsGXBLYtVzRuuvy - YUZd6EPiLvDNDVeYcLL4kCJnznsWI9e6gp6t/cI6+lq9tBwvwzm+nusCgYEA5yph + hPkSN+kIcM1bJ6vALz1XMPiNnJEbtQOyFLNYuITqFE1oKG8UTZK/+6sCgYEA6zCo - VWmTlMgYG6YAblk3OI08lp99jCOB7GnBWuBFg3s0OmY3w+kjjT1AYzT/nr/Fxv1p + ao/w462g+cK2kBSrB7J6tJJpGTs+jrN+W3p8le/OKpPI9e+8nwlnJ3TVThhAkeFN - kt+deClSILqR1JdAQhhy7JYZtvSj3D/Gq/C3dBFSkdZf71k3ZGWkt0M60M061OAE + rUhIRQeG0LHVrIyodzJXLPZNMYbzr5EaGeTdjoFQl3GvfMoN/xX7D87F5SQ8QhWp - WKvnFg9X4R7oBQs9z6JECaxCKL//ndgU51WaKFsCgYEAs4uDew+yrYyHqAFVKtPG + oMvKEEVwtdj5vaEcjurefCCFrpokc/r1FGxGMG8CgYBTxnrjwE8c7nAvwBIeFZX8 - ICq71eB/DMBPhmRmipAhZxJ9C6r5/p8wdo+KfukX8/RjOzqjQFEe4iiGlDOaSc9t + 2VUZ4DCbJAEp1IiBKyNKNj52P58m10P7EqOPoKb1Cf9WK1C4pKVKf1emZ9l6YRxS - ff0WIEFkilHjTJLsZnKyk3gPZqCHapzXXF580oGPUt21ST7bOd8hCzt5hIsLSX+u + 6+CZR/07WqC2cPZS1GEywn4c2zlbVAiilYygtEETqBvdiTVDO2ioxl34yOyGZIzr - rkALSaowitUicKU+LXqG7/sCgYEAtQTC45ehMcje2AfOHptOWsJ+x5RtQ+gqPW8z + Zb2/W07lc263OKN2wY3VewKBgQCNq2i3j+8l5m/iIvT/g0Omxk79uHfQeAxtvxdt - Mm6dALDh3TleQdO3O0rTuNwvr6iMv56BpbnmHcp9vZNbzxYCA8ARfqKr0FESX86x + GTI0yxfgM9dItdlj3zEg5lKa0ScL+LBmofTOiAMgcQ7p+mx8KHm0nsTPAaCGcBxN - TMNbZVCLUBiHV26NqdjOe5Px4sBTaY9i1+0FMIkjT+5b0ldTN9zhWpHB3Rc8m+Yx + 3rvK3IBkSVnRDJEzx9TMp9wy9AnMMOpV7ovQE1QaZhHBtWvTdwz/rkN4cmdk6ZV5 - uFWYOjECgYEA7uVh7iyGnHQzlEqrdGmVhLulCe9gYvKHUxX9RrcEDIIP3hC8A2HB + cOMyxwKBgQC/EScZHM1YPWvUatfF9oQrzsYBYTXk14RQNvuW6MAAK8C3E2REnypm - YBqF50ODzPo2G52GOXL6CjENZ9PQXTG9o2nQtB1k7us6Dxsh9T5yxp9fEAV0R0om + iZDeMOyYzxmJ6r3sT0YIOmRdsaSWof7Rt7x9aeMmG69KDJL0jUsUhE7aK7W2H2yB - rMHlu5JSGjo/eEGPTQhKYXkJGBNZzBqMBgQay4vdmS5vnBVP8brpIao= + Ax8MWdGte4MuzMRXpjZh6R3P1ZZ7xI9/O8SeKVnZB5u8G8zxvY1XUQ== -----END RSA PRIVATE KEY----- @@ -831,62 +831,62 @@ vector: segmentSize: 131072 total: 1 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:MDMF:5nwmwaa75p3mjf6yzi7ejaecg4:uxl3eoj3fagsubkfjrhx72s6k2kbunkimuee5gt7fqtukzn6anxa + expected: URI:MDMF:2wd77rwew4gfhioljcisf6jr3y:4nzqf6vq7w3f5pw5yv5reh4isfsxmjpkd5dvc3swrcwm7zz7m24a format: kind: ssk params: format: mdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEpAIBAAKCAQEAhXkJkX7At8/h4BgBfLWyC/RKG9CuTb61kYLzwE9AmTRUAjnU + MIIEpAIBAAKCAQEArEKiDSb3JVVLiePghhefdt0bfKlhe0LHgxxnpBjkRK9SNScI - 7wnv4DvHj2ZGwCIrEWL+4KkzNcBWH91gQuIfl0cDFOhrLljGrrTzWoFMzhkWQm1p + iwNB+PdJZHJcUSCrGv1l4dpEEhXpykuhb9ckeySFYh93rtS2c9Z3pKYJ74SzkTIh - FUevha0DvD76uWjuEYcS1OMSIsk5hxdK8tbQSLqk9pVgfA8sa4Nj7lpYrdNSoyJR + T3x2B99H8uIeKEtbs+3wt4N5gwedYIfhxfhIC+AZcMD+BF7JyAermj9p81kqfRgv - +DalDZyvf5/SbKOnu3ETiUhg8CbyG8AHy3GLyRE5YmnwdloPzXuDR1Xrr5VmFnn6 + n00kRQpozitb8Vs0upKX+T7n2F+1JDrisJjTT3yDxnzwQ9fCGXa7rBOb80tNbrqV - 7+3pvDD/STUNIGg1bb/IgQ5TlPcZp6rGk4sHqQHa5YJ82jA6Zf4xO45jUZZsG73s + M+LpObdZ9Z6TKZbTKoVK5KFz5zSCiIx5usnTJ7MeFKzZQbPArsH+YkcAaSA0ctjH - huVzGtnBBUFl7VXW7f6Cl95vk/V0tZEVTr/LQQIDAQABAoIBAAZJNYSnKg9eGH6z + yIIeTXuCA+oyO1ep1YLRZ8YDqW3/GBSQ1sbqLwIDAQABAoIBAAEOKXL+LZ5vbtaJ - 0robWofKmQTnVpYtwaJZPv6THPE6MCysqZUabDQszJC52eIpmcqnVWaiQVmqNcQp + 7Fq181allSEG4p+AMbu2ZVYmhuN61XyY4B8F5pYRYcQ5RSKEGjhAnCqtjSQ1OSjx - amOr/53hx8jfy1By+OR4fC+KgGICd3Rob7cDWcZbaB4g/zDlOrUTnfTtvshpnq54 + nKwUO9d7SkvnGHMcP3/nLPV3xPDQNc9qaLTVdN2oewbcZsXlbErmZsahChII7mhk - j9yQ9l0+gQ9l6JXfJxHnLbkngx6okEUbHKab9KCk1iclggsydveoNS/4vDpGeZNw + fLgbRMm51j4QGJ5LPF462czInawNebeJxMfuX1ycZsCd5b/hCxhhn82CnA1LeGWQ - G3BySsD+tUR+XW+oRe8xLU62rAT5HSTLjGtlUWMtJ4POoGeRPNiohNHeVcucCdQc + dl+87lyCJvTb9gTfKxzKI6X94N46zYz86f6KgaMDPazZz1cbuRhMJsRpqFWltYEM - jSnuQe2LalAHMa9bpII2qSAUbKVbEy+RT8DsTj5AKxhiN0F0HdxWGAFKlxNHcaxd + UneIEKEIm/56RtLnlAKZOIAI5AJGpDoMWYzxzlD3PwXyFfP6PigB0SDl/nAGA2Gt - x5pWA3kCgYEAvH5OoTBgSgEGiJoHB38kM5xNoOs7W/OWCurqpnJXSpADC+biu5vh + wdoSzC0CgYEA2ajpn6JHoRnYfgi1gXvkWnU6jZMfYNeGO8kjSvTRok240eVMiMR2 - 5pqlBXqBTrmDddlctI1JQXXcYPHhE0O40m7exA1Tzj+tgoLozgbejEHKNINrCs4o + l9dOe7qRglNuksW6gCv5XlF4RtFJnugf98PQ7Hq0yGvbdkA9P4+jnwuJZ13G6lyy - 23kKrnmya+JTZN7scMzjP3v52SRw/W22Sr293TCfSL7WRmRgpnZq/z0CgYEAtUZD + 75cFDXpx34SMSfWMnZvHO7cQuPe6bTSj16LGFiWWVIWjSpJo6++zwOUCgYEAypp8 - JKHMtvthC2cuHRsoGik+KvtzJVVBaml3tSgF9XkhkHT2+fZ8gf2fFhcl6xSU5Nd+ + yO1NYBxcuVhVa6hJR6s4ri3AKkrMR+tIFyClBBgSdkGFSK4f0fnyaU3xxWQjAZFu - oYBmTZQLKFwLQotj6rJ1kWvLeAaMR+FV/87zaLdaKmPKlQtL7uywN4JSWOjWFKmO + ZlrNDj4VMQ0RkIWl6RjCWt2yzv5ZRS88Ft/kG9FpI18N6QpdpKqPSgILTCThYj0r - JjxOKZ8W3hYz7UY88fZHqvhN1+yXvagIf8Ft/FUCgYEAtTvolFk0K9OCmbMnURDx + Q89pbHTnIEcYj77Yt5P9XMFPq02WKbhFm3JMEYMCgYAgaHQQnrsHk6+WZHmSA/5m - GOKPTUr/vvCdco/e3/0OazW+iCIOHP7LnHNSecsJK0151cURutQh/Fu7ckb+9wvl + MZo8RjGf27dS44nruTQVdHkWV4vjuUznItm+tnK/8ug91k8EkoeYsYy1Dqhljq5j - WAecDvsVejiFtvfxqa63KjpTllxJfpEsfaGLIKkIYWyybElfIzIMycyFNUAxl6p2 + 17Yd389XICgXHU2BT0PPhIo6582cS50g612HOjoGS9gPkw4S0YUCsSk+QTRy2imj - XLTFKjiG3mYHFpWKzGMNi60CgYAGt/TzHaAVxBljr85Qu9nvpkmslCc/YfqLtB8A + C8tutL4Da7p6ef5BUvlyuQKBgQChJoBa0WXaLLUUN8658wFWoDpVUM6o1RXnnp2Y - stwNrhClZwBkYVNaCglkazU3kkq2dJo36CdihrMnKsosDDiG0Vh0LFedOjjmzR4/ + x755oywMI9GAHf/xZH4MhJLqDtxJQwQtJcw0p+zzNxHhgmyVG7x7yhuPyX/4J7oD - 3e69mdYYrhwrDAEjeNhLJmRg8ThGCca/+go3lrLlRlNkXu8RVLxxRMS11QoGuHyg + 99HYzphyKglGc5hIgeG3XCjgR/V9zmm7Zh7UxaeRZPuEWqjGXFAKlzhnaS8nwAqd - J44pBQKBgQCptTkB7sPmteFe1Yuc3TBUiuV+QiVuW4gMGulaj20jGebegZcOPYDu + uiwHQQKBgQDCqKXQa8ivL9IDscqW1UNK3HPf3JU9GNy1OkcIhz9HotoooXB4ZASS - xvrUThzeoJPJEiv5QQNQIIdnmJAcaTtPOh26ivMKDBlOlaP5cIjLhzC0fObJ6BRe + iV6B4tKev2scNA4KKU5KvY0a893ghsH9yEDeN5MJP/cGCceh9+IhYMbDqmRmIv3W - mZxl3hyrfzE0i0A2Xr7Izw+xEViw903r3qYtN0/rjP/DbIUPbwIX/g== + 1uWtjpmyRGk/rFBDf6jxWcXKEVXAvXq8IQS8AJ3JaatJzwuasiIDJg== -----END RSA PRIVATE KEY----- @@ -912,62 +912,62 @@ vector: segmentSize: 131072 total: 1 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:SSK:yxecfym6ggu2pxfyxitnxuzyxi:tphao7q3qc2rd3jc7fsflqgxljohmiqwcvkbfp5vzdxit56mcyda + expected: URI:SSK:eojgidosziluemkir3zkxtkwjq:yvdxyopvbqryaeslumwlc7x2xuw2y2wcqenys4rws5xslaox5nra format: kind: ssk params: format: sdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEowIBAAKCAQEAsNY9rFTsXaFGu9ou853zP7BO/Hxej3pTn9am3vLHBDh6oXiK + MIIEpAIBAAKCAQEAo/r4ebWzsWZmdDKCYXv+1Fh1/0xyPWR8do4BgQjDG5vsJFX7 - XA4xCLGVUl2pRwhvJyZrNTtyfBQKOEjYHucrgAhZKpCN6Yob3B3oy09DEAhU1bIl + gLwo+/v5JuvE3k+aTMDp/VNFtgvSd4t8WT1QTYJvVQakqcIL6mZVVg7ZBc8JCvHm - VTaRVFthaQqU2woNpyK6I9XqAWhh/xybcaNs2xexSS22lnSoL8VnmU21Yqtq/n9c + D3wtc2my4nqN1VLw2iUduLmFHY9SPMxRVHafVWNaD8MAhDpEhA2Daz442uG73/70 - UXAHIKae0D+rK1xiWeSLSjBwy/EHz81MrfKz7zoLCt3qEBsCQGC9e5S7CXYrIgjU + 6x2LeOqscoTK7/NREfUbeTK9Xz5mFMnyGSWppM22+mDf9/3nCTTbfq8WLafMcLW/ - ZnKH2ZmPpHfWA9R1BS39S/q3iMrJgTsIPz3R38pSfpCbxhGqiTN5IZLHAHxrpEyr + 4X74uJMRE4uYzUzwiC0HyDZnsNAuU4ggjzljTryCK6emynSNHGAX1dkBccvd2Q0w - ClRpAN4qpfFnsrz5RBbBn2mQNR5CSXvzv8aTOQIDAQABAoIBACO2XJWnP+Xnylxp + g2nNL5rbsonjbaYHZ6BLWRUGbh0PMm9vZcL0CQIDAQABAoIBAAJMkyYKtgalWcek - I3bFFQktbsIsTr98lZNP2vrm/jyyuEdQS+br4ciu7mhILIXBJQt7xYZmV0hKFsdH + iycwgL/LrzjZgqsJcQTNBCih3bFyB9cxFO4GxVjWHTXOXbPjwBU2Kbm1bI9rPkPS - pMfW6TDN3s4LC/HYV6iELM2UWAeOmy1d8Q6surxVyQ4Y2jeDJ/8zMvLGQmAe572I + kvdh2JHDo8m4hn/CUp4yWd0zZ1fRYa4zes8qBa2d9GYAQ6OTboDSfyaPRFZoI2Yq - 1jakqbj0Z7QO5JMtg4LEQ9gQursua5YVkQTSGIXJQUbkeM5DujcktyGkTT71vlmQ + k7PYV27QLiu1J1lTQ1FqKpbq+Atak+OEn6n/y+1aEztaF7G8o37W+RDJUUhgv4+P - NGJz1JE8VSTWAGFcoWSh3cEg6PwiJCXXqGGhP5tgGfvUT7c5vzZSnYuHAJpwoTaV + SrOVXVbP72XCW39JPUBUPu5Zb4kqmWPZwh32nFZhWgv4dsjC4y97G7fluLhBu1F9 - IoF7ICL94mRLJEOA32YNwk6vMkZ0YQBv2li63AsGsqlvMCvI+7K+RFyINju/W6rs + rDaX9OC+jQkgOq0znYAYWpaQQXqTJcluCOuk0RZHbT0ZOLdCxUl2Ar0CCZmDLXJH - fkyws5UCgYEAySvEaYRtzxZjvuX9atInCqsHG1aZQaRArE9DhaO6a8zMkZND7AP6 + +3++gJECgYEAyMrvmQRbCZBJ2934mksJ+i9sYWBoh0E8yPFp7f7MFZM1x+PLGINW - BkfUpqpPUFg1NVg7FfjSIZDinmo8RpeJKoLG8b1Mwk1ewPlJ9caKcUF6fUAimYwz + Q3qKLjzpFThlAHVCu5cHcWnnqCbceE4aaDUjJSTEn8MTDsvTleF138ARl5iZUCGw - lyxgpJ484LY0eh3NtZrkrXmFwbLTJZ1nX1NOBYyHcJRqDsYQBE5uyZUCgYEA4Qie + EA3qk95XyLz8QzOAY29i3SGdoCF5w+RciNDMq6eDfvOgtpUakHG63XkCgYEA0RDy - qInNOpmywQq8CUbCkNZR26lsgfMQRlD9LLHyeZ7Vli/uQzVXm1Yp6O2Q1ISedcqA + eRXjpZAr6whs1Z+zDUU8zLDeh7Fpp1xSDElqR/79del8LkoRQNf5e75qRXcw4YSn - 9xTARqDEL1AOAh8/6gYcwQTiWBAJnyP1op4dpYDx7HEGDquFJr/0XhTQSyNO5Jz1 + cAxGewQ68d80KGi43j1vT3cyyM4qdA74AslMR8avtr14DZ2B4ZucgOPxEXh+Sn4J - wOogHAyFoigiKwchG0/Y6FomBjtrzML95IoxYhUCgYAgH4YIl2X1eIzK8ezKfu5P + y6uXA88FezhW56WRo5QhwDcUwWD8cdbq2sffdxECgYEAuDxkTc2OTp8i9eec5M23 - DMpgui5UxgaxvSJ6F4/wIM7VvB9Pc78b+6JgTrfFi6BLeWBN/OKJC5q0UyB24UG8 + cOS0mECiiIAWavX/0Uz6K02wiiF8R5iyvSWhxj1DTKI7Iol1WrSAMtMOqN5HaueX - 8Q8VkPXN3Q7xX51IysBWn28Qywn7XODsFeEEyGPOOiodCd9MTYSQkuQh7w06Z20X + GUiwX7N16nVOdkQVdy+xPUV5ntR4z+qpJoHuXBEmEN0+xdZqllIsr2PZwoUUueY9 - UrUVu7/w4TIiU8xA88lLFQKBgF1IdHSLAx8anYX2TDJQOdFOdopnNgq8Vm+/nOON + 9ZdIRLRmVkTqkL0M674+zFkCgYBInxdrAjSCdVMcIGAV0JSUvoFBJQsXnYPy07nY - NGWEGSfz4IHNt+41jpP2/sWJ4CIV+tXxrS7Z79lpBxWMHLOHKx48RxOYOlTU1Ds/ + dMQLYELQ65rsRWQ63wrVdi5aFO4Q2FCFNm5hGvYSwJLQmpfk8vgwZVppsFvE1tJF - 7iwwQpjv1UH5ie2hPsxNNncfQNH12s3If90At9ibaGaLwwaOV+0hiYel1C8CWbsG + vYDWyeiNlMEYEadlI/W4O/WoOO+9lox2iBM8CnR/+fEOJFAzvyf+KeP2zGbEcIBC - KihtAoGBAKNwLwOEH+N+bJr1Ce626JnAcvJpOu0rPZ7Y2Nyi1fwBGIiME6+93/0S + IP250QKBgQCwFmCfg4/EBvU9lQZJkGZOdEig5Y5+pqOSZLkOYxTb8b1pbDF3hfZP - YdRBLneQB8gfGmqCK6Jso9XUTn6yQyXx1ARSGwvQ9A8omJFfjLKFQ5Z5qeLID668 + CjfKpC+u6dUVLC/glR+W0csPsAPVSftm0cHNIHVLaHExeeS6cFPJ37Tz4TsOWoxp - BEgYH+Ob4vwTfNcMJsamI0XglzBmwmG8mE1bUUA8xZAwh2rHTdIB + z6hEblbGV9C0pyhAgu1dAJAPRBpzzIZax5K6EPIGqFhEkLUUL9VocA== -----END RSA PRIVATE KEY----- @@ -981,62 +981,62 @@ vector: segmentSize: 131072 total: 1 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:MDMF:ca6fonfjmpenkjyz46fvnxuxla:z2olssb2426dgptfsifbimefukkhlbab53pz7jxv5cxkpumzxhcq + expected: URI:MDMF:axafhf5i3oi43qvlbi5crtcn5y:6y23ckrgvhzeudwqh7tl4lx3th2wogx64c7oxzu6q7mbai3wfuwq format: kind: ssk params: format: mdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEpAIBAAKCAQEAt8xN84sxgxySP1tLtcuXgZ4WxfZHd+oO/MXx4v0mEGpiUJw1 + MIIEowIBAAKCAQEAqaKZBqNMZdBBYFHtrwyneRRmT+bj/ZMXJXupA5LgTIkFAgeu - Qdb7WJtIg2tBMS8Q5GgAEZ9c4qrE7MJ7bTJCZn5j5oQszrOQRLcLdIK3yOevWlqh + lOCUknGhdmteXd5KJFpCY+/YUYdo2I61BL1EzfnCGTMyQUy76jsNGVUTewV9yFb7 - wSiJ9qiqFxM/fGmlkSJGAU+V2MC3aSl1Ak9+0BgPBrOozj92NoCeOHOyLnURXmwI + ji34ulQ+va8O32F5b9e9zBFJ/9YEvzlEXUh5Ui54LYcJBSK81Pi4YBuwfMhjcAGF - ZNcoQJc2hxMdoVQKiDPo1Y9UbvZ0CyW1/iy6X244CdnP0dUNa5JHu72TEU6+Yedt + YRSGOV6yFNu+yVZ4X3M5BG9dEdSS9D+RWvSQvvf4it50MUUZhB42ZuxYceEQQ3VD - +a8gzE0tbkktg6T76y7GWf55fqS5ekzJgGszellICVbGQInyDBG7B6rIYZr0+DdC + /yzbeAA7VYkA4x9Ogi1vjaoM81edSpzeEL53y9uq31sJAa2TWJ7+UuvUuOHjHZa8 - ZVPK/4q0yEAUbAg4fF+QB25HnNBfVOagQpQMgwIDAQABAoIBAC9XW+K1wSCMzOyt + wBo0k/ggvMU5gKYf7Uk1omcLu5bjawUuqLQwmwIDAQABAoIBABJt59Igy5wEFBYy - xtACKzmTLzl5SIpOCuM31yiI3POQe1dZDOyzA5Wcla5oA2g4P8kdMptXaXTm2IdF + 0G/EIC23WaY0n3BdGpan6KTKwDOQb0rZKs5h18JRNgl3gLkR/VwVskDJPecdnvZo - RsZnEixVNMUs2V+6Z5gTb8toWg9RAd0riAt5NiQG6Jy98/XHPoKmCdMPnUCxzuwy + CqKJLYBzMfMq7LrLa173K2UlKQikgBXT3WCE/hDANlDFRAhgvqC3/5a8Ch4RUlK0 - 5fUc5cSS1df7kajiNsAuG9LdlhEZ3WvE8sfbbFPIQGOBT2UHOohTtgxCYsE1/Qm8 + FglMZmHXSpcnpIM6UumQW/eVCoeD0GzG4RcUm0VAPbNXDqekyVZdc/AYO7FUVW7r - ngPrlQrHkZpnHrRCAqbMvGjtr3KEyrDyLGAUmKVxTDz8Es7wXyzVARQnsdmYySxy + ATcye36CT6AqiAP7OCOyk9LqIxJluHHqvci12Ab05gBsfR6ihwNkGH0zYDVuIRV8 - dStQR9YkO/PS325zygR8Ynp6U0bZb7s8bO189hJPIBzFE+p9Xm2H5r+DQruu3GhA + 6OBxpPsHO6fdp5TzxKEhh5Mu61dMjhjOYE7gp8BiQSTaPtvHxcAZiNuAb6L3TtYS - 1bTtFEkCgYEA41n2ArxU0HPqlCxHdKkI+YbVbebF02Rp8gQIbuzZ8l+agsFcUCXc + dIiqDAECgYEA3WkNFq4ADXjJ+3ClmYwiv+AX/8apD/YOwcozcZlUUmlzywJ4dVrM - c8c3ZVi7rFnDZ+PlCBLoG0ZN2oVGQu8B7jNc+dXzC5AdA9LLk+Yd9D8hwAXEOg4t + kFpo7BA6iJWeFuow99T/IgyPT935D2miFPW4uTG7JxNdFdL+QAykSggiSs3CH+UO - r/tPQDvGN5f1BDO77aquSN7QC03D08pFw777PUEF04DVa2cA1VCgsEsCgYEAzvVf + TGIPJf4qFZYZjlhfzpyjfiJkL5qdT1wDabPMhVN1eC5RWHplIuzv3WECgYEAxCLh - l3gc+XW44LAyVs5Z80uhZkkVRTRdJHvols6SPdY4DU3Bv8UyDkZM/Vc+xQI6+wx1 + hlwB+mlMsnxYlzD/Vbk3L4y5y+bYjuPY61OdRTOnVknnI7WQQwSBPqK0eR4aUIRM - 1Rkel0Uqf0HYgmTZEjQ0IgQczxWrhra6lBRJHqZKldI6D6NcuokTepBADBKHryq1 + XLlLCX+vvBpG+I2i/YGzkOtTqk3qwyqapipEjfTCVuWKjGXZnID7r1ESVslUgXMT - o7oj4YOoZ29Ck1Xj5aPuhYroBXp1n+dMB/WSIakCgYEAoXB7IZwsOc1mEIuUvgFe + gqjk7aId9FRfAhaXJrJSXwM/agJhF2cWLB/qs3sCgYAFtMRilj9oGXnTIhcSevsW - DxowqhbJ+P7/wEwe1O25IcPDiv/VFlCcR1Z6PqwQsCUZfcc1FlOen+d/VyF2MAdZ + SNc1f1AyGhxNQEHNJq92pEMYs0qZc6qb+ciEdPKdPIXjf0udx35/ySUUYNsfW5CS - /pRYfEvxhw9xmwpvZvlr4cmGpL0zhuoUhTdWIk2PxmBQKwi1dOHTWollf/FbkiO7 + y7ZkB3UUT7pxaouk6O5+/fCsTts26TdSHqDXUNKS1dh4w6xMbdsE0CwW0fxF5FAu - AHG4I9ntUi/U3KxKyi6zvBsCgYEArookJ1NmZECTPfN7UNhQ5i4nnWMPbEEAOK/D + NDUMJpd7bm4oQpdCrCqOYQKBgQC6mfD0km6zXiFBInpqhYw3c3pke798lYjtESsc - dcQbc8lBln64YypEz+McNSCqUG5UHbvheGnp8bukXpTCqx2wMHkUaoe7YC6vbTqY + YLWc1BLdTnxghenVSODpxYRsQs6IUgYQpZ0VUWzRSjLBYId0JkS/mJRFz9GNkugF - WiBNlmq6RmZ5Dw1APBU0903GpifOhL1pWP64Gg32Ld2YcTejrt01YSzIBy7DGqtv + NOt6HyYR5FrXTkMl2bUaS8hl1y/V5LToN2VuDDHxdZ/abcGqaDdj+8QGSHM0/7eh - 5NqHdpkCgYA1i2pMoIwKh05MowZDPejnHLkxPHv5mlJiOZPM5lHlzUIHRXJH7MZl + jVAPJwKBgGbuMV/ccKzDXzlYhj9ek20gBfq/kdeSRY/BK1oIWhF0wb4G/S0BQSGQ - IvwJRbDncvup3DDa0ZHYKLop6WN/l1smS3P1pTbL3ngcHSR7BCQ6KvfXDmjcSSRx + AJs6tBozgrN1zxI1PkJ+J0u9KNtUhBUV9KCVSXv9IxzOtEOacFXmIvJL7hDYKMwe - TxYXbqWoO+lWzU0LaRspV28AgMcOHi7+FQsiM2icFClBL4P8R1UISQ== + 52PBFl9+WDE20km3OYtlFmezu7EvBNxEnEAAXZ5VO/0pxiRCLoq2 -----END RSA PRIVATE KEY----- @@ -1049,6 +1049,156 @@ vector: required: 1 segmentSize: 131072 total: 1 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:CHK:gsxbfakyyvqrv4sqvzftmoq5sy:4hinuje6rhm4myffxq6xwaocimd57c6hfutcbqql7pgd3rdhdnta:1:1:4194304 + format: + kind: chk + params: null + sample: + length: 4194304 + seed: 2NVWOXvtwcAFf0J/g0fDC+ypQbnCU7awZoCxcnhpkVM= + zfec: + required: 1 + segmentSize: 131072 + total: 1 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:SSK:pnpjd2iuvxpslyhy7xzlbnl76q:5wjat7x422zdadbomr45uwbsurpj6ezasplybg6b2zkl3avs2sfq + format: + kind: ssk + params: + format: sdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEpQIBAAKCAQEAmx/L+VzqYjeGT9QCeDb6DlBzSwGM5dZ8hkR+1JeURf5GCdfo + + URGstMbQQWl9RvHNywFYBoxXWD2Xzk/OVAl9cTKtagDGT8MfR0X0AfXQ7CfzZeV5 + + 3ThNsF0X2qd1DamOiSIGm+RP8NaqginLYVqNozMk47iN2CkSnl8r3XpatrIvBDiW + + iY2o2wWkfYCcDAbSb0jVY23y94t6JApqPSKNQ4UA7BI+0oWt3TI7ucA379drHOO3 + + 7d4G3kVwWxO9zCi/4kV4Q90QkBosqTdC2rBR/Y83FMaM2hT1a0iypnz+0wGGHkD8 + + PQ5gmHS2fwmN68vOWk6j6nv0GGB9WukH0Q9ZJwIDAQABAoIBAATz1phHKM3VRfrQ + + eE+25EmGtKzwB2aNxXtSk9YHt4Ul4WiOYG1jf2cPczX+FIYqUXlUuVG5iKRZkGTW + + 1BiHa7OB0bAxBs55xUrfkd2/fQRny3PfsqynD9vnQkwJzUk6tkFB49d1n5O/LBcX + + FyuBhO9xQjvpotBZkgLvuOp2Qzp9QxIP8RMecu/NRtJD8XzWpkeEI1cKGnPl8oNV + + H8Tw7X6YhicYI15REqQvw/KQMmgK9egOGGtAgSZu6Le/PCfDMsWWTXhP9gnjqY2N + + duRcUvoGvu/kaIM2bmp++4DnZgp9mumcAwwbVwYNzA73NrmCsswDk+ZBULRd3yCW + + R9Hrr+kCgYEA2DBW333pWiEzv5lzjmMQD20zZrbrF9trBTxByf80OCLWaSEOPvXr + + zbP8zFAAWOgJFckFsgygHfA5yIcLPLaa0TiWtelXNDNVVbgEVo/MAWKOmuujKh6O + + 3/GKEbln935UnrECDg5G8GY36fyyoNWjezfug6olZ/goiIJQOsNSCOMCgYEAt7C5 + + fETBtLILfHVu7IeLkUW72Q6foTzJVTkRx5cfGPLB1TgohqMqDTICURIL783ZGz/i + + Cxk042QRD7S3j3C+moLjXQMceSdWgY4sdy7OX2hscmbNSET42qyYFoUN+tLc8FUH + + jBVIVwZ1JfohDD0DdZdMMXuHp/BElGr1+6Uble0CgYEAiJPJXjoSgRE2uxW7rjmh + + PM21Sm/HB/RjoRQXUAC9QbWolRQABwCf7v2FeKIWBhTZIH017u0Q/rj0GF5QWBPY + + rNK+S8BVijHf+F5fxzvjGwDjrLWvB/30L0BOBLKIHxAdb3/OF4kngdph+p3dT8SI + + GmEUevOz3AInwU3qV6VrnxcCgYEAsNfM2xx+uG2orTuJfOHJtiRCgueXOu2AjzGQ + + Mm0FHUmo3pNgQK6Y73czz8TmBQpSd+96uWCdEEXoPwymo8vRVIOqTIOQR/tdRwEP + + Qfan7CZmMYVTIL52LmB3U0bpfI7A8geKaoyaxl2LLvKuGlArImx0iDb7FO01uQV4 + + p7n+4skCgYEAh9u+E2U4YtqeaSaL0K1TUlNPCrS+iKDbikC3WXv/anEDAik8R9Yq + + PO5kQ2AeNwcuIeCwY6F9h2TAIS0HzZTITW1AcVmoVKekqkGPsxzNV5zofZR/BTZY + + OpNYpBwJmxFwHPNnN2hcco1QVRN663FvaptxS/kgtUwFB2+hLp3Tbn4= + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 4194304 + seed: 2NVWOXvtwcAFf0J/g0fDC+ypQbnCU7awZoCxcnhpkVM= + zfec: + required: 1 + segmentSize: 131072 + total: 1 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:MDMF:ihnqgpid4zygk7utpua7n7beqi:2qppzsf7y4hdbo5fmo3ns66mvshiti3govobsriwysvy4w6dovvq + format: + kind: ssk + params: + format: mdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEowIBAAKCAQEAyGaEmDD0/hTK7WAWTEfXO3Vkgq3vi3UYydqC1610aQ32QOnL + + GVtwlXPM1KyC7nm+MmBUx/f6nhOCLmK3Q0+4uVf6D/58u9I1f7K1hsuvZtuzoP/m + + 5OV2jEBm1R30hZUFZ468b4KuS65WNH+LaW7ylNziALdIuLNmz7WPta/UPpjOQfWj + + vq/XXLenVsHngzL6azBUr8U11vS5ombDLWZec+5Z5WEXPJTZ0ywW4o0VcVAK9D6W + + a+K32XanLNVpCpBhVTNQ4Lk7q1O7OobYoLYJBhdiFIW8jMTknoYQicHjTxmfLTz3 + + zWlCfbBC58KHogd3qVMV3JDRbYOG7I1/vD65+wIDAQABAoIBAATvcWiGHCJ9xJmf + + +iyawFQ4iecl/XZDxf6CoSJKpUlJDL2AhH31YIpttaevL/JLkUGQWcYq90MZW+Vk + + jPrdZcE6x2/JZq0BekvQzOOq9IDl/ECEzNzqQccmduHcwP7hMqbgPwfIAh7fBkR4 + + t6g7EUJVRkOaP/I8iNWotQdWczWvZoMrEgSjIi9OR2yeosrZB9EihCUml18NH2JC + + 8MDBcKmkSfFSfwYoLPwH+EoVwhotBzNl/dofMGCeG1DzObKXZSC0Fi3nXILVX/Lp + + 2wWaWehN7HggHr2SggQm/4jA6RoFAlP/qqeD8CBpJn1eryeXRd+A6qsIvK1C/miT + + aJPVr3ECgYEA0IGp73f4g/sb7kIYQmylo8cTy9LqA5tK/SwIHyc3nYtqRMkcG9Zj + + gLpGSvZABbPiwprawwQcxXz9TwMCY6PjUEfhevvo5K/ObQmB4p2cSWflNJayhqh9 + + TYzUgpIypjVlMmhQDRIX9X1hwOxmEt8ueIlDGNlaLb6sFKHhNWeqIIcCgYEA9gwt + + smtU2xVmoXas6Q5VFZV3X/5dfhs6Uko5XVTS2cwhzxDHn54mstbDfEqCU1G1ZDJ/ + + 3EUH2aX+RZjEU3eiAiB6TNq8SfIFqDzJpnmGUR6y2RofhjLA1cZZu6/L2fqZZ9nc + + 875lV7tENrebLu2a59g+l9NoLBbxgtsKAX0ku+0CgYEAhxCHWTU4yZ3fUO6FsnmY + + rsflnfHpXx64a9mbBTstPqOx2g8AY1P0Ls37fNGZVVhaer8/GHbQgGlf2U/Uu2DN + + fhKiED2gdosfx+gRuA9qzu47Pl6kFLCOQq8IdfBoWNxbylRiDqV62a43pXY9BNqH + + ytL3oOAjF2DdLZxTO3oEbX8CgYB3fD6M0Jaqtd/bNViO7QjgrG8GTO52GR7fa3Ak + + JNcoMXuRpOJsX08HtkfEiiJz99AQ0n1JKLTBO10ZyzA8IHKqeb8qp2acuk2I/8wl + + bgqORkwwJgF9GBSRO/vDq4FhX9MznZcxPxrT2fssX0mbJoP9ZwQuktmZ36J1G43m + + XzGBmQKBgEeliDU1sEpkXMYUaBs2mzCEe3u7DqDNOaXjvT9mJQioUNoMFMdd1JPG + + oVJGuGKJAOAWbfeniw7wraIXfSp73jlxWULKKs7kbvMWFXcJ3OclOyKtFs0RycUd + + bTkmeb5G6hZBu3y3HjYYW2MX5E+2TRPHAWm9CvnYaTbcGW8BsyBT + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 4194304 + seed: 2NVWOXvtwcAFf0J/g0fDC+ypQbnCU7awZoCxcnhpkVM= + zfec: + required: 1 + segmentSize: 131072 + total: 1 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== expected: URI:CHK:scwepfpkvhuc3fc5jkpmrodsfi:wui2ixaxyac76mbxtr6diw2ajky75wehk57f2kqgytddnpo4rozq:1:1:8388607 format: @@ -1062,62 +1212,62 @@ vector: segmentSize: 131072 total: 1 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:SSK:sovuio6fbomgdor3sypj6ywd2i:l4caqcoo4xflec3tbpt5jseaob22rqzeseb7fda6asms4z2ybwva + expected: URI:SSK:n7a2ymjpmh6yzczjz5dzttejpm:zx5xl74ellbif73polmylwfapy4ptr26vu67x7ecwqrhf34ldqda format: kind: ssk params: format: sdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEpAIBAAKCAQEAwsvOEZ+3GvfC5j4n03btkN5wjxOcjfhmetfjy+lnIvdWZWpz + MIIEowIBAAKCAQEA7Qad4gnk7mpesxcpiKQfXW7+ybDNH7rdWoHlMsXSaE6r85/4 - 3TrCZZqwhOhJUZnTh1maowvEWJyWNpcziA/Z2GBrKEA9GhToew/52+TzSmIEzneU + 5cZkBSptDFJ9+H65qlyiuUVOo2gnX30Hd2jRv9Uq/vVwgcGTlG9yTeqtLwNK6fRt - vM9kR5Xr5KHPsM72FCpemBgtKqS9gXODjZfdz2+wUu7Ki97vSp1uT/1RX5jyARDA + Z5wIYWdver+rACxUrmg1uwYFcD9fqzkTtRoFTNwUmjBZKDAkJA0OU+HN8ehZ/9FV - GhqXkGU9O4JM5s9UGoP1JEmBiCWDhTzDhmpRtm67YW96pBA/pGzp55ZYnF36BM1i + UJdscqxIbC6MDDwTckVMOvpNq2Cc9maInHw8On5VwEOsOaDGYK+TaxsYjmwhxWl1 - ggzu6gIuJYmPPP6LZ0uJ+LQknsXHVQxsk7X3zCoUBmcbbM2GOeoC2j5Jd3WQ50u9 + TF17Io6wdauHSLP6ZYjv2KibGZhiiyb1NBKD9GnSuHji7oQyAWKaXGGKTU4F04jx - DbwMyrrCbJNNJdicVxn7crWk6UWIQc/hL8RPfwIDAQABAoIBADaQaxURtXMW4p+m + JdtwhYBysZWHUf7FQKsVIjGiyE/8/f04FBlX5wIDAQABAoIBAGK1QTXbkfuZz6M8 - 2nYH8qypOkNBnZFA+se/MH5eTzcCrE81HeZivrBCP97CyELUwWVA6qlwMtwVZJg/ + b58IXkl+Slv7JYljvAAPnUAGMwgeTyyvf6tM8eVW1D/v8Kb2O2LPnjKSwtt5KgBx - Cz66He3XuDxqnhLvt109UOJRA/sacLk60s1+lFre+lgtISWoG1LzuVKGNySiR7j6 + pJTdUZBWeUfhNb/LuiZ0PQFmzEWKVP3WPWOLDtBlj37qaA+z5nYVTt76dHRY6AH5 - l+dyGj4wTWY1oEPEuyed6Jf8X65Uhjnn/fCQsZcMl8Sj3CbaPPYemENE5fd6Hnna + zJO8aN2nv2qw3MhIOBzNVRyoqiflorQQ7su4pXe6IrvHzmzS43zgahJby4SSmt7W - 9sZVfZRg3zZsN4oExcggu7kpEKGIe2atVrPrVmutFtqhj3VHDnHdWgfci6G6ETin + OebkYXMsQ2Gv6jc0aJj11CcpCvutO3B85OE/GEZlywWyNKo4sMD8O9x40ByVnT2m - K9TfDuxv072xUWcXhUv0Hn+fak9OGweX9fVa6AH4K3fmRxGuHHQAhXbQTUQO0AdF + IydpZOSF/z7P0RZOCKyZdDoEZ+hipc9taog8Drdu1eq+L9tAF3W7McIlLFv+Jqrz - sNPQqX0CgYEA8bqlaep0XHNVER2Hkz2/xjmHQdhK1fhpU8wHocE6TejUoxuWHGYn + PrdVuaECgYEA+zG8iWbfT3DdyE/59t4OX3MJMj3SYDTQ4h1ePLc84zbuCy0Z5ajh - 2JSS0HO/Trt0M01SKFwYiVSnHY1C7SHCjH6dvIxGQPYkkEvUopB8TFZQ9xktbUlU + mCnFSPdel+d38bUFHwh5BkFV12qMwXTqMeRIn1s6EDuccujxINETPN77Dl7FRF7k - Mn2ocXMXy1GBi/IuWY1qNgusdlPi6kPod18joNgZ95GEe0J/9MJ5weMCgYEAzkvX + EE2xNfJ5Io2L42Izzy3YJVJiQqBnXjMdD1X+Z8hKbQcdjmxOZTu1accCgYEA8Y98 - SDSJCMuwqCgqJFpB9kEMV3SPLnlqHoeB622taWrD/8DDd7dt7JiZUZ5VifUXJBzj + 9ODKRCocZ8BfFyHs7D7eY0Q7r/vp8iqE4xHXZGLyDDozphwmMl77Y6a1mjEEzsrx - TWALBnSnaie3zgBXOku96uovvgtofk6f/P7wSu5t4OReh3vMGcd9fcysSpxJbeuu + 6ckl0LVvq1ESt1wKoGBMxIMlrv1tkfHvgrhu6MeBe2QSDCUWfd823VujytEESyj1 - U/Rxlk6ZTe3lAlVMM7++DOkGidqkpmozE46l/rUCgYAXLowgfTCNkS3uR0OyNjDH + xwBelqRcTXoZC62FlGA+uMXOGs84RPWDfeQdoOECgYADxn6X7hTjI8YhkZonLLU4 - BMtY4DJFFN6c/6sXsx0xTYve3I1nydA2cAEoZoFJPqblKJwhbLuZp/mi1uI9NYif + mAkGWUmFKqYND/Xvoa1nmNbBEj92ZTBm0hHmA9nHHLJ/zoGyMrVm86pvn2lYKwKu - yqC77UPrhO96uxr4QBz7gSegmtSFb4vYj75wqtX0VKu0zRPu2KX/6tyuOFtBliOc + F0lEI+HehpbWX0voe1v3qT5Ku//pBCgXWqOUNP2/GDOHCl3O+lhqTy+s4q5LCyef - Fw6mpTLQUC9BVt5IjcH5ewKBgQDBE1g8uvaaJdGDwHuYpGTh7gV4AJ5VV8tLIXYl + qGI3exorQ1UdY+FVwiz61wKBgQDHJ5XODsa0DEP/Bgtf9whufia7kLXlEbx/e66z - +vN3GzavsiD/dczKyBOOwQq74Ig1A1h1vXL0Ks/ZWaz8f3MkG2l3aJEgZBr7Q+kW + xzHeAfWtPw72FJ8pSEXakseGqINeOtPX+47B09SNWfokUi4wqzSfj8Cx1R9RBDaD - 5x/McZSjC/mxAduHMR8xUxLZjaZn21HAP6Ljk1KGDiXs5ho4wLdF6/5znQ/GtNRy + f6txH4sRQB/hA3LXtAB33+XagRkZHlwEBbn2WOwAtHmRty46dl8/11VlpRKvR/tw - 9GpFlQKBgQCH3NXTfsc55BZu+tooYbsL3wZ9iRn5oux6+MfJcl3pTxtW5twBELgI + /3GuAQKBgBz8ER+tjIMqlVSwLRtgJ+Hi4B/7I94HdOhpEuXynkPxYkOUSKGxSysg - Kg+SjF19Mqx8pgaNqnbmltlxcVQW0zA/mverUrc6HJ4SgiNzojqLEqmm5+JVPnse + i572W0X5fIFK6m/ubzdtfYx828y7p8LM/BofvQpy5q8mNL7tOzbnsR6A2O0k3Amb - r2qBE2Y6FaBEDfQknOhBwquM/nAXk4X71hl1hQZBvZq2R7WKbc0UgA== + t0H49VVOSMlHAYNFWL0CQE98M3bWZctIHIFo7xyGzHpypCQUfI23 -----END RSA PRIVATE KEY----- @@ -1131,62 +1281,62 @@ vector: segmentSize: 131072 total: 1 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:MDMF:jc3ui6y2uhid5oy2nt7kd2vhpy:5vww6u4wsitqgb4mkrvy3tfvks3cj6t3smfuryzdzjbegbamdl5a + expected: URI:MDMF:jqzeglflgihje5znycp4axh43e:cju77h6dvd5w3oarj5m543ff5ou4vlrsipfmd7ntfkleq37tltma format: kind: ssk params: format: mdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEpAIBAAKCAQEAp7VAtzy4jNgUD9wLg0yAbKnIk/vTaDNGpnPbLjqXJP+BNErp + MIIEpAIBAAKCAQEA9+tyPYSnh4G/0SdL+ZxbCU+UCQn3ZduTfDstJbGfNGdlZLgd - h0aPmWCDUOqnyLwtgqDdrl+juwvx5cF/wRqRYQWB00eCG5egccx2VpVgu19K92da + F5mSDgO0Ius38nmOmMAx07abuUkvdVkDNN5RZKfOolLZlDl+eDh/XBb55AaSV8I/ - 69IGGsRP3//qwYc3CD/p63/MRcM+VlS52XdI0hSMpVqmj1ywDRXTvN5k9leaBB6c + 6asiblwxGA+xOS31pvnhupLSXYKI1xM2Pc+wLbQEgS3MJIazJAakNySCrNJ8Yp8A - Of4MetXRX6xQCEB+KrEQ8uVrdoHXEi6ePZpJRAwsP0N+dAhaMBS/TZqdR/MYm12w + MTAHLDdMYJa4kdvQJeJN0GBFRNLdbdQ6VCqnn1rK+ky0khEskH5QkNZV8pjGFAIJ - aipJeqxTHFj3gKD2FIVizB7L2ViILd0pS8cf9h4B+9ZsrVkdMk1sysSci8LIEH/y + 0RMcbysG18eZwvLsyCzhtsST2f3v8/3B9N1Dm49Sf2OV9DvwMxGRamO36f5etPt2 - lpCaLytW9HfGzKmyRSBHGzZCfrCR8ip2sM+y6QIDAQABAoIBACGEKkJUisdvGZdP + xrrg+lTnWKzEoqlINSrQ1tYNOZMGHVoA15BTnQIDAQABAoIBAE+LfDKP0v7P2x88 - 0Sc030eYKONWRRpCgSCb79ZN1E4LGB3EyOYFloY/EQ9XTh/iZ6//CT6jk3u6t+XE + +AwFJlJ09X728yl7y7Ty+bfb50R1nls0FaWCURHtD0ma5e8HIIETPYl70DgharhA - ZY1Ii3xZ1ufMFzb/dwu3IoFMSjA4K6nFCJkveJPZ3uKz6Q0zQi7OYyfy+vaIPgmP + kJ5QbJYan1qGsaf00Ia7PeXqu0/16dN9kGslTR1SuC/LrSW3ANgL2ei2fgehv80e - 1jKdUbrWa4NSWg41pmN/FLlessu+RFHj4ye3rYzPZ7VMSy7MXpft7F71k/wxJEPN + LWukrRbk6QMXkiXwEB9RgDPvI3xWRQQT79Yf35VwUoj2ldG5bDRiwciA2NeNbNNc - bg/1DIA74LWlDtsLmfIhv608H//GE6EqA47kkIpfBfgaOuhcWn/Iw8k5fuj8/VkF + KUdKtN1acgqZMvv94uN7HeBMYKyttlfUM9+EMoTDhGJzGNN1vLcB7R7nUwiCJume - C9sCWk0g4XL9/yvoi3JdI/Mfak6OANqI7mDKosJAuLtl2ZUwoK9vxUwLVuais8db + 3/4nm9+pOJt2ETl1pFY/G+CSAUeIrJU47y664UTfHhydt0YI5Xaco+3burj/9xxE - /xe8/vkCgYEAzkvTQGFg9UEpgbif30HhU/TpdUJO03Vz6P+RY2JsIFjs/ICmG2md + qieUK9MCgYEA+Kon6r3VpPd+vkhylgqk3vNleuxB/lZmmXMH/oR8WdU3bQh65q3W - khnKoWY2tZiXE0630IPgm3Y3blsCAmyrRVoIFzSD6FdUfvYY+Bn1g8tHlbXxT2C7 + YcZlRh2Jq4iwChz16aDSelWB/mjDXotPLK2cSvJZL+W2c7tzCF/RT4XiphkELmAB - Af1VAFKEJ3aNZd1wzcR1gsWsMZVFOLkfS7NlnLWsFo+pWPY473x7RjUCgYEA0B1X + l2Pj9W9FEZVbxHBZHRuaeQQsJHGhM6MtnX9TCdSpX0Apd2YbKx71mX8CgYEA/zuq - d2nP7wUdlAT8OkyXVORfGeOVgpOlEWwXNSQpbjtW/ZmAhJHAouWT6le0LLjx9TBk + I4R8dSVNkliH4it6ABhX3z0t8pTQqhDVd2G+ubQ1QjweJ+VA/xxyQF9XoqCd5Bqd - Td2WauOPV5B2M3JSQveBiQtM5g8UQxywF4Ks4yhhjD4WZ9G1vWioR7f5NG+cda+P + XY7BpXRtVVQTEyZ3ecAQTeed4pVlDuJXuwMqAxSVb2EAZ4rLVf4DpfSuSvREqZjb - TollPzWX6SOwXwJsmwxZndeWGHd3DhmCwPUZAGUCgYEArv6hN8ajAciB1hlv/Gmd + fhrW3zbRzhBQF5QL/WferHruii83JDKuAF8iyOMCgYEAkBhCI1Q4Lm1A49ElnW6z - I6PoeeCCj1vdtDM++EhgIlxsw5C51x0TXgDk416aYBcNaIJo6MdFu3pfcQxgOwBF + lYKjxrSLlW/J6pfvBP0O9huJD8S/O1d3CJen7haFxYHiySl5ExYfgcZ1GtDojava - lPHXVR/mGSwjcAOAkM0sd9zzX2rURRpv6DMmbLySgAtPzK44Z0QUzpayB+lwq7pV + iIBeNkvzhL7vmGcCRNMJfrSN30RV2O3HXkwDOCFve736PH4CFcz+GaxiTAgQqtSf - cti+BF4TmZvJ8r4C9BvrUlUCgYEAnARTLQ9jNdIE8ZGnMWF31cl6ziLCU+ixx9Tb + RUoX+3VhZJHQtaDUk4tQNM8CgYB7GtITU4GcFyP1JFJWGlY72YH4oM+ao4CJppjv - tRgOAzhzJ50rLrdBzh0D/ZuQVDK2GVUU7Rbgi/Na449GPZ1HtDJuprmVBadqTkG0 + feu6MltF2S1KXN8erR/GQLZKMGI3dUbVq1dncGKTt3uDzxftV2AF02NpuFkH9tAN - dXuedpEwR/3HuD8L2xoZheKS7U964PMjIQJ5p6Ba6Qm7UA62MqpYiK81M9RjqWtQ + 2ZbX6YOyNv0089LjZSNpVj0C1hKQIrQrfNKK0ywa0e9vj+7AiOr0Ek8fw2o7QV5/ - ja1w980CgYB1iRfNAUsBOAA3wzMZ/u3JiwuQaAWqnwiy79UsCK4LQ9zP5uJ9hiJF + u2NRtQKBgQCdjdTxZw1n5jDO62w92iJVi/Q0g+KoUEpeeEXAXmctiDNaCNspsETb - dTsSF1VuupNWNdVKifvQypOLrU0AFrHqVJiypKXA3F/1fqQE2RWXS1B+c3QZJ9Ga + qqIo2ZmoLA/yptOPa0vfEYg1yTpQJIf8ZY0AjeBHRTw9ECUFeqdakQdV6IX1dEFd - ze+uCK+a0b4noZE+n22hQbc4PrItHRoQUpB/aFJ2C3A1IP2vPpHa/w== + r3OSCPSREX24x6CQyUhZMAtMn8D9mAVMlUt1Iq6HtXA3EzViTS0PSw== -----END RSA PRIVATE KEY----- @@ -1212,62 +1362,62 @@ vector: segmentSize: 131072 total: 1 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:SSK:eklrgikjshlanhotc7plpaonca:4vumrwqbsgy43agvgfoqtorya3bs7wmjedjcwjhlhurloybghhuq + expected: URI:SSK:n6eoyebsflmwvubrh2crtnxxxy:layy5tfeichjjxoeg6jehn6lkxtcxpke4udpakwpmotutsdxajja format: kind: ssk params: format: sdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEpAIBAAKCAQEAtvnHGO5pobH7ONT5LdgIKMJhbMpq/BsmQ9WQCkfW3CZ28tDn + MIIEpAIBAAKCAQEAm5KsR6Jp8mLWjw6HtwEMd5rnzsyCivFl0kRnZoC31r38+JeR - LloR1x4rfWnkXNtGlgRUQwPzlwjZ3bWd8RHNEFgeiXaZvYmrL4oJhIyAp9USqa7l + iYgM22iJ2rzYVOjZlSBtq/3ts0wZxx5HkFpqNaermhD1XcpPWBX094SvaiCxiRFT - b8dOU/ZwTp8nxEw9RvHbalX/APx1Uyv3mbWwmzmorQ61grERI1+lMg4uSd6ySpML + I6o76DZ8xaDgpbg6qOle7+zIGyR83ARl7s8gNCdHNWS1/eAJ7VYtWukyvAMdOpUu - 0HJxhUIC/dIg6Of/hR1Edm5p26bUCgjNLQCxXrQB6NGfRf9IHbLlR3g+qXwEafVx + i6Mjqy2xgaEeEHBdHQq136NwZDaQiz52EodJE/Fi+hatP9XluKwkv/UXwE3lIcBi - TwNHl8yaAMNml9INDzdpNcBTLT+Q+ILbiNid58pcO3jP6qfREp0S9bq+Emcltgf5 + sEr1/FeG8CwRAk39NKk0qf97vAPvfrzPOQ6WFFV/MBS0To7KsCXBY4Ve2zNVBsOf - soegODgMqNJdqKs14XdtSz0f7qJxIWnI4DGK3wIDAQABAoIBAFcI4cUApttEg0ip + YL4u/52a26X1yH5SO2oblE/cyFCGWiatd3IjWQIDAQABAoIBADmRRdTgIapCrriT - uWsujtcAewIaGKCZs25h1/Wj7VZjr4HZj5WzPzgxgCNUKs1meiFipsgHyacGjUdS + HN84MR/VH3Ajty6o8w+ipkyE1wJMnV5z37Pvtyo9fb2GYdrRqyoGrO6W8S2GvIc1 - G/Iu8vl6yO+/K+sF4Jko0lUr1gi/J/Txne45Ag+bMhmbx/kuAJnN8n4WsMkBzTcG + CjA9dM3T9Kj3G4SQR1oGDfbFj4+K94cL9SLebHqaJwOOa3KHQJWefbX0fXulvdpA - O2zwiTSUzSCgVgN4ATxvwu7X4vm8dzD5eoUZGerFLfAFvrbiULCoWooPDEVeOse+ + emOrG3SREEWOtdVy4NmFKRVZ858kSCXGbOZOl8wuaTfzBQCBSgWBM+P90ERMKRgL - SrZdm1nwmaOTTYhDJK185nDubn6CEV1HDs1fDrVGAya0K7T/WtqCWW39b+FxOyJz + Lw25mqpXBVBuIsS2/EHucMaH8F5V/2k2jC1w0bJQ0rIjM8vITCQuLCl1zb3sHjYh - 8AIiDohn+WU1062p8XfHTs5g1mXP+EvRBk75ePjSuTO+HPR/eOoxqgfqGEJ/UyIM + alCQyKR/DjhE3NnY5PRedDu+OikqsEMvFxEXIlUWbAgPybe0BgWF25z4VWMROanW - nK1epy0CgYEAxrViiFNlvOIjvYq838rb8cb2pwmYtoTEVJygBoaJiI1dJyHtGMC7 + uEZ8Ev0CgYEAxBmqfy7Q/wRbOdlsFcSBNkDtXHIjJYtS9OvYwJcOj3Yusnw2qMSC - sIN5K8Q4tddqX6JB0WPE7HE3Q3/lLZI5SoU/IhrXQTCny6q1u2l4SwUoxbiYW3Cc + LZ2teRvbRlbrI+zIpXWCVgiB7iaw2AK/WGohtkmf0oMgl5QR+lxqqwzwze8rKWiT - MLvVaDrPxPwm15yN9NCNz+yl11DcQT9A2uLj3l0O/OQb41DXACPh9aUCgYEA67so + gtnEtGNBPLxuPmEt43thL64nK9beN/vdlVEwEsZcpjRgf50bMEXY91cCgYEAyxfq - IhrIWRLC0MQmwrg/5nK1wizN5XZwlAgiB3FE2a5xPUvMPtlRTHb4ZNOmWs33ak6c + sD3X9WTE9jOnJWpzA94z0ccVsUYD5+vSXRBhwtUcPet3KPc9M/PS3XTShaV9QOzI - WMlImIzF4EvuIVyFdWZQuPzfJJRzPRFVwCSddAvKSvh/NUr7GS+8wrMivkQ5P/86 + z/z8wsFLyN8JzoyOZteIAd6Cx5MKQJ3KO0+F6jv0xyIeaDaqcVzsycw6vT2ITQtq - Zh0w5ZMOTixU/OaptmkDSDpG0i8aA3ZdkmqyPzMCgYEAkhjgpizzG2oFLyHndn9X + 33VGVB1GaXYCNwMBhUx8VjyMcbW4xGGPa4fsfM8CgYEAqZhk6v+rQpIa74oJPz4m - MS/BP9T9dAyvsSorOkEGs+CEAfaetVlXZhN0Lqqpq4EDk+bfj41URyeCo11Qai4d + XayDW8teeC7pfOaoG8/IiOw18KkagJUK3Lace7xKxKeRTw2ObgKVySAsdrHBid++ - c13+qhuj8ilM5aDQ10dXi4jyjlUHqAtmuyoPYQAErOdbw6E2ei4wZhSvZlzsZAiW + apHHPCaqcV50hoNJlRPuMKbNb9zjoDlQMf9ybmvU1NlGIu7ax/1BjQH54KFAqHxM - rZiuQ1qWX3dzzbEtMswvIYUCgYEA4Y4vFJLz6ObeqctGG0MZQXO5HpaoXEs75Sjz + I8IGaIZjRF7SAiv2gqY1wZsCgYAw6cm0OLDSgTqOsVIISOL6g4GnfHNVBq/aI4m+ - BpQHARK9H52LTQe7lqKvgipSHsi9WGbnirzuTalFHR0KObnBqVfBHYA4M1Qn/+K6 + sDtbWUg8AYHpc+JhqM+YVpJ9baYFBQI4VY3qufMupckO3ftN+YrgKF8HAfruJRKX - XiOq1QMDCUFE1sVsBel7gADP2aaF8QpR4qtDwic3pO0eVO6QrQ1GKrI4WZzgEzgK + xkdSaq5BZ447Oy9Brke5Ml7TRQaWx7EtsGkHySU0MR/HcAnluM4ZVuvcVw/w/C42 - yLJ246kCgYAx3v6a8bivaZVTML+tYGXt8A0oCL68kgTH+ZqezJiS9MiKT1Z/mLJM + j739MQKBgQCFbLq7w8IGar1IXBuG9hCvVFUE0zx9ij7u5u40I8F6yW5Rt9QYNiWC - 8TS39Nr+HWZ40dqmh8RZSfGxEManEPqWH4gxFpQV/nhnktRJFQmTBkRWmqmdCZQG + ctNwMCkvbhiDTvYvd6titc0U4hplRmTpCeQDf5B9gHKQlA9wlk5sfJ5cN7AZsc+0 - LeAspfI6SmfEn8RdLTKyNYu5Mnf8C6W+/gTADugS1/LcROBc3YhdWg== + Zl9LnyoMqX/Mug+HcTEMnwQDHQugAkto7jQLYDZezXIX0ppxLp/zlQ== -----END RSA PRIVATE KEY----- @@ -1281,62 +1431,62 @@ vector: segmentSize: 131072 total: 1 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:MDMF:lfisprnkod7r2ueddiduvj7pzm:rb2c64xi6s2n5uy6yvtc2xd6xl5tmprgkjykjuew5rz6yxqpkidq + expected: URI:MDMF:7sj3wkjglezcfjwsbnjanxlouu:3p7qxclmuc5vz7vbnivohlfwcvwd6kvnnikbdx2afkf3kfxripea format: kind: ssk params: format: mdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEpAIBAAKCAQEAxOk1j3k4247ayUyyJKmm0CljRFhnKIChD7N2AGkIW6Sqqr7B + MIIEpAIBAAKCAQEAoDSGK++imql5vyeH632kPLk9l4Tzb4tj71be7xXT475INrFo - UOboZuDbg5+iv84uhvACB2JF23mGH+Rf/OMIettMByRWvcUdx76cB5k4KTDqw6Dt + tUFmxce6eT9TKQQx06OAdiUfOeaOMPuRL6+qxijFnQ9dkLmpBhOdqW98lOQtr+hf - iECh8f9Oa60oGDTsSmMTmyeDRVlsM3s3Cu4D+UDbdog+Tz9ael+y02Q2LtFVzfvA + t8zMEFkC7PYThOpmc25zeRI6U78PLlpOXY1A8dKgy2JvGqqXlKu1ffULBhyzZay7 - K2OEI8Tghu7PW8ebVSxbPAtnpxQi+yh8mfQpBdyQdlSjcvdeap9WQgtv+zdZb41t + VxWsArjl3b84vfGyAUeAZkaqHff6rlPrsR54muud18tWBMTfwpBFxUvnD4PI4bLn - LU8uXUyLRuPQAShlowcfcGfxx8TX9EIuCqKapWoY20wCfscKXsZYhkMZbN6Yo7+e + FHR23SlqCdWZ9A7s/M6z6zVgfmqOaRmsHqXjdq5yXKBYhlrF0MrBIi6V3s9kKUZV - 9vfmeeh7wV/YSndl2XrnJyPxbghcVYOdusrvDQIDAQABAoIBAFlzZZjpJRqcYShb + OJGC4GC3j36l4mxplcE1onK6o8sLcjAwbPoHVQIDAQABAoIBAAxohQvaH6oC/l8D - 9nswNG7Qtl8IV8hu8nuq9zqFfD4BZmRNZo1FcCK4GBBJlwnR9JHo+sr26iwjHvpi + 3M8siA/7+P1HWuOE1FSxUcsK0cKN9mHmE8oWKrOe6J6DfRlsLb/KpiSAc460gMbi - 6PX8/s+synNeHydzIa2pGcFb6cbQiX1YID+quMaxx6KjlRi2Bfde3bu4beo1jrEu + dThQTtXSSpwDmKeg+apy0n9RF0Eg+zjosqE1x4hsnIFl/dUJoq3GHEOAWewqnC5m - Uplc+aIjw+6rQr8GVShNS/O6zOBj/TufeiJ71ZTjPUux+n+cbBY7wR//vj3UMO3n + 6DLuwdz5B2M5WImkNOFa+0+qLxRl/UPLytzQ2+z/zdHAPaS2spC0MFO2J5z1H3rm - rkHVb4xj267g60deSJfNsNjD/dZSlVAcM+EatqMI9dgXUtX522HQmfrGPgEReRDP + N7gaVtGPDqDnFyqpN7zuDyOD9+7sXKgV4CL50g7+b9EwKwsjWNaQeePKWlbyNL90 - V0jhHL+SH8lOAQRsvXrrPehfc02t8aEhATJoxBXwPFQsgT1zgkRCctJhcnVLOzR/ + 6AkWT59cahVhqzprl2k5GWURkxi3QryfBjGJcBPv7ZkJceuhgrhNg88Hga7fxLCb - XmAstBMCgYEA4jXYrxwoGvJI43tkTwqv/Z/lfLasezbIUrzgrc37b8c7U7mz9EKT + GiksSPECgYEAv5rT+9s9B96Xd0h29pJ99sNgLaus5yhGlwcIeDFCUk8jvMbzGum0 - iEg2c7sxYCJtCH2uKt1SkJNgeH/NZI1UqZjDFH/F6AvHKLpIg9FaLLm/85lBVsA0 + n2BCZ+4aT8BBYYx/qMdosZPMISmznMJkOEPPlCP4wHlpG6C9QGZR+uGiw8ZyzBf2 - eXXU6atLEY8tQD2Et562/NcagZgan+RGfYGArPANaYlPSJE5L2S9DlcCgYEA3tea + /s8CFXtRvlRY3e0TQDMHgKcFGxWn7MGa6K8CR4EGxdbXFmjRzEyBIXMCgYEA1gwn - ZiHg/3tQvk3mbfzTDTUwp8Y8uD0lh3ze09607nSoNdR2dnXIMH16OjfN3i1j+vnz + sYiRy5ZTUdvI/r4YYedsNrFlv5M1tFiIccS/wnFpDg3vBfYgeI8zO/iGWNP8OLWz - WJJ59rUEf9YPnUjNGyZ3Pe6dWE/i/hjGps3IATm67WjBYnUM4pw+cy34Y+65k3bM + 8C/lyuTgRQ++HoPahJ0XTywXZg51qZIC4GxbieKCbiSdRztpIb77+waJUN53j4VX - g3Cj4YIgVIXUYO4daGql4GJ75kWpA72qKsmlxzsCgYEA1REE3MM3n1Hwh5v0umKF + DOyK75hfxS1uE3NbK0ZbEQDGf6zN2fxEOlqmYhcCgYAEIA4+VusKd1VlgQ7moiLK - q+2MuXBSe+f4vb28HtkyaHGPFuiGcJ642Zey+kUqV7N1YZcHksZOe3DlX/p42qoo + JEy2zwJq+6gBampZRB48bW3Ei7gCNVPpNoZXfH3eh7Igqoi5Fon/gMIdWKuATYMg - QWpq7QcAwPU/DMSRgt+RASmgfHEw0uZNRs5O0h2OoqZqZ+TJ+i4bi4GMLN64zTu1 + 3vziIKAjbLnBmYVZlJphP2hktKoWENIFjGlsEvqgkWpUZN1MPY0EzRPEEIRMCaMP - jYeKTNn6uBomPGLVKyfGzxcCgYEAie2BD34guYEmNOQaoDFAoIgvmWjF5HNUa0wK + LW1sIrAFpGl/FwSlVGRXVwKBgQCtiDU2DU6GC12JY/JT9LG3zfNBdBjVc/d6OryD - z7Ck5IMoKklbGW9FfV3s7WPk9IO7wng6+rOO8fiQ1F82Qu/wo8FnRNoQYbzwjr3f + 38rHTUKqjklWP/CbTR1wZVAl+9bj8wvqkipuj5fy5YxxGNyz3tfi7BAcQWTLEQEc - Fxd/l+KXpKKWL86rLwfuT3RArfnwuylo5GIvzUCxqh87mNNJOHvqN7w9XAX52urm + CT09UFIGEdEgyt206i1HmkkBMxsjVCr641rQXGxoYyh2xHMJZoS2CDblk6dgLtDx - DJ3LEkkCgYB826c+IHEeXBLuPe1+TLdBJ5GTwjsuCQ2ofrXVvJG4ZLwRw6DE1i/L + rkRuCQKBgQCHK+Dq/DQmCi0yYiWz4gEYmgTPgCDOOEstKD0YbfFjvMesitgQYXd4 - WaqO2bTHqOWYVou7lHvEtKFJ58swPS08tXMWZYtsqHRtdxQ9x4F6udELjT8VOlaX + uWHcTMZcycOwlWXZR709Snrhnvy4xuKMlYK5/lHFtnK9n+Tzs4KtsDRXuIkIHJNh - SPuf/Dgbo7a/QDD5+Ki2hLn7YPXxXLJcOb0n/STn9KM224JviRfGzw== + JYyzXjQUwhLbMHY+JSa3kNVOjMkP8Rq+po8xC/6thsDaGJ0CoIQzYw== -----END RSA PRIVATE KEY----- @@ -1362,62 +1512,62 @@ vector: segmentSize: 131072 total: 1 - convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:SSK:2ipcnc5v4xte3a6nbhfiehe5gm:nui7axbsxlgvfocho4i4nvauimfl3uowlgmerf3fifd3cqgn3aha + expected: URI:SSK:jlpnxcvtg3ohti5c224dtqsfi4:g3exm5bx4ctu3iuew3ir633j4dieg4p4fygpvpb33cvll6jqj5ka format: kind: ssk params: format: sdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEowIBAAKCAQEA0FX28PA6wQ6SPAn0F2jEjuxV/DFMiW0AFizdltf58v+KJdbx + MIIEowIBAAKCAQEAznrHSzm1uPCplwOOjhgL+PvikNBWyjaPdZEGGBqWLIpr7gsD - QTAckR5k8ufUiaP3McGxddMeefWeQfg/WwD0XnZmKjE7pPc7fXH2vYYyjlOmARfv + ek/I01cdgJzAd3p2oJpVj+Dew6Wr2LCTlGFWaPLFiCaSlibE7BJHqhQwEDazq24f - 1CufCGlpDqskmFvD+3W3mXnsbYTDzfFWHA5r2B65FiudlyItzgK/gz6mJywdQbn2 + eAPbkxFT9Y6KC7bB5Vg4uROUdKqadhdN8aXOC2QqrosdT4IMzMn0Vxe/GrwanCY8 - cQTRZq4M45Teh6iKe2hDr0+CTErCEWtB1r7izZE366shnyx4zMH7w5dYgQJdCgTI + fzCrcp6DqAH2SEjlavcSlCelKgt9C9cHVN3XAAFEsVKAlDnSKE6CVE3UTMTKmqOH - aWhN/hb6y/tgPNUZurYBOIihxHgLfhTJENf3jIja+W/knwUw8C3O6B9DziS2YHrT + l84nGMspHD4BUV8juSsseKYz+Y7LElcdslZiPsfiEDcE9Tv28zWFlFG0iV36NIg/ - 3fYTF0XONBLs7y+o0QXbaRWyI8znhrl5ext0BwIDAQABAoIBAAR6pKl/cLPv3UL/ + XzFutNOioIOhDhWpzUSOENUuMXZG3cDINbAEcwIDAQABAoIBAGVOklfTYdjyo5LH - L8lFDlzIRfz7DlsyBbt0UXtJv2zzA4RWv68YGrUgAymZxF8FMG5YbLlMxa33kuR2 + mPsYy08Hbxt0TRD8AhlB5YaQDNyfseLinns1iChBVuVSg5Bbkrar4o0sXMALmixA - Mt6BAb/6Ka4kitS8IAJNbfGbLgETWVFSs2xLV8r1gTW4hjvkVS1V1ZGuJmAgZ5lI + PriPpZDqhIaPvl5TeU0GjwjgzNA3tqHG70O4SNR4rQQPQqYKrkmzpmkQNUekqRKF - 5AIMaVMnLfGFFIlISdXRB08KDMZwyW2fURRYqZgKxyeU8W6EfXDnOCRAxeGi3v4y + zqVgn56xL8vhz6jB+zvDXtIYgZhAjtB5gyI8NEZzzWfD1VNLwGP6Hpi2Y7LvPPxM - PFn9Yg+y7Up8sUJe2YvErAuLRKTQ0VcgvI/r/IejD57SSO7TQYNauaTx73DJcGbL + tjCkGN/D1ejbyiAHA23EjAAIeLF3xjzSViZv6i98lGE+hFe//EPDs/h1C+Sso2YW - c2u2S1Tf1wwlEldDu7JBZsmA5jFIe69ir4CQDIGxMFdCMdbflJSQXLMcMR0H8pMR + u6R9IIPW6hwGw7+wcmb4djEB6IkmrXwMDzRdPqbCLaN2DEeWk6iDBkgbggcqxxVQ - EiV+idECgYEA61xskFPjGkuhj9m9My+FB999YcRHuXh8xuhEBCS4+OvbpfV44QJ6 + uque8OECgYEA8AEj9fQCkzJ4R3g2Agj3eD37J+44cs8XEeslh8lZTFHezvQE+5BY - 9gkqdK/Z8QXGKvRgZRYZdwrEJxZZdB3xao8uIpJmaLScVEC4Mg7Q+3PyFsoeccmW + I0GAX6qsetKJYV8d/SooIyIaDy+LULaxBiipw5WQIAclHhFSTiZ9zskALzm3qUaF - vvq3mGZ1bXbuKiyQDHPflbzbYddTBoQE/DSZvlqFzo+qo2c6rNhJmpkCgYEA4prb + vJPzpljTCPskVDtXEdkRpfZ/uOXKQWdMHUQm4u38i84zLUxHBdMIt4MCgYEA3D2m - K6vN96iy4UP3GV09f8/xO8S1ovYwV6DYSdPdkb9fRQgvwTyPXlBcgzjt4Y+uoabM + hesxOjuUMQ9gNhRVauq1gEbZ3V9RcMaq90g9xsAXB6ax2YkFAy7fUWpfdn1LIyKh - O/YAcxcCSry8T9kWIeL+VM6LiFQx++AulVfWGjnOjyJlhx8vCvFyY8Osb6c3GTeq + Lt2aCVmvqUupsFLNOonDLmasOEj/RUbOxUg/TWpmVq1AExAF3ZQG3EyB8ig4uEFL - TLqoX5ps9gBarnItIq7Ii56pSOuyatH6fa4kR58CgYBRqXDVpvWOQx2cfs0BvIQo + ZJDgYQ/IWIofPgZ6GY5LGR1GX6pgDbqXCJxD/FECgYEAsYSGbrMu/GUGJga6G8M6 - 1id3y5WjSaXpkd8/nMo9PACrFX/KeoTVZxq+/+DbmshGUSI9EKznO+oRMdT50AXa + F4vwqtY+llyqeaxttAOvsw2TOYuv68oWBu254AjDTo1O4+CQs+JskZ/1mmnWJ7sZ - ljFIt4km3Tu8k/QVEkT6aiFePOTRUEOooe8fxrUJtREvuuSEHZQ/LRblXMOm6Bme + MK7+8hU75xSh1Z2GPRunTj3JjySnveLVpGfifZWRckEf29WQTzk5HoI2cjI06S9Y - tFV/0YLJx9lJ9uBJ5oWrSQKBgHPrDZTgdSNsi90KPHwgI1afk+KkNNphH8ejwyC5 + UwVHpe3VMCsyGz0iAyLWfbUCgYAh0vINJSbFS7shobvj8lF//xXq4na5MddfG5PM - HY3yHJUeo/cwuJJhf4Gs/Js3Ofj9b1p49C/rpEOBGr+p6FV7XekaI2ygzVTwkEPb + MHMUYBHpYed0gj+b0ooHhe+tUebFOZ9JhE0Q3I5G0ND5vG26bMfmC2ytpEBYElzV - Q+30hkLYMKGXhSQO8RoxvaL8IgZnYFmR3pHRWE3bTogQZiBo0rQBfM2NrJ5SPdZO + HZhjOlEHRMpPYymTcxVupe1bFGRJn/WFN17OaU8akfPkYbhEzn6oF7/kF1VzJlHl - 38Y3AoGBANIWVHTeBV3/pxZvzTWj3s8xjcultFOrli8p0W5XxFU9uMgv0IK9okhn + x6IFUQKBgDVkZseTqRYmO1dZxwEqAEgOZ+4c7RJ7vaYp6x/JZcmDD7UvmH/QAJUG - TYoEJMUvHsIsy4bkpUELcdAD1jmMUtMgb4T2cS5VbBqatigMjMKeqeaNB0G3dMqb + J6S3uHv23lcR8rwtAlbHT1jIW+kK2vst8M7W4kRuXafj8nW/mk7mDMXzXU1rD56g - HTg8gd98E/5UJ7gomC5sE7upU5HZa9L/TPODa2cvXZqwiTCh8XA8 + xhNZqMb2ruLqkEUMRpbklMHlKxlZtk5NS9vslPxFPSDqPfwctu3C -----END RSA PRIVATE KEY----- @@ -1431,62 +1581,62 @@ vector: segmentSize: 131072 total: 1 - convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:MDMF:6clvxlbhtt7czgom6ofessp7nm:3lg5q6y46dnw5c37kvdmtyg5bgqreaexhcaah36elawdh22tbm4q + expected: URI:MDMF:l43byqsidupuz4k4kzskysj4hy:fnacy4ixivqrmptbpd5tvybb3uhkha6zxucyj57ovqbiof7dvcqa format: kind: ssk params: format: mdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEowIBAAKCAQEAwdqgG0qOUObb8fOyQ1HoCtvuKs/D4TieYsPuaCOFXuSwqWH0 + MIIEowIBAAKCAQEAnYen24v52vWuFDhJps20/WRqmksmi74optyIDS9EE7fL4YRr - 873/kE8jUNjdklCRxP8+V+sU50nA7pZaR6oq2/n9Gu5y0Utbg5U/OPWTp+Osk9gL + w1EnmATjh/09h//Yu0LMjEwSwfOgfKK9VK5FmsnWExfSDYV82Gcca87JRNdmvEvo - G6oIzJmaij8Y5y1Hb2C0WTQ0bu9vMzgY2LtAcApCm5eFt9EKa7VbU0NgJBgj7V0e + fsBMZOFVbJDLvgTNha8Y4cHiAmtom1fTSklkEwml/pVPvKnTPVT2rFTp6S5iOun3 - 8zW9dNtW1SLa8VtAgKE+9Ts2CrYMzhYWjfKg4MW7v7kaE9SbRWCFdmkc3lJ4bl1J + W/akECkzPKXuUJ6erpvzF9ksQloyL7d+UQyhUjyL6+sQ8L2RTkoWWhDiTagXiZJT - BmK/AYkJsK8dvWZWwvT2rr+vOzAkz6ryRzqwObuebnRcfWhvyAD1PC0llPo0TVIh + tnXvDXugY917TpTYfwaLuscpbzmSc9eaXTBk8cC8ZGSWluUDaC5PVbheSXfYFLoQ - hwGS6m1KYV2xr4gx6ZmHwJ0BzwjdCsmx3x04TwIDAQABAoIBAA0AtdH1Nfpd022O + mizLMZMCz/Pih6kNWiFb60T7l6TCNr1R1YjhsQIDAQABAoIBAB8p6A4pydsEQVTk - 8itk0FxmGyGtzo6juXzLddYJx+jkP9tLGEVli0GQcibhA2x/t5j2O/LLigGBJ3R5 + scVa6pQ6WlB9z3lTvC0OcafSEvCnqqDJlpwEIQYU6YJMmfCer5yUIW0b25YdAUHG - cYD/SSFFl+ey2QQj4PWnBAobvGjVZl6DIV+cj9S72t6G0/J+gLbX6DdX8pc17tU8 + 3Be1hjWR+lS6oKZmIwWYmGnHdc+1oTBc//ibSEGoxkJ8/qFvx8zLj+uRdImv//jD - xohHEYMfLqksFTD0T+KHB1ZC6mB3ss01jr+9wEOvByW81xWs8ybA8L4EJULHl54b + ThxjGnYdsYYEucqD+jMm7Mm43rFvWScA1wAFlgKCbylLXRZK6Z3R0sTE47SZrNX4 - ToH9b/0k1xqzj3e8WTn0BG5quXH8VTHH9BFou5P4tPBBo23uYf7pmaD5YAeVlHaM + FGCRssorYNybFgw7r/7ewbYyKJpLtp7r5QB/YcQc3/ZZKVOphSqSwpRCTDLTMH8O - LU5XKr/fUiuGpu8pUJ7VxwT7yTi3R7UsOrKoVPrAdq+FjE1HrqOOi1XMmrYUiaWR + MwhbefL7jz3VJPzu3Solm9W8udhu5A2iXhXUj3IL86N2ZncEBPyh/dRWwKFPxvvr - gAow7EkCgYEAyCtBvim/eqv0J5637JoE5VwEg5OQVYe9NbEA7WIUFu+MUD8oq3fB + Xyy91rECgYEAzyqf53UCCRrKfBaPgLX2bl/lN1cdJWeRTE5CQvnBnE4Qhwfj3nN8 - o/fq2z/F4LFuhni8ow1QNeZ4ReJnGz33VkM/p0owCVCI+7F9ySM+fHz9NcTwdBmC + 5yM597eA6ROLOlZmFyAyKGK1jtkS6ojZDnlAr7nVq52+kVqCR86REybbbaXjzT2f - CpWrfhSsfTNcNkQfLQLUHEwI72bWpfQC+1BgvzI2fEZfeqiXt3A4ZBcCgYEA9+x1 + Jb7a9+1pQZ/bb5n7xs7GYH/LX9I3MCPhSRS5apf2G10GPoFiRe6Iv/0CgYEAwqm7 - Zt9pQCgDwlGkz4EVs2TObM/Z0ouNjVcXF+qMj6aHbKKoW3S/DkzGlbfZPtMS/Uqa + Q0hZKxk4D2+mG78HNKgh4WVR2JTRrRR7XaVTXWZNuVhKW8BBEyniFNyn9+AWQEAY - U60HFkyCQTgmZwzLj5FWJ+GSCpj28/PamiQOfqBoZ0AH7OuTVw3R6C5hQyq5RklP + jUx35hyiFz6DF+TlGZ1cPOTyWespY9JLi/dIdACc9K1lWAcniO7z2ksYKycPI7IU - GjOfqwDGBtoEp2PdRV1rj1ez7awo1pZmsW3OmIkCgYAFrxzFzp+uVxWuzlYAtPrw + 4K98qhrbfLJPAx7rF13uKcp68snYQ57g1wWQ9MUCgYA7gcnmyVRpWxm4pR5ZYWtE - nGVQay9NDna0AJu7Ie7aG+FLIhAAlnz8L/0OTshKsh8mWGVa5/TgIvRFX8F3x5Gv + 7yS/TbWgjexNl9kuteEoTcAvmVOaDWBeYF8BSeOsj6GZg0HV+LiPozL1smLdnauD - dGdpU7T7frr1Erw0qviKRm5WSYpecZ78t/VPtjyTrZKvw81y1MK7LvmN+sibm8s9 + nc6361B1+FzKEc6EY9CGSM4U4+bYiI/TXsw1FSv73rhAiWGqDLEs/OhlQNP7bwMC - 4bFtnHppmwH5FLKCNgCT7wKBgQCaFmCxW1FzCmurrkqcnUH7iT+y6UwcS5firKox + ZAKSnM3jtEfb4nxhDBCZ3QKBgQCWZyX02lVq41VZN96T2YjrumxTBkGyoWlP3V9j - txk9fubUYhP5I4pLPPR/wRBIt68pteBM+VFaTpr2JgvYKF+sD0xY5R17cK6r2HeZ + /3Tl2UF8TydEtMqSz+2KSOLOtij7A4r0wXxyIvVqGDaZo5UPsXGu6wYFS5jzM2yD - LafEk7XP1kAWxCODC5fWklzo/fjA8nczdbpa8dQiFgamcq7nmbRsFrpBkaqgFEIn + fFBSsJaUxdRjq0N0nYtzwkmuLcOYxOM0puIfXBjxw6MguibSKxT03SkZpbKerIb0 - LHQm4QKBgCEUp089t+7uWuya+DBmdM3z7THKgsE7FrM6Xn/NTHDH/3FEmEzRoiY1 + G6zgMQKBgCU5nTQTh70j3b5mcynwsvOwmOD0YEd0Vgps2VbNObsBB0u8UkcUiZGz - C7UV/3xKqpSLwDORmJ/0N1pxCyFqN6lW3zMsWM11eBaLxbWsoXm8WKEsvHOTXg+F + 4jloIGaqTsf6dsc7hdHdkL2zCiBtbSsur+HXpVUl0HbtAgXhsrCu0uzLHInTvtg6 - hWkdjQMGG/nLYpdPycaLs/qDqMybfLJXG1ppCa/aLJMMS10r+oIA + K4E7oqX5emi/cyUG6Emx78GDO9dfYm1dz+gTZnAek9+aQyo66F5i -----END RSA PRIVATE KEY----- @@ -1512,62 +1662,62 @@ vector: segmentSize: 131072 total: 1 - convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:SSK:5jgelx7ioj3fl52fobc67zccg4:e5adtxw5e33jwoqi4xqj4xaysygxsb7hgcls5tjfx7grnsgxxv6a + expected: URI:SSK:rq7lxvy7oorzoamt2runjfgvky:ebenedfynevee34zqofeu4vtv2bgu4dfqhqy34yry5lb6x4fdy4a format: kind: ssk params: format: sdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEowIBAAKCAQEAizP4H3C26DWZsfCkJLixuU8B6rZLjWAGwoqGdvJTZu3sj2Df + MIIEpAIBAAKCAQEAp3GpCG+2qiLvwO5aCQrOrjr8Uzt1502l6qc+LwbJ4MDgH3pw - gqStygSVKrJnHcwRK9rOmo+unH4X4WgMb0klH1whL4MzvDOwhX2TuGiC6/W5PY4O + re9knaikOrghUl/qOo0iVCImZoBItDShEU/HZSXDMMXyqZVHNnpnLNn2EQOgHXdw - WjQNv+2zTMRnwYxKgdMOvVfA+7bNoX9TnNjjNOyB+wkvZwxx/Ck3RCJlty8XJk+f + OK6uDNONDJg7rb0zIpr5sF4HoPukO6odIqogsA0S3VUhYPL9YE7MHEj4IL91AhH7 - uLFDpRa4Cu/4adal3yEdxh9OGhYjlPwgS/HI3Nl5IN0S03XZxNAtwI9s2yYHPXB3 + u2WdKvxR9ZOZ90Kqsha7iBhMzy3/JtdrXRXP4NRJPTGDSGo88bb33P/i4CN/GQ1W - bKqZHXn1Om9BosgXR01lZbVPUlQbDkjnZTNIIOF8SHQSNTm+Bo7qVjNrhClLMIg/ + Xu9WspqE0Jn5yOO5YOrxvwNJK8l4dBCqMmvGNLucul7UYDsZZNHbixOlrrRexbP9 - aWM/BBnqfgwR45Ip9nueoRM1cmWG51NEotO+lQIDAQABAoIBACqkH0gmQ2lLbgrT + I2Ho9Gsfx1hN05vKNlLZ2Tnuef18XeLgrceTpQIDAQABAoIBACA8Nqsv3IXdA3SM - f7yd9RciTCCFegxTE48JVxpdrc20aUgccSs4XeIp2DXNk4fNqJ7p9mrjQ6Y9e/w4 + PmuSt87dfrGkVxKwRWKLD2LcxvUclJkiyHoHxgI/EtzWEV4rJmveu07goy1lAXol - 3sJCQkRieOnwg2sN3G9v4c3V+fDlAzsHZn8cPfAClO+ZpHzmCDbPm87FcGDLBR+I + zqNHTU746d6kIQ8KNMM1ZdMB5AgK/2Jk2ccjw9Cm0nbsAMM2ExfUp4CPXZ8ditTM - /OhpieP+5OwsyqAC8HHBgGP3M3hS2VlxE6AW6F9axpAr37v/ruJghlYz0TRCGGLq + r166y1+xKKJRDwO7y9EkYlGIr7IX8XY0QIr1FU26QItZXL/Dtz+rInVx+UReL6OV - 5GtkvvZ/ZF6p7NSX7fAfdx6zSezsKQWH0ikB3QVFt2audF3CzRJiBJ4d6U9K4Dud + q6bA/28gWCOkCeoMqlk+LGU0WWAFryoJr+Ob/TjqkjFX4nURFXmVhJwaqoVMZHU1 - 9Jq5lBCosN+EmPgWfEQzBXo49elyAOSRmYxzfSWRqdCs8XNlG4qTtj8oYrbHTDeh + J2O96V89Y7boz7VAuoZc9fxy1IH/q/p1S/DsD69EN1elFs+g4TchWg8AdaYK2xg+ - AqMMlpsCgYEAvWyzp1W8aSgNNs5lU9WLPr/K3PcUnFeg5Vm+fch8UG5ZcZCpTcKH + nyCS/fkCgYEA6GAbr3Mf4++xg+Iq0kV8lFRrxH0fSMhdLeYJuQjsgCjzkLpuMUX/ - xmzADJi2eLehJtovBhChN2R7THgpqOQgZZFlQcsbcYcNYXprGdFW1X8zhofuWXzs + JbkP4LoUmv5W1HQ+Gq7ZAOmS2VseNjaeWWrZwdoR7s478OTFbQwIEVxOpyzi3MR8 - vD0UkG7txNrxZINIQFzLwbrb3flQmLpsrtpphniXIGuQGuOMxqIIAy8CgYEAvCCg + WKU6KiTCzs4ijjf3f2oyf7Jl5/aEnJVhiZrTw2iNhk3jnqZA3h0AvE0CgYEAuHef - STFm/U93agdd+w7CErkzuYpdWmbqOG9+jN0Tq/8/k7L8TJyfkX26E6zBQlSZkS73 + JTQi9e3rGr3KWEl0vhbjB7W6isYdh8+xZtwdYHSVdogQuO0TgPooVTkCrJI0KiEl - +HR2aOgVmPxynDD+E9IwdHLxP9wRbz9BYsjptL2Vbm02/eI3fXVmt6osHRudzAFc + xghSqYF4OAh8iLHGtexrTahw6/xkxAx5So3PHev0eackG2ZFVYbLcRxxHL2n+jew - ggkhD8W6fUqoNB/Mbdhplk6+tlXtTMLIWB3XeXsCgYBpZMHQqPNbzt0LUWsvafE/ + XAQ1j+T8RdWOVdXIkWTOoKq+lYqwrB7JCUr9gLkCgYEAox7cMGRbTZF0BkVck/Ct - yJamuxLMqjTrZzOF6LbCSaOafFK24TWKQZfZal6cbA9N/reLOFV67H1t3q3POp6L + TB6a6/p9XIUyS8cAqkBmbGzS1ZTZR4OAYUWwrKtTTZ4e69KRyf9VW7ubFzNMWPgs - 5IniQY/TasEXK3XLt54Iy+1vPNJxGADf+1wlwJKqpOcKdcENjpQQBleu+bjOQWuX + Xk6Qf/EJx55EG40sPalFfJJUsCvlMN5I/5004GKf0baIMVd+SJYOzu83dAbr/lMq - Hg74sr/jWfWkAFejbSPoIQKBgDRksb8wrwolM5Cn9JiTB6HHSoyF6HHg76JACvKY + fgMOhky9lDrW/wZr4L9xRb0CgYEAj2XeP0uSSd+Tvgv/ujYQHJ0qC5pH0w1Dc4oO - L35bXA16b6G2jQosBcKs/jXG8e3pMs5TQRb+a+VriU/OpTRH+Y605FNwqrpc14z3 + /EjsRUkbzzLi3P7fBIpyxB03aPOWvZFbDeD2cXKGA/kE5jZcpJuOpqXkcm6X3pdb - f38Cvbc/W21hryqVo8HK9vY0VsIWLvlYKYkG/GUgga/im0CMYPunep21WJ1kMf+4 + yosGkNoWCGPX+7y69Ut95wYXICKG7EpSPJXBFYUKXzcuGKfB7NSSk+9njFRuFr8v - b+Y5AoGBAJWjZ3WiFf6b6AWlY6Yg40Pya53TklFWehWXWvZj6JExxZ4wDDJxFoRY + xJuZCXECgYAyyV2Ql8HWz+iF56c7Tar6vjU1iaHHp0iTS+OiZbe3iGbBaHbCpNQ+ - U5DF6w4JrTHy5rSeyK4qRjTLmNgU0uyVzsVskXoPJ8ed2iXA1FB2P0co80vPncQd + SiAOcFOhAJjw/0Ez2Gffvmk09TnjuiXBUmYHgZKIWIa8EhAQCklLDUl4YmqcRA2s - XMlWTDK1zVRoimOiVhbX1wPXkKgRTCTxL/SBhz2TZB9OYPBZmIjA + HvvNdvNPEnpyoXKOe3BmANrjkNz5Yk0gtOXDH5ar/JgweQaTxXoflQ== -----END RSA PRIVATE KEY----- @@ -1581,62 +1731,62 @@ vector: segmentSize: 131072 total: 1 - convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:MDMF:cxqvidbgpxb5bl7kcwcl4bjld4:drfez6t6wd5dwbg5le2fp4ksugnjex3nksqksss5t3pb6pngmula + expected: URI:MDMF:syhziku625dhr3x3c67oa7d754:cbp5qtz6vphepl2tmyo4b55o5x5afrdjjjyt5i5t35k7fp42qw6a format: kind: ssk params: format: mdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEpAIBAAKCAQEAnoqIdreTcHst+7JUmKdbjqtjcclyrOWbcPrSiUb6gLRQxua7 + MIIEowIBAAKCAQEA3NxdDuRuM4xF0bmoRI46FoksTeyQwtzCYuL0eEhfbn4PTB0F - 55auJN9/DObSrvlgUFRUFIOLpjwPFs1X+hkrtUg4UZQ5j74bq3R2bjCUxbxgVVgY + jjcTcDp3Lc+E7FOT1zUU0wp4rhvOZDTpFjrW8n0+iBZzZCgka1L1sEtncPDgYb+/ - 9wd+KixeTvnVrIwVAhKEgCKLMcj7yBW8RYVAU52l+yIu7Lk8CyrNAM2/F0GGo3NS + VDtSFIoSKqQLmq0L1hdA5oLgXWEE918kLlNjN5l++W6OGfzB0ZaiWbJfQpOyYWvv - UaQVyCE7aMgZigkyW9FgRdHdSdfvwOO2HFJfzSZXWR7QIAysaFkNsD2+SoJnrHb3 + PN5P9CFxh/aNO4924oivqB1iCRJbP1szGcEuZ1L8QWISZK4oauQA3wO05UGPNrQU - 0be+8TDU8+Vh6r4u4D7kugvKUhwR7crj+yuGaWLBc0XXOLUXGdA304/l+8dO5Bxv + XIEJDUoEtTSlsuOV+LZlFWkEHD7cwtlG3ZF9u2r+iHf0xcOPqsmqSIpOjPqg7Uic - fmAS1F6SqgsYPnHncvIML78GoaqUTBoKXbG0KQIDAQABAoIBAAWN3k7vapCdcMyB + Qh9r/g1WwKmlmzgYgg2dWt3QUTv20WYZ6GOmnwIDAQABAoIBAANwH2IMy+sGgHcQ - WxNFLgZVujGw2viUQNb1+I/TNGCI1flz46fEPf9Vc5y4I5p6QCiBncz2rfr8c2hg + IPPsPjA9SLtvzB2/FPw4FIkjde+A4S5p9z0BqM5NmARWX3cpopADi1mbp0mX34xv - eFlZuKU8vZlbt00HfgNqkfbACF7pillEZqJ/3mixPiVnkq0SnsP1nLNQUn6jjIY+ + cb9prBCQmDQYcV8Plug1cuNLJobmC6rJ3X8WAKRBiqCE9t9HWVCnAljJlGzaW/BD - KbF9+WAR37AzFUBZPcbj4IOg0aB0RB1BDFLAhvRchHVStC1v1deC8XWWMNmxYVye + 2ArWkSFQ74k6H4EnuOv7yyGCpXlM+YfJJmAXSVM593BuERTM3//Ki5ctMwR9t/CN - /zT6CDRotz7Ygp6lxXZYep029wS0vIZlDBTURGuLqDPXeDe2iTn4CDT04S0PVRzQ + rxzDVrGhbIOKHWWAKfiNUseZVuy5F5QR7772QU+mycAvW2qVLeKs9lYpLekwqlLR - 0jA+kZKSejHTL0lFXg3c8sqTOH4uJrb7d8q/+H8nNX2WtM56O5gsbYWTRCBKVoYG + TWYjE3HVeuhSbpQymHc6TsLiADlSVgGVC4mDeg80Q7vLEyM+BS4vkouJOkZl10ZA - FhXwcpECgYEA15kr2gAx4YpR35sokgC2I/lPe/WP6SVmaKU2FYTFTFaL6dAZtodz + KdJT2A0CgYEA4suaXxcc8IrNTyE7lS1NCAXMYaI/Qlmszy3KfR2aIFzbJWBRTlli - ebXJQNZr2dfn8FXtviaNWulaqqNNVTgEz8lTtwV576BFOptVGwM16t5eaMIfw146 + U0WWVbvVkcs616IKEb6jMgK3gJK9Sg2+IuT8W/DBr+q68/iTSJcjOhWD/OzqCHpB - fK0sxGpUqMRSdMr81pzTwwHcdDs2j7B5fE2HfvOh4KVRX8Ck4tdcH7ECgYEAvEAt + Y5tAkLA9NZf5QUcS0eSaZzWJdn3s30r4kTV/ZtfSbsXR97mlb8vRWaUCgYEA+U0g - FQ8IJ09/9m9KL2mLoU2x2qpXqCPXdyd1D2Jokm1HUdgCCG5BIOGvue4SVQZ654ek + XARhs8QDChYNRjcBnZQz8usHL/LMXbO5h9NNtdfqhjYaBljHjYgu7cgEeaj5TMPr - 1DLU9CgDr3z03fH6FLRQdir5MOLx4kih2uTt4iTo4Nsa2Z9pwO+axORUvewW5GNC + rVKgVUmIgMun7mBIAOTsQym10tM6ARf9wIR1+yYxbtDgoeaWSje+InSaG+3hYVe8 - kYL9Yy1Q9rg2bav7mPqDlcAUWsUqGmsCEurmMfkCgYAgtNXYLmtiwa8F8u3GqGD5 + o/T1QQFQPW2ngQQ3OZ+YRrZLCqVw2qRLVQuiI/MCgYEAo7WKsjdR6YSYHRWFF/LC - OBr8vRXl0oykl1uLDCc6G28CO1WLQSUdc5xiP6UA2SYQaZi1XffXsMrWVAupP+RK + VxcwaA2hEjj/F/Ia52OV3OSKQBmdtyuoYSmrEinrSTllOUA7eoGc9b2mTkYeIzV2 - +Um/3A7RcUjPST0x6dzGEpHT5o8W/jZ1L3g5G8BYEeBIY3rTu9rMHH4rC8iNJ8Jm + WWPnkkpg1aZf2zpEvrJyeDwNsWYmrYXqa4cm/QpqtKQGBYvTVvVoSzYHCyRs9uX8 - PwStF5yZDbs4gWsCFpWdIQKBgQCEmxxmqikPL+Qu3uQ+E7YlEQrIwpduvJipuaSv + NX1jgI4r1VAwd1xnwiJi9Q0CgYBS43/k2FgbywovqlFTjSpuWD5FgDtth87HQOBo - Cp4pD0te7q836xp7pB7Z9Ub6l875y0YjqA70Uj+OXZJLyYllDkNjig/xDNxgjtNc + 9qqZ0WZapVZV0eLXffYMfTpvsOzixylvAU/py38lQ5FcQoruMS8UzaN0q2JXxsBJ - 00hytZdJ1W3LgIzJOL8oFMNQ6b6ScQ1SXRhKxYAz2z2T8cMQVt9cHGr6Kcrnwxs5 + 6EDJ9lLtQ2nMqrxBhPMkxZwPuTH8iY1g/islJ+ij4/eTf/FUqWmZ6TZeHc++Am5B - 4jf3WQKBgQCpJhKfp0xkfSXRxewWn0mKl8JiYTCQaE2hlDJXnJMkTGNGSB+OAr/R + opKQXQKBgF+86OH5QEMW5MMMKBhL09uzyMnM/sweKLo5fNYL+7hiO2Znb4sz55u2 - Q4/O5dsO1Vr0jSx3w1oW/7EofSQJvuByVBpJm44p574B9Q4CD+nH/KKEoX+ZJSUC + I3xsH9+OMWq+o9Uy4q4QIqgGApbfrujOECe1z+j5mZBRTIpWTtW4W13sjT0Srqbw - A3qs9c2jolDrPpS8IjJERrmZx5zTfucCVNOR9HGhUJ8iCKvEdEp7Rw== + rGIvfueyhQlQqxcKobvVGIdeU8co2tG03Xg9jlFhrq8vMdnxg457 -----END RSA PRIVATE KEY----- @@ -1662,62 +1812,62 @@ vector: segmentSize: 131072 total: 1 - convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:SSK:ui5nb2ljhekqtnzsjjxd6kvjha:jhqo47cpqszio322x7ya7a32ecn7y3einrce6mszdfnopyjlfita + expected: URI:SSK:ad5wtml26s7jjv4gfwlrs2653e:hx7zlqd7fp44i6ky6lfrrwcqxslkg6gd6x7moxt4kxkqvm5d6r5a format: kind: ssk params: format: sdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEowIBAAKCAQEAoMgeFJ8lHJqb1Ygnb/QcCmHbv6DQGIzEMVEwigLJfKzCF2t3 + MIIEowIBAAKCAQEAtrhCulmdP/BXGRpy31rlHCP6E1vlAqnbaxWG3o3o0VlTxS2a - UgYj5hkxcc+OlUUjfOEH5DyoN1mTrieK7Cn6+XULkdhyc6zD0uMEN9oc2vcEEVnG + Bv4x3ac/KhnIfkznrXj5G2o+lYNlWNt5DhKIInA9qHsPlIHeXHh8dtttnP17to8Y - fQl46IhMyF+7/aJZ11cWH/bCSnSm57jF2MG/bq+sa6LjK2z8ydthTWi1PC1fFC+p + YtMjGikMyfSF5tTCCYYHxXxFknHZ8Rf8eo+JCuOzFqhO7RYSN3Rs7zYTwMeG7rQX - egDOEzWtr/3P2sM9E7UUVRHkeVrDqcOMgxJq9gYtGI2GCYnx29rCdh390GAiU2wr + PZgrt9PxOYxlGcluBsphr+PAm+ImFg56vk9Xy0fbDMXI9q1Po3W4eo2RTyDdmBOl - R3YIKqRmJfRb3MrkMYyjsP3tsS2Z6B2+gGAdr9X5ydpV0ml7mKrPEvtEE2OeGFiv + y+v3MVc5L0ai7Tw5wiagQD5nnJDPwLcEEsE8kj6GVa1wGEUG7eERzS+QsoBLlmkW - oYh3Txl5Hochh0LOyoHfTbkmzAHz2T0LHBNDRQIDAQABAoIBAB3WveRoW7AZDno8 + GLThyO9/vCp5cxoZ5LBIKUfq9W8D4bH5mauzxwIDAQABAoIBABSh5o8KXX0B06P3 - 1Erd9DVGD40bGHux5jhb38UBOukRO80yY9jcbF2oB8neMhFIXUtwDPGiAzsQfAyq + CO+ltjyg51/WPvGNDUTLRSkjrHLZ+7ab4/T+29H5TQ2g+bMhVgE6/s4Usi/J4DYn - aIknSl1xCDZnQ+htZANXn+EIsOm/Rak9rs1mTGLlZtCaGc66ym8hSakhd9HvH8mp + b5pM3L3JdNJDNcrW21gPwZDeTZiuRZD4xDqB4vMqSCe/NSY6nL/cjaDEMUr3EN1u - /Efbvz4YyshIGN5mkeyZcw+1csspugnDc7CIwvroBrqdJHs82hXM6yscWsLsthdA + OvP6pJvr2FDU/Tc9CHOQnFhqqn0DGfrevRyBBD12AnHdfA/hlvquMdgp/u+MDeKg - HvuY2WemSxDiCaow1OP4pMWcrnC16G0chhKqAefTufiEdcJchM34dFjlEOW3qsFq + 3McZw0oVDqHHb3t/GwYNvEesMKC9xeA08YIMQOMYXyUxZ7VILiqVhUc5F5Kw3l9D - ghnD9P2uG5pI/vYP9cLhLEqTNTTMTq8OK7yjl3GufoDleADxzCIPvkaAEnSstKEu + /SGn+DxYggL2+C/JwwQzbgHRCWulZFe3l8wDI7cOI6ajmjJBQDMx10YYeZsUHmEq - Fp5TUZECgYEA1tXAQM9iq/ZQOwy3etyzRRP12aGcEPpsBCzT+aucU/sRPntU/iEa + EkEGOuECgYEA1d+v3L2mRJJrjblh1juJilnSMnsIiuw6W6fGBfR1OCsmaw8MQ3jL - JwZ+mwty5LzHdHU8ssYYA4CtvwTgGxpRFxdSVjqwMRXXMUSZA1bt1oUD5Q6yM72R + 7JH90L9kHtchyz71lCAf7V56txXxkHvn7K3zuEn9qBTodH5GSKYJFYW35zWwARsz - ysf0NeMQgbDbbVQhdF7cKawOMRzwzzUkJAcXECTKwYs+rvCGCRWuj5UCgYEAv5bn + J2LifWzQLYyKg3PTcIm5z9WMCmBovZhNQivhhqRdGeRrZEyVWg9aXLcCgYEA2rWs - pVOkx2CSACvXXvIBZZY5MRzzoyHpgGHOUX0Z3lR5dfLAfm59m8j6VDDNsGRYMN07 + JM0ywx5F4S51vifdM0PP6VbVCYPfXeZ9tDLkdA+Ztz80/IpNUoQGmfmTFXk3YRsK - 7AMZcVp1J6HBuUSLuE6wes1q2FYu8yctijmgMEOpCXT5z+YWG0oQ8AEiBRSIlz2s + bheqltMJcqC7xUm5YMJLGlmLkYRj/5s9Xux01pYNt9TDJowu2It/d09OIPK7+ioU - 8NmFgUmyeJ0BWMog4By1zQtyRZoxGIgx/VafuPECgYAyvBYD+DXwMGIwH8ew3zAC + +jiparuYULKNsEJp9cSRXyZ8qOKB8PhX+GsEcXECgYAsDynOgq9G/xbzGlaiaJ98 - 7zzPIYhOxiT+M2v3+VwYxSEEZXHj9gNMFg+OI/0FIcPkr88e1QNUyG2/v7IBFIzz + Beb8iUYIQIQBL73mqiafzJvcgDwZhkAUWzr7jwIULGOE2FKFEl0hbE5Be17JUg1E - 7BEIxiFX5jWEsBOGo1/VmmIaFQdmiq1Ee0Yj97StPAwF3Klt5v0NZlGPrar89CrN + P82ukGeWAcClhwH5o2LJsUNieTfp8m2GVqOsDQeR6pr6W5kaXPUPcMGpvZS2QjLg - y1LaACZV4MFz5N9yg8lOpQKBgBbYoODnG4Qm8OISWElbJG1/v2wq3qa6WYTUpOy0 + R+Ps9d1MITdScUhvRixqXwKBgBlQB3Fm8mYUvd+3AdeVQ4uoYIrQCu4D/jke8ROH - tUv82MsG2ot5E4NrMOavNyfsn1OcXhPjvrn0pnnGYTp9gQfGYmcSbcZEaK7YIicU + BFvOZmsH/LjxxMs1DpKJiRVmJxutBoMBaDP2jtRed/z4cGUbd5fAH2AjI3O04uB2 - fhSjTNny2ANBlatFZsWn7O2cKDmYwjGqTrA/IIgfeNSkrczrv4Ym8kZ4f5hETWm/ + m3sueL36+O8gMFfNpV4IprE3hrwIXM8s+aapuZI1aCKrPRo9utl5WdouBP3/sCbH - VaaRAoGBAMs7UVWOCybQ5mQVzcu99wWBNfi/pIRt9ypnGbLwgvH+wrAsf6w8N9tX + NAdxAoGBAMTddDwt4yuRavBVlZSn+b+DNd3vV4skOVa+zV7Pdkqf2MLsoksIwQhz - iDFg5j1nHfBSr7bc8QWDvzp+rqNJcwoduDpDQAtFfYZbBnzKsp1nTEExV/XZP05Z + DT2RwZ40+Sh6JIBaxKERIRTUE0o9zfWwWQ+csXAGUxUo4Pd7FV8svwfKVl9vqAIX - wMhBGlQ3wpWMCYkZi4TaP/qGvIh+3yt9xiCdXa6Wt9uzU15FiaDp + x1XcKc+/jgT5lvmkpv2TFwke7S0jNvUtpb/j81L2QnYg+9nFUxDw -----END RSA PRIVATE KEY----- @@ -1731,62 +1881,62 @@ vector: segmentSize: 131072 total: 1 - convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:MDMF:miepn2lr6abe477ynqttkf6rqi:kd52ue5gkfi7chsmtk3tciijbib4fmgimneyybfazljrtkczbawq + expected: URI:MDMF:jfsmvwzuduc5flywq6or2sybg4:uguwgnvjuxfn3ivws4zpt2dai4kzoucp2zr3olid52xeln7u4jqa format: kind: ssk params: format: mdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEowIBAAKCAQEAsAxxyLdBwIlwo8uyK2iZ48xOXOezQkGOWqzvVh4YJrtxEhJp + MIIEpAIBAAKCAQEA2K/v01VINZgu6XRXhKx1Q9Hl8zjkve0R5lQmY54p83t6eF7K - naYXF1mzk3SJgTMXC3NsWnA6tAFRGBrM9vbdYA3MmX/iq1eDBafkswPv9Ggsgijw + iYqi9X9pFxVmG+3pGwMwmA2ZO1lg6+FWhiSazDk9O1bEdK/fM6N6Pyrk/s1imIf0 - QLTRP0sPD7YSmLsO+ZpmuB5wFq8ckOg4GC7Q3uRBKpqQ0/kSDW1hoghN9XTjaJzs + 4lUkZH1tW3cKuzsaCSXRvPKqMRXqr45ONhhNXpJtxJcsrJXJrQ+37PtqJnbQHVo4 - t/SiLlYND/5hRIElLS5Ruq+GFmpz41TyRQouP2A59Rz9ILTolA1jPyfEv6AAtPLY + UzqCBOm5ytUJp65EtQDMBCdf8up4E5HvpmOzjfEw9SoJ5bREvuF32t4QNiBvkkjZ - C1taYo2oVS/G94LKMqYCNjWWYt9T4lDBci9OFj+whClceDapFvAym4yJeU1WclIq + Xv0G2EtXBsO1ujpCQaRftxVCFermNDt+qL6s6pnYsfgpNMBmMgI8YSr9aNQrMDdD - rYHwQYGMH0u5RDJlu0jtdXJWqDQZEhZLIs+KvQIDAQABAoIBACVsO8rGM961CKH0 + r50e0MwnUo7uyqQuZkuhC65pdaL0Dc5Fqi4dVQIDAQABAoIBABZ/5kImZ9IQ1EYv - 9LaYCXB8V2MV6MvqihN54/WLN6iSG07TZaqaqhlvWsY7VViGzv0C5/NQnJXzmrS0 + a2r+UUrSf7MKpE3IUQR+lmHfqXF7z9Kx3Qv9FkCxkyLveOPLh1njsecH+nI8LKEx - S8Iqx3O58zZlEj7If0RWRH4OVfV/KIjxoWKr3TgmYTj+g/T9/IiwGupEJCEaT8j4 + i79wC5bLFr2Tm+CV5nJBNk9az95ZSzSVYWsi9h1tHK7TpIyebWynvaiF9gAUy4Kh - 6CWx2/opjLW8/hDlRwJeME0klUfaGwz7CkveSUP4p7lDZ/MoOm3J+7iC6wIJLFAJ + HyPk0BvSzo0MOXpOL1vF9w4naPVHTfC+7GyaDld2OeZqIkrLJtXBxMnq/C6WTRRc - 3D3GlUYtUw+5+Gx3zDFXT33SlaTtIUubAzloVUvAG04o3bYwTCNpXFZPv6JAH6gp + yurMaAaQCtp8ZgjKaw8vdud+KMfXDJZqNIxqjDe9PN9Lzh/sR53y0BifEYKeyJ28 - 97NjrvCPsPx0d2xZA0ObTXXc2MqNkWo9BFOzHecdKHOIL7Y1G2UCH5+y+dtEsm3d + PavJWKAIUH74aku45FFPjZEIJ/a/JrHEyTeOdcn2a5Gx/dZUN/h9zVtZRgE23wKA - mu+87FECgYEAzuI7Ur664wi2tx3I3i7vyhdOJrzswKiIC24yuPW0eS/XsiRcWyK2 + D+Sy2QECgYEA/0V9P6NRlP/qz6+FfCKAx7xfmlABM6qvtVgZ5kFOACd2diY2Io+y - rt85bBRcISV7gEt4g/tixdK+EHJSfpoE/IQM6yqTyedUXmF7disfH2dDaQTeNDh/ + nVAJa1no9JGu4ufJsRYaaa2ilE+rekXfpqXBngmlazrRgLkfyy0TkKDxMgP1wxkN - WlypZlYYtiPtUJs+t5hulWteH1oLiUeJhUwlG9+aQzuBec5qyPn7rDECgYEA2dgl + owHh9Hgr2/l09Hb8La/1ITzqIQJJgtJVf7ObYRxW2fQD8lTUKvz2nEECgYEA2U5B - 4jq+VyqfQ2NOWQsi3A+1IZsX2yoL1uN4CypWibF4N61N/wkL9oyEA8n8JclPAXZG + rBME13Gxx/Y3fMhGJJGuJv50t/ECdPxUXOzg7WkcNtHXB654YvFmsr7WxsXkwviQ - j9TmhIJCq/PSfmc9XLZGSudf7AZh1wRQwDOqONA5o+vG4n9Hlr//2zjA7HEFC0dE + k5bnewj5BHQuj/U5B/G3kVubxhbdiPldQaVP+xfaXTLFhUodEshzsA5mlYg76QFA - Wx7mCbBJknl3PMFuxk+DusQB1+jamGsLXZp4wE0CgYEAwCA8s2VJLZpkBL52UlAI + /iKUQ9W1gDvSn/xK/ARLIm+GdWrw+JEsRg+QTBUCgYAobE6bJzeiCqyaWscekzAl - hAcMntEIlQpt/R+Dn10fEwQpLdiypDgiq1fGfeaSgH3MqaJs8zS7z7ccpy1kCwqB + cPUKsKSgE+VjKCJhzfGWIKmnqAFmk67LLoNvVnuHTxKMp/vOaRuhpHdcWQlkgXAb - 4vfG/4X05aYdJeElxOHa71D4u0i4CosFSiePceg23r+Sni7uGZZH7B9fs4HuALkc + KaBxcEGbq2LFqYsZV3gDrRjEvM/MJ0l7iK7JUcZQPT6B/92LNpPwwX6p33zYlIop - r1u9gpsvKYzTewkFBkuRO6ECgYBO705S2hxMM2qAHYSvKSTZfmuQoMUVKfgeRlAi + gL2YMS6nsPZ3B2vZqtk6gQKBgQCY/+dvP0jWZB+XOb1hpyTz1Hp4zAnkBNYFBjBj - I5Y10HOSIR7o8Zs/HA1d3huaiYYyLmxFA8z/aL/F1NSJ7tjCNl3kGFCeknVzVuH3 + 6QiJP8t0sZQjvWzXxT3YtlNESstBl3872zEKSIwD3cV26GKKPF9SAd0QwMKkEWbe - swDUE0c/iViIi7wh+LI5+ieVxSIhwxIWvmx2SEVwaMj239RG0VsXGpzcYkiLAAaf + tIU2tlmx6vB1Y3RK6EXD/K+vsubzrEVVaYVYqZyMOBKZQCqPfHpmOX3DKFOXv6cb - RTDJ5QKBgHNl9ki2heneOnzApk94KGozB5S/GTnQUUzDSyMp3UGyrrLkwLuBMEd2 + gRPI7QKBgQCLtj/0Z4oOc/o/HaMKnP6Nsu7vCtbB5iYxtmA0yhBTB3Sf44/1mENE - 26yItmtWuWB+Pjd8n4+NUYB3+s5zeK3QPO9vXe3FX+L4J0UYCW5Lr2Q0FvK3svdO + LZ4dVfA/ZbZH6M/+6xWnnZJUSXSS07dVO/kjX8obYRGEMXlSFtF2hsw6P1fkJBLU - 24eoBuTkWBv08xz4K34aB6d1jVsz4AMW3vwQa57wky6lbhE57UBJ + 3KPOvN+NxQaCi7zW2xoifSGPvLwzHdMwGWfioVoUVhWXSK671OHFuA== -----END RSA PRIVATE KEY----- @@ -1812,62 +1962,62 @@ vector: segmentSize: 131072 total: 1 - convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:SSK:hwubqcv3edgfopyatdczdw3sai:dinqtiuqgfnr2sax42d273glnd7szfjomomd4rxk344uwvhwaira + expected: URI:SSK:edbq3ekmt5si2lpahrwyh246gi:34snfcth4337nbu3cdgznrsrzmcudje5ot42rft2kqxyad4uyanq format: kind: ssk params: format: sdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEpAIBAAKCAQEAv6WWmbzM1iOYxPnAwlmMvTCfP3DkF09Cppb/5h/+RUWw9LrG + MIIEogIBAAKCAQEAkFwLHxfDZEKh+ekOTF6YXW3pl2TxEoCMoMegV7GLQJrmj+hD - b6/iDSyJZWwyJyzgic8WS93xQLxl9HLrkmnMzTOaqA1yyeMLgt4tAR4UknHOsDqu + fE7WFpWGE3DPsZWfVU+79VlU/o56/Ymx2REZQCFB19UgkGV8FPlGDjsYcXPU08Wz - /006hbW7D7SUrTGrqV3mtFdHvTjrI51TVpeJfeql1A1SVde7OV3ZmqTnHEFyY1fU + peR2heNrx2NKF2Y3v6CV9Tb5mJ58ibfnByNyjswVstEiL5FCRGswKGos8eMewmJ3 - KywYJ8f8Gr3xl5qJKBWaZI2V4LySKz4iAO8hBwHiuY+gdQ+nAR4gke/YrJOIZHZo + 0AnC0s8txTEGki/mI52a0QkIlzgJTa0qu0Cgm0XnnK7xi1w6Yli4VIR6VbNrGhwc - rq0vYSvD0QMNNLWZoOLJs0DT/4gU1rSomjAv1297CmYkHtjEZwGW4m7rgo4JahIc + xChrb5cVD440H60LGJvqNrhAoAy+6Lg+0N5tWPuD3ZnyGqNjkCzqKVznvhTyD62B - fppqNvlPlUvnThmSJun+3rZBY7c17dD/qyfVrwIDAQABAoIBABvmGqzpv7YCu5gd + 4Ap0Fd5tbcNgqHAqmyMIvOfsZYmz/Jq6ql8ppQIDAQABAoIBACK4IGyf+Hxakj5a - NZL1X1ghTmV5ZTMBflXrEHirOqRR92dBE2cp5xH85EmH/SsXzN4y7+9+cUL3yi3S + 0PeJILgHwVCKFHjQtgHNQUEWEFm/Z4hg4io5g7/2wkJWtX0OcT3BaYE+tPRsLCRi - Vvna/g33T7HcN1Qtgbz84/dQLjV9bNXZzSTsVLMnWAJ6ytQFsZQ3z8B8HjztHsnx + Q4XjWOFVnlJcjfJslgUtVq4BhIV0yFEOkYBqjB7zbW6M8Lrj+LB73NUXHbyZEXbF - +rJV4BWdBaP/hndprt80ituI2v4RRnk2hJ7RCwDp8XZzf+98MAHeB19PsmKftPxQ + 5iiPW/QAHY/eQIyUMQ3ngbOWpayfCLJ8M/LZBorPb+JYLLgAygfOP2XUH2+kRbeH - yaXnv2GuYxayWYWt0wlBwD3PsvJaJ/OUHbO6/sRTN/3IaPt6Go+D1Gsyirv4vyrO + 2X5gOhywDkGSeoOsBCI8W3f9Yhn9HzP0E/htrZE5hhKim1ZcFcvSW4Zu6jLSMOGp - MUx4ygsfxqWg1trgyjMknZe6o4r3PPZsDC8R9fB3xc7FROA8rCP66Ok6Re7cnPyi + oRnpYcyFg2Nx6JIiWd0w+f+JP0Nx9ukEJvHmJc6zYZptxFCdAPW1ATMB2vIao479 - lAZbWSkCgYEA4iQavryiR7yEgTP7kbpVDNp0TgPyqLJxajhXEz8TkOM9Z/+0tmh9 + 1HwchgECgYEAx9mVKo9CA2kqoDO4nEOjV06F6Qc3me1C5KUglXHX0JQzW0Vxt5Yz - 1YqemuVsyNfccem218Gz+OX20asedXYb7TArB4InLn78vKGW59vxK137vuhj2JWR + ygqV21jokzksiat1D6cGnOMPqaaz1bQu+bxaBO1LPku19dvDfi5VdCy+BwzUVkmE - FCj04DiQMPzKWmxPt1/ga8p3rZPcEZ76ZMGHXSG03T4kplryrhIVHnkCgYEA2POH + uj6EKqX7MsKkz8DgOPXRJErBIkMFxK5q2qmLaJ6I+QHE6TAxzuvKTeUCgYEAuOtB - k5dp+0ybvK9uQZIAau1rYmzJFlxgcfCud8a+e1iYK8EaCG/J5LMvxkynGuJ9vNMA + jVNHpNgWAWgmmva49j/LmWy6pBpBt2uuxXSmKlk4PvB7oAzDvn5HPKw0zLZ6MVsb - CckXroX+etk72LJgJIcxgGWIajNtRkSrcQ7nWo6TEYt2glY2htPWFrPz+RgeCDhS + /e+YqbLGxfAJeEsVZ4ovppTYOdClVaPlqvnf3tztav/JWA5mi/YsQNIJJRIGwLZf - ceqLwqfC5YsFWBeYPNImwcXvlU0YoseKMW1Ma2cCgYB9Q16FNNv3PJdxMigxirM9 + n0lyY7Eby2GVH4cCRAsXmDFKnfWoT1X8b39JsMECgYA/e5xooo0jrDqAHS3dZZbz - 0WwHIuyxQVbNbbPd91yRLy5+gwfI2oyJUqWUS208u0Vi3ADp9mQIhOl5Ln5Ktke1 + Wtwqw8IjwTxoiROqpTka5pjRu2N+H9ZfrbEgtkNa0OSW7sIGsNXm7DHDgFLL5aqu - 1K6hFBk8Ch9ZJXD/sbcfPIoML5HPENox/pXV9b75Q62a9NAbVUJsstQkE/kc0aEF + ZehqfD5UkZRBfwfAg1NdzgCnGKoyprPkvYsaSRNccnwMCoavUVaYIq7rBUNF0Onc - WqXukpMq0hdfBpXSkjWckQKBgQCZo/2DnFtFyH8SJPrkHM2G7BR8Y6YU297BUj18 + f9Lq7sEv6CH2uPp5cmkXCQKBgFbVqm+p9s+y5Qp+FPrZ9tsz8/C0/SQIbGmseGKi - PZdwKtG5Sstw5hoIiI1w1aAR/gwlyRfh1jObOPF7dpRXZhuIQuXflAgDjd/5P3Ba + t1DVmrL7jKIIvHacp+kW2Kh03AaHSSrCs0ak+/CBGoFRiNiZLG0mIi9sCeegUj4q - ZL+a9hVY+3c13nBHE4YuFcrVwSqjj59zZTMM61muzcE/HZaGnB0uZUrCZRLpVH6d + nnTx+88uFCd0g7UfwYIi30Z4I5GlUlvjSoMD7RBhX3xxkp/PMaI3K1nnvMSclTDq - elYASQKBgQDVbzggm0ylJamRWy+apZ48Z5OBpAbMS/YsEptxboWnku2Iu9r+4lr0 + bflBAoGABvqvBpYQj+Lv7WWdC218juhICDEnkcEJoZK1ElS8AIFiQ2m3oaKR/+Qh - QZX+sXaoaDUc8kHXlQDcz2UYj/5AANeoLWlu6PoqoRseSMWaDB7RQkF5nIa+TZGE + ONjJGROx7v5JdaM357QPllCRuQjS3UfU4XOFQ814GJ8TiOzp86AfhT+9h766nQ0Q - TgdNQuxRa1w5RLwu/brRqnQ0SGdDqa6ulIJldkyy0aTq9XeOjlxhqA== + TQo+tOB15TiaBC0QMChvFNibHytFmGtsMGvAfseEEWYnRDDADlc= -----END RSA PRIVATE KEY----- @@ -1881,62 +2031,62 @@ vector: segmentSize: 131072 total: 1 - convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:MDMF:lmh64vic5llzathy7v55tgjjge:ai4qfl654ats5tpjdmy24jfsfxirbhkt4sgb57gcomkfrqq2cfjq + expected: URI:MDMF:h6zjlq7mo46y6zqu5w2whkpmcq:ykakuwcliizsy7n4gwqsntjvsxnqznzt62iepifwuy3b3kwyffva format: kind: ssk params: format: mdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEowIBAAKCAQEAvliwv3nQaoYx3rPj5XY8bFgTtUOTyUaH07dYtIP0gCZejJJp + MIIEogIBAAKCAQEAu0lVdI3gH/PySDIEdhByXGIH4UxlSn1ZZs23TgiEZhxVIeeu - /ZVO5JdeOOIzXuBkCB/1uN2aFbDdoGNbQbEsd22DjCDwMxan0MuRHVOeddHY65rL + HA7K5uHfXlstKt7phM+6x8j3hSHPMpq3svwdVqioAr4133F6W/ITVvZEeBKGrmEl - RwjFX2UKyCRtQliAlEzgniEYXOc1TJmjtAK73ftbO5+MeUKwFY4Ba5CbGx1XX913 + kd4SI2gU6yvG/iDJmf6Pq9qAq5embshOY8gC737fufoXMkaE9fiOMXpzf8sqF2RL - 6LynlAM234/g0v+B/aBaRhzKVn8mS3POYKi5pcAxkpg8Ywj6HwND6QuZvHmji0cY + w9D0WXBiI1g3QvR3Q65SQTY+s4N96odAAD8p5HgO/LQlNRA5sFFJVLfCSlmx6FY8 - tbE5UUHi3NuDwJG9aqxt/kxwyhPwMGG+Q5WJIdmKP4dGMjLDlIw4btSL4uPe6dq4 + wIYQnv3VsK+xrdo1KKNOpBBlldYth64nhCpzX31OWDn2nDsoPvZgU23ndXDPcKBK - KK6kJ4OCgcm0VJqpSFOiOK05fPaZ7sP7JkrN1wIDAQABAoIBABgsZU3752cP4dd5 + ztgZW5rJ9vt5HGbbShKFKBxGQCK0ud/IKOILpQIDAQABAoIBAAm4RlpYc6GM5k3X - mxCyIlxUFzSm/2bJaUiO+Vn7hBqeRNWvZnyI8LsBKjspJwL+llWd0XQH2KC2lH7g + ZLJg66J+RvTrI1WgmEd01TbUS9TF0yhBjyBvJxog7lgGCNvQ+lMVedbdB/WNmeSB - /17pZE9KfjFWoYqrbuaKY8SIsRAfdV/+iaBc0cwapfLjBWkumi27Ua9jXpe12UQA + MZf1LCufcKrFxuN8DvLfJyBMAyUtJva9XXcKzKuwPueum7L8LiJTGw869ZMSOYXF - IxUiX7+CQ4Tf71QbDwe9wBpsA/a+XBvKB+BlLWXRSQQAIBmUQpmxzPBWLQKryHcT + 2QWmL3rQ/Zj4EQSfss5WMkEAnyZqZMY1wIZxdnpvQ79iGXy8dJDMACBfJ8pLTeuE - xlEdyI4PQzLMOaDfQ40YRAPrDnYoUrK3L3HtXbKKp0MeF7fx9aJWzHD20Ssenj3j + lyPERg+RfDlvCCP3rmcNyNLcUdA8caaEzKS7bOhXM27ZI9WZ2D3wR2WyohCl9eac - eOFAOkDnG3wqvnqi8lg+O63YQBCOF4AUXAMWJYEmwp+e8LjTc3SLZGLM9L1Mi4iC + EKAquDoDwe4EVOv82cphUuFhcTF3xU3ddRADtP7Cz3UtuITc6w8C8TeVB/pCpUp6 - ZaJyHBECgYEA4fM3i9jvhn7YsV00goS9qj99lPzthtp+Ty+jh2rCEaZt1IaZCeQi + 9LDQlbkCgYEAxzf0oesOILxQecpnIbylDIk8khnBVeCAmXpYWFhjLu96pf+Lf26F - l//MXE0lqLf+UyA6ELZVnGJxlq2jJn08LTwpRHXWSBGrsO2Qkq9HyrDstMyu3Fji + IBiWA9jwETPCeztPVZy2KDnt3lh62DmPUW4HQN2SVINVTo3Xe13Ve2DSh4crYjuk - jwJFKfHpMrTG2b4kF+STa7ySCOOFIVNsAjOcnJdlWnzwD6tyZBbveH8CgYEA16lM + c+B4Xs8TyXCA74FL19H6EbgSDzf7p7SICG8IHMuxcoiVQHptdOzJiWkCgYEA8Kq/ - hLtmOAdF7qG50tOdCPzmnncCvvPIJhmPlKix6ogzeW9NR3BiqfZAETgV+VisYLgC + aM1D1tbRIKeICmqZATkwJJG2MOB6FFcBfRICpIGtn90+ztVdjgMfZ2FQJ9Eeg3gj - 6CjLfe5266RGVWNFZbns+hx+FzCtmHmLavr4lwFS0NtyXm0IE/4g6zybMgfwDNMC + ImjrnTig0I9Qlxa5DcgUqW2ZueR+r0Bw6dAp2VzOy7nWHZUyv6RGQGMA5vModGNC - 32YW1aTqCdy4ZhrcJpLT97DNl0OnaQrhMgY6vqkCgYEAhfg5tReZXbuULAXBfqnJ + FvBj8mqD/XOpt5VCw7t8MX1r+GgBZfAPPiiZjN0CgYBMQAKGJu2VYf57XxjyNL4H - 80nV4iLdixm9zqHGaiJokyKE+IAd+Xlk8Y7f0tKDQ7hkeVEgXIxf0mukQd0OYWHb + ek+QrALv16nhFI7T4aC0yjxrZNADyk1x53cjqdjY/LKncCABaKXf56w/uiXqtL1C - 7k4/gbIErZKcpDkXgYGgJZQlpUW/YDLrkjOcYrRmuoPpa22L5QbIShby14Zfh1T5 + MZbdIPFtH4d7NZcQRO389yYdcYMNaj6bi4MG5sNwCnuPMDHTPS81sPpYkNjla5fV - M4z6jPZPSAnQJNpY5vOaZW0CgYBqitgjptU8DtPMrZc5AZROEWr5lIAFyDf0IqKd + goncW6pjaBuYPkO+yRKqYQKBgCl1oMfTJK6sDxbLBZqVxon5ahvCplpBMYazfmQn - Za3n2PvdHVCHX41OvDowh43LjrQyYBYHjcfiYgHcLl8U5iMtu2nIsnTUjhblAf8P + aCEi3eA+YwWKqDVAwHY0w3Q4iEMpvRO+c2iASuPi7IU6uuJu53BQmzz06gYS2eDN - jgdrypqYViGtZp4cCmtG670cPXGpVEHSDgRv7bY1wxZSUyi54cXYUz9uYFz/dwGE + pYf2fwGFoCc0fquZByksZQlkNkHmn4oIG4+1XcuZ01D2+6twbvKvopwGfscq1dVl - DjHNaQKBgEhlpj4spuSaUC04wjMxgzsJOqnZvBm59INCnEZKXknqlVHmOpHB89C5 + dR5ZAoGAMsHeOltLyITJ8Ti9mFmd8HVhN5Z10sCKtoEBagvlVs1e8egy1wh6l0Hd - YWUE+uDTOszzR/UxblvGZuT/4b9L3bm2Ie0M44WUd87dkEhFLFMsfuWxm5//TIh3 + wPj+r3UHNfoxiWQWQffKMw6sAhfAuuRGP8gdcglzq7hYeVpbmw8eDn44OQYYC3ql - qL+sDc4sZr0nhD3nwomLvuBSj0sxqUzm9L2uV2hOMzM7h/3RPK/y + lz8eD0KbSm8MlWvyGvASgtquvuDK36uMGEBC2n+nN8Gt1+FPGHA= -----END RSA PRIVATE KEY----- @@ -1962,62 +2112,62 @@ vector: segmentSize: 131072 total: 1 - convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:SSK:seuvwjkc4tkea52zq6j7qr2uli:dv7fdqnxuilecqhi5gpyabpg7ijkmmm7e2sl45p3er3d6sd7ytma + expected: URI:SSK:rwikq43zza2vvcy557moizudru:clcrnbchmh4ucl745t3f2cwo5hwhchbvb4pp5vbapqo57nbh3iba format: kind: ssk params: format: sdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEogIBAAKCAQEAzP0CgFCh4OB8OgcEgvXxu4ijaX/cSKfyZYG/tnJR69EPKyH2 + MIIEowIBAAKCAQEAyLPhUE4G+3cl/Qb1SSmciZrbAJ8qFKZutcWqQQM12THlNtzg - CuE/JEcWvCAQvnUO5P530EmY/RRyBOR7gkVfArkx5nco62Z270p4eA2FrLWGu2iR + gfl3wTvQsPdjBJQ1DMUTyWb254Mgdy/KJPGI56NY5WcIKMvNUe5QUt6hzH/Cn/uF - CJOrcol5CCesormsrqKV3t6ka0ofiQq9AydK346FdUJLDSUyFOqUVQPbWjgWJ6Po + N7z0hZ/590b7dsr83kir52eIjcyy2Bv8dYwUEybWv7NvRW2pboTuqLpg0ww80+vT - D8jv21Kwup5qHjhx495BfuJx/foBAd8LmaRF53lGSdsTUS/y+yL3OOvqaV/gAmba + oCVvgvJhet2V5eXmElmzb8HbVEZklGVoUg0fGHEM+hzXiv+Nvepwm4nC2+Yq1cay - hAoAZpTbWiYIn8BvGdlT9TIJZqewT6ZHOAjnA+j9T1BRGztZfP+T+whdSkFWzGe3 + gB9h1sDAgZSERoJrczlfi4pbrj3qgHMZTqGkzKCy3d1zmKYIOub5TmCnzMAa0eHT - 7xziwijyuwgpYtsUogOlIc/2/fztqs224r64zQIDAQABAoIBADWgAJ4Buf9iqozh + G7i+h5yWRm/k87H+tRdEkenZoPlgk0eSVAsNbwIDAQABAoIBAF5jIjJnD6eRaD8v - lhgOcAEPwzQPq1hkeyB722PGr1Ch/bZaaYu6FjMO1886EjdI1y8ntL9L6ZZXWWaX + 14k51ZFtT1NihyLBBs3bkO8UOG3Vpkt/4uGdVfF9VO702Q9dN/mycVTFZJaKN2l2 - QQo4zJyhRwET7iP6x6Vc1XwOiYg/arIvLjXQr7rEZOGxw1NEgHyk8tD9bITWvL40 + AyYOpWjyjCsOomq1NfEzF3lxlDwdVYVxfzwwU/rHuoHNUxOR8QwEtzuTmEe/ndg+ - jXK8PjWSiq48u/aB4wKexVQiMKl+Z7tG8zPUq2n6a0GsBy3ZPnallwAnIxGFZEL8 + iSMq5oH/QP1UwJ6xLP5569dUF5cIlBfGVuyMBfynjHrtQXjUCKM5ZJ4onCnzm9f3 - 4+5VG+BSGTWJsd+5ahGsA41seb8+sMUZtKpbQsNXBVqVM0OgceZPUs3M82D9ImcN + amS5B0bMGpWNSnOogrUIk5vlgVJoydRkEWook0yNYxx1/EFanx3rpzE2V2IxHjXQ - sricXvLSh0z9yesPzWVwWozyy25TxROE1+Fy0P2a7qmoN2YX7l3INX8FalutAbcY + VvKze/lCNCddGWNvgqevsBMeGUsrl9VtYQ0fvWbYdzNrKb7WtYmXE+WkBPm4PM6x - 7epzdR8CgYEA7US5BLeC0oZVXurIVrjIVO4+qvpl4TeIbPSs0tuIWNqy5fHoCyDJ + Hxe7k2ECgYEA4E1gmYaHberFF4ye3tZ7Xdeke7C+g7Rc7JfoSM6MnUC1cnOklipr - WRIRr1eGwBKBheS3lJOVnBfPgFVef/r4TVIZmj+iXL1g/9zMpP9nll2tw7F2gxGA + yhibWkqGi+mJl/mWB07uPz9c/81LZWVGLVJMI3RikGRfIb70BPXddNIaAXceMd4b - ZwZEx7g8CV9dqCgYBmjZZVKwLwDDtkU118QAzys2cspHJ/rtc3Lb75cCgYEA3Svk + cLx6rQ4aWLTijQXqu7t3kz3utgX+9wGbVtB44W9ii7TGs7xVERQs8X8CgYEA5RC8 - /YI/1uscIrFg3NCECPtyL3Bg9AC1YyipQKVOfWAQwG+O4Kh2wrfQOgJ4C1KN3mxW + ZI+4lMFFSsYe6pLQMauLg7d+hJjcr/sQ96TDnJCMCkcDQ9YVNepfBWbF/TZWU1er - vOxXjnlQrL1KyZeyw5YFuXftDk8vLjtvs+GLwKK+vlDtgkLGOKmbPU5MS8cf3d38 + F8RCLN+HtDrXqaVREf6ESYOccF/rKCavI97QTwD2Oy1PD59Oh7gUo1djjRIPezuL - qoZGjzSgUpg/MVvJgB/zAPqJGcBH70z48C21pzsCgYABlQS60FJx/u1QzbX6Rg8n + 7EnYvcr9DpNMjDeKKiMocAlWSRDXpSHaAx+q/BECgYA/DaFlJws1G/URvKcAb3y4 - 6dLHJxZI0yr4twTz/vzAwuyQdfV7JYPSMTmm9qlyXG06rFTBC97ihJIgo/EWX2EK + kaEcYD/+GBqzK7TRmraukf0v0lBnIj+wzSAGzsJp3FmgjjndjhOtVeuXwSc7tq92 - evKqwaPehHDCJAHFU+Kn8QX4mRVWOGanyTXqMwNLeLRSK7pFSKuybkO4fIPRklKS + mBbtNI9slbqkauB/8HmzmEhVNx4W2KAQHfvCYB+J5jd1ez9UTMu9aYCMTL0yxJHd - lr7+oqYhS9H/pT+yFmD7DwKBgHMCI0ZMF6RLh8rmj+bjKvV8w0i12ESppajVeQWb + YrdIcB5ctZHR/tRO+8PykwKBgHIvVooWbp+QfFcazbyG9MtdxQ0iwimc/Z2n3Lxl - sC/z52IZ4KMkFvV0Hfw8Um4Y1JrnnUcKYxE8Nl5M5Hnlv1iDR6DFIukA9hjFYXWZ + 4LDCCVzyKzl8lVQsAbPymE1x8bRX5kzRo181CjOYhXrmkrQSmKUAu1H1Lob0Safq - gFGAj01pycelr2vBjm8XqwbwmbqGd5+4yTIofIHWl220PBi7BGLq5KYWXZGrZfuG + 4RIQ262CF4AlHINhCsClxlVDJH58n3JpGWb6sgy69pSK9w+sOPMoZF/Fyolhh4i5 - 2WIHAoGAITXgWLFrWR2e7bktlbW3HJRLEIPClA2GTrc9PLmBpgGXRw9Gb/UKXYsa + F4XRAoGBAL9lDVciG+f7flylMPFkvKNcdgKMxeTAYdupkmYhBqg1Z1sKpLWzSLHM - 0/YUxFVbKAA2B19zUq82grmS0270oiZh3H+S2BMCNUAWsWau6m0CTuN5VBE/PHVQ + HgHel+wm8sDw2atJ0QCgwgDWzkSerzz4MujYEuntM7EReMbx5ArhqsGo1abAo5iT - WZiLCks4FQkFwChLM1/9rueao/AivakRd5US01AbLEi2PUBkhdE= + P6pdPFMUBWaw0rVgpEHdcJylb7mIQsorLmN3KaGrFs64/rGVM/4E -----END RSA PRIVATE KEY----- @@ -2031,62 +2181,62 @@ vector: segmentSize: 131072 total: 1 - convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:MDMF:pxwqxxop3zz7inebttwm64lqwa:2ntxvchwrq7mgea5te4l57gv5jbtmtx7wjsjps52y6cgrajswstq + expected: URI:MDMF:sj2qlco6lwavwmcjtuxohhsn5a:nn2kte7aa2q42gq3fyy5m5ojvm5qvpyeqpvwzswwrti6ushqg2ia format: kind: ssk params: format: mdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEowIBAAKCAQEAwjdBh78eL6MVW+dm9E0gy2M+TNjXh4AVX+5UJv+rOpom33vS + MIIEowIBAAKCAQEA8pEDu3IwxvOnY/TFR/sK/m0X+v6OW4nDv/P0n7KmikC1elek - ugdkU2mr2jYHhGlHiU97iVi9BHB2mtZBpieQVAG86sJiksN7Y3s+SPe6hvebXtdy + NPE+KZZYciD3RvYqcmT/y6av37t6sV8kYDDRBNO6sYcAlByy0qqaBJEdp/q5sGRr - c0+ESruPwmkYq1nrN2PRPtxJRrEGv+s3/P13Iwl1qvgOOB6BF9QFE3eGPfPfbezg + tmF5zfBtZX1nvbmsy8x7pGrl8p5uhfbUy6RfJZOv5s6uIHJQMkae32LfPjgyGkTb - O72rhNSSXuJt5EuX7/jeoBev/S5+ola8kKxI+SK9Cx0jEJPxmNf6lQXd/fihk7rs + Ah5mQgIHBun7NISW43ESNu0XPLatcDezO1gAAjQQCfjsLBcyU/bnNq4zZTj202VL - GbHpQQbm6Y23GDicQPqhrhDpWA7XyBluV/VSAYY22f2dVLH8XYt499fBwmWGBNFj + Jl9G+JMdPT2qkEduNUyeY3B0Ot7XXT6SXFd/IBkoeO4r/LKnxsn+QaAUPedlTBcx - hjIZO0PmDJ4HQ7Et0HOXGbeYv2mAgcCrGVJ9VQIDAQABAoIBAAKb/HDm8/Be6AwO + lk91EPLLCvH4H4ZHxgOLAp9QZFfOdcx1NWc6iwIDAQABAoIBAArGNDM8RDxiEDpZ - jVcN7DlfUXh111t2MJNT3+SQPcwxQwFwp/Gg5MusGUd6v1obkf75xuae/xcerbFB + YfXribZ5ZApLCkm4mdBJ5sC9L7aOX0E66VlMqeUw/2a6XiFxx7rjD5WdJsy6SB3e - 3KrvUCSYy2F6EBn5r2A0SS9wyJxEml1JVrvO3y+j2ngZsl+m+x6I5EhMbF2bRkRw + yv/Wy0H6oZ1HENiDWdIPr92qEHYopdzW6Q3l0II8Pq+2XUhJGgrHX2qTMPmQ3fnn - 1BU9kIqzd1W/NG2zlzdrPVA4JGETrjdNrZbegc5EFaWFeg2VveOdKwrtGSupuNVf + V9Zfy7YglDydS5C3YyaIioADpUTfJf6hUqmbHEg9qmmHOOYY5rDjXxd8L2gaFMci - yiggTXxsTXcfcy9dqdxe/vTNZw5hQFnZrjycwQ9J/dUSotlVo5W85SqpsRVK4m26 + tO52tHBjHGS42DZVY4NYN8cYIAqqgyDuXIUDmI/RCxQst/s5BjUnLvK9FAXsh9pL - +9mA2yvpDYxs0+fb8C80KAq/BLP2Y+sAnU7f0cKSsLGioN+Q6g/71wf9zrCrFPix + XQvCDRU7ENMt/FXUzR34lb9k3OL15Y7+tgDU/7njberBBZmgvejcC5fLN3P5bMMI - rrDSe+ECgYEA1obViMRRvlDpFuBLPpoMr00tvXmTJmwCvqCksoL1nRFIEuNsWQSw + pIgbhfECgYEA/UtYVco6NbKdYq0mI8HQEieLOeO1ecdjO/n6amxHBvtTYPTYmkus - 63gETyZuRFz+xXacUq4eOSoLcrrW2CPeMOFD7rlvSKbG1MpU+OI62UZ8A1YJC0ap + YvpEQ0+VRcqPQppMgLxZOkXFsM9MsD24wb3CNb3NtgXQyMGFwo4wP8NgR4csoLlM - YegJjc/1rn3r2HEjSphZWt3SWXz0gbVM8yVvOt8c2xyCcxxMRRlfrHECgYEA58M3 + cVu//PgupPlAeMlicLPxcvIRnXKQmQrOYit8iWy5VZ2G6bOiqJ65JTMCgYEA9ShV - EF2pqKc7USkTdMKHKogHRJgDINLDz2NZf68WlZkfv7EXekpw107zjmqQFPesdG8Z + UmTZjvqv83tNnA8ViZbeXyc4xJE/qrF4Saxcb5Cihyvh4vc6uBGyXZwA0euybaeC - oZ8KjVVU97my9/eODMXFQ5V0Mztk4Fv+cGAnTenuAtJ9/weJiR5Uou8rpfQOuBMp + Xqbemc8+FjoSu+N8WAbaKdzy96IkgIJJn20k+ZljqGA6a1s4OSoTcIklx5PoOCaE - bYnwZnI/BwPdtS6B/jkS5KMHR+Ri3lxlmp+EISUCgYA48Yl0yEe6cNeuTtMqRtHf + 0zdoP5opOE7dsM9jxtvfWTCxi1ZCts/vRJBX5UkCgYAGAoqnBnRZH9LSK4+TG58n - JmlhxgedR0ZjO1j8WW7AxnmPKfb0mh4sIqtiJx1V4ClwWM+d0sILAnIPfjDRJpQv + Px7zka6VpCB7pNPHQKhyxvXUgBq/lnoRoySJgFLnZAYAK48TIuTvGAa3ykNkjyJa - /Vt+3pH/guV8TkjH16UvT1pTuF6mM5d6eZEvp2fbbWlRBpcLke0GBaN0RYrRc0J9 + HnmEMuu1nO+2Q7k7w4nriWQ4bkGl0p+4tNeaVf0tVuirtQOL7wkUlB/M35IEv5fk - uA4SXm7Wanbl/zjvjpCqwQKBgQDjSueozELEXWXmHcOwActv4cJG+lIvEaTpskSm + BmofDKBdIq63ztZWL+XutwKBgQCCiJI9h8MrVSGAhCPDt2hhVTpb8ddRGoGK0mnY - 3X7nrimd5L7itzjdX9eq90Vg2tmtwvu/Luu5WlOfM+aaG5WbXyYsNtmkGP7Arlfl + 2HRzVtCjJmNk5PyX65xMKXdqTpQ3vJw256TYwrctQIifEDYx7JwW9DVOU0AaSMUI - u9cwKVi8OdVJlQnEiRN2S9thwO3ihyBdBifXQPohFiCMPRVNzomB44UTc5+m9bTL + pSWt3NVqXqpcZTqffV7SacP66y8XTrMkf3j7fIr8F0oFDbfztzjKFZpDNY/aJQci - pN9/ZQKBgDUfZQquX2TLgnTTRm79FSh72Ji9gV8wKXyAcvzI/N8nhgE7xPMm/ih/ + O7UBOQKBgEFTXq2BFYBWQ4ot0TuvnVF9/8/3nAYFZEWGGu+XRcoq0DsgIwZcb+rk - pPMAjyEV7vc7ISOBGZ7Jg4/p5p11jcWmLt7CSQY9LOLCMIedFl0+Ole93LivyeFn + LB79AN27BJw6gaskoGKH3muylECFWju3CQ4yDXSLFhZhHH/NN9p8Geu7eluWDNIJ - sy74rH+TP8PuEjQuzAK+unwzY5udBT/lTdYXbKXzPR3ceoarKBn0 + Z2wWnZzIgOK6W1QY1ho+x7+BjQDJ+T2NBK3ApbviaKA5dv05dcf9 -----END RSA PRIVATE KEY----- @@ -2112,62 +2262,62 @@ vector: segmentSize: 131072 total: 1 - convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:SSK:qje4blwekvqst5si67jnjomuhe:yjjgmegtlwedgwnrsfuu76yesdqpdz645u7gbtnfywsiybvkul4q + expected: URI:SSK:74k3o33ahqgozn7iq5vgnnxuau:t6wznkzfsjrbudipdvh3c5xcnpyndge4amgfhajynisohhfhyp3q format: kind: ssk params: format: sdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEogIBAAKCAQEAsUqS5wnsfJt9NEGBruv1IF4ekvr0XtyoP4ckfT/tkSnU7txD + MIIEowIBAAKCAQEA7Pj4z8JZKl9I36j9UWEKCdhdrgsFDo6B9Ul2KcZjFTw1shEM - rLJzVKbBENAeKXcLkoStGXyc3WtNcUHcA3kJi9H9pn7UBqaZ05q16mMh4Z/s/l5Z + Li187RthvkMtN/ih6MpBWXx3YabO7wyI6TWWyYvfcTkXnkzs/bGo2ludJHyPucLE - xir5F0XIGBgjOf1H0BDAqlzQhkvyd3TlJ0+mhHaP2KoYZ74TsZ97gMrEn2Q9ozyG + +YPNTLLYvvya+iwnDBk9y9gSPVTQKwVKN8FnXi3uUHN/s61pMEF4zIA5GhqtVAPn - nijnYKlOLpvnBwW2fg9kpZi+3MuLOFxaXdTr+mH372DaVghBnU15jr2DE+WI7VrS + cKvpa6fWrNay3+qiyOz7HqL5loTUI5UBlyuQedQH0vcqYRDWb0eLZAYMFM7Q6kCK - k0n7+4nps+cWhgP2njydCCqsL+0sBsI4DKHZbY6fhMkqXSQyyTjs276BpHgpgLwF + wWSwZEcu8Fvl07wCqgYdmhdWIZSU6PEzZjeHR2gsUirZc0/nyvOXXMEOQxGgkbdL - omAIiS2zE0AzS4Wof9xGCBjsRO3sI8uECOKa5wIDAQABAoIBADEIK9qj3viTVCw4 + ifYBFVNJxbZjPBZIIk3RuCPKOsgz4jOuE9gXuQIDAQABAoIBABX/5l5BuIEF7lnQ - lbIX5eI+xXvm1eDKa+mt6YSOQpisFgy9dCX18HmP6MNKm5ziJJwv/2OWGBgQjglt + ckJo1BBZhUqDSWMYxcxD/dofY2yoQoMXSnf+NX484yEeH9RcNIoypK7he2/JMNlI - qnh3aBF4UQtT9jWkq9Re7ELXic5JmZS76V4qElvCW9V2D4ABMXQ0veQf6TfLF1K8 + FgaYhZ9ZnFMCYUS+XYFCShIjgssAj8Z65EoWlPhzrhcxxyCskqw9d6g1LtROZTKc - TIfzulzWIXBNkpRWeEHelpyG95wQ+jD76++GPn35EAfusRShdwRDOk/DXNp/cqoj + cuk9y98WPCA9G/Qa7qYFN5xWPXbeKQRoBaDwnNIWkNTav1wXY5aK/w5IR2iqRXjb - 93GXHTlCZGwiD6yae7MLNKcbsXkQISuRuhR195fHw1PbGmajhfw733LfgrGiNWWD + YemajUleU4Vbn8ge9CLf6iEj9ILtGeG2N0NzmYvWUNvbxN4HxIxd3LJHfLMZyqvu - P9+hgsvwjlxE94FsArxLZICO6VMUBVqpEPRktqeeSlG0WTZJdDB5L4sqGhWJ43hr + yqDuhSNTh7nKP3sYxWGkgkLcfoYYnpz3syhhJeAA4L9WXlvKtv92234bBzS6TbJB - zh/0FqkCgYEA2bMoJCgqJhcuhmMIk7dVVuc9vFbxK8kqaf0NMuc/D4Tf2hx/WPMk + +DXUAyUCgYEA95419jXoLy0BJ1i0rgr31UwAGs7w78yUw1mUyZeUbj3xxAu/Fv7R - zyK3Jq/UL6hUj81UrcyQ8OAvm4QQ7PlbAOu23rW2J/mwytmYkIctetYkrNeXA2DB + pcxmxpzMa6CSeDxHY3DWuXqOu5oXV8S8MeFI6EgBlBiLWYoGVP88Ad0iTYo2q3FI - Hvylcfl9HeMqUfYWmPb0akY7MpTpyqNyZJP9a32OJL4SJKrJPbWZN9UCgYEA0Ht7 + Thwx6jyMCLntcIHJi7JtYMMJSLD7ctSgqh29iDyD1QuUmz/YZbVkclsCgYEA9P6C - t8rryTxg8oVTcRgznq99ikPswozElEAWnwCHKRr9+b5H81SXtZM/0VHZSWjcLk6F + ru+W6NxlNZN702Ha+c41cN1vo16KAhlc2Li9YI9/Q51hezJMrV2ged8yUfYxFiPG - qUzmxs5bZ9PapnBQXUXo6ds0o5uXkIng6Nx7aVHG4WtYNLWE2I7k6JYw7C4nZ3Fb + Kqtk9y5CkNmt0wNZ4oA/8d3WatZL90cf+p0G/yGWb4BVNFao2Y/pMJtVFZV/T9A+ - qNiYFI+D1xazBZ+MiuVss8BYkIhQTZ6Poq+9gcsCgYBLojvS/AVQwIMQe32yXGKQ + 7zDuyPPHjaG0swiQ7NX5eravsq47v+SXFBocUnsCgYA1tlPuPHNJCHIfntZSin6H - 07wWIBqf/L74ncslIUQ+bwqaq4Xu8GKceFIrZbERcakXYN4Hl+fPWAQSQrriqetd + /hxntEv/OFlsppnnwMGpyDYRWJry2gOP+26v1oNhNUuQWUMDBw8M3NDpUNuPZlWM - EYeyLm1/y/cJMroXlG9PmvCZADneGZJe4qXUSDqY1KCSYy4MrNfTyFyuwR/MoCaR + XFn8SOJOxaQ0oAQPm+3gWZ9/QmPpfIE6sFMDhG671djzdrPJYcLoImZ5JirlFcpk - HP1RiAiHaWXCSXerMdlulQKBgFGZc1P9jYoHIt7phj5GvbWHdHiQm3OOS0bHStNT + HF9olffi1sg9hPPj3B0V0QKBgQCGveOJ6uOIto5DZRXZMByK/0qNBHx90WT9uo1B - DpPtJ6j/bAP2gSalip3wDj7oVv2c6D3ahp0bmbUqu3LXlOzc9wvJK3I57Pm6rZgW + 9HjTPpizyz7tzsA1KSU1Yff+8/QTRSGcHh+tgpfBqrbbMyCgXgDNOUDQCYRGP6vq - 7ArN4izKqgx/W46zZy8N0fovGmcnfDu7AtNRVMXz8X/q8cRPhdtZFpEDeYLX49pG + 3aoXb5WZRW+XFYJQBcIupX+qG0qlztaOHs91Xf4Ge0Uyoidy2kwXnZoMH59k7ofY - NMM/AoGAQQqeB1BWvwZU02Gaj6W0SrybX57+reP9ivile8Wjk3n13OEj+2hjDZ+r + 2nNxOQKBgB38kUiRxm2LGo4pbfuGhgiVf4aDRUDT4nwlaRV8waIu2qeaWuETaf1k - EPEl+Zdg0R5Y1k9lyPeJxqYXMNA7GeOrPq/DmuaKd+em6kEUSaVwc67gvi8sOMgS + bayNxEOYFNraatkRVF3AGnOJUHLNwj90HugILF6uXDc6FhlgQj4R3peui4iaDYvH - 6u3USnDhiv/Kclzua98XFIFRgrkBcI/YwkCjJ2NvOJAluBT5s8I= + OPLuU0F+oJIKhUnRB0InZ2Vbaa4hCKgvEftqNPR2EkT2YkHgqAao -----END RSA PRIVATE KEY----- @@ -2181,62 +2331,62 @@ vector: segmentSize: 131072 total: 1 - convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:MDMF:h7lcy4sijoe2nuvvf7ulplahce:yxbndqsxxqxnzd7qd7mtclmhhd22a4jmoluosbitgzg7zuxrlrrq + expected: URI:MDMF:5jvanzz7djyl3ritv6cey66izq:ognbst5tvfuow72k2lx4zlxbjiwpz2sjbtien4kvfl3ksiybhnha format: kind: ssk params: format: mdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEogIBAAKCAQEAzmdmHTwMgVhsonzBdU+1WNs7ZlmaYUdxR9YZVL30INgqIEnB + MIIEowIBAAKCAQEAtJv82wDGagqVX1ivYTR7Il5iF3N5NnlBp6aMbCRRk3wcebLT - Bn2DTiNWI+lTMs0wxRvFAmFP+kSJ+XCvlucfMRp65PWOop92xIYwu8Q4Pyv6xkdF + VpTYwlMfo8LHn/b6trMcyP2l5vczc0EfkHYE2zy2f31SRRZ/LClZqtb1mu7lV5Nt - wck/OMTBRUNjavAcuAsHgbR+LfT5WEufuA5NpKoSdtVcnxYNWaX6XhHOcCBk5dm6 + h1qLc8jGt396bvAQtQnCM8FavIQW5MZUxpMwizWuvNoMEcjOva8PXr0umvq1wROd - K6QhsbmijeCwNoF1n/6wuiJZ1KN4IajckNqmjaDta8bpgerrnfVnmCLLiHIVFiOw + b9ZdTSJMxD+uROl7ksX4F7zOS4e8Q/+s6lyumiZcSF7ZSF3BifO0d1tzRR/4/jv+ - IorlRCyPkgMzAIMhitcOm3Z0dEzFoE/ckDVOTYjfoh/PMSe+3+/BlIO/zB7JJJr8 + EtB4zMf60b4m7DwClMKzht2dYmh3Km1rWdVgxZEKncWY9iNJ9Ohul8TpdxGrTWQp - 23TyCqBN9wYAsU2e09Vn/faXuUmDFgY5DXVU9wIDAQABAoIBACaACf+dAlYkKMtc + bdYdjRS7mNiryzU6hHF4gwzaAEjQ3tXytehvcQIDAQABAoIBAAlttUMRMZdNyyN8 - Sve3YQPMjPVn9FB984brTDlO31k7CQyRxVQRGGt8UuaK8K5ysMyrg+GQRktP+o6R + 0HCjwHCrlBv2TS2KiGKb34VlVfuvK7HJijfhMfmr3jrhC1ih/s+n3q8gpfCk46lq - MueKf/p4ToEjvrHd3dkFkNSNYtKBwRq4E650e/r6VHS3f7VkSW8Y+5L5mGm5HsOW + JmwwhRHIg+lFURSs10gmaaxxNJKS7L0bXXxTitrC1s2zvxM1WO68JJVc5dA+mXVc - A5pg7KGw6ZXJ8adpBR96QsvGNYQbaPaYqToHBec5Ey7gon1C62PoF9i+yw17J0G7 + lznKOFGkJH87NAyy1EjzVe5ggPANMETFU847/4mNE2BYINy2eGX14AK+2ghkQYVX - dG+C4GJB40+Wp8bK1FmDTWdoa3j7+Ws3/jfceeMCIzw/q51PTQOyPRHRfLr+57mX + YFfO1OGZiOoxbEK4O4Jqd7KezCUlNGMygxveajO/mUdrRfD/DxTdo1FM69USWSTG - RihBgAths+5wEHdPQ8HKG7y4HJJQiMY40Lusl+kV3Dg6eoU8rDLd9/2kBPcT1rLY + ZV+VS3cKc+zxaL+lJVoYyVNJmPi+n9RR/+OtS8HZXqcwszGHQQbPRXWdah6mNiWK - HBf6y8kCgYEA3ZhGsyOX0b+HdBas9FZhzday4L3ypMliOmh3TuZ6dsOVfn9YFRP+ + Vpf+ODsCgYEAzA0MDFWu8QVeI3yn1o4lKiZTM55gRIBxpFwVe7WCk13mQCp3XuMy - Sj4svjlVu/qrJFm0DcPS+5zILuUc5vByXLS6r+CAl+/1zJ8PHD3vScK7JiYJ0D6f + Cm1WP47fMmTTGCOsNO0A5U6ZNf4J1yNbRfITCLaKkFw4mOmq9NqPz3DY1dfHKwUp - dp6GZ35otQuL1tcY/Tn3dWlLJxU04UoZruQ3Wk63kFuk4SMVLL7tWHkCgYEA7nNU + bjVB32rGrS1Qs7Gtbq6P37zXUGLdrfL0qs80wqztkqJKOvv+il64x1sCgYEA4pcj - T6x51orxtFXjTNh2ATXbhL7Yern583o/0lAJregsglDhOdGqrrTt0voODix2fCaB + yQO4mAEsz1h3RUrnV83OOooaTAd6i6Vfcdlo6VEnyl42uqbrmkzmzpRrYBrmWaTp - wq+UhCvjh5ZQ7IRwgcK5X66VLwbq/lBL/d4zn/EDnx3JRJ9hC/68h/u+ToVJSGCt + ctDMy3j1tmRO1Dhpm+uFDjGlmIOeQcLwFI1kx/7kvqWdRPSVP5mRskF+WNiFV0ER - jCYbL3eh4BZa5wN0SBFUbQokjnJ+mNM6yNomnO8CgYAh5mvad/V/5xcn0VhAQP7R + dFptBVjLT1lRBP70rzCQgIY9loM9fdGBam0E6iMCgYB0xc6sTGimM90wz8i5J3Wr - aKkQ7L40K4LVgKnP7j6J8L3sDjtBbj+WyBA8QbU1/tEzzG1ZNb4PNBsD4ZUcV2iH + Tm107+DFsv/WAICm4DQOo8D93Y+ctMZRY0rlapzemQaZHOkTDMLjd3yEgpIdFXXJ - ejadNXE2zUUDOsoq/eafmCTdXzBdJVdr5DCXoKUQHWYVRe7Svo127tbKcdoXJSjs + bIRqCxT3El+tWqPkJiQAoeLlVev7+aNBF6dP9SontvQlMbw/yBQ8BTTvIvUb9BsC - soktTaGTehGtR5qzr7nLsQKBgBNqIHs8N89YEMX2GEOxfCotEGqGf2m+qrNASOH+ + mTvnYNFAhjGW2dlMVHLIWwKBgBKypiFQTUs9zZTOmAj/xVdZhEsQWlsrwtEDNH0Q - 0krulHEn1K64e4UuBg8ffPV6eUsyd246jYUVbbkkbAJV5jMqf51iwZLKpWd/cjCB + k7etGrt4SsvcOlThQ6qIVNP5ZEjBcwImeL/Rm3URke+xOAXFyZUCQ8fyFH0YuPb5 - XwKuxPS3oCOONoCbhQ4tWRlbkNPryzWWBLCgtPVh3JTimx0jDBS0trVCbTxUNn0U + M/fM8NNKl0+5XxeAdKVhAiwSse4hUG9phtWKHjzOAgGHiGlseIAik7J34fsf7q35 - BgDRAoGAEjnEFmrPDEI5Ut2gtdgByd8GmbBeISlADCd96zKWm4XoOcNFkNOsD9a8 + kQ5BAoGBAKqKtCu0OndLo1ZI3voVGiQrbn+OZ/7Hhjs/Um9UuOlVopv8rrzqRpAx - qXvSVKgeMoKc4p+aTOEn3g003ryCE3n7jhRDsPsu2lhkDHfWbf0D38BMFfIbgeoH + LnC9rU/zEVG/TOjkno0Kj7/Af49z9mxjtRzxP2J/yMujOPv6Eer30z5NjtLNyJ2u - Q2C2l5PnwhrNK/+7oYhF6xJaTVAUcY4JADQqeVfmJx6IFVhQSec= + QYIiihHEUULkdZ88FIHfx5YImHX4E9lEeFG13JaJnMYbmvViXtyk -----END RSA PRIVATE KEY----- @@ -2262,62 +2412,62 @@ vector: segmentSize: 131072 total: 1 - convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:SSK:6pah3govdilsqlc47l2zrispta:7cyrr4k7elzjyvn5j3bnk2q5cbns4yxckiz2nxsjbe3xe2ehnzza + expected: URI:SSK:kibmgpkvunbi3wmq65ohjuxfoa:y7gfq6oydjcaeui74rjoupf5xst7hpyjuqyvcop6yhk7xxmpnaza format: kind: ssk params: format: sdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEoQIBAAKCAQEAwe/MhxnQX9dLfyb33PwsTTdoVU/ccfI3kCzjSQBgcizQBKtU + MIIEogIBAAKCAQEAsbpi7SI2V3v47iSl/b84TmUZKH6fJIaznI6bDCRiJce+rOCA - j+H9RtfuGmaL+foD0NUtNqLn9ztRegtBc9srWEBC2To2m9Q6+u09SRShSB5uHmDI + LATvooZutPlbkI1QI1RxCmiZ0CPjemUpXwUIuLclBUIfZDBXnjZeNM+W5a2C5Yt4 - CqAaQAwE6LOEmcUS4WVtXfVMYMiPcEtWEfT8+n40CWd7+rBSg10436gtaMPrDyK5 + CzAHQl/Hhr7Xtra5csf69dJkSOwVV63TpoN7r5TrCoLrReKlgqHp5iYndaHIsP2S - /5tzpqugLM2/N6cLlTzsADA3VpvpKe2NAlNCCCx3y2/UFPMPMnFkKmxgzDd59vW/ + lfUaX92Z74xtXEKxi/ZaOZHsMh5DpxtL6YSZjHkXXctmrwmwcAvs5Rc/qppOydqh - h3TRKlOjf3i6OYCcscncyAu4hP3DIty3Y9EK+lzGRwiekt6ztopFr3gog3jE+/OL + H85l5H+PlSAJjP75roQXD8T1xlIFNFb5TUy9+fP9/NMqUmoES9eV+cCjj9Sk3S/S - sL5FFzJKsq4EdxWejKp8q/R/Xqr5ZQQh+M/k4QIDAQABAoH/FNSZ4QRrpaK+JJ68 + WhSSFXoG1DOBFpR6iQtuJD9qRjL2JkDD3PjiDQIDAQABAoIBABMkz7+Zwg851hY2 - aehmAWIrTVnzi0B+VEf358rYqYMxXdUekyH2tjO/zdq0Wmwd9S+PA0uDweaRvyG3 + wd97f7HoD2Xuf69kSAgKz1YnPCA0LAx8kSnMrVBNGTMqset34UQw9g0oN7s1Bm16 - 478keLVoL5YQWEZk8/ThU1XjKpgUDJ5+rUoPuICeLfvXLbYtOuJmQK2CM5z1rLM6 + ZJKs3OPirFzs4qs8zs9GrW6UVr1uK22U0I0p8vo6DWis+VjfxUmBE33zl+RH88OS - D0FrGHOStxd+CZFxb/5+xDflTUs+mYwbvDsHq1RxoKkg/uwtL99GyBopIoTdcDJK + QHxM6N+Ak7G56ORJ9ciErshg3zq7EKFjUrBMcpq7L6m+yzy1PBqZ1BcnaUI/o3WA - D/P0KT75JONOcGSkveuepcStk44xh6Aj11i7l2BHH5R6mwluq0fmMPIDe1T8U6lt + W5pMTZFODnpo8MTWtmFQQ3ZUUnFfuBx2iwhVEP7Mu7BQJ0NG7C7GLsIBL0bNosk/ - P8oJW4yPDVsnc6xqep97YC1+TKZruLHIZou5n1AL4oA2VgW13GUmy0LhZNcCtPkX + +7YbhKWyvvAiRvu46JKRIgv/Dn/iOjZqDnoM2UOBNWLbq6g4gBR/MRdgH1pM0m7C - xHLxAoGBAPJjxaHiP8DUQIefTxM/lyLfoCSHJQEjmq1nStnAGYH+zACkmWQDrpNJ + 4g8oNzkCgYEA5+bJ6Yf9HBBSROGoZxMppo92voPSZ0ykJSVRzOIu/8wFqQHlNXhw - krvkqoMmy0hzdlgPrT8VOu/rHv/R+TLdy/3mI2O7Ezz0pPds980Ol1i/oPAFqf93 + /udWfHCIqwmqcj7yhl7wYdSLsperEy/ZMnhRb2+KVNyAZBlRlWmZ0kX1lahYJ2RX - fD8gxOFyOim/2dyS3unEhyyycaifMQ4JU44DlQDxSu9s7DtLNUGRAoGBAMzTiiZ4 + 5s9+fEr+bU54ibHZy43d3+W7//NW09soMHyaLuhnGuhEy4/lRVo7xHkCgYEAxDJv - Io9ffFuy9B+xNAQsgA2AUxkrptJd12q4Fky7Zs6SD4HjWmGMcsYen2KuP9Au7PZy + yLooFWeasa9usQTEeyGQLuh5Fm3TfEXUluA+pBegTqNJ7BCegYl7tI5+btojZyXb - /+8Yp6ZBvqkTkEJ6CENkehyNutYOCGPw9XL5TK0Qfw8vvhhg8L25TGU7voJ+1JEO + 8m6LFdPj7OiTA4fHX8c+FgRpZkh/4HtbX+pKbKkp5IycMG7N9Fg2TOkdDfoJl4Yr - Avvhg9WzGS4QSP3iszl703AD3B7sot4YQMZRAoGBAL29yGlu2IU0IceIt7fToZXV + NhLap4xV6A0X807fiTG4saL905AT6MOojbsznTUCgYBv+3tfIQLxrVP83Tcz5wYC - BGFTwW3g1yZCo19NdypBsKQYNVMLZs85WrnmyGueJKd0awGIVA/7qIVCwqNzVOWy + 315I62EL7u+I3Heex05IyZ2mGjsz0eBGxzF1T+Y/KaC8IHd+uZO8uiVnbWP4FO/+ - pgr86lsZiHfA8poVHO3SLDt21p7NcEPg3svz9OqeJlWkLwDxn7nS9BXTIhHje90H + Nimk9SjIh94b+Dn0O5VC+/N2fF9tTkBAPcxnetNXtz/vxglVCUGuH8Lj+v7fuQG0 - A/c5apywRf6if1HzD59hAoGAIk6YSCM9HqiOqslJjHlgzgYqGJjS0ld2ZKvlJfHZ + QEc4BZPcY3LtFaRyE/uuiQKBgG9ptNDn5ZtCGjaMyO79JhZGGPqKSTjTZSVNAkwr - glatPJJIWKgc/lPI8Zg1eBDZjWQeupS+e2y0v+spJSaqtge8lJUiwt+WWL4W965n + S2cjg4UkdPX4+gnVaMo/oMySU0hf12b0H0dl7Ci8ab+3eyCIpFkcaD4NLZDsfBcb - Xi+VgTNPJNsJSwoJqK19t0MPgMn/jqA7LbczHrsVz5pYr3WmMU2lN5Dd8KwQB4Um + lOffqEqBDrDyO0JmVW+XcUhelNPW/PLYAhLjPmVoChHA2G+wLJGzXTCmwKeNdEoH - bFECgYA/TCLA/F7YHK3hw6cP8IrL/L93+30E/poQZjzlWPFUBysoO2YauvxD6M4u + 5GeBAoGAUEAdLLTRoEUXSUnQ5Hk9OGgHdc44QDZjVfbN4sc08mc8HEXHcMDIwpMx - 1N8ZdMnWEKoEc4oPLiGLsQQDz2ICXy8lr2kFzcKnwnAXE0AYokGYZIQlbPRjA4xY + LNWapDRO2uixnzkX5Z93spC23CerGRcH6co1fRqO4N8iRgJCvCZiluKamKlpzkBp - GBHm3ATXgNJH0IzSHSX96YM15rVvD+uvcjlTbwWdVUYpBDENQQ== + HAMlqkVaySOcnAB1hkM6T/MzUu8PV55f0LkkbBbqF/hOk7003W4= -----END RSA PRIVATE KEY----- @@ -2331,62 +2481,62 @@ vector: segmentSize: 131072 total: 1 - convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:MDMF:4hgxxkkiytc23y5asf23wbumgm:t6yzndqbn4bhjksnlxp26v6f4mxbet4wpviz73qpp54fvwawdwha + expected: URI:MDMF:7itcabaosxi4au4pdhtyjepqnu:emhaqs7urtgtsastqh3kzforde3y4kqm4j4yjhmbnoox4iqqrrqa format: kind: ssk params: format: mdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEpAIBAAKCAQEAwnocUCzqdON41li8FEhElOcjhTUDbUVm8eMWNncqLvADlobP + MIIEpAIBAAKCAQEAv8HiUgJ/Iyw7fYq2drT3Ct0M5XCZPK+5tprVC9wP3jKcwo+3 - ZfCqXIbSmrYeaLlUY7tLeC2ApABStBPRgCC6PbzuhSTS3Js65WCPgJIFzxfgCkNc + gPQ/lOFvyJah8tP3Qdkth978KRU1Y+L7cG5wWFcH3r3KoNB0tUMsI0HaLRjbpTyi - zYIozajsqbEgcHxeajYFUu1y1BeAIcN9Tfnm5ZuhjqD5naZEXgpfbD4x5Z8xFoxs + mU3MuLWlXJOtOw53BOpwFcFWLdXZhODNm8IxEvPKTZYeJstzhUmew5AujZkpnEEg - 4LgcXAbMtEmwdj7cLfie7YStUAlrPJwFFAQUjWyN3Qvx8foTKSqI3H4jH3heVct3 + S2gF9JFqwB2LGae5NmS1EbKXPHVSAUvGhuMBvWnXD1+LNKhLO78VFE2C8rj+zCNu - boFcyXOSd/DrcKLAwfxWFrHW7sZnatJcg+UarRmXfBBOi6SorjabVUd7IdStt4z5 + khbF8XCsf0TN7A1rxuQ0xXlUDuAiiS7Hzx1doncQuGSCApKxBzAEKXvprl0UgCcg - hLtloKW5BkAKI8yPa1CiVcK0Wg57oumVPQjgjwIDAQABAoIBAAHJyDWkvd0oSTBv + sRmVzYbPLWoFVkbn8dJgwwyzDXK/I15/aIioNQIDAQABAoIBABDpdKBu/++GMyj7 - 0P7povrph58HG7UgAtrtUANAwHooH2E2V1ikOgog/FkZgGBpzhDJzwnl6MSgZMzz + VuRZSYB3xm9l4t3rUaG9PhTxr6SVKiYurqx83i6vQ0CZqGbWMvRnxxA4plypNjA9 - 3s9WvbM3hForZR9oSOKhnNWEiVLN4skatOh39aRsP5sQ85p29dlJnG4kuK/3ycup + EJf15YqlAliuvHQ6blCeQAJMCIX5r0V/d4e1yNxxiMgFbj3LJMwWMRR6HLOmLKz/ - POZx7fpsZWyfmDFSaAG/zePQ9s9Ev4XPUTAJZ6YAUxutPXv1HddvPLP3419k92lR + dqLKGbHmNm1pU/dv8hxLRelRigmK1VJtvyplgYvJ64vaEz4Z9Gh1zFIHpdeiXSKH - zoAPiforCfdJrvRcCqspb7GRYbFKEKgLBrahdQpr+d1rzlOHofl190iGxKcgvNUl + 3RRCCXVwUhwr8VikpsSjPeSfRj4BjyqlZr8ibEkvhr89hsNUxjrrZZ/stZX5x3Ea - tcFN6bev960/aan71ykvyRjDWwpXZBBudtXQsySarYxnzeXMaoqfdq//TM8a+ydx + 8QqRUrfegEn0a+hq93JXN3Gn7psmrL0Xy+0/IrW619MjgCP/s9U/VIAEhVcc29B1 - 1/FckokCgYEA2Vi11HfFV03/pLv5qeDf6Jg5Xb+uFLJrb3hm5Pdx3iNiGXJ+8NNy + MQj1OQECgYEA86wnP5rqlw2z8rEoXwHKPkrk4gVQpQ0zVwBJQUE+myTHMaaMuArU - dE2RW8wUSz82RibU2cAlRVcpPEdQ+PMTDVhWBG9Yuw2ZHAvY1ribQHWwOgC38Si8 + 8zpcoG1TROJ5FJVERPQLvVttAOxa2iK04pAwtaCiCmN3CPqwv9j5DaLVcZlNqpe4 - X14BZR0eWC5fFKLAJZREacVGplcNF+LOYWbpsR/iZ9Lw3MHGnrp8ivcCgYEA5RAz + msFKDCahqgPq41/+yakb9RG7l8TZn1ZkfgRI1wIncphtlxCTPrUXCSUCgYEAyXVe - 9kMryCmtGHCfxqKdjlCiMMiTKjVMW/v0cNwZpH7lsr/OEFLa02DB/xBhqT0W53Lb + XnfyAj5oN34k3nz41Ee0DYUIIVd26wUKVUFt75DwQ2qucHU1Vz1ajGXoON55y45d - S4e5JfccEy7mbsW0EXJQHmtPErgoYOueWKOpbVmB2ZQuvx1miZdFBabGcyhs8/Mi + avd1W1aZMs24Qg3to0UoC6n2wX1gWyf8m3ibIbTQkArcePPZNQqbqJqICJvcq+jb - Dke//9ui4nSpCPAe3UL3V8X9OL4+DmDjyJz/mSkCgYEAomufngJPL7nzE9kBbsjE + 8A2zk2h1Zp+DrLWpBUHnW12jpDdFKOaiyyWdHdECgYEAsJCN4Ajg85N6UOEN38ns - qt2u6PcIESFwFeIlCnA74KQSeC/O2ws4md8phC8S71Ryq6PzJjJn59SF1Sz6Pr/v + QjcCosQ3K2HlUaVjb2VXeBOuQsvsK2+t3pDrjVOqgr+X/NIsJcqwtwUIdyLMskNz - eeaMiU3oQgicZZAY4AUex+Hq6r2EuCwX8TCf3D8RYRZuKU6iRrLxGRW6gS3GdBYi + zresk+9RezWXi2obqOgPj1HuV+I95N8LZReqECPuAMPV7+wfMwDWwT2YMODy0AJJ - 4jj05E+OcsX5Bw+r7Qwxa+sCgYB0x+H19yDnF3hMMX8DwfwZhjpqLJf6uNmJO9bP + zwZLwYBOFTteLZhVGZselgECgYEArmRnmKeEW+TiGoecKu1MCZc4iiuK6jHow3HN - gyb/mkJ48xiXceZmRboh07Q2mBKJRSFQTI20MVt63DpW1yyKiIEYQRU7MfBEGVvN + jBfjrups0i9bagZMcoSuCbN93xzXmhpXS+2DLdo9K/lhc+zSte97xv0OmliKPN7U - TQMf4LY2uzlp7g9MrnZd/zzFkSKa7KW8KhBU3SEZ2ugiymix3WZEtYf32eXBZtw6 + kVFKGVeI4+hDCoEsmfng3YdIEwu5bydYnOl/di+K0ZdsSOnIssBmInVg3xrpR4q/ - dvBIoQKBgQDYH8P/gLrZku++MWbQZ6HtxIXz6iJPy9JTaa1q2Fi+9hIqetsb95uB + idO5usECgYAeKvPyn3i8jPHyAZGcpw6MRpNEr7NjdrsrXE8lC/uKEHdHBONh6kLW - UBVTlGbaFKsTkywQj3cO4c9uptHq7VK5DkM7O//6cDtVvFZloH9W/QpoQ/t2XMll + +B3Tq4wBeUjQkRjeBCOIH0fivDjDGQqOKDfV6ROqJfqqsMCEc5Z2hMQWl+wNHf6c - el02/IKmp7dAx9kAdloS+OgYGB1QLp0LsCxjCrXKAWJXLpkp9EaCCw== + Xbqb9Vbs+foYJImZUixnhsBUFTFItBrgrjLPIHyhXW4htQ6Y/6W3Dw== -----END RSA PRIVATE KEY----- @@ -2399,6 +2549,156 @@ vector: required: 1 segmentSize: 131072 total: 1 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:CHK:acgfc4hxhztews6tdk5i5dmzxu:qehbpfrrj6nw32tdhkgncqaqenvlm6sdiskvjq4zasc7g77pzqta:1:1:4194304 + format: + kind: chk + params: null + sample: + length: 4194304 + seed: 2NVWOXvtwcAFf0J/g0fDC+ypQbnCU7awZoCxcnhpkVM= + zfec: + required: 1 + segmentSize: 131072 + total: 1 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:SSK:fcxs6wbchl76nfcmlc4wufptya:gtizcwbv67zb5g4ezygowytwf53zlrf544vgsglfmpre24fxdzva + format: + kind: ssk + params: + format: sdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEowIBAAKCAQEA4PxKNABaQ7xlzJS4kc12HHplV34jhtKN5QBoo/RSVzB90lnj + + GXAa1wMFDwTaruBznTbM/QEZvMtdgCG5SaQqBZJoXjkBcm7RKeD0OWDDjOIWfD9e + + xnbqOjko6XBj0ACOwSwJ+sc9d39Tnd8v0OOSKaI6wPrmXMzIxwqEBUBi8iJsWwjn + + Kn94dZBHzRZMuNr/FSsXN6t3KEhClEQAG2JbYrmbg4efC2pvQArMi+xw/0MH0CGd + + qDBALfB/kU93PN2ZEb+D7iGic/3pphLwmlDVGI8ZcS8bTcDnNj73TPabshEpRJTQ + + kQmy+ErVHyJFndHXQ2XdfNNbHptiEx5WmFRJ4wIDAQABAoIBACQTix8r3/gqRBVE + + 0xRyIpqDbS+qkYN4zM79PJ+ZuasIeAyHDwQ7toS8E7oU+FoAB29HY8xoD5qh7jQc + + dEEg5VTFEB5CZtR/fOO0Z4UHL/mDIWw6nyBqM2SIWOKXJod/0g7wrbL8SC4as9ZF + + /RKyWHQmSDnnTDwc4aRlBRwbIc1F4bWnbmdoM8M4jbw4ORs9Vj+CzGqkNDKaFe+M + + CYmyZFrc6d3N1bpKR6HG4zS4nwwIwHJxdwV3A8Nd8oABZ4ZI21FlL49uicdQR2WH + + iZhYfCjqayZWDIKQNxphaXMRuZX5lQPxlJY/3g/e6kEIPUc5R0QLw+KeEvU6gCQW + + gV3q0rECgYEA4mxMFUbDghRr5rOCjTt+kC0WDsJtLWHMLb1o7Gk1eCz8puhRBhAq + + 5vxAAPtx722HAoJ7/VkNpawsZ8+O+njjwt13UqPfM/04cz9LAsX7Nl7Iz13HX0z/ + + eI1mjqvZ+I+TPUbhv/aU103BXpGfUzDFAJOgfF3Tr9OFOgD9y9k41nMCgYEA/l/r + + w9/D06OmkAo4CP17cs3LzJTZG1x4HSF7Cri52BEhRj69ePkKEB8oWhhCg+RUVb6y + + qPBBHhnVyB+n2qmRacL2C8zvScbSjRb9adbVYNk1t3CfsKb9bIinEyYhyBnV9Osi + + bIPz3KKArI6g0UjTJCDz+G5J93geLJ64Tf9actECgYEAl9FzpmR/XO4id1rv85Dr + + yPJiMt1M5TwI8rZo7vOQZZcMhUGKal1W1vBWXhI7EAZJm3YweuxGSUrLr4OtY+bB + + GPz0MBYu6CYmvqe2vRJQ4eDmFpzTvOPc/FEbbhhum8pxOIoZfmRw4niBas6LnPU7 + + cqqJ1jn3YZKbZwwZIKDzCl8CgYANJXHu1pKtTmjeStjohgkqPr6InSy3QEko4EEW + + pcNdCk2Y7scmCGDfwmx4c4aEgHlnUQq0tm33G0i0Jgarw1WzjJLguz+mpXLePdDs + + 9mBit/cRuu1V2NuCD3HrCG54g/VNyhUmXI9u2Ksjv99J0aSv2lAhh7mk431TuwQ1 + + a7wF8QKBgEKYyNDgiDmOdUaKLnwZYi0UfAHZ9m4MzgaereIScyHNxl1FiRAkkte2 + + X/Crl2MX4eyLxjFiafvb34l7wAsw3I1G39WvNYDS9tI98iryizB/4c3YDWf7NeJZ + + C5iTznHcOf8+8bGuKjjEVUp/RZ1/v4L1huCqLYCKak1jFNoJyGxr + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 4194304 + seed: 2NVWOXvtwcAFf0J/g0fDC+ypQbnCU7awZoCxcnhpkVM= + zfec: + required: 1 + segmentSize: 131072 + total: 1 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:MDMF:srwpg74natztaqqh2rm4p3skji:dvxhwilixw57pkcto5aviohrba7gbwyscnlsdvjaqhgvmcy2jezq + format: + kind: ssk + params: + format: mdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEpAIBAAKCAQEAr0Vb2oFiCV5paQeKQD9USqx9ZKHux2CWCjzUTOCGpgr/uXLk + + 67/XL0H+69Z3CNBk9mNRpOR4bDWlps9QL6axSz9Of8E6PSO3YYEKktyzSsMhQn4M + + AaJvN4RBgmOGcN1idUvslv7wbNtlvMfvqEUCSaylrgErcNsudn326/kkE2ARWfJ3 + + ulvZ6h8QA0iDKgnQsuOq0k4jQT+4XLe6RzXIubNZJM5AwBCToCZFUKQ9wuqKtbGk + + hj55NgvsKUj/EbBEaxDzjVoP+hdSAR6XoCXkCKHzAzdJNyLZgUXy5CajeOHFDsm+ + + gQ2ZjRhKVtAz75QWNy7eas/Na+I3zsJMzfVbIQIDAQABAoIBAAFdkQ03KqryhJqN + + dRGyPJyhUo/FEDMW86Ghntt4zgEUqbCJmTLPVDnuxxw8wpbREtGgQqD+KQRaIvp/ + + 99ALOdXhfiHSK6Xmyuq0TT9e1KRtjUCInVzU5bjBFnE8Mm5bgdpylzsHl5rC1ycn + + nUcfvy27mIXYxfxzhLLmdn+Y+bjkkyXJBqfG87C31Lzyc+08kYZQh7/Rx8wuXiqJ + + 31CbD/fzAiKWKWeKopRs9nmvls6C2xMdKqM/r1XI3Dyy4mDn9xzRg5/uSAZYOmeo + + vw1wvTYkPQHYdPZJ3pxQ7GMhY04Xc4sNtxsBlIA1I7MANw0jNVEZd2yMrt4luxG6 + + gFwUkL0CgYEA52BooMM5xybo0uKSNa85RYWGOepZQxPV3hz4dczIpuiPeekQiPD3 + + etWw7+nL9aygm7HQyc/QxpcfUGCPZoiwRbaL+94wQ7PMleOtC0ixxeLWQOS/UQ8X + + QklSaBEiUayDiZJGknQdjWlbhLyGGfzIU6pDh5TiS3fJuV0lA8E36PUCgYEAwexq + + UugfChyoM3ZFIs8CIjH5rLrmfFZG9RQQu5xKKjWYpyfe+y94TLIepKv2U6b7oeZ3 + + 0BnAF5UB6Bc2/MrX9UmI9yB7a8cloeP1eVzH5g1WISJfVKon+Do7eRTlGUb0Pykn + + C8Rk6LB5LFEWjsMTZc6AMpo1f4gUtV27fdJS/f0CgYEAlBmMvyJXMFeCfcHS7pP7 + + J7nhAd80RZBDu8l1bAmpgdSoSdNZ5x2+exyfBeHz0IwvvZji2Nqxevwuagd0op/p + + nKXNEmnVIPDMikDSeb+NMuoQVDdXEm6DZ8WA/uXAvuCazYsYqxOx+tsuXldByw6X + + t53rXbR56O6C66hoUe/ydqUCgYAFbUJEc656r/adChBBOx3KKy/bf5d3n0p5DUiy + + l1sT91AATYNV8CwjqVBmN1G7YY7lJvfvYOkZP9g/0HZ/eIW2nYoxsD0D9Ry+fQyf + + itMlQvZIExgr3F8l+Ss05jrLDEtFgTdQgvx37ohVjydcc2UVkkPQJrScjwhVUvwu + + NzaPWQKBgQCr1nEFMNY1EV1cucmhZlp5QOUhybUH0fqPfRjRp3sFuyu3BUOeTmum + + lkKoA+Rs6Yrs2U3OsEQxOl7qV5Ab350wiUhzqOZsCVEK4UkAWweeY2IlsyBybbg7 + + lSCo+6eJWyStRracQofwUwsXklCZlBcLAL3HVyH2R/9UuplVU/Wmvg== + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 4194304 + seed: 2NVWOXvtwcAFf0J/g0fDC+ypQbnCU7awZoCxcnhpkVM= + zfec: + required: 1 + segmentSize: 131072 + total: 1 - convergence: ZOyIygCyaOW6GjVnihtTFg== expected: URI:CHK:n2642ofjfr5qljptdwjbwjz6fq:uwtjrewrev42yvez7br2574xu5vleallwnyddjaxqquvouv3dyra:1:1:8388607 format: @@ -2412,62 +2712,62 @@ vector: segmentSize: 131072 total: 1 - convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:SSK:jr3l7i3eegfx7gp2umfxhohsiy:qz6pu2l4dckgar2wouwl43turrkpokddxy4dqqmtdqe73rksf2ea + expected: URI:SSK:m36gxdp2beqzfnymsb7q4lgmcu:2gqyjo4z7f6z3kqbbtfasnmfxdledtkxr5vs6hrksaodejunrdra format: kind: ssk params: format: sdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEpAIBAAKCAQEAsDyWgH/oavnJ+/I+kZuiO6uqcHm0Eetz59LfgATNImyx/U+b + MIIEowIBAAKCAQEArJF3xtkkEXvoJbJy0WIuatA3Ou1SLV815+BAtobFPBhFO4IM - CPm4sICXC96mHgjY63lwzbBMkjjJFd61G3Hs/jzLktPivgYU2bGltqxXEkeyk9h7 + f+wF+3TLsLgo1kvWz3OBYhiElx8W+ZBifi0iGQfAXQJNOerRhGFgdBONN5cXCR28 - YzevjA0yZzg5mck8zECNAafmjHvGZK51l5z0DVpzfcxr2SmBIWMCFOonetTfkui7 + QUvvmmM8gB1BWfJRcoMgRCsGrfEJE7r4OQ1E1iPwjOr36cNvssiIT8LDeb2yQ7hq - 5F3uEsgri1jJnNmNF1zabwlRxNrOWk7DxV7tv/Nb8rk+JNBwdJHsCd+u80tI3s2/ + gzN6wDpFxt8aXR/weo0vjcw86A1MKby1zn36ufgiQAj42f/8F4bXHh8wGs7IBKBz - l8WWEXCClcTaW0Ta2/6tYouRG/f79G4b7a6RKu0zuv2ntF55YXbnxd5CyuYn8DPI + 8ZkgwCaI8UbksX8YCQD/i8MCJCrlQMaIefxlhR/r1CVB0hWII9jwirbT7vviaI3g - JfIpfBGRdf+b3+gDxXzx30s+Y7kbJGdN6AJRxwIDAQABAoIBAAGeyihgosm+K3ZY + 7p6/s5Xgf3eB08ZTvfD8Kizenoh58DoZ6eQ8ewIDAQABAoIBAAE7ImsZNnAnfZ7z - Dji81q8sswooZEl8PglSPO/qjFNSSRhmk1+OC7n/dZjA/L/90pi5+d1YXj6s91RQ + OG108VbSuGojsj/fm39VcrC6omKM28WSZmttMBe8nA1dKvFoZhZhwQ0FlsCLOLs8 - jmLDF42zxMyLbRjjcU6wtfSNlANbEftpZ1eX4paF7ImWwaBYCQAtnaIGrRdNrECf + A+/Ze5JJI16mew2MNsVmem0pjIrWeZQXkbW3iHSF+7MAQmyVVW5nMA48blZET2fE - 5Bsc6Woz5XNW7UNg+U6oRdeCUyICAsHl4GwM7TgHnAJ35ksoPcN4YVjTxIFtjeWI + ICwsA48xf3BJ5s3UlpHka1CCV6mhJUoDhbQjnIOy80sOVExDlPuyh1OBXp0hsNvz - 1LVus032XEVHyiMzpP6y8xcwDT/qE0zG1L8hxph8QbdfH5Skjx2nZDfLYiKaJQrT + 9HNykDeT1uioE/E3HEQ2Nsmo5YNXoQD1oBnk2aOZv9loFFGGVXLJXrYo9fJ/OmcW - O3Nl6XO54fOGMowTD/yGkNR9sOynHdlGQ6uMlWqPWyqPr4Gn3ihqu4/PFXRSzMGT + 3IVz+xIIaVomeEkM6CH/3yp3J8n3Zlv9lYVthpoR3PGdU58wFPD/77BUthLakPMl - F4xV8QECgYEAzwQ8IvZGKN72m4NtLhPvmUvpBE8LPBDHzxah8ozQeZO5SD5InsMy + cNRMg8ECgYEA1zz4Pc9ldZepCZy3ezemUv48p589BX5ehbbpqd+csraacBLq+RGu - zkrkOn+qHymxyuFMhmhUMTJhsqm48Id1TSWN/UwCO+Fg9c8vhjlSq902o0D3U3jG + IlnVfmqrXD2XYnFzWHxExUXuwWYJEDXQzrOxsgsHXuQNv1nEipSZhvpBvH+etoFC - 3V5tg56UxCl6foJ+xletlkTKoZIaDvb+bxF9R6zlY0fpvTlE0nC6OkcCgYEA2e/m + tROEmaa+9MA5DJe58OHW8U48oXQac72bNvB7jF0bF9Z3xGphaaEARLsCgYEAzT/N - uCbgxZ7vaYJZGgYMxpsH0hzrcuScC0XqS7+EnlQoVUnLqxThgUbEwE1d1sf2P7ir + vtJMKAjTf4PknxhmCk3DPSqwAb988v+oYw3Pqwzq+45Sg5eOGImum9mu34Gwxecp - VrFCBhAn/n3MBwSpfgbZ3IvFhSvGfHlA0R6GEnG12gg+rqQHDQdgMIti4rrlSb2J + PawdlHRBY8vli1KHJfKRkATh6Rd+8s1maiW6Mpcy2d8qCUBKX5Kne6sdyk6xIkPa - HlOe9pnLg7tlglfFWH/veofZNc8Zdhovee9DbIECgYBJw4CKFKa7OXc1wobMvF3L + gJutO1CPLUKFuKdxaFNpj2pIlo7D4TbHc+sQS0ECgYB7x3NXUIMfmiU7AuY9tSYw - ibjlyCSAqpoHuFDMVFCUgYarr0XBDFy2FQltrr+3iuvHFrBl1Bbr0L/vIXq8egfa + ikblet0D4MWJDkTYTWF3IS41j5uTuwgydwkhF0UO2djKY0YbN/PwoyQIEp7ZtKkt - DV+iucqx+4TJEaIleZdzlcc6NJPsMkTp7BOpqn/nxb/YBDeYBPXdbXWmTKDsZCYU + hgeFxXPqrSn+xigSLh0Qk7DkL1xdxn5PVjcmic89P6JPTJ5BGg+bXAvgKb3gm8S8 - /W5ec8Tos18eBaH4OiKhUQKBgQCjpYOuxerEGfsWU/2KD/7p5yGxQWv/AvC1elNb + VpYmhZDEJ7FewnLc5RsbawKBgHaMXKzT0HLrLiWfq4QM1psq8RK6PjC0RlogOkUE - e70ekn0Sxe38Uhqe00AMUkvjapVa9dUarNGx8dHGRDm/D14iNwzCkeXIgL1zXC0y + LCdS7cJgIN6qwcMAex6/a5bi1JRqANMDP46IW2Bl225OO4s6gMLbXxR/oq5g3r9+ - meP814vA464Fvz9YJjCxYwjmzYY8n+jlb88Oxx9NlJq9jCCwuqhdbsLIp/ErgLAj + jP49gHyAvknbnVl4Xk46tpksPHlbEbBounThAeGVY6EU7ZbhXr4cGFMFoLPLLQaS - tGkBgQKBgQCmJWwxwTB9mU/KINfcBGkgJv+3FAg3/G3Eq++6H/4h0mrQAVj6Q/bB + BbZBAoGBAKKMG+MRnM9d8AlvTWeJF6nzY7H3PXQHFgs1ISINqE5v1MUJ7o+i/OWl - KKFzmem61c712kxMo88uMfR2UFSiifxVX/aEKpmOSuG24JoBE9WXlhGFPSP5r7mK + rW6q9BKhgdxoQKEmaOSvw/ecS3bxoDyAieXBkTjbcfvETGnUv6JFCmu/tolHCX6M - cfUEQMpuvj5DD3MNaLBlqVxk0VxabV9s+8MrLM5DZ0udhgA9plmP2w== + PHUyyYABqJHohWyLm2AvQTKcSt5zK2GCFXtZ9fMZQlHZUAmiNRpm -----END RSA PRIVATE KEY----- @@ -2481,62 +2781,62 @@ vector: segmentSize: 131072 total: 1 - convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:MDMF:irkh3ffvh2eaitliu6lzh677xq:thprgu7uakbkjzsjahw3ub4j6upj5ejplibiuxh6pomwtfsvz7na + expected: URI:MDMF:mo34r2meiiy46vq7uatrgadt5u:5zscxdgegy6z73ij5zqwq43mlqmko576fxeyhuqc5rvauq5md43q format: kind: ssk params: format: mdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEowIBAAKCAQEAmFXg9LZxuJXGCRRquQXn2G5E0PU2td44P9DUpPyl79Qe3iRi + MIIEowIBAAKCAQEAzD877AvjU9kwf7WSCxcTCtMR7gc2Q2F6GL4y0/YnnGOiq72u - dqjU8JpV3qsM4OyY1kPLZT9DDjYBGGfCOrEo+QO3/ZXJqRGz7WvcvcM4zXoB2RO1 + Zf9mVTfdfn8TcQT5h2dOYWyypEAxnIK3To/myFJ+1/34c0ddPBTHmexUpr2czzx8 - zEOgp9GXD2QZ3s22ZXhG3Ii1hFMn1AyPq614lA5fhvehHj+2cDOcdxQxVFgVIS24 + wioJdG9upkW4tabxQU2xfQAYcs8sEVmZMcTILfGdN0vcSSKKpLx953rX09jxuaSc - hAs/i5cugaJLfIe9S2MsGHogT1lyC4OjPm3rioqaJ1PMMPWYd1z5+/HmIekVlBmX + sbDOwqKvt3wBGDcRZRuJuoCjEBsKKQMhWDSzzuZBwF1xn/oqr2NagJ8s7oTpj+1c - A8hngf5NMwiHJP8foS+ag+byuWe1SY9jc3KOytrnrIw1vu/KpstBVZV/FxdKn1Xs + srfTXwx4oXdZmeip5xMeI/nZzuIYNlSd7q6NSAK6W0CRpiK0glK0ixOR849YmCGc - M89pj/JrxBFh3Gr5e4bnFDBJecgRIHYRMK3M1wIDAQABAoIBACG30IUZ5O4AaMcV + 8Cnn2AAw4ABmLOXeBHy/mUHO2Kdfn+QrLBbymQIDAQABAoIBAASoa4Qq/ZHyXyKx - t9GgVwL21VCTFjsHJtgpNwgVy/zbrMFquEifchKXdq5EmiMm+2VhuCF+8S6yEWf/ + IBsXJAodkqgwQL/2zOCcE8+AKq/sbhpcZB9eMfX/D718Wz6QVmV3XmhRNjNnOwpp - f2RSVklX41/DydEcVAEXQNLX5TjF6qbL0A+YYHUE1TTY6UkBq3+mMbkaoWLarRQo + SRIRAjpuy9w+pegbyEf1ZwY/B2uEYaYF6IggCnZTOpLZqQhMKJkskv/kB5VfXU9U - e5x6VxgeXlKXeRgi7hTDt7w7wfdy+DN4yyeJiH166EtQyEIvg1u009iSwgiRa7G6 + UQP7dh+QG0hWd01cAjfsDDNXepe8Qs391Tzxf4/qlHoZrpYt+B1xItk254YrrPjZ - XT7hy1zZZDknfgnfomZFoXSlrt8duX9OBgIYisDGny5uhEVcvSiMhft91wJj+Y5j + kjcR5JTonutaPi26x6ULLS+VLeSGdFAFf22xW7eDPIX22aSsYJ7nzAWodRPGqBcN - tioxT/+sHY6OlPgNMqU888P7P3DyL36I5TmbQ93yjhKpdAP3e71MJhoI51UdgVnv + l1mgcHgpSFnP/TbWm6+FaM0eIiTad1I9ESpQdB1Gvw2uCNqa8sN2hLgDztkWH8HT - KAxEBQECgYEAy8CTIaREsDo7lWlj6Jb5mH71piFblifW1FEPRqOVjASlTcs3aemW + +PkOSaECgYEA8PxHZENANGqVmH10WltFAbzmDMagGeLC+omaMOekdBNI0cSTlSTR - h5EMIzXpwRdHLPrd+3wQy/2NATfwGwJqgxoxnnuTpb5DAQF6CK5845b521+x18uS + HEeELWPCfRRo6En2VCYuiJGdi4vss6r5ppI6vu66Q68eMjB6+sozIA8CsarH8N9/ - sfpbij3NXYcEdF64IQGRIkf17rsCVvcCpNRfC1UZXnHqfqh04IorZ5cCgYEAv2YF + aDBG+Fy56/2vrK41ywS5L0qTMbw1lmRuGgta9eCtsYZvjGUyXtzgeZECgYEA2Pj5 - mFC+wk1hSN0RrJO+ZqyWeWkdJwU/aI18mo/ro+oxYIEJBVIteH6Lt5E1lCx4deeQ + 8yurZUZBhbQLGiL6qR2Cs8Z1HQpeGlopFN7o5EH7arTy1f3CF7McYrkyefnwAzCs - AwPgv1G7zo5134Tcf+o/1kQMhh7qd5oNlGi0r5jku38TjfERBZIl7e0OWDJ50naK + JcNl+ufgVtx33Bkm4yNxirU3WlzYbtC014ERvdLbUdrKspcxmn1zSwrkHbg4gpWu - mNsN3KqqbOtHC4GipPjnEOYJzJr/qNstPUvnbMECgYAQwquXtdqMoI2sMbotNNYd + qLtNl1OZGmpVFxIj27YcdhXWuaMRyKV3PQZFpIkCgYAtzycbDhWkYSZyyFZX3sWt - TDxKyS2ugWJznqNiDSzNEsjCSHgrdzKRvkXAU7wBzTdmpNBD0qXTEe1ab06J+j3m + YOUyRIempA6AZavj5ATE8+2BwqZzUX5Wq9mabz5HXJvcnEKxGFj8KQITxtOGC9hN - wO3Z+pJfrPH4EDYIpsnRMucku492j+FmUJDdI05UZjnglLYSyP02U7MQS0PbAYCv + K7rzFJpfx2gsDj8ycUFqtK/Eajx7s2Caw6KaD7Zf/+dnIe6j2xAAx2JXr/lXz1uG - LGURGpP2p+pBNvw+SD9fywKBgQCm25xZE1uaLLd5PDDiUNMW07NDGR4vHGYREffl + o+X0m3MpLe8CdzIuCjq5oQKBgQC13AIMyxaO/VMgESeZEaaNpzmNG5O/8pereNSk - Dz8Q4WQ2i4d/ugqmFzxaxh79lF9X+o4T8teGMw0VoCCmwj8wzNjmRODeNCmYJxdb + NK5528Ay0VYU/Ov5V1w8d0QSruZ4lgxEXsIUitQjmgkwxzgr++JIQ9oQeG/EelSJ - oISU6SfPRZOYlOaQAr9KUvXEcgy+LFXbuGy3SZnV5q9DGrreM5fNpZ45X48ueBVS + qRpIw/qmYj+xbz7ZYbsINCm9q1JaScGqlcvUQfK7DFMj0kWR+9NhOq7OzBq01dPa - cM/KgQKBgDGJMoBvSYINh4vj7re/HTEFXhCgQoFxiewkdHYJq2AOF3c2h0mnR76H + p24qGQKBgA0qtgklc9qaDCEhNT66KHDgRTrrBRNf36IAWD5mEfb+mpVMBYBiq1ej - Ub8BegJE4MiLZTjsqQJnJkBBtt1YL8y9SZH497KSjKOEvMbgEZHsuuHdV+KRC6O8 + 1UIae8DbgJTzLNHxlBtFEpCB8TM8xtVfoW7XGWlQaIRK8Nr1VvmhmsCQVnhK4ZKd - OJBycdnr64J/u3fNVQ9Lv6QcJMmAU9drhub/ao0z5r9LNPgHDP/H + p3IMoWvoug6cZpr8Hs6c7j/u5zdwbbYA7/mDRer6LoqmA2pYYnpw -----END RSA PRIVATE KEY----- @@ -2562,62 +2862,62 @@ vector: segmentSize: 131072 total: 1 - convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:SSK:thrs37wytqak3wamf3fyjs52j4:n3irbt2hs7g2qthkrpsllqhoxspmca2nah326tnwa4wlbg64kkxa + expected: URI:SSK:bmqbu7cppakl53iwet7rwxdkc4:hc53hqy2gli66tvjvnl2imgg25zjj4lylooxmwjghaxrq46ch3ya format: kind: ssk params: format: sdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEpAIBAAKCAQEA1VrS+13yFy5YBzglh25Kqrqs8v2CMoyTtmlr9khQ2FNU1Xta + MIIEogIBAAKCAQEAq12cz/r3Jv8sUPbOvXlldAhifcEj72eE81sugnCZnGS67pvm - gskWuszB406PFKmxU8q4yOTCavWERh73Bo+1mumRjgNONfQRBgHSAQfDkiKGxvOu + d8PoyNgMlFXbTKXTsxeDVcoOdUcq3q5dxhhbKwQVt9XscZJvV56zgDTG789tcmmC - t+ctuH+NvAcJFh1uW5c0vL8OxdbosTwbVkIWz9qrj6HTY6yAWTzukcd3CB+rv86o + NqlvCnoXVXThrj7RU82qS8V5vez1ovaVo6tEnZnXHTJ/JdrLl4yfP21W6nz0YkC8 - Xb55r1ZCRcbRfCnjVSLuhta3bt5yeGbabWmTgQqPZmwuMa2oWPWO2wzi0M6kCB5U + cMqSZPCYczfq78S46njEulLYaEPxV82Rl4ffm3B24l1Pl5SmGsSsVe4SOBR7WFse - K8Bc2aZ3Xa/bMqpohmPwXVx8Pflt2vzcd/0g9EULOyxNDwRIY/eMTboCGQvKmfbu + JpZB13p7iBzuMT2SIQdQAkIE4ILDCo+s86TSVFDRfJpbFX1nD1dntGSUmtvtIjEf - WxJotkJqLqP6WeuZauK8HCssXielksz9ne3CCwIDAQABAoIBAELmCzDJaN8O54g/ + 5uiw56hwZ6Xb9tiSIOLUrPHMWZLCZUYNfqRevwIDAQABAoIBABeK003BWS361XBE - +TiJgz0ccp1wkxIRlUGFtdYQH9Vs77VOy/clYYyqJoOFPwUOHm21K5LGdBXArTyl + D+/42wa/ViSXoaO0sY+rQa764CebsRCxy0491F+vSr8gMnILwBM1Ej55dVYIUmvo - efSjPCD6aur6K1xsjqfxCy3Khu68B8G7aAX/JY1r5X/XPuihythKRb2HNPUg6W6l + QYrC8tdshr1MPuD1cKV1cIyW85OjiBI1S4XN7irezhDX7188Uw6zzQb+2LROdwqN - d7bo2ylKmi/b4KIo1Ufl/LJWNoMjhJGKyD0Ib88PNziILrUP8cd5QWqaBEGH1df+ + 3M3w7ArIxURGGTCup9SopYIVt+Cb0yRM2xU0Nnuhk2CPemNwcfzulxzLOoaNT4eV - 5FTfQAzU4OfrY64SK/loN4uG2PeOLCdX6/rfWf9m67hhJevZabCHGg3rIIpNzdbv + wxbhJH9xiutZY7+gk5CFesouMgy5KvBCLGLw8osMgdBhX93yNQULknidIA4Ko9jQ - THjCxvrjiuozvtXMrZ6Wh60DYRWdUwcTTFQbWRPPiIijQki3AkqyUExG3y0Pm8FH + 4+4sesH79TmQWg/z1LSV1XOKepOoZzQv9RY060LRQAAzdUV9C4iCAnKAWiw3mzPz - gM4GBOkCgYEA6V8A14+T0vhX3y28haL5z+7Gl0V4Zvk50YEGYusVNYPfhTFeKweg + PZ9rj/kCgYEAu8NbvINqFTuQ3iy+xLmDygNYZ5scetnZsxMX2JOGb7DNof0rH1AK - eniVQaDFBa99rjXUoSVEHT02QGGqmGw7mgEosqJuQs3e1/OTF1hp1UKwscwYYQ0H + YjGku2xxJHta6S7uXqVhItMf4n17eUpb01+Id+oeYgUb4BjTfBW1sThv+Wl4zKvu - 9gIKOhkpLORg2A2MpQcREg+AgbgluEHaIhlnLaA5ttUIzv5pMx8F3WMCgYEA6gr0 + iQhrTl3AO0K6UrPYoTMx2pcXAfEsy49YL6zYajnPwwXDK3AQijOmBNcCgYEA6aS1 - E3gLTz/KqlSPoNnCS0etSUkpW7iCPzzRyNE/i4bx+WxISdGY5fNowTy3sMzwN5kX + j9r42626zK6ZtPpy4OcjbuytMZceyvz6dJnUKRYm9ykdWBmVHxKJZ+a3VqihaYS0 - 8gOfRmW7M9j8Ho3ocYY2Qcwh5kb4ayCcb85N5zUlLdddm991eWP8pz3ArO4Tw9VR + /kECd6/HAQaPMgR280eDfhVHZouKi442Jh4GiC5TT5gGzAEuoUgP4aB1AQpd1D9t - 6GoaYE91ZQggZxTRwD/duTggoYGJURC9VgjL3TkCgYEA5zhm2Cz8dMn0Hj7ti6an + DwZ0iAuLtRQYNN8j4+FsNrcCV6xxrS1LpWwy0FkCgYB5bspYrCEipEh3+DZUoqpi - Rtq4TsbY/YWvQKFK15U95VDslMYOHCopWU7B601ECFcQ+huBucv3idTNPMrHwM9z + LzGwp/eOWHBcSV/luNt8RrtnJYYLFUfx46tnb6Xo80KDhs+xNIIS9LotT/xYIEgs - 2imNzjfbcTsSsPo3YakK6u5xrSeffADyQ09QHLIzNrRsM4RxNk0jH7bWRzBRxxcP + 9x8adra5rBYwI747BQtiF18LzjPLIvL9ew1zPFzDts8sB5Z2AtceSRMfNWxEJmvh - 7jsnHHCk3j6CxLwTNUBmiisCgYA443S0jsdg+gaPJILM/GFn3wJV//yXmN+/806i + QYchhEwjFAn6gNqhlu+rNwKBgAwoJ9JOYHh9t6SCyTijd2rAXBWfdvuHk3CYbSe4 - 24nwplqG4DUqDFJ4ApSB8/pKdWYmfYX+g7bha7T3Q1T1MFVB0ve5Qp8y1ClqEME1 + AVQJ9QkTOJWm7x1ox4GCfbOinpNw9kHsfAZiPQaOotDFbrMF24+p58csJ49PXP3q - xBXXj2l8HQ9Z5hUt7onpNO9ymWQgg+em8LN8mZPVfQYzSDI74spITUZRO6VfGQyM + vghD8M3JaUEgJp75sunYgX4GXg93JWOMwG97uk83tnK50ZI/3nOSXirVrCyImNEI - rxKusQKBgQDBIgHEWMyfNawAjnSk0PJOSyBydLRQ4W6XkSMLmHqF05WUii4i9Meu + qjQxAoGAM8Or1b0Ko1SQft619DRfKH6pHmGcP19ymidhcAkjfvwhJbrEkoGtkh/2 - OLvi9VObTZyi89IUcqZBs48/3B7FQ31IsJoHPugnzov2C7JHg7+EzFDPupDQdMcG + TBowy9QLaHMnKIPnkVXGNWij5RidLxUqrg8GzdYvrKC3cDOZidr8GNwIKQIaynh2 - PnxRNGkN6ZvIfZnXO1ZzwGuuwOH/n7LID2uL9ZnHtFuoouaOaoiJ3w== + /TM4NRU628hVBPL+8v81u4EvNvNBTgxuRpn2CcI+nVO0arCHg6I= -----END RSA PRIVATE KEY----- @@ -2631,62 +2931,62 @@ vector: segmentSize: 131072 total: 1 - convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:MDMF:y44dzsyjkmisidhfgleytdtume:pllr3w4u56xxl24zfdm4b57ppxzv4pq7g7pnx5cfun3gx4iyf32a + expected: URI:MDMF:jg47nspwliweckiaov4ajbts6u:yegxrkzkdb4umh4rayql5rwybftrtnghh4g324cehqc333ryscyq format: kind: ssk params: format: mdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEowIBAAKCAQEAoTdi7OsNEmLySMALvN1FoBGSBQEf3WFpXZVHOHxWycEGkHfj + MIIEowIBAAKCAQEAo6KN758FqGA+L30E9U8AVEDWqCu0k0dzFt4Y6Njr6GfYLN3a - s5Pkm9ZXew3EwB3+KpKo67IeojMvgdhKUKd6kzI5opMJipp4b3uqleN0eB4a7U8p + PeeJcuFQe4NaohPfderNWlDSZ2rRDM0Ftq/MQjHlBCkjY/PolqQDARbXrP7rmDsX - 6QmwwncabdF21cEyJ/FQmxi5tNioj+fO4vO2yzNRUpS5oJqNZEIBjftJPdlhBR6u + zyhG0sawySrgld/sy93UZVCprF3Lc9/CVNN0iphImfpOPNtH2FsN8ZCqi2fyuL4D - Un1uYb5fCQ8vgnNNvZXVdYF+R285k6htnaM4/kn9YjONcaHYQv0ykO3rsNeLdKUR + dxzVn7VfPOXKWx3OE6p8ME/0UpwEwAB/W8GT1CvAV43spMxcQIAmANpXKb0agIpG - IlepsqGtMBMwsN+bWdz5KUVEn77DvaMu+Z3ziveee7+FrMj79gSAUVPifga6toF+ + 18WoEI+kgyIS+evPQBC5pXRAmoeRgyPVnNZAjhFNntDvP7ZotLJ2ZLAsu+Z8xy/R - IrD7Z6Ryyy+7yYciWdNi7hJForQHQb76o0TsKwIDAQABAoIBAAQpoMbFl5alcmRk + TWe1xGydtj4+4m4lMT79VJH3KiGtggEzr1ETMQIDAQABAoIBAETFMW+uXntYD2p3 - J7QZMdkyYEb7zUhTNxFrr3qVufdHTMrseuHMoZxPz6jFkBJ2fnUNSLTz6kGS48Oi + 2VibXiEKquw8igSHt7e5mbBqUiL3WaPpjSoNH/f295MhLjsFrRlql+lIJFUwUDFY - WOI16NQlzwIpHMJ06ZjV+cTD/40IbfaO/YyJq67hvLogLJrT/F5hUeJR+Z2Mnebq + DPmtQmPjgkNQYr5EKND+lwCjL/tVm3/7/dKjM7irpmq1KXPzixpW3UfDMbvuI25M - poIKppTJlEHMcEc60Qgegi7EfGgkiw1uphbOmzKhbipYGVRvAGofjKxiU62Mh5pz + cOijgcwpmgGUb8MlyTkFc4O9b99sIRrR7c+Zf/02tPPo1gCXIyEBKfVoUBDajt5l - 6MJOjaAjN47wV9JaoXKmO4xs1ewUAaAXaV/DdynBEM1ZBS4pstdCCxD34tz+OIUA + ZceA7hIcFjlUUkdR9zQbEw3QaJQ7Ge0AZBQtqz7t1UwvhLn/+cMD7MzQi7TQpDQT - wj5dKbmj+7v4V9J+y1T9Y7qNCmBmCwdAyyBzCvVZV01SACdOkC9yNPkF+bAU6L9o + wO778XL5Wm1TgZPivT8t8bvvo/hwtZdPAXOt/gvXJNqXdD2kDjJhWxXI//YSrDRi - RPhmSFECgYEAvOG5mxK4nKRSKqZzxby/t+jzUTeMnMAUMWGp7/VBMsqJ2MrZYcxs + 5VPVgu0CgYEA1siNix3V46t8B/nfBcs5kiX2/pMTWqQePcWjVQpOGNz4PT6e76UY - mXFKhmHTIn921rp3Vvp/JzR39o1nVdtg74cmH0OLU4wuBzMwFoG/A+h/VY86v7MJ + 23+urd96KuUrn3d6LdJBrsfEr4AKJGDBpETKZnUgZMsgCQ0mnZIExxXYBZ/OpeYw - 0dMHrZWHTew3Zw/fkF+GDkJy0lzA3qoJqi1VEBN4DTwLZD0gQ8d/WTECgYEA2oD8 + qG8bvB1/Yldtq0Y+14XxgC8/XJY1C1Plyl8Yh17NeCA7htuir1Vep3MCgYEAwwlI - GuvALdyxs193OssSijAeEmJ3ezP3SWbkQuA8UGD1ULe+AS4T96W/WcKTLoREiT4+ + ycE+P87Q87+hmO3xHnRD2EScT6+jVtIr+hSQrbHoqPDjNNgHnSNbRzbSvkeG5IS0 - wIIjDdgEXNXiD6e/0vzDj30LSHZELyfyKcWYX4o1G575JKd69ga9Roa7n0UE/mHM + l5IpwcDLSiGn2+u0AiaP96rtyk4fiYhKX3jis9vanMi4VKORuwMJuIECMET1Aw/l - qOqAXbfLYqhHdTgXdDBERH9VtOAAB6aLCl4dxBsCgYAzrl+murygv6Vr3heXZ0nd + NgUlRlxnYqmQjOBn2ArcSS/UcrRAyfbI57jTycsCgYEAgZjRjzeZb55xYH6sy1os - /HN3KYfj6/qaeGqTKbwpNZn6I6bPR6v/YCxQELxAmDfgES1OM0RPad/ZKl+38krX + irrNph4od3C/rpYqT43AQdBTGOFIFWGQ9iC8zb0ige91uurklfFgII35Z8viUsDv - v1cC/uxEc/q0JaFmxyGI5DjTJFmi0k5Bh0h2io93FsciAAnf6wM3K59XR+HOCyCR + FqdLWTcjLK5DzjJZMoqAx3+usPYUQpX6lic0nPVPf48xZT8le/YeGjJoEP2xU/xz - 282GlI0oseE8EC2f3hpOQQKBgQCHSgnOmV26h8U3LMrkCkyGZ1iXRYR5MinQtvZq + kwB+VHAnmmwYfu7X4uOoEXECgYBz4N5IUPJFQwHO8Lp4fFbYO0fcBNfCWJ55hSHv - OfDeS8pYmgv5KxCN64BZEVKUIK1W1MWB6JHPxoqc+Ikp7FGnT32+YEwWJ7P8Bp24 + 0awsJxoO1iCIUxoi+NDQvPf1adXxjA8oRwVcQsoF3302IxKufG7pPbtOiaAfPMTD - I3I+5ZIQchQND+3gWzfibRXKfa+j2eYgSGIGpQA3K75i48IR3LjIOJdWkMMz+Xhp + eLVpG2UF5hPu5cg+Do4F+1BrkWzpRtZuhBwjc99RNWHW8bWBHOLI8QwOop4j4OZ5 - iPChNQKBgEI9dKOoGJzpC4CEN9NjbS8RxjzznQuW2pc8hAVgGSJlhFhTk0XwjZwz + Fs1uhQKBgCKDVJojSw3IOoDC66KAN+XQfZ+Q/n5pEElbaUzoOCTg1q+/4mX0HZ30 - S/PFv0JawKa6xotQ959iVDFxHksBVw2NPfH5kbsEvmUaId6mOJjhLQDtXMzyIln6 + 9AcIaiXmWAjMLmOcihMMm37yapIIplhF81Mkb6cYS3i7boFCa8Ioi/D/ryxEt4HU - LMXJeJlCDZLMMBJBXh+rc7p8osbH/DX1vvfLTn24YwFhU/tNQdfq + kOZqm/l1lQixd99wjf9i8ozjxFMop4WQdNCNzW9ujyrdBrPtKtYx -----END RSA PRIVATE KEY----- @@ -2712,62 +3012,62 @@ vector: segmentSize: 131072 total: 3 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:SSK:o46hswymw5hlh7xpdzftdc6ztq:aeqveubp7ukzywobdfhgrzt45ogurmp2wtzjt2rz5nfnoxrhyqrq + expected: URI:SSK:uuwofnazzi2ehgwkbzwflclpt4:75jxdbmcilhxb75i6jakaqfne34gu3csvfhem2iqkkkzvcqnnwsa format: kind: ssk params: format: sdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEowIBAAKCAQEA3JZ7Z+cKR4dntNRXVn0C7Iahr9sZ15MqHzMbTjHX+Xh74A88 + MIIEpQIBAAKCAQEAvZZfYCB6QoKGf+YkZVpLZrQw23Hhl1UboV4nXgKLzfCljmGu - sAkLXEdu1cVT1mmIm+ViahPuqi9+XGMaPCw66blfiUIH+2tMsgVmR6J2VyOsGtXb + tBTspcr6eWSm2kmZQuYVG6jZkHePpnOhTGAtbPI1EvH855zUVccBhkgtLok9WSoZ - pYVq5BYk3nblqGbCEfFhHb/DDNN8PoFwEow8QAjlFa7CWWsnyJQjqvvfEMWGcOI2 + Nq9Z0TOdgDfvtB1MKeUq02EuiwgAoAU9j0cs/uL9upZn2wlrahwXcrZcmoCstppV - v2OREt5sYuFW9agQ+9LaPKQGK8DXlmXsNiiLmLcqCirdmjSl7tB7eb1Jcj7Ck2O9 + P/Zi1E0kQ1WvwJlQAuK7i8UsqwOSw45jNOkTlN0nnnA5UnLbPuvg0czEEu7EaqYu - J3VS+wlD20lG3AadiwLFm9wM/ZptJltXlCZvYZcnGGXVO9FNIyL+tTg4p3l3e4kd + CAYMl/ZyX9V1U8hKw6lmtYWKpGBoR5kJowBGfYhZM9Yr9czuYxbWi7g4ZcrgDusJ - X4h6BwsfLr7LsDBxIE3voef09OpH8bRINoVUxQIDAQABAoIBADgcs3GfzPabFB9k + uwq8TeVpAne1CYhZZOh4XNrSzehmDnkcqhYHrQIDAQABAoIBAAFj11osksjnDRZl - sH7YuAiwyqpwQqea0Ok01+pRNY5JPsGlPpvNAS3NIf2Q/52YJN77P8iaH2j9QdiA + +RFXKqNbodoSCS3jXVr/BjnduemuIICdPbsrRhrnFJQMRV7nWDzR1AjKYaH6Bm8Z - gSjzW10fAZVpzZwAFHdodjcctZu/AEWnRwNY5/LzSxeoCQ2Ibi+gRkMKB7TYi09f + fO7C68JXOkVjyc9m6nWgimXSJKapMe5z7RBmE7oBb1+vyU2gQ10xRXGcTkuNqPeD - H8IoGB9148hbNycF4g3c2SHihkC+dWSAAPxNxBVdyXPlINxb3O7tMB8T6wb9PWjr + YlKpGm7Z+jNChAtqk8OI5jEcniwxzxRVo2ujm34LLahbQbw412yuohwhkK0m3kJ9 - itA3JlzDtKQLx3TiZrUonJ1y34f9nWff5fYUkJrIAMu8CaCU5t7NTs76NnH5SQ9v + /nIaDuNhKaSqrAS+n8gflIvpFY9yf+k9xWnPjWhW9nLie86Y45Cn46Mw0OU6AHSo - URAxZQcT01OKT7/vz2+9QsDVtuoiLxZdmiqJo727z5w8JTXU4CrynSj0D/6xllD9 + qoSorW0BUZZhRPIDsW1B1YINHXOREtxvqpMEeOat68Q1+UQCYJrgMQYgrpMUbKzd - GNEpGNUCgYEA6XZliEDQMrvYEFJm28wVo5GgxFStzX56bKgF9J3D8TxFEUNzALQ1 + 5JoRZKECgYEA7/nFVS1F0a1U2ITvmF7p66WT5unNUFppm8jk8xPQXcC8SX+Tl3hc - n+t/yjpae9q4tAVjiVfl+3hvgYnyigOMsvEjCBbGoac2FTFCIxrrfMbeX3G1O1Eu + Bch502NgKIchDO0C65/EgQwY7Cou742iweagQhEDdojbBxmQdQXLzkJlyzxCh6YG - 2CkJfwvSx+pVvxMadiHk5BKIpYY3f+2xdjFJtXPIma1dIIZv2M8hCXcCgYEA8eHp + b8hWzfc/TlkhgJToqvMvF9mPNkJjo3vdbqFQfee9vPUsuJcH4Y/H370CgYEAyj8/ - TXSBhNXeEXhCwgXbr7mtLbgco/iAAw+He3YG3GyJqXrOmPAaRXSTgoSyClej8uv1 + J1nNACfxy5eNM6FPriRFFcHXdrR5PvqG1yvAAgyTZyKwlE99pWF5a7zwfFye6GDF - o9VLhOmdX8wwfuPU4OKgxXRcU6vGh4TusSA0V1kPz8Dki9DfvK5aYrbTWOC/pFBX + RlwFOY5U+rP0ePG65Ho6zTlxyG4oC9mF9JuAnKEIIflAWudG7OqHJr462BrEURYZ - 6xDs4C4pukKlKBKfUemtDK/cqo0LMijM2Hc5oqMCgYBCIlTuvRV9WbMCJKWYm/6B + RizLb6DrI37zlORvL1uLPr/b+okVUJuih5PIDrECgYEAhrpZ+ZozSqbfrbfktE0F - QG6fTzGQ5cQ+ZXaSbeKkwqL6GfZI+8O5Epg3rEIXlcT+0gv5SxoOG3bS5kX7jLfd + U5FgWhIFfQllpVrCf14ua5RboYAIos+mCnElRHLUd6x198XRD+xg7HqYO27rbv67 - tOtsji8ked6bMEIA+c49oYQ621YwgHXZq/5RrALAuQQjRYEYd8+EQC/PW+764VWF + 09ThQHZA1Xm8Tl4h5jFc3O4WLGYmi/XAQ13crkITvq73yjLP9boWRHOWncXkHtLZ - Gr87lJn91ptr7ElgzIQaTwKBgQDa7BFw3SXsyHT5ctNZMFwpq/AmFSE2909FdeS1 + 3NSgVi+XLNERTIkumYqZkpkCgYEAk89ff4nw+jE3VR1A1EALtPDLENineRjzF+UP - xZloH4RpNJGQsp/UhTKNSvSpj7D/yLjG0+JKJfceIX0zG5otAHFqxWpbAHnrZlFz + EUjnPlgkjpbayLnD0U+I5wWiGLG0bY7z/rUYGHV+g+9rN80rUvpF6WEXWG2xlN94 - VyaIeD9rVbaFJUObTmLYPYkEREavvVgVlXgPXzi9MFyy7Efup4TMms8qPgYIHA1r + OEpB17cU84dv0j//JP1OozEaXoBJhB2LgS8Ry1anIz0QFnxRCiJ0hPrBcbwoOM5W - Tl2H6QKBgFiAa5p7YO3HPTG1k5lLKDyJ/Fof2WsA7j9VswKrnXOJXLlT4OmY8WjY + HZIS/zECgYEAnyX2uJ5Awnef2hAQH0gXNfnHjcIy03UJOLjOFCH/z2wJEPXZKzYD - Eb8D4FMNdzyAzUcPg7Eku4C15wK3ST/aDY91wtgOKOic7hqcpVuFx/HXia4DqHjs + HXimoiodVWyjAKu7HzAwCxfp6PcC5l1llevJFjIOLyG+MrJ1JOKi+EtV1ETZt0BS - /Prv+5FhteFz9uTo9H8pDFLuQA8ut7UF3YRgF4Z5ENkmu0MKuDE0 + PKkxMCV+ib0PtLyC457p9q3XrtuaAuEH9uJNkv6+e9eg6eS/cvPWaFE= -----END RSA PRIVATE KEY----- @@ -2781,62 +3081,62 @@ vector: segmentSize: 131072 total: 3 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:MDMF:3xmus35ftdvmmunmebcz7dwusi:eplir4flebhq37sxzvwp5mkqsdlvi55kvlgmnorhkobuadyvq43a + expected: URI:MDMF:tznn56mtthkyxbrmiv3q2tjlja:7upsseycyxoyeoals5xnhczxb3iopgugggwievjibkc5duezlb2a format: kind: ssk params: format: mdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEogIBAAKCAQEA67z6lT/5WpiAfDaKORKOUhWJlJb55p+K6v8H7QR6gggmrQTN + MIIEpAIBAAKCAQEAtbFsdwrCBiI+8Hh+s2HB6/6N2wVgjm7Nivejr7FzjmK8+UgN - MSrBq9X6qXH4pZZIx9PmAjIdIoTQlxUsKM2p3g+s5LT5ADfBbc/1bZiqFWjvPZ4y + DVycRdnygdzoEybhzXImonLw3bJh55D3BRK9xlIsXFLDfWDPAyr9NZW9316GhB5s - GggvXoLbJgrjnHSLc1/y7k7gqOzE/pmGzjhznJ6mO6MlJ0xzP7phMYr2EaKiKPDo + EGnvvM4Vr2+XW0X0fymcOKunwaNTQ00ps3fPqyepW/FLUuSrX8wJNuEOR4U4/vEo - 9HrUsxnjh66G1lfxrJIxy0wavsnjoVfbtT2kb0oj7MkEzx7kI0h6g3pyKP19yBFs + 52bmxq48oYeWwd7P0gMR2q0eHydqA1sL93S27NwMR2RaRV4s2gxTZyrrG493gJhJ - S9W1Vm4UL1OZWIf8uPPrvjh5Wsd9sUMAmxxeDi9/W8GCt3HSo/Ge8QrykdHFaqkH + g4xAsXWiieUboDccYrZRO99zOAB2vyYb32gUh0vs6xlHvSFQpX9IEftdL/Vlrbtq - YGkqG1jlPCgxeayzL5EfotG9qejcHEW0zrW76QIDAQABAoIBAAFyVJlTiKyxJeIH + 4y+pSPsZiJwCwJPJQqMLU7OksDCEZzJDFV04MwIDAQABAoIBAFrFjJg+dieFVWdi - 6vuPAmznhpjGVRHrZbdWdE5/NLSVPI4wQDAZN+ddi0m8kiQ1/WFYirSgaRmxS0m6 + 832f0beaoXkyCwatoZ+TT6IXZ9FTT/DER80MnwAgvhCV8hWbX8T5igavoNlJZLNB - DrN7ZkZKE1YInu83kwog+LvA8D5BuWzH3+fVUrEXnXpTc8fIgU3mcf+wtTk5fBCn + T7+nmMrrQ8FOEd9iDZoaEI2ERWtCOLbp0fgzTLPJS7ktaXMOlHMxMRx42aMaex8M - PDK0xE+FkQs++jc5BWCyke5zZgTVVySEX/o1d9byHGLur/F6RShyHKvU0IoZ8x5l + /k/shAIQmwJVntmHZ3zBHTtfHXip5EbdX/L36TAfCBjbd0ArsFP6YP9F6vP+VWok - 7OUkHrPsYCuHOdePE4+YGSklhJ1I6/mW0AsH8BaiczjXMsMQNO4VRO99gub+yUuQ + HES6wtixAX+ff6MGmU69PYlthJYWXrhS2/7xGdY6Ezdrm0GEvn9GviQqnS+6guSA - qBzn6xhsKrEnVmyttIOGS0cxsmaqMCZ+JYntsJo7/iKtQcAZQz0EaHwZFynPJIaL + Rkq79eFG61mq+Oqd0xEe1wO5ul10xngH70NJ5xQS37mio53Ijcmgv4SH7GsIy+86 - 39al/rkCgYEA+EVMnwff53RGe2KwLNpo2lDzld4KrWB44S6/dUwIJjKVi4NnSK7H + 2VspYW0CgYEAxizeJnFH7IjugaXoPOUxz3nXm/Cq+ijmwz8pZu9KGQDm8Jf/WYYP - JcyrEMJnsyWA8AAgXZTOGxMSQUEDY69Ih3+OY4cwU/uTZjCibIS35fLhxlSW5kyT + 49uNeTlfEPf/qVHLkzNVKrHSqaxubwrOONf8/I6wqe+QGHO4k5elll2bI3wFFdTc - GyKP/n/AYZUgFWZwIgZNluoRsv4j97wSNly/221idngY56whnf9feq0CgYEA8xPL + tPT6tJaq5MTdySfuUrbAx4WC59m+2Z/M4N2HxTXBPvO+AAqZiRLsgx0CgYEA6rVh - 3Va7XCqX/vrT/m2HWmBvHIvfRK69uEe0wIv/YSPvoRD3no66+CfwbU/CX95U0O4p + xCOIKLRN1fduW4C+FuKAr5umFnOiCRya5gDwD6WY1jPAaSbfQVc8INAVV1NExQiE - 8dnyM7tuKb3GHl3WHOCTq5cvXL9qOd1h3vLU7QPnDxOqnUGEhbVGzhHTC9LWTQYM + oMAmEOrt3HuYxEoRSl/5AAXWoVN7Qk6b20jtOshEBKkqk19Ew1rXBkfzWfVq9+4c - h4TSxrIDYVhlWJlq9L3S9nB641HemDrXI70Wya0CgYB/en3cTpvOaarjIgpaDY+3 + 4hrK5hRpC/MIi5tR8AlTM2eDvu3/WJR6uqCL948CgYAQaqUYgCfaI1nGqay8Zqwi - QcfBVTDgU1/eKDXQ0ciBbInTCBbZgDzrkMrpoRjEKOaq1TXJN2YZCtLdxLcr0U4J + qNBAncY8JOlA4VmXqlj0C0wWQDEqBF4KnSRyF1uVt0WZjCoWDpmOiN6PqbYYQsfk - nRqMylarWMsXtrM/y2nt3afGQZr2B62lSjrrr8clk//UXTQIlHn0mp2Z7dqkEuK7 + k5fkgBmIak0AiY2PxG82LpjsbpipP1HtN6IRFa4gd2J8CG/IsFT18kxu0m3p7z/0 - HSa6UdE0CXioRH9CdGUfRQKBgCRyzHflAHUigeYe8FjPTaN0oFSUeKcQ2KvgPK8+ + nMVjg9l3Uo+5xycC4OtwzQKBgQCktyBijvEaZ9cMJzZanxJIazMWiqxXq1T3Ag0v - js2fGNh69dZVqp15R6jsc8XyTZ+ChtGYD6RIL42cwi9dfLSZzCrHobdzkFca5gkL + B09yG6wT/4O0B+S8LWV0PbQMcdKcWGsDiXXtf4HorxC6CKTzxkCwJGjJFRY2pYY5 - OnhLxILTPRsVbuypsPNHYvD77VxhUtGjTgOzP6SCH7g4UPxf1llTpmmdphYHhKj8 + sYdTLoKVpsbLYBuY4eJvdQUyh8pHLuM0RstIBuDl0uyXVSx+wXyTYb0SvGHsH1+2 - OoWFAoGAAbvvCI2VggZqX/FoEpotLrvcSOiKn6havN7Ey7v19Hya1zXSw+ED+GEi + I7+2vwKBgQDBAbeBgU7reZHXa+LOK1McFVazYXCyi/CSjDyQOtaIRJpfoU2dHQfy - 612CtkTSB2CoWdpXsdD8urKA+CB94My/WRvNK+X2mPtLSRJkr0ZrFBRKxEnbJZhI + 0SpcXb6mWC7gAf92sT4kDXJyts8jNWkvgKhifhtmd7TGSqsdNdhdc8yB+Kl8SmjP - m7+WuWG3wkeHfe0e68sWkV0DjKAJDDVuCHJNQd3DRqdybG2ZRQs= + AOLOXs2hS6NMCDlmQDBlEyh5/NHhUVyN9FCEt+6l0qjE9gS3FyQAjg== -----END RSA PRIVATE KEY----- @@ -2862,62 +3162,62 @@ vector: segmentSize: 131072 total: 3 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:SSK:ff2y7e2fpz2ohrx5xyqzgtbaoi:nmq3dhqyhvdjgdufzhu4lu3k75dtarphkg3d6pm23qpaaz67t2gq + expected: URI:SSK:i5hhjthb4gsivkzvch3megrely:hkqc6eypopec6otxa63muh5olpihrsnj3ssnq7whrwvc4fssoojq format: kind: ssk params: format: sdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEpAIBAAKCAQEAxBHmofSaG2NHf7287D61VUYfWhJyps4VJohY2PG7BK1z7wly + MIIEowIBAAKCAQEAryhP4uLqtk4R6BJnXYLo+rwmWCcdec5O7Cg1ath3Gvt9RmMb - 4gRqVBfOOMi1VvHXySLKaY3RJXe4U0abAin/cRgSO/HxoYcsH5kYq1/iGGtFuBZ5 + vVVKwTdC0ZDyqx0+Dsr74oUNXA1oZDr38/3Xk1OLJmCcb78V3patv3yJxOiz+jvb - 8XbHeMxxASJ+UruHydwqAP/rZHj/goQgN0hZI3P5oxLF+xDkys2wBZu2W5qocRAo + wjKtvyg14rYIBoEu3/bgUPasphmEbjSmUEbBjfihNr8e0bi3OQjNNlVQgLUyI3FO - F8YhcQar9xOUyrwxtfsoR52a72vY/EWFcg8DtYm7sX7ZG8BH+QT3IpHIfUvaOTaZ + 3PX8awmTkyRSubAFfRxOjecxfqj214DQB9j8F6LrsgGSna6IaNTRBI/UwtOiNOPm - 6tGDa1p2EBHheHUUb78McOrhlTfZOqPpSFFwksuOZL1Mv69mlPwQjvQ6AMA3CVsu + sczhdoctVRZOVzKlwpjickL15gwfdldPsuSLm1nkqXfoW9n0zV9V/QwyXJCdhasT - SBIRIImWdHWeeNlsyEzIiLCQ9TUw+EtSjKM/5wIDAQABAoIBADxDQD7A/miyj/Q8 + mIoDN6ZPHunL2XDRN2IFjholYyW4cWj4VYVnfwIDAQABAoIBAA73K0gv6iz0Y6xH - LgfykitefR5jEygfqTKJr70mNxQN99cdcVj0gHXOR0z+q3XIqUkhz1K4CvNYI6g8 + 8kP3nO9bZw1OHkMbgPvFfbbo0thf13bNnf+hy3bRwWhFca50G6rI5heXFaqhTKOP - yEHXBLMO8fPIvjqmYDJqDMIHm2dj+S7Ggb5sgoynUYhGwMrO5sJtT9+0yPW9ltLX + tELJFAO29iMrywHzOiugBS1gtya3WTVOqvqfOOAlz+DUe8AOhpJFNipEwUCZ2opN - p0s2imcyKyUrDPzIyXln1NU0cc0fZxqcCRKy7L6H/uOtgBRZ4PNyDUNtSxmbjmfE + /k3KldwK+79BOiFiHmmFmn8DcBLnA/iI08hKJRRUOtn51kVe+MPAZkETXQthMq+X - iqm58C0lkeMOPa1Mj6cqH8PD0aU8y+N7Vdy2PrxOXHv/+6Y9eRx1S8GvL4wFmtgL + NgJ0a6kfRDV+7+aQZfhdV5raCB7nK8LZsm0NzGLvtasX944XxFH15kKXaLkKsvvQ - WHUwwfxqNXmL4JdsQn8TSw78p3I47CNSk4QXXdvDDBrLc7ejVZU73JgYh5vsQt5P + zV7QNeiQMfshxlLfCbcgA9NVsByYdpJOfchx2WD+L/ZEpIM72wVfCIzpqkXJTPGi - ZfXEMfkCgYEA6KUtfM/DpaehB13qx+23ArYEyHQ8Igqma2pNoSj1szW88BxLVzBb + +Rqil70CgYEA7fgz0pOe6mGElk2TSQBjcYjGLrCGttyKtl9OCmsFgOlYadufN1eq - CEJ+tb9wcfQ5IJO74mqE9huvNUmeU2vUVDwusUEe+rnOUeqkL/sSkX51GAqVMkWD + Ahn8wQtuEkQRbf8t+0bMCZ8/XU5jlIUjO0bIDgZ4LDPOckV45e19fnqkCeEnlPnp - 3Qq+N5f7PqhWPjE0+alm7Qd+SMJoj5FBWs7wntX/A+17Q84ZtF1NT6sCgYEA18DD + NAaM0SewY98gsvjHuVl1r9m96pVzlGiyC4XFG3T0XLSzg7EpIIKey6UCgYEAvG3F - XtoZ5p0lrC1UhdI7EQPQL+S7/h/sgiek+M6wiqe1uk2b6pkGTLFHRRZFTLXnxEfI + BqXU6o+BgamohqPHDR7qY41Lt6f7XZ/o0WCkHJZq/2/JFycwmHgRU3esj91zbHcc - 6WWIQPkhmmU4eX7uqn6+9TV6a610PnseW/JFWc8vVRHwXAOvSzkO77kldVEDNtLa + 14TeoCsIPIZcSdqA3rwtWolLrSuUvvmDfmKCkEgAJRsHi0oayYBcHE4lW8UHWcVT - aQyXXcHsnNZXWWrKKrLlGx8GTJORBLUjrCzxxLUCgYEAyO9BZnecJ8usjUxUp/Ft + FYjc4JkcK7viZXBRIMEhwLPk+DsSjX5CO3ZKDVMCgYBpt/l11IUmBRq9F0uWg+ip - C+5iGzApb817B3N9MSDLdcmIMmp9uASP24ZzIk8Cs6mYXca7lEckJ9ypa4D2Ol77 + 2KSKu3utoz6wlJh8Al2YjpHrvVj3Yiex9U+Xh3dn//tqTZJk7mfY4nlo/1k38wna - uPVx7q6sLymkRaQ/wyE7XGa4g9dAHXdk+Nl6iVG/MtL6CiU9+BSUTU0XiYg//yAa + 3LAlovQiVwWhOIHkS+STmvJjPTazdW8H4N0QUjyHsem5+NHp4vdonyhDHhAR340x - LnBl6woxhBbtTBcKpHmheJkCgYEAjr9yRDKnimaVA1smnjffbr2II/gBzfyPPfo+ + l0Ug1I123gRePgdSXRUkzQKBgH2DCMiC0a5kZLl/zzfQBBjjTPF+/r6Y8EDO8X/2 - 84PFWKfn2/D3ZPuEKH/uuK4ogb2lL7+TFaFgyiRLcFziRbiO7m1XqOOOMOodjC1n + RZqdPyxiw6neeuo0oCXfA1zY/7dyKA4O/VPnFhdq0DKJj2nOIs+5wGTbMLt87G/V - g8xCyE4FchKhZi/l7i49TKzCNOG5768IZRK4n4bsJ0TFnFrEkgW1AgG/6DCGdYfn + Im8E5sPQm1fWxr0N+U0JaK0WMu1DGTKw9Z/NnQwsnINBK2kL/HWl3pDSmGsTfP6q - p0ZBXDUCgYAVJ0l923qS5EzG8+AP7a36yooMTdrL0NxvIskIJVoO9jBqhQGOxk/z + rmztAoGBAI39m3K8RjybR/46YG4+4eOOFeo81amjSR1Pf8o/SsqpRpqqCzcmGWBO - cbWB5FO2e7k4Sn4NjNs7KiXCkZ1OJCVvBSL/YXFKGtPcOq9CW0ZJIa1pdlgIYqTS + hRbA04KNjgludOcMIV0LYAWqy9LKfaHKcbhdh9Q0PL+VO/fJsh/NljkusRxHxTXj - YmWzzF22o5WkCTt4I6tCvmq3b1poI37GdHmS+wG41rPKbc7mY+RahA== + /bIeG6yEqFtg/MYe+zeYUZvgGsZP7+7Swbp1ScexuyZes9H37v2D -----END RSA PRIVATE KEY----- @@ -2931,62 +3231,62 @@ vector: segmentSize: 131072 total: 3 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:MDMF:kgahmjtyuv57zk7tcfzq2n2sn4:26tksbkcuhgw72qmklz2ihmycmpybl5u24snnay5kln5wumrllvq + expected: URI:MDMF:n3gsik4lgjkekhb2hnvkhiieya:35q6b6riggvqzsmyso2hldyb7hklnrmu6farokcj6d4nvmufqzja format: kind: ssk params: format: mdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEpAIBAAKCAQEA2elUOcI4tjDbw4ZqGXI8mBbGAhn84A/Qm2KH9LbV+I13CmFT + MIIEowIBAAKCAQEAhaXFN/J9blEqNROOLhzzOY/yy5wAQgNmBtULXPLgYffFop+K - Ck9RjjyHYQa1F9XFJ1CKKE/glUOv+uXQxOq6+0bjgOAaooKS1zxeLmy4Vc6EcHiY + bfZcedR3VTGArQAz2eOm4KfP7BNSd1udZA4OEIvub98O90osuj3B9cFGE9+P4jv4 - m/puBZ37mlGOUROZ5PktMWDQ6WB5kycHwVrzIXtHB/JuHCA8S9jWAyhxqlsYpm/O + 1GfynZbUnBNq3s71P1TqXSTEJ8DD1L5HFY9coRXSmh1+TiEwhaGDKsBWcMMRjizm - C0RfBjrUyZXR5KxBS6IEjlC3KYHeqh8Jlre+weXOpFhl/z0SKzyEOHA4H96xFmKj + AjeH5hzQLhY17NLjP1215Mmw4T+gXUYWKfSR40hfXOEx42uz0W2nlMHR8vcDoz8l - qVvVOF25uh1ocS7A6+M1b19UH5TRQ0dj9hAFz8B04LaHCBxToDgSIDqKGmFd5vsl + UbV2xCBmMa8m+vl+UwK2rLhjWvkZZP2o5hpk14kHMYoLSF0U2aQPgP+7rLCYYlhN - 50ohULKmi1Dl3Tf9EZ8YCjL1KzD0VRBTun6lYwIDAQABAoIBAAHpvv3DrYg1p3TV + w+ClF4B9Jixb+Zl0FhLZeJ/bzXZHdY6AKPYhDwIDAQABAoIBAAOZgpWDrqov0yBe - e+dm247l+W15uVxAyZON72Sf0EwVZLbvQfGWukARWmYNc87Wi/cNwFDBkjIa/Re9 + gvgDUppsd0sm9qt7K559KChcPaaV2wO1eAUx21gdXLYgNl7dKwha2oNMMcxM0Zii - J3ux8UmVBqr/6BmrSDjiO29weZ8+p/jbYHecYLouU2I4X8pESSTN+KU9+WwfqTPc + ZeZXKed7MAm2VdEGSpcc6Qz6pC25L0cWJhkT0FPl8fBe5i7THzA8qW/8eLC4v0Lg - 4Mkx2W2Vm2itYWVgPsmOP9orY17uRYRjHwwC8Kj3cWLmZcQI49o/SYp4rjgNHFpZ + ecn/Ca0oXnrv1sI5oEobyWHM5WcgZ/EFWaixuz2g2O12pb23XSXeoREg3Hq6UoqH - u/4ggH+ewO+5UKUJuQPCxFL45UAzl+nPeVhjpb3PxVzd+CXxTWVwWyJ6PqCHe4h+ + 6w6CfeRI3rXyN/E6LOywHv+gDll6rBKilBBOrkwSwoTca3QbZEf7iW7FC42mI+Qt - kz6mGHMglt/yQdZDYWnnlVHMtfTFbvf4YXqser4oChkhmLASr7tr1aL6aMidpuV3 + g2T+T6MYgK81hDcu6zpMY67D6ZS5qZfIaDduZKeCabt53BTzJDxSKqeMVjCb4iqA - Abq8bNkCgYEA7PYc2+3ehahZSf4cidzIsL2MIU5EvO59Nu0o7mvFzeZY7bTn+04/ + RDzGEmECgYEAu+5FThTnzfP8K4Nwuc7lbUQuU/1Abpw+bc7vQ9QWE43sl++YIpp/ - csnrqXucj0p0vwRj92oEaZQqrGDQlk0DEmOufk9TVRwAj2clWF3d5M6bR5HTxb2/ + CJLBiGuhW7RQ1fjDk16M8phBIyeViLkHCkbxsUkmurlQIHq2KOGAvcr6Jxnhb5mM - DmmdPfnw/Y+PCJ0ZDG8l7405Rh4PwzctZZUv/ZetSxdtDn4TJx3F/HsCgYEA62tk + crsk/QBW3GS96x+z0q7ZKNM9jVSqVhtmZ0nMzZBPv1VhNKxTEMRuAS8CgYEAtg4h - U+k36X1cyu+seJ+DQNl4+zm4Fb7cLSySKNT6f3BNey5RdN0A54gPMBd3G531/MKH + 8QQ96+FW+PrcUwuQ88KC2xAVBt4J7sa8sQ/QLJHSXWnQoSSy1CmpJVAVUpAB1Dw9 - EsWW/SfE8m6a1nBvu+6Op/GzFqopKFl+nR4hbnvmWBH3KOu/12iREUEAh0ZJR6pX + J5ypVcYNuu/DFkoK9Ill06wP+vSc1RhjW/ozaZgzWIIUpC+54G0DMj3Pz4BUL8tO - 757eoUm4gjK4vwBzv4O+8clskU7oUGILW5KT6jkCgYEAqUhtd4Somq2ZFC4wbyDG + 12RqpgZfbI8rBXwURuldtsTCC46TR/SzxkJVJiECgYBnf6DhkLfdABsH59qkKiLG - UtUm3chPfPWXiHzG6AUgK6cq0q6Rp8vPsg6kh9CiGQ/k9W2KiP85Jb/O+JS1jxp3 + W39cOCRNBnWHSikRZPNHj6kWQBi8LfP6R8CYHhZ+h77hKKClP7RGQr3U248J/kS3 - XlTOHLhI3R2DHO9gE5ADbGlZLzjzpGmYqxAyYEtFqa88TLgGZAangEpQp1Hkit7J + Tzz6kzvmJ/rN+Gbr+s1JOUktUZ6LNLhZ02FaiN5NgJnrrMj/JdZpGnVSqacpxutN - VK/OuAj6qRGUPG0++4venC8CgYEA1QwqDloXtGE0EZ9W+Q56LLziZJB2jI9eGC+m + xSIqr+iLijz/okwY9uVSdwKBgEzR7dSLm61a6p4pDKsmKEYTf6/8O0MokjxlM93q - 0fbz/1J1fA2Nv/GlOOMDw6TosICCNc0hihZwrwdHj5IS5A96vpuEVG5CgTda6d4b + 9Ea6SXANZHF60NLhuXP7NOQfzAXIXW3Hl1SQO97zqPhQygqhp4wIAL4+Vac9oT+A - 3DqBTMgpy/fuMgUvZtSFvBSUUteDx6xbykl+9n2N0Z3vXUMefOnQamW7r8C2MtCX + dg1Koe/pA9i8IszmcwDSQEwotF1uhpgw0Se5bK6cQuUPlGbPtjGXGOJTiSZFxU4V - sLZ0z9kCgYBuvHidGdCncYfx7hPjMaPH/6J2YjS7yhiJ1QK+1HW4mUrqa7sHgUUK + U2TBAoGBAIYIgp8UZtxpIfj+u0EJqtQGJFGgw5UgqOQLozQ+OmWxE0Z4jaCgxhj5 - F4MTGf3o+q1HUKrv/CrrUm3a5CoEd+uLJD/3fjCkdCnnkAf7P3XFBJUv7sJIDFij + TW2b2rNesGN3FlaFrXPW7QXw6caZ3/C5/KWcmpwS9gESCJWP3iEbJ4q5Avh6mxAo - JIrFWM8k47vH2jPPyzr6FMJSUUeVuO4Zz0tEOc0VufR21dnkpnZAHw== + 6KnU5ietcfrDZtjb0TqLo786IHndR3aljbqjZzWvZS7yncfIkpVg -----END RSA PRIVATE KEY----- @@ -3012,62 +3312,62 @@ vector: segmentSize: 131072 total: 3 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:SSK:ym5fhzrwsby3hvapqocblodfmi:j3seystjreodtepfokv4qnp2fwjb6lwg7ti2cdi3f777uym5hycq + expected: URI:SSK:qucnax3vhjild5ifkxf6uxl66i:26feb7whvq6cm4lxyajiwk4hv3cry5ongo7d4h4vjpgskvgmy3ma format: kind: ssk params: format: sdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEowIBAAKCAQEArLuTmZEerhhv9SgwmhcPcMoOFxFldMayyd5daEWGH9o57hW/ + MIIEpAIBAAKCAQEAwa1PAiv93FgzQ/4f7aI2pEWAyJkqtC4cP0aE0ifKCZxZ/M8F - F4Al5eTLwnur6RaTYKKuPu+etjSBAdO1HBMKEmhuDXeYOgWEWAe9LluQU7ffmRRP + xobPnNw1A3aWrAdOC/4iew0y56SRdejGPrrno7d8eDwwc4qV0LqWs/dGs0yb3/KN - Xgsx4HaSc7nZZVPhWMERAeZMOUwyB55kMby2Og0CcxCiOOhUyC2KgeN1jPRN0El1 + MW5tXzi/VVoo06eQX17YdyQllKqf2ebpkopDYZ23sOXOY1krUFWclGGSlmrpP8J0 - Hsuq9LUgg8Fe/m5tqZ9TDO3UiJiu0IhLPUrO9Os0cLLntbtOo8S0vt0AzsyZsJcO + arLxeDZiWnzE0jZuZqPliQmCozUzPSBbi3cSaiw+hjYsEdt1ooGQcfBfKOwCx+HW - 4CKzf1pNnfF0ca/F5P2O7YHf6VR1P5HqUA+4tdizOu1sYqHIRA2IeO6x6P6DbFRS + jjyjck5rss2jVogEDgfo+/HgUwkoZ/h15YvMSt0PBh0Suh7R79P+WXl7m0SvM5qr - zRS4b4UOHQ5G7rriqzUztWypGuJmbI/Q6oLovQIDAQABAoIBAAFUtA57TE/EYojY + w/xSJKpi+RIcsLOkU89BW+2DiGM3ZeOghnlm+wIDAQABAoIBAFEiBt0EDL6HfEJZ - 4MaWXGXx7JnEmZh35A52a+S4hzZgg+pzDCUGDzF+NoxLgyRsqkFUzfMwSfPQVwxS + bIqhz34VV5OxBkCgqFihc/aNkIdiJhhPqT23L7WoUdT3krrR/JHtjgg6ST7co8rf - TjaTgy2aoXrX+81T19q/+1Ce3690pUcK6l47x4xAJaDS785rAi66oa/daPA9FDQz + Dl0s8uiUbuH3ZNyiC4x6/bK6PbXSu+GevCMe/VZMcWqR8FRp94LcOpX+YHfc2kXw - 7HfOPJIy1PZZf7Wto2LKdQl+Q0+mpydV2UnldRBTUP0h6maT6Ln8VxSVnwnaTyn7 + A5zNqthzt4W1XzYjHo/yrTtDfKLhvwEp7NciJL2cQ9DZeQAOTbASjVd80Gps1ZLT - PdeZQNDC6I26lD3upg/aun4NS+AxigOgXkk1qi4hJHTCeQED3/qQsnlEMf7n+RzM + XGgUpbwm0AzfpjK/6I4qtw2XlhlSBesj3k5PPHqNB6H5ZtRPUGFKeKk4vRl3TRdt - bnzHrxBd1v3WOK9uf3hpKep+GJH/VOq2YizFOE3nBsr2zxTlIlUIjMnz4xoBPzqb + XdKWbGtawWe9He4fpGc2BMbNE9EScs9q4qZ6Xd+90bgGN3pa8O7xSqdnHtlujOEr - A/qlxXECgYEA84GSzuPL34cbXy6cWbndxyipPlAqNfMsnawavxzgRqAjM1201O0A + WwAevaECgYEAyWycuheE2nU08iCxicgiEzJd9L6HLhBvyeYoZuhk/NsSElav+7Mz - TQv+Ohw2YAacnewx3oq+CB4ilPIUydz6saMNJBIE/nRyei2PksjvTa7xEtC8PHMl + pzWohc8sv50lKo9C159onkK6u7dNqTSZezoDUn2GqrazsYgRP7LPB0jcF+vQUK/D - RHN0kVPDP0snjjIsq/H5u9ReItaJNdMuglFtUGTPt+KKWjGI08i5FM0CgYEAtZhm + E2gC2ZrTutdmaCr/Ddl78x6v5WLlKrVEvnJBP7pd/XmjuVvNQyDrQFkCgYEA9idS - 45pdRtyDQ2yqkWAW0htUF6pVpMgyip6iAgethd1tBNUS0t+Zn97o2p3kFd/UnyJF + bitt1aqndeKAtYhquVL3xJKwqASKEbWfxRYXFX7fRqbrmCGT6fg4rs2rn9YAB9wH - jAHEMJeHeg7i2DchHq5YBkp00ETprWpNiMZwEV4AkONAqhDNtPZTMPaSda/pm4uT + ZnQ2e0tZtG4liewD3Omf5nvekkvSHJFKscm9NS+XLcv3s4UJuCnDpDlmJyFRXSct - asBW5jyimXQYs6tE+ssmmyUTWHIKAZupIQuBo7ECgYEA69wKzjiZRcbA/X3RVZuR + UuVAYcOImpZ+Qtvy+sA1IJBeNl0Zgo4fbUWvl3MCgYA8PuI5vuMbvEbTzPeNMHEQ - tJGu9LuDV0RWZ9bHBWw71EzSK7PNLxzs2LQQKEshY/ujgdfBKhRrIsPFrU2aUzim + sNXtaDdijcQB7XdUIFpkTtn+5jLI4/alIqV/MFJAFa6SJjtl5uYRv/++Obteyr7F - 3p7XYKPPkIRMSgmNcpkMKcuUmCv01/yUEWxfcVCX4tux0arJ2DaGNafrEoWI28jU + Xrqzp5vp36+rf/k4xjCqCx7ZgMzT9V4xpcCEeYyuq9KTgZi7+brbIuiVgZjtxz4C - 2Md0QZWUGUHlzp0CMljO5NUCgYBTtVboYAXTXl7bu8G8lbCvVY2kAw7LkMVLhOhl + gIYHm6SVNhbEUDL6yxPSMQKBgQDqkfxOelpXlCGzCB7pX098vaDZFYT9CB5e0/qm - Syi/5lwUuCufLRdhzJ1F+TZkpvMaD/BDI6VOSOtYZnhG9tK7k95buAK05q9ZEwF+ + APAMjvPMy0KVneHrw5yYj+wuC+vJkZcHvlUw11Rryc9CCMSBn6y+ImqudUyL5rUM - pQqP1ucn4rmyK2DHpCyhC2hj+50R6Hsh4FuuchD577xbRf3cJb08ExEh2h+mshx6 + iZgh9/EUNlwdGflyI3KJrB05ytlTcQMTDN52i7RAxIsbwahh5gp6trjhC4VE0ZUH - cRVnYQKBgH4AkaL2yHqbeoK8G7q2uaztH0qSL3wavov6fu0AiQR/izujdZU8RXgw + N7ImGwKBgQCM1L+omSvZLd3HyvCEot+EV+bzUsoExDK6+k6a7JgPN3nH9ReHDH9I - 034LgVHK1oi7pD8ANLWU1u1QmqluELg4npGerVJr9XqELLeDlkgElOy0eKzSm9j0 + mTRuonZ9zNdeulse7E/7UXlHH0YtgC+XeMQAjiiyV29HuycYt9NCdF1xTrV1Ao3f - oslhbAEFxzYGJqE4py8GPkI2Axxf1BjEdJjHIerOPk9VcmikLnmV + 2Hc2JrmHgccuZP9y/xYTgXU4qD0sOrOpDD230dkkt2xFujfapl2PoA== -----END RSA PRIVATE KEY----- @@ -3081,62 +3381,62 @@ vector: segmentSize: 131072 total: 3 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:MDMF:i2mcgpkadj7etqz4mbsa6kucsq:tg2gfrsc5w3fb6hr3e3aprf3mt26azhywaafu33nqfxutq374m3a + expected: URI:MDMF:vspwhm74zz2ffbihu37grhn72a:len3xv7moahbuptdlfzy6otoguaybmsfroeaonep7koan6mcxzfa format: kind: ssk params: format: mdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEpAIBAAKCAQEAqrBi/bpPWjUv1YLoTEr/CKKvYMzIu71EUVrrNwJ/FfNJlJmj + MIIEogIBAAKCAQEAtyEYblHULHgUI3qeiIBkJGYKj8bmbIKBz9VdtYWbyp0Elvx0 - T1XB0KYTbzUGUqWW7lFMAm0UfOYaz/IQjLbFpWDJh15g7HavTvsiT0p5yeCuM5Hm + cE9QcLFK25fSCACviocH7Z4yNXPd1yamDPVhFUjcI0J1sj4UfgR27iMYS6rJvHMx - SRC2VY0Bmr5euc7dG8UWIsjqa0bg3LVc42dgAtX32S9tIEOF5Ftgk45CXMyGevgt + Z/KwSKz7TW3eDPVJbHl4Y2MstlQBhkTbzzve4XCK33E9d9hMj9OTfgG75XP9wdOl - Xjno4mSYHJJKjO2vYnttbMl96KSE2N8KSTvASHfSQE9WP50m5hMsAhYj9u5s/E8U + Y0tBLoQIPRi9DzVqPMg05oV75B5o/GOWCSzLCTdGXpdDsbFsQPO6o/nNDUvXwVM3 - zO/fKkddFEW6qPbk8CSndeZ6nFCert0FyxTIJxiv5qsHQxui3GEsSY1U/HJo9HkO + 8yxEIZW1oXw3+dmbmKnZi2/RjHvRixzW9JjwW6kg04P22ehjYVRyqAATMoaTDvNq - CbtzsNfR0mMTzF6i/hPkNDfemBSNs8p873auZwIDAQABAoIBABWBov9oNa5ejDfh + S7q1JPe6tTpBGqnhoFjQH3hHsHylhAyXMOf6pwIDAQABAoIBABjHnaqv3+n0haUX - Riayvl6WrPVL6DDrgIulooRsXpnj7Q35q7+HxSNmgYVeD31jWtiNSr/1gYLZNWCl + XoRR+zsBo8Q4wc1FC5O352o8ngwYmxpjJs5brSLSmrKEJKN4lEhGZUg988VP3GDU - Fdu8/btALjRNunWg4KbZcrG95wl+M0TRKcxj/C1cVmrqeKH9xBNHKmpYmVzJ8fQt + lfuC6JQcu8z5nTt1MwiqSf2HOi5i+dFKNRE/waLT5V1g1H8kYb8P4L5yGQbC70Hf - L9aBRHInBpMJbD0H9PtYXhtJbegmMiFUIKKr4oyj4WiIeYn35k8rQTRI7E/kVGVG + 525vR4Vx2RjLFeo1loaPtpGCYo51nOJ9UKkaQZ4sKVgNPL1K/WVDnzmaVklz5R2L - vx1db0Af5FTZ0zA7YuhpE9XA92g77vnWBb+OfbW6FVJ1BtTGYSEM0G/E9CbBSOi3 + i2dV/4irCO8HP1ZUagV9JHVpGUT1HJDzBGaVF/3zmFa6hrOrW9KOVuhc/KdybLU9 - LD1zIpJL7Oqqt0vm8wSzfzxAZ0R8+xAJU92RCS5IwHRsgApB72w4DwuDy2bLW+HF + Zhk66KM6fKVgtGL8TG49O99nhcT+jcp+xyzKq1fN6V+dmMFS4PtVXCdLMRfaV/5S - W5qQ5LECgYEA4zgWVUqP7ChgLSeVs3CjQOodfKYIrn7a07W6qPGe4fPhTIz2dOa8 + NuoLhgECgYEA4zPbX1M3hXsH219AvASkDnaQKio3AY18T58XMt2+laPPEIhFDaiY - +nCT6/xrcY+jVHjpG2p+uvypJuTp1RZ9YgreUnmf+ORgjFfaJyn3vJVMVHotdyJP + J53or7IOfBMphwX2+V0LqL32b+x4cbMTnXN4tUomLVZnvJKA4+1Z3qtTf6xv5yif - MM5hKknaB7rBykKCQhwYvehOlIzGD5NIKslkXZAX1sdU1X+ZjQjCArECgYEAwE86 + kZ1w2GFcG9rvqCdUnvandkQo63FLWsRKXG4zYXibXof5izSbtMaM8ucCgYEAzlcs - gt+sPleCnES+RhaWmFoXZi4gM2KjMaNE+3s3mLKk3LNTq7I8qgYbXoCgmcgWPwJx + QqTzcCujrjA3Pr7Tj7qkdJZjCPOce3k5MRZfzqpE8f72OkMwHw3DR2DXm9vP0yev - 8DiQ8PS4VFVTZKvnxfiLV9XGMP9+HkaT2jkm11k8KN85p2wsMrJ7NlKce89E1ubH + OrrN2u0LhK/3fkZAhcTTkUwlH4WqMLGmIUGlUqMyLvSUEW777qzcupbpGYG6K+KF - kWJQq7qOZM9Hl13Xb8iwuhJD3x9UyatnAjnOmJcCgYAZ5E5HMdPsqT0saBJa/D7e + 63Ie54KpmAaly7HWRhWsM74XcY64dHlopnpzgkECgYBhWq0bcZsO1SMOuwgQCKUL - Ks9pYNIkcDgnX9IBZmcggFXwDzAWaiSmtSVmAsGLkz6dZZnKkfwW+qubzwIGUiW/ + lX48sw4S2j90FqVoJGAv2ps1aE6+hYl9IEq+TjuqqsNWmhWz0EzYp11bpCYQAj3b - glWLOGjOR9fopiopxFKCntCv36xGoxY7DYls9DVwJAvpLGMDfYgkO9CYhOIc7D+R + b8k/VWB6eNXGlbgo8mFZ6mvC/26LzHpjeOULstw3C1853HCEFQi4wogOKuOxJv7+ - AJn7P2w4AUbdfUjWFWVmQQKBgQCEMLHerlOe0taUBmjokrRX6220LjayO7ZD86AC + EDJwB6/7l6Q0I/y8P3/R7QKBgArb6ZdkSO95THbpUK77qfShdPAxzep1r6GL4qej - YdN4oivTDW2RU0aB9QqxLie3LaOlElAxuSBgkUd3qONXCxeZrNxTtz2yBp2xv//3 + rs0YhuJZcanlSU4JEmLaRN6N9eT97KnhlN3VpcqI3DSIC/M+RYgbAsUi6q7/Wmfb - /Fsnok5JJhBidmf3PVqWn7izHmmKcz5xQCyFrwocX6MteDMTwtdAQDfpUocczTZU + pZCwx/5SnsxAFAAelss3D/NosVyH4lZeviOe67/1cZpDtKwhjdt3QJKYWTq//PLt - gFnz5wKBgQC4GpegOkRD22AOC2xc34DaMx3djUjmGWj39+aZNf4zWE6odu4oVYrJ + 9NBBAoGAQ/Y0z+5u7MhB3tW7o/CxZWdlFWDJ/qB/fUMMcULP3bT9Q1JaJs/Bq/SJ - QIQT8R9TnujWTtTUiNjsH23JpOSca3TRIqrMBKu9kyQX+t7YY6sKxJdYxOxbfOIn + MhgeCsc3dR1jmZvgvxsosuvNHUvBhpksoGYtJvOERBN5DpQk36bDg1MP0pye/WYX - AxKOgRDY0Pq8Cvarm2239hPgcjC5l9eynj6jRfU1Z/D1yU6id6ouYw== + +hVtQVrSOU5JvQ5fWu2zQLDTHLEXPaR4F1Kca8ljgFfW/Ay/T4E= -----END RSA PRIVATE KEY----- @@ -3162,62 +3462,62 @@ vector: segmentSize: 131072 total: 3 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:SSK:7utfrlscj2ouzmjl6ktz27z3si:vvxvzep77g4qvrqnaa6g7vcsvc54n25cc6fjuahrymuyi6lzv4pa + expected: URI:SSK:isonfapi4rlbrp46a43pqvzxjq:xitths6mep4ltlbktkwgzegtoaz6efeko3d6pvjgccqtirhzzgiq format: kind: ssk params: format: sdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEowIBAAKCAQEAuBpUR6IyfX4Vi7ggWVjSMQMNw7PdlEgxamEeCuoQBNDEJB50 + MIIEpAIBAAKCAQEAttp4OhdOj4JqoNbsQ7aoF5SUk0aIkziEJK0M7DRCqtgU2c5e - IlxZId1c12tu8fESJ6ix82aNSvXLaPWwIJI1UODmj8TRlvlafxrqeSlixi+1onwL + e60BpRd144CXkfc3uzAgEIOTt1F4MtXAWkjki5Ie90pF8L1/+SjOdPii8IA5BHis - Wdh99r6fpM7mf0gKfz2BeDh8ZM/9o9MNhwQkxLv9aP+tBk9xXZrfBQ+i6ZMCskr/ + C+ydELjbQY85z2GVPvCwB4VR0aT2GsCbtDaeshZsUn+AH3BPsUUP+fXJFsIGtkpr - ZoL37k6H7psMLVIaCOuik2mS0k15FogSSsB444jSr7A/gospWdj6hpiXLLu+93+n + WpC5pd/AQlrlYWj3tw4a6j/Fh2Jofez1ShDoHH1GQ2IP04ekWLza0lrzxNqQuP1n - Y6Qv4VzTDTN6HIb9ApYbUy4VaUEc6hXCP0yoGWCegaqPclNHuktTphuhrmfKJ3T9 + hPrGhr1+kS4sOLM9LYmYRrmIcIUSF0P75EdFbmrSKddENTk9PcUSauOYxpvdk2vt - OK6bvcp01UaJdh7ZAs7JaosNY+v3XnhwVADiFwIDAQABAoIBAACuL6jXQmheeEhY + HFuWBp5i27YUxqcEDKmolqoKhbh5HDmdjslOWwIDAQABAoIBAAGOFLz/EGlNWqAe - IIJ64g7YvkHHsVQGnJC23wfElPncJs6R8D5e1pN+iU4XXw6HJFjSLSWeyg1SzM5x + M8l/oX5R19GeJUXbPS8dVEwjRaMzoznBn4a9ueiFgo3PZ3qTok2yjb3rizNhO1HI - JN9tX3qkr9xQ4WdsOcnlwe0W2pFjFsDWDYdCHzfqdAOX/ZfzjEgZG09Ry0R4+SzL + r5IU/JyTPuArftzFP8lb/NAnLSY5G5hbeQXA8ApXCBuj9CyR+orxJmNp/CrO6ajg - L0BB2gU4+hKSoQyT/ih7ekNjvVVUFc0RR/OgQcUoEPmW2AFh8qV7R7kOmz4ErG+l + lR0Qjy5Eh+H2Y53g0/dV9wTRKkdJ63El2MVjlTgEKdytMWK8S2RyjbcOKGSMgvvq - /g0vhJQ9hOYzu80NXyam2QnFcyAhgS6yhwQxYsy6dahpxS8eNKMDknKmRRXu14yO + vgw2k9M7dx/KHVxGbqgVHefSkfUFCQne1wEPJS+zk9eLRcJEXP1MY4JgMbpMnH5B - NU2ZGwH12pLzITkucOAN1S3zWNqhrh13ADx4EjTQ5ip5zpIaQLUFpW/x5OEQsUQe + m9BVhPvCocxfjCDoU4CZW9mQmsUx/J+De3icdIOuZsJgMMBJ9uV9e58x0lStdscL - RbYC8ZUCgYEAzR/sKkR9pQnMGrMAzFj5zgssWqwEwzOtwaTL0/iKoMkMuAo8AfX1 + gSHlccECgYEA1HPUIQ0jat2cJe+SxIwUWqpbjQQN4Gd1R7QtC6UqH4WXDDCe4wfD - ac6V7EvDdE5tbJ2qWTKeHA3X9oeio15cHWam4pdMVRGTNhu+XOilHydZbJWnwYLk + K3OpaU45BNm2VFNlozfTIMyXRBKM1SnBi3040TWr0TEzasWcneuCbhdP0VUWS//R - 8eFV2vCfB8H+y7fwVyFBBwoFZgHhWLeHX/OTCHIhV8xhf2nJUjI0TEsCgYEA5cOo + L2rRpThtJeV24aXWP64XMZcorMiTSPBqq6f0OksExzSkqDrhVckD+30CgYEA3FV3 - He6KOjjbrC5ug4jmfyzbhqkQskWN8r6DQrz1/52qPAHeiF0RV/EHcOzvhGWaxJYV + 281DsFgqPhJMFJ4NNJgNbVA0mSnsHiQ10tnlioOgjRhRWGgI8Z2GSA0ZxDxyS5kX - BdymNTmKCi3Xn/oTRs3EJ6ojG2Z6IbliV0Zea6jesaOMPzmLHTt8NThKJr4FTsJJ + +j4U2u0+UsiHoa5tv5/CUgutg+HEH0PTvJ69f8kyLCinBD+b7FPHvLJBAzN54/dV - s7ZR6m6BgooLOlo4ZpMzarxFgdid0oWq2BNeCeUCgYEAzK+/JUplKmwFXNskv8VF + 1mvZ7FWfdgYlCNxWcSQ6bpjDLy7gAkEP7lNJKLcCgYEAwbB8FEnrIVG7O2bIswJW - uSKTJwOiWPtXtvTwZFwOUXVuGLQ1vyslsmhwWHQd3RBpxsnp88o71fjGeX5Nf8Io + yDYKU2z/zbfk16Nvce95kNV1WTq2kJsSF3pSWFxlZYOrVAPYZM7PYFbGDdyvouN3 - HzqQ62lYxUadZI/4vJN2Ogk1BdKsrMAmH2vhFXGo77/Ytoac8QUA87o/OtRDfxjc + vdlDRJEe/RBTJSPWXq9I8V+1eE6PjmhC6W0EhxIDiIpEMQLFarcoFCEQhz4x8Uym - oJXZMcNZnFgZLmBsgXYRk9MCgYBNBiRLtHXeQsVRmVcu/SvYIl+NawvP14VYhQlX + o6ry4XZle8wF1g4gQ9qJE+0CgYEAjfdX8j0w9wfnt/TsJoCr+45ZYGzEZ0fWxpkI - zCTjhiVVbIL/T8PKqWCHOMaqqa0SjgWKK4gEe7+M3gVU+e6QY9aIPX77ZoU23QDc + QSJ6vyQOp7radv8ZfCzGX9hpGMLl1gX/qBKmN2WTuZ1RnwCX4FdcyqaRl585UffP - pRhuGvRctKkFYPMD37cp2C7zgewhlPxEJLCdWGJOMpzE+Q3DRUGNXIQonUd7FZhK + DwKtERAfDsrmylr96WkWEmQ8dYaObC9qlG0LjjahN1fANxRZci8ooyg040rttSYc - S2PRCQKBgBStiNUaXxe2p8waFGS7s7aXsJHnffWeRXFCpC3KyUmDxU8StujRkT4l + 0K/DPMMCgYB0RI5t8wbyIo1H82Wl/cht43SVXT37rP1lkz7kYISaCfU251jiHWON - i5UPPhhu+uyN2sGSVv6I2T3qYPMMMVAgAoc1TAK2louq3YKo0al5ZJVhL7A+gFUu + BWECvFup+k5IL4Kx988CxZt7zJcAbgx8CI5ShvEJ1Pn5sl4flwjoblzxY/mpyNYB - 6VqV5cEc46hydfQS8oAlC+wN9R41CEMZVM5o5eSQUxQJX7EYd5CK + efYc8zrCkd6ILhCrKtbxijt1C6ETh8ufOjoLKWFISLkizKwi0kYILQ== -----END RSA PRIVATE KEY----- @@ -3231,62 +3531,62 @@ vector: segmentSize: 131072 total: 3 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:MDMF:l3zgxf6kuww2j6ncsc2xocsdrm:xxuxnhpkg7jqnhf6jujj3q2qeqagnoipyhvpnohvwyk4dden7hlq + expected: URI:MDMF:77ev4xscd75db247gnsymtav34:ssizcjz5yooirldyp5n67sq4ec3h7adkjzb5i7nf2idlgum6ve2q format: kind: ssk params: format: mdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEpAIBAAKCAQEAs9bfSRdiJEUd/MltLPZn8tsG07oAfolJq1sdJhPCmDhHUBt+ + MIIEogIBAAKCAQEAooIJdI38ekWIkn6i+Xn5Aht38L967mmHX7N5/PZdYJl33Oob - m2+meR1HGVeaywpcqzdqOoTtv0x/nTl7hDtZqW2N3ropb1XTnaPvg++raFzJOfdm + ML4xSK61X7MVlvnDnMxGbIQ2fFdbBnpVAMEb4z4poKB91pYF8HyoogV9XP+mJbCl - +dIBvruf66X+217t/p/aJ5if2tgZNgoMHw/mTjLfPM1xDAbO4cj4YHWiNhtTIZeO + mPzYpbEqcPt+ROyIOgSshE9c6ID6hdG489oqDkOoavTvnEcfEmCF89Cm5PsEE4DJ - vtbrWUWFRy+2fxG9DdKbGhmNw0I97Fk6Cq4JayhYulHIOBx6lQqfy5hCUkYKOOiB + nL5V1CVZQfPBmPcS01KaDvNE065CjRyomx8dav73zpVguKmGLpxba3d2GnlTYI7z - sBqSbXlifnQ0Ez4tRghu5NakvQVJ9cWfm2p/N0nqsZ876Yytku/7tjHbV8E85Q3j + MNaxZcGl/seDX7KiGeI2kUC8pWb3ezyvPHNbwJOAPNKPkw1oVkEZICwnv4znEnXJ - 1X7l0aS/qIpf6Gd0GGyPznWWY0kPqsoqV3sfMQIDAQABAoIBAArmc++t8Wah4tSq + ZlH9qqmPIWmn+3PgWRqovpYhEEWh2YOTZaXtYwIDAQABAoIBAAy5g/CwIYHtfz6B - z8l5IOlNIb+NB1EkGJlAg0aGzZVk2duu4vBgZs5x+hh84Ra76Mx+5hsoafGdnSGG + U1Ts5muc0lfJQTX5O2lp+KEEetan4wkq/PVkdHV4K1Q9W0ZsHj4L3MPTLR+wCLz0 - NaiY4VEd4Qq2LWNAaDxmjpKoYPMJJrAzAOSU+EKbhDCoBce9m/7CKRqby1qcHQET + HUMm29PVDUC6RcA6FRL59TZimai2N+tMRAXmXoxy1/Dq2xNIi3jc3tP+vDapyslp - wFLUp6CnQDUi/Z5dPkZcpENSdfPB/7Df4NocNjpMBMvvKuP/MK8ecuGNWpy/Qn+X + skLIkBx30xs051ePAMbZRK6NEFRuQexOYnUXXJPqPb5pkTYxaYmgER21L+CfalYc - LaTkJ2NOEBslz9cpZXNY9o26Z8ty7duSITwzoFpc2USFS1v4xHvpk53+PgPxcyU6 + sqnMhE+Es7YPueHv+fVr/cw8Sdauyzx4PkH1K9iummDGLIA772jerX5SA2oBXwTQ - WHQsx8Yh1oME+KG+A37TXxTMgDrlCsPtWIiJEiM3AzK4Wb49ehPwu3Cw0S/iYKUG + gV89YMdeVCyOa4LpD64OSG+NKda/H5ZcBi1lmJMDqyegTOALpTB1NC9bM20PpHZb - svx3ZIECgYEAzOKipuw141V7k83cLJdDdlNqUJx79wE9ADwTey0z3VwNh9xxi/7v + ILkMQKECgYEA3xvNtJybLTLZ1bTVcSVhuuftlnBYqVLHue3DZpDH/6L6wSgUbtmH - b6Nb35F0/IN7DVywqElkrIvM5+pJKv4mISFt3TPVT0oEumdN+HpQEn3psVyg3pNp + tWpPTzOKEvL1Yc2IFhnl4j9/d2TFs91Z/MOZB7XFsoOySe3kq+8X8r7XlsqCEdbo - fD9Bhinbc2zLvGZ3n4fAFDfXysjjCyqs+orR8ksKen5UH2uVtMwGUyECgYEA4LSh + /yFxN/espG+GDdT5/k3UwGuAZek0811po1+EXkkjqKK0Ayhrtaa1z/MCgYEAunck - 8pHvpq7ZCLGr6C5WgXvmuFc44TQyXs7TOVvGO+In8+VwCJpYz6tT+8eur7hJVScb + zSg4wmYwT0EgQw+HuKWwmtYUO/9QgsruVFtuCr1XFHCwEtR65qnln1aratJYMoeg - Q2lUXrA2tZ7pIUPf1hetoJ9fM1ACeOyTrPXgWFUOhyk8MRysat9UsLtczOnD6mpP + +eFw7VJE47hTuhIp1kTFe0zHy2xyOme8kw3Uu5kRMMGnaAjWveQMRHC0H6CpXJEY - quUf+kv8t6qHrHrh8bx4IK8tcOMZiuTYEs0WWhECgYB3GqDXTKWe/EiUia2etmhf + hqnz0gs3jZe/rjKxISZdibyQ2G3yp+u66EveONECgYAlvl9jcaby04p0k94UAR3y - VuqM5gsicjPV+RaSGpr16ddrzXism4zxZxO3icVqLbzQ7bs8eT3vGG4Lu6TBO3FK + b7AK5kCpjH6LXsTSwiWDgr/nE2+5fQVvVGfMX0y5fe6zOAEQtBrm1pUqzpp/ni/O - /TXyy3kLWMoa2ob3FZOKzGuX0XMrMKK3ucYLijWqiep+IUsVEENW/YeSuOlTyoE4 + Cg0Gd+LVG2B7D0rDJ3SmtVBliybL+8548uBjdnv3aYKFLoWIVwRp9QXIt9YkYaYw - PI8DvR/gSaP5h/9FVP2wQQKBgQCJOebY84Suf4MtiwuX3IyZwOfy1dl3tt+4BIj8 + ZEVRahAlRDkt25W1KlnRrwKBgEIq9Uw38axKRupY2fHyGN9VLI5FWXjQ6OkAygH1 - M27JbWDG0uxrZI8uK8w7LAQjbeDi7uH4dh+/P8/5dJWc6g2NeqJfQFTsSkVoQdoh + T99PwQ7nzhNggxxHPcyFHN8TNWj46A7ECSauAvQr/MoSl/YJAWr3nA57tS00kp48 - u3qJl1Aq/OS0fXVSQxc+Yv3WakBqLQiALjMsMTGhnLQEgnrvnRCjrTeMBDS6HO1T + Ujkf4BHFJnqsaEeKHTBMLh7rDC5k7qcauALZKCV+q+5M3r48twShfWTP30PnSrQT - 9glbcQKBgQCsrNn9xBLI/RlHE1tD/5bLSHJRsnuc+5ek9JKh7AdioFHYPTVJ9nfT + +A+BAoGAcLzVVavz+whB7FuH3wcbfofHW0pI09PB88qHHbm99voYwi1XmiNl0r2A - uDf/8f6acF0fNMcC1ZbqcTd9LMO0dH/yCP55QKtPPNVmPdx5ilEJ5m6yt+qwTRbo + a54LNTlifPhPfI1OIIV83XvyRitgl3bCdww9CyqD35mXSEKhjp2UaKYbz73gxddF - np9zZOmcaT5orVWs26R/aN+zwYoAMeANGp/P1RBCoTpo7s0NctfiNQ== + tkA/rbjwYc0rvnZumDEgeMpSBvxHKNiMCHuwvHbkeDZu2hCBe8s= -----END RSA PRIVATE KEY----- @@ -3312,62 +3612,62 @@ vector: segmentSize: 131072 total: 3 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:SSK:tq5socaiwunkijgdq7j5ggkhj4:6s76nfeflnfuzjsh4jf7inwtjmil43xbskvjd5ykfuhp6cwqma5a + expected: URI:SSK:6lnma7gxkxs4gz5wuitsqs7loy:uugb5btz6wxdrtmtqf5uc4i7vnukhj4wwezi7ygwlkguee6ponka format: kind: ssk params: format: sdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEpQIBAAKCAQEAvy7SCw1WfwAdhbT7BIR0VN8omAinI3vlgeuSGEtuPVVjNNCa + MIIEpAIBAAKCAQEAvsUlCU9RU0cOaxNW1B8P+ADl+e++2dVV4YDfwTQ/tfAV4EeV - l33uacBw1wtmul+h3aJjWThEs7TxRyEHhyIJCq/9hE7uwuHyUWcR8esOYFJqEa1U + ajvOZ2qRveGeXPFoFiRs4f1jA78AhHDu9YXUlq0ID99yCZUGAa4S9ea6eSYybScr - fDeSAvTX+hWZSC7Bge3dSQubZXCC6Xa2VRkdBCauWxszzHvITWIXb1QbkyYPEg0X + 6ZgvAxJo+FAkR6DKvrjGEK9ioav9NitwVlJMhmyrQbLQz2kHu9TYa+WGdxP+0R9L - bDoSoMbw9tfFys2GrVTM1RZX62qagKDwhlOCQlMDGvPqDRlZ+qBddvOKpQq80C5e + MGgpjkl5JClHMK9YQFWbC8OyJqtua7fj0nvE40yYxw2v78wyad+dxpAieoROu6ys - /SDhlcFoYaz7Z+mV5fZ+kjSUp7n36J/Fha8uYck/FJm1PhY90ND6vRuiKEdZYXfm + J/FLcYcX5/ijp+ipEPW1izlWNtGsKLfZkulO1OHaTllztLcY9RKcS0sIZAEGWEIk - OGJzxK9Wk/hLtZptgDqa6BG898oRqo2kJdXY9QIDAQABAoIBAAL2nHFKQLH843Mj + aU41rUVAgnjz6yNu/NojE9Hb3Yogywb7fHpvuQIDAQABAoIBABw+hgA93RWOVKUn - KmwBCxns9CNYQlmstjZnne5GRYSLVi1rLKa7or+UHpNQpr7rEsFRUfxhGjx4XmsI + xNg9DRkz0NjTVRddTYzIjthFCxW9yQ9eqdGDr0iCb7eEOvUZzMZSeEhxQoKgecwi - cfggL0SdClMGvcg8CogwHg8b8ZWuldJr9iPR4RYAXYJ7LUODYWBYuofNS/7IYaXN + CE8TlGkGj0YCWBjxFmWTRz1e/sbHD/o3LXo1TOiQhjgVbXmpZbsdEj4QKXM0SpYF - qtUThVrL+iGs3MPRS5jQQ1u6QjIekfJc0yEGCvg8huj2grpzZV/XMBaOyFVjr8aA + kmlFYA0a87QTbHT44Os/VReMcP8aFlGhs48JafEMxAeHRk47deugSML/DS2OlywL - m68odNuEcQwRvv705NhDyOAMJnkcZJbK/H7n+OCY4M6EDuvNpn/n8N9s3yMQ7r0k + 4FNrAwA6AZZrgE9+8qXHoR6J46k8tvRI04Cx5ADmbcgoOrLBnfheUVturJOZGxKx - uo4uql56cIc2LxYFUAGQ5HIaoEbceBk5JZCCJVCh/IWGqDJSTgGHn3x5ip+R+jC8 + fEftj7TqTtHY+YtMawSVEd4tC6V2XP+vqIVduYw4q0gy/gPWKquKC8NE0BgpgXui - hcaFuesCgYEA9L5cJRCPk7Zz9p7lwYrnMJlSjXnIo9855WRwjK+9G290zxQczh1Y + suZvRC0CgYEA2KwFvZ39x7+ALjuqWd//jnfA/RWGh01632PCatgrlmr+XMHQQLZl - twjeSdziaVXR3t0GGHYBF6Gr6wV/dxLHZGPlccfYXnIQTmku6SiBz6B6YBrWLTmz + xrpN4bQCur9yOtF6jkF70npK5n/LnbtjWZ6CQK69jXnShUvdkgZJxk0iIDT1o+5R - y4mFR0D51WVBdFLskG9r/hoUQ6UYKOV3YxxSXu9BTeiHOgnm+m7l0Q8CgYEAx/nU + 3r8hi0qWxolhhyLgpIoKuCIU8+KHRnRud8Pw+oW2aMWuDPYVSr12sNcCgYEA4WWN - m9Mh4tvf4cJ56Mo2q/R8habcAyWAnp+MJr+qCiZip7kkKxtnWlmLebRUkZYIAbbN + 7Q+h/86Oe0L1Bjv7UgaunvvCDpNVAGk1IFj3+FnVFf2PXNW+ECsXBviXNvGzfNDm - JTat+UygnCw2oWeFyIAJBWjzcwano8NThY7LGOax0JU2aNocu5gzz/aaN3EyqZVp + UoWRZPI1+ZBU1vzlidxY3ns7l7Xa6NBVBEKfEWwSeQ881S0pF0Cf7MxIP1xMUts8 - Cgk/qyoF9EPYOielRii6KD1nulaJCbL4GHFhrbsCgYEAprs0fQ+uMHxAvgd8EIE3 + tx5koWaCf0+my5kouTqHFSelAQmfgpYetowJge8CgYEAz5+HoLvUg6Qt4B+sjZLo - hNU+9yC7PmBpycvGHSHwG8uvcQ+LnCND99Wz0fAH0qjjhAdhCrMBhX7fZwnkz1Lc + AE0g0WPfFahZJdciZd/fZLQCKkBOnrQpstSz7KPiObFadKJnHgoB7R7ixx2OsAbw - wZiIjB4QWi8syq4/hhnRbYgvNl+x/zdrNEMop+UtDmKf18ZSYQd3M7HCkl7beajx + nOAXUIQhf4BNCw43s8Xyy+L94H7fI8crDJd6PU+sS3M50ZTKTuE4hFmkWk+n8QuB - z3RQ7VnjTFcYIML0NzHroKMCgYEAxiE9CPZyyHXYp7ErX/2ZlV0yUqkzqtppSMAC + D6LjOC0JLjy/HAxzOrtzEOECgYEA18YTl1T23d/M0L1pub0kPAM/md0jijaLEiil - +BFFw7CsZkkFEMCh8d5uVjLY5zWi0S/wqUI3tJy7NICJz/jlj/Vq+rU1H24kghhw + fkENqge9kR665sGMAQhvM/I5OJUsIZoOVAOgC8Y/25jLT0CtMUvrG5lXlEW4ulXu - lA8aIp3O5z4vHkub1DHEg/NscCnzbBngbFUlg8yrAYyGm3fURGLtrhjIwNIkDDwJ + fXSVuOT/zjrDHsTr6GGqd9OcemOOgWd1+Uu0RDrRRLVo1NHbhW89MAhS0up2dFno - mw4bHSkCgYEAiKmrzboctKbD6gr2t6vZ1tXCevxDp4IOIO6u6qoMt48Gggw5oZYw + LxNiaqsCgYAFitD9jsQuSigjltdWGBnaQA/cyS9Y5VzU5y59B4nXW4JOqfnLmMps - NU2uRB3KatsKLC74uDs6zPTc+jap23T8185V0w097Xn0gA5FXIDINklp4dyq6Erp + o/WGAO7CKBvnz+7UOCOd/1ZdsA1ny70ZbLsgaf0Ku3+ehS5XNy/ow0LBFqBEw0aT - yjE/73oP2P55us18FAG3WxsR626Mes2mDVbB9t5e5/sVtNSAJMzQ2fM= + LYdUwudveoV5ZN1SOOR19Zzc9+30WHRP6Nd/UtfX2m+EWF/A51AuRA== -----END RSA PRIVATE KEY----- @@ -3381,62 +3681,62 @@ vector: segmentSize: 131072 total: 3 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:MDMF:omvs34f76c7mxp6moabdea6gte:5yjko35bt3ks3eeosni22fyshwe75p63ilzso4m3x4f3w5ay6qoq + expected: URI:MDMF:l3xpgqv43m3df7pr462evj2mei:xeixqrar643gyxwyyz4ccxjte4oznzm2ka347klrwr6kwaktzjgq format: kind: ssk params: format: mdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEowIBAAKCAQEA9ext47GMBnIkW0mbRLDHZlltA89uH+UMkyGyIKEeyA+/XUNP + MIIEpAIBAAKCAQEA1HnRxcfB+NKQ+Kw0dgtWgRUXgHcmKmQ0cu/44QlykP0BrFJ0 - djLlthN1f/3/A4yqgf5SGzxAuZ2QtqDaYCqmHQ5ad/RnFp8QV2hAvEjp/tJc05ZW + B+jmpHwDROh8jG8tmVyW1caWlAZpdIUvqHibCB6CJhQ4ymKGkC3BBTlAzUSUTxEf - NsAZg9f9IUSqOPlzwGDHDQGtVofowkP1Wuo/pnkEI57uqdbQoPKtznQ80KcFBLDg + 8vlx3UbPvvMZVjZrfTxfkcdScEybizqoQVcrxeA9iU8Ud4ef/dn6xT/rgtoByDQi - KGc5gu159mY8q3WtzEhCUr4l9yueJgeKq3Er9tb1jXKW/WqZHfaejmt/HE952MU2 + E9PWRwByAqnF6fwsuKzQOzMDB6vMPPjMgo0lqJCgXHXZ2XY7ApHmQ547pucm6GkL - TyaOGWxHfudEh/W5U6+TKL6Q7En6042IV2x6xz4SD+k7F9eO9kSMSw/+Tazk9rbF + bs+Ox2eGDeizTbmy+JgfqriJC0pb6cMJC65gBrmG3qL4P32EHfQFkZy8vazMYbNu - g8NahhzvjzqJhNEz8/AaXxsaQ5mJAXRHtBzFJQIDAQABAoIBAAMbnvmi2GmcCqwu + 7pPM2CozaJ6XHGCOQFNxACFLep2V8DiNb+1LAwIDAQABAoIBAGG01zMlBdzfL406 - qzBvekuBImntNqte/F5Uk+smHQ35M6QVUx5SyeCX0vZa9lyUnzbi4ownGZ2jK5CV + 2zEBS7k2MsV/hQxvYfMMyRzq1EU6I1/T1smgXxd6c6JnaobFxWlFu5L4rFvLiwjr - 9PQEgbJzkwTq3lym26HHJ89ZWGpdCgUUys0iZged0okoqcIwXroV+J4D5WCaJDyI + ChxlwZz9MopCOE9Q+WIpuB4n0tXR2IV3cYKxFJxVqMi9T4RmqA1CCwylZRKBF/Hx - iN1tGsTBc6IKJNoROfQ2wkFa2XC0YqSZ5y25kowk1zwd2S43XXWpejTLFGeHkeSx + elf5twZadNHEjLveoUMBzyCPoURdF3jZuCjAJCX54NrD+x53hK8gKv4OOjp/k/DN - WfPPIVZjPnokg/LFjZm8WWUxGE7L6ZDSseV6PUDfnBKkRVSVHHkulrEOowe920bj + inW1wk3p4ZEzy0/A7S5EsP+BlwRkkT2yvdEaqCsuOeOEwIuz6ZCK/EZp57hsEBi+ - 25pm4PytfpM4W7s1o9+Z9b0qpFmDIX9MFxdcPXRZH5oibNiEK2Varxo7OS/SXBgW + mbqh4Gdvj/MxOiWOa4MwThw9eiYmSGwBpqwwdK6qGUNj8LrpTREYAIVw9lwrN0cz - BupkfoECgYEA+ggnF4eR+42vOx3hsRsg09nKOkfmzu+8gM/oHWJ4TL2jvq3+2duV + 8iQgYTkCgYEA7L4Rb1Iq9QcDDnMBWFUDdVASILfePCFwbmPSXeq4I4d3V/hRlYdx - zXuEDc7hc4J1Wq9NZ9MV/t2gxuOVp6W6vzVGHMiGmTAixu2S9IcYeRe9x2K4p3nx + 4nNxva+Qkl+xZZovOlcFJyLu3xG4YYqALuVDnpjVXIKbHXrsJyYRaPTNc9iLjz3V - ULjg2FRDRytvUPofpE/Y6hHldcZjBtsF20kOokuyBYhHG0K4AsGGruUCgYEA+8ss + cGQJVT3/uIQRkjjvws37FiumntYwQAkhEVqVfemE9an2D2sW3zrSW/UCgYEA5cJs - H/gK+IACIjywcocrEdKXyd9bOELhW6vPS9/IiUe2ZLcbjshejk3NiCSNrBA6V+PU + SVF4P96iNb6gKfHPvMwTmrmXqRpSc3avkM6WBWthx5RJIPp6jEw06rpXfclILKPt - 3jnV/2jHeEldeE/fd/kDaJOCPGv6lbN+UHmk0R69GCVVai1zTBWmjAq2CIgfiAwM + a8V3TOlZpYjjC0M1QgShPtN3zUPN3L9T5qs7x1ysppn/NNOLGgHMkrXDRcoAW7xW - fs+gYFf4+OAbpo58i3mOUPCpG948JgrRJI8eGUECgYEAzkS/i0fKhQ5kC48hS+yn + 4rBqobvbY5jIgAut2WH1A7MrYo5F7YnNOY9N6BcCgYAzjx168hk67gEDZ5aWZ0Vu - bl5z2RTMMtfQWSwrv2InAJhKZ9o/LxdaRESrsoCDublce023u/mGYdYQ90N1iPLO + ija4e3LiA6JZ2FGbdKAP1NPwC7uw5iOuXtgZqJ/C0SZwa0j32rXblScS2+gdDi3m - V0Pp7YD4mZP+fMItxBFXfT66z6x/zZpqHEAJLi6Fukb49IMEa5d7yc6t0DW0KEm0 + iLXWV5C4KhWgMQI3cHoAMriAD1wtoRjX9mF1+B+2TsUI5G+LLJMPfAg4tYsilxpl - US26JuXvnWTJ1JF8ILnrFIkCgYAL7usecMEEWfy/5qRuKR3PcG2lMaK/HdxUXeYr + jiXamz4CxrY5G2iKy2O+9QKBgQCnQvjOD085P/xan+G9Z1pSGUcUVpP/+TeY8wgw - MGXuq6lnSI5TzAc/M0zEYQcd2n8JX1DdX1xXCH47oy583zw2EWUp9aO8fVmY8rLP + pRQ18cyHHH54UaCxTjEfyHQ1EDlItjX7RQ/qn94xUgvngQ/edbxlHlGSzw+o6mhL - 2ZQIHS7VEB/mMlU+i+AizvclnF3yMq/86pYtOr4f/W8SC7q3WYF3MJCzM2siWmzj + /tBP/Dl8N5PAg1g6oKCrFUOJJNtJ7TxbXw7hmv7F7M3Z2abAID3caazl3Kkvmigb - EK1agQKBgAsKaCJYbtgpbPJ39EF93XbsN1m/y7yDdyPUvQbim1u36mAWJg0QwJY+ + BSY0FQKBgQCAM7Q1W9SjcZ3b5M8f/POYzgb3N5QU9RzvOJbXcs1jEB5LDdOkKorL - BmtEHqZ+U015m/5dkHNKNgYPHPE5BedaPIGWkPDU9CpQ0g61J2itSfJhqMSNImJr + mx/6lLXrKN+mkdJxXLO1ruNzlfrns8gL7WeRUqQ9h3OU0Ru2Bxe/VFja0+ZR0V+u - RuH95SDlk+Z8/MW7iJ8S37SgJcja1dlTqiQpd9O2ubyYh+BNSZyZ + 4TiKtcxIl9EijEsaGkE7mRAmlOnyvOBuEFmEkVA8aaCNkzJ4v1AGfw== -----END RSA PRIVATE KEY----- @@ -3462,62 +3762,62 @@ vector: segmentSize: 131072 total: 3 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:SSK:c2hklyqlte3ar7norf5nrvowaa:2pd6rxd6hnnkyifcxce4pf25g5fzcrszxr4r2vcu3vhest7m5sza + expected: URI:SSK:4zprwdvwwgfc3ai3fumd5zr4oq:e24f6pm5p65dacq6hpv6vtcospblpahmjrd6uabuhg2zbahfkkpq format: kind: ssk params: format: sdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEowIBAAKCAQEAoOFuH+ZLysnPs7kK7pVWeUJ0C71ftfcmncL5FgGQV1MMsupz + MIIEowIBAAKCAQEAzrq9a2BWiiPkWXUt6BMAdSHLqm7l7QFCjYeTEGJzFIvR1G4g - gWAXlrsUT5toD8s7h8z63S+R6e6+I4EYxc4DoH0icTkC6wIBXa5ODpJaMQkFa+5M + LxSdWfLuVX75U8M7A57Km73BfTzHLkQEtv7HKvI9dFH+Q5u1DhrdHD1yaDOosqFI - kDu4YvO6QUfGyC2J4YE0wySXNTMSnYOEmC7Cfouo0uVzvxR95bP0VWgXJLxMZ1u3 + 9v/xk9ih5ngCUhOZ6oXVXnOnOpUYeXg//NTHNIAZcKS+YLExeFHpKbngDRvlsk7S - N5nNP94syH56Yy0Aza5VjlhwmB/HdqXBiV/wQfjniNjWgkz3yxl8U/FIkDHEtrD0 + XFTixFWO6xCsHVCNiq7cUe+1x0kG6I1gqZGWFG0NkTxycZjUjC67YNXzD7HGwOfn - svGzoL2AaNPKlfkDM+SnaB6tYfozZw7xP64sLy3wOVkWjfc9JEWxo+Gco4nMK6pu + b1DCgOq+OB/u8EIHcDnw/N8ijELxbWC51Dr7zPF5j2ndOK3qtww6jod9Dihc+kcg - uqcq+vX2m3fAAetw2brletd0zpNhqw+9WtjiZQIDAQABAoIBAC9FDU5iJDLZSSXN + ka4N7D8/KOPOzkD5OF5oZ9SW2Dfnl6SiasEHhwIDAQABAoIBABNGyGjdx9QDusQ4 - YODpEBdg5yfr5ItaqwX/m6BTpU2DIWAQcw+4ZDXtkfIx/0lktYEZQTxsFbteYo+c + r/om43EIoLQYuSnbZzhJPwZVF8P/saYsSqX5Nx7vDxg7ycXsu3D/+oaBMCycYTpI - BuNXvMkS+2O5FJpoZG5aIKU3azitJeKoieZ3JZ4tbrRvmoCGoNSZWh9cSPFgqD+P + L2Rc53dytRZGmv304/IXwSxj8moS/xUBAwu9G+qcVaRm5lh+6Wg76IRxJlPJGUoi - vQ3Z71uvPVN6B6BFLRio30mY4/Pux72huREGa2EvfOhhQm9/euIW1JqvMcg8MIti + U10g/h5AH58oTXQ4sZM1mBdC9Mhj+Wm9dPIsra4v2x05y1qxhHNqHGe+n7JLrDNZ - WZavUcLxdpPTsiNKVIEEJUtoCK9xbbXuEmKKuakJfAP3epcPgBxIJBBo6mrPxIkc + c1jZ7I+8lxBWP9Z8vnE+XxmOVENjXndSVlQsHRo2Lqujk5I6I7EcGapJH+xv1UD1 - VVSgNRNGACOrILLxEvV4Oy1fynh/VJD8fZyA2vppcaQERn6R89z12zGvanXGwIu3 + mdKewzqZb1MwaK5SMmRVt7q3QJ1MmyJm0Gk4MRWkGzbpj73zpKT3eeHTHxsw0P72 - CVRcP6ECgYEAtvCNZ+RSXocG1Piuds6SiATFFbvQYoayuxwlrkqaIvD8M9V6Yj7x + yLEKnAkCgYEA/zxyPOrUAk3AYIXKt/BbSt5evDg/S/9FOMwLiDrRgiI2neB3NFfW - 0aXY7RMHF70hjGue+ZvR0B9hBqLGpwddLbxx/UcIv4sZGVg2YF+v5Ar8Ge/pdW9L + 0qOSKKhvpJM/2s6rCpHAbqI3cG50E+XUMqQKt2gSwwpxS23iky2qtetFdcPv+c+1 - i8gbbdbapWzErQJtX6BYyQMh5KkeRARWfove/JY4nvXgUzYcSeSogsMCgYEA4SGY + DJ+wAODXAsBpMvhuS45NEvZwdR0D9oiKrnz82mBrRiFokukuBmc1qbMCgYEAz1kh - TE+QOmZgdjtBxyjx+LX090Rjg+sqI8HyV55gA1C7RcOcfK3CbIJOMhddbT3samVK + IbZOJu4+qY2+k9iir5gjuk9EcJzvTJ+PGI5d/gJVp2jUUPaAqM46R+eirhhBas6j - PTxjQ0hP9v00MN78hdyW7XIQJfIvEdtbnigivUzILgI9ejgQ+NnWBZ0706/i1Y7m + WCxX7g89rU7rGm5aUCh7BDHxWsqtwp5k24eUwv8tgVFqOe17CXf/722jRJ4LOKCI - t1hwUToHmV9JSObQyvd5hyhVtp1xWblzOXPGY7cCgYA382KMP9yhZJLGWDijxZIz + 6R6qvSKU/BssmtZugasUhbjH02ctU+/IGmiIWN0CgYA/gxcaOYUQHbDlU+Wh59mP - X6IXf5XATIolh/pOUCrMPQAlqkj/+1hiUmMCPyuQKxwzokbA+NM24CIAsZAoTaxF + w83nIEf/7UGYZI3qFFjV/RWCK7z99W2rdLCGFYPSfCHDnPHK64HrBcqt245e9S3c - 7LjAShV238gRZFVdLGbTTDjGhgXVEPD+E3mwImJE7ftJHtDsylHdSMP493B2RQ1f + fB1+jhM9HXgbwPhEj3SPWEAskdlBXII85e+yCED4mlCTMmafvoVHVrOdMN8vlcKM - LtBIWHmAxJqTWJ1WTETtmQKBgHCKM7DKASZAcS4JNzuQy0zx4JAO3tReJUWUuUl1 + sOVqoduP0/hltkiRp1UfKQKBgQCvplOY2XUvKZhPzlHpsRVwJzPs+oWB2JAnmut/ - gTeHDuaz/zEQR2WoyeAeb/ShBOK22aK84j4LEvY74vAfOArOl6AA6fOeGkuJ5UWt + 4+rf6V5iKT3jME8hsUJR83oufUG9lzts0HPUqXiYPkiP5XgAe6pqjVxmi1fTjJbo - eJg6nsLpGcRT7KAJfQR3ciXDAdiRw+GZUyQ3pv7TdDX+NBeSGG0pC5frInOg0enB + Gdz9q2oDKNMSK4mVJeDcFUbANBpRaD1TLrV477jCMMsCiDCpDCKgfT5aagdOGcys - Z0YHAoGBAJswQoQ6cOsi3/nCIjRG4uqPaLCg/g+qJsShbfW3u+9j3sFwffaz6kYj + Am2uyQKBgEAVR90u1U6oevjeyFCMP6EzqiaJFRQQyE8BAtHNHlDXOZsULZxpIvy9 - ggqC2HYlerbj+7O54rlK8A72yHfw01R1cD3BbrkPG8qKNhn9llmhRC1xlHV7j1tI + Y2Ib8QctoHfaurErSX32rSrXyXqK9mKnNUk3/DXicHhw7zPe9SrOL/8H51iBRouO - j4R4oV0et4AAO1wbH83sESajyIYdpe+9JjqL5al//PXctNwo59OM + uV3hW4DnDIJE42SShQcL5WNdFBrzXG/OAnyDnjmfO8o3ir1YJccJ -----END RSA PRIVATE KEY----- @@ -3531,62 +3831,62 @@ vector: segmentSize: 131072 total: 3 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:MDMF:vssudzh5hmbquhrlqnzyrl7xgm:fkkgisy5ozu5kfyuzfd3ookqkqj2citrlbbt4kgdsmwa5znlrhxa + expected: URI:MDMF:gmqhs227cyrfb7bkvjz5nkhxlm:hqbl4c72w75tn5w6dqiha57cwyfgijagxyhab53h2vccgqnox6ia format: kind: ssk params: format: mdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEpAIBAAKCAQEAyKkGhkde2GQ68zuzyMAmrhiOyFo9uu7/JdhVy/XLqgWaCCj1 + MIIEogIBAAKCAQEApFyk3JDYamuKCyq4piIPAGKW8xUEjZ7CSH34nTGucEHBmTsZ - d6ciBMJj//02/GSm8byqB/I/g0+hqJVRTDTICMkbFoIRR8el0Qi6AzOlBrvS4ztX + xzvMbnnIMCIDB3NIQeTTIRaSMnmdEbtDjiA5sJ54lkCSkHOaEHGXKVivI/NNTcnB - BFxgotuwuvXP+8F+Am39G1ei1bcQqSyDRQhSBa+CLc6d53kOBEy7TUIm/yFYIlFq + rC/Xp7DESLWQjn64rw4UD8X3Bw5mFnj/i3dEQc0CSnwRAAgoWRU0LkXWIZaIPIaV - uR7jEvUyqgbPz/2vIGPTvb4cG/04RxzGriYvzaiL46SZETkRJ2L+WEpBdf3w1/6V + JMX5cL4bgakkb7PgyQ16CIWKA6XdzAJA3Ef4TozRACeB6YcZuC1p12CSoO5kGvOt - fKVRJBRVmt4ifoBV7op77qMl4bAP01ZQiG152zIF8COSWxtPCgyV0TTEmOj0g/5b + TLYd3OIvOgkU+TNoaWxf8k5ajPF6cXKDmDCDjwTShK0lhCRXqKqas2EwcZcp5Wxm - TsglYWnTCMAClcjuKoNh24ROmrfX2VrHqMsEMQIDAQABAoIBABVhANt0pjPK9gbt + Jz/tYerlcr2ZLQW3fJ2I33hzd77RxFfOwYiE6wIDAQABAoIBAE4oL80io4aXdKEy - QPnuExDwd+H7z2DnztJy6q046nKady9QYdrWOUcliO8AxQd+F9VgowMGueKdLN2f + w3Ncpr2MJDObPvsJD2HhZSN6yHRhEGqJDA0NhnzSNDuPMNmOHEIZSbxmO0b9RY90 - zxId+4QIHTU4NWwe5tlPIzZtHbOKdm0UaPCDgR5I5tr8jqTFmE3c9x8fJq+7efB0 + +P7QnB48fSMVuZwvHIfNPBBRN5dkztG6qvnyFh6LlArvK7pW0AOOMkP94yXb1vfA - WCYWPVryuJ11ypgbazVlEX2pQytiayLrYfx/FW4xDlBjffq3cijqlMHV7LdVxWqa + uePb5v1TO0+oB314Y5dY2eNXGigF5Hmp5rZIKwI67ab8AywapYHYOzvrYMj16Ur8 - NFUI1oEGvJEky4kns8kTENwV4Cy33vTNSsNS9dRvUgIfY2fdjFT++GRz1HYtfWVk + emmxx2KEXnLbABWPDUT/3T/4+h+T7pGOfZOl1bJdtNARGGXKIUYal/n2FFIan0/f - F+vSKr8Xd75fbKLiu3dK+5Z8ba9KapVqIucwKn0YQqHuZC9p0HtNbN7a7476cdvF + pzoSL2m7dGyUFYBiPFWO3LYyVKkHpvXhicNjXhQ4L75gjAD08fnjXl01UB5qD70u - jMj2fyECgYEA0N8aO5QHktxURBL78hwx08WgrFuNaLafc/i2wGTwS7LqhhyeStLX + lqU5VAECgYEAtlFX5quPBPAf+HgkwDZl0zOy4e9P+lIXKth+FqiHunSnf4uVbkMc - sfTm3x7U6IcIkpb9l69B7/nUA5zdghvS1sxn6r3seh35GgktlaapomAbGYyClPVF + tqxBNTr+Z9NvF6boRMZCIIza2QTjlCG+yfo7c/+/r7o6weaO4E3dhlb3D3ZxRVDy - N0SHR4GGgcYqjK2/cYDXgJneY+h/Ph0M737wjGfxHt13cdN/pkrnQwkCgYEA9e+f + 8LxRDXJPHPiDXeMSJMt3apL9PVQxXxqFy8iobuG4zq+byOOm3xHR8msCgYEA5smU - fnz8/e5r00ct1EEBxkKOTDJD0ZQYBj/bcMPhA7s54qiZbZL4wYR4DgYfFoPKZIRY + 3j+GQ8Uk/SK9HssEXwkRJY2BsEDlhnyFd7rtzrBMjSpdSC+js51xfmEfFf+wJDjs - SKoACQfkdodwFMy+54Lzfu9VyrnoukDRzwgJvMfV+xQeafbTRnYJew9ICb/EVNCg + bg+LE/NOnQkqj0aP3cAnxRqxguYLDoue31/NTxHqF8Osh73CVaycbtGOlgZVSKlV - 1ClNSOkZwHlvYVwxMTS7zGda21gPspX2rjnSOekCgYAodO5J1/RXl+GiheLTFG76 + wbIopIZCf1qT5Wqmq2iaHU4fexYwyOAt3YzRV4ECgYA8n4+7BBDqc24uEMUnpO72 - S+9BM0KCo8zi06viPCrnHrKaY3StnYU17O/DC9/FYlJgwmpANSwaZVORl5K4HteJ + 65nvxsOxWNqbRKGopyF6vo9zudZWc7p4g46rRJKTs5qdIsLZG5OhfzTGNPn3p8Dp - z3HZYAwr4x5a0qhHsk5tKxxUqIiqfY94kwd47De3b0DSmtzYCVK0kBkpVOFAkLPu + KGvcho4WwLYJA8E+lKW0pfZBDgFcKy6dHgFVl0z3NSt5bKf71CxBI99RJU3FcexL - t7G0IHXtuovmOkchWKTOsQKBgQDc1a6CBfmmitCHhwK/9R+Cx4C/KuN67WAlPHHv + ds2LEUOCdqI2inxScHp/QQKBgCAlykffH2vHRWzBbwigDP7T+4B4oq0TjSVbqRfR - b/Q9RYFU5c/fdHmqSykCbry7mtvCJpSfqwcdFNkxFayvAKrrd8rt0DtZLlar6Eh9 + gRi+dBth4FaS1EHL16hcDQF6eWXCTCTUo3Hm/XdgdH46vQWNo7yQCFQMiVPAXSQO - fto/ibG7IvWscNaGDre0qKQnHOtOvYes+ulK7wUQr/ozknUZmiCICsaq7wgpdD9t + a0HtY+dliV3rL+vRPIUvSaZaQXz6oYh2sbShQxgMXPejEI5l5rnYTQhPQgJpP9pR - cr4zAQKBgQCLZCdKGTbqbgzTFzG98anWEta8soj3EA14qrkapMgwm/zrCx/Hmgp7 + qEOBAoGAWq8qBylHt6iCe/Y+mp3MkGbFXlOu01Nz1gHucIoAQdZYrIRUQShHGBhi - k1BxY3CJmB8B4i3ctSLcD412NHPSaZutxGjgVJTw03qfKxscJGivMhTUjrPYTOTD + SdXiBD08R2GwJw5LvvhIAf6U9EW0joklODKtl948OANOZMwfBjWDTIUeuWmHzXz0 - jhF611YHyP2NSdUlhHXd5Z/3Ova6PdE26BKiGUqldqzx75/VzHlJ+A== + YU1eZrwemS4Phq6UH9WGffDhW9UgKI7lysZjQDX9FBEIsCpz7kA= -----END RSA PRIVATE KEY----- @@ -3612,62 +3912,62 @@ vector: segmentSize: 131072 total: 3 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:SSK:xpybgom7doy5akhpaomuipjhge:vwvxgsbm43zib2z4ibgn4o7pht3wh3dazpo7kgtn4fm6pauzr4rq + expected: URI:SSK:oq67w7mdvze6t4nr4yxgdvs5ky:wvz2swuip6lw4fk5wmtcfchrdai4vclctok6abjyrnnblenlm75q format: kind: ssk params: format: sdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEowIBAAKCAQEAtct5n8YHDPlic0Drl8bCk1oB8UvQ3S+LgdeQGwFzjew1agnu + MIIEowIBAAKCAQEAuVhfMuw4NVMyvfH43pOdmBQ54Nk24hgAl4ho2iKN5cHjncRh - y8vYFrvqioJxWlxm1Vt/ZwrcPORJ0f2RqMLKxxqTc7sMZOmypCVobwxMpCwXK9DG + 6E1Jx+DraeQXeonpI7YwPG8rwBbvJbksSQAvGZldCrnHBXahcef99uaj5ozx0poU - kAqxLaGOLqC+JnoZ6GFR1/BSegR9IlSpRkz6QJbIquobSC1QKVwXpPy+mhVp3Efn + Jd5EN+3v0VJhKdF/BZ5h6okxG/aBX/9mzjg4amNBNpgCcdudU9NeukSfTKUAONfb - pHFocfbh3HVZy9fvaaFUGCWw6VdTv5qJM5plZLhMojdV0u+A2UDW37rcKiBqIvgj + 2RBkWLmJfv1ypoEVUUSWa5x09qDL45PkGWcnPgKM045pzbCBowPSAymU2XWU/Vso - Ulhz5Bye/+DkvhOss4yMPvJyO6yz9hhJYXApYxqFa3IXXBc2m2kSIrMO+nsuXC5r + ELkYP12JLOgRY51I4rVwAXWa3DUOlE+xqRO4IzndvGceQfc/gL+qAoF5HvpIsOS2 - x47fXtePM9voaxxUUfhej208vk1ErYr3P0IqyQIDAQABAoIBAA47vbxmzaFESsMd + tF6w7nXs9Ykv7qYPU7NknyMGhtbU7dTuPnf46wIDAQABAoIBAACIHF3NB6w/fNnG - kXyHAy+0f7uvzQzuRqjOMvIMV2rklCuG1s+VuIfSI2tACoYxvxrkEKnliagKUyXJ + o+wfiEgzZqcaeGnVn8rPfV2C091g3QJK2e0Mq7WDE6nOGbMkSiLsFt9Vgk6ewjjd - 6bw58RSs5d/NJKunePU9WQt9vefqLE/BxzQapDPfhuFrdCvQykO9j+H9ZsW3IYF9 + ex8KfRgZtIafWc7ONFBbVgMrTjUPGnJ8NqqVJKT0bcev8M9p5m9hsO43TojCAEUC - Oaofh4XkUFaCYQuyAYlVdKP2Jmmrkr9TGMAaeHUtMEn+ttVHW+SaFrjU6ptmDZfQ + Oo1E5ASKHFnHvdoNwznGVXM7VymlHkRvTRIv92BoaOYVWeB6HCaRqdlCpYLht4FO - 0/I1ON2yh4LYn5MRk5TZT5k04eZd6eGHRTuLDUmK4ZhNbo6IUPsUVSEJy88K2eSF + AUnaG7JVdyFG39xXV4FAmSpJGjZLSNgwNnl64CyoilyrcTLAkRgLzD2DnnWQ9V2j - ytdaWx9Ux9Xtt0K3Dmfqyujc75NvtkYWRKlDyNsL4xCa7YTurIs8xzxyVvbARHhv + 6S4dumacT5NhNdZ7ISnZaTPjFi4tSUaBUFBB/d9gLCvk/XTGaIYLf3BtDksY6G6o - dVgqrlECgYEA88FhwHy5kTo9LJHrkkOJdWy2sNMrgOF6fvOC6Dm/MTCBQNaP/cMh + e5fYr6ECgYEAv2jY+i7UsczKZFsYQOt4xDhOHN/kQayXPHLlxFm5Sw5fIQZ0BrmZ - 8lj6gb1dtlvkoxyICXXqJGJCjdCkZrB2p4Y1A+Mvb6mz6CI6a9e/wHq+2ZHOScYH + t9MmP4dFEbBkOGI1b2iwO182cJrIn76mWAPiopEAnE5MLZTFeQdKK3ITYYJitRiK - 1xkGNDoEytaKdENoJyf9aFM4PxaRpwG8eYvOcnpT8go0x5hN3UB27tkCgYEAvu1M + e4HbOIrR8BxTMthjW878oR8LUQIn3C64HZCM9yVxyGx7x6oeoG7Id98CgYEA9+Ol - tr7ihrEJJ/BqRNxIgYLVLkJIUI6JeBLy07H8OnzzIIG58e4n8LQ0bJHnWStFnMhH + rp3tT/6bqepdFAHb4zyewdHajc3P15TAYU4eImSGMZlaMyARjRaIlAD5nzdeykKP - I7SKdTh3IaiFfA2VD9WMW4GQijPLItt43dxzDPRP3RUtOexIJ/meANTc3jMUnqQ4 + CA2vdgVOjsCv9tOVG5w/WUUS/PC+wjm3ZuSit84jghhEHVOunIkA997H9DnY2jcF - KheBC3AKv7qBTGam5SmE7pjSQCplUrVptzKMhXECgYAxcAtPavyIA/PcUkwhAimi + r+ew3O1P/ebnqH7Amf8JLDeoF8RLWhxWpIMM0HUCgYBM5blt4UyP1b9ly/cVdcRB - 80WqX2n3XcPmc6UdTHkGlPviFqJlqWn9KSbFoY6cKc8ZdfPxV0UB1BwDf0mYujmW + yIERNp2ECOuFXH+Uf5tiXPa41NfeL8hiwpB2K0kDT0MkJ8hh5sQORjUfzf9Vtgks - iJXAEBfS4exnLGoE7WEqvLpwji30sIFukti7Rvkp2pGCOxmot2eh/R7vTLiF0shT + CPuO5gRBx27xTPh8pAIXLDA/F1vCd4aDEetZbuPiu+5s2eQo6SIzNL6eH+iVm6ta - LpPUjBLyiDdkM/O26Bg3IQKBgHN5KB2aw3y9FBGQyWUOadfSnkaVFhGKs7/oje7V + LU2EqOVqaLLdxymguIEPLwKBgQCAfyz/WzYM5Xpjle0x1dTZ8i6JYfLc1vcKVT10 - RfzF13IAo8qbxJJDGzXS5L48eqTBSK1iox8UYJD90IXf3Rivim1Jpna/rotNfAOL + Mz5DrcAyLcAoCFOQw0GBFxBOjDFCv0XNcuqlTxLtxMxyMjN04IWmDLxPCayYmbqM - MhZSqP7IsQrISjfLM/HCzDajZEQyhDmI76ZQRGADV/IyX5xYCSsZSIhAW/my+NYw + R7BhfyXA5jtIyHwXAJ3T31PfMa1LUIJOMNfpbcqtXuhu22WTbjSfCyrDRymYSGBR - /2YxAoGBAPGyQ2qwKcbAgGGgSXqRRN73vgYfEijr1axCi7TRsv3C88NlUsnH80nu + Xu1hkQKBgFFuDFlGPsREXJyvgNiMgYdTIyLQRq7nFr8g+kYw03xtgjpcw9j4WYQD - nwesTL65aTLb3lgm/h3mxp0byKkNr465zzmiDgzZWOQk+VLDmZmCrgq13RtpbUaN + 3EkQcurkbHTY/6qUf1oh6vgrY07W40VocEHPuimSfZHzyeXbbUuI2kXyGoJeu6cc - OLqlO3RYMxwsqIwNHACz9DNHhBWtul5DfKon2UqlZiYBoZ5U8HZY + ibZ+MhbfpLh9ynjyNWreSn9BoWQG9beNcw//7Yah6beMRe0P0Yd1 -----END RSA PRIVATE KEY----- @@ -3681,62 +3981,62 @@ vector: segmentSize: 131072 total: 3 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:MDMF:c5xr2pp33zxmelaszjzkwmjc3e:4zbsei7gh5ea735gyq742rrppnjte4jf7cvnolweu2s4tl42piqa + expected: URI:MDMF:7jicgfnv34e5aim6u4agso7gsu:w53hrgczlrz36vuxdwum234dzgfra55yqsmqmx6ighnfarzex3ya format: kind: ssk params: format: mdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEowIBAAKCAQEAswxWC+ueq16cftv8gT15QlxXK8Qxmkjg8o7cujkbm1uMxpgn + MIIEpAIBAAKCAQEAj6EQEncNJm+aeBHyxTcfVnki3Oh+MLhMHYXSrabWXYDln7aX - Ft52fe4hlcnv9l3z2HRDm2E+LoudhOVTPHCyLddoajUQn8vER6P+5uSlS5bNgCqt + Fl5sSGVWDOU77INbWexekm7jJArTRGHEafOBiYF35P72MVQJv7ae05tS9DhM/tVF - 3k9qRYRq1Fw/1x8fFyEbfQdbuIh1OKIA8G/5HlSPpG9ItAcGdji7Z7Of+jXA8wdR + 6rBYb9BURjMeNoAls5BDNX8QwctMZGTF6NTtx2q1mXj6I+acxzZagxcunZDQZIQ/ - 40hswOHIYMaBtyeDH0XN9XLRAPYFJHgJXdOrEd6pO6+2EBLEcvQoCZ5aDvNiAuVG + YqQLLqlBxtY65UlRrpMchzVfqFv9ghYXKGxlC0OzoLVERuTYWMKjsd7YCAZkkjFu - JHuxpzkIhXSJV0JiQJLJ+apJdptW1+nr2Kt6Q6+nPZYwMfsuCwuQc/xcwEJVLDwa + SPBsFTN02ng/9ap6H85RwpuomYdtGjgM1CVRGllbPoY1m/8Hk/MZ049SNeByg6FD - 1rZXd17+jNnCYb5M/exaMemazvhGHZ6g52LyywIDAQABAoIBAEELOjcaYYnf1PpA + YP830YZf3Wwru3xo+WAS3YWIYrKc9TOZHTYcoQIDAQABAoIBAA7aR9heUaX5tjPC - 8HoC2wpAgWpk26Aw2YdEXutH07+cgoeivpCQQHt/BrRjp8jYWL1Jf0XzDaPbFF4y + yLlRN8nlm/p+hlzRFM9GwUsY13kdimFc7JHbxinUZN96LcYJrWmGp8mlHZMNatyF - 8QoD5rbAii4LGP70B1n/OZqndWUAY6cr2f3o27JlaGm9GXQM2j6MyG+jPK7M48iv + mbNKgvOZHQYOwZ8cmCd9oeizBWiLkUFsWkm93l9YX/v5xEmRhenkOgxZrVNvtmDO - EahHBTj/fy89PiwoYTCRa4NAvd0nDmBoLaxZuQtdbmBQkCUtfkCUt12VBRt35uFB + tP1HVlF1Z6h3xLJFYk1i1whQXjndPKMvlDV+nXdvZx9Y9x6GOKG6eHyeQ1lYUmrS - wj06udcPfyBLmWC9qFKo4VFB0KBvEgHPXesy0Q01x964ZUMzz3K/HOt3YiKdAvpI + jUei8O0ohR5uBXNVETDpZFLExOhpElmejowrPvvNc8A1goHkHpx20ddZGKW97aIV - sxXNSvqCKseDA6rLQhAbR2CPFXUmb2xfl6v82FwOL6T73kLjQiXiy2g495M/g82I + wbgPiWA0W/nyWChPwBHHfzv1QXY5n7akiP96KqknvoAQUUhEWYbVl3Ra/4YtjImB - hep+OAECgYEA2N4xlve/msvdBIRT4WatyDE5sBIt4h0Zwu2SMnYBi9eYi0sgbiC1 + p0lfLHsCgYEAtr1XIHTJpQB/jaqKrNCh6ieixDDazjAnlWEaY6Fn3HFRHZehPzfL - EXx+UfhdeKjMOke4AQFRjDrdflnd2jVnVjVI2rsdI1tjX91h4LdMjAJZd0I+WYCR + Ylnu5Md68PqDbb8ERrstAgbJ8lBZQboPBsq93Bqj0R1+BrsYGWuUXj+gFjj1Z3sr - y3hj+8v2+TSg1L7KcerbtoJcg9irOVvqbJ8DI4S/F6tgccqEZNTnQ38CgYEA01sh + Kissmrc43oIOGcBvEuCIkK1QgVyB6o3Y6oud2iB1S7YyPcLLBZPehMcCgYEAyTXL - RxYfILK2YXoM8aEEXCemWpggFkhdnBguZ268Mk4hZ7+ViuYsDaBA+epWiaIG9PgQ + yA17mcirXDuRkFqSuT2ApVRq+ddOf6TYxOcP5RZFEuSG01D4XSJ7iOsELUSGE7O+ - tr6C9fBpw8GbdV+Fg18W12N+jQ1231SOb/azOS6dWzuJ9HritYwlQ38taQFaOggf + 57N4MCdDdP3TRGOWCt8n2vP9jvCHT7Gz5LVuxzw9oPoe4ahh+YsdmkWWUWxgGhXk - CSQQQmjlc0WbO0XPsXoKZ2sN9I5bBbuSdKRUxrUCgYBYKt1WVxrawA73CyVe+fOk + gifqThihSTQFRM0nH/MDpDNy2BGp2iFFVbS6G1cCgYEAlyDPxY/QlB1tYAQC3BlP - 8/5UCtAEoXgbu6I4SamPRPOLjdt9amay2T4x7RtzNozxFL9GCVcx/6yU9cwwLo34 + Tw+olQiybIN2uRutb2g1NSKiKw8T0+yYz6YA5EP1cQY9W632I2j5OAvVSAkbSDhP - imk4I+JQwZLBIqvsRBkmwr3EsnXOxWqAok1jzSR3ZGIOnBKKBcWViaI7KBdUln3T + 5RYXHskJYhA6AecJbzyBX9DO3JIOop5CfIVoRivxZFO6xaFYOwxm5P/w7ItNBmZ5 - 80G/avSVluL64C67H6N12QKBgE8Bd7UM7eHZLBfP+dqw5+JS5/phd00dC/D3kREU + VsBQs+zUFOGBe4J11Q8NoFECgYBcb9Z6xZbvA32WFde91Z5qc5LSYYHz3bI2eekM - 8cCUOCSCFzJuy/Tj/KXvFR4ptRQJTqYhHO82STLlwmjjphLvjqhBBuNPLypYf04X + LIrk1+JL16kJE73GK71NHYsBsOVXz8/4aj7hAGjBKosQdB/ORs7sjAME1AOV3TGj - F/O+GxApd24uKWTX2G4csirYWJPsyT0vf+xzLaIjWN2VQQgEqLLz76mFNT01Wo/D + 9KY76bT2a3IcgVrhZcPnx+hS59MOqNgd43CFCFOwabGx9f3vc5lMqkYsdZDuoTJV - hfUpAoGBAM96jRnTD8ohGUoQ4sZueqPR6zUlEzteKVCbeTtE1gHZAz41aunWeOLA + OX32aQKBgQCr3mKRSwKzmsBEnoyfebbW0WKi8E7A/LeQYUsHY7zcKy9XNPewgqi3 - QCPU/+g/F528QPN1noc3Mlm9IobfkA1nPL1nKNqjfT1ymif38FXYpy7qRWxHz6Te + f4cgFzPF0InVVI9h2k14VW27Yf6wqvpDaPYmdZiX45b4186Z/gSnjMgLthkEo8nF - YPuPFms7f35CqG7w4Qqt9kBWstI2kXOSJQXH/Fm92PkU5eYGEz4U + HFeIFIz6LwA/o4f6cmHIL5+r0GuOSQ1RyQPJaC2NGFqRLP0xjw5HSw== -----END RSA PRIVATE KEY----- @@ -3749,6 +4049,156 @@ vector: required: 1 segmentSize: 131072 total: 3 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:CHK:joatus45ult6nt44xj2awpeu6m:jr5gh7rt44ppdqwwwq7v4feibbygqfbdwxv2siqwijkbnhvdej6a:1:3:4194304 + format: + kind: chk + params: null + sample: + length: 4194304 + seed: 2NVWOXvtwcAFf0J/g0fDC+ypQbnCU7awZoCxcnhpkVM= + zfec: + required: 1 + segmentSize: 131072 + total: 3 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:SSK:zvtntitwrkhj4cgrby3ffhlhxi:vn7utwydkd6e6pwf7iato5o4yplgpvhclc3hbt7zgy5irbdrmvka + format: + kind: ssk + params: + format: sdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEpQIBAAKCAQEAyK3peTMxSvg10hxjiJ7bUENuiH+c8rXiPG0C+aQGCnlzn0IM + + gOSrSX0FtKBdlMgAGdlLJB2R702xhXX9xbED4RnB27lEFCEVCMZjlRcxl1NBivNT + + gSMIkO9+YRHsSQeIOoZ9QVlfCSk6TfSMyRnYAERZjbITtJxW08P0M4UY75plTVrh + + r8tcubELsYGCy7DjyM9oX5gUY0KEqWAIGrxyCS7TSHsBHCor4f6hug7t1pdJ5RWN + + Yn76apmjnXnhZj9LA8AWhqUqM+cS3+GJ9F8AaIZHOF182vDC9u6t4dR41+OqcawD + + CYCJH6wZDYyUxf5LWKJauG0VH8w+EV+7xlgjeQIDAQABAoIBAADFimxI4hujsLFK + + IqfX1IgOelJUC0pzoJcS5DwJgWxw4xztqBamynHR5T+4jiOQUU/IIh3Vb0Y4SkJ1 + + HbCw6Y7oBnLN15EsP2R1PtTH7gzi6RGbtep9M/86rIW7B+mP+dofwkOKEG98lRjP + + o+ryCn6VxLJiyOic4UiXLE02ac4iDBrI2A5S751PzXPPKl6rriVF3i45IrWV4bXD + + iNFOQeq9T4jRrpdhEGlks4u20Xdrk2qZrfYmjlktKoypBwVFkIT/7phuL8sibkpN + + psfnrUhldGFIiF4YE62Wd/2axCqiJ9xX2eNandSlSH/rugwM3haGDDCjlneX7D+Q + + tc8jCjECgYEA1rZB+QWGt5z2f77byK8oJntpGdGxQEihzKvV47+7/5nNfLPmR4BZ + + bHmvus2LpkZ3OYmZ3Ay6hbReJjR8ZGgEknuOxqUPp5Sxo4EdrSYrOmtdgc2ktGwH + + imPGrKuwB+oDlKKpxKrr9f731ggF268NY55ZwrJdeVahW8D32S7rIlcCgYEA70Td + + ImY2fDnD7d2zxG/I7RiPfzlDV29cWuPk+t35HBbEnY9IHh4ye+UlUCcQTWp+tL/1 + + BtdF+xQvk++D6F8opaxOKvELH8GFQe0L52ougNXOle/jzeRNAg7xk2w7BdtDI4Yo + + nKMlqDHbu7QGwXFww8ZWF1N+OxgCaO3ohUlbZq8CgYEAv4Q4mn/kZ3k5oj7C2mHq + + RVEFMYOKQFXJBMAtfAV1EovE77uj5xlEKm7sYYqgSwNFq2vicpZj9YkqBZgBcKob + + kfFmLCflK8yFGtu7dcu6+VP1RygABvLpUvamqzRFQvnokbb6CTOQX486z42+c/LT + + 1YzUcccZe3bbXPVl3jJsh+cCgYEA6f9B+KtXq8PejplcfscIDIARjk2VQ0RAYQ8x + + V/qP7l2B6ck/sVy86JfgFvQtKFj3E5QLcKZF5VgHc7kxGqc9nFDXnX1g8KyUwzWt + + h6M7WXo/8DjMZAZbHaE5toCJdJ/LmElTHGUdpdEk4PweAz8LFhu5BFT+RZKkgLPy + + y69DOTcCgYEAkyRNcRcKTUAJF9ETcURZUydYxmVp61zG870+xkNib88wR4hMrUvh + + lCFlZsUVUuuCeoqHWZaMmwwmgfoj/xMc6r7h6b57Rfe1tTQhvhy4yVC3+Q8nR3qH + + fMB2mBedtoJpcuNnohg4TVmLWS+iVPwq6LLYXmqWs8T0gkehCx/0AFY= + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 4194304 + seed: 2NVWOXvtwcAFf0J/g0fDC+ypQbnCU7awZoCxcnhpkVM= + zfec: + required: 1 + segmentSize: 131072 + total: 3 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:MDMF:zg7tbgey5lcomfxvkbxcy7bdbi:fvkr6u34kfm6lnojyz3bdrpdvvpyzhtusee3lx32blskntogde2q + format: + kind: ssk + params: + format: mdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEogIBAAKCAQEAjrJ3kuOX436Eyzbgusynmu0jgJDkonYVq0aYKRZJS+zSIRR7 + + GJTPJb0VzkxMgKsf4BrVrsbybsZLB+I/JDzKhmrKaTVj3pw9Rz190vqif/JZqyY4 + + g4K7El3P7C+9FhgF4/WfhOTpVmG4Qk7XBjzWOQcBcwlkB5fcSx0dfK8ou4t+pTjw + + pohbW1l/ewQFqj5QgMOvvyxSQJOpQ6fNGv2T5s/7EAur6wVkp/IR0eB7QHMOUwRs + + yQo3SoM9Nb6AL5ai1woc5DgjFYr3q1Zr8FbLk2n4otUQ8Mn9CcFgitZx9ylG2rSn + + gf5FJf57uSLrc1jnOnv3FPe7Bxm9wa8ltI7RMwIDAQABAoIBAAyYLnSUMN/SPVwG + + pzwYrw+KyZZ7c127CJUtR5j5j67E8EdQj7+VXSXqc/Ly1tFU1gN8hHEG7yo2xzPv + + 1Jal5Owk0pU8y87ZWsRsZkWv4fxjKMxLaDNQ329mDa491ZvbGYrLe/C85LLyltKh + + ACdCFfHcYlTdNp5y8u12qUqyPQKca5oJoMXnkDUISKO2t8QY4yZy7oOu3cljiE0G + + NOQvkyU8y6RzL6X3/mWGR2dNNvFMSx+Z2MkNCN22WI1cq76l8S3TwTI6Kb2HweET + + 8brtEG90BQ5mYvyzxXgRhIuzcnIekRoHJT1FgHwNq9EGPAWLjH4FG3wINUUdr+t6 + + haoJIWkCgYEAuJC25X216BEtCXfofu5giyENG2qtWZ8C/kw3MLr/qB2MR1P5WKHB + + k3JeIwFVKu6A8o5obFsz8ZN8fUnEq5PcUCLTWKQw3swGjOLdDcU+ANUeZhC02cUH + + 6EqhlqccqwdOeqrjzDIh+d5OT2NECUDIZyxteZAqOxNMg+3ZFmEE3PkCgYEAxe1U + + aLJdWSoiyrr7ismFmUByJ3bYYu5q9fDqfLIUBH2FHIoUymLARqM0kcpH6obZjskF + + 8AXW3DYW/CkfEZOw8rS0JdEsZV/inWdOIdHfnfr0822QCIKvw5gF5tMN2xosYaHR + + N3SvgLSc4xJGB1OEAS2gKzqGerYPQcBKQ/flBosCgYANXCxhIGBylAu2i7+AsLC8 + + YYAZY/d4bVJCJjI4jNDE0p686w85ozvn+HdoAUiw+uLKrBRTA6cW7Z4tU5Gw+dsQ + + 0fSKjhgbiJlQyXtG6+g5FzREHyF4QhL4da6MwTwKBVVg+83Rki9zbuwsQvtB0Dax + + gT3LduwXqqX3RthYDl3TwQKBgGc2FmEuOc1oUpJDJS6/XaKH379Ckx4r060Cf1Sd + + DCE3TzWNr7/F9RwguSYZRJ1AyqxRmX4LnXph3mSKEQB6crhtkM9zn2IRuTt5hl3O + + cnyQRDG4fXZip8MoQFOY1U3e/SvAVThE6cwE4xbqDYh+fYSbxT0lnMDatWQFIPUG + + jPB3AoGAUgp8W9DPRavRwo90Ijmqs486/0YOB5MKfb/tFu34W1KTCtG1R30JA9oh + + GP4M6PRNYq2+CoWu3vzdVNVRmFV2TZVy/TJD/pF/jcSBOurnlTiafdi5CM+GU7I7 + + VM3I2EaE+TPEB6ovFqbtkhb14SPPYVt5BwK9EREC0n4kMRvth/E= + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 4194304 + seed: 2NVWOXvtwcAFf0J/g0fDC+ypQbnCU7awZoCxcnhpkVM= + zfec: + required: 1 + segmentSize: 131072 + total: 3 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== expected: URI:CHK:mz7pzh24u6flkfmfvvokdwmriu:5mzo57de2ywsdqkdvm7557gcikkpjqbmj65mk235t2xyoejtpbta:1:3:8388607 format: @@ -3762,62 +4212,62 @@ vector: segmentSize: 131072 total: 3 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:SSK:uhmjy5p7og2gg47j7aqgyevfcm:wxz4blxknv2cbocdqh3tpvh4s64rfy7npvw4q3lptmgtxb2kzpbq + expected: URI:SSK:yrwzlwbgia4maufleime3qnzym:ymybiqtqxpjuo7rwkvmxqov2baz3ikpyes565u3b2azlga6k3axq format: kind: ssk params: format: sdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEowIBAAKCAQEAzOwnD57Qk5sgnHKTlvoqACw5UfQNychMYSse/i6zOVKkTbAb + MIIEogIBAAKCAQEArKRDPNNLKhOIsALx23rZMi3gZVwZHDzlNuem1GlsnuCmGjmN - hIx58SmbcP2iJes2V0DVmGbkGrvRBUqPwNBMOQ2B8BWl5W//lEK4hdxtjant0VUa + kTFruy6QdNiecKO134SCK4XFB1pLsOrHlmyiWg7sulwYLAmMWticGZ41aZxANQsK - KA6spt51UbcXQX0wiVE1+4DUNLZ9mgFONIMXgqlvVFe135hAb6ULW//afG4TsCkP + em1AEoqqrtWDKKqQ3vrGdsLYrVFdgXk+KVavoy5RynZTURXeqh1AYiUujTP0EtxJ - /lsADBhu+GAhGsYytnY4A/RDw4E9k3nN9NO9sCvq0y8tleGLSDRW6UkQd4uXMTwL + SxHyQeXgtz0SAMi7RnyY6ohdYOlhAj+6hQMBRkOOdXpMyt718QIBnVM38d9VCJX8 - HHci/ll7v0pTFNjT3j7VgvXHU7ZYBTMAo+mLbmtUbtF87FUf3qi3a/Zy8sirkw0/ + tH6fDIB8rxj/xYEu6V0jdxomYRtzhAK+C8a/JnJEJ4L0N0Dt9GgB826DQzMa5JS9 - PJpMaoyeRokP1dcr/OtdAW3DQaA2zSAg9ObGowIDAQABAoIBAFHA/SxsLcZVo0MH + GXtyucnOmsmFjs+qZh3YJf/BQSZeCKYsAjcZcwIDAQABAoIBAAOFvyN0vNyzv6m2 - Kv6Wu17qRcv+U+nmsSIq8+hwdSwvXkFoOvI8oQGnmc4QQjpihoF06kIs+l/4AkHc + XatDPemSsGM7tlLHq/ZAqBGUbGrEyE7SVsucF7J0LZlcuXAwlEDmahDWRAy1hcQr - J1HDSEWSr/46hL7uWcaqf7dX45Ua8DgNfavxfsvsAF4jb3G/IjgGYEUAdqi5DY79 + KNMDuL+f/Kouv9rXf7LI0aKKLP/QeCo85UQAik/2iVX+NZS0/4tf0CsofZWdCoyI - alfk3OJR+oppm7OiqEJiVA/WGTJ+eO6NZt6MjAyS9TORTf9S+EC4aX/YbsMQZSMP + 5eufq6oSLglfUa2JWzIwXO8ZCsfxu0UfdY+dp+/PWUJT04P1pkGxrWmOjXyCKSd9 - K5Ixn6cnwqdJSX1CPkfVhAQ7KJK5vLFX0DYUicILEo6Llbh++6iWInWimON+toPL + fRvX0UGQBc6OsJZy8SWWoqU+0KiVdS58CIV1WdiRYdad3lNPs0IFbX4wQ6KaX0Pm - nMtH2qAj9ZQTVYr8Br4Xc1upAkNr9epxj0eip4GYyFl8nY0YxFVHzp7HdXQCPHDy + w3Y8Spm1gM/UxKZ4pHuA831Go57bMLr/3aZPeT8wvTaG1KMh2qK0mIyYd8Nvd4NS - nvaoGoECgYEA2KX4+QrMSzDn6Iy2gZG13bHNPNXnje4p5hvqclmP170PTxSath0z + dQeJDjkCgYEAuB9FGZWCPQBtbq9xDdBHm3585IJRUo5WzhEMcnbbqfG7TMYeNP+V - XA5m318t9pXE45oTRdti6+B1d14pQuzeHfgMq7F/vMqw6Oa2xSDcZjtzzHsPEPLA + sngt8B79om4ZdAHQnlmWwNd0AHQpGvi+V4OsO8VEpbtiN99nSf5O0PompWRm+lP/ - cYtXdXJSvgwv5X+j+dFIhPRXKW1Md2Vr3C5ag/QKNduBBU8f6URm7MMCgYEA8iTv + wgXgVG9CVniFodocm4ouwy4sBrzvsOVBBtCBpq9++4iS9xX6FSMPWYsCgYEA8Amo - HwAOg7uS46+lxQyMvi2tt8tk+0b8XDEUgvkmn8dKzhSWEJmZ18o0brDUB2W4D0Ox + u4x8XI6qbrXO6w4LNrB+vjjOV840cgQQCSoWaShyCqC/E5ZolEIFEQmnQXSwpB/n - wN5qcB2Cr92h34hMSgEzuD4sA8dHA+aYlPmTsk889Abc5p20Yy/VEmYVOetfZ2kq + y+XTrPTWazEBhKhhed0Wj+SNAkCSrl1Zv5yPemjmuOUThiR7msDJSFLZhT4HwbEL - 3Wzd+kGCiHerApkMqAFcbUjwIbiHA7d28jfmoKECgYEAvM+JOKJsgWtR8Z4QwMNY + O0anGk+Zfdo0e4y2unOFCvoOujBQuUeeJFairLkCgYApbwABzd9NEveNXPW4AhLb - mKmIkOhrMYrLATx7CsV7Uy311ZnDa8vvIt96UFoHGMxWF3YELfGROLkaJrntg+Ij + Lw+z7I+YYGewX63JZG3yRG+9yyepDYsGan0L+C1jjBs2O+JSgB6ortNv9rP5Wdib - gkLX6Bp9lO+hVpkb2JlW+9H8jc0ByGeHyG0D/9tuuSqt43lmUyZN6XF5NSWIatX9 + oQn1OWNFWHG12RRJVm0uIdzogzuZQaXgZ6of1hm7k8uerJKbffEgAxftPD9EM5L6 - Nps/T5iz/VQcEaBv00BF4zkCgYAqvbo3jpsBRaq35dks3vo413dCafR5Jh6FZ2Rn + kZlbhUqyF/3alJt+fjFKGQKBgBd/2pUB2+rzzJuqEOfSKCbigIX+6bSO63N+ElPT - efMHYPYjSh7y7ynonRiEMVI7vAixKRHHKXtALvVSdZyNCFHu/idS7iZ2xEYUui9U + Wv5Qh1mjAxHX18Ur8XMJjuZJlkF7HiZICcEU9yjnU065bVGQ43SS23ss9y861F59 - nHklkDcCG/QCAPRGTbsedEZq4tEEP9wBGaZU9htEW1skKj/Bp/vYjndUfG3YihnE + 5U4Glw/i3VZ0m+U6mnoKImF5ASllO8RB1nos8MnxYtH1pK38QToh4O85a6235TOJ - x3k+AQKBgFoLVbsfhKHT4DnW74LWb4QhsdVns8/6DzM31yItJzwHN6sn3M4JsGQq + h8OBAoGAW3c51UXU4Ix/QkOUmCreh0QYruqMhdQSGIzPTVeBHm8nUVDqEnnbABNw - 0Ee4oRrLejIvZfSwxmZ/s1R9YzdXkog3d1pEO4UDonKYxEzMYTB6mqJ3YJuj4Qdl + 9dPUFRvkagDCHBM0J/G1SISKNYUStIBVnwOCVc6wQopC7K+SacN8Us7oG4tZwFOe - lGBfFVHS3rukb74jqQC9VO74C2VrsCRA8jX5irtO+NnXKoXO0KQK + C04YAg1FeYDYLuSfEDFxsooV9OV2Hrkc4dQ/27X1WwzgJe/9s1g= -----END RSA PRIVATE KEY----- @@ -3831,62 +4281,62 @@ vector: segmentSize: 131072 total: 3 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:MDMF:nrmxas6u3e43bjtlxsbroc7elu:a2pnhh2akkelw2ywnuwhc4grrahfohcck6adx3fwxikndwemm2mq + expected: URI:MDMF:2fneabhgwzyxbv2toidi2mafga:ultt4xt6hlsliy63zbvdzgp4jzu4fez67lfbg7ogrpzarsharh2a format: kind: ssk params: format: mdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEowIBAAKCAQEAsjzS01v9YJCM1/d+W20MYjPQJi/bTRA0bwLaq8W0M0XRsrRE + MIIEowIBAAKCAQEAkq2D5r8pQqQYCgRMs3aMkVJKrkLY/EFnNLqM4xXj7aDMwv51 - n2ifkiAuHaORaKmAh8iSh6fXkN+PjDSF9Tbuq5K6/KK6IwNL5lp/QE0UDaLYbeNR + CqMNw3A17DsL6jCVAFqCK7prPGpfBhFG3sg/n7oRbrUDn26Ow0IxP/6JAAe2+uc3 - lba/jfB8LCBzCp5C9nv9NMZP4xgEhrrUQFEQAt+sMajDWanasNmjj73K3SdQqgkt + YnF44NnKU5+8hNpyy8GJmtabnNx/+0oaKnrr0R556u//NKoW+1sBB5lM9V8B3GoC - nwSxXdOIJ6HdPhMR0OOj5G3VaObgF8cTAo6F5RVhOiyJhrRT/21WdJI+3F6CwiAb + bozR/kosFoxzjCYSKROg+V8nQRDW/aqqDndD19Mk5x56cND4++/mqWx49nTsAlVb - wh3HPqBFILwtxm8uvQYf5TSClpjfH8RepQQp7JddxCStZp+U23Bx3Cpp3vM485AU + 1RiygnfeP13BrMiUMP998a1iZtnpQyoAMVSqObFh3eJogmpL/lKh80V8PzeerR0K - ngYfc6gKVDvwFSD2mllCcKigMmXgneWti1CyyQIDAQABAoIBAEbRIdb2isKuTDeW + /iryh/zRxXGyj7HWE2qa9JNKo8qm2ijFoPYGEQIDAQABAoIBABaVXbIo66blkgf0 - zy6WMkBmY8J4a0LAOIUO9kEfiUyB5iKBu242zIfrn0cJcUHLbxUEHSwnBOA74zYK + yoR1M8ZqN7Jl+3e3cDcHpAqQw8PCFtTNDeNB4KwfOkYxfy+jJEXmKZWnAbzPGeta - vFrEm6mx3/d21EwLCEIbHMo4lcohNKrckdLRTGSh80Q5FFxYqzRx6RXp4V3SciHx + vKEvjtA+e1Gi0Dh7csKwjdmvDJPzHCK8d+QR2Y0xKzw9425HHhauKUwsT9BSiOf9 - 41k2nAz4P9tvOUbL1OdFYdY/y3V4enGaFADsfzGXP0pOf1hlwx0/BrpgvcInt37s + bFLvEyDjAt1/7zjCGYDA7cMoQNekeficFlaB1I7XsTH2w6w5SlYmSLl8rtkwn1N1 - gT19cv5Lu8tQR7AtYFaM37OacMcdNHgjARfgzzm3YcrTVNie4Fcz3WMJi0toYzGR + kaHgZp+YnotGg7D87J1WOsGR1grnK4E4EjVodCdEmFaSOkpVeCI22W6ZCrKdBNSf - LA7AWVmuWyKLFkRf8Q/eYjydWDP4Qo3frfYuKZkHTugmKsOTKJ5UzxoUX7Yy2kot + h0V3vyEfwrGd+FH5YJME51cD62ifioNnNQjzKvQLlxXCprUe9JFbhEGdRRR3GF8S - WnZKm00CgYEAyeK0Ua4FW+qFjzdJke7v1jrWaHTNflh8KLfO8xqSovXaWHUVknlJ + /WZbJ9sCgYEAvHNMZ6mo+K8gul5VyJw+UNg3fhOK9vTKwkaF/x1GAyNQ/pjkup/e - Z233iZalKSlynHol3nUDCXIdfwS72wL2rvJ+VMYZsfcYIGH7JlP09quU+bFZ/zlj + f6087aJ9Xn4avuv4h/yhbbQSPreCr6lSIX6fGDpxNXcUWH5eKLk/bsOcmneUh9Hj - nOl0LmsfbpHc2go9FIkGBEmlYD50tpwV98ucJTYNxLQicapBsAzNYI8CgYEA4gNn + kvPJ5k/20APSoAsnN+pxiWmlSyoRo21DAtC7piULchyzEsY0a10DLScCgYEAx0EN - 9KNLfJkkLiabROZsGR39OhXNTr9uTuZp6WaimkrwzmZ9vhg2QI8Ckt3iqOOn62Fb + 5tCBYnuKgqm9ooB0PgANWGwkYEBP9lhPtNAIThANI0bHI6yM5p8wzr1exLMN0EXH - h19GG1HjNwfnlzhZZJtE03CZE5DJ1e8gUfKnelWwC3vMfcCibVqRSXBRygrtY2cu + KB0m8c0lw1/1iQMGWYmRljmteORCwdTb+txiZNS6bUY+mzbRx6g5cPzR923EyjGG - /9PqT+NX2qrZ4qd6OdmluxfJh3Vj0rXxMyXNsycCgYB0iN5Zf8AsLJXn85wOFwRu + lu66pL8JcfJQR5BlkqwVJTbM2S5uSTFjP0z0JgcCgYB2lzArxA67gKnd3mOpfPmS - fwwgw7uSwPT6dA+LmL0oQA5HnV5UbJqIj5uh2kmAFyLHXGLbpGOaYjrQhSUC6RUI + Mp6pTm8S/fVi0LKeWrOmYeEkdt7pupVwT3qaKLkwb7cxEpoyKX8E5F7e7Ojm1m0C - K4Xs3WUbq2xL1QMqPrBaavTVpSA0CSaM/t1HpiJAqwX2/o3/epD0jKZfhe3NMxAj + +wXvX0fC148MKWnjwr/yWlMAuePUnPbTkWsq7oNpYB557MrfWz/bs4n7hRvYdnfH - N27ss+UCtJBlWEgOnXU31QKBgDAS8WXD5iaWnG+EnrpFGPEuw9I7GPSLG3eE4zpW + G8gaxBEx3HGsjOKL9dp73QKBgQCh966uNkVGYbg4+HOvGY3kLDSs8NMs7npRfH3m - LngLQLVmb5CjrcaFpNKAh9nMsscKamGdDlh5To9CCyzLO5h+vmELLkRPI99xgbps + M8jcc83KJCmSRRwQB80r8OGNMSOEboQyhmf23FTbGTDFHBFYTSxsGhx6Dcp6N2ZN - ltsaptuKdbC57NK91PF+BqenM19Vb1XTSZ+8h89nT/k6DnGHrgzhvmglvBnxwWBT + 6EGPRyD33MbdctVZ39Q5lTm0UKVL2rBWFl7ftm6eEmPRmH4ImRtjMcWYsVZy5tOP - xjE5AoGBAKPPRpspLyJE3pwZoskhlrTu1bZmIWoYqEOZJKIqJvgH9bzG125egzHm + TCWWlwKBgGMiL5S2zJG2SSo4iDqZW9jbnxnOAbAoMh1X6CDlVRaj5XYe+FqXFEQ4 - 54miHH0lerkxnPOZcJIYgimVjwwx4THC1wRJ7p1q7Xa1N3GSsk0zF4Bp0h21aS21 + aK4VpalIw8WCzgTJPdJLJi/KJiWywco4lvX47tCs99PwYnP1hT541XRKjcel1pCN - tunj+RR7TE+8q5u2ZQsnpXg6msfAAxUmYhE4xLKeR01yZXlgFhaZ + GlE/aY2p8fIYY/yDDSOLkssn2Zkye/mc1jteKlXD38T6/ecimkm8 -----END RSA PRIVATE KEY----- @@ -3912,62 +4362,62 @@ vector: segmentSize: 131072 total: 3 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:SSK:m6aebevnsdiwaxdtu575uesj5m:ulkkl3gmdo7ww3gko7obbbweo3k75xvyriwe52cqe3g5nbeotxaq + expected: URI:SSK:an3ntbx3uwl5pzmgz33s36tfoa:5eaytstw6x77wsov5jgvxt5vhcneykoe4myucajylkgznt76vecq format: kind: ssk params: format: sdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEowIBAAKCAQEAr0tGN3hBcT0GxrLDM7v8GwM4wuVo0tOv3p5uIpdH9HtHg5i9 + MIIEpAIBAAKCAQEAsXGGPkzcQLANVVzczL+QBOxxyJv5zk4UiSmqIdJavHiGYDeI - rz1sy3MTGHrEDtDt/fABSxSwu1R7U2CeILzg/K01Pcn8AekmMHno8DlDds51ZKx6 + vp31Od0jdn1eTpbyixEDOrSKHRYwsDRavQ2mRA2NOJkWe0jYPIzFV9tvFBqjnUcJ - rIpf8kd7N8sKpxMeWpVSM2P/wr3c0TH1KeMwT6xSNrTNVaj5z9ejSMhRd0nlDNCx + NaOB7mKwFMHKUsIjepeCaFey98Nqvfm4Ba4s0MmvA/DPpoZEGyjK2AoUdgzlzfFx - +j5uXsL3ZTGTbiq2lREW3eF3pVouy/7wK/MG3A3XI9mOHpN9UsgJI2j8Wcbn5xVj + sC+XO4aIFKRcDxJX2f6lAOsKnUvFSgGw9qmEnUFI9kcw31Oo4Nt1Tkfix6ioEv88 - nBYxOxZQ0IMw4R4tRUw7EaIXtDI/lV3/yp6Pq7UTKllkqUPWZ0mnMd63gRAE0kJF + y0PiJlMDIzbvOJO53dUCrMiAnv3CzpCIx5USKaIzoPu7fHSUmMIsetvKHoRHjs2E - pIUd3QqxpujOF5jXmsuq4QL5Ei3eTWEfSjiXTwIDAQABAoIBAA9IVG6A7nutOApa + WqpwBsERKvOxte9SFUDiV3my5ssa5rq6DEZ5yQIDAQABAoIBACUVxenFZI78fHzn - Csis3/WJmgjr2pq0LUYPBp0UwVzgOUOQ18tlDeIi6IxJemRW9MN0iY2pLSCijz9H + wJnmO3Jb/FfiCW6NsQsNsyoIbcBQLD11vdWg67yhNCUyhIBGWgComJUvYGI93gUl - /Mvv4PxGgY4fLk21rPMBqJGg/G++34nlEQDlQ34qReTUWoZnFU4NF1CMVv+nG+RP + nAVBEgvNDUPT8vfnPJJDFYeMLAX2n1VioFEekCxDYeukqOVs/79CZRXrplLT+74i - zbGsMUm7mqNTBKqDg9wJ+cizpjPsY/iXH7NF3J9y1I/MzWcop+RsR+Ep2A55MZS/ + r8w4H9OvIy3eqXdzPk6y0toeGTKmm7WTWheMXTfiKpsn3noJBj1+OyjzfkEEV0Rt - OeaVfgItESWSqWYW48mtQM+D7p88CCbGl5vFLb/zRktQ7PpCWGnNKJbLc0476hSW + k2MDwupzLgFg9N2i+CEutoobxgQf2SUys6SM3Nf//+mtBd3pciQWusOKW7rzmX5G - hS3dsQTGH2Cdu7TrggIxknR3SCS3xeoyKkIbSxBOTBISiWdffNdplF+W1GZrhZ/m + kO9GWTkG8oJko2OECRZExBpPdw+3/lb8HXnqLC42Zc3fECSeWLo0evilT/k9jjjw - u9a/h+ECgYEAxWuGk54Kth/9JNA6U6OM08OPbddPa/iAFzNmP8N3/yF3SOPfzeCm + I2TC+BUCgYEA7/P66VQaCuskhT/9NDIVB1Mwhpqxvy0SO3yyASEIkCXGk1m58RsA - xlUheBYQ0uLE7BgkhapuIYCQ2e6BvPE9NI2IN3sU6HFOtQNOd7NfAsGwIhVaQmhp + bPV79oo8bD7nUkbQBS+ChTPpwaiPubg0pyX+TTAkAYNtlmP9bQl7LYR0tWDjIuol - s1ktQB1dOIudvdlz3zX/E4EOXHpyUsYQbB6V0F4k4gC50L445NHIAr8CgYEA408C + Zxa+HbAvmNclzQlGuk+bgevT2Hq/0/E4OZP+XcauaQfAmfi2FXOhuC8CgYEAvU9f - NW6L/WzILo2OhCaGF6PIaJONy1I6+Qbif5v1bOm/0kehgGf38455GQPulKVDCbTc + tXFAbYbNNYM19LW7DXTVxQAk7+vn0b63ltpysqxVYGRxxReEtiHIgIapBO/aYABo - azKLH8NRIuRe18io+uvwDfwjJFfRWfMgOXDkxBuY1fT4m5IPCiwDux4Lz6Msab3q + ZMJg6gX2zGjcfBOzWKpnIYx4jP0cUrvNHm/by3/2WNOAzE2dYpUZ7+UzvujtI9Y0 - Bj0MnJcoHrvyD+Fn+LbuQAQ5mNd6tkKowVs133ECgYEAhgc7BUr9gKn1BaIshw35 + JrCpB3q0B2885LVchDTU5XjYu+PVo5dFIwAH94cCgYEA0UsCwL/dk0Z0bVFZ/kvs - FOemn27Wp7m81IN7vnxpIhfJUP4LukzzTKENKObqIxH7mUHGwcx0GmCbdqk7AVhS + sZ3rBo0pmnGqpH5oGLoCaRC5+s5ZdCa0IVWhkXITr+rSA57GVK+S7bJRItxuuHQm - MjSILwprpmcOhUuqYQ+wyEFQ38LZVU5nvHAljWqiGDqJLBPOW9LfypEKe/RRWyrG + YOCvxg3GaheD35hJdPC/Iv2UepwOoeaPRzK8EtMZQPvv+b4slddX8WOMPRcb+LY+ - iXC2SxEvPxQ5EqOiIo7dmCcCgYAmiZOfSXG0cofx1JAP+ZQMV/k3OaT1jqhu5erq + 72HZjVv9xpi/cs1PrLhWB3sCgYAUpRJP7DDVgOziGBQLQsJKXmJtoG1myLg4NG87 - pZ9TasHZvck0wuu3wDTpt8/wJaCa+a3RAs2xgeS0nLEztlJn0C5vwIqYs8bLkDur + AUme2JJa97k8gCsV3atK8OR/yFRtQb4gtt3wx4O5mPnqgg997N9gVjxTS8sJ7rcY - YWd3lBIyXAj2HyormFC9nZd1CX4TI16U1i7YMYxcwZKFfLqq4SC9e7nkHswwMFb6 + yaQTljncR/x0y0YNmSsB8WHqQOaTkOmRCpT0XtpBMU1Xt7uGI2jQOZSRMPB8baO5 - CSO2EQKBgG2YRfM+7nZ0s9i0v8OgAEwL6q/cQISuEIaGdWQXxhWW4Ob5u+3KsbgU + hGhaAwKBgQC5GJv3sLa+21PIlXp6UuIGMFp39iwfAKchhnhXP6rPkC2pP2WB11xa - nLNbAaycF/abhB5uf6IkDufzeL8yIeu6d/+nq40t77bM7P60tncjhoMiJtOVOKxJ + W9MuEko++3e7c7+adAoPFdsjf+a15CjogEURZwskiRpBYyBugM/y90sWM7ddOsJL - 283ZwYz8hzRvDiH+FGyq/+cb7b2FI4/RW6TzDBMCNxX3RrgqC88Z + 3GD4Y0aRJ1+N3R+l0MAv8JMAo86g2Xl5J423ZfXxnosG9JvtvPGndg== -----END RSA PRIVATE KEY----- @@ -3981,62 +4431,62 @@ vector: segmentSize: 131072 total: 3 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:MDMF:3yhl2op3nrsrbevgpgb6f642me:twhn55j2i6br3mpinhj4ipktkk5bi2enitakg77i4h3qkihnusaa + expected: URI:MDMF:5hiosdrt4b4h3x5fx7v7cxmyqy:jkudr2qefit3qow4xs4eldwclbiv7m32w2cvu3igrm42zcak7noq format: kind: ssk params: format: mdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEpAIBAAKCAQEAxotYVkVtYuRGxjNB4HJsn6tz8UMc2aCck4Sv5fBK5tpsfX8L + MIIEpAIBAAKCAQEAsU9bSCK68rb90LSqN0IBMqtLfLkVWlUdKXD4J6tTtFiFWuoB - 0MyGHo9sGTX3uIgUYvrXjkVZQxA4NaCIoCaB2rUOhPbSYuySwc1ZwBleZEFKfHf+ + Z6v0kpepHIIKVfWQc74ms5wFX/M5TzPhjuzf+IXMMBC2d9Y4p0iBwIj2S0RIPxH6 - +fm3dV5eKWg7m4tzXqpUIBPD+0f27Ch+xvy2fkkIJPv7vctCDCsltl2G5lOjhNF9 + d4iapNToglT8eyoF0I7r4S6iV7t3Sr6q4DqtPlQmgCSvKbDuNYuP7eygEyR94Q2+ - xmj2dITiZDVCMnbIzX/E2R/IRUSA0sWWswJ1MbvbMPKZfn0DMGxuQt5hfVEcA6hA + 3DcfwZ59W1u21humiftuoy46PSDMR5/RFY/0KfMaIeGnqO0IeWnqdSsJlyhy9jrq - dock7p3UFgwz5SLdhz/DRz6S/Wv5KAyVMuJaEr0PnhNfuRLh+Gf572NK9dFgg3F/ + Sa27VycC6p5FnpzqkTYlY9WsZn6AXvrAbVX7wmNT+3E7LfO05GIBNDKCK10Ruxiw - 3MnU+L/VpkjrmSnydAWZetqA0cHgR0h0myK7yQIDAQABAoIBABQp+f1eBPVYjXsK + lppkD0oGdei+cOLTS+L/MYS2xpTZcPHQybQopwIDAQABAoIBACxz30g+AM2iCA8/ - 4N+Cg3dmXmqz6BHOouRFAyfBQxwltgO7U8f/g4YGRyJa1acpM9fXFa4Ca0W5N5oz + hB83apJ39IRv6IUNqrJ9kpFreCBSQxiwayrBJx9ra2fsyEeVuaHy7cQA3S4Zjegn - rPiF3KWJgplcp2KKV//Egx9UPD7GlkHknnHFC52pCTtflYEvNRb5ybwtwbdLGF/k + 8yhAhcRKUw9H2V9A81IpMPKCw/DJzS0Wxksakd66TBKE2QnN1shbVJLPfL7vDnDM - 33ZGW2LhOCloaYldo54wgh6eqkkJKvyZPjfIbQoFwc+17Bd6r/HTz1cBj5UoL0qV + TY3K2C00rrRYvht/MrF07GAzf0xc4kt4dZ87/o0tsryZogpJRzQvwb7T8fiMNMyE - QyVRX9K8NqPUAJIDiGRffJWgTPUOnuuqxSK45CgzMhqoAvQr1Np2DFo+Hc9eSZgu + wYYQCA5Tw8/iYKTjWMgevzi6qy/AiLBTT+KAcbaQpPF6IeRoCfr8BsTRz/Zvp3gb - WtZGnvUY8pRsOrQUxgN6MH8BGUNvdjpK6+Mnhgo+TzdpGviAU3DpBqC8vlTfm88s + UUOWC7tdOKNtE2M19ado9l7JgoXNQ3p6bIydDFQSEg4bQxElvtz7SyukLnM3N3PF - tQn8P6ECgYEA4oLKTu/U85TVwSSeSBh5bIXPcLosrsQQIENlPNOjjQMTAnmGqibO + VZIm5wkCgYEA3wgwDFsIAm5f9n8r3SR2sEJgPb3iF2/x++DxI9SJRodM1qT6RAXD - YHssf+8W0xIzJxA7aqH6mOjEQUDfv7YrNtzO388aqew3/rNdy/azjq3GARKOCvFI + rNUqLSXlUUaQvUcDvIcwlP0hpETuBOjQwgCd8QEWLPRllYcWgCkBtoe0E5GyUEGw - 0GCU5wHtnG4vaVbpDmfWWlNAy9MLGdX4P9wH8y9BJjnsdxsLc4XlI+ECgYEA4GR6 + j3sDZ4EdPGQUaJpWFR7ZKEGNVjFDcRq3cEGQUkJWP1uxrVBQU7ohtVkCgYEAy4T9 - UTLLp1rUMJnT4UN0qK4QxqFPx9iDZSQldhsGYXHbRUPXhI0qL0487/ZeZHnazDt4 + nvCfrmUmhKSawXWTVSDVhMoBNAyt+KCOaIevHscCj97PfuJNhLCOrpYQuShMGtpF - 19Sib8dHB7/O55KlUlXshyEjzD2FsV58t5XjVQrce3EDZCTOb3NYSMC76wA0BQoY + 5DqohfqBVosXRmGFqjxJbpsQQaQCL7zKy0e2kojgoIdSkmNxKoQup2oew6SWuUoM - CjihmszeP5ny/aYMLNPKkftfKGer7BjZDn+3lOkCgYEAwORkaGhwzqXWik4mxHqj + /08QaILHjd7t795xIFPimcFgVtD8Uy5OaA7jDf8CgYEAu9KyXAVhZmK7T+Pi9bYa - HLmu9+5zkrjAitkZ43zPcIxHqfnXphq58Quzz5bJtyFukjuOfbZG8+R1DKS0Zkw5 + ee88C2LYfzJYD+1sRedbv9h7fhurYxOTqP5PKXxLdTm+9JdUbzVOVXojFaqy49GI - 7NSJD6sMp9vTq4EPxVvneP+e+NbWQ5dKTLmS1E6eDHMAyRIMEgp3TiBLs8ebUnsW + 1IgeenKW0T70OYttCHsAJU58+Snuh6X6YaqPwF+8VjpV8Y1fxyOWb09dDmQoTpzY - lztHQd7h+i2lo6BSViSWB8ECgYEAo1WAE4q94tuiiJ3wNJA9YmsRmwPgZr+bJQvi + NKISPyP8bBj1NWZ4bzpF3ukCgYEAmJDcs458lfab1mmy3X3vcayIg+AO4N70d+Kc - mM2jH1sZGJoBTmLSygxRHvpeSxTHxtGjbLdCZcrQUTu1B6se24ff25yrygceQbVd + fv3gKHlVkVadQ+gP5n2YqIY0iSkNTD/+juXuOWmeFat1SjyHQCOrhK/Xku2I+hJU - YuSfzU9SniftKAACo+153bstDinfs6tdRFNkjqGBRRpyXV94jUi8svYelfKgmgKc + D+l1kwnrIkvveW/0gMPQWY4y+8ThfItnjOjPSxlm0RKiWePt+CcKQur09n/5971J - PImKv8ECgYBJ+vxGJiBWnxjV1iIXHfp7E/TfXRIEj88avL8w1MELyM9eUPI/drJL + 57XpPi8CgYAK/ypz0j0gYUgwvYr+URad1OtaF2rgsYNt5XwoJjgPNIBNH0i0K971 - 5v4zHPDOmWcy5I5ivmkbnNASHBszag9wXb8/6srJwHzRpyUvjy8g3XiqxhrC8LKm + gIiprx1i9ew6xWyuKp3E3dtClZ28m1txgOrcMuP6L3rEXU3lA6E3ftDY+l4wKYK6 - 74w15W4Yual2jdWa8DbyD411kkFXlFxUNAfk3vef2gl21t11UoceGw== + O9zVS5GSIqjhUgEJQU+M6zo31L2hKcOYY95zO8/JyWfT+fgBC6mW/w== -----END RSA PRIVATE KEY----- @@ -4062,62 +4512,62 @@ vector: segmentSize: 131072 total: 3 - convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:SSK:vauvfigv7juzb5ja74dqsaxzg4:2wuk3rewcugoewhacjaa3zornswzz6wrg6rv4mdrfsajbf6mckqa + expected: URI:SSK:p6slmwqqxjwjrnch7c2myu3j5i:kcvumf4itbihkvpkg46pwcdcajyr3wuglmiyw4gupqshdlnqp5ha format: kind: ssk params: format: sdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEowIBAAKCAQEAmh7Awenbu4CQjcBW0LVpPTnI1YznrSBeitZc/88XgF1+C9jH + MIIEoQIBAAKCAQEA2wplwURV6PZJmwtqdq54eKXN3yhOiCaDEhdGyfEK9qUtrOAj - Gk6apQtQE5tqFs+8B0ss8j6avgAXJMUL5MmBYMsb8qtF2w+30CNcyBPOywNFoVfh + d36v7ejCM1pxunmjqM0P/zMAkpuo2ezmTNri6OVwITtFrKYqhvzsPxGmysJlByhD - 8f9E/G45mfC3gxtTI7LXAf8+oEz6H66PpHhPmicjb++CNlr41Nb2YA4JIUieb1R5 + 2V2Qjhngbc++q9ly8ArD0UAaq8p/c6M41YbIkafbtzrUUYMdx6kH5OP0T3fppNO5 - jUkm1K10tcgvmOKSrU2N84qQq1zas241WiPvpXVpYAzr/VHTc+DMvdqk4imDn+dm + F5kwnxDXEIwSCuXuyF5CZajk6RO/iMB/IeYhiGN8Hr8oZ1pgknLO57OlbnAL2XD6 - ntkCJyV/BAk2ZVqmk4hlSWwrbKilK+2rH9rlMqGGN1NDLmhOUa+w2Q1ZhiTlvzh1 + a6qd4RnSCWSzWHRyK0N1/AiuvINBBN5b+l8tdz0BnrPYL8EVhA5XZ3Nlz+9y4Z6v - xqaeCe69ZU2UsBoPyHYTogcf0CtcPrhV7hk4JwIDAQABAoIBAAmifIhi10K8gczq + p7vfiheAqTUUDcSBR3pf5G/AWkhIH4FSg+wa8QIDAQABAoIBAFl2Oq8gCPKYPOHd - xkKb5K1YLG71NRKEoIRrbDrttllm/tc8wQ2q9k31DBd9sr8kU2vdTj0Cnufb15aL + XMNSaRPlrFr7rG+BQ0FNTnVGRNMODcSw2uuAS7ygt0igJRkje0uDTYhOvWojt2gi - 3vd5hWYIrIGaJW7RZ7tSSp2TZ20XkkXI2a4oOCbTuTQfcUl37tWfe4N7cm3RAh3y + kMFNGSZEJ3L7MW7dgzsU7CyqOfRQR1EQCf4qb9MKEJbpJZgsvPv7eZTqWLpXf4ys - 6rXsc4V+ht+biHdfbojXu2U722RCLgUt55U+aE4tUQtZ15A2TbcI6Q/LxeqGgHGX + WpcjcKHE7EE+/t637Z7Rk87Rp8QYlVS9Eu+bXFl7ZF15k37uoPA4Or4RZD0Btxnj - ZJ6c63EwrYmAlmVUB0VuaXk77HC7qxdXADe2LT10PYGQpGSa4CHJxCf8jCffsTFa + +QoSi3UacdvU0bkmoD+H/XBFnyEPNPKnWtxV2lQmp7zLQ+2jvUa1TKyhQyFuLb9U - pOBd7MEO/tX+HwfrQmoGwJGFcJkQutWytz8v6ceLeqDXhnsXKifQuGV+mAM07dW+ + QhqGikpWnVB9YTskw5Z57YJ3w8pdnUbZokpblT1MFOc0qUNlstjXFZ3+q7s/ZNam - 3Dfp7UECgYEAuBIXIMsXn0c2kdecPbZGBDgOCA/LH5EqwKidV47dmvc2hJ48Drrv + QYiXwc0CgYEA6A7yRw67Yy/UWScJYqA+c54RuxmQhalk2jockY6Ur04EHA5TRBzb - bqtSXGB/XH36i0m663ax+KOjtgoRTBHd4Xly4ZnRAG7Ik6DKxH9rAzrUylwGSg4i + VKhvX0Po6fu6NNg290BT+jFCZNsgHiE1sIL6M0rxPJEDsNUe580tL4C+tncsqTc3 - UTpINmk24m+G//sSNKF7idTh6hv6e6oVZ+kIGd1YF6CxsKp+4QhhHOECgYEA1lh+ + wdULPGYh0IYgZ9mPWuuqX8TAels+P73FFNzH29uRDNpDIYUAE+AcDKsCgYEA8aOh - 2HfWVHdOUdyN3SMKqL4w9UF8Oejef8L39/mKz4TPkBQ5uFYKYLF083rTFhzRsPqA + PFrYqE76tkYvtUZFgjI8R5Q045eAG4eET2BsJNRuP2aAK/bJ5ngwlo2kHlLtBi93 - vPijsn7aZBNj9D7pwn+gy98M8Zq4svBPMCuqXGBqM7QAHY1ahno+o3jlxDm2ddRM + Ogp/k5wXi3JQ2y1vhjGQAu3cqIP9vVVYDZhQzx6WKy5P1tVcKw2r/KyDDiqqGcjJ - b6PQmmVVq/alzrDPpz/J8KHRXe1OvZhOFC9hLgcCgYEAnY6Wj3Jn8OWC90lIKqa3 + A9efnVp/S/nESeHjOfiUhMUzpsCExa5vdDC0/tMCgYBMQ204EQ1gYX2l9wBMm2Eq - verBT/M82fNnVeu+anEWjQvodZIANFeclO0+nWXX/rKy38Enp189LWfcvPhXH/b3 + 2g31sUcfxjXQyjxNUdBndHpBRivzPJCQV/KSGl1XWFUvvMcDpu5yUPIC90is3jko - JoXPaP5BoQ4yz/LFPXcXgXc9J02n8IGyrDaoEzLyUNZIBxrA1Z4X4b3/9mUmfe3z + 00KqzLxPLVFLMh9ACtwIuoTyrmPNEMqQNxXEOcRvJUVNG+DS/pQ1eRHQpF/mztUQ - TrNwRLtrKSZakq8N1c9XWOECgYBiEbFPl1zP3ppN6AxcTikVVZeOzvxofnw2llzf + MCa2iIg48xoQ0AbggUx2lQJ/Uh5JrTkyaABvM4Kms/QtqxFnauvzDWVvI+vqCw+5 - 7yOsmMZi1G4oQe2Tmf25XMvxhRQH1kVKsLQs+c8wFJMZ8CMB42UNgiso67Jv5HVG + sMqArQsog8ha1PgDiyaXn7aO1otK+W6X7JIfbkRrNhE61WACkPxFAP/aO33FbtlU - w+O5Sj+tEkEvRDpT5uB76NevdPxfYtfqCFhsG8sb18i7DbikfBIH7/Gb+PSa2HF4 + nQ7H+eTDPT9FE1ySFkyKPUZCiICzz5p3pAIdQLShAHrDve+8iWJ7KzBB7uxY9COZ - 2MisxwKBgAeQ1tkpSHFsm0VilBRknflbe1HRBAaebq3sYVeUR5zLqA6MZGgTXGU/ + XwKBgQCRCG2U7oHU+R6eznmLIUscowU039DUNBxe/loZDOpx/fN1aZPJDlaII+Sl - WB5sBZANJGRj3mLry6QNH8e4+L3+2yObkXH1JsVG8KdYrOV3pKmTTwrfnU9sRqgA + zkcOqXR9ILw/NDTlfpeptYJ+ahBgP6DjX5saA2T/uf8wgr/B+Aox0j9iBwfUahqn - zVP6zTIlJDnL7sQUOpOmD6pvzgQlQBN+zkKwnmZS5SZTX8cTEFqi + YVukY/sfdXp1xdWsllk0wc2HcoW+L90K7qHi0ihnR7oK4s9e9Q== -----END RSA PRIVATE KEY----- @@ -4131,62 +4581,62 @@ vector: segmentSize: 131072 total: 3 - convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:MDMF:ez7wfw7xb5vsy3shceefdlui24:4higxbbkczkgpcisvsj3u25ybbw3smyek42pib4h3fjkcolnba3q + expected: URI:MDMF:frye2ctqdffyf437do4uq4ngpi:lloe5ajvtkpmyflag7phiacafx4pfwt2dnriayshfj5hvno5ltmq format: kind: ssk params: format: mdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEowIBAAKCAQEA76JNQfiGWQy7HAqPN7YjkBenSybWjE2v8uFuz/Qufb/EMysc + MIIEpAIBAAKCAQEAuwzavy62iNdYkMdbRNLSzYURMLaw3ipteYaD93gk9JfkDt3F - FODyFVmToK0++qOzYa4NytdsXGfb1HO4wEovJhUivW12lzIeB5niC+I4hCF61Rs+ + W1Fz65n0IQN4fRol8po48mzPpi2C3Uq1N9FJrs62fx5pdMTzMolXWJbNY4xAXJ6s - XqyxGrgVQ2er2JNYRireqKhJ6VSttaUqyA1EA6/sp/xQQjRSCdqMK8E006379Qy4 + z6ciaze0VpfH5TMd4ruRZQ9hLPcGT0zODgNnGxS0l67+VVUhmeNXGqNKUwnYtUEh - GSbiJBbJVwVuChWA7G8xpDACMMNC8BjS++33mJCgm7PW2uxmKwSz5OEuPHXrcMSF + 0RMym/Mi/LmGXNk1hBtWhC6R4IhKhZgSVuD4yr0FEAlxUOsCmritJROknA8boSzu - q7hPahaSXao/SaxUvQAWBw40SsGwiH64auoomNKchDzEAT04bwPx735OO0DckYXp + UFrUrrFMpo2VBcwD7kfm02tejDyXfe9aLn9kLIrbSG3YmTAQM/0NkVsu4PUr4iPg - hpR6y9s3FwrTl5r85BzPRvyUVov7CLfMnsjfUQIDAQABAoIBABL16xoAsZSjMrzd + ULPeyUzoDhYFOnEAZf8aXRKQL529SRN4YTayIQIDAQABAoIBAAJTqlvy+Y/Rt/cT - wwY79aVlkbmbCZfRX84eczemEPWnMj2AODkYsV7qFwm8G4MWZ8+fR30YvXy0RQsS + F8pPIhKu61QTDbexyOtYVkdrjfAh/JMHxLb6WCoP3/bSK3tI6jxumTNA0cN0MPrO - 2vfwBroDKxwE6MC+2OxuCxo4nJMr2P26qZ0xGdRM43XRYqIAypfGtZZvtmVta083 + PVtcpeFADoqdxvuOIKVaCoVeWN86ZSmRyr4JivbQ+lQSbsjl3iMOKMScUJ3l08UE - keKBVjPafCWwi6MpY6Je9f9SSr1C3D4duSMLfAwCdCntxWbWF20gYuoJWvNb3P5Q + RFLtzWhLlWSGp4DJvpGCv7hj3B6UzRIpPiisYUecyoiA8+YxueCRwE/GjdEBtAqm - eMj1uL/2z8b9uRRGhlbmWvbJNXWyEdSJFDmlBCS3fHwqPVos1yBsdKK4LemEospH + bnucpTpEgCE6idbD/8/zzHPk8Y9wLfivGi9T3M/kRWbhv2x9qrmaSFTxp2eARISx - HREWCNowBktA1C1AWjSvvGu8Qjdn7xaEyM4P7EsVN2Au47e5bEp8ghoyGmcuU9E/ + ohWmvlvl1XTF8ghvOifsIGOGARXk89AldWbohjB/zs60x3+Ein99WJTHSvPKBAN4 - Y+O8DBcCgYEA+OAvhOECk0nW33/cf/qOuUw/wiBsxBfLL40d7PVT71OYQCrR2dGI + 54ftTAECgYEA6M/nz+42MLOSrQnotL8L6EqAI6vB9h/Rr01eRmSy/zB3/o9/PpJZ - nRRJK+fKe4CsGqNp/kF3Zef5o8lm6Yp5/ZesazU4vVhnw3vFvnnt75WYKYzdApHy + 7gYLpdF+PQxU+ZpgYL5qv0gPppLncWe9M1ET2gqImr2nS6KOpQ3Kban8OdGtSoL9 - 32hWK1SAgvOUtNcBnd3I/UxdVm8o/dhY1W3BDfhN40Ngn12byaMVcEcCgYEA9n5k + qoSdrQPpOM9awK4y+Heb2O+VsVykcJAOzH/wOz1sDH6GNe9Yldw4QYECgYEAza4k - CZam1WgPeXWDKLrPgt6SUY1HCEAtQLkH7UlMUOfoXpMeqKQ0PO9/7PCp+g0MHbzY + Eo9m39btQBS7z3Ne5zwT+XNusviF60LA8B9bepx5f3uKkcuSE5NOdjD/AYtBbEFW - KKQs7asbao985c7i5EbxUopO4TonwJ9TrhOvrhQxRHji1Q4CLFiChhUSFjzgyECb + 6tqyGbee90emW2/ei8KlqkUpTwYH+9+59QTQR2cZUFYMDsqsXQCxKJxk+VG6ikJg - lNsKatxzta9/Gl0VFHwSBNuk0Cm4KCxlJOr316cCgYBfvf8J43YWK4XaHVo6ca2O + /leB+M+/H+w3MD6qG2VcfUTmlE8NEjoebSGVgKECgYEA3IG25WmRaBVdcom1ICTn - Y2Lzz32IQo8MEAG/MvHDVClyJgbtAMrJgxBTL6yZrnqHFO6lvZGtRnynIcfReFBN + aU/PCHoxDyZaG3jjNzc/lpbYwII3mhNSHDEbrSW8NKROg89lQ5x3TM87C6GOlwoT - 2ped9q+JSAVDEs6T5FxAmxAai/JKFtOUVpMvwCZgOkyu9TfN/5BewY32vnTKkvw5 + 2NwNOnLJqg7ButCv3MMwHShonnbrdGyXSH+tPGc86bL0GRWlb1MSiKl8Fe5STc8U - vytRsIBmOXlmVaClBXQt6QKBgQCysggl52CFP45QWD/AjEWZs29RzeDb+2KTFFDJ + RTtUVTe69CaOhd06AU8A9YECgYBaTC2CCHr9onoeO/wII3pywilyxn6/C+SfWHsj - 1iSMVsNfpLpKOdhhAKO2Cva+/yx0do4iUHr9xdj3VJSQKX7VTRTv6LKslzNwclEA + 8GBVAAVHNpGrWFgVSAKWWQRbRSu/vx/Nk53FNJwRq98ZHY/yg83/ZsWv79HpfltB - 1ua6hYr9/8E6AZDTw0rEl4voMTQoGKZxsKYJuE3uPg8f9rEsi5Goke8Wtdf6z8x3 + eo+GCXlPj7dTdx8c5YThc2fRHVRsBqBWiUyCU5JxDV9dSuWbiXCFfo5MZjgy3Fkn - ihwo6wKBgG0bYUcYJGUEilGGqHicZ7g4Wvd+inwTJMwYy1+vFKQbTRF3zy0TgGqg + SCs+gQKBgQC2d3WE8uxl7F5LVyM0+ckCPzBQX1cdSaslQ/PmzIMkjmpm2nUVnhGF - Jnv4bW/vAd3N75/xWWeiayYINrvtIplDcqzCRkX1cY+Wf7j1dJDzPh3cfbxuYm0V + Ww6CDF6GOxLaRYloV1o8txTaKAKbrOZAUTer8RqK4ha5KVSWe+eW5MazbtBPGF0Y - I/kfDbWrMeggSX2KMIOXCHHMnrDet/MvLrThZN39sB3fLTPi9gEr + d24lR0kEUqH0i6qQodmnGsFHf4l/INvbQ2VlBtR1yzrBm7fki9WzJw== -----END RSA PRIVATE KEY----- @@ -4212,62 +4662,62 @@ vector: segmentSize: 131072 total: 3 - convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:SSK:ynlgfwgubnngeblvcgsqorw2xm:3jkcozzh54l7dqrawb7jugq4ycfbslmd2urnsbboef64j2pg4oiq + expected: URI:SSK:xol3f5smh63b63cg5zcelc34zu:646qwuwu2vkllhe4rcblvp2sdwc4hlffoh7p4gaxmirmtzuvdiiq format: kind: ssk params: format: sdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEowIBAAKCAQEAzaUt1gNzX67V0Rg1IPWw6l93oU+UN3I9DVEM2odXsYA9dSmJ + MIIEpAIBAAKCAQEA0O7WP82q3/+vIKJl8tvzSgqFP2N9NpcIJ37Qp1qgVJMZoipl - v+DwcK7BQHewr95kxWl5VgZ8dcQyz6Cg498eWd7gZ5lvKdypT9vMv5fHxPkAXLwa + ulRnnmf+q5rNBnIMBtSiKwE8KDStxVhsQRloCb3CTFR4RMYIsMyxgKb5KJWLN7oF - ek+lJa4T4Nw+QGeBsOUw8RQ50NvFS4CA8J9VrJ4ZJO9mEbZp97AsUA1y6JIDBwtR + zezgtxzU+czBUhuXy3hsW7W0wLgrjJXsZboey2exrufDk3VfZ/8mMOVB2BRO8Eiu - pJH40rrT1P/zSwH9naxSSFV/g+8arJSrWEUECzIRgEu9TAljHsziDZSlYJqNFIhx + e2idLhWvM9hSxLz65clRaWJR3MbBFvXTR16e3urb0eBpRTKqm6a42YPR79eYZRJD - 22LgYzIxL8GW4ujW3S7mEDq58Gc7iLTviH3VsEPdAtZHTY6kYUfT6bnObREbguCm + XGHpqYk8K5AMaS8Iha+37c6soPsyNorFTRBU/CKsDb00v7NHutkuNVcTk+pY2vok - +fsIeKwpgfay6Ap7RTXcW+s91067Ol5HkYvbYQIDAQABAoIBADVAOSLKiPU1dkur + 1ymla4hgZBfF9BFl6BKj+UMSrxwaPOdYAFR0/QIDAQABAoIBACakIX5b4pA6iNBr - S5Kp3HKMXxOH4lcLP3Dz1HLACj69+OweYfusWUaskgFKHRglbA3Mlq1mh5MNR6UJ + kJjduo11TCcI1rHXtYOedecZwQDbUtiV6EoRjbdzituAjg403goOXe6/s/lIouHW - MLBhJeBavNxG2IjMCZHS1m2kdYf1fJkG4opalmav8ZjQH1SZGPXAG5DJzoDdb/Tx + hHD2yrHQhWSaE8M+cAGI3gRJ/VdW7xBPqH6pIndjTvVjO9bccRLJ0xqUDNm+xhki - pTHp6IsG83bjgRhEFqObXJYsLV24fkU49aBph9rhs3tJa/KnH8OOjDYvdhHtlDcz + aj33MfZTF37ecOKvCMXiX9UXtXmJlHHry5fMyaQp13lyEU25IHCVbfgHeeQrT3Hn - 4e2j/iluQ8dQwrDFSKI+SURkkFpHNHghpim2KyVjjvCIs5bdZooEbVvw3FbDCo2B + jS2fJi9Dg7cDf14lM9oMW3T4rH83NabRkz9a392RSnEKubYMWL/aHcUiy16Iur5A - yzhMuO1xbRPBDv31aW0Uss4+h+hRnF1qL1EjCx5Gw5O7EO5PIMPiIw6RitLo68ug + MaWxofqcFvF6jxJK1giJaooMCRtWD6EqcWLkgc/ZTdztafswj0eIVXum8frEDxr1 - HaQ6DVMCgYEA0kqruSASilijTxyQsIh+PhZsmCg6N1lfCXSxBHzzme+zBZgUpxtB + G3kGsLECgYEA6Yi6QkzUZaOoELRPQA9YVr0+sx/kcsLA+8t0+9qpUCuWr2p5XCPD - ZecBR9Fhl7vNb3SE9dBRzuBH9qDWAxp//aXEQL/q2iSAzdLIqR96b4b++qvw3W0V + YH1vFUD1iOCHTVq84bXPxTsuS4xQFopHeyhMk2A9pB3/HCsVkIuoEUpGCAQwsmMa - mKL0KosgwSnkrjOS8hjq1JyfRdvsgMCbzGuaRKfa1VAqnHzgLHMERvMCgYEA+lf2 + mVfqtX2iLxCy6/KBG/W61kiZq0PzpfdCTZjuTMqK+tq5xmSZl/pj0zECgYEA5QhB - 1S0hcL+gFcNOCFOy+wVVC6++S0pf8aYRkYx+F3kR0/SjGF6vbLX1vgWIr+hvGD/t + Epc8LnBOsgNVarkvj6wJfsKGjyFDQZd0KtJfIMkkc5HjSU+yx/dh4BGDC2UJ48UD - i+g+2JJ6xpsk/Spoz+7SDigN6xnQ4ebxqC6meNdrigWGji27a+7eUxtYBo8zoJdC + +aXkdhXYXvSVvhW7nrNmOU3cCy6JT08X4BmzwdFcs0uu72FADoPkQLS80KFJ+xP4 - HDi9SqQA+WyC0hg5zSnFXSXZSCQggvyjxtWNkVsCgYAuHOepEapfIe61s1rbCyM7 + NQ2OdE85wX3/Ac8aSSIazzkPK9/Ec8TImclUk40CgYEAo/zC7jONIiIdrj0vOUiN - tCkd+HxDlNptNWR3ynqUf+ZuzJmCx0xA7zXtrLFM14bF8PQS/xphVfcR0tT7Gz2D + O7t//8BxZrSjVyyzZPdS1V0GXv9hYPYsB+GM01veDtO7rvH8mHJXB6RbCenpgypu - vmzZkfwK18RS3ezYgSmU+TJCf5+yvm/k557JEXceRHR76p1Hb0VXV/zpEb+7wACq + r2jI/OQj5M67iUgnyGyJBDsnmhF3MIyu7ObzhaZG8M3FFjIfv0Z6gGZSohUBTpWm - A9JxSamH6ytc41k5BgOjFwKBgGZG/+IyMQJWV7nsc/n08B+cGxXONCmgdjhMx8q2 + FV9CVuITXbuhoFKcGEBXQMECgYEA0b63aUEbGiQ7zYaECRLC3b5di7q15ApAP6dl - ImHGpeD5hpSTQopggMikjCaKCLFYlN1fAiYLGjv/8Im6BN5GzOzZsm4FuxBAASTc + +XljKPFL8pLeJVtZjQuelMc0zZCgd/kLZOtpyELFPmCbadMZWYNN0JjfNVZO5VS3 - AklGgXn/LezyhCrhiVVcy4bKhKYshebvy24uOPOuQHhDS4IleavHpdDSabH6M5Mt + tsGS/6KuVHyxHgRf12st85wRdrbeu5NUMbHSje2oJO0wRgXWOreC+dd1b2aj0Kmc - dkwXAoGBAM2XgNw5f+/hR85uAqypyi2/JcGVGBhVY5i04s/WMpkiPziLwueZDx4s + VEourFkCgYAsQfMB51KrrMyC49sUsYm9weiDBGQJuTATCxX1mSC84XRMiEzf1pIh - 1+yIw1MiAFimmNRBo2wOiG61P8hBzQypp+iHWYmU93YBYkL5w6HZk8Q0fIkVXqPX + csVTLMXlAGSZP4C1zqvWaQkETCjS3AfEVKnXI095dYp4bADi6QglwYV5GFvpJnMQ - 2bexcajdG9uVDrCRJrFYD4OjV3xLUqD8z9+IXLLc97jBNi6lrdOo + lQwWUMJEup11XmTe3KxKxjPj5MFRXBo7lZg8SJudRpgvE5IHQRfjXg== -----END RSA PRIVATE KEY----- @@ -4281,62 +4731,62 @@ vector: segmentSize: 131072 total: 3 - convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:MDMF:eb5wv5te7y6k2lgjrnn6jtqjyu:mt6j6a7paoezixhnuuiaixmllwsvlgqp23zcwbkezizchadbttrq + expected: URI:MDMF:po3s5pnang2csh2auurmksfhwi:q7ltmms4zwlmtmjw4o4dy2p72fae3fzzvzn7u3pwvcnqpqzwo33q format: kind: ssk params: format: mdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEpAIBAAKCAQEAsrjGRwDgqlBtkw08/NIN9iaKIWT94GDYRl4ldWPbXjBl2NDg + MIIEpAIBAAKCAQEAuteEAZr6K+1C2iJbXOS/KguaPV5jI5r8lmZwenO7XiEnnj4f - I1AUVAZreZuoyptUhmCGunlnQpdp30toXO4yEN6J/weLf43xVOtyEPI02cwbJV1m + iDaXFF0Sg80UhsKmdj8IRPbRIkUMHzui4/g1xUXHJcosuIPxqamru5dDGj7DcRke - h5XLZ6jvOw0DFgtTtCuFWQIc+O+hWjF0irHZdV3ogyb1Er7YDSJNfo4dntIdxP0W + 4TeOfKWLWNwrf5oL9I9NJZ6hicEE2Zg/13dZeSDS6Zyyn6NQb5Txj3H5ntHMlcxv - wpda49rcqGdzmaJpoOLwTQl9+c4UxHRP69NTwbsIRAsQsXOqVxV20Ghl89/uVCY4 + UWGCESsA3A+co/AwHtin7Up+29i7NvqUXgn0paBpE9b8ZjQ00QZVksEGTLH0KwHA - RmM7IbaCAgNRpCfG1vgPwfNJmKY0O5TdASqQkUg/qiCDFEf5RqAsR60mNw/1fFbb + cRDBnHrY0WPZdYS6swgsmto+gtEt8z6J6w4K++eWphQdeRQhQNfdIqNDfoa0G7Sq - alHbDIuvZ8wUnmLpjLHDN5MOXKWds0cfPsfN7wIDAQABAoIBAE7XCQzAe9tWAIhq + R0uRaTo3TbHbzVWLPBEzS+J7ELggWja4uR1h0QIDAQABAoIBACsNrUPnb70+g1ib - whkrVqJcDPo/UWlef3nHRVH8O4TY58zWE9IwHM+WR2oNe0/pZseipDx1mtI69ibd + cR0TMr+gA88fWE3kkU6g1UtKLsMudaAfpYlwNtkA51rKn2+8G7qEpMWrcB5q9bOe - XowEPczIRurcerLJvIjAFoEYP61Gh0Eb60Nrlp/DW8laa56ZX5Lu0fPaZUqBd1XQ + vNa2I8HM5epdz3dHJCEZ0VI9NT+vdb6ycKyp7iHnzZfQyA8zsoyMltTT3FpPSWxQ - 1D7sxueqBgx5LopW6vscQ0BNVA6/mCeXDMfHZm04GqfFvCInHrKdpRTIs3f9fedY + iml+fXYNa7xcGMbzTX4gvpb0xvXi8gcy5vWEmXaUBeQJrO/5Q2elYxW8ovyXvI+B - XUDgYVIEhZ1KdBqD5o10s2yroPs8xSvDCZpnmfwDnbAPGO+KxTsQvsLYk/j2YnQ5 + u6+Ul+LCBzqUQsf7xspwydSPDX2lJbUeJXJUsCqC0+1PdRbNkXs7T6sjZ3kvTI5G - u6Cf5iPmB/HjV2DJU8bs4JOPHf6hVP/QB5a+96fEIyjXIXoGg7IeX/uVOagPB/FS + ftGl9tXdLkSfJa/yvtUoKvhWOgyn4JZoc8nuNetQhrQToTkAqYn487dcmVr91HWf - IGW2ykECgYEA3sKWMQbbuQgso03SufeM6JshGLZGHxOt71twbiZjiW6jHfAFrZBT + FS9oRP8CgYEAyC9oX5OUmBHw30gTXn/0doC+SPbuFkgLvE10xzSnSjGhuqegkjHl - UgaUVM4oc8XsS/PZtY/bfPIIXY4tebt59rejW5SycE24GOkEKSVqicgf/g8mIto0 + IPOT/wL7dipYfmN2CBttRYfyaelgGnefhCFSBkJYufIZeUWQhbe0WG9TPSmEp0t3 - yfXL7y8Jt7dQJaRVr3+N3t9K4WEu9LDq8k2SnPEGuUkAK3fRpVeLbK8CgYEAzWPt + rBV2EvLPc+G/Qm63vrtHYlF4SxyUFd4RykETwVJ6LScPEj3hSQGXZ5sCgYEA7u+z - m70fHq0KnObhErJlnaBOzmOosWqK1MG1Qh1wqAfIpR2Mp+R7whHqZ9JzU0d8rr8G + 8eYaTNC+uttNFB03UDHkylV92daFF2dVykqdcFgk7NiyYOBppSgVsxaGW1epOLbK - 4rbNjTaPQ3X5xtS6HU2jSu9wRv7xY2IBreZ6Q+CrgnbDaTxQPAZUz98Wrqsb4BLT + j1QAHmRPGCKnRltzV4+jb7+hxohHhakx8YRgPb6c6HZ1iW8ZBBN8+oRLsE3cSEhe - 2BgP+E89edsLhmUyLIDWefENWKfyXCiEgoimgsECgYEAmMLamo54ecB4VBkXbL6t + cyoiwDoAWzviNRhIKR9cUEs6rKoHLoPiUaTvsQMCgYBpHNuFNAzWPLVpyILDIBTR - 3AoePUMqfT9SpXWQeYlL80BzDiG+0xLJgNPQPwQNy68sZ723TAJ2Y43bXMUWvIdr + FJDV5zLk6DehTFqBLxiYUK9HPzWFDkXto3iWco5vYZTN6JPVdfFOjS+whSY9P4q2 - kVzH4xLq94bku/h4CPuGvywFfIXJAlefoew0yTb5tAo7JUU4GZ0gnnmEcWDjAZyd + 6ngTaUsFeCYAE5LrY6aCuRHQD7jjzzCrXyl8kZp0kpjG3TQGJng5G+Y6KmtngA9/ - 0kKOS6Aim0fLnQOTOo75pzMCgYEAh0cC7+mvfofgjpkusx7W+OvmG9/d8wTGbf0r + T+R7oj8c7mFvhqaAmyFQ+QKBgQCM/NYyY7ObJgWVXrfxqXetE3PMTHvxYVqxP6Fo - wmEbm0CNMdt1kftWW+tq5XjiRn62K25cPaTDW/gMghVJL2FbOAOzwp5T6B7wpFGf + t3SFCQ3oz8kZzvGnqap8PUtUdLp+o6WMw2U6ibf+JtyLcITz4ubulqYP7vQ9E2RL - 44cDDoQC0sogSMbV3cMZx1QbX24JzRr5dsHaeuTOC91vCNTMKC2vld9jt/neEj8J + /e+IH4SYyuV6Dhs1w4YYkJ3Uz2yvHjzVOcS9prv1GbXV3Jkf4shm/K0Hm2CXeuy9 - j+QrL8ECgYAz7vY+kTuX9y6PoWLmiPg3oBTBzF2kvenGJ0XTDmwzNHi4dr4kd9wT + flSNHwKBgQCR0jQz/koDWvM79/3w/NjFEPKfeEmdEFEFkOIQQiUf1y/qP6XJ++xn - 6UWKBxPwZE2GqT9c5/eS9wr29QTDmi+7/gpX/DkmXvSuqK9csXoIxNkt64rrNCNR + BEvFjp09YGjDAZdxrve4AAGgB1KbeBv3w2tKxg9Pu1452YFic8zgqzlQ9Dj1bsxq - YUJzBtHwB3UihT/y8O1zc+4DZbMOuVHhGQ/L6AbXmVw7DmLsoJz9Sg== + oXB+6DwwyyO6v7MGMZaUTcLGhlzulAFW6eNugvKC7ZZUHzEHdSKDYA== -----END RSA PRIVATE KEY----- @@ -4362,62 +4812,62 @@ vector: segmentSize: 131072 total: 3 - convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:SSK:uelnsh3gbs4pgkpzjfl5qp4t5q:a2os42cflsgyls6axuhesdc3r4gqiqz4fbbtwoqq35xghhpfzmia + expected: URI:SSK:c2vvrwbmunt6tov3q5eptobzjy:ppk7ttf2hkwd5ff5rzbqlfjox7ygijqzguzz3egjy5quph5lmrma format: kind: ssk params: format: sdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEogIBAAKCAQEA2HVfU9IdqaqWU99YRYrBc7wZTye4X7IaHhiLzVtn1iCYFyvx + MIIEpAIBAAKCAQEA34lcTzYiozXhilksLDEew9TFTRd21etDcUnHEJNt92MdX6eW - fGirEuBLGzV+3f6qrfRPpLLCHi63UTK4UJj1kgxf0KOQL9guikyqCJ8+J1pUOfld + QKH4V3u8l1cgAVIa983CuLeQcR/Qwr+93aQ2YO0GiC5rYGch9aJ/ybTSjN8MF2QP - ZzfF5chLIjgkv60liYerUebiE8Mr2GOgTqguLL+g0VhhAv/PgJ3MYZygRl/c+9gg + aHjc3etSS5WoF33OjD7wd/D84AkOmXA6On7oTcfteyeTtCql1ssnrIukgw2qy6Hs - mt+CbXNqHMjjYtkR138eB00c+QmauOzbFZiLVkTbLiYl7SZFHnmRO5P3A7OrDKv6 + w67kOYeKu/omlmmKHX2YAwFjR+j0lBNS3lV1S+pEz97kNc5mBhyrx+XFWXxYvtIa - rfSOm50CW6T8Fe771fKrCOTqVdRw/I0/++ahSo+j9tY9JKPT4qSz01gZRYUfInjm + DDaObv6BsTuNbegqLM1fvhDUbcAq8ymnr/IttlFGcGzu1w+1wo6SZQeO23/Mrb0y - At5XSR0aD+78RTFe+kAXXnx7L0OdokEwW4Og0wIDAQABAoIBAAMVpXaSDJX7MG0J + +PJASekB6CDr0tIrvU+0gjG/SroDE6oR/UcC/QIDAQABAoIBAAHE6qWeP8TuuSPC - XaccZ+Y7fg8E+oT+8zCCyNp/fN3FSa5HNwfWL9HnfxBQ1UuIvfcu/3J07mU6E4+W + rYS5OM1pyP8uTdyTxmgs1ZxnrLZvouMuBnrpehE0ygFt1gBP3/KRJT1LOqOI+G22 - v1nUflwYWIB3z9xI3KMaQR7FD52XCao6MWKj+uo9QlthS8zELbR81XHh+fF/ya77 + 92MIy/z7zN7vQYToT0HMV1phCeX47lwRbyvJAedOiv6zjW26+BNur7GLVAnXGILI - RHaL8EMOcGH29lur8nSVkOU3M1nb5mdclFd+CnmkNBCxQviK4jmotQIUT++dGWC4 + +06ZalZFqR2d6qxBrPSK9BWIbDUBtv9u1fe7eOsjLkZGjYBsOxrTtiMcDILRvzHr - 9oBg/Dwoy9dvqjnmKPkWqVSwqDx+TRtZWyHxkUgkkaKz07MlamBqv50iTCqQxvux + gWzEMSdfCEiNdmaz4j+JjYjqkFiE6rKDv72XvFWS2NUSmhcLG5lKKqeUk55aAEWz - tLnRupSkr/Z9myPBI6v64sZ/AsEznPNbY2sRymDf3S44cYmB1UNr2rz0jjmTMUBk + lR5xamqdyyD0z7RSJfg9xLn33uFybhzW1smVcjpKP4ylBmxLNFGNQbXPiPH0ja1s - ChLrQSkCgYEA/TQvIGHLTA4uDEBLJCYmLe7QlOHfy/71gWMWFc6m/rsRy4FSDJKf + jlCsU8ECgYEA6jDz0v0oLjUbVceQUZpFaa1orQEH5OCC2Kmn8xPR5APF9GwM0bgI - 9M82myyNzPwUfUpFV2rx/9vYv8TeGG47rEOeS0gwDKVbfl3zCsHmuWGiJOCQhDts + phPTOp/3BM9KNMdJbwAHJHDuaKwTZvIYyjFijWQYP3iKRZi0uYG0w28m9aA/4eJE - NWRwojA/v+8PxZOLAbiAkbrNW5tXefBSyZTJ5A0o+i7iqdUhbbBUO7UCgYEA2tlO + ZHoo0LEQuyxp058FLkrZXScaxv5O4piMKtEg1HTq+JX/brI4OB/zx+ECgYEA9Fpn - 0rMuI0Vki/3E4qbfIvTcdEwQfHtm7zH1+8HwNbJQ/VIBxQuSqLs21b29RPOdC6pZ + hh+j2q6+hhCuDPbxYIUyf7OOx6lCEs0zlPWXfyM7VeArn5ys8us6jQHIUu6/rhf8 - 5XdMiD/rcdysqcYlRljuHr6c1foTZiriwjPrT/J/iLtiQcu163yuEWhkolgTuPvo + bP0k1sOvqCMyVfCh5iC+/3zAEmb2UOUQx96ED5k8gU3DNJJbI+HSGt4p1b+D9C/C - 62qnoq92h2J6hOlp+PSeM+SlSnEAQvxXhyWgD2cCgYBkh2d+j9VLaQXXT1+GBq95 + twjkawoXTd5upxouAuscCmnw+Jubt5TOUkY7Lp0CgYEAgUAnMYk6xdXVklAj3IWy - 5StjMRrNv3hx2olWNyoOUO+LwNh2rXBcnjir+1CBZkQsSmSlhIx4bSztVphnUrzW + TZLBNMpe2vj1/jIUWVnU+20BsdZ4dL6HN3G1oKNsp6DoKZzbcIGpb3lMe0SNKMHw - dDJQ6WRKYQyma16nkrysNZtO0OoP1hfsSuh9PHLTHXNBmobCNCK3uVb3XAGrJEN6 + 4JbE95gIse8LEUIobEGjzEDqVaHt3/MLIBEzuYof2821UnBvYY85y+mrI6xzSSg8 - TVyq8p6mVh8gFsKi7jNDUQKBgBdIHvaTUUk3TKcH7DYggoR5gCpvHSHhDuZLblvG + I91rqxYkILJYWXXPBVrNJsECgYAVcy4tRu/CTZ6p9CLjPnY369lf/molOsVzExJZ - GgPcYHlSjBWmUYfZws+iS8xWDlL7YGzk8CNeiXGnhEbbaYO+WjazGIQ7Am1QCqeW + HCn9XiFiS3ho3X8NH/sWz7Y/GXg4FyDwjFREig8ManKLusDri6pYkSHnO6SZu1H6 - VmY+6gplxOIzBbtznCEF9g6/R/nZ8sF4qzTHbdihRV92ZWuyulHS9TKiKuD1b2pV + yZy8Jc5651GgdsyLXNJty1zOx64UrHCiUqSChPNAwari/lhVpz/h5iTiHf7QYb6u - Ol3pAoGABithsvuEddWMd823bBdZ2/NLVVI+5S0/c18IuSAwieEnhzPl+38m4M2c + 2D1vvQKBgQDUHknI6sHa87DWtHEz17b6HNeEHlP5XBI+THia35LtrI5NUxphFTMp - 9+c1LoYJ+ksD1JEA4YWnVIni65tjF02ZE3D4QQlai8NmXhpkFXghdtH20DAw6wRS + HSwnZmDufBVd2Y8f+0+vIuyZKvZbp+YloaTfonm12kNSQWkfwKL2cF1SteKcl8Aj - vAq+5yRl/vZkgfsg+AMhI2br9I4VXnIu28b9PL+tGhkB4Ob4JGI= + HVrd38q2YwG+QV4AHmmc4xU9LySlNECFbT7sJFkZlEDtU4qVCqk6mQ== -----END RSA PRIVATE KEY----- @@ -4431,62 +4881,62 @@ vector: segmentSize: 131072 total: 3 - convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:MDMF:r7b7a2hc7ihwnb2kil5ugrkgmm:to2sfzyq53oc3z2om2aanz3o3wy6ouh33yobmy4myyvfyrbcsdeq + expected: URI:MDMF:5zoan46gn22bdnxt3z5a7dahyy:o7wkn2sawrdpukt6gdoh6wu6xuxcu4k556mycqtfdhhhoqwfx4ka format: kind: ssk params: format: mdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEpAIBAAKCAQEAyOxN6WrKut9PPH3jDgMCNos5axRnF0Niw1fXsX6k1R/B+zwV + MIIEpAIBAAKCAQEAw01Lm2+OJNohSz0WJZPv3bY4ntD0NDLz3HN6LpNW/YG2+QeE - YtU8+LSfs88/XeuTxJ7ttbzyJhTVYs+GGwwVVpYKWI64vtjsTOwmDdGf9r8az0eE + 372TJhEORe/V9yNMzxuntGHI27EtsBNy2Ux1IX6ZoJyTn8WmwTxmAM4cnWWVhaFu - aQ9NupA0FFtA4BywDXNvXZyRki0EgggQZ1iS316X6AkP06AkeojMsN4D/eUK9xFS + rQgPXwhR/DGfYrsMgg25m0t1y8eB2n7cZHkLRDDePEZdk+54ZbEwrC9hQYis+xt5 - LNNSFUlwIDAPH4JaZnCDYOQKrROPI9/779YebUDecZhMQiFadIDrbrKn6XvEFx2o + CyLgpnz40/4ubnwIcifM+QZKiYEVz+tmlFyZPJ3wEdjNAvthQ2/MH50SCxn6JdYU - w1mI27HF8ZFmk+PVKqMwfG46gNw5+JHBlvhICYe1NzOBFPmSUHYI6FKD0jJeYfIJ + OhX/IvJfRgO3PPk6Av0NhN6pJ3ifQscfwjzDtdiEbeiNUxsZggQ3H+taIoABtZ6m - bANJ+m2aA/PsYebaFg5RhfyQ2bDjwzkEHqFm3wIDAQABAoIBAFQDErTV9Xzb4NrP + oaghx0k10mkgcPrcg2kw+qLCZEYoSCyLHsxpxwIDAQABAoIBAD/JNHrpNd8iYQJe - XIBAW82Iu3J9rnl4sLQzZ7oM1UlUJR0yy1JvDTaE9/4MW1efKENfnM+P+MRZk7vk + SqfuR8a9V4PDUibkR1JGYu7oT16PqY9vHb4nf/JMWsGLwfGsFU+FREI9N6lNFlNu - QBPRIp74z8ylqLQMKgoj9+lxTGy1DbW8Fq6DOqIWp+AXI/JRrH+DU/6Vd/ziG+9v + HrIK7yyH2SwkR3DE0KBHFjeIGb9saKfS4D9iJQcQRBqeqGRKHB0z1115iVkLaYVP - BcTgsVD60ZOxLk/zty0RRF10B8FCJSE7YUcj95c3JqP9+Vcxf7OmEvFJ34dEZFWX + rrKf+AaHAWZlQvXoSmlINFHgTZ7lrvcBM9iJ2ix3Q5C3rV+wdNrC1j4RRmv818A0 - dadZqCf3mcKe2oGjJqpFYBzMZQIzLWD/4K8fchUAZcvnNwv/wGLicYcZvJlY2GTo + 5nVSuCt3rH3j+xwCXkpxzMa4AJo3+RKy41/2uIrV+DETUOnY1uLeHXAOVj+Jycp1 - 3kbF/REWWGjlEMoZtyXDBUxLdpOnFcRo9dPKt+WVdUlhJ6OYDS5Q5lTC8gLprD0r + leo8pyue6+G2nyK9xCgnSBXTDEk1Vn1pGzY83deMpnkdY/wcAYgBt1dbA+59xp4m - WR+srhECgYEA1QEenkUkeWQWzk6B1ZmxryJtY/N0KecYN26BGHKzW0DREDfA85VA + 2OMHd9UCgYEA08JxY0V9DN7XqBe7UmACuMD7YBxjkmdkFDmJVCyIu1WnL6IDQ1CY - vXCY3yUl+ZFkwtY5fwCKNG2dSMqLSpmnBaNUPBbk/O42ILoGG2zOp9yh2Aak+/Ub + W57wpnoUMMNDsbVrK2D2YAL5A5yHxcrD4xS6QYhCV0DoH8MGghrIs9yhRH34h5wg - 1vHZ76y8ShZQM+YisrObMZt1jQjIUflKEBiGzjawp5oTCK/GHoAA6icCgYEA8Xrj + wSj8HxO0mU3J09oGvFOBlFeEJd8hpuj90hJ9EUOvFwB2LtjOOvozMpsCgYEA7Bqm - 0O0GPAPbzJXp7TS8rZJgFtkdeTv0MLlq7aBvvHEE2d+XYVcS7UAIDN1pIIRZySci + BTjtxtXRuQ03WM7XmSX/XZJvWEOQAv393LnZDTVJVjHKmzl25vF2/l0Kiqak1Jdu - AqoeHmHRnG+mF5zdnvjVopLtNmgepDM9OvCo6/6kQA3kr0PPQj7axhXsA8OMDnGL + kw9ipuLmT673v47/0LlTU/iYP/ZEgvbpkZgve9nWLnlRN8gIwJS+P9mDbyJloacL - qauv335pM+bPVOWX28iT8O604WSmxEOhLLjCKIkCgYEAnZ5medfQVcOq3J9blCRX + qeVhEPsUxSeWnRCOEsJbEU7PmznJwPjl5fZwskUCgYEAr1Tku1RSyPBN0VDs+bSj - R7HCIORWYWuQj/RFs0GtVylviwC214jqj0Ry2y0yHKtqVIMRqNlNa95xNRwsVte8 + LEQlHpwC2bqfg5tsGHTTNYEi726OkxLNQ7ci/ERCKWnTx/U1afJbrH1pntLhHCTZ - sH9cJdsLN99OTolZW5H4ml65pJHGJGwMXdI54xF/g5NfZgg2ROaDQQI4ylRlZ8OA + 8lA8M3xVqZcFWx8IaXsxyLKaGHLQ77+W4zhDIJwZQYHF5ZI1V4Mw3BlmQlEwtNlf - +sgreQ0fS+bHjvYDNS6jfqECgYA7KvtLI+iVJ/ThShJJVtSsSuNUddps7C3HCoeS + J6vFQCExfLMWJ429m9mDwJcCgYAVvApGdswkvrA0ucu7iCb+uSm94moPlQCf1ePV - te7q415m7Awxg55Vl4zhahbqKsO9L+N7d6dtllY/2HN/8aWz4BCohwusexKW9R8Z + uuIJPjuHDMRa77pLXjUXC1eaFeccjugl74ekV0TeVvwFjVNtUnIiS8MwOCpsZ/Bu - pAIf4QLp1v2jnB/agYAlbRWpTm6w001/Q1wSjOzGFNXUXXU6Gwl0zWhwmbLrAA8r + b7UHULFPy+k3Gln76HVvCUX5KBB1BhyjwjLiTkrGL3PE964seenKORgRcQtqkT8Q - 4BFi0QKBgQCIKDRNImt0/X0MtQH0aiSkubyA/RwOe1nq5zJRI6w1A57hjSj60Okw + oVQUhQKBgQCGXXKYWdX/mGxXNEnaSxqpQKOmQwnldsPggnRA+u3M5pDIImzt3nbR - RRKqgc4e3XhcbDPkdJiCZp+PywzNmty1nKsrHxQ2XcxqCLhHXn1D7cBXyzldbxfM + KNWavPy6v/v3GLHSCkMvb+oor7qk56YiHpfmL2pjkNAiwZwcdUky05sfHvhs+9d4 - tr153WIvd2s4LDVic1/aAddikl76evm2ei9poh3vVqUDzkCTDwKpoQ== + qLxi7Wt2LH6aidBK4feMC8ReOlu31JCKWjcZ8RwNNPEXbxjtOv1Eeg== -----END RSA PRIVATE KEY----- @@ -4512,62 +4962,62 @@ vector: segmentSize: 131072 total: 3 - convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:SSK:ugxvnpmo2gpclarr3zdqfkfx3m:vkjlbeaf5bez7c7nycx2xujehytbyc7e4ehyajrv76gfbyhhjkia + expected: URI:SSK:5c4mykwk6eqqcbh3slodybxfee:uzkht3h5m6ianotz2cgy74zxirzuaxtkxpcy3ghbafoime45n3hq format: kind: ssk params: format: sdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEogIBAAKCAQEAmbutuUcf00SZfCrQf5efeqlRZ1lt/mfZGtJSLYIEf6BGq2+p + MIIEpQIBAAKCAQEAq5KchP+y3p8pRzcw8rMZd1Bii5k3b4TNm/2jqy/3jUt7JY6T - u5kF5g6mOTcbFgnv2wb2k1V4djK8bvWRlAcz+l4Y7T5GKO2t30S0v+RvRwZt5Zk2 + pNmSzPnTQVe0iz2Seyg/2qynbX5GcgQ3Yon/XSXQSxkKmVMSq2emWu61G3k0y/fs - MwGJjDyUGCsB47C5mlfZzLysgyDqqSzIb/9AzrrOPR7nHYuMgLe/MM3FKH9IqFhy + KpblK/YLGUiXvaGZQLeB0jlyaaAWMKrFAFcJ0Fg74zS+8rfeOQxGMtedKJGVBcin - Et93dQ35qXKczuCVP1+OE4Ws5OjtDGCf2hkJwrKRzW25EIix9jXLC/85YyJY/ORS + Q7OgC8JjLBPKlFgossBzZjZRguTCd+6gzz+1j6aHSgPVYjjNQ+g0U//zUdIImSgt - 1+KE4U6ZkXRTmJR7Z7jiNwavgF3McQ+WbmY3D/noJOvLvcUdc/OUyXrUKBnfQ9EX + tSXBDiOmvRWG1oxdtY5xqoY3/H2HKJUNK++Q0D7BLV6bQrYWBusf9JxJbBQH6ite - fAenLRqN8Q9Vye2vckqQoKQ4FkvDtJbg6QAeAQIDAQABAoIBAAH2EYduz2dEBhVh + Vc5RMW4b/tsPJHvbhtozZO/ByJ0mAgy1QFaVxwIDAQABAoIBADCBDLW+0fME3PcQ - S7BT6SH3QnUClpMxC7B6dXE8SPlw8T6vVlbsVbr/nLTpUyCVeEhswsyHV+th09Vw + n8plHpRwCcP8Z0MkMLpiTMRnFZ2A0sot0gifJ9TB5drJsDVTDVe6675m5BhcxA6U - M9+V2MRAs1XPnTmFRVXRj1OlTepBAs/4NFWzfKE/OvDYIJQuab4XXES1ja/IxbfP + qZG2gJZ1S7sHU8s/xH3nmgyIAnRHYkktiDsMGLLCZqXZs2g9SKWWm2FysykwREWy - 5JUPQeFuhKfdBuyHtIgyVFChfKJXiWUn1Lumb07GLbtmdLe5AAlkZ6G0XyWySXX7 + 6WmssY4Qe7HCZh8ZIv5OvYO/F+NxOSHPcpX/ubbypIlIu6GhhTkqfpw9a4v7IGxk - ai0x7rpm7wurLx3TclSZPN7eKesqUKiIEgvgBtTOJ+3f4q4XSUBkYcUmer9p3kP1 + 4i7J7rhqC3+jq5iU3oeWGwUXxN/ZVwAmx5V1iWgz8Q+0Khc55od1BgpLU49hxrIP - HMh+BHDrshs9fw+1JiVxG5T4nHZWO0YKcPzJWq2jxR3KZ5KKFGQCQMg/EvCPUXQS + qmY6cbiuKo7KlgyuqLBuOrrClJDc546sGnbvudQf6ENRjDhY4p+udwXUSBuCLCYx - WCQ0xrkCgYEAyBbFh19y/eh4nmOn+PMIPI1JPrP/nPFxmxzfefLr6tfAjsNcSjFB + P4Ud/i0CgYEAtlnYKLLuPIq+jwMxefwJtWhoV9qX3sQw2716+HGCbemdFoPLBXia - +Ms9ng3kTeC71pGBldFOQCyvu1fopFe5EVNK0z1m7VcCrnxIi444mC8aXBqTGiVn + 9VGZ4dbx6V+pa2KE56x4s9fe1cpFOVfaoY8NR2hOgy3n5OoqyGv8xnTGqdFGDDDp - oFFxaOvjLfpK/SAXMi3CT4a/cYuPLShHpoaoDZvezVHxLNkMRD8gsz8CgYEAxLDf + mC9a2fKZ5QCtdDCosMthnT2nADr30NT0RyHVAA729zrTsxC/uTDNYDsCgYEA8N5Z - sI5YCTeeWFxZplaOqKv/O4gxm8QlwAqE9sCawxVjY3xYgR/pXCJWj9RbbGwXwVeL + ymG7j8dzWvSTLy/FAv6hF/ovPh2/4df33EvNjLPgzhgC1qpuOH7Yw1+CbCMOG5Kf - gc87BfdHqafPuOk2FJdyGwHTUJtIcUCWanHva406xjSTPjPuphbuvI4ydiochdFP + YXjEZBBwa4G8mF2NuTriY/ndezhysFGoWagmd8UDm66DnhQMUoYJJR2smJ17XcRc - TLK5kI0W7N7GFOo1BzQ6RNhtcJ932m2rPFQnHr8CgYAHt9kmv6fP44fDlFSGZdmL + mjk9Cx0kvGw2bg7LSq9wZjM85AIandVV/1nfc+UCgYEAitKpWoqmDlc+LZgrwYdc - fGe243qYszeOpC56pcQz6t6ioyaMNho1XqGh1ydXWbPlMvesr8Y084RT1bBDpp6c + SMwcq82R0xkfbRq6lIut1UmFuw9Ir3ia1+pwsVs3PgkC7OrK7akDFz9fuPjNbJNy - 7HmWbGfr/886q9CgkXvdYvPBWcUS3R6CMKIPSgoZW+5IlVRPuzQjnS8FUjzToRoi + sY47eMJzCzEWmtKfEYgMn3VljQDyR/Ow0pgynTwxZwL2Cj/FHRsozFGUYvuBkG2f - ck9JNxoBEYgcEsNGXqkEQQKBgCG2c1DOxRYnW1On2JHjKiaM/H1WtbIOJ65H30xv + LswV4X6DC8KwSmGU5ELAB+MCgYEAww895dApxXD+8QyZWSA1SoyMRs+bjJEpACsW - 7NbdNqDZsk3Hi3cIR6/1ZQoraNLxz26bd3FpVfYlVjxKdMOIxb0NTgv14a/Pszhh + lXdpuWU+S4hUXCVe5y+KOQXSp+HnndqqaZQUbviFIfrJkRZKHFQcXFxPyWbYMgOe - ePkFRvqsDkTOH+yF57uX39xTEXp6Ss5Jn/a/yBsnf+obzqUCda5RLkjsfF2LCJuZ + 8yRiKqIInv2/preTlwzmwQD78geujSvk1hw+XA643kI//fbLGOtkec9Ko1c02NaY - jO7/AoGAbj3i20/BEAphG7+M73ii1sTJdL96G/A8QDNba53+A+rmczO6YgMrd/FS + MxdEByUCgYEAhy17HFcyGvj0QnwgoTxWB8AGqivfq7ZkCrBZ8a4jtNtNbYBiCH2i - 3Gab0l6ZEBliFDGWF6VtG+YsCByRpZsEHbYCux+E6k4K3SOximh8WcWgB7KU5lQm + 9v3uomcjTPhZqu7W/wloYhsf03b4y/V4ABQHpoiGUq6NNAQoSu7/V96jW7K3HOT6 - VitHBiC8NJHt7PnVgWclDzGIZiM86v5c4il5qD4AjdIu6S29JaU= + b7YMYzPnRAkYmruFnL5Q371e82JeQl7aK6hkxglVF/AeXKpEKDkMw58= -----END RSA PRIVATE KEY----- @@ -4581,62 +5031,62 @@ vector: segmentSize: 131072 total: 3 - convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:MDMF:t6ljkehb35cjzldhi7hzsnquou:5qg7zj773gjqejptkptbjkcugmwywuvmo27uq33sloafub7fck2q + expected: URI:MDMF:sjzzvmll3obdgezri6tldxtdsm:5rcvg2egftpcrsli4giqg6txgz36xdackev2yhqdubbhncw5hisq format: kind: ssk params: format: mdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEowIBAAKCAQEAr2+8XTlh6He0sHac+kTS6tN7tzZRndDu9Tru0q6iru0Y5az8 + MIIEogIBAAKCAQEAjw57luJrvanJrEOiDtCSd09DWTIrDWKDTTuRHlabqUyu7i+Y - /3QKQ5b5PBylSvc7ouIJK/OLu0avfnCO3qSp5hJF11PdZcQSM4zy5P7m1WjloBTu + O1+WUn7yWdcryJ+hQoH91gmexQ61krA7RqjkjFBRthSKeVhlEA7gFfFVtBTxHsqW - i39L0wRa6TA4/Uxkxx50RbemP8x+JsledWzQu60TjXOPj2uuMveVLmb0t+xCpx9U + nUwyKzlfIH3OYZjppugjyaHX0uFEVFbFPYa7iQHvUbWzzHwlBFbM/fdM7qfwV2cN - QppR7AxeNe7ot3cWGqgzs/QwOKcHCOA4vLmkA1wh+uCL/G6ZrxNmpl6vQgMML7GL + IgohXyTeeM0JHj/FO2LlE7wbbzC+nxDrdTmzbLy1I3a1gWiQeFJKojuY0ONtlEkv - qV0rR02lL3qmdDwW/VYl3QlA2pPvDkaQe47wg9ei3gP3vgx7A96R7Y/Na9AQV9ZP + hD0TS8lp3VQ/L6zFDdAmlBk/GnmiwBetGRJ8CniXIgUQbCYUg0G9NTPBSxKUvykN - Az9l6q1eB5ow5EpQM5MFc8GVNwwUqTpvVWSOnQIDAQABAoIBAAsjfAErQIUi/Izr + zKWXzTMRt/xkhL4UVH5XcBc4fEOzxYbi3akk/QIDAQABAoIBAACtU57WyhKPA7da - qwHU1tNkBAnY4Au2FUXqrPkhb2DN2vPSLOoHMxOhhUeExhXhZp7r3Qs2VlvYnBHa + toOK2+Wtnxum3evLvQtjDQj8+SO+WfSYv8zomUWDuFWRyXYqcY06vdjUMgT17jAu - EagfKk5aQKbwQzFP5pvxSgayDHPmShYE3jRrK6RFNYRytFuYuxlNXLKEe4C3ehb6 + aCakv/Nkl//yefqbstGsbPcKi04804U9zlUuNndrLHV8vCwH4sxJcqFzktAYNhxx - WA36j7IqxgAII0hG3PONdqJQlR8MPO8aCTnpS/2XKP7/xYejTXjHvLzGP19Iod1m + Yf9AB6DYHzYq2MYsRWVs4Vvp9wuwfaDvp/rJ5sndknid2i21ZLukrNXccmdhcOHq - lRDRwGnAyz322g8tNz119TOmklxmDdZAcRsgUvA6xlGxy84TDrEHbSdl6gfEgh0D + rDqv5GyyWgDtkxoOFLausInih4Y1IF83MtsqNdsXJ0USFv3jMyyHdX4nLutZvXWr - VwzTnO6pOXw+kDIs/k5MPN9RdlE+7mh4zb1g2aP5bGjaAD9wLhhe3NHgcfaPPuOO + e2c+08Em/cMYvnnRRnL1NDHz6NepQqHatLI8UdoZ673VmWXS95pJUHsn/wqlKpXD - vmbnBAECgYEA85L7VPF3JmCU5vS6mmTure0IqrwL/shodReaSYWvJA4E/WBdkeu4 + FBSHzHECgYEAxriXuwvlCom/eIALAfjVweHM5HDe/1OFN5E0dZuxbke59oqp2/rx - XH/AjiZVEbvhXQt8wGuj5Jeo0SpPrWnHL1CLbzhhBO5h9Do+5M02c4GG2DSOFt4P + zjrUVxAPh9EIwjVi/av5b5kf7TXJNfDb9vrKLZmxn1uohlqxSKmZeEmZPrlD04T6 - kvMkZLx5POqwlSxUGC2wp0KgsMFkQFy0Yg/6f0iKTB8WN96IrLXGc50CgYEAuGLk + EeNVH4HUdQPoVZM8xzUEKxuLlL2wgmfVZiayxg9Eh+9o57cbQmugPy0CgYEAuEp4 - i9cakBTW+dwS3bV5JG+Ajokakm24vzjMNkAI4su814d+uvCz4B6TtmDrpw/biFza + B82k8CvOq2IOTam6Nw/72uTxrvi0gBOsHuQ1iEHocB3zF7M7IMdnlHfAp+/4B9Qi - NYThUvTv6jOMtATtkTqYvrat7e4hf6/y/qqNSluzKLPqn42I193gEIyNopw9V2Zf + n7JWs1Zn/L9O7MIHwDguYWs8kSGNWeUrmv2HE80U33yrTnA97pejy7480X9AWjfZ - r7ePLJwr6EIvuDVdHG64AzNxnqw4OAsh2963FwECgYBd8MY2UJqflohXOvPtMBhN + eSlKeAMvl/jC5ABjIUu4C/dfP3DDn9ktfbDonxECgYBdPjo70v36rt2/zdzcZQTv - xCmfj78gmLKQ1nWO/Zw6z51lC5GLAdqs8iiVqnsMx+V3OUL4A4vGUiet5B+uxiko + v2Kjge6wwWDNzP1ffdmIVHGGpFPFW39gdCw0Wd3frY69ic1UGACng6L+a/FotQaR - OmxMjPX+LOJii0ROgkcJ7V7QbBSRBTwEdPoIUBiCQhGwttQILzb+i1fmU/ASUq7P + YeXB9c7pZlmyCRYMcUAIuAgG7WlM86VfBVtouEOXUGkQ1lB7bH3zOC6LcWJEHjJ8 - U1JNXPDZwvOSwKT9122ekQKBgFsbEg4+pLNYeLhQk0nVJxxns7+54tVDPavOZqjP + hwX1dy6B1i1UJI/O369GVQKBgF0Jy3s1CLXvh9DiGhJ/Q3GU8jNaLeGF0apvRA/8 - jxRw0sgz/Nxlnps4wIe27/lGDpUcO/2BwMv6lqjD+9vfK3s81sg0/0+2//pVd915 + akI18+DWXelzJMCxqss7gdnGvlNFd41j/X1ge/MPqgrBEiSquE+aUeN7Kt9dYRxI - bAK3uJh9/YoEpv7ydIn4yOr2BCExRkpOioHiUJecTHPaej1YP7flLVjXg5e9eGdp + Li+C86y4Rcu9wqZ3i8vKaO3i7lwdof8XJ29bPGXewpVU1Lb0gSAZbM6cYUaL3/l6 - blIBAoGBAK8DxEooYqNeB8cGouSkj/6d2kezseiXgcmhpgdrn0h6jlrAWo93wKmE + nJXhAoGAZhDKzgaBYkOpsTDOsWFneBDM1+FjNDT0T8vGXBTod+8I06gVlcUq2i9Q - ZTA5+yQm6EoOKXYk29esopYPd1mD/GzzQAy2Cl9eDPwQpykdeShwrArroiZ3lLHz + ufLTXRPLqQWhnDldp0V2YI6wivyQ4ab2sRch4f8iY+N5aAPexrAd6wYxpakEGQ5W - EB+CyNV12Y+MAncOQQt0j5o63oR+ihlhxCY4sRPfdsK3okqdLdBo + sg4iw5714ZTy6ZcvPxoa1IFBkOaLhfl05/Zw6jq35Y88gweZD8w= -----END RSA PRIVATE KEY----- @@ -4662,62 +5112,62 @@ vector: segmentSize: 131072 total: 3 - convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:SSK:b7ro2vtzpv3ssm5xxgobj4dc6i:t3eqqhurxqmiqbp5rgd7cqpciingkx4jjo2mmyfwqjhgaitqwubq + expected: URI:SSK:2cwnw732f6pd42kjvbin4zv5nq:fzrpbc322pmjemwlhwn4mv7qlfmnxey2wxmcmuksyudcjbomxjaq format: kind: ssk params: format: sdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEogIBAAKCAQEAn1J0hRBe0PNPWhJFeZBcam7MuqqQ6omOCa9dSULWdDIb7h0m + MIIEogIBAAKCAQEA2C1mTlS/6c3YrjFVNax9bCyDp8WR+EVMdHQBNMrIxOsTkpXo - WP+Wjib8Ih/YtuKX02XgluK2Phj41G0d3IGKiEfZ/WyZ7ygNoKOYNkv7P9r/acZz + 3fvN90vWuxcZNOY+j/XxLIsK551JdzC3PFUPICwBvBWhRNiC3P3AiIk2q9Ju/8fV - AGuR4genl6Ogo5ZMkMtxBHPdGZ8CusO+COuWnGhAm7kfC4FU+fPj+9tHlPt97WaU + zm3Aj/fmOMk4Yy9cTTmXgk9YEnpLnhJiLni8au3hZpVHnQmfTMKv6OrwKP9DopGg - d9bqgV4qfqHVZmKeKTpfTM9r2wq9j2Hrrpqe4cH1uXH1bp4rqHLFakW6hUxOx8RM + p31pfZgoUi1Yfc/wpOVpMyJUNMAgrUOm2s5nnoMcNsA/pxVdmS/Zob9q/78Xh4VA - KdW27vhFPcWfISjAb20cBo24DrQWliL7ZRJ3eyiKsftPiCR9hjyAtS52dsW1Akf3 + 5uYCBVj3V+XKBvNg8xlmkFDp3CrsgjhG10Nz5BpB0ucdXfnGaId26IGHoGrlpzLL - jA6laqgHbvBBmoZWAoeIPAoY4t1TZ1pwsMNxXQIDAQABAoIBAA1Z55HPEWMJQLkX + C1G3bX0y7EHHjBV86G59pVdb4zV3TSxW0mMPTQIDAQABAoIBACVn1agAKiT+pVue - luLdCiGRL27lJEfDRzfgjjy5cSdDm7uUjcYfhQpckfx6FrscugRpIS0Dyqnhhdin + 1auv6RPqr+071oIrG4ua9wp3fD32nzBiGCUxCPadfM5qtMXegTzPxad7d6uUH74s - XD1CTc2l18q48x1ridjQXM0QCPoM7CJ9Et4SJaN/aLf4alnLGpd3tPzeiMTA4oWs + A8jAvxlGBBbTd1A+VoZ+se3uMDOS+fnwTiKmAwfmUUPKLaOb8lC6gmjd6dNoreTw - KZytwW1R/zgNh2B7cheQLKbKdXEykDidTYl+lDB6vn/p+bU/1jTziqsuHwqNP8+q + MWTxJ0kpWDMz6WxW2eWiWmXnIR82msDgdVUnhdUOvLdD8HTC2HV+Vn9CFL68k+tq - pfU4SHK5ViHsjBok3TTkgqqAEAB1xRqnkI7Sh318g7Y/mLY/6lfSLPTxUY1r3FEw + Iat+G1BEeFp1nDkLbnH2zDMLW8GUt6z24HwjFbHbVkgKa6t92Cg8l+pV1sNu4jt7 - 5veLvPtgzCdhZOIbNJ4r9CqJ6dbUn4sFRAwHzmyghmJ2coL3AcjS64f6VEYxjfZV + aU2+AoKYRnWSSLedBkNjSw+nasK3Tg1f5/6xq//hiCMPrN9OPL3fAkN5QLjRW+EV - FzR64TECgYEAwxt3O2I50lcjXnUTXNzvAl6y15E4NFA7TVeqO5LZSi2+pHalRofn + c9HbIYECgYEA8XWQQU+6k5/rrkNJh+UoeeH4GkgFUSlzq5H4b5iQdQwSzY/FrZbW - W5TIPOlwH6hxrmomb1gj5bJxpiQA7/8fvyMEwvIoaF78fWE378UdWIGeEgPrNZlz + 1R/S1YkVqq0fjmZjCjTLItBpnbfCt3lhMQ1RTBi3fs6spMjY09aGdJeFJn/BGK+7 - RBD5ZKeQ6GpPn/w6LSj6y8jH19N7DfYSoHdVpV4agfOdS5+0dk5Gen8CgYEA0QvZ + M/o0p386LOI5VHJtSKdfI97tctOTJ2jvrL8mVJ+oqaA3PJzHtz7nQ8kCgYEA5TIU - zjulCIv7E+JhFk/zuiVv3k56lgrj1omkaFSE16O1VxNQyLFX8EBVt7jUXp0w4Zq9 + d1N/Z4GlNnf77mEishSs7Ln6J/VTcNzZrzsF5fd9NMKoEPcO9OSFa+ZJKFOyMwkl - +B3PWwZEj67GyJdI7krkTY+I/nOdCoVotRb5kj5yEWBtIQp4OrLzklWtA0r+pDx1 + haPY6WOv3b+LpW16/OvntHPbfirk0ruAwItdLva8F6vOYDOrEezMY4bXVAM58p/S - 3/SEii7iGLiBdlOlpMVV2uacyu47jFpuV+kyTiMCgYAWZLagqDt+uuWiV8mrJOiB + qYWXfk0mXr1kq+Y3Jk4KHwh+HFE2pqHqPybJSWUCgYB4wOmWsA/H2jdcXAw+6QyX - 2yCnwVE0H+lOjTtKryYlb26sLbn2iG6zgjYhV6G44Hp7zE8xBGrKWFrW+NbqtNuN + /7k7M39tOoS9be/Hp42+633PzbH3gTMJPLQM1FTAmXnpliy7ovFgBMh89rRrW0mO - 8pT/Uw/0OsK8GUZ0TKl7mRTteGmsszoZm+Ej/l+RbXJKKIb82/E9JoRZbzp2dcHZ + 5XEd1FKYGTXf5w4Ayw1M66XMPiHMfb3qXZvNlGP8pFo1cFBVAFclMkyfm03BbMpI - jRjVbCGavL1XCrOJyJ4qPQKBgFBQiEbW3YoSFc3G7NwgrZg35+n2JtzcpDp5uWOo + IwRBV+NaiWR2bJW4f73aeQKBgB0o2/W6h9ERa4Wcik7vyMxDWSTdHHiM+8q2LnPe - DT24FOS2dBQXJp0UappidZ1AMVaMGC5qbY8gMlktogvRK+D5fwtZeR2hl5VCOj9Q + 3Ic/j4xw35UY+awqjtcFe3VIALoBheaUy/oVlsBtqESpwyX+lYId42UP7ADrnhvh - 62PHgBWzAVpvZk/PDwuKxST9vCWnYPZBQGbCqnUq9fpbGsnaUyj97wF8U/6Rg9Fc + Hz/kYFXao+0VZcRoDjDzbN3hczPtJY4v0vBcvG79RZuhNI8JCrycBf9wTbWxj+3s - s8oXAoGAEd+/YeU9jAdC1F0gOMyJF+Zx4P5rtFpZnkpUuEd4826cqRtereEP1ILb + 71sJAoGABKJgNFvvI5DzXXkV8V7arT2jv08AlPvf3HqJ0V6Fw+VV1sfcXEGkRVKr - VSMAL7HIgHao6Qxm3rLaP6w+m0Asy16u4nD2YZ/KAwI8UCMbugcqYVkGBZ+VDhB/ + VleFaNqo9zt9atAvSNUTidx4AWL/xlUzhR97lzdRQyhbJYa3T6mU1JOPD/zHJAnQ - 07H+p2YX3nxrQCZ+SO2SeIdvUNLGkAvl1p+Uaa1OCa8zGLMWt2Y= + xiQGOW4ll8c+ht5wCademva3C6k+R9/pk11W6JjeTISMZ8W5ICs= -----END RSA PRIVATE KEY----- @@ -4731,62 +5181,62 @@ vector: segmentSize: 131072 total: 3 - convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:MDMF:5zirha4u2skivahf3cpsl5vxgi:3kgb7qi5y4qsx3mts7jltclahdupgxieleycnpnxkg75mvrvbv6q + expected: URI:MDMF:75ls7trk6lfklxbrush6ex24hu:tho6cve73atgkhgpvjd7446w6c4wlta5bszbfv6qhbs5ykjltwvq format: kind: ssk params: format: mdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEogIBAAKCAQEAtCN2kwnkjeag9FiUvRprIk4rqgVzuzF4saeUdM4jH0LaBYzz + MIIEogIBAAKCAQEAo3aCyqOOeNyiAhFyMMUjsC8yiJHP1N4AQtNJnaj6fG2p9oFH - 1+tK832B0D9E+6M8PDA4RSjlM6V4sqboe4W5AxZ8bihGGHjWtpjZQJB/QtMSfImG + LLVqXIj+9X8wFQvU1APqfp0ZhdyJnCIOmiZCWGZxAdBAKMupEP7KcARD8r3pSeiK - ac5cOVoQ8JbWfDKDpexdDRcgVgqHeLZG6+d6zS1XziLw0GsOxvCvQkA0XdABibNh + MeDYJx23rejrtKYjJeJAeocIj8x9vcCNXpimlN1uOLkHJk7FEvxn0qmCblfqGKPc - BZzvStjGNOM0W1NNpAGe/tNLo68ZW4ELEjgbuFVqcVljEPR2A1jwzoLl1+oM+0BK + xFBsBpBIPKVK9SZ4vOPa02sKz+CgoCLxRSjyLj9XgH5+FxU/O2LeOnrq0TVttvsW - DT3DygKkkSO553HnjNFsLp8A/9yHrfB+k+MNb2Jqlyup9FVHA4oQ39I6YTWvO1NH + UXILD0P2IsIZcQgXszyPoCkdqFZ+GgzJA09ut/inlDtB2Z0kvNfrxaqOtTlClrm7 - S9Y0q5bFfjkEdLtiDtTKMyfJwZdb80IgtaC5uQIDAQABAoIBAAJk4kOMAxybrxHw + 7zYclePeL2mYP607xbqjQzfQAe0hrlLR1L9QQwIDAQABAoIBABSgpRL0PnYzBWtf - R3HH8xqOnWfyEJqxSqBZ0NBImRDmS419VRROjT11Mo9498q8XaWTInxQ0dMA5PzC + Dzq8o0By8Zy5UHGWIuX2lWtvV6NcZV565DlDRWzwSW2VGMNhzwAtjv8I98pkJlC3 - 2R4jJdVTrC2unVff60Kb/28rPHW/5mP/U+j+FB2zA7ye1JTr+vHulUICR6y9ERXa + yCxHWAehbxuhpM8LxZskQ/XhxpSlxtWG95V2wGAjSGVfp2A7DOQVdTnhEMFbYUy7 - nlCuT+SAMMWNk1PByH1+X2XrAocoodXK8BnXzmuzYAS2t4gN2xERoZigsoWJ+4u/ + ipDIO6aSe3vDTEssaoMhqz+yw10FctG1HANZLxStQl+SpOm+iNBUuSE/m4uz5OFy - BsB9gjFmLysw6E40KznwpJECbjRhR7MSzglHSs5fdfWvH+XfQO08n0KyVbtgZqbu + ENoaDxgWyar9OF2ZvLZc+hg/QaQs7pAP+z//QyGNVsq588aBA3Tq1J4tfTDBlyKl - 3xu3h1m8HniJExZcrfsGj0le8JukdVFS64nfKBn0xbzrtfg8Toq/MEGaeY8oVywp + 364z8g2RN5Ap2hW+j1C2uT0pUD8wjkaRPkdGtsieoutCX2PaCRcoQaHHZ39t7LIM - zZhfcjECgYEAzSCvTGyCG8TFQecRw49rl3xGdvODlJ8/vkqkeOwwPHlvKy1DtM/Q + YjcdT7ECgYEA4/R5aPJLWd3soy8YpMzRQ8EtzDK0yOeNe6RTFKfESkBReJuGBbH6 - kcXwDCL2xhnvK6uF1TCH07CZKkdkAV7+SUyd04nuvAuRBdCsvkbN7D8oV2jNlp9p + VS9nETazREJapY7y8tekppNSsN6oMCWtJX9EjcLELL4ms7o3KC5IeMI+TwUpGhov - xW0Yl5Jl2V1fouuO9s5QQYXeVsAk6X9xMIcXoHDJscdNXbqaDT6ini0CgYEA4NA/ + CZTiem5b40mXimteB2oREcIFA2EucTua/CRDqvmiuhLhcyZejk97jlECgYEAt5LW - MEpc4TBQJVfjl13d+hJdM7mESCRS0i5g+uykCLb/8dsid422bcKXehTFFKEIq7cJ + BaDblWaAeo1gRATyumg0vX3MTwXOR4iyz9yJylISMsSLPbSRUk8qLrinBdFftYwz - yAP/r4lQsOnoZ2nXBUVUqZjBxz5SjYXRZ/QiCsX8StUmou2HxK/a5ccJoTYTfPHw + GrDEXYJ26Wak6b1JAQMD9GY4hnbrHFcLB8JTofV7c9Aem1hKOP+KhuJFd/aibBBH - Je8QZEWAFz75r2ao5kWv76AO+5NywA6iRMpzzT0CgYBDFl9+xTZAUriY9zOuG+f6 + CZKQUvUCi1E7fQA9iQX6qvDmwDxgvG+J783GbFMCgYAk2GI7bVZymyVhpv4jvRti - YWDCYp40K2kzmUH1cnnMLYMYQfOU3Sq/olcCASVoYO8B/1UEBp1FtMpDM5oXgLP1 + CTp+0/9Wrd63inMHVqqqmcTRasn556+fzz6okJ/fO55tPjLUv7hUWGG4RvUGe0CG - 0SMFHmWABuBlYHw+tvV+QKG3BMXIb1auhSG34N+CmbE/nX7iZVOGOnwfLzRjUZT5 + XBDXnRCabs3QpRu/OePq6PKrURk4p9zMfq0wvt/JWB7Pd9VF+4XwydyHlFCuasT9 - ZBVsGbc9d4tsDi14C3Yv+QKBgHsQI/boTg2LJ+Q5R0GdxZxVnyVoYUwobhnV/4p8 + Vls9qoX774tTUnNcK0q8UQKBgCscedTCjS8N7nhZgVUYEGUEmfYyd+vLAkG8cbnt - LZMDsfmP7j8pmPpechMG+ZdAS4HMEZOm9Lj/XudpM6ogWu7ss9qe3zyVFhWYcjgI + IhL4qTtw+v5XzJUW8GIejWMJY7/AGDRZdRQ80m5H48zc3is1qRUZeIbjoJ18N6Pv - gPYKyP+hzKOViSOW7CmqGdBgzKwxuDbbtcpd7S9MbtugQ8bB0PxITstSPJd7q0Ii + 2DI982skYju7RVsTcFXzB7t/mW9ldzlhSTGiRqGvRxg5GTp3xAGnJ5nX1CQM0ckW - 3N81AoGAdwEu+rWuqM+jIANV78lKGxhTrhq0byZJxNZI79GDsYdle5o4xGw+dGQI + e1XvAoGAfdS4gbE6hbLmXupDiOeT8/W2FL36QCtdXMf6TfI/n8CEsAiuKsxmU1PO - Z2K081N8TC8SZZCXrXEUMQPCfy2mYgxZ9bUcdJOTozdT6Kg3ohRoTTSi2ySh82vo + dh5f3uzGlGSuk/aFxeDLcpKkG5PkaRS6Ltpy3+39dSy0qhSiZQt3cddW9lFj/YAv - HpZKFLMxIXKOGjG4pR64ihn7hURl48CNeFkho9wMfg9k5UZsaoY= + XConXKubaYCYzR4XfkFJTbbS3oHv3263xIaZlplH0cgb5EBO5v0= -----END RSA PRIVATE KEY----- @@ -4812,62 +5262,62 @@ vector: segmentSize: 131072 total: 3 - convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:SSK:77tz5f47honr2le442nicoq3gi:jnocx3f5a74f4dzcfegxml5dtoegc3iskb2zsnbchz4vzodafgpa + expected: URI:SSK:saskbmnnhte42buehoshph4s5e:sgy7cr3nt57e6rap33epd7nx7u4zczj27mwcix4kfiuyyh6fhg6q format: kind: ssk params: format: sdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEogIBAAKCAQEA0PWN1WMZgHQHTKfMolkQBZGcckQOispsmQB2SYLH9draEKYN + MIIEogIBAAKCAQEAwn8K/miyxzuWz3uOVkrgIDi+EAvbgh8KVfA3QBrwR22RONhl - gfBH0cPhxGmnfHazNjM0BZMfmdpbHhZafy4ynGDo5qyhwcwM+GV+KL0HrKU6jish + t5eTvdFj5piqIFfGzcxcAizKI2XNVH7iKHWoJJFGFegL404HgW5Oz11zPIahv7qJ - jBH15qGOdxKnDInYq7hMFaTyehGsL8lfeoHCY8sGFSyWcrso0YkvTfewv/azVX4t + a2KtCMaZF4WkKeirmcIryLjROV/j1nCsnatkYDc3BYUpdnGNHoOzDccoX/Aun7AQ - mtRk3OvORylCvHUGffuTRcofERYXlbUFaX/Dozq/BOTrMZjzukTGxx2CzwP1/SXz + IM/FavvJPTb5vPhWpely3yZluzO+rqyqRoZBJIJ/16vp+8AEAzLKoHiHvvXdG/FL - cm6tcH5/f+AgP46frbuasOe7p1O9RiaI0Pi5zYY1QXJPMAyM0X22V6Z3OWPM5/IH + 9XJr1NQ1Qai2orM5ibnda4cvCgSBH5QSRN7ZSkM4avQMB2ZA3iJuiOQJMOwC+7Rj - kcJ0Eq8h82cPOZYAJ+kqbzP5QjY3J4sVFB3R3wIDAQABAoIBABRot2atHDOIoaHi + fMpzXMkx+eVj8TDaeu29VHTdkts2j9UR+etj6QIDAQABAoIBAAeyYlt+BjEnNQkh - DcGZk1AH7dDXRthVdw/mlKcPZ/piWsQfg9g6ILmjOSzW6O3mJhDYJW+Z9A8x3Y5t + 8RiOHv93b2IQLhAgrVaISo9xYXfaKKiQu7m/uFuHKUZrXTQpdRcY0r1NS3SKJ02E - vn8Hgxf0+yp0mAP2qxmjyBOwisxZAwQZwFgO9QaGpwSIRNqbqBb1lDDVAH3dtgSg + Nev29//2dckRJUNKB7cCCAFhx9kp1MXTGnQS9BkITu+k3MHB0OSlT/lCAxmbp1cJ - 1XuAqvzWOozc4wDnuM/mZ0FlPNUy1RZUtdZVGmfzzgo3jc3/p4OAxadKDiNKK1du + n3Mf4LmEBdvkkKb8yGJgQNo3OuxM8coddIMZp9pLJ/Yin6AaKgzaM1SX5HUgVQz0 - 0r7kbKR/EWyU305bdT85eui8yeYZosAZzCuGORi9MgccXTL3oXmj/RDiEqsWSXpm + vfY1P38vkZRk1/CN0nitzUm8/PVvUncjF1jnaHglYvy7SA6XuR1e1rNWbNkJ6Dhi - yaNCyXsIKQ97x9WIVpSM2RoSlB6zWr08KROPyvzc2wUWWtWdfQ7cDUg4IrUr6/8o + WZmRGZSUOG8F5WL+3Ib6sr1996TEX++8Lp7IdE4W1Wxw+kMr9xpySb8INg4hgcm0 - o8ZIGeECgYEA4k38I4UV8p0xwmiLnjJido/0EWvKWMA9Va22c4PGetMhOiCoT6uS + nzwrKT8CgYEA3avzL8w/de2nZogn3tZBVxEgt3/Wx/ZBkMHdQwI3kiceG9HH3L/d - 7Xly7gK3t7rWRZj4nZjCFJmMhT5ml5T4J8r4eGXCcHl3X7FIAAIUdsaVacs3OLbs + D9KdWVJP0IDXrqC0ymm3sxnKLz/tct6l+JR2p5pI6uPqcl+HAQRFIGV8WLodH9Hn - MslUrYXocFluF7lecagZhSJR4jItl4cbthYucmWLO1u6Z44gOoEs2pECgYEA7GDm + lifonyLFUmUDE7/rlMEKfwOxJGMFq1y3aiOBe7v9izxxnu8JNZg4rNcCgYEA4J29 - 79jqcyqkQAVEGIoDn8LTG/HtUgpJExwFCrCa8qEomZUTx0R7s63GCxIT3FIRQmtG + HEpID4gWhQdsNezOYI96fM/EYhkqOm67E9z9CBShqZETvoYwJClzJQyW53ghcuPp - KR/Y9xs+LMutbu/nW6OsbeMJUORQZ6HliCBXiL5TiSBe3pbf9Q1XpB5DW7r8ka8U + MqEGcYHaFmFzuNindwp9SwWxvJOX7zvGZNEwPOX7I3q2/9zUinMuPVdXtGeFkIMW - l821oR/2jL662bkajq9CpujY+iibreGe42puvW8CgYBc1u1njQOSApcVUFpmzfjC + 5m1E9IqIewebyiKkJwKLYTMyEbL+pl0wpd2vnT8CgYA9D/HEd+n/TUDwwI7jFngX - 9w+Dzhq3CjafXaKKBTd5z//Dnv4toQ+nyLkzl33TLB0XdEgaLz7/wHZ7ezwPV5fu + SNOPWLrMiGxVOOH/ZGv0aawkk6wPhhaaFjVb9o2f7O38364NmAOPZYpJa724B9cG - i0Af9G8uQUaNxWbqSfAnQhSt0CaZZ8HCnAHXJiZTYPzfUrbCHdpKWegJydgWX+Eo + W7c3wgtWEQRzDxd8UzXLj8kqE9KUAlleBo5Qz941LTgkx5hYeLiwdk7krBZStw6b - dDUdzTavZVQ1g4MJPVEvYQKBgGiPnNgv/dWf4TQooCyysFO1XKkZ5T7LKfP4Cwrl + QT6Y3BcitLrDwiryRYVPLwKBgGNUnmrCVre3oO3XaH04adO935cOcnRHWKtaiJSy - gEUfoNP/K9aTppyem+I9xudIrjXROiHq4pC8Tk6GcluGZ7MTvayGJ5LOy/prlRsY + J5vJM+y+4ZJh2SxEwEzkEl/ueixKqbfgCe9sUzuOgRR/ix9TnjDtJbqVMp1zO7sd - I2BrwIwB87VGzB6cHk6MzIMBPcQ7zEIyTsvNVcSAgirZRLQlNriae5B88hCCo0Q5 + 300vDy6TeBYSXFOVuB8cXwbCuQg9UIU6UUIreUufA8ASLbGqqGSltUCqfX6ou3i+ - ym6lAoGAIlS1fLRkigI2PHmPTpXhOCzJ2v3Okw0JS++Yq4Pha9XxyCgqobXNduLJ + XokVAoGAC1g8mfAZ2jAqhm0+ZDnETP7TduBqGMTNBtrNyF5L7SpYjfm5Z3WdoTQe - lpugHyL3hEgPLUewuaomGke6uEN0GoJ98WDt57AiJ2UthNI2icgSucLdo7qW9A7S + OhwKIGVf0gtfQoIllKNtDymxpM4f5rmB0AvnSY4Nx3OriAVfXqXrYA0Mk2OfI3qa - oXWH+FpHHkYjAHy+FjOuwWBt84VNrHixTW8WRNSq0c6aFVDajGM= + IPDclojMdyMF5Ch4Bf4WQvbE/sPbXFrFnlt1FXUIv6KkUYzToN8= -----END RSA PRIVATE KEY----- @@ -4881,62 +5331,62 @@ vector: segmentSize: 131072 total: 3 - convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:MDMF:6ybmjtp6o7x7h6wbcut3ao7ih4:kdnpi3rc47su4eypnjq6bgx7ixc6qle53slv2d7hpzhdygf3emra + expected: URI:MDMF:drwxx3brbk5snzvxum7edrm5mu:io3zsepo4wtx5zvsgabwopzrwe7ym6qymgfpt7bdzayyj73orqfa format: kind: ssk params: format: mdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEowIBAAKCAQEAw9r+RsfnlBqGlQp5KcdVAy0j2X44Ut9erFEoFPq28VnT/IAU + MIIEpAIBAAKCAQEAyiq2CDqbIfT1f7icI+VESHA6R/j5AthU06WgxAvb7xHe8XHn - A5DQDYg5+xpYijOeF3F5dkb/IvqWDIEmOqEG/dvku+1I5PtnqOclqohI1o3a/751 + z6lN8YvL77zK8Jg+B36rZbw2hIhH2tQVWUx7EbXWgSFUJXuYT1DSgM4TDhKQRwwn - kbgB5mFET6wv5OCwcnG1I5Qo1XCdB/n5HYjs6otvxUkW6P+votKKu16P8YnMO95B + iGqcLjShxhy4bn1Bgr+JCJHdPZhhszn+prKCPK9X2AtLGbBo3FEIbbJTxJFNlEhD - jT0MxZUhPaqc58u2kXI5+TJH49QIVmQ/aMWsXGuuhzHBPqb+3G/2lGKfBSC3Z5Hv + hP+mUzHaLjCkDE2CaBABKexSYE8aLFxbOuE7IaRwRbJaxx2Wm9rMH8BXM63Dabve - N2IQOOP4z69D+w+re2ALaMrHwQMzPFskuI2pZ1MHsRoFi31FmvGPrViFnd7/0VvX + 6eqjiRen6taW1IIhFI4Cfcno+Lp+cU8AR7c2Rryxe/G4SzaXCGNYmFumrzx7SQv3 - I1MCh1qJGkgTmMZgvJNST3kBna5BcCdL9TcuoQIDAQABAoIBABHQEo1MdB7vtKrM + RndTofKUfsnixPwZ0dsCi3jI5xLVAxt2KoGvDQIDAQABAoIBAAPiNGR1OtUlWnq8 - e42VsAEsc1S+GpBK+XjRnsQds1LLGTEfUvKqEooQiDlywXe8TxYRv3rG5UCAqvHz + IUIzbFRrI83wEQqtweRUBsbvzf/n0VOJq+XpJoFuVfKzxKF74MEruAlCZP3eFgdI - Mw9lAtZG0AxZfeY5iUl+0FmssHc3CqJ054t7wUx7LzPR1L9LwjB+b/uO55HV/qox + Zj5IVbhvfbHnHPaWmeIVtzExZrSJ2NSvB4M7eV1JGNs/J3Tshl6bR7a9szVIXCQc - jXsmr2l7igxW4+sICijUXkLBTHUqqu17sYEpHkMFKBfL57CXXkKg6nPwTt0CWFLo + iCJzRJPAtNxVGuYIUgSYiSPgLIc+FSo0Z4gp4fp8zHDK7ZBFF2eKQAJFA1n1H3GB - Ps2Jj6rbkqzzU9KSMUDhbkrck83yiBNHfeH/TkALj59E1e+8n5b2oCDW5YS9nNlj + qpBWa5ig0IIAJGjaphQqPD14pWSDAIxGPeESGnzJbiSKau+5gEcwTwuMSREdoGlK - 6hwOMSU/Hxq0NH5j41Krko61aqmojY8b3QRBT78YVdJinKPmzPwefxjq+hUqmUNk + FOQOFKCw37ILB75F/h7KzJjaFzC6LtiZZuZTavq78HDhJVSILW/nepJV455XQZso - 3XbWvxkCgYEA6yvyLeAJ2cyuDXk3LmaDuT9oOHcKDkp4uT+IfpbEk9lEapPzBTpv + UgLRhe8CgYEAzmJ/1YEa9zt5xigz5q89h6BLEIf7kI5609dUBUleVvLjFq4Ys32a - wavS5FLsSllr8yiv/cY6yf2TX8DolO8+1/rdXPPjvm6XZv6nIggYQRdI0TtbiREq + fi8LnijUl12EcST/C6ni4tgiw3Sq/l3lZM68H+26pLnK8BVdHMF6cFwR2Xo22lUK - 2JKbf6H4a7PR2ShbSTibpjS40VnYYR7oYfqmAgeBwKwqZ1pS+npiThUCgYEA1TOh + xmbD/nsfmIohTJVMpMQC8ClQgOUpnQJvzf3plvlol38lGwOC9rbj0f8CgYEA+sSg - IXK4FoMpCuv8Zd9RA+nQlDrZklckJchYCi/wxCREjjVuEw7NWtJlZsSBqYmnfoZ/ + +uzi1TLjGYa+x+WnRRVAF8Oe0yAXA7wY0MrnKQsQPBcmuoj3abi5mv/+JfHgXzfm - dAI6e8vIhuW9YFwV6vLao1ASgeAFCZgZsXkl9sHNKqKG7DTmOigPDrjvYtFTILRo + QJeun0uHcDHjyAQ2tDvAna6sc0DuCopE7Nuvn9IiDoL0dtIttK0qsflmf42WVo/4 - mAw+4zAeS89zMkr9i98MKdhWTPwNJInUigOSzV0CgYA7PKqYG6LflcsR4cKgkXoE + pDz03rpuTFaRbMv6eZAzRNUgkkZ1FcUsWR3qpvMCgYAbJUGDJ5QQaLY/phINiYci - o5AhCPsjdmbKYtKC8H87rrKpFfNVEc8svZc1pB2Y7MVgTpNmHRSZ5KHGsNTlDw6J + S6cT6Y7hGJx3OJ9Igrnx3ciYtxVwplinuDBjASPVNOuyphcVxaaeB6eq5bGH+3ms - YMt5qoVnZnwEmYiH7foOC0twSL9Z21UrkGJS1/23Q2hMhvnXi8bJKuaS9UqnzB1E + pLSBzpb6C6Xxph21Jo2gMbv3SufkF8NvDR1CX5dsTN7MX+bQ1Sc9x3FbQskSabui - 8Nn4EOQCIFveBMZ6CXHRsQKBgDHOQ+AaeqLXtSjWBDqQNs7hOlbGgLlNHiatbNPE + 8H7E6NEk/Ag5YWDcannUqQKBgQDtavx+lYitEWCx6kD2QRf88AGefjcA7IDdqFhW - a0yG5HUMSlCtbo+/Au1FDr1aaQSHyxKAysTM0GWjGeB+4qfmX+ky9X/do4+gNrBd + VcRFt5PHUKP6N3MHRT104qlcg4RKokH9JZ7OclPohVODK3ofafMTVy0ucWrtz7sy - Ct9gWtuQ6FAZ84a2gP4Befrtx6umOaD7i11rikhPiCvBlQWt75t+7HpDj5ZvlHVB + BUxhpDFaS+HoHVXomYqytc21NfgAPI7L8Gpl9Vw4Kj3FI9og/cWMhbwwwURZODSk - bHQJAoGBAJyLQXh3+DPK9QiZABRhkMUsaa4beMczIfRPuOilQacgjRoteMuBH1IN + qw5ewQKBgQDIOuJpbnGn18SSI0zat3hz6pFpKsyvk9AcMyLiVD6QJCwbp+fUXe7E - QXkTul1GOoIi3VEuHTSXY7zrqx0fDM9rm7HzmQl1ArOR3t0n2vZ/ocgxU4B+bc5o + 29Zck+c0djHBEBWivHbO75dHqWp9ERW0riwxSu7qUIgW73yxKJK4tI9nXa/yznlw - jsfcdXkvRK2MCKZiUL35wGhEPdsUv91uT3TT0fpqCHNF5jCDxVBZ + MULnyIHM/0c07fKut3vRS79Jm+6P+bjUh6KMCIBbZJO1nE+96c7S2w== -----END RSA PRIVATE KEY----- @@ -4962,62 +5412,62 @@ vector: segmentSize: 131072 total: 3 - convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:SSK:s4d2dnoiozjyzljmuemn7jmvpu:q76u6oplidnchjhsa6ds67xrk3aknnuiwlnm6yaymkfxzsvyhtea + expected: URI:SSK:62g6yna52dvu3dd2ccmtx3uvhi:3jrudt5drwfjwqbh6yfph77rfgmtpxflnijmdqzp4mkrijq2ksnq format: kind: ssk params: format: sdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEoQIBAAKCAQEAySMnWzmnU4vBL/wHXVsmXG/Y3WEWQKdhwGZ6xxmN8m/E8UKi + MIIEowIBAAKCAQEArbOweiMtfKQfeHVWoAU+zw25AZs/mD6OuXicSRxSZjnCapUR - PwzeEWeCdiZ3QySEAYQWY6dENVZz9TAbZ7nlwuxp8uV34JleUFPr+b9jE2rAqG29 + tekXCsB2fM3CjP38HV7ofiFKoYUSHoD9VJ3mXyFmOynosWWjkVyeXc2o7t8z8rsj - jrsqU6D4+/CsXGsP+qXhlfzINNuRU1vetPJc1VmKVEXMV1caRnuScw38/evRnDxZ + gWKuHwyFMuPZo8EEEx8dWSTMMprQkqV4tfd3xZelgNMBc0TLc0hvntzlT4g+CzT5 - Ehw0b86T/4LgZ6Nx3BVQQCDbGC0asf0Xw63ghCVnaLLxFBpM1ZGeLalk/xWy5e0w + 5H0VKvRHCDVtZnkw3hNmWuJNiSBkBRBBfoyJLa0HizR/o0kQBI1gkXBmSva+++Zk - 3ICnQDzP66ErZojRAgEVcYhDi+M1sHGB4yyIPLMxewHIr51IyvweI6QjeKWftxiv + 5pe07cCyd5cYVA+JD1Fvb0AcyszctyOcIFlpW8Ysthjd/UhXny3G9zHNuNqB84Xy - dmWCFZwJZFoZIGskK+XGwIhuUV6q2fRLcHp/VQIDAQABAoH/XimimH82qAz7okyw + 1ykwnMIaZq0/r0GfS8BF1VlVqYJlYhE4lF/RBwIDAQABAoIBABoKDGQa3uhG8ELO - TL52syzQUleGOjLM7arOTApskH+zXjeN6/aOi5dWfEc4ONmhbzNIQEfIh9qUT8C5 + EEjX0HUYoQnZHJz1j87FAmTBXqbddMQmiaukACTH+ls2OzqInqFGh4LU+cuh17gD - ruVBXboZ0DBXauW3aG9lV37F081kZF5BrYDqOJmnmaBgRyH3Ko8ot6JxXdFtCRWF + 7TYgn5bWOm2XGD9ztaQGZuU3/eGlSzPRkv6D7QdRiKw61PcD6dj1+p/Q8N2LMMYz - BsSPDnlgrgPBIljDzYFCuSYJn/xmbOvs2TD52pyJCeFLH+q1idC5rEmop/EOvwYv + ERfyO52+4Hwh5Z9Cil9DVhxSD/wueIM5zGWm6vOqH3mBhNK31D9/QMiWUdVFb8UC - fJj4dDrCgDWXNU+orDcVDXYp4OpzyKDepaxdSoD99MLX8FtUw/11ssWmXSR9CN61 + uESDRYcEt3R/r/AN8hPMptyQRoFXJLiZLt5ca9j4j4E3bNq0JQ2qfdn0nTjpNEQ6 - tD6F30Rq5q6K/iPNh1QE94CNDMiLEdBI9OgyPfywva2NIehV/YUeb8NxWYPztphe + zgacohbJ5ute31dBOf8Kj4VEw0zQDzm/vNoCKhj5WFjG378ydL9iMMxe9mrl+/oM - T6olAoGBAOVCMRUD1VJrhC06AFAczTeb2/YVF65lFZhf+YTjRnkWeUjoC5U+nM1N + c1llBF0CgYEAxOn0k2pDXC/lqW1BR4vfrzxH0xJHdl0ro5qmxwg5wgf/pxbbA+od - FP1qG4uR/hT28V1o+ZN9p5jEpcU7r1vW7ifMNUDC20GIi8xlpKv3lYfWK6J5VL5M + 3XhrW77Qj4kz+N2t56tMgr+YaVAK/iyWA6c+icG4HLEirKJWA6wU6ZCPHCpY5e4q - +UgmrDcSongRDAAbe/0r1drH6RlIC589VXRvSvGcycft18Hv886XAoGBAOCZPl01 + w3tEUH2WnwlrGo8AGdzd+8LaF4GyyP09XqHtcamyErhmmZ1loPhcO/0CgYEA4dKx - Yf2frZM15apNjy/1LyUE0YL5EzOQ0eEDPO/D6oN40qkiNWp3WA/prrj7lWObork6 + lwyvdbYsMzkVJ2XskJ6XE1it9KPgtmXkMCCvW8U+7v7QecNBhm64o47pMDwR6IiQ - 9d37gFnVwfXK8pP5+J7TPTsAsAt3yHrZrCQg7YiIJNEDTs2cirHjEEFv6d+pTOoq + IkBatSxV86qQsw4boURIbnQ7bSZAnK1DNie4WED+jc8tt4MERT6r3JkaIbubi0Ck - eUcJvyQwNLIfkJ5UTPcUIRH8Y8K3jGg80IrzAoGACOnN5rdDb/TmKqv6nyK/h83z + lFzG5dDXACRovNBQ43k9xrktREY9h8isBltlNlMCgYBOlfwo1PDbGrZyXor97cGw - e1nOleUwNcBlfxknAEYzaPY8nQzWI9U/X6rkb0S50C7Zq3wNWAKmpXXfzA9J/hQZ + osMbZqEkiNyAp5jFt++tExohagqwTj/rAkL+U3HSxvP57yaXXZLkX2iJJwusEskv - Jkr2NxJcW+vnI4dAI794fNOC1spI1S1A8+EtCOcckfZ3tPlclLdDlUH4ehcm/IXx + 3hAkVC6RLNRkx0jCoGucJzgmCnR+FwX0C/7gjK6O++hFqiplJ/NjpYj6dqWOdxqF - 8JjzHPmvjqpcnRmrLPkCgYEAty7FmqgLgBxYKZTv+HLBsk+7X+oKJ1SWwJwBUhCe + 6OPlR88sj3FK/zju/A97VQKBgB2kxdE5RhMirdyvgppgY9R8LQLKIlO563amG3VB - BsA36XsF9kScZHVqMbBafS1UrqUllwXrul2CVcLuK1aXevGKQZ/wdMseynur2+bl + 5SMb2m4PHxjMy940zKIT0YKWcBdhTeJhJkcgIcxRuJr4oCHkT8nIEkD6w4KNsAP8 - a6IfmhfQT1jvUOu4g1W60GRCz9T5kpOJztK4Pv/COvVbsob3Lx4PyuebRhkGP446 + 5NMY/RFqf+rWFQpt9quHoYmKEhoOi0w6fZWPe5m2LdWTVvr1YGmkx09uFQetDP/s - WNkCgYALjJBQ6eMDg2CmVwtXbP9ryHpgKZXmEc+BkBKbfmk5tO4bdiydTX1xXK+t + oXWnAoGBAKbOfQ3zJXUQSrBlWJ+mQbS5agy/bT0DRdkaHuYmXoFZ0nokQ6SeUeTp - yxVjNlVGDuowmFmLr6Mo9pE//+GPjp3udcwxUizdq35V7MiaYMeP+PvE2ZDldESr + YcLDmXiXYzXmsS+I48myk578dACfRUjOxlifK/Clzq7ntc+FfGx2cp2vHBIbOiow - qB3KGfeN5jj8lE3Rv4B2vHthqJlbxeFfATgRt60udOLT2MYoQA== + iZswZxsmQpafxMj+NUe2GfxDmtw/Lwt9BbpMqSsBYAsnpaZcWWWK -----END RSA PRIVATE KEY----- @@ -5031,62 +5481,62 @@ vector: segmentSize: 131072 total: 3 - convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:MDMF:54bul55haopklyyzawbnetxp4m:ww5wq3cqszi3qz26ytkcnqtc5msnkiu34at72xaq25cfdw53rl5a + expected: URI:MDMF:6ge2un5djzopz4fgfmi34eajbm:cqgc464puzka2edig5ofmwvciu2urojqsrdnt5jen4zvsdnkfuja format: kind: ssk params: format: mdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEpAIBAAKCAQEAvGBGNDSciyc1taYbXJ9nykEXJztT7lLadtqtA1HtdWBprQmC + MIIEowIBAAKCAQEAwzTPdh1mTkZYtu3zkzFo6oHV9KW+xbdnCKojoA3Vlktcy2br - 3c6tPSaal2HiMtbKrsXxn0kvgZrELpiQ5sRtuuP1484fsp56bmMr+rRlcnXOAz3A + KJZBkjATEcO/S4YRGRdsn57efyKpMZTMigTt1DyNHyoTYULNV1X6Pnpujz3zHgkb - 9d9A7aqiImS2nkwzg95aFwgXFAl0U1ca9ZJqHBGgIt8atWfJ0Ws//G+tKjG/CwGO + FCPwUTp12C0XhM2JtOY9Y5JO1dkECFzd3GLMHCGSP/svLa0Xwh9t54wrdkvl2IKr - zacycXbhARoneAlzAYY3DWlY7BYQfmC2yvG7lJcTrS2gzDLPzn+ffHS2YXWkj1Z5 + 5VvGrb/ir6E6nFi6kL3LomJ6iuTHiLx5MX7CYQieXdy99OCUbcXVOUO75ZIPG9hW - dBerjFCzgkjDeZnPyxSUk6BOLUhz0O5k6kMpd+N0YeZUoEjuHxJnFVkTaG75x8I8 + /a2cXIrKdgMVh646WV3QMgC6W+zdAHUvB63UyWCRqC2eUicu0v9VLnTo4+7WfOt+ - gUACgtwLV/7OYKH2Jsa/LU2Gc6HuQtVbIVRUlwIDAQABAoIBAAQHRQkRxPU+2emw + o+eUtZ3s/wuGhEmJ/DyFGZwrUrdEVsqM4biyJQIDAQABAoIBACkNVaXy592FSMnr - Cy7AY+5R1QZsQN+8Wtqm04NaKatUa/4c0XUf2dmQbUbme3ld2YjCR+gBnyf04NQk + v+JQLU7IEE1bgAPHnrkBQu25ixYI8lJqagEGnHKYfqIpRvUklDrxJKxq9kLJcMiX - RBkDQ1t3S4dbHZdqxzB8I3EhkXfQqB6un+VCKHfMqdn1cGhTNdjQlPmgHSo7B4dO + EO7ju3p7Y3hO2nWFXXbFA5QZHmAseJDz/EhfiH4kq7zTOtN4gEHVe8qRbdfmREVX - ug9dWbKO9+82+TG+7fRyCDTESJ4517h3mNTMeMLWZG6KolroqXn4to/iKf+hjj7c + 9maPNnqiCsY+1nymHs1525yq60b8r2q/mYyXCon6lyIUtuzl9fkHQdTvEBnS1zaZ - xvuLrgswUGM15BFaiNLh99DAN851s4HCY4AgpWZvpJ2/D2iKCB8B77kO+pBjxDqo + P2f7NHtA50qgFWeRiQWa68TNzv153toKDwo8sapJ/Rw5IdTOibttC75rxNq7t/3R - lxMU83akFBQEtBrnXdIh24uzQnl9Ls9+3TxMAaIalw+LedvZYH+xMzhKJOlQ6Elc + 51tzyw6+9ypoLLzRFWFo4bveI9zMxXLx1+2hy3vnt3fAKAL5pEHS439sDJ/1aq5z - bb3+NMkCgYEA4KnIMHU6jool2W2CrGHnCzF0f5uraZLNwzsDQQnLclDm41/IakDZ + jcNXXW8CgYEA/Z6Toz0rXojny/Gg6ya0JMhJX9KTtA3oLkQmieEjF+xVmO/TlvO1 - /vB39xzV6QdEBK/eGzZRhpf6pMBeSlHowrClAGH6qK9hJAE8DHNUrzDGFCt5BkAh + /+thSmAcj0Pd7mmt8hueNss+VtiUMoRiveoYDfbp/WFnqbUnGLCW46Nj0FSBFsMe - rSxqKiLQcl/dP98JC9YQra5R6IvOFuyO/TPe+ihrS0IWZMa0vaNkwH8CgYEA1qbC + 8vlvkzHejA0Fp60T+bivnLseYHX/QbaPC+bUZYRhYY1Qm37iY5NT4PsCgYEAxQnf - lYc4wOPoxaXYhvObtrK3qIotYUaMuyYilHlVfOPohpr5wd7N9fZ0B/Twn77uIx9U + WgJBDAY/9qIkgvRi+U3g+yobyJ35yLUvBiDOTDXyv6xwOHFnDAg/1SjmdyMYOksB - VhwjNI6V+sa5ZoFjqi9XD2ZmPZTbyEpLpbLuYMecAvI+KC4EmSxv2sGdscr7zDvk + dtRgNxq1Uuk4kxbA/Hokf5F/AlNcF2BVQ0+qmRimi1fUCMoXp/PIl6a4hph4j/hM - zdHGCjFh1DqC1y1XUMve7TTbb5/n2oma41/OX+kCgYBVbhpy2tEWjM/Ru0PaeywZ + 7fmz0d6iElX1AkcEA7Um8LR4gBY3iLP619UEj18CgYAIa7t7MAzAlssbempdZGuW - ZIfxUme/MJTP7WvSWoAji0IRKkYSqXB78kMcE7n/78Rcp+ekn2Ym8TndVk1Eo5sI + zQ+inttIny2WW6zr5w3DPZWZ/lyIJo9kb+xLC+Xm29oCkH+2CjS2nQj02TwScVLV - FZXY7Gkdpfshbtq/vUdxivF3kARobRChQmdoeG6dX3jJpe1Rs+gJs2TwMeF/dBr3 + +2/RBuG+B/3pJJqntzVLWaF2yVd/6fqdFqsduAornEMTzitbn0Y1bgEUMtbG18jo - i7b5l08dghbz4V+vUSepzwKBgQDMwBFIdM4MINo+/m3GfMWBxoQt/nA/I/7F3iCK + HEHxHPQeyRJkF1Js+/dNAQKBgBNnc31zt2AtxWLOePYEhzKx/rP9Y5sQI6cmYKkj - JBsJoJSDIX0wEwm/nzEbDeghWQzq782QvhJO5dvmdH0RbEbXZYTUKcdI4p+rNENo + 1e0favaBTtPgJxvCPDcLvhaBeENVW6GOLKOAl9bAbPffR8YVaT6+31klSG5s6Dim - cX+1TXJh1RS5WvwD6EFiF+IGYCtDq7YbJgiUXHqG6LE59AQgC/g/qHXQymVtLmlS + wdAt40jZr2HmNQovMdPtcUKgBU94TmspKhJC8IcJvAUrZTPQRTNzMmK6zWFDCDL1 - jmbbUQKBgQDA+CPfuyqG0QO3lfdsR/ya6fP/ybPSn0ovu1crEO0hruGppgipLAAI + IWvVAoGBAIxc0Zx74SEjrXDFfALloWGR2wisDA6XXwqdWzThVNwRrTgr6u8V9icW - VCSyGh++1DLR5+3ynGx8iSvhYPyYyrbdflDA8DXxijr4JNXboii/nlwmYsv2korx + iR5gFi4Akp4pmbYVUIKKgEOmXbOdtp3wkcstFFi128uECJEuveQeK7WqEbJZEFSR - Vd+mmSHOiFyKYShreE8YCCqbYtPOE2TFfoeHqTNkDBXsRlSHQaHgHA== + BtHC/5tsHyVRg/eA/yD0sU38mzoeoU4zpMJDy3SJnHrSMOaVkuk5 -----END RSA PRIVATE KEY----- @@ -5099,6 +5549,156 @@ vector: required: 1 segmentSize: 131072 total: 3 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:CHK:jvi4htauhe3qxdruqqxptzg4hu:r5gy5gjgwoqxeffc2g5gsgm274i3z7jo4d5klipof72zw6dmcmcq:1:3:4194304 + format: + kind: chk + params: null + sample: + length: 4194304 + seed: 2NVWOXvtwcAFf0J/g0fDC+ypQbnCU7awZoCxcnhpkVM= + zfec: + required: 1 + segmentSize: 131072 + total: 3 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:SSK:m7vqkjrvlu2kidmivmjwfbmtxy:4t35eoohvrpaomfflyl2pjxvpjb624ytpop7p33zqshtmxz25rta + format: + kind: ssk + params: + format: sdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEowIBAAKCAQEAlsm5+iJWoZQm5tDEpkgBP5OnGXNlk7p9jgc+z7GGDnvDBzSZ + + i1tLXjQJnQc0aMfJo4tRn48NbdhTWkFYSRfvq1oolO4xGEXxOgxmcIxK2FDyiexw + + UMI7Tiq5IUsCju8vI5AkYnLyPuPZLS9g2CYkIFXGeE2C2ptCXtBWMz8wH4ErzlAz + + Wq+B/erS4JYLw3Le+QCXjZhvDX9P8o+8+cKNb0+gJQeX2pa7E5JBC2V9t9ayMrnC + + q8M2K6pNUpl2prJ9VeDdoY0RBgA2ZcalViN0BYZWuBek83QmfB1Qk7/I5LATPTg8 + + SssoUGalhu+TDLZBVdeF/JwBDzqW7A+kc1NJiwIDAQABAoIBAAkiWzO91MWg9eJR + + jzgLcJfrV9oA1YxnZaeu5K0sMdS6xouvMgXxF9WzDUoH37LhN2PC0sT5o3SeAB6d + + ir2Sx5/3rDGpZCv2QLClg6cZuIb2EFsuiXc67ODFUcWkh5klABQFbU7Ra79HiiZk + + kldFqDaVO5qaB206roCT2kTsdPvTfXT1LBCKa6Hr/LWeoXFDNt5xxB6y+VaFD9LZ + + O+ykSIcxYSX6AYtglnSir8/EP+pmAM7XieWj3MeEgcFBlRjoT5vUaJgybAB9bR8s + + Tx9t9jQ/fICIDTWwZbpxp2fut438MFutKqnP9S72QSuhMCdosV8T05EL2lYUInzV + + Xi4gxdECgYEA1K1tlFXeUsiZVF46pC2+paPJwO7KmtfyVX/iKi3GyYfEQpR6ISth + + 75PuwUXLww473ydqHpRyYShZ8uZWy86/D+dsnul+mt25qNAnMLKuddabVy+2NX2s + + 3kUdqRE7DMH77RKxsfOKgmMkwbPz4uXasnblQ2GLu6aJbbxP2dLQ2+cCgYEAtYDs + + 32RZv+Mw0yMXwd5mMLbNLYOmfdVxvnvlPMid7j1QRsdrD2vgdgsbie5ZpyWMLF0u + + 1ezmqjxdCfZtVS1+jMLtMF1TtELAMbicFVk7HDgYN6HhYzmTeY4XZNxFkvMhnsJZ + + xFAeiV2fxQIpfES5M9W2YLpPokURE3UklBs5kL0CgYAaMrbh4+X8GpvQqb7dhIkM + + jG2I56Fri5hdceBhQ7xODPxfGz0kItzwjy+E/V0JTRKQ/aDz3WNtlnPmGPuuJWyh + + v+dAeBDRcOiy49lABXK6L1J5XfY7Bp0p0CfEMMwuWSL4ZCohepegUigv+EPdumTD + + QSQitbxpxCz/qIfJlE+IFwKBgCwWX/M3XfGVTvPKT2gBDJOCo74Nf3CLWzCoyZsF + + JA+NhyVaJTA+xOwHcK4FXnOSVEUmcUz3WWQ6e2MDH7WT8mxgoNqhoMZlfGfXbtpk + + rU4Cdid1Q9klUCQzlo0iUCgMtLrqfIGJ8JDvU/K3vrn3u4DSxZUjTFqfKjGuv67W + + GhqFAoGBAJEWr/O/5rWmYdEzllLPZQ9e2jgHXRMrlu2LozHqylEyJgmiZOvqEBV3 + + pj6hGSwX9Jp0cdqBaJEoFWgQh/ZYPnK9gydWYy/lGfIUXwIEfHJ2bG6tZiMCLw8q + + 0YuPnzqa1EW/R0vZW6J1BQHJBo6bS/Xj6NxTTGQh3S0dblebt8Op + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 4194304 + seed: 2NVWOXvtwcAFf0J/g0fDC+ypQbnCU7awZoCxcnhpkVM= + zfec: + required: 1 + segmentSize: 131072 + total: 3 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:MDMF:zlfyvqls7elrevid5czlhls7gq:bt3bw33shbxg3suboavalydnot6zm3ftie6br72swiiwmwhn4zua + format: + kind: ssk + params: + format: mdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEowIBAAKCAQEA8XlSFU/GAVeRwcn+hm/CmqMu9fmkYi+Zll0SBiQjY7vCtd9I + + 8eZiOv4J84wHhvNKOho35oAvZqCrAWfZ5vEM9fLW33Sr+TzmeXeKkl13WWlP8u/J + + oXTwYM4oiqa7F3OHwzGrbrAVF09ByG+9ukwZhhBf2LHNx7/H851WMJ1TQ9CES9r7 + + w+wRwmxlFuhA9pqhImo4upleKq7PfymzEovIWEVeuaCNrfoU9I6f1HXKcVcmLauE + + 7A7Jtu89sdZC0jeXLv4q1YuR1j2GaZJQv5MuH8kIpQyGhV0N6rBZ3FZAlc6+SREA + + muUveyC6lUDCkznoOeaDYZmMu++l9th5k0AJsQIDAQABAoIBAAEJDSIMQoQU9QoI + + rK+04Pe6xWPGmz7Uh2sOoRono4M09ePDvlNTMo6gMji6G/onJuVS4XR6jjl5bOJH + + qLaFyBFx5hv1KxuZeD+DFLQF9JIMkowvHQU1NCamG6RkjJ7QHv/mQZ7q4FxGObj7 + + Sav30ZAyl8adFI3Ls7bGsOzqb8X1p8magM+yerypB2PktePm7P1Jy+x10kAdc/vv + + +h6KR32NPTo6QNNoIPUGsn+JDFdP+cq3zHVCZRSZhwsLAaVpF9Gx1qZjaVs9K8zl + + yFUomMfdlWFoGHbBlqx3GW2DH7Rxzt/qVLoU/ShKiZTu7mWeHSiSUVyrHlzVkptD + + O4Tw5tUCgYEA/jXZa1W3mY2mq72xwCasHlyj7fjme0QmVSbmzVI3CdbF8jDP/8OY + + ZpFi1hc5ub3wnEYBrCPyxtUXBgPwbuHrk8j8e8TFVoPIUTIGkvZcc7o3LycRwIAf + + Q6HXcVUzT+PVzLxIcoqkKDeyqCZthqSmpvzKAVl2XoksjBRAfT2v0GsCgYEA8yyE + + X/aYSZ3BH3RzvUsWsfOFamimZTxws2JRbBzkqMuJpYEWp7v12RCU/XTD6+5UORok + + rP/Q/z43SkqM1HhHP7hPt/ge1ZrcHmDsGdqWcRfM163UoM2e4QLnlREcXYBPM2Vr + + 9KJHMAEz4bgmv4gTANeNiVUjJvP1t6TKSoZgJVMCgYAzAd8UWGi0mOWehDuMULYs + + iW4jK9QjW7NNVrbs79g3Uy74v66cpUSJIBby2kos6N3EnY9sWPI3zz4FaPjvZsl8 + + J9Hxi7QE/gBNunnzNxep6O11uqMnOw4K5ghypyPand6ibA0lXog9wZ9Jehxz7cm6 + + q/JkfuzvXxrfKJkgCCak7QKBgA/1cvaNS29BYCQ9Uz8wB1xEXBQgrBLmxYqwQCG1 + + P7hoKy9mamM1ravCL9T2bck1Cef5dEC6RTALGDvS6q0i+6IN6YVsTjG8iQehWr1T + + oB3p7vKUoOiwteWUeDhLOC5WtlvsIwqZ/8wBuDLvD/Pv7TdX7hz+LmFnD1AvC2ua + + qAKrAoGBAJyIAekG9AN3qgHNBdGx/g8DWaviWbINlL8A/7PV11OEXZ9NxhQfxYoD + + fSzPe0tpaHMmS5g5G9+9tgkzrPIy4oD1EPwV/vGOwX+r3lL6RaNZTdpa5YT/yL+D + + UF2ApoYl77aXQk3OmocC6J3nq7Z7WDjWAcHCrPJahFPnWMzKWdHg + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 4194304 + seed: 2NVWOXvtwcAFf0J/g0fDC+ypQbnCU7awZoCxcnhpkVM= + zfec: + required: 1 + segmentSize: 131072 + total: 3 - convergence: ZOyIygCyaOW6GjVnihtTFg== expected: URI:CHK:5anac46dpfsjumutzsoovvtlpe:yuc3gqe7m6s6xp7k3uodcg2g6k2bac32gt5amu7gdx7k43qnudxa:1:3:8388607 format: @@ -5112,62 +5712,62 @@ vector: segmentSize: 131072 total: 3 - convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:SSK:l2zvtygxn7tvpomu4vmyjqlzpm:bp32x7vsjk7gikid6dqgak7hjpmfyq46taj2qg2tuzdervxfesxa + expected: URI:SSK:dhq6qvx4nkx22cugbnilxv463q:hzssbukng6i6a3zkco6dftck77in7go7gsjf2q7trdisy6htwusa format: kind: ssk params: format: sdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEpAIBAAKCAQEA8ObaMnm+1qCV58BZl1ULTwe2tHBdVYTsb4WPiCoJerK1SvT9 + MIIEpAIBAAKCAQEAwVi6IPmkilRG5OvCNOPDKwHWrNKki412A//p643I2qdy1WqR - Rs3oZA34D0ViSYkaAdz2lMaH1XNOZ++8NTY4t2U++fkIbOe0xkjHQ1dspExAytwH + N405gGIlMmWpYnpEY5gpD0F+TKNjLEjpIWY1GLbzHsBOkgQR/6KcIBaNjBJ5GZqH - KmM/u4U4/h7IAQYg077x229FiaQ2UPdxwFOjG2aFFmXGI68qz6gqr4E2XrUjx09p + AwqHNjnHwmyOj+GIY0lLkqviXUJWf5Vbp7ngeWXqJ8i7eX8Q0PK+deWt7wSwQHpG - wfbY/D6aJ4JusrY2512rNbfYAx+OvkQLIkZ3XlbUiYFMwcS3NnPJbS9oHzVGrR2w + mc2UHjmn8zbp+5zBkvlVsxBPdTB2r9Yj8dShoUGZw3hDpfWEkUhVTYS8awHjSDBc - AijATGTxpAVXNZc6dt60rlbhFGesxRwgeKHYUeqmvmwkvq25wMDNIqAzOcXwX85i + BPzFXzH+LeK7RooBXvhg16Ar7a3YQh3bBPmtj/yuvuFNSwrItjQRCRiXoBARXy9f - K9m8WyPp6kh1lZ3UVdmLJa4ZnTti26xOv1RfVQIDAQABAoIBABIE0nFQFewr2sqY + QytfelzIiQcfPLAmZ79bwEPWHFvRUGYuhhj+TQIDAQABAoIBAB2qBkye4K7UhVgE - 4pqlK9FffFUGypRo+t5kmRXQPyFEWLcgmAlBwY4qVVGfGPjzHlThWDhMmUBn/Ydc + D/TbaQtFKfG0E8jReGTes74PL9zUShsSUZtrUIIxDLHxxQ414h5BrzMHAmCxxLp7 - sTExuxBMrGc6L10l/6mNLApncaLgaUBDMO4EunGmR1sKpl8dPDtaXvDQ49ylwcJQ + qUEVlFnpX9289ZETjMti4H9P1oHCJ9BU1BgUWnBoZwyeaTUMDkfla/Hh60YRsoG6 - n9uI5fxYsL+6IRXuNj+ODpNOEOkIclvP9d5V1aQmClbfsOxY0Mwvh3cfZRItBbfo + oahXLWiSyV03QARBCYx1YeFmzsvX8PLJmD6mwYoO0e1mABE5ase59vC4GT+X82LR - e8nRyPJRQXzUwvQ35e/e5TM5ootgDSqaaDRp4m05t5Kf/w+7Br5+eiOSKW+bgJxO + Rwm6gb2JE8Ko83lleAKnWOJ1ZnBXjh1V13TYDgsX3JDTYK+PrEe2/YDFm3LFf4o4 - OPE9tAajgFkil1zK4rDLN3mN6ZWikfSeIKPzSGNHCff4VBgLhv+tZO5mgnqBn7N9 + oWZAMnG4kc70e/MxEO4iZ4S4+VWB+8x4htSn+1VLnwXSIbVqbazcgtL7whbvMfY5 - 57KjeEkCgYEA+nYG6wnPLa7mScwDus/Lf1B4OOH36k9gXYhEoxi3A7loqCBL/fYm + 0WVU1WUCgYEA00wfFvlPBXpvYsYI6Hw3atiSGtAh8RrkVVIgKJTF5Q0frK9v5EjC - 7E0+4Qdkymmff+4G6HikvdLN47a7M/ClHncTnT3zwwTEC42VphPOLI2/uPDDkOHW + tSUkXj/Y0ssq/oIUZavKmK4ALD24rg3HV1ewe8Ih/M5MnBie/fPp7Fa1b+7HQ8m7 - TvYXKOk6TIUzUip3hCBiwsnEIOoDxzgb6/VccYvJJb0n7yB5PdV4OfMCgYEA9jq0 + 5n9GP4IAvH5DASxTQadQ4yneLudKWnbG/fzFZm8xDJjltoPjnpvNXncCgYEA6kBk - uAufrh+CYXBh/YUIUgO2kWsZ2PCHvH6ur0EmBkhhLhPcpLRHuFTZJbOqWpn62lkr + LC7+aYyjSpgJfAFbTrRgL6QlVmQ/RgjI9Afn0mWSJWcDm6t9RTJ27OZapaCadyvm - uytnc6NByKkbrNcqzKT3B9+2TynOcROoC3+9kQKMHiEQDwm9pUx+zdu9rolS0nlv + OLsGkIRWKP5l2BacBR1FeHpPw9MaZb7QXt1WT+dL/i6gi2j+U0BxRvIjy5bVGc+b - PgcU/FvSnKmqg4W9SJ1Bo0XysZpMyGEvxB3QS5cCgYEAxBExST3cmf6Y+JxlLxEM + y80fjwvA91az8i4DWpeT2KlhEyBJmnHGumdzZlsCgYEAzouHjI6R9znyrewFgzUB - VRZBhwYedaa94XqTgLoQSzIR48uksaLIxaOS3cZT+MDGw/cqIUKQdKlZ1DFwSzDP + evlPANTZiPUPpHOOKf0b4UZN4yDvUIjrg+VVwqfIzG17jqQbSjN+7HaShqyi3cls - khHVoPqmoLxSXFjyFZjbhbVRqQ2RixHAGwA7ESPDJ7P+gQwNk7lmluYsSzfmzUX3 + Re3a/28KiDQlYSUULgyDatprq4oO0S3e9ncNdUEgdSE7YGcyz2e9wwEHRnQjE4Eu - Vbg2Lg0n4gs5/9CEGQvLmlECgYBUzitoKDi7FAcn4DkfxC31cWnz89tXKKDXfxpT + DdNMJ1Cj8rt3OU19cGq+ewsCgYEApIJOFz11jBipgLRfTMgDILXKKwsC8bX7Parj - KjEagNtXr2eTIrSA/Fg97/+AbQBFK+kv8ecToOsLXZM2mHUZPsgGYjq8UT3VHFwI + vYVjx71vMncy8HsxwYvcOyjXFiRA9lpNFyA5TvqxK57lVSkjru/Mnvx+0g6KJlQo - edqkkygHSIPragNzZ0FVTZWrA4kPDNwPlQjZUhbb9mPQIMPsupzcyz6nhOllKnP1 + L8cPW5QbKUoDk4RLv5mtM97PRqYJyFOlnS3T8PiXLtykCPtJfbCfsvPY6b6uEhm/ - K/+NyQKBgQCD65f7J7czMm+3RJaFLEaNU6bYaXpB77Th9Df94xevGcNVhcL/A+ya + L5+BSqsCgYBsQScpNkJMelNcQXA8XB3d2jD3HN5s8qBFi4rPaCsrL+/Gq+qu78x7 - JRK/D/+Cq7iudYAaU4YdyM5XRjpbdGGl/PdqcEEgfnNAtYyQ2Ygp9u1hUsMCwGRh + zcv+7hi/1R0G0dnPLeZzSUjvV1snksxnAjl8BpQ2zfkBxRPRsfVuNNgoDBPHZCT+ - oFQ1FT15a5FDE2a4uUPayq4I5dupJATHboLDFCosuUxva7+GAUVf8A== + Aag+rzAJ8Taa+D8ES3pkoDJ2QiroClMmaCZbpy5SUMt0QGldlMAnXA== -----END RSA PRIVATE KEY----- @@ -5181,62 +5781,62 @@ vector: segmentSize: 131072 total: 3 - convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:MDMF:b6plxmkzdirhgwthfddux7ijjy:bdnh5zgxw2uex7plu6bzomx4udy7zeebsoquvdxnh2lpqetov34a + expected: URI:MDMF:3b4gpaaarpdku2s24mhbrzfmyy:gpvx3x6yrifjb3yrzxkzekewe7hvuyjrnqz3t4jh3bvby3bwsuyq format: kind: ssk params: format: mdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEowIBAAKCAQEA1Sg41bKZHsJPHX4rV4lMju4vmgToay4eoUhydPQyJN4/K6fZ + MIIEpAIBAAKCAQEAw6dMaIBLZvPlxEoB+56rygwktOgnaWOp+NS/v2vBMyb1Ih06 - 8rtcRsN2izSvodzuSm025GA4vxICN2FFMZPF1bAWaOJVszozTmDEJoO1v76cjptm + qT2JIXlYUHzPqAn/a9XCde3Fl+srlXenNoH3WUWzKvvbfXf4JutygPoJfiAUsZCb - Wqw2PZKSZK7eh14AzSLc9Bz6yKKS8uBTGOch4XTP2MMkQh08knpQAs1IzohqkNWv + w/MtxO/2Tp0U4KPDML/Lx/uE8e79OHwByUvwdgM+DED4vbqFAhRJGGU6v2oy9oLs - F96p8MY2q7JyYIH0NN1k05K+UiIdxL6Yp/C7EeaJV2yKVmgNy54mug6B8+m42QWc + No5y20W479nGhOF8jnxet0FiWwX7ONZtFimqBB77zg5rpBSf7Yo7JTGaBZtL41W7 - jfyvq3nQu3FsaAjxaDbaYLrOOVV6meWpk3U9kUJoQnQ4EAMzUa0OU4e5GcVCl6v5 + YsyEklthyBS+H2DqQXiMONYyBgTwuSRBUYbBstgm0/gNGRY+aIJnnVtq/PedB9a5 - yfR4Wq9GqgtSLXSie3z75EcF5AqjxxlxxFCyTQIDAQABAoIBAAcxyFlOIeTr4ge4 + 5m23qm7q9tXui0UaIGp+wo4/aG5W8D4xd9yy0QIDAQABAoIBAAmQf135bZY0FJae - znWx6KeaWnj0WXPkppwC+foAlACyj6dFjxGmSUMKLeIc8SCheFmCviuPI9svHGwK + po+qH0xCgTXdxnVys4+wOMpvBlQNkryu8Jvu3+oEwFI4877LdFLNcZLpw3fMfYYy - GG/H8RF5VAhOO15FRJ4MnhI+t8+0+0vE6vt8fIgvfklvrYscHSLPXm3O3JgRBKy1 + QfiabGDPFTXj94Qf7f/bES8oaffiMhjHELJIzEM9Fs2bhLaBkuxqZ9gYHdTk4biV - 7ZgVlQsrCijizUJ+AiFfh3vQufheioR9OHF9ZeS6WP8iEnoSklnsOppWjX9CdysS + 6VFxlqjyOiGHuJEv0co6+yLH6hIK1U7WJgsKMkdKKGQzEQt8mAqz/PaGZN9t+KO4 - 6t+3mryY/LTyxhwTTfco/dhdUG8tmL1RIhuI3xTrhHdTDuEAFP37+9QoxFMMUGIE + ovUSDyoCzyh4xiwK0lzhnoI8051Wg1s3x37mwimqbT9v3zU2IDWPaPE9is1xG461 - 1nysscCIG4lyHRX3JtTrRIWjXj4z74do5a6E1qpjImQGFzHNwCBqyJ7n61Fw+hFO + z+ViexDRQSz8G2Cxm5LfEhc89wi/k9vgUb9nB5f2n8eQP2FQvNIOcA6VqI/bkhNh - +lU2vYECgYEA1S8IU2F4HqQDOSDDf3/ZavZp8AcHdcTSDgmcBQ8x5Wd4VP8MCn91 + ja92AQUCgYEA8V+pcG040UqLXPZHszQPRbj4LtkDBwBqhX1HqzlQMHYHiiLg/1r4 - wMN/aixfHMAJiy4Q0qfsgSpCaovUrw/uCSY0uCXLhc4PC0KWUgjhA/RPtel+LN4K + Woe2hPL2nPxnCLxxBN+sDz73wpmOBN5fk3Lhz69vd4GvgX/5sChXFQy3o/P3xGCI - 56tnaWvk/IMAGzCXwgXWpFVjAkiiV++CcantUS//6kPeAtirsoatvfUCgYEA//fS + rcepn35M4AJrL780C5gy/WK522/4zp33sPSBV8lOcAaSdkND1NUSALUCgYEAz4Jl - V7IFZnt9WfUNUNXjvfXvdeQFDYzaqMhRkecoGTrlfUOa8AByS9IT+QpPefo8xhjq + oQH9K1hRtRoTbKB069aSneXtD2/f9D+9m6u+6D1eSZfYc3jLiIXZ1VTIeXT81Oww - q7shAdsv1PXNF5vQJSAZ/wfxiqOtu+rX2PyN7RZ7t0KMqm8Mkngy9dAJ+9QWbdf8 + /IkjsHuD1LraLJyw63AcKPAK0BfJwhnLIWBKoHWZJoEMvY9Xqq3wnef0+Jfa8kBp - BOlSdw+8qz+Pfbnm22OCla1FdjPQkABcqP960/kCgYAegJE/ZOXL9ImlheOS/Zb9 + LaiZ77fRhyGzPQICOmIBtuIZYiYjZi7gG3ibJy0CgYEA18yAL4znDG9KM/3YUsaL - L+6uckMF/bhUW9mf+7GW8jwMZUWyxtPxVceISHr/YRa8fEXZ7j7vqD1Cg2lV9wCG + lPlvomrRAxSDJ/++8L2YDQupZ/4RDRxnCIFnVGvowqgC8lOP9ByJt0PDvU8OIxox - /Jl0c6vwJDCQ2uEpMa4IY8935sWv48FJroOoWNC1tISyXzyHfVBdyP3WmM/ppxJR + dyFx8/3UeZMPt4cUVENsv9wT31iCvybTbBMjev4veuOOsyyOOoODqvj2U9NDLm8b - 8w9Km4SRX06Ht7qxW4XGdQKBgQDmR74kxzO0j0SmuZ/RKZxKOgfEt+8T0bSmRBGe + ATFI5pSLNSsbDPLMlV897jUCgYEAyp4Qjf5bLg+2+JbVkKO8huuljfgMWZ5rlxsG - gafBiwsLNtcdNEmfjNALLQtzYX1ret8kwKVhViAiJ0DsDHGl9MtudWcIo1iZxx2J + ERLJ/gquHj3eZCH22v+Xi+6VMcNBfMaDrpJZ/uEcAIPStOzq83ksheydIkOYBacZ - SS0mLyP+KxECBAX7f8fY/eD9fkDvcXB5uq9GDhJevkAJjEX0+gFxRwFG5jasVqcG + 6SUUuUkambY4sn9copPk9sqfMH1WlGTATozqgl+Cf+gwE8n6UvePpPtwvZ1vwz7S - I1INgQKBgEuTunjdt19PXq0PPe3F3Ic8smXw3ggcEd7fRFe4YsaGxr2kfiz+Jrgr + JQDGvqECgYB99QTs804jni1La1QdCdbumonHWEfMrM60csGkrGOjLiyR0xUwQyyr - E+t4x+0wvKflh/M3R9KGC5k/8t/jyphFB47GOk9carUOglAy+LJu/3szpBWPdPZ3 + cokRNpIAp+kNNSgM5QRK7WKXjdKrrmPndnV+FUr4GN+WDJKs8lumCKHveE7Cn0YG - HGLxO5E9bjBSSdWKRB5NtfVZDiCxU19iPeCYaCy0Ji4k/teDN0a3 + I4hG3f+HgjdN6HXLN1wgCgP8yufvRwAbsIYkm9Ok2hRrfndxErdxHg== -----END RSA PRIVATE KEY----- @@ -5262,62 +5862,62 @@ vector: segmentSize: 131072 total: 3 - convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:SSK:ekyljbpy76pxtpbkuch2m52nla:f3hirwkvbj74mltkneiipttxccixja37xjx4vqg3oavtfwexvsgq + expected: URI:SSK:nrfalxtfzw2mju7hd5xuvvqvhu:neuq5ftw4keuikytduj7ql5xmo3pzp4qpdsn2zt3magaivpzyrja format: kind: ssk params: format: sdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEogIBAAKCAQEAiXTj5P86e7L/b9Y1RFaIq7xNt/yjLJh+aqDXonoNLNv9poMN + MIIEogIBAAKCAQEAiPrrD/TaDA1eRnPu7gUE2shwo5Knp/+WbuH9TdEkY0HUa+G8 - gqhI//0CjKAP5KSMDxKNS0PEneoqCBKCgGKm+Psu22v7w3pyJS4vZjT9Pq2LMqWa + j1ohiQsILqaE+EdJRhmpHb7RE9ZtoosSTss43tlfvyK+XevfWy8XzUExXU8OhEEA - AeXC+i+swUp4B+bQemek907lY0mQo7v6QE2yOCa2URxtYYvOP3F7Q3Zu32qrv6DS + +OYCvGPF1Bhmp1AXJ5HHLzieTiuY7dXUfsbz34D3AqMPRHUx/mN84D07rdFT4pDM - B3Z6F/4ZRqiOTZoTDV8++zrNrSp0ddOBcN+nT4LlDaAEE1ke/TT581+74KE8wUsR + At1y+vMb2Gjjb/iJP52YWlcWl6D+381JsYASccx3lWuSOMz64gtfaNCzyE3TpQ9S - h2yKUkKjf0Z/x9BwGaOvfdN9Im0FTNJ/W69KViOox5wKRK6pScWdmIwrtiaXZw70 + 47OCIf36VD8B5AW6E4qgenZ6tV8Auc+QKHmzFkxfTMeGOZOGNL7F0xxoKp8FlNZq - nBRsrnJzzqvQZuBKOCBhqNn4vysQ4NTtMrmnoQIDAQABAoIBAAKSGWer+k0Gm6Oy + XFkN9syy6+gTdP4ziWtzKl1Ry0sqcU230MCY5QIDAQABAoIBAAdIdAwOpDNE2LDd - JDbjThUK0C1JCp2H5egFRWMi2IzmjweGhex+oeGKZXokeDJKKDBpTr6B8EhqcQog + RCRinu30/0wrJX/groJpwJqNFqayXtV3lJt4mtTbAc3dK6+5tpMkFSJQPXSVD5I3 - 8SiI2nSSRghmgZk1+LKHUEL0v2kPrShvqRdbU6/YJRP3BOhTlxc4SavdDDSKKKwB + W7tVwcnTe+xBMCb1PhRujhDrOPExnV95x0/0hsu2cFPFElwt/XUsoo8Hrx8P4Vsv - 6lE6OBi4E0rQYZ9PEmFktEMeZj5uukNzv6aFOAZonK21JecbnabJHVHUYsAjSfkA + 5dLxyBCnnjqFKfAlXQmeB4sypkQpC6YpiaI80hkNB9a2aXZr/vc0IW5NUTPKe1h9 - MNQ0fJfYR5z1Vkekd1ewmg/AmSA8V8ziBx5Bc1EPXMgJIiVgZQ1FpXQq2SMFB3XL + Hya4OkFs0GMuapsvE2oAhpESU7jBuAvA3t+mtFvStKzimYRS+mDKMHp68LWsGs15 - Re0RX65gja3IQknA2H3bDYTnbBzCivlFxjk5//JMsXktwylJ/mJyCVQ1cwq6CTmS + KHTiBSCWIuWx5vs+r87UmD/4SjDE0ANiIx1bXxitS+NU+R1vfOS5xFDwKheeO/Xn - udeOyj0CgYEAtxEPS6cR1UWbFAiplP1W0ngRaZJKLKfRa8TLHDIlt+wIdqA78kyM + BjIYKNcCgYEAv5JRvXmS5jY8dfY1SfzOcJYx4uGeLx96+8gUOpRNvSprZM9zliTb - 2JcVdyvS1KqC0h0aptJftJR6OsZ2urtJfqkJ0OG2iqCghnGBqHIhjDus42xc0gPl + RCJu1ums+nozq3DDzzF6er7bCMHxQ501GH5LiY9tXQEkgYvqn7oFLs867N9KuEnI - yONlW2KJ9vEN49Z6PctM+uXl8YbQJWbDMHgSTF10uyhr6ddxiPsrFO8CgYEAwDgR + OyXTH0QKk0Pnmc5UaYOSMWtEshV0sS5wtTg7xFnDK55EZ3SNgk72m48CgYEAtwx1 - v0VoOEoSumbC1Q6AOMV2MUDCjquqm5pfHESl20SpihZN4UEbcDWJ4CNiHG/yRMoZ + SpOLGpJOCY/mE9wwt3xEd4r1/iBXJvvq5Z28+C7yJm0o94P4SDW3TquFrBMcjywJ - cPvOeS0+utzBHHUOVofq94D3kP6G180HhDMmr1Nvcuxr+NMr95On5BdRSxSbcAAZ + MzXMcS3w1qODA9HkAw4+XzLIhgVHD8u0+DdRp8FDXq1Ym79BiVJC42LD8YyuU7M/ - K7Xtj6iHQ9tEqBDKY2NertRRjgj65fO+/zS1rG8CgYBMFmwcDoGL+hU9m2gYg79d + g8SAJtrGTK8nyfuSIm3v8jT2Dj3A0GwFATnDmksCgYAX330ONqNGywV30cnMQZPc - VQgvr9zieJHDUBT3UCR7MEBIRcsEpyp3Lzx9vpovR/t9pxkXsyKSJJA0854PeJ5Q + Ves7kdAroSmrTMCwmCCj7TBa7LtDv64PbJcRcydaQ3ZC7BeKr2jK+RPEoJ6XRXUD - ZaOtzNKZBbASkQTJ5T3qUjdGgxiFNZeBCnprJCahm4khZFiEbIY/VeRfoZ/Lm82O + a2Gwb846I9VPy4behsj0j2CRejYOhytLq6gGom0K8xBei2bbi0jhnbN+2cuj9NyY - zKkWUlWdIGzR0Xjf7Tz3wwKBgBxJI/Ntl0SRQehELu+DTsML67SbvwWXpWd4c/6I + yLwx+Nmoit2NYunrjjmPIwKBgBY+cVJqs5C7Driiv/bR3yms9DUCsfn7vBuEqXrV - 6480r24ukg9PsWX1uvBMxKdCofgVdWD27Q9P5SdCTPiPESkSnzUEuWmQyu7+sNh3 + vEz8h3ib80qAwv8jZ+8rcMcEW4gaddO/SeTHDGlI3XbtXqPwayvuY+fFZGlK++bd - Xn32XTQgLlNTX+jyxYX/GGtgAO+eVBXmk6rMNft6TMQelGnDua8od0fbcnBcSgLs + 8hJMrf8nWYkzqKcjU/WF3wHPcq/BLIq6qkgOdeKDtnYZGB0O9wWb4frBDllFhyYq - Er/pAoGATVJwz9ZT/hdfo7kGhk0PA5z37eTmzcEVn6AI6SzMLu1L+bt88x7nWMKT + +tYZAoGAHwFRUIo8mJXHw1HoHrYcCkfd541hl6IAj4YYMY62SyVOYxlaDUUVxEMG - K2kIMpK5N7shLOSITl+i9eZxM4FiEPGGnW888gLL6oYgoYCyiQqlkShusIe7siUr + KX+wyRsKO4nthVDnkzCRXkbAXAdCatt54PGRRxg9wwi3xwZ5lMVa806AAwnXjKke - NrHKOluBZrazxXNjUl4Eh/o2heK16ZtxsFSelQZMMj9p96X+7lc= + TZ0ofqZaGVwqE+zo1gVwqM81Vs2QsKSP47yK2/k8xS09dHWp5CU= -----END RSA PRIVATE KEY----- @@ -5331,62 +5931,62 @@ vector: segmentSize: 131072 total: 3 - convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:MDMF:osegaioo22obqkky5yolffimiy:qlyeat5tp7tktzlxqntqqxxwjkn6ayi7zrin7ekmumfsfezqcjza + expected: URI:MDMF:sg2kk2aboiazxpzii6niebisru:ddxs4jaogxyavypn5lxfqiuymic7ghlbiesozaq54ubfiqpjyyaa format: kind: ssk params: format: mdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEowIBAAKCAQEAoVz4o/FWAv8hDRiYcj7Avu/oWS0xUFJLLm126YeKzo+KpEFY + MIIEowIBAAKCAQEAxz5hpr1q3n6XotU59zDx5oXx7hZil0g4WOX6xHhi82scQ2dE - gJVWkIlV/blL2JQQMMq5gEJvg6LqSkaZIrD6mr3lhIRviNXhs4EWsAkZC/mch9qW + Kfz66yEpuGBktOq7Y4xhXC1w8BXY+rt02f1t3tQpmZ77mAZWbUmR+y9nFU0MWs+e - QlCelLb+nJ47yL2gShZnSaWB6xM8YDyvB0UydzoEhyzUGEq9x94oNIDEDsDmeq8V + SHKTSKynUT/AWy86PNdQd9bkvhIynGq2WUwHCpE9ZjilkZOZTrgjBs2IavmkkH++ - oL81berDsOJX+c6HuD3IXQcd5mkj5VIM+9gmlWnv3oI4Aw26ip0nzveoDlUycdKN + DVje/KBSnzDebi79/0kBa+O90BiBIGNkfNMy8nsJJn3NaVwbsm4cVgVNjns0adDi - kzibaMqX/CqdBxthrCOcQicGM1/FAd0cPQaEXqu4Dhz0lrdAsSYCO7LGrlODDStm + ar8fxlzyKYlwP/LvODp1u38I+/9sxStKtzlVYiCD48ILrwUtu5YBSAs89xnnRIrx - Ar0annnzaDxQ4B4blF5t0yPO38imcP+Gzla3twIDAQABAoIBAA0Eeg6Hvp+ZQ2pS + VJztUMUfXCY8czyThxEj7QGuac2HH/o1EWWDnQIDAQABAoIBAAWoxnJLgA0Y0GrO - DKJSy6vboMvo6GyJZwVE0W3/gEQvskiT+PbOlWAtpCFG3IaJU1EMWbCuK17cOrhp + HqrgR4ayOEh4d01vvjX8Nk69UqYyXSem978vkdr64qn7hvDCmBcARkWtfr87B6I7 - P5tb2au4HBb3tCO+1WlsxY7H/RxJM8aF7M9Gv7RRmvK5lSsZmR/A9O4tCvES6TD+ + j9qfQa8qHJJh+yh6yZqqE8zsMEPhH+A2dHr/UiKm0MvUgTy1zzQqM31r66xgbPKX - VERq4apapNje1fFrvimsk2PA65AhAOteCtYkUeA7HReOttNr94q1id0QbUCzv3Dk + b12C9NI+hN1x73X+mMStI0WOsfjD2KgJBqm4Bl2b/jBEnr17DOVGAHIvfSyjrObk - 1SaXwPG2Nn6pQEDnUenhBeYFmsBjGj86mQkuU0eE5gU2L3YcAife0Xbadw7R31Mb + hChdjgBQaiW5L6qv97f6B6CTqdk9Z3sFIDQp3oD9veim10nK+6c5akQQ37MFERdO - MWta+wfM+19+C5vGCF21N6mpVNXPb82ZtxJ5ZhpSRgy541TiR3Bb08JwlaKXTSDx + k4ariqAunYP5NcjUeZWEu870eZFoQR3DB7ucza0Zt+FAvw99f6ijvX7nteVUpZhC - LSinc9kCgYEAzZWqzsEoTqF4//9UJs+NBlUUVhyrlLNm8fSiC3TUzFbbzIxxapAC + aqLrbCECgYEA76SYmfHhhBHv2XRI/NwY3/ACcW3xxPDpSLilT4NuFywDUrODDVzj - xE5gQdpxpYM/cfl55qsdSD6ngv14XWShXJ8D0lAcdm7IG/dO2Oquepj5ejJkLkm5 + //s+/b8PCUW5kXvR0MXGKXh8ADNBcbhym3nhcC3XGqCo+M2n78ms0yRF9DzWDjn5 - bQQfSVuXWE0n+FkjQSTJRL2oCa8kktg0z08dVq7EJRiLpKO1S2ZE3+UCgYEAyO8j + yt0TQsuEalHMk/sL0gz2RUarcmyt0RrfM7WQJQHdZ+Vx4EaQjp5+S80CgYEA1Nfe - cytcNwFFltLUbo5NkF1xwk9amcC8XZWkuMdqzSwl1HMNas8cc1g6ZCMg0l160s40 + PlcxwbJ2MO6PLWaoZAO83wrQBes9CSgUk8mB8CEtCvA3sC6gEl8ihZSD7RuLjzSZ - IUbAasDD3GLFlZEq75tWPvIX1UlEtXAd/5pf3RcwB7N2xbp87KAFcf5Sabltgmvo + nVQZQdLmHbWKxSnhFRK1gfIqnFaxr0HAtrDkRt6V6OSAV6rHziNvYIgtXFKsw29u - ioqMn4PuUWRDRQnqWPlqJTg1oWSbx6MNaSwYZ2sCgYAMQ++q4i9LcarMaylUH3Hk + 04222C+XB+lDtGgIxt+LkQRq69CKSFCTQ++PZxECgYAOfqosRZEaZ+tV/86aXMW0 - fNL3yEIcXw+3Q8cfM9s2TcBTVdW2a90eZSatByFcpJX2cNHrBy56DvLjh8fUmppd + ZdQAAGJrQxcZKvH0yUJTbHoW+nymxkOULCI3PuMt8GW1AwRB2HSP9ZWqfW8r7bgg - 8kbCF3F7R2S89mZH3siGG/ZWagc8E73yWRqcv9ApvoCx+m92BYHUjhQmb8KY2Dle + 51JXcq5cEfOmeOn7evtVGhCRIUzhN2iAeLa9h4nO1HvHR5wDbH1I22lrVl99El8F - XPP9JfQh2nMKYZIBa5qUWQKBgQCFqSXwt5g48ryizo4HGNwZuz8wHW9MNbxXmHKh + xameU2qM6jflFN+RgMyq0QKBgHymM4zU6dnjVx6PB6DyJxnzqnABWBSvUJ6FL4/h - g+3Um5hykTIMqcboJ3l4ITH1Hb/VONvOguz+VkozcPS0QIPKLY+agZo/A+UTuIgL + ikyEUWm/hw2SMMKxnnkWojCBWjky9+fQsb3/8i5h/HQ9c4kw3MXOei/3AbZ+zorv - /lnkjUci6EuKzjnJgcz9fkq+D138UuG2PuG6Pp2qQMLKywS7uPXV2mU6fd1uWFVU + i7EJeEfdUmCFLuDFldu1xML11CHcp84Th4qSTGQgszr7VnCJyKXULX4PMnzpW2WE - b8OwDwKBgGThhPlcWEAud/Vxg4quwujhlOvLEasu8V6GvdAlIoRXqjJa3sS5R7X6 + 7bnRAoGBAOMSiSaRliUYjKBF2ggI7tfVEjslmKfg6MPb8Yuvt5Q6yJ9U0qDqghIw - 2aLJb+4dFq9KrBqX8my+PxR4HqvxNTALhC/JhZ1Pm9ME7j9jIM9ttXDDuEL+SG3e + tFgDh3H9OhFiJSx4SoP9ouKE93iQ3F1XPZa06uJ5sWVxnSL0tVDhqXHcEM0JCEB8 - QqhyjNyRBb0aWtwXSGHEFOzcDWTJPjSChhfVAXTFAXNhsc/qpeZH + 2KGer2nGo5KcgBbwxILp7v/rZdjXJIvCz/ktqUneyW5HQtPEffnI -----END RSA PRIVATE KEY----- @@ -5412,62 +6012,62 @@ vector: segmentSize: 131072 total: 3 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:SSK:i22zmzlu6qlcbwjcxfughllk6e:emj4f7kmqnwlx66efhdzuy4llfuayx4iasv3vhjtl6y5x5tis3ba + expected: URI:SSK:zyx4vx7xud3ibmvazav7xf5omi:biaeawjobvdp7v26ed4re6bfc2mwis22aoolyz6n7ciptaqktpua format: kind: ssk params: format: sdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEowIBAAKCAQEArG9yGPwyUNWwPrsuF30gOZiPK0j3vSM8slD/gs3lJmk0Yh/0 + MIIEpAIBAAKCAQEAuIn3ExvxrpPLR3mwsmSlgk6/qESjM+a4KTrWAwDF6W9CFl0e - IZ50MDYofU795rgGYQcQxEqWCVPkf9g3/ssCv7shHIdfAkq1v7nawrqDXAd4CijA + +jdauS+LT4yWOwXQ9jemGfwUN6/zNqZKwHl2IfK1nxdLmi2yDs3OTWPSNQ/BV1gU - 5pVmxttULzFEYTG+0ATOBr5PRCZieeU/K6v1Y+5gUi8iC29MuFEa0aSJ5Z1fU4NO + FTpRio2ZCJJ1prONHEvuABzY1un7CCmN8jKd+VhCPXMZ4AS/RI3nmREP/Hhj2nkf - cnIoNOOb0GjvxZR+PbGP37zReRLGPzpOD21XH6rtPoGgPZef0qQRqrPKYwdLmNEe + 7PMDP1RCvk4zl7YY+IqKCk7IixK/4DH9FaV4DRiQbk0W2DeYRtaGjf62AFaKSVY4 - UUf8YSBi2LmzXpLhKPW5V4lYQ09sVa69WOW5kQtj+xGINTngGC41mLC0v9Rksl7b + 0PnVtt450CWGIl152YyJWdP83efIxT6MbEF9rayPZqxfGgXswFSAAKxtTY/EFllg - 3bqgNWXYXFqzEE6E7AazjbbN22KNqOVhJkHEFQIDAQABAoIBABj7q6ENlx+pmje4 + OJomAqQYTZLo8Qh3HsM6YsTbrbdU9bbZoNnnkwIDAQABAoIBABkeK54okziC72FW - iHDMRvk5TpLzpzsuygo+3Io/2RHD39xQq18clVJv4lVndrdxFbGEo8wAz3SqBDr2 + 2UvUJ98DtFHv/b17wvIBtc2KK7tpTw0s4aGHKqHC10By+fkmAoTw8CU19X6FmXeF - oYRHtw3+54j53wzWtLcCzzxz5/jTNzPnnC08W6/3kp6kyXa4jaAXdh85fwQNeKqX + IkRD2WYnhhHAdvmRDLJieu53uXigILMweZNQxnaIbXMIPwamBn1P1yD2zDZRz2m+ - CZxC9ZKFNMzrecgE1+2DiLpzl6wd8ETSrXqNeYLCbiecDkSG8H5o1h2ZSJu471p+ + AORy1U5yH+9YnQYBP2jSXoVWwt6ckolyduyhqJbyFtvzduK+RCXLtYUYSz/g2qTz - t+l/EJZU0Fgk01RufaeanWLCd6gT9lDWnKEzGiugFzX3NG3mh0ussBH+WBE9S8hL + TZLMeL7GC9ahigclXmEOhxrrC6b7+rNNMidCWR+S9JwSXZUUOld4+4jMDjC7f2+Z - C8RaN/q2UMfASiobyO6nZYc2BH4FvPEhJtWVRfCMTT3V6nsNEhmdNrklvruJRc5X + 2dQO3LcwTi6DJzuEIE+NAw0P9D/43VVjw0LXsQ+1b/Z0Y60SbTs6qa23KtvrnMyk - STpnoakCgYEA2v+SZBSmihk9cEEF9hYNPvmcChy2aTrCsz9jYyT2/tEJAKYnD6Ai + 7tZDxbECgYEA3gRy7JugmPf8Ir/7ourB39ks9gY8Lrgx1OFd1jKDPicmm1V0y34I - AEnemI/3aDjBi0Y0fnsgH9j5AnjDK6sEO6l5gAaTkHurVUrdM40efLmwnMzz1uRd + YcXqIVgwkui/C/PXigYselySyIun/hRI0l970W5Koo0Gizy1J6vgAJKsRpip8Amc - CFIn+wZ2dQ+kXEwa74JlfOMfUQ41PX04xFdKksnf6dGG5CHFaAqQET0CgYEAyZHd + iXKdjOZlJqBo89Mey3QIjUuq95bdwKM/w61TLb0us/tCNN9F1QRk42kCgYEA1Mj0 - ZrQgEh09yyWj97+9W2Pjk6zklEimrxPelytwRQ9WfWBDjfWQzCG2h8EaDDZ0oCTA + 8SfE2UlERW8qS9nG00UUVxXI82CPXGrcYgW2zT0eM9TlK62eYDC2vmH3xZKQ1b3R - 3FCXlzkM5gnaMnLHNjfl3QY9irKiqs/wzfLz6NpyeFhGLTeaCh4p+VYmmjHZgL/h + GfBzIxFSX2WtE10hnST3CuwhwGvxH/sLgnxYeWvJZvtLsHLgG+HgKZlTkbcnj3ub - cyJsDB6h2cErqJJnH25XRZcsQkCaF0hKR48he7kCgYBCWBZzN0ZUo9zW+vvhV0Dg + 6F5HdijrpiBf+gJwHoIE1aPeelYtqiOcuLeWn5sCgYEAhsEoiBhFt9L8xJLGRzI6 - CSJadeRU8LY3M0barEIfZBhEGBHRTAPA7p/+u+6JplgL51LT1l0fCM43D3qg6gg4 + DoYg1gseyDSgeld3vyTVqAnXUvzhcQnESKP54ddHVEPUgYq1Tl9E69f4d6TciEkD - QtlKDbP6m1yGVE265k+MHX0Bo51jRn9gm/L8uzJ7uCdkxrGKSYiRUwUTuygp3pup + kjzGSG2q+1KhoC1uvu+BfJeJ3SeYLcuHqZ1Zp0XIK1O9oBCKZm69KhW9ZZ26Zswv - 73/qBDpPTWh+CDUTlc+bSQKBgCPzkUKlM+cnMgNOtl0U5MgtG8UWHDrabmhhqdza + TbOMAv0Ktc9Rdgn2tr5+BdECgYEAij7BvQg8gXtzirUNwtgLsGmaLHYv58edfMrE - kY6vuqRoDASA3Q+bn7u81FGDUO/TPlbNRQxiz4skDLfcwu1HsQbn+wgG7n560h9Z + wj66JKAHxl8UQYt8cTxVDl4yD0AJL4UynGq3M1pmrSovB3yjgShqBMOjrhOzRjbh - iuloNOyEChg8h4vwb1oaZI4x//I3xxVK+Wx79jAphQju+9eeTZCK8wjqDtHCQgVb + pHZLOSAJawnrhAkuh476B6zhObPIVRVXFuJiBWfSqk0wbgs1cuzAXVkpC0yAQKEA - YQR5AoGBANaXcjIHh5jqKDibUkICqiRdoYWrRsp/SiLiUx+07qNrADEGwH5CUVGr + ipZkmu8CgYABb10dBN247Pr/knhFFQkXm8Zt/iYIo1aZfNEziZpi0CEewYY39ZBM - v7tgSKncoEb4cs2nI7aukJWGbsgTs2vXWusa+ZgrWX5PpEfd1SalaCjEnv26VHkD + 6OHKFNjCCsXSmxWDhWvLC8CHIk9ExdI+M5F2kdM9E3TSvqtjVL2e6qqTSTWW4M2H - Rbl5L8TlLtqJSErOyTpIeK4lK7L1OPCuZ2PA8u+gWn8C3m0JGr8Z + AsqoAnHhTsnxjfu2TMFPmdeCfbbbZHoeF3AgUTyWT6TaV5g6n5bw1A== -----END RSA PRIVATE KEY----- @@ -5481,62 +6081,62 @@ vector: segmentSize: 131072 total: 3 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:MDMF:upqygomnuz244nfqnblom524zm:3c4jgtn76exlzgdzsf7qlqao73lynmwbotm6qcjrsxfcx2aipfgq + expected: URI:MDMF:xyhviiiiqqoljw7poxi4uqun3m:rdzfyy4nplb47auypcxjdvxklg4wyseiitzsclj5aeogss674aeq format: kind: ssk params: format: mdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEpAIBAAKCAQEA4is1+B0+BAGOK+Hf/2sc/nXx3f+IlgE6wPnKZ73Ikm7DLeSX + MIIEpQIBAAKCAQEA0aImdZWIU3aUzKiTP/h9QMteuUvRtjuHxs5ysQQQfHQaoyHk - u55huWfH14DErWaUM0eER2f/ZE8NdnqXf1225u4dEsk6yLJuznnxXIzkZwtRGkUX + rR5glq7Wb/9FrTOI99ENDR1SxrI6Eqsd7FvWeASY6fTULi7gSIH2bAs0Fwc5cZnE - 0txwKX1Ff1ipzThs8bf70m7BFbhNWmTUTZIBYEnhTpTMm0FcIWv0hLe6X8wcJXf8 + Vga2hMnTUchzGQCewrskGFG+JkBdLP1YiixBj/QI4GXOgVOkZQhtdjQ63vVWy/Hu - rnvzuqXezIhpI6Mk/6MOLMYKx0k9+XUU7atdEI4OuuuXUKPxoxNuOBhyQ6wUmIQr + kmZHy0PbHrl9n6e0Q/FCdqho1D7BJ3+3CbuSnUcHDuM/+kk/cYuSXHL0BZLV3vSc - NSnLelLPDHkYRGoVcx7p+KLOBIfXWs/tZcXL2c2jVH+e8Hn44tVv70y8QCsAYuYN + orEgF29LDRu7dZRMHT7ziP/e+LjK5LpL3XaaNkNfs5h8aXDHRIEjuOlKaVSHvArG - OZS5Xd77pdD9vSK3Zjo08ddkP0HUBkT7rZh93wIDAQABAoIBADzgpbPF50H7yzF7 + RiHZ/4CiHObaQSHUJ0BgEFXwCPsKw5BU3woMOQIDAQABAoIBAARXlyyfYsPACCAN - qKgfRFwoEjUPycuaxB5afkVjW8AyqT3KqJ77WFGoIi4bPpVwJZcR+oSf9SoibzzD + Np4KyiGaXa5V7knXt4wBZjYLYd5e82WEzNfsXL1Pm/ROUEd18roXkvRv8qMCzuSF - bdD+QDOzx0advMF16gaQ40tmrzofXTLFg06iQFyimBjZnEcdl5GO1O1FG5sFk2iv + CyB9VX3i3O+mF2R3Y2C8NtoqAgWC3cTuTpOYXbkvPK/k8ydeuvI6sfFXkaoxx34T - +Edy9ATfjhJxUgu+UZa7cMNikvuiuyyg3QBFw6EdGnOgPTFzvaeR5GsAetQeVxPG + GTZsIhWhYno9a2Fn5AixmD6eIWTRVVPyPotXdO5krHnFkgq6ZWvjzZEIg0JuV1yf - 21Y6KATO4T8s18kJhi2W4EB5AVq5ZNZ1cwxe/uSm8KVJlrowGvVIC9tbRk8lKUb4 + 23S6yhqlohLRPGyU0uw8fW+O5gQCBp16KbsCwYI3sKxadfPZfl8Pw7I9DDsWSg20 - pNXsiIzewLKwj9ClCbwtO9WrYU3/kED4Q0goCm5wBhDU37xZ9XK09fG+moxr/xRD + sORw+9RWs8l6egCXa++3qYn832UMeUPWOarEUVO6u9RwYnVpVP5CCPgCEprV6eRU - wz2G2DkCgYEA8dZ8SvrSniR3LMeSpg2sr8NGqgNQjZYez2IfEhUh7mgpkJ+lTl3R + mDKGnRUCgYEA1X/3H0bnVQK2zayV0DjvZGkMDLgAMMxf+0low85T+DWx8zo11n13 - vq1gXOo6KnntYS1vX+D2XT+cg+mqHnoAi6Hyfurxe6Et+Sid6/L8/zofgiC/uq5m + S7n2CUOIqCh/C4EFw1m72ZygxRO2haK7q4JvO/a19KQhyhpzXrdS5cbcvyfIJwtc - p6oKIFMdNGv7+NtKlWpQJ7vez+w0KVJpD/k4sTGfSXl49u9LJtX4iYcCgYEA72nS + rTjX7UbX7r7SWCH0jI/1/ckWDHWs3fc6MJiw0YBMXW01LOSfH/XCj+UCgYEA+10l - m87ykiPWqDLaBIK5bQoKJs52abreRRXAYgPEtATI+SsC9s4DlnXGXvXdCO7ZQcxZ + +2GvmryjypY1Svn7AaXkT9j95ZkRSO5AQZa6bczjAsiYv3BGODmsIiWGK2cOggHM - 9WjDi8fwSWBwFMdybMP9PqXi0BQFc9IctGrZIgzmnBHAVYz/AIvTTLk+MgVSziQr + XHtULBaD0nFaQalmHo3OB1/GafpDkbrQZbraLxIEy6VVS5sXA6qq2Isf3VrFo/gJ - oI2tCF0RrBur5kkEpAcPXT142IvjWvpjsmLunukCgYEAwEAVEPoyYvt0Lgn9X7px + K53F9ti+7yLUi68NcTAAKFCXZjwaoo/OjH3s/cUCgYEAgSeeeX9NJnIz4AxNvN8U - NEyVqWP3LodPuOc08ggQsFjn6guvuwvESMPFXjfpw4ioF9+psVvCHkEKaKdh0NaG + guvBbFhLVTntvnhUNk+1IGxrMDbApvbTmi3vFv+Rxhhpcq4krF62cxh7cX1RZ1pg - BnrYruKQ1Ao+5NrQKBlD3JXVJHpqULqB6vm3ERlhlyHc7mlN8lfQnrWwHDSXBt53 + qYqIe//tZwd7oWWK8Xt5XKOGmuUYAfavo+LFTTcUHcu2N7ai1/2m1FY3TmZJoyWS - nPYvhlV/XkaNzihO4vGooZMCgYEA7A9QfRZZMOUrYx74vpfSkwPiLI9ITEnXnRCs + QB++p54zlDkid/v9/zmO77kCgYEAkmoZm6m0/e7vgSupczjVKoqUyKXejoRwewi8 - ZzhF/CX3r07MlmNdQD6SQNF1hrhS+UCvtnz8yldywjbXbHWXikzY56uS7w2+rouO + SPghM5/qg06RGsGtRUbiqyksU8+9taCShzQXPW8H7ea06hZgM1/qKIVzL3vlK9ej - iAoOXDeSLnKGTRQ/3t7/kdfYzmNXWTBq39yxrtxtb2C9Zsu6Sq03Zf0VqZaMrwjR + V/5U5KIcRPrTCi0WZL5esa+oKembwfzSaqOGElkCLo0dPRgEPm/1R4ZaCeTsptAZ - wnMvyvkCgYBtOO31JrwBiPAUfQozPJykvOxZwfGVK7zPPpAF9JphP3bTisqTip/7 + QeB/0PkCgYEAu/Z0j5XOD2jpzUIIiXOkV6mj2XoGEZu7KPSYWVpZooJ8rkc+Eur0 - 7cqCMwbYmyTVCkfrcyHq9bZKtE84ZI+T8iy3j3km4nILniG+y/TWXC+1OAXuR4L6 + 3y1KkBtCbfk4AtTtPNcBMIr+SCZhrI++3hbpvA6uBAyWdMHTeNwCDsBzP8i77opG - SearomoSCffhCN3P/NiTaBRtW8dRyfwE2DIhy1pZoRRAx7WPGeI85w== + OlssRXcBgtxaG0lbrYO5KpyxCSv57wwBmTX/VnRbxSEVnXXU+vpXkHM= -----END RSA PRIVATE KEY----- @@ -5562,62 +6162,62 @@ vector: segmentSize: 131072 total: 3 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:SSK:7nz6fgbygb663pqzi5dh4bvgcq:yrckbpnw46pkk5theuzl3d2inlxkrysgqpzpcqyt5477s4nq7xqa + expected: URI:SSK:5bjpib44tzcjlbiysarvufdaim:iitswphabeqczmg6ntnjeuxc5rfca33lorjsauils4bl6c7p5tva format: kind: ssk params: format: sdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEpAIBAAKCAQEAsqOsKOGJndmNm0TQIaF26G5Q9lgXUUWbFijpk/+/cpGVCCx+ + MIIEpAIBAAKCAQEA30QPR+SkpkThBIubgnvyvrEpBWFE14xSJKwE5x9RZqvrRdb6 - v6mUaPZAlfTH6okXKf4gx8462H9rk3tmnXfk7OCpRwXWhaDzfItSH42RhPZ3ri/A + Kf4YygB4Hp/Mhdw9E94EP8zfxLkYE9njeAzxPsnhckRZz1xmxY+jQTZ9kVs78OJR - xMpGytIVqLRfz/rKpP6N6tXe67I1CgOzGe9yZnsJ6cdi/R1RnJyOq7tF3mfqYS5b + fqsa/HYKP/5yfOKxMFpUVJ0fJKzj6JVRQAnvJVun3YXA0I8eRkmDz/+aByvWQ99X - dUkvI4agEjozS8bPBYdMTdw3fv7GwHYDcP/MkiWD+x1IlU0ngmPfgt1yzF68bq/6 + xRzsvr5xsFmtAa/Du6AZBLhlP5k06CnqFsHQb9a7HZnyC1OyfMCyCQSQf/N0VH1X - 8wggmTVwB3H7eIHRU49VgotudWftpwHsEwho8MGEQbJwdy2YqQiX9MXR1P4Pbvff + iQ3KLyWufJhq2UGl403XaxNi3Or59brp+R6wIQzIUn9rzXe/ftjkrqp+TTT9FyMB - ELKvm0Idz5DZ1oP+BK+qHEq+gOPf7PQrGO6OPQIDAQABAoIBACLqHHbtBeGlKKkl + nUbO4HSU4of0uzXeYINZDkiN9Q17mpx5FSvzHwIDAQABAoIBAF5vIyd9hkbti1+o - POyly0DIduh+9Se8TAB7xJNZlAiHbbJoR+mb8lbFcoAclIpBexaJBc0ngJbZ6KOt + zTX7x0jxFjC4W63wJC5utAQuMvgCb5kyvM1WNJX8bNJHNPLJnOvyVEnIFj3XLF/5 - pbO3QDYP/uXTvUbm21AHRujF2aA8L84KpUmRI172yqbrgiJ7KOowmnpAjM5SSU2I + IUV98+xi54C1eGdE5hNaFetXaPU6abgRgfbZ2KhAJUW8EiDQobGaA2Fms+2HUz7l - xZOXGivvdlOL1cwU0+OhMb1c394E66nbcNycMdt5i0QbIC9NhFwNG8TQL+5dwxvV + KWC00voyMmZ4VH3iiyOfpKktq7CsB8gPIV8a+BJqCrRjRnI4n/+/8qfd+Wa655QC - mwEBvxSkAYE57eERfRpr8/oQVFn4OGcfHU5KAEuSENi+xEwheMLpb7lCxR9Xq8jD + H9TjuGGeTKn9GwofeXHbwION+ygPOXQpez2YudigZBHK0wNQeJYkvDNAS6NdfUdY - aUms6POYvhYpEk60kwHem4/jHQV32S2P2dSzcZufaeZU69hFbmFPuwMS2YqLzHQy + OtDOiQwhLQYOa3XeLGkcxLh9Al0c0cDEWLToNqPfVKTfsqKIiNail7ij+7EJ9rOB - tRac2CECgYEAyUEWEGRNje/9saCqHXkHtSwTdi3uUr3fGSBhdKJMJPov0rfgojJ5 + q0suCgECgYEA61GSBETDtsJLxoxMgGgdxyW2Wc/GdFBi/f3cHy+BSRHROshP+H+n - gxxfV3QvSO6I+YIEUI0Ij7nHBuBey6aRm/3AxWw4Qq9rPW8rbLtjkWbf3TeeC/Kt + LdT9H4U86PK1QmQZts9djBKh63AYhKgyRyMqSvFxpqywbznnkt3qygPr83VIXXmY - qFMfPXOeeQOUuFaoA2m1Xd8EXLbPr5iaCNP7ae80MiZNtlaW8P9kEOkCgYEA4zu7 + 2Yj8SKwjgslq2xNl6kTPKdBv/r+Es1nhYvSZf5UTbgdQ5nlB2CvCp38CgYEA8uNQ - vnG2upHww3sWmtdUSGfDWBhLpxtOdLGDhebFFKYwIOEoPf/ec36jptC83WoQofDb + an6u2OtcmsDrwOdNZ2o/3uddDtAkRuI8LFzD13rGDYS0ZE/OeIbDxaW0mty3t2l0 - dqWRRZMEjozUxRPBtAS4TkePdf+7CDAE+9VFiVLOaZCFn6kTj44FEL8YvTIgR1Z8 + QbjW0GJ1wUFoutcdJmRBOj604Xb/NYDkF2kiqtYIlnDyVP4nIi44+0TIfTs+zXE/ - 1qouidrCY/y84m2x3Tt0TyZGj+xlsk+wzW4F3jUCgYB2iB39o4XF7i5GGvF2kF0I + s0xXnwpZB0jfuQxDT3g4U7fARx0K5kaWQxxzhGECgYEAr4q9K3QEr/xHPMkCdLO2 - yJ/hv+WY1/l5LAgaEKi2MqBOBDyKax4EKYbB1E0xMER+Z6Qw6Q+8ztc45pcObNlf + qw4F9v+ZYsFo52KN57Gvd0vUUk6F4bGQjA8b+HyTUI9mCi3URNxyQ4DOy2xmzecP - vZF29WkhZX3M3hf+X1OiRKve963fLZw4AlTo9ZrFfWVvOKKV+AF4+yvvi0BBFKjM + AqRH83ojtxuRzpdameP0N1kvlgFCx4BjNrwKv0eygekxTlYtK9LC28WDFn3WR1jg - QEXYO6lLTCIDHXajFFgUWQKBgQCeG1HmkPizmBgN6/cuheT+/DPPeBgrjbRpPZpl + WspvC46w3N6WSifgp5sVbY8CgYA9Zjcy5JlgnobHXBN8rTwE83f36ja9AuLYxGH/ - 8MvwMjIKrp9xhDcj5Vm5GERRSxuHki8hvtH1tvXUuejRt41v1FjpHqGTWPyqFb9h + uOeM9i1Qx4YugXopP7AHq67vIvKSO+c2ofozrWAlHVrTOIPW66sNhUKGaGV1agK+ - +mMHybYVfZl8HgieOhMMM+riuZ38BRGXy5HWGYBoUdKbOfgoFtY2vEscmT+pcgly + 5EXuN7LuDNlFoQXVfyfKZQXlmm9y0bkPozHXM290Bvj/N1lgonxitWW21GGn+poL - 5rrugQKBgQCUZd9VqLDMPsx5OOIzjYNMUb7xxXfp/XiZkhiXDbAAjVN6tpFGPMes + lwqgoQKBgQCiXSbwbmUtTSmgDUNdHJ7tkwk0+jP56Yv9feH9FUWVJE8UF5InBB62 - Jrl5+XW7wGqsyJxTj2G8BAVBw4DIPnf9CVxc5Gs7DtzNitCn625Hn7Ss4WKGfk2c + 8KckguXC3OcVmXgoPhMFgMwHJE0Mqx6FdhQPCM8oEMYYg+sM9kTHiykFfTmdt2hJ - XGOTIuMlJsgvnbE2GsVtIohUIKyBr9lpPKaY+HQCejnYtkdu5o4RiQ== + L5A9omWNxiWRKgIerKOvO3ljmYsSXWp8VQJjvCm6DlznDg5SDSQrUg== -----END RSA PRIVATE KEY----- @@ -5631,62 +6231,62 @@ vector: segmentSize: 131072 total: 3 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:MDMF:gvrje64nislvfxxr7ot5x6du4a:g3yh2jhto3csbxtatoh3zgopu7heazuhlgrcngasn4l6sajcpgta + expected: URI:MDMF:pxo4i23yfsjuv4ntm3xb2rwp3u:3vqfgh6u6k3qh457kru5nxe4enc2zkmnb6jmkm6xj4x5qshby6ya format: kind: ssk params: format: mdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEowIBAAKCAQEAubXy7c1CAjoBJS7qJpAtDEmNV7rwyHYbDfIsEQqvuDckkrbW + MIIEogIBAAKCAQEAxr4IL8ukM5awbx8SFDOqHXyWEEm72ceADkaBYiwJIemJESPx - osGY+w6/CXVsXB7A3N9rABZu+pd4s+oIZqhENkZ0S6tuKVs8pgj0Wi2CNfviQ0EZ + 9kd3Vvc3Lu5clWbBeyMvys6LAjcMz7xWlvDZ4/rUDMRVzY406P8c0dSYGdnoJBLD - 6K/2BrsvW7dzN/NqCqX9jTBouOi4/wbGrRQf9+Gd5TFMaiNHNT8rRAfWz2Bl73ZQ + jr6CDz0JfiInMKGsBaVUejnKoW4l2PMCj1tbl4v2egLKSW+XzNCfFU7kFJHfOK6R - UkDek9YIvS1anEqtjeX0BuxrBviy/NpDdlSYhDGTm+9SKPaX63MW6BUfNQYcZKmV + MrRq/HVMt8eIaHQt92Gziukx/AgoktQhGxmxUJwd3ETBuVYFAqmPBT8uuf7bcAfV - nAXvw8vRuELq81lgwgXEtnNdAte7qz6n7wpcHi570/+sB2z2N9wPUT5UASavp8Ux + JCfjrxxH4wAwpWXuUSu62Q1yOw7FqVQ+k86IUNFVoJQMlJHQscXTu9EBAboAqgM7 - ABKADrowXBKk1X2xn+ltkSAHTsGXb0+MZHg8kwIDAQABAoIBAEKaq3A3M+vZgsyj + A4rcgVMqo4QkqvhX0/trklqyU2kwslODo+pv+wIDAQABAoIBABJCYlPV1K5isVoR - qU3AWq+z93HV9YJnvWdAiiZoh0IR4NePpKYFugicrs5FI2Jckz4EEPuckBvm1F7u + k40kmVlf2WpXWTlvJq4+abkLIlmpxaHmrgkk/sC63NM51hVp3UEtdet0TM7gH5QJ - Win7Qk+W8CGtb5p8guFnh7+J38/dsTX+tLyb0yhx3NfPkQ6picgc5TVMfdqHeMXa + vIuInFQ29tAnd69M2e1FhwXQ5O/hwSi99UFCR9u+OAbifLxElkk8gTBA6q4ohbSo - V8n+VELSU48+IZJVabYCnFFPYG7KHWI8ATeue3+Bw27kre+WCkWSfWW/Lj8biHBy + JaY7IVqhOCd0kIDtDkLPMKvPVY19bG1FKiDx0BSBm+O393i+ht4vIahZEPhxPyWS - FkDLxBGr5rVgZSodoAyFtZzunN0k3/U47ZF5bsUAQ0bpJolnXY7qGXWh88cZfxEv + hF01XWqUyeG2EnMzmAOmdaTUck9hbu32It9IWD+Lxv7mN4L5CIeRKVRTEF4/nV0+ - j9FTxKP7SbWDhdb/yh8BVVqbJhRgzPZuK7d4t4zlNsCpIc69vxBw8xwkBNKTiKwn + SpMCg5U4ZcVkR7Lm1k3Fl5qUZe+W5j0sb6z1r+M1jKgFwTmDMKrmU804qABDw9LK - 7Cf1xOUCgYEAvGa8iBLEyY3Z7nsSmtRBmBt8H9nNizK6n5JaOrHd6wj0Zj1V+dS/ + PlfXe+ECgYEAyqny0/ff2y0sq6TDGq1+tE1hgu47fI/arQhkNR6GmFJH+KZfQPSU - WIfce+RyTPAX9cMwlCv39/C4UlmuHSTiH+r815GRPMg0fAyxz/z6IBQWB2pdTlTr + Uc1TF9XQnEweLr1/U4HZE7bGGVUfbhAALmvZVKT5E8gfYOoPkm1fDQoavFJW4+26 - URvDKE1wdXozOLZgyxRJoGuTL6XRVBYG2FDdCdOslrZtMe2Sw0Zjnc8CgYEA/FgT + ZSPu4gl0d5Z/AmBPk4BJRdHsRDDDKvB0BdKiqKpSkMTgnNklznVPeVsCgYEA+wvg - D4Un2FX8VPprr5wqJupsoNYhDBmaJv/NC81QEC5fUAX6myBpm7cFIwKANzsgE5op + l10zUPML3sX4EcduNfRR5kXCAtLlU/cnB5lEZNpEDesiBMVvXxmYw9D9z9LUiX2w - BqO1J+aEZs09cYszZ6eDbiKpkpWJEaOFT9+YjD9ViElRCPuTiHIK991eMN9N79Yf + dTcruHv3mgsESyytThh7idyBv8Nw8bP7BSOP1TI090MjztPYByOjK9LqPLSD0xhN - r/MAQyFWuPAn4OulKWn43EwjtXHfjKjnANQ3Cf0CgYAN6ehyhDBUUk2N9zjghlxx + 2sVgH0DN+THvD2aV4uCEWLgKEC9bD0dttC+2BeECgYBDsEmTdInHCaqO1aP4iBP2 - x1XbZFJxvUVbE4vmWcxx1y91fYIj+TpIZ4A5Bh4K4JBkbg3gY37kqLp0GntpW5f3 + opW8BlfF/cIa4t+dQknQHEM/kEnmRwo23C4xms9nNKEsGUyqlobrZ7N4iI7L0vpM - k3so0G9RddeqcaWQHra6N8GIuqo5ZrwaOVqoV0++3U97GLz9QnpNhqRQGIblFtta + hub6t3MdoUyhsOtsi60gjMxrM9EjpaYI29yQkHne61wWbhaF/GX8tOWFzQeSkucd - jl5Eo4VTfBWEYm88TK5+sQKBgQCvmv8QsuJKm3PxEx/zYmK3GDYmKz1uNTbgYu0n + fsGnNeQHyEoA+SIAd/wIWwKBgFdVf3FI1ARSOQvr1OvidB3C/Abet6qh0XPPZD2J - hGZuDEdJ4g9G+uRjd5b8iRX+2Yd1/LcGJtC/hpynCbbzmCJaxOkisL5/As8TVk9E + fTiUkd5BsVj1klQEJJfiiZmV36hhGFT+t2/7eFyXfovkY/nqHHgORPkANbdwBGB6 - iV6YYs67/AGHlcNSlcJqQUP2EMAk4kbE4/9PuBiotH+b94DFdDi53caP00H1mei/ + SZxCVhi6u6dFHT8Gj8o8Go65wa5bIyJ7TYAx3DXXwDGcX4JI1uHCTIXq44PCNpDb - 2+69ZQKBgETEqefImLCG+Drm+T+VsEvNbWUNQcgxQ6O69z9UiJOg0FR219m2XTVX + lDghAoGAYdL8Fl337G/8wO6e1cIbxLWYWdtQvr8ibsQJCYznbQkligQ7+Cb4Ch1F - mBlJESLuplJ3482qxJfrxZnVT+3ajlofD2zDg+Lxcjzf51eUusBFs8sZseVTUbti + ceMke2mC511kxIdPbpECf8eP6JSqtAd/zrBASe60N+QSvaMcQwRqtoVnduERfjDH - oiDTLpGCKDpyRRiKpwy/1kuLntP/hbzd3rSQBnxmP+QkQ3CukfWH + thD/UyoA81pRl/hGX2ddp7tT/Fu9A8/6FLJzd7g/k5ZgjkdjfpA= -----END RSA PRIVATE KEY----- @@ -5712,62 +6312,62 @@ vector: segmentSize: 131072 total: 3 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:SSK:v37qrnptbkbw6ljh2n2ffwej5a:sfuywdbrrrzyfr2f4zigib5qmmviexrvukt5et4jucjwykkjtgrq + expected: URI:SSK:dq7opblcokohsb2hhcuej3keke:25zjdhlue2o56qcu3teoqltk2blks2qm2n66rmd2qcwfujbvo4ca format: kind: ssk params: format: sdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEogIBAAKCAQEA5dJPx3tIzQBSP8THjUZXZs+1DZ+gxWfrgC2bhQy4OzTbkeQx + MIIEowIBAAKCAQEAqF3wcmVUimtXuQl8xGtHK4q0zCOrUBkBVIMakmXC+jlLBnKA - V6FasL6vDDNczZujTKklEIiC79jprpfRyjJ+CA3VkekebkrX7nbqNG8j/C8kiQ/H + pyNjX8S5JP+nrQy2ZfsNONSGgNS3HoReEe/U/y+hb3hXHTyXpYrrSEgbKRgTxREQ - NFL7RSUdcGXwLo9xGw5A0FJsgO/dFvtZE/U9VsPoKhTyFzN0K/ieC5zl5D4EDYpx + uoPW8ljJWyPhe+h78v8EooMuz+5E8pGuROQba9It3b84rg7TKRSAEHS+qN5v1JTa - jRFd4fNn+jP99AfyZx2DodGFS5kIAHKzDPdqPzvr5055VQT3yyzYrXlXRWPz6JMP + B5SkfPvrFj2BrKefGupn+ac+lX63B4a6KxQtVrnhHZ+HZJ91AU8sNXnIRLVc2jdT - pXIyGa2iRlCFCgU2jha0tpopuiQWi+EgMKrT9eRZbEvA59JZth6NDsbN0gkP6+2Z + VLfUKHjwpxEXWnpZdv4ypUfrjj6oUfn6K3y/vPM/BR68NadLUx6+EYX9Y4ZBaCLA - gzqzbJtyn1vNQILeeh/na6oAe7w0T+pn57tVGwIDAQABAoIBACCAlFjVZi/b5kXv + c2pNzT1gmVS5KYvNipt8SEZu1wjj88/8UxSoMQIDAQABAoIBAAD+kwu97PlR/Sc8 - eftQYeb/5A6nrzCL6GHp0U9JQ7rX2F+zIolOoAlUBmSW1P6dDsS2PTAv5jiueCoB + Ntwwd6+6ZpXOanXd5pvcQrUq0Yi++2I1nWxp9q2LYNORiKjqlQ42T3juimqANyJV - faF3b5yK/FPU4MFfY1dtyOSefTvanPOnYBhVzgRy4c12FTg4gBn/84mixoabpaxs + 7Pn6pRL/x0U8XE6xe6zQN1Jf41HYu6R+P2TL9WJM43LncUQVLAbpT4VH3z1+U7Js - 4qWwbrrZHPnqmWxPkhPv5sYkq9yR5gsx4G0g0qTsXHOCJkP7Jdfjf3bol5NEeXpW + vq8sphEli+dOIz8l78SDmOZf+3tNrejZqHPfN57WzZ2iI6XdylU047fk9l6XQvcQ - 9trCXtVd66D3VKGOHD+RkVLaYnG/8K2qhJSlqAn/6he8MdoQ1yKF900RPCHQeHb4 + II9o6LfgB6pXeAT4yhMQwK8LGuUTkwQBH8D29cAe1Vqt2uztJ4NfiEd+n6EsH15m - 6KeFSiSOzCe6L4dSKZ58kemu7itZ4IrIEK29x3e9D3BJfk51SYlGq4hbtAzl8dwO + MyAuZDVWp++T4m6w6TJbcFBL2LioMkRb6zsdb6XK2QsTDDyEqSGfBxNPIpsRJTqP - HM7w+FkCgYEA6B77+CBqYwC6HDd00NyMj8nC1Al9COaEWeN5kK8k9jm7zynsQJhL + mEjrOuUCgYEAxvyWv3vRyptCOZuh9e/6Pt4eYoAk4gLVFuxKNR+WXjSarPYAVtt/ - ngvxq5q/ZlG1f/BkgespLw6mlrGGr1OGueXJVnGeX+3p+RqcnzPJOGukiFc00+j2 + 0u5FbccjQQcZRTej8rTVKHbRA1jT56mPzutKFlftpFuoUGoeDQ0Y7g2nVVXnHHIR - VK+mqZDbNdmAXmtda6eF9WSxrDULkXCF8JduWr97nttbJPo9UjIR29kCgYEA/XbE + KeBgifVwUIJQRWfsiVt2Gq5ja++ccGoJHPcnCHokqZ5UIAuuGNEIR70CgYEA2Jtv - 13/ehGUnFW9YFs/vAls/xhu4B8kGIZXlpIWIt8gbZPvx8R7d17R97H+u5g1J7i/b + DOnIJZ/4jU/IDnYeR+9Fbg6GioB/s9e3j4wJDNZXuQ/ju1l3kBfkYLVaXzgCNgFj - 9jKJvDhQAIdl/Ar3Q/vrmMTEvDiQuZidzBb4Z1r+C+1SDdYH+q1jHvAX+9arhpNe + UECedrt3OUxWP6cp2dXM67rQ9uw04Fnl1IG/Uxp4icxvjqEGI7IkxCFFAXnRdA4y - qCOlBT/tOQRFhzQqK/JnZNfXsnf8AOAYwMZ5pBMCgYA/nk5c4TWHUOmxVhm0LN5x + oONgNn8jCLhuD9PqUEXAKGiHkLr+FygDye0tn4UCgYATVn7L8xuLRhVkhdRykzTN - glDdoIQebl+T616kIvy0Z3pr+wd/ZL5E4O0ppU4UEwz1tcM2QGeXOCK8ZoeNgg0I + oUZwqiVrdX0B8kqv6PbzBse1YV7dEg3VEOTca0royejRyjt7nclNWmarnZlSXS8l - 4kveX2GS1TgtR/fpQl5CEm6T16Lo+Y6aA1JgYw1Rov0l47NFEDMM4L45fohfIkHz + m8Yib78fhuzPi9CJ0ikHEXqel2+TWx6B5FVdcuXMXS2x4QyiuKm8pA/zcGDSp+tk - gO2D/bs5/NDsP5GS95ohcQKBgGda2wGVHsOWC83tzVngCHJJi0PZYb2q91kSqsXf + zjwT3dLsTP+98YSk0sOsPQKBgQCRpKGcyyy6r7+ONNDNeHqP38CNadLpGdHD+Q4B - vdRTQPh41DuifovLCd46YrNkj9UUpvlJumiJ/fV5QNj6D8IlI/jzo9WsqzdDSHVE + xSTors65Lofvlw6fopD6vbYQRDaoXXKLqYdjSlW1/zAXCK1JPUrWTfznqpc7Kvcw - mJ5suFNcvqztretGcLjY5q7G5sLFrT+a6VuuqakqWL+9QcUR3597dHVN//DLcMyL + VjVxCWF3NjDkdD0Oj1/NSJl/jotZP5qnN3uf6QiDeo72sYThiKTWBsLwe+sRYuR5 - ImcJAoGALfrYfEd+RWcE7TEwfWguVwHskdBuoGgUkYwq9Vq5ECUQe1W7xuF0N0XA + R7LfiQKBgEG5Zdbp6tVMxI6CPdDpVnMujkoGHHdNwVEX57I2a4Est9dXp37EzSi5 - vvSN5NqFm+psN/nJc99riGjvtrkXyNJMhIhI/iExn21Me3sGttXCU3qVPxKFO8TQ + /HUwzsS7WR1VmSxpffg/05yFBW1wpPW6Eydd0tip6V6u3XL8CJDgxLUIg5GMWFOk - 8zDQ0bRW6mxlPdUL88+X4UsIsiCU5zEc2ioY8gCq355EZw+6rok= + tvWYmVQgapm+yt4MNe1VVui8FL5V3jxJrlBJoVherf4GYF3N57Qo -----END RSA PRIVATE KEY----- @@ -5781,62 +6381,62 @@ vector: segmentSize: 131072 total: 3 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:MDMF:v2ryyypaudkr73uthfwax7ilca:b7eyzlxwml5gsvt4altyx3tirirv7edqne4krburi2fxl6zge4ca + expected: URI:MDMF:tvewutqp6vuesamnwdvshuhqiy:43e5ze3qcoatis7rfqugjf2s5oty73qr7bcfgschuesycfsi472a format: kind: ssk params: format: mdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEpAIBAAKCAQEA5bwGbQ9bJxRdy3oA40Wg3uEWT9EqtYIzrqBVGqu6dANN6ALH + MIIEpAIBAAKCAQEAv7qz/SkzzSobYo18DO2SgJN2j3JO2XWCWFHBK9bO6R/SzFmP - /+OH6LZvCYn2O4ztQ+Ddas+B+SbZxTfH2BH1A1N3aB1xc+VpY9ameBWLIwZil5OI + OY+Gx7WFU/jJD0Xzs+bS92HcUgcnF+vC6gAAqcRaqiyDIuAWumrQquDcUyShKwcJ - +QODjbZmcIYx39r1gGt9OkCltv9+GMjsg5BxcwiqRwMSfS4//LEEHtb8R5GZ2mJC + QmWBX0Y566teaguVf3RKlLA/s/Oe0b0CVV0mFmr7W4aPjszTXTH54whHAuO9hdha - cEfkoYFsr3RUzPLU09Fz/9NNYp6q15kO43u+9+r/T3dGu+3FDB2uZgfK5LPFAbNy + vRZzqXWKsgH6KGp4RFMh84zwF2Cq34YSKSg2+cOl8at4L7lW3nbQJF6GKZke4hd/ - QrVc4btfVfUVjG6yWLpn++DWjcWDHNW7xCBeTS49ebyWXjiXB1AxD7uT4zxxhF7e + FtSAM1U/ynJQcIFfVODmTVLnj5xXpLtT4j0Em+EpDVVuk8vQKOX4k0L4Wkc8JZ5I - a3eirX+8SKahoo+8z/EICeZdjD/tugdj23SUywIDAQABAoIBACkTIJ0AOV533DtE + sICrCqCEDdmXxOGRsFKWEnePXfl0QdgMDhoCbQIDAQABAoIBABEcX6mAJAQzNfwe - sYLxEI249dnIfpfcUyw+O2kc2iXi71tzn9mnD0Yy1BCDC7TjAgr4We4+crEe2qHR + 0LYzUkf7oRuejHnsz6QD29UGl92x/jVIPYzaCcxzSexbaiTc+H8S4mYQi5dIgHfV - 0tfVghaZpkhFt2Ku2lSA7NuckndtLVSHit5m2+8K9S7aN3GcPve6gDXZmCdrb9qz + OHEmFz5kweCpFfpRUcXzosl1RZnSouLnqa8NcbUn283JXqThLBzoAICc6s/WYKiq - leICAd685mDy8ivSiJs/9Qokiw+qaHdN3lxYw6sCU+lmrkDVMM3maDkwcmnU7stJ + 5nU+2cIrbGECineollPIEjCYUa9orc9Lzi2k+SSvfXUd8bWJunCmNn9fOenSYmzu - mMrDYNCfrf0MLYUWl4TJIogyhfcxL2ai4eqOOj9ArwL52AtZIBNerhr/xRwSYR8k + dzabXlDvQnaA7C3ym/2NsRl5w6q0px4lPqnL8MJowNsBYRkd7pgCRSQqd3eBCeHD - KJ+5CnsWTlI7wbB+bhcmOUcMGcBJVKvn+pcGF/2PRtejpg6U70lOlyQPuFXSyAR+ + qbOT1SXyjCklnse5bjEP+rMUcMRDktFCBw2WTa5GZcYgzoWtDL8WCZ39jJQTImti - 0Q3uOIECgYEA+AagJ6FyYjbZa9MeGJ9OeZlxkllOw7VFwSOw+XwOtIsUK5dRk3zH + qiU3tPMCgYEA7NY+pUBRrtkdebwHiv3zvD7pKzGvUxxyY7Uc0rtm8zIvty2XNU/A - 8RnIBg2nJA5k9YtMjOvCfx0kgb+Lo4TgJv+vkZ8QOEM+GqgjJqTF9yuhJI2As9XE + +3Yle+7ffv63Z0ub/0yTTFw4kSl/jE8iQwqySSmDKZ3EhC7dWbbzOjHIvtVvcCaR - 9/sOervr99Bisq8ME26+bxturhQ1/ySepAI3LAkjj5ErnM/kaSujxksCgYEA7R7a + 8EbX/jpMpASX10iYonJXAwtrtn5vm5frKKjm3sxGNORBK1M+OllbOosCgYEAzz4d - LV+uaahBfvBFVSDGZFu+AmivY8a6C3xAhW5Tei0IfsgU3XYiyevU4Wo8nxDKbvGE + quPcYInY99yYp1g992vuBj2bSkD/9u/lbCwn1Z7gVFJDBVVec7FAXm+3Kc/zi9s+ - rE7De6aHSEyJhHFiGM/Bx4CSQ0FgAGUdNEkI7bAiOHhO+z+ITyY0cPznVGejul8G + Leco7DBkz1IHcVGchF5lGaUQrOXV3mHZEbp4mXfNHLkwMYjFCwZ/70ktXm++AmlU - pJnN26yP31Uf0OFkfPsVnucVAjqAJhC2WgCzW4ECgYEA6XKrCcI+/ExuBrwWhsxj + 6rFzSVtQyLj6EUBZZ84G0Yq6eQGqZydo7WjxbecCgYB7hASp5FB1UtAXg+OfLnBm - O7b+m+Ytaa3UMv8aEyj+WlbRrFnn8W8wbjF7AJ+XIyvdQPRVIArD7YsLkogsscNe + FZ0/JKteOfDCZVtB3/CCFwNhkgpRCGYJ/wTvjJXMwoTd/0W9MK+FXHc35Z+aik7B - i7Z7lQ/nX12DNent8/CBWK0bJmF9s0bQ9yu5rDH23zCnxVFXh27kFYX3figNztGz + DhwLIfZAxwINOe/A8TQKfppGREPZBpSH7jqJYNhFlgumgDryRZVxhgxH4crNJ77B - 8+EV9v+/FeFo8FcIwSjPJNMCgYEAwnDzPhPg5NlJY/tJD2aB6Rfl9vm7IRl7xCFO + tsypF3np7by7Hq/OeHmmnwKBgQCAy1CyuINn96NAfvbb4Uo5bvjxJe5RWk35ECPb - k1wF7gDxn20Y1wWhv2y4s2O3dESDi3hXcChiWooTEzFX7xg+9dOftqXyyl3YiFpi + cyGab+9oV+tQ8DoP2lNvnSwOry7jdvCQpH1ZM8Yi1g7MDPUhimx4YI4ZdYjReKvn - GVbukGJHnYDiW1scvrK4fBKW63rVIuX7f4xz53hYvi2CmnnJOkd20kfxzVMFdLFt + iaSTc9GkDS73SdFzRanScv7gFr/WTdG5PWixaS+uXs8CU6R8j5zLMtUqiK93BhX+ - pt4+NYECgYBqUe8ZkWThpIjfjv3QihGOw7YClupxMMdDpGAZIfPQyDAsZuO4PB4o + nV6VdwKBgQDBeauKIaw6AaWFY5ALiKmWylQp50hF5S/dFocCzVm+isnmIhogjwu4 - qPHQY9v0Uem+RQRViimIpDHaXcO0eDKY//EjEGt8RhSZK/ijsuBPAXXGSGSMfoa7 + cYG7zKtAOU+6a5UhZLw4j2Nx/YFjExHpY31qEap7CAZ1u8VD/MqVzohweh908Yd3 - xA37opft89tUJW7e10uzICWi69TbsTVFjaQAMy+XK6c0/FWAoIV3Mw== + X2X+o+t2k+sxaRCNJZwYgpxpThULs17PoFfgKS7hYUrbwk1ev4SLiQ== -----END RSA PRIVATE KEY----- @@ -5862,62 +6462,62 @@ vector: segmentSize: 131072 total: 3 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:SSK:7wrmdyazliwoctoaz5tlxewemm:4bprnjqsci2ewyha5hnq6n2yxstw63dzlys52yizqcuy2qhbp7za + expected: URI:SSK:4r4hejfttcshuxaatgims3x5ge:25nruf367dyhnsykeid2owcdrbtbg24yuvynqqswvbs675ieaa7a format: kind: ssk params: format: sdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEowIBAAKCAQEA2QfEx1iF21VtPUDgUhUHJAKiourwkTEApPN74s9pcLrkypzY + MIIEogIBAAKCAQEArW/N7PGvnQp/v+hiC9FjnwBTA2sTB7SRXQsta3Up3vBHyQFk - Csn4ngvT2mLDw7LalYiIkCmUBLO3p96e4LZjgnIYfDGFQLPqXT6VR13Ufpy0fHFN + RCcpGL4/oQA0tXH0QkG/nIyjT7VwUHWUuTs7HuKIVhfg42p5aWrwW2lWbZpCed3Y - k1+aNaWTN3m0Tr8yMCMuP0aQkmjcVzsA29wV1Vg7dcZem6o4XasmSj+nazsLBFT1 + +AJCp3/pqSd+7SDV+2Tc15lKLsx4bmN7+hnVpMkTDj5bnpTFPkjL60SL/jEduOvT - jahae1u0XC2kisxemcnvMklulfaSlKtPKq7HV+///EnGuSZ8q38HZ0yYciax2p+r + i/K7hJoIiqkccaA/+ldlRwVq9GTom7RG+AP6l8/tbOUyV6heYCFOTe5jPEPhcF9p - UJ16OrHMHr+WuCcSM0zaBjLpcmzJCTch4gm8B2V7EXIPtvNFa4E1ch122W9/F1TG + Knsvwu03rF2HRjKqb4TcdqGjZqJbcMnPIlJRbUDwHOwKRpx+2SqeQFe/jrF61uZi - Q3KSWuxextAsdfCz9CjhvuQJtdpYQHjXzLvi/QIDAQABAoIBAEb9eItwVFCbT+Ey + J6EJdliKPIrGU2GEEz+PCwQwvUXrZ78+lfKU1QIDAQABAoIBAC//ZhTx2fDzC+G3 - YG3Y/P31crdxvADyE3DhSAu3ppi+OWphBXX5/L3Nxp1vovNXhJJXF7x4LTeghZl+ + VRMCMri5JUAn7M9QGH6QblRzy0+oXaVyHiaCU+xGEmPjI7MnQzrm6TlQ7o1LoDaU - g1+jqUcZBRNSq3CvqSCZAQFYGtLTdWIjOanUIsAbid0ijS0Y81S1nUILVezeKfzK + I8GTc8wbUzHIT/N/7vPOpZAWudWBQG5rh/Uy4vFqA+dNR+ImGTIGeturz4zeB2I9 - iwxfoDCp7MEogvfOJSPWgO7WhW/YRRwDG3Zcfgo6fKU9lUshJu97uLj6yIJkh0Fs + Y/0WEzsUbpKdjUt4zpFrIgwNH1UMGsyjkdgHOBFW4j+aia6cV5z1Yr/3vuZ61Jad - F90MIWxs4TeEhK/QU46ZsX6lXCNOs47myvtdvhgKcyETnPHAMnZ8K6fbzcJl1u9t + Jf6Mz5+zyLNqKB9iIM4HBwJnohvrW3n62o8Fxu10ZiManu6DG7X4xCUrMk34mq7S - /maxtoqMTqAyIy2otR48LrASvbEPkrLO5wg+Jolg7SzWKQXGvgplv1Ao4m6Xw1zr + F/M5CR5a5ZN6ZFO+uGlI5Vn9kKXylYDyJqezAfS6eho/Nm2J6KNm2lvr1URI8nFQ - FmtlbM0CgYEA7YVl5i1GFlDniM/BVX6KLcVqXSe5ejPLvTNcbDSp4Y0Kg2dfYynA + T/Jn+e0CgYEA9GcoSa/Nt2Ulzzalmyi8Q7TfcJCMmPVCsMYe8J55ncYmBwyyUy7u - 5fNDCWuw0t1cJgTFt6wfHwwbTynqlUo9B1cDwDHmizGtHOEiQh3O+2mpDnUGSwWe + TKd/osu+nWsuTxtSQ1bbWMrKoMghMi+mfc2cDGIjvfanmr4jYxpiiUy8UZdOGkRo - WAfu4XJaoqy9eRvaDfXOhMCzLjUWL6I96WFmpCSUlBPA9zewQxL69TMCgYEA6epE + QBovn5jWNk5yhC6PksUp2GR9ykezD1KWSEK6RrlzrqfYTF6F3nv/qNcCgYEAtaqY - AnMfEGOHtWv8Z/llmRA+FS355d8zBJloSKIda7ogVDmcRiexTu/26yuRtRHHG8nh + 3NizH1tF7uh54bqgke0fxVMN8egv0uSlxb968l9B7LXkQkP6nQ9s/jcSnlDUpu9g - nq97xW6pdj0boVDnzoWB9HZnsiqvS8spc75B1Mh0Su28lkzy7BaEtxcphUa/5x6n + ulu+IXpeIaIDMff10gdKDEfSjNKHwBIgLe+BD3xkGYbuGSMeffO1f6apCNAvceoi - OsR/QCiN4CSVuLXxG0+u6PidH311ds7uZasgZw8CgYEAmmrif42phjKdBH4E9D8r + DfxgyEHL3HGnweOdJXdoMf4AkBV7TWRzlMSDXjMCgYBeCzvDbvSXt0IfRAXheIFJ - OGyjJOMBm6f26g9tI8/tLf0S+7EF+6MWjKjlSUehEsXk9baekDWvmfC2BHZ80wgL + BFZeOCEB0o5A+1t4d2KQxWhomggcXhiwQluoxwGoDVAafIbhBpEMz6u8xoWPjCpi - uyzf/GC0wIPQRvk6238jpKHhzctZBwclFZg6voko0Z+6IvVvgynuVLIvC3hp7xfs + ijWbxj++nyTLNJLlVYfJEU/9jV0uWlhLIkhk/yieaP1Dw67XaSq666BDr+dE5CCT - ZkDziP1bNxXMmyyyRDkfvK8CgYAysYV5rm9OAvP3OmbiNadyC5YYyvT8f2m0FncG + 2alYAZB0Cn3+lPiqLciorQKBgFcOb761ofEO3k6E3ZOMydHkXmtDR9V7PR/FLqO7 - PrP3k8fL2QxoG9QOUm0FvFSAlFC9UfwmgstlFz18lXO2ey0xkbd/PmXss9l3qJjc + BQINIBx9detDhF+rusGARs3TUnTFFgd4W37TkzRu3TUe7JA/qf6ElKOjaCZlCUK6 - L6Bet+6UCn+zZwvCZILwlwF1k8alFPyS/ODDC8bri6Iy/KM7EwLKFI8gsvTRAbmi + GZEAPT/2Zzdomv4kwf9wMGTmzzW5y6QEI6UT0svLM2c42l/P/wCmBkMF6FbIIpNS - qPqFuwKBgAHoXx4HW9do1WyGsCaJ8rsAxKyOx6wjx7MJO4NoSVcyudQXE2rTkaUA + MNn9AoGAN5Q51YWtiagbvyAzvanQbUPumSaU4e2Pj02sUj18XiHR0H05lzcQPa00 - roqaJRZPt/Hbrj71YLsow7/LvpR0jvXFWPLgaiDDwQxehP/pOVVzW8VqujrsSAlL + R3NRqz0/V77zVH78+hiUwaMq+cS98a9en2IYNbIQRzxGKQx+1f9+0XlpmJUe93k4 - fRGCTZKGaWj5NPVjyk1NHrBqIH6tVuRWkKdLCM93+HoCz8AmnTGK + wC7hG8l53bXeMj5NXC2W2RtzDRcfnEiwcIYUkfmF0DWMKlVdLAU= -----END RSA PRIVATE KEY----- @@ -5931,62 +6531,62 @@ vector: segmentSize: 131072 total: 3 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:MDMF:3xhseftp3rjgmnhahuvptvg2yi:7dsywoo3awdp4uzwhgepzokecufh5kaodtpefizfv3nzjtdkcsda + expected: URI:MDMF:s4hyspsx3kyont4eaad5jecjzy:43ak77lwoa3ent4nmpsb222hgb5zcwpcmhq6ct376qn2747u6mla format: kind: ssk params: format: mdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEpAIBAAKCAQEAyJfHAmbSrehM7PZPfuHaio2hTr6kvHXblMiojGhyZ+c+kpca + MIIEowIBAAKCAQEAoV5+u55DxsXqpuB98wwyL/XrkYXYsqX0kXcJ+uF5XMFkSd9+ - x5l5yzMEgNUMuX1hwawqD4zUfh4C+ReWuXk6sQKonRymRfUFfHy1Nrr4tAYo8Lm4 + YneCtRJJHeQvilh69d1RZcisuVpXxKFa+IiTZChXWih8CTtRQQArp3XlSllYlM6N - 2SE9fSTMXoy69KZgHcwMuowMGt7x/uubaVXlFEdm5U7imJ4HbzHyQ7QmbB8MRuQE + bSQ6h3GOU+M+VCo78AJJBLQ5MIJRJFzJVOSbtA/b1qV/tZ7NrXkA0c7wQz/bwwO+ - m/CRTDW2MTRNPXNAfjYY5Ll+YSUgaz2H6AwWJFrJ7/c/Lfn7a22jD7UDCqN3/CoF + hlXdAuwPlcR8vyDkO0ZapQBlq2Mp872l6S97vjNX4mmWmJoMRuHwSciyntjdn0Xv - TzTjmkgum3DfsXf97q0qyV8ZswJTXRGj6qhu3gBJKZ8VNi8BsqjVZ118+1oxGUJJ + QIMy/tmwrM1K2yxjjLoXU6anMTNrxdGG8t2N8j5hKprX0UWsiEAVQ4otCzJvmYwi - 54xlaSVF+yUJBvTIbkLm5goDKxkBBUptD+pSywIDAQABAoIBABtD5h0BdOmGhc4/ + hxCccJiLchsnSVRhO56KTUO6H9PmqilpKIdPgwIDAQABAoIBAEY61Ig+JHw9hdbz - vJZ2hIoIrkBR0Xp8WphineZX3BUbbXnnaBBxYAiqqpYIX25nCH6WtDDg55Es6yKI + 7Azb2XnTGx9986YooOywNKk5+TI7vrSB7sTXA41ftG+scF5TDMy1cigMstOGdJ1Q - fkg/niapQdn5JvCjWVeOa+NAjsWJgM8Xr3Rz/DOiaNUBM/hFFRN3xNMmbg7I9wO7 + pkF1W0RjZEUKSpVP+hiChP1AS7bUdL9qt9Vwx4JMEygCRg2mRei6beH8t8kbZkof - aqhqsHSNMANDMbGk6UXH+DITrpVLdTTjhi8iLbpXTz+qh0tzIP71OBqfI6IvO3Fp + kcX/Kp6uqjxcUd0PDK+7cnZdjGaPkuAaiMFe+xpw1OQBVz+g7AMMMfWzSpPdU5+z - Yj/CaTjNBwovuClKMKQzmLCS+arNqX+ya02EmO3IlqFA0hzd+7QFHuNORqu9et6M + 2mNnoPzcePbFojXbv4YQb7IExJxp44cS407IMrM008s2SN5XpNEsM62JdCbvCQuC - 0eDsNA7jEpFpGFZCiiC5ke2uPq0hsidVb2PitADYGFzJE/X7Ey+AihLKNABaVNDN + LQJXvlccg6GzOv68QFtiShAWhskhsAcDGFDZLJyLPCTTUsav5MJsheoD8gCo1ogi - wn2vsuECgYEA+LJrb8zDD1fZN3Q34UKpZTqXiUJj7a0CnqTlWu5MP+kctZhRcO2i + 1yNVFqECgYEAy+ZMATNtYyRuP7w8sW8FbK0uHRwtA1XaSfde5Edv9dy8bVn9iDik - W41PhJH3h/UmMfYWqTp6qOuPIsUPttUQPtYhfIyePuM14h1tgUzd9XpwJC/CAx0W + G8pnbYWoUOz9ol5PWG5A44WiuAoO1/wLBsbNXyBM015SN/zwh0YHV7G6I5cs+018 - 1gze/ouTTFbN8qygL0dXQci/SMNYFwyQzzrFva2YrHUFo/kHLSl4/2cCgYEAznu8 + 2RiyPECdQjYNPvO+JaGJ8KLLwh71rn5Jijs1azQNHWOoHqz7hSjC2iECgYEAypol - M0C9oj3JT0rJaZ+tGwDEoy5A7INP+0Lt+BB9tfPKyUz+Iw6642QlVeS/Xt6PqET1 + 8eS5bo+Tet9UuNirpozlBbjZxad3mhlNbbkn3yCdlo16lTv+BdTT9iUYwqdvm7r4 - CndcIKJ+hfq4bVMIqgB/LqCdKZyGed6D0jbZKPK0UBHIjxH/54Bv/B/zrX/d/vDX + f1qLpNSDD1fSQtkFwLKVJMl2eHt/kFxZ6i3Qp47y+x/0CXRrwUcBrOKXRDisY5uo - P7e2dt2M7GjYEv8hnLR5UoIBLfHIubmJnvtqhv0CgYEAzdmw//s2wa5vR21VC4lH + KahCoRJnkcoFXbSTUd8feJZKdgu3d7j0Qd713SMCgYEAvihJOdV8brnLGCW1dMTV - +VhEMgLX/9Uiw/mtNlTknEnxz4Xic1ze9HTFCvBfORP7p4MQQsb63HMOKTN/zFAT + ikT4fK2KTVIEAndxR/RXtjPmhxUmHaS1aDWbv8im8NIUuRi7Zv1sBsTavEilD0k/ - gE9xrEwgd+FNqnm9ODdNyXCs/ebh6f3b9xT0RzF0nM2E7odgl4Gvge4OFsZKVdm6 + /1Hoto6pF3cftpduurnUnzRhJFAY40Wg6dbeYtLf1qASOmOXMgE1Y/ZvkNrOxa2B - yyzUnCnio2zBXHY2MHWRh6UCgYBGji1e7g5ec/Jn78wnFXLXOUn34IQ6zRv1ZYdf + aClP6Gri0EXgxLsO12DsWAECgYAhBcMNjFRVGv3U1zX98wL3YJurtRd5yfQKn/ko - LnNmSynN40sru4rMzJmdYg6qYi6Adx+sNeD7HctSCLwgTzE0tfq/eg55+4xP9GLi + 2zcOfUhyU5kZXe/nj2sFAcLpZ1Ufsvfx+hYsxZ6fD5dr4ee4IuOAXX374VVHeGYH - 3+8QeO54Nbtsd+ATwOWDJ3/il0DKLo2+rg3hTA8tcR30T82yeFDEirvQcT//hpCq + b3RE+13LZMfoCpvNov846K3zJrGigqqEL7K8gj1zW9RIE7i3bYC5rPVyDDLZRsI5 - DIr4GQKBgQC7/Hwv2GqWVHp83LqlGFUp7XiNEUbWzJeDhRtC5UrM+ZAOzelwmxmw + QlgctQKBgCPvT5jiJXsM62rhhO/7HxrtDO0o/CijpZiXQkPaRuchNdNh+KaleHg/ - 1yQNH1n5LVNrTXlXQl6RZ9OAGMkCsKaRdr6BGnkLaj2KyfhfeQh3ieqnbZ4AL40U + HxYQwPLnDaqxKt+FBwgq0Mu3cRFuAN1snsjx6dvpFtIQtZi/IbwSUf2zEGfVwsCn - dBIMMBC20tiqJ/RgTG13RnM3ZZnIq3o0ED/7y0qR5Kub2fgEvVTg1A== + jafG20PQo9VmtwInm90Kqbz+qTXTTCTzwAFGHiq6tPoSggTp8AYv -----END RSA PRIVATE KEY----- @@ -6012,62 +6612,62 @@ vector: segmentSize: 131072 total: 3 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:SSK:xryj47onuklcp2ogizxiopkdtm:2p2w2seg2xpefulijcksejccsrbqsa4wvcvbowjxwwkbhvuz5mga + expected: URI:SSK:32tsxhwsbircqp6wt3z37xh4ga:f3iewkiw2njala45sk5dmmjsl7po57q44kutdflgfudorwxox5yq format: kind: ssk params: format: sdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEogIBAAKCAQEAw4YAcqbljcbU1/s5lRi7p+A78jT64FIpGLyq54HyAM6PWbJK + MIIEowIBAAKCAQEArrji79TMBXxEgTKfQOBjShN0/1TrT9AGpPdxIt7VEUP/pTdP - 6UbDZEbEIknszxXZru0go/ehynHZ8RYxEDGz/tuLBwhevgqSwWA6ONOdrS3N8F9M + r25PjSpO90YXJsQYExyf5Z/rp88gHzC8hfJ7JBFsRi6FNG02303zhAzRydNn6AW0 - mvo4+TOUeD3uUIhUf0E/u0yskenSeTTuDGTrDgf4U2jmMpXjVmIfTBcPhveL0Z/f + X1+bfjH3ELdf+skvC8mcT6jZP4mRZIQOSAC6e426FeLrjB2qbHVj37pXyKSi/OvP - Yr5qxzY1wCWYv1+e+MUppClc8kLJ+MBEL+mbtDSqWl/DKgfGTR4Q1aeTKbdfVVus + Hs9esqG74KyISQ22xD9X9aPE5eK+1S0V2Rqbh4NnQg1qoA0iUiAbRJyga8TIpRuN - ALQO7HMSt7MuC4s8dkqjZtn8YhPq8tYJirzUdC8JHC4m/oLUHpGMt3/vlaBQgJyS + KFcprOLd3POeIExviCPjVosH7Ta1SvDH5nZ0qsV76tiZGBzads84wFzg6RMdF84d - KCGaNlFQohfa2qR528DfTP/A3xR06Tboq4pbBwIDAQABAoIBAAaWTzZEfHSjHUy+ + EmNhb+5crWs0ImtXzdSWDF6+ql56c4wYPUANNwIDAQABAoIBAEwKNum9qNUyUfYQ - 6Fs5fSmPpi6DJJFztXIGHlMGyu4PUJrmN1MoefDdjubBr9OCg0zfmrayUNiVruw3 + e/KWNWAFu8Nrx8VCecHN1rUgWYZcG6RhwBJPZdu/8AH5xRWf/gJDUOt0f/DWWdp5 - vWSz62a0vu9BgZOewySIhwtVyKZ9Lw/ElQTCwukMGOMOKoKWfTiPdbGQpBW6F2yK + MXLyJtl5o6+fi2VXqqvglvx/P8YgdXYrFWb0iw2O3UGvLNxONmVg5uBcUcAvNNGU - YQlLJWIi2HXS7PlST9l7QPA/Y4BQkErTIH/MA/362q0PNRc2XDfzN6U4F/SuM2Dj + D0sS1hXzhmsECRM/ze3J4R97T70IAXVoW/1IBFKVRVS5YHEYDy8IHRm2TnhXGMu4 - geGHropqC7q5Cg1CLtsUU5XyNrK2V8JCUjnhdHKAAHEhUPE4K8K03jqHcmeAAWri + tp51j//PQwR6TB6vh5eBGqBQKSY+Iygb3r2Doy6ef/nFy+NmL1vX94IbfueGNb9/ - 8rXSMXqhKxcIJMZcDu7pSEL8yGRnNCIIlnlRKCevbgJE31I0mfA8zfnwvj3vkx2U + mNB9obqiqikzHV9573x4oahvinJQixuWkmPzoVDKtUsHlO1xXyulZjVmPuJyl5lZ - guMZaokCgYEA8oMxmelU2iymtUAVj81nlqBtJJ2f8HsThaX934x3i7GtlzpRDOz6 + NCUGFiECgYEAyHt5brRA/EWoFSZ5/YdGZa4B/p94qj87iWCilUkxrIDTYtM2iMiv - BLlRTpSR9+w97RCVui4y9Kn+B1KRFHvBfATcs9v3wVsbIu4YLmz1CKXU5XedRFxI + oGDp6waCiF+QqiDAFY6Ggq1n4wyMiYKwfD9ggGhHLcdtnaRaeiYoOBNPNAMvLixd - BzGF2CkXk5LetyyUKY0mIUyar1+pQ3hF970kmcOQZdjrsLee4+DwJLkCgYEAzmXL + saw0g+GOn9P0QpHWanqIFF3UBhGTNLoWBKI02FAhPvbruMTpM8Cvng8CgYEA3xs8 - fuJw4rf+122dra4psLQsCk71aknuWvJ6yCpQ/CWsjMmWt7HXhY2iMiKY7FHVr0Cq + MBPH3ggBENqg7jgye2fZ+IKaQWIxFKa8kIvHjv5dasY2nBxM6xcZoIyz2XwQIfq3 - GDGItWzVQMwyFWzGob1s3dkNXsDCsaMOazKLwmbg1aEo7d8ttME58pbRHSNooZdD + o2OU3nt9N+GQ36Y5m46igYTGkAlXTqMuq2ITlq+DTmLwLPLTXRRGPuLsdTtLJOAa - mzoNlGTSSQsKTepQ+cLcYaRRNp8FmqAzHdbmHb8CgYBQJoQSNkfRA8jlRpTZvi1q + Sh0Ec5JqtFVlXNVmtsWMlrPaXty9ik2/S69mRlkCgYEAsT7g9CvvDFo1KUXUMn5n - XwMzgtUFiefd2AqcA7TO+p5AyQlYmEnZndX9fqTvp6if3UdfDT3SFwzaJrPEbVJ5 + kbvOzaOF1daDt4g1FZEZlq5qtQORQktTYpJsHLqrqw/6YT8FM8nHSD8xCr5sfaKK - RrIaz6yGvzGszbw4O9KQVR6T6ICVw1oa5ocx9gLQx03MhHNDeF8Nyl+lbpxmrC2T + j76kfcIzs9iOJAJLb5TOmA0SSCTMkKDu0QczgqlnJA0K9dPj4k2kg4UUz6y4HbSr - v3OFTlk/D/51nXpqHkHAIQKBgHma5lPC7MnXqJGa5v0OkUeoUA5eyR+voXz6Qrcu + hLs3x0rIqdc6PifxGS0w1qcCgYBdWwkZWP2WA6VmhwU2CR/ekXscyJGBcHP3Hzni - n3qAY/KrT165rIbmlPq/AaSy7piMG+uXO7nQ/rBn3tZauYlQBxWKreL25X8t1+/2 + BgtP41H1ntE1C4aIDJd1ncqX45jgjweOf9nIKsYfvuwfGXAbjlijd4qatL3qss+R - 3vtSDAQyKOBFzzMhaZfxnhFx7FLQwadyg8+7u14H4DFZ7g3J7nilDKiG9xFMc/GP + eS2XLQP1peK3/DfDR/uIzu2AtHniCUAW6QN21Lp/kQgkC0u6iPkmCkYC0b0iBRxZ - zRMXAoGAcYUuBGp2DFGsR1/1OSk1xu7lGn9IgNGiZlrB/bjuhs0t3/ZJVi1l7O5w + sCBMqQKBgFysV4/bJM86H6lq6Z72bJPoFCrkvdEDvWPyGcRAsdzp23AsxzZyXwe6 - 21kFvpJbKHL8VJGk6kJdHT43zmcHrFPBkVK84mLXHXXupRn6QUJhG6il8H8XmaVs + jOclUXpeQZ92YTk6G0+lijFZTclVLVyF3OuT7lJn7iAt1nWP4CW1UIDn3AsYiB8v - zjyWVUaMjc+MXydts+MWNivvZLqdQ7L4suUwJIecdhFxMGfgfGo= + TnXZ588sEdQUC1ntog+ikIZNF5am2fpC7uBLUtvHCzgbf6Zg9sWB -----END RSA PRIVATE KEY----- @@ -6081,62 +6681,62 @@ vector: segmentSize: 131072 total: 3 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:MDMF:6fbvodb4cfvfyztxliweqitaau:ub5jwuj34npt3jixo22rqzoa2vpc7bahk7sdwcb3k3nukjez4y4a + expected: URI:MDMF:chrm7iotsma3yt3dw3sc6tr4cy:dmhl2yrc3bg4vjdyuc2xvcdqflt6zz36mpchvc4bulhu4c7dru2q format: kind: ssk params: format: mdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEowIBAAKCAQEAzbgi2MRkM7QQKadt//8U6R9839B/oFyGFcR6BBn566tGvoA8 + MIIEowIBAAKCAQEAz8YPORhy3pEpkHDKkUXz3UJXmGEQ+88Xd18ingrmzACZ52f5 - 62J4O8N1Mzs+1TE9ObQ3meejr7SL0Ksr7QI2rcfpes+gUDY4eK6Y8ypxdDmG9tnr + 6R6ZCw2F7cAVgrWbvPZ81jClKgmatIaFqvN490NUVGm/BgyLd4kDVUxqVu8Qb8MK - CsD0RpZP3N6rFiqCDSW+oCuf0s9UoQyfKtdRCW8NttcQHAplNBFTOUMfiqJdknLf + dQJS06N/xV8ZrJQLFp3mgccH+dY0OIZPYMwRhchACbPWlgDngIwXkySe+O89b1Rq - KMWsVcH6t0H/a4jkteXpHd/HS1TIzlcBXVWJmPyhGIFGDMaXhF8aZZRN2vA8vaed + JfCjj+t4VMYYLDg4es55CFum8VUQmK9Gno2hZYGoJUumIlS6bhQ0OUq44Rp0lKqy - TuoegnNA4ZJ3fbhlbRjpbKsugrmLitxJwKHuTKuHWSbYOF6SrF57S6xYC1g3q3sD + 4pB0e1EV0MnowzXb604xFUTaWN7kI/2Y5DvjgiwKMNWsdy/yECz5gvuesr9zw1OP - mEhgDK2YdTMp5npn+wMxLsZFIaQHNCXaBNQp0wIDAQABAoIBAElZORt+0pdYwVaI + VuXnAPt9W9p5L2Uq0xHjMd7vE0ns6CRlUUHeSQIDAQABAoIBADGDa7fJv4ASHFNX - uwDGq2b/ch+/EHJV6v0B49tog6KSnBO6V345ytLMOxJ8MkgDWWgkqJp4a/Vu81cA + SbK2dp+s8mZ2BT/Y0WkJUxzSEL+fSg3nDZt0BvknExo+CvI/+JnerI+3fnim2sb4 - YRYNbv+BQu8l7mwLGRF2d1RkMrWU+Vk0k8huyeoNGAaRYgDyQRJ8/b3QMBkTEYKm + As0jIdnc4fEO1S6qGSdWj6SJZhMK/AICOxD4yYe1YBrHJ71kd0L/xIF7ToeHeCDb - pG26/crWTNZ/UeAdmL3622iUVT+xRp0cQphDRuF7qBaD6E/+CYbBAZhvFEPhyUEC + QvnYj7lvp/EX+gR7uS99UxXl5XuBS1eZkw1wc6DW+Auz3G211ge5iewdG7IceQqK - jiVz3VgXUNQaheo+5mETcgebO9GInGM0Xx13j4JuSJIkOkrRwcuP7BKAoODvRZzy + clhpeLMbi6JOieeAIwQFzjLuWr05h8oRSU7f1pjBsy1oXSc1jqbFSPqD7LMv7gQK - uFtuEt7hhQ9Ok8hGpkmJhgf+tUWZfm55IsHMOaYU9n7BVocG+kAsQrmqWHI9LaPi + qtdz8yLr6zewUlVdljoY/Iuq8J+pOXnOLwtN4/aeRxHGNVd5/fKKZy4GhtM9eCIh - zGnW5bkCgYEA1y9/8wj6Mq5vzm3Z3M3zhOBoYpsZ512zKlxJ7Qkj5YB71VgdNOEq + +wVkE60CgYEA/dOEl8KGyD3jpEvIUziMpe2MKO2rWtGKmzM3atXAMyoQamL2hA/P - Ke97PLotZ3+HS625AYJzxFn2vXKtw/TtwZWODz1eFy8OqNlUBYcy2Iv/rLhpaZ6S + xEkNC419qtA0Su5OP260vJlORG3DULchOv3PxirUOnRAgQ/WjLVNqmjypqDch1qt - d2jEchazdRvsHyy02smtM4mPpjl9Ta5L6OCn8ZOSjDjRCcFS9GIjm28CgYEA9Lz+ + 56KqA2KMdqkRAzGh2rVbcS2RAGJAsz9qK+RK2GDf/mPvoAglGBJu2RUCgYEA0Y2T - mrJAP+Xv+4Kr/whln8MlEOAhHc5XbQyBaloyjXNwQPjaMLxsqmhQSBS3J/Nvtozf + ukrd9AjBIQk1wsuQVXi5K411RSy1cjZKDMoWjkntsuEqmw/u1JztYzwDcpxe7oUG - Y0IsTas42fpZhC8CfCsMmozETlb8RNIRHB06vFgjsr2JCpEMyxPsxw0biUKWxHoW + Af7VBMlfqC4/ip3+xc/H7g5VxYfKeRfWb4Kof/hBcnw1WdkpB1jkOyD0crqGVNFQ - /MnAxMfu+SCedLe7fgcUdFCHGrQyjqdUTTEnNd0CgYAliTuggWhjftoyACeIxMQ9 + yNRr5deCii+Ptw0CNI2gfVumKV3rI3r8ecu5lWUCgYEAzy33+ztPcmB5z9//alYt - 4YdT7ApQuZ+PPBoJJxcD6a2wQXaWytA4EHZG2ZMiArTvFpa3FLJtBeRsl5yIGil/ + Dz6++aVNCXshrJ54bF5+XATIk12lo5OYXbnWyQg7e9In7MVo3wPIyGxF4zrIgriH - Iz9smR/Ym3TLL13guPy9mW44CtMYgXi7K6NY42zaeMxvg7TolVWJL+3G6TBaZJUJ + ijQjhLfutKuw/udjk0RlWWQCM5n3hw+wBPLT3KFyz2QeO8ImP3NqaQ3tXifUcbqg - QmcJVNyzaRXNxdNanePoMQKBgHx7HK0meROHKtcskbs4VCg8o7+/oFh+uW0X7UNg + OaVEpiskieseazxasrRRACUCgYAUtpfZnDjgPFVKvKxRuuljaXF0OqFS2x58UZ10 - +VGFI9WIPcKINGVAhYUENFy3r1yUrpLX95zRuCr28U0QdB0f8Fszui12hP2kM1uv + Mi2Ulv4l5lfdx+9lmLVmT7JPi+FeVkvHura3qGgKKbgiLYfXabhrcYNM6SU8XvzV - ak6eLhod2XRsbqtkSQy9mAHqwrDQwJx3KfttDhndA3uucEkb5MV8qBtnCEgSyAgz + POcG7zUY6eokHXEOxtyYc6N2C7XngGEp2MVpjKncmuLrxx9mrnEkswBzqlLpyOhX - NhRpAoGBAJ7HJVTQGZcT3DF/IJTg4JaINvsOtUCWsyegm889udpzlqoiZLAb2Ot8 + 6alIeQKBgCDaVMa2HCfRQJ5p3fBT9aBv0huV8LsyurN635B0rF6qUhLA/bhYgDfc - I/Qis/9wawNFKbwNRYtWyuBivlpx+jnkaTLc8mqqoOafe9CS2b4Uy7AIbf3m/F9h + ADjsEsgz38LahiGqE52YtrfuoJwX25MdFtrDjXiGVpIwcByqDu0lLK8ctox1OufO - 6X0XjnZrqd5uZiuaa2URuEw1srj2pMDTehP/OraDy/bJjrgzZ33p + xRSKmBf2aI6Ruhs50y/Hq6Q8m5Nshtkq3Kmqd0oMEuRn3pynTtHO -----END RSA PRIVATE KEY----- @@ -6162,62 +6762,62 @@ vector: segmentSize: 131072 total: 3 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:SSK:rgmmkcpc2g4srtr73wk2z5g2zi:mp3g4z5srhmtfxj5rgveq4gxbxnlcstsfa6kxi3h4exbz2f6ka2a + expected: URI:SSK:fqulzqs2c3wfj72rw5qxe6a5bi:3uld5nrzgjvcmsjvesheeuowyhnmhmz2fdlqzvrkzyck3g7se4ma format: kind: ssk params: format: sdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEowIBAAKCAQEA1gC9/WdwsSuvx6Sl7Muz09YTew8YjtmY5JqPea9mlo0354Se + MIIEpQIBAAKCAQEAraetHF9xlUFy1ZkIoBHJ+5WhRkta5xo1WeN4/1UdPkqzbhUc - go83JHmyzpImu3GVB6Z8h0rQzgPUgc9S8/79eQC24lrO3YjgPud+syhk0rW+FyiI + n7ujkO8D6XuSuhzDLoTUjzYvO1BVkIegZilF/seP5EJSkGlbhwjsiLmWx+U5Ca2g - URGtTswtq9adXVYNq2YsxeErdMSEfVzUHHTQ0wuiXbBihWmKg+c+fDgxgJms8mAg + 0Yt+1dhgx2uVapn5sxlarlSkexbjx26+fmiibDrasOu84R8YvMKybia+H+KX3+Yy - FKgL8LZsic55ANUGgWJTr6/M4uab1rgAlvFLl5Ke2XPvTjMR0YGjseLU7koMLByN + z55rqjqn2B5iIHVtt/IOhHSU/8AIBwig315Dd+PfseAiocZhTEjloWpj6M9ctIHP - UyKalXDLrL3fmQjE6H07ZirzANuje/AzC3ki1u/32xEDrMpWNukirfcHdyj2cyjU + Npp+WoytoS83NmqduZOh4cLRMBNxddAsXj/XFrEforfGWavcxsUpgKapDecUmZHh - jrlbclQPC017pYhzHbkuPD60wSJW83FYP0UxUwIDAQABAoH/DDHqAs3rBnEB8SPw + meWvE8A3w83l6PlDmFQ3kmDoD5o1PXCB6kLTqQIDAQABAoIBAAHw6+md+pHy+LRW - 0GOpm5vLedv4fvmqkMqLdCKbh/hGUa+PwHREHZDh6lBDEwzgZms+1yvrQ4O0/UYa + UxUOtD9SHgZ79iaqKzuiKGZR/XmXnKM+NNZFL7VfZg6yLI2IFz3Ftbu/aaw2KLSO - 7IEl9SQfFpRE8KncY/CbaTAYJiQdjgwNdDCmFProMDZmdlwTOQv/3zfJpH6Aze75 + 6a63df2rhELgFsdxM2GL7uPdBvMDIsMu1StlqWowUn8E9mhORoIpf9MXo1HWpRiQ - xdEdr7RSSbMCF4YTIYjER2wNCpKfGZ/PJmo4mtFIm4MpZj8TcuQjIher7kuJfHEF + oaOd5AEHo4UqSa+j/E/V+fk3s3KwxpP9Gsgh/83xfb5U2D2WyXGFdZiEADwWu7u8 - h54QgbNdQ6bmXuHbGb8k5vAqlW2O4eHxnIsasY+EeMR6uUTbEadPYEZcgG3aKq0V + Pee1mTsi5W1o8lCoRI5LIVdIdLfNMUpLlXx1eAKhv9N6IcJYn84yTbGKJSm6OzO7 - smHLlkes9i65J1DgjoTNtr/tcBLyOqF/LBbEEQzAg48DD3gyv95RVGdLdtE8ZMds + lR+KpEZcT9fovahhw4IvYyC4kN8IwsnISIeAOl1LDEPnSC8ItRP8Jse7AuYuox4C - 9yFBAoGBAPh07zbXEDiW83RWYT80qwuV7mKgmRkoaH801YoIchEUfz57hVCG2OVO + ki7LEnkCgYEA11TWaXl2ltiQqMP+UnhgjLixC903Yv4pVRPHMNZG6azylMNhPDZr - hhEQZn8D0fcHVA6mekD1Ba69IObkgHq1QiyypUgMz6fP2AamviIC+NGRnptd+w09 + HLdwvO1Lc/3WqVsFssHGYJFtcNMM2gbcNB6k9Y0KwzdDe+g780Nj5oqpmiYC39as - 4UPiIcMudDKrUZkw5rne5YrKBaVsucQxlXwtUXA6XdQ81c+/5vjBAoGBANyABiDl + 7cwXl99wQYOOzwVSjaGAZJKaKJKCGF6NhAz7QNh8oH+xaKPyJSxh6m0CgYEAznPO - vw/E0hSvJ7Rs1KL1wHfG6pXyNEswIQWwA+1VHw2ET69cmCJd7qRlQqDyvkTBp2XR + Oia3vGx5hRBa8VLZ6g6JLcL5sp8Ql+DZE2xh0UJ4arTqF8XcKoONGG0K48KPHAQW - UtsqUvcQtVL/G0MmT8NIXSE2vvFEvSPvoWxrJhB7M9jaBl9BYIdeZxVzYMiHo6Qu + NF4VRAoKtEZrjP/BqOJtjXV5CPj9NBVWI9x4XLbsh8ZIyk15lZgz5TSFrO5ENIWs - XNypZ0m8GLPunOlyJivMLcJpHcHgY5PynHsTAoGBALX556+yC4Z3QW9nSSjjKZh9 + YKk8RujLB2S1OgeWbJwBmakb8AOUDmiztBVUCK0CgYEAr8g18IyTXrkT/nFhH/nc - wzFn0Vq01vy8tN652toZui0IiZd2fOxO/DEJYxkKskGNk4p7crWbAQOAMNYMbPHz + 94OeJE1Gda1+GFG4/gkugnwI26BTtE/ISP0HL3OXcOz7W+1OTYsaYqLVcJEZoLKQ - Srm0SwyfnYSa3e3ZOQ9uP9I3JwVC63tCZHi06uerYZ4vDr/2KjffQx7JYyNLpDBH + +Is7pqio7IwkrvX6Wq/c0crIgWoeVpRtPwKpD/X7McAvyJhTuALrSS7UYeKYCUTG - 5OYjxy89ALZPrIbSVpjBAoGBAKlQ1WPlhzUAmaCwbvioqQ8JTmWrJO9HMMibiH/p + ydG/GkSgGHWlYgLUHbyJglECgYEAsF5IOG9pGYQF0EInnu+rkAN492oQjKLMtyLz - jNptho7GjrnFjDy3jExIRUV5oIkDextABTOt6E83UUUOB00k2hLGOl0KwMxbUDGM + 717wtac2XdpN/Z8fNgaKG+rTmb1VKpbnLTeOrUBy4o0iRiMbmx5MfsNzcdHb5Ymw - DJRIIs59DG7z2/jBJvJLlzRtiF/zZ8DmqP/4RQvll8Jy86J+uLjg7DJgrSz2tQAi + vBQVkwcGS/t9pa3IB58t/kn/RLuL8t6bYzxQbTdkct164Kcov4IK7+2DG2jDLAgQ - R+5pAoGBAKzHFNTmBSYYZlWVeiV7J21iGle6Vc8DSZHMzObd2aYGx8arnTdH1Y2a + NPDfiEUCgYEAhlf3Y70hQaGmH8q4sQ1Zzv1KDk/k8vxcTEoqB4rTjFuV7ueqCtHt - qglMSdUsUA4jHP880xFoMimcXnQOLPeKsAtF6LCOJw1HeLUY8P19fvm5mZnOy2S3 + Oa5VB+JofexDmfs9TueIwTjFsFaCPw6PSqaish7ZTnzOwr8mzHsyvp56c3c5sQsk - WnCEuBI5ZlmF6l1fnRbfdgC1bA2Vma/94XkWG3vqmiHAouh0vZfC + +ILbxcckY65gmD2ZKru3gW1oOpOpk24n6szu3UXuiWp8+AXg6sTsoz4= -----END RSA PRIVATE KEY----- @@ -6231,62 +6831,62 @@ vector: segmentSize: 131072 total: 3 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:MDMF:6f5qr4yrplexsp2cdfrozual7m:qzniyjbnxrtvqgv4sn5yq2mksiedndju7nw6of3ea3ku752clhuq + expected: URI:MDMF:wif4gkouxqe6bdypqiqjmkzy5m:lhv5p6hp2ok6iio5rar6uu3n5u37f4k465o5hqxzgsaxly44bowa format: kind: ssk params: format: mdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEowIBAAKCAQEArNBRHeEXTwth42HXbxwTZ8SuuF3H60KUQRFx3zXaJbWAl6jX + MIIEpAIBAAKCAQEAuwTMW8VXBzjm++6G+QDQQBS9NRUNiKNN8Ni+TFmSH+mIYt5c - kCHuNMA6ioK7L2cGMHUfTDzNiExP6a0WLVteH5voQcgnJzmlFLa+WZCsyi+srvr0 + uV+zZnZyvSiHdvcQo3VrM7lko5qWORmfmt+pLhO/BbTl8G1u0+RNnEi/++1f3Lji - nYm6obhgj9ihr79XVGmDNFxKHPiNmXPV8eqU3SLuSsU/Y/dSA7NLUna070nwGgG+ + UyL+U5N45+HhQpIIrIPqp9JfbJwxNMFZG20JGs+HQgCJs7APgp4EQgccGlizJw7u - ctK5wxKUnbQZL6bVBo1qhkOCuBHTJBHCoL4P5xv5cR6lducyi2IVtUB7lZmUFwPe + 7DfJJb1z52y5t7rESKlFPh0hhYR6/3/nHa1i/kuigSDtE6HZHd3cx+BfVBE9zUqy - IptvuAZp2l7kYQ+/ScpaGk/NIV+69QsJ9dsjZJVE8mk7n7aTvy9Teojdcq1JhQCQ + gkU0sdiDK8OzIIwlFkOf9wdkl27HwfRYSkbqx4ke0yejsdEtnmqwknd4d2uDnWBY - 1U0PKaqsEklzqkLHJ3xWlMGLfncFhnwwubCYQwIDAQABAoIBABe3VwqEs5AzfbGY + TdcXn9+3kGknnt31GVz9AMvjlNHhL+SBq+wBMwIDAQABAoIBAAD3+5UsjLtTyhd5 - 4dnrvnYFNf0zUZZlwrbTUA9T8qYuLIGjuEGdhnVS1DXiDxJITz8jM7JgvcwwvN7S + Dunv/07PFe996DDfPwwR3Arrh85lr+ZwRB/dGmbs/Eq8YEiVTB0LCE89T1uug30x - 1DJRUa+A0/UDJOxrKs6W7bSY+D2fIVG6OwvLtQMwrH/ROQ9HcRKykEEFUV58deJT + Z0Luu+4kiNznsWYjqblfRCsn/Hc047IrCW6rXMPldNzgH7f7TE2xQGJmvzr2k02P - VU8n5FocyxsTyslLTcQYPQQKKnaUMy6haRNdr1Uc/NuBc6Zd4VyqJ2ltRYKFJV9l + hzfzLnbhV5GThEU1iitLIQ77yHdb7UvvlJ2F0CNpsc6bWN3udjg6iUPeQ9HXQeCH - MXzHtBbPLuwO5nSIuDS+5pnVhpGhY3m8bYXcsYdpyG/952nvf1UQNZr9RwtpzWLC + YakYdqZ2iGaISE8Tk+GngrMY8UEDrJe5loUzR6k9IG1NwLbIbbf/vE01aMMIvYqZ - mkAh/GAArW8CNc0BWeqsnLsLfP0H6V4L+agzZuRoCKi8g0q1p6Gye8dIdXcbuiJO + VTeJXcxZNet8KEv+i8rL5hXgTKg8cJPJ6ex9HO1+aZ3J9TDS4zFl6Vr4YFAOW1q5 - zk7sr+ECgYEAw4e8zi8f2Io1aLEa5ZhZnmw4gHKiP9qO4tWBg2iAaqAgDqdhSj/e + xO3D09UCgYEAzXZw93h/CZ87rBnP0Y5LZWFZ/+iXZ07+Q7N1GM8IWA960EkeEb4/ - u9W9peL3MS56bLKEDGYUlICLIrO8JVHDpnUdS34HJ5gr6gjPpUu5ieo5AJvOYfHo + ImgPg+YD/YoOi4SWq8Ckv54JgpMUr5HLdo2KTmB2TVjEgPCsPOIxO0SbVhkI/kyu - i6XJmSuQYYUkNd3Ps0pyss9aR8S2XUa4XAh5tPBuMntENWhhTlBdpdkCgYEA4kIZ + FfdkiPfDzFuBmtdNLDRSqy5wX89OAgVN4N/fkO/Nf7rHgnluQoNLynUCgYEA6QT7 - dG0tQKpsg6Jp+e3mlPjYElX8t0legjHpxIsJE9CKPew+DaPJbLSkoE1bx8edKI5X + TM1vhCM6UGUuXC7NJfSstMkaNAmE2FOdbcbWgWXmc2pAFE2Al8ypq/EJsVWKVDeX - 4LMvQ2PKS/BugSC/c5z1KhJwUtvHdipW6sNmaNze7hpi7NQkBZf81BspNmfJ57Tt + G70pDe3iV76mPFOUGhjIthdb3J1eJrP29QBkuCGLd4rCcbJlhr1v/f6DI303zYCZ - 0GMlExfpNcWZk4lBmWvNIpZ8UFCkWL2zfNySkXsCgYBhylxiXm019oGZt6H1HEoO + cXZ1loch5psTFaA1F0TQxiAg16XZ6YXX5sWUmAcCgYBlnrUU0PYULjt3TXTp8nT7 - Ep/7ldmRx/RYfGHG4BgBu83spkfhQ6pZFSBBfA8XOOCfxnSGYvN+BgAQPgYmQAtz + +YBoAAQSRpGfrny1/n/j/hQCPIewwuW7ALjbxcInfkbfXn6fCDLzyxhtCo3qoDN/ - D/Wz0PcxFUk5RmjbidDkqhESPdptX/hnB2aZRZFzRIyEqEf9qolM5qmHZVmzsu/3 + uVW0miUo8ESQeXjWzBEJfU9O8Cbwj8BygN+qltCynHenu+Ehged5XwiZepDckv8H - j4GXPfxPIRlPAMJR0Z3UmQKBgAmoAogahLzmyRzRGK7G/XlMKYSW0ONNqU/rK2vs + v/J1XwXGrPzMXX7ZStMLmQKBgQDdbMJv5PeHFQKgysUXC9Idszc6Q68Gq9T0y9/Z - 9yU2WEAOThOs8tLF3uTMiGc9WLK7aHq5iwHYR3D4QO8X47PedgQmp06R/LBJXE5G + JQ1IwNAP9HMX193OYckJfm67eJGOHZUV4tZUSiy/PIcy5Cjj85Eml2PPbCq/lFuj - qp89FfKZg7FR2Hu4ody3kAm3YkGWUjP7l0B6W8Sku0o1qGwQ0r9wJrwSxQDYj8l7 + zM/ouNeSrOTArckUFIeLUILFAoQ4X29wBiUO+TIZtFqaPja0+ct5uaX7xbog0fKr - bHHzAoGBAL4OvQ9MbyUvV+pSoqR2EGCKjpRfKD0aiQK0OYOAuHZH6jPNkb57fmop + dC0TmQKBgQDI46RUW9S72Rtp9DPSyfqcAR9vpdS2uUKlLqM0h7gyAjGe+QARst3p - 45WYpbuXK+TQIFbtNApszkTBeevrZOT1sDCldMG7l2nRY8pOZsqUWqfiR8wtVVmo + eeL1WFMMZKbcEgqjJOL549WKSt6P0LsmHBgKsYX2ZeKxC9FOqdthuwWLUwMoJwA2 - 72jkBD/xAS9wztCtm/b45WYKuv8s7SLhTD/T0aav/Yv7hyUxbDYq + z+U+YG+nKqIWQJh3wcUEOE6f4/FbumdaTmgmb/vvnJZDJhaBSMvMlw== -----END RSA PRIVATE KEY----- @@ -6312,62 +6912,62 @@ vector: segmentSize: 131072 total: 3 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:SSK:whwvpucrncmbbxnhld6palajce:muvfanna6fyof4h3zzswt44gjs2opt64kbm6jipjsex6qj27y44a + expected: URI:SSK:opwonr3pv7h3vcqn2on4ucve2y:necbkc6fuo4ahphse3nyfrcsrgjywctzxnwzt67qxzwewpklmkzq format: kind: ssk params: format: sdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEpAIBAAKCAQEA2LyHGvyH5SaY7lu7XkEwcE9nT0K9x2fDp/yA5TPlJvV44ru8 + MIIEowIBAAKCAQEAzDTLbYcIX95OIeuVqrdLTNNVsE4d6/aQkVUAHpl6b7w34/5/ - JykYGquQwKkMYtA8wVxIC8D4jHrWbwbhM/UUK8b33U9G3hgYsi1ksVDXO0AZTKNL + jvACFGBSmiJ70J66fDpAHrnTu+WxZQTS2xpi4CIla03wkEiZ+BrlFekODpot8Qtd - GyCNjuF0bPkeWMd13jwOChyAHaRQdXGGRbEobALDljE0+n9ihEw27Tbw6vv7pyog + CqhGD30Xv3KYIG2qo6tHQShGG+9GY9/hswSzfjaxMeIJIjy41XzTPdrCFCVzrLqD - u6XEwURFjDCTTELMIIA2jfaUThQqjF9jMNZAQLJw/w40BEIlyJ/y7u9XGzbsj1UA + y2vpEbDYhsuujSiBMlOO9c2l8R1DAb6s3ci4X9X8ITp3NT7a5KlA6/Bp1obEiqfP - RC6nTYeM0hN53OTlNcZry3zIO58zgPufkSUV1LdqYj26mUx+ILdtG10VIMfDz0fg + ljOYNZUWC/wyBUsVXi1g1iF9wK9EbVcWAgYmEKL8d0wwBNkGGpyMGD4clxPS4QzS - WGbjKVBVRCw8Eb7NhGaSgjGiFfv4GFU1/yeocQIDAQABAoIBAAIIR4nhs7+wIazn + NEyh5iJN+sCymNWBkyJj0AWQT3G9a3I3G51ubQIDAQABAoIBAAWbAWaNSV6QVKa7 - hDg6qy18bcsHmfpjvykPuYuHVcG3JCMX0Kh7LqA5EGEMHMMpnHjm3dVADJya83CP + t80K4QdH2ddQHaQnjYpfwfQVFHZSvVoF12yODBCRIFNY1PtCEC5uzunJAhXrVTZH - AL+GhRdG1PtuE0DWepcjd5SgPfputrtaHc8jRPc3EZWgLZA2eO+Z7NBHCuOsRHsL + rp4TGFm8tjg+2Hatd4SHAHjcf+VIuDAgtrofKmUscuVveNuTBxcdEYSpXVtQ8ya0 - ctoWXTs2Y0GLlH7OVmedmYIWhYJl5ikfW+0nCFGpFtgAigsIJ5RzSrQWTKNVfJFM + s5Zdb6vsRmrvIH8PGafKmGXfRmqVHAmT0fmV+aOrk6C/Q3sC5JqsyjVkhHUj+OSI - oOgvchPC9Bjs/Vc5kDutFcidkM2gdDlKIjmG9Ux5BnDfPtkpMokIlj3vlhKNkE9a + fmgSzFAizPPEKMVBwbBDHGkWYdvPp+szWrIWPareT6JtnCNL9mDFZVjmVSIdz/y0 - kNO3US62Eg1OGs4cTHcKHAkjwavxghfL5wglgKaK8xhvtshr/ym2uLdO1j3WUoMJ + phHU0D0sGT0lJGdKJTRv+6PVlL/y/6BRZ8b9zVPAHrIpTRY64ILa5Z6OW7PYlhQP - qZm1jtkCgYEA991xSuwkr0xe0OacN55uFlRRc8aLeh8mV4r8POqkFjXZVsigZ+1m + P9a/14ECgYEA2ec1RSbVj68ttDqNFwqTFA+1wf5sYtlzEaCMUhTsv15xdHWshJTx - FEMfJFVj5J58Dvl3A7TgIGM4qP1ldnE6Fb+GBJzTRb7MTjgPbUwXYmBPmhsz2pXA + ibGyWHBeVsoPI1pxNzTcGJZCrS5J2BvmdngGyYXf2XowaHGFtXkg/Zmwn5GHvC3X - NHIXXeUYOMGrcWGLy8Roy7ppe3Wudea8DYMskkN9kHBCwGzhfZ5VM2kCgYEA39mL + aHM71vXU2P6Ye7xvPWQZipK6IWGudAFKvkfvyeWjELlpxgaj/9ZaR40CgYEA7+iL - IYR4H0CexpYca0VUA0DbyCh19fkR7u92Zj5F6VF2GqmBSEtRa4XAoObYvNEm45f5 + /x5nf/+5dV0NLk9xw25oK22r2WlI2qVyQairiQYSjUgR7U1D7t3BKjKZe1P4vD75 - gkx5oLgmahibdB/iaQc4JdczqzgD/I92LfOxWfMcdfxNku2VLYtTFYNRJDdSmZHt + RzSrEHgxLo9A/7S6MKdJJ0fDjt3E/8UVbde/J1QI7icHimeBndsBo6hJBVWi0Xoi - GVS2KVOtxm524PZ+Pse8YdB2tVJh/c6vCpU1k8kCgYARyXta1A1h4woe1Z26RA1E + VbekLOCBlSY8C7b+6uRBebDl8q3BHHFICk20GmECgYBF9zIokQ6TgykGrKIu2stc - XvKla0cREXEv8RJe0LvLuDuLhcQ1EQ01QQfYFKShgFoIvRA0XOOEj3o+bki8si1n + 7qpqrrm4h5+l8kn79RILZFTDkyEgtP5VOwRL11DDRz/TFzAxDLz6/AxOtQUq6dJ3 - 6CGW7SYgKCwDJPS+dCptbdnohjE3a22qldFldI5DbGqALW7ZxZN7ozn0mSJW5aLz + CZUMUfsNRmmSr5jCKzGHnDiVE9JkfseilxWIsQh14FGvsVJ6gNCeqPwwyb+NKfkI - GUm2iU9WcSfpJScdW6JjmQKBgQDNWAuIcLOcv7OnKlbhlJRv85Rp9avYO2ZXEDZF + 3epFhoF0VkR7PBiehgIY5QKBgQDMWUl+KljAt4MyS+thSfw+GjoS29zoWHzc+NYE - roSFduPnq2zcO7Nx9h1xvLI/64FIMMaC39KHO8aJdw9LpGAWxrecBuDwBQ+rJJNd + xXYvRgPhYcUbW5gEy9Cwb986JIGXXxCYLW2UnrxNy2nzJO7/aE6wbblOZOpbbnVd - rfoYMKsAFLW4vdcmE3Pg/Th3B4TvOW0N2qbMHGYB7J2C2ruOrb1C4W+z/+HCaVIr + VcsV5cehi48pvhay7gxMaZihOZtxUNYUK1Nlgmn+ME4vMFWcoIaA8EQ93PDDmF5j - XBrs+QKBgQDtHOWbd+IvyAc+j1G8yL0vpoxsaD0K4bUzywCeI3FwXOoPU3eyufs0 + oGJLoQKBgCOlFH7EUlugO878go0qGU0E/t948pBGGM214H0iNkxGEgA9YhOfHVWk - pqiJKaAVfsIXTerk5SCUMwBnp61feQephL2X8Td0oi4dCYLsRz3c6a38F3jv44Rc + cNpJQcFULt1ue7yDapBypWD0Xh97bc+nuTV0u5mR6jt+E0qm91T74czMCdVKvAzN - UQ0gnsCGH8EKXbAsDUD2qi8XsN8I8UD6hi00JRjuliDoWVcR6a63Ug== + CfYOaXmNMWOJxdT/e+VJr9KDeF5VDnUl2EX/sGJXQETZQRZ5RFNf -----END RSA PRIVATE KEY----- @@ -6381,62 +6981,62 @@ vector: segmentSize: 131072 total: 3 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:MDMF:wij2d6hdexhefscaqhtvj36ojq:6njb7giycq77cs3cpzj42bprznakf2larkyfpp6yejteu6x26nyq + expected: URI:MDMF:cwkenndny6nlpubc3mddn4jdh4:op3xzbo4xhsmqwgsjfi4oqztbb23grvho7mlmzgzaq2qudpucota format: kind: ssk params: format: mdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEowIBAAKCAQEAwSHlr0hpeAdvXcYr3LMj8Tiht4sQm0SiVnK+26BSHx8GdRkJ + MIIEpQIBAAKCAQEAs6M2GcOwvLQXv7fPJMCmLUSi/ulwirddTqaAIKVQ/sxEovqb - pgpMaAT74Oukc89X0gHCGN76yYmf9FertkpYAtlgj9yf/BwIWMnPHqFkhDqSxi8q + 7gx/yhUX6p+K1OaQj/Y/le9oaJR0W0bD42ElSjCopcyDgg482S56w5vUJ1zGAjgD - d1PR6xOrRqLwbdtfB6gR9uazjaQ2nnsxz8zQbJ6TZmQR9ipJP3FmLSZLZ3jL/8cI + PMGBB5vSpkuEsuuuoH6sDJEW1fLDxraVt55vK9fwAHeGYnvGkTfwi470ArIj/EkA - 71c6/AaxlVxiKMyATykoyWkFxw4JxuWD9ff/E0fxJUd2dApvFIQPAK2RZUo/LIGH + JIYKOsfZ25rJkvaQcJ3iHJlNGZyB/mmFXpl56exkPeHdgAelgmHOgb02uIzoyMZo - fwm3oHqIFl5UQDrJArZwWMA0bCTukXIIM+dxBO2n4bBA3MAX92K0baJvUaGC9L/v + KK+uzzwkCoLtWQutRIowiYUH1HN8PvRhgEQ7k6HbIYA/jxT+AuqrNCW1TDCqLJjD - n5jzdva1Us7zx9dRiaDFTHhDdH3A2Zltd2GRcQIDAQABAoIBAAGKcouDuTEvpH/K + Q8KTVt5TxLTYb94+3Ndk08czGjhmit1gNcjgpwIDAQABAoIBACNytxfRdnxeW3th - gQ7fGEE1pPnQFIaQ7VdhmG7rwZSisIr6aMNRYNc3MSFT+fKhVROC0h8ft3TIX1Ks + Jba+b2xqaXG9FhDBi1+cWpdWmAuXuomgw4lvnP3/OJd7gTVvBCLseHK5ahSNCwMC - M3gmPHtkgPQRiVkiCgmXeGqSSgPcIdqDoEaV+uunDepSakGtyCKz+zWK6m7twDs4 + DWC+yFGCFZ2WJHNTJO3EjsQv6WcVFxvT+suP8crTFHftWhPGj1Crfn8CWIvCmqCJ - PgSOPzxa8RpmP8aE2KNL9XSUjLyUpIqoYVamR13QKHQwImBVpfjcHzOHFrCwfDKB + YjT4Rj7UH0+wRmwDudTpQYYAoSUwtg2XJv0H0CUL0+S+Slu7mL0ryHhYngOK9qiS - iVhVqDHPff8zmlge0nSQSt+ikJ0l5XIL+WTDWbHewahbLHP/hdN7QCbvAkENUreD + GZ6Sf+ztYWTW0HTHun9jVzobInL4DxpZJdGJi2+j2WLdBQpUkEnmtIu+8EPHU7+o - GeMV5U4NOYMJUv+UzdL2PHGAYr2WiNtZzoaBbKO52l16wCT9FLlOT3bPxPBrkx5H + 7hpqMw1s4jHmPcrFBVaxfXhHH9FhXyhdWXe5BgX+0jbeaxZU/DRV3M1MxTvhYTDI - gRYTZEECgYEAxcd8nHpF7WHEf7m5oe8OCiAD4WzQ89yNIw4UIW5PjcC/hivjy5Ua + VYKozvUCgYEAw8W3d9m/t+qOLX0nfo04S6kUb0/QaGvxBa206SzED+L4zUqI1j3A - KeAOFPlIeLsbsKMzyxU80VjfGwdHsl9Tj/1ZckqWW+l5H+rwLULfy1ydeRLDQGNl + oQ1Aub5kRMnzuJoXa6kZa9JSgbaMEXe3sDhxlv/yMOuwDDM78xtWXdYkAkQa2tx2 - G5IBiF6+Ps1G8Qwnz3COcyPmHNNAuqXmJs081F3o5wi86dOHCM7XvkECgYEA+fw6 + 4O8jkf1+15p9+mKkY4JW1Pnu92kPZwyUqnoeJ/bX/vcSnYJRwkc50SUCgYEA6ubH - Y/S6y7TJwAPX7L7JRua1C09az9BTMpjWWqBh/pVjBo7jjFLOKfGzPZS7QoZqCmhV + wm042jqwU4b1FmMHSxa6CuchWbQsScwpkM8fgdocDeorKsySeIYTt5axRQeehYVU - m/tvTPi7EHcvSdDW0y2JJ/M3fmkxhohSTzdwAqEl891mGslSTPtWTdZ8MJyJb/I+ + 4rs9/vLUzyPOAPWYoCZLhg9b6GxwRKAhbdfS4thJF0DblgbhDvWIwrGY5g62HEf1 - z2q6ViS8+APzjf7F1/6VpfCZF4QdhcRptqQKZzECgYB0vp1azH5MckKIVnwyDydN + yGae9+q4bN0Dsrdx3y9YFzMcYqIKsb3CUJo8PtsCgYEAt6KkoBVuknO//cdh3oFV - aLqBrTbmS9Dv2VaeqTvCY/1p2Kx9NoUcJMqLLN7PjTr6GEvxW5bryDbiAHkc3FI6 + BxOIiYkScoCdyrfP9ND67/P1cYuyo1O1dtxZlGGU6DmPFd/kjCZIJC1bGzVCWbg0 - E4ViBo8csAM0iPy+6tOpegDmP+ILNuCu1o+bDLnl3kw66z7wnvMnGhCyAS0bP+RM + Y2XulrdqVJ0fu7HrT/SapNaTXFTJ4/XcxM1MTkq8Sj0uYklY7cZ68LeoggbYXc8d - ESgP/2MERU8mAxuZYmdNQQKBgQCdbqxzMLfG/EcmZwU/8nMd9MNFqScewyryLXCp + PHPkCZSvswfLPFfbnSL2hskCgYEAvRg8lJpCGwMFsKfColvjoiHQcDhxk3nD8UBl - SGIOi5P+mFRTlf6CSdZAzP8ViUMU5NotTq6sgeSFHRop2ZzBB+ddwn1LXgIzoHx9 + 8Ymaznha/ySTzWdTPayJMNAhMfWZOdkEZWTf2l12zK0BB6qtS7aoM2onzWmF0uip - qQMglM4rA15/NhRfqNWUVaSGlL61QpEt3SAWijJ72zkyTqXYPluOUrSHK8vP539P + IHiN7ki4RfzTB+nPwLANgNVgxUnwdcHD7KgXrnGINzKP6I1eIJFHM53Uat4RB9Y/ - 54UpsQKBgH+D9NjJvplum1FL1YI4CORhVYa/vYqE5b2VKxUUyruGVDzgxKqkITj7 + F42hk+MCgYEAn31DL0OuzeNEaR/Lb4nOJFJWwCGO6XN/oAujp8bqkuRpkS8WzgWI - blLBlIjS4Cas+s2CPauBtYJiENZEhCDJACKjU72YGYnRU2GbGhHb49DQcBhvJk49 + fkJK7eI3TI0c8k9hF4hpt8y6Nh7aG3FZdXTHMBS9PRHaLgpHaIFCihibSz8a6kAT - JI/tLgL+DSThPLdeIkAIRnpx9zJmCkVKOd4xAO/ta3k29MYOr2IM + DIGjDMXH71CluU5SquvYcb1OY0ruKEMlDv4POOnUQiy8T4NPb9K9AnM= -----END RSA PRIVATE KEY----- @@ -6449,6 +7049,156 @@ vector: required: 2 segmentSize: 131072 total: 3 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:CHK:7egadlquaxmh52avu4kcwamo5a:svlhtfwkizf4z7adzqc4f3ijpkhhpoocvemtftgwatzbhxhxsyxa:2:3:4194304 + format: + kind: chk + params: null + sample: + length: 4194304 + seed: 2NVWOXvtwcAFf0J/g0fDC+ypQbnCU7awZoCxcnhpkVM= + zfec: + required: 2 + segmentSize: 131072 + total: 3 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:SSK:yfkgi475avdtdiruhn65b72vai:v5psft4tzbi7bdvht75xdp7sxosfnlpr2hrloo2e5zkutvox6kea + format: + kind: ssk + params: + format: sdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEowIBAAKCAQEAltENlm6LjBC6ZIcNJlDZpnYREFWSKsK2G/FxzXRmgB7wGILu + + gg5bAK2RIv5/qj2iuPc1pvc0XxAEiwzarWW/Slgbi0p55Jnj1SU2IO/Go3iArELV + + pvPT3pUHSTx3+mRYcJyWgeofvC7yzcMoR9Khb04/KV+/zkgfugivS9i8phjt/LBP + + hzQ9SGRLR8CDTBXGnXJd7Ym3Fu6pbnlS9s6VJGjGoymqyjJts5Icu10BGNrf8cUi + + Bkfl855JZJ8cwpyoK9HT8tirHGlhuT+/EFmt0dhsTrUcFZwCqai44L+KWdrTckyA + + qrlrVG10WEjRba4Z5dDFla57E6dyhRtXTZ6NAwIDAQABAoIBACjevi/mBSsP3XMg + + pg+cGV9i33zts46i9XbdF1n2EVDnEWmTEc9s1Hx6jLpO/YnE6jP1yjRVCXw5ewGz + + mg8jY5NiDRTSOfYZPgSk8OY8FDh4j2YfNobnzKKlADR4jorsZosd5CuQpsj4cBQS + + rvfHvLfNHJC5weDE6tQfRmHnejgIXvYlL0XVZokLzQPpLmvqjA1ueuQp29pUjUQt + + hxcsYwtpxf+g/LGhRxBl3HPeKID44bdsN8Zz/hzkqE9F24vKFNcuvjMnK8mM6oin + + tu3fFLrYhXHzQIjQjtK5UNMFxbKZe1Ya48njuEBwGxraYP8WL3MjTqPyyjDPb4Wt + + tFZLUwECgYEAwzQHBOE9Q95aosur5sXawVMdZggKK0HNx8uoOjgEsX27gL63emSj + + QcCv5ccPX3fbuYKbUA6W4vQ44z5/0EqJ9sJ44631FigwD3U3+ZZzeBuzGwszgHHP + + Byj/CoiQfCV2lZdwhwWxyHRn6IKvAuUnfiaVzVHE3EJrZTkx/qD2El0CgYEAxcn8 + + PZL8VMQuPbT0+z1z3x8NWlZwUwl5y5mqpkji4XaFWoN7AYp9Sa0MARO/J2g918jo + + kDMGR3tveMX5x9Uqq2WbDU0dQTmErb7skB62qZNcNA2k28/V3e8/ijAwi1EAaIcM + + 9PaQmZcFnEglKuE3D9m2j+poQ2+KABOtd/uJ5t8CgYEAkoW2ExKi8xOvgu1Qnku7 + + dUvXEGROhcPCHAuhvfmYhEY1fWEqxgNOjCd/oQF3Z5jHZItF26Tn23moTeL2+7lH + + r+Kv7W8BPd1yndfF6WHmUKyyF0WkJfDHjr9WGWkC0z0nswfWnnNGzImcCWo2xfyO + + VWHPJiwPkamFhZiWD2Rw8L0CgYAXPQgZ4+8ptnMIZP5zlmDK0kcrWgSQfQiGV2Op + + bd7aRqacX95P7AmUYnSKm9tVsfWyKLTKXHRcabBLLFeQlwcQZDu3cFwDkdJ453m7 + + 5R/pBJtMsl2wRdcG4FlCzy6k77twjI2FKoMKyKesGP3k79kcT6QXfJ8LbUt1ftpe + + wnsNWQKBgAyP1Ehw6s89JJMrVQ4AZwm0xsBBdLQGcFKxuSz1zfujc9GT56nY+sbr + + 2PbSIwCfNirtErxPvznuciUwoHJVLUiPryQDR9g4+fq7+Y3TkeJXhzJM2GMM+se/ + + JRDDbe/EfL8E498RrE0BvyzLnPsj1FWHSRUpqNFsbI6Qm+lcS//2 + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 4194304 + seed: 2NVWOXvtwcAFf0J/g0fDC+ypQbnCU7awZoCxcnhpkVM= + zfec: + required: 2 + segmentSize: 131072 + total: 3 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:MDMF:qv5ksusf4pk762lpi5rn5l5eqq:pbsbizy6dyjzjhjr3acia2hbfqlim7zp3o7ljxlzw4vh6x7smk3q + format: + kind: ssk + params: + format: mdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEogIBAAKCAQEA2iBFBQhED9S2ED4ZeUbf2wLzriDl4rFDVEecZdabAKqh58zq + + EPS6fqT0pUAH0zQV3RhUOEXLENcro6ilO6gpcDminsEsm7jgJNCvAfRF1JCTgMQI + + eLuDE7FvA+ZCdvHQRyok3RIn41qi35qVftx3TzQjsJX7qKDhnrfhuKmFIbMYeQhv + + 5PZ6AFWWvdwFQ/3fpqC2yRP/dS+7Bub9JbXm1t2qVaFEzFj7mwSTnhP8SMT0I7rE + + WqZqRfBx6DFzlBshzovCHT/QhEk7d/QyviH20cqPqeQjNAImsyfFKjnxmIBM5F7D + + eXsfQSbFlkGP7JAqfW990KoBh/9RxmBd/5vYEQIDAQABAoIBAAiyz73PIhO1ilsk + + dtSYyHWN7RTNGA3Nvt8eCfUftUe2BkXdrJnngIZrYpwybP11rseF4FnsIph11DYv + + FAPIhXqFud/12ScOnNWrAsejq6M57r/sUWArLiN7aG9x38WpiAJGgnjUcAXHiAY9 + + vmd0OEfOzvuMR6BmZgjz0UsRa483589VqSk4y1v7v2XjqJ6ubWPlXvvhU05dwMEh + + UmHHGi08/MLdsDKaKUMj8Z+sq0TLu+hDyew+fYEcFb3lNwwkNgPxiShJOp0LkhGs + + Lx6nt7f8FBTYU0x4vcrvMPNGz3cap9pJrmHuxhHrxSC56lQE7jYBiixgXBiFKflS + + 2nKagYcCgYEA5TzUGcMPQdNiCGOGdzSWsmrQNhPvA95MK7ChRI3aCzi8/f230oX4 + + uJVJ3o6CJ7rX00iplZbASWD2EmKF4Vr2ko34Z107Un+0HdiFrkYPcjl6iG+JL16T + + Lotx458Taa3mbHwGWtH82z7evWZtTG37RqXDQrIocRwJrkNPEjcpuS8CgYEA85da + + PqKeI3r13pykjZYI49XR4shpMFF7xFdLG3eJMKgNxLL2z4vvuicP5oP1QhG4wKqD + + Ie4gCxcSe+4wseBGMOiPVBtSITrICkDVVuM/r8W70ae6luVx4c38tll2fF3IKeC5 + + 533+roN/OUL5h4d4UG30/g3AvxupYggaxVjIsr8CgYANQ4fCNdccJ+70LU4Kd7CA + + gk2p011xC9u8a2vpW4vSOmY1DAkm1Tme9IRhrD07r0PtpbaqQR6/IC0cwzab43eA + + 41YMJQjZrSnu0Chr/QHHyiuc2VdGtmItv0PHt9yXsMg0Xri/aIcI6IpayyJn2bVA + + UTcLFOPiJ40n2B0rIKX5YQKBgGMFRrEph+FibapVwOqxb+G2HMD0uRXkOczBs41x + + 1ToLRrWMDpqmBwiEMomBYOS/sXvYlL/pPetkMKZiWDcmtUHSd9k31fYeIA1S96Z/ + + cHcyiTwb09TdZqLlCnLSAUFjGigz6z54UFx+pewQFsGKR1VirXHNA2psgzmPk9pf + + Ug6fAoGAfqx8mQe5FKrT9gvO1TXIwFZopjd4PGhCF/QZx/1YhVh7wkpocnXJQRYB + + LIwKrkybc6c4pVziJUKWop26kM3Hs4G8tZIWy4D2GQqpzTvrabvU8SKOKcoUssof + + pMF5GV25DLc99P/xQG66qK2+5kwqU8oeaf8mElO3Um7YAPLpzYM= + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 4194304 + seed: 2NVWOXvtwcAFf0J/g0fDC+ypQbnCU7awZoCxcnhpkVM= + zfec: + required: 2 + segmentSize: 131072 + total: 3 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== expected: URI:CHK:ej56vl65kawat62wsxuvbcd2wy:tv65cgghaddq2p7dyvimmff24l34wsx6mhzd3dyprfvrdo52qtma:2:3:8388607 format: @@ -6462,62 +7212,62 @@ vector: segmentSize: 131072 total: 3 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:SSK:v3zxebrrlgskiey4ueqrqz5hku:nxccancsocy5iugopecpdzgqntekq3mvvo46s7nm7msm32fcqsga + expected: URI:SSK:6h7uw7l7w7kyvwnfbvdez5iqii:doonvzgovuzwr5nzwltigr4s5qnsmopyeh6a47dgi63nn3rdzhaq format: kind: ssk params: format: sdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEpAIBAAKCAQEAvWKXMowTtKLHOE9bzZz45Km1vFUmP7E8y8rrImnQRIrhxz+v + MIIEpAIBAAKCAQEAsugmXmLZaZ+2aa+sz48xKVllHO5++98qvP9YdGMUa6SdPETB - eFGcEIPQMyplPeLtfz9Q6DUh+XD+t2kgXe87ME06/a3ZcVL+omlZxZnilHkobQYQ + lzIIrtvff8Krl8KjaeRy0TW7MpSw/QO/BK8aDWPdAmLhSn6gayt4Zy/r3pCgM2Ed - l/lrLl1vtKK8721KE9RY+r6lVp+DKyQwtsuzsGYB2p/ZUzkozqveQ8yT2JOjU42R + a0lkTL+d/+J/tjgGf4u3h+SPktv4plhqfU8ZTHi9PMFzkIdz63thUC+RSSrVEP1b - vLxDTd4gkW5KulWjnHp9sS1Ic+v7FfU/M5mNpbyg+b0Fo7G4rghfeV5zg7UMoxDc + KHzIXU4Q2nKVG3TCTQp/yW6ytv5OBqCu6Mci9N9UF6fNecoSKpLCYl2iNYLHp5Z8 - 2T8wXkopUBAW6EJj45SuGcOvACEt7uLSqEutpHNF6kWF9nG+XRlHg2xpgn7SyyhZ + 1WPwLLdKoCWwDGT1ufkyLb6nroYB6V38IHgb7FKFGXOzcHBPd3STCY/oFiaTvmgn - O9sfi6pb+Iu9J+vH/XyWmVgHFzp3CZJHMFxZCQIDAQABAoIBAAP0kO1WlRvG8Yu4 + oknsBEnunVa2kmbyOfy26NOZ+qdqmpPGVuoxMwIDAQABAoIBAFAtZwiUvz+tUmgh - xpVRA7a836WPDrUyVa947bfCh33C+8uuRhMoey6yHhFPf51PBcBMWXt8DplX1Y4N + 0U4Bq7QOuphRL/p75KDnxIIAZ0Xoc4jvfVzfkPGgWxTcLt9n3KlXtrcYn+jGp1z0 - lUY49p6/4i1Fqf6uqdBJDH2uxNduf1xljceqxyUJAQoAAxuqB+vJmdEk1a2tN69Z + oVYdjQzkLMdlffbPMeBljmOcH9ZSNWFhS/hpXzhgBZSIMtj8WbkuadVOcqOLzm7q - OmY682YJ/1xqTb7p+PL2DnaSiXzysTI31WbycKFlIrpQPXDSkJ+/oOonxWlXeISs + H+tBmCJj19cTEVH7ylEFrbJsZu2FMipm+Eirn7bALbUkPlNlzmcjUT4Je6/OOny3 - 0YIw66xFDgF3MNGfYaeqmNFnCt54nfTLU2+U4koRH5FNRsjkeOXnhTmO4dt4x3hu + rjMEOhQwnfhcxTL6IRDeu7lJq4l79+1+7HlRGazt4QrH9fjAWNJaWZyDJEZ9wmge - FpBX6E65q5Cumw57MSflw9JvzRmR7E9+KTnyMHF1PIcPIEcBCZ45ybxZcr3Y9nqF + TQKueupXiqWC8w+IL/3w+8arPYG05iSxCtdb7ehzF7FVAKQ9sw80YahsbClQHNTH - vrRvvaECgYEAyrgeuqOeJQZMUZUzxikadlTFRVs5sK4LihXk/Q6LuswX4QDzFmLL + XAwZl/0CgYEAwHwVf8AXyOIDgGAozn8IcmICdlXBf3Jr84LcDlAgtNtbx85fZGT6 - 9KvAYl0E4ILSptZ8rW9gCP3oWDNWylJUn5RNEX111JsGTLHA5QMrPzp8Ear9wCd5 + r5aMwpEByg2RyhDCzx4S2A2+FGhRbvjUFYV5/Nak/5hN1leTVAItnzWGTSkcJuVp - 8lX2O/2nSM6oBgryAAWskuQHcfmFTf8tub3m40j7NugvRSRjQC/TVBECgYEA7ylK + 0VC295BhRKBjdmZVBc3zUIwGKanD03NcKJ99nxXGm7bmmJRrMO9bxLcCgYEA7fEX - iJXxuv8Xd3a9lGgVRCIFPT0uiszFrN61d35r+RAG+9F5sQG8nQzxEYtJNHTMzx1e + ef+UDJeHRxI4oH5lAEGwydwJ0kIRB0DfuPKdhXFEY4ihW+rppY3A/NvEUZAil08T - 4oOTViTIMRLXHVQXm49Ogf2WY/v3ro+OUIEU5o6gyaqmyXxwu/kf0wxFDP6lxeSk + f8MUJEaIF8MeaOCSvY1M+z0ynykEAkFxkSfTnCsJqMh0p/2aNfrWVDm/60lbsGzd - YDSPo8XC/iPvacH0yVMZtJjxfJJpua7nEjCozXkCgYEAq4PWdAElN5w5jDkZogp6 + nT0xMwP7RDn7VYCQpzFSlbgOTwJ/LEFiNmTEE2UCgYAo7LjtdnwYG+W+r7M9ZEj5 - 2i1k7wZ9LBBFsSJPKRBahsRRW8zq30Dd4XhDgLXE/5OQWRpWSINYFKOHJsDhKLM5 + eNkpK8Z+QGevWI1NBcBOc60p6Djj8YxTNOEspQQKX6Q1oCarPqunABT/5cYaoBEH - 5/6YqjilLimvzcoDM4BX4dpAyM4Mfbyov7GdcSpuk/pNTTeLgxtJ5MpLxlHgSJqj + ml97YG+oYEt8XRZX8Dae+RRa53iy1GgRNuYP8MSdgLRlAhDlsQoggAT3ar7WAFsB - fGjA5gKEkfMms3BTDSapvZECgYB8x21Uv+7EIq2KrdARqxBVYO6c2dv7nQURwYyq + 0Bc3cbvOc67HlhbMSrfqNQKBgQClIQ4z4oUyf+6oCiM2bsFVfkFctdIzExqSOBmL - ULJi2wLZxZwZRw+yXPs1rRc/oCTvdqJ3yjBIBJ7SQ8MqUSKUDfvnBHi/p8m9MLcO + VwSu2T6m/OlOyya/eDMYyMPj/u2iqIRVxGK0EibcptLx4fi0h92G9p+tCV/42MYi - t5pBBG9NaJTmkN98o2kAQumP8xhonHdKnoHG77phwDv8UK63j3zc5eMwnG8+6ssy + AMvAs7WOZx9efoeJMr2P0kw4075IICVkvFTqnLbCUKL2YbUB8x7nPMbhWlA7vFyW - iWK4+QKBgQChb4a2GAVwcn3yTMx0lfTftXuwlaDwk7eva1Jou5lvR9tByAusppdR + dzQEaQKBgQCrN7DDrxZapHLMnyL7Pjgm7st0CZxMxbcY/dkmw/7yo4X4tUiTyYoU - HZiCooBEEGDYtmM+Q0huLJ9bl3UK4tTFO3DyvDSkubYvBMhw5Or0iSmtqkxu7Qto + oQqtcEzYKFrgQ6OEtq8i/9yVmtgE8k70XVJqpUGM9VR97MRcdzEKypGBnUKOamcs - 9T1WIZPPFJ2PoIlbVZOGPUqUw73Ar6VK5eAFI3QVNN1AoOLUKzFJvA== + Z28avBZ3ygIN5L48UK+vLMayI4m4RmUtMtU0jFYkhIEcuhTWEyGQcA== -----END RSA PRIVATE KEY----- @@ -6531,62 +7281,62 @@ vector: segmentSize: 131072 total: 3 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:MDMF:nwr3zieggdkguobo6suezt3b7q:sbswvh7qr7y4vgl5efbgn5uevhnimhquaajhutbawnw7gadufclq + expected: URI:MDMF:e6pfkqn4wpggq2lnvc2josmkpm:hcfjov4qzb7fkiucisoes4edaymbp7qzlmrn6rqqbt6tsov6ojaa format: kind: ssk params: format: mdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEowIBAAKCAQEAsiFWSx7bRsQ/ImDm1eE7vRa/IiUg0mgnI5Nvn12+/sg3S8eS + MIIEoAIBAAKCAQEA4AffDXMatwajd6NXgtaCi9gCsfH/r4+0zwHKTdwl120zd/yS - tHnURTFwb5ALFwE08/jOYmiH/ZZqD/xWDRCwZswe39MgH8p3jV9ZKFyBDm420hg/ + 5HZYFhO6TbCAuipADxUbKgaRTf7NAo9NY+T2Wke8UFfds+tNfKKlg8uGIKC4IsHS - IgRZ313kC0V0PPwv7n6bvJpGLke8820VHlNCzTdrbmSaTZj3AuOCNz/+h/fBZgtD + E0ay3GVl4nCr7y/Y5Rg9ECmIE0ROM8b+iDKVfgD4ztigPQPv8r3l/XAzU1yx6bVf - XY8uI/0hynSNoaXC37BiTeem50Ml0Ehjui03891EZysLx48WJYTp1tK7X7of7luB + n2Sz3QzCHlkjbmS7UpcEWifUG+8sgO8Ee0zBsOjdFyaOZMiaTUPlNvosXvKQUGoh - hb3JV+C7asEKqxq59DZMnIDo+vIWr6NPeE6pqxMe0bfGj/b21foHhYRUuIg/Hd14 + +aBrEung6ZW9ahWKeBaqpGh5D+T/pqWIzXBX//W4gMS0v7+GAl8h/JyAz4yW4zac - 9RNSlzyEPqdf14GbwcGIEW5TH3FzNAtLHWImVQIDAQABAoIBAAwGF4zVDaCafRd9 + ox3br5okjgEffJbClMI8mU3ltxDTWbKNeps4tQIDAQABAoH/cKOEgofKwjtLGdwG - On+r5ywllavAnVVOjfvHBzUW7x5EFgVxuHuhsJwmEOya6MC6BmDEhevbGfjaVxjy + kZ2SBZlM5is0PqNLjoAOyakvprMcdsnQNemTzhgdJhcqod0uoUIf+JOd6OJYMV8A - o71Yh8u8kgXyOpwivsymZ76DdfurIVyvoc1SRV3AOPUw0D6QmEytM6Z4tG2Jzp2Q + QRUCNc/bla5OygLN8txmW7j4tyXOk6so9Igpru+Vk42lLea8Jry/9vKhnMUo1bsT - 2qjUHnF7QO9v74F25o+Fm2PO2EfFqus/3X6k+3vFvpvgs/XEvJFJRiV5luQ7+r6X + GkLm5uFNE0VFS7Fl810+IS7/esTmSkv+HNNxFUHjBDEx6ntLH4JYvZ3Z6Brfum/N - PyNHjpJvW8H1KPFK6z0qElAodu0RB/FlGXRjUfuyCwA7/va0YxDkiSPTtx05XGPN + Mq7KBj26v+ShSrTn7uC1cQlxKTyRdudkpRX8qiNqPrL98b/KrEGaCBsyODx0JgA4 - LMeZYDe3iBd0SMlw51uqMpx5rYWRRYje4vUAhkTFHP7CH0kWHn7X8eDIPZppwxix + Q6mCt6zdniHrRAXhamFdO3f8MEx3VVYKVy0YyFsWjo4DNhWVKYQHil143CoEwXku - c7k2dSECgYEAz3Orb7naad8pzvTS1lokmpRjTwXK2DM+Z+Fi4mHbjxO3XZLyrcfF + JVlBAoGBAOdJUfNvYyawktf8LmunH9WAjpqDtCm8/XJomwAD1ozf3P522+yg/Dxv - ONEXzDIQMyTzkSZGJK01/pZ/GhPoWZ6F3ebWEyOqFpjcmEBQIc52YFjn8CChklW8 + 2bwW4b+9iI/hXWuXJ3RoAVRfSy7TVg7xN6WmnZoNzv4tRGCGAOqg9+VYy57IC8Ya - 4KuzOO5NHWvjGAniZbAfDLE+yz3DsWi422DBf6mAjTS/S18E/NmvrnUCgYEA29EG + 7HIgcBqbcVvj1TMIZitRDl2t1sSbzdunFJlbHKfq86n26D7RRERxAoGBAPf4E99M - azLy5YdvOxJvOEHsV1ODY8OBaCAVLZaWlkcbJAlmBjvazE0Uygo77ZWNl1if0EeU + NiY3TaaI3UxyY7xFf2GrVlwMENllj8RpbbVByQVJPxR4Hl7dj4aJcd9PFnAfaDNy - 9YVIpkDaD0ZwzfF+JQYfMRp2YsGoJJj8t3ppbwDKMJZ5Vbw768z84JP8uYGFVT/o + SIqgbjcbibsNvpY/daFB/vtXUS+XKU3TLWOGHZ+25LKVOZFJhZfpSqcZHjCl/Tlg - gld86oUcKT+rwzIgx9iiW7mPrxt5I5/tNazDXGECgYAbVia1Jkx9vwaHWwOdc2t3 + m9Ou6Bd1MOTcnUWfECIi+8zVpTd3sJQti0qFAoGAQHrpdQPF0cCCf+KXkn26W0yG - Yy5i72R3cOk8Txr0seh4xiRXlFGeTMDUZ9k28zHnS2s9KUn880Y2Mia1jQIFLTp+ + 9T7omIZO5nmRVPS8+PNkajD66UKMb9EDE/QRJeKSUwKSh+9RGZvxWvNiQ4C5ylqn - rzhudTiomaQX/AGTMt2ufIizv9kKq3mkMXwAeIZ45gqa1FKdC8RLq9+WcKEk86PI + l/AWmh9laOl32a0iTkdoNTGHOxIsbiONbdfrSQ+zD9o50wtxaHwllCpl6NRDFQzE - ZMuawv9JnDXI/NBvcVARMQKBgQDB2Ikyi9GL/G1oyI7wK6KCOBGMLuK1smU6uKu/ + qmiDWbEgE295miG/dZECgYASV9egfRLEYPLtjtJQBWY7VyjFINeSl5HngwvPi70B - hqE2nFsucCY5OFh2+6NxlwswRmVYxWdlRM6WXmZuRg5AbxBxEf77zHxOBr2C2K80 + 24vzSCfSa9BTVDB501EJI+CVCr26kImtN5DvoqndnHasxqT8+NTT4vGug5AaobSJ - Fm1YCHhFdM03gDHPdgwi+B5McR3l2d/u4bw1DIGTFqUgE9q4oiA7h15ga3fepLAJ + 2DH4zp68Vy2bAcVQJ4HOOp1xG9ZPmEXustGYaqLjSy6XJ90ZqVzXGjbOk5wMWhIj - P3tgwQKBgFdt9NBBXmDeny2eDxXBz1OY112usnR2XV+nTBwfu+XqnnurY3QYRrrS + wQKBgHgsuqxEqTLQIqn0I4E/VAKdBuYaGngSUB0frVKn1N9O3HtJhG50L5yhWUUf - /XeOEP2jlyMpy4c5uBJ+JhlIP0098quZ2wfC5TqsMINWBoqWHX1JaP7cStnYrp6Q + UvHCgaP06ft5YLVUbTkelhgCbG+VzsFAjRFhesiHp+q65gCZeQJN1Q3JReeFjaw1 - ebmiqnqsF8HVebpYPv2wplJgtA1IT+EKNkbHqsanQkODJqKdtuda + ECQqk2PgtN/3t0VVrmuyLBcw9RETnN7f2mHayo0uU68AU1pk -----END RSA PRIVATE KEY----- @@ -6612,62 +7362,62 @@ vector: segmentSize: 131072 total: 3 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:SSK:wtihnlmjxihghrwapnvtsjgkjq:gmz36hpd5vi34dubmh3mk2jahhtitk7zbbusnlknklah7t3toxjq + expected: URI:SSK:zily66v5xodir4wz5conizmonq:6zksu33nods7hita33ju4wbvlnqyxp6qkl6mmtqyl7bh6gk46isa format: kind: ssk params: format: sdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEogIBAAKCAQEA60U7XHhqto/vkbXa2WGs7b2ZtVgmubgoW35dfluhIYB5HbH0 + MIIEpAIBAAKCAQEAttP9lzqc+4EZYh3s6IcBlfOpaL7svgQRLuXBYEwd0GNz4FfN - Cf/IAgsj9qMkxuaNrol3Fs0O9vW+zQ4WgbpNV710buUwg/OZssv7qPtQLjJGRdVw + v6L68CgaD7pI6vgUZBOnYm7c3z8zBdWibpH3V0qBufWnfFdtYU+20ed3wWWDoV8L - zB8Pw4633SGo60J9wO5C7vl14zfT5AZV/BLXTLUbOOA8SScIgcMiSCzJ7YLQCNjq + z60tj4GNEHxwggLIclX3IalHySFyOTY4K6lmYptxtFBksULuYqFIZyX724O17Hf/ - djzP+9uErB6eT0yhKYkueqJOccV+CxP4DQbguruMpOVgfZSqMmvThdO6kLR4Zv6T + H9XGx+D6ffpUyR2D+hZpXGy5ly06pN4XJsdNkgmv4YfboUN2Tlm6Ldg8b6ni9P+d - I/EJzUjpdBrywAug4xhc2S3RlywiyD1A0DcnHRWHO1kcB7UplEDNA4M1nzei3nps + BhjdyBqg6E7RFKG7Z1ROT+GzeN0Q61PNKPJ8/5fCYPoZEeHWWfZFNO/c/fAo7nsn - ABDEvrkuP67+vsbXZZL15IoMlJoStkh3c2gQFQIDAQABAoIBAADNhxPj0ufM0dyB + 9Ei/HxUju+PZmEU0jcsHXIvL9LQY/2b59S1dnwIDAQABAoIBAEQ90HP4Lsw5nc3f - oVxlGix8twbNwTRlR1LHAUEmh++2QClQx7QNCCh0L+scOO5Ygd1D2Z+qVcrboVf4 + uaP5cIAWGO++BAPQ5NEKdSmKf75ewMvGOkgDf4LQlRm1wK3jt0i7hUjadJrnrhXJ - 6UE7CNd92O9ZD/2ZrnKMbaZXIa9u/diO659CN455Zrk4k1gSLWpR975YTZUK9D94 + bf2zgg0VBGLy7Hce8vbVmDm1GiAX0hATuAbmbxEXnB3BNQVyIHt81ue7lc3fLBFq - 6b4fYyMEZPTJaQYij8OhRnpQLH54EUrDwUfFQGAArRR5oCMHh4TOFz/My1J+HifR + yYCSlGLN/pz9PPhlMTGjXbESnnWKi4NVtX/CZUUEymgfiynfdXCf7s6wlYUp0WiN - d1ue27YWUDeDWd1U4vh2qIy+V5tQFnisgMw2QQjFTxkre5nqn/vikmmvf0vxlccy + LIHBRNwW4FMCvp9Du4W4BvJufWp9v4yCoIpL2SyGbYE/P1dMg+8ZLVHdisSN4+N6 - eHn2wLJmBbzK8WaO0gaoLXRDBNiZBfY17YAxpUMOujwg08VbrNY4gNpLgOjmsvFu + V5+xVjcx/5IX+D9FR5wXR3LXZx/eMa+dsEvAPmQAJTYUAVKzMNOXoixCSsmyV9qj - Tr8qeTUCgYEA/mWKRmRzkQ8y/XCvnoF1jCNeztkNBQi3ST/7CF+f20FSbvtlErz8 + zRB7MFECgYEA8fAAPiq1W2miLyTR/bKMHLIbdrEfMpGTrSVPTw7PdYkiDbNzpTJH - Lv62DdyvPnDMvu28nk+MiQfel4maDG1nRMBnjeXuFbetpeJZc2tAh06zVPCQPACr + uUwxwhHFerrlPAnxsGbbVZG7iT408a9gBqL3fOCP2095f3wg3Hxu+iWtEXLuaGJw - RI1Wujd/+QI49PfmBKe2nV84GwxxEC5aCvviawjbIIigTLy/IU2LZAcCgYEA7MDV + hTOKgN9mw9LtHh9WT9JBeQgHU3V199TYNdd7a435jxf/+77eUN+pYZ0CgYEAwXRy - EX8Fv4bEgGt7TXF0vkkM9h+iGtYTjdUZmsdkNvzW40CXlusYzJ/INZwgAkQPn7F3 + +Q5+02YChU42JAuCnOITnCLifK9U38A/YaaBtMYTpVd32GSefCv1emA1KvnTjddA - 8y/dAnW3woJ+vCcozQOa/i7sAhxMZNV84OgFTGQE6IKw74aUYCdT/bCAhrmIaOfC + m1br82oCz1US5YsZsfdmdnB7mA0AGiM4GTHfUBw+N2gol24lCY2+wsi07U4z+3K+ - YnGkino17EMyU1wskF3mKuOEbO8qQgC2gkt5/AMCgYASbDQJSPj9hkZBCEoPhnyG + HmGcAP8FLXR7/slX/FwYlcJXz1qxYIh5rWnnhWsCgYEAj18PdcevY3WM4+0o9/O3 - u4EAJcPFm436ZgG9537iF+bqVpZJNxpkJNn2QwcF1JFfOkQwir44pjM+ch6Py9Rw + 7kVp2wOJnlkAr4m9nvcC3/8dDAt9C7dpI5jQn9YSNfHNaK/n5wZ9Eg9jmCgiDdtE - rCZTplUJiZWvr6aeryOrKM3f1tP7JGlCu6GONrqzw69wPguQRrz4xI6BlvMRIuou + x4oJqZoWBfvp3y969c5Toa90CTQXrgov7e+mM0qwRnmXhNNDPdg2bnfgh4fDGdOr - ZXNOIQQNZReGtxx4Qu9XPQKBgCxiIg97npo/K4tfmufzww0BKNrjJ0Kcq2HFd11a + MPT6MbmX20F4tAHfEwQIB00CgYEAiAedLNnvfjC1xwzG7zOUxUIHLfwtrCURlkA1 - z+C3GZnUvBZg0G9b7O6P7DhAhiVL4c7HREl3xBFE4XloZe+5I09PgJMMtw2YMCcB + kTGm9PlvKQ1HPUcLVh8G/uUVncGL66oXSOOnCENb9HRK1FOqXsSrLM9NaQ6DKt3m - mCyv+3OTPJRKyHoWJVrDwfR/x6DTAc/uugfzzTQTjNWvy/Lsh3+201aQp31kINLg + /XhfIZKqgQVhvZF6w6wDHi5JYrBhxwbY/r3+F4k7F8pXwkHL96y+sNe2LR0Fqu5s - T2f9AoGAXkW10qtVSGKG21BovxbMbVrl9nmP/jYWDV/hY2JVwAXho5htZQucbva3 + OO9GGD8CgYBKoK1W1QcPbScWEehb0BFhDfew1dARf0M9BQDo59n1IP/fJLAj+irm - A/25Q7WpaZECWeJLlafL6gQRnZBKWaMC2oIm+4+w6I6JsSRi+XLNL8NdPx5oSUGQ + dT0Qq38vskUzH2nlohVbOiLRnKIqx0AJKYCpfJNe3jYkzTDenQ2hONwhK+0I4OWx - k230y22YV4bWpNVg49VB8xgjsMhoKhvaP4lR1pn0Dke+06y1U8o= + MHe2ZVX046Ou4SvrFWoDJ1xKCeF7sjw22SEaF41OWyzyarhsK5pDVA== -----END RSA PRIVATE KEY----- @@ -6681,62 +7431,62 @@ vector: segmentSize: 131072 total: 3 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:MDMF:hsemzrh76z2ha6lrv2glaydane:nbbwvf2ivfojz5sb4cnlkx7z66bp2ou3teknd7uobuvbi4dcvlna + expected: URI:MDMF:nk5g5nxt743nwaoqk46dl2iy5e:jpwpdslpyhgafsbny34auvuyh65h7hvz3fioq64efhijdz6aij5q format: kind: ssk params: format: mdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEpAIBAAKCAQEAx70raDxNkfMamQm7E9lJRh9UagQv80WtuuKhvq2sidJ1glsO + MIIEpQIBAAKCAQEA6bFJtYwt7TfdVJpjgXe/PTdvk7MVxPiS8qFSmtFILf3Fvof1 - 3IdCAe9dmzZR609K5OLJnHvR2EGoXu4VNLtSXTFlSDgO+tyRupAuFY8Dp4P/EQfR + hmCdLayS4QcL5EulLTtotvmFPb4rG/EHEwY4VWEwulw0FXcoi7gPhTYwbyf/MPBU - nPqJOqIGid8xxjhMAvIKK7GW0Cd3iVXfJo4remn7zxBWHexpFy6N6zE+cARV4EeD + IBI5crgXrkVRBY2dqVKlJnds6EjZ33jgs3sVx8q3eKUmLencGKhalUWGiGbEtB9z - NkSv/bmJ55ZAFZyViCBuVxEhRJYKqmPeAmCSgilc7fX8LO0E2aZ4baRIrXZxvwvV + YgpZetlWdXObPyU+KSKB6kv3RhHsBTS42WxQho85wjMiT58y/duC5IvDcI+ibD2A - kW2CoDVl95eGw6vR43pF+DODH8HiOuP+uq5OJRap5KUgZiTtQv6cXLqoW54KPUQW + Iknql5DnN7TdanjpufGYh3x0nUhOUlK9whs+U0ip4P+YooWcB+ySlgsHy2lSOYvO - 5Et5lPZTBbBWRsBoQJw0Sq2RR90slptjqthj3wIDAQABAoIBAEL46HGSarYJy/zN + fgWFWt1Wm94fFyUoRwA5ZUSU22y8lkVpVHVj6wIDAQABAoIBAAgd8fsjU6kMqrG4 - ePdeT4XeImlLzyIkVmzH6dDsHeK2eSVEz/ZcueK5NmtBKvWaCDQ34L8B+2omFcUC + lxeKdZMGbOgKwrPPv9LmSuc5Dl0X68QSwYgZrPvxeuc5W8QMjeqEnhN/lo2EIHKH - 0oR0XNkXo2y0Mz2lMI3cIz+iTOjhxugYdY3LqbDJvCSFfISIwt/n9UYSTU2tNhUM + rFY7KL5rx3SBpVwlgmav+Eyz4CTMP9nkTxe6FP1owDHREzUH1opou3ca6NEnTqIA - AH9Gg0iP+dlDkoSFDPWza+2U/OkEyqZmyd3buZtoCVIkYqe7DbdjRRBP7r9FuxhW + zH/kw5Hn9vgddwuwjBuIespP2udmkWo5UXxD+3irKP7lG46OBYF/QHQLrzklXeR8 - jQR5J0CbK84A9IRvCjf+uMKCnzn7zZV2bGzn75YX92+ul7pc4gMLAL/amwJRI9SL + fWyUgTxKD43D1RtyyYRhV/7hNTrEgNgmzEMmFVTpqfBMxOu3dmBuC3XfORm87rPu - 3EGixH8RhVsOD2itM/X8gCKQeFI2J5ll8pQxRi95AkZ4z1FAakcUWFxq1FJ7gLk6 + lxh1glQ6YlES5zYyl/tSNa1CjCP54k7z6Z7mIvXKb6LCrZYw3gjCayqbec1n/OMq - t+3Wy4UCgYEAy5tqqU1SvtDJUVWu3vtpeHUZFlSJ5wVHCVSfUWkqNaZ44xKvSMQD + trB+xwECgYEA+Eyq+Frp9KwiGwmAZwIEhRicm9e92fO/v3vQfR4Mz4u3rFSzTzQM - UnQcWsk+KRP13O8wVtUhBNxCq9zwW2xTvRXBFwomx1bxt309syxBakSiAVym91Wd + P/LXRx4YqDs04WJHOy0+oO1tiDoaCQbNQLqKy2xZBukGiUX+K7koMLieC48eeZEF - gDrQf4xAGRKqtvB5TJsvTIyO3BL7stCpGq1LRZ6e7s8beeWYYRu6PFUCgYEA+yLw + TFUhbuTDMQWjIMlFGmWDG/L3yU3HGPazUkJSetf3+Q40XkVhPUvX+GsCgYEA8PCm - Yomx2E0x9t/PVhqACE2lKMZH7uBH6Xgvg586Ps3nEvr48YuxYS9czPhQdS8jfgQQ + jwSP8p+PkikseJ2q0aKjS1RUZDqZqF+WeepQpGQSnMeTOEqh34fm6epggBbVHk7w - 8ioWuK2vRKAIMMDJJQnfGCrvAq+5MpE+cFo9T4AKi+CjO85zHN30pHD9SMEqbnSH + 7+efTuAx+GRtcgF8anGT34WGb27TsfuAzV8jsqb+mnttjcG2f2RGGP5KpJfjGeo8 - gDVRkxI2CYVmVh36+d2KteS2zbNi+fNznaK102MCgYEAwA5jBz4bzkhtjd4v7L/k + EU9hZsCfLZQBd519BPTrrnePNvlw8Ok1iQ2pIoECgYEA2KBC9YST4uQeqUoD3Vq0 - Vi7GskyeJB/TSRbcjVOQ8DiOkUsfspjKtW03DeAEVYUxhuzMgSvbUJVgAnOO+f3t + SM5tK8Xwm+t92fiSr+X8tUInT9Fh0vMM0On0CdbnGjb1bsGIdceGgW5DhntyZXeq - 409w6wW1XJVDvpxRpgAZ2F7THkvCZ04IGlvgLmAiWkREafndwYgkjqWLYEY7zAmN + sRNOriVsEoxRKIiJNOpIdyFKubj2lIcCgVMwZQhuhyFs7djLUjlIRqUWq2kRD+WE - ac+LUCl2q7cKqOoM2ZTpEF0CgYBB6fSv2DYOcIxpoGp5zfDGvSJZJlmg78rQE1Rd + E3tLbGNps79BzxFmwcyestsCgYEAmb1U6klE+NHrsJ3pLIWeq+mVPMnwl4v05EUq - NoCCFWbNy4NlWmXO/TBdN9teNmYZYBXWiYd3J1b2Kw6bRS5GA2ZDoJkk2lxAUnDR + JVzoXB0m6zdFr1Of+pwjMftF3DW1g4Nnpg0r0A6qlA6w72AXXWxfqO7wm0Yiep06 - 6k1nPVMHTYlqXBBIhlT8iA9idhid7wXVd6kWcdQvAY1PkwTZafVLMmFsceXLdsNk + 0ND2XFbGexhrDVsf8iWvvN72DhSE6tJVxc3bHs+mQlUAoqyxS2pkwIy1q6R69p44 - n10bwwKBgQCPgfYyoEada42vJjjgm8uHpnovj552f0cZS4ROg8mqKaBoheHKf/Pw + dN1soQECgYEAj7BSQZHlzSfYK6SgZTiJ2qDYxsIkcAjCicijT+5h7FJd2U8pBTaV - 46Yc81TGJTsEczPy2MFwtTuKOooMaC0jjLKsgaM/tyf+zmJBzut5Cfi5BXteyH00 + 1/A6UbqwmoFlCXHuGUIOpI69V9nL4BYrzBL53RMkpx2RSvSwUeHW89wsgZh+EB+v - eRCU+N1i36oKLxlPisAbyOwZffqYfFomVQv/x5vSfx8njkXgA+Ic0w== + sgq76XdP55PL+2oJNJEsB4vdxQbp40kNOfmo/bEIF/vurNYQklKb9FI= -----END RSA PRIVATE KEY----- @@ -6762,62 +7512,62 @@ vector: segmentSize: 131072 total: 3 - convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:SSK:gndzkdmlzvbzhjwkupooylrs7a:32kgw3vndmzeufhfruyu62dsun5ghndbnsurgdmzty2x7m4h3ida + expected: URI:SSK:2tvuk7uigxcngmcxx3nt53a2my:ssk6vy6plxp6lp7odlqhybbjxs3tsxe6j2ehogpf5b3ml7wgkevq format: kind: ssk params: format: sdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEowIBAAKCAQEA3RN6UtTgAInFX601J7cDvb4/trLFwkAX39f2+4lUMS1ZnXwS + MIIEpAIBAAKCAQEAwHgjo5YUy7LGiFX6GhFgnv3iB+yzyB6jjIrCjJe39OTI3v77 - vdvoeR3Fj2L8pNufV3R/qki7ZM3Rf510YVpFJvXt9XzGNYLluthu/r56aNz0P+Ga + DWqgLyKSUApAaSuZ2bMqIVcEvTIrBuWczfEhPm45ZqpvR57UqNnAe2z2CDIlTNS2 - 3MQ9dOq5QGKTxtF/ixSzrZCxqkwCiZXaoLfSb69S9PLDeGBsLDIusqmarRzpOPJ7 + Mg7UWNaQigrvrM9wOnNoANXb0Gsm6n1KkY3JTszsAH/P8vmRXvm+rZsjeLwAqkO7 - uUTDwyX8c0SVhaDrcoCacfFMo9wY7mrSJuhImuJLDsru5XoDcQo+DKMBQUyWvQ4F + R2eUYme0hZUldOHHZSlmz9FAge6ajvw/lIIVg8gbk56EHBlpJq4BTK48REm/hzVu - HliF292ZQmHaq2OcRkgMwWH3ZhWHar79XjehYLvGOBQEEoXtgm41cYQI1wKvgRvr + xQV0JGeak3zl7GWALoJXtkS81NctORsg47EiTkTxGEgXS3PXqDImEKoLrX9Sfl/f - x9lCw7zBrmJVa3HrxbdtXDW+egVvH4RyIRI4/QIDAQABAoIBADGnrlHsfmOgjjRv + kjvpxOsnTwqBTu2fBhUJREoXMbJbioPFwwPIpwIDAQABAoIBACFiVVAxHo9MhZot - MwE4mh6EHMtsW/7FZpdgapkUv1RMW1SECbGbMxwBE96g3R4qNh/uir40l+KMWAHR + S5HM9NTvFY8pU+/AvL6KbP9k65ALRPpFAPfNSFaUqQtAE/cKDIgRxxt8VAKbGpJ6 - 29IB9IZLtqbs35glTnQpKMUPA2+KMVIn2iC78xHPpsxPV+HQLFWQ0MqrNTyK1gcR + Lk4cZpdFGCjCJEYoexuElZnzBuPaCtU+ShH5t4RnRy/igLsZSg6haOdIMPYAOAJR - IYn3v8xWFMvvuvfOsH08yEBY1+UJrl4IgKgukzSV/FD0arZcnM1t26jkbxM9MbdL + VCdWEBZefgsCIGg1OK1gJV5IfAkbSXbgUKQrt9FNu5Nbt5rRuuLg2CbB77UAXsiA - rfP4LKGbL4PmgQsgTAgn8dx7uutAgh4AJnk6hZVN9UE7k5F00vvXSO1F5i9/xWBD + tCynDL4kgx4sgDmU496tU1Wt60flwb2Ka5SwnQwkVDM5LLaubMyKsL9clFow1Oyf - 6eo+zlAOuWKGmOpGp0G5u49RSblr7guXcWnwPdQmMfYwe42id+Ew8bcKnnMg+10U + TwSsERNPxrYzkWUFhYxQmNUZggzdixiHdKsqYCnUzPJ8s5DV2iKSqQPVW5DR7FFN - /tTlcAECgYEA+PfNetLAxhT4Dq7cM0sn0C6Z+/REuf7YmASQfZ29irkYo79zb64h + MC8ZM4kCgYEA+hOBpsfBTt48yBCPAUkeiLwcF0oFrQ2Q4wRZ4ldtaGlD5J16Jk/4 - f0eGEeukeJUCtfUJ8BqY3848DU2P/0ITb/dLHfoL803MH24uj1YHk3totsN/NHuq + HIx9BPvfsm2Q7EKZC8CfiMEtBVhjZZmJTWbZz5dH4SNZwkaxL/JiaJ3acWylZmj/ - WZreeISQ6K9Zw1sBCmCWmybPfO+y8mSCaF+q50GZRXy/V7qbKq69EZUCgYEA41H/ + 3BUj50Ysh50OCfgosXLUE3vALQQRGfQrYxV8SRo1UegeK7tX2pBZzO8CgYEAxQdM - vUVqeJ3hZC+GguAwbOY4RQ1BTcfcdfu087UGjq+Qly/LKt/y05kHnZUSKWLoQRXj + GHlKaUz1OgPgDIj3goXhm/c0qB5Nh5k9LSNsz9kfds1vEZNPBEES7pQAdKa4EVaw - udw34MTxgI/L7htBEdy49EO9MM0Nfs2noG3hKFmwZE2gBUdIWeFsKvWhcBuYp0mg + 7C+k4YpcWDAz3Nzt43bJu0CXi6wmAKEh9ocwk4f5Cms1s5uG8XpNcf9wcH806xu3 - f253Y6QXdTmiRzvW7i/psNE2z9iNoXH7tyKs/8kCgYBVGk+Qxm3Cx/QrALagifYo + NxJgaMDjB7slDAW/DFVhPGum6E1z3fFYlpI+L8kCgYEAhzGgd++p299dcLMy/Hjx - AWX9a/f6JBThkd3aMotR2geEIbNR35HvsgEwKv5jgXwVupcVDeJnzlVUrsikFnAS + Hu7DKPwFkYax+2jQxwKIzVeLMr7H2IqHEbgJpnYcezOsk211m9ro5F+63RbptXWJ - e9OfgZOILXWy4LTlpiCc1zhqENVwmT0XuAqH47is8ROb5YWriGyyyEdwi3b9yEGT + uuSNgCLC4z3fOp5JECizdudPvt4DlRfSqsJrBI71Z+NKQa19ImF3sYjHXg7CyAsu - b/A5cID18bhuQok7w9M5KQKBgQDRHR2lf7XyP0qYXx/eRV5Gz4H0A72PT8v+vQ45 + oYRuCn82sC8SkIXZevlq8tUCgYEArxphIo8I9rSSbFDtWbaQYcuiSf5VKeRketJR - Is5ldBwO+GhtiJZZEO1wiTGr4NDHDtvunibJHmMLYTy4TVoOlH2QNsBTpE5F1+nc + cEA/gCkysV66CyCj5OAAd0/JZ+KTS7WD3yQooNlaYHXWYb9nG/SCLIynIlaIH58U - Kzh1ZgxeOQp70Jc+F6Dp5AwelURYn+KFV5l8j/cEX4BpByMw+eKARfWmPhAL9E8a + lAhpv3PkfMHzJABg2VMcaOffgdtLqHclSShnzjE+k6xarGie9dMba5sw5tuO0fyg - qUt8AQKBgHfLj3vvSqU0Ymyn4rq6cYtr9R7GutQwDkhXxLxoY7xC4YwUXP63V7yk + ApFN+yECgYBG0t1+2RdeRK6z/hCoiX7kr+vWeki5uoJ2dpt1OfrgBv6ti0qtQFXZ - makdfVSYpqW7ego5oi+Cgu9QYnNN4wFP6+C/oQP6tUhDlX+9ASrNuiolacWr8Ci5 + rCcE6TzeFy/z54TG2kGwA4CesO5DtaTO6pXCHuMnP6KSm7pCCjVv2p3+7ZjxrRdG - xpbfhXJqw0UZ6/p+2j0sBTn1uvZ/JrwTs4FxT3Y7U3lF8KZVYKC1 + t0FDgTvyfy6hC2ecVKfdwmJtnarH5D8f41hUSzAnctzaVGJ88GJBCQ== -----END RSA PRIVATE KEY----- @@ -6831,62 +7581,62 @@ vector: segmentSize: 131072 total: 3 - convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:MDMF:vlk7vnwekb3ppcge6ohmnjp4ja:36supzzq277xjc4t4uhuu64aqljx2ws6zptqdafwnc736vq4dppq + expected: URI:MDMF:zy6rnxe2nwn2v6a2whohnb6qcq:gisq3xdaad4fzcle3plmfosulcioxsahzvc64aaftsjdetlkfvoq format: kind: ssk params: format: mdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEoQIBAAKCAQEAsoIhuv0XgpVHhsgGW5OHmBGfBSTQcBaZa+GOKITgaz2t1rXm + MIIEowIBAAKCAQEAxZUYtjBWRWXbZxD/kkUFjn1ZzD4xpFq2Ti0mp5z5OJYISpRN - Kdjne7sYzw5p04zphNmzqlam0ho63dsDf/OFUPv2kiG9yzXuGs+7R0GSLjFr0z+V + ssUP1Xb0YnPY9w1QID1/ZG99xvMyTxDAZ3/fZbHpsnkdi4MRmGS6BCpa55VTRSyu - 8R9zGdyv/G+4h0g/i3ZJamY+py/r3SK18ik425F0W+ASkcvwntJsFGZegifVtMAF + VKqt++uTyanxpMVX54pCjfkWsf7Lqw2n7SZ3DUyVd+FL5gka9Z1GwxjUJl0kTaDY - r2LArlNIje6cczPK1OTio2CxK1lMw4RbZ4aAFXSEiefU9r2CNiXgR9HvsJT2l08H + to/EQtEImLqmPZ+Rhx+FWatN7dr8dfiJugGaXlkEspkCxXHKbqYKVpWD/6bkAVGm - rL31WDZccU57gRjN6iG6s6Nmnvc3Gs9WDZRmIl/LAqv1bSHlViBCXUX5Om/fCOIm + 6+ILTVwT0jFWIiY6ERAJ/74AUEudGKKbgQCvdowrlfboyPL9SXumYPGiWCn2nM5J - CoU2cT/v+hG7MWyuPfUPvDTfcAOSuZaDwGUC2wIDAQABAoH/K1kZLSSeEO0vRrZM + tLp1px8P1WOWYrSL7SQHiNwLhwdeGt5YKOfeXQIDAQABAoIBABv9JbHDUalFh0nH - KkYpJ0R1g/TsqPYpS4lP0Ycdou0sycxiQsc+xKJ/48gcQBh38fWWgPGJ5nt4JWff + oOiTwfiAHc0ev1IArqQO5dOnGy/OoxCLhyEspLRQxEhBEGpY1rGmfIoZ+BeLgmQs - Rwhb13lYPHmfx+PQw7IDncoj4BPK8KxVkmLmEIxcMBub3pOCMDEJTaKGlGg5XqSP + Y5EVzmvdwtTvLsYBVGgB1s75wARfxRq+vFhOkFRoN/iAjDRS50OrtIdfkn028puB - NRR4Oi7tkrdXIGXl3gDl6LjzOlJegK71hHViYzbaK5uTszTn0EKRzYk2fUmX6udx + 1Pi1cvZtk5vWjLWisxC5jZlcBkuDtNk7tUeADjzUa09krQUHNoZ0pXb3T/39shrV - yvV0LoNRlj6wSUVvXew75Y17Son6bh53/zPCU9E2QyN3V2BMM5csr2aszAVZb5Yy + gbD+oz2H7K/rKUHhj8FGt0DPZUPqQjnIk1nnugIdPqWmwM29JA1jjOKw/DLam5O6 - eSRbohbqjQqnmgmhF0o/3eKLc8qquK5//f6SPvfi8w/62dNny/lbOQevjTgPxkoD + Mx8VwSBc0YeAr7jsx9xBvqHiV6esSmRcydQn5wKMNCXnbAatyfkBxbyPNRIp/R57 - 7wFJAoGBAPbtss191JLHCH5IlLnIskhebYSGuHKTCQTQ5YUR9AZPHk8XOzE0ZLp5 + yZyFYdECgYEA8hqLCZpfCnx3fSUi86+kx96BfBXm5R4U7wfVii3mTBj+Uex1J0gP - YhKor0mOnxOWsZksYGDrUE6kJwoZAx8eHEkxipkjDXagLqExFH4ZonAycsncfjD+ + 89KaOEDYAkAQkgZ6+0RqXpoBx9rRSH+X5pDjN2l5Hs9CY4ZCqeymt21aw1D+KV3R - X/0yS3wiGYQoHqzjBXMEpnKPX20O9wkmKT6S16har7VIX3Z97LI5AoGBALkQ9VsO + laqF9fiJz/8YaLunOgu1WosiPrBupIEjO8k3rogbw2tV4Zt8ve5zJRECgYEA0Oxc - NKzSS9yXMF+u1B7cu8uqStba4tBeYlJvXUyVvodlX1KlAATwI3lCrGphBJAVfZc0 + SJ0oFMveC8NzSXllNJACPUEgTC4qMExOUWWd2LT1tmI9YxUD+5shTsWRjNMzBL9w - Kmx/HXv5AiCAwWDsDYDYQz6ZTgoYD1f13qDlRFubjL3ZSNgwRsLOdQT4wO5iarY1 + wZq9TBFz2gPlpFN3HuE3Psx1J3BYPHqFiFvm3V/erZ5mB0Tlx5TEkTI6Z4rZlr2c - P6x4ylwgGNvf/+noP9y0eFvPBzdgq+f3k42zAoGBAJdqXJkrjr1OdQPTB/gAfGpq + OOvEEoqwTDO25V+AS4i10XLUp/Loe976zMylNI0CgYBrlqgbCGMcAdwH3Sz/JhsQ - FOgOIG6JgR9F5Wg7ARMZUvGWwkJC6X17T0s3yvzlCuDdKBxQHO1xfjYq7JGBkuty + Ry07u2/0eb3Ly6t10Jf7UVATkAUwA7IzJHAsd4SG23mBqyeT6f9rMv1/lxpSIYGb - 8E9lpKKQ3wGd6doIGZPVrkj0dnUX0v3CDiRZwfXlhxYF8AF92GqWMGbRSee7JHqk + kN+ojFKrAmf6Wnvdj7E26n3fNmr8bxjobfNCL8TujeqHAH18Kh/ZsOLzAOzqZgkG - vufS7ZEbwuD79yXWw9zpAoGABiuLkpqZpP1p7BPaWAZTKig/1p1520n27+2Fp6vw + VJFOGmZcHaL4s2Rn80NwQQKBgQDN0k0H7Gt0MXPLOv30wHeH1Oef2O0sn75IXqQ5 - 12HStV7q262Gn6OF+z/+0Zkkds1Qn57snytpxz1ZFc5VJC8akCYlr8uar3l34X3g + ZFahC4WV7Cp11lpaIXYq2FCP3/E/GCrJUNx0eC0d9wDhZqjP7ygx4dL4y6Dh1AKB - C0s5iThZa+b3p8WMRmhtvFmyzP/ZAPQriEuKq6GiUopYVOsaXfhiXuU7H1yIvrYh + V6iVJsGFYas6NhH5EQKl2EnZf9zkuF+TZBGCAse0Cq6AQhluUHxunyYJXzDR99Y0 - ZEMCgYBZR/dtsf1MBQSa/avlpRuWXT+kWspp+470z8KND6NL0tXbL7GmGfHoe02N + tNd8cQKBgGhyn0VLhxicircIRSq1USwAmq1447wQ9JUFlhVlGDnScG0K/8fTKqA+ - Le3krngmE4hHaZovTSBqY3vPVuamukZahuf5lIqmO334LAkG53ggxWYSC/DDK7j0 + O5nP8tMyJoTRrTxofYQLl9PwENP3wCXHQNM8jUsM9g5nHJrc60cf9OZooL7o+y8/ - ML+GW3P0/8+jrx2j2DDcOLK+7G/rUtdAU16hO5JyznlwQLa2nA== + Ntkn/f1DLOpYsnIHg+FLPBomF0jX6fZU0uxymF7XJU17ZW6ALVrz -----END RSA PRIVATE KEY----- @@ -6912,62 +7662,62 @@ vector: segmentSize: 131072 total: 3 - convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:SSK:f7zhz7uiaqui7hqxinamtovtm4:gjz7wb6tw3glspfgdld3sqt5msdyxbh4w5xmiqembwosixnylpaq + expected: URI:SSK:dgbwqgau4tc3ndunfr4nvp77xi:vpob3vkn4ogz7vbkdspr2kge5mezvgf2sb6ghpjtboafjih3patq format: kind: ssk params: format: sdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEpAIBAAKCAQEArwKijDMVsmEEfQt+ahwOoN/RR/zh9EmYnwvBc57S1XqYYD+8 + MIIEpAIBAAKCAQEAvE+dyJ0fXNy4h7KR9nekUSjgkbKUektsFAbSPJdyPI5Oidyu - XSuAV2LCiraBlc+cJsf8aYpkb2syLXXQZr2s3LavbRN65Te+ilBx72bqArgynHhM + CPCCc3p2dtyWu7IvB7psDEqaZpSc/brUxGxtW7AJWybtIK1MPCZ5ZxnOYUDkgaLn - avS/ecDupQg5Vw79mW5evsBEb0rkbavrX8gmkP0d2rS9Sy0b2zku9Qp/OiLNODcq + gnkukWTB9+ugXSD2oDqK90DKCGQ4PykUynehYJpgS3B86hdS5MzFnrTcchzU/9SL - NzT+MBH1i8v/HVtfW0S6az2KAH+zX0UFrzM8zPXsPCIJTDyoAh5/FJGnmhIrdep0 + Oii6oykEtrp+u3Gx/Nzx+n62AWlcvwcYlVN1wwHBRBSx1qt4lTUh1crqR/cLJnLu - mu6PJp01jhU8pLJjj6orOaqvY26/PcWEe8MbruDSIUg4Zyk/tl/dAtplKtfzZT/6 + nNFuNC9rh7ucDDcml0HAERddsEwXxgzNLt5yPmGSlTTP0q01mlBHfP4FMpijPQP2 - 7D5kLV4vFsULiBaRvANlMuSU/wg67+vDdrzPwwIDAQABAoIBAAv+frxkDeMdQgz9 + KQUULouYCebDEhWa3A93XU80230BQ8oX0bGmqQIDAQABAoIBAAvCuSr3TScijc/D - 2iqUhK4i2Ll5y9SNrK+NwzLU2jc2QTYreBHclt2mT5XpHyVwxo9j2lkzWmHGc3hp + wkPvUu7Sq7vNuGIu5bAWgPjRyIupo5QOmTvrsWn+4vkna66LQU6tQOQ/oIb5jxh1 - ICDCdBPmU0yC7sPB18Wr8LsLDxOjoxhVKEuWPX8vKUvXLfLY/KlkxoqFK8uC0vfv + m6Ys02OfieYMd1DMIe+7w2dCAFaok9zYzLakVNk1vrt6FsjaLyzwmw84F6YQhEbF - NeDpGzeJmV+xTl3WGBgkqaKylviZZ/QzVLnJTT96xouxlTYZqpbw0cVIkceW1JwL + jQvcDtMWsR7lBpgkHsQ1Wb5As2fY86walvgkbcM1pJSwan1HLDUGNjTGyXQZn1A6 - Ww5SCFTgq23u3b89lAaXUncdN+NIi0FkfrGmLylypohBHhhgDLVOZkrcnkWwWFBv + HBDDpV8aHjoObUwUtmcYZKw7KIs3Ft681x2ykUUPcfVPDqCYFCqM9M54QAMg14QL - 0RlEzzarNYsKFZX2yKQ4YJfzwtAlt18jOeLJjrOhjdJn0hZ+WzZ6x1G37FDAllFu + 0hw5Tl/nEnPuiWRQgiCmsmAGuvYJvHvkA0QzjBo8B7S9wC9VfTaAHX33+IZ9e6NQ - TSOZ+qUCgYEA3MqZBsK76J8lxsnvostR4W4kNyK3crOhsfIlQvqeSG1/DjpoeoS0 + 7mXgCrUCgYEA5gLa9HZuKSC/vJuwj5yIp0BrRKyACOl7YYej4+/TjGiMPqqHDU0c - vGLeVwT3MhlziHkPu7TvA7182ILc+NL+/TcEmPrYwEsKq2eI4U2C5W/fcLFvpIxn + MGTi+W11tAnZIwr2xbgg+rBgxpWwrlCbwYb1oayhHQfbsMrGc/RHSl48sjrDzgQG - UBejLIyNDtpXjOd7nu2mOJ88lQdKJRiC6l6W5FXqYq9ipxGmrkmfY/UCgYEAyusb + OrbBU3uWIHRRsikislaaCls/LUFfVddEkdoA3PqWE0cg0VRInPe3oqMCgYEA0ZaS - nt/TDSSgt8AO3W+KLeWjG75CYCH/4PIDdMCo+u7A6q6/38PzOs8i6f7OsjtuBDLu + MjUpd9Nh+hcZPyLPLh/toNvZYmu8e1HRZKCDl982kYmQL1t2e80S0D3P8NshN10d - CgZZx+erYRFICHUom8KgLfHv0i42w9rKA+1sKEE0RDZFsLhz/LqixDick2Q4f4Vm + GWkuW6VJe649+FMshBwMXLlVQsLHrQozwz3BFil7eudbmJ1PU2GZZYsgB3h+sy4+ - JIk942MG/LH/Y5UVbFNr40vg+T9hPBev4+iDSdcCgYEAqKZotW1SM6I9LNdbILLF + Qr7KVtU6qGe9+KOZ5eulJruyaGpEKWF9vrnM8kMCgYAtyIG2yWASFa+0pjTV0S2u - 3LhRGXx/PDJSNKaOJ9dfyFs7Thb3b36mv6+Vvkqgt7gRNBGlHvBaEjVPg+KR/87L + RPdVGxT9MSRa/HnV5CXyu9i2nJD3R9MFmv9G8M/N/2vWOtd18bm2zKbmwGMDv43R - z4eTD3es0VWA1OTE/bRDZBZMSrx+VuaYk+k6TvEdXlcRwSOgnglRira3g+6JiERs + TsDT5p3HPoovPZ2U9Rm/ptRkEahp+IkY5MnEiUQPv7eHRALhBrXwu8rugiWs24WN - 27Fc+RVXcAIgDRXCiCbchXECgYAwEn7app/zTygcIA3le9U6hlqb6fkDmUprWipj + lpw3YDXBLpZMtH8jp3dJCwKBgQCAg+2db8+/tBRt/9/xQOz9gYJ6kpSXryxiCed1 - cHkX6ZQehQPD2UI4PnZBBTKmmtm3ePFXwqVmbIX3Wwa7qjXSoMsd12E/Y99pit2t + 5p8Kb0rMryeEgncCrtsMafqp3BRgGG6ReFd+xrlqZ4uES6wOTgyeht5rE3jQ+GKJ - DIRBDSF6v3jHIwunZffFkLvXVzjjTREjurfEtOMk3m5ogxsuLJ00nfdQVSmN+Pac + I8LUThdzY45c5IkRvdUL2OWI7y/xuzdeQhNcb1+KiCKK5famb3pTZ+Cb+h1Vqnwq - gasIxQKBgQDIJdsrB0MiZyL2QDipKRs4sKpfa90TQZekuBH0eT0Y5gEp23yUbnqE + iU/MdQKBgQDVlggsXCqDvj9BPvdyMy+PpKKnYlixAt6xdrvLfMCImhvkYvJvmM2A - 11rmDavNEU7Ce3CIIHcnRfAFcv/cS/BLwyaVzAODT8ma6rjoMKbxM98VnOnxTpjQ + WK1SaZqQ50ETTaxMyBF22NOpm133yadWSpBWIA59MBi4PAbVkItAKO7MDbvSXNOr - E5h7QBI3rG/fzB0ysxrZqkat+bR3vP7Lmq1Fzgam6GO6qVzMEduOow== + Eqxj2sI5gwHV99+NqMcW1psLcFDb5E/N4SZQmuOKblpaEJTlZ80L3Q== -----END RSA PRIVATE KEY----- @@ -6981,62 +7731,62 @@ vector: segmentSize: 131072 total: 3 - convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:MDMF:pprkcpehlvoizup2rw4gbjzqgm:bye7dr2xg25mogjntkaxnnmm6hriqcs5hy53byf3bc4zavyzhs6a + expected: URI:MDMF:6jynferbnzteludtwg2vd7thvu:oek57a2425optdwaggj3zxaffx6i7kzlbk6sbvje2lphuxdkoakq format: kind: ssk params: format: mdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEogIBAAKCAQEAuPFtYiEqdnlCHV3RT3LRqJ0qXM+QJGnIc5mSteC6g4nX59v5 + MIIEpAIBAAKCAQEA4110ZTjzqHn9RkYJDatXxPP/GuqSc7RFisqXYr7BnJ7fQZo1 - afurdhb1QZupk1KxPcje0WFqLdREZpHYHvRX8M6cp7mupo7HhoqRPlN0s42t3Ost + W9Qij0tIMi8wgAkb+r+oeLaTMQqZtnBbnsGoWEk68kHFSglMInkTq/MZDgwPFbKe - LJnZKac3OFARCDuOBHYoCGezFqHmJiQRJl5q6EYmiFBY/JoHHna1gsaApEmZvZjX + gaMdGQU1k0IAALu1Du4RPUWPUTsuxxh8u5dPWelT+SH99F7vxv6WZ4pk0N43hfrc - XzcaGKjQVWwT7DrYbEgdxNr2YJJjWEpOWuiJpNHoEHITXpK4E+4k0PU/LPURuQ/l + Khfha+oTgBK0rVq5DmmZ2Fd0EHw0PCFFR/8BLrmyZz3jvuk1K7StW9hqXWqNILJk - kfox23kh19Wf01U7/CDxoZ9Y9mfijvaUtS8F/BontkbpWN7JUwiuW1wygc/rQK8t + RL3FEy0rkYqxZuOXQ49inJE+o24hNYSB01H+IVZRUDB6s4/ZZBRHhrj1Mr75KzpF - +7en/dQ9Ap6VX9eceu816/N+zuakypjP5hUyTwIDAQABAoIBACwZqc8iAHmivZC6 + y4gfd70N1vVi/zOVj8iMMCvEBdXqDUdPYZTSFwIDAQABAoIBAC742jilTPV0CmbP - G9y5kOQHoh/igMkmDl3+a10CWwdducW4jxdmI0M0A0SjRULzj38fpH5CH+sQuETL + xkQML4xRilUhvBLqXemgFCmC1lYInoAbn0Vy7JblCyvPAvqYpy6lFOWndn5Nvdbq - F0F+W2/5HKLkJJDj8BEVfr/hb60XJjPNQoblosKLdJ/xe7Y+WUWYFUC31Z0aewJy + nIsOYDypGGP/QYabqB6BHBbMmNMFm8I0TjnjHHpUUK61FnIQVYYZmfcqHUM/clkQ - TEKddhmwDKUpn6aQZg0uGmc2RVunJ0r/OwhgTzjtMkds9w+iYu9DbB6RDLRQBWXJ + johk6LBfG4mfQ6uOR4Q5iZInjc92jYi2XAsDMaGWefVsiTFGpUUjlYCZ39QWQv8k - eRYGg2PhJ8J73Bk0oQMAAEOdgLWdOpbkaf8TbQHu2QMFE3uv00YMJivAaJ0/dGmn + JoxsXWLM+qyxYs+A1xyqRa+/mJLpuUV2uQ6H7rDtRlLiHuz8euOibUQDDa/ovXhW - dujUzeloBFPDfhEEIL9Ind97Ix8lbCw51rJgKv78K6yJuV+zuQZjkLRpKvPpMmr0 + nId9SEatxYs4E34JBfc1ibqasboBVtrWhTel1OmF7QnNYZKXBYl3Zf6t3oD64m0s - 37VOzKECgYEA6E4NkBH01BYLS9OI9pV+DL1DnMeAyJyHWV8E71Thn+OMZOqpVjmO + lkCOcfUCgYEA+qkiX25dgXAqyDrivkme/u+PnKXhxiA+2wdrY93j++dhi2mbIhxp - RJkqcy11wQCRO8JSKg/FfPBi05F1N0xAtwKYLK16Gk+NUWqthRDqM0tVuvxqObSg + 4KwySBQoCkHydawkdceMikS1UKSakRlJl8EuIntv370gT6v+22QxpWb/M9GYkxZL - U/1BpZoLK88chF0r4XAFoEMX/KIeJg8otV3sM/+Y3fQ/j9i3q96J3qECgYEAy86p + Vju5/8Dm8QQ1W2LmmaGjO0ef/+J3UCTNGncAQ+ZTNQSmy2Mi7uppg80CgYEA6DVJ - gobkIuJ580SgABR99rcQLXHeyBa9pl6wsiSHCpwNYy1TZ+csiGNlbEUSRdxJOqZm + xH1/neG1mlzg9GX8g2Ckts9WtcbBK8Nifoavo2ZwxUEAptNomdURkSx5vFaupWkK - C3VGKOJoSYr44MElp65HixaNoyQKBWxVLJD8BSqQYqvkRgGQxTyX5OtVpvgEkxmB + LsoDpg2mEfZzF/QQ8sqvqZGsC4bvajlv7YGJWFwkOwTasSYVQK70wcfy7aPeIG5U - dy7Sy4iW14yRvClTS12T1XCk2K45Ujmd7vWCGu8CgYBmA8DY/8mwSW30gpSnFMch + LpXbFixMFgsPbbIH3VdUfiLN4bzkS2iPiYomEXMCgYEAyijOyBjC4ToNxx927/GA - +Qt0EfhwIK0fhia4o2HhwR+qQZLTlrrvTQPjSJdphkJBJ/jFF9/2GeqMVlhPTGEu + givDr5s51Aj9qLj7K7gxv2CFk2LA82nnGoTGqMtY360AV1dWsIcYGgwAD+IxpwS2 - /SiulhAE9eJtWpeQ0/jFRdQEJUzQwo2V1KW7f4ZgWrd/ORtICNWvp0clXlw3Anky + DeaHxte3CsQF4zvceCT+xV+kQ66vVzGL4SiagmKZ35h9UA8b3Jw4gf7qU/3aLJEB - DGjp/Ni4v8YZ+WXPSA7rgQKBgCv1A9xyKYxYmoLcf0HlKZHnw+Z5U9qGBRt3+tZB + um5vkFOamBAAVdjGu9ni76ECgYBDJJVt8XPjLQ6b0dtiD9NSEbHPAmjqKsxUYSyr - SJsCM2T7pqyXUKSOA5cJgrpsm6K5tvKrtZkl0+ZgwfL/1ZZH4YhfMedI45xt1CUL + tTo4HzjgcIlFs799K7Tmq1uP7+iT/6loGhWwACZS71YcSQBVk/HzMCH1O1Ei//Sz - lD+tAX02o8Jxnf7cZcpq84tSnPH5I1JIWBCsAhS1bc1OgHeV1EfJxtQxJ43TfXvH + Uk9qc0ounpq1unNOvsga+DvwJv/llMFWrxIoeSqO/Se66k2H3OabimjJqxrACz3l - mesjAoGAa+822jANAvu642etlFbbtVMoeFFFaqM0+QM9yHazpeO1cuKlTnMnA3XA + 4UJcrQKBgQDk98mfTuvDdcUPQPK+eYK/RNp0W6jNsLht3s2823K2K2+R5f7a4ug2 - ztLTX1YhnsUOOHNMyJ2+o27mTy6GhKBTa+rx1DSYBxYvXKHMUUNQnomeIR9W0obi + SjW9v4cT+IVDR5Nz1XZiYWNuTv679v682o7c2SoalWne6RQzickG2s8h6k5D2qSj - MWsw616EtbP+KuDJF/fuWX55vyf+pON9GGkAnpnOVH97rC9zXTE= + LyRqPtyr/3ty3pRT5ki10yvLvNbPusfrUMHYvRv8slJt+1NsGo7svQ== -----END RSA PRIVATE KEY----- @@ -7062,62 +7812,62 @@ vector: segmentSize: 131072 total: 3 - convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:SSK:urwhxkkv2pwxp6kkbikg7g3gmm:4tfyngd4lxccrddmiw76s4jsbhmyljitk4u4ue27lsdtvdydcr6q + expected: URI:SSK:gzcs2in25goucrttjikx67zehu:sbnsmodidyov64f53aqpdkunrpx6siqw6v23h7vs375ftre3i5hq format: kind: ssk params: format: sdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEpAIBAAKCAQEAoJGKHKNnOybDXGm1fzftRDzJEBez+fkeIFKmL6GIi42O6zlm + MIIEowIBAAKCAQEAsy9CIECvk/doQ4KfrSYIwRNZxr2JmOvfdTA7fJr91c7znNob - 2gGLRc1u9cvKZTEaFfkEqQeG2E7+MNdSVjCPTWpIVZ5Gzae8Ziy2VfFdXw8wStsG + 42FTrDmG41G/Kxfx7S3hF6YBbmT+Nu/LDs9aSO3mPoKjiLldKL4rqp3VXLGp8pnH - mhXLH5hZJTJGNec8Fksjj8TgFxuPvgR21xTXXOLt48oytNLLRP75dgp1njBN58Kh + EDq8oYLSu/mRaR0yjtrVqUsj8nhtwxGg4kMbORdNe9J4XtCCxQvMy4h89qiLp8uj - KY6nK7m9b25dwRhH/9KCwKOqcwF5kN1pGNnB0X9CcLVnKyv7geSyaCTuk3WfMQ9k + 5fLGIFOkfYlaUQbw6pFxT0THVGHyRx5Mzzrdivdjeo8RFJsqJoZdrQyEEAXKkeAZ - B7RkOmlbgMTgyTJ7yyJSmFA+oVZO08CvkWr1WwmraUoLEQawJ4qx1MYuMKgFfxSZ + +njTxR1jrUQAx+RxtZygV5X0urUvSt1cVI7fAhbfID2MfszqHa8JAXu4DxnkBCNY - 0qVg7RJx6GHIwoXSq83l3lmxLa7Od1xBcjbUSwIDAQABAoIBABbE/LZZ76IiOZLp + ukf8Y7tt0VmfM4zWb7wmubQ3FzKmp1vZ3QprGwIDAQABAoIBABSlKkpLCa/Tvrig - xJyRRDqoegSnr9RzYLPJtJpNiEzt2oX9wlmI3YSdAK6nYwCdiWrzQJdto1AaR46K + kUNC8ZlFYH+skPEEpE99Si1WMk7zNFBrNPFi4mAilK8WWR9e9+nq8ldmMh4FFuE/ - gjkJstCSEUbe5oB0WFGO5p5iV1DLGRiMXa/NBlxpIL8XFYDAVTN+HUFedD6ioGwc + ibbgHzft6SxkovD64ofyOVfELbQraDhijXQKQHefeiZcX+uriIq7HgkOdkrWo1bp - OvP+Fxorbfue6TjeKYgTtjFog0xWqHtqC3LtA2FHUngGBDNdjQxcW024rStIvwir + bg8DylwumifdHS9XeOm5LVR6GmPU4D9I73OuxyiQZGY9OBAoEHa/5QbnqK5Olqa/ - rlp5dQ/Il++E29XYybDVcQPLk9NlCMYrj/aC1N0lAdiY4eeDLeeM4KacEODNiIxO + 3xdA0zsrHHQpJk2C86hR7+qshKBSdj8Z8HvsnMjUrHST/JEv4ptCCi7MdvXtlQB/ - xzgJpLFN966Na0lk1Yy/7uluLVZgtYqGPOe2N3p5qehF9diTnNT6nU5Mqki87bBg + b79Cs6Zp7AGZH08X5fSnb2BgEB2+c6Sz3gRWBnNcNHXI0mty02G8MX++fTwXBJm3 - yTCerr0CgYEA3tPgSJcAGQ5vSza2h+R55rRR0sb3sg2i+Itn7CXXyJpDJAruqf6R + 7bHPrEECgYEA1GkxFGYgHtYL4wyBINP40fwbQRVjp7gQdKP4y6/SCNaIojUTfbGn - DZICmSYHkNhJvW1yvNXEsD6L4Ory2pbNcX3lFk12pN2m38MWOR1jyiYzgW6X1WLV + U+GSDtEUchrFqeCEd7bJwZPvTI5mz4LJVtBiikznqk7YPZc1IYIRl/qR0tLjPaE0 - hmBCmovYOQZ2x4jWNCiiIMvwo7dxFaIUumDPB+iH6kXuo7A+G0R6d70CgYEAuHjs + vLPNbe0hUWaJCfuEGy0J4vOKjXTqNud5qen28dqHp7alw4+jbP3GV2ECgYEA1/SN - Mo9EIX3x0s10MI+Jo+Pgcm9gfzU1AsCL3mAqG3Lv0tzrRObzjt86WMz45Efvnr0d + ycHbk0aiT60e7d9ecwz2R9z2ZQLDcIFlPz7Oa89OUqMrae9V2B/yO2Dabc5catGx - lEFMiXeQivHcCYThGLc+I5V3bTLIAlTDw1p0iMu62jSDP3pt9YVnzk4emMYIh/VJ + 5Yg7Z5QGahJ5ers95XWy3ed3v9A7GDZwc41HTqdRbwsaM3b/YDcU5lD02HFaUqeT - JF4NBi5LkZBZlA29NRcEeoVkBGCMYeOg8ziNGKcCgYEAj6ovet3QdFc4LlgyS19l + LjiEn9mJ4JlH+LbCOESORqxAoZjjYjQC5U2+H/sCgYBKjf1/FnVxvVmAwRPVzPEJ - sPcloi4iWSwtnO3UrQ6hF3dOPpjF09iLkSJIhpFcY2jv8i/0wAdbbv6ElRkmRwTf + 6z45suM+rDmCZ0ddXwIOvhZJMO39cUy1AXi9oJ9XiZQVk0uLpWndeypEKbtmXJaE - pIK1BzIegqFeC/ruAxkN07HZl2PEhRHZ9W9uwdHUMMAYKQHyiWKBVX/nwMZvJLGB + 1TGxL1slCPWXcKpib3/zYyyp3gGK0TlsfoO9cL0AEEhLa6+rxjwxH6BjFEVdLhQj - h8EO+lxT9RntiKADCvWVuEkCgYB2akEMj4SnjyYtMG92QJ2VE9FfA/nIjooR0zG1 + Eo7txvFUaaR3JsSK7ewfIQKBgAq9K4vb6wpg2dNyfXZAxFaeT2T1dP9C6useVCWX - tLsy1Yv3KpLnru0HeGoG2MSoHTlHB5S2N1h/Ib4qQukBP0gTSoVb6DU6Zo+XV3w2 + /vXXgkKTwKXs8+zicc5IG7SYLXpWYS1T3/hfoQ4HSykyRHqzpqhoSUktlrK2ilME - qZkGuuid63mYxOlS4qjo+KKRZQXS6HRkIO9xWURvE189N7iOHNFmKLw0Rxm2OJ13 + tIYRxffqqmviwAJN2uk1H2fgAyjXEnea8eVtEPEtTintFK1to1GaYUBn9O6+PWKf - o4SHHwKBgQCY2Cv/ga46sW2RY0CzDnlmWP5758x3Ugy40+DW/nItTi1Fza1/oL2E + whOfAoGBAKSFBWcsn3qk42K7ZFdurqj8FbipiYNZYXU6AdOWxWqIMzZjEi9VBPNI - wVjEvfr+yMTiRI7jGjnYBDDeSrPHq2m16NxdGLQnLY0xWEdyQ88DYlibAjie3zXP + rCun8bXlBFUNtJqLE17n0qzpcr7GJGcy6pqFcE+U1Q3j8uxJEqG/z5ZG4phRxVd9 - eEm8719/NRLnYMRPFrKASpdQzijZNPiX766G26/xZur64kds25WXLw== + gWP2U/qIZivdIymYn07e8hlpTEZgJcWjkVJ4AdB/PPqfa/m8t8Xx -----END RSA PRIVATE KEY----- @@ -7131,62 +7881,62 @@ vector: segmentSize: 131072 total: 3 - convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:MDMF:3zv76egtkwfc3hezy6dm3hgh2m:t2n4zvvk2wm6p5tdz6m4oqij5wowsxnlpwhwz76wynb2wwpzmdxa + expected: URI:MDMF:uvvcinwvc2gxig3hpi2os6ajva:yphtwtymxs2t7kslcwgdjmx3xu7wqzknff2m5jte4meilabbmhpa format: kind: ssk params: format: mdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEpAIBAAKCAQEAksSSDT+xAhppAtpeEU5FGgowdf6KfkR02XCuGjHam/jEdx3v + MIIEpAIBAAKCAQEA8mW7LoF4muvoZ0pfWJW3wsY3VWXLF/4RgeJvrBQy8k7ht5Lp - iBdMDGRswFIpLeLyM+a8gNs4dcLn/G0e9AGHikTDucBCEDeW4uPq0B8+INNX4h9R + 8qM0z9SJ4EgryFgXtG9e9n4AN2dPnbsEpozcxrhYLrQZPG1F9FwcppKPZqqGoBYw - cUitHOA7U3MvZRFDfLOzNpfYdKrKbhHE1T5NLPQiqVhYuSD06T0y5mZr0O2ZRn9a + D1qeUHTqrOvtG5xHmoTCD2kySrzn2JA4pbAXvVn5EH077LgayDX9jaZGofUsx92d - bwCnxLrTu0fOEjQNukWvEkpcy2vq84bRqBr1zCMzMkeFh11HOhPb22BPdnSlRvWl + THw9argVEYhVGv2NtAGvMLgyXBSaH6SZe5X5r5YduLE7d/maZ0e6i7tktjLAyZmu - PNeBrnxYVcs8QNFqM5ffVUMyqr5gphtjAmLiICsB6BkJbTkEeXISwyCOek5m9R9q + eDw5QE6AVfy8/2bf4hCbMACnuhMSu2Mb29zgIn1iOgr9sgxRcnaKRiTl5AD+Dc+q - X9GZh/yUrTKbhgNSiHVXvz/Cpjw1P0YifEJo7wIDAQABAoIBADUxGN3EX5qrh7OJ + MqekwTQP9KfgV+JM/x2vMxL7IO2YZbdc6f5Q6wIDAQABAoIBAEY45pNAetoWwcs6 - AN60x0aQus+I2Ri6Jr9Hn1HPD7PHjSy+pLll+CHlo6RwIoyG29EDpv3sdaH4aauK + poiZRxUsK1eYF9ApkJTaLpPhfijoZUezTgc29NPItPC+t8BglO123kH2msVyLoR5 - wNUeWMk78tO3YjoOa5j/kXKsYA/1iLxjLVkpRdRZUCcGb/7pKtRfLGx0y/Y8j/Ek + a418fXEsco+FKVJyLbPvA3XWO6j4eeviwaWRERAp7tqNtrErAytmjnm8dg9kzp3U - b3n5gm7wbD+DzWQLFbgSfggSxrCJX5mjcPBJOmp5la2Lp6CsFN7SargMBr6Y04hf + mjSV4Sq/6AG45iVb6JZb3cqtgwTj2+7JJIzk7MsG1NsL4CH3bQTkcfK1XHX85WD7 - 9Mzvq8COvwcYT7gLmLZFUU4zv3rHZwdOTsWJsyDZjABmYGiJmry94xbh/ar25hLR + vkmN1D5CigOJSxeyltPpMibCpuZmyUbauVYN0X/0hPCWeEy0u5cnyDL6eKqfnF5U - O6OMaFWrFWXID+4T0LowwZo7gX1LDam8Ine451njORL4rNGB1E2C6n00HaqXsnu0 + OhWnrghTqWcVStXbDwlXp8g0kO3hwu3GIMrcJPjQjCJn7Nfq+c/IiFEuiwdQSPof - P0Oq6EECgYEAx/DklSZgwVh9c+UsqVAQ3BmqtKYyvaMFDoUZUejgKdi4NQC11FhK + NdmBYgECgYEA9epQwTVK3JUYgFDmmR+PjYTAR1TNL5KOqt8tqsUXEcZWNYa4Xyfw - rCBhS/ITmLePDzZgZm+613wlOaQayOi0Jnu/LFRjLv2UmWFfcUAlbRxofhVfni2S + 6EJdvSRaZCJJd6luO82QsqhQu8t8/pD6vM6qs5gNwnQ27bYHdy23gmP1iXyHTW1i - 5tbL0NSnroGBsvIp9+amecqIIpCQfhyeDc3ghLAzscx3cXC4/xQsHoMCgYEAu+sV + a7nq709lDardbJsriSvk/YMCH24SFRgyozW/3vrZl4Nii9daPlxXDIECgYEA/FZ7 - suCzcaKaqtxi05Ma8VvkbLqn+WgKO7I/qHp18GvYrLwjgp5wUtMDyj9MDI4FCeDB + 2jPP7EF4bRy98/7HkpXfgNeV6dntjR6h7Zb4Lija1IIS+7459Ttgni7fKNpF5T5t - 2BENOTceVP6TYl6lSow10sTO5JUOsVlZaBfcPYzR1G79eX7lrQxCHDgjjxW41veO + TKHj3cgnjg1hBCcba4nK7sGvX9noBhcfEbzD8BGDUd35PWY2W4/zR3mkx/ueCtxP - y4Bf/U502E6f4BUrQ1s5xYUwgf2qlY3Pf8D2ACUCgYEAs5Li6iZ+7gg5HJcvhp3X + vapwWf7vuOEj6+GZlKk3hL3/PrL9ym1MpIs8l2sCgYBRmf5QDpIX3jWyJqZOe2WU - lqciz38ZwYKh7wmR1SRP+KWhxFDv/liSMIggeuJfwWDThzkyWa5t5E2m7V87g0il + TU/Mm7w2pAhJdSNfPmVoVYs32cuGb+eF+rfGUrDX93Svi35zw8PXNPkNR/njM1Kw - TI8GA52DO1gbV6rB2uhe9OF35A30RA/wiY1Pny7vr2a3g23GTdWFnYtOu6SVcf7n + oleMntE6DHxJpxSVHIt/bhIFHFh9feWh36Cw5oSe42r0Zg0tSG6FHRrwOQMxEsWC - 4cQPq3zJ4R2gBW3VaZvHiFsCgYEAqZNmrVjgFXdqoyzlcY+aDJuj8goucn5UXbJo + 2QwhPtZDa6qgwsZEWTndgQKBgQCs8cU+/uXRoemctudFtGgqAkhF8PwRY8iZQNZw - h5yauS5ZBOdyE/jt24/YJ7Ye5mVyXouX4Wbhy/PVR1XDok1OU4tbNqurF9L6w0eh + lVkRofShU2kZWv40IncM00klobvn64pTzFz1Yzog9PB6PSdg4/bO/rZo9ls82Vn0 - yrFdaZ2d7FmMGwtML3CUZ+qxC/nKJxKWpUVfWbJm9ptc4lW4CLxV0cxzDZrfSL4D + +TA9eHNNh9pMB6LXzGhLo4aZfc2K2gZZEtigBcddKglJoLx3FCc19lZbLagdth2i - tYFnfJECgYABXvwJksbkZxSE6Jjqi0nW4l3JeH6+Ac38zDBj1fG06ILNrh7NKaRe + ZL+pcwKBgQDxoY2CkBAuOcyY29WmyBWUEO13odEzC3uUuvLu4mqxRYXvEIR6OMy7 - pp1JF44m5sY3LeVnsbAdsipwAHr7AzYAbXAkgmJzVbUciB9bKbeHCAVCyBqqKID7 + BgPMOFSgqsA+qUTzahSAcXgNauS9oks9F6uAfDNVVuBNok1eBK2XwaoepxRFL1TN - EL+sAqigrYLqYZ4T2ev/KAH590IZqeYAXbewOrMi2moS4EPitIsQAg== + YBqj+On0RnnKTqRBNSFjmtopTDDtZUzu4qzpLzBgDP8Zn+dLSrgoMg== -----END RSA PRIVATE KEY----- @@ -7212,62 +7962,62 @@ vector: segmentSize: 131072 total: 3 - convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:SSK:mwxshkf34ewmdca4gaknxkwgca:rfahufbykveekuhceto2ynwhckuswliwgiwpo3vs4dibfyvl6xga + expected: URI:SSK:qidwnh4eshzips6trdeog3tczi:wqxqxiacoq4zlm34kw5y36u6n5qnwusscm54fqu32cwk4v2df5uq format: kind: ssk params: format: sdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEoQIBAAKCAQEAmx/LUVxlqGwQnbjQ2PRSXsJIK2wojqBM6z4H9ilFpVUafxiA + MIIEpAIBAAKCAQEAtbn+aJL00xu3hzUW76iOnI1++uKgSaLljxTqKEyuMdLq5ba7 - tCy+MegxTsRbUYPAmfxYDxXi82GNXxT87OsEjbtk8UWOqlNrqTiskZqvoHSWt0/q + zavatvLxnDiscV7FKTRQOh9c2NEnANA0CDI9MdrCDFM/SQvvU2X2s71F1T6HrgWg - peaKTaYi9PWGLfBXuKnJMCHyz1Cke4EeLEH8P/Sg2IyYbkuZ3y6Bp/mxRwfTtxnB + 6bII/Tzu3YdbOatvk1nbSKWnDTuBQMqdLKojT+58lYe6/n7kgo7atmxSOuRG8bZN - DuyQLHGAf8xU1hR9Vcn5G1Ku1f4PgYIknhQbJ63BZpBC4T2VTnrUXqygNe1xzIxF + sLBE0Lab2EI7jYyO2fKr3PmXxfCMqFvIVA7AjTfATs0ekFIMO5Xus2w/Sgz+plCU - 9SWMGCyQdM0k03DMD7Nhgjw9vWL8OgMUqULu7aE/cR3foFECqK1qpiPZ26PKleVW + 6hC1S9hGMMLWtWGTOIwwlG5hDknJ7xY44F4o9M5S2fDHkdXChGDSpsd9dQotFMZx - mDvfemDIUMSOYP+mYVv3PGXmArCNunhKQ4ujGQIDAQABAoIBADIN6VJAiUD2Vco+ + Gc7iCNHBf40yYZPKrdZqaTnonDUtsyZsoSPj5wIDAQABAoIBADWxsv3rDfOiaOPG - 540KEUYoVJdGWDPlf8xsgK8qlCGMO3eFVYpN4bVC4h4zd+/unohRh6yeeFPmR3LF + R/Sf9SNEm5Q9iea3/uP75gPqRD3seANPrsXiVUlhFwp1pF4LBm5aSqohwik+Ayw2 - 1/MuxpJhRGoh8q39KwE4m16EVmVlGXjfHa0Yncn+cMswKnLKWdPpXVTdr3a748dC + WGljjrlATb2ei0BmCly17+LDtfJ/+07r0tO6CvXoHxvNdqLfiKQdFLGuYGGEh3hB - W5UWWandasVVYJ4+YNFGNWoZRN3RJM+1hcTTuPOr3a01g4U8cTLcjOcitCBhW7xA + ZZdg3fYsHRuBczrm+1WoJ+9mqhVEBM6fx+JI/KDHkDu1aKWEeP2Cl7Eol/ZglY6E - ebpCsdqD1xx3yMBynU4a+0Iyvsijj5orPetZHoqU20MYrrQMLVXIqt8IBYsa5U16 + NEodDfhGOdAaXKCAYR7IHq7LBqHLCK6moWXrM/CQs6E4Fh5mal5n7jeRzutg7ZPD - 6ZOTf/NyQWKLR7RURSbTChT7WD8PCoxbamdMi9z+2y2ftfr9JfqB7LDEScOnZUoC + /tWJTWRPWVd2oBYnKeGkgWJ4ADGddtLKbaumxZ1jPu19fKIoU5Sw2xn10vVk+/KQ - KRvnCs0CgYEA1do6lBtMfy3XrxawKLIzHI7Y+qeQ1LsSyYeBeZECLNEeZrybwu3O + YKP7SSECgYEAuR7MpUtt4/jsoJrvf0idHvRVieBTxYVQ5nBfsHghDWtag4X2OkGM - w7MsCovzWIRKTjZtz6kO55iU3Wx368UPTdCCrjRws+diO9/Z8w3VXJP/KV+T4AWN + dX3QU/q4KMbsHB11xg8dPp9VJEH6wnqYLp+TAjTwetZE05SzcyacfaOZCLZSAhew - 9Dn0eAefc12eEcIKkDfrcVk+8/rgciHtltcfbHcqEGUyB173dnfA/OsCgYEAubJ5 + P1T/uIvcg4OYuJyYvaciIi7q4frBANGIC8u5JQHgbyDZCayeMf/YRPUCgYEA+06K - NARl2kJ99WA1xGi8/rqlHjivnhiTpo+UGqtMi6yAlvV1trn/tAYNfFHuR3ip2tJ0 + ztAsqIbuMSKztzTzQ6ljGxANMKjmJDHTwqLZ+UnknP+R0ZpGBFK5Gt46EjfNkEgH - s8mHEeOzwyhwH86Pkzbl1KJ2wUdX9+BsO5FqZ9wR6K/AwwUipkkk2OgHuHrDLnde + 6liyE8nM2NMuXGFUkTgCzCAjpnsTR9Wh02bWGH8ZgUHr/Apq5IeN59QJxWkKHlJn - AQUWAYW1wmnIOTgSZmo3zmvI4lCYvx3IYgaNDwsCgYAOSf2eBdDvsoV12oM8xONr + jCEgHtMRmOl6kbTJMpMoAkJ6kPi+N+o8dczD2+sCgYEAh5Nb807by1NaEYGHF2QZ - ZhQTc3zW6gUQWDCLiefmTLbGUJXryW4GX4Ny1PUWlghM/5AIzxgC24we22+L3mfu + 1jrBjrmhAI7TogD4w6gnJMnTv3FT1HR/JukesvJy/0I4V5rnz0bwdxV/6I791IKu - YB9LOo/JRY2nyIZMmkEGZZEoF43O6zAYAINYPdImqDu2nguMpV/i+/6b2MiEd8Xj + g67QnpQg7wWP4JkOF65We9ld0bidNPUeWjOpGQItXI/7QHFHl9YYtIpB8YCQ60WJ - TU55NeEmpUxZd7v7O3c2rwKBgQCq113W02z5Pk8v3pHY3xtxpzmd8jzv0GCWzmVN + aoIoNUc7lIetDF3Eef/S5yUCgYEAwmbSuAOP2Fpwne/zSBEc8cVx1fiHy5GMXolw - m+dSYSP0vmLL95cegqsJgz8bFhH+tbyUY4YWmUya8asmOB2zLMCJveZPr1lpPVmV + /4rMxawkvlJxccw+x49ag+9OytMCIM+n19/++ZHM9hn/LhVYvvGuMEvYaCujEZmw - /BTO9JKtZnSLd0AHiCeUPvRLbvX+2+bqPUmfoOo1sKh6q/GRs4sgJ92rCMdenQHr + EoHlspN3nmbpb1J7uAcofiKn4F9OJYCne14Qo+exIDHU0CwizA3MEFtuxwC03TpE - 3WcNPQJ/VLI+fiVBRc5VkAUsywrhmG2tE2XKTygARtlfVJXImPRuyO8d1opah743 + xPe+tzsCgYA2lWPwtLMK2u7mstYsYZs+g2KHlZzKf7mBtZzmmL4IkhvU38wuKMuw - lPsxHv46zm4S4sa9Ik85Ktlkmg7t5+fiEQ3DyQwgsPfRGETgGyATniQv+4AUL+JX + xIlmsOIHjkXsfPT0tBKD4mAngPVPN7Z9sgJGCuWqxXoXxQltlBSH3WzQEGvipt/T - MkSQKCxm8NVx+/igyLPDYnY4P1fU1GHYhf+UbdW64Foyb936Cg== + i7pJItBHrPl1Mw5m2baRRIPQMREtRqwHR8VRlXy4vCVg9Wky1RGCHg== -----END RSA PRIVATE KEY----- @@ -7281,62 +8031,62 @@ vector: segmentSize: 131072 total: 3 - convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:MDMF:uug7cdmfgrcrze5rwnoonojcoi:ihoyicjkxebylo4db75inmmqmvredrvxbbnbamrmuzfpjd7cc3zq + expected: URI:MDMF:qxw7uvecwrhqibsdufbzs72voe:6goxtexixkeemwwpf6xu5is2723hkpzvxtoy3d7772h77jn63ewq format: kind: ssk params: format: mdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEogIBAAKCAQEAtljs4KYJOc/c8zXWR2/St2f7524LPSXMjJQmzrMlE/oDswfB + MIIEpAIBAAKCAQEAyGa9FcNFgZ+FZa375YVsROi+KaCNNfZDjSW4xOhvrAO2ch2Z - fksJITRrzIbMeHSscru+XgjSKdybFgPdLf8xotGTgFLRb0jfnNpclcqu025rgEh2 + cY0mQ9iAcDMIE1NtJs95HqLGbBzkRF3+XxLN03aqRaTai+wAao2xZXiLstZJV2tc - pnITYRZYTkrB1wlKsajFNwyRbdIUWL8/wDQC0XdPMPlxcCWyV9qDBYP2FzlTSDmw + GIFmoKx1pGlhHpk3sysQMJTIgg49HsGxf6KW9TFLMkeNA5kvE0tSJghQ7G3SSKm5 - fYb8QrUr5DV2XErzpsmIMHqEgpkQF3Q1xp+BwwxXKttbUaHjMDD/sn/gCL5MAlnv + HTq5ZFF6Hh/i2zy3G1hzwvB/kzGmQb0Aj33C9IbjbtiI8wz79+TBecEMILwUHg5r - LmQo1TwTKZGmzKnpqmJOXTDwDGAYaZc1MeeMmQ5GSxaWB7xWeHXIFu/QPKgGngY4 + NdwP+9+i/ZoMbQ1RW5Vzfvcb7w+/yp0GtvZO/6yUihI6qM5ZZ9q83O575/HY8R6t - OpzqcQeLFMU2jnlnJr8rrdocJjwMCz8jto78RwIDAQABAoIBAAsEDJdhCx34zpEg + LKgSw7A81WVL13vwDHZtcOwlGg7ElAITsRRgRwIDAQABAoIBAFTWjajnbIb2IfKM - zPz20UTZiCXJ3O2eQCEJUt4vg2DAEgN5nKOxC8WmQZGTCSeJFjaTh04YpqhMa/q5 + R84AgHfhshOMWQulsC5ScFjH7/K59aR03G9ianchcipNqFcKI/Tgs21hSrAdQROi - /rLTKl/63OLRM1bPkQwMQwXyPyUwaVfQHjQy9bRqoai6f8L/tt6ht/xPNY/sCOT4 + WRwlqUYi/+2Q30aKCBkT2CxVboqsxgrAtBHZwjk2GW2bgRaDlb1/dxCiBApu/bgG - Sh/Qjzpb71GhmCFIHbc3UQKpygRKt4mPvRjvC/AIm7+704WGDy1Z0Mo7KUD8nvtm + Ft8qVU8C6csk2FFUKcY4xlO8hIm6KpnYoHJXmtxPctJTYmRS0zuvfV95trfooFdk - 81DO4SNsdVhF5OTsALZZOSsuUbnk+hivdP1l9sF7nPprGjhpGolAWQag6iywoLMV + z/0rHorBq0xMEQR4CiJgrNyvdVPXnlljpmjvFOPCBwbGjY5MfmnYfS7dKYdmyKTF - /orCZ5+Mqr8T6Qs4LA2EEWW1OA35CwqH4RCAZ2xQF8I+V+NCtsh3vMZnerTFRtV3 + 4yndMKCtEwC917ncUo9NZ3dwsaG+uxaK0F8gx3STP0daE1Ct8hSiydCgdNDl97E+ - /1LhnhECgYEA7r7Dh1pb4FQXErgxrJxM9oCBupOMGUQKHPPwb4AKzy3z3PcAhDCu + uJJUbzECgYEA5DpKwzgXb4xrTq/03DhzX2X4uSIh9/kxHW17BwmOQ++q+lJtM+63 - uF+CppX1HWlOtwFZ1tJzH5cUp58yVuQtVR/9DxDJkXZ93+KeggDmnU0PPye6WwlL + ND8tp9w6LiUTe9dWPFebBBrQAmsVpTEw9/rLcinFEJexTto2SjDH7dt7PXgt9dxs - vf0Mprco6BLSPxiZWIWhH6MsaaSvrfrEDGLfVcyGZHFy4uMu2qsA6ncCgYEAw4ay + ki6GoxgRq/AjL8i11gWlz5ydH3/hQnfbQx92Ddcwt9nc4Q9oiSkGWZECgYEA4Mma - Fx5PQGqvvjdY1Peow8I9tQSCVzCvpfGiROrTjPaFdI/qU46vRrnAn81KhsbgS8FO + rhAPxqY2kq7jq82HYLywXknqL52pe/GAXn65PTjVREBllKtgY+woZsXzw8L30jer - hvr3F+nNeDvAUJjWOV5WUH0hBXHeubyP6Pjh1H+LOV+7W+4I13stMjNDBAT+e2PU + jsX920uqpMunrgn7wrvrxXbjtlPFp5JqDK1f2uE3X3o5MLMxD2M4PLHgXAkFh+7u - KYNsdNtbrlT5YxmJk60BQoK+xtpScury/zjSILECgYAmTw3o5iLf+B5Lrqqp29qt + NYXJqaz/tiDniiTaf8EFnDkZbFyX15z0k7K08FcCgYEAjv7O9P3iASwz17t7abec - oykt2wcb9sL4qlvmSFFztRfwWOIIVBd1Fj5MpLtUINW0n87enZ5Db2atDupw7uQn + 4frcGfL+4YWqdkuwN7qO/pXdxLV8YnuBIiUrj+72LQ9h48gJ6gjhwXKjPcCmcTge - SJ6+kB8H7E9+YUq16ZcXnonXxHQur2sr7TLefX1e38ZEwZm5jpewD+rMeNSHwjk7 + /GCQs9jj9f91QniKZ3Wk7q0DzIHOGiufgv/Pr8RW3im5gij5dT1YpHn2IFRZaPH7 - E5JqngriiyG4LmQSSmY3OQKBgCzOLv1ROsP+LqueL0MORaQmXNGgaOXmCDo0twSn + 2VSO/SEFD0xbjk+/KaEgr2ECgYAcuAUoGes6EQBF60wxJfgW1uSdl0nxPW5q5Gbn - 8zZ4P3jIid//8HZ6loOIHa3o4Pk7IO2ZkQnvz9/fgWB2xZB757emFO0UfP9/EFNI + K7+U3873gla4ENEm0wQyZTYIm783v18OxaLyQo+RsGdC6AmfTo0H1HGxWLCXATDF - xSdW2uaY42xbjbcjSOYaDR9crZxE8hdZQH8+zTGT01o8PeSTXpiJMYKMARzIbkrC + X90wRLfjXeUyoKIy+hU0Q/GLMKfhPxh2BBrIr86XwUpzrtOvoMSLugvSeV348Rea - EJThAoGABIoKLYir150APbI1PdpIPjPAiBfrqdSF04JVYbsO/SxTrbqwcQNP1/Lx + SymszwKBgQDeESijXcYtztiM4LJZtW+lFRbuUDk96VivUNAfWgb76uonN00wzigB - sllFJhrX4ATtNJgQhjdkvWqOBohExNlg9RTUSEYNF44VND8dEwnZ6OIfDlINaS3A + xQrq4VIkmMv4EAHs4WDzwiR112BAPiY6Sg4YoCaA2VEH0Lj7vlGv92CZ8YNyjNpt - 3mK4MFcooU8DcbI8y+EwfjyZJjw4I3WobNrw2RFDfd3ZX3C1qtQ= + +zAhdRunSIgpgw0ovhCe9tko6PX8Ui4zdvd94GxQ8FWmRovm5LOBqQ== -----END RSA PRIVATE KEY----- @@ -7362,62 +8112,62 @@ vector: segmentSize: 131072 total: 3 - convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:SSK:5njnw2umlefih5k2pqevitaq7i:etegoak5g2anq2wt7zvwofeozb6p6nuyvefyqf5t6yydrbxqy34q + expected: URI:SSK:nofb42x37o3rgbt665fas5kzie:k3s6wi5ohmsqytd7lsbsagknyg3elwphviakrrsazcsb4l5qgyda format: kind: ssk params: format: sdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEowIBAAKCAQEA2vzC/BWtWIvTZQaDPqY1JxY5N+KycdH/RPC8LJ5rgTX05RdX + MIIEpAIBAAKCAQEAtxoNXh81HfkF0TLH3nkvVnHZP3E3SHFzlYTVt/+0hxjqd7H1 - ddq6lIvkA2tDMUabvu3UTq+mB+PRSH1zBZ0NMJ258oXMCgLYQMg8G9Om1ERDi++i + ML5/AzYtDjsFamK/GM9PJw1whNyqqt5lFwJvFku+qk9XxGpj1JtHgirNnGdhs6Rp - ELHDI3zXzhK5Inoszg2U9yg2I45bex/w2oF05aN9jyLO/9hlXzNp7Ruvhy+BmK2+ + KCQE3Wx7gQAI04Z+tHmqqYdZsEefsC0n35K1d0P5n3zToi/sNeW8bAXONuRvqNxP - K0aQSJWrPfPEKeObVcFC/08PohZJS4UOPTI+tox1PMrpJbA6F16YtQXXKq2KT+zC + ocROo60hKIu4NvnnZ4mAxLyNge4DEj9MAOAURqBW7ZoyEjaZ/RJ2XZvXhm3+oKd9 - lBJvk/geeAFXS0oZY5OqY1TYkSlRd38NgPN6BXBCHOX9euinUnRGZmnWacicRd7e + FyYcUeRzsSLTE6H5cDsiVi9if+juc8xhd6yA+0Kd6Nyl6FkxKo24tCGQp5s0dDdp - hhZF84aRGxAyaPlQRn7aqWLokjaziAhM3OGDPQIDAQABAoIBAAvRknBqdxWNTlZo + ZmlVWSu9EzXLL2ibYC3lKRWKTpln8NiczWf5LQIDAQABAoIBAAYvKZNrv2oHPqSN - eJLcA4hdga8LdBgCfmVpHK7HygOKNvJaRSUeLe2wcxjgJBs3tVYjnc61Wh+Y4wWn + sAV6F1i2mK1VYBYgytQae/NufgTwGP848fyW+og7vLLV2H1631RxsA00HYBHSbZi - h5qo9DpIeO2m3PE5YBR2+g+CZ8GTAZY+059VCLQUm80KY6WBtINWZlDEgc9/cl59 + s4xe5yycG1D6RA8cvslwAy7IzlABh+G+5FRYPxfRcaxuOV4XlVD4KQT3ztYu3Rxg - xdD1JarzHOapuURDmIz/yFq8oMeJ5jrTu3W9WjweWkguik8a3315X1lJvVIESZsM + sg+Rj3J7R8OUvjsknjhFzaLiYVAmQTDJDuUofboApGD7iZLwvNRBHM2LjLyjzoDU - gCGjLvCGyUG4jJp1DgC8rBxSVQ6XEq4llsWQ8W4825wvUA/1cMt4aJpwDs//rfET + v81XItqMUix/PjeDKc3u42/R/+5P6CV+j5iVqjG7DODKih+jm/lMxzKPfGZM4XZ/ - a/ISkqVQLx/4edmNeLrPXSJzfTkY4/7RybIf26FzKQVCzKYPqkffl6DyJykFUxHQ + 8DIPmoJDxZtpDutB6C2rEH6UE2oQreWG4vYLCuQkittaTzAaLabMRHfQF3E1HgtE - iCPJXiECgYEA+lgPt/FmAqM0HX9KRlSziOEoBk6MkxDNjaRfqQxh7bPBysZEr5Ea + XkoU9SsCgYEAty2tYRxDGiLviTyPfuLTgHBRxayZBVivzdHGph9aZf32oUsTlmLS - K2Fn5nYsKD22TXXZS8Ae13c+x9zJN4cOltR9DrdLVjTJtWPiZ9xHhqyEQ+g5MFNz + TiU3IcsMt4x3Pv0Rm1V+Oj3capC3QdGyL8hQ/qmDMC88+tKd1gbxTTliDKULoIQp - Q9Gp5/TmMES7wrnAZ8GeGBA943oKrzwSwC/zVDIaJ7jYS8sjG839PqkCgYEA3+9X + EvN2nrHKi40YjxaxKU5kVD0bIZu8x1VQnk24vnJRZIQu/vlck9gLd6cCgYEA/+SS - AEHHqD82rVXMtDbP6VHeAjvdAVNt9mY3L+kjEyYhwIBhuRwHlbXXqEIAvWgkq90G + uF7U5OA6ZdRJL2KSGkm+WGxaFoJZbY8z0YeqjU0YbChnpV7aSNM1UvytzU48jO+p - Jw50EJunU8fPOxsMoIeeYiSSEWAHdTo8PQWMRP9nJe4QMGHFkBE/6hLMjpRb4SsA + artSjltckOw2QsQFOnyhgkxPyy0w+ykzyMt8Kacg8ZVkJxFmXf1Z0KsDWFurplMV - +ApYo+niqfrIbqFVa7cpDiKeOUXmeANGeHjQ4HUCgYEA9x2+RmCvxaK8avGfq9Uo + MzTf8eE0ZvjP+2te2uYr6Vmoa0OdNfoJ/mHCIwsCgYEAjzzBmfFOq25sHsVjdBYM - c9Ft5Ovcr79CaLL9Cq4CbNWoUjVsz7F4F6JLIZ8872wbbFMMcE3xI9e9zSQQLBPR + yx+JYejAU4TxHCGQk7BqsNxxcdjSPUOTLhY90UgE7raBPJkJnoywwvxCknYNRwOh - Pun5mHEumKX7BmbWspcqs7HPzgiJiz6U5Tktcp64KqVugkVBvCnPmQlPTiDGMzwl + sWmTpD+LXS9jIMN3NriBEiDwAfFBcUhHEhGdTSS7vHodnS5iZGlvXMvXnmU4riqR - djjfBRl/3/4C5K5ctbGcbiECgYAqJrMJqVgbo0p3dh8CDQ81q+NOKFaBWWLpbnQU + euhNsWaVLOOMGEeH0/gZp3sCgYEAq0X5MjA+/KZcT/XjujSWp8O+BH8ZWUGLy7ny - 4J1pjVPtGD1Myqni1EeztDjPbjr43rG5yE6wkZv9eS7YwU6vKNf3QUr9WkYNGtkb + rAbLD+KPOy1cGiK/pcjAQzheuDDqdEahNZAFtMTP0yxXMR70hO4QSB79tXcc9q7g - 419z3V9dFGKXuM+nPpf5R3CZpfNlfuK/zbLBp9SyijIQIO4jSGbB8mI2BaJMFNG+ + O0B/bX2wniIos8GAq948NF+SUJyi6iNn6Cs2zTW4FkfpJVX7WjZ/I6PgB1NtMUiX - +37VwQKBgAkDhkwYoPe4SSLDuY0IGSGIKpcwoWhjTf8XqYqn+WnPzY7PPLBvY1D5 + Uc1q3HsCgYBdqsVcc7xtNsoa6VLWs0UT4ddnesOGNPh2OE+wD7GAZubT8A6Ndktw - xmQBviFK4VeTqzTbKnCRMa0y7ghXuNfQHakojNa+Z38QeGviqaQXEzH/FeV4oBgg + 8P17MAMpFjhW9OunA9Vnat41iHdqgZtyKUjuW87O+1pRp4l9MvKal7Pf6pfQJ3c0 - DegYr/wrCofy9kqDQswSPlNb66eGLe+JIangrozz1vBf7WOML2m/ + FyhWqljYymTn/LFep1x8OzMC/YUxclpLu4gUbHu5bfqdd2mZsxW+Eg== -----END RSA PRIVATE KEY----- @@ -7431,62 +8181,62 @@ vector: segmentSize: 131072 total: 3 - convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:MDMF:d2flo3xpwvfdgqmaibnsktn6om:7z5zabf3g3qktv2k3go7eli3nwmb2ocu6dweelykbgie4aenjcoa + expected: URI:MDMF:h4khks5ufaa6ka5lyr4nz4hetu:g3mjwxhi6jswmwppy3upio73yth3jfx6toguraonmjysc2boat6q format: kind: ssk params: format: mdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEowIBAAKCAQEAsYUiF6mG9Q85KsOd8TlnbDuYrK12pWx5GRU3HGDyNLkQs0Q0 + MIIEpQIBAAKCAQEAwSglyEQntPe1VGZwA8UR04afb3YU14LM+yUtPmF86CFqtkYF - wfXHNn+EBzkiPI/S1jJIgdvytBKua3DR0vpUBnAZ5Cl3hqz6yp1emGLZEDuhxaGx + jERO70XwmKJDI9qPskII5G02KMr3L6L8fXvBAgq6YBdJlqNGpxnS6YlzrPIgxLJE - t6b3KKBGsCgKVueOFU7pv4IgkZlNqmt50MENAizbR2C8iaoaJWbdPE3YACv7/OXB + 2dcxoezU+NQTA/3YSCGukmqBqrLyHOtzlpyU0zo9W3crsUbdM9yCfy8N7+98vLiB - H1fj9trQTEa0ueL6JjR4CdO4pLy2LOFDvpCDQMl7EhwsWqtOeWKcmiTEV8Kouxn3 + tGAtjDDMsg7rU+L5nKJRwNSV/eEw85lcmj+6Xj/7JZOLisTI4MAspt3yG3YmapQi - AhWD8M/pfmrFyT3uGneX3Q0KQQsDwvEAJ+aR16AYpryreqGR+OnhdEz1BKesizsy + xpDif7Hgz6d/zKnkys/BAw2e7/iFJyK9hbPDtobtJkUK8UAs0d4fbcrQZKg9CsM8 - qR1UJ41JqIwf+dL6ueZmMQaMbIUVOQhEceLjpwIDAQABAoIBAEikVBketC0ft6L6 + VDgp8bR9oBqfhbDHQfNtZV+82M3qewbgL0BWrQIDAQABAoIBACB0QfDlvrQx4KZH - PW1yshGmKYmvyfdTdhJ/jfe87CALAvx4kqY0LvrsH1jdVlc1+27PUMBjAuQRKPKq + Ne/0NzwOxRAhy3uwbweNpg3yrF2Ga9snZbw9J/QdEMFcliJakUVWwg67aNuuypyW - ThJpgWzI/q9REKo5qr6yuvzcpjpwTHiU/CZM2qLzQzneiKybQJcTna9SToWGGDP+ + 6oyc8/+HVOxbTVKBqZffB2iU3zpCTo4uE9J0TVMTK2+Jlo5XovTvr9jLC3Fmcra4 - mvCDrxEOzgRdX4lt5BkeCLYenJ8ksTytV8BavWwu/5CtzrAXA192tKdeW7iabA0s + Ouol8f2RrgiFu/Ij4Xvaw5RiEBntnRJxUlvU1ik40uDOc0usbkqHTMS3drBSDU8a - jOXxopJr7jBYq7Cx5xsbOxBGwpxLYP5BhVRfAGwxp18hKyXcElwRmmCrZ+cCI4+X + 09317Sta5iS1EDcDDAg2qOwXScy5pzVAhUapN/2HSvckbKJkjL1KuaAq3Jk5y/Pm - TZSe/HSTOKTBG9laXRcJBjEOdLrFq6Sa74ilTdFm+PC6WuCIzcqWuIZE3hW+jVbb + FaHls3slBPsrfeT/MW61PcSgraz3xk/hw1ajeB+swxqQfkhwgAFe3RxJbOwi9uva - 4YOo4nECgYEA60+jL+VYDA65Ig5DSJiG4WDkgQNrq1vW914/QY92cvk9PFeRRoGm + gWdoKRMCgYEA3mDRx3GqV7B1oml6tuPoYwONKmXSIDh4HLDX/qxHtBLHIInNp7vc - d3ef+MRaiStC9OTr9K4Zr4oYI8kPtlMgHjxvvyg990OBO0xifFeHVuJqt69uUgkX + T5ntId4EXvz3DY1PcRqpsMpGLpkMdoeqzE0eS8a5BAK1I15uSZA1sQAjWfu9o72l - /xykZtYO94TF4qPWZ4psMmuqnsOk3z/83mGpTwHqOatr7bzYPCBb0HMCgYEAwSC9 + +wjjaqAuBTm2k8E0XJ1osJyvGG3q0zb9lvBmhblV8d+Xst2iEATfXZsCgYEA3lxP - awIJfJ7pOqBSgY1YUl2f9TXqecYA8dfPaGVf0c+g27KXnM8iINzkIAfyB147kjg/ + g0lADCRcCrt8/SlSVxuECrORryRCKISjYInOzVdcsEJO6vqHFcdrJ0UJv3oPvR3b - p1t7yyFgUUyTVwa0ezHJPZ9Gi1cYxiy4lOu5lLFcH4oHHNeZb0Ke5H/rHYrupwo2 + 185qXrBJrxCM7frpPTQ0rIjXYgwprTtyVW8nxCLpn1IWNH89MNiHSj9Ag3pTroa2 - HO5/IFVf8Zfp1X+qyeTEdXff7teBetNn3zzUFv0CgYA1NhkM56v1bg7naJpGfFdj + Gbig19hgl6YutX2b00sCT1hI9Db5Thaxs4mChVcCgYEAg2u+rlrLa9VaP+iMYEei - 9+k0U3Wxll8SKTnctXhvn3T9hD/R1dezBFYkhyKCCkpl3q6M8iHU1EGJNhpbfIiy + j9mKdNMF8orM2U/d5qFUIuSyD9XA128bjWOPk+NMvAJN0xF/MH2saVGxVlqW1fnp - za/nZk488AL1SdyriY+NUj4Xs5Aa9Pt8MRnsN1PDHT8ydSIy39Z/wGEg7dUGtw2T + g6HT1L6VmvwqpsNo9EqooHlPax9ufLVYwVoIZHxTljz8XKfi1RUlyLJgfFSBYd/u - rDoBJ8mzqNQLOr0bO6YHiQKBgDrRtd75Z9pEq9PnMDm0ysmLKkSMfzVHUNJXYBvz + 0GQ0grT8SNx2H3wCCeuHQh8CgYEAuZoVqIEI29mxlie/AVVvbFQEWCZg0O8T5dwo - hBNqoRtIcVSY4VQQ8omu4c/Mq2gFKZ3XBwT+zU71e4ptyFoc96WE9P9LL4hr5mu0 + vtjobE+ih2EhnFN3U/97enDO3SuWXYXBzhV2hgjhyCWpbK8F5ldgLC+gkC+UzgsT - v3jB68TPTQtDvr9cEviU3Q7KWZUWTxTQrncyiV4TXmxfzaxfuFXuhI1BpXW7HU+o + uSop6DY4CQssi681NUNXUesP/26o0MGS2E9aui/bGFnXHRh2a9xtVitb4bTNTZf1 - PxAhAoGBAI+2VYkif8AfGx7km9UfpG+21Hn3I8igw63doyWVnOeCfVXfFu1bXVpi + xeVes8sCgYEAsILzo7kk+SgV6ZqyubbN52t/q/0xyZoNFRhlNtlJyhbCE7YwUY8B - gj4yJ/wR+FR8qTpTn32UIHFTvcxsuZc8Omtdnr7xBswWoTvY93fzzP1vQ1FsoSEf + qmbXrbuJeiuDgopMZ2OHUWh4aLMpBxbYGdAlCz0vPS7a8XamQXizQSD6GZK6+AFX - +/zZiVJT15jFe3DBSnPIpGJ13INc06mJI4+BPyaoamt14GamfYCb + tcrIMtrx++XsrD6RkikqaQmgPq320rMJDFETZOzxXMq5Hr9NgKL9Nyo= -----END RSA PRIVATE KEY----- @@ -7512,62 +8262,62 @@ vector: segmentSize: 131072 total: 3 - convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:SSK:n3oliuimgv2di77yhho2sqq4wm:omp42y6qt5ve7nxoipcwo3fe7qbllkmzr7kzayaoyygx7pf5m3tq + expected: URI:SSK:smbhexz3nnaz27sp4ycnn53qxq:v63l2vvp3z2brgul3vhlbudiybaoxo6fa6qe7n6cqrmbkbtq3pjq format: kind: ssk params: format: sdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEogIBAAKCAQEArjAAPTxIkfQW28UHNQ4YJKyBCcOHmR9S0eZd8zStGJzK+3bu + MIIEpAIBAAKCAQEAnluBaJX/nthl3VTvSPm6f5LZ7ZoP1pXr8dmUZYn8cyyAmnLR - fkjxkLp2Gmc1vUGd55lwpBPmKLA7g9m2yUAAF2+hhPk/jrOc/3TvcDo+E1fv5pVY + 5TLc61DWPVjD9A5fiBeF8MTgU7h/RZeb02MaWbXtriqarpvvPqOgsb6P5J8viZ6J - MeDD3Ls5EN4kfQGhbipUgfNBs0JFO6yONn7xaYnFCWYk9qiRguw+Y4egFNFqoP/o + YocarZ2ZJoBdR5kJ1sNbSf5YAABEZX8KbZWU+vhGyx+rRpjcsM+31ZLxSEg4kmX6 - +pfFRztn0ROtFvWezu6zNiltHakI+j5yidF8i+kBlctZxmzFBra3diV6tYbtTlBf + W1YyVBfwtkYLhRwaC9s53ZuJUfQa4H5tynbgrYwh21vxw2C8NQgwPLh9mNkCAJar - utEcVw1h6wAUITGIec6N9bmY0ObT2nYJAC5e92/TV8m66+bO9eHPDEYMypE83XDT + x++YD4ipkEidwlHGSaDQ/miYN/fXNbwOBVk/VOsPEyVrPLQa/dfsguOZV5ugIOJK - rFhPJ/MIVthxGMeOAzcmKGdWIuF2f0y4taEX5wIDAQABAoIBABAAZ904PwBteHYa + fu9bdAQQtstfGeryekVWvvq6fi3bmetXwjiYuQIDAQABAoIBABZ5XBtQtm9/vK01 - /QUCNPSVhksj076c0opmy8WuVqJ2sOz16YXfZJWjk3rsdVLcBsoCXgcsrs2ZFvaP + waP0tTAn0j/zTm4g5tRzEal7dNWPqkzBIOLLXikTVuRr9ZtscshotjyeZEvdckqZ - VwvY3clJX4CsNwsAdBFBqEdail5Tiz3XBWGboNKTvnPOvHJhZneM2vOPKb9yfJK1 + IqdUeEflFu9R4pQHU2PrawHuzpMeuGtqkYrnK7UaGcMqEpL3uDq/jPQqYajWYN6a - UOEvuzSzS88Hu6iPJsLsufSBvpJ63bDKoS8TQGUsWwYQvuAAJpMFRe0uKwrCrqCb + sgstYHBhzgJD41XomeGKCUgJS677P8X3+g+i0MxHx8fgnVLxURm6QCXeLFzkgqIq - GYyPXazKd0Kqv9WIU5ljLZdm8QdfA7YU/2icBN5hoaAxvjTnd7Ojyqs8qqtYbFvC + yt92rNkTNa9wyDdYcvAcx4b5RLOxXrkkklOVc/oJlteM/X07itbuYDZBQDOYpZPP - 2yyrgLy0pcQR51tTxQRv/nP3Xgtq4rJv7Zgqlgo/g1TzYCslQcNGTzdCHmEfru7L + EkTckkBSbcoPy6nGfcHDintuIWxKe3jbWRcRl1uCN9JNx9rn89DQzEAfeFMorROs - LhHX2RkCgYEA1GYXozyWDWt0SIvOeaR3Ebi2BL5gIf/zbK45/ZEO/C1H6Hp9R8ba + 2DSDI7UCgYEAz/eEfIZ7EJ8VZ4CV+4oK6s6+UEKuoTW4kN3tk4BoBuUD/kaeAkMv - X0V3fFYbXNa/f6+mSU3IQNrxlfFf/LtD6/bSqE3a7M+kyG8C+A+cF4MsSTgyu8jQ + q943rKeDtcOYOS2FxSu8XKX8s7Ad6IJV/l7RMtjRy0ihTsklGUnQYx9ptkBvWYnR - wmEeqHry5KOjoYIBJdfRN91XxHe8+tuEEvYABU7PKU4LdUH2je9EuU8CgYEA0fHX + M6SwLfOS4etG/+4VomUx/YGAsffZjInQbAqiZKhVeX7qFr7YSiBfkTsCgYEAwu65 - ipNcSj3XvVjh6oM5tLAtM/c9eUE5/AAV+BnS4imDm3i5WgnwtE3VCOH6aHB/en/p + vwTOWT9pazDHER8UC7tM6uxOCEGkHZMcGFlDftLaB30dEVL6pCeXW1J3XrFVFe6x - nW1AEoL/Pk0fJkY+CTv5OfQC53MFH8IkNocMCwW8APuRJy6waIvKMU5XDA34mAFp + IdZdPuTyywtpWfJrlMPGsjHE+R1x3AB1NaLEeVGIjh+Y0Zt8nDNwdK+4u8Kl8B5a - xGevwQBXNMyaBY6Gbzq1zMhGJUCVHWPFaKnq4ekCgYAZ8K4KXafl063L/mclLBTu + p22ySmcPpoOPFv75q1MTyrs/C5VaRuyU8SKYXpsCgYEAkGZbzp8N2jerhAdrnJF4 - sSRpx+ZtwJi2OUET2td9rPoPRoZucbbR0+YX5VxKJmAU9BrW8Qz3/sVqjqQudaCB + DRvqVw5F9Ne5RJVj/bP+BzODN05PLmD6O8r7O13A/TdHfgQWyxYYHvh940JZMfU1 - /Q8VRwzpxyJU6FnwedeSd469EoP/szLryni4Euv/SIz/eKUzPfxrWjkR4Z3O9WhX + wn6RoU2dNhpDLtJJeSqgkALiwtIwvqoL4WDrl6x1g3p6/P+SdATx1gTSmD/xBT03 - +HtgKpPac5GqrHe0NfiquQKBgEWSlVEQ4Gah89qFl+g1MGxWbcRozHBgUyzVgnJD + w50KrvuXBdpSreJrieS6lrsCgYBJSTUeIrFtjlCU0xbUUgnYS0ekvsirg/ougEM8 - bIUSKNDewt25qZC2skBNUsRFc5lOxkYrLC52RsuIlygB4xEAVOkFmejFTw9lMMb5 + yDp+8Mi1rg0CmV7P3m6iD8P/Hs5tW3rOzOfroGnDenvWLDTUDjKiheGXAsHuw2FN - Hd6ROepBc6q+aCtdF9YbFfGit5z36urxSWb2C/AtVWU+BALcO97vB3/U1RV2OLck + k+8n6UZcoHZ0v28+znwF8paSSKDYQKE2dyBjppGUubtPGvdEuQwk2Pbf5Pu21HU+ - h/fxAoGAK10WdXnNYJ0v2+znHY+g2cV+U9/1ZO1hBIwgJKEdz/v05IfSEWEwhgq/ + nxIH5wKBgQCJbSa6TDgwwZmqOCfMHkBLXH+up9o30VcREGHu8uZyybq5q3va03LB - ClmJe3LTaJxw/J0iwRqIZTtNUPJRn9xLYEouv9wEdzw9jczPZ8u6pg5Il8OJu+sc + Ket2oM88ot04aF0wAOIJqo35FB51Jxgi9GNq1LE1U0TlB0o12dz+yjFGWy0A/o1Q - +JXvMICkAIGi1jeVVQF5lAeHfmmRZYQJCUAmU4N32StfQiN6/uU= + aCBedc455WFm1R/BeP5bjaKLM8suSiVUIrHGTTKhtobITgvrmIZOLw== -----END RSA PRIVATE KEY----- @@ -7581,62 +8331,62 @@ vector: segmentSize: 131072 total: 3 - convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:MDMF:4yx6wxx2gw3pw5i2dexq6u4mtm:rsvgkzh2bxrmeuppl273ps4qgqfnxokff5fhpw5ekuqpsk7fzdrq + expected: URI:MDMF:cplunzl5zzvjzx3hurccgxjaya:ew2dgbe6r2ivhbdn5nmueb3tsr7dziypwpxzqmzowwexv73n4aca format: kind: ssk params: format: mdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEowIBAAKCAQEAlCwlHOJnRd6uaZhsH0MGDhZxoa4G4+PyQHE+/YA1B/LbhbBu + MIIEpAIBAAKCAQEA5rZ0EoSYGWTYYnRw8LBLb7XKPgyB8kENRSRc8C4zkQjGlxay - FmI010z+XAbGzzhXBIdNaJ1TtBx7HXveH2NzudXwlNNQbUlTAQdFaSkWCAURXox9 + rBEdxXypt5KoQeBXEyfwoALfuuMA7lP11Tn6TnO00AXvMEuvqwspkdIptBddnkGG - be1RrzNPHmadiTkkRh0WUPmkqjxiKKo6GdVHbrFjUPADuzQYG+1fFwG7trAPueaX + 0hfW6liNrH/cVbwOkz3cdxhf7jL3hTTw4DeJdGepV/ey2K1qHvbd4ckgwjwj2hy2 - zFGM4hdD6PMx58W33rntyx4MF30QiJ1FdQh/hcTxTlkyK6duVLTSV0++BxoSpg5p + 6PWzVVIXJe1m7crNmB6DG374rYlRgh6QnXBP09zOi7eo1nxGVPbu/jfIwpRGC6W5 - P1NrrojIR5QVXquBYF+KkKUDrs95xgwyyjfTtpYkjzrNmGaUOv+UYVe+cqgxLKuG + tZCr+ShQenXofqbZ9po547ErafUaeRsiWMSMyzIT0uLvoKv3RAS93bkfqM1USLR5 - +6t8eZAPDPS6PhlDDuxCJAJnnC6HIbpLupJ8uwIDAQABAoIBAEe713vUYAsDc4zL + 7f4Bp2yQcNjlUUnarZ4M+Q+PWTQOZLeX9mLrnQIDAQABAoIBAEdxj58eZVVTx7gx - rgy0dgn786dCiTNq960bJlOz7fibKovejm1nvg09ySbkYPuRWw9mMaOkBxH7d98e + U7oM9cdJla/CQslIgLn9CTStMfXDMHAgLMMg58W8lXfN2AHSXVSGxTpfuXWPjz2+ - SLsJes1NNdvXMei2xuiIjKIMsg3P5kjP2ymM6y7WuEcPhtUYROdszZEGSyHfeeYW + TT2y3wLFTOQwOkIL5gHDCqPn31cv9yMnKn9Lt5dJRdH5pDr+acsJ2IgeybIjIUgk - A2reRmbgmiRlDmljHwjmMlMBE8+tUCMvCZ/dDbA12esaiTqgQgJDnBq1xNBlyOHY + PUVJnWypHyUpBL6ZcOfWzZ36IQVUcME0PMjykTav39K/uHXWF7Zk1wy8FFdtsh+7 - CItjYwI9+6SetN36csBYjHwfRPw/H80B1Alx9yHer4CUhypn9QfQiFG/ROpbawRC + gpq6hsKzZG68vbwySml1V+0utsiNArOTiudzUEY3NHrrjAYgKgFxX2C8tSUzWTEd - 4YRz1Z3dQiOofBxUf5xix+f69tF28Cl4p41mqFyNkx4iy6PScedV0ltl6yGXJStg + Z2Yl/AMnfL2/jtQjjWoNGaG3WZdkPIBfrgvF45uy0QJ/Rm3K25E+EyQNkz2CdFi3 - k8y9OAECgYEAzF7o+qce1cCE2UQeZz7opq7x26XxzX15sQU77XH0f87PiRG4iqvW + vMKui0cCgYEA7qtx0wbCw7wPML86BIp2y2tIo8kmGeKn2NiCsCZl1bW4lkfuRmdN - ctq+Esv0LDhbfWCQY/BB6xw26STCt9IRcHXik+uVk5L9TKVxPIPaugrIp964ssWW + BYTHqTUaB4PaHijsXrLcxuGqjGQ7/OUqwmEz2BOZZT0ehFAkkSeA/418ZoqwbH7K - KScnersk/2ObG6TepuEY4WBsJAPiGgR5ew0ApLRsENBrgqYJrQC89PsCgYEAuZrF + AylmDTtZ9JF0hXDVF7Wwwx3RG4O0OYROie6Y8RXvdKE0DFkcoiywzN8CgYEA93cZ - e3WzcfUBOr22CBF8JGbcWjHwN9a7hSXoCZ9Rl1vO0vq0VGMpiYaGM2+i0oWDW/vD + Q9m6+OrbA+KsThqC/vimTGPPWUVATF/yPgwSBT5pwq+JEjIhmmpaVTMs1E7h3/ob - uZEeJiwoKYCCjbYMlHfOCRZYtiF/o8U4rB0jcQ5o65OiQHsm8jw7eHLsZeGggAhP + mHegt+g51fJwc9M0h6DcjGJ0ek7Qlv/TrQtHCzljqQporuUgqIfaU+82Mxxemwdg - YnF4OTduJteUrVbx10qKMoiQvDSzFYDvwVh8i0ECgYADHxv94BmXeDZPPzwbpZlW + JqfwT21KCRBvMNRtQwILRCM/S7SEGzVmzobrGwMCgYEAtkJIfu1XyF+RfhlZ9ePD - Gmv1R+aWlekK7CKLMOdkIFuJI20nKRLAdFjc3qKfHkk/c/8gl6XaGnc4Pmh++EVt + Sh5Yb3MJXJUgtlDIpDn+ZFAMcP7nL+5s+/zk+AtsIDcJVyTLNJoETQBB3EojIUHk - 608HpVyGgYM+7XP6UaVAnDOOZNd7W4s8m619sWgSQoo29OC1udBweNGOB0Un0pOs + AGJ4U9bLumsNJd+JvStcsErcp/XbOk8sd3Oi0hHz5Pc68zgyEpQWMzpO2GMgOxgV - bnlpCpxv8U8DEtgo/U7liwKBgH2f2iaUJd7t2+UsXrbbTtE8pcyOnG7O8qFOZN2O + XfHN67Vjkj4UYCWg3xufvCMCgYA7nmDi3NjT0VkUlY6nfnGi1erSqpUwz6NPAyqM - biUqSLTYZ5HuhEDHQrIxz1z6bUym/XTuWh+wJ4bfqn3MSHt9E4FnFKhByCjK5m7o + UkIhK0k1ky61yIgZ+JdswViCicKXQF1XnTKGPBd6+N6ouPCF4HZiB/JB6S0Nw/KO - UgLFpBI/HMTUFipCxmXiM0tKCd5ewYx6DMt9TxsPM1yXypzToPJPKNeaO9RELwMI + VRI3nQrqlcxknmUA1UH/SLlJFQOh2+QJTBp0OENG7cOsAvGT3DE0qD0+ku3k1DfB - p1OBAoGBALrcFbLv0zzWjcrLZj34eDB+LMVot2rISlJtr6X961ZhoOWOfZI+B0+M + d/W6WwKBgQDS8pkBm+pZ0EbZvpYTy804jUtErk2Dp5M6m+KEFE9NF06ncb8voRnQ - LLmBSpPooKYVqpZIv2kWama4YCrcry8vgG7MkB5n+jkiGmNwFrNwJaV+OiLARVU6 + sYg4P6NuweJnNlQSUIneNwQ3liuZTdLz3WDt99+DLlcToB6+jL6ppXs1h0dYpkHW - LcEtObxXUb+S+WkSEhylyQ38zXDR+LgGWjPzKdU3duCjL+/G//O/ + WEQ2KDzRt+BEcdPe/sbkF80jhkZRPos5gMmy5rgcDiQ83aA4RYPoNA== -----END RSA PRIVATE KEY----- @@ -7662,62 +8412,62 @@ vector: segmentSize: 131072 total: 3 - convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:SSK:7awpweikdkxbw7x7i6emo5d5vm:iij24bgytahmjjidcerztczj7pludytjfgd3quyfwzvindfion3q + expected: URI:SSK:lbjianktlydwc7qn5oco6sp2xi:mqnj4jq72xxuwrff2f4kyuv5xvu5yzzkhlgxumm44n3uwvxeelsq format: kind: ssk params: format: sdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEowIBAAKCAQEAvrLo/Z177W8famI+g8Avpb3jmP3yEdB+gNhoddoUFPxsV5Qu + MIIEogIBAAKCAQEAuLx+Rysw67iIOHq3IvRgaYAn1OT6xugUvhXPc5gvPAS599bS - C1JLzjMMHlZFGEunom7EijiTzpKLyknkWiXGdXWfQ+8Q/o0aIoGZHVp5j6XhHbWW + rhPVP71CazMQHzCPTx8KcDskNnNb08xoeae5DfNVCs/E7PBtgIy08Xrdv/HEZ4xm - mRZcLXbtc3ZT0a6W1q6niBKiXYBn0Q7nsHwf9EpPWn0pMa6Uy0nKnD7d1kqz08sE + ksJBLydt2jGvvFTbCHmNuywWguHHtUig1IrYZVCYMWUABGkiZ/j4UqL377jjY1BI - JXlQGJUUXlEqVZW8G8D6jS1alFpRm+lS3LYsUoHREPgRMzmJp2T8m9JUm/fwN2LJ + Fb75Fud/3hiASG07BdL8hzDTLqmhdXDwQAeMklhaB4Qw1A+Ubb1it/SoOI2CUzhZ - 3rdmkbj2XjEbbwz/412b0WlSfk04F9OVNBcZzKAere2wIGqTgcW+TK1n80yab+ik + /fqAJcQe3YGbVtJm58VWy4dw/AzcGitya7MxXHXrjitWqnj+CVM0myf3vbvaX+8y - vrbybiuGOWQXFNOim4LNuoj/kbAZhybTKjdedwIDAQABAoIBAAs1OrDFOiZltJDG + YQlv5pATfhJbCmh0vj03oiaSZcj9IS2fooV9UwIDAQABAoIBABCL4YuiVLloR9s3 - sdC06yTKBbMIbqf4LnP/xBjBHRMKrvEKxxYCDtO33lhEgihMy5wMkV9j6eDNoVeq + MpwQ42nPrsGk2MlkFCeKcJBb+y8XBUkrlqc844bX/tD3O+RvRwbBMwAma/Hslzb7 - gut/DHkubsWwXe0hrLQm4WSydf71i3osIdJhJQ1eNjoZOneFbZCA9vaHzQrC/dcL + QghTe4HCX8WeIndOeaBf+fz/EkmU8BCORMm0WH5Ou8olVSY7O3sg2A8BvepvKqIU - EdHAMD4yxXn9BzHIH/vYT22VdRy5PYmVQ0b8O9YEQhC6AtQpms6T5jKPNAuRSRlD + JUOkRAmfFGKoNz4t5IUHicZtDmQMNmrsb/jiEslKqcroE+7R4bQHAQtdLCwPgQSW - xjqvRES1k3afAZjBIddGrx8ie6C2GnJl/C7tKsVPQVrGVh0wEIZVzuBNLtWLwFp1 + AHh+Ft1wh8lGMHXi1EIiahVqTF7EifNJ3Aoy/O9LybXEMYwy+fDN9WtCNa7Xl5Ab - zxZ08RGwtT181e1Urx7dWrWjbJi2KHnbPgHIaRvTdvnBaiUX3p7MJfnquyNFt3mU + K8QXgtTLg2kcAWfsBy5aM9vghN19bDV8OPgXYhFqK9AbFz7nGBO9VhLwVHU0NbpD - WdfoKhECgYEA9buO2fFcjBMPswktFUnThHJjKOfsUuHFWbnfmUORTS36OrFHOhso + VSm88sECgYEA3a4QFEE+2qn2Z30ar2HMrEAh6/w3KSi2SKMfLvnPl2M0B/YqLaTi - mmG6dSk4rdxnOfJ/mu3a7EElicM2gnhluN/D816DD+80BLCZmce/4zLab1ZU+dW4 + YddrdlDcKJGUEx2GKNmJFUqzFX++BD5RE1olNjw6RW6D4RDpCwoxE089WN6gFiCA - UWXZBk1/ElO15dsGCSQe54nHMGH7Pfg5nDfpd1MVEw7Bwakf/YNtZ38CgYEAxqqx + wdG5RXflHhF4HUfz3lrewelbP6VM1q6Rs2tA0+Zn+KTjPSIGTl9mAdECgYEA1VY6 - DhLOq8JqDwbKOlwVOyU4BEk3f8HcDVcgbBzF7THHf96rVf//iPxjvysjtamwv+ml + O3K4p2TETW3n+T7+CcLITnSMp8n0hWOAX5ASetShbq99ACDQ0mzcE0/KWyV+2sNx - S3WJbuQzVB5efBIXLer5RSWL6k01YzctOYxEbDGgiXxHXG/STZITnUC3pL6aY2Pe + JxvwBBgtsGsGsdsqGNyRHXQw9LsY4ZSF1Toda+236OwaMDs7WVVIuFutlD4tKTX3 - MMU/kzDdiT39A0Sw2plh6Ns8jpmNHLj9+vJGxQkCgYA/y198qTJzkwdCXaF8o1vs + u3UDEsjGWXXohZSE5ajtIjd/IOHtCT5YGaeDEeMCgYArlW5Z3R4TdbkZTbJyauMH - SJ4BoqQxqDdJ4f1wlqAEP2l1D00EgsR5v+FeRUNXr56E5rXGDPYG26rZJvrhyEvw + trA0qmjZ8cQs8c1OuhTDaeCv9AkE4lcT73uUTn+Khly7iWF4JJTcF8yv3GaqhOoB - QPdoGSNBYcJJbWeTCs6AN1WKDgmlipx9VUmQX1Ib+euBLulUOjJjvdsebnGBVw3t + yQZp7Ft0jS7mkCGRZxaQ+lJQZ6zHzOojsS0g6Fqml76q2xuqSuli7JNhJwm9Z6MD - xn4v4jvYZL5cfoG1mQcwFQKBgQCkEqKJWfT/m1+WK2hmzFfocfOSbpl8VLGU/ujT + yIF9Z95nN1vqCAd/Xyg6EQKBgG1wsqbUj02wL9PY0evXGNNBDSjSOWXKAJp9FNnx - AOxh2aPGsjJUo0j6bF9AqbMjPBKyXJdb+6VWRPczOKWV2Cb2kEHv3nNwPPWjjBU4 + OsmwUrBJbkKmkvmfxrZRdGmVrqHjKST6/AHdtXKPNPwAhnQCkp8dgA/L+1OdsZpV - muSDanUINvCEogFQeRzj2WgRkizVeswtASphOJEt4FkOEvPwhY58Dlwz9RK6rvlr + GcrIRFRE6ppbiHKngYqx3TXzP4+ok9GikVUNklNKXWJJcnOuWRf4iEsCG2tmhCOE - AB58aQKBgAd3SjZ60ek1q3DoQtCZKDKQq2q7chXMZx2OrOEQOR+DB5XEks5IwCvp + /4QpAoGAEfdRRaGKv4w0qhxS+RdEaTrDYzEW3xnB56ky+E8UJEy+YgbGlt9Mz1aT - pIgb1CCgYdcxtesJBO5eDIsoIbYOSxBPMnsTFfsOqAXDaGNu9pIkcEmmGDjehwlh + Kr8SBUp84Qx6EDa+BA60VnkL51E575uNY1/xICufxwM1WTWKnplLHmExSzzYv8me - W8kSZXq8nnR3zMzq71IOZzYIt3LUfLql79jycoOZ0CRMZKMxbhry + hm/J9LA/rmqRTZFd1bqk+M7qAO6bDzKD7WNhAt+kbdaFOlkO+9k= -----END RSA PRIVATE KEY----- @@ -7731,62 +8481,62 @@ vector: segmentSize: 131072 total: 3 - convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:MDMF:rbwwosqh5a53jcddvkl4jlchey:xc7yci2psgn6hygl6k2qc3wjaglpqxqy4gk6ij4vo47z4acmqlwq + expected: URI:MDMF:khtxspsd2whqf6n3s52i7osu4i:w4z3xljy7asjjjev5zicdzf6xu6xvvr3vkyghu6w7bjitsthx3pq format: kind: ssk params: format: mdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEpAIBAAKCAQEAv11NTvTJICddHij3GyfbhtkFH/cFZkPuqF7XDcjha6myDbHX + MIIEogIBAAKCAQEAtlZxFsekdTXUZFqXRPIXWmfAv4LevgoECO6qfY6ZN4XxMx5J - IbH2+jS1aubjf3hHKvKx0fl3CvdJMJUsz94Qrn6XsQdy8hu/lVpGTUCdyaw8bIwX + ET5EOkkqBE86YqD2nZukuSzaY1I1PA4fHkwiVbdXw8CCEah7foRaxaiW/a3D6AOO - c5o0sSd8d+yRgdG1Y4vz2optCHm1sKKMzxDnNODM2KX22Hlml7fbdp8q18ByLzh6 + L2JydZls+CYbimMxmbGFQF449dRtEMzQ5wSmPw+bfHulD6zHYLWgV94qSUWR5pry - ZgjowVxL8coyDBldpvkw4r9m6mXJHDIi3MLOqvtdcAUIKZ4q0dCmtoVrZpHJl/YZ + LpVtH1owGid+xQqJncxt1G/WYoIPDfEwHzLhTlwHYCEGLDEHk8UPNC3m0N8Skevc - bHhMa+eBqpSxjpS7uNZ8szszEAK+1aG/fKQPi4z/SXdUy5v1beYEInDFpvXmjDEb + MPCPk+/68L7m79SBN806eeH1jkuMIOMYMPX8+5PzT1Z2wE7IgvjzXQ5bNL8fAk+t - xDnvKhJhBXmaQsSOzpUSVA/u0OcGxwLHiRSfTQIDAQABAoIBABxMRIiat5wu3fz+ + h2wvcClECFXTup9qWyFgu430AOX0eu8ITV1B/QIDAQABAoIBAB4YAT05wQ5pTn8r - A06LdhHKiVC5A14kSQgyYBxMepMkaK1QQVcc/T/yJ+qrQnSA2YtPEL8TZAhl2Xea + pnjGHg2ZPyo8jse9vnG89l2XqfkMfcUqk/OpG7ik967TZrb9iwZzOEopuXeYC1o4 - 86G4faCEHVPjHVsSgeHo09EML1kZhGTr0XL5mHWi+Gu2eqznESrrkO+d/TId52F7 + mHE3LpmIE4+m17DTZmJ4tMSXsSf4RHOoFpEChhKbumzwWS5LddXAg1Ye1vbX0xJp - pBVhs0L1RC18W1SXHTXtzQENV/VH4RWwi/Aa7wFsnBD5lv9bxqPp3D5Z98xzB0jz + Q3dFgKy6xjZS7+i44wU2pNqru73w2ct6ELmNp+2hZIaQQmOwRwXQeQFLi+Yyt2In - +OpkpxLm5bJ7yCkjO1cubX946cfRn/Hj+oBwkJFb4OsRg07u9JySTSixQRtO15Ph + g8tGnKaGTGHkNCxWIyMLc5KFgxPpxtFu7grJdkfsGQ40ykMPHQjSepAa7tDAvW1w - R6cibGldFdchNkr9pc4Y+MJcCxFHHv5N6i/LpKTCKPpFUV0LtwACJ/Z+StM7CrCX + rQX9MVkqvKFQUZCcSi2LUwWXZWnaMId5FPvWHSdy8SS3KWyD55Pg5K0MA1Jvp+iG - zgNToAECgYEA8WWVHTTBJahw3zmJrw0ygNUsT3AdrGK70xtzhhUz4+HgoDBVckyH + I9LlFdECgYEA/Xt8lCn5mscVw3mF9s7c59hyIT3NMj6JSWKNbYVRlpcic1M+aOjJ - 8d2yI/7Fnu0AdifSf/QTkAPGaXOXL+5l7WZ8X7FXRvPnyMaFo+WsN4gL0hW49WfE + 88TEZxukiZd+J12sd41J2krsvVWkfANMwzdIqnaxG+1/l9pqfFgo9skikirUd8RD - LJHRvklv1kCygQ7g/MEuDidSeFl327vUVg8v7ZxegRCdOTnFg0KLv00CgYEAyvDj + SJSoUDqKzB5yRkD1eyk8vEBxioUbeXi+Km22A0wJ+BcY6/J9uufvhZUCgYEAuCYP - Yqty/P9Yp0LS8nvptXmhtqRweQqSJdDLIwpPAxmDpWe+47FRq9toLB3vAojq6fyU + Xn0RRbSZPr+8N6WlPiDdB/hJULTlsbV/hf5TzBRMeqOD6DerBASeikBwJN+cbOKr - W3SMW/Bm7hi4Rj0BUt/WDkSE8ZvzlBxcaLZXWs84vVpV3jZjzkgXf9Plv7RKN7ur + 69XBWy+DhO7CgV1GBuxaTo7RTl0nToDnYrWjOy4fHapNOGRQ07TEbdx1vKo4S1N2 - XFz2gqZ41N5fnyYkm/IJRwOk88AKdeGmhEv8YAECgYEAlBYJH92ZD40BkS8u86BY + FwipDdQiQw2For8xFwj4F0ZLR5pxtDaNPtb/4MkCgYBOS51QWqLJpyLWzSuO75iW - 9wfPIvxYd8QqDRuuBvdC2e1ba2m7QV8Jlqq1+bb1bMVfnxxW2f/VcGegdFhgyxqo + WGnwUJmYIm7fZvyOTrbD0A0JGDZXy0fN7wJHYudwxIVn/WwvRUoBjlEPrmtvDsng - lLZmXh3guLov2s9OdHkU6QwglESXLpT1l5Hs5ZsPbJRL7Tg/dU7c/fnJceMQ0E+t + JqxgUucj3DkkG4f2vnhwufHeujIEiG/L9HcEyQBkSic8AgaRM0yaTUGE6tZwr9X1 - tw2iDVX785lJmi2CqT9Nk5ECgYA5PL+lMJ355Trn0d0VLwW3fVqy3KYsPWMC72Sb + XwvwesU9h0zgXHdviwKV/QKBgFoLUkyTv1RkYOLMAo77Une0vh/diowKSJ7C7x5o - uWiXgzayDBS2u2hBhFxZNQgYOu2mmOpu8Ow1chRVyvsONF6PNTp2Q7ULP+TvPSCD + JDWQX21Ac4mjXt5SG+viYnPFW8nqdMKW/TtHWnov/bAgGdPc0rPDJhm5dzTt1zbv - GAqDPjbOkQ/u4IA9ye92yhjefMcB+RhXsJCGQNWLlDx78pIYuacMNGbtqJhKrx37 + NmgDv3dUBPpkIxnCNKK7wF6GpYw/vWi59WArsK00+XmBH9Hxss4+syTKIntKiXqG - 6kKAAQKBgQDXLFKt8Kg9dU6/IbAP3nUg8Wqqzxglym2FZ9TJWMTJdmND2AQhEkYc + ywvxAoGAILJ2oWur4lXGipG+Mk9dceCSUNzF1+8eHBL6wcC+iNeEJx9vj6T7Sgv9 - mHhUBmsDsooidZ9wjdDAQAH+tt188V7hcpphOvaG50/ln4VB8N7C5LosM0fo5Jh1 + MlQYizRaU7AmTOa3jFqbDv/tO23apagwjQtIJVMj8DWQbp5p+HM5dU7kFwGQRscW - 5Ry0vnhr4HldVHPmfDA3N6ClcM4n9/ex/Js1BmMiHzF+JGGXHjRWTQ== + iMVPpVMHqS4F4L7j3oQ07Td7xsLob0mM0FxVlgK1pVUTkcpQoRE= -----END RSA PRIVATE KEY----- @@ -7799,6 +8549,156 @@ vector: required: 2 segmentSize: 131072 total: 3 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:CHK:yjow3adqfodddphsctxenji5ya:cgqbbbawmgeeacol6umzjofyxzqeegjnl2s3utqjchh7vabkmeja:2:3:4194304 + format: + kind: chk + params: null + sample: + length: 4194304 + seed: 2NVWOXvtwcAFf0J/g0fDC+ypQbnCU7awZoCxcnhpkVM= + zfec: + required: 2 + segmentSize: 131072 + total: 3 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:SSK:ymk5gokxdb3hzszs2nuhn6sqvu:kigmqblfazd6wpwi52p2t4lzb563naw7hrm5r2dtfmxheedra2qq + format: + kind: ssk + params: + format: sdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEowIBAAKCAQEArVHLgrQL5qjt1rxV30gfK7+AFv0fYYTcddOFF1g+03o751vD + + PTkhlPNGkK5w1gveH5nTq1FnZyH3qLQtWyxvY1pnDARDs2xuTbIQbbojeLjzgk5P + + 96nBujrI+TatjbDI4vuW+3tRQ/WJiE8dEV49r0F147kBgNC3al01TsFQZP7DR6Yw + + Cqkp2WYbfEj6NtrsVJIHykzDifzZ22pcm3SWFYk3N1VeWIBw1OSRh/0GtnoqiWdo + + yztfATT1d14uubxHIXL44hz+M8EnOZ0hwevJoFHwnUWzWP1LjrHxPhUwteTg/MCg + + nzJcVHWDVnxpq8G4+WTlQrstvTIqB0L37pA8aQIDAQABAoIBAANBotXlKsa+Acn5 + + ETv4D5iI0+VFWDjtgB7j8dTgdD27nDMv8YPNoP3lcZFNAGhlIgVB+fd2uL9NT2k8 + + tBAfI5tK8DNfmSNeiYFYM8rRUSf21vcUdZgs+QDWEz9AxxNUcPx4HFFunbZDKhRa + + OCztXQdVRDEadsFu1SNgdkdGhot/M0vQrl/0He9DomctxnvnnBKXgZo1PLHpsmdx + + s9pWv3rHT/k62FQ/d3TDfmSxc43K3OZRfyER5D70eOlpomJJdJURUNOTmTBpPlLk + + F8mzaB85ojihNtCcQjQvEskktQhk+Ejbik6YAXv9+jSf4ncchymG6LYX+S/G7yL/ + + ScmmJgkCgYEAyaMVMCAT+lLom01LjJoMPxXFFZV6CVTZGuDwxhkt5URZjXWLIL51 + + ZdhdBhoBd8hGm0Lkb9yZ8kXwpt7FhZLjt1up0yKMRhCw7r+zbhb5o4AZ4+JwzGcw + + DSyp+DzinJgun0Xup0eUIo2ov4TaPYfz+OUxjj9yUtnWI7KOlziupOcCgYEA3Aw/ + + C7RJZJm1AtsZFI6mfI3A2ObA7+KW2/yIJhfXtRLbpt9H6CIxotj8+mlw52C9U241 + + VLVQWHVC8t1eWeMaN6afCLu8FNxJLBzS1GvFjjlbXrplBric1yuSX82aogZJltoz + + TvicIWICOYGkBl8O1j51rmbMQwfs88W3Zwermi8CgYEAwcEq9/6rE8ydZbZFlYrl + + n60Mn+vtw4+7uz9RPhot5vPh1bOQiFtbtgzNfrJ4nKBfcIw7tF3XtF2OnNrOFMeM + + d8HmE1NMVXtueUzOX0hGg9zxg/AwkcnJ+67ieP4Qh4cYrcXmSOnYJ8fV0osXpy6/ + + uniKQPUopwJZ6h2HNTqrXxsCgYACsVusJv6m7oKakFfUOpKq/4kWnmxKAznZY1O/ + + M5d+LcbmWeElZBW7anBeGCA7lKF8feLFMJrVGkpBcpgO/Yp6l91mW/XHQ5LZqVij + + JNZ8EROfKyTFWkkBERVverKjvPP1lqH+G2i9t9dTINUDBvLFiGokQjnJsDUkHo1K + + A3wEHQKBgF/7DAw9FaDTbdBS+FhWErCYHHZF1ZQxUmcb38VuSCvf8fkmnEpN0hOV + + niDZyim8rB+ggGf1symleU3LqU6et1oaP22ORbiAG9uliHaKsDU5xC5YHY/LGI4/ + + xQe/x0dlLCDV+ZIdmNdXeG9AiUntkxkCzS+xLZDXuQxDgWn2YN+2 + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 4194304 + seed: 2NVWOXvtwcAFf0J/g0fDC+ypQbnCU7awZoCxcnhpkVM= + zfec: + required: 2 + segmentSize: 131072 + total: 3 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:MDMF:6rqx6odplnpm3q6mlfvyymo3ku:3t3xbyqbuz6ykrfnjmdccldcobpsx3giwhz4nsyeeiswb5tlf6lq + format: + kind: ssk + params: + format: mdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEogIBAAKCAQEAqzmFksZy6jN3ReJ5UycelXqDDZkZZ2F2S/wp1LNMo6Tepcw7 + + g8X9/kNE0+5RTY61VFHSibeYwyV+LCJMAhIH4eDKz9puD1zSCgRF62/qofsqgu0p + + v4zvAoccEOfp1zIXIJ5YqvgsOLR3JLxlCQmbXOrhasa7OE6qzpedghx9FocQyPMH + + erzwWXknLxnoF8guH6EAsnBCPOP+Y8CSCgmnWgCE0XgF6cBFKGh/cWicM9lqZS6W + + 4sPRYYqXDzyYArYqHCSJ8iqxzx2/O1rIgo8Di0xZ8IC41V/Har/y4oBZjMDkaHo4 + + ov/gCsVr80Tu0maK5/NaUT2XGZXqra4FG7fqNwIDAQABAoIBAAjmuaHyvSCdwlKY + + vnPrMbTVpKB9WAu+zlaO6mHLXG2ZcZWu810bWuPv/VEDL6jXhWe3xTkxmThz54ZF + + 1iu5Yj2E4SZDFbuouKaaqEPgEpOPKhuaVrRFkFtSSMw8MjTkvr0MXlGtCyd7gkIf + + pST+IdyHvWY+pJb9x/Vrfl24O2yDTT6TfyhyMXqUIgCo+1Ntn25V4lFrn92AJPRU + + lfrBLf0dCMSiOJHfdyllN2UQkD/juqos6QBlRPi8BVAq97a79EvYMBD92Z7T1Hkk + + 8BKggbHmmoMlvJfPWdSrncMujLnPcEia/9fIxUSc5rJHR6o/l3hsLEwg0SLxS/Lq + + 9YlkMnECgYEAzBhyN9KtPRiNFhkU36SKCfYgkTm55A5RNi/vnRYuRBnZ2ReORy4G + + /16yBxTy6RpxbddYC6XN9Ame5NJX2sv55LUWGVWheJDWpCrTT4dGHbkb2enKFPZ5 + + 9d2UPior+xO3F/sP0uNC8LDKrWgB7Ji86Ed8xitO/LovvTUfRtBMI68CgYEA1sUJ + + IlBsIJk9eWUkhHl7sObBFNScuVEPx6QIr9+1KTiPbI8VnHZxSVR3hONpzdCZAsw8 + + WDILeii71C/MU+QIKN1dV81Oy1aqCq+kwrnes5jpKWjAbYD6SJIYqcPNbPKZ++mG + + UM0kUqgv2Dpp3Co9YWecBOjpBdcQTSnkvmsmW/kCgYA2V8QBzRzHicP3QFJogf0n + + Tdu6D27JpG5HSVg5sXA8Pc3dmgIOPdkrIeGxNQjAvIO7RX1yDIHcGruuHbu6zFkL + + ZpQtxrkpyxb7u1Nsd45Z17Hswe1Gy6IJrygLrVrsjYFQ5059TnnCcLBmn6zzfG/A + + QVidw2ZSsJiJfp2HU2sSjQKBgC5J508TAEsCXCKG7xjySft1sJW5wVGbrAf+TbUC + + RTxuKVNff2vqhz4jy2LD1PD8DY5x0Gu91YVttBXme2Z1VmDgXRbodBwVQK7u7lbd + + 0qboxRAcuKShUNBFVLV6MxNRMmj+CuntXO/HuhAjft9p5zLQLutL+7U7hhLrfZag + + 53KZAoGACJa7CP/07PoJg90wbxAqLxxQhxTNB226nSFhhQI04CqTMJGJm7i0okhR + + qjTG/lfoWNB5K7y0nLhS91cOaLKw2SM7Z9Ws/fMdC7WtNTsbHyjqXPFyrwMsnjKe + + eih8ic/0hKnIkV1SOjU+rgi4fVeFISzZ2qtjuc+CYRbXPKayMhI= + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 4194304 + seed: 2NVWOXvtwcAFf0J/g0fDC+ypQbnCU7awZoCxcnhpkVM= + zfec: + required: 2 + segmentSize: 131072 + total: 3 - convergence: ZOyIygCyaOW6GjVnihtTFg== expected: URI:CHK:dnmjqjku5vyrkeaulae4bt6juu:d47airwaqedeji2omacwsp6yfqp5mpurj776gmzvq43c6dovtdca:2:3:8388607 format: @@ -7812,62 +8712,62 @@ vector: segmentSize: 131072 total: 3 - convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:SSK:v7lmtefejcs67fv4qoppjrdg3q:3xvveqfckxsnp4sxb474uewywjhpjv6wg2fooxcx6tnrb5gxrgma + expected: URI:SSK:cqstfgln3ty443es7iodypwzky:bbaqquydugrdq7yuoxb44oathv6x342yp6o2j3eofdqac6rycdfa format: kind: ssk params: format: sdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEowIBAAKCAQEAkl1jyJ78S8yDqSPYJzglfkVx22MRTC5Pb1639llFcu8fA+Ft + MIIEpAIBAAKCAQEAy8EGzJwnWVqWP6vX67QnnsvA1zli+zGdsr3x7TQOjyQDo0cb - ttmoGV3T1R+8ZujVF8b32qu15AOoSfdttHZNqCgk5wP2vIOrcP1UtiDN/L6IF5mW + 2q8WgMqjZxIr+74v11UVbv2T/1rncbdQo3EVmkGM7ZdHIL+kbQMRGVQyZMIHIqpf - 5GM85i5DCLZJnoHgRzA7zAsIFrJgVSwSKZM9RAFGQt1fQn3lUMH4W6u1MYDh6r4P + FNA0mGKkLPUUVm7KL9NhC1fs7FZSs8n18Xm7SBpZj5DgihoFvJ1I9ulhUhLrme9l - F+++MF629Gm4CV3mxjTtQ+fRHoZ5tAviJ+ASJ1J0ygFVI8YC52TqnH3w4fHqpu2q + EEeYUl+MFZ8g959r2dzXoszOllains88+DzAM+X0dg/muYVwQVmSl80GIy+DNyjz - RoihiRwYjGyp9oGLNfWebzH0Wbja5Zm8wCPW+IxOhEsVnuDgw/ocs1xF7wsmlAEB + Y05f+vmcje1LhdwGxndRJ4QHfgO0O7NdFLeD130ZnuQMo8Et+PHQRvWbJyMDTfTg - IpIFpEc6xvrdTWEsggoNoePhn+fmwPr04sn0uQIDAQABAoIBAACp2CNOK3kIM4IC + zWqJVxELEVR0ylZThQA+Ubs6xlU50kliJXSICwIDAQABAoIBAAPKeEPqS9kfvwKO - 1vDrtpc5S8p4b7SQoFFo/bgOgolwMZ4nhzoAmiVAbvx0sAO5+ggpM/Fqc/xnrmkb + 7JotZJH8cdQPSfDT1X6ehhfHC4D3nJG3IE+LSRHls0XsUqErwrFw8HsmK1C+8jtt - bDth9+aTct2o7UwJmsGzANin7iA/YbWLZC+jRYV4KO3FicLkY/anyfWaCKDaR/+m + Zl9Zb/DKHhh1jhSP5uaYU1hzS+oP/239BCkw+SmJ/EqnLKpcWVVWdiHDOwc9UUCY - REcYZWjFJiMrmQDfp+88R5GRGFT0LaIg3jCSSr5kwbgQdZFK+bwbYvr9YAhF2wp4 + a1kDeMvjTEidhhqhAqZQZCvOinRVGUcIm5jYGDfeB7WS762NEwA6r5Dlm9S+HkYr - wMZVctrU0BrpKRVQYphJ6DNUtd0MzBsVWRveowfhMoQnOK1jubXKangOQ8+a16p8 + iAoR0bOkQuGDDL9W/7itJLizj8lK3Xuy8XUI1yHVfKCRDqpYES7mCAYWsGj+EPxh - ov7LqnPFQ/EsHtWgpw1ssMzrhh2KMbGJaVlXtSFDv1A8zjuY1y5h+HHTj5r2Kn3C + 3OwVyRvQvjtdyDpshSHDLuP8ggIEocQzkW21XpIbWsuojBLak/I2EHYd2lSB6nit - 3OPA6WkCgYEAziSu1aXxK/WAr8ZtCnRgfbWh1bmhOY+kwyZwFSez/fXik6qQUiU2 + kxdNwwECgYEA72Q78fCx0yIJ4vhcqUkWUxdnw5hLQB7J3+29OUVVUT6byabMZLPU - MesFBm+2DKl78MipS3ca9l6Z+tfaW1JxEP+ghbaXq74a0eLeHO9nzBzY1GhTYPLv + SJ6Yy8hAy0OR5GR1giplB74KWDceX3wH7OULwuaEj2ImiIqXplkVAFe7mh28Rwh5 - JbYmzQG3q6uOu/ii67gnkEeoPeio3p639fABvi5Mo9AknOUBFOq0q80CgYEAtcOI + jv5phhggBfvjZLub/LGkoGkBFu3E1ZBa5gwy7L10gE27HWRYgfheIUsCgYEA2ePY - 4yNoHBKMrkQtYEzP9dPXCHFo27FytVkI9/Obgy2477Ps3KsvMdgOdzSNP5ZgLNBl + IPAPmT9CS0eIv4XJgXYZ3NdzjAOHFpyJOPhbg6WEJNayTrZiRTys40GzmSwKffPI - UgETJeOWRai7saVhFnto4Kuo3RBAbO+DgMoSMCcZ12qEaR3j+Oj95ZqWuPw/ld1S + nVEQOHbAR/eOjvIpBT4adgOuvkycylIIQ7DZRShOtwxrdg/a9bLxaHNDvPL8wqXx - 2zapkv4uVu8VVDO8yyb71zp27Ftwij7FjDfG+J0CgYEAgzpm8isJNGq82SkAET+0 + QMdYZZxWTsPSTXiwzBBiYerZmaSj19xdielxvEECgYEAj//dRzWf4f7xr4PySSpb - jVIrC9t3/ySqRnEZuN3lfy4gZtCVvzVhIrXyJP7IbZcXB1k2LIxN5bijXUQ8BRae + sXO8yR1M9q8OhBK/5jlcjth4YZ5iCJlbsqskAkDdKOfmVFpRjRDvYO7hzhqpvIoh - U6vnjDeIphQHDsXVj6X39cAHaHBhY75C70bdvHPzcJ1t58uIK3a3+Okk+QQ7PDzd + QlCs+Hotdwp1X2Duw/OF/ITJpnUIkjn41RkYZL8SVEcmi6uGs0QwYQWI0EAKTOTe - 7voyodbngwDlzdsarS4chaECgYAGowoniQ5vH/pFDrY9cvCRAFg0tbdndjZDCuo5 + qM7huyJjd+JKEe4Qh23dQW0CgYEAkKMepT62HBRR/YbOz9QPn1C2elLK8PamhewD - 64o9IvlCv2YhtJp3jnUQwzl5HeuLF1zrvqBNXN8K0htwZCKEaKMuuPXkhIhlseUy + az3yAcGtpoaedoG7Whqc6X6DqfoCPPnHAib9jX3Gxf8fMuStNj2zcwOey9QvgF5T - Wa6KVZMq+3e0QuQlHZTPwnJIdOV5emhhGsDcXi2g/P/hYDY/kL/XXwoinUAhvCMI + /hs3HyFSn1AvRX/g6ZiPh7Z8EMF75/of29B4bXsKD98Niz/CnLODm1w6djNET9aI - eKzqPQKBgEyP2NxMeNwaPoEjltcyYyYvriDJMAqPyZ+ntLhHJLMl4Tz6yR2VROWJ + gTPlvAECgYBSUMCKSAZvVLMQghSrmVUFas74O6/9eOz2/T3YyG0hgr2RehxZ5iBx - hQ8mnSGJJfKgMf/yxXDZfoSiMK89pekmCGGQsZN32Pp/zW13n9pMuYRhTDbGZDZb + c8GPRbhAvd48ba8DS/hqTeFRX7+H8RT+hroM+dv1NT4blEuGJDClo+ioC4cJZTI8 - qEVa41y+2bdvz2P7q2yZAudDhc05o3FSX7QKk5EeuMtIkOihso92 + thRcwEKwbjeyXjg0JZ/z6oAH77Fhg44r8nC0zDwonRkQegf+iJVaNQ== -----END RSA PRIVATE KEY----- @@ -7881,62 +8781,62 @@ vector: segmentSize: 131072 total: 3 - convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:MDMF:oto6ivpouf7ay5wjujhvb3w3za:aezel35hpiz3bzi4z5an4lmx4c6aytncahbkymrvyddvw2tgefrq + expected: URI:MDMF:nyipqmlsbqj5k2vi2jbi7m2ymy:dsdawym74j7fgry32phhntgqsyc2k7wgf6hbs737dsrrriu6px3a format: kind: ssk params: format: mdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEowIBAAKCAQEAvKWc+29gXtr/g2Ar3x556n+9SMJK6KLTYs2VMJJgc8xugZZK + MIIEpAIBAAKCAQEAuzWxE9RCah2cgZfL7Ki09SgQ6djyb/xW4AuE3a7LjthXWmFp - eaAYGH8xQlpPjxc4DZO4lIp+2fv9/t/u64Fonmvz7nHFME6j0DhQZL9EMZxEwzGW + TL5LBTkGD6G1KYBw4BKgrj4xl8qpMheIvLP1rV2wCcfx8z+jr77YLrh4VoPINNND - hgtnTmc1HeeQf04l3BnHGJQZDGfLqNVKa9Ff+lfESfuUbFGqmuQWXH3ljZD0j1oG + BX2+GlwSSYU8R6pWFnh7BR/RSMNQ9sXtTDRxnJ6xshIOAukOvREFRZGaO/sjLlL9 - 6uSM/oIMiuwPr+HW6Dfz378u2+b37xTN/BGN7NfawDeEzjm2AtS0JsUV4mEtNtJ9 + rg8R3oPvlSXtn7fdAXnxwoNnMeX9XEHoxM67MJ3kmBWb74kdVYcD3CimBqUQqkDs - jPcgCTcdAys2hWTV/tSJ8h3NUYeglC4sz3P5owwJWsyO1oWD0PeL3DCn+9qJ71d5 + ZFWmlhVGhtvB38NDlBH3G8KG1ebN9icONcrRTYB8y50FieochydTSBqD7jx2I8ca - ssGQBduWir1cXwIBlegGJrjSCG//5TsY9W+s6QIDAQABAoIBAAFP1oZiGSW3uKip + LMl98BfCVsGav3yeBXetiWC/6FdyvWOjass/cwIDAQABAoIBAAzSVhDmGjBbW9My - ecygqeDhWAfiQAKbpUQt4VB36B9OB+OzT5vGavx6n/VR6vU4CF4BzboMt4KdD8Be + WsiYG2CpAFOLxLrvvOF2WIC4Tn+3iHALuOMFK20tpSEf8aDoh5KJJBEa+FmNiz3/ - vsrY+MkHP6hEFsa1+Uoophh5QwhkSY8g8GbIvARtz88ALf9QpA9Ch6GqX/032JD5 + h6Fo79wSTRK2a3c99g980iCNCMzgFK+tgmsXXBRBFw2K/wBnhaLfWImWzsYdfmeQ - QL38tAHp69XG39qb+8d9eBFXF8pS+Wnpz7QcuopJWKX0OPdSwM7hOV+rg8igkmbW + UbrE4r0Xz1LDUstXO+euCT7lBHu0DuLrAFaA0Js8aklPm+eeOXyU+j/jqCOZVWRh - IZBv2NM8dczjhHDewJiExAggvoJlEhAuX2jOyoGCEFRgcR3Oen85S+lmEX3RN6CX + dODy0KpHS2PcTCNzimKnfzC3DNsCedkIhMvqgIoXUkr/xqQ0UmZIVO06yPCfarZ2 - GJJsaenWdUpP/5pCIHe5iThPZ7osU2AwLp0Y2dyc8ytDbwaoclX57VnmMksDyfWw + wbXFip6A3gsux6O5899NKpJ+C3Mr/h6LbNb2p12YGe5xJ78kFR5IkSlkirDzNpJh - 72xZON0CgYEAxoolFefkRReFf4icko0KKtHPa/9O+s1feSuAzAT1KIr3yzsFRRu6 + ENybGnECgYEAxQdb6ELI0VyeCGIcJRP1ujtN4+FaslKssvzKU8yglb68ubi1zrLM - ziVF6jUk7Zr4lbk6r7/bxgeLRd4+2d2TxzNMaKbBCgaRxwhmx4peXk9Ycoy7AcMW + nSGtQG+VexfaqLspch8JpVsY/Mp56d0Els8J1fMjvgbwuoojSKpY3s0Xu65rf7Zk - lNHx+zTtW3CBYsUPYBehIwCO3WfXnvm83tWwguJHX/cBpowASif/Eb0CgYEA8z6D + /Mw8Ro9lEtCPPnWMYfWNJ6xYQPAzU4pn900BtznycuY6EHkClE2R7CMCgYEA8z38 - 78pyXPU7x638ZoBie0jNxv0B/lVi08Y6VH+oHW/B/RGWDF0lwNCTZQkz82Zl8nyl + LHX1ry4kNlJxFt68gw9DlzKyBAzLyNXD/lir/JpKeOpmVQ0S3E1orCugkRtfchXp - X+KewZ1d2oxEIhkWrhEo2AWUtv/ax0IRuzNmBekwW4hlA0/4JSTvaJPQRH71+8Pg + UVogQwO6E9HhXYBdn9+19Y1cITeoD9XmuymsYq+pxWrT4ctUjaZiLHsTfsq5F7Xl - k1ct7hgVGIHozp3gu7hYeEVmPmmS3+PAJmkXvJ0CgYB3QzTT2+C7wE1pNt8XCbI5 + 4xy3Bzqf2YuQ+f560GRaRmOjVpcuJpqFqVrELHECgYEAjHa8nQ7PoAKJX6yiKATc - 1p8K+Oqwrf3UA9XyuGesWw5O/r1Drkyg2LMO5a2xLY52IjamrFGQu6dl6QNITFoh + 0FHrK6TDRhIOsOPrUma1rUv3u+flJWDu4q7ZlvB1/vV4m4Yi/AsIk2womj+3PnSl - JyeXFdSP+TJIpTtYUj4t2OwAo5kSjeZar2L0y+5pJ0QR2N5LkuYw6Hzpcx+LV+mk + Cua7Ol5GgvjrsfE9Sla3WM+aNeEZHkloIZlw91TPV+R72qlu1X97jGcf29vim5I2 - 0iid9t95Ph+3tBHYef424QKBgQDqcbC8p8V+bycFGF6TdN52sP8U8brAJhAwyXhj + oGWz7W5QXH2ps4ixwAy1FUUCgYEA8tlE5sLSipa/srh2jgXNIfBgZBlKH78Cyj2a - BP9GD/dLMW4L0KOYqe/GjA40ZNeR1i2Ws1gMiN5yzIrGyqOfdg6F1ys1DnkRYE6y + E1tGQslsZvJnPqzx0p86TQK1qYoxrb5wljcsFJwo8FbP8UESuGZqzYDXpZZipYTC - vaFxxQXE0zt469TiCC1wADfWLQBtfqevm3E7cJ60llGLA4Qdqloq4cjgEuVrQZpr + esRtho2pKx+v8TPG9DFUvOIYIbOWPjTuEuR9W6tNIq40DVPkHCDE/JfH4NDJU+Nc - 6xLjyQKBgCV/I3PWI7ShyxxqcW4naKUbZuOgQUZJ6UoF+KRwWmSi+gcdjLxcfqvV + ZSvoxhECgYA1yJCfzoM9MAv4yDu8bLfqS8hUopUHrH+IREbSLverA+zQmh5+5hcF - 84EgdaIPgh4NLKDqaj2MHFkILKhDXfgcVZ5evquwVH2hwwOyf86uI9a2yrWcJsy/ + W/jOQdK/ow2wmCGSBo3TQla0Y6Zm0mA97QQOyo/N/U2aWWnV4FHa91uhI05vljAP - EyhYqdxLYiF/mVCusAkcMdPKOb2VhLapMw8ILcREMswrOnpLEUAi + /SQ8XCPJD4OYbGIi6lbdwVzSpZVAvPjiQ3nWEAiU5X9tRzBbHv9Gvg== -----END RSA PRIVATE KEY----- @@ -7962,62 +8862,62 @@ vector: segmentSize: 131072 total: 3 - convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:SSK:ey7ogxmnmp6rl7p565cp7r4x7y:opwwoooqzrp64hmta7syc3olj3nalfmjilouuitoimkxqch4kmza + expected: URI:SSK:cq5nwf6ie35evzc5v2wkaucyti:us5gy5xdddvyhdg5ysritrzcphfquycmpemoviqbfou7heowq35q format: kind: ssk params: format: sdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEogIBAAKCAQEAtEAuvlzk8pp9nXSP7bybiM5SFZA+DzDz+yDy7iDHUb97RseR + MIIEogIBAAKCAQEAt8fnhPTCcY+kGOiWih61BQKwwcz3MysYGYWkvIvoH9BqiT/3 - DA0jVF2aNdB3Ju+EdoBsviU0lMjWu1qdqMFPOmi49Et7Owyj4F4y96xHA2t6tijm + MoSzJsx+qcEatHIHAuxqH8r0nAODgQ4Bl+4xGlU/+Px+l1KmR2qDl8ThwLFQDhYK - 2Y40/tZGEDB45LQsrPJ/qFEfb6oqXXclarbPe/ZWKzwc/oAtF9sdbqOrnt/3YYoB + kUtlb1kLgED7NrLLrplw1ElI+wdHwActeOfpDAZlxbZGohyq0L4n5ovS5DxjNEaR - uakVX/2ZwFzlZrQIL8YJzZXXkm2LOqJ5k96mZzlOAoPaJcfYePCbLxASZ83IlDRc + aFA36mDDYmnSkqSIdEGepVD5iBx5F34JckySpuLVBTEytSLKCp1S4I/uEG19KigT - EBojklzDsKXBxw3KlnKXvK09K9JofYAYYFtZacCDiZkDeUopJOT3FgI3IoPLjitH + 5oeDlgp/xXlkhdF3wM4z8zJjCC9Gla1KATJa/5Js5oX9xMHi4ZNpfZENSR5zDw4w - reUQqpekexPp1SHgg3FH7+OVjrkhnaaF+DP1WwIDAQABAoIBAD36C5h8zGP2Ztaq + c/GLMxc5m/zNK6+lmBiUoEixt7kNaDxnf/XkXQIDAQABAoIBABdw9tmpW4zfGLv5 - 64os3bXOaz7q18vVYy6oB5+FOcOL+VE+8UqZgdpSTOHQCggjNwKf6cP/evLlk5/b + nMwAxy+crH2HvocnCcOlnYHUKZc3PwODJm1p6iz6e/R8lkKqYbUQgSfNfB5TP2iM - 6nXJ8fn9ZArroTWOhRJykUfDvq8YV5smuSl40hQFjRWn9Ql+QhY9U1OGgS6d7e8x + aA0gO2Cju41vbVkxWFa3IhJPcUkiBLdLPe1S690EQ1iIUVKkgyDh3veg4l9sig0X - NnZY4UKYUsyO4NFJNTgMqTQPpsT7XJnL8lntHRs1bKUa0mciOR7YRBsH2sXgDIJy + DUiE8h+PyFbr3T4LwIjwHEhGkO9+AZujL8/2ikzcqFOwv7AoITQYVM6c1amvkoom - hxqUHjRQju/CSI055YGMxp88lXxp4cALqWH26ThIM1FryrWRZuhbX42jB0TjrAK3 + Ag/p42iyL8ywof1oblSxBUO8kDTFxhbazaVQARwWQHIiX1AGdg5za6sq5OF0QPRd - 1VfE+DPsp4gnDd+p/rSUzkpTkFWct/PjRaMM3F6IOqNJr43zb9+rLs5N7oEPwJw5 + ib/hoUNJeRehVFUMoNSYCNdYNHYLHboTFnemNZvPLSBGJ+A8iBJAoPPgO9LSEyyU - CWUrQzECgYEAv3BP4tWXRCfGVpMTeR/6bWtitLlOI3OUuD8jpGBLpwcI0im+CMl8 + diFMkGkCgYEAv8uKz28ps1feQfDHF920ONHxIK1JnvOTMJzCWUOcXZv/9hVlt0JB - Uyjvo15YqncuBPLYfckC12D7yzmZmru9a+tY1GmEV7cIgUTFaT8NlKbr77MX60yl + 7fLB/AJt5GkdbuHx6dLkju6sZGPxgENtgyq07bf448x3ouJ3Xt2RlfFHbo7AFK0e - qq6rBVWwkRNt0awktChn+beFAewxF9VTCfKPgSlp7qBb4Nn2our3oi0CgYEA8Qn2 + cVUkfskr/ZpfCMi0tg8GYvXF68MJj/tWx8/4yMz/zSWNFAlzao6LH9UCgYEA9U2P - 12lQG2xmhjLQalKfsxEUQG8BojJPSck5bLGpwV1XL7/ygcYOpGyVeNOFKUNzOo3T + bBjKVEudYXf5AMOJeEIwnVxnmVhUFL1i8Fnulm06HvQxCzFEadvtvwO+1J/JjGYH - RlhbFZSKcPuRV8QJRQqj4uVXa0j+g+zpCOIocB1GGVDfxgGcLH/04ng/ttBXMOJU + L5vMGjhLvR8YkoCFCv+puFI4zH8cVCvrF09o5+D0E5Xrt8pKB9Uvv2HKbXC674Zi - coD78uXXnVxIPP9jQVIF5ddNocMZo2uUAyeKEqcCgYA//3blWQwxn65hgNeQtY0N + SAwpoaaavJCsvHnvAJ+/OuS+KBbZV0ye14rPfmkCgYBMLV3u0eowL2A5tJZ/JjGk - iUm9KvmhRmFgWtM6f2qrEuHzCDtcSqdCUbwS/FZd3mvHAbw4CLvnbqeeX8om/T4s + t84b+nfZSElX74tJxQ7gJ0vcw9bomMpy5g6iN5zKMe3c0qUxB/B7zNRv8zpChYWD - 1seicwfoHus789afAZIzsL3NKy0C32O+tJe9t9DIHxumbYrzo1JnG9/eLayX0Bvr + qXy/Rmj2oYmLCoP7C+n9Mh37DXvBOplyzix2pxRv39aLOJx+Cy2wNInuAENWCrAH - hmhNAKBGQtuURql5+1z/nQKBgF8bYmV+rVgUvqNm+2tobJEYRRhjdI6OMVDY8Cqe + INVhe/rF0npcUPykgAVGGQKBgDUy5ejWk4Kmh3Is96aPwY+AI1TtRlZ+TnXVANEJ - M3ATp2o037gq8O4Z1iSVuW4dqiLJgTq5dD8gnDuWV7P8qveuChpmCcdQRvTBDvYt + X/HlrFYsNTqtK54doTjs0gUAxlAZjHNpwWDqVpqkVMro7nGNMryTsFfBNV6Xy7tZ - Xm1Wb6lfitwzGG9KkdKmReWZcT3doBqKIF+oJxp1Jh/DWWOVvLQC7yPLupsLwJw6 + cHHhWm2o9N7+EwIR3PIPfjwv14q8xTHE2X6CSEqewad6djfXbTyTgR3mnqoNJuGt - BrXzAoGAcdWwqRl9uFjz90yM9K/JrjoNrhzuPoL/Nr28QkEpKmtgrE5+8FZaraIH + 7AQJAoGAYtfnWnH7kqQbesu1b0adJZZsIZhestP6CVxk1EmsPjO9RUBjl90f9arb - pMB7jzeLs6YUUx7ClHA33L8zxiChp7AsDwIPT18rf2oEkqaXQgDn7r2NV0n6xq/l + H5MnbPNIkOKgLLojpyholHKlBRAeO+B7gh6zF+PuBeNp9oADegovCZOhOlcG7jn7 - 0LvtFkI6TFDveAh3C8ANm3gJXtjiQrwWnU0UGuj61U7BdAKjJKY= + VOaPPT/jxE2ah9k5mdZJQNt1tCLbAB9Mw1Emb5Ku1lhewvWqWog= -----END RSA PRIVATE KEY----- @@ -8031,62 +8931,62 @@ vector: segmentSize: 131072 total: 3 - convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:MDMF:rxfm5lemmemurohy4gqpuephha:tdo5tzgxfnklgaecvgyttogn3lyfgabplddabhdp4ilawedyja5a + expected: URI:MDMF:eo7strdhhosajkhzp7x5lqukue:fmqoufvqqrgeg5nxxzvvypblc2c7r7tqx2xn5l4gzw7uyuitqhyq format: kind: ssk params: format: mdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEpAIBAAKCAQEAoec4LfFaZ5vWVohEmozJfp4zPpG//BmHK2FBJFqafNZW3vo0 + MIIEowIBAAKCAQEA52AqcL8qR3Pk9UWi2qJWey4+VnwOLBN0MA/3tN/VxWszA+D6 - 4dsnXlgYN4PiVaXWqtK0Q6S6API/QAmmuACVMNxIc488T2gzDX8/8eY/FOvnrNYy + kCV5Gr1eepPAHYELRdepXeyGEg6x80gFlJ8r+wrVGf6RA3FffB4NmbA3ZLjqStQ/ - xZ4ob4MvMfUThcJ8azqa8HOZLHn/kgq5CsGajlnpVimVpiuD6k6l3ualtw9745Pp + LYptnXoJT27VqboqLXgXUZemGuuW/jN+NfMJhmi6PDpKLhCH8smVer/w2hsVT6Y0 - bvij3ZlifcYzDqc1UyK3D06XA0ShyjFD8uGd1rggkAF4E9AFfljl38pUJrcqU4ui + ltVxkmvQMLA7DHqrgaqR2CADWYloyx1R1n7uSf93TK7Y8R5U3FihsXimfQ2o/DnM - EZzdjU2qY1hrHFR1BY+agL4PT/go2f8fhCbZpTyJ2EDJIJGVerQ4kkp8J+uNm+ZW + o5nB0u9QddlplGFeDksVbYSR5TSp79OrLi6MTF+2o2A8PoM9gXy+0lhl09At+V19 - hbnH4y9zwt9/3eYbdIdb4D6Ujr8jjJ8pOLk0NQIDAQABAoIBAAaG7euVCqbmECbB + +r/VzrjkRXzNSz7HrSguoU2Ys7aY2fbwhlyPCwIDAQABAoIBAB5t+tbuMADKtgt9 - hdNQ1rS6LjWyplgRZv+9z+SZZ7E3UOqnXdqdihsSSroLB2TNjUR3JYy0+ny14Qy0 + 7OT6jGDR8aKNLo35waz1L1DrTqGSr4MXP8zdE3gtUfN8DWcJSt1qIh0MrOCgwc2V - 9uaXIdMFwA2ZZv4kJMUoB3Nta8cG2ODnN8YvLjeLEd8eBfEppvg86IJfuEg9parr + pMo+7tlVkODve8yV81wPAHSR2jnVygEtDeFkPyCtZUkRnL8gO/N+mPktoSERkpT8 - fL9Gw9mpg6/7q2YdSstEHwkuW9yJfLX+shRekV7ouu7zDy+CTQltndurNQUu4Crq + WmEnsBKBy9snmAxGjKKlLr1rPfa+cMDADzUC3vFwO1f+3RRM7PrMT23GsBuV8cbK - GJTs1bp5oXZtBZQoDAQykd4WJbb0XfJtx7ALiYEBngX9V+NA8rUpZcbMKXhOeZHg + uiTXWMMjNNbndNVVDFG9T8TLIAC6BKzzkXSWfT6oON1nkn4UZW7WVAuIikoBNb9Q - spf2eWUVWL7VfArN8LqNKbTqQbNtstZlVhoRAjShxATqjtVv6FSwdStnCvr/VJcB + +nkYMkwJ2BmCaMnKjMshBU3MuZFWrzF+T6LbhpeKRfm3DGpDh42peU+qQi/rFI18 - n6dmPRcCgYEA2qu5hoauNn95ZyMqIenhEndDqqMaR30dPvQceTLxD5sYPMliRRc3 + ZxMUhs0CgYEA729qc5i5/6ENSPqThf0RMsV6FO1g8ygn/2ZZMy6Sz3Jxp8pv9t8a - r+LLtFMF4LGM4isMxuZlRcLea+josMcnlMH8zLwHbGuavQ7Oq4D06Ks32IGaJRO8 + z2YcmxBiLLYj4mVSEk9zobL64Y28W0eFP3xd9MqVe/l+HuRFOrR/iJaz9T9HrFyR - 2bS0uZQE8tUYOxMpkQactG4NjjaDadhYS55+qWKiX4nnH5+7U2NYSCsCgYEAvYqo + cYeekAL8z0VmQ6kA0Qd89ILC9h2UNN+RszYP7m0XuLI/PJ5CvvmJTLUCgYEA92IC - KOG8d0gysogLFUi2dtU4Igf46CZ2jVgQ8hsB52R890tT2Gdlteyc/Y1FEO/kIadJ + OUzkdF/GA4dyor1T1z4nSKKg1LFzY9Zg0gtkYLR6Sud+TFAkc1hvbv3J3gxBTF6n - XqtHGccW9lnJbulq5BwR3++ueyjQsk6bnoNvgtBXPhE0gO138iZ+hEQu5pgEPLHw + V9/UoVc4TMf5Qk67/Lu+KdVGZxVBVuilUg3brvXG4ON5dBncpuTH1GecRsMTkqP0 - TJf5p58vxElrNz8klu5sRxvB0o2AB0O7yKJu5R8CgYEAnh8bEtoE08et5BSbfNaA + v0BIG+Z4djzJr/go4VLpsJ+94ulEQplc9tJDhL8CgYBkFuslD38RQT3QeA8bP8Lk - ODghqBw0/ojMQx+GD2X0xpIiHqKI+ujlDbx0DLsUPvxkoY77uEAV7zIQX/uVd28r + unBiNykD/JFbzmkTYDC2z1x7i8BqLrGCaWkj2SFxF2LAzSIVzWjE+5CsoRdQAQHO - gfgcc3dr7syIojk43O9tKWnWAisFadYx80MmhCMyyN2qnd0na4Vaf2YtSy7ELB+T + nCqaneUHQjBasYnPFI0LiBQKPT20661RDCRYhycvbg9l0UwqFTtC6zacs5i00ZCS - CWtcr+NxAqDXjhiU/qGRzu0CgYAIVQ0ZZvsC/2CDKqnaEK08whjKnjEZ+37grcto + ndLjFG+KIdkVegLk2mNu0QKBgB7vB34oykxvCXC5iDEnYYuBvyHLDDdsdRRf4z2A - 6TkHNAquUFhqPflhqvonx0sO+Iy90f3OtJbWkkL3J3FMd+RkDLvYbU/tSBkMjZoX + pS2eg8hICDf8sYIm5dBINezpNWUaVOydFZaTNHwNaXLMK5+fzlimzaXoN4Jplvqa - uM1xIbmEF/uH42iPc5PCOsEZD/u3s1bN9yxZaw0NgvC8qADyxZ5q7dRybhf/ucGK + twS6wQKwDyjgbwIDi6VYy2bhz9m/XMRpglrSx+9pDINPkbUTTBuE7haouptlWAWZ - i2F2nwKBgQDRktqOutgkLAJedrDQPZu/PcKROJPFhranOrwsgigyDEQafCDQU+66 + J047AoGBAK7a1AGzFE18nbWjTKeTQI1GWVb1A+8nUQMkyntaRBbTbKp0w9ECngQD - ilfG463YuwWv+a9hrguhmiuo5N+KNnKB+DxVAp8ZstGjXoy/dCK4GBvPg0T91pZB + 9HMeoTvl58e2VAjkQq+cLxMRgAb1bvNN2BCcO15RtUq1XrVbSIxYPwoZgIbKSwc7 - hnYS9HmebrDkcwjzaXfPGxJz9+F9xTRFPM/wv0rA5OobWz0v2dqObQ== + LIRZ508gOcjxs64Ajkn8r6mmk+nYyjhuoRUFplDW2pc+A46X9Xq2 -----END RSA PRIVATE KEY----- @@ -8112,62 +9012,62 @@ vector: segmentSize: 131072 total: 10 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:SSK:xck4f6aog76yfi23xfnygmfxcy:qcla75enyyitmshc3bvu76dct2lcszvrqyyp32pwg7qb2j72qceq + expected: URI:SSK:zphopulx653rikxed247gbiryu:on4kfty6f4pskitmrm6zbbqufh7fcnqoklpozwm4mlhxnmc6guoa format: kind: ssk params: format: sdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEowIBAAKCAQEAvWvFaTZQXoS8Cs4I2JB+1kWVoBSj96SeANqONT7BzkdbvIYr + MIIEpAIBAAKCAQEA14mWsmxQm2A5nZu2dQ7rdAaZgb9ohE1EcCIjycMMU3wH36UL - bgVhPLYNBc8o1UHiVDAO5Nnhbnp1MxOsxx55xJ5bTXxdX2JPAH4l/zHDxapwEu30 + DhhwebvfmqkQEaiwomYzisZZER3IECSwqbLegy6tAoqrj+aQLszjcSWVi25FzzgN - AnfQ7IDcGxLw1NeabcVzmf48NgWswkXhiae52u9MCaE1gRluQ55pv3HYW5YgnWjz + IGMtej2PgXfoJO7G2rYLvo44rCYUmBFU2+jKcV+1UF7r3wCTzhxzp1o2aQFBPBg9 - T/vP6nMh46L0vuzsSKIxwVsYakx+zSyTaAaR0wT/UdjESvM2znvlRqSFmv2K2rIz + moMvlX6nPshjoXUTVWjlomiKefoO1pT2FR8cKSfgKSeuyNGXR7RzrdP7Tj2hNhf6 - tNTukqxFzXCK+c5/X88BR4aFtW4bFzkIVVDbaxcbicMlRQvelYhMZRP9yUpOgl48 + AsLbBY8RVsqD/Xec/pOm0RtlHdgXmbcSON0hL9TczYH7YLU+4E7tGR5gXFUj2zy9 - IvicsROo69Yjs8n3CQn1r+sx+nmO+4hPfHMB7QIDAQABAoIBAEmUAFXTHDrq0mRc + zwEk7vJlMgYGs5OdoZhNcr2T3BNnOLnuc6DOQwIDAQABAoIBAANtReqZC529JZfL - DgGZdztiQjGxctOyJRHt04mJPB0ViOPdNieBfXjounxEMPdNpU3QcSiiHbgdZ0MU + +UWlqPiOx8hU6DM3sJUbkKG1poHCQCPL/imA+HK+VzoPxrbpKVOFER111q/bkUBH - 5GtgQiqG3K5nnZl8hXWKitXrDcHNZ8VimwMOaaBEmbsGi3gR29HB/hqWL+tIHwhD + PQ8Jes49cM5x3Wi8fq/L+arlUn5LFjcvGqGg0r43IS0foynfgkN7dk/cnlwB+jSr - vs0K0t3hUCb5cOAZ8cPgV/Felo1UQeu7dwVYrWHxYLj9YoMy7cXICSDpRrwTXcjO + ybK3YgjHjbWss0khn5aV9iHn69K2467H0QNG3Ppufx3Gshxjv2p5MdWX18oVy4F5 - SR9qNv+TP38xBO6+cFhGYpZ73fDP/fbvKMwSq+EKxeJjpWujWctByAohJrHxum5p + tZEj+o4nTegU2NPX0BNuroP4rVUke0pX7T2M8tYWaZsA+tl3KqCZafO3A6lpr8r+ - l+bCrTFPrEicG+r3nq5JQyzFjHRav9TcHbxaRKfkUpsaJx/kATbxxfGOrUd2kvVA + imVvLpOFjnTJ1K7Ruhs6A8kVpd2VJsNXBE0l/r+I53gm3HWNkEK3pZLGZK1I0MD9 - 1857dr0CgYEAwHe76JE6dwSR8KUdCwz31e0kpau2Cn851sJGVTjXIR219DBMT0Lg + v2QJx7ECgYEA5f7B9OgcEhDXvsvm5pOwoaPDxopOWaHt5Os3olT4FBfCrMf/C6Gw - c3Qq/kn7IzGG9uTRQ9KRJpCdCDBi0EhVo2cEKsb/YAYEQHx4yDPZx0E8BAdwx3F7 + 5KEI/lc8WcKdTX+suBBkD/+M/6ua2VxHCnatWpXBRo1WHEgoGFGlvFcQ+OGPlm2d - lwH8+7f5fy4Cy6r1rhIX0QjRzeBkwxXwru5fGa/KZRaa9dKrR607SwcCgYEA+/KT + w2LPtFov7YYBDNhW0B6ETS3Pe9vGr2spr0V6j8fhP3BirvmKwap2gPECgYEA7+hZ - n1jb7a7QgKJPNlj0r3SaRL8gnj2sWztW2kd82TliUghfdFwzEpu+hBX37c3GMedJ + 4DboBwIWI4iQPJqr19b/5B9t6c4HRpfLWizjurltDENpPkHXTjpWvcnxYEpgyROB - MS0eQx5ccVIZgchScLoJOTG5z91n+s2N9Ni+PyyDEwBFeBzMG3iN0xWUcYjVkqIx + 7PSavd1yMYmmf1EY68MNRkmnITMRDIPQyc7mop/O3xwyS8BPZ6+8nnOfn0bLmdNT - Ea0pOuWj7bHTHwJ6xc6NZ+3HiIlyJfU083ZoqmsCgYEArZKNjRSD9FfTwYE2awPb + kkkVEM4cEZFSwUUFoqQHaHtDsgwzRBcGXHmTAnMCgYEAtbWbA6VGWDeaXJG4Mb/J - 8jp2RU5Q0rCgGbSEt1CWepAPytNPzl9Siexm5YMUkE2XGMuMiay5KF1csMjqJEpH + s0sxZ/DpigNXcp8r60L6ZNWI5v1z0XrDyT45XskJU1lg8lPG3/2DMOiUO4MW6lfv - qSA7WtSx9AgZB4r5ZhuUuCR1mnCXXdZTDgFGBECLKg31iXV5MO2yOtrIUvGeDW2Y + gKLWv1TFyLntqJaRpvUK3kxjil6bFRwxoqa0tybx6tUOi1l47SDPIjLpVFAFH56o - 7Dme3Exzq6yyPSUrQG3SvjsCgYAiH1x2/GXs7vw2L8ViqvGYwcYTAX+9bsTlJkhB + 5mMcO/CNU5O1Q8zABdZpneECgYEA2mTeTGovVxHjLX3IMCNthBNI51ZlLI5NuUm9 - D+WM1gTG73NeIw6Xupg283K8tl3dbGGxU1cB6B7FCkWCGktwEQImyOFNkcL/aM+N + 6N0sgnMCfkNvryko4yHgjO0lOs76xJFpmVgi9ex9Y/M3Cne9BAKQNwgdiO9/+bCV - Fb3OeIzYCfVeqyfJoK40pHuSVOH4FhdnOXiYDXoCO09Ip+FQ4QStyrp3d4YKNgeR + hOFAu5JXNGvqrWLn5i/ouSXwjYJZHjNuxKCa+K1oh+WPPDmlI6XGyKpNueu5T6bW - 4buTOQKBgAnWPN4EpoCe5gBJ/dgbk0mPdGv3g9CNTXD4hJak1SjPybJoEh6ao5tC + N6DE31kCgYB2l7HnVZoOtjgsu1S4PIok8o5yiJhImn4b7fXfWbT5k0gnBqpsL/0C - g+taaKrwfzU0X6BDkRnU25PNRs7/vtoSqf7XZOwUSjKfpZ5Vv9+58l4CfhJxiMpZ + uMk4uSMDC1zWRcQJSAZt/ulYvXSttth411Ljx4rzOsh8990/Br7xSngbVqUaj7t8 - qiEpJQsyLoDqGKzUjFwiwzjSHlu8otcjb9od7GO422TmuJEknJj1 + F9DHEhPnutBBMj4ky9BvqY6FSHRylmDnbx6ByalFkoKzHr2kIWzkNw== -----END RSA PRIVATE KEY----- @@ -8181,62 +9081,62 @@ vector: segmentSize: 131072 total: 10 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:MDMF:brrsipa5gkkna6tojwmc2hcrti:rr3nz27m53mcrg5nqhknc23fhxg2nz6t3etlc5gpv6vca6fleddq + expected: URI:MDMF:mnsqs7br4xbc6ghtpfl5ewzjzm:jlens2myp4td3bcjogvs5gl5cbvnxl3fycfwqyi5qqxp6ddk7etq format: kind: ssk params: format: mdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEowIBAAKCAQEAvt+w/o6xb3qswBmVFYjKQGV0W+i0kAlL5cits8Bs9HA9tU7b + MIIEowIBAAKCAQEAi5KYNdtP7AeY7JsnuEUiz/VridJtZnGytDRMoVNgpVUZ7+uR - Bdwx8NryMtkQR+mYuKsrfkOZZnK8c23uh4wgEOjIF2Tnmmu4SRpcUTqFQPNplHVV + H9RCE0FiwFBBk4hLw6sXCXMHyBNDlE5omYzuX7El9+/Nfwc6oW5APwb79cVcMYMU - K6l2aqbRM9KYP/FJY6Al3ZHmTrVIvdXP9vAWBwLWimUphZ0+wz4Si7y6SrwARtRb + mMXSsTeyQ29kvLt8WaoOGWdF1SurdDajd3AQndcbNQmYy4WMupLvGE4ZCaogUGYF - BPqoMfSnznsWZBCkKEIe/ed0/zoQcEO8+GznupMm59Kjw5035pz7leo17/kf343q + qxFqGwHLJvIigHDxERWC0PKQ6Z8PlxeQLC6GLVJDZCJRwNHfGXxkGN1Gue+JDh6b - aKMTPIWZGwxPiN9MmDKV6Ujlw6aYYAA+nq9iTUDwmGJ2DjgnYx7JWid7tK0J2Wqr + xmUi8/JftJwrU2ayamn64xnwnfodzT6bH6wcxn29pjy4BdZeKA+z6WA7jJ7JHLp7 - BBHJUrFBTSfsO/pmczmNUJa2bngUKq5WjuCaowIDAQABAoIBABPZMQ+XiQ39pL8p + nTEfjaPy48Ski45+O+if5ThvjFSah3YXU+PPDQIDAQABAoIBACzflFVYbgEuTiHg - Kd6eZeHCaxIvpa8guFrBvoZlqS7WCSS0eYQnfK3+JpdxCQdhXDc/3Xr4zpffsIcU + HmyVucQPnSQCBg9WRcS/PdXuVxfA3SZwX8fSd+316zh2dSboPqeppa3xkFJosyUG - VGyV/rOjcUM1g/wD3ZsEebscqcSySzVb6iprKdw3UqPf72Me1THd8nIS/O8MXO8Y + 8oVPtMIKU/E7ZZ/OJLELH9fDuJVDf0kh4ijeDUfR5tvcgBBX3Pp8/Kx5Mg//ys+B - r9KO7st12Rd1I5c4XdFxv/319y1UJzdbLEiBy1Ongc8XEdkPABXMMpSAYZGKi/4t + 05uOaaE9q+8o5zmj9eN0Yy+2yED9OntDYiBg0aYuB3FpbKEkXPkt502NHoIM4wO0 - l5LZ8/fSasgcxmJBR/TEwN9BtqkL+EspDWjva6p+PFmJRbNvK6QuwQ8T/1iPaJSY + retY0PPZsMfkIxYy5LC8buJhnXDApBWYXdQYbs9tcXAfUM/Vwbl2ds0Amb/PTnoA - Er94JMvb7Kb3GwKqnjirgEvruzY00EaXh0X/5gD9x1OLHxjdjAf9EZO0jWEyRm3u + gf3PNPGR0B+jNR5kjbGUJlGIdxRKKHbAb1y99qTAU3XreaCS+1C6BnTAYI3HbtNL - NEFlt7kCgYEAxiD6XZ1GGWVumCEsAyWhRWCGktqy3F5iMclqNBn4xZBaSUwp5pD0 + 1v5e7xcCgYEAvXV3a1I5/PJ2X/vgLsDb5rhitMNyvxhepObYvMBpl9/GS1zmkxrK - NaRHx/E+EPj6X2OHVDXT6MZp6w7gHm5z22At7qnezCa9PpyN2aFLPrqIyWFRzj2p + aBgC/4YaLEHaIH1gJO9wBplPHtuDZRgr9Eb6X81qBqzhpWE3hWBwi/+cF5blbHlB - PGKWI9zeB4ZTAMKht1Z6uGRMnos14Ej1fw1zMbCQ+C20IFtG0arV3V0CgYEA9qA4 + IQem6zOraUMMJiIKUCbShgqht5DLgHupEett9N54A8+OxaWHh4MH4bMCgYEAvJfI - lw3/xH+2DQZqbJqyyuXF+d+0PJlTeTOBdLfHwhWMH3AtK/k8tAxwaWWxXnJx6TL8 + SL2QJmFg8RtsRb1e/CxgvXNjNNRHNrYCEeBJuSuZShYUz7JRnaJVpaMafHId/PV1 - 6Up7JlUhZUG3PtzbV9mHhJUPB0xYddLmFFGwdZz8IAtjEoFsqeR71puGamEP+nYX + nZj/OG8FlRZMORYsDZOU1EktwdrfhXX/Jrp5QjIGw3yws5kNcl1rvmR55OKuJxDQ - uqLD4U3QeHxcGpmN+sK0dMOXQ4Cs069gOYOw1/8CgYBvch1iixTjNCsBZ6daHdCZ + YJEHLx1EgXYWKnhv1WzD/T1tYU4TvwprMJP1rD8CgYEAk3FmdYwxesxLGZnQtzH6 - NbJ86IezbWPOnX0f0XwdpRUkJbNr/h1gDwhRb2F6KpKrFVEKDT0lsnXhwnxOodKJ + MQ1QK/NrSpKxnU3WYNaxlrNdA+uRuewAl5AQTUHU/pplIiHQgA4jNc98Bry4/iUY - k5BCr0qjiyboESe5QwEQR9ypahSZ7hVD4jCR+6rokKYfx1svxXVCQyjWBXhIsMFm + l+vhEEuxdu52URledxs9m4ZauPUDKS8YY5cr7SFyBeJbAxY8xnHgJtcBUfWKmjwi - tioVyTvCXfL0QGOVjILAAQKBgQC2m2lrRx1C1EDqof54zY5mtv0Ah8e/OtPYoO9Q + sMJy+T1lUznll6Wh2vE7YgcCgYBqr6p7i9EKBThj7NF5OkGLgkdPpQDQF+4ZQyk0 - iacpqKSovnlj3tY4hiFRmM9cnCaFwZAL+G74sf3ZKHBS5lquUE2MOIX5JGk3TGG5 + l57dA4753Df1rriA5h5xTy1ijOPt/6WDe9OVRyjvR+fiu2o8W+prlOIvsfOUekXW - V8btPsBbxbKkiBn6LUgYXe2HpLic/YWSVmPs1Z3vKD1WIK5EppfRAOVmQMc2sdrw + 0NJb4hT1bYpAbyquMa8Ly6cxFhLSwq4+koxv2KyyV+z+JZeOMrNEhQVlcFe3UNuG - mvZ85wKBgDDbaHWS8omeJTojnyx7OKHo4SjRjK068sdKXsGsXlMQBRWBXwMlAvda + ZY0q3QKBgHPo5ViFrazcRe8YvW3e+8EVGhOiI6iaBHwWtv/G2h3Sb2ePjf1FPdnv - vbcX62A8vx046Jo3nnRgMwghh+bPwltz0r/OkVpcZ20CgkomNt99/Um/EcpxwJHU + dW199YvKZ5BU0wUjGvnGT/EIY/V6KK98n0SodFm+yI/tI8uREVjyLkA1BwWspuZe - sqPw/aQdIv5MC12w+Q0U/zSHAo91JtaYVjP/15uymD0bHi35Ux2+ + SYapkc7QNeQdyIq+Y9ZV/ks7Kq6XgjWXlivXE6URhBCr9xcn4jKb -----END RSA PRIVATE KEY----- @@ -8262,62 +9162,62 @@ vector: segmentSize: 131072 total: 10 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:SSK:oyn4tomwx2lbijba3xhl7rgaiq:hqgcwcw62dwdm7ayfobetecvgxau2r6gzl4u35l7vgtb6oa5zrxq + expected: URI:SSK:fo2s7nih4drxob5yzbnmv3v5lu:cbyn47mjceyddva5v3fwwfhmezre57nsrm5q37pxqyg6lt76vmsq format: kind: ssk params: format: sdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEogIBAAKCAQEArG5elAgFslXu9Sxyjf34WCmIruPJRsFDEoMy3dKGIQYFfKv2 + MIIEogIBAAKCAQEAnZs6x26D5CRQj7bukVe0NoSa86YeTbow12aYJ50262ASxppU - /gvQGALQDG1PBrHMNCVKoFcphjuP6Xsy7kw7siQ0sH5TwbtTcKPyYVfKq9SXkTO1 + 2i0am5SA+XcTLinpy0bVuSN0b9kXqkSnAPWZ4fwbFCsgXsLvNwTiGBo+Bkgwlarl - db8Lg1Ezi2rA7oV1sntfyS1SBqlMrOhby6mHwrjf0pnjwSkwvLLMjrfxYatgEAD7 + 8VVMz+FJLYpGxviWNlctkZ3MFEkHUs5SRnXLAE6fz2qhzdXWW/y2D3h34pqG4ETF - HfrAKt2z2+uhnyJx8q/oQCncoGH6zw0HiCeJ3Qf0IsV5nhfTWAhLrRqnwkCqB/yl + 1KbU/X7OSmLzgo5hw6OsyxZYZYp/oyesie/y21TeE+6LyClb3O4mI80+vco6A9vp - PHR98xyZ6RcFm0m88r97GH3JmWuj427xwydGYWYISUSR1Ae1tbExifYFUP25hTIA + xncn1MJHDLpfY6z1sLOfC7NQ4N7J2BUvabJ6HSJVEOL4LbfoBJwo+/tKDQV48MfU - Kz4Y6mkL/1h3JEFwYopFPdJYLUdS4baAm50WpwIDAQABAoIBAAmEkBjotnPJFYcX + C9NPhbV//G6T2JEg1sHcMNQuOSwQVe+hEmY8WwIDAQABAoIBAAlAGuSEjzCfXZn6 - /HzE+4vWQxKwRSBwM4UWk9y1rayt+eiPT3NncIWaxiQhdn6+mrB4LH3cQdEEgaWY + 8vrVdh6lDLIPuIzBEb6FFQduMxvb2SLXk4UZlJ3uyt1rzTAcKs6+tCrqhPWYNBQ8 - JNANijADmprxZisn7WumyQ7Be1Dvy6v6qDYHJRoLBebYriycVkpTUA65PzFZ7/8N + 4r0HZCY6kPjhyWhPlv7GRlAVjsuLmJVHISdXnFkc5xnJdMRODJeP6TPvERMTJ18r - Vl/QDEvdy5EC6JT1cpi/39Wy5pKG/WTvWHDwddur/nocgayi8eCy1zyI0QCwBpPk + /F+u6IVSGA+7p1ePeKG09yl4uO5YtHVHEynjrRbfzk+2c/6ePkWSeCpVDndCQ6eU - gzL1/LAvjfE7NRYCaKkFBUdHvXmE2wvEGu5XBheULRNnzIQ7dDVIHHZArUVULVYo + me64LjZEPMFMRlGc6GhfqcT+0GPvyWIBocHZd23oAL8cJiMdXykWhDlCX6ZkkcHr - oXn/7XCg7v2aztXN4DiE7F1Cbu0ievqUyvA8O98xX86JGBHIIodzIQ6jnt0+fkNQ + N29MILURnGBYFOuozrYPe8zrZTGUhWpWGjZQLSYleyBM5BjCX0WR6cFBrhcdFLW7 - Prx3mVkCgYEA4jWllsf+sW70FZmgeE6Ah5kddKlWg1C9szbqecHBOoLucfndbpC4 + Fgt8ORECgYEAu16bOPlKglvktH4ro4Ren1JpCUCW7SE8bV7hOG26MIWs9UACMYPK - Xn2ZSVGXYdoj45fewama9thdSzk7Cj0Lw2Zfxc5rb4i6qgfwllw5rcCDHD+HE3u1 + GdbK2Zkj9rVD77IQ5zWNNmmobe5jTqNpyOxMETsV6uRiGvhfzt1AqQOd2YR8vUKd - PT1bey4yDM84XYIqiQWSKok9tBhLViF5W3BNuZrEASeDcnMNfjQqrF8CgYEAwyOn + ujdHaaU6owcWFOVDDZrBsgftMLcHmxFQDCAtH+ohfidA/OkVRbgs+OcCgYEA11XE - neuOzrdAnaZlE8qpc3yD+/dZIacJfkSEaxDMT2kVyLrRTfJ7U6IXGGpfYEjT4WRA + wB1JfWDo/dIPUwCHUmE1AkJBEo87mp16F8e37nBXmQ1usD3h498ACRbgrNbdSqM0 - qDQzsJok/R9BbpwzZjrc6c7kR+Ct9OpSVJBVanLuoCM7CePKB6OrYlhP71Z303Zc + gJ0PIKLysM781PqE7gGAn6WeudlLZ6/b2YHh31HDD2THio0+MBVKR4Rdv1rHxe9B - PjaJkO/RgSqI5KfCt4cVe0VP+ecP82laIsznOrkCgYBxKyubgpSuCfc88y2v4n40 + nYz1A6gpxbYpFt1MrZRLRKGqa/F1J/bqfx6rbm0CgYBiUhS14urcWQg8RnDzz0Qv - 2Go/GhS4/2TYSuoFXeSgtC48gSfBj89dHnLYlmQoxSxdSXZc5tArHFWYM5qQ5beD + 6ni/qCsKqAQjiEQ67iljyOGnmD0Oao+k23d6k8excBEEOLZx/UHqqar+dLebzlh2 - 2yyg1kMzenEAbZZ0ctE8VuqA8FtQaPxkFdU1jAfoFqd5SIylHk9gzmY7Okg+X+LJ + XLjV2eF4bvukF21/Cc8iYYl1WPZ0Af7udo98un14iwFlWaDEBM9bcplelMzi7ETK - 1yZba80RUsZVNLAUal7K+wKBgAn3QTE0fYebHkau38yh9gN64Xa1zCyGzlpPf3/E + +B91vdBxeHu7uzu0aB8BRQKBgGUqsIMpv0seaphFRlnSl8EGVmc3RWc4z+H2NlRR - TNrlYAJvYA8eCiRcS9eoXxSYw5FoQFEW0Wj8hlUTCpFuksVuzid2tHvjQp8WdHvz + yoJFWYJYozY9/JCYRmX+z5OkZtcYEiSSpXbJ14dl17cf86/2GL3oi8f45MpT/tAT - HxmfowY0pmg75O588lzEa9iqTtZS3iUjPeVUChwRowoiczRSRsuT36DApzTkNYE/ + i1DmEuR6jpzzetIQTpOHBpxORCkkHQmuHbaYHPf8exV45vtt/mbCJVUNXeNmyAjt - e7OpAoGAGU+zqOQ/I6qe2f3awSezBFQK1B43YSFnYyWFUVcoHN1Gdir/7VO5tROb + GdGJAoGAL2huWld1IJOColg2ye15MQ1f9Pyu+9aaghglEtGrvnd3VgcH2cdVbsHj - 0szku935s5fWqRydLXT7W7tRpfR7DB4IIVMr92BaKWEHPZdNBWKYddpzOzEp9qWo + 5Sb2tSgT4m1N0XjT33nYCyTnwLS0txPqDU71KlTLcCfH3iPO50ppWbU6qTi2yUFj - QoubFJCdM83ubtISMH/Ovf/i0rVgSPTGg4WG5ow6Z2RJ88ruJEo= + +MmT2Wy5/VgrsU1Z9hw7+FbXNeRnPNns4l43PE1TlZ3uQaBSF0U= -----END RSA PRIVATE KEY----- @@ -8331,62 +9231,62 @@ vector: segmentSize: 131072 total: 10 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:MDMF:zjjxvnxrs4logkm6fzgyg7cxy4:ogp3chqbwuxtt22jfoasxh6rqylndggift47yum7b4q65qts2pca + expected: URI:MDMF:6nqjbopsod7vfscqp45ndtfhna:ef7jrzptougwgmcofxtu3jrjgvfoib23tms67m5ssugrmp4ddqua format: kind: ssk params: format: mdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEowIBAAKCAQEA646A2Y0zoJFYAnAslkocvo/RcuvT8xyjTrjL/3GjWQS3/udV + MIIEowIBAAKCAQEAuyDM4HLs7yvKQRNJvNj7dNcszbLrH7da2Qzj+OF58jDB2Wqq - xniRm0WMMIq+bWlhskXLyM7/KNDNCV+fsnKsUgyaXZrmSQOiUFzBLWFhAT17yCwT + 9ZQQcLowoltdfRZmPIADAYy16w05CGa1YeFI62ukt7KdZZ0sbGipQxaQShZzeDjk - T9Xw3mA3P3uBSn2PrnkC7XW6wgV5jT+di9td0i7C43k3QvV/X/hXZjVFWpbSBKbo + 6sSFyN/iQ+v9ZX0GwSSLCwULZ3JRbXPYO5fRSa1WAd+OuLQ7vC6zyPglC/IyKEXG - f8NmBjG1gKX7ddLoSZRxg8wc+UJJOMrXF/z9wv0hPMNG/Y2pfcmNiBNlh/SJFu2C + i1tBPe5GKboT/1iKUjB7CRybFtERRQh0wZ6L5ChnLjpGJHEkz/Z2WZcSwsSvdcb2 - +pl32nrq5cCnJ1r+S7TUd7ccaX06iaJWVsnCPAupy703Qpw4rByXj+4ZTVuN9DyA + gRTkquIk8kPMU4BFZofuFEuqVZVe27sOeklsPgeCiFedifX+S49UMuil5zVjr+VL - hrUbkzaVK5Udt67wRW5QkXewq7bC28+e68EI8wIDAQABAoIBAA9W9TLK/GDepjBp + kG2/ZI2IRU3JPA62RSlNrvtSaM9Ltb6TBnpZawIDAQABAoIBAByL/0VEXbSKOSNz - IADc1wNdN+1icyfX+8CNeJIW0zrhpkkeGoVgEPy0v5I3rs/5jAhWfZStpOCCZx/O + +6fwGsLzKoDCM+VVlZSOx150mKO5CsbjY2uYaBOntntQ4Ie7JDiZjMaD6U1k6ded - +jDeQTTfUjwknA4xg46f2QqnVpsEkdIxde593FThnNIbAl/t+P81LTN2i7tv14Bo + bOae4Clt9VBtfV5+5kPZiZd++InHjVurTkd/AyvRn7/IihJyoCup1VOxNCasBWkN - c040x7wqGbyY27hAvCiX8mUbCfxXSa4+PPfUzrPeroaDawCkXW2Y6A5Kny3peX6m + pvtHCKSqk/gwMHfT+FhCcFHkgn5PnsfPbZRDm7jCIA2gaqmSISL4gHjJginqKiSX - m3dYPGq2cBRxyDr87e1enOBJeBXCa+AbQiuExINS6kTqw01pczAe4F2ZmlRsa8xn + HtE4E+GO5NEkzIiNQl3kf+vcEBOTYgj3YeAKZhqO9XDGLDbZ1XHmWV8y9n2RWsjk - 4hjMu8wdi1WkxFLMswjhs6qaXiS3A5vduzdojjwNyOVTSl+/Zfdi8ETFFKTb79lN + ySZuMuvOYUQrp8kmnAKe1oRGrmRLexLwHM++raXYUCHEauGOirRd+e/yFHlWTA1/ - 2kRPhmECgYEA/tjAxk7fJwCQD6u8TM0Ae2A2qDvHqNJObR5W/77LHs6OIZw3GXdZ + tuqjfXkCgYEAxV/pHECWeSJaZSyY4ygO2DhYlPPw1YXZWyZnp4ptodyRj0SPHoqE - nG6mXFPw7OgnbSzBB3TZBuYgwBbWcYnp+5Fp3zsVcK30kOBmDVqedEPOasnaMHg2 + ucDjjnQOIvgXRB2ohTrEuf0CTkJcnWtrgaWXbzHxMbLX5xp6IXeZyyU7Ly+JEzY/ - voS/JmpZa0KvsF/6hmIfVK93mNWQvAlzPA3/F5waOd9ScC9Cr9ZYCFMCgYEA7J9m + DIZTrdbmL5vfq3pfTHIt3ea1bHG/Q6CX4ehjnV5NmJrbBpq4ibWHbIMCgYEA8rXC - +cSTHcfLl7QvZvm3wi43quC0ejLPYqYl73wVSHWEWutTTYoNfJSol0qwEfVlvT0s + Kx4ASPmvIMQIyRrMR1g3BURiRBBdJ6O7WTFDBfUiJjnzqSo2OViZbepXMxV0h4xe - S8TWlVGuVq+zlMnaqnAHfLswVgY/lFPJt10nHrERdgqCJR5Kxj+Do36nhRkRZfqh + gMoRaZAxEWc5G8UkH+tL2WpneEA2w9o9EjFX3aUz5e3IKVIys1AIIdof6r6hoFV/ - GyztLKe9xN/jY3zhYDZhZZ+tdRUd/VzmdIcAaOECgYEA8hKaKsTIm5ehQAF1P86K + /oQMherBDS6mxmyS6GnzPvp8UmuCz3DD9HSpmvkCgYBotxPkC1hJ+DHhT6HlkpEd - 4qalxG/kW6xI6sWjBhMJhh3WTH7Cp+ICsOE6DQF/HMn4iW+1e4u2iyMVgOEwmXDT + ofdNP4bcoeDJfTytJMI5h94qFoOf/nmgW3ffUi9V2i3t05Ze6OkKi/M3NfoRArbM - XS7nTjAlUX8rjGJbDdxCH1Y5QJ60Ls5B0f7uQ2NJxOT3VaYVpoiWEi8Kf5Z9gN/J + 19/Z/LMsXOgzElcNfni30I7v39ZnvPYCXRn0Nvl09MvcHFaHJmSzP/2tBUQmSwOJ - IgZ5hMe28bn76Kw7wCLuRBkCgYA0NyDEMSq9wZ8dxPdI5AY25XgHTzrEVH4LKNrq + tVN0YF3mwvHFNT0GwqqQpQKBgQCr5Lx6qwnKpUM58nzCaT9KPBjjmxX5XJmNLHHQ - NBmGOdiRL6jcTYCYYz2o1SRxchOXZO5ncfJgVPwByRf512lXfw1H6w7JjOtu0eaL + boooWv9vkVWXdnTm0m//n5tYa5aXNXvsvK/uUpfd2nxgxZObI5sZhTl4ugnPVe4w - fhTp4u0VfVAm3L5nbRChfYt+BYAfXuU6V/mmhwWLclR9WctqLdXkVQ4z7gsGI//+ + x0+Sg6Eo8+nyEewkgMbxqrk2GQMBOeynhkAUTDmjq2mkWFsHTZpf/Sk1ej0vy46M - 6uOeQQKBgGmGvJ1yr5UCw5/rzSoRwGlSFGv9BF6UpglnXEGYtTUKaY3HISfYbd2Y + wF8qiQKBgA78rBWa8R/AB02hKh3CrIyoMwl9WwGAWRvJUuxtg8tXehMQIX8c7Wtl - seHdl2dQT1ws9Nv1sID7bP/Ea0L5Vdzk1w+GMoBYctAyWdyiJs6usl9dpsMk8Qpl + TYNLuIgitz7X4kPIxhUkFGH5kX9t2hdFaK+y5cTLayZ4Grf4poLVLFQm8rm/vApZ - 78pgDYDloeGjA90ff4tRlw9UTkx9vCWBXa5c1OlIZxb/6Ln/Uf/d + 3wP28XLiz49qZzx1LYE/n9k5V/bpQqSib/RtP0lz6Y7mN9RpRZKd -----END RSA PRIVATE KEY----- @@ -8412,62 +9312,62 @@ vector: segmentSize: 131072 total: 10 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:SSK:qy3zzvtmugghvguwl3phedeuza:a6zdrpyep66nwgxthmzgcy4u6uoq56qmkcmrai7yaxjuq2znywfq + expected: URI:SSK:x2ypcho2ngz2yydnvdfg7zmuam:rnym6enrwa3tqqrhmgyo2wqdhmxfjn7wpzjrkzjtqibazembi62q format: kind: ssk params: format: sdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEowIBAAKCAQEA1RbLEX5lSZVslOz6qEIa6TZ2zoidqibxCZ5UPldKiTNeh9G3 + MIIEowIBAAKCAQEAwfmkPrdspB9pzUy2t5MhIH04ydpX+d0D3pHDWyQO5bcyu0eL - Ecv36wClMPmBF813yJJudyNjhYM56M5ohbaceo2E80mZr+bP6STUtKlfJh/sfe77 + prmWkEvPlcFW+ze0WTsp7bBnvl/ZNm/4ATxhq41cB96l6uROwL2jVOC0sJYRHZFZ - BdhKeYh7XAlseN+8D/x5ArxssiQAKNLAa0ked80aaRlF7CNF4rEwqKSlnyFXB3q7 + 8BplNO6frfwsvGCVmml9cLj587VP1k6tk4eHglg1pK1rVNg5o/FzkmieGGDW/ynM - QDQGZQrzSlpLvh1g76NcCQK1AYMmpu4XD5CUUx+HGKo4bFJGp5OhCGMCCNj+uocq + 5FfsrCeWbFh33pr4NqrHHnR+VDADFwhYzHbMlWLOS6zujSoV1Cbum1/BIBxPijIc - mDAktiy0uIhI6K/vxgv0FVT1tTmeo03NwoWNDGsVNYBdW7LhDEoYr9/221H6wRp2 + MxdsqJKWyCWA0mSbMXsV7a4altzIWax8fEbx8oxCiBPTyNFhQNW5sB0NcwQhpMf2 - 4lEaFgANEs3oioM+9iCQZSGVprOpmeqxzJCBrQIDAQABAoIBAD03yyc/dMHrF8LB + mHEuuTlMVw+RVYNBtk2vrZS9pOi2SdfFInghGwIDAQABAoIBAB/RcYHWIwHYTfRz - QlHMjAasCv3S6djUTzNANVujoFpCU8oZScrnGlZ9XPfw9lFsShlpWCsKE7FrvdtQ + /nl7WWY+aYJ23efUZg5Bbmr0rbrnA5vZHw0Qs/lkbh8nBpERG0oTHIwVhoGhsFGF - UV7404Ox3Jw4bNrIKLsGRcWRUzCUw1B6s8s+FEdOGoKagntHa7P8CJfsoh2bkiAo + KV3D3VOUzav+zMw3JLyUqYZCkRvG6fTQgxERtgM/lz16DY6IRfynttYMNE1SiE+O - S/eGjiZE2m2PQTNR/uXdmekZRCuu1ioJX8zQXPTnF9GW41ss0zGh4ntfRevbXWUu + OrEQp1ztV5NKh74e9R7cJt81E6XKHPYAR1T78RqRiz8eTrfkECI9Ad73043NwRXQ - VvtfJl3RCD/wj7lDY84SteLp5ih/IZJbxBLjKM6tancxkU88T1vIun/Opn4nRDje + 273Ph9X0EXKZh9RQYNuLXlvaTfMTU22JpPf42kL6/GlZSY7Ex9ldaQya1V1TG8zr - vt+V3EVIBPEuaIuAIbxU0Fmrs9nOW8k+D7gK+rDM1gPxKR7sqt17D/Kv3OB5Pcs5 + qmcTX94hg+TMXmt6xx/imxt/DA3JOqTpIrA14acmI757uUX6FmGHI/0ATQBL1Cr6 - A7v0KFECgYEA6FZwpBPed8JYWekHatWpJ2gmiLvU6EHUGNv3LaaOd2NLU4SMXd30 + 8oY43bECgYEA1lVDh5cr7aGRIcomfrOHQeUQgP4EaVpOfF8VigcTk3GqOxfSKTND - e+WqZKizF3UhuKD/T87YhZ8I45pipoTo4flHPVz42xdeLEF5pabQDKk3KGNEFVth + BqDUiOO14ETjs/qV72poFXdruRNZDEBT4w9MwZV/J9K0eUtNTnh55X5BDLIG90nK - KM4EpLlLntQjAmFlNrzfSF8l5ENMk037cmGPkoCPlBTdYB1A3WQt43MCgYEA6sp/ + yAfcdh2PvVeDBeA4hoLaEsumgBy3bWr2WfY1O79sPXvoRWJT10+tnesCgYEA5685 - tG5Ns4wSa1wC6qtt4jRelanwfOucb7yONuN1f7DNbew2cPdKhkOk+whmGTVkJGnz + XDtNGlz8tSUJpBXmaAAoJQzbvRjOtlH65BujTFBWiW5XXH3kURUlCOYShyM2Age1 - dB862tY50/3zDCll8AyCRqzsQLsl3tQAUo2H+0J1NreKTaGtLpVPc3CflM/zVx8g + S01e22WnpgSlE+4DbrV0MVB3dBYLif13B3+iCOPZ77m2uO4kqB870K5wINBMWuf8 - G62lxq07a2yWGCiZuPjGiyqtiCxSbAQ2U2Nv/l8CgYAEZV5MPHQBIBQ730TcqJ5C + /mv69nPhCS8Mqz99JOGNl2vLtwQzrkfNhldmTZECgYBwTlUIIyodZd9KOUZadW+W - uJ3CCIvGuTgiIEdU/cnESISsV92wCPsPPRE0RlzdHMI+lA1AnVFLde7dH5auP+WI + E5TGQlPFcFBX0urSXErho1lzhVPVysqAGp3C7K5MSUyW7eLKhJLtTJnhbEXoqXxL - IQdQCepLeu21OKfsknNtSeZZRUeMf+Yet4cu9rKPlsPyz5TyrDAtVl+JKhzQzLDt + KaUqek8aasmuFMr5Jx+YJMOpB0+nG79peNUH/w1mRQied5KmyMHDv3oK/wEOEFHt - QRtOUlBlJN/raaJIjhSwMQKBgD1LUh1zclt+JMzcP3KuAEi+bTbbH4otJDDTY7kW + aZkTKYZp4Rcf5BnSZCmw6QKBgCXXKLc3uFAl/+BWPEzghtFVtTjX8Mvh0WFV4nR/ - lnUYXfjlYq0JEe6NOEPExIquMo+DDWhyQrYgmQYr1MiHAjKxwUzcFe0sLk3GwLLM + TxyXwoqPyxUAOtpDadkaOsx3o8qRF7tE18ldwRQMjinDJixe1qt3SQtczmWrUFWZ - egRxLBJ1xehQXdq8Zfp4G2EJDLjgykwPgCimzs1TkreJ2d+9Km/oW1ciYv4J93i7 + Mw3gqSfOXVm3C6Wp9EsRMp8pZk8ytM+ZM1QteQPW+2q84+OyMz4YDR3HQemlMJxQ - i+A3AoGBAIL1rB9kv6P1z83WtT0KSM2+SfML/QTHF2UpxMm9LYpci4kYxX7rWiii + ihUBAoGBAJnLqJG0ckNRnuaD0SM93Stvx5xOu3g2uzy0zJYjzjIQfu2gNNx/KnuF - yJbGoLajm6PFv8+gX8gUnq13FWrilxTmO6XrxECyblueP0M7fm5C5sl4etqMtxJq + iOOJID+Y5HgvqxXjIuRzxyH51rYeojVcouSIxpkbShDkURDSS6l+pfEtRcH1G1GM - c+DTtNGadm2iME0cE2acZCmg3x8UfYfzTvDM1IqznNO25XvmoHdc + CR9cLUiNJqUgtyLxrFljyKKdfhpQ1o7o7DBhJ2C5xwr8WEko+L5m -----END RSA PRIVATE KEY----- @@ -8481,62 +9381,62 @@ vector: segmentSize: 131072 total: 10 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:MDMF:s4yxatb23wpmyoehz3c33zqqbm:dntfov2hjyd5o6hekmkkuyjgre4n3spcxgu6oq5vcukdula3j4wa + expected: URI:MDMF:thtsw4tgtfmq4m3ajpgxuodvum:64bbbja2hsofykzx2imjzvtys33xhinh3kuyg274bkyuknwtobha format: kind: ssk params: format: mdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEogIBAAKCAQEA2fHpyFtfnt28GF+kqmI8E3eCoYsvVkhZPfDlqR6pWt9To1Im + MIIEogIBAAKCAQEA5PUmN2ICIhg9Oj1fJgL8KS15MQuycqxSPnRGryj70RgoY/x6 - +G7NTv9XTsidpsIq5fG3G3PA6gQ9jRrKbgD5x5te/HXr6XaWzamyWlCyDzIAHvGv + 8MguJ8Ofz8JvXBVEOzMLT4kYihyce6fVPTzNv54/ln4Iw334UDEVmsY97hNf8/tl - LasAfZONnyavcivpN6Ul42OITzKmN4VYOneEv15JVC2lhTc9cYdeNxgT3C6wol8M + EuksNVofSTmUOQxPrJVeM0ezNGFJTVEPTPPjhPjFlA+ANox3h3x/W5GI1zXJopiI - /Av9ILns0C2IGg2+2jcvxuEJ2LWA0BCXAWFTSuVoUbNDjt6u+kBtGQcO5NN6gi1M + llMmpo9Q+UhcP5R250bchMJ+wxK79bwm1HH/1tfICLCeXnoX3JT+ndeptbfks2j9 - wcR65baYaFNLY/q73tp4JvPp/sCWlByOxRfK3cuZH/zo09mlN0IEkbvV7T7vNluU + /aQEz1x6YLLkLJFWVZBrCTwbQ1GMmKHe/7fwis3yUknn4MtzwzeN2a+KbNPCa44W - xu8izxAeme+Ywbk8fqps1jQFZNLtivSiMj0HdQIDAQABAoIBACKqdA/ehM5CoPqp + /77Ono+sGlZ+JfdOmAr+eGgO2Dn1sMUhZWDqQwIDAQABAoIBAAsQabNdchrxruvE - f99qt6PgXwKt864Vh5MEIFYu89YUmVDsLhmsRMjGHDos9nKCBjZupObfbwsUo0EF + kXeFx2e6AdRD63CtMSBBgDTwtxKIp1MFnW9LTSewxWVF0RnTEUQHGHHUfzIVZd53 - 8WE06NtN2c/DO+51t+qSODWzCddukblaTOMGhOdJ0ygXjFGB2DB10BnOiO279ddK + 4s8dxBeRbyM3nfbMfJZreM7M66s4lnd025KJYBCH9WEVfjsvhB4j7bRur5NFbERn - mkGBJY1rhF4OJ/qRUkXJYnwrPsFxz9cqzzT6faKWwnHQvH6p+1al/rXmPgqk8t+D + OWUPmBwR5YJdKWX5bcFHW+Qx6Tn1EQoYnOS+e4gYbf94tK7l0J1tqJ9h2LOo6yMe - QP0UFm/YllMUXvd831I+arCDdX67wTxwsaCKzUOTOIngyy9zWMlanaI4EqgoBkrp + PQctpdwRKAOVneENboh1pdxIHYXwziayEjuAo8QaPkF2LxroJASgRG3o2aDg7BUo - E2wwDgnx89IzZ5NTncQth+b5QPLEGjUWoh+5fGgRkVnVYws9/lGiQwU8Q3dBob+4 + EXFfvhxBzFNe9zyCVrBp7PfH740R2CyT5vbf7W4d1vhN1uqhRoFLkVWW12DEAvD0 - 6Mym3wMCgYEA+E9HM47jweg8hrItW92x6PC77ZmDNtuU7GonE4khayiNiRWwleea + 5No/hNECgYEA+0wtYGe7c7p1rFJHw88ngj1t9gWRY4H1JcNZ9o9/7ZOFQRHG3d+s - 6jix+XKqOihr4CGIB0fO4qLW0gkMRyezzKidkMWDGWC5jOyIqp5tEYvBHiI14L36 + HdQOmNJ7wJlhIjp/NQQT5TiffFSWAaPlD1oUJ63N8UupyEGlQsONnGoA66vOkXVh - F8JGJbWWWhpUexlpj1z0W/OJouRq1PnMBuCTC8nEQ7K7fiXzt88F+ycCgYEA4LHj + +Pb3MhD1kH5HDG5uMp/BEIJMVDJnf/O+8McEEbvNj0Jxf3ovM0oGohsCgYEA6T30 - gJRhT20VhT42cGZbrhRPqasmuZYC3zMYArbu0Ji9AdU+10qYAROe01r3Bjdbo8lo + QGc5/ZDXmDZDbq4kaBL9WiqaGj5RLMluYfLDlsW0pxOO24U/A/wP3qVzNNjlqoFN - tG/3QuU7zvktHTi9ag9XBZuYA1Y9M8xbZRJXYj+7oSF2neTI3uSwMjsyEEUvN67B + TApi/x1TUFwaQJ3pjiKBRTO36+i8yW0Kyd7JnMws7ai4uNYlkD0gUUynmaiYvEJ5 - khAeNhKW/QgPvDB0PepF+YbzdkfanLuXoi9B+gMCgYBNSuAu/FuJEHFGvE/CONAY + QA7sgBuDhK0aI+hSoGCU5gPNv+ErWE6v+VTLmvkCgYArS22J7XU7NAWwAaEBmEAL - YlcdLpvZh6BjtudS/WyZnpXwBgBhqSZfoiZEL50tXUe3DLj7Cy8q/OVBm+9mdsVQ + TUATodPxm+M7dVObig+VQ9QyaLilYzLJFM7K/4B4pzQ37HIcFS7EUCQSDJSnhbAi - /2uMlO6qB6G6bCZeddIdlBMY/i0nN/uRSbfsJQoYIfoKF270YUrvFG/TdKaMhPUt + G/fa+jO//bQrnzu0q/JK32x3Letx3hJaDVp7UrasBUWCW8g6ipF9oaU64FA6mCju - btpW4Qdmy0vxiH7EyHxkIQKBgBd33PgoB0Xhcdb52XvB5R94dZ2WB7Roi6I+Vuqp + XKtTztJUezMIrmlRYdCQvwKBgCoZQZ3iQ+hNnWxe1vsCOZYDX3FH4Tq9Zr9zuBW7 - qqXU3iDb4fVgkCHEp9kRbi2TCJpBxhLaguvUv3ttoR2lOHtkYMVwK99lWX0Ygg87 + 0KvFEZ9ae12KBl68v0yLhmjSgVmuLvp7oXS0oVYO2boyBnbeKYEJHbhZ8MFWiiz3 - bC8R0woQUbBKHgTRw+lrL15tq3HYadVUo6MoK+b/uY0BTpLM7kQSqUkYVif6m+rP + pmJDxBQ9cOID3RHUxqGF+XZVpQPN5761MuDIlot7Bw3WIBvMcvO1Wgy5Iq60vTR+ - nsd7AoGAXiBveJk2fbLBwgXiv4PFjI9tWI7IQ1ZobMsPW1k29Qj42U4F+T1K8BtK + pqVZAoGAbZurdaGhKu9DyJroKTRRv4Sk81vsI4hNgZuyE8FW35vfJf3XaWVIVo5P - lOd7UKFJr6uGwf4zUOtNXH5lur3bWh4v0iJYXro5vxomKVo2ZWNIJ8VHiPIpqBDg + aOtH2mudq2vws0C16/W5fq+vwLa1/xY2X7Ni4ubRF915BHo2zTwR3K5dUZKG8QxP - KyL8fAZQmRj9DHqUlYnfshwmAEG5zoJqzqWRL2RPRYvq1qAclI8= + hHOaZsh6CgxL2cLCiUGIWNpKkcOz69GnZZoY0KJS0KazKTDGdMk= -----END RSA PRIVATE KEY----- @@ -8562,62 +9462,62 @@ vector: segmentSize: 131072 total: 10 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:SSK:seadx3svf77hkyujx4ebons66a:mobkn3t3tbxgwhjria6yxho2is2c74d6dis6xbwxxogxybehcyna + expected: URI:SSK:eb4hwxibn33xlwshgqgpridd7e:7ca4kdczkgf5dowmqrazebqydxt2cmb2biui53d3etgy3rlcwkiq format: kind: ssk params: format: sdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEpAIBAAKCAQEAo56YIQmQb06Q6InhqE55cyTRXhi0BfUYeScWrgTXDB7K5/PJ + MIIEpAIBAAKCAQEAxxM8zLQ6lSfHQ5U876H/kkiJ4+xxuvH8HNaT0EhF144PQyLi - RhRxTE5K1KJJ/cHOf1nxauXWnWB5N74JUSEIPu22mHgV4uJc8ktJ71XBpci6v9U+ + pYDMOgv9h9nN8Da9w4wviZLj5kM3kVhPF1ElPq3JNEW80o3J3w/5hTX/B0OpU7t9 - yoPM3X5/XIor+zGoFeFaNFAIaNlnItPWtrv2FiL00smW1O3JB0EvLwN1MKKwJNEE + SXNcKGOftT9tYx0+38LzH6bbqQnYAB4KFfLyus1truXeJMzIu/3pLoINNT0My3ZZ - sBnIpmmUqhJn7IL0DuGZTDOpIGyq7+jwIOEzoQ7yN4hwqR8Vj1xVKorsEe/WRdJt + 3Fjr/+Pwcgxq59o4atIJlzLILQo3UwpofAwBhD1X0aNiHc/BYxjMdAY0uJABvwxk - WrtmAIOTppBDenye0rXtx5xjBbJd3wfnUTM4RDbRLOvhhqSWdwDis6oQmJBN/6kU + 7JfC2XckIVdTCtbOz4b6m/HoGU71/vdg6gRTmXy4m/18b+yuEE74omovDXa1Exkg - j70Tj6AOECQyghdAZy1WJP1X3ay5wubPgm6t2QIDAQABAoIBADMnbsWJcXQzOn/R + 1DroYe9s9zQUzLZABn86xUZwR/c+vt+qK8ab9wIDAQABAoIBAExNj9bIV9H2wrYh - N9FAc50JqkGCdKoWJigejesrDTa3W9Wn9Mnps0BZi/CtqndhA9fx/VXf9LiwREWm + PA9/cMWBfzS42mi0upTVHCfPo9F4lln9w5B7Gww+r0kETx585ORQVaIuBqMp7WEM - rtAEBUlzVW6WwLT183w3CK8AfzH/L0+xclerXD31ggkjE7wNmtD4axTG3tI1Ahcz + z5fY1uU82CtsdXDgvtj8Nv/7j8oZgYviB6YBDPhAIyVl78f3HDPI9cYSfww+BSga - 5sGrwzTJigRqzTLWAs83VHKc4KMrAebPS/SvRATS8f1OA2rHHkeTlHg/jRbr4Gt+ + W3RJQAcgmSNZ4PkK8v+3VUqpt2VJWUHOHp4CETT3aL7JZpyjt4IlUb6gROey4XAg - OQTsHomH2VfM9VBgB2CdWxluioA2B24OU35h370+KJl+6NmBxeaxiBn/Vx2hjXBc + KubV/WTjrHFbS9RyFiFD+ZpD9eXnGand1yUemMDy5N9gAM6MAdYHjNFaO/fq9vGK - 4QunZ4Vk0pI6ioSwym76C8rVdwO9wyi+IdmeEK+lNbggY43P0UGE0du/v4Y5wFQh + L6Rr/Xf1vmDMJ849kCEG4rOpLwMkol9v6L/eO+xssuvPRHHW/BjCbFTtkPAD99vS - UTWjey8CgYEA0c4t1QCOyjsacTwk6VgsLROte54twxWqcer13mBlQJlOpzvcUZ1l + tW80Co0CgYEA1KavoQqX+ApzTahmLON85F2RkZVMAFmNa9PYzbXrr3pmrFJg0ac6 - LB5ck1hY3oqWTl8aTwLuKgkvsK0lhiqN631XsePwoEOiwzxLl4wFtGSTRvHwfBbX + V+vPYBTlaJ10bDLYEsD+rpYqiHUwXsm99yHBxfhRHdvQFe5CfjLVQqh5p96PMta3 - EGWAAx5mWt6r2reZMjZeKymp9jWbMC8mbR+XdyyTvzCc7eG3O8auFP8CgYEAx6Ue + Eraiy80cS9287VTaVV4IVJarbn3BootOyK/VGij5YNR3COrrr2kDgiUCgYEA76gV - CD98ZE1S0GPJ47KZeHpNeOu85zq0oyP27g3b6vahFxfBbCZNqLA6s7yMLKRyx6bg + AIOEuGPteM19cUYRzJh/Q3TlDDhc7jK3n/N5Nivtfris33AorN9lEtlQ3i/Gk/aY - ev27ja1ANdCvjAlATSM2AVKIYcwRE9Y2x+uatliw3Y9SnxadhpIwDrgNJ+OtAJWq + pNUNFS9K0PW/8u3M6yoGDeh+w1wLb1RHUTamZvOJiTSknb5vNZWTqFx/d16WvhDX - iUmMQgqkeGEyRaWBCFab2ED9b9hk9/3tTB3/hScCgYBmPADLXWU3GEvPR854wlVs + wP4QF5YKVVRdOGnrm2XdWlsbwE4IlMWna4NSVOsCgYEAvQx/AOVhGzN3NGfshiWr - db1AkpicCm+u6R58CR7ttobEKQA36OmG8RiNWCyd7IxHjkIkpDnn0+ggQI8bbJsR + x33jxxB6Y6k5j83jZWZA5F0l4DbQOjK4LKfIUbviAzJP6Uz+SRXolR+NKok8elhS - WFemQHtdrPegCT6Qj1OsTqIRnQ1hekO8IqmZW3Pm7cByaKrG9AU5JSlD52VCuocP + GN2a3jwXKTtc79JErNrWOw96MCItHl5CnVFew15StKOprTiNbe1N7J2SRIVqWu4M - /6fwE5G/RXIC3M1L3ImxgwKBgQCt+Ie8Hj5yVSMmLt7eCWNNJh5ekeZSBMkmJI/o + GWAwTLR2l33rYTMwWl46rz0CgYBX7oX2MEtMFG4XOt5h52G0feeD6qn3t95xD27M - D7GlBXeI3Q2TBanEppTwzQvFVyQiMJwK8RI/ukpq2sguml0rGtTTwCzSM/Zpt9CS + Y1sAA1Iagsv7F331H+pH5jCDtWfY9ku/fuRT94wt611IVvQu/LZH+Bw6tdUEPhoE - 1A9EePLejycrNJTekINKQD5OlUrLaKBr8+hCIG4D7IbXRAq1zmsNvkxa61HI/MCN + tFaNw6GdFBGqRysqr/0DcxzZwXzxs+BV0WI6JTUZZeDmSAbId7Gl63PdNUR0wajS - BNMGHwKBgQCtrCi6yB3pVK7Ie0n1KlTJTu3PDlnJVFvUs+tqXnf3HiueQVkPKVMt + C9bzjQKBgQCxJZxGJT0MUrX8BzB/tRcdPWpFhnQtDENMdeunArN27KbL04dSmrZZ - zG+heN753a+dWnOUl+R/oKzy3DtHIcdmGtC9O2/fs74hJiojBLn631im6zGYYNJD + 1V+H8GY1bBooSyULm+H2fouuavnHN4VHp6DI2JJ9wcxNsY8l8NOrrwh9y3sexWFJ - MocXDdKBilZtVQQ6J4fGCMGB8lK2PZA8pwKBctMeOq84CwlOkICElw== + 1I626BJlwuFT3Vd/0IUpJjPwQSVj1On2cAMTfsd3y0B56H2GiAXcjA== -----END RSA PRIVATE KEY----- @@ -8631,62 +9531,62 @@ vector: segmentSize: 131072 total: 10 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:MDMF:rqdwy25h7srkxzovybpxvke44i:ccx4lh7qe6ce3j35zhdwdu2d6nehnxv4j2eymk5bbiwlq2webngq + expected: URI:MDMF:5spgmqz7se34bzntgo3wgnqb34:zbzy5bsn6kkh6dz7leo4x3cfoiuhnotjp3o3n32nj43rkpoee6ea format: kind: ssk params: format: mdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEpAIBAAKCAQEAobqIH06fWJed7sJbSsOvf+tg0vaNlWqdItvMgOOnpUvRCPj2 + MIIEowIBAAKCAQEAwi+iQ/lFevdNoN9SEG0Ie3i85CHwJAp+ZcVxyybkesM2bDzq - f9lV9U0nK6fU1IeVwHv3dLfUOYGPzVaSyiX5GZZo6IQWxBQ/8ejjjrCU/a4Y+TZn + 1QQIfD/LZ3Wk3BLxJNviF3HPVAEBjA+EK/vI1hsw912YAmClkt3/YQOIVHZ9DKHk - BI5OB4iIWKnEPsyrEdtImiEewQA5BAgajzOU+SCJ4B0uwKkpW7LRhIv1lqDKHg6L + iNejKGC8ViL8YHABszBlnPshVB1k/oXwBO0izlwP+9hUGXbHjzuksYkq0FLlGAJD - O0TuAk5TPaJDpSnpJq1d2oxle2XWvQPyy+sQxdvS8FL6o/yy53MDjP04nyB/pq7X + bNPvAunUeC6NrSsA0lgPDoVdMop1dktN0tOhLxEauDOQu3YpVHLHhPVRpTQ8c2S/ - UlTXo7z/e0Px+2YCM2sI2G984ocoWHdZ0FpFHmnJk23MIU23918JcrsCtIRM52Ss + C4JsvoQzT8BWuilfCqjkua2Ul0oq6mTSV5nVE5GFxcxghV0+5M5csQtiFm0wmxTp - RY5HR9ObfL4YwU6fiVZ1E3kJMYAcCJ39XbmYjwIDAQABAoIBACk7eb3lmRmImiLH + ZQFGGdw8bGMbF7Ghn+k9wfPnJroSGODuTGyUqwIDAQABAoIBAEH5wiVbG2awgGEA - mWfPyRwnYemXI1SvOD2tZQ+NOu4ZDMOpWYsR2WjvUSe/o7LFmIfY8ydmQKyinAuB + jxbCnMeqmW7fMwJjyFcWkteFgspM6gAzYEv4f1OLrzWbDGSzUNgHlxUFF36AiwCF - YW45TS9ZWgjBuF4oPX9K3U1BNtMQQlyzIoOWVk10YTKdoaNTIeAtFG77N7CEAoVF + ww/Yj39jJKte0sc4A/lW0K4q75ZW3Zy9onJ15VrSJxsS7vFrDMDPWC7SShwUkpxB - HaRZxcbYJV6mggdreVhgGCufVS8gClTplSk0mKSI4ZhyUMtOxbEo61Uv8R8cM3Ps + cG+UDCfVsp6L/OLb7uh0yLuDEZdOgnkDP4PD5v6bhccxm4VJK6LmU1+ozeFVoyR8 - hdI2nI3nmbjCtk4ajHIGa2iBn44Wpg8bfzbiBA9A3lDbExZkStgDY4xuPBY/aXFz + Azp/kCPwBjfs0qZ+pfyR+0uQlgcOqW/JcOTeGk0E0BKHxYws+eEZS/FpPEeKYUNs - wCg2WuuOMywUhPnU6CpH8Yj6gV333wYCo5rfBf5Ot5KaRjVwzAlz/tzdu+cjx60F + tgLZAcuqqhniijsWpvkhcItD8wn+Bl/6ieVMDdo4qx1weXjlTx/CVKZOcsBNVULK - u1znuFkCgYEAwXEKT6DmC7aiGJ20jBjtZNXPfPbE/S1I9Gt5IE0BW4MvW33sIcd0 + PTDUQuUCgYEAyVVq3ac/Z30nvDJbrvg396VqY6SDQF2QSVbV995ZKRofxJwpeWET - e9skcNNRdMgR8mgm3e9XU4oa/UkhuwtLdGH6vcXEE4r9039X6yrpUdZEZuPtgr8P + QNoIBgoVIVxbk8bOs1ojAIFyPkqtIuunkTlsaVRfdopx/choA6mzRHed4YQOV4za - 6qWVfSN2Td2XL3h6UTM8ivjJCb2CwVlds1a7g6DVD7bvz4UGCnKWhZcCgYEA1gf9 + Uq7xwmmOCS/vJA2xNl+BXnDd+K2dERtLl8iW+zzBbZ4wJN2vvvpKn10CgYEA9ulk - UGPSbweVXgUm1vR/nCJfLVXAbnDLvX3Erig3jX490fQPsnnk7MiOoDmFZh9jAHdL + PDQzTpa6SiOO1VxlhN81X7HFY+k/FJe6Ro9tVnnOCGJPJcum+4yicBATKmhrwLQk - 7oiAZrOdjn3kh+uhtZiH5x4x16o8CAU1Ii1P/dza52tg7C9iIxoXgcUFf+9Gsz3Q + H+mmqxj2VBu7WLCgNpEEaQPmVLivE5I74wo7zAM9952r5Cw+cf+xtfNNYL324o8j - qOWtkTwdr04VpUovw6kjzj+qDZUi1kH6/yxzk8kCgYEAqSxWCyu45HeVrZeGhZtr + VBp3FEV8U41uy1nsmCyf4LUMBCyFVTca25NCK6cCgYAowDpGLQD/YGy3gfXev20M - SetfaXda8dv/2JqBNQmDbWf+K7KlpykLKyKM7QsySsKKR4hkrWWa5pl6XxbtI+qN + mhWjn3vVflqjDYl3hzDCyf/eGsGmSMjN2pO/LTFDtF7w1U+nK7pj8s993j2XEN20 - 07u4kOz7POgqciQFXMqLgKG18pHVbqnvnpOvd+Bin1hy1vYzav43LYbEMvuE9dlV + 3kucMjC0XKdf971d6G5ZkGCLceA5RlA2ZiSW9iiCoYok4QSafdBAnlW/bNyaxsyR - A/mPRl+K1hJ0CfXZQZvTHgMCgYEA0nwK97sjoQNhNpR0bOMIeEEpPsldNH+DLnh4 + J0+wAIciOd+CxsA4xo5uHQKBgASjR6W12UzdmewwlMs/LAz94FPG1A1XYT7yxqXy - KxnsAB+NpmOR6GCN7PsToKjQ8uydDUFFEHF3bQjpQs+2JqFpZ9B4nqcIN2L4JJ8S + pbwdF5iiuBfepmlNL/Po6WM/iN6aw57x1ZabJm1YBAHbd3bu7GVIlHf87BTzBzrx - cOkFCNDhCsOEDuJObdzkDz/2N6nV6sI46VDuz6zCOLve962sqYw2ZUgg8bigCPvc + g0QGv5A6HvNvPVEI236ubkKl7tA8ng5DXP89euNa4bziGIaXN/2RiQM/DtYV7eQ3 - XoSVqVkCgYBrCzwmUcWlU9j8FqHxcYMxSfgBg7ntxyNsioidk4k7DUDMp6bDgsTe + 9OM7AoGBAKE1BocDrOsN55olO9cnw4nQeBzXvlstK9rZ5AYa3FKnNs/sY2euzlBl - 0nBri4PvTmwvyNdPGYC8E43z6ysRIZqtT8O7USWuYFTAiU7FcWxpNqV9JdNf7Pe3 + m402fQ91RfSF3pBbUX5cXVdbNt/g16bN7bJ3tI4A4XSo71Pr6+nmiaAt2WDH2IWy - GwRgN/Iq5SwgHzHAuLwTAeGGzpPWcHNBeKJ8VMp0HqSH6OCZOQyxtw== + X5cw5+CXCLok9331H0W8Im3yiuTU73WPxGWJ7voDpoxpBGj9X6it -----END RSA PRIVATE KEY----- @@ -8712,62 +9612,62 @@ vector: segmentSize: 131072 total: 10 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:SSK:ben62gmw6buwgx3o4ndfvqe6da:qt2ru42whduyofgesstswdxtlskmufvkg3lg6lzxdt5bxtqhv4za + expected: URI:SSK:f5xydhnzscumihbschybuyocvi:y3wz7t7pyj7id3xyzdut5xwsvpmzxxbotdy7b2azois7fxfuylna format: kind: ssk params: format: sdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEoQIBAAKCAQEAvKXoxfyr1z40UlLYMQ64W0KVziNAowZ2wFjQLz9wn5PvJZs2 + MIIEogIBAAKCAQEAofW4kcitiw1sFYCYY0wIJXe0zMw35+vVTFDpF3kH6dTKcemq - xOnCyvhn/8brO/rSksANWIWKXSH0X7a7TNrw0bMa88yTG0LETjMyhzGGXWzWmFUA + oHylBFmDa/qGwg79GA9TfpUVJ16EgPKEU3mBdls1MyWrqHDfy8JsQhHPS9fUD+td - YK2NqSKijlokMQwdOCm1MfN94zXerBUW4ipeKm8bw21Ailfv6V5iv1YQamxTv33B + cLJFKypM89gtEXCt16zWe+36pL13rffT73hkoeaqGI8vE+xZazd4ZnFWd0bTGpl+ - BQVQN2UzDijlxAXRRMq/DG9rVGhuelYtRogyfQJWzlo3cPTHEAAPjmFPG6rWJGR+ + ZYmKMSbGNY6MAQfWPCOeQ4EK+ReIPm1dKuFAC2H3g6+BGNCArnLSSCF6QgofRvXx - oSaUuLRqQ49F14dyvRO3VVwQa3u/cno3x3ojDpkson6WcVLCLJHXhZwebd2FXWvp + xy/taEULnHHRf8G9ELRIKpT7MQRediBwRU4C6dizNTmVKigOEKIQ5uxOCjS9gjX4 - ih8mIFJFh85yUYU8owRzUIUsxwPzg5ZmLl+YbQIDAQABAoIBAFAj5yQct9+jpFSI + ng2ng3pErVaOB9jkjROOPrtF6uCOQY1cT0zTMwIDAQABAoIBAELu5o7VNSd07hi6 - ryEAEN9sBPniTfYzq8UAtcgsmiqgjMqcCoNSjxbsujmVhp8fac8/2SuO532zC/6R + 0v+igfFeFenXcjlWRQnrnFE3kzYnW10VeQ8nRBlWlxIucLfNcvqZBuQW362sCa2y - QTZgGEftX3jMon3FOmHCLCf0qRENSIjEK3nmoLSGayowLwnLDKqsRTZoK0WXv/W4 + zE4lNoQ/8G4JYPZVY5/1Y0Ew1A9fjIPhvPWgryZGLpRN4F5HR4kNJH0GHmIr7USH - q9T+jKxYMSIvSmi6/MdV+nswE58xlOCwkHXz6teCAJFX1nnaPqeTDIwpgZ4DIjh2 + 2d4rTsd8KQrKTeX5dQDy5T7NEzNqeBtsb0Imo+5hsZdCI4NMZJ1wYnx6OE8uYMOu - uml8hRHql+e8k6BM8XJqp6wvNBzPZuECuaiV5vO4Jf5TXn6hoFoAr9mwhlybcX0Y + L5MMl3uIkHQwWiRuilods5bW5a5kBmzeE5X0hsGi+czYVXcD6NvS1ColRi8Hpw59 - FhUzntVTM8jEXnr1dRNP6fuou1PKFhXYj3ISQfeMarZBBs7kg9BpMVofugXlGnHj + ZKeGJdaX5eCOGFpUIkM70e3wIboCPN0msyEsWPNfgeuhAeJJ/l7BZK1avr2+mhVs - LY2Slb0CgYEA0YVMksfK86JQt9eYrSOTKxwvYdLHXoafgNIOGQ+hm0zhnn5VwAh+ + E5LyzAkCgYEAuFIs0imReqm3/f8ahvlun7VPNVqeOjr0Qp/O5LpDv4V5x2RLG6fR - KmKarXvsu/Uh4tZ6w1AA/oxGwqJT26iQW22r/kGf1mD4ct6xg3mwyHa5KfFRUCHJ + MBUUM4Lf1Rknn7EHYfYZzl0FheNooQcIMO+wwZ5P7dsTDPMOpcjH4r+88xxa4q75 - kH3rGMj0cTSd0N7R/6i+6I8GpchjOCP/aVivRsflGxW7NNwKd7ySBXcCgYEA5n9A + fiHOmfmlcXn36GKCo6km+PjmA9t72Yd/ylRFkx9KMJtNiIpI3UNABrsCgYEA4PFp - kmDBFvjD5R5p0wFzAMS9778rHAH6eCEwItndmhSXd35UHrkEIGJdAWyWK74caiWY + iuCwLtL2MesiMSWjNFhJ3766g7ckQDBeKEavXt0ZrteEXAUXuywqpGrrhiLG4bT1 - pLZvQKmxHyb2Pe11WUduxIo11xwtrHoYzQuOokKZgG9bbCU4hCVQWG+9pyshb5/D + LbXfkoVfuvUk3JBSmTVLuCRtUGdNeAleB1Q/HC7WO1++yKHm1yFveeDN7+fJQ+y+ - DLb2OJE1iPl89sdYGdhG58d7aDGsfWD1oki32jsCf3HC7tDDqm6eszUe2scnicDe + pdoYgZ58NtjBQEEdshtaz2fKwxCyb/WyI2uMaekCgYAju4KG55oVXouVyPu6iOaC - jNuQlq+8aN6JLx5sXlL0a4yjC+w4sEhTQajwoJltf/iqe/2QcvnDMKh9ewrJe5go + PaLyY/PitAUgWVzBiL6ThWu7VN0eqmTqXlvBNLDx3eOJmMcmnZAZKn1knFZvSS60 - 9DoZZ6/+9udoAvpgGJy/2cnsPTpFHixWMlBCzHarGwVN9rfZ585d2j4pj7Xr1cJd + VfM9RdSW9u51hzUivI7LjYIy2x9fbK5fXmxv+y6wlgWSXm6XDbbJc28b9lPHMvZ6 - ZM4Ju1v5cKxCzWw19xMCgYAezWadqRxkq33SQow1zH3H3oLbZRqntYP5RcYfAiph + IeYvBFTcoW7hdnVzt5LU+wKBgGCMHZHHKLegQp1gX9eaYPdZobOQKHvaQovudqtw - Cttq9pDbQjJQ+ZQgOpie49r8PGX3rQGVDJhE53oEsJT8B1XAIhAr3PIlmHN0A1Ve + 01qzKY/a3uukH/BtX4wcfCShjp1XzxgkhOZdqp4TFBQ7OciakHpj4Ctve1e3JY2d - TbQhu7/l5dt1nV7tUpFvo43mUt1H97NTv+P9mAmhGOanHYXsN3ZAaFL6tlhdBYa8 + wky1aawoRznUC8Fwj2lPbPS4lrE5zwZemsAfpw7fb+rFSBqnFQ4KbYPWCdB2M1Ry - PQKBgQCf0ZLxESRimOWkd/SXcUt+EcRTfS/leJG3xN0k/k7fvjyvU+YubDEhOSDo + l9zpAoGAKRl7T81Ji0P1T66Mi0i0vm+l32QZuSd5pUyBZtsycFcuVz1F+lDIWhMA - qP0KRIlvinOdhyCTTdXCZuQ81ao17nC2m4rpb6BihvuBNSTdjMh/ZtxP0wWzT4JI + fuai8L6uzFNd/ErekSIQXL02buGHqp19QpbxRlzNYY4OB4OMz6pZkWtP6H97hXxm - juCcjfvh8EkQHSgSFsZEYDjYKa80vOU1UiseUyhqcALkggOsGw== + lGTDX1qzm6DfQpJjw39B3neYZC7cwAImcDyvr1kb/tlOmePvv5U= -----END RSA PRIVATE KEY----- @@ -8781,62 +9681,62 @@ vector: segmentSize: 131072 total: 10 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:MDMF:57zjw7jihhbipyoi7ueb5n64oq:5nrlpvno5obhpucb36o5zbg3eldl4u7kjvnve2vcoowlsn4bo6tq + expected: URI:MDMF:2hqvwmzuxctume2eereg7xj6vm:qucnebkcgyqkipwxlxmt4k5nagvufpmhahuvfvmncuihp2llobzq format: kind: ssk params: format: mdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEpQIBAAKCAQEAmLgOSaGtpZP6I8vuyivxm+6QhUOn7IRcs3qUrpXZ2YC8vTHJ + MIIEowIBAAKCAQEApvVF2hYDR6x/aIKoeIrLgvEerWkVwxeTUYP1rTP3faTPXZaJ - L0Ks6cfHxm31bRIheM6f71oOc9BJnUouhvyEIgQmsX9CdkdUOIQJ3QzXaQLWAmgQ + RyDqHMkwSVlkTIdhorRROoxWgrD8GhI9Uc443ac1iFj7MKE61S2xnKNplhnh1sQQ - NLj8Puvgvj2HDvkHx8NwtQSgU/B+tS11raLvbPTErUU264QRnFNXrIaitMWs4dtD + OmjkCn0z9cIqCdaqDCtLwKnLKEKqbm30JoCi/nGc2Sh6FGvJ5Wa2gViqI8bbBlIy - cOU+Bbulv0PKMaSO5zFKGOv9/kaIt2YEFeANbrgLhyo0uSz/xcvMgPp1OdcV+jtU + RzO1NcoJjEVmxlAqLgH79XDp2n1TOJvm0V8WsOiSNQREcMcbLTcl+cwTbpsKkE2d - 0rL6JzwAlLHInGyzPwQNlm9bsIYDOBsDnSFcBA4wylsZnQd+/BAzfeMi08UjNNoN + gmkzej9Vf/PMxHrGxGqYOUPsYwX2gNNiEWx6pMN+INUgJXuMOlwdgvEDxiCz3Noq - BOF3MjDpFc708Jk734OQCz+nVTNJIHbw1gzhLwIDAQABAoIBAEJ+POZNSVRi+hHU + WsJpTsJKmr3rPrPvKuhlDx7EsZQ9taxTyB2OXwIDAQABAoIBAFI/9t+PhKIkqsez - 7JrFCFTqyazkWLxvowcYM51SLIB5f3PmteBoaO3+6JoabTX4o288k8E8ljdRtIOR + xodL6SJi4vgPEvd/f8Xiun9PYJd3P+kdJhfycSMpQi6AaVcCQulC59luFZhg1HGL - 9XEbiBJheVFmBdOG2gIjZ0ICIdYcgH6avZefBWEGBZv/IQthXURacXu3UHFLsHeF + lsXcUEtx+n9nRqgYZcFrt1oxbuzRZ17ETDJaRi2crKJfuxIJvNAt7C3H+BunbArn - HAwmeZWYevuwO6nOnnZQiUdadYQ8MtHeBTzLbragX8WGtpzlZuBwZiKrUHN5yPSJ + BCaLrMCo+9pHhIzW5SmsRjDGm1rv6l7gMMb6cJ0n+K1Vc3dy1yTDLqXcpEuC2OpY - eb+oNN4Mw5afqQ/mlCqS5NagRLL0NyoniQQvDOK5M8/bz0TcCS4hRhbCbgl0V0yP + bdekHv3S2mCwDD2jSpKBhX/fpWrKEoS34G9IL78tYSpUAxAJqPaHJGf6a/GGJlVO - mbLsfYde1vVe1Fpe3XwtDlZttgb3gRezz4HQZQF2/Xjn8ubERy6Iqgj6mMCwkat7 + Wol5ZnfVduocG0RYfB/Kv5Mihwt/d7XuSDivefP0myr7w+m3XsOm00/BUy2eVKmP - r9lIqw0CgYEA0B/R/ukUMtzjFRCbTQgLss3cAKLZ+iPMITpStcn5OMJKgxRFwS9y + yrUDLcECgYEAyGsJwXMkNYw/hIRU3f4gekJrwHkWvsJsDkxluyWfjLn4XQGiylI7 - vsj86qDfs5yT/JW+4/lyXdzINSzLz5VaPr+b1vb5SNJsS8uHnyUSQo/UTblX9ExH + /pix1qj9LSFi5TDLIMzZ7UlJgENNbcYy8jFx9AepSMHFhUvARWncOG7Uj1uHLOA0 - KgaOG9oFJYNND1p66YXqtcJ2S9mqOzxz5NLBZu7Feir6nR1Dp0D6FE0CgYEAu9l8 + ySD6V8dCnqaCUImf8chKiWr+joIdoFBx9xS3PirHCRQZaKjYkGABa08CgYEA1UKx - InQeL4+vw2UlIUrYgHbp1Dlp8vY4frso0336H7trEIfttVXLUTYXWwzSjfHkVezP + uf04DS9E60ZXQf/Yqw4PZOpFgPce63j9mkggebnmrOcNMub+4BUdW91FinqT+IkW - 64PrH8DvVmy3CWY2j5cldnXBsCnCTvbXp63x9u/h/47wg6U+a4NjnHlD3MSth6AH + Qt1sI9TYtucBg2tZF5LkIRWnvR1TeSgY3qZsFh1qv4Q2zVmTs+ONUv/e96tnvyY0 - vEkc3lpocsqPLNRraJ4vVdejE2sZ36p+tpCmeWsCgYEAmotWa1x2ZEKD2UuIls3n + s3tdeznWvWpqakE/62hnjdLIVcQRHmlLDAwJp/ECgYASPEL/+gUCZkdlPFEofbXg - qfGVcV98T3Ovi+j8LAN7rfsQS3+NQKPUJ/mlXTDyjDQz67bilfTQSQS+IkZOXanA + yehZ8+qQ4snIJ0VeWNcCi+1AMSTpub/Bs40C1g9rKs1/wwfIbTsq7u8kH3uNEGqU - 5qFvvlOMztd6FVpgLfvgME8PTlvYBQ9zNLDDa8kcUzvJyCHe7XNE041APJi4AN6m + RNF0fbn2Z8McFL9i0XX7IIJwpMhQ2fmTj0+X6wZxvv6+azdFXY8Cn9yXhNlDO+6S - DH+3n5CkUVCC4pItf5APY20CgYEAtB85eWvwWdikR27f6Il1CdF8KyQWZIMV7ucV + p6zgmC3R8qU5M5u4zzNx2wKBgE5ggp0OWUlPNA8b/Pm+o8zKEBJQn1a0e+KixuGq - oZ3lTbIPWl2MYFlwyGFeic7EwpjUQlP9lq36sYr1s+AwrlGVNaBPqsQFQh74k3D7 + 3HSgRA0LpagtiUKlv/KBMgug3T0cdNgCNLo+gZ9G6yF3lHi7fahDIzC31HPUrr81 - nmwbXJXuFXeBRioXrU3iIPLiUHlCj46yfCd7B/aWuqNiIDFbAIjViLFpTEBhIefA + fsfp68+TMejqoQQd/1SfwTxY/Hod+oR0NHkTWr6mm5GNhYZpCpXu/721n20D2ZcF - 8tvG1RsCgYEAxtXdD3dbMN1GumMwUNOQj1nwyKqgdVmu772TAJ1y55ZN06912Zqt + 3Y6hAoGBAIcx/SZKMhXCGSx0/i6hS7DqXw2rlMsO0/jFPq8jyWpjES/BV31cbkqX - qW1v9Hvy4qzU3MpH8p4sXyBbUzxKFMt5VYAlE+/eSC6qzTEojpp1e6ACzpeOqTEt + i5QiyVspb2k5cE/NveUrrsMP3HHQpY9gJUr+GineG+aWr8Bruh2nCrar7ppimhYR - cChVzJG9ezlgp2ov2vCmvp/IH59dpdLDZ3r0o1iYRAYFu9r9iNVct/Y= + 6ek5FVvZPBvVVGrhIhX6XQ7RNnRSr0UkF9grNkPn6QQNWcj8f145 -----END RSA PRIVATE KEY----- @@ -8862,62 +9762,62 @@ vector: segmentSize: 131072 total: 10 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:SSK:q6z6djixuqozvwnooe22vx2bzi:nts6mykkgnjqovy3v5ptbckgevicuo4wjztqddblcataz6llmima + expected: URI:SSK:zxil2jxwwoe6le5k73zleft6oy:gamewch4gcjlcb3xb6jzqqa6a6xumhsagoeakdf4ranxvqrq3hiq format: kind: ssk params: format: sdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEpAIBAAKCAQEA3L0cVo1jjawoeQwER817huQSRZUAA1G3saXarMyK17tG8E4h + MIIEowIBAAKCAQEA1ga6vTinyDMFCVRuC+g+QEjCE4RqjfDLT2Jnq6F0Ld09to0h - w9i1ieiCaxF0+cbKOLSabELsU6S+AhoBZRwdnk+oSU+YqXIfCeWw1cig9hIRT7cO + wIk55Pl7BCUanPbvA2C4Iy6uH6Zd8ixtrzfcI0neGUkfwn2k8PJ7nYO1DJ4E7IgB - P3yd78DF22yT5H1Le/hEjW14cWWfb84OLHd+MuIUy6ZqmiAf3N/K++/2NuYos6Re + COly2AABlm4C943YRx5774qz5cKgYa+1+OtYZv/inbD8f97uHnfD8pql99TNKEZA - 6EPMELlw74Jq1GaOV9HqdZnak7lBlD8Fhc3jsOS717/1pwzbVUpa9Xk0MUihLhC7 + y2yE1sZ93KSdTHidOXcj+EFNwIpBtnw1OSwx8rQl5L0zUcYRjd2J/709MRmYLaXJ - kQGcHoBWQ0tQWAbjhsAcSZC2kUENVdH0MYpg2OuR5FGAmTqFrdXslYA8U55NIU1/ + F//mZQdyETFGuD4VOfvhcZiUueNnDxkCjM2YBNU6BJXOWeJsJ8ypgubzzuhOR+pj - 7FeFH5Oygr6lGmcnNKVEl3u3/bNgeNwh/nAyNwIDAQABAoIBAADmXSBgiNiHLE5m + ERMo7tBnqZQYYcYe4Q4zhWF1IevGmwtWXAU4pQIDAQABAoIBAAWYad9AZ9w9iKca - BU8c9X/KG+WgYwogbKfIPc543P4JJj5iNdKxu6JkH21RT7vu8Aca0WBXRisHH7vW + Si8WOQ5pU/kDpwHH+SniNjiiQSiqqI5NuUOHjlOQFU29XbuJOpICrfSSp+tu6Blt - yvepuC3biZuW5qVA263eJleQxFCMfU6Tt/amoDwzJHtYWM1UfMN0x87DH1ETBtCI + OFpN/aBG1r9EV364Q8OVIioOZ+8bS+nNJLt/IKSSAJ0x0zvjvTxD5LADKTnbcT0+ - CK3CD50kfSI2u6DaOFcIVdsAvO0MsYs/Q7b6pRii7A/d4gRyTrmWtCOaMrzxlbAG + 14m0pNeog1v35RDkLTUT/lhftoOBxEUm/6XQOpfhp3RcUypuf2po7mbpFHvcVpZe - 25l7nrnSuuuBFrg9wTF7LfbYuDqqQY/a+2JatcU+WMEPVyw7V7WOUqQJJR/QUv1d + DCFhp8/DWho+Oq6yBMsPavP0yqQEgFulbigvMUhq1a3xqnOO6T6qq/A0Ae5u8y4Y - S7zP0vsar8iVzQiuknnFvS8K45Ir92gmPkovsQ0H7VIwuVx7hzA/MpkQcDsgpnNF + ItK6gzqhXkrtkyvawAbCwZfre45NXaVFcHFXT6DqHRKQE0GtAs0sI3HuRm78e6Cx - X/iIOqUCgYEA6wiv1VMygji5eB55Yz9WmUEQB/WkfgYrTnypcXWpG/c4GaFckVre + 1ef3a68CgYEA4krJaRWuujRee0XgMPv+lmFYLC/bs+uiMITl0zkt6HCNj23eNdmO - +wyjVRFW11ZpsqDJ+yveAU+rASUW61HzNNMZ43QlGBXSAyWkInkgfoRDJf9HSkyY + GP6J96V6tknKzdCoDkicf4zZqscoCCqm9WU+u/Bfra0pWY9mXVYSnyo46m98cX3r - FVd1BMGt+XGJRxXz0mU0/2Qear0PyLwLStfWSGw7ENCJah7XZ7dIMnsCgYEA8G35 + dIqcxq7uet8aMzZXn0z03PgT+ytOS2GMucpMBb/WnklS6hF8lfFCTTcCgYEA8h+2 - NIxCPo4Ke1WjuxboL/VJ5B9+22ECFfWvK0rc3+T+zmJ1EExdDO8FxxqTW7AVz59w + dgVlyjDmXUf79WVI7J1Of8NEIGYUERaZGAzwX5oTRPSQc7iN2pRwZWBZbR2LlrJ8 - rtM9/tMxUn4YEHkL6eNefNeKxF7lkPRWSRK+uVs0bTsygJOZgRheZHGJK1lXtAHI + 23jV/Gx57C9V3eblhYRjR4wYTKkFak98YKyZnckzMCaFMWaiBYz0aLADsVn3CB2A - ul8RXenK3KkNrGuPcJ7fvh0qAxnLWFLW+zSZYHUCgYAORRG/5vQ7GcyQ8XC3SOIu + yte9NA5L8Co3OlWo7OPGR+MwhGptYCUPiBtUtwMCgYA/n3tFWl1H6RVvX1QLMa6A - HdgmU5CwIhnBAyqae+VPkFv0mmpvXNAK+AJ2qL3YByQVt1NsD4bEF50vTZwtn2Uf + pVnfAo2o5mUxcwwS+Q6ZPZvvaZqCVWqISHiN8i6wNcsZVsMJUQz/J6DDTT9KHIPY - wO1idOvHoZOFo2Rqv2XsqIUXKn+ekDXvnca6CjRQ38bQ7RFHpeNo2iBKpL3vlxMs + luCugoTEFd18Wr8TGvIdYgeikjnQxvB+UcKGcgSG81cwcuTr2v01a2Jiyeg3dXPV - cRxOe1u+spqVOdgkMOmOPwKBgQCbKP2ocdPWduhAy/XMKW5SdOPouoKtpR8peNJB + gLUjIK68zizLtqLqnWxgvQKBgCiWXFXIbdnI/LTiXkAyrFjNvdz49LChq/d5XEyF - CCEexLPEETom+IEcdayu33G1vB93TBf2WxEpQLYV3JY/Gz8bA8bYnmlJbUyNjYGZ + zr2X7GcAwD5Fz7G2dGjqD9OUwlOOtBNuXCCmZoHLJY+/JvaMzL+volsncjrx/B2Q - yuUWzcs5qvhejeKEs2tHOxYgyZmV64jU7cFRcC2g1eCjIw8AySbvk/am5aCbMWrX + kWe71JLbwjQXyk035biu2M+gDyMTHwXhyFuzkdM+oGds+JZNUG24jeeEl7UoQURF - 1wwceQKBgQCkuYGUkO+fIrGac9buxGSrTKJZoSv3ktfBK27UCW148vv7ZOmlrNRz + oJvVAoGBAJvblfRFyShYaeIj/mtQhBgdJ1rHLmKcQNlgERnH4M0uwFOxLqSkgZWa - r9lVjRkPI67Ays37ChIH5WYh23xg2edkA6YcftpxKYsRXgavkab+kgSPJEqP1Ptt + uRCOGQRlIW1+lNMqYFNSeq+1Qp3DT3Y5E/9Rvp8aPqNjN4oCCFmaNNHBgUkgqDS9 - IEALNn8JYEzLqUJFj79ljw7ln4XkNWTxzMnJ4GUPQxJIJA8EH11jZg== + EPG2GwgfWxGicfJ+ntis05EMbNxSY5MdoO3kJzRxcoZvlzrDEK+P -----END RSA PRIVATE KEY----- @@ -8931,62 +9831,62 @@ vector: segmentSize: 131072 total: 10 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:MDMF:j3vcaycwlx5amtj27qsn2y7km4:p7cg7f66hemvv57z3jft3zu76755wpjimxo2ttfq4rwocd3gdkzq + expected: URI:MDMF:drsoyrobruplipvl2zet24hvhi:zvyy5uqu52rzcae6erus2ytl3dugnsf7n7im725vxutz3imfi6qq format: kind: ssk params: format: mdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEowIBAAKCAQEApnFcrtpSauyUmwqA/oO81nSe97hvTppQVo1V7r3yjryYFNJL + MIIEpQIBAAKCAQEAr4dAM7CUJMARGjCtRYqUue6rdSHxf7rn7zqNJ0k/5DMs3dO2 - Tkt5BcwTHCr4NNanfsxLuBb7GSbmMxcLjg58tvHmLa0lcLD8gt1fqCdrhX8asLMo + cSdEuzPmPclqEXoE/XUTDszfZxkNIzKpcL084WTuzcecchxPAfuD95ZlWdRIK/2f - d/XCbAlkrsRIRBI+LtDkqqSJlNfTUDNTbumHJjiZt7wYe5sOVoeWjSwp4uKgBr2c + Q4OTeIycpBycdEEfG2jbt/zNbUoBfRpQTTq1T8DiK2An12CEAljdGaLgBHgn9/2F - trJCpl6xY1G7x7+L2OgyfeKqG6/Zuc0SQS/Q7HJw49+WsdAQ4WDbapb491dYpfwl + 7xMYEDIcjtY9gID8Kkqd4PQmh06renaK5Q6QtwmVoztj/U2O2bdhAtUmejt7bGNX - JtliOGv+YjdS5gU6YuqMNyx3Imi53COb5nWkbCBWpk7CuUJja/CnZvwvnC5YPxRc + 3b2xewZNBtM2e2iJLZ2TgK7b1XIx07/Fw05RHH7bnTpRwIeAR4e5ECGegUKVucaB - LhxW9qfgrsifUH2ZKXAPQQAFXD2ddWjwPPpNrwIDAQABAoIBAAN+ndOOAez8yqH2 + BM84CvbVpOasZqcssIcflEsI4yGUgozHlIgk0wIDAQABAoIBACfNZpySBPXUa8xh - tn6hhXV7PVs2JCAiXU1z6jn5Av68NvU49RvPudrFTiFpRYzWdO3UnEJhOSRuDKdF + j1j+lL0Yxt53xPhu3JsdztZCwO8xP5JJqMw92FMO8L3AB4JRBgKnYpvvjxUk1BrQ - 9Jgm9bdhnNOYrxCOpr6Yp0mAimFjKcxL9q2OG2bpS5PfyySivWt+N07d5YWagnVM + KSX8c2q05YXaJrqlerD7ZLBm9TKKdZcsGspHctBaKkb4ie2+upwPigtNkxOePXot - npPVk2DaD2AsMtdligeHEUIlizuYPdXsKLr1u0bkg3IXOTsPBqhNbyIMBXAwgR8G + 1lm831JnbaHiWwZ2x1h06CYhDeVVLZuU5Yyp33l7X/hUlE2YaxOtL7ODZKUs2qvo - 5alaQt3jIj9Gr5FOYKUEj0sJSxVIF5nrJYvYskxa0wRdgGSqnfjsGyidAa4Qj+Yv + 643v2x9FzZB6g44n8HWrJzxx6rtZt8qj/G7h3YVvQkT6D1LjVRD/IBVQ5+Un99k4 - I783RV0LwG1Cxmdyelh6Xs0+Lq/4tHGVcmFiGSuYGB5iqBMLikcc04xK3wd47E7c + IxWuMZ5u2jKitpLlxWMuMKYQvw+BWpp7s9nTp/NkPKNfknptWIgsMY5yxwDnPA4g - ywtUrAECgYEAxAhTcXHAqRfIMHEigFVZhF6uhIV6LLvrzHXIdLx25fo4zDk+zZaa + TkSfa+ECgYEA3MUmD0djOeR/YiXNt9ETkSHjL6ADPhdTEtH7BeRoRrAQ7FJXOiJB - MdM6HRexQxx3iXRLEh6oOfuA3Uv9jJtZ/751o92DetWPaDaT4yRCpblU5gcYBx/S + RiaIJBasJrBz8npqyfyFfYBJSe9lJDz2zSGqitmJ6YyWnnEv4gr7XDERGhkfK9Lv - nzxJoBqHPTjUF/7YWj7Zj881Ufly6tcQS2ellvFlivR+YYnrdKfH/vECgYEA2VvP + F9hCMQLS6KdoO7jBG9thNBpmEheSlOvaT+LQynQ8bUOdNMXk2LPuR7ECgYEAy4nl - oQO91bhNI3AzFFMYUIhZCliQxYVbnzaeiKKBECC8mE+p7lEslAFGr4PnjHIY8op7 + CRp1YdenVXO4qttKVXJXWh8OJ82dyNgaX1krHsCnph/P7AlB81VKZ85Arqo690YB - l683dLGL/YREy9eBEE87lxmhHfYbL6W9gIGjQlxC+tonKtBXnm1JZSNX78E7qX/h + Lu5BNJgsWcmUIt5CMNXWl4DDqneT6gwagRpEmiBSlEV5ByJagkrLKKX6Mis5Cuu9 - 97vSrMru49QTA8kgaA3dUuS8F7yX1UMZaDX+Vp8CgYAJ2ZU/xQR2OqCvdm/SXPeD + x5nbZ/09pX+3UVfD1i4DEChErGgzCDicNmKLWcMCgYEAlcCpk32iIjgL7GCmTcTl - hDJmrEJITyT5AA4Td4jN43XJJTM3p1KWIFPyNEeO5LZI7NP81BeF2lJOTEwwLXon + 1/G7sKeC65BYypBjDVklHqX5pMQp5QYtbs9eU9SJS+kvjVBatc60IjBuBlf8LHuq - NI26rx21JVfwV5W0uxSyOQ7ABCk76mht4dydM9gJxno5vm9mkXPjGvlF5i/VBvtl + EfV/QJZVhXXXCXzPtS4r2RpzdleKHGkFxA/uvl4jAKvl+XTWkPXb1sL9b9JLnPbr - no6eeACvK7vR8Nko4mlVMQKBgEkRSw/2oQdKaGwEWLd5Y5AW9c+7jBdKSE2SX+LQ + bHr3lA0KnDdcINsH47MRs9ECgYEAtbY1OQxbEW/jX2HB0x+V3HUJUVb6X0StghqU - thBE4QFWrmpV0WWDtE5mSh11cZt/ICMSnNLWqJe1sibQMCvaZs7Zp8bZp7PxxG2B + aN2Fpp3ezmwGR7b4HxLdK5Gyo30syYfBFLH2msrkhYB2dS6yL0EppPZ7ORwqfMAz - pu808rM/SLFkzj+Mv4KHShVn4PWO7tiHxD+gDIR8E1RPdVxlZMRr7isQk/32C4Fz + hWD7MBJ9RwxDAcCEx1+YwoBzvwhhk8NlGebdP5iRycgc1E0jdHp9l5YrwTQBo2xO - vSdDAoGBAL23mwBwTlgJ7/oHv8M241NY+Jg3NA8UrOlXz2p/FW3ZZ0eVyLUlSXTk + 4irWN9sCgYEAgFSvJKzRr2lpiAAjFs2iDEv01MDAcGRmMTBephSGBhNA4UjZZom/ - rErPe+e+kPs9nMJH2Y2AbCPB1rEgEb6/x/us6/96DY4c4VWonZiTs5kj7Mi3qHr9 + W5gbQnHOhk9AY2qHi+z8Dzg1e1gU6pbVpNui3lSbng8sFbqDmpbhGyxXtve2eZyH - zXr+WAt8L1p90N0R7NeOIcnQlodHBZlAZ2CQAll3qZ9V9Q1i8Ug4 + mODJTFY/6VgevSdbgXo31ci60X7rYeGj7njmVGlGF8M9gn7jatE6ASg= -----END RSA PRIVATE KEY----- @@ -9012,62 +9912,62 @@ vector: segmentSize: 131072 total: 10 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:SSK:63x4hc4tkqbgq5gaol7tlcmjxi:da4llxjmaowkwuhov22qb4ko6v2p2wrhxffdfybioaw75jb35z2a + expected: URI:SSK:c6l2mn3jlibs5zzoy2bpamqhru:migiw43ipkn5n2poogkq5n427lfhkbtrlwplue6mjh2lxorkkxtq format: kind: ssk params: format: sdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEowIBAAKCAQEA2uybTLwxSCOAFQZnsCPI8a3RWilM0zXpUNnRUffFzFV2nV+i + MIIEogIBAAKCAQEAsud691onrSR8mmRebEYMCb9vV9fBoPjt9ARGVUHAd4jSapGF - hObdCkLwsmqygH1Ssd0OQM/qEAzgdLscK0h8+jEfCYTi+YUMjmYImKeLrb8fwFD9 + bEa/lSUnSK/8pmEMmflmbDQGw7pG5bY33K23wPqG8LP5z5TqK9junZaBIJPoJV6S - GL7WGkWQ8cnH+BkavLAOI22xrQIg8JigdwQWnz5qxvMSIKoypndihBphlmJMNO81 + jaNJ8DVh+9AQ9f8bdt3dgAkMdRLEKCqKj0KHBBCphvoJ4qROy43ljaMcS5+MoEi9 - JMp5ai4QyKyZ/2K3DSlsMufitd1cxKW7NmWtbxztss11BwpQk3HikRNKocVRhjN8 + q0cUKawtXGvwvitSfoLhYqlOij95W0hwD0kCW0tE1u7iaAwcQC0vZ4b0uEUNPq+Z - lqRRFYsnuc5+tV8laJy6YcZqWIv/OejsebMvTQdwVKOiGFKjk0YdT1xOgp9Ml1gz + UbbIQtl3ptlWjl+lUubhE5VFCnK77Keinw84HD5Yjom6uTsddorr4vhuu8iCNdP9 - JQUZdCYHDYsiPZvF2bxjbmKRNx3rruTMx2wkiQIDAQABAoIBACFNg/Rg3nhUWiwY + A1SVt8Aot4oKt6UeNUcAM0PDK0M+3Y+XHkMceQIDAQABAoIBABD8qz+Yyxsk04L6 - nNZZIzzMjb/S74pjtZnkgKig8fh6+b/H6A+ilPZ2J2pku8G7DsTa1Uu7tSX653wq + ZD/OH97+ExWpahx9fmSU0lPOkDaZYndVdXB8QD0qX7JGaYwnu2FUXb4I65qCkbBG - aIcXEFf49/k5O1PszvOshts+BYwJOnnFeDL2+NfnRDzbzq0pmH0ipQvzqGcin0Mq + jsPQp9m2QAFTaXUlG94JdVC3xW+BM8H2mp5BwqfA/dqB3VZqQGKXOuypD0p/e2of - XKKuPwi7dH/OQzAv4+OZ3qUs5DJ9enoWq5czHaP4HI5NDX/D1TFjEBhAwI+YHP8T + 7fOf2r+PUHV9QNqJBOVNhh4efnWMmj/Z8dJSHnUVIz7iF5m0UFcS7ewKRRPu7rST - tCJH8CRgDgZPXw84EIJqsoUUVxaKzU3azlra+jtlk7kJ1mfRMaW+6f11bynUG8SX + hE2tQv9ezvua66F6iYWN4Lfw0rAOyUA7CFm0VKDoEB3PcyRgD1G38XBpk2EHwKJv - ub22LU8WrNNYRzN5STrBDDteb4S1iPLoEXca3LCddMNcTLfH87f6rnHeWUXR1o9s + mKu0pf+AfzeNvpfoRh+z+40hdlgI1OVfIm8RJuhaA3D58Nlsf1iLS7mmbkEJjcna - f+ZYz8cCgYEA9Zv1DUnqL3o9czlDUI1k2XzisID1EEBnSmSRxD/gJiZDfJxr0o2R + kKBnkeECgYEAyXN0Zmar93SfLx97ZcPcsRI3OTQSeSBzS+W/imv3NXOOmVawyeqm - 1enzG15LA/aK4v6KPpfVh8g0BhukFEzp5EpNHwOSHlcpz8j3k1OQ4rF2uUeml2p1 + mFvScTJKo0f4hMWdAZxReepNywrrn5PzTlnQndnJv1n8oqhUtgLLyi0N2YJR8wxm - OMr/n2k4stu/I0fnibR2QBAuzZdBq1coCC5HFG2kyYemMWDO/BD7uLMCgYEA5C+k + YCGxn1t1z9DVBSGTAlx/2Am2DrhQjeXYsI+uGlkRXScQJil+9ki4cBECgYEA41kW - E82178C5TcEVTFoDLpX7sFMIVR0zSwEC/65gYwGCjIPtggB/M3TjTCqbfR9s61Xa + Yg5gWoqsXSKrJDWvYNQcJCLbSOIUEBsXS60bp2F0rpxLFgPs93A+I9HCSH7qL+Dt - 7KEjNuM3fhhn4M6KJQPZDpBu2mkd8VU/tbvXNx72L2REMfXklsf1yQKcryacbkVc + dgQWgAry6Z0bzZdsXNLrXkqHIFaOXTisBxodvMWRH6dbJLyFVt1QYw8LF2UBIRAD - ZGLgOqKG9kRAS5d2E9+m6IGwuA5LfLItghBb89MCgYEAhk/OF4FHRsVjW2KCNEfO + 5SiKN8/rxRCuyxpf7CNvzyKlrMmn6JrfhNmwTekCgYAOwrjqr+c10IPBbisaf8lx - Ub0gvoMXANcnZSBQMnD35ATivP9RW2g9yyxP3LSY80bctruZ4BbqF4HdKUXuWYei + 6AXH2TrpSSlpjEIGoHaSog72yVVW1iyyyTeYN7kkUaeyAtDIR23o4vQkRn6RSMPx - FEyplf8+5camv9FXykJVphKEKVhMetsl1XP1jDhfYDgZc3K75KtCS1BON/GyYL+d + H7+bcVPJA4zxVigu1fGctMRpBZV/m478yDs9k/QD8CdLovQkniZ36+49EeBFJWxF - zbN4/WvkRK0grjoRlvi2n08CgYAGTja3gWC8rlOwjVxcTsR1vhlFZxX83CC1uuJt + M6HsKE6PZsdWJIA7B4UMIQKBgGIItGIsGNhyG8k9fdbrX2i9jjT24uAWvNgFFpKH - VFE/iyQjY+XlSMQ7FMjPKwI+8+ZbnnS9Qzqo4qB+8Ie2U57HpRKTb3RQvsTgDV4E + XvlaSNpSgv5HSxOXzvPbK4/fSlTDBSJyuNEV55FdMfQBa7TLLrtGH+aN7G2+Vk/p - VJt+33EoIBouU0As1nu5QUQ5JtT9yxbhg0X0+NbH6Vzpedb+d5iyJhtPCr4VRQsy + rxELkHy5yc+Zi1XdsSBGCF5aK5Z6NXPHe3J9sgkUHItwIBTPYxNKuW48tq7Suber - 4+bWlwKBgFWNAe+pO9XXPhvE4vBf7IsafB5NuHCtR0VI2FZUowTXYaT3j3ZBoNGc + tx6hAoGAMpcvnPUjv1LLf10MWHoXxYlPvk1kL9INWQIH+dlOdp/3IKglf2YRN5Id - e6xXv0fAvcU84hHYq2ib0FG3aUsSq6KT913o3yG1EiJmRJmqZvrLiWJA5Qlv2X7T + bhCrwyIEEcp5iyNxgxrKYDJwEJblJV4M3vFEIkoXOPFNni1mazfBb1tnuWCty7Me - kOEiH+ip1sdBTvPNU80974Q4WvNRG6Ejt+DrVLOaXm3otSB3ZB5w + gXy3wcxH5qIGhmdiJdubUyZ2Spz37spBASoV82HzeeW+x/pVtVw= -----END RSA PRIVATE KEY----- @@ -9081,62 +9981,62 @@ vector: segmentSize: 131072 total: 10 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:MDMF:uhpq5jscjr2qolkmoqkp3fgeny:svwlfa4b2yxv7glkhga5ogkhffhhvsbqzpkkcdpxrnf5s5i2buvq + expected: URI:MDMF:immegpvupjky66rdfjm4d6wef4:6j6gmacjaguy375eqqxrigvknrb23lngyzhdv47ynty2iihpqmiq format: kind: ssk params: format: mdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEoQIBAAKCAQEAh3FSORVvmWRXYrmV1q2+M56QzBqfEihkcBPBbmIXQSYF4uGK + MIIEogIBAAKCAQEApP6Z4uiEAWfE/DO2Q0GeO8AlR5WgZpEd+/hF3epJmGYyTvRQ - Gm+qQi7jeFidT3WzSZjy+4zeVM0HGGaG36vQosK20qwn0vy918Wnp6KqJckLiw7o + Gx2ZG6szk/btuLkI9fStqs3paQFW8aJG5I4HJjM0w2N86KIDgtexHL/zqf8ogCV5 - +S7aUJEZAU/Fkj48d+JrJYK4bUyqVLudzWzxmTDFh356RKFGXgvKewT8nNtRO5ak + w8LWGUPwyiokbUXFdo/td7pbzbW1AYNOLohpWtMkhalxtTIWm739TTPDGemWGilT - uaiKCQVKcvNYfXuq0cacwZvUWRHFjk81Xax1ZhqKNoXzpv82TksOmkX/wFPptbtU + 5rHOT3NbNmPjGPG645CYadJNShR2F+B0AWNZnxANcBD8IgMWvIl8IrlMkH1mUKux - dHLPgu38ErkYwYBcOMo91uU3cZ8X24PeWIH8s4JKhHipiHZM15qACiFK6V54TXKX + TRf4LneD1X438tQ2hCsLPcgxXJePyzww8pXujXVrv6Y5LsVoaxEy+qGeLMIcz6qT - VWgCYziX5ukwpHnYUy6bwB8YQBF+v5qbXwoPEwIDAQABAoH/GjBPNbpvWbmNLAm2 + EgilvtiIO5GVO3mDR1TboVAk1sCc9DMPtAzV8wIDAQABAoIBAAT44zxSU4ATV31e - b0wo+tIuLUj4eQpWYVVwkWdmF7LCcJwrl/D/esyWLy7zO+oGQLTSRtF2K+94777j + NZTrSlB0pur0WGQe5W9tePWKFPOxyLxWYn+esbmCvEguPdW+RcXbvMwT7n/KmYso - VVxjexUrRJEFIka8bnxJbqCFRckZ8klvwr7Md8eWjipeiWh/SK7/CMG952Rriva8 + n8hNe2usSV/GBMKh90cfJug95KLv3JGYD4ZVvcv/HyeIg5aDbsL27WoZRKD7Y9wK - DHyEOpqzlv9dpOeKM6UUAbV1It79UIWBBUY1iVez4P8ng5PmpTKgcdTwM8NnXcJH + z/VZCQvCpywcAiA6xTGmVRbZg3ypX/FdANVoh9ajiulwolhBss+f7OKrqJ9pnGF4 - amscOThUY6QPJFhHmSDQN10GiPQJMPrqRDXrqSNRT3cB/+gEzY1fwXiVzroevJx8 + cqYBWju0fmZM4KE5nXE6ab3ea9oan7nXWIz+TTWxFJkLvg5/Kabffspokxc0j3nM - 1RtOc3Jq/g6Bt/ogX/N8vvR1uIn5AhkYJWUHVdnjT0uhywGJWaO3WNIHm3hNaq9+ + DuMKJFTKpJqazitMoFsxGfaketjwwNZbgwleFOFrxYxtP257rPHF/UFqflIpnh/S - XtyhAoGBALborYvkIcyDsKickGZCHwGQDSa/4Q6XSmmDL4NVzpp911uWp+FLt98C + 7ZF/zYECgYEAtiLFm2E4Naav/4agcOB+HHnFE9xT8ObIlEM+jTKwbYP6Ib9Xecj9 - wPe3NLqFWesH0u6t2m2lqXFO6/gGssU7eaq8xrP7yzBQeKLlDKCfzPrEbLBhgcdF + PnxLXmlC4KTgnkHtCPGYdUD4GeoOhDTI0VACgQv+HzY70DDbtN6UNak46AYcKnX/ - CmD/xrNFLDioEa+0NYFnXGgdmRKttSr1IqReFOdjIXsplGLAcH9zAoGBAL2Q6gkZ + LPFJhOWj7/EQ1md/BzzmHIudW1k9o8LZ1KIF/uy5pUi0i2R9yP5Y8UECgYEA5+g7 - 59CZwABeZ/ItO3wkxPu8IDnx3hWOKUmxgzdDpSt5O238GTNjSeaPZ/BsN7BbE86F + H8Z/RIODQR3esb+j1CntJyYdoEy2U8GZuFg503hyleiOGG9JKSeBAQq2nqnVUWUQ - TGzfjQ7pINDAaztm9714AuRIQOKPBOZooJnW7ErkYU+lvwULsnNa0HpJsopkXcns + AiPo/vmk91GpoQvJycjQT+C6tgSzzUvjLs/SnzxAe0RwbAjn+m4OCy66WBxgSopY - PoeNSjlMPlE5SXAM+YWs1xyMxxtx4nuP1QnhAoGAcv6YVZIJGd5Vi7xbIJ9DhST+ + NtE7IbKxZJhXVOy/5vjr9UfeeLrGKoVU6uIdRjMCgYATy7W0jJ2CX0qTuDsp6Yxr - z7TlFtpRQ0Lh9U1WRlUFt6RhScjkAgZmMZdyRC4gmR5jJAITiMoVXJKE0nvLmyrI + ZeTAotrQvRSh4KkkyZSZYpXGIzjLuMelifbbHQ+ywNjU+o9bwH50iAovLtxDDEWj - VGq49mFAntCI98jPhpDRO3uQ5dd300N5wgAs+Xps0fYAoJnI5eGI/EBXg6HIfAiA + UlHjWr1VAR0BJL5Ma0CqkGjp9vgKuWZxqQv3kMn/ozDUTM1mqPzNr3L74bgsW1o3 - ThyEQfFWFGvQycE5OTcCgYEAnU2WV20OxzP+do/gc78DIJYme7p1h3/kSUDJlCRg + nSCPs4T97OgKmnJ9bP+XwQKBgC5DY9gY7zapzbtlzBFFm9ctbgQLVImwBAd9bb8a - fUh91CBqp27Nvq3CkjcoCgLTB13chsBoVeP/2oKrv24czZM5OxlOVP58EUSazVO/ + yp5nPuSs+fvh54RwPwoIKxpH4yhTsvfaVhbXkpNMFTztbxn0F6p3uIerNHtWEkI6 - CUmmlNMEySIB6/7z2vNeEkv7gwmcJkYK8VLWZ8uT3rTJ8thhaoKtkjxjsKuFRAFr + b1gY2vw8UPkcZbrNzbtpXP9K2eLE6og1AUjdrwnUYkes2zOmoNvTtIv9Jp9A7gnV - yCECgYACtlo9j0uqunY12qwz2cKbZ0j+fKafX52WvJUdgyOq15YBl4WAFJlBeCNq + heWFAoGAAd+pZlqlxP+/6TR7rwPRRScufTKaRGJxsEcr+f60gEkHYbIL0yH7Aati - KfelZTKQcyCFxJaNzDPgK99OpLiF/kzFAt5RQIuwabdkpphOV3RPe4vWDNG2S8ug + gKsp4umaY4bP+hGfDYYNyiibguR0LAguGZSfx1DDaJUxIgSYoTHoWQWYibcyySUN - sULT7yvROK5TPDBFobyIpmX1b8M5sXSVH3e+f5hWgKfdVF3MwA== + uaRuiTWmWRsaAvWPEte9UmZ7i3kQMziCnPqh7feGbNKrc1+uf8E= -----END RSA PRIVATE KEY----- @@ -9149,6 +10049,156 @@ vector: required: 3 segmentSize: 131072 total: 10 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:CHK:72amudbwylfpsdjqfzjifywpey:lebiq43z33o6fznzhkr6hppzil25ngracqgvm3s54v4h2nchphsa:3:10:4194304 + format: + kind: chk + params: null + sample: + length: 4194304 + seed: 2NVWOXvtwcAFf0J/g0fDC+ypQbnCU7awZoCxcnhpkVM= + zfec: + required: 3 + segmentSize: 131072 + total: 10 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:SSK:52f3eczj3vq6r3llloe3nxoybi:mpy66ok4hjcwr4ir6gizd6inhojk2bubjtrfkavenk5wzw7e35va + format: + kind: ssk + params: + format: sdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEogIBAAKCAQEA2vTVaerg1p/nTfX0peth4kWZGnYMPgemUQUDhqXjwa5r6A1f + + NhAwXyp9Jd+cIDdr6JefXW2gW0trkfRNaHmMCPV3ZsNBFd/nQjaB3QM8mR+4H3QD + + kLNVvBQuuL8a61GXfTstdKcQKe7GaZ+Sm1H73d63SSfe0KJx8BtsAuQtM4ZVRjxf + + 0+k1EgFb1wkv4kShOcrVKWRduoOIu9XBGqqU4d9/QWNC3PMsYZCnLI2lEwrQAojn + + SP3ZCQgB+Mo4sxwDHIzOOWDW9OYSVTCTIQiSl6Z/vUmLIGrG8WHp9aHT19E660ie + + NwCniPEKK/O79XTEo2e4qKatjGCwdSj5QTE5dwIDAQABAoIBAAKKccmLXLWQ4HXC + + o2ajfxzJkvfAI+86Vn89MCfJWAXA2Oa19QNjF7SbAR3F5QFosztdOw+x/HjivKpS + + a+2I74uREaQjIue2k+/sQwCGD5d1S0UuKvZsZlPK5inlqdHOPhRJcgMXBzR9XVcP + + b3uW7XXLJlRWfprsL6dKIiw8apvc7zF1yUS1OAopfoOcc+HJ3GNMUlqp0kqtLlX4 + + HwXo6sVmNo4eOjJfllyTcUN4TVgzc2plSg6E6qBxnfiAxFwyyDlDuEFkTaNULPm/ + + UKOiniDWb5zf2sBm55dC+R15eUHf8rHKcH/wqmrPnTKn8nxzLSeC9qNWPDtGlZV3 + + j9JmEkECgYEA/XfgM71+9nQ8zZqJ32+ascITSDWvAgAtsZ9A+ze8G9+lJOB0MinD + + IvRxLN5nq68MjGrc+xZKY0NWfOAh41gbgLLvWYENKJ6AEQiDnGSDOSFBxe2Xc1gf + + BGaQMUYXWjbI8BoXG+dJcOjo1LxhxW9UWSpVgebNvnAUpgJz5E5jRjcCgYEA3SS1 + + zgELO/afzf2PeW83QbfFzkLsaTGCGgxwFXyeduHq7Qik5xfo7kX4/RykmyLB+D25 + + kmKCmdZxQQ773IUzMnMPOhxIj/9Gw+RkzACZoFYOZjmS6b9Sqb6jJBXX/PZL7f9v + + A0cTftaN2BnZ87KBf/GKrHkuE3Y8moEiDLbsBsECgYAX6MTnXIqraM+LfXZf80Ee + + X3Y+K4I0qBunU6RnjhxabMBBOEL9sF7N300FtH0G/t4qKLJrpPCjaGiyItpPfbIq + + c7aMNNYu7LSb5rezeu+95ds0dnMA2GEkoyAa5ceyJNTTgUKIyUpuMio0VwjJ/PRx + + 7MJgHItv2Va5SiXwdUx8BQKBgHIPHx4zd6Hj4CSUpU2SyUNCD+oEpn7TJDFfPOg4 + + MFtMxqifDr6KnH9Y48VY4qWJVdY9r9sKqCXEbwGJQupIYVGh+raUI/DxT4R15m85 + + 2ALUn/SluVqKbY5TXz2bbp1wQ1Vrq8xa+nkvHFXbb4i8BwMAh+/RSKyNDVD7TZ6V + + MkOBAoGAcqeFeUk5BFlRDEUB/34tiUR4ZwGzdwMgQ1RQ5uSVsgSHpp0ArFZCgq6W + + C8KtdvCh+hmwzF6QXrjcOrcHpRdOM1c1k90HOnO+d6CoaCDsyR0HSc6fIFYl8CS1 + + Mz+6Ugx5sCeSa2DEQeUSYwlQppTEUrDyjz8y97RR555E5lof6S8= + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 4194304 + seed: 2NVWOXvtwcAFf0J/g0fDC+ypQbnCU7awZoCxcnhpkVM= + zfec: + required: 3 + segmentSize: 131072 + total: 10 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:MDMF:araofmg4r5w2byo6viyya3oso4:6wbfrlkpzp2w22keeo5frbjirditr53g5oc3ku35swuwoipu6fdq + format: + kind: ssk + params: + format: mdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEowIBAAKCAQEApE3alS1Y+qPUVkfT9Jou7Pm67Dm9hikzQfEbQS+qX7pvdPae + + H9QwenTW6cfhMmo6vkbc8wqrG6eguctXWgQSG+rQroBu4dV8UD9lx27kYVdUcVK0 + + hKtU0eyDpqMiTGma1ILQVSM+/e3QZPRK+HmIlwvsOYCJSFGhMIPLekgYyVuxEFe6 + + 7olys2mhVNrCpaXoxubhXKE7dOvBuWBWs5A9Vws9u/11hGpXS96xzJldN2bDMyLS + + p5qygPyIFKNhoUEahGUD261Dd1hu6g3CallXEXIKl8CEQyPbFMWgJv1x23omjk17 + + UTtpd4QZsS6SZwVsSZLtg1/R2hDhIN8Fd45XJQIDAQABAoIBAAEQcgiCVS+2bPMs + + Hu7YKtKlIXVTQGuEi8zzC1qmPOPG2N74k/ifzrqUVCoKfeZuMrg1zEuUt5wDv3JE + + o2m1WgqtQDHJKi6zS81XQ8kBamBJCQZ84ydy1qdPcWDccKXvDy4uNLxAcLGDX1Sw + + EmY+n0hfLuYGc50wzir6x5AgtGxldX0GOCb8MY6mY/8mS/OSRlVGrG1KdQ5WqN3Y + + Min+oTPRhnErxHWvVikUdlZ63vtoGrRUWg7pdrXqnCkwxPH1ddyBkzPIIg0A07YB + + 8oJux1n3u3hO56VelHX0/yzFNkviH+gvybPWBJJu1krdLcRgqm0QsJ5NV/x40/QY + + osBcUpcCgYEA1Mzgj7GJ9JfFENkUxvHXzzSK45//DEOiW+rA2KziucHTZkLs4Kl6 + + EqcrXj+Yk0DbTypOW6uVKdKNPWl6XEkKmMvFxZQrL0LDEwAcqJzZlizII/D7fkE8 + + KMjExMg2K1lXCt50nqCW8PsyaSVpy0xNrZa7x1Iey9HksRDpBfczgAMCgYEAxain + + m+M8Eyij33XeC3YoA4eADXAdTSrTEuIvUUjWZ0Us1FbdizwniqYiVucI9AKNzGxN + + JRxFr6az/oBiiBJQi/5Qav4Iqa7G+4YKHhi1QTJogZ0yNJRQIO/CqXTbvUEN027m + + f3EwIpQx5JbxU0hKGyEhhEZiUdPReMbvwf2/R7cCgYB0BdBaCB6DcUxMx08AuVNE + + 8gzX1qAke6vGGdRTTs+/H+K22r50L3MTQHnwxRPXFYF9RD+802xchSPk2+GO93QD + + ovaNpx90gR4C+gimFf68VmY40mcMi1zVj8FY2SBPukIu9uL2qfAiK5NsqK1p3oxr + + nMd9AVUxI5tgvyuNyR4XKQKBgE1m/jvNgHkAMSwQvCNA4ep/5WVdwhu16XI7oMvz + + +gH21NdSLO+ZXuKsrEXbs2XamiyzPIKLz745ScMgA3XFtkUcEeHUGRBZoRJeKxge + + FNyzILmhFUgBzF8ZhOFXIbW7A+8IPrspV/AymFcrxNUYOezlzHpAFcB1clIZlUoi + + VAWBAoGBAM/HP7FbfXWQzWY85grp8fVi7FkYSaLdpgZeVUWCXj8YfkTyk+8gDabZ + + wf/rdHzjVNUvFEf287nj4M8mi4XtB5ywgFUvehXA+PnHJ9+55GeCOQOF5ulma0z6 + + MwfB7DKFKTfl7bHtGfTYJ14xVg/PtWFBJE7pnOcLFMimPfNWWCMT + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 4194304 + seed: 2NVWOXvtwcAFf0J/g0fDC+ypQbnCU7awZoCxcnhpkVM= + zfec: + required: 3 + segmentSize: 131072 + total: 10 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== expected: URI:CHK:tyt4hfvn44igztqh5zeiusggcm:ex3gzzn7byvhajithwbo7c3p7qwqxmnactaxxsxsxowkafyjrf7q:3:10:8388607 format: @@ -9162,62 +10212,62 @@ vector: segmentSize: 131072 total: 10 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:SSK:ridp3gpk34rfwdilzxclgeeb7i:vo7rrapooixj2qlepe2myiwzjus2zxge63pw4dbvzllfgvftxeoq + expected: URI:SSK:xsmblf4mfbwrri47s4psav35va:amlpbidxmdffbpreslrugvw55nhhwqjyaks2fri52ehshxngffwq format: kind: ssk params: format: sdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEpAIBAAKCAQEA0jmBeNgAnc47WkR5CqDDp03oGUREmm+fQ/xNjzKEWVLDAI+K + MIIEowIBAAKCAQEAuSBNunMdTApYjZptHL8f8fM+QhDWIE3B29TZFEhL3zjRgFty - vZx0Z63uFqv/QxEOomWsrjbe23W6GVpN/60l9R2jkpkrh+F4QSbiDICK5xH1tMnF + +KJDwPMViLL70ELTThuoSUscHyHAvFivKUVYvbGRg5WcgE9r4V1V/3pbrCvEfMxv - fYIvlBTya58oITWQrdSg4lmFIJVcTA5YysMM/tzKtaeStLCC6xbj+AGXCZlSAlAa + RwNES+aYzSK1etulmzdBk0K+aXRQxe2Lm07/o2iQ3n4RjuQbYbaQzfZS9isEk+28 - 53w5p454XyJtxORWSuj0eZnOOWNgbXh5VBIR7xWQO0NrJXHz9ZDyM8at7cxNSjnJ + iMLAQ/AOIiZEiY1jGIZMR5k7xutoceApwzFo4e91+Axmh2809Y1uYx5SOk0/bv7Q - syMRVT/ig71DoZL1RGRYTQo+tivForH57luGmta/1VwWA51uswN1WPmgs34BAvtO + XqxxXejURpevB4HzAyQiFKQcKz4CEsU6+SKr4dovvXdEHodaFUZzSl64sN6DqKuT - fNsPi+T6BYdmRPyjAgYaE5sfsRqrMCCa8pXFIQIDAQABAoIBACJbfqMC3lrsGnwp + i9T5c39KQutV4nyD134tdZy++2oV2RasLSphdQIDAQABAoIBAFMZo5qW8uc/26lQ - Zlo2ewjWrTZP7MxhWMyyKUoxeJA7+/MVV+30P3wLC4yOBf/D2Q9GgbR5SEPiwjb5 + 0Trutl3LFT7dxOjCTsup23oFy/0bSbvHETB30lcqJxfyVCQT4zt0IdIow6pb4eMK - hiI2qKueRp5OGooFxJ2/8+q/frQUFdg7BQ6lnmQGZmDwwoy8OziFtdjjG47/NMtD + IjKx/NhF/a5l+dcFD8Wduq1QVRdPnEdzE156om05yYyH0JQiRdALeUWr18KJonp+ - VFSoZcZ+IZe9sDbdWuV+uLmkJ4yhmRGHxGvR1EHYC9rQlSsDx6mtFShOVnNg0XzI + m8TvLMTC+wjM6X/NeFcf9xdlQ69ZMVEEf3Q7mzYc1L1e6SpaSVV2iTZODn6oGiYx - wDE+wxVclqZLqXNkmJjc5yIfjK0SiBmp2iXoBJCC3Q9bAaA5yiqQ523v5EFdkhDD + ZPc4djYLwZC5+uC9gDx7pSzusAol74/0hg4xJAl4ak1omlVoDfviuloX4AEQDEIl - sUiVaqS6WLCzrJ+5C2DgEyPaEm87p3aN0558JTHP22vj2VoABmoWzx/JoClmSH7z + O8tREPUjUOBIYBMpgDyKapqv08eGMD3Dch7pTYFKHYX6UJF8rjih2aA6a1E+Ch3Q - MGkSt0cCgYEA9Hjw4ucz7bB6vLj64GYoV+Di8h3MjXyDRQXttUQ86O7WfKNNZsZz + YBijUT8CgYEA/gowWim67ErYzMXCqjmy2NBpMn/LUU+nWGw3inDZ17PLKdshXxXe - 3KCe38SLwEJOBtgjer+Wlme4znJ7heALRAU5UmkWUELtzs3ybSceAz0rAQi4h7fu + dNuhfF365DLhH1Lded/nr7dLDd+n+IjMD/FNDG7xXJXPMZAlCwTZK9t817C4MNRI - AI9NSNMcYQsPWh81kCtfAuOhCTnYLuOB5HnD9TjoL31fGPYKjgGNxrcCgYEA3CMn + dXKliwACfjKt9fBiszvy0TP8Z9lXWtTw6SEvGNj8ftYY4BFKocpF6/cCgYEAuo38 - trfkY3nDwWFxyxF4tUk3qN0kcA6CzPV0677hhWY/d3Z9+u3Iv41hQnylcLA/o22P + 7BZQ6qp7G5epVyFQ1WiGgs1SzTdxyxdHpVM1Yt9Ya97/oVLaTeerKXOAty8Qh9a4 - HNtJBSy6SeQrfmml4MLvz/IUIzGP6yKr4NLa+szGa7rsahJ1S+g8Kaoc8i/5DuDP + 5pBRnkJBqFRZhvSwXdRVIY2b7k0NsEfbVvk+fnhDSAP2wIVDFGQ6vpthgspeCEZm - S7SUdMFaLtMqvjjZxNPT/rOyztwKhts8BEZuOucCgYANmNBk/lDwDlm1N2CUrHnf + XkrQry9ZwG0IvmopuOw8EHtmvRVb5tdqzhyaSvMCgYEAw/ocPxI+T5eWFLLjb+q/ - 8V0N9ERVNjCi3SKMa2Ar4GTDh92dMrps8e4EKg1PwyBN1yWaBR/d+6TWrp1aI8zc + HB/7Z6fKs9mdIcuqNTTF+W+MZafU4MPAL6pXs9fUe0L9BOsqTKD83UOrtPI7ZLIz - mqHGiJu6GQ7a6q9qDLvpmDRVGWQSAFPXaiD1RPCWISRYcdXrz823/msNdU8lxHeL + qoDejZ7wuBoiEvw+d3ewCfNzJfoAvjqmA2UEbGz/f1edeEOQAPFYayeNqpeymjH4 - +o3AjMq1IXbxj3Wk0kdNXwKBgQCsEEplVh0M58rrZxgDqndX84+uzJNDhwQT4bNu + AAFHkgWjFD4aRpFQX+vpcRUCgYB3FqTaYPSOmP21g39Ka64aTXtwjHnLHxW5O8c0 - 5LbvhvkKjjJwJNXpaz9fMYA6sXg8bFEVNA1CHzDIurCIUVmXcabyOXwl+gJMvr/r + toVh9ImRcu1komtRSA5vi5gjWBwJWvz10jMH/+vB9PahvBnKC/28SZW87dtLKNPQ - rcP9jnt1Dxjk2+KU30PPKSkQ4BBi4bMFsHLtQ4gS23koT9VfNFcaWSjk6TbNK6Ug + FZPbUBJDKqSeCXPk1Ibbnn0E7QJR7f7zOnc1HdkBiZkHVOYFcmh1bREMq1HbbrBz - jlBwdwKBgQCmc59up+9CcHRFvk2qYgp3R96APHeL6u+BSI3LHYyOYT6T+29rpFD+ + mra0GQKBgAYljdd9NGkfDCfPNOhwVTXZz1Ro7kU+4wId/wckMPd95MyWvfU8E/Lc - EX5g0nuc0ouI3YHVvl/RNXXz/8mtysHWUYoSj555qZqo/Kt2QC4VfYFW/fpX5b3z + o24QaZSDb2VrRVJ1dpgNcSr8e4zVZ+4+urhQNsFItvhRJTVnZq/YFITB3EJCrZXe - nLgpUtzY5bLRxCXoB3LsMy8LHl9ehyRmLeIjOiXTHQ3vSrV5kMYJEg== + BI7jHDLwaimv6siVxbcJfwPw7DJaTbCAB3BD/u5pa5NSSVEf769i -----END RSA PRIVATE KEY----- @@ -9231,62 +10281,62 @@ vector: segmentSize: 131072 total: 10 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:MDMF:is64t3zpeyh4dmxepkn3x7m5ye:nwfs2s4gjq4ca6vsmznwkkjm2cztpdcuunfiaqbhol2xvlad5t2a + expected: URI:MDMF:zqldyhgk4msrc4wk2mbek4tytu:nc7rmvy5fhgxbszjyshadrvfn62rqh65euocyb7ltfx3qmzc6coq format: kind: ssk params: format: mdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEogIBAAKCAQEA3FWV6LYIi/XT4PKt9oQ2dKE0FpR0aPm3oSJJE/VHOzHC0KXe + MIIEpAIBAAKCAQEApLgLaCjEUy8x/xNFZuVGYvJnPKzk2OTDtvDaddCmrzTSn2O6 - D/rSSos7hQAidFBxO1yuZFyG03D63Yx9n3FapREPYHq0MYA+YLst9ikuvKj6A41I + gAX2K3cG4V+NF/zk7A60ZoYjAqgziqk6ybvR5f1zxTk+zmWplhVahGox9iMi7Vp3 - nqu2Jm1wWSLC4sG7uWdbpIK77HD+OLaDAl0PntXS44IVuuorY6WuIl3wJzKZeh5v + atPcu1RNBSCOj1usI7u9/+fkxJmML6EgNtz7p4W3ZbmA8udv/l5U37S3tEb13kFE - 0sIxGSawM7GquYFfJx/9DM+BWtvnDTuZEG+V02XvZ6/33U1CCpWmYH8Fn3W3Dmne + mMKrokeTKbW9T/0YDkq1ukYziY5ye3eYbDqduRUu+bqmmAL5JPdb4GQATlDzHtvL - jw52BEEVPLXX1eo18TxUmPWhinmgSfF+dLsFo5n6wfwoA3a1QTrsA5PIqAGtMHgo + 5ouc3if5YErOWJCv5ZbM/jPATNqPbpUQ5FBA7kZWIepcN9OG7wK+0aaVUKS/k5Wg - DuXDj4m5UBst5OX16DE+pPmyiXifMIg/gd/13wIDAQABAoIBADH491cgci/MQZHz + +tTjDshkFAZ2x0MDnVz5L8qiC3+wIUbj4VKw4QIDAQABAoIBADlNaXJ31CyYG42A - eKFAu1kYdsfoQ77LZGqXbBuqtc0nLBhGhmb5bFib25P+w9G9rPDZxHPeyHWMWlmF + F8G++yiK6Y07HHWzx75JtcYMqyACgU8/s268JDJkuvkGc6Ansz/HscyE14MiHqQb - U7il7PkjNWmcauIPRBaMXZBHJuKDMLE9igryxw1QJPsSd0EWz4ztdEuLmzO1LPOP + UT9C3rdi37Z5vrawuTlj/lRYWT8mZA0sTqTURVLJ9e1VsSKAIrdfpa5z7qrSO+mJ - 8YbHtJNBy+Ltzh/mnJCtMyF4TM+WrpzffEQp8BhJqfdmaYGlMrtj02etg+l3XvSf + 5RoQ8F8L7owt54UZLGXSTTZxuQK2qb/g0DvuWAUXe+WU9F2rv4GVLzvsuQf0AbRw - /+qUiDwK2MEp2KbrQoDCkAGz+xUUVcCsBKhdBH9Lu3vWI9SoXx6ljeylXcQaOO10 + 6UdIfIweIa3GU0u44FEoitXGLq+3O0ZAq/9UoNRmTUSAgAAS45OsKWGhYjJ4qto1 - gJnukBV2K+HYIO8UkQqdm24j+OMA5ekWjXeSaakW73hfh+sMk+slEFwlFnnuKxXI + aVig4iNylqD6r+En6hbklz64h29dUiHdwOUc50duIarMnaDy2geCFHtkWAlkWJjD - J1YhUYECgYEA7luK61NwuAEBsIdRKqomu7wuk5zr/9EH5wUhAWCP1L8gMID/Q2Yo + SBQHEK0CgYEAwIdu9AvSV4FKSxDefLbU/jTU/NkrrirRzezylnFc/xsjTYIVZOWb - ktbSY6xJy2jqa9MwqY9Iu8vyU/GGfGgi7enbp6vPSQeO0sYCqPMFjKBmJqwUp/Eo + rH3R2myLRowBN/WgZUtAUwH2f4HwmJwcmMG+13RWYI/+Nquid48bGkNTUkhbb78+ - QGSP3XwJs3zK+Skxl/nvFSJndvKkuAfVfI6ynPz0I/qB58FjIW/z6WsCgYEA7KSI + LybV3mdH2pfRkbYlHpuZCZpf0lZWlPbH2WgtWxbaFH1UL+LT6AWcKIMCgYEA2wWQ - nPW0U9rNDfwCPckbdWNjysn7jl6lSNmNJj9d6iDlNx3CLAwoGr27zp4YEqa1i1oO + ujP1CLDD9b769bJdDYoLMAkciAjtg2qMLLmTAfjD221tcEt95u7Xq1qdpaCaro6j - ZxOVjr6A/3Do0XXV0j3b6YNGruAyOa4jgBvDoSz7CfVRi4Vl4J89jaNhuqnOHEut + Rf+0QFaG6BzgKTiZA4o0wznGJ74WDtKkaIK243MrOW/4+qSrRP+romKrdrB2XphI - PFenBZUUm++HillETEyp+P1aYsKwovlsQ00E/l0CgYBsMXlZYEKmAy71JjcdmqaC + t8mr7bURtDxjEAwZcWSO3pyHdcPWkXejOTgIW8sCgYEAlfGknxsJ4a7HDrl/nb/D - SOULdAtbz1I69wUITwB6nVbLLYKw4UpBfOl6/NVyU2k1EGPiU3u8YtLYb6WQCuTw + GIxLCPWWSFn+9pNAx5xYojIfh4D1apRMbsW7B5Mb0YC+fjeliN5XpY5UzS+FE0Ya - AVsHPOGWUKvv2JmUftth/dzgaPPnV3vh3sO+0XLF2jt35c7xIS349ejpATLrpgKt + G3phSGnJ0AC1Kxz3NohUwlqG7QF/fQODybNEQ6dKVduBkys5s6HZSZpaYHVvjyq3 - y0ggImHfgvI8dHe+0cZxiQKBgC9grZ1HMAhN2RoAp245Uk8JTBRwpfWWC19vduv+ + sQGquVON2wFU7MqK4RxlZ0MCgYBy2YJBAgnV2suHS/RRbox5ExA2yjBZ7USPCwoi - ac4TMfD7+0EYWfsom249hrJNQDGbISEP8bR3fZomv+YXwmxqSBoTV1ZxunyD2cWv + UdWSzR33LHSc1BlbRZd2VXghaAx0yHs5s5KTwkvP34R0WSdzwb9VODB+mqD6eN6Z - SVZ+i/AtdlsJpSD4oLk3ybw2fPZ7TD61idH7S/oAVdGkF6FzA+C+0JbPRdALQdqj + pyG8N6JM5jiLRlpBPkiESHVdMb+Abx6CsZAkgDSebKQNwCp/WZnJhg9KY71aXoAK - k+ldAoGAaCrRerbYUDouYIepA4oK0Vw0XrS4Jv2fuzO5SORU5kGp4ZVC/7bL5JNy + +yT27QKBgQCxw9dLlBiQLQuqDt5TQPkYzNSY3j2seEUqYMF8QSTvWxS8jJN5ofCX - obw/HZD3NMmUcb8ShBBcNriX/IPM/S/B8y6PF2yLFet7lUNclyN5LtT3kvi52221 + A/yQRxNyunl9AJYIM8GE+4CBDszhSu8CFeoE8cDhHSIMMFOR9dSEL7yXlaykQ195 - oBNL0mNYOo+J9w3K66Uw8M+df1t+2Gt3rYtprjWDgOZnlxleBks= + C2iuwWcu1i9I2hWDtNz2d8hLL4XgEZxNGRQNFhC8VXEwbGL1dizZAg== -----END RSA PRIVATE KEY----- @@ -9312,62 +10362,62 @@ vector: segmentSize: 131072 total: 10 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:SSK:mzceoqnz4uohtzae3qcgxkxl74:4mbfxhucdkbwcvnum5rjy7sncthwxppglkygqub6lbydf2n6wrya + expected: URI:SSK:ezd6kflfw6qfpryquupyjxvg34:dbpmolc6skjs3f27go37huyn2zganyrgpcdijrqlhegvfcwdmvyq format: kind: ssk params: format: sdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEpAIBAAKCAQEA1mIefLi0nhjyHfmZ49dm37vw3r6V/Q32a6MSqvsi7hz8rR4E + MIIEowIBAAKCAQEAoNZxep7/aicgkAheO7wqXkoQY3FFMdTRxZNUrOEQ3Z5AAe14 - ngbxyY5ZwUmLhfSWvVszukjUcHz3FB40hyDRYMOEFA8JSb5ElVePtPmXxA51G/tU + ill4A8FRMZkT8xgpHzBw6bNDWO4vVQb6G0baM+6yZrzG23MVWyyKkiTjnnUdWU2X - dPlmLnuZCGfiKlOWaQ+8Xu17HRdJ3HSEQjaJVpMMVTh2r+xe+0IEyCqdHUlNp+qY + 7WxGgnsZF/evK2cLFXDU0oe/CV6JzqYMJbyeIgwdL3+OmqJfjvtldT6a7ZphaOhh - Ya+DumJ7TOZOdF1DmFdpOpnm01YK9bDM7iLlCfb27RZWxoN7gOKVpR1lFHXCO9Lu + i7IJSJFlb2sbdcHWZPCyfPHZFW+6sfdTSKKcn/DfR40lG+TYzVnYQfLQ15D7ZmNP - QQo2G2uu7nE/w0zEMk5gOdjuKpga9j7+Zv+E1NXTwF9sadv16VmpEeAqeeICNh7a + EORKNzHk9zOYZnPXwJS8IyufjaEFUJLAguqVZGPHX212GpGW+jzc73gSC8oDDBH9 - OUKj/jeU5d1+zvYBCG7x1ZgLGXykiMJ74DKYLQIDAQABAoIBAAoTrZwuMGs3//Vx + lzD1Rc9aeb59hGOL/LtyNsBArAqruG0h6RTH6QIDAQABAoIBAAvAl6Km3xFt4fx8 - vI6NmuvMUTufGLy+0cTocuGvkUpA+Y2Hmi71Y5sWQljIBLNktksrRMiuULIC5bg/ + TAxv26WvokJt6qkxPJHECfYm7PFQqKsrY7kyP+mAVPM7lQBYldqkUr/U3DkxkE5V - 3Tc2zzCtsAEjXcvmEiI07e/TRZN1HIMWsrcW2/s2WxCelW5o5GqGz1Nk9UL+S739 + d+2J0BRm3uzQYvRylI8oskhq/yHbO2WE3LLZzE4o+gStEcTpXt82cyqeBijEWmv5 - ihP0rUrw+YTt9QI66ZIE3eWsvxrXt5+FKvyclQVVikfRb5WMVceYYpYoxIqUsWhn + 6J3SSjjBK1nG54/niPV866U1SaNOXKPP4JF48+BKkNaaKXeYlA84cAttD+xmO7Tq - BMkZO9mK3I9x7V8u3OsAAWYdquhNOBf2RtFXFXASWSKr5FSjWo/f5QP+Bf6n8kX3 + W0PoxFny6hiGuDXYKsVlBnCujD18eNx8aOv4T9o27A4LJztklpUF+BuzqVtfhXhd - SHsRek5WweDkSjFXRwAE+urB34TRbMfcU+sVIUOFZiw/ZvSqRiJ3BnfaTNSKfJhj + CZ8Y8Tay55PKIi2qv/DLYPT0fjMC6cbNoT3NJdTl4LEB5NWk7KS3Z6x61ArPHdwA - gGnpTP8CgYEA1pndhW+qbLbQ8F2/R9cNxXCIIYKnj6/8zpUhdrJJpPoDdADWkDJJ + Em/km6ECgYEAyyrcRHYRBg2keBW/5BpIYLJEj4WJ2VKbB4jicc+wk3s+uGM9lwMt - NJ2q7jHgRUrVKw3cHniFx7cguJSEx17b654NOH3Bg8EZIpBBFjTJLuZEINNMvp9O + hMPHt5r3NihmsGABjqcARrLLE0Jx2qmyjWrJX2rfLKtY1ZFdyQ5hKKHg+ubpMpEo - NVGEE0EaPVn+46cR16ZD3GXTS7QOULQeaP1IsRG1110QN5ikq4ItOncCgYEA/71/ + DwKOVZQXxOEfJDqUUrI8fM15+UUwgkBgh+dGqUb3IGdim41jIxoxiBkCgYEAyqmh - 7BzlQBLwBZClWeKqwS8mkK38gP5D1nFk+uS2jxIYJsb6cO6KoSY0gSvDqqfrGfgf + w5pP6R2vzbvfTtauSCzhBQzdOoAzHY/aNq9FY/isAHVQVLf51obO/FRWIJHOGTQn - t4HjOqWhP3sJw5kyNgKJTE9Dq1es2Wh68MSk1QMhyDrNXRZDbfayz81M3PySqsWk + l/OuYKQFZzR4wWIm+763shxmGxOGHQQMPs1JwJoksl9hPJx18O5gUZYhQIluI6zB - BvxzDdi95ZU8fFdShT4JdpWyhu+MHj/YmvBjx3sCgYEAqXcd+KaCtZD0lCvjxm5r + prOXgyVeLqcm6Ios/gkT81J4XLxlNEKC1BwleFECgYB1GdknpJ2fTZG0nWSjBvsc - 4JOJ3LSZb51xDQ21PE90WoRYP739sicTqior9ieKzA1ZIsOyJJnWQy04+KnH5Mzi + sOOPjbqshk5RA3bxfnIaL3kxMhI3zl8YHPgqPamrj5HQqyV6oYspNLiT+0JAdHsz - 7ECGfirIqyvMln/F9iw/Bvstp6JUw1932iECJFZPy00LPGkNbPdONXhvkCOi/lYO + w48Z7jGAP6rOPiE+V4lssBFKzHkw6jWaoTCE5vzkP5WBfjoriAwRKyXYpSaWjKCW - gagqRDIRH/3MtaqjtxB4eOcCgYAhF0QWKScAw3KLRcwfdVTi6lbzIZAqoLvmY4XN + 9JjnzL138d8GJXI0s05FUQKBgQCFF1ebnFiEUDGnG40wOj4kOgzggy06AP0QmesF - cQquOIkne1eshTEq6OaiUCdhTZj+Izz3YbclP4k9zY3V4Vy94FYjqZ337cBP4VUH + ZJ9eYu2aM3C44kVZxBhkj4IsS3SdCqpB2Q8Yej7uIwB2h13wj7QVbR8FAxJdNc5Q - EmrBpUYZwoIQKXFQKTu557aqYYQY1LoErWW1xPXNXyIUdLgYxY4z6erPyu82esxs + 5AJeURxuY8L4yguOWQ26JqzZtCc3mHloX6LNxpmOa8lah3u6rP2EGxHeXP7djhxa - P+6pQQKBgQCDnH8yqABNGRB0fZdgV62M2JF8uLnJnW2fAyWzj99lUxUUZ59FO47m + 7c0RsQKBgAn4Gd+IRyRU0Wh07FVh/+LuxC9Fc/GbIjxNbdodeuQzuOlU7auUtIId - SGCkZ0n5sgA+H8WqEXiTRQYSVAWF3hnY5Tn/cnv2ju0oT0+98pk9sSN/FrR3NtnZ + wVAXoiatL+sWpqK1h/+4YGb3xEOd8QV8NPO2mImXr6fWJ5VdZ0aX8DVbcDUCvra7 - JnE13OS+wLqu8AeOv60XMbKv2jMWlQUO4vY3FohZsTNfZ7OM1RHxjg== + w7HwiFiwVLIGdyNEk6OuiaaYGyJyJEax8i3GKtEeLOCagh+//eyO -----END RSA PRIVATE KEY----- @@ -9381,62 +10431,62 @@ vector: segmentSize: 131072 total: 10 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:MDMF:c2d6zphb6e5qchmyes5tb7fud4:ns3clwx6gdzeguxgu4tffjeadjt6fiqmxmqmg25kxqvoa4v7qbba + expected: URI:MDMF:hnueoftricmvuv4q37m3b5mzee:hl2efj3ydjv2ydjk5ilxb3kwehm7qi624mefyi3ksmh7o5me2nua format: kind: ssk params: format: mdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEpAIBAAKCAQEAxlbX6B6G4qcs09eVFgA0AvaJ2DdABWBPTKLRUvVdXg07Y+FA + MIIEowIBAAKCAQEAnBWZCUifbysHRyel1LzvRgjn4gw6sQo8ifcbgqhvHx0GyU7B - UHe1s/LBiG1s90Ghn46B7eknPFzA5cFV4iud/bqpEw2Kwwys4rRWwXdUF5FALYPV + rAOUajUAzvCGVkUtbQPe2031Sx1LSrXkra+1D2yGXlQ/pyLJixUTwAkF+V+sbF0Q - OIOBSLo49dtTYl6AM3bAacg4KHuUXSLECAEVmqAwVuxUT3mJHfs2e+f0ProurJsK + bR1TGztpzxQZV+W2SDi2KFCGxR4b7dN+0yZwXCPbZDYhpWfpo9gHBQAlI9VPEWto - hfQYbxwpMpGFVfwjDjehK5isWH9jgmXggSusD5HasNrLrTvqYORNjjgUwZvP+LBl + 8Q8Lv5iMRUFSgo0redIGWfY0lIH4+2/FOlOG30Hl4Q0GwzqXiSv1rGjx7fc9jDhN - QMNoV76bZNfHMBzT42edEVE+i2ptjVKbpefWTwxaH4vpRJH09mmuEVfydYygeCxE + vxf6tdJs57MVbUf+l0ZZ+eh2aST3MzfCZUO6r/DIYzzauEYyLG6ImRImXnjAhZ/D - UpYZSweufsq1EV1qbAyRNL7PTrHZWxoPy22MbwIDAQABAoIBAArd5dA/OyDk8iHp + i5N7lz4R8KE0A8VBh6Zy+SsjtXjvZeO81q54TQIDAQABAoIBAATnndc/l7j91Gei - mfuagQvRtsWEm9RCx0vjUDiAeAaeQmBB20FTzCR/viNiHdb1BUyikeZw+htAPuz3 + lztek4idyNxJhOz2yiJjaD3Rw/9+4ULMAtnVrqEsGLd9XJn96FibuEdScBHunKTU - qRbbHYaYbDxxs/+x/8qGASQL2Szw/BHiVLoKPnjRHZJrqC4BQ0eBwwYPGCb9Q/Vp + n0LA+dLUrB/vNvqDepYFsR5F+lDU4OgdKuIO5kNoZCG3JeO6oCDYCAtaGnNmcpC6 - Lh6WjJ8sFd0iIqmeEFyqD5cKia/I1xxkQZC/B4xjCT36A8YKRCQKMcJ++Ik79c39 + 5mpFXomwBm8j+f3cDH4S+1zSHW3+iQJiYkD5slH5nyAIWCYkGBj3z6gzOhCOuzVr - KPakYqcvjeGiM6rOu8NvrKgMqHQ6VagIizXW8WgmgqTbvYOdCV/MYfvTXVVJ598r + 4/WJdXrrxv7zENUwz2PUYe8aEhtyn0NVayCnS6oxdONwUowySPtLoh8G9EW8NQb+ - C80LhftXOdjMJsbXm3PxD1PcfNmSyMFw+sMdQsBb1teUUZ+uhx+Wn1cNTAQk82bT + dbBehYK92PL/XH7zIachSdgcRvoXXzKSErhFXdVh6NM3wYZEFVh7yBGWcxN/453q - FKYeqV0CgYEA7i2zKGSmBJENnAtChBfkhHoE4Cx7qUugWhbuRHNKZV2XiJuNwHb9 + JAWTvikCgYEAxk9VdT03f5jM95yHhBMb7+E4GWtt1iB2hfeS1hH6xkPtmBUR1B1+ - ui2QVgCrCpxwNc1ZK6TiytW7yg/E5rjxa44TUh0yVdEYSyQZX5i/BfWWOWDOoVdA + +nPc2aU0kv3VGgzIElUWOrZndf2eLNbNzslfECnVg/8EwUqZhFJjlS9GvpmE4BE6 - gKniPiU9eT6DeJGKRN1p9Rtkx14nQISB8I2ocCtMwapeCgpp46sqqWMCgYEA1S4G + /z6kSt9locLP1GvPllhbKm2yi1uQCbPrqQRL/LscJrubV0awDkCdx1kCgYEAyX2d - DSnsydcgx47IVOOv3zUovKfEbNv4VjOJ4Pt7eVTJh0qwJOnFhBTh2FwK3lIsN2uk + wCc/HDuvoVcHX0Pk7tl4RklShX3mlHHvWLQMAuiN26edH92p48vscw4s8DBB9K5U - VbxvJT9XcEBy+kq96cKimQHQKMTBdSwml9LGI6dHmfsl4yMnhqAfMtPtFDiJpGUZ + 1mqA1civ2f6zZtcIvqoerW9vxwSpqVrDwq3HBTOscuip4aYbIOnMe1pLabjmsrLn - 2/l3Ybg7TbhX6RqNlBU08Zzz7VPzkKX8s99KBIUCgYBXwLqrfTm1oQPUpEljhbIK + tP5rkgJeUx0cbSfCBpQjxUlw1BsnM1qepU8AThUCgYEAvSjVekRPWO22rYXomenk - JTK6rWj6XQS9bIlo6tlUM4FrMXSunqio+bSeGyzpge3NxNS/wcZVWR4ROnIfV7CL + Xxc0fMLFfVdv1u/FZ161F0OaMdP/MpaEFYBJLG3yTTfEetmwShRRZOWyoJCvvVOT - IhN4Q42SFLHQrYIzuIFY3rz0cvhudUksnmre3rWhgCjMOUMqUDGDvw4IbmYj3S5K + 8uiQPgm5efPaZEm1T8uK47W5xHsJjPXCkc/9xNF8zyTVO2kvFNjo9Pq4MUfAiBDP - xMZ0XV+wUubG6ENPQHc9ZQKBgQDTLRFPjv2DALn3FWk8NoStL0LYh7TcRZefBNUL + /GN12/farXOMhF6P9rhaB9kCgYAZLtELlhwmLDOMR7NNLdAsJhQJPNrKgmzSOtc+ - 6vNowOYWQJV3K6C+89S5+IvHqk0k5VvYlp7fnfynNSDw8oNpAqcBvTsQd8BQq1jb + T+p8ZpJsVKunsu2r4e3gh3IIZw+nRC6oSdFmZtnLtjC39sJKCjshVB81UZje6NA0 - wy8GeJpEXfctJ1DrWsktF6TeCBfJo2FXeKubQN52YiurveMME2nsApfcvPIlk1he + wcFxHf88sWWiJT+Ywn/jHur0AL8csI2TKoVJT3B4lNfbsK9oYRWDb+VhLS+eFIJl - cs4m5QKBgQDdteelQs0L/lkYXldLqcb9KhIPLzxskJpjdlt+W7AsF6v1ayOKE9Y+ + iNUx7QKBgFRzMgeWnmVHkdMt4WqqaNh6GEwXM86Yeaw3f9WLLKOtoR8QoVQukEWC - 8x/Li2b0rowY74UKz3xDGWN0ZQucPs3aDsxDG5rYl8nosDL50q+TNJ8D81Gu3/II + DplFDDniCfBfD+a/NoC+a5azXI53NnL1iN8t/G7f81xWgZC1y4yeDBqNSyhS3vpo - qVDgw7XuL15mYWWBQS9m4Jge0Ww/MLxQGIPplbToFUSo3StI4QJsXg== + L9QHJeP8AZ2cEsSxKmei5+zHuvpN4HuvndyAXuWh1V6TYSh50gVR -----END RSA PRIVATE KEY----- @@ -9462,62 +10512,62 @@ vector: segmentSize: 131072 total: 10 - convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:SSK:exwdnakezxzb7ca7xxirywsstm:hleh3a6wfqbtfgshmhhhtpcuyxpgpr5yz5gktvtv3f6oghhnh3pq + expected: URI:SSK:zdubpnm3ft7fapqny3dzeypowq:o753jes342uzajcwt6awhn4rmrqn3cszsth7kfd76zd4ow3yc6ya format: kind: ssk params: format: sdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEowIBAAKCAQEAqyZss4AeZHIr9oEVMvbdTn70HiaW1CFY1TTKDkMdurYflEf8 + MIIEpQIBAAKCAQEAu0SG8FEKO2y70r7ymRrbI43J051onWTZGOZIMga5iRApsZxe - jTaQzzKsaSdMeZIX1/5BK/pmCgLWgQ2sxe1WvvWr+yzcWzlOgIMAZbYnFr0UGeBY + VGIotwXn3+3SkSCChu7Z8OnTx0iRJuAtIi5uRaOzfzFfDqsOulG/Gglvx5kRDVff - 9sy38st5pIQWsisWdu7CjvEOv8sja8HJMwGc8heQj0gxYl1czS+eLqOsQ08mKlEq + 54RXY5b9kejnLu/uqBdanMU/xUmaOXumHgYUdERxKdrAZhEyJ5F/AhINA+kgwexU - ayPF8rHPYvSHrV0kUrnA1AxeLv+bcF9mxL35c0E24+RNLLAdV5CEhProvmYkO56j + +pGBtTFiamb5S2gqfxZ9MyR2LZfjbAvcA/PRCGcs5w+j2Kn1rSBmIU2c8uED/bpz - FPD4swnT8bYYRdYq4lyda8HzK6QjWAW1a7Kl3llcPhi85kTeBnkSyg5LwTlBVpZ1 + lEhu7PTB5GgQubsl+1PYwk38kSqDW6UQiydUKhCVwqXOV4fbkzDVdmNarmPToZJB - aDH3MUeVyWcr3wKIB/8ESV0CI74EfQr9p9mQ6QIDAQABAoIBAAWj+Sxjxiylih8g + yfY1R2XNRhLShnHm0WPJaVAq/I1kToKUUx0btwIDAQABAoIBABsmvmHNfixLgZf2 - QzJ4KhRAkRtOxoUEedTh/eBBNEE/aBRHGJLPV++qMf4vsECn8M76p0tzn8oH7KmZ + s3nbWPZ4slCKPAbF/mwLx1/pdbEXtNPZlhup97lBk/L1qlf8XLBvpQ22+Uuli9YV - aaamwyd9OX8oS/VaMsT/vEM94PUqA1f0eb4dglj3PYGvZD+YNa/y/7i+B1BJGbhf + HrYcAUT7jSTd5ahcyM/e1lRSFfDckopauU359CmuVKl5GTvG8dVRPYQJXUufdkrr - DTpX/OQKJIaiWJVPMiQlg/sxw/c+XlgSO4g1fpebSJoKQjpXufkDcC5fGisTu1H2 + UJR90S1iVv34h3jE+X6fK8kDEPwF6vcDrvIp6Y/coKzIiyWR4Wjc9CXYonLfF4fX - IPMz5EYbYkg15tcPdHQZNG4QoEzD3jKpHCXKfJiVX/+LOwvk1+vxuXE+Lw82ktGh + 0XsHV9cVNQ3RU9UGNn9hnH88R3f4YCBqJqh77qusUHW+RXy/XoHc9Fz3Lj7kGcJ7 - k5IQTPSSfXLgR2v2iVuoEH4mneamSuyjxi5jay3PCgiKhYyVkU97xIhFA5FIcHPU + CoYBNwLKU95fdi9TRIQsnJiEoQzIb0H26NDJFjYDPqLCoj1C4u5SF1pT4WaazZvu - vV28A2ECgYEAxdNzI1uBGkFX2SJnKeG9ApwNb2ix+onQqWYjSNvJigy2Xktetszn + RtjHaGECgYEAv2mybf3RZ8HA9J6n9mOHOdU/5/q4GwxEj0tsIVAvf9WdvbXgDeLm - R1SjEPhILr6T+hVmW/gtOJz35fm4xPVlwEdFnkVY7J3X+EyX+evwpNw1RpsbUKp5 + xuXw0j0N9gfIBnCcrEZ/FhR2uJmQfN7bpA5IDYrp3aht/amavkgDv98WcG9RYJMZ - VgLsDYjAy9LPFOPV2f/CxM8Thfcsz/lryUuHD3AU7UyMB6td5hFr7EkCgYEA3XrJ + aso5RxI7aAJuJn7oN62fKkcCUS4S2bMrTUzYkrfcaqZLxBRABLctIFkCgYEA+nTE - Rf17q3VkbT4u9XORCqWSUrdPH+bMjUzWW8VEj6O99V8W3ScyuNF4CKb1rga8G8ZT + VG24gt40Ogehs17u7GL4xYhB406LyZ82rMcI8PN/4TxUV/wQyaZeMKtheMmWGuV6 - cV0UEI5+yRKOeovRjhW7QbS28yhWwAlSiVxsQ21iHYE+Ep1RFg731rGMT+HT/kLQ + RHU2l2o4alW084ZAyYD4cY56nIJnC7AUNXXZrYKaRP11LMNCvtK+T7HE3jY9BE7Y - oRPxjUuVZ5g0CWuVL+RqzFplCSU+UOF5+/00P6ECgYEAw2GGivo59QtqTxrqVvQL + FrpGXbyeMz5G2wfV7KCW6kZ13C58IzLF9iy3Go8CgYEAmq3vYrMZ5Z7NLuCHGrST - sEMeBdWaSn7IpjYpTTE9yOmrSFAaOGMBXXLbJsyAxiIVll6CXP0s9IgbUnikI2rW + MkkBu5T/8duYC7QHTWRe/g7ByeyPgqk5lMF8OmjcP1VKbunRseXGDTG8PrDZ8g6l - 1uPNf3awT+nJPwOu6fg8EScoOxbAEJh+BBQYvXk+KVCIk/I96PPwkl6OwrYP/Uwz + r41a7Ja1JkpVmAbW5a2MWiENIQ7T1BcLEyEX6DbzirlsCe/D+Dp1xNRdKvzwfrwq - R3kf6IBjOsdqWbzHnY3BUHkCgYAs4s+b0a2YqCf8Q9f8grlocPngranpizr1gBcJ + 4eyXlvi9RfHciDdVBHqCHQECgYEAurwasZRI8JH5wJZ2EoWif+7e6nBIJ9ElWkNy - bkdg3QyIiAb4NxN+hWVQS5YK+O5yqpUKqpSAboCfe5VInMGRjDHxNRDG4uwB62HA + AWo4mWYDn4xammsenSqEqabt+p/aYd1cxvPZqxUQUP/r9XHQliypkAkaE90KNWWn - 2OxQFgEGfcT4vM1MLShpaH5JSjlOlHf3zTTtL95NqnkRV65akG5ckA1d9yBT//5a + +6ANl1d77BpJpgFDn4EDUeoKDV/FKJQcev2Rf0wla7FwJNh3wICPZMb6Exs5hQjT - 5YwLQQKBgFOdazTFzSvJO0lGDQnF8zvDtLGR+ynGuP4Mi5J8xoUQach2L6edFP0u + HlOChbMCgYEAoWGCOHmr3XDxNKEju0R0JVuOod3VkhkNwL2l7P343QOOO8lfkSeQ - agjkx2clRLGfWqmyziEYha8aa/rVZfr5bPCz/i1IWDCNZo5ccg35RfJtP3sYbrtG + F++6HxzYkR/50JuYS19bUNS5KPYrJXqsG7HT7St88os8ORcaHtmmORf4UlMoJkwl - 8ZiTTs7d+1WlngDUWDHa/aNGLZM6gr08XcMtPOjDjQsUhkXmFEsZ + H+Yi5PY0QpBrszST31bgfqdoHKh7i0BW9ux50XO+51GUPVwXQpayX74= -----END RSA PRIVATE KEY----- @@ -9531,62 +10581,62 @@ vector: segmentSize: 131072 total: 10 - convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:MDMF:n74rwqiah3chzxknpktu7sf36q:xdwwuwhs2owcnk64t5e4exp4vho22y6vyr6mb2xg6htidi5oufya + expected: URI:MDMF:xps57pmsgffe3kshaz7ynqjity:zexdamdswofh6v2t2wc5estpazeirbynhst4q3k3ffrq3w7xau3a format: kind: ssk params: format: mdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEowIBAAKCAQEAmaf8JzMPBSQYNxft3tCLOeMX+r5fdFvmeYvk+n/xdgueZIHB + MIIEogIBAAKCAQEAnDNgEwMYgFqRwwZJv5J3wuv4qByLdSfvHKQrLKXUjr2QMwFF - UNm61tjPaRPAyopGlBILzfrNjLDIEIJFi9SXF5esnkH/NEdfhqzYZV5QZQ9s+IiB + 9g8ohHmrV1toiqIRQDiqxr71WxCUMJk/84bBdkAoVGIM3jAfl/q1jsL15JtFPeQz - Vu8RUofty92T6XE/pNJgouQu20YLEWDCSeCBqxMcW/DJBFcNP1KHqK6tAWlz8917 + mQoa8hkWQ3GnxL3MAnzEi8knIpmtJkW/npG1L39CTYt33kfUbm7+Z8VKNjHdEQ7W - Ty5Wu+q+NGN8ecuQIoja+WdTeID8mqUybsObh1QOqIiGriyNhMq18cOngoJLcjoY + qsRxfmxz4CCkHQAbtL6dZdVbJwEi6Ea80/jyY6ipr4+JqIhtGpksvpKhfhpPnetL - +EP12LhP7yIqvm2ScSYZt8KkUvrL4rruuQ83ilHs7DktbQyqPkeDWpsliGWDtY3y + AN/aX41amGlOE0EEmdjXi2JhI28+fYplXgNR+sBqsi0bVmQZsRb5dwVpfbNjZftj - AK45cxJFw4GupZiAPor3cS0kXB0EYq/cB9chjQIDAQABAoIBAAbe6pkMIO9Pal/I + 8WKEKB/vKvm1UlmOTxCCY/bCW3QAYeSStV40HQIDAQABAoIBAADFFA0j78P6Lku6 - S6QrehZID9HwAk+vLlKgDUigQPlQ0q/W14CYg9DImBmwPu4vmbFUTz/SJ6/TVdbb + xTRHgYWZaiFR+rH6H2iRupC+xHxrnMFTmUesLXPxsZF9ptdAEzuwy86s9EKdo01W - JGX4xxrQdvEKrXE2gdCBWRnLSlgChRJl9DahcQpaNqlnio4lOL3ThGu0PV4jtyn7 + BAWsPVna2RgJX6zcqdsy5iAs/88/oKi8bjCr5xQYYY61ibEjilTczo8tz56RCVRt - RCTOi63NKHb6ANsXU2nR9Gv4B5dpAIAErMMSJf3WrVD6DfwD/0/2JV8D4KFKlhWk + 9ZLPfwgb5XTCYjXbPsXIkEJsq3/23vg7ji2Pyq6GmhrSB+mxe18u8ggwEwL+sMZu - OXEwSodaPG9HLxqZ9p6pdQqv2yn4uQBfdUUJVXgTZ1DwhNWvbNmG0S3VAcWbhGje + 7VFAXoEkJ9Qa6AI3Vwx9/QuGs8W3SkEA2MiEjYWrVZUYSYTdyQ7W5b4qnFPBj3GF - kULDQF7WuXbte4lSRh83L3kL7PgGK2b/VfyX4IP7urSbMum/bZ1V+3GKkcM9cxP6 + gDzsDbNEDNijyW3CJp/SEfqSZm4/3wmxIIA3f6Iz8jZc1Xmo6LdwMwWOm8W1ltDE - q9yxPfcCgYEAxSt2Dl8DkxY4+t/Dt61+MQFVWJip63T+KQiietrRzFlGDa2vrlKh + 3J1CGNUCgYEAthlbkmf55xW/cw/0OIDcMkaiQb4ld5jdLrVz5ZiSUQjpbeO8lIBL - wdSxDaQTWWAc80XLMHtgTxbI3SRNr873qYoV7lahbf8utKn0w58DgAQQzI7vfeso + epQzSeTNanJ6p/iI0xZQ9kSL/9kLBIiVMqYhCFsnzqXcd0TX59WQkqhh/Fup/nCk - n+QNHweozsyKdCepnPUmtu/EtKpCMX2tj1QQ08zn7k8rHlkfr3yRu3sCgYEAx4DM + L/Hl/eZvDg14QYhBoZBsuWgfgD4LIT2+eIPFtQxv22MJoBWybFy3PYsCgYEA25di - sQ4ja7k0N6IT03fJK/Eoj4LUwvs8pVarDAQiZ4/T7hDZxdAdRjH+Vy4H2Z1ywY/D + GV6OHMhaE4AENe8MFUlmsFq5Pdp2kDySDDFYjriKJ8Z8ejY6XBHcWIkg6qCTykld - BUXCBLfQI+wr0DjGMBpTxBEM+t7ujTUbauMkCMI3fZrV5Zk7UK0EChBYgGhUiYis + 4dTkdPg++bbA3zc1yYf2wN8bRVISOUlioafrhGsi5kL9HUIFWxtfHFgDutMQGxRI - YaFyL2eiEXv4SomCbim3Fw9pp/sbBBgseEA45JcCgYATUVfGvq5l+dZpVgUh+OCV + ROFbXAbfm9t1cZvUpPlT8skoU0zvHKU4L8MV2fcCgYBrAWWlF8Jq/4Wb6KEbXuWG - QpEvFf4H8LV6JbttmATYJaMEchD1Xmk0yXbzZDD1H8KWXy8yN9ROy2ewqv7li7ye + CampNkIwEDzRCMGNBmXchn8dGvkizm0MH/AvmOr4hUL8V3iXigKTZF5cPr9Rr6z0 - IsZVTK2STl8wGjq989Vu9HcE47g5ORII4FocwS5b3JRwHvayRx6c6870+H11xd98 + sVix31b6AM8XqvWwfvfQpm/F6ltvb+ObZOtAkttph5LF93qRpRuuq7fvFQZXR0AY - XHstlTTgF2edGJRPKEBLAwKBgAgvh7aIDvn/il3x/4BAvPdZmMFyq8ooRs+945zF + 814HcMJ+SalLT9SkBquK1wKBgBykLiNo9dhDOZx5ghMWztin7kDqVGcA6538iIAW - mqfHJfnxpQ3RwTG9IWNwVxAdvrSkcmsH9rL828Rtj0qm2bLlkaRM0syEUyNmF27m + n5pd74comGvITuxbWAYkPKrdrukfkKM4BWRMTMp9T8LNjLJwjXqynvf3sHDQZZD4 - TPczCNXVgYs/I0jnIHBNRWRXY4iVHAWRez7osKSpAoIEbF6axZFjp4El83DSkRiK + OfvXjYHDEwiR5+juNQWZZUMk7GDb0GFLk4L5Uokdor/it2WdL5nnKt9SlY2C70Ur - AguXAoGBALZyw248cG249/ra4PwNzto5bal1mIzEMfG2z8s3Ra+ogKKzNQN+EbtG + iNoZAoGABeXyJCB8iv0+FoQG+UERAN1q/z6ycM1CKEOoaXlgWbX7bV6C0CNK9NFN - SgATLvwyPiZrHseQ7QtH74STSHkhJNunD7Upjag6lxiap5WEKoX5wC29sYyRvLTo + IcXQOwahrj0N7cq430R8tmgugKp37R7mFW8RB1B/nx/MFG4fSLduuNZ6/a/vquMP - uLsRkgovxnv8O6qdnvAU92Pz3465gFYP242QS6wWD2+Bve9nYuIZ + k88UQFA2FX9bV+T+NUIiu4BQwsNsO/vVENX/cSAZWOV3mTtP7Tk= -----END RSA PRIVATE KEY----- @@ -9612,62 +10662,62 @@ vector: segmentSize: 131072 total: 10 - convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:SSK:3vrqukqlq3dgfuuldeucfetaoi:7ruowcqpadtqzzqj2mdj4a3qguqs2aznccmahwj5z5aqepgmjmta + expected: URI:SSK:j64uavihuouuyskf24dua7pc2u:obcjb66mh6msejksq2b52dhz6vffzxuvxjnoxvc5eq6bp42zadrq format: kind: ssk params: format: sdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEowIBAAKCAQEAqm3D02icCWXSTGKLQrscIevvZJ12qzlGc0P9y511jEaKw5Xd + MIIEpAIBAAKCAQEA5i4eDHhy6sVcQaAK1wSU0kjdcM65SS2aB2Rl+SI1Y2o2JYCA - RTcZ4isNCRK3jnRu7TZkYfp7j0wdzMH9M6/BFMZOJgUUCl5Jn7vn9TD0KzBIU5Vt + MdXYdmxkBJXkrrZQ3OEOu/KTms8ovYlQnN0Ayt+9GuUoD8x2kkGrry/bIxwEmTqu - uoNQ/ZIxuUd104FC9ywTg8qNjnT1OmhR3BQse2rS7trn1biaujRt62k4SRXZKsRm + kSqmPbiloRytbyfxffo5k4+APjm99AfnvHeOaXlxSy56dzkQJaomI6gtrQqlLLhT - oU/gkkuKm4TddUA+NSksO95tVQiiur/BJjJk0vV1OOgOtp6PKXw/RWAEhwbdbH0t + /lSPlackJd7Al7umpBQkX5NWBkqyEGT1xV+9Lz7ttfaaOeM8WMG0PJBHy6LxqpFI - rbj5xqH6agLX7Iz9sIH/5zw1qJr4Z6CBUp/9U68Yatt3WzhklroHvpZ0SK3TTlte + /V13kwFNkHw5FXaYzsrjfSBmCOBVPQT+ZmAd3CRNs5oxtSldKa3br07LgNRvoj4M - 9BufbOgIKzLTIpfsne6sbIi+0LjbGYKKWo5zDwIDAQABAoIBABJCZ1gnnYweNh1L + 2A05bCiNoMm3NO56zAltjXBtrd/l3ySxdbWHyQIDAQABAoIBABGTdWtt2haHrfD7 - 84qnPFjgC8p2Wmf233brAm6FxLnONwDEdiv7vtCt9xwRPsxK6jWM/c1HhmRwbcLp + 48AB40xAUJpfyqGmGAQW/DtfC5UVA9/utTs48T+vrJ52BKF7neaTz9B1qCQyy9FX - x8R5YJDmvCmzopWHy5CLE8t/vrE/34fg+xwgBJXeS2iD4PpTn4aW2NJmaaspGbrz + Nh7YOEqFdZbjZyD3s5kc5xtoK9M9PTOnGbPPfiSp4AnSmwKpGeVM8U8NbtUxiwni - wU14ddmVNNs1ZeBOgnlPs7UklUyzFyJXAGENaB2MHqyJlSxIo+KkIxEjYNBBTfro + faU0Ot1ebtJ3ENZgNtWtbZ6c5an4CEcxTjug8Q2WDqNmLqsGyXRTpgXZNPIFLDSj - 3VKr5FEePr+4jU8oHSXcspCIZG1UUqUaIjGh1nFPxHeEgerGOgCHDAmh56Bnc57T + rwAw/gfJ6/9vmsxil4mnr/fbnSL9/i0XCj4y9vvc9AcwX+l7v5MsNLAh65169Gpe - 7pY7B+A4HgMo1oCCGTnhRVnLkX948ahl37kHSq/1jZnnN6ib43LLbgGTwbtElg9o + SfkNcngcbVWDSIlSAjAJW9lJCKZtvwCIHUYPhX1lrcQaEPLzcCYnlX7hg5ogEx+o - 35NJz60CgYEA4yul7Njf7kBPRty19oOxLmhusJs4l8w0rvVlYcPdgXm7qhmdg++Q + p49JTwUCgYEA6nbV7BXTwuxAUNmog9i5Mz72vydZSyWPDk3r0j/uM1IdXKAntuyw - 3crpuN8M1ld8PD4jV1gunMQ4ganCyIAkn6VWR8qmKG4WtUNxY0AH9wO2z0+y9lKt + lNi07UcpjjoDXSO0+YtIHCpr5HmE7gV7FDutNdG53mqkXImfkSiywRM+EPF+fOmH - gp1h0UIDR+qqZMKxsIyYOeOstqi1UjptT3bM3N6gcnePgZdrj0f6t0MCgYEAwA6u + ajEsueZyi3erqJRZrsa7b7HEAQQyZS54x1I6viBo2m6nFsppqV6hMN0CgYEA+1KM - qjL6xywTxd0GE2knEr4gKSVvYjoEO/Pq2lefd1+fQ9KNIlmfjlUwj6bqFmf0Ej6a + BOXsb67vB+oXDc/oBEBq5HV2TXjXQK3ZmZjOfcoNW6jTpXDpJXXiQoMWfHT7W/GG - CBLINstjFMTPAHupJU2wcopeHQpsfWSydzpwklgQaczUn4mM882vO63/pc/qMndV + DmNNJYytgsNUfsaxTLCyE+oCsqvgbqay7JckKNham8JLR36GT7cV7TN9RMHDTKKi - 5J+Dstt+YiMHHhJZK6XemuNugy6xYteI72+k2kUCgYA+RLyai2f2OpKAbgdCpx5u + 73iaKLrbetVOgmFLP7Qwy7edhAvdUMbA+uudrd0CgYEAxofraeWtkr7DUvKKu2GW - BhoxNprwoPzf6Ev93F5fGyshmRvgCk6/PNuL3Tf7mMdpC+9MBdPhDLggcpP9uYJQ + qCrnekLSXEwoTv9x8GzLwM8GJ7lBB6ZxewfoY4Y/TLwYvxQOGMN0Qs004Jh5E6a2 - cFWSIC4jbumyjeYKuoZ0YwQ9Fy+K7Wa6IsGpRlr3348NR4DFUAR5+bph5ySsgW9t + ahKB/zFgBlIcbHLoF1zzx4MIqgYiiZigXi3XZm4Yjbm+M5eyPMjwS4qlogqwtXZd - FLda43s/ZR5k+0h0YdqLWQKBgCnWyPww8OrU0lXnaXxvCuENZCoyiopGg0egQohg + NMGPFhCRWGwbtbOdNpn3OU0CgYEAqFg72FAXFxxrmraQKL1aIfbwYwXXb7+BGB8b - UFAMF8EJrE9QYO775gFVZmeNK+GRm7KojM9LDYGnwkSjq9yBiS/artf6vlmuxyYs + wgocTyAX4Izu8EP4uBIFtB3Q4x8M/CKFdH/JvlxEIXIr2BvJyaAWOMaodfwxgo0B - J/vpjWHPCn7a00cFhugkZq3zllx6HM8aZPFg2a59XP6TFrPohY8OfrO7R092DtHC + Dv+SxhVeZDU6bbJvz1fJRTEXOQY9hsjuMVBsmtnHiLj3NNhtKkfN47ejuD6mSaRI - a5JVAoGBALrst8gw9BjZs780Pi3rB30pYa6OMFsUCNvFK+DeN5dinJw7CbTIFaBh + wgsvfLUCgYAe7EGyWS3kXBXEkwRicBjwPi4kwPfxHTXz64vjFKj+NbimV2TZ2O3o - hyHGBSDpO8qZgMdqC/6T9+Wy3DN1M5jypztzLcdxSjeVIzouWzsNcgndOXmMPIjF + EoJk40dOQWZI6GYDJJnNJ807MTi5OwC4rjgxc48mCb44ZBIoY0BUwNh8clpiFCJd - jBoFl9nELSboDm7orelbBtme/U8jiuqCKpKuKr+GWBAT+f6Ij2D4 + 8u3/e4LmEYJe5AFRT/Nuz2xvRpnZqQSQAvrLB60U+6Hmu4ToRMv1DA== -----END RSA PRIVATE KEY----- @@ -9681,62 +10731,62 @@ vector: segmentSize: 131072 total: 10 - convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:MDMF:nndoruti53vxwabaf74wzqhw44:5pglds7tvdgweggqsmku2onqhlrtslkv2y7da2efvzeux7js74xq + expected: URI:MDMF:n3rxzmvfupy7vucfjfe4pkdgga:o6s2fbq5fryvlt2vbno35pqtqj4l6pjfsf4d4es3nz5cubjukw5q format: kind: ssk params: format: mdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEowIBAAKCAQEAzrBrxgaiiBC+cMqgaoea2MZkQ1Vxqo+O5FJFu7H5WkJ666XB + MIIEpAIBAAKCAQEAxJKzLV9c2HRBtvz9Sh6ZhLopsZmlSaN5VmFY4IYF/t8KMFfR - jvC/cA1pp/4tmyC+Rswx3Bbp30Q5ZdM4SfhhghY/kTYpQwrV2MvBNS+hFcMZgXpS + hgY66UseCV+wFwjjiY8ala1w6daStW2aP4W6Y2TUvgevlnHTzWqfP/Rekeyyj8Gw - 9FW3Eo3Mh4kFbEox0K/W4QHYEW3l5640xH6UcFHmMAQE0zlRo0bFqUNbTEk+Rn8D + S/GE7wieXiahP9OwHENyStN8dbWkmvnIrYqBiPyozVjpavYMRuhGPjKpHMNXSkr1 - Ldh3Qg5XKy/6jlHSRspYIqNpTHxcSsn9U4dKswu98UUYalmC6T98RGRr3W9N+lUV + yYpeaQ4hMxosd044ktj6ObE/ajBBiM4l3uhQzwsPg6rjWpc/unGPmXjiwOKB5iO8 - jOADONPT/VsghXWu3vyf1S1NdpeNA4r7GaJKvgkiLUMHNy30EQHRAXR31CGPW+NW + YZrUUvCMchBg9oOHyf8ovjuWB4NU2yV1yhNVt+xbN89hJlqzilhCJwHMqf4sIbUa - 570wvEWXiyxmZQhY0lKZnEhj1/CAS7mcfVkkAwIDAQABAoIBAATbWGz60u7nldII + ScPV6AycqUz8pjCqPWOvTYHms4jrDkuXbPiVswIDAQABAoIBAAyyJuMi0Ppkg9Ut - sORP82+MmeaLJ3SekvkChej3MajRTxoidv3o4la7ufPcoS24A0Ceo71MPIqmi8K8 + ZWbcCvFH0137Dzs3Z1t4hj3IY1iaIjWL0wslvkq+V3wUN/uqmDuUkZ2iMyr2C1vT - x+HVGFV6OFwtLaMJqiTCBPQ+/kYIo5zLRw++w+KHunqk2Z/Fzo3c0+vNo0oljvV2 + t7u92gEF1aZJOYJa60YpOB6VHIrTOLxQnERDaAWiPKex7E85hmQgWsTkAFFbW6EQ - vn6visVo50PRlFtySVzQ2Ow1TPvp4dZ7u2YM9zO019tcDB/mFCFhMFMxjBx0WfWv + yfDtJ/YwqsoTWIUzpuQNzJZ6PZbqqNlJXNt6nnCqecng7H7Pt7tNQzQ+WA/WHjnt - +WRY7b0DtATaqwE2H3F9rOFFCr4Zl01GB2ZmVFQfoAM7GKhOaT48QXObn2YEqHvo + A8R0e5+fUFUO7M/V7PZe40FdtatNGOv5MIRzZfkVvim/0BScd5wnyhRQU8evph9q - aFL90BocL1rWGg7MOEE4oWN4xORRkzjJ7TfibQ8WUpTWhG5Q3FGqOv4mVxGT7a6b + wopO/r5wQDlsYIBFr3AuAB+8gzlPC884wdK30+qPUHAkUIvzwYLbDYLVnz96q8Vh - NOBW7OECgYEA0qDdy1ALRyIOYiNnH5311FgA0ZApBFw5UWe4RfzRlgRQt3787mOk + St0WxD0CgYEA/nDTsY6A6Bk/c6fj+Aj/i0RP5F1bewED7Xr1el5ZhBFrdAvVSPxf - 1e1BdBr8ja/pkS2gTXAoc2wcgzSHuygyyLn/AQ0inZ2Sg8KdAU7mE/nwvKoN9HIX + AGM7KG7LCi7nY8d+HZVKljvEp6tDFjrWrDW6gcPmA1Att4ofQvt4BA2djULbtmNU - xwGxxJASxZzMVxZnIzdGMqGPVx0hCalOQwftq6ZzvXFnnlBOWIcLEDcCgYEA+zZT + uR2tkx1LS073jK2u5qRyO1e6BCcK2UjGh5EU+4A+ZYLSF/0Aozxo0VcCgYEAxccW - G6SOedIx5l49+DR7PTEPubVojn8lYlHKuJ142ZiMTtTpGhZ6DXbRwIYtWf+WILqI + tJUQMphzAE+rRaCrkVeU+3yR/LLK7Ob/x8P8n3OwrJMQr+aJWK1MRrmtmzMLnGgo - HnuD6H5C7eY7JPcEIzYHpXQU03tBIRjioh7ToynIXF26k54kTBJ9fst0QP8ReQrH + hqzeaibCC/1ncvqKpIw0V8tYxMEN8I8svoeUQuCmUF4gDRkXwEJfheE3qc5aosE6 - q73SmMhc330nvlcbZH9/auelPLCCWz5uAt+67JUCgYBITYJ1hW+ppm4rkB2ZQ98c + QnO1CcSRZnod+ywuwTK0I9/OSNr+G2CUHa19GQUCgYEAjWmXvm89JcIiid0dzpTx - Wm1FgughoAro/+LI26WSir6ujsACkWAHM5+RXKYveSCDfpcVnhe0r3sGKyUgwQbV + si1dWcapOUvvKuXT2RbnGYe0+OI6wD3Dbyu3jVlGb3pyD/qoFTkMI0NEoQuGVayN - 0stPsBOe6XVfF5JP3aarWtQh33pU3El/PfypDg/zmASpLH6RHytQvBb5f31U1LKR + 81hJOCXwiJbfUcrqZQfuRBJtJj2qb0v1oozkE4eMeWaCHyXIt1deRa0ULYqldO4F - 3gnfL49xi5lXRhfu2cSZdwKBgQC1hZ+wDcxWAqjECb1FqMaUhOsUCh2vOfjNfsS5 + qQLxbnZwN2rl6X8sA41nlDECgYAFs1xbLiS+YJiH/MPiCOSJFu4rZYbLsteYhnv+ - ejBlK3HXVMnLbAptyDnwoAQNUD4vEBpjzGSYjwPV29NI9qUqvFPyHlseJaX+QHkj + 5Q5GBk6kWsTTXSC+VphpPXbcj1cZVgM9BoSOqLlVISO3M7OFVKk5kpnnae0d7vKK - JJtQ/1QkSiYTnOYlggbkpCcxAB6kFEILu3J9q+pQI6OgSlkk2Ww8133yyKipPgdI + N1w1pUYF8QCZgAyoNQGN2VUCZvlD0a/9NFqWgnzyaDivAbIDTZPVqODIRs+mOF1s - VFpBsQKBgF8DwwJQQHINYbAau6m0D0+3B4piATNnlFOW/8ghWWBahSuXBTrKy0Jv + kZCWhQKBgQDIO9k69EE6VDVP2iO5emq6A9+9QegnwQlFF7iHN9PaKYFiyHzFueB5 - X1lU7/MRrcYZZrnAO6FPk3/FpA2ONwpaZhq5uL3p21FacaWttTc+hAwltuWR80LI + eyNxHSuDrjmXIBXXHjAbX+twD2gLA6+xeORj2yujYcUCVJ5G21/FJ+7PKFdUPh74 - oda1pqGc2MFqyFCmyz+H19dceWJ3k7i9WXj9NwXikFmymlWPZbe6 + IMMuQtmXaJw6zwUUsqZIGxllv82zcnfBQbXzInue2+081BCPh6wn5w== -----END RSA PRIVATE KEY----- @@ -9762,62 +10812,62 @@ vector: segmentSize: 131072 total: 10 - convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:SSK:itdsr3oaiy6ymh7mxq7zhznjge:p4hxxunt4fm3xhqgtsk3j2qfplpxwvwi4xakcps52n4pdvy63gvq + expected: URI:SSK:oibebmzc5klthxwje6i32qsa64:4jn2csiis5vv6ix2qndrvegu22up3a6wciqg2rrcbf7m6sxzxhya format: kind: ssk params: format: sdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEogIBAAKCAQEAyFwxvZiVH3Ayl6wRlOIUHVILY/pyHLqioAlzaRKdMx9/13KN + MIIEogIBAAKCAQEAn1VR+MsUI22ssr2wErUH2SszGxF613S5wZV+MY3I+5E3xFn9 - ULOZ/npHjmdej4OCQ8mwbfLQLp1+r2aesrnpcJ4rAKFKf5fPrenDmF8WoV7UCsoC + Mwf6CHMiv4sdNTnU2dadtx+lF5S/MP/bpjqt4shLF/71Sh0uNktvixdE1gHm7nKx - 94tRJh+Am+9PbX0QiGSMNwWH1FLGSyFRfs2PNsuc/QrTbghgdvDw3ZvKsBt8PR8d + oUg1huUgVgXV4D8cpDhscZ8CSZ94GMa5JXB6s7e3EtrAMox93P5qoBVYNUKWRTyQ - 9AMwkWBgGs28EUXo1Xf/Mk0fu19obpLvwObUJ7ZSHsy/ePzmwrcPlfL+61mZ3MPI + YZnYvVHE1xMZl+1QBxnUBGtHjwmiUG/BfafAqVYQyd2e1hakp2keLmk38GIq62gO - BLhaRsiSshvwDpzfBBEn2pu9HhdSUmLBhYL4hvf914JnCGaoMcqfNfCZIvSe0foC + 4y3dAUTDfImlENH5gbMiEHpmUve9QLxWBRDqk6VnMc2L1pTk+DA5l5xFrFPCU9yv - MICzdbq1IvBfrOqt8lZCgcgPU+jJqGoQEa2laQIDAQABAoIBADmnge6vWfXueLh5 + I5M7rjV1MFL64SA4+WmsjaqCv/VLkjXT4a72zwIDAQABAoIBABYRZiIUVny1swaq - XKPNfIFFax6tYinPMN3BanLpVs/vt/9cqLp4vA7ky/N33leIvbLY9kplLS/ExUAe + mRluM2ETx6dHG4F97EBwqSLJ5X1aVqP+ZsBLqYjEEZr/9JKrqNxnCj8TxfTnKDfs - 1PrUEY8FDJXFU+UsX6gJVO3jKuVrnrOuFrV54vOH7B+y+NWmP9wnpstsbX4VBZd3 + KAr086KGZUg0itqyAfWJKzDTjzgo2UhLYGjbLHa7g2gGtOGzPA9OtU5jXJi/2o1r - 8nX9G8FmTPnppBaNFYkUYxM58dTDyY5XWpXch6VOmRs0sBUD//6jicFmlRGb5owF + 8LbLxmLf3h5hZ99YcBJMto3nhukRUMB504vpl7TIRJ2K2Nq8Qn+9oK+WkZ10emsr - /yS5Zz5Bh9jdEKzb16jAlAG2Qb/bTBN8S4eoy3EK/FzeYZAXmUT7PlhJFCX7Fz20 + gSPNH7hcZntVEFSwPwNT/xs75QnFT5wZJ1+RqPfCK+kPocIFsGSmih9XZfPeWVjz - r4xofFOPJyJpi0EUQQpAudh3dkm7oJBoqvg2Hdo44JHC/C7asoB1kXNDsVQ9i7hc + QD/uEGcTl8fxvxY9pnwp6EP+NW8QOvcBudwK0qVaWYPE8DtSHlXYOLX81ZfKqKhQ - +BUQk3MCgYEA6hBN9eJBMgGCTz1fJVZ7XAcfL2kG/NxGfTpfcaOA2SkR3R+GiZlN + TTITPiECgYEA3FS0smNgBbFjAlIcr1rLBYpJZRCqC3XEjpppu5bVeRjbhEdJ6BaN - aiNGGTWFpF7L0GZQNtyIkNPOG8un5FVvSC5hreunWbFvzPyrSNwv837TFsXZ9RMs + g9VrgyRiIK0DykV67ebC3wgABsuT8WntWF/Rsm/jdr8Cdyq37bZLP0trRToKJekt - Qlywmjd5b/k0FShXOHYAoCJTzgs3gUBZLBqUbVjL1a6YdYFGgsyECWcCgYEA2yNE + hEnE1ogakMSydW8sG5fCWUNrmLae0XqmzkvE6D46WcLuSKBYKqmi/zECgYEAuSCo - tL+pqVd7NW5p1yqXjKejUzfURXiTAF426UVqisGZhWkXQvRARLT+qoDXFNdLxFWa + PmYYGqwK7G9C5AgjEvSeG2FRHxruYlK472MMgR0C+NFVQN2yXqAZWGN88bS8SIXd - Kqy/LQrvehla/NiDZbMftwa9GyGRq1HgQpXDOASg6ZKYepw0YDEnyHKnldqNGwxN + sDzVPxU9+kfGYcm2HFejQSbg2RevvzXJCMAVY9bvZfqKXtlGMLaH6puUDgwbMY7K - EJ4e/OATBSprsK+hIjvGcwYDwgaUK1I5AC/gCK8CgYAr6Ukm8v52KjBPO11JPPNB + qwSIzlTf8rI1yIsZrT0gI9QXInSvro/N/nAv1f8CgYASqKG80amaEdGeqrF/MCMt - rZhdJaAI+i5DOhtDz3/RvdG7ITn1QIx0eA+jlRXwY1RrUXaFBFSejw3gyxFBVgHd + Tu60PlsIKWsB4JW/qyBc5vwAEcFyhCZr2bEHJBejSMOfZ47ngrlSBe1qpebbdOsC - kc4Dee1Yd2BZHaHotl5MmSNy50Vfo+wuuwLqu7ONnTv3KC1My16MrEP6qMIN/ot8 + puqtP8h1j+t3iAiXeu6YZ5yn+ihN2ZdfMpgWyuPlCqNKSqXjmFB/GrSL9Dsy5j4m - KbRk2z7KZMn3aXxX95RhywKBgDcT7jjf02zUqAsN7Vw/QEgB+nL4HUo4u/njtDl9 + DkiYmlx6qYVgZSPSSRdioQKBgBMKteN9MtuDeLgrFZFI+PqZKK4eS27MoVqBsb4F - UQH/Fu8JMueJLH4YX8nLCEQcuNZoDY+cS5CupvIxXUUfxibRlq8R6oXfMhW1RoB1 + zSJ2rniTZ5Z9dzxecVzzFsXx1jALfOsExtZvQ+m7ej9StSWjKgqoihYqZoxfZuc8 - 09NIlokeZ2SpziA/OpiO+MAZZk3eEaCTnYZBa4Zo2xhVjFJmY8KVSGyD6snYqKr7 + gra7Q/KUW6k35g1aqQ5LpGXxftaRHm2K0NuQRVy8UeXn0ONN10F4Lkz//ZYR+plC - XXvRAoGAId99u56Llk/kW8pz/Q/aySCfyo63d3GrTc476h525pUrQVfcF+DCiFXh + RtlXAoGAT/+tvp6kB5UqyLWI1DNv7Cu6jtCyAm0IebguHqf/I9pTLH2fKLd1vy+e - 7OKY0rfmWiBhlSlH36fGL1ZlMLRdEGy0iGJA7gchEJadwK7sL68XX5hdlnz/I9eR + aeTjxfUV2CiCzZSMMyaym9hf7JNR1VzVzkwt0Y5L0DLhrTbDGdgDRV3nI1GLTXz+ - NGoe4uHWgIEoBA7ieUCTE6GCG+3HUQbtUBDLrtNeqTwUB0tlU4Q= + Ro+Go/k+Ql0hJb0FF9CQz1kWDVprTooWe3S9llqMClMjBKkMPMY= -----END RSA PRIVATE KEY----- @@ -9831,62 +10881,62 @@ vector: segmentSize: 131072 total: 10 - convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:MDMF:7ch7z4jgu3qdu42esntv6bza2y:or7w27w6p36fgpykao3cm3mclap2wd4aehb5xybudejtpt4nvw3q + expected: URI:MDMF:nmtnmr7nerwtcasdqjo2mwnxwm:uol7bxzbeeiurj7e4bmvkvc4izwgzsurrxm326clfbiseuzspbaq format: kind: ssk params: format: mdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEowIBAAKCAQEAiMBOTLdfKpmY4ssFaNnMW13QvLdN7iprMGElW6uHnXrFDKu8 + MIIEowIBAAKCAQEArLV+zMpxyAS74Gnjm/bgauAaMXK76/Ph0vy8Qj6b3zyoKJKA - BH9HO+o8QGDZYhueAE2mvNM/y8FW1Qk+M5EP3hxeuAWkHJC7zINOltv94cnSAw72 + m/3Tm0AbX9AAZnxF7hkJTfxdQrdAvXv+E7xqO39NCaNxNqO011MI1wgY1FV/+2cH - BH5zif+7PxEIgn2ylla8xlUJ80BQdzEVP60yMMyJNDTgP0zdllHQcjCgnKJGQqXB + dwITBj+Kc+Rr1zNoMBu1CRQriej8rK68CFpG0OwKUfRbOX8vpIhzaEHtPN4C7NpR - sL8AUDj5s7Dl0i5SIBkY//aSl7+CeR8+sA5k33VGI+SnLlKBEAeD8371wwtF0VEn + UFEUbPOQsdfNsUJzzB6ilZ6vsxd87DzGoS7zbuFQMmBjt7+typiWCD2jEvfXHxcF - zDIQSlSyX+IPQ0wmFXkOXaXYtVvMuiDmMzYA1DOKkPnayCSYCvSe18XdMSP/Z/Cr + 6WH3EkMwaxZB1bmtPbrHe9FnzPP3OvP1zxjjbxsvCnYmxt6lyoN1wFTpCECIkM4r - rT+2H0fMmp0p8KatNHXoWXJTirLqtKGiLOrLlQIDAQABAoIBAAjUxGmzYNevLh6d + Qwz0rTVcgMfApXxg9EWILEo578qripethPURXQIDAQABAoIBAARUr6o/w8j8gkqs - RkboY8hVtWRugP+bqSrpZyB1og8jLcT89SokLw05OfVdW8R4bJpv6U/h44mMvYcJ + ftuVaLznIA13RK3e2XM3CRsLe98+6sBaJXOSKAhN355MugT0VEKWCgX4N3gGM5Zz - 7wSk/k2Vbu9628eFeD9DjohzAgD6B9AvP+d35A26IFU5DB/jLqyDQvMa7EbTdS7R + 6nW9dRC4/N9/uRr+X856Pkir4weBYSNdytyEYzASpQyKJGtAHZbiWhe/WuaV6Vl2 - UmI3lNluaADhVkb4N4oc0/V/2utqfiuv6ILMTe7JvPqfJQnxvjJZASzpoGdLf+CJ + hHccfNzkpJZGExdRglFjZlV0qxfjNostFNHIv3EZjqpLgtBp365zMzga2nBhshhN - VqnKOBSJSjVniiB9LDQBIWdhdsCfHeVpK4IL+oyLlW79c7f8GkgjN16iUxhm2Vk2 + dMRN3zI/Wysuzemfz6IC25F7Yq/5HAwRKvuaDc2rJaS4Dn78WJSk3TA//8KSxE4r - 2Qq8I3o7LIX36EqbNTAsPT8K4s0u4+yKYYZWKzL+6vc6dWV7HjibsRUMasUnco7l + PypI33b3ljafExSBSIB9++C04ORKvhsc50BYPfw/g5PyXIoSzjKHOOuLFwVEPsVz - 4ddRS9kCgYEAtzt7Ox4hMPdZh25c+gFo9ACAywM3yrK3rwyqzNnPPCIDJES/CfJL + 9DQnRzECgYEAz9iKWOl4z8LzTbBvw1udresxMJ2jt6uiFx1hyV8OXy4MulboMBE3 - lBQ1sprHOIZq9ffP+us/8eBdkWdWzGqKFN6JVEHE+98xKfI4Ua5wf7RQF7jhePo3 + LLPOl247/zyWbw3FdvnUs3/ckPh406N1IVuSE3LZXw1Wt7sael3mFYT6RVD701VF - 5IDpPDE0etijV198gxD6HJp9c2LMERXogj3T9tjqAnB7pMYyC5RKHIkCgYEAvw9E + PX9Bq3xE7hYx3/ALW6k3H8BTaVK44SRCd6PZQn2wKtdk/EDx5YJdQDkCgYEA1Lj3 - SBODIsy+Arl1B1Enl24kNjh1c0q7VBgqwiB49pSA3IaP/tR73hPrQivEoVTjyP15 + EP3yA45nJRf3Z/5RxeOGYcOGS3Htdgsom1AQnK5gNUL8lmott1pEcGuyQSq640je - zPojyekttI6TeY/V6AH+l2m6zUcKVdYgXWD/QewtYTQKg7y/AADRIBEQY1MP6mCY + /HkoFcqFvybQ9W3TGv2rsYQmShFbkHGLcD9V7XMOVsLSov1MpL50jwG/pTDA2yOc - sHu/dy6OzOwuZGR1NQk85uMVHQh600kvvnIWq60CgYAsYantndSoSaFT3nWC0Mid + cSmH253V0V/CjcamAONd0+YBI0ycIY2KEnRE0kUCgYEAzw2RrMdQ8e/svx0gCYaQ - IWoQwkzHOhanvce5KqC5jft403X6cMfBrEt9YWQT2usZfNbRjh3E9nVzfLZXeQ7N + Cvz8cMjpmoRhohNEIf4O7CSMy2jeP1w3EdJB4TsQi9DIr/MRHtf826BpkwXkIDl6 - E0HsOKn/4AXGhTcDAd+Z7xDfTha++MyE+nyD6d8uSj72MNi13mzWdM0iH7ISCV5x + 6vM1DyjfgMBh/gBnfTVji1aAl2L4q2wL4RqPygyvAlub7dFND1AAOSI4NfkRcj/T - /YvT5KJ5yMkKFj+U8mwpEQKBgHNCUn7oxoOH4FjkaKUxYCEKYO4UwUX8H2Zr7d+O + 8ymHuqRJRjRzRpRQJen7iYkCgYAI3HOeR5XPRB1T1D3AHT32ylWMuQJdHi/QHQLi - l2qpy9M9mkCxDsi6W4JfxQ9Oltv5jjEJ9e0orlnuaSk8jF6aVWwibH7KDIIb2wp6 + BWHLxQ/I6DNxaJbi7mWvcS0JvefvE9gGGF3tGnSb09gcgSisFSkTyfd2WmbAC5rN - KYMrZ3TsYCt5AgCOfZpKsQg6Y6+Q9owBG1Ba1erp0FLgB5UnLYZcF7CcHPy5egP5 + YDYKICLWxmLT201YB37/fgknrnI6Lq+TnzFDmr2PbTfDhCTiIJaF/yzI9aYDV8wK - 75NBAoGBAJxOkzJ/ybNiMgg/j7RuDLmizK70XVHsakjQZrqB3JbGj4nxPtY7phg8 + nMFJKQKBgERFAoAThF/5RxZygP5Cr2yytKde1V6K6tkx68LlDct2ayQ3NVioz3TZ - ESAtpj+7n9LVd5GSpKsdZQYZW5SYUddMhHAbVjHhUKqNRwIxTy+m0ZNYf5oRMscO + g6nZzbnYZye9UxFq9Mj29Fk5KRjC2M1u8deuxBccFwsC4Tpe+YyXcyqE2nyaymcc - c7bVOFekrtmcMIy/tdyfeqOxNFgwmDU0TGhx5G3/GcjrHLAlaG9M + 3HHpX5Hj7T3pvHSgGAA7bB82/z4082OxiEeUnzxPfhO9YY3eBz4e -----END RSA PRIVATE KEY----- @@ -9912,62 +10962,62 @@ vector: segmentSize: 131072 total: 10 - convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:SSK:saislx24hefommttekhesn4rvi:fl4mgktxt7j55najjipvgj4hlh25sietywncuqja4gcv2fycmpqq + expected: URI:SSK:ghiqefrydh4yklwo32nwo43ebi:vszy7wnlzofynf5nswobaym4emv4yzdhwswmhoxzwquimn7dnthq format: kind: ssk params: format: sdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEogIBAAKCAQEAo3w7C1ayXwHGmcYs9KeJp5DzuGucgPSHaydNDMutbRWm5Hd2 + MIIEowIBAAKCAQEAxHHC4JknMYBpghJ7/ZyumMb9aDChLKIpFlACpvC+CADLX4zx - Oat0ZPsrgeVurJUT0TgDPS4SeLDVSIb+7Rg647yqEp2K5M1avn4Qwf57zaqPfubc + lJUGu5iTIQcFOi3oPuX6rdL3/d3dhuntUW0Gw3R+Jym++vrIi2kMzMuTcmQ8PjgO - 9ijDN6tGxtDiV1q0q9ZIIMaoM2GpKpxRDOcunzW/ttzno537aaUI2qbCnMvmkqan + 5jNCE/4hKdWSIZ613U0uAJ5inusERXBaU60P2zss+2hvTk25lxkkRtNQ8OQCwXk9 - t0xfBsq/+JIDvEu5sLAKFBivnY38L12tU+1tlwFQ26DJbXmApeOyrnaRlgH51yCp + TjWhnk0y0jW1vdjYOMH11uHFcEG7yIcvyKR+a5jZe16XMFsM5vkhMQ9RhoDud5/B - q3sb94VJhdkB5TKjxVVURulo159NhBXfMU0lK9v3f6+2+YPh6NkkfgNYW4HbIYfT + jj7AiJzWc98lqkTuEfc4NzQecNn9hT93GpqjM0yC8re2Lou4YBT6nWlE4z2n0F+i - mQgVlDKqRD3duzVHjfcTzbofihrml3y7uV80jwIDAQABAoIBAAR++ndhsf7E42ZV + 8ljO6mDrwFO8UCs6Qu8cRzsFcfp9WR1lmyw7IwIDAQABAoIBAACFQ1HwYJ/gn5Ge - OeVHOj+Tz/ARcuNlANhkjfeSc7mH7+gL6ldvAgDI8OBd+UDhH1i7S/OBxzDvLrO+ + VkEcIC5RdgHHrIr/ZwJ4dc6sG/ojc7vYwMR3B0yJ9qHYxepwjV6qogPB57GphoPB - QAKF47thno0WghGwHWpsYx+X/7r0kYUG6lVmt5UErn9HPVeeVKZTz/X+y0oRHyrf + V5qmok+e8xKhfrBlkmXO18oseUM4AXIyGIAC/21+zTxcz4VSnq1ryJpse7Brc63H - KPzMMCRha7KmuUQZqHwLoDELMazy2er+o9nH+AXh3R4ZChezV+N8fEVvMPD7HNXW + bXCD/eMZK2EDLCsAnmRIccXU8Nr349ndSmUAbRVXqMW4bRC8qS0d5JsyEvHCbHPD - rqLFUtr+ScEc0b0ExhISzOBeHupOyxZPVsm983i262yfX/aEzytohqy0dtfLW+7+ + vo5xJCRYdDtxm4uVML8JcAxVFLjvVWe+9XoebtfYDe9zHZfgkRrjTRuzWLda+XIy - gT3JRB4iSKVG2nicOSqyLlshjkVMMwAUs/ZzJM5+bpDQphtTzVjvR0dsJVovrnae + KZDgDkIjm5bNMnjuw411SB9vGIgbYF8WeC2pnZVuS/o7b69XxgxRtImYDtt1v/fG - PDz7CeECgYEAtTibsN5jvqsEDl7HSk6qCsTQsiCAF9dGXRZti9X2KLCiyoEgAEof + tBWvVYkCgYEAzUxKAaXxoPpkIcGNA74PZ2F/FEe82yJ+GB6vM4nlF3k59OwGvaxE - QzbLYBYFA8OD7B2bl3Bc9oASboE7OHChQOoesg2kH2fg9zfqvPuAPFXpT+y5KuUc + Zz67jGMAZRdIbagWdQ6Tc6FdBe5bp2j34LOxUVFwU4bbFelBFBSZtFXIIqkCzjLD - 1LF8ptUW6JyocSF6wDFbLkiUkeK1An11J/Z4ZK+2cmnVym8PpuBnn9ECgYEA5vIW + 7QWmLrMiHvrsND9Xlu6+esFjQl7dPb6y/wmFurAY2rPatON6Kd6MBGUCgYEA9PW2 - HM5saxFpX+8ujNFgBn0HkyU+ZIWM0fQ3+2iqesgwfjNQUxSB/3rOzSaHh5s7zsmC + d6oBDUP7kbvaJbhzlIv7DyATgt8oDoDFVOg45ASnvuAprYEJ4Mbs31b1SsC0UQf/ - dAlMqDyKvt1lIBD79YDfgVUvYezNkuXKm+U9dvWEmztZcQbPbxeK7g/2CbxSDokE + Y3pllTd+DTLjvKszCUIobQl0j2g9mbkR5qc8rrN1DaCAavxqNiSgjPLM2XKGy9Gi - mDx7Dx4VinMqCI4D07vEiVSLQozggIlKoklsBl8CgYAHCCZKa6a1LE+g+x6CjKDe + bxMjjBtoqw7Ln1LhyhL1I7qd0QrdT0zY/zfU9OcCgYBHCj0ZsOiaAcsgey9muh/u - gBqU/tvZkPni/M7NYUUG+Sun7fC+8iFaa1Li7JfPOJPy4oc6DhsdWYTdktgobX5k + cChfRiut0JO9mPCbbv4dT0+k1v/GJpRM/cI8ZA3A7XucpmuO+gpAGvhrkv2YQpRz - VXFReWQH7/DzxtCt+phUPwUpm8bnmjJPMn/ivVwBNKr4kNMBiCjAmAJj0scxTIry + 5vpW3011OdcaD+r7Hd3KL1zf0Ygs/hgaLrhAtK/79GxD8B9JFThIlh7Y2qbINPMP - PQcY6RSMRf0MuNiDoiuDMQKBgD9yoPqXB5g+t1mA56QOXbhKn0sgv0x0mGSSGNM8 + maXy4fjXxSDLM2QUlPPymQKBgFCKaZA2yVm/PHvSNAuq9fWlgMqcVU32aYk7NaaR - RSHoX9I8HMRGbRSYU7pu7GsoDb1ZBTsF1wadY2zefErb/6zKFB1/Hr5jhXLnKMu9 + JAN0tGLB+XIet0y8my1jvgryCVeLNaFToQrK0Bsu3EowT/t/USNotHZiY76jZwtb - pi5Jc34GRyNTQKf/qs6OmgTAtTaDFD0S2Kgllrtruk+RXKHOA0fLb1sAQyltDpEZ + eUxHnPj6CL8kdxeOO2uceVYVndRt/OZgeJOcf3Gez7x2195FFWzF8xXEaLemIMLp - ZNE3AoGAEYP6aG6CjX8p6OOBfVoPJR3is7V0TdlIilaFAYdsjRghCGt7jGnLMgX6 + bI+JAoGBAIiVzbyeGxhf7aCqbILLUTGxb3yLf+SheEiMdVJ/CR6bc8uNcKKH0E3y - 05y3mUtlc+k+rCUiJAvUr9SHQvdsw2cXH2rBa8tskzy1RrSRO+n6mOHd/2UfDjll + 7ygPXZpAiajpuSfqA9e4rfTxHJbinHpV1yxOkH55QpPm/4h3mMupthqgv6dPN+aN - CzcGMyyU6rytmuv+xD++59PzIjXtmkh7m64CFrgknbzQ/9r0oTM= + PcOyLUGJiFlXKXKHMeG4YHHHLrEOFxbKsqzMrgLWANmg7c4dN9Px -----END RSA PRIVATE KEY----- @@ -9981,62 +11031,62 @@ vector: segmentSize: 131072 total: 10 - convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:MDMF:e7vrv6maesiv4ax35aos7sq23e:xphbdz3aavk6tulpxjegzvfrrz3jjdv6gni246yugsucul5zgjya + expected: URI:MDMF:unhxjfxrkuzsuds26fq66slezi:bqley4l5syeh6vyws7chsveukz6rytkb2ihrquvsvilbumohnrdq format: kind: ssk params: format: mdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEowIBAAKCAQEAsXNKD8HGXeQN/FhCv5e5iAFetHF8kHPNMg58SsA89Nug13l2 + MIIEogIBAAKCAQEAvCBlTHQVDtHf5zH4GvRbJxJPXfdpQ9voK7dqmz/ggmAS5bzF - FKw+K8vECd2VjThRDmYCar4UbyIedtiHvJz8/HM3rCwGRw028oJEjaFyWtdyBGfm + z78igUXiZT1Z4AR6wWgodyYnXHcl6nbpvSPBJRrVD7eI1dBmjV3QUBPAoQzTYmzD - +8QB3rmPIWSj2IdMloyRRPIeU3Ae+ZsJr5dgfsv72lQSA6wgEBF/05nAV5iFDFTL + HFTLCR1wYQLS768Rrp7lUt0S5WHU0a+MxslcWNHi5DrhxKWUQ0eYX1t91+86l8nS - 3wCQKAgSVWPQJIGh18QBVvppkCvRNjdCf5QqCkRzB3I1UUecU0kaHyylPutRQCjM + B1CfhEI2stSYlTFj5vCI8DhSYMxhZclqgSUx6fz6RFmqRA5zZ6ENzH0D4BktOFHx - O/HUf3vb91VCcqrwghf1B4JQXB3kG6HTQJYseVKM07G73ExyyPNDvFMXQthfmeMe + G3Tx0Y4LrDqU2U1BmXBpQKzabJwLwXTwghVGI5PgaLEPwc9Br3bOmVsj+PXH02Lj - 5EfGq/rVGG+Iqw4tTDHgiC0PlmQ17JCDyMmpsQIDAQABAoIBAAFC6LkX01cBTxmO + 4vW8YCd/Q+CfieivfUdhVwk2PNWXWyiCpyNcwwIDAQABAoIBAFhJqdCd/83zL+2b - 36MXYyeLXHLU3sSormdIynNY069uefxGyhBxA2m0k33ki1q0kTbEgieudsGt5ONA + 9VCNEgQ9oxK5zGSE3So7C2Rtr2rwNJ4tn/X1wPdDOVMC3l10LLn8rFTyinFqF1i4 - VucqoxhQ4e79WrT8YV+2uLcRxCjZ22mapY3/DWpvEtxoYpXbANquBeiUaMN8B8ar + UsypbXkA5THZk/WoNqCsgNk70+ChGMktuslef+S4tKdKgHzsv9MgDgZ76uTMq2h2 - Znr0Z6iBFFGftjzR0fdxRhGGeCiQkJSHjMe02FwIDT2QSzSmGSk9IFefjXoPXwi7 + xw35rQWgBqfOfGrhvDlw7bD+yoneFRNeiIwphYezgi1m1fhtQFgAT0JhBqKCJWPu - h9Ll+wuXGa8GUGHGaVrfcWTogTw0mBOMMqoP/5FqRiiMrLn4svNXlIGfrkXwQf5J + KrB55XmZgc0dKTvp8Bzg9/KnhV9zstpeScngbjb5gNgrWqSG10SmBeApYQdsKD3t - d9NfUEP2ilhqvGuYQSsXTu51BW39PhO9bBpXHMf7P+ZN1KVytul43y4dmcLmOnrL + hFwUEb0To/8MAtOt+/T5lM/MIBBoU/wyy3Z8zQgZZePgwu7NmAg5Z66p/MGWMfDP - eFsFQwECgYEA7Uz2+Ep123Ny/tCy4L961jfoDJBdHHmaH8pte/FEPu4Ns7Kde2eW + ZH2Ayb0CgYEA00xMPiXnt3NkbRl2eByVJN9TmPATawagV3yaiwZSnwohgizb2osM - /qZQjniCWFPZkUbyLjYUuR6lmHUWxl+F7kypoz+5EBcWrHRTzlBqtSx9aIhRZoVQ + tkG78UAr6cOA5QSg8KNHCddPO0Wn+U3VVyua9+ZH7u1gdjH4ApnpQ2jJewmfVItJ - b70q+eIx8Zu21jy159+qN8yS8W47gSDrWfmKYrvXjAq34Ey9mIK7EwECgYEAv274 + EKeKd8qX7m0RjU1IRmTsgDveABdTgcDdHlc8D0KjuZRTHFnHt/ZzLlUCgYEA4+0m - 6NYtCzWcJpu32dKmkniyOkS7T1a9yWfn7EKzAHfwfHjcVWRQV7q0pYPNhHQwuTic + 1vliiIcM9RxTEOiEZsWnEsMJBBJuD6Ukg3Et23rZKhIqLcCDWv/fb7X1kaDVPYsO - /89wknS33KNLyPZ2EpGkqyw8S+SdYQyz1m4y1PIbWKM11ZlKJvb1fHr2D1WACrgF + qPr42x5b5c6gZq8AV8UhvDqxOCY1MuKnWOzAGleQTUkx+BOa0N3LNIqCxSzvNUek - VaW3CHBFDFW2cZsKti/X+9SJgwq5DTMi8Qx/hrECgYBpZHxvzApKPB0/xQsdPI3e + e78qMhNLnXxXuRiZtkOtG7w+3vlhc+0exqmaRrcCgYBTmQc9O2//A9eC1qUphl13 - 5JegNOHVysBEDFDR8lbgKDRXsiW1cE2krdMrY6RofF0t47eeBJDxowXjD2XdFwHR + tif0BWAZYwjDNFhMktbTd4WkZC0jvQntffpmy7XUCfaQJZGrQ15SxW3ijH+Vwjab - 06SoB5424jpEv6mVASxTaP4N1jVo9h7Ccd7LesW5y/HJds9Hu5PLEoXUyqOM90Tw + A3SPifuBy0bz3Hc8SDqi4e19EWSJZYYl4bOGC5Cq01ozZpUmzL1JSuZdcN0oI+8Y - Ah+POGREI2KFMTAnszBJAQKBgQCh0Ib8IZZfpEhC5luo9wOwSe+1i0Wdkd/I8Fi9 + Fvl7LClsvgNX3ymGXipZ9QKBgFfcQB+YTJpSbPVDcOXQq9EuGeRKmHwgWprfTv74 - f7/ZRIj2Xh842xuCnKJ4SgodzS0mU7F6Fnm8goasLSgxTguONKgxvKmXKT7Suy8E + LvQvG+1yyR2P21LF1ayrWLlFZU3u/7y12h4lSsmAaCaNCTXMQN/dRBlf6RvvcRD/ - sY+sKp5s9UDbNcDVYOku+K0nVwlthhGUTQiDTItBGu6l5v1N9PEnwIcgSp8Thkch + WmINJQwVzhRSAljHVqCvUA+P7bn9HvOw0iQxefGAUBSC3iX7WoyZeSbcvOtCGZ39 - 5IOjcQKBgCyoihvhDkNSRQcUoEtZ4Fn6tqqUB1dI+Dx03FXeh79QjcQw9ftS+wEq + HQJrAoGAC/jVQ9EFNzwXDnkv/pHRj7i0N+3cv9XXCX19bu5AKmylBgTvgj4xZ3R2 - EK6VVdQYg8ZR7lZboDA86JwqTgyn+r3qtf4M0r52AxPhZqdw4rqTtVEQuEM23Aw4 + KFwmHdXTAs80D4qZPkun8XiFw2va56y+2e3lcWT2MrTKhs/dwrodYidOOg7XGCMJ - z8qYs9yyfUvBhDKhiguAQexiIBiZKneKNK0a2GXgGYF/asPJVlmg + gKrOJbH7cf4Ts4VTksouepDk+ZAdbgFcKye5wN2FajvvY5kSK78= -----END RSA PRIVATE KEY----- @@ -10062,62 +11112,62 @@ vector: segmentSize: 131072 total: 10 - convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:SSK:delnikz2uniwffxdizmeesqtea:nmpnpk6ve44qcxxiskfv6sflsj7szj7f6whrxlw2i4lfcwjhze6a + expected: URI:SSK:r3q6op5itaypcvquahlmpudidq:6esnusmt5xfk2kr7nyqoleb3k22ks4efmltbaah7ldhq3mfz455q format: kind: ssk params: format: sdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEogIBAAKCAQEAlzf8n46p2/xzv5XGAf+jp9qmwEic1wMRtOOF46tHAysQTbQM + MIIEowIBAAKCAQEA4s2Oe78bWzPT+ETNoPRutptgpkUKYL+xxJTAoatGSUdsqQbk - gddD9yeXYiMWGFSRzmsrXM6cWRWLJDa1NWTh4yrhtFeEv/a1V84wkSdfYHKeqEe+ + NxrKfoXADGGWz2OXsraRc9ucRjyjlbjFNVIksr4Ql2EaYPv3IVdPXK6yK9ZRHp+x - ZgtcAifVrFlJ3kUeXLvUyHGSCeD0fcVK1Z+cIXm3uUZwLZvx9mH13Ihb5XT7QkkX + 3F/sG+d1fY0BdXfMyafiPHGC2v2E+Mfa2PcGxEjA/3jb5pZtI9VKew+6d6ANVEtS - rApCEGAf0+EE9eIJ+S+XYK7/9UOOzPcPl1jiCESwXkvYOEIzqvPfWTm9kaM8mtaW + eiRo7oFA2Hdczo3nNJGH0I6eBO12ifP3e1kAWoxrRY+kkx7zp4fx0hgxKl5YYJhp - D7GkhTlHv7628i20nJVQu0OUeZG5GsCgNehOJ6JghoKiMysEwSiGnVF+IeZmbRwa + ZHhW2VeX8KNv4OMbDnUyQCgTiEHFoSvDCLYNJXwAuOgziyDMafzuYI7ZvNtm/yPx - TmW9bLf5HmzLUwVu1fjhC+NY5jA2TN5rqKb9sQIDAQABAoIBAAdxyMionvVe6Gpf + mpDMQKDF86d/PWTst8CNCXy57inD/5WxtshRtQIDAQABAoIBAAesEyPGmab0NR/k - QtoclheYgLLQ6FK/+o5NkKz4hMSTyUfQBDkcqkKHwxDAYjE0saBoP4BgRgtvL3RL + hfDQRY/mtPZS7UOyE5IJXSdq5MNuutm2t1DrSb1ncJE92nnoATYlFmNtFgqxm1Zu - ErD0HFhBB09H6zfBmyQdWR/+QJhokCW/8XBH4SHUiPUFCMlN3Qyq7RLon2xH1DWA + fzjrSMoFUDuqtLEVfO0Zp/N3UzG7Ikt1dMqpZQWM07mvfV5LP+tBAymiLMTHLcpN - YdqiMs8llT1Eadee+KKc6DwRBPgm7qgNzhk7jqv0vKPJ7XuI1H21+v10bKX3V9/m + PaI4X9Gs5Ph1ztLsgXZXmf+/m7Vw4IrdU+ovT3qkKumBEMgOWn/AI/WqVaWpBgF1 - 6hZ5ck3AuwaJrK0uw44YX1NjY8cf+QPZVhSNQaU1ow5K/AJwaQ4JGo3Sbxb00ZL/ + eGTPP3pr39cb7b3kPJkPCTyQgKHaoeRvX9JrpGXe+z4BNqiZI01U2cVsAF554AKH - C3hhaes2TJTvd1yhRsXCDxBwwepoxEDH4SgKFwjZB7KoVscbsIP/vL0NY8FFAUDB + Va6bXgzuIgXbVEFBskgQ18pN4KcZyBx3pGQP0OEemH28Zpm9r0EjpmZKWDhq9BIJ - njV0n4ECgYEAvT7E4EebI3YtF48I1KwgJL0i8gcU43dZ/Uy5t0GOjb3O99Cu9Nnx + RyAjzOECgYEA+RHYAi1qlnPVBIr3ZItxSVwjd2+Brg9xqzvkL/sBQg6LVmjxuFNs - 1q0JhVBTylHSX3Vs15kjPIyyyfIFnMMjHhpxFAEdDI16mwvP72Yf4TyCxPjrZrTY + IXYzZxZzcv6Dl1Up8jdP3NUUFaZLMKvjaLx732mUgGSesalrqshM3j+wxYrsmpQk - GDqnpUSKxuVmoxrtRlyjDLqlFVlgLoN977v0oPnxFkhheEdxWBxeesECgYEAzI9Y + J5hHgpDYrKTUpj3BsPJFBq+E39WoEnMnWipLIimGUFsF1bZ5rT7kc6ECgYEA6R0a - 13hMQSqYtOThSrknfG9g7NiNcACWBQyHu5YCIIqbvsfHBhLj7Y/CWQWyh9z6/tKl + lIDBaboOjLjY2JFdTHnqbCg5+f0J+bRUAby4IYENe2eR/omdqt5aPbksdDoP1/nL - qm/nd2BlirFnO51E7XS/zsyUX2erSKRmGH45cy/My/IJ+5oz/Jmjo1P4WSN4hQpi + 9U2uAjMIdbYL7zSPd6bEIVT3Q7UPUbMhbqln18RcPLvjdk0YfeK+eT3Hg4XZhTqL - VBWvSMRlyGkDz219bfe/skF7+HmLsW8dg9EW7vECgYBF3lrJgyZf3U2gmQplmnbz + Yilmntaq6AlCcpZ2taL3FaCXUvme6XRGGEUi5ZUCgYEAs3IQr9jqz0Ta92/rt4vj - mXDBcqPfpzzuK9mVMvrykdVL4Rv3AlArNg+BzLpiw/qri6r3nm5H+Jo5vMUdr13T + bdgtUVKMGszTt2vqBkuQZ3g1GWd4p7Wq1RzlAeOh//qw8ioQk4sYReFanBJ4X7On - y2dcP1z+OW2+uIm4lTfH7JNLLauba8EskNs8RSYHcMKIDXT0uVbpaC9yxmCgS6O/ + nwEVOixGKo7T6upGQQAYqZM3l8t0lhYfSkujUcVr5k7HSpJ55zNVWfDBCcdUVR6T - UuFqXV0JIQf7ZEUQhsjLAQKBgDhtOlaFipNXSrRrhnH0TR4YIyZyPeGtZ7SQ1kg6 + /pk0EoPaWjCKLqROW+xRCaECgYA8ecptgEGtFhG0PDg1ZvDXaEGCsaToz9aIq1mn - gu+zDG898HqOb20ygKvJ1IuBu4LbXHN9Vt4pKxltAksBgOf3kolbCXqfwDHTl44e + 4be7KWm//AyKBlWbAHhUzvdTZ8S4eRuKlg5wj6DAOOw7sF0P43m6U/qZ3B0PSvN/ - E37gqp9/bp2G1dxSDT+ahCEilbYtPR5wtN9fvavgu/pV+4mAE9L6GVZbQNt7CSs5 + a/9+oHh1YSEPjcyuy+YyOe7KlizqPVfvrWHsDzDjZZOReqttT8veFn1rj0rEsd+F - XBghAoGATM7bx5CEEku+WvoYZW0Pl0yqbJvVM9JJqRBNz8ZrycMvSGvHAgTaYBW6 + aFo3SQKBgDtU6oa+9PU6s0iffu5Ev1GRT5fYE6QckfPP91UzOSIOqVscfIEzzKvL - CjiI8208ZiTI64czcHpFS053FNlg9i5NZT/Zm3G0sgFNBJTKZqeeVDa2VgmgM9YW + N86/xD6Plzzft63kQExQ3nuJnzhsf2lRNn3tedhT12GwHV2PHMinI35cDcFLzamm - c1dRfTY/tg++kjzuo0HNU/C9CQNZFOe03o59o/nf4hySp5Gk7Iw= + Ew/1GbD3WvZ90IPb6p742gHoDqYUy4CjVCPLl400h0sK9KEIt3Xu -----END RSA PRIVATE KEY----- @@ -10131,62 +11181,62 @@ vector: segmentSize: 131072 total: 10 - convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:MDMF:o6dunuyiagettqdylpolevia4i:rnn3djc76kylmrcbcvcxuuvsjfx52ql455je2dkn6egie6g3bjea + expected: URI:MDMF:wmnu2vkxtpdducouhywbdxiavy:tbrrl7peldu7poep6u2bhrnyfiixnzzgtj3bwrvfbfoedrabknnq format: kind: ssk params: format: mdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEpAIBAAKCAQEAsI8MtaBEjERTpqfLs2sUwwXUQvWP+Jr5mbd3qG0miqToAZXC + MIIEpAIBAAKCAQEAys5wie0saDumz0H4gXLEWFKZ9iXAFs8R0S95CwxyQn9EMifm - cvsqY9B5UoKq+JdIUr0vrQup+5PpmCibGT7/dE4kkJZ1it8U7/z3ntp+Uy6OFNFh + zlAuS45VKipB+ysNcZPStxlyjenCyb+jKNg4ZHD9DXBEB/iDflGIuJ2IL7byY4yK - ft9bFYRYu7GqGiYqtT9avLix6vvEBpNvgxJapdkb4uA2PaYuLtgWssRcD3z7c9Hs + DQfk34/eiOms/TsiNc5YJjYQ9vaov5W46vymFhnOLxL+i6CnFHUC7eRSJPcjnina - 3AHoA1xwBbfqzMespx7XSbuMSHG4zP31FKSienQh/4OYi4SogsOuTnGtl4UEFfNR + GhopcwN215sx2j4nmad3aI/2d2sAtlVoP1inlZeNdbp4ZfIe8YMBwb8TofO9ga5s - Tfd41CUiotiN1OqR0AuNHjInbmk4DYHVo7MiuzKlsbT7KcLdXgPY+HYhG0eriqST + 3+4Hp5bfhiZJkC/RfFlZypSIJi4PJdVohPxfTIFCYBF6DIPaL8rkxPzwFcFDgiCx - L6flnt82s28l1zN7Ed4d3+5e/7QUk2kzYp6sbQIDAQABAoIBAAI/L8g36+dlDzN1 + tPY3JjSqU72586e2JMK+vygKMFS337YFf0Kd2wIDAQABAoIBABVAb7uMTmh3w8GZ - uy/jUvZQYq0fdt+RCVAdd5ZbHTxycMlkYH8aFyYCByk3pHlZY4A6DBtFpLog3b4j + LKTH9Xo54adRCmF5fmj4vArj2X9NXcSRuNZqwYcqWZNLDVH4D3cU4fJM6NulIMPK - 9iVSGeoe/HQilghYYmnTbEtHOIhSdVhqebUlnoEdmAt7bVC735tC3SK9rvXwkkQL + YJsRmUsxKdtEJeTd7k2I1rZdz50MYzb6TacS6jFhHpUjQ3zfuvR/dG5AoSN38nPt - KEYgu6qUorg2ZjpOnRPXiCJqQUmpJhZrXbkRvAK7d9G2DTCAwUYMCRxKdTqXLoX0 + CQ0av362Yow9Rc403f0/S7jJAbCcw2vo1UZVeUGJKIsc+ALANrsCOf0ROEToCh5B - 18FagC5Bv4XlNwtr9jozi/jjl4v6fygk7BgGeYg1Ls/ciPlXd5UhMHQBRlbch5Sz + F82zT+qGZ1DlDnCyuwUEcevWkYqnGDNoECfG7QGSFDqq5CcmLgsW39SdtfWBcyUs - Gd32jHO/Lg5dMoJHeLRO3Ido4f0TxL91YuyaQ3Z74qDoHSypUEUrHKLL3GzpMaIE + PtJlYaH/xvi9W6Ncuf5YIrUv4alkZLJ0XBgvuajTttsuwMOpN1SOSs6xvoT4AsUk - dkr8a0ECgYEAyHj2hkfimNnv4l2H0pOJFALX2rdiKKZIQHFaFReI/1DQXSY2VqIa + /py7ek0CgYEA6FFs8usZHywVZGveO74qwJTLUFe6LcYEnTqDxGuVMCTeVMf9Hp1l - qXavlVXXTjOUO668yacQ8VgoaA5RhmfjaTDv8g3TyfCNLH2sbxUkNfLKyZCzJdtV + oE5CBhBfFe4HbNEtUw2ZqJBQDkbEP1E6oz5Gf4LWlmnEbMsZBqgd/UJZ1hUwe33q - e56eUn2TVemRIH80wlsljIEm+7bNwRm5DAM2kOWkbe5VfO+5W3D0RikCgYEA4XZr + 1cNdC1kw2O6OeULXCBzt/CyKZKGSjJ/JLUza0+Ha02TUTSEOGaFylj0CgYEA33rg - oCcDWhGkP/FIX+F4/HZyOAVIWOAKbszLQO0RWAiCr3ixCTyYwtWCjECQapKVk2kH + m4rTpuowGvJ5ukWYEnKfsHoBJKtZfWYdpUnopWPs8v8tE3zx0vyD14FE83ZHzGlx - zLuZUAXls4GyXC7jRK2a0BuAo295EfACCsH/wbfxb5tneWF25wiVxGbOuCzgV7at + xxdYT/AdKvIQ4EB5Q73E6MzNAFvxbcVHFx33JwjaR3/bG/EWs8DgGfkOIeDyogft - FtGwXwaXWg2Xi8rVnstQb4nKfM1XrCP29jHoVKUCgYEAiUy3Yu5W1mLk9X8jZ+hd + 4nVQuKcernxMndU4pu832DBtbpv9rXI6aRcV3fcCgYBl19xFGZ8ntTGjlk4ULqeb - yNPNrGFOnBKOh3xauvlcfaiGnFVwf9MUOZ4s0TVyeX+/9UROzjla1ECRo/qygUAj + SR9gFzU8/8PiEVbWcrsyIdd9nzZth16XyfbTpbWpbXG/2GtgL2QfKzSNLaS2hSuJ - s0at/3TS6YqT1bXY5FdxbnVzx6sP10yp9jmDq3GP+BY4rC4TH023oMxPu7POpYMN + iLFrELZ1teQwNVDBRE3xSncLjLp2SJr8HurZIL5zOxEmQ5D0s4n4tKXuu439K8cL - hpmoxIJTJGtIJ4Izy9nHo0kCgYEA4WWv5uXZtfuZBsvCnQgeGdaYDWVKlH82LtrR + nteHb0l4xojzTvxZbBdJmQKBgQDa8rg7q7fRQIAA5q78IFLtP//UFrQoCPiUMwe4 - /9CA3E91xtKTujY4Sd+FqY0KU2DD5CDGSWjqtlOO9cwdcYb2cbxU3uP/0GQq10Hn + eMDFyTDModS30yHZZCyHZs72+Fs/mc8vD2AmcUkiWibOjlxAUhwpOP1f7LSMp0sP - 6LVVaGbqGbd01KYZZpLwlu5ojztd9JKNrBhpiDZgrQiVjo1yzlNX0IoiQm5OzasO + Cvyp8bJpeopgxcNIOR9WUvvVlV4iAUK/K9D6GEGnEYC+4bevVY+Q72FHjOzskY1I - w8XVDHkCgYBfZBsi/1j3tlinYmF+by7h1x4QPwV17tYhz1ExGGWGl2TlXtnsGUsf + iKWT4QKBgQDRmlhPCytajGNxHBKF4WM9iqSeBr8FVD1Q0L/OOqGHd2WaOPr/mK3X - o+39jbguFTOYUpxX8tcHyP91BhPyNXKcxxL5R33W/Px6kYCsxYbxN9v8GMeXde9Y + Po0/Ruug/l07GIuTO+aDClxbnjSnVIQkqJE0PJcS7uefQSDpikL/UXAZKgO6fBD4 - gsDPWrg7yQ3M+bi4X1VqKfW08rlQazXW0RskZJVeOyFzixLcrrgqSQ== + pOAT3qspB/zHsfYhbcqhJ2wWNgcl0OBtcFL8qa7aZXj5j7xYQWx9IA== -----END RSA PRIVATE KEY----- @@ -10212,62 +11262,62 @@ vector: segmentSize: 131072 total: 10 - convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:SSK:z6kjddndo6ngbjqvakwxg6hrc4:47tmzwejcrygejl2zxhtxpjsoc5asj55gxrozdawxeyu7cvmbjga + expected: URI:SSK:bliwpq6aqv6np5ekwyzd6pzq24:y2pbh5xp3magahwtbhypx3gbgs7bw4guuvl4rgygv54vlx67jzua format: kind: ssk params: format: sdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEpAIBAAKCAQEApaaORk03LSPd5fCIpaxIReD1spllgA1fX3zvs88oIepVNl6T + MIIEowIBAAKCAQEAjbfnWiPN+x+qVaQBm2BqDAlffuX7yL8H5/WbFYqDVhUvewd5 - i98c8MDcWafHDON3UE9q4VDerCoowCXqfV77g62XY15V5G4L0yMOGGsV21r5BAwU + 92mHczn8gPkatrrqfDQ1uDAT+3XN93ZNbLXpVfApI/JR9Z6xbnjd4ngrCI3UenZY - DkK/bOPllzS54Jzxq/1T1eXxevyYSwYd1IXJbociW8aKoRV9/zlPxMTeV73qyjIA + QdaMgNAYh06MkErLWuLiqNFIRxssdc50l1w7PdlTETjPfMkwQxrNx6NQYUaYqkGL - ygKqjcjZ4Y6WTPwiTUGUPD8oXW26uKoIyYOAneBACWLxTsnfzg6CjPQJwj4a1nBE + Js2HJg+fvqY3IzZtYlbKjTvo6X1vTk2qf6EkpsVd6hm5Gaec9eLDumTGHSsW9e17 - 3kJtbrnGkm4xHKtS23qTANPNCRDZcVfny2ut/+G3cFtoCrm44++nBQB3bzow40gF + vKnDNQGcEMAR/XZLHPoKta0iJO4gEdg0uUolzcgAAxMbJFQmYDg2ZemEFp3ESHe2 - YLiOlDrXFtCAJRrNX41TSNEeeWmmQqPSBWj6fwIDAQABAoIBAFKTN/owlLBCYGO2 + MF/CmdJSXvPa13D0DdfGFq4alsuIDx4a1JTo5QIDAQABAoIBAAJdD9G0CSpoA8o7 - 549a1f5LmX8p+5B9Wg0yLRV/z1w0wbykIb4Ifxc+tLlWqyGwJHKa3EcsdovxSjYa + 2v8BY6NhwKL4KPPXI8WdlgGM9tXHsqwFmuYib2zfibOI9AYaJfD+WesBekPWWiIH - 0I0ls5BdEQneZUfFWcyq/WRLwW4DJ/4N/VNsj5s68eDRzlT7N3fKhSesBBgQYeSI + ahEnE4YoZDdCQlWrWOAzydeOE4GoA+Qq9xvZ/SvkzJPtHnEFnlCcuhUAsIjnDh3E - TId8F8EqyQRh8QpCuffn/G00zDeFLfPb/RxyM1164bQ2pix1isVCF0uiLswER8Lw + 3LLtidtlNXpzDRrSrChiWQ48TgnhQi203dLAQHQ9SDu1GDux3RdnPnNFDB41ipjZ - lq6c3HyEMO9jFFruVEghg8GbuoSCdxfNcrqJxbEUaOfEhEeUMB9FZA25ehSSt79u + dGGPCOdIuEpbP9B+k/vQcr/xBUPhyghW9ysASn2I0Lnx9Ge+GWBug/iq1Rt7x3bX - GZN9dz6XF0QGX96jOTunFkLUOY/duYty7A4/zbexajraqhG59zgZbo4bSmtnkGJA + SO/qY5iAVc53F1Fp7tzPeThJyudZexakfNYtzOL/fnQ8WmWLREwNBwMGCsm2+FWU - Ru3GajUCgYEA0ft0PGzCAjreUMMtOaP8dNN24PXap0cgzM1znYJl93pSJtKeNNsM + V3yKVn0CgYEAwqfRG4nvbtfUTxswgH9XL1yH1aY4870wMcQ76PWxT/4Dfo2YtJ6f - oqvJmK+OZnMMaaV95kFBktqkB/yguvV62zqGhIfTv6P/04sAyAActwJ96pcKwsvc + ZWjTV1Jock2I4171HTRxApKMOhS5OmlgVUj6sjVxtPOSyM9kj9vzWn2ZHUvCOj6s - yPB9W6zjChmjJXvBaXUkU28fBx/Sj7VdMCx9R9CpQa4FPXcfJyhp+w0CgYEAyfP7 + kJKoF5a01IZng7qzo5dgknjnRMt1eiFI0Vhpb8R/nNDCtoDullqSlHsCgYEAumFJ - ivY1vUyyQXEIQlGC7ANQzl3KjcGmomSnK8zdX5DO0R0g+fMK1WuJtAa4d8w78uD1 + bogBLfkJyMtvK1RzFXDEc35vE1O65lGcW+Kbo2YX7hlkKy/BmwBtrl1D8xoaJBnl - vCQAvQekd3Gz1lcEVkRcx2kVvno8t+ICGFlmOWtlyKnc2Vjuz/cUc00H7QGn4Yn2 + +ztKcBrGXGta2B4n95PY1u+IEYom42koCyu5v6DFLmpIQ2JvEu9yIiTtNFTdYYyx - LMOy0sDIV+MMzgd7UEm/MvJ1SVKEbNsDRO1y+LsCgYEAyIOsLX9VjDeWz9xxNVeo + sQHG9dsIOXAsBOwbDn3YbFNXDhvQmEcPLhPuah8CgYB3Om88nPpJPG3QnmjQ7C6s - 3g6IuK1NDOvZIHkYbFJ2+GmwRS5esO50FGqi6dDK1H4MXl4P6W5rJcbvWEkfWyjL + 1dJlrNDJiqIQeY/wmz0mMAJX68cTKu2bIeABZnqPOKqWCj28y7hEyRqXIMZr3sug - Fsm+ZpQl2hzLUMCuEE47HW+dugRd3EI8JQ2xR3fCnoR4zHRu7ztTYvD72hvDQEPa + sXjM2ytwmJjZ4x5Hd4PRc4jrhtHK90SfsRTAjhDo9AJHj34kv73pOaD+ZFjqm6SM - JwR05b0Vw4hfrKAx+XyYJ4kCgYEAlXc1nEtMyqWQ6E43xp19QB/UFmfkGbZRFa9Y + hcjfKs63cK8zNjntYkDSLQKBgBKAmfOZGNThhjEi0PRyO8KDIV19zbUTeNhofac5 - 6lndHXWXG71rQpJWWk4UxGCU9lT5qXBFbtFWmpClcKF+cAxG8XH3GL71kNv3REDJ + hc3g0rtWVfVbllK25iyLIbW+f53Z3FTme+tJHSwLlEckJz+Ss9ISkWV8W4Pz6n/B - PCwuNCEAW9sb0OC5HsHHKO7CBu9KyOnKgKb2GnUD0cgBGhr/cRSjpZk8pN+lkssl + ZX06jpifAHGAEhrFHoV5OPsa+ac3emRiEshRaC2bjyMl9UGpCJUoaNoDtN+JHl6q - SEZU6TECgYBHAQ0oBwiB0NgRaa9t/EaNJn0SPTO7L4PNYDZ+CL4UOpKIhkkC0CXT + p95tAoGBAJbsD8qJ4q1Nf/76a/CSR6QRjkvCiDyleC6wQp2Qn7hOQwbIUwb0qDHP - yngdS3hZcqQQ2udtZT6OPc6zIisra8ByvNG1wY8VkPcie3mhn/4VcctvT+B+XFVP + ab2VcrW1bhIHCL0Q90VwD3lPlR5V54aukmNGJKY8cUqnNnMH7jSF6ER0y//AowrZ - 16vGbTKDQNZ+8oQo/PrPzoZWbQO0TmeFzqBnq7JqX644DW/wremZWw== + 9ZRTAHV2uZPr4N8lnc3daoX0FaZl3M13Np7vCmRXaOuppbEEock+ -----END RSA PRIVATE KEY----- @@ -10281,62 +11331,62 @@ vector: segmentSize: 131072 total: 10 - convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:MDMF:rtv4atjpdmdz65zmupx3n2574a:zeo7cvdplvkgafvke5hdatv3gpwjgj4esk5utqgclfmvxm2kr5ca + expected: URI:MDMF:5duhoqd3bucjahftqmhyms7dbu:avli47dw754af7h4atwg5s36ffiwbxzaw6ixatwm6nyohsjie46a format: kind: ssk params: format: mdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEowIBAAKCAQEA2HKyf4IlgsjChJH25B47I96YP0wU7iXFzzw7HCkPMUCGRpz0 + MIIEogIBAAKCAQEAlPJxm2YJfiYklq8Z/UAsKStUXkHfOfpaljHsn/mCLvTN45rT - HsXdjfiILjXPIFqSNDy/UFZMChO9flw51+ekB8FjOJUtTcFNuVcozoTZ8g+awt+/ + f2UWUhtX+6wFiBep9Bq3FC9jofyCHoIrcT3RsA7W1X5OgGZdG3AsvPF/RTaKpBtq - NGcwTsdvjcR6rUJt9ZnRt3/RhV6WXclfM9MxdaZ8SJ28yRhxJpBy/tGLgyLgbGET + fPD041dOuJ/ekjCEtQXtPNLRVETsCNkYG/9A5ixZl8Ge6OmELFQpHDMSVx4PtOPe - M/SJ6csQ3NLBZ1aJIJqgV3BhhcloqAwr9LTy9EKLGkAxFvdTYX64s9sy9LBKnC0l + tDn4srKQhOMSEoOzngHf0dUtE+atdLUUs+gwxGaByFOcd8h0UvZYq/JcQUKsPkTl - 75o0JKNi/3+t2pk3otg2PZNWh1mdHatXXbqsWNfC3IpfA1CUfG/kxsoIZZ20SAsC + rvbxFuujFQnhcIshhC1da8wWANcP/N3dSx0wv3934aoA58ukb0VzV21QFHyF5Bdj - wqw5HDHPK3P5asvqJxwZBhaeXesStJ9/2eb/rQIDAQABAoIBADy/eDSMOO3rXZiq + E4Yf5CboLT/dZNpIEmvokcGg5MHEEO97+I4bswIDAQABAoIBABsZRwEaXeTFJMgD - hNICYBPRogZF2qv6IvnmTCq7pV1r4CPKYkOOwf9aDRJ3HLJWaSlLEWDBT6cWYj0o + gIt6Zu5wky949ZeTTHLiD2aFmyFW6bScwRj+98Ildmrz/6ekgofGaoOyIYLhsXzC - Mj3T7/gTQT88sxHbGm7VtQi9RZQH6CYgeQACpA7AL6Fozwt4lPb03GS1dX8KjIY3 + ewvlzuYktQJvsfGbbholXQZdO6YIh83WrRehMTTBeDGP6IsZZ7OVqfV0d6BIz9bG - AcbAU+XSu5f/2V/RQdSSfwvgkNjVOShfaKrRjyrfe3DvSLrUK4DjuqAZ3OwCYqUP + RKQnWxPlgsFg+Tvv7FuyTi5yvkX6C3g+cGRXxDHVE31kJy9aXz1nK/NEFDzbbHLx - V33+ohcLoMqipB9LjWLV3WyfTgi0M4jqxe1TglffjJVq9divTmfs7KcyYpyyah5O + PILDgS8UUOP0/mnGyfHgcXNSA7vhhiQ+FKSH9gRQGNImX7A+/vFxc6AzoQ63kq66 - ZAtLq2+FOOhYrBE43tcQ7QR4rnxw3ehVyZroSuICpMyhLF5fXYpsCo8z+kVpIsZv + ygiDdQv9c4+O0f+X3+/cGyaKafz+OWKHg1UeXGUv4aTnYuxelo+RwV4dtaRygoEm - Mwd71fsCgYEA+laMTyiMEWyZHUbVnIRst1BQdT8c7fVGadahPhIYzElziGir9CrA + dtF01XkCgYEA0ATGNf0sWHpxkdpCHEgaxLge+m0OerTIZKNzVQ0UFszsRCj4LvK7 - uLGzMlyuDAOTH6aW3pTFJYeNuPtI5OSsYyjL6SrnJDGoyBGlhu8aHvh5/u+Q2Fvs + Nhvu5rCY9T2C9CpJkgb8VpwGtBBZEFW7Gvfg0C5xQkzDaZ+AhEbWiSucOH3dFPGg - eDgCtcEVtGjxlQQq/+BLJgYa05RmjpMeO6rlNR/7Rk2V/9Hig7K91DcCgYEA3Vft + 1IE8WV2oALu+IttuKizhQWyjSpfWa3taRiuZgWemEalLM5vC1qQTiGkCgYEAt02R - NIa+8zm7kWON0amQ2CqWEdmnUFN0z5MERs8vioMeWFysaO6jubmuI+1DtDkSzz0C + z1g6BoMt2OExNMB9S7JyEsL0X+Jnt3pMa5tG99lHl9hc7vqxgb71mLKb4jwBi9U9 - fG/AeFYI8ZcMQUeGJFsMU+s8Tkt17xkrCGvnOXSYBRwHvMhEIg7cCgB/DsOqm3A4 + tFLIbTGJ82dXlFHsM39KU/xvJJiBlxfHwlfRF9t1amp68oBJrMyuQASJYs+3XmQN - zwX0djLmSx7UVv6uJiegeqXY2e8QDxlaR4CUITsCgYByXloJzBd52mh1ZKgwsptM + GAm44QKSGdCaxsdBOuQT4LMIQJKjv9bKE6gG37sCgYAlTTXd6JBTLWHALcs9FxD1 - gIfRmPzphfYeYm0WA4SKyD/dIRz2FxYnCyA4MPlfCb8MZbplhAgxpiVMTpk14XcU + xa6IaZX3GwP0R/sefUHk9MpJTq9ye8RmZ4vngjNrhqQ89HhM30PQpBnvoB7Ydwce - ck3+f5hMA9f9V3qNE+2WGqT5oI9HGXAGWGh8ivMUkiFUmCvg7KLIg198LD9Sgcn9 + RuThb/KPWQSRpDB/h9RgtJlG6AsE/m9ArAwOWmUN/JyT05VlqraZ7Mk7Tw78JxqB - Lo064RqWOtn9nvDihCWPrwKBgQCJkGsJOTGWAuyTKJdslgFCh/0q7OXyo1u24n1G + CsB0HAoDkMATeRLvOmzmQQKBgCg7LjWD97hWMknXoyUg2l8y2zai81/YIUtz3DIB - 8N9wK5uBeV9h++bfuAoFpCFu8gXBrP5NjjrFz1rRo3nnXGd/UuLviQS6+GU8i5zW + 8qGTXtNE+aC6BRuk/eJ10SDmarB2LQTW5oaQyOZTWDWFhYIH/hhQ31P45Ph0j7Nn - KBHWAKO2kTwx1RmbPTb+NF7DM1JmNrHn4KCVkX7VczyvMKvVZM11THvgvpZxe+VD + 8sx5rluc4z82SPVUNyp11HGLhYOCEh2khJ9eIRLpZg8azIZQaMx4fuctSCNi0Rdf - CSOHHwKBgC8+QcZRLAm6JDG6v8YNPi1uOw57vJ1pSmq2rJtpJll3KRUiTCF/iF8j + WaLhAoGAB+9gxfCGBv6EN9tZxPZHx5YTAjIJ4Y82ZRgMnGhijzpdLvS1AzZ82tj5 - PjT+86nF/IORZV4M32Iyo/j+X5DDdrRpLsQEjhcPZAk5SRm9kLV9ZVupMFq1QuXi + KDv9+If36M1jJNGSQIJ0bNIOGbZkaqa2gHAYb4gPzROct/msPq3/UacauNiUutCn - IIFc0UBhrOeJWd6uXSZSM9ZVUUj63GUA4+S2QUFcJNenffN2q/Ua + Fu3UjLZ+kbmLgwU3vM+RGjup7xbshlSbrHFlnnrBW/dFHJNXYgE= -----END RSA PRIVATE KEY----- @@ -10362,62 +11412,62 @@ vector: segmentSize: 131072 total: 10 - convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:SSK:ctdk4rtj7uhew6wz32rj4pmm3m:cxyc66od6x5lkqyw35jsdk2nyszomusd2i4d4jxiyo5lor2gtntq + expected: URI:SSK:l2haupf3wtffxnzmhgwrab5hbq:cux7vo7b5w5fcqfqldt6l5ijiskdcjid6rlcz7bjchvmleb5xq2q format: kind: ssk params: format: sdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEpAIBAAKCAQEAu3XMP92QeEeRzRdTYGqful6psWVg3hWcSnRy045akcnup2m9 + MIIEowIBAAKCAQEAzZOs4RtK/VfGmRuKSxE5vLsWwJtV0T8lRJZl7Rg3EwTrDB1K - 5pJhcgjBpMuvPoCzhoyvaIjMk46gTbzwi9rsWPUpNgtNl3D4oASIXszo9bVD+PLD + vK7g9E8vsWmNphYSpOe6gZ/iOiLLMB5Qhml611luN4VOOckhRqJFmcSdIPjbmw/b - CERWJC+sBXvb50C1Y29fpGEh0P4747+goKmmYKhYsesGaqfl2k6VVfXygyhrtYwK + O5z1985q14iV5ZjN2VMdfG900+GmMKv/h513BlxOC7Vp0BIZ+PAj19JyZXLm0pbn - knlrpkSVbeGDgnfp2WDI94tIE9TjEhl/m3On4a7XA3ggkikfBHqhE5rLsKIPOEz9 + 475i2D/pAyIkzWGsNPU5Wm2GqEqbFAu9knp7G8bx5CF5J0RuzmEMJMQcnPZ68Kzq - DvubZlxZkSTdrwn5mOWE/JqyhSrmePogYNjzrjy+8qKG3+KoQgvNVdzjAWcMXpRt + PRfPmgeevEI6lBRLxD3QpUJL2BgT7hKMKInIxP4GIFyWN3hMiDtgxq/2UHWHdH2z - zYZyPS+sv0pvqB3omWZlkIJurqI7STlkN2FU9QIDAQABAoIBAAvvs0K5y/IstHb3 + YGl1j0ArwtPLK4GT1PQ24vORxw1iHdY4w66tjwIDAQABAoIBAFSfiQci5oQR+VD2 - rkJsZ6FJV8rI5sMdYydGhO09mjzAO+cDD6l31qaZMiNZKN50+XluydiBJW2b3k80 + Sr+q6BL+CpgfeTyI20z4Ah4OnUEpgZ37gtPXwwcef5nuwt3O8T7LmvUX/RaEUxLM - 4ag2F2iOq8IaNCWZdutRfpFywL6sfRiD9LE5ELcbJfvvaBAwiZw8Qj3IRYv2NEAL + L8acrfHuwNV+/NwBpL6ANtlc23eCqVeTt+G6s2+eG1H8ygN4mqfutFEQSk2b8f7Q - OqIgS0zKS2OA0JbH/BXLfSzNDVUWiOZKeyRjhlHp+tRUDQFFPa5pLTBP0I21xAsX + FoBbO+802PWt6FA721AjfgWt/eQvIhCON4Jt405tCmRlOQ9KLJmbCyXsiB4+gSks - GLF13DAjTOFcCIaGEtuAM5OZ93qKhDmGyBpjlPTz5kRJoMIChk1gOPLXKgq1UeYt + iQUAPZI2zoj3L0tAd9J3KEcSRCYrSRTv5I2wBCuQtQVDXWZ5rCxM4T4hoNj9JWU6 - m1XYlAd0qs1F39Mv+yqLr6nbKt34ufywddNGt52wivjScAtpiiqFVldola8rF1Pd + kqKEU/nCQnfc/01z/jR1rgrJpTnfTtimqoc1IWC0sEbuM2Q+O2Z/szmN0hRQLCcz - lN84qgECgYEA119K7ntn2XQ9/NKwRFvaekfOcFFzMl+XEcLmrd9oeP1ffivb6ZIn + hxBduvkCgYEA0Qet8LyVH6CzVLof40VcpDLu3er4qPHC/Y56FNmtpscTUh7QtC46 - gLoCgv5BpR7huTDTgTvcPgiasgt7NXFo8mahmNEghUBbcDGv87fMwi+sICjptV04 + ecUyNkWbxw2i4J7J/6x2nMNhPIgf9p85Xx11bQlAaEmBAEdTimp/v3GBxrY7vEn6 - g3or1oQ5saLoo/W9kXoQ95dJF2rGKQAAHQrXdwxizWuyb2Xr6eE9BHUCgYEA3tKU + 5GFuIo0PkY7cjtvk80GsM3iXELaIg92qRkSTf6X02WydJHXiFWZ7wwcCgYEA+8Vb - suNvY1xl4iRe/ayUyfQqJIkTrRNe5g2r7lls7hwbBGN8cbxtQiyW+F4CxP8xuB65 + JpC43P6LB4oHbZmVEShjmiy+kKRurGFkwOGrH1cVq70x+78EriyrMQso0Or07L9R - fbdKWPpSewjEZs5UJDgvC3PrvKTbct+0lF6zXvND3BO52vQ6t13/2gOVh42CpLAY + eiv/sMawoGgpF23UeGDWAr3aPg+ZNBeOaH9RxrqPxR04zkBge0+j8fppQ8ynAPnA - xt7tmkcr7QibCziNWXK8xCQe06RuM6C/Oxme/oECgYEAzYKYtcf76HwLSlyg5hnf + 8OpsO3FYZNFBi0qUizRxklE/6mw6iTUAkkvIdzkCgYBI2r5bW782CNK4Qy7+DZze - +B7c1kBidAbS2JfqFq+/uPPNU0/2oIJeP28/Rk/nw/Ab4+K7b+320xrSwmJCR1TY + dgofOth0Od9WdKRERCJsMJKhWrAvPLWQ35RCqjxDQpN0aqPJAxlMRiTL7j4FvTVH - l7VnLbMgHQa0OfKvuxf/wqxKysU/fVhevNavThsOEnspEotDQLYBysAJdtbkD+t4 + 24KkAEd8kbHuoO2THs9rsGolEjr7w2U42GSEklnMx9hDyoyf5FHalrtATf6Cx22j - MD5QK8Ed5naF5daTrrDG0KECgYBLEieHHZkpoLeyuQ5H6R037UtFg+ldJmmSmIiU + lB88rGEMrviOTq//+XpFXQKBgHQCrkuI6AW8rGde7KlN2Wg8ihiigXS4r95ySjCu - hQxuLIntsJb8ur8UzHEQvJuyQ0g9ABz+fgJOeAfR6+I/wMQYb9VpxmRl6iUFTtlI + S2F3iR7HYN61V/zBzGge0kHh4dWtGmgHGhxkkUJ4fGa1Tu/g/vvoa0WpfliIejAg - I5/LHap/OyYi3qXpoYHRseNvB/47/hha6ECk+dWSxpN19FerCz0N2B2KsJtwSXgk + apf9ov2ax3ASLeLkAZEgZ5y8Ej/a1VKtUg3Z3ncmDOOYC/ZQxfw7wA3OrPJIH+lJ - MT2gAQKBgQDORaQBkm5324m4AsYoSpn1EEr0gzMFve0X59B68kNt6e2CAVfKA6A6 + TifRAoGBALkaj07vfKfnf1oKCsgalurnPRF5/zaIrsQUQ02W4ZKLDPCa+0jcAPHy - om/LcctyDIRUjOnjogmX/iyURSCJsR1FyuuQnECtWuoos4F47utK6doZd6fgyqC6 + LoqUyfmGy/+Q9QFRGVTY7enh65sRadx7CN3TAqdqHyo1KIN63UaJ11irGdQZbgrr - cTRwVuM3ndmK+eIKrIoDb69S/AsaiVHm/FqnGJKg8vCywKsc19AI3Q== + 6bNSRq2kPWGyWtURUP2oo4q9jc0iWfmlIO0OXihdf1P4C5Uxc2GT -----END RSA PRIVATE KEY----- @@ -10431,62 +11481,62 @@ vector: segmentSize: 131072 total: 10 - convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:MDMF:dq6npcjrjc75bymi66mlncgga4:27qt6q3aekswegvwpzlbsx4joc42dp42d2rutejhmkrmg5v5luqq + expected: URI:MDMF:im5wulgo7r5kaqlkwexlxrkw6e:qztfutuv3vxg6dzi5zn2is4q5gzzavvh4sttlaliyhz5oyi7ihqa format: kind: ssk params: format: mdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEowIBAAKCAQEAihmQMc0MBheS9Zi4NwC4zU4cbWxSoMU0L34nvR6fItQmfURC + MIIEowIBAAKCAQEA7EmomKj2wzFXeP4lRtDJtToKB4s5mP1VP+LP4XESDux60i5q - Ez3mvVk4ByhfXFEXUEmAn+juNbxDjB1D3YtgI6APtoFxAyDXMlMKO3wwE5UCZ+OE + OjCMqXBnyRJpd40mqPRKjVWrHLmmZuc5+aVqxXrcl4TW7ZLBEyR2WI8WuTaFnEeI - YvESyp/Tek44InOtBCyMy1VUPZcaE8hWIWG5JhysUbr2Piiww2GYOmdW4zZbLWJB + gJr8obAKiyfIeCrace9IShGhtFNuon8JTMILE3y37jPzhBMGL6OEMguucOi1ct7y - hXSToyHgDvn89nbjh2xG0x1QwhYTqZH3m0HxQFuFIdScZ6Qro1tDUNKzpmUjmXPO + Pg8Odd6aiJ1Jn3OYa7nf69PVwam9FWvg0Er8jMrphI+2dpNETLkyHylZPTkk/t9e - cDUJl/K2r7WGNRvCypMUBatIvXM8Dt19ezyvOGdi3SvPFyLe6gVHZoUq+W5FiE5A + QUBSNW4sHfUOqwoBLYrw4B5xDaTNSsdjqmxdz9rRovBs925GwRCOEzM0A2VCUkKg - B6R1bUB0T0q5Wja/8TD++YksCG5v/zQcEmA3ZQIDAQABAoIBAC+5VdNgAN+6Fdc5 + ZeuIOnSh+mDhgQUoY3CSAqdjda0ba2U/mRP4WQIDAQABAoIBAAo1jAfti/zmy75d - x696WGLas4g8/vEANWCUQDdi9aublRGFHTB5G9wjkPEoSowkmeHtBL4+SNPZE57A + 778hrAdtJqwJBU/xiS6Jlp0JYUP9Dnj4ma0iNh+njEptJdqtIfmclYCKDiq/cDvD - HkvZdofZMJTpdpyWJMgHWmnkKNkbjZFJVt66YLwVL4f8r/l38DqZCq7Z9hqytRhR + u5boYWa/LlsLGay54ab5tTXSF+OgghEcq56P48C6LhhxW3lqs5XG5o8RuDSGAtKZ - CzLOCqXZEtPLwH0KostiVrEYNTafeD3jgn0C4kHfpG4cALcEQHcQVLsjmizXLXSx + Tc3v1+HrinhrrPa8yjCCfoaZh9vXsuc55Z6Svntr5EmsplpTUrgUq8QgYk5PXpV6 - 18SC12aUnAJXG8HD9Ge2CDFfKZx2RM3RRJvq5MZhe+WbQyidnWdwEObEVhRkSpgQ + ERadGnNOKlP3rpy18BxciEAdCFHBS3genlqfefIJ/6DSUH6lrlPWpGgoUC0YIQ6i - m52mO0thkxMnC9TtjtNc4R9ZP6Q/+bep74GJjX2nbPdlGojszVcHUli5kyXwwbi9 + mbQ3PnRiqQzLbKlNbNRf5QUQAfcqXSlEC/puwzj/5TNcM6gGgAaJ73X9gh4+xsLL - AFlheqECgYEAvLp7FL5XsIH75LnegRnjTugT7nu92kdXyVqEZ9ONS9/XcgH/pMkv + 2WmgvBECgYEA86Sm4AR/+yPEl0famqO0ThcEoRd+oDyxMn4KPVgIqsgeN96FZA4A - nBZ4vBEUYWqeYlsZD80tZFYE0unn8niXlpdNsIlFZV2BnxME4ZnGdtFTsjFg1AJq + vGQmPuGlZtsvvW8qQ0XqaGi1E4CSdkGyDYiUchMmuHTXRn+FVQRNScio0txKICuc - JmosiUN2vZJ2mWPv9I7cb1C8Sg1JYnhOESHn5dhG2py6Nmvwxpic82MCgYEAu1M2 + MHTy8av55F8GGa3m90LyPvuqi/FyRheV3dmJ9Ou2EJy7FbU8o4TNr4kCgYEA+EWB - BfqBXD+Z1qTzKp+pppR8HIeRI0f/jYsDp5TD+XVb/HBeE6Q+9T1fI2yDJeIS4WO5 + 0t/gMEUg1LLNx1EH30kt/5ivK6uJUN5nyqrtKT11s+5Amri2zOhS4uiY5L1FJ6ap - CHUZLj2bwYLOSre3A+co5nAhj4pT1C+Iz3GsMd+m7GPhDnpYpPTohE5YIxlilKKQ + uk2WewmyVZl90xnHyb0ggznjhM1Bu0v7538anx6cmk2vztxVtVp7QpCPaQ2zVRxa - T4B0qSk2AZQuK2i+YecyNFkZT4b4DJ2tNkvlOJcCgYEAt+J+sAxxxkIwG4DagjGm + 6lVjV7WYwxMCkCK17b5C9wGtVrZInuxKm0pKflECgYEA19XPwvoZihg4iq+rt3w4 - H6jSWshoiDiBGWg/oCYpAuebtLKr0nRQFiZzBtMhZ3WJ0s1uEs5YTu3dD1/mpoLH + OUlo33BZy4eYjhtb5NX875XSNzoYPvesrTenLeNlTEX198Hn1aq1KoM/jiRDGyG9 - OGw9vydQ3V4JQOQ4GlRJYlW81d90t72OjdVfhXKdTEJbmkMcds2HjFI+02w0t2P+ + owGQR7IxhgxzvM8xBYyHD0sES6+8tt0LQ14G7hKkkCuh0tPcnMSgpyz4+3oL+o3g - tISzvWhISRLyALqVQ/tI2X8CgYAymcSjEsr0xz1gDMiev+hM1hk8f6ZF+IHgkyeW + RKT28pJxOiwuC9/+9Pir4ckCgYBtaaRnHIaefziSxCHv3wQLISMGa3F3W2dunjU7 - kgnqDbieVSAkgB59kmlroTk/93SQK6bk0PTPV9cGC7Z72mp2hG+455s5Me15CKoV + mcxeylke7LbH+POGpjQxD7Shyc+6Q7a1BhB1NLbFBpnu+IOVoqW7bz2XfyWitz/S - FyijhD2L52L4zTW6wWk5rAwE1yuY6NzAjPt2YmpzPLrIARBEU/Zsy5CZueSxS7pp + q689xK3bSrVaArw66h88HJ02/PS1Y4Olle/r7XnfLneIseNfXOQCG1kax8aFUzkl - S1EM2wKBgETqNKqw2XnGKu+PCRgjnkhLLWNxMV99w9XM7Il/d5OHxiQ8fSLH+Wnx + 6r2doQKBgB4L13+ad44L1O1wmL5bQbXilhK+y4ZCeI7kmArASXbok6bCjf2KhN73 - DyO4Bt8cRgEQef/h1dwuI5MSpL/F5AmWAk070Q9u90atzmCPzukbbiJt0D67Fzug + rw4VTeImmWMn16GWznvekJyff6JkG6f+RMUSeko8GT+Q8A3lb+99clJxUL8E548U - B4e53WwD9VpcHTC4tN93fagbSYci1WF6+2YIiO7rI2zFARuE/jL+ + GT672ONV/XbMLaIqGlZfblZ0qElrI2744IJoBr9D0uP1hvxDz1z5 -----END RSA PRIVATE KEY----- @@ -10499,6 +11549,156 @@ vector: required: 3 segmentSize: 131072 total: 10 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:CHK:i7vkx7yjzrtlzwnm66a7jn2dwq:rpz32lhxxu473pbze3c4a5yrsy6yoabfdb6v6o7plv27w4rlwk7q:3:10:4194304 + format: + kind: chk + params: null + sample: + length: 4194304 + seed: 2NVWOXvtwcAFf0J/g0fDC+ypQbnCU7awZoCxcnhpkVM= + zfec: + required: 3 + segmentSize: 131072 + total: 10 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:SSK:guphc2riwixq5yh3hhelvznxvq:cb5gfzv4yswoqct3njn4irc7j24q2cm2byqlxuawaxyjrrvns2yq + format: + kind: ssk + params: + format: sdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEowIBAAKCAQEA09GH1R4NN/xGCfZG/AsPeQ/UW861L5FKvT+F9FWmEbotN5Dy + + DsbBxeduMPEti0z5wZWXbuSrHq6lZMD2MxJ1M2QL5rNQ+eR1Rpr8LAQO24PE9H9X + + TGJO5aqeIo5M/GLFNGG4sDCGmwkwMGetNAtTFRzqjq1i96GlWBIYclTRLj5Wey92 + + EaSCO+CWjyiYMaYloSd0QJCyfUipfrtA9O/tKBXqDq5pY7g0elCCq0KIjyUBRHx4 + + UmTcl1Hpg0S2i5BpkskvJr8M6foY88eoWxmYLw0MFb7ueeh+3e+NwpSPsx0lP2Wc + + B2zTT/F8ClWoHR+u4YqCSOmePKci7qRtx4pQKwIDAQABAoIBADdzwjCr1mASvi87 + + dyfiqWFTIJAMVGioi71xlNr7VSeM6uuCGax+ohnyVWmgqgCu3S+tvuA8IwQ8SnZP + + AeUq7t3OUkNKLGfPRFiAmIXZZh5Xp8cuUydfETKU8SMwx7zHCsOE1bniakrKJAB4 + + E+LtGAoN8OX7RE551fRxgE7mH4EQPNwTpl+1aqAvl2DOukXjcyywW58zBJ5VuiVf + + OR87US8E1RLKAJjxcMeh/J2hJV4KY5LIzViWBsyQH+3E4938qLubXlKpZraXzoYG + + STv3PTQd17LTD574P/1aFAirJlAobqodbwkQdc5sUcQjGi4tcK8ju8UPEllmrpuD + + kVkfQUkCgYEA5zqm0feBtoeJa2BxoNmjKH1p+zaA3RRps44hDnriOrZZ8R/xaa98 + + CjhVJMH4lthlFNXDtH+2VgaMs0AdAXxQfhwMUTUgX7Oxivd1OG7XBDT21ITlnd14 + + f9tBy6J5fRXGkyHRSZ3ImCRcgObCA0BkPPXQtdzdV48O6pQudqEgQN0CgYEA6oKN + + Ot+p3unrLUP17oQqyEHnCcPRrOTQkdVOZ+rGeQm/2bkzVOs36q2cWJcXrOFh+L1h + + OKnpdTvAYpgiDsBt0i8CybXhUr73l2c/uO19z91VUiFVLr1qNRcW9e9bJNQd9opx + + E4NbpGzFINo96wB1c9oVOJKfBKzkkiUpCOGAAKcCgYA+u1bO2AtE7fiGPSAWt3Tg + + Y0YBdYP4drVGlWS6fPQrYZV9KWFhfs50J1xSIJ3EruidgnEZ4xwgsp4xc09rO8LK + + s+lTjso9rI6aWRBgQxHqfkQI3BU/gvpSFbX//RBgsyuwdxhElJ37SMIf5nr0Tt/i + + +f2pmUYjnxg45ALHBGevsQKBgHhMXkyMPeTnFEhVK4yeah/uhqlgtWe+vSuCQ8VV + + D1k54hu7QJTYUQfm5WQgpfl+aLaj14KszuDftPIe3qG4nt2KViDJV3wOEI6vXWXt + + FnQSM1l9VegzLI9td87TaWr8ER7Op/D9mn4/eeQ/cDHkO1whzG8H7+EDHAHIZEN1 + + AifTAoGBAJfmHCZGx+d1X5Rtx87YgpWzwVDDXTveZ6CNGe4hUNAEVQCNgVzol8Ha + + o/i+J3IqLSb5KUclhFrnH+fMYelJm3JQXFy/3c25cwnQLTJV39hIbREDrErGN+SU + + XvQJSQDtqXtCjWILMbSq+xL4fvUYNSqtoqsvYwV9XdkjG/pfaHZt + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 4194304 + seed: 2NVWOXvtwcAFf0J/g0fDC+ypQbnCU7awZoCxcnhpkVM= + zfec: + required: 3 + segmentSize: 131072 + total: 10 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:MDMF:fyqcwoevvyytv6vwchxcwvkcdm:fqdtjlkf2yctrwprgu2eei5n6noufnasi52qvqzhmoalaxgo3zsa + format: + kind: ssk + params: + format: mdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEpAIBAAKCAQEAtptKt0U5+0WhbXHsP97E+R4e98MMZ5zhSIEJf4w0SX1PN2JG + + NoO3CpdtmqZSHA/QdiMGM18ThzK2RU9OfU2ltAsM9VGFPjE6qVGYu90vpYy3RTVj + + en3dXXVix8ib2+6vZSKx+BZXxadeK1AMFEKl6XzbcnTfTgd1JsE8pImgHhUGXH+p + + eM9h0MxIMXvYdGeG+OghjNkPUveC25jrJO2ZNv1A1AxAX7Q1vNgFPrYe7AhJNlfA + + QYq0TJYXqywUYDlVxEtA08CRM3urlFJIHrdYgQ6+7PkGqQ1V0nUdosqHeXxAknrI + + 4U1WAObvl0zGHSG6UzfboLxPKHqOtpxyFIrptQIDAQABAoIBAA1CGdr+orwUt9Nw + + ybzFbuXA2HIYGPwjjtPV0pu5Jp64rhoQ/5S7sXw1DGyFwOvFVixi/0iW5vMSV9uR + + FjDQ2hlyvUkegcnHMewolCn9CurpMZv2dAzpEJEq5wt8YZyb/Y4em8St0pzzjvR0 + + G7xo2Q+qf2sVrepGGQJsVfvYUOYoyXihZA+INZ2Rj93jhFHDTs27vo1sksj4OWDx + + urXat3Q3tA7AN/nTzdKPOSoslqbctY3qMtJuOWi26a04YWpa+QYp9bWEEKuKScRl + + LbTDqj7GgBEC3ODeWyI+PoQg+Q994H1AETOZ8bDQ9r+C5J9cC7nXOKZrpF624zbO + + 0ZvMEeECgYEA/z6I+iQxjywspyPHvhbN2EeDwEjCtGrSw6ywO6YiOLy+tjjxZp77 + + drsxc8yIfy2DAN9L/zg4vNYD7AUVFE29/h/GcPRU09VbcMUzlP4xAhs3FKg9Jq5E + + o4t6Z3kJKVzFjkN38fuzMq1SsIw3+ntlPCqF4k681DabW5sF3XFaLekCgYEAtyWz + + Sid2pfJoyHudbIWKTQZwXK8STw84VE8KNCviEGPooBoBSs2FInIgdW411DYQQyjY + + PtLakwoFOnj78LxBPElq8NzVKRRLzzUcQVw9uXTZN0UGTf9ztauZ/dKysBjKIyzI + + NWfz1+SyfND/BFwQlAINLMpnxkgwjLOk56wMge0CgYAScfJ0ISlzrz2K1osYsY0u + + k/xxaNCpOQ8CFPinVtoiP4GIqZTIVbTWX7CzLZSvnBpbdceIKgfvnYerBrL/RJ72 + + PlWY1A9NP53cCGQx4Cyqek0AsSe6I93R88Jkt9pxosKkBTwlwIqyntPa7kcdUs1+ + + C5ShRg9fRpLzi8BgwFBEAQKBgQCYqJkYb2qLilJjAf7HLUyJRZu09cz6D0Kxq6xi + + rk1hwhVuFh8LneGiQ6TgnTvLJkFJ6arOOu0r8QdIpP3DvPdXbA7ys/ANrLhAABIM + + PPnKMya31hYaP5rQTDgwhUaiWBdtWG+NbJepVhycw4w9swuygz8+HXyAnz2wmjET + + VqqaRQKBgQCJxWxMire59h1v1SMAf6x7eZlNMfBf1VTmLmQalt33xCk3egdD/lHx + + qS8fZdcBgxHcVYZ2vWxESyfyZ3Iy/dglRIKRy2rmZIyKxp2yQNURXuEkKNQdXLrt + + NalUR6GCI8RRgnZCHmFIB09WQ9BFnSUzbFhoLOYQIg3Dm2HoA495yA== + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 4194304 + seed: 2NVWOXvtwcAFf0J/g0fDC+ypQbnCU7awZoCxcnhpkVM= + zfec: + required: 3 + segmentSize: 131072 + total: 10 - convergence: ZOyIygCyaOW6GjVnihtTFg== expected: URI:CHK:d2hbvcmbex7fm3qu22yj4qnkh4:2xwqxbawwgn773hht6etox3oypvqqjv2orktnthfo2e7vibko7ha:3:10:8388607 format: @@ -10512,62 +11712,62 @@ vector: segmentSize: 131072 total: 10 - convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:SSK:5v6f6emfnb5opnhyodvjzoe46q:4dtbhyqrxhribcthqybarbueominfvxydbofslw2jazx5fqsddwa + expected: URI:SSK:cugmepep7hr4ori4mvbkkplac4:5m5v2l3ul2gmekwefhahep5xvi4o7frzr5t735ruioy66oicvf6q format: kind: ssk params: format: sdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEowIBAAKCAQEAhAOLr3FMSznpE2hrSqzEuImNDNa3iNyqhsBFGT50ZnvAbsis + MIIEogIBAAKCAQEAh3dRTejzms9mArre9F+NB3B6BmFfrOo/fytWqAeT9blc36bJ - tZ0dNfyWVKsa1H+e+5ctyYRbpJ8Lpz/clzy3aBxxtEv4dlJeR9CWUpY0r4wnYMTd + x9ErWL3PhkhxJBaCQd0QxwKbM3gV+41JIJrrwMCdw1D3yi2aAqPo8kogQg1/MOMp - 4L505REujkMbEw0y79hRdUMB6dakA1gKU1TlwObDLYx2ItTVoO9uNYvOLguYCAvx + v3LF2OcuzY716WOMcd9KFJDgIMd2dQpO0sEmxsg8r+PDS3mL0ZFSMHScpcn+tq5W - qKuYTdaGE67026Y4uQXlUlFNJWIUcr35ZsMPgiP59VeyV/GHtVqRbUwEx7Mkfjoh + of4qhCFEjkvkWEtVI8fG+OcA0JAQJHQDzjNPuYE46q8VjWwRSSt/5L3U4IwF8IDy - Nz7V66jVcZnyMZqxZcD5mb/F42qRtcJRLpy9f8lTlJZsWZ+WQcUiskFUUz2jX8Ll + v1w52Q2Q0pdO1OAOVh5FxZ4lybu/bhQENQGaQV0ViBSPtYglei7fYYkGhVEpZlNr - XRAgKZ6QX/wdUHB3uPdYmZMDLpammgrgexD/NwIDAQABAoIBAAPKLxFk5duaMjG6 + 3VfwUtywsvqTMfsHIDUhgycAFqyl839wPal/oQIDAQABAoIBAAFJ7dCXcEXfRkGt - o3hFiD3P9q4LeY6Pq/ZdJxHeQOzMP3r7fp6VsxTHFCQigVoTJXCnJ2S+/8gbDlrJ + ZHSfuhhBhi+sWnwrKFp934+ursYuMsd/7ziC8W4hNCm7Y7QrsKaOw1uZVz+om1Du - ZfJfINjEfRMmKmAJmOew4Kl54tG2hIIUd+/CwYvt6vGhXyUBEcxG4BfHFt/5DhGD + 2azBPI/zZSzYBtt6DpaK16sCcIcgWL7u0lVbcq1rGaNkAkvrogjtTmeerzswmpxn - ToPHG41muOwVOZlh+/wE9s7Jg6xYrrQ7u+l0S4MAsWV+IPkKkaNJE+baJ3QWgJOk + cjOIGeXXPtiPiqqbp+xg/Q6v9VOQclw128X3roNF8WWtDKZBUuWAcYvT2Zot/aNf - ryN9A7esYAses5cSSw4JMgintd96u7tB59sey87OJzrpRDAxiQ+2dOSJKkbJJdqS + 8KND5bal3af2m4dEdEGK3d9cvKuPsukCWkYecghy6K/vgce4uhA5I+C/NYvVEvWv - NBHwtLYPLdrRRMYGrvwZrs9LFTheqs2DSrl8qThn2mCZkUzXkqEyu3weieC3Kgam + +6Bu0zNNT1ITlE/adhtcdYD4EClV+ShPWr+d304HOVLZD6NWJsmuWOa77ZewgmT2 - gD7XUNECgYEAuWvsqr1pHoTof6ozJliKACw5JLz5w+3Os/Drx3/W5G6ke6iYodPx + jWFgOaECgYEAvPQP99gKb3+NkU4TilgK6vDrlt3qMsGTQKgPwB+hTJ8MFFPKyA1e - yfLFDLkx39Fu2XXrN47JKuuUyEXuEnvMgu7GX3hxtrUljyTRoMo6S5BcZ8s3ocCI + +pSudVtCmchRg2E7YAU6NOnSESuMvXwAY/r2i7annvKqgegwmP4dr7K7xEwXILic - WliphC8pcTcMDrthUTs78u13UZEx1HqDVhb5b46kK3n3LMHUX/YF6QkCgYEAtkNm + w9J8G2mAU5l6XD/55kUPIyJRE65d0ver4j9krjH8Ok3HksL7Y5NCoKkCgYEAt4ii - a9mD7fJ8Dcn3dQmN4c/ym77V2RV0pNIzyTbiC0BVbqOQ1y4L6nRrU9kQ5semEho8 + 5LYOockPa/coaRFNSTfaUhKc9kii95K8iPyRMx+OrgAZRYuOAL14j0R1Q5gt/7Pp - sPfcw3cd3uz1LJuwyHz6I4GbSlKtpEEWUfDWjyrnCnv1Hxxxov+htrVYNH/WsLq8 + R86SdRFVuGCL0Ec3NVwZHAeJZVxAv5I6vpkKQFE+UN7UAp79DY7Usbo2oYNKLbnz - 7iGIC4/9LNboSaaXc+Bz3umC4s0qAIYJiCW+9j8CgYAUF+v3vLrtgb2oSAtu9l1O + QxeEKdFHGz5YzgZAT9plDG6YDzszdltxoyy4KjkCgYB7e5BX6yLevN/6fqi8d08j - E3zFzGzMnLKvsUX7wpDJBGxyshyIPO2Q0uwjqtYKySlYC31H8gM+0XS4F0vrWNsa + PLDpljrwUpr13R718nXKCgKt4hiaZkqUvcfJQAulTkke79MKrD/expOWzvwZ9MiY - vUFmCylXgV3mmzjUUdXrZmN9I/qNXs3n7H/CQVIeYLa/yfKL2P1wH+e0QSXDPtuI + jjDDG2otsO6HGQNxaFhkMw0MeqF+q8cfHhYnH+pSN/HECbc5qhX7YvjTQNdmAJ+e - ssipHC4SQA9XHFIlbAXL8QKBgQCuNZQDB+AbIofCYkYdXul3adyZUvlxyhk4pRYM + qskUIexw+dWb7rq0107qiQKBgB8KUOJ1mAvswVr1NjRu5K9Zbucqlrlgl49fldtl - gGHkoTRHUR3THtcS3P3tIfAOtcudR+i0ueUQC53IgzMA1TtPFm28XFhC3O3NrsyX + O/l1gOAUzDFi5OVjJhy061A/UKhKeU3XthVulRzV10+me8Pei7Cd5bLq41iDFsRd - u5xJMZeuJLcxam2Pf8lhKspJO9vIBmUpM3Gmo5U5M5zJMOtYPbRi301UXQeFgpg/ + hcpS80MiLKE02n+MhJR6dfrjBYyuQmI6e/PGnUwSQ1q02/OlcCmmGrmYvZ9q39FF - wtxY3wKBgEzQlxcbwea/3VeYbkzm0l3zmcEzcR443J1SKfIDAPW+5i/W0VoFIbM7 + IvChAoGAJCqcARPYEZ7bRbUiWNHw58C12bjqA9+Z1aIBmBSkuyIFQF7jDDoOpzON - t4C07jgAM7kg6RfcanG7Qe0g4beY3+mLopNM8+tYTEWIazg14fTCDMrMbm0CIyxy + e4njvXv2tY0C51NVQrnNNqa4Fsb/HEE7mv/hAApEGBZhcJIX3nKD1o+FtCIMROe0 - rYnVQaYTNsadlstOnJVQqQ6KLcWHadx2SnFll+RPp9h8UHZZeofX + xx7f3h/IErCNOmoImhrv2A5Cxi6cq+ZN3LtO14mPdkq1lwY+9i0= -----END RSA PRIVATE KEY----- @@ -10581,62 +11781,62 @@ vector: segmentSize: 131072 total: 10 - convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:MDMF:aqh7dflybkehfvjv2dtwm777qe:glkivk52anvjmc6b7ayncuq6fpxqsgel46uxyqqaajp7hf367ica + expected: URI:MDMF:ynoa7zlnwraylieeahe3ndidf4:ukho4xxl5f6zku4qsknk2aetdzbkqkq7v4ebwiqwh77mdkyw67na format: kind: ssk params: format: mdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEowIBAAKCAQEAwMil8D68T4BxfJHUaEsFhPutYwqYV0jTnZHf/+TPS3SgkIbS + MIIEpAIBAAKCAQEA8nnT38dExgYDxNQX/t0WtAGZd4ibBLy2OpW097OaJ3ets86R - OjzNzsoCZAyGDTA8ICfWOyY7W84hTGq6pSyXdBfokkpTiMC1kZFJKwqowUv5o6VF + qWNH5LWKwESEIQhRAM1bynMudc8dVCPRRuMBRq+aVJXeq7eGufhptFvtzIFz+/h5 - 6qH1yiOHR2KZSGxEhS6D0bIaX1PcUH9bipGT9AR/qiexc9RUhHBVb57tpiEziM3R + sehSd+D/t7puCQw2uCA3E/xf/QiqVhMrqnBZKAWzflrFlIxabgMjAxST7XxuPk4v - UNpT5roSh7WxWuyloKDkzxA2/1FVeoxoa3Xc/vlIaTsbx19cJCMnamnnU8cpYfZm + HxjrwEdBc1UNwGMJ1trhMWdWrP3FgBn4BIA8YOszmHagsjheLBmI+QtVbtk9EySd - 0WMOHEG51v8GHKnQ/4si3KvDQvrPtG1IxEpM0guzZmEXG5z0eFo3C1zKfVFxPAbC + +zTXZdGSN1wq/2WuO+YWBhLz3NW5dnFn9OCTlBYqzGStpAD3mATiQ6RNVoH+hUMN - eK/J0xKUbbq2P4+FgkK9vCR5UYUgxPLZ9GGFCQIDAQABAoIBACAbit+HY0+OYdhQ + hNcepowHlrnzjLW0CArh21oFReznFg88eEpYBwIDAQABAoIBAFH216Wh/P/5YYXD - ZWL1U7cBP7BmHFc1LuFoYTk6P3getXs8qRi/9bsCFAHbwBvEM89bMyfoxywUGaGj + 8jaPctC3Z7Kt5UT1K55jI9DFj/r+bCPHVJrPOiq4KWZz4rwtzP/56yjkxZRCRlY+ - iPBni9XvAXIT5PO6vMLAwsHjZZXD9JDXvtxEGy6OWkJ+Xm8ccREJXTT4h8HmsqPJ + Y0xUiQZlbsRgAuzF9Y9gxw6WMqy6J8RJio2WjGYEkzx+kxqQ2+Bi12t0mNf2eWnu - glKCynRyp1yMfdZ/v4/LMb+EZaosRNbLBzhgW70NGI/eY0S2DNjpNfx1McfYpKkK + QgjzwFeUkcfJFFfyF9FvUwMEDjss97odJa1TRP183kBDSm/yC2dndnXuS4HTqAnK - IsBqq5R0YdvwmF+yaU5BuKOYe8Vtu5S0m1/LRrTNHzNB6Hv1odwhzZU09WaVpZdz + 6ybRDUm+3dg3um0wAoe0xZhW37OECO0nS42GK2DdHW+Ln8Uuw+sMWT4lBqq/5JQT - N77vj5s72OnsZsM2EeCEZjQ3x+gD9PnJ29pBlFtO7YLMvWjASOav3S74Vy3+aPG9 + 9pNyUPqvbxkV7Y40JPbSFHG17eUBDCRtc3G91SgEn/MY0iZMdYeqi5xaMyhknWBi - 65kmD70CgYEA+9pLETK94JzNG2kLkdG+ecyIy+DuLInfAKNjfIqz3JNZoizIcj5r + CTwRGkUCgYEA/W8qJcXUoo6TtR3y1XO6vC/gUBc7zxj6UaQAK/gw+m6SUp35Q9Yd - 9M2haKX62bW6DBbmvp5vzvqoK3hyM4FqwU9PEr7p6AjF8ffoH/qta/BaV7rf68iQ + pGP2t37T1m2PMcWl7CWvG/ctkuY9B4JvaAGrfD/6XeKdnqm0qOt5B4yy1yykfJ4X - b18J9rHhjIp52/Ke3P1tGtiNhP2u4kzh3mMtS9+b3Atb4vpaXlYug18CgYEAw/VY + hnXQ0Nv0nxWJ2Pi/+rwG7tLnbd3mU6AQmp6Kp0lS9I+Rrp2Q2qnOIC0CgYEA9O5D - RmfQOOb6LdC0apARf1NUnGu/h1OjtqykgVZMl1f+WV60F2jw8OE2gd+XMqE6qWuc + BpBn0mpgy3d+55vZXufTSN5ZS/vc7WRcWOGCtVPHD80k5uNkGiR/XVu6MmFWeRdy - YQHAgQL5FYaPXJzWpgKo5WIE35+80xwHdwbABfGVfccCOGW44Ypm6hhHne1PaN7z + N8XeGZl41scKnKtqaIbr1tWbRB0HthSOQvJOSXk8cbEbbxfchUsBlW0LNmMMQIC/ - wsRVOKHtb0bm7hBs1Y+ZEkpbNDdHB4PumQ0S+JcCgYAAk5FUar9Qgktd4rGqFcbP + R7388vtSdnIOY3gcHprfNT0W35f2KxhOA6BLBYMCgYBMhl2WrEbJkv286b8ifuB6 - 1I4DmXIyG+asw7L4mACtYpDz9BJJYKcymj3iVW7rjKTuXicNDKPI333/C3mHcKZj + 5IX6CRnxLdyf/EJlBHtdkzexpKvYtPWcZubff3ddvxVG9SRlyvc2HYvwWH9DHjqf - 5uCRdGpoo4yAb0bSu+olsxkh1kWo7n6WIquNKv8PKUn6HOYML3BOfWxlf4ck8XQa + kCmEyhjCcqQffaTkgL257t0tpfhA/MejvT2BY3lY8/r8vhfSESaSxLJG9YMP6zw4 - 5DM4VzyuFkCRlm0ahiv5FQKBgHvjr43DsJdpIJ66pnYA468WJhZG4O5T6NtjRxYm + Q/kgDD71Q8i8ji1oKW/pPQKBgQChvwJ2QEC/vM5lL2mX69y1huSJl4Ri4FW6U2+E - U4ITtdEW2NE8HaiNGoL9s3/lA0t9p36FNwnZsVT0n8qztdl7MQDk+aPQP/dQbz/H + po/ZzRSFA9VdwEan2PhfH6crhApF90zPNhUA1M/vDgyc/7pKgucVvYRGi+E+xf5Y - WrvnQtYkbbjuRvcBI5O5Cf5EvMHWw1JOAnstlQmXUAUPCV/zy5kOvZ7Dm/qaZM0K + iYlXjf9zmSDj0V8oiyrlkdg4t1os8pje+MEleQCxBYso9vWi5GWI0+naCJFhTjCe - wQW9AoGBAOzDDUkowmzcn1Fs16x0ZhRgzh8knWwKkoybULqpshDf7jj3gZOBBa4c + xmkknwKBgQDLVkyCu2lqLdJXEpEjnaYitlEFo61g8U1509LxnqvqjQCyBUZRf0Wt - 4VDUcbhdxBtqPv7243lYbkpZ+0FDYRIVu1iIVjvcNIxDMpKK8ZcQ2LfEDWT/3k7J + xc+QEPNIlbF0PmX9dBFVazq9bQ0bNZk1C2Fp1EJ/62MNGrc5PzNuZjXG/abfX+Zu - ZUGmgegwepIT23xmS4nh25sT+2m9q0ugPpQAD7pCd5+Aci23dvQg + uYxLWlHgLHuky81onVX0+H2S1y2RTUiBe95xwIMJu6gFEB1iVq3a1Q== -----END RSA PRIVATE KEY----- @@ -10662,62 +11862,62 @@ vector: segmentSize: 131072 total: 10 - convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:SSK:rdtcycjlgs4jrn2xfgrbbumely:rt64kcear75nsev3r6344qvxzqo3hcooigugckkreyqld7toy57q + expected: URI:SSK:zlyggm5badnoshujgf6jckdaxu:cjtasx6gvdlsxn5w6wf4xltilhhqq5aowc7tblsewkmvzplkjlya format: kind: ssk params: format: sdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEpAIBAAKCAQEAppvR9FvL+QCAN4iaEysV+pxecdHZSJwM1F9cQz4E38XnUhBs + MIIEpAIBAAKCAQEA0rjXHJzn6mxSQA7dvD/ZGq2Ot8GytcTkwcCzqsSS7zWKiR5l - Le1eR4SNNUN/xVy+hPyiNBBU1eud0YTnG23ACoxN9qyf76+A1+uyXAKYcoUFbs4W + 6n3krs0tEkZNteHntn90oQV/LxB3yzoITuZmuu+hT/RO+uC82NV/F1Qa07mJlfDY - 30MLp1bUyByx80rHrOsQUH8Hohk1qtP6RRW6e+tK3htm/OtvQwiRhSqscXW8ycFR + qxGvSfWVtLgvpkh2ruvO2N8B1ClRKcB3mHMnIU/ajHY12rQZhKIOqjvhrqXPI+We - G36Xid5wYLx610JnwPOYOGDYqhCYLxK0tSOZkQp/tHXt42ADNGMVLTe+kFLX8UKR + S0CjiFbAfYViaCsb1wabjTxg84HhbXVl2GKjujecYhiwxINp2BsNAVvuveuEOvBK - znc5no2LRHdhDuG3mUrb+0WUWZHpK4S3uB/WZwbMLOCoy4fQs5Z0nXsbV1NrU77j + Gx2CpnvQ2QWv/Y3mZe0o3TeCy9xAEpMmZIxHjFfrLZOsCifxB1VXcjxdGiVpmhcn - LasfAaVjORirFVdKMPUVmJARKGFjQYvigVaw0QIDAQABAoIBABGaLNhwSmCIWQOE + 4gxFTldCKVwXdjz/KzyG/gQcKPRiVJanBC9GJQIDAQABAoIBABahHwjFmOpF47dZ - /yI/TxcnJiNIVHiDZCeb25ePGdy6f/H/oi5IAcn0iyaxdvJXFhnexxRRFWV0ezwD + YVqcCLaiuNbnCEgY8vATv7exEI5703rSNuOtzWcwRYzW2/WSYw3oNiAstPHa9OJw - mpcfRUbYA/Sn0E32cNpfIHzwGUMgIq7OP0RfRP/tAJYT0gkuQWJXg2W9xgSuPSlL + QwAmIhYlMc+iTvEGPYGTu+hHcfIW1L2zdbE5Xve0VfVoakWTNpumWzpTCKE+Jqcz - NAnQfd9RwJsusfbOuPaQFS/Ijmd+KPWfIMmSLwGfkNbOA1xHYbLNHYQfrYAosV9x + MiS/CQ68wp2e/D2WZb8moCiL8bqNhfo2jhuyXuq+QJqwCFhsgNN5QbDT+SrYCTGV - W6P8BUXzL9vHwkHNmRi4EJt43QeQj8eXoHIsUhZ2ZwANVg3Lw84CAGqjMbxyFJQ0 + sg0sudx0bcmuZ9+1RATuBuzgBLIVSbwyWcAtM9YRoYo+DKdlaFCFFAbQdo/G+tnj - iZ5nFNUSMRLFfNxMlft3DAqN+T85ZDFjjscaNC9IVcVo/ou2l5LWNCyzpDnyuTrP + +scE2KjiPE/mr1//Wi338gbjQZ7OjpVcAE7iBBie6Yrj5VZ23Kg0hkFQYGz+PiHr - bAPfeLECgYEAtleGb/OO+CDTnyIqcvsr9NkE/O/lcnZZS0SB5YhDI8/SRzWOW20B + FcuAsakCgYEA3xxtkdDHGNpBx3JmCb9aGqSCO7YhMBFlo1CQhRVVSlNJwSv/KK0g - rD+YTp1MQid069A9pKsIax6FW0eBWthohEe4N0CS2N3Jw0sWVIbmpWabYx86NBAz + DTYyhwD/bIqC/jny3Xq7RCkaVgmQHw5hMoOBqmNuqzAdBHHUL1xvzYKMjaZ7U3J3 - uto7OHSNoHZvoVg+ANy2Lwj3Prl1cKCKLxPpARrvT89kxBPBv7dQZYcCgYEA6elH + HLSl9xbweADUO9foGOjZCJuSs0bgL69gUry6CfdZRuX7+fZscYChq2cCgYEA8cji - glTu3KT9tj2T34xrtYY4PqhFgbdCHSRwKL9zHypadvGutN0CsAyPRjWm2Ie3pIs6 + 514ojQpIpckd7TGyelLQbSZp54YKWGN//WSY15Ool2f/bmg+DVSAt6fK9KEvfuSw - KnJg5Swo0T80NWwXPLfYzWAz+cJVMs5+vGjz28+A7FhjeYvNRZ231VDuhcc1AL9S + GF1+ZaeYDg6L8J40NMvj3zwA9B3o5Qa1VoNuxC2gZosh+aG0GLsS5U8/EqMFwuUU - JOq6QDiysDEBSfVstQP84j+FNpETk0p02JLhTOcCgYAxg2raE4ULE77jQ1/LgTDa + 4Z6LXzr87o4wh7yGR+2OsXuyHghPqg2pT2QJFpMCgYAqn1uvR5tBfDCk0Y38vrmP - d+PG202u2zw8GAo9zdaNbu1msMBLSzpdD5fIISaIADbbodxbTqYmkE8eDjit9n3L + 7W2Tyq98Z1ZrZLC3O+QXVuH4LVeJhclhvMDaWa5yJePwfVGQTioIU3Hcjecih7S5 - Db6UIlC92tvi0AzsPwV6fHZNYDlp0cx6PLBAEEY1AHQnl9KeYVCHTSP2QF4Hi1B6 + 2bWjv2sc+QwSFUzb32Tcddw0E2HsByoKKdiq44783eutowmL+K+9nTrhVODvOynD - oClxR2MchPCT3dmKubh3GQKBgQC1s6X94zYtpekEEO92nyDoQJweaB6eNhoggzax + pJpF3SMJEFaa4iDFbjV0cQKBgQCkCHPYIAtG1IlA4FcLSsIZNwHsazlCN6/hE3AL - II8v7XmangEls+0rjoYZdwHlf/+yzQhhArqsK1KFwQAwY4flfbbnSsz1PfVq4ydl + yyneZ3Djd0zV7KbciE3jS1Tn7kq4vhGyFgvgj3kbYEcUcWBdyU3Jb33+ICSW7Jwu - +m08GgO/FKYpO+U4J90u0pCG0QkmTHhl/wSxcJm17ktfBUvtjWx63/b+PVIkf5km + G3EUaxf9ObtNDqWOeaxyIfdaf3szJBOsldFcRDrA5XqLPB2lwsciJhdLRLw3VJlf - x2pGjwKBgQC0nYYt/ntilVFsdCfV3LihC57LLwdejM261vsI+YW2klTzyxEFzrmq + ITEBPQKBgQDHVvKBfDKNKVhqTFkycRAzaWgtoVljEIfjAm/qT1pXthhaB3PY9NzJ - BnA1KZD1zGWhth5ZMgsYyTPCjWlyisE4+UukFE3UYeWHqJPLJvpWNjjznmrzW5wk + qiyd//b2I/8mWIGdRs5BqzY1hbAGvC0qDGLYZ29GFs4J2s3Vxz2jc6IpZmIbw30v - oUHNGL3VfIz8nDIyrQdbv0z8IFhNbKtnfjqlM+c1+Uab+rVQTcH/3w== + PQKAnbv2N/7Heauqbt88TPY1G3dz4asI65gCD/7SmmtqhGqCKnZWBA== -----END RSA PRIVATE KEY----- @@ -10731,62 +11931,62 @@ vector: segmentSize: 131072 total: 10 - convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:MDMF:ncobf2b52ul7stjknlrgtksjnq:qp5dwo33smgcad246gki2csqzcsts6e4r67y6mrqtinnf2q4lydq + expected: URI:MDMF:gxq3orvuhm4mtwvw6y56cugcly:x77co4v4zj2oicd56pqyvqzd4kcdhuyatx6itovmdwyylcin4xxq format: kind: ssk params: format: mdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEpQIBAAKCAQEA2Ihx4l/I4u2ki8Ncsn0/JhHldSllOmU9BMtgDdozGcGzQ7kg + MIIEowIBAAKCAQEAvJZwIBA/tV4AONyhJyebcdroWZgshXLoo/BdnTDTi2Vj5S8K - bX3zgf2n2HL/Hn+DouPiGoF+3lL9xBnVasEatMeuHuGrJUeFSgVFzSnlkN2j6vm0 + AtGOObP7foNLz40mn4faSV1uSpFvAKKlNWhZoRUlkyh0hMIQS2WqHUkGFrJ/S8oA - n0cV7pk2G9yjaFGZHNppWYwiidUq/tq3b6PilOM+PVPAFOilXafAj2V84A46wkgX + HvoZFNaVtVtXGlDI88wUSo4dR1Kbm1480t5M62RyNos9FmG5W5n6RffhuJZBUPE2 - pNtZFR08r+Sytt2ZfMDjn1VeJggIKbssKPARLCrzhgEIVqBoXEL3isqf9JplbA4H + hAuMrk1wbJDgdz9+DI/n0KAi+kCPpktBjN0uTgiXH0xFJAImONLVkSFGUVdDSKAX - H+oXyrkNaOOWsGuq91kXDRDNAR8xeYwHKs0MOUOWGRkgisdv2EHG3c1dZXVldNok + IKCr2LPc4ZGF38gbDspg+NuQR3Zk7/I4IuGevZwXh+YMgpL/oAPdYZe622yb0lkP - der96cIwyRNPAJo2+DAdBdfYBqbE3innNHmQAQIDAQABAoIBABk32uZIeDb7Alzy + qmIIhpMw9iMs5bgJHiXKqwVl7DIYp3qiPirN3QIDAQABAoIBAB52B4W2T4QNEJN0 - BcvBNnzn51dmg3mc7NGwIKG8YJMxXKrDLCRUX5XWGu7khb3hj/fanoIDxD4ptZBZ + Ak0KbGTsi0vayj4tPFSBQQMLMvA4yp+R2dfGChIEofe3tL7BOYmFGQYb+EeadsfZ - 3zctrOnnuj8H0qI8MCksxV0IR3TrEKTUgYAYtqnb13zAjO92AWUQ3ZAmTVIhgikT + 0ujt16RZf73lnUR1pXTNkWJfkYMzBmAIhb0l1SKfh9jzov3A1LnvBHayaRpEcUFG - A/t76hmjqvitgQNuYkQEE4h1LhP8nUPFG5IJCxCdqRBK2pUw551+MvS/9GJs75KI + 7HjRqAem1tKRP6cQ1oQW5UJN8qtXcBlXugjlQGu9SoDKG0wNQ0e+47WlOqFribzs - o/26+hUU/pI299VOwn3qETJaFiwka4XJpP/sLaGAQsdB00naexzzU9M/t6GjFPCP + XVJK2Ei2jH+ICeDAzNY6KlwsOv+GtSxdCc6hSapVRFKPHf/A93iglSKcFziAg4lh - zG2Na9ppr7QU1XdOj2BsEDqFdUxtQyzHNE+UKKNa7OBQK4f9xiQqwaI2Ayiu8tpC + TVrhLwzDrfY5RSXInipYoP+cBO0Bo3LkzF+yo5nVkdBGlH4CTYpjsompgwltECWE - QQRVknkCgYEA/j5RUAfktQg/yp9YDIWRgC+RxnZlVVEuAwrpmEXMIGxnrJ6cA6fx + DnLLuaUCgYEA3Jf2n7AKjb5LGnA49Hg9lrN3kWGsmJv3AXmNPQCR+4752lcWKPi8 - 6IjEYOjfpDPVN1NAXelSulSatHyLj/Dnol5dkW1q1Br3e4MeW/7IKNnFVbF/UHZG + h8Vbeef35VjgrIfoxIGNf5YwAhZy5BHrz+IB6z5icQ7ehHnV/YsqnoDI6uwJy9lQ - IRBFtKFPTAIRVWULr4BDKCq+ttwr/Eyfe+S8apBCwgFF6iv3kN7zKcMCgYEA2gdt + QzoRMj8xlcXe043+ok33sMbYBDQTwWZ8XFdl8BCDzSwzxQvhz1F5/ZcCgYEA2ttf - sK2Z4Hp+ibkf3FMfcPCCCD5q3WVAzyOB0v9IFJzSmu/ibyET+dJ97ND8yWmz6DFF + Se7r4knHGKE+un0F3q6YpSFSxN504W2LO/gXpBDBA59BZubKqcCQSxY5QVgf7hYW - U+ybbV7BhYC/3tLo3gVTDs4x5OiRhX9+NEh3Uwtpm//JnINdgj5Pc8u9xxb3w5yH + Wi7JtkLIYBDOaNKRsJIWR9CtIxN8YQpR0XQxMcyYW7JclhGFpS7MTg3HKi2imfmF - HTy+qEKBVUfOh+/WkfU2o3fO0YJHIim39nT3PusCgYEAgcL6q1csAr2wGVGUleeC + 5yVZU43BbDg8dqnl4+oZpDjRuXCQW5csrO+GJqsCgYEAi6EY941LsNrR2SNNudje - KKOeymVZON9TFZh3OxG8qnvJuk/FnxQTorRTTobsxhjyZOdnvca9Q3606xN6A8BX + SyTAO2LTCCo42FMjRoi03sFqf3z+RuLjGyGePHTLYf23AR5qBPBoK2labAffo2OA - 6QYyyWvID3OoBnEYiKmULU1gq2kJat7C0lNE0HlYSJnxkN0exrc3D4QpjJj5Fi9h + my5YvpnXX+7khIBGJl3PlVK5WpIbxU+B0XvQ5LhBX6dG2ywXEI8/iELk+wwnsRR5 - YtGO3PC+MdiGf4trMpSoFRMCgYEAmYnxrSIT4wlgYwyDa1z+H0K/z55lE1Rit3yB + BU5A9QrPErC2+DQEM+FD0XMCgYAKY5V/ZfcOk9/+nFDk+2BW9MTMOeu66rBzrwaH - yF0OHbXyejnEdA4PSzb4hvUFj7FoiHNqJxfQvMyl66YneHt+khudyida66D8Gc8W + /zvoDt+Ks3mgT95Y9ooi9lgbcPp7C9NdzpDGtR7b6JBTy4Mc9aJXIGHHo3opBRtj - ySrfHRREYx9Wk2nPSBEpUpqAItwBzzdDz0sf2M481hmjUAeOS2sr9yI/+zqLbXuD + LPfU3FhzKeFZQlWsxK7wGZlVuDrawkyH727xF26SG41LOL9v9UHoWMYj3mML5f45 - mYP1OdECgYEAiHu1wNmw4FNfN5ZZDJoaH2sHpMsjzzitXXVM2tZcRaYlu2sz71uH + 61jb5wKBgDkaU/ngHe1yZHcp4xODgyS/d/GX1cxcsdjHvpX8wr8g/uqTReGEFblV - 07AtMk5Sdh8E4z2GFRoN3+q4wJamXg24Yy4UUZgwcupIqZ6hk6I5LNvmTUrj5oG3 + cJJz246P1u0ISESiLcSDJEaIPEXaV9x5bOHrUfotPrLR9K2q+gxpx7iYZwA8LSvr - 01D4Ati93EPwD5XBEuhLh+c1YqzEoz0opo/5kOzwR//vA0RHp99GBC4= + EuaRLe0uq7CAE6Emd10jQw7CZFkAnlObMXHLsxGpoU7iL2qHU3hb -----END RSA PRIVATE KEY----- @@ -10812,62 +12012,62 @@ vector: segmentSize: 131072 total: 255 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:SSK:rqaf2txzrv6oh4znpy63d4s2jq:ojyyneb4p75ywfya4xmlxilckvy35mh4his32bmnzuiybdz6o6ba + expected: URI:SSK:c65i4qdeg7vad3lj7aa2nimdua:wcjda3bhjw4zccgjeof5ddovqd4i6h5ab5giecvj3bwiuxlxmyrq format: kind: ssk params: format: sdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEoAIBAAKCAQEAyOV62tMTsblQDvezJSsNTcZ8hPHsiSB24NanhEPDXCYB4vjG + MIIEowIBAAKCAQEAz2I20p+CDDfJxxNVFjry9xBzepPa0mCsusjrd1KBfvCMx3oE - i6kiHsfvbyANObMHM9UMV/zDdWMbK3O316n/ok7wqc2IXZsoZLsyFP00e6IPtR76 + wudp9YJCClLGhwuATEr16z0endWApkktRRratS2RGn1bNQ0cuuzjLnuTsT7sXhwy - kwCFU4koMnLGEUOua9MQ5AJCDG5aYwDXzqL9KdPxCHliBMtRtwoW+dH/je+HRSpG + GXPzjrUiYMHbIuef2Lrby2kuKTP7UCqpQ1VHS2rqaQ+iwIvah7VqI2haVewh7/vX - kWbji8vB2mQkTekJ+6ayTiKLyd3kd1sR/d/61kKnITMVTgcG2wQQb7o5rKV2XFeh + 3CR7BhMCFKAmTpNS4+lKa2UPbpXvRJ3kwjfkpoSYMvrRNEHORDUnQ627X1St5F7p - IUX689Dh3NlDj68Ps6OIAQvcYjiL//bbjyTOHN0XVJzTQGTnPDF4rYnxOplCGoFZ + Gytfsm5do+RpcI6DYDPLjb3BDNR3dwgJytxX2udsHIqsUu/wKLK6Y1AL3201C04V - 67ocrsmeZyEO7e+JFaftiFkCI33ixnuz57N4gQIDAQABAoIBABvxCEfZi3KGHD5L + FMlrhkeRFc6XfPzaRocL/y2jCL0Iq7b82FikAQIDAQABAoIBAANzzBvXge+4IgjJ - XFV4lJvzKzbQcSjXZOwRCZvp/YjaWRvTpHB2AI1jGzMLtFszQnaxP0tuQqntNAZf + Xpo/IvpP8Mwyl+r2pwl4/MqAuh0l3gIYGuovtgjbQUQwupNW8qzSdqOS4eaYvkqG - TLwdubTo9tcfLkQarvZnr5Jfb0ZyJsN3FhHjxNIhZapdRUgkb4+GpoxCoM4a0SPW + X1WOK+PmPUsmmRUaAcdu2B1W+09xD7hFWeQolP5kL41SzNYY6wVNnwkU2CrhbaaY - UQMSr1R7Km9BIWLE+/i8hK/lcdOwiboUmtAdQsS/gJ8TamhO1UL17hf9r5rSUTsQ + /zZavhQYVIypMPlpmpq1v22VeCOEAiSiKicn8LvrDuB7HsZSV0mlHMq0Bjgmi3tQ - u7BHTYfC+PgR19grdU0ARLdpYmAY9fLktFNJrejBRncLr71292KSoEpYQNGOjcA3 + P8CCofImp8OnuK1+Zpm5hyPTKvsZJ1j3D2TPOT9R8bjBy8NA4rFpZBOfrQd3dX4Z - 3NnDiFsuYIPDLceeybQRV+N/1noU4q7i/KwnwmX0IuGSuSUULnvga2Ug+UepvgvH + 7qWFHfn5PprIZEaRmEgido4dn8k8wRRDR8IFoxZcDtv2PjlSMVjaiPt2Tse+l3tn - HcDTjMMCgYEA3uCtXIsqTYIHGmvKYdMIL5kCGnCMW/glh257KGSmpNlE087eYTON + PxlYpUECgYEA5F5hvzh5rGFjgwwhNnP0NFRE3N6/Bs/b+K/2EwxUqQvD91XtpeM+ - fgY0zzkhimxj5DKF8Snr0x3lapSp6H5CFeqrm5UkzNLVgGRM+BGqiHlrFSTHOanZ + aiLKqAhY9X66PzPgZifDDKwzybGbdlyEYXS+dl6qwTvPusPoEIGs+gQW3BGffMai - JOZsX3B4LxT8vK+4gRQnCpEhJ57noEYgj74HYPPgjfpuGsVtjJA2Cs8CgYEA5sCI + 25jlGZaoklr2ZOKaC1ANsXsGmDHWuIcq/IzLTRedrouOqZkaJM6kyjkCgYEA6HnU - N3oP4yehCCQHcRG1AqV4e+65iKlSbpFWF4ucwMVAf2NiP2QSj7tkArDcpaRJIkbN + ktCyk4ughuJ705e4UXuaeGLsrj1nOBSZ9asA7QxqTFFSi/DKHXjoxNbUd3FZVJ2g - od4kGbJXF1DP7I3Fa3ezB7AvfND6fEJLNRB3/5o0GB4L4ewO8KWMhI0iLfCMbitJ + c36knYO4VsYMwgaDORSAezF6j2MW0INkULQBQl03M9HmeqT4LFMW6Sto2NERIvxF - 8DgKCtEudTB/HoQCdPKbnze8wlmcWIZLrYUN268CgYADT/eDnpXcXQhZ/iwd1BMV + R6T2mPWh1IP5PDeoaVRkuuqLcJrk36kVzc8ZyAkCgYAKoMHXzl8LQLUK4kOhbyAM - EgMT/YQ4gbGdF6lA6m4HmSsKstJfQ3Lg4pq6UbEL65x4cb/H28Wjd5hHQzpbODUn + V2elB9DIFmBcYIQJOuetvlhuaFdZAwxikB/yVgEd27n7OwTUfEE9k74NQvDDP2cB - OjuerlLDsIZ3yAXU0f5k1NkgkVFcrAeMItiNepBusrMm4r2tPW1vHMUPX681lJU7 + yhcbFyjHOWtfe8KPEhnkwM/3ifJsMipeIe13lWVe+lDBPTKCGEWq3tjduGQPzmqX - TamyaS13LregMjr0kdgbxQJ/ZsFV515ztLPxAa8JoVBBSuxkusuT00eTbalKrTF+ + uk2z1seF2gTXq8JluCA/MQKBgCWTc6GbbBHfMr46o2srDdbV1Lz6uGjdce6lndEQ - nFk6X3/iQFhP67GG16vqldiSuLDO4UYKzWadYcSa0rxPLYwgLUxH2U02Ph9HXln3 + p+Co7hGR33bRH8otven7E3KO9rJvm/yvDqqLHOOhtXQzG0jBoJbJA5djm89uPWux - FduVVygKIpD8Fi2iZWRz1AFKh8S/KDnMPwTnq0ftU6l1bp6arkwjwmglN0aWbK7T + /LYeXQraNZfDTH3VnFFp+9N3z35JKmWPK3DD2zl/b2ylTmpgArwXpxw3XSEtsmGG - fQKBgEfZQ5/jSejcKXJnZNp1otPJi8eB8vTtXShrcpxFaFF+ua+ddjWbZ+5KbfUX + xRMJAoGBAL16UQxn5k0B7R/bzawd5DzYX96m51DOi0MoZPEYXaOBCMY2k5wsvt7F - fCRMDxiRVwFROTDdH6jx8JPzHkL7RoKifUMhrMP94Y0m9sVUdvoEdU1W+eENAYif + LsOBVUZIz8Q0vAtYxC2T6uZRj9Tgg0ufjUq3uc/KJGvY3ZBBC7o3GBdB25jLzF2f - aFP+pZQChMarh2EzqWJbQ4RP6wVq4jqr2QUGVOIpXON0fjz2 + gmLCud9YOcaZpKzcdDx3eHRDe/cWUoN9d7F0A/b4JMib0dV0Pgr9 -----END RSA PRIVATE KEY----- @@ -10881,62 +12081,62 @@ vector: segmentSize: 131072 total: 255 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:MDMF:uk26bsxso2rrhxsgampqgm43wu:e6ndy2c7jfgkd4dcypcrisdojwfvptx57x7rs3svwkpbf4owcldq + expected: URI:MDMF:jswgkqtxgcqw6xrjivthxqi5aa:6a62r4e27eqtgi433tyjd5ijounpuelqd2rgnsz5vascudnp33ka format: kind: ssk params: format: mdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEowIBAAKCAQEAnwlavQCw5YG48BGsuphgKXVttgWHX14DwIQMplv1x5SAq074 + MIIEowIBAAKCAQEAuGqzbMA9HFkXV2SDqVlTEYR/Q9+wvjzjekq/SzLK/A8tMWb9 - CqCwL5mj0RzltwDBaUeKRy/lOWMtgJZMUQQekacTRgl4qbwEqx7L39lKy6H6LZpK + HAVggYq6vNEqowspiVVIjspkXCPAxE00vlKQOpikcdKM4euG8hi259oFgBQFx2ER - 18tt3XArOFy4oqqzDIEmjyE87wJD3mCJjQ/NfMIC8LzcDKo5zb3tDRaalQUtuf00 + 2HmEljXgRmkf2VuYWdoUA2ApMQiuoXRM76WXYA6rzhWp4PFdmEiv2zFOzOSWwM1J - csyNhQrBNkF79L1TFGB/iSN/uGlRUk2x1PsRowOvQgFqsazswtARwrl7KyOE7w5y + MnyzaN89Q+bgDjl9He5MArsxsgVOhMTNDWHmvRY5rSWzoPv+9Bs7Um6uO4nX30wT - 58l+ooVYb2KjwL8KewaUdn/tfJHMzWgQcYjT7aqxaeqJVoj3XMCif8fK+UACY8hL + kp6adeYfhndTz4WN0vimYhyIWThXrgaeTIzDRTzAObG8V8RW2l2z7g333PNYVFUL - MHVj/op0SbGvT9dqbXWoYrsmNv15ivXofa0zfwIDAQABAoIBABkj6WwnR8+ACjQp + Cdjzo30/b6otmHgQVfb6a3icMLisCG6J4UbWxQIDAQABAoIBAArTrC4uY21qjJct - Fx0IKWtkXMuBZDz3J7CvLzC9KMU7/HsYKK3FaRSdPQA5iTa8r9ZssLdAIwRHYVIK + +JiEvOlVHvdK5sXlw2bOrx2OK+0cQ1INwr6LppoMXUC7GKFNr+CscAVUEWF4cYza - cFX+SLbNqoZPyPtL9ZD3dVMVjnVSTbIXye0DA4MV0D9AqQ9N3LAFWosVvgQqX4Av + HzEs8ziWyrwU+VOaf90Zltlp9ciKbw7AyUBX3VvH6h9wH2Aj2MADsIjvMxPkzNh/ - 0o6yCNHH+Z8Eu+RkpG6Zr3d6M0WLJDXkcjljCxXWHpobxCJzWCqmQnWppXXDrlKr + 6wlnGR9DqpH/jcOTSrmnS3hsqWdCyxdm/S+9yMYJkBzgzDFBkh7ze9nPL8lpiQpa - az10kdD0y5YxU2NspahEymW80Jjbrw6heyKLYoVIpzHtMmRtxhd0jydqdstFT0Jc + wFDhfPJOww51nFfksQiVBlMjh7H0XmC7NW/KoZBGGsp4nNmIJVL+d5cehZ6GB4B/ - ZiSfM4k0Sr0FLPcxJ2EbB36QmisZW94eOh/er4WxffubJnjRSRCKshWANwCOpr9F + GIhVz8OAqRtLNU6GU3e88dkLZM6sqoJrw6KOQxMR27EydgZP616gOMEYaJmE5Ghb - r204R4ECgYEAz3BAlPM+8K+vBsESIdmcxjw2ECYPySSCaQIQiX179mT/c1RnT0xL + dNx22rkCgYEA6lGKGvs5ljaGP58jtENG3hXk9DoQ9peVvYPb/BlRzEFIY9PdNamp - OgKk0zGNhPyVbQCa5viK+gc3QOFxu/py7PHtTO88GVHS1JOJGl5l5JCyzVCzhE3f + GGpDg9STFq3DWL5/o9i8ltG31NZbiZSG2fBnpV6oUbUYTg0PqBMTzIwKSOt6PaAt - Ml1AB7wA3x8FS4fuicsVFFEJixGccjIOthGBdhg6x5uTrLMbsWVZNBkCgYEAxERi + mDzU2MpCR6HwLBi1uvM60CQDB+gQg73xPp9d4gvzLgrd8p4iaTYgEkkCgYEAyXsb - FMcunTUB3tNr4vJq0IBu563lxgL4y7CqVJnsPuLrNrbrg3UyB6IPgLqBzCH+xm2/ + G/sgl6OUUEo1UwMAuUoIUyVL09sqxDzMq+ru0dAeOuavOBCjX9Bk1UtKH6nwDxv9 - BJ6gPwFtzvDfNG+CnRwcFNK9csPZZY2ZKqIgjxTRcWjiKhcj+SFxw3K5dfESzsqZ + vpkdgw6hsV/hBJmY6GIVhR0QBIPNIgo6Y0kzq/pVrC0n0E6ht8ZMf0y4kFbFPQb6 - szW26V4CgB+Kzxs4WWEyzBV3KjOv/rXwY03oV1cCgYEAoqT37hG+4sZM7HXLOsE9 + 8eATpd5ORQKvk+D5Iwndy5kaKZUhLzi0UYK+oJ0CgYBTyA3ycct4a0x7KSKyDLAl - ++xP01+UdvhqS90zjCnYXTuZUxr1maZPQV+7TmAG/yNwIbQcwEZV6W5o8zUQkPvw + Lnzr2mtAUJkI50HcFQ2LU/hXQWTCEETW5v/2/iYNoNnNPGgVJKTh5GCvqGmYetPw - yjlx/yWAsLWIIea/0+355DlUCEljR7Qq8XlN8AKHiGnxI+SjsmSJ1ZEoc2LOkHcR + zyWwGnViqbbkCYWEmjWlGJmA0zmlGUXUPkP4s/EY/c0LZ1ZrXxazX3z58b8d6+d7 - M84L/MVIqSMhqYIRj4jQZVkCgYAOpbfYKyFMdDdGhOrJTiQwmVUtjynVxEUDFpUv + da4y6gTsfJQ5cNNq/SBgiQKBgQC462ooNkblplcbkeB7PghOB2q4lUSRP1hzH7Ji - qSkbbF33gGFFN0rbjPmxNroXHPZhorEdzCTTbuzeA9X0mNnblcx2tV+UIA+qZ43l + H4/ttevo94zeEjdAW04QjbeMdDZGR9SOOI3jmWxCFdO6mxbCQjOqJtBqtGVz1ptc - w6HAa+JRn205jO6PWjKeToKOzcYEjtQ3rquO8QgovbHjUPm8medrmbKCAMeCr9tX + QPVR5ML48cDW6TR4LWJMfCfxIhKJPnzXvmeKFw0TPbHUMem3hPiyQuTGQX9hjdPB - 3emYEwKBgCrSOr62QCLvyYbry3CpVzF4/pnL2aDIZzw02hqbzwxR3KfDOon618O4 + 9BPt6QKBgG3t1vLgr3+pHCL5JCKyS62AP+n7Fr6RjLnpzFPHLW1MtJoaGeFlEaP6 - au1LFRWwjecTDDH9ZWB//tMp14KHBtDob02QGKABcG9x/IbmcNupO3Y510aQtz3W + zVuwxVY54BIcxMLa3lZiuQBxUiU6cM84wvs4A2bHjUmjFnL1FyUY7Y7Wr/gUUyB/ - ZBpV+hujw6tWBj8PfwOX54nYe6KQ4sLpiAcQhVR2Ph6rDkTwQkBk + eBKjH7ED1RMIqzY9ewqF6RyxsLnahfSW5BDfM4XesQCYgOSs/f/3 -----END RSA PRIVATE KEY----- @@ -10962,62 +12162,62 @@ vector: segmentSize: 131072 total: 255 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:SSK:s5llvgygyune34aqpulnw37mo4:nap3zaqvcvwif7tjgxnsftnz3qxtv6bwlegkpzt2kmdo4htwqskq + expected: URI:SSK:zbygwa7b7p25n3zxovsemlz7s4:5eklpk5ywpbaqzmobixzkurqsccvzrsobgveqgvfa4mnslut2oca format: kind: ssk params: format: sdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEowIBAAKCAQEA4yNOr0yItq8mSCiYSrl9srFX+CWr2DFp9vf998X4qByc8Y4E + MIIEowIBAAKCAQEAnDp7uPJokCdVpnjWHvYheSiWbRyu7eLxgIyLBNB2MpvIeYGp - vSQgWhLAKtvbtvsmoGLUrfSdqgm/Zm+ro9r6kiKnu8QX2aOp/byybxOP03n1Hg9J + 8niIIfv5LlsYmmw8pZmAwN8O6IV0AMZvT3XENWhCWRfayIOzTx14NDyR5hGoe+Q+ - aVdR9TmerZNR4JEqvwICZBjnbjtOcBs+0EU4SFhRZI2hA+Gl7xEeRYtHBunapQSn + pKVESHfmNXsH8AqhR2qY2SCeYKOlNGRBUqj6DqAGDiI92UR8FWg9xZBCSN8EdqFQ - 0JQcyi/pkCZ3hAsoMVw0A/Kh5B5a49zJ1Az8OAzuXBbhQUFieVcMyyTiBMeQJTJ2 + RSKNM1AEoMa3OC2rJdz/BOlplbQrf0gFUBBDW1VzhDSWz4TkwhKtMkD256RG61q6 - /7uaw8psaSQfcxnFVjFFh7RwybbrQF3wiBs8FkndLOvMHgQRJqNXjftSdmnuOq6U + PxWMkY8/dI5jbsWoTGRjV9HFBZDx3RvQhBvVFZcS9TPjjHS4wQd+3+8s6SASdBu9 - EATRXcwnTdoI/WUWVeYub9kDfvBcZXSNP4auYwIDAQABAoIBAA9wUSMPM7FSWtEt + PtqRvUtbwWwQVBYBr1PlJWuO5eYnfznC3eDNIQIDAQABAoIBABZ8kHxRX92H29sn - /5pmQ0n7GryhauEC2TrSBNXSA6KbPa7mW7uzeFN6N/+k+nYHwJdhDSdWm8TfOfHi + P2KbeLvwrJ4t57vT04D2ObhKrQihxZw/no+I68dAdmBGumbXt276hj68nG+5bbYC - uaryJNsvNqSuCBgt/+b6uLIq65+Q1Mo0EpklBOTpnR4boZJ/EewlXQSkWk64aNt4 + 2ditEb3CMPKT7Wi5FCEKE9go87MSzZZvhti59PcdUuRVvhG95fLak/+Eo2czhr7g - rp6F2Sgc5xaYaqjF4XtMRMXhv3RghHfPyrq939MWKRSwr419IDlyUDR3ZQCh1Qm1 + w0o7iip6vARix1Yzdky6gVGjbvb/8Qm2W3TdVfxnhZq33SkSHQkN2VbMihtkqqBr - mjvBUR0eQ/eUeIibu5DKfCi/NL3pI29ECFdCKjbS4aMCyuXUBIXAlngTe9SjLfcT + UXAOf74Tz5xKTiRRHZCMP2NU8PfDH15CMyAnu2dTl471haepUYzq2nN5HnYxNTwl - Z7AoLpYC1hoV+965R+jn18tf8q7WSfzqiLolh00iZTK9mEhTHq2IaFiQwKeh80Tm + CJn7AI+R8assOiODwCIkwPWIylmk9BsWJfVwS7JvNBf2T0frXc8VKrI85BDzZuq4 - OR8KkwECgYEA+SkEPYYggcnkGnro2fdGX8EU2GfAfCiVGrtfQ5d/ZGU+N6qErfZ/ + 06XGITECgYEAvvKJ6tO/2ggRvdmqoKBl312g9dH751nOEUbezEA/t+WTvEALfmnZ - evKGoGjLQ5A9CurNsY4XKr0t8p4aw3Jt1X+0YbNBVFeAFgmWB0wFx8C7x2cY7i73 + LsFjTOLvlLFRo0R/Wt/Pb0ZY1qT4iA9wsA314t4I+qu76+PoCv9DZ2VRXvt1504h - ptkOnrbP6iHxGcaJFdzzslLpFES9g/KNarW9eva3K/sB2MBFcxlg54ECgYEA6V+H + /LsgJX616FhFtux/sEOvdIx6n3VT//5yshFhwo/4aAYGIj60y1kerw0CgYEA0XPu - N5WTDeURfdei3H21Fmb1/aCEfv3LqEt2pJUajFy41g6z4ckO7Oj6sE9x433M6VC4 + 352ZqPHUu8sq9q3yBWSqXxpKAA0XMieIPxOPviqtBHkqMj5W9bTdS/eYsjmvgSza - 2+uhesL6AiBjKkqAMc1YEBO8pFRjKgAGxbuUbdRCvLNk8YBd26HPTMONlW6oBZDu + Tb/dV/dalqkSwpAFA11HCDcI3ejV0252KO2HyNUIF2+bzyVU1W2FyFCz9yCLY/gC - ZqfZ/pqEyJsYAFcdYTlAKrARDGA7WwMVIAe05+MCgYB10oaVzWpr1ZvPRdX81Kjr + 7ruMWTqde7PVpg2KMBi/2ltGhwgODSxVyidFcWUCgYAw+XIoOaFGYbVzNSXPRvR6 - uPNxjkaAr/QqavaWkPqF8DZmvnT1ir4n1q4BBu0v6vJiyjwwvV+JL2Kd+1Punpr/ + AsCq6+2pG95/jebNClmNaCOpL+ACz1E17cHzUW1TfNtMfeAQRcElcCyO+QcJlrQ3 - vd7/4HOBPcttIGVY2ANXvXVOyxsH7x/fP39hYFObhSdtJ+xFcXGwHvLnScZQsg9b + Y41CX+J7sJplWTIFyAzYsyLYsrQ93EtZUAFhvIsZibJvxV7GrcWNpg45YdVmnjN6 - qcuLbUWbP5xU8j8lOZgQgQKBgQDQ/OQ5KbBcFBO67x2AeO5vFlsZ+uJMWvlDR/kC + unyRc22p+ImQNPcYBMaa7QKBgQCMd+PLtDZJR2YUS70UkrOtSkW4Yjker6jOyhRl - YCg7JFm+D8KU4pmEHQtKUoq535FeKxSwlO2x4uNCfkBvwfHVJ3/CPfqD6rI3DXkD + uQi90IEYbuoNqCFJx3JicDrHzEgXqaz+V55qUElAoUMjmNLD3tq0d7RKnsxIb9xu - H/1G8XumQryV7I+gvOHIa6Lh/AtpmKV1tsDoWPWqNAGlZF4CD+PflnZd79uXoEYN + tl5KIhS7Iu6rja3HNRxzqywGoJza/ol48e6+KMFVJNYz9wCmIPMJzg0OoihKTWF7 - vfkKYwKBgEX62oKSPcVOnPnbRe1MjANJ6jvTy0SeUQ+WNhaNJtYUJ4HcQrxVOB3v + obrAwQKBgFbXt9Cww42MReP5aOzcn5udgNQ94YtGuWpfn7xtxi5pD2BjKjH66OcN - WuH5X8m6gVvLiGDfPDSOPBovEZGAFqWGofN5MCgbjkbjW3/Et4d8OKgV0/88+/r7 + 2iUReVkhFsjKo1G7QVG4p/QhK51wxtFlZiuDU7AW6gzDSMGjeZQqB0HEtFoPdaPK - 69J+hC4lXWC1vWa65Fi4FpmSfrQniw43acPJbAQ0Q3XzXpS0XESX + c86Pmhd09zlzQYCIaA9tZ8vbUBZ0jHq0rBUPuPg+kXVP7D/px0aS -----END RSA PRIVATE KEY----- @@ -11031,62 +12231,62 @@ vector: segmentSize: 131072 total: 255 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:MDMF:fsywl3mmaxoprbn7uogusakox4:6qfednkpeowa6ikzu47rwoztwxbyhc4ofccxrtbdtdsyhpt2veja + expected: URI:MDMF:oojuwcpqotfqvmyuhc245awdpe:vzxamqps66hyemt4zabbos7cufrc5ugi3ii54vsfqeqsmjc5ezjq format: kind: ssk params: format: mdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEogIBAAKCAQEAqgojWWXz9+4FGPFmfCSbDcKXDLQrVfkbafr1MZPV0NpncKgE + MIIEowIBAAKCAQEAsO5RooSEJraTk5Cjb4wZd8vjOixmAQwBXC54S0di42ixXyhH - bJaEYKqSvPo6pjLm2OEm6dJ6R6ekJuT5rUp7LYY/DSg78m4VJ0tujVEbrrDG5BSk + xHCLhjK1TuCiC6GaHdYLXAc8YW7RstLhxdQ1NGhCljxesSedcXqvkcfKKyxWhe63 - dNGNaBTgM4/40VUwzWJwbLr4gJqdCrgI6zYk1qptGXozqRwTY5yZJkWM9A87OkrL + 06gCAB8y3hWTEgeIl4ZBlTo+xx1phtpoC2ujGsIgnrwLcwJO+FSIRUiLjCfC5OI6 - Sbzj5HoBjn3J+LbjzmzFefVC7Cb7XZ6lNokvoNxxTum5CzDZXkc7LHIYrw5mNdZ5 + NACIlx9sKopj72B3N+8TCmqUji821/+jmoiqMT5hRsjmjal1KNqeOczC62ieHD+R - 2OScTrA29/SXYBQvTrRArfVXEOHUEmRtgZwYRcPComO9YUzLAXrLTRQE6at2oZcM + 54vi7EkFjZGRHVpRgwVyEvRhy7y+RJW6ezfkUKW+JhLZo4UfAX9AMPqwYS0ZPYQz - mNGLRmmr9RuT9yAalF8nDciGv/dlu/P1Gp5N6QIDAQABAoIBAAJu7TUTDRP4wcI3 + 8++Pefs9ULQF33UYcekyV5H+1Cdf1NUwEV1s5QIDAQABAoIBABtAOZacbnY//K7n - q1jU8qwWfNaUQHeCCyPWRDDlFb75br5q64J85lP2aCDdfHF5UoDhh47g8aa5wSyP + wiR2IZ4P6ymUmQlkPflituhxUEvSXi9X1uXsp7C9sqs5cfv0ofYid5FvE9+139p5 - 6i74g4oLlpzmI1kTieioGKLbxlRJ/tQ8o5YK+djp5yoNu7v0Wf1I1P67vG3yTuBP + HIkJzEAMJuVY2wTSIy/NQ6liakMICzOJtwqEf/pg08bc79ABFQqxhPxlAjJM12oL - pGVuft+Pv+QIseZDLgraCGRtQ97DnNl5egTnTtVS1hNu3GqjDSfTK3qR1LWanqYy + zaakp3SBneCU+fZ2zo71BiAVslidrUWUPs4pc722tln9ZHVchXp8RylDk577GoRh - ypWbWgUKygcFQv20mRTNotYmjRWIBaWdzSrvIA8pAUw5U6R+FZ2xfhvpL9KuUvIF + paYAIOAWdaPsSmLTtju44Enpb0wmHF9xt+rQyDvNgyRbsDXGg3laU1V7ySCfvKjL - 48WA00WpCO88az7RKJL+lUuC44qibduLBc4YEY96TCjKo1prN7nOGR2RR3qd+/vC + P3P1GpJmfT2hIMq1uBQWn3adRUvsxlfJATWLEJH8uIgGnRj2rqI8QSJGKdrU9Eck - gJOZPyECgYEAwoSBJayvhAGIU5+KIggwSXvm5VqlZSMoTtRC8WZghZKxmgX47FCR + 3UzsNmkCgYEA6zqoHJdD527EHVtB2NHgQgGG6mLVv1pE/+w8SNUQMYc87ZQyLZVM - nj8VbVbVWG+lg8fVGXW6SCys4//NfbxvMP3IIN1tngb4klY6TIs3XIB7PfE0HtZt + 7PZkOrZGOlL6Ns/zQ3j0OP3ZhjsXoA30gFLVMIby0jmVmjSV+i/CwOb5IvvT6vV8 - TCoI5mEwCWX3VSRPTqaLBS2y8Ad/4bNMyPgnQX1eQY3wDSBfI4CNQvkCgYEA38j7 + n01yQ6rGGbUPYqoMffS1SxkoJMZA5pIe887IVuwjwWcmCfXgKIhpLuMCgYEAwI3V - zlFkhat8ZndJoFoJEYFX1saw+rI8OgXcKg/b5jtsJ/T+R/DLe26YYxFhnglOcKy/ + kpHTJzCC+MPtiN1CqMkAyzvtk3ON3D2tSgqpppAnWVzRTMzJqVVu7qClzJM4xp5D - LFm//z6rnky5ZCBF2WdYFMksxn20IXGwsDBiv2Cxn/SieWxQRfR1g45yC9TWNm7w + sxqgt3Wrh5flNPD/BS7S9WqNFbkwFhmjh7UzijDJYBHJs0l8oo4XjFgHWIGOt7p9 - +wuErTofXXHDyFwLnoaGV1hgfEfcvHW5mr/TLnECgYADdG8GyEZlxdEyCwddC2Aw + hNUR1nwusFYgAlmQ/gN62ycS1V4Fd3J7YkjAN5cCgYEArldKE+604EnDRrLFSeq7 - Le8v66g8X597pvF5cCQOu0hEQA7nw5aShPRQeNZZN8Js0MPMK/cfCQwZEJYJwasH + mJBDK2LXYzyHWVsAj0aC+wJt0PP+gLRgUFyJis5fnIi1dHyJot95uufCGe+gIftV - 57oCO2yS/fS0RKvMaDyXfAC0XPBcC9rtG2IFFXzQ7eqyrG5sKzEU6nbfJIL080ZN + 1OoPoijSvab6T1FcOxK8+HX3/srAlSsfE362Cpr+ujzsy0aXfZ8p4yAhFahun7V7 - 23p3A08FQwwcb5LBAqt/oQKBgBEeGYzFkw/adzCLTVlzqZ/qKeLm3eC/Q3YYvqeF + BV6kM3BPS9+kXMuEOOZpdtsCgYBplhzz91TDG53mDYIWV8Xyye3Og/kdrvKuP3/j - AQgSYYqI5e5wz8/IPOXPDY1+Hr7lp9Xno5UNoSkBq2iqQ02G5yjn3oHsWZv5S0+e + pv8qX8fD/9qhc3ZtjXR8E8l26Y/rkeNrtgFFOJgrjUZZhoFA9VEm1BuzOs48gCil - 097ZsZyPpOHu1BEVyuteOQEIrb4KLGq3jdWGTaHjMtufls/wcFQ8EV1QTeUoiCL7 + BN7TOzdhn79rubNHbAVLpwW4Kar54qBrk27pn0T22vNIdpbEQ2I+BCoWqCz1N+ii - K1cxAoGAen+lV70vCiC98q/oBwt5SAvuuX0C7fWwjA94qRjvIopt6aJlBj3cKb2z + l2QX1wKBgD0xAfQZ9D+irtHqR4tCSCznbqmGqCzxA/g3PJeCDf2cNvdtO713i/51 - FizF02dt7qrswg/uEiQb26BdUz+xctulB8kC5ZfLD6EssUItZjnvYT7IqvHHzJdw + SHdZ2RZI7WitU6ovMEyAXP22tFXa0GL9UHEYszooVegMx5/xR+/z0fgWQQ+IQuAD - P/GIEwZ36zKwM4G4oCb/JgIykAHelSgd62FxJU0ACScXPPx1CZU= + xXlJCybYPBqfhAsEqoDmQp/IrCllKC92U4oM45kDEfHxssZ5aXvH -----END RSA PRIVATE KEY----- @@ -11112,62 +12312,62 @@ vector: segmentSize: 131072 total: 255 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:SSK:6dul24kri32nxlqpfkq6wpfcsu:ykq5hxiipdpthhz6ceqybs2itiyjoeoigloze3uvplzxnkgviwxa + expected: URI:SSK:lrrasvfrww4coivsbcorx4jtxq:phdrpge3wvi4fiabhbo73jcrponuzqz2js4ow5vd653okjkilp2q format: kind: ssk params: format: sdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEowIBAAKCAQEAslPNDO8333eMl0cGe55QqUzk8Pjr1shYREPtsWzTQ+viJVrr + MIIEogIBAAKCAQEAlWJVkT5XFuo8cXgSi25HvT9nFeNuNilLT0u0dZq7uKrANJlG - 4sbf8sZfYfZM5i4wogHLRAEdAi+U/kX2iX2Uvowg43FMOE3OKWLUYHGLQSCChhap + aNDF7WU4xWXHdiCod/qiz6o1kjyaBiSp0Epm2xJCJvBrNlEhRp2e5NsAMGZhuHeq - hNkkvat15rC2O7YoPff1xHR1QfSNZDDeYUcfDNBZeg3KUJNSU2jTQYtlivA8YAj2 + 6Lm9QyRE9qp0NtO9KBWyfh90r29MeQ8jZlnSNHfGE7FHYrCTtBr+dvT2WoovzbI2 - u60f1T4hCn7Xn0TKBUlNmzluewyM2Eh9H7NKqqeRtwGC0IAOORvLA2ymbtZDy/lw + ZhM/63HVX+PMMIVYQWTqugBtznDzUNP0JM0rpkCn+sMbSwEIafRf48LG92bHsaD8 - RyGjBVIf3owjFFzB/+UeG79rf2mPPqRwRJdhS1vkJVyO7zKNa19dWIbyXsEmpSTf + yIrDeNA4JRbG9DlPZLDEcvciIbs2kJxMuIKjyLq1C/LU6PMykcWPr+tjBaH4p0Op - VpQVxQ8qDjsMHyVVq92jkhP05AenMtDBtCn5nwIDAQABAoIBAEmAYfmNVi29BE1M + yUeB0OXugqACzMMx4ZgpKpHWYPT6gwgqjJS7WwIDAQABAoIBADn78efPUWGxKSYI - IJ9uSxflEk5Cg06liEAm8X9aeB+8R5uXBLgVubPC0Qi7MMoFStVTwPjYLqE6hIJj + K/aYJ1uDhUF/RpPaYoOUiKcPmSccjD++cRCVXQaBQFCK6anmEk6D6HeAA4xQXp7c - yvCzus3pSxsEFWL1qt6DFj9kPX4MDNCA4cFYkRy+YdvChXJKK/8Sx5GAYN5dErQz + 01do0dTeGPRhZNQwGKD+5KLDiSbATtUaCiqMkjTCGsy+LB7uAGHTTjXbguDn7ECf - sk1NN76b1+2HZpbciifIAp82+hUQLOWZCwjJ6TJ46Prt2KEfqbkL6/+Hq2dnBUJw + Sifb3JaGGUoiiWl6zdJemgBVpFxO6F8ueGWKywCHX6L/5VLMQQvAAQaxEMkLiEYL - 4GKT5ZFZSrcCK5Zr6NHrb2fjnCH5O8r14tF75tY3d4yY3EX/r4cC0dR1IGziZk+j + Rm+wcAWuK88XNETTglgevCmuZDb+0H9U+9Vbh0wR74+tb2XvBsxw9o0O88KlFaSN - oKpK3xv0zlrqd7mSe2I5vZLiugQisTLKcZUQtfiLT/lEtjj9HE0YeVZAWP78qB/4 + mrCrH40p+ShNumfE8QbHz8ch6CQTZpPLdC9x0Ziv1oUo6CGAOA1IGmz7rMyWjFu9 - mMJoMk0CgYEA02zrxFd7hHG3VdxvnvC9wzKYswroSricXKELa/xP6ChGG0z+nprC + 3xx/JH0CgYEAwoXBg2lz4YooGkw4yEdzXffJXaYxvj2BgJKUZyb/ZxRUHxHl0LYY - tImBbXhJ+nTQkHWZBZDkHUmbWUDcq+BxcQeBSRp5uzkkaxlXPy6boMKxFYaHJP1j + WZvPQ5UyOZDqmqvkfMYJRiN4rW6J/uPsCp2NYdXQVRngt3THqKeHii71+xRuBObB - 4mcHY7Q3UzP4LWN0NBft9fTJ5pX3kwWkLkQMy/ST5z9vGjbfaIrwfksCgYEA1+yB + qNNdfyT2HFuwbxELM/1a8r5QW2ebDU8eLlDbiYOEC6Fn7/foxM1uuHUCgYEAxJiQ - HNK3VqD60A17zMfUGb8s1qwzo1vCZo2VQy617wOtk8VzvmepB3bceEQNZnYcoydH + wMs1eu/dG95kQCiYX0EjgYmhNtQPhbom4+9t6oIXl28CtCmVhYDfJsI3/qVYqXIl - TMZkVmiENBvkieD2O9DG9FkLc/PjKkbq2FVpyqNYlevvrG0OEMLWOK/f/fLn+FZl + OqTOW6PLaA7uuojeQYmq9ehqldi5BHavs04DQ6nDncBIDqI4NlNG/EAWxQQghG1j - bnVj/O94oRVwfUSEk8Adsod7sQUoFIsVK2VnjX0CgYEAtLxv77A5TsdHSobehKiY + PDWUn6wZMBdGyAKCX/fiz/8mDBPnlTm2NdiGqo8CgYBgI0cesZGKGIP1a1Js+ZM1 - D724+5VfbkDSqfyhnvZZ+MQ06jGvmDYELAFAOyyRUSF7CYL+BNwPpVm/C1V/Tw7W + D++/jxHqme9VIhyiVo3H3i4tJOVWH4ktUGpBVo16EftA8k98s0uGFKXh4U3mYbMZ - 6yDXTH7tgTcgAs3u33wgXhUQ/K276csTD/+zOXBduyq6BVL3i3DJY3CXCB87PNud + FAD6J3hNdvqu8NJ0ske0rbz4mII/feSckcoVuqjAHzi1y9Cjo0W9zv1cD3p8O2wJ - tk9GATRbG1wGxgoSgXQEknECgYADj8IdcJhXlHYuolpNaWplNlMOA28inavaNzGk + LAE0l4E9VkpOOVIbYgSkuQKBgAZvO+WjgQeaDDGaUMusyHftqNzXhVhHDo8A7b7u - FwwnMh9V1abwGBOgrOQ8E5tI+l/EjSxO5uLWzgiIN4GQiKZnHC178FARDI/NrbfH + Gjnfsif5sSv2ZHdvJV6eYrjJ7qH1I3TM6hgjv0eTnYqraiLY/6h2x+5JnpyfydZj - 87i3//PBHVApvu7BdgVEkBoYvT34SayIouUQUf7iYVEmr8+kBEI5JKT0qYoctKKX + ikXPq6BhJ7qa4p4ckak168jc/rd24RWaZ1fmiRiC2oU3V88OTPUj07n3eM/wiJ8w - wadwnQKBgB+4kUNjyfhbdVY31Rugb2ap3qNDDp6weMAFkv/LTRP7FpBw6qgDHXcC + jA/PAoGAVPnLKxls+vDRSdc0aWW8Ql2yuiw/zU/GhncTfwfiKIrU4sF5GW//n6J5 - 4YjudF+rdaeK9rW53Fs/3DLDP/fp9OcmUwa1zS+Cprz1upq1gPTqe3Q1EYJz4cWT + w9aw95QBMmBRPS2dSyF0iqvK5Ny+EXUzVCAVi3s45o/XlFz/RxkrljibI+5eOBGS - 0pbLs8DGeWMCWN+gsYFw1JhFOpwZGXbwhv9POi7mxhW7WDxLS5H4 + lpe0EC6J/AhY2/6/5213nDPBoJKrqD6kCq3cA0V4ylvIaXoSjAc= -----END RSA PRIVATE KEY----- @@ -11181,62 +12381,62 @@ vector: segmentSize: 131072 total: 255 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:MDMF:muvykqatscebumrd66c2doacmu:evblp5vn4guj7dcnevua5qxeounppfqrorxe7oicure6d3bl3bba + expected: URI:MDMF:g4lr4g47o56kqvgsyikfj6ksoq:qs67p2rpyurqsuq4ivtebxyxyxi3i3qqxvvfzlhyk6nuumu3nzca format: kind: ssk params: format: mdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEpQIBAAKCAQEAwbNQmkA45byGaXBRjf9jElUoMF4hqAaDy3QrD5IyKu9AooUq + MIIEowIBAAKCAQEAyocKsAG2eamQBV0FKLNCo04WL512g7j/7OeCwMofs5xmj+3C - /Sxnj0l7NRYrhVH5K1Y2J0cmqINQRiOIQAjxzcb1JYbn3MRJJ3KtI02YI11BFd6C + 2StysnM5fjeS+zGHd7VOanAx8XbtNwwh5Mz/h0c0wi8zKhro9900FyVQklsa/Vnm - G2ayuykXFsseQ1M05fBK3TtSDLMdblT6EX5FAx6OgieA/e5zOVR7ZsSwRTH/j1sU + ckOt4km7N6BAaW/6qscFcf3xMM9rzbvEv2VvQsY2WTn0PbdgNHxBQKqmPyeHhfqi - cvG/41u2LFbcT/SbuuBf8eBeZzjmVaACGmQ701o1dQJraJjpgQ9QbA4NUlJcXRou + O68IItuJ9Atnb5nJYm09+FAjAAIOueg8jxCoqJCvPD/LEsKKu4mxS3MJeKG3iqag - SunYrqtqRdqT24+V+SMPWDHVDTGkAcdlMUi/vz8UqGpNmONo8IQBqr1R6A41liNg + Rl5ZJhMbxJ0bBBxm4BfrmSatcbyxlAERlyvRt3w1Y23zXhJBnf/krfJNlwYliu8c - jf3zQgTWVYsvResAPWbkmHWKzKVMC/gGdhH2VQIDAQABAoIBACi5/psIIM5xBqPZ + e8sH8NbmGGEdJHw0kA+1Inm+DJTcO7EJ6Rm22QIDAQABAoIBAET5M7cYhjwt9roU - uVQNW/PJUuNkj1gIUqKvALTL7N9pIaJqNIE52mZesViWmjz0YNrzS/yTMbYhsfml + 4W5oin/SVrT1pAidRy+38qxUyfIiCD3pQ+wxI1lJ66EkLR72UcP/j8qpFiE9lvEh - U+r+1nSJPhcPV+XroWP5cRzonjHlVB94gsunGrJOb+vbdjf6oTctgFgmtlg0Ot5t + 5SMme7nnEr01VO/4hTHw+E4Pq76EX488AEMW8I+5+5Qos/cUp2Jk8GJPkUZFxdml - YIzYC0OeI6GLE8yQW8q0kCOp/FP30mlLTYzDYwkia/q4FcjxNRR81x9Jk4C2/l1C + oo5qbbdoiNbfaWJhYtbNIG31EDQ+V027Zd2A67QzJy8Sl0WdhvkB8jQaX7DOe6X9 - XLlthe5KW5TlPZ+xeZDksq8dP16KCTmYdXAZIi7I5WLQTsw+MxXnCr6ybZqCqUEt + l2aFWxUH+zUx+T8/2tfH16OYPCLPsW23IbU66Wq55izeu183Q/OzfgS/tXX4RNpg - wzJvKOWVNhwehdVJwpfHZFX/IEGJPWgSvYda9jvEExf7txiJVkifFOulLyVZPJcO + i6zxihOVPpMf6+4WVBh560vW9O/2ehIbHwTsm1yVWcqYgWV6YDahBBkO8lrqKG8o - j1+/61ECgYEA+4Srm8nxYVX2gbKzSAAjFScwqj3C9UzS6kFaPGfmDguXPhZdhGO9 + vHOGg88CgYEA1yRioqjxZm+CmF3y7GtXJsFnyBem55OSX+CFaFCUDashKohMw+Pj - 6j8tQspirKYN2Ix8/NxmKe+MrTwjyAIdnQ3f96dpR86kIwx56d+N4ceK0Yfg5fSW + KWN92PiSpIC/PliROd2sm6BSw/RBe0kTXWZAJ//TmHo81dCdh2B1KKSlsjZecOPJ - HtQgecMh5TMcjjFe/NqEU/17e0eZfxSQbVoVSDKcZos5c7uOgJQxnEkCgYEAxSbm + N0r450/L3QtlzTyC4iGU4ThnjXLmqWK6yM1XFn/UyhhgGpZsr92jrjMCgYEA8P1e - 5LN/1sWynmWXfntUO6pc2THxMrNMTHaAUBdQFrDghHoFybtBbuM9ecnpTwvKMxlT + T1XTv65k5YR7CVfJHhhcFWkMJmRDFU8h1jW6A3VIYXXAlx7k3Ju0VlB7FbedmyWz - gGXWRG8CRfTPUU5LKlg/22/GTrX6hDP+sX/Q7pTWbJZ4LwJhfID9ngr4v4almGUl + hgEL0Q9Gt+7y60+3dMmmeFk9fkwto6g2WqMcrxr0cl5YqFtHjYp9Q1kFL37TuTax - MJ5CQdCEu3cxn8PO1xu7/7j+WT2FogEUeQLWka0CgYEA6GP/z3S6Iy4zEkkTnz4J + /0s99AKkXefp53tDy71ilQjFVsGc2LUq/CrR4sMCgYEApGTbMeviOiHvKrpvS5Ri - LD1GmLVyEgYGhs0VW+S/ylBpUMOHapBh5DK1VhX7L/xJpMDBpzzY5HxiZZnAkcdq + De0vfkgEc2PiL30Cs8kOuLsRJszry6uxAwlROqAGfckbWWqX3h2zLV/+nllgR/J4 - pzcvrfovq1pBi+S2LCITTP56w/ihErd3kUp8KyThh40/IB573nLke1olIpXYPHO6 + 55+gWnAzoYmWPtOf67gbDilxq5G77ItCUAvr0eS5pHh3G7KnWF/MwaQ2DHHGK5yT - sl7edRPWMGUJE2bDVwgWAokCgYEAwJdpFN74skD8ZVnO7SLjPUoGW7I68hFPJp7Y + mai+aSTY1mx10xsqhd/YmN0CgYBh+WAiOO6Be1Ehzp6Gyd3GEnk9axu5cAGl5CoJ - Z+TuOsxc920QPGot2HoqMs/4l1xoERTbimFxN/bNXLNy1vVJ3jrJXr7JFVkWOZFl + gIZDaacnmEvYJIM+/T5v6QBhb+jvboBx9nLrZ56EoOy5pgsbu++l9gH+GtJjOrv+ - a9X1rys8cGVpUFreCrcjigEj0E1jdQTRmLXw+cQN9efRVUX9yAry0zPPXDQKWCD/ + VVoQBpFi/eBlcdbBQJB0lPh6usExB3+OHvTtAzX3x5VcusxxRGmT1aEFCGnP4Le9 - 89q+6x0CgYEAkmAFLhtlILXl51kd1uxZtrYCLvq6rO5Kq2lHsc7cnVjX8zGlLTX+ + FVuHKQKBgFZTLR0b+xCu82iNXFdgFnpkl2AhG2puinsGRhqhRaafb7Vt00OL74Ur - p3CAChirnYbBXlhn9sCf9OzPGMQrlW6q6NEQ5EJ7mc0SnZI46jQ/dJgziYvNnVqB + TmyjiGA50sd4dzJuL0//CZsapGnt5yDqA2wtFJVF+YNw5ZJRVbFqihulfR7zz/0c - LOkZ0bTT8fboKAOtaev/ubjKJ95aVdJTXjiuGwld8IbBUUlmt2IJKs4= + QbIo/GGC++g5f6gU+PNlJ9yDlZrhKSWmaECyGDGVvqrrPCqNp3e6 -----END RSA PRIVATE KEY----- @@ -11262,62 +12462,62 @@ vector: segmentSize: 131072 total: 255 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:SSK:pvkyxg2paj3uake66gidgapuhu:kk6x3uwm4tkx7jglftdiycsqk3hbarvi6gunbzrpdyucyn3iea2q + expected: URI:SSK:nana3u55ewg7j2t352qbjqokaa:s6i42emhgzsmtxxhro63znk4cklea4qmjn3hwyvha6ckxk4ptsbq format: kind: ssk params: format: sdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEpAIBAAKCAQEArWejWyCuXYhCalNiitA9iZ5EEFw+JV5x4HoaJWasXjxKdPNG + MIIEpAIBAAKCAQEA4DV02okqw3N3Kia2PrPCZrjkTklwT8WV2zts6g5i/0UKNTTt - qIWeqMenRSweYgvdXMEVzll+meFCaV0evEotu+5HBb/zrevcJJ6ISHKjwMoAh3Ri + 75IwvU7AJJhMsCvPSh9OrCeVE3aqgUyb+/oQ1EKD/U+hAEVc1SNP3YaNlAWmK0kw - eCrWB+81cncxRghY5W6w6U+lCkFGv8F1wPafGm5RIhTp2T1IFDRDQ6mjzVZFjt7r + tWbTmBk0WTIm4yFwuVVAXEOWJF+437l5Nramp+LLrP2sS4hAOwm6IYbUZNiL2xal - 2CCDFi9aBt1V3xNefH8V+B/KFaeS26ECwlM2mcWWjID7jP2oHaG6i/cN5l4fwLjq + RO3S/vkpCFx0SmZr5M5LbkfO+YazS2abU9GmQLXY7ZajS8pzb0us2MrqCbsFfZud - BfZGJ7BY1YjZDiOIChwp4oLXQME+43jPW7mSJ/h0G+T1YVaPnc8L/7iZt+DbtLOL + 04+zyDLuAJb+RCFx4zDJ28zK9haqdhdGZcHfNjIu6dn9StEIMwsUIPJEXxzU+3Sd - wAzmrBjIyMGRj/FeljNvt/tt7Cp+kMIhH/AslwIDAQABAoIBABLwfOEZIrJIjah7 + Z13IGWWOOOxnLeXcTH9MBmnkCrc5k+D87aBYpQIDAQABAoIBACnzTpUqJ45w7wm7 - TwoGUJJVXO3EhW0jcaCo9W4cVrs8Lo2zfIYvgfLBS728Yd0nmpfk5vLQx4kbF1vW + uwsx2qyaKRuNyZ4fZ9ngfVEw1myRWCba668Q4R42myRtu6GV3N/vSirTcCT3ZdEy - teKu32vlTJCONJlMZ5EAV7ZB/yyxY3ln9tFVLGdVcyr7ZcBWbQ8yFdSFxGroUkfj + CpiDsjzo4iXWDZeNouA9Tm7yQ1DAGtaMB/lVDz+s7ZrH07dZNSx0K0noGnJdV6vC - Y28eAKasYeQtEJWPoe3C/43GW4OzuD+c4NZuPNTX3xs146e9NgyJcCjWrqvSj8Fq + mg+27qlIedf5EdEDIhN5zzIBNoHUmfEK9PCzkvJWsjpmKYRl5KNWHhCkj303wyFo - rw1lMQ2LFC2r7RYsIG8DjkFOy2NVrmu67wavFJc5dPYZrpC0E4dEg9QZv1Z3II+E + LhkwIjg0Hh2zjwu21BsMcCQ7AAwwqPCr6QR3cV1bhuznBu6TQHvFOEVY11fykUfE - aXUwTUQSIgA66Vkgv6rfFbvuWssq05HLb5wL3v2xaV0pRfqZsUigi30goB4Dzdbp + Wb4etekeb7/dEOkXf+9MX+O0TDMdZn1LG3MAqEh3bmQnwK5HdedpxVLY8B8uIb2w - mHd3tSECgYEAvVuUW+Uv151mV0ee49E4gYyuauYxMWCfr8jOI6iCcaLIbyAglMNX + Rq8yk5ECgYEA6xZhNg4+FnydGZhNexvn9D4pKO7y7V9Ylsxi7655YOj0PMSPu8d5 - 3AiBAC/wfWRj9FPjNpAt33UWWEc4c9scBIdQLZ5dtrL7WH6QzY5SC2Qh8Rk/6ec0 + Rlk3sH2io301PuevGhDQlToxMQJinSCBbS0m9Wh25puBDV9Hv6/isXCLSykM4R4n - opmEP8XKkOi++zbZAoMN9/1XAw0Bsjxg/qpd7kCJjxeZfKpAc/oTIj8CgYEA6m7D + fr3iXLlBy+yaAX0FSmDYOLtdmRic+mLftg2vDPS8Y67QeFxxBsslANECgYEA9CdW - Nuu/swPh+da9vCu5Caao8fHbcccVWXQoeWJ6xroKwRxndKkhYGQ9FE+vawWxuFtn + 40AXX1DUuntO7UUdv5KruTl4ERuDfHA/bQHHSWQhHbDQJih3PSICJxvcOu/yAUj6 - qdFDhwR/9Y5vIPKPMdlYNpx/aoGrCkdCPa+V7pxgM4eDAy2qTU657mFpCYqfKxLP + s7Jz7HLbwHBt6OKReJgWAsPTI9xya6ktWP2AhX70cg+LAJUEqjPNVrdmO80cOpCx - r2GjDYbmtvpis6CKTIaOVLOYmCMtzYQajf67L6kCgYBb6BQ1GiNDcrkWicOb7ZOQ + 8CI32X0iKpyKosPO25tyV24dsgtCI/64HC8Ir5UCgYEAm1FO6qrgRHUSSk3AqxyV - hXiul/WucqhvCHbNJd/SSeEg1qYZrkp5mIMMVThTlCNTllfExuwM9maXCFJlIScT + 1F2ZTg0I/OFoo0DoANjyIp/mdZucJwE0U9EwJO/smz1nB2eE3aDMXjtzMqETzCIk - J54J1kDECVEnXZ7otjgqITq8K7Yy0C5i8UIYNecguGbhxXhE2Nkx0XunFmwJV3b4 + wJ/7RFo/bD7DNbWErtWi3X6w2PiHJNiKWYdU3dQb74IjgeY6r5hqYpkPzs1fMWJP - hDY9CoP5uMmdkYDhCbK5vwKBgQCqcgerCbKiFEObY58ljeCF/M8+warcSXPjSwoW + 0XUa+WFccjOWUl8o4ccnbHECgYA/b8k/mnN4LIRZP5iuRXsJP46mvavYphFvz6pr - XxyHGkKxbtZlQobKz3Z9KwaOWGCDeqmfFU/0fbgPMBTqLpEgHDb/1b7qEAbsfSzT + JD8nboC8OiibAYCZvbBZgP+jwJxcgR3Ceudr1BwM0Cl+jP/HGZz+cur1Ml34YyiJ - LrNi/f0P2gnOKMh2VrPmdppo3omlRpMLn4BuWFOsW9WhZirHQtl/Cej7TDCECPVJ + KrSt+uKAkFFJgW6I2pTLpzNE7nIbNWVGQ13HgJ/T+oB68e1ZWfZHiTDmBc7Vk2U6 - ohg0qQKBgQCU3ZdxbrhFetjRoQMuSKCROwV063g+DJ3zS9h1L/r4MEke5qCLfzvs + zW2wUQKBgQCmt/ZrSCin2teQOg+rqcmicysXn3/DkYYwafeTJa52OJV7JQTWK5iZ - OpYTB+LUMVFMy3G0woaAOMbqK6MXnnD45zeJQ7wEfJvZ/OLQ4sHN9DCQoRdcjQyH + vJlkfKHpqcmsLrcUJH2rRlcOgzrjXinaXGkFtvCaWtCuUUJgTw3n7SF3chOUjmRT - /hb6ob/JXdp7s5C01+DI7+otJUTGyZGWQZiRCudkHHTa6xmTAr5ixw== + SKL1Ay9Y5Dus8UREJ6V3JchkwgIZcUgmWgRuW6YHdjCscWNOiD9Pow== -----END RSA PRIVATE KEY----- @@ -11331,62 +12531,62 @@ vector: segmentSize: 131072 total: 255 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:MDMF:beerzs25tktl4kl6ervv6zm2jy:4uu4lyc7p4p42hzzifnojvgbhsdjq6haseowvljxpfyjulf4w6ca + expected: URI:MDMF:2qmpqupyhsx526l32tjy2ouhri:taqkf4pffhgu4iy2ddhv4ltt5opsx6s6l64exee4c4vmte55adzq format: kind: ssk params: format: mdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEpAIBAAKCAQEA2ZNtrBKh511xQFY9fhekwnZ6f0G/s2atWfiW5UfXMoeh9HDD + MIIEogIBAAKCAQEAsCZMCCaVWiSLYmGr63IC88oMBAiDDohGPCEAS+pgXNHpb2bx - klb1qk1Cnabz7wgYsDMUw02skm4b3chhzi1jTUFH20X1RqEuAqLTsqTl1s/QQDlX + 3rRv3jqtMgAlKF2bQ+zGXNRKC1Z+b0XyDxBxLtDmXVhqz7FTh4Wsv3NKnNA8Lu0n - kO9b8f2qDzsVWMQFHHfrysRGb+Y7KDK3tNM7ljD5shJo8072QP7G8ORC+Wscf/WN + m5/49C9EXv6RrLxk1UQJ88R62yL3/apBgS2bBC82WlyxjyBjJhv03B/6AYzx8kI9 - Rrb/V3TjxORL8KckOB0iIwmflKtStvAGAYsSHIgU+3wMOffSKC+1gOeSMX8YUz/A + Z8Vqj0DnstVpaKi1vQnoJtpYvTxiqxzK9PS9wKV5hGHEIVqD05Z1RoPKPPHkxV+t - 1bBJi1vwnSz0wwC6YNGRwmAYoM8ta3Sx/8Lv1yC18DPLrHNPzX3CpdFaomGfwSMc + 1UxbggtnME5+SfQ9NQ6w0t/+aNeDRWc7dGaLyJhR9FtASuaqjsORutFXOauAXxpZ - yyswXSCEs/YrXrM4Et+8G0NajBYGdlGBsmnepwIDAQABAoIBAF1fPJi04lBlNH30 + 0Pk+OLZs8jmvjyklucoNIQ7fU7mNieIO7EI/CQIDAQABAoIBAAWQUBp5zYHZYaWr - xK0BPo7Jw6YrNDasYMaUvUUmQH8J4AIEBpodwY3VXDpF9LdnFRlAwq9R/TZWFJVo + 3BhFs54rpZGDC1CsMTu49x9uubh00OC56a3VGSt1wv6vTn0l57+PfPx1oBkXlErD - MjkGF3CHDGxYqHsoHpO5BvrKc2xtgKSfNyoW3rGKN9oTdATFEqB2AnXhJ41ME6Ub + dM/Q/yIxavVL06PXwAGp0TkC1Tp6wUviJbweo2hjDc/KTqOcF7s8uOvAKvk884RJ - puTuJcs9t1qpNer8vweDjyLAAtIAT7vcGSayxGminW77qS1R8R1rrWPeOWUGfqJm + mmQQIrNRInBcOSeCKF+NxoAlamoeEiPh3YU6T9PMHeJkyB8vtaB3td4+XQTX3Qg+ - MNfTkI1H2bUZ8QFoGz5GEytQxqD00LwZHXYWUe/PPZWuuDOhmx0QGd3JIoqyURRV + gaajC3cwU2fV6NEU+apem4dsn4TrbHMgJpSLWxof3XaJSdsAz8KVV/T7hG4UZbFK - +U1vuqeE8S+/9tmyP1jfp7L4oSSXEZerOYgXB5KX19JmXJ89cz6hkfe1VlZqBNjf + HfEriY5xSNuv4u64vikNPNi8hHaTaU0KAa52VB1wVzyUqjfSaotwCL8acjg4Lizn - GQO+a/ECgYEA+TXgaa/83Z4BKTL/pvQXczppRbxXrY5u6qenqi0GFCwjFTuFaLjr + u6VO+7kCgYEA1uc4dVic62SSxJxoPoojZu0DrhEXgvadFAxN3X6udkYNdCs1Ox45 - R3LwDxL69FU0H11WX6m+aPzse9kfyP/rBi2iQGo2Cl9JGPHFPwz/R4TTB4f5ZmOM + PuPUpTWjvLuaVbmEt6mU10tv83A+wxjWOcJXetHsWwPROv4G8xPO5vUgVyrxysLm - o4t2Cwrb9EHQm35sp49P8y2XSI8aroyxqtRNMto/BszJHeuD3jqpqQ8CgYEA34Dq + UxCMP1FE2rHaQpXNYtB545V/SV9kV/+iVxmlGJ1VzWKKs40GXVwESJ8CgYEA0dXb - d7tIR7ajpGsy4GpHRSdY3iTRFOuFnmajZZ1pMw1KWQsIXBgQHPMvtUWFH/hZ2S0p + 2XxB17O3FK23zSFS/yVwivjLFq2IEoCwhHJ6dLa5VrRH6NvUAX4xYIKiWQBcFsUK - 2sjcKww8TqDStVtuMF3KVp2YlLvU45l871qOf+xvXww0CW2grShvRu6rMpMQTOq7 + fOMBhAe9Y5+RCwwuPmP8ZnFPeaS8mjAmQIzuXxPr0pjXM+j//D+2eAL65oFm5ivU - eLCWWOxwl2jlEm/re8ked+ATUW2vyI7YXKwRAOkCgYEArUm0YWk5eNT806wdrvb+ + LB9qAtwMQ9yNwtjotal4yBl3HP2ALbPnQCbLz1cCgYAYjdKllbpYKuWaEUTX4HCr - M2bDevVLNmjbYZnw8VlbZ72FK6d2zen/2G/o02KMVEfG9aROgjijKZftzPSesIKb + EemZudo13HeWEtHSvOayHM7stwMd/hYMWXuyZK6Qod7AbLH9SiL3dmcUKX8CS5Qu - 53Dl6MqyByZYytqbIIumGxIWN59qYbMJQVOhYm5Loh39s5IGdcEmg98I2jCACi3V + hUX5goK+43DEjMG+hETfnqJTU1TNFfe7BekAUwjK9Ac8FGGjKK7EkhA5Ee0lINAr - AQedIqY1u0G8+2wgBvBdtysCgYEAwUAce65JjwhSciXmdbgvK5Ib+ufmiKokfJPO + o4J5jYCANwIiAbr4b8sNgQKBgAmVO7od+5/PPFA8csVyfSjb29zs6dF6UVmO+QDD - kFwMzAGf2WH6tnZv6Dg1dg1IUB5Swb+VQwENrYME2g+gYQNPQS63dzEI7wGBz9G0 + faYw5hv4lcQjrfX3fmfK74EjDBGaJBV6BIq0E8kl82jOwJnm1RMUn62NgXOFOWn/ - /thUAjQTECHjFIvftBkULkbLbA1QuND1jCNTvEukBqbB+rEe8YcyewAac/vdVBJ+ + Ra+f6Egw5LshK/eoLTwj3rOCO2HNpJ3zPVMuG31J6Et6vn31ZGe3CgKP7TepHKmI - 7ZIxmZECgYAczaW0DADRxSmrfOykf/rL9EEKd4aOrXaiCWkREmMwoaHsDBYShq1I + XAx3AoGARDvoRZiLJEmUDQOw1mI8NtrvD2oKuO6YkT6pt1eZhqPN6SMDUK+ckx+C - cYWGZFONdHFfI5RxA9mFdpOV8B/TnUtrc7x+x6xDpvktXq9bNAX+tniMKPRgbbAe + nG+2loG32i9F8IcBCFWqeEXO/C6jvb1df3tcsxtaJFI9L4c/Nhxx15MQAVhgpwaH - j9bGVS3+s2mcO5CIXy2EvDjxuOA7VPYXvBJslkoXrbcSYw03mpjW1g== + zN8TUe04nOhf0OZZebwoDU/ZrLY7OjiW/vzDmeqXOjINS5kbxmo= -----END RSA PRIVATE KEY----- @@ -11412,62 +12612,62 @@ vector: segmentSize: 131072 total: 255 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:SSK:3nkwaseraq27epfjp5z72teznq:jqyndcqvfaur22h6yjzqfbe4cjiqj4jofutjx22hwv6ap6rqzx5a + expected: URI:SSK:xann47c7l34cwixkcegyjwbrzu:ohhz4cila5kkv5ruixjzrltgx2e5q45bx2bfjpjqbyi6mt2zfkha format: kind: ssk params: format: sdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEowIBAAKCAQEAodch9NkFF2Wgkcv29QJtIOgoY7hxLuij/0LY+NRojsrc4LSa + MIIEogIBAAKCAQEAvSRDhsaKiEoAW8+6Xt0bjgxl/FPZxd8iSWdGcibGedmJsvK9 - 2NDkM7JGwZ807zMTpf6p1FhSEmTGFoOd/xPq2ZuhpF9VejQBlUdEfesb8yCSEsYD + hpAFDrqSFSXJ4pLjabqvhnJlxhc1xjlqbB/JuP+9eqYxm2FSzBZr5F+mezE24zQ9 - CohXvhbYrpQq7EIPfj1EelttDCSZCBEG4OLO6FVEqX4d88K7DnUJ+17D/9+HG30T + tqjCHxbm4JQmOgTeotoum4WXS+mNp0YQSm2NDil71mmN288mZMGt73GN7sYz3nti - qSN3886dtcO7B/MboPNwMBpVKWbTpNkPlRDq8utlxAmtJKSQ4BESCglelx3WzJX0 + ubABLMONRwZqN3tffAgzcTaSKON0k3pimr7CughlTGSSbnQ/Hd+G8jxiSTuW1/8i - msQkNO/2EzXabml1IAVRKANknqd7kL4RqQmC2Rl5atSmw3DhOAyleBN2iHf99bZl + p0gt8NvDw0WaOxbyG2JsAxqG5HN0ScbUlCMEwiobSxc+ajAy3oyMJGbsPMqTPT3p - TujyACe5BxBTKtycVykolbQGQ1u0GFdgRKFHiwIDAQABAoIBAEeulWwxCWfFDBs3 + 0ZKiTJKpA5M0uMhF2rQd4F56FKo6ZjkYSFDTGQIDAQABAoIBAAI8Xc368woSy3ew - l5EKu5I1MdqFUaBgy26eyaJg1lTUtoNSizlYQJNDNcLBxPzjhyLhSpBydBuQhgpn + xQe4PmPXkq10AkscAAhB5ivIRVvAYUFrxcDDwanyztipv/3jjzhubzYijEBv55m5 - zn2x8TXkEHLRBPek/ESFtej9zznfJcPp72PlYtOfo+ajWuWdFuantWJqh0C3Hw7r + wp/KxzQrVb6PtpHOVmlSMVSB3iWiDcxYXD7LnOVulfkWpAlwieWYuNwexqYsX+Yx - F7xYySMvzUMzSIn0qMxs+3haj36Pak1cmJi3NhcU8crs+Gy2MMsJEIv1OBxbqqJz + ZlTmGZ8OvEeEq06E+UCDDf7Ns7tPhRP9SMdt7QpgfyL8rWsqvh6i+hNaNFXj81Yi - 5wcGW2nHLE5rP8hkEM9LIjGSNWY+eZO7WxwfwRwg9y/Dy5l8T6RvFVHS0Est2tMQ + xp7S4qgLJ2XgUhyAuQuO+tJxlWVkMhWAcQgKiEWDZGMTDHhm8+v0elc4iup7J5SR - uPfD9zbKR5gZiSuuK/0IVXmWEZYXGY8/bdgnp5CDIihBhp/lWmEyOuhe7ljLaK8y + 0a9KKlwe8iBwihRyvWTXgf9puiVoGmjQPT2OwXqcrYwMBTIsf281am37Yo7as1QZ - CsqPO+kCgYEA4DIyYQCFyPZaS/6CHOh7LyrF9F9vOcoFSNh0npWG0gmUIO8zsII3 + 17GvKTkCgYEA5QHevOUc8DqG93qXgcT9k9x6JWnRC6YmVWhaERDRkPANNCiTRcBW - VPe8W5Pf5J7brBiC9/a000lejsy71XTR2Q/kP8tdUjOtT2vObUHee6dp3LUOHEHi + DAVgL6O7lZav9qq0CRXmF8X6Ukdu1MYiYuxxxGgSoKY6WqIcaKGqwTyVUlSe6HJk - aGvBDCJ4Reoj39oJ5m0H/1QrE1YAN4xTUMDNxrjuC1CDopibCbbRgNkCgYEAuMx0 + +9mSbnCbE+fdBMYhg6CcIeOvoTeXpFerIeXxqI/N6d3FUbA9MdNhlaUCgYEA0295 - LfG5fybpybc7YR+Hd68uldhIDH5uWiepg4PKA+Kg8sWpc9D0LxpwFVQBixM3LNue + /nPiDcXgv5sgF/wBL9sR93MQX1IqyhT7Y3t747odvkxkBg4aQFpZFM/wUEIQFBEY - wn3SJNac4z6+CfSSCgX4yufs1PqUT3UIKMr5zuYTDefmAXHTDZXY1Hati6zwvcSs + 9adWSQq07nifLL0mrkazF9wrDBJhyppqMnqZStzAjdEj7qxYb3/e+RTf2c6/r0YF - GjSsb7RznNaPhlCfgionJ2c9seSAb7NZI7yXzQMCgYBjiAVzqQ677BqkWEYdXVyq + 4LC3X5tLd2G4d+UZCkcoSltdmDs8ncOkIlvNVWUCgYAVB2Dus5M+tAEkxIsZDX/D - 0Pt3BRNU/YohD++eI9Xp01TO1kMFXpn//8fAhELGtXviyDMEsKMQlicDkILnPeiX + jiFhQiBCE5W9jgGHQ6YayxBLU9aCNzEvlWbJuR1GlTm/StmRZANm93UPDSQuQaty - zAVSCQ/SGZ0cgEjxmmeST/2gfUTZaKqCHyxiHb91koAAtkTk5ozBXvWMrQaFoqeu + rgecY3oiamE9ZVl6ei315JxJnR+idK61ObtqjMiQwV/YSmFVdvAfZIsCINq56pr6 - VxpD2f/cSA9YlRVnV6Fk6QKBgQCEICIcy0xOHftfbrN00H8h0k2jczyoOikaKmtn + V+Ui92GPMiAmaiqUYra5SQKBgCjvHB83MDyaYri1v7DlCRXKw9+0VycdMUuOZF0O - jW19c6aRjUOHe+lqWCO1DBgCYJ29Y9TRx/XcwtjvHOfw5D0aD4T/Re0tpW8ulEVe + Ox4LmlaNU5AYityKoVR2LYBcSeCYrsxgaUQa3oyMrcRrmmGDLokgBvV/WY9v9b9w - LSmIhTUwZxIrDD/S4cViuuuABwklFR3bqrdzMnjKtRlu9evlu8+8u3L/4pj1xCxC + HN1xf5X1N4+trjFoADMY532zmUjFtb2aeOX5mtKyCJSttftXa2V56tTeIw4oIk7E - gc+jEwKBgB2Z2Nv8mCK+ASTIujUAvTybOr7X5WK14sao+6kELQoEuHhWOf2tJWnn + lyxBAoGAN90N6kHoe6KuTq19gc1MGSy1yctOk9LskE9U/9kZoNBK94XukC9jTFZa - OyukYwkK66uYXiKPfxBlvgInhlU+QVsCEMjuitetNPsmq5CwSTuxK4oE8Ym1L/I7 + g5VhtchWKaQmlKIhRE/m9LIIoRp96EBmDMV+VPqNu+xqGGHCLX7hofEfYDst1YLH - T0voyKT0M4i5IvCiIuE+Larc1wqILLdkyo/uVQQYvgxQhiJAZFoW + y8TOOjHiY8XpfqOnftV4ZCRKrwR4BCIIsXPKJ++p1U/mK7sw6XM= -----END RSA PRIVATE KEY----- @@ -11481,62 +12681,62 @@ vector: segmentSize: 131072 total: 255 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:MDMF:5e5m656gdenkbp2zq3n426riai:umofnc6r4xbnizpkrazduchscmzuv2id4ejgkkjqsr3etgsxhi7q + expected: URI:MDMF:opuv6wbozu5pmpb2qfidrw5wfu:x77l2jh4dvrui6plhqzcip5z2f37bdwj3wivq74w3khga36muzoq format: kind: ssk params: format: mdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEowIBAAKCAQEA3YXz6bwSWmLFCn+Ekx7RXV0ksXMdbJKfMUa4wyjhOpMYsUb5 + MIIEowIBAAKCAQEAtMN6z3XTb4STPwK01804o0QpJgxm5B1pxh5LPc8DEZIN/iHn - XXMGK1ajhBsZiBX/akLBd3L+pmtHqK4W7x6NXME4Gl+OMs3SSiwFn43Dm8bzaL+U + V2wNfPlqoUGNT6/p63eUrJEQYzCCt9A6YlzjkRrgaYC5sqmSevkZ8bwX1QsUh4ts - n9rOCZXxuPOpQv5mNW8c2/DBQ4PUqSoLAiRWXz2vlpcwqeBq6323ug+U54/ZaMp0 + 1eVyb1idHEpvglwvZkxlq0JOXcartlePm4pGlj7/DxkQs44NGOivmg7AqzKILz79 - D4sGN4w8OHkZpOY/dQfEK9srzhoxI9R1XQBbXtOfUvL9Kpu63CjG0RCdr1d9QNrI + VX9HjFTUYliglfcPsDUyulNm47qRSEfBKiaHHdszul1hLhfupVg6oNF3Q+s1ct/U - qlocK9meDvClqRCh8bvUfQPEXdbbdqlLK9dHCtF9kaR0q6YvQcV98uN5EiqViKPK + 5avxFEfd/1DLz3HRyIkI6I5f2bshoS+cQ2nM7I4/y2HWEb4ed0yEztbBCauVKNdP - Ip7TY53Lr1G8tiVsLuKAzFQ0eODqBIpvaj0n+wIDAQABAoIBAE9nehhgwUVj3RRX + 83D7GSJAfWvQoAjTJeAqqZ6N7MNMAkfce3gTkwIDAQABAoIBAC4/IOl47KpIUd+6 - xCpGJC7uub3fsP7fia+MlaLi7uTjoDi/Y5hDKEV1n1Q1sI+uruikeBu8fRojH0MP + EohvocDrjFeGrsBH4irkzz01/EP/iQLuq6BLLbw+l5BAFCZCDGfIxUnNJ1MpMxhR - 8AmTboF+gwE1GlAMpeHPaM6Z7rFSfaKg9YHdWPhnpocw1A2/Cd0CcJpH8MamJR7k + 9s35k+Mo7Ccx3tCd37MEjiWxiKth1VPEUQj8VeW01yVIyfShHyNeAljpcuE9Fetl - AqEobEtkXaHBnQBvgHPcEvTfK/VaWEZEb7cF440CeNTjzZ+67Nd4LEEsEsQoZT1N + xYD2xI5l+Z1kPUii3Cj2Rw70HUjvC3rEAfbdnr/larJKPajWHKE0uCoSX3GqX7nt - XoUZuceQ6bljNIVOgQMjCPiM1iJwELMDXKBjVa55TV3CUQjsunjwQUfr4juOPB7J + QtmfG4KrGMz+xXvigLH41zNHevjnvnqb+HBVgOAdHo5Evh2VBBb+1I9l/8tdjEYS - KLeCp0MWaqF8AxvKDJ98KDQu8digdtgNp1fX2ERGGqcvD5HW8vkH5CMevu/Ciazo + Vx+NAal+kpHXZETFmflAyLRk2whcCyQtuhye+ITJhMAtBxvMkm94RC6izoRRKPDy - RVlG9uECgYEA/FRimmOq8Yfs8LDEvRpICz87M0L1i0D/o4ewMdnPHGyEliQYssDi + XxOxRWECgYEAy9CWVhzXrr6R6hQKXRBuUa8mV0ogWj91kWYF0rSMbm/SORBBIUrS - iNpmddLf4KlHCAlaCBDhgPX7HWK+1oV/Qy7Oz8hDPHBtmgixHXj2PEM34EMDpTkQ + mX4Cwj2YqqdSKmNw8mYvBL67lV/sQ1h67Mv+g0Me2PojqRUnh2TjtcxyRhhUGkRK - MhNRAbM6r+sf6KRWD0xeMTz7IE5qpsueruIu0rkvf4P03gpuxb/x2RsCgYEA4L7a + x6DZUrh0WgybPsK5zMeT4G+GtZvzeBvAO/D8FOVE6H0zytax3OwUTKMCgYEA4wv1 - IhdwQ/5Rl3ttTcZhMXRHs1kEtSlRhuqrFFDC4HCQ1Qjw3EsDilt3nMjWiYpm7bMJ + m8WtVAsurwJVKmtxyYLsTMLeLnW8NO+STlwqfBQXzBxWYhQJjJmfYWRbSTlUPUWJ - eLo1MtzpnUteJenfkRQ/FH6k1q3jv2e2EYVxb+8Du4bzXSEBSvIxpKMCwA7GYA7y + UCYUNiF0Dusp5dL3yPgpDrBnpUQ/uw+Cza0B5V/Vrh23kvkicABw4CQSQOwCGZkV - IEbh1DJYpKQEtptuf/fnbNpO9ia8LhIShx6SuqECgYEAv1DT+i87eyoOMmg0oxR8 + CL8bnAxI33SQ9B3PHTqunxrb/NAqq5FvP11OHFECgYBTwmwWBZJpwO2MSiIcLuV3 - L1rf7fwE5HKB4WGN7B4y9GArHxN7Tn0ExbKiIQ+kA1kVrDg69Qank/ntTdiCzXAm + ckiKdO8ox42UbF4WQpa3yAKX6uMpQGueItgVZWT5NPwiaW2AYJgQFiZW8+3Pm2wh - j6+7yrsSj47G6xVQBQKj4AkvInBtISbk6rLOprVX9+4UIXYIckz61eZgmZwbLSAR + JpB49zuVJe9DzGrLTJ38F4Ia5mKhzNECi0rkoONIIogmWbYrvxU5lfvBZM7A3H66 - zpNb4RXbt5k7XecXGgRwwKUCgYBS/gc9OZyKbzqgDsMhSlWP1pm3n/K+F2D0ymmc + 44VlPPd9p/6B7It55BdPiwKBgQCZj0wld/Q75HhFi5lYYGUMOo1heWbWG3EYiHP2 - meosyUSidqfDIaxQBlDYQ839gm9Z7ZhczZ5hhvR50mAU7hVR1MEqh03FvPbyMpEo + paViWCCkPwI5wX2X54sBTuPiyXBtJGuzlp2S4ttg/7JNq3tFJHp4Yd0nzNohxWLd - TTfDluaw9DegN1Tr4R315wBX/dzBkiNVSfeQzXqwaaUX7bPTa685IjCwc0NgW+od + gsbGgSO/aH/xWqjtAY9WOW9TE4x0DbJJQSAGUdSztV4YjVS4WykhmQPyoERL18hb - nWufoQKBgBKSqG4fgLA92xcDwvWdk8/rxwJvnWh37u/lX54LGqQbtzUBdFeGCXLN + HdsnkQKBgE9xTWmSxMInj1Iislc4Ru88KyGMVgpU1/IEQVG4zmu3js/KgHteBjPq - E4ddAUDjpFC8PFwNGOwoey5dCqYFvG6HubVv9V86lPC8FT5MuB72i3gslMTE7+2h + EZV38+EW/RMblqDu6883JAEURnsroLU+KWURE5EilBr5psE+3WHkHqUVDiG5ghtb - 1BbPPB/Is++rzW0kIpXh6yyBdQU+dAZWv9Gg3WbzdTUwhMKNtpMr + TRXTovFp+oZ2mz2O2UL2oA/iXJES/fa+B71ZaGdqMt5iR4a6/yW2 -----END RSA PRIVATE KEY----- @@ -11562,62 +12762,62 @@ vector: segmentSize: 131072 total: 255 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:SSK:6hzvdbfv7d3s57vij5kqamxfha:fygqhkuaj6e4cg27ocjx37rej5yo6cvn2x72dup2jbamj2rv7opq + expected: URI:SSK:7hnqxsumt4npusjqlbhmmo35qu:e7b4rmbxwbgrhguhjfbcc2pywgducrmjdzcs6opoa4z2dywalc4a format: kind: ssk params: format: sdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEogIBAAKCAQEAnq8zfx/wNk/vmIqwnjyBx7AAcEMhCjhs+IcJTmX+uH7rNDbS + MIIEowIBAAKCAQEA7VJ3LNata8TXtyH/4NhdD9I9oIEV3zGvsKLFSzIjUWqOoM3X - ZA1wrioSBatNjZl+T7SG6OrSLfzuzMaLbhzL5gBVN2z1jWLn2jLyLEwYfJG+BrM5 + XZdWCl8q0ZC9NTZdDUNDQmoHCLNGVQjbeXkMI9G4JMGdVxafHivxCpa4A1HAcPXw - n0nYtNd890a0ATM0dWNKkkJ7zl5bidwFZXo/YlvDBHbTkiTr01WiqLj1PzSDlixa + 1Z8L09YIXQmgPCxOiCgvdCcaVNxr97PiZgndAiGr2XQ5XU52YWm1M2F7bPv/67FB - 1oDTjAiQaJrua0Qtarpqgdgr7YAVow3Vo9ObyOJzotFuHGc4bLq0PL2SVVt+TvmO + kaPKlS4lAOVTQ76WKS6x4z1B6HpVQh9jQyvYYlfmmY1zvzIJvBMmyfMxLQpgBSe3 - axvWjAbtwcwyeFJ5S7pjSIslbY4Izu5IrRWf2xFLXaTievlXxfxfkyxCtawDeWV/ + xKfSHW4Ow1Kjl9WxhfBw71hEr4ZxX6GMLpFhW5aPtmUCZwl7FBLQosC4IJJYgRol - kE5VeH96v2FYsK1J0gNmQlI/9b0uR/OZdekNKQIDAQABAoIBAAWNjsMByDDINrYP + FSc5RsdvRnfPYbJ97+qnmpxlRRICBm9MCXiZhwIDAQABAoIBADsA88xcG4XdzNwl - YLS1KuJk8itKThZ0FmwhjtPP+BEvg4NC3WaLOxdL73BJMeAuzYIldM2ZfIORoAfT + Qd7/LDQQy22qamuxiMLb1T2a25kUax2jz9XfGG8/tf+ggspGF+CCRqiuf80z9VqS - YZzI/OGkq34hMYX6yIgerEgStfCnP8DMk98r43ZAnW+qaUTvoo3a6kPAAf/h/br7 + 9y4+YDxPmf7ZfGr4ntr7hdRiIKICo1vyacxS3LfwUOgAyqvrQCMuCo5QYoWSv+03 - mIuEZBqD94kadPIjhdoNFoBAsou9URBFuEKx80BDXAde5AfeV1skUy8Dgmcgai0D + 9iP0c9Rh1r4b3V9LcLdLdtetduhjTGNfPE0w3GazezdAmWmVeDe7L8Fts6XrDdYM - em3Kw4e1H2W4KclFZ653Hnu5tST6N+vm7oVv4RIHrtgxllxedB9lVnV24aErqvrs + 3x6Z/0rk1daMO/1j2LFfu0bbtWY62m7++ZlFbG2vNnFmZqUF2xxpCEbjIm3PwshA - wuyluWCLJriax/Mi2fRbskk8p6ypY3Tb+0ivezE2z/cJu53/605T6M4AWKq+jNAC + DRKISHqirfQ3yCMjNgn4kHUOFAIMK52IDmplb1I2gknoUKdLlXXuy1PuEty8nOu2 - RLce05kCgYEAx56qM5G/V0Kgzvgf5ClbxPoURb5ACXMLpiKBYjtPH/I1EX6fB4Wh + QuPwlNECgYEA796GhEQhbkiP6UYCjaRkbpJ7/YEvwYo5RCdSiNvfLes47OtwLkK/ - eQCnSwDc+xlDjUOm6dFxm2mf70omxXFuYYcISlPhdIfn5opRZwbOwJ9GVp8p13Kb + lsfqvqPclG3O4AZDDTW7QCqiu1cW7tx0W9fEz1t+yHJclrIYG3GcmN1TtXVreKfo - g3WHIP6acDUAapsiAjGky5JSC/RtNl86l3sgA1+VzICCrzz061waaZcCgYEAy4C8 + MPgxNP/XzmXMevD5qPzETf0yyIQTyHaRA2Yo2/kev2jRl0gqu5kZ0LMCgYEA/UgX - xkem1rskv286Naez+qQipZohMKBvuNxlFWGD31sYXnDoJdLQbRdoo3WpKWwCTl3m + Gv+QGUAVEy7us714+7w53IABy5mRq29pAVAfm4aMKEogX7LPjYytMEUmWdlnSuzd - Kh1Wl7mUSPYG0lGoZUmbesDJql2PwfJHfwnk/+JY/cUhdSeeOiLeL7CpsboHt8T2 + rltH24XThnGHx4gfQDbrXHL1ln5MsUtL9PyY2yzbvubr9TqZhnwOcPhvChfH8dXT - LWxMViXE43MPlubwwZz00/au3HP1j6xAC8wHlz8CgYAFsiNNIWWCSeZowW+3hO6X + 0H1BXTs5JBDwEZu43GVNgxCMNy1ck7aVBy6MVd0CgYAzK9isjNBI86fnzuyqhOB8 - akNV0h3lpyC39tgWQ3b4hGK7Qw+qmUeIOlqLq1Si3Y+t4jZLCaziMFtd6pG8pIXv + CjnzScUDV9aBqJXd5nIFHMInIM7sv4aZxwpYIyLic06H0i4pukW5GZ9fseONj3Av - xniYFliiiJY3X87+z5TqriDFq/j3qs+BKsNWT618cia25AJOabg4Ds7EhI7xNDpp + S6eLyOwSHPuNlm64JBORNN4vvt3vfnp1P+1XbiD+wg7OR2wrVckXDiXwSuThhhHH - xBufvQR7N1eDRIwAgzpFtQKBgHZhkW8Wx2squpnSLl6ADCbFzJHhM2WCLvuu2e6y + lNqwmsOpd9YGnPmoza+JKwKBgHHthm8fe4rQF2q8lqSE2rGpNgGoFqalWi/Z+kqb - J3CLIYXu0F0QYcbUUz6jd6BtAHpuDTJ6lqD0h9pZpGY8smUZiKTD+YxtmO8N7aFt + 5svHVq4cwbkqLlAGcjSfNiP+NYcvSnvOFWF6Le5wjNnEsgHpci7wiuV5xDePnggB - NBXWqkYVovzv6w+OsQm1D0IgIdU5cqvB0DZdCkf16x+xgGRg1dtoKRh9LGBDp441 + wyP7ZpDVQFfbVwl2LezE4vWQQuDWBOPoI4mzRP2jHMle2WVRr+7/d4KuRdEvtJM5 - RkUpAoGAExnRJWFKcoi1gBGSqV6aigqdCRZp0CeuXVeEo2kmLg4MxUqm3kX7bdTp + beiFAoGBAJZxp7CjdpRZv8nv9j30zj2b7C2Kn/fBusD9YeI3u1P27VkZ08SaP5te - ek1ILGYhhgJH+f7R3lDKwnngc/+vlWejdOHkEuQWPfsG19seukueSMgaWjkDqfgf + YrH+YndbGm0I0CYVhozsIVQK31p7iSx3jEBZTmNbXsdEduTFb2n/0pXQ8Ods8/iy - M8RVPErerYnUM7mIpyiQXndCUxwYmEhnliFF7DeEhI9doJeMrFc= + JoBOx9z+6v/JMYxoKDbPEohDL9lT6/TbWAUr/k2gv+90rdFwVVQs -----END RSA PRIVATE KEY----- @@ -11631,62 +12831,62 @@ vector: segmentSize: 131072 total: 255 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:MDMF:swtqq2rygd7pm2rpwqnw4ge3hq:bdl4ypfus77ukcuz5izjgg6k5l7lu6ncjpxg5oapvgsdlyz37ymq + expected: URI:MDMF:5udu4xzy7ffr7mbrwuxxzus5fe:h2hcxihkyq2fpv2maprwi32kerubawnj5yrpjcomxj5e7n7p4yaq format: kind: ssk params: format: mdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEpAIBAAKCAQEA2PpVfdw4iK4vvihOf+4suynwzX8Nqaxpx7G943fvHi55fkKY + MIIEowIBAAKCAQEApwF50oSlhVzG+vcDRCc1vYwk8pdlqMMgnqSHn4sj7zqNrXZr - dOsUbNXK7dK0vk7DYpMw4ydPLogEV6gOkU8Z6Aw2BAp4uol/C4TooP29UAnPlMka + TJGYDm5NhltudMc+AXFOjStmNO6TtSsU/ttQtG7SJLI+fu6rHf5nr3c2AW1NgLKQ - AmhjIlloh01N5ZHfNdcoueBeuovEmYsaTQn/zqKzYKRm7RESZEGhaxGOCXwCjhwu + py47GId5AHbdyLJdVqUqQuZeHTHN+OqgHxEJ5AjYB8avGvAtOjGG/1m95d/VHe38 - pVZzvNnTZs4OQmkWWrMoOCOulceuzvXnHLvOvYqMs3Tu6CHXsDQjGqtfgsDL2vR9 + nB7rCIZtNomiO7eFN57T/RwBFCMazD7vhTYyq+bRb1jg/STG+b6kLl3jC67r3/Fp - 8lmUDatAYWkIi0kbs/2bEb+Ckl/NsI/XOq3K/LYGthZS3taCpLBVBmsMMc9cDnNa + RqXBVO9WpHjwbGgSdAGR6eNt6jm2eXXrv1z4IFR4DdzgnaCtWZOIKmfdk9jLG6ZI - K/i3dMT4fBPb2l63uDmUbE811RBaTEZp8PCb1wIDAQABAoIBAA1wX+Lbmigjux2e + xqwznoBq3ahG020HxQMUWHRljEVI70HgZBMSBQIDAQABAoIBABwK2uZXAKYinITS - NdJ9qDPwJ+k1ITMCVKDuP4fQZ1/pWqlj1Y6JpZUMyYoLvsR6G2bxHUTh1pPYD9t1 + illcziDMUf3sHxVV4nnQ/bbz+a43YkfQvRan0eUGb30SiDsSo55BZOO+eFSGBQZk - XhrzmR6tbEZkIaXSxgAq0L/GnOGmtz6MwCQoXq+DddJ04B9udrKegM1rz2I4LZ/2 + PAvJTsVlYGLqDSVqNRB9wfJMLaTSsjNciH6R/DlTsiU6UGZdUN/2LuD55q63SLM1 - +io4YCmRG+bJ9/1fTSd8MH5xsr2f3USkb5f3gw91oezyPD3qoerksFKqwQ86bswG + zno49bS1KXUwzwFSd/2wCE+DRag+BlETP4hCR8Rl8puZq/2KAT+e+wLLNyNZA+OK - 9gsehcLZIIYOnSLpgIE1faQznV6P/aQ4PXdve68qKLpwgARfQwl0mYNRKnaxLLM+ + C9L+PgUv3Ac7e7Ll9hvhx0o3e6WKhus3BMbWyQN3fDUt31leVoROXnkvF1lORNtH - VYzljc95kERweCcbF+l5BKx8eNmWiO4opXLmrd4ODwPX5jo5fQR25N+kUH8nt/oq + aaCZxdSFvZDJqTtEZE+XMQND3Ea1aneiWVLD95Q/TwIlQfd6TpvuONxZ4npoyk8V - CWsv6OECgYEA+5t9nX28UTCe36kPgbycXWkEwU46OF7b9DNUWak21H/ACiCdh0Kr + 5CNHU+cCgYEAvlTtbjqSHCV0/3OjvXHNjnH0tTjPXzwJO/BdKiE+BVnVyznVb4UB - Pv04iL2MFXPND9pIRfo4q2ysAHgI6lMyz0VyuHCXujlOFKhd5gCupIXCw1m6QPFA + ygcxDyblCy1zEBOxSszpKOc8kHUh2IxMVe7hLVIxjNjxZgKso0GhOJzF/Qb/tkKN - rEjWZ7GYSkryMcfjEnQV9pOIZBDj/16Kh6EberGUjzhdq/he1YBuoKECgYEA3MQS + H7EdXU1d8lpWewztR1p/ZZrZ+dI/+m0kKlX4cD2L7HcD2WXbNJ3hj2MCgYEA4KBH - 0P7KfsZVoFHGFYbDnqS43jbcCQPQ3wrQCgD237AQs4tL1erm5zxoCQuB+W/aYQk5 + Rhoi5tsGkI/dGR1KFLKDehd+tTWUqO77hs/CCxEwLilVu43Bj8PmvAG+MazXjG4G - zZhwF6rrQ52o06EWOF+OMsL1Z+KcJBKWpwjW/ATGcEvCEJahuXA3QddMyspdZoYp + N5RxAUsPN9VhJ2jeUAReCLCkpEKHJ/YuVEAZjwDTypAlgIBIc7tLCPhOYarsohUZ - dqMjnI82RMVmPd9gkg68OFNBcATW+m+nPFIRUXcCgYEA2BJzMMHe7DZ96YNNDtRD + 3+LPhdvBoV0CMY9ZdhTXpyaghhqEuygcEoMgWXcCgYBSmdT5E545bOAbxPn4y5zk - 0DA05jDg7LIB4FgIUytvK9Q9vjS+M398go6Bc2ScHXwiGUASmw3Ehuq/V3O97EXg + BvymcWM993YidyxXjlm2RMiODCle3qBqJzjZVI3ujejzvzggOFGwGLqmDs+DhU/T - t4FjgKMomcNGm5TvdmsVj7JTTOIMgmLscEfo4InyR7LPBRMsnRdWGTgfhBfBRPgS + s3oyCwvKDpSlKt/1chQf15ntN85eMP/CE0GlLmBpP19sw61uXA4R8GRNETwG2Lrr - rWEcsSQ5eTklsF6OSnmOB6ECgYAoH36n+1bEObnAPHx61xZgk+GBiYjuHoJstyNe + TKgnPe6tzvDytkutyB8N9QKBgQCai2u+PYk50APKPlDeUJqBdviibbvNrRmkyRfg - XhSATRiL+SocQ+gZaLIjyrKhqgGPl0SpKCZfNtIxZMsVQ3atYjiO4z4E1nu4VqSI + /twAhUji2amUqsk7worjW0eiIcsDYUeBwe2l+CB2R6baWHpsDzUrQW1lXihjRCtH - 0SN5hEioiixIJYhZEpsIXV/4j1TwWDva8wV649BiKVpOrnV3tjPhLMh82nRT6c0E + 5/otu2H8AgTrTleK2JediklTRSgds+rjcMdaz4F/JeC2fGwOo/Rjml3jJiegJM57 - OoopOwKBgQC37/FRGYrrg4HQYAd6bERCK2NqnP51qlvygFL59jPyCGQbp9BH/7wq + piABrQKBgHUOJRCcLh4UEvI4qsKalh28maFzjLqv0Neh5ENTU2W5Mv1YSISjqRYK - uApjV4XdIhno+PTx+S7BaLM+rYXkZqWMFxOYHs0qlFeOPj/M36ebR5WOQB915Ff0 + V/hKLURUqDl+1/8reScAjDNXIr0gXf9UGkdcyFWnPjkCEuu/hz30fPegYcnfjLLk - tgo9VkcVUQU3GyJrk2QzHFZ+C+oMkCHcJ4oDOOWXaJ9Holp2KuEMxg== + BBH7ePyDTDomBJK8NN2NybS8jIx8X24OQI1aPRCK0yLFJcmj9Lwa -----END RSA PRIVATE KEY----- @@ -11712,62 +12912,62 @@ vector: segmentSize: 131072 total: 255 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:SSK:qdvgmo6nnk2jmb6btg4kbvumpu:a4xn3lgnhxsv4uj6g43zves5h4j2z6dl72bkxrphat7tmm64vwxa + expected: URI:SSK:v6mjfmtvbo4eibevptf5g4neuq:pqohdp2lrbontgxcna5cg3ni6bgzljvxkho65xjkvbsrgxozjm4q format: kind: ssk params: format: sdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEpAIBAAKCAQEAyFl7uvuYjGO6H/LbFpV3AKldXqGDmypEPa0s4/tVdKmV72WL + MIIEogIBAAKCAQEAzHCyGaqHP6kzJgCNhvnbkwEgugxEUDACv6Pmt8k6W/EZNti1 - bUFmjx8n9JQoS9fC+TgCMHaaLssCuizHmIpyKdJ+setRz4VMWz405DA8RFKnzrKc + WnlINd1d1AvgHV1OSW5q7xxoasO0Bjy7UoegZPdYEvk/neqT3ybW+4SncCqCjo78 - UuM2I3Sr8TSMVez+D/w5OLxdHVcL5IyAJi1ibnxYkniq6RXVNfyzaezxLHBSUAmb + ighVTq+MBx9BVLnQQ7NT+6clI891veBldr5h43bZlhKfrf1Hfz38+51sIYd1aS9t - aSn2MJwe3Mgx0VxLd2Eg1RDk2DOFbzrT1cZa/gdfloC+b6DJXtqbkyKsiwTPKKBW + xIVXnp7O2aOKEHgMMLjY/bqNBmR7idiXvB+lM9U+Lm6lqgEaAq/Owb2CV+a2b63Z - NoYXzlC7KGv3sV0VCXSkawrJhRpg++eMNVN1vaVqShnI7xgrRxoXF/TUFJygmDtG + ddnDUDksw21/SPgJzH4CP0N2UH4KZvMGqRqSbE2fNSGSeXXWowRop594EMxT+sJI - 8n52G3VoEw+88FuoH7uEm8Hn/K81eXZUlWmNTQIDAQABAoIBAADN3DuX8QR8sLY9 + FLUuincZzdQeM1fhSQLSBIYAVsRvAFum5fygYQIDAQABAoIBAAY/XGX15N+waUB8 - HbW6ocIlVJt48jbILQ6UUSJ9qrbp1yhbgmQSpOaCnV3wtzQk6IKRS3uACrrWtvDc + TFbnXEssAebFmLHVocPe+546r0afgcyAB1y+L5N70hH4ke77yrhqQCjR9qvqkp4N - RFVeFlfauJHqVjLcn8lssc/s1ov/HGNn3JM4DFYr0nvxZ6Rt68RoBYgMLMXdsVlM + LZFKVT+4ok+kH8pQ4Jd1baTuixpdpjM6keOa+RZoPXB7R0kSS5fCC2s5kqQ7Qwcc - zuKvnX41CI+c2b1PH850WSfSRq/mA1IwF/awMMc6ZDGO3lrlQ89nOzir+unY01PK + LCWakE299EzGgWw0/QIZsBk1WJhWqBncC4rNL+5iviWWbWh+33je4syQR31R845H - P2rfZsoZUaJCXCMvtyIryFu1W8yoifhQPMa0pnUO7dVqykySQD7xxUHx8hEnOtFn + nyeBNpsYDK06j0RljUHqJ5sbe9o/YE7lKj0Vr/T8sTw2Q43riJAF69aDFjr9C+FS - mzkCYyYzPz3Tx/7X3pQblYmWFgMirsQxPuRmKDagmt6vb2FwGF4NlHVbK49TTWGw + C0vaua+r181Bel8WOI2g2lSOvOfMydUoYNwBYaZp8tYH7eJQXmp6FB29VCmvVdNc - 4ARheH8CgYEAykyIT2T4X3YowYs2Z7skp/DuGimNRmctPIfafxhA738WjYWHS/zw + JjPuMAECgYEA0Y3w5z851Te3PxRWVrGBEpuHBtTawzz19K9VVEAWwuJr2YBMeGrF - ODwoJAg6K0obNkFdy5G97SwjcAR/8YohxZ+f8E+5iX0xumBy8yyHNCSnWQqekyiG + t+8OUHzN69A8l9VlqWyx9HgMYYz52ULO2UFh8A+1CRYxfBck+TY4Tl5HEpiVuNwD - btt009dVUbbwCWrdo7x+x7yyHsdU2yhYxEBFjJ2n90ufuWTU7xUU7esCgYEA/Yh5 + KJ8WZ9L2AjpE6i0TkYO2uGTKaPn3p2wsfh1FCiFBcSosNi6yz/DlWmECgYEA+cCS - 8Fkn46W3H5zjlG+A9ehwNNCndNmW8odxBxNXH3OwAggiEZy5uO1Ae9egIBep8qve + /3ZHOr9F0+w27YZc0rFuEYDGeOSypBiES2znii0GnU2ACD9Z2XgEG2VLwSE3U4To - by8BhCSy0VQQZcu+sgExtuesA0kilcyLNi9D/AQaWO2e3FcAiZ1BA0v12kVJawR6 + JH/p511I6y5VxHyULkRsxyEnXozevV46DJEkurBbz70S4U7jXvGAyGA0kdvPEuPN - nngseOudEXOZun1c1jEsetHT3KgU8hFIuEkLy6cCgYEAuJsAZNsyIAL2jC/atOw9 + EbhNFePEwO+hCVx/8hkclSIBLSnIYN3vEjGZBgECgYBZzhCtqaTpQWVgvSB7Krr7 - NhgRX8R9TDrJOAyNIh/i2eqyjPDGF1y5ZcfXpZHwayKUFH2v9x2HINB/gjBJBQTV + 9HcbcGEIRrnJUNKqtoSKpGo/3gHnoSp2txZVXAcLxkQRdbyJrTFeaYw0yivQ9hab - br8Mt8I5ALNDVt2+6BPBSZ8NK58aOBXqH22afdpp3EjBYQapPUq8ss6KCLZDxD5c + eK+2J6UX7dDrMyf/PUNIIpMm3wlbHb6ky/jYKcqQDdS23vaB6AaIY3lzH50IvQ0c - SrKQBRK1fWEAX7EY8xfc4oMCgYAHsssxFyP076U92n+2lCQwU1yE3gkXrTu+JYqz + RwLtYm8fRkmINt8eykggQQKBgDZ/xFQEnmR+aqFlEVNhl43OdANTw3uMBEN0qiG/ - Ek1E8ThY93JBYqbpDJs2p3d/QfixG7LnYWAEaTDc1lahIKyrrwmZajN47hGUxt87 + YQMw8hmPWNnz4QpoexTzVMWPFwCdpv6X/xWisI/Ja6PVv4wdGFOXs3yZZt2R2z70 - R/gigOVj6eM3AZVMmG/O79GJTS1LiJlIkpGXImBklUQHu6LEBj45hIGQY7IvH4M7 + yTwH0fESBDWwPkNwlbaj77TIb3ZiyVQNkJyvODcV02E0kyLkQe11HyaY0IX6x/mD - xUwMZQKBgQCoQN3iNLM3Zvfb5TA/nqbMQG54zimhqO4ahTpihFBpxxXXhefeLXkN + Yy4BAoGAX7Kv8GIypDDzgEGW27QA8RRVD7Cb5mDFxt3359RtZCXqH6MhkvOcU4d+ - hHVsNIud+c5ET9pgNAUbWhfKfO6j10YUvGlh5aFjOliGBGQA8sg/QtS04mxzRpK+ + XhCWb9sqD4csSwSoR1/eT7g0uBoOPuGMHyZSoAcSGcoDm827Zjphh36DEBmwCbCY - rPNAU46OgbFDCeN/eaCOs3nPYujL1DQkZ5XZ5YUU+k76hskwhKTbBg== + L0Hh+TszyB3ds+x1+1NbAmjvPxYiFn0K5tIsbkdHfdmmhuS6sd8= -----END RSA PRIVATE KEY----- @@ -11781,62 +12981,62 @@ vector: segmentSize: 131072 total: 255 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:MDMF:dl43xvmgtpakgi73zcvbgc2ngu:cgtbisxge4chvp2yeop5vye6e4e7wnrs32m6khaq4uplipbczvvq + expected: URI:MDMF:a245vdxigmfg5vqawu5hddp6mu:su5w2wqnksv23tg26ffsuosxhtp46zxxyx3speyruvj6c5uqaswa format: kind: ssk params: format: mdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEoAIBAAKCAQEAnY/8dSHOzMGDE+CG0JqXyoZnSRKczgax+YHNHqy7BAUor+V1 + MIIEowIBAAKCAQEAmtdn1hwB/oRemDFMiEzAVbcHkLSTomQDzvDP90KoQ96bmprv - Ca+YKZdlEK1yirkbkGHm6Pd3gzytM5D9ah6L68eLg0d/HCWS5z34j6SvkHR4Pbrb + 9q+PCTkKlGcxuiZDnMBA4dNzcJG+CPPgxR0IBGLQ07ncNjtU0zTfVzOz46p7cbaT - OjmzKgCtyLkGN/MUoiwjQRJyU9IxsyYJ0bpdppiGh+i+4JamgyeBkFRE5ztsT0YQ + jZcHK088eMKAySWaj55DgirTXIMqsmolE58mIn60OOfLkhAAslOej61MagYYj4CK - RBPxewMI4sezFbiUCx1X2k8n4GCIAXG/NrZc+SfRlwp03KJQbDCJpEghU42Xo/cH + 0ID5Ij/ZX+F6d0DECTSo6iA+B/a9tt5EV+9vNS3axFdrEQb8aiWynU5GsPUGEyVg - xYDYXvL1EVEnw2KWHEoHQE2ZqlfVasYF31s4fk2+8vUkKeE7nAODSHjHY8PiFWGh + l42n3LUpBvA0kg/macI2BY+Q3xjy2OdKo0BC3SCcmiIj4/kXXwStwyghlo6gm8DS - lYLzUjCbjHmfUO6xwpnDuJSNMFS+OpbOIUowUQIDAQABAoH/bSs2YHHsNzJc/4ix + IjWeiSVZb/yaI1Jb/BtBaTSqC90OLqI0xCjqPQIDAQABAoIBAErmCTc7YwePVgZ+ - 4Bc81LYLGjYrLxS0e4vT80z6xu5MIpN5ZByl8StUexmyIyveTUuIEiJkTCneV7w9 + Skvf/GU53LH1dzhk8qamO6KaHrR9uH0Hly2XbDQE4IY6iIZHvgrTwE68Lqn0BZ1l - 2SkRCWxY3bzL9VSTVGU7s0sH2a7ZIOw2uUEBQjj2L/0CsgFaaoLqaku9qxYYGWhh + AoO2cEtW1TalP80H1Bc6CxKuUsS8kWvG6gbiWDht4o1zYEJsKyBvaK5NMuIcHIoi - pU7bVHKZw9Efb7zx4i2dN8Mreoo7CRt4hNLEtdRr2v2WJ9/Muyc9c4w/KuW5rA6y + 5/5ezF6BNYIVNZZYoU2hPyC2rjDWMxqJaP5gwCszWlC9sAZJXYVrh9DusFz3S2A6 - TyO4sQUhcQPNV3FQIE5Roza97xsepJPMGd0mdwZMDnl+dg+nuRY8z+4gN0vrIcMe + H3gYZLvowbuiOS3A71LXcTtqmBRIRDV9xvpVt4l6lfF53Oqq08+g3+0btVRV9CB+ - Lbo39qj2uH1OndfTYaFKSj+oa+q5EdY3fWhxSyKFSkKrwKp1majGqCrKwa2tEdCx + QUaFBPDx+7fG+uoChfN8xz7WNY+aC7rA9YwABac3sLAxv4bnkQHWfz1m4t2Wm1EU - 0wwBAoGBANIe2vaCRd/ydkEWE/+rZIXEal/LMyo06O5XcDNL6LyEquaTrbYQujPI + Lcmn0O0CgYEA0uKcNd0kgeU7ks2WRq1TiobAxS3t+x4/iTE4BMcNk5hrUYlHRN+h - i4vV8LnE/Nl0UcjakUwiwclspoj2Uc2TJ7Wt3z8v90HuPEZjm1Ppb3KEweBFlhoh + maQjn8gTHiMpPdF9aESD8z3yPGWKlU7o6hC0GZL1PLfqFA4ZSOkmEGkDDul2eN51 - FJ2lwagDZtbG9aL1IslmTmkDIGEbwzOVRxbrahHsLdmhgViW86ZRAoGBAL/3SEXG + Duzs0lc4tv+PrV+9rEQv4rDsa+lMSohsRsj6JK29bHpAHi6UL6O7XY8CgYEAu/d9 - bpQZ0VOHjVuGh1soo37ajkO340dMJXUEejvZtaSHZFTsY0nSjz3AlUc7E0gFJT3w + I68Z99L/d3ORDo+mqbIOnQfiLLlxfr1G9dwIltCqO1kOQMLqMbNjU1+EPs+TbYGn - S7wcMBIWp0zOykGKaI35Yf6ekizBvBUVI86oKrdnl7Z6iaKwlBWPLNL/jnKzLyTx + e1C/N/UxKLH4J/m5Z/EJqDWvP6pPXsjY7TeVnl29LPbZRcu5aX0DbanD6zN4Fl1A - YSnPoMv+BzacmWqrcBRw4PLOkW5u4XscqWoBAoGAE15GvrxJZpg58EvxsfqBfJcb + 3b7nL+czFw0Rn1hMuwrFsHtMFUbKRhyAJslYbXMCgYEAuMnvXdeYzOXkjN/vRaFN - WxMm9zgDVJz4ubHAlUgBXNm2BHdMQqO0wUIKO4V97Sl8tG/5PrRheoiqXSufZLyw + qf4oXt+/QCOiQwJI9w7BW8rch0cGl1hqj2nf+XvlHKxs0AmInVwkT3nBkKDdjbXm - x11sm613NDuakL5zvethm4PDP0IK0QPFm7aAwFT38MpMMCY6e6gTiDiCjpD5kFKt + rGvUlPBMSldSGx67k0MRoqGSF3gF4yXzZw+++RWK0fggmyhg2NmrKDYmBO0ad9kR - R96RW2+S1mG9w4W+ldECgYA0USt0QLlASaz/69B9ojNfh7rPRrdBA2vAsaL/ukGp + H/muD4Paj3qUQp5IJXKQlQsCgYBxC9SIPIw6nvyj465O+pg6oOrnCFG/ojwfBEkE - 8BKODYwtjOMeanE5bjQA3rvJhAV7VPL/CFudgmkECNOceyE1mEK5xvOlmQMuZ72D + HrRPt+lZziKjUla1U3UeNGj9uauqBXsr0BFg3ycUmYxsxmT6nV24e6kNeilIETVd - g9dodqYlSE4cda1WFtgrhRSIdAckNVi6sWhsUAYdPx6csK5yE7Vq1xtRkoyHJe+S + 3bsvRqM6wq9DqdW2GsiQELTS5N6JXMZhVqoGBl+UsnhxxBJJv53LmSvV9AA9EHEG - AQKBgE8joc9ob5a+jxj4E9Pse6tjqum9E66i7IRZBrI4gddtYqKUW5f25xfcYUHo + Yru6/wKBgCU6ccUsqtnitFwdtI12V9v+zhr1oHYahPf5c3jzjr4bhDWFFXZm4pag - SpVkuyVGIWLe9t7Z9jYoMis9/ykseiqzMgeuKCSZRG+h76kOvCrRIzLRJwKv4MPM + Vi3DpaMGaMBqfFpst7MMrWOVwLPGhEKiBVtHiblKlioxSgQf2rzGWJr1c1o0EJJf - Jg90SP/U7CyTvbvpN6oUbhDXxqSRdyRVuQqgBr3aT1/AJ4ba + V+/ON/eItUzjfe0PmWDrTQr5MxZDHc8ZQloCo4FVHSXhrHIEPscL -----END RSA PRIVATE KEY----- @@ -11849,6 +13049,156 @@ vector: required: 71 segmentSize: 131072 total: 255 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:CHK:sxbeisgiikrymzuqqb7njizgjq:2ljbf6mnq5mjc6kljctyknsznnyqle62unninepj7imdazch7o7q:71:255:4194304 + format: + kind: chk + params: null + sample: + length: 4194304 + seed: 2NVWOXvtwcAFf0J/g0fDC+ypQbnCU7awZoCxcnhpkVM= + zfec: + required: 71 + segmentSize: 131072 + total: 255 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:SSK:ftvoiodksdpqeepc7uozm4osom:tbrsewof5gvjlf6lk6fsgssjs23rxgzdko575t7saubvyhn7rmbq + format: + kind: ssk + params: + format: sdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEogIBAAKCAQEAmv4eV/sJqOOb9XQF6bLeYQDp9y3wcNXMJr6t5VzzYNZr4mr3 + + mA5y18fH30HcmLIHVDGr4nqZ0jT8pBQOPjJ64pCLGQVTddkflwHAqVfaa82XBM41 + + 2EVvmzzNnlOadGjL2QAwFo8CFrIhgN/SfnBJHZpJcJFOIcYzlrgBgSmrRerDT35i + + 2UN4odKlK0EUXQ6XKUR9eswoKUjic5nuowjgnSvoKC3qzBZqiVks+d8DFHSVpt3n + + UzAhwp/LgSC3RP21kALqr/fWZ+6iZtR6rsTX6ZeUFVoD4JaM5SdTsmgEASjT6vVk + + vA32+CVjPmCThnHX5LAKRaUGuA2Z21+qCYWVxwIDAQABAoH/NBEqIGQ02oeb3Nkd + + I2TzT6L+9gp4u28XJezofiS7ncxqcaV9h5dS/Sof+uAlOyaTT7VgCLUm93bVaElU + + f5B1t6bXE5C2eOB3vELadgkNVym5keO0MvMgiwXiDU4IlRKfaEan4Owpx3YPyztl + + exQ9e7RY93fYx3/N1NP7rWhSISo2KN8yimvmYrRe0SpVL+kFCpaA55sZdcGwcxSr + + dFISGFZ1dFf5zkEvNIF+HOFE16AkM07PICFaaL4XfdFsfNE7J/T3HwVrIlsaukMG + + u2Rky79ZUDhxaZZUMutvIuCun6apcq2sjQhPIacH/zzWrbdThARSTRFpeDVNdfAR + + uSjBAoGBALdxT2ixGDF6wXVRZ1sufAmbBIeyYkKI+WJqD44dykDGBlTcZk8YYKB8 + + VC3poAIeN62YhflJHvQr32UvLu34mRVS1AePkBZ2Za6VWqSgK0SGRjQv5oIvBgQ0 + + R18EedGLZnjsgFTSx0eHyQEmYuKJRgfFXI3q/WNVhbRrBAH52En3AoGBANhMEx+g + + X4wxLLRAcRdjbjdfyj68LlmF8sbdkvPDyPfnyp/SUjwAa2ZzJqPraoTD8olReKVO + + tnNsytDJSF050KBRdfe7PHgKBUPxIUQ3B1dLFWilPqaB2L7osWuVMDnl4j/om2WN + + 3up8Ydo09UAfPai8EntoYQzyi40fRT3oTJ6xAoGAbossxDT8FE0aKZ1tgEgJ3Sv6 + + Vd+MUPYD+mdZilWvXMs4Y4kRahaRnARwId7IWp5lBQqFqYyDx7Zsf6goSqVlcrEg + + LpI3zSF58vPz1ILkr/2ObsJy0P6PTJdIbxzeYAT2MmaqivMdvaA446WDL2pzthkb + + xjXWjjaqROe8WYh6608CgYEAgaziTjS88/TLY2m7I5WGD4bLXt89PojS660NnD2F + + 8DK0RSs4CCcMPMjOornSC7TaZL9GgHz3X64azh/O1a2CyYrtGc/USfdf/sLC+f2v + + 1gL629kt/W+dfZ9ONzyjRCLxiPUwrSroOVbG56aWXpIcSlwvDHOgs716MupLffkW + + bpECgYEAtfMQhjXgJrGfS513PistBD01BCoYuurvk0Ja6Ssy0wBNk58/FV7/3KVE + + CcB3JwtQiAfLvTknbvtc5L+a56FZ62bJrgsfvv2X/Y/fPLicvWXuEFoKAxD5O2FV + + cXl8ozjDXcUvEjwAZaLH2ui3QrHn3jX2iYl7ohKCYiyXqDbj6po= + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 4194304 + seed: 2NVWOXvtwcAFf0J/g0fDC+ypQbnCU7awZoCxcnhpkVM= + zfec: + required: 71 + segmentSize: 131072 + total: 255 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:MDMF:xa3yy5ihgbskknv4nvxaxpt37a:qrgqgnkwo6qzhzge7as3e7ywnt2ddxex6oktg42ljxnom2n5ymha + format: + kind: ssk + params: + format: mdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEogIBAAKCAQEApd+7rVZ6XKYuR1tFUM70FiekjZ9EQRcinHLUYZsTriAQDlgg + + W+dW3OJQG9w1HHBwuiKKk2ACOCms9LtuuU0wNnclZAFX4xDk6Ap27lVTneil1mne + + arxqyVeZBVDT0A4q9fuRMDGrV1kj2F3RtBgiXiZd3DTT4BqHRFOrKfXjDOD7sDnm + + CFZyWiDTAAipvr2tj1dhJU7lNnUjktg6koW+JxP2c0MbQqJeIy/99G5crdTkga21 + + jJVpgRJyH+cubw9SjqRRKCKlEKkZlF8b5X/y6Htn95gfHRURoltK+XogdXeABIso + + JZ2L6OHVnIrUew/7nnMl94wTJPPFoQiJGT+7HQIDAQABAoIBACemtzwtJTRzFDKC + + DqyNwDrwkI07Mot1vpQT/hF0Cu0PpI7tQZT+lNzZ66jxR6/r8AKKwcIPjBBFZB8f + + lA0PNtR6QFGq4Ym5zuJqJ/p6orGnfMcnyR+OOV+2hTGIXA3K6TmigJc69Fi9ygwN + + h1TMBSEo/jxm03Qpm0a50nuGGBfKaLk/4Qu946LJoMqZMnyBWMc/6wYKVgiRhsOA + + 2pf6ji8xgCCwhv8pwjJSKIwl4exMpnTQg7h5wtY/oGzYRM0DhF/8rB1XHzeogbjL + + dbdM3Yc8v3VwxJuUdFf96cfKHhID+aDoAEB3sfRpkLqBAzXDdlDyMErVYw9THMUK + + J2le4zkCgYEA15GXSPxWeNXfYCNq7f0L6o3LSoDMzdo+yuAHwGoj+waxidoe1Qku + + hGRDm+41rkqe39dEGuD+13wGKLwDLqXE04b9VS6gMGXhFMIS3yjZHXher99pv5YY + + 5vKwOI5AYUbLKg9QsL+Kd1tba0w57ydX2q1AcVr6jPixFL2HL1CP1/kCgYEAxPwT + + omBIdOEJ8QrNxBhm1p4aCXsQ4nhxuUbNrmoaNgofATcZHEdF0ha7vQJg3LotBcHK + + 7PmqQ6HomXgTmxsLEBRYIkX5CEDQuwA0WMPyUa4OTaOLZ6vuQgFnteGe8Jtu4SjT + + nUw7eeWnswZ89q9HQtSeKcd/PpiyFcddDOAR7UUCgYBetF+6eOGkhJF2MxkvJRSv + + H0xIlv1jEpazmmjNZ9QW3IHzBhi1jysYjtQFFUoQIEhcHr6U8HQFRz+NdcwQGlO2 + + en+hhLJrkNapv/l6gP+hqtgufACBYvfdvpEcx6IRGoD3IXNZs0yp00D+iqaJIse+ + + Eo9VPZsFg9yIOBvD9ai8QQKBgD2ysremLqulHMcJ2j80YWmRZZhYmoZEsWIVwjCB + + /Sm169Ymms/Xpw/RnQXra8lW6ukltNiarnC2krMXABUR2Fo19RDvF7w1COu5eavf + + 29MnkEVTF0Pmfx7fb8txGqZEGOufLQDUssBQZUFWo+dkKQ7Op6dwW/OQQh8+LW/t + + 8s99AoGAWsgONxn9p7gU1XVOIXKpEUD5fL0IC296ibYQ3QA1bc/GALkKe9rRFgTz + + L0WbHRw95qLp9KnV3m/zeUk6C8xqZZlKI6KRSJxQczVYoUJYqWo3Iduckub3oW29 + + uFXwuZLgnBU3W92hYS5a1CFZTu04UEzTe9zLR1fwDl9KwShyG1Q= + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 4194304 + seed: 2NVWOXvtwcAFf0J/g0fDC+ypQbnCU7awZoCxcnhpkVM= + zfec: + required: 71 + segmentSize: 131072 + total: 255 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== expected: URI:CHK:r7sv6u3ihzys754twjltuzzpza:tpyoxinubkl4kvg6hipls6ezun7t3yyqj6ix3vp6tvnzqirzzl4a:71:255:8388607 format: @@ -11862,62 +13212,62 @@ vector: segmentSize: 131072 total: 255 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:SSK:wnyhczogkxbycoq42bhpklu32u:svvkovpiwd3oiabf46fcksxoawo57m6nh3ikefmqyoovgozio6hq + expected: URI:SSK:y7cqgstkbqw5xg64xzpveq2xle:2rt4ydyudlifphjplkxroltqcbayzsnfgvreu7qufhtk6or6r32a format: kind: ssk params: format: sdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEogIBAAKCAQEAtVfoulOwIeSHFEK1T2CUkNg2LeYKROzkIJleLBSecaIYnRh8 + MIIEoQIBAAKCAQEAumqCuPxXTpGEF+uFm+cidAyGqbwWz/TCmGRlr5bEPrerLg/j - vYcqjiNzLy8p+dhY7x9vzy3cgxE17S85kbe7VZamHbiXWT848Q7lcyhApzOiFlGs + CfLcq0IeLayQrhhUdGf41+wn+Y88qs4Cp5jjdcNVn18k6mkEnxh7cHjkCqucSojf - 4KivxU/oXCyDUmKx7TmfovB4EMm4AFtWq38YiztRx9pbJ8K0p+KClFmzdH9alK4z + cTlMY0C/NnVBIjub+OKwkoMWEDlzY170kRgtyHuXzHfuxCwij7huovRTRgktqxF4 - Br+0hrVQXZUDMpS1y6z4+Orz+RP2V+b4CFthawLD0SengqZBycd7TCZqFaPRpJLz + +IQGCyhxRJqgApY1oTgXG1Zt2fKFz0CcZ5RHbatjwxtOCANZyowwfGDNlLjTOZLX - rRnA5FcX/LcdAIeJTSskQuzRjRMHOTumj+mTyW/5oUKQPMZfzLfknzN2ipYKXis3 + F6I6sVx+77tfMJTmNEbk4+ZSA7BrGdXX9w6M9LdQLzLWRgf+cxjB3TEC+6WU0MK0 - VxAyxbUSuYmh66mc1w+3wBN+oS7Mhj16yfbpqwIDAQABAoIBAAJ8bHrKVSOXJHBC + CrieNwcuNvV2oQDBEyxmiiAM6jsXH81dNPkkCwIDAQABAoIBAB/IpT0xGRm2SdVi - ABSJ1NwDCQP/n4+HOaFmHGic9fjiKY5xVX8dcF5a6vJwNDJyM7l40mmUBrU0Y90W + PMeWIxOyRwuNnD4ct0kQZR4JELC41CDoaId7txAkF80lzQ1B7LRkPdNi2nX8bBWb - /cJiT4rtDnNE9+VO4FoS/KZHRzZKs5V8FEUWZ17qqU0KMBHNu661iYYMUeO5B1SC + RmyY7r9XbLPdnwewnC8cF3/XvNns5Jr4t1Awust5cKCyYUaa7z8CN6TjYNGnWfsp - z5oHKZOkZadIaCjXKO7cRE9XcbDOeRSXa8mqCl/CZQEwdr2Zd+/YBaUhLdl7FZ19 + Z32Np9C08e7UzAr6k3H5ujNigQhBoP64Fi3ZpsGbZCH8MWQKY6TBLwsvlBMeiHfg - zdz3ug2wVQKRRwyJf8wa1OIuMrSALwsTUYrwGmxU4mDo5BjkiiUnNjXbBRM0PRT/ + 0gliCDdMc2AQREKEV8fAptVGJg6nX5lC0SyNP4e4SFWPbWJaF8Mb6822bU6sb81M - sUo/5O8tPkaHLbiLN+FvLIGHE7DX1JvYRxafA2CNh2txyB7/krGlGZ7dKgc0S1wb + Gp5rrEdFnom1Z3/fCAIsFvWEZfsRluazFq38gkZyDnLGZj3JH83mVtW4eCsJVq+m - 7c5A0ZECgYEA9Ptq9f12lsL2tLcO1P8fgFpEExt/wGrRQcP+efRNCIg5fgT9fwZK + buO8RYECgYEAu2czdC9YfrH6CNNnsbdg9P6qVTUKL8pQE/Kpj/nY5GOzQI3zB82G - LWCBtNCXJFAOLsQNaCEMixrA0gJMfDOiGl3h5AcEbavvoJYsc1U4i1Hi5B8uQJhQ + qf3Maed+gzrfDCOMeMphD9yET4u6J5s+qZF0JVusO02PQOfBoM2DPYWdxDQoGHA+ - r/pmFR9aee4t0wAOwDOIMQjkdMBw3XnQXZzBamz6F1sU0iTbud70qLUCgYEAvX/K + P8nH95p1eTeZjuEfsREt0Mhi4hnoiuXKlxNcuD/Xzl/p3llFDNiXOSsCgYEA/qbQ - wqW7tpx1IT8FmVG6bBfzhdx8x0gXBPSy6xWQ37uLhvM2zf3x3HyIS6vhdkPoFM9C + qg/3giRNBarJvaw3I2vHc0NMq/gF4pIxjsijekkMjt/Y84zQHovPtTedzcrEDYst - UFVx0QyWEi2o9slLps564nh4bgKTlaWwI4rlcNQi/YMO26g+ms22ocXe+LU873dQ + w6AuJl1ZNwAfBlTJnFPtZRcunDUmeBjJwkMDhxWdSRkOSbJD0FISIJiGB/fViwnY - Hw8M2ege0wKyD4JV4j6bYYiiPE06XQsnVYo0pN8CgYA6ZmD2KSkHAY0cQXNItVTG + roxqfi+48SPzMTDwevDe1KQD1iwTUw3NxAIikKECfyMjNoKSXgVjWX5OJSMtPwCw - HT6TK4AF17Dws49LdUCT4x2JfBkOGeq+7H2fJAaTwn3PCi+D/jTmSEdlCOVAynI/ + vz86sq5DQMB8v04/imtIRlPUSb0szBMTg0BYJ2BzqV6dS7laONjAgA5qJH1Inncs - RNgfqsiUeGNUbdhE2jDzjV7AMOquvWCmwtNo/6Nq46uK3D2n9eDmh48mgeWl9m8E + zpoylhiIclO5IJUF85WVd/9RyDLM2N8c9mF2lJAl3KTtkQOiNPTwnZnHQdLJQzMQ - keTNwRLRVIYfHmg+4/aA7QKBgBYVDzaxg3dbMhcGtgtQx82S2PDvaab7UptkPHlC + blIdplkLos4N7uR5t+ECgYBzz4/EV+CbekDhG+wF68VjwYeCnw/GgdTDVvNc2Vin - kRhRTYgTTX6hqg6MgIF47RQQA7pxEIQ2AMZglhhWM8tWV7d/djhv23DOYg5dOXJa + q4MfkyQKl3aq/bCn3LRSvC1vb2WPu1BhuEBzqAV0DqlmBDFJsUJMXkuxgKx5QZrg - 3DPStKUgIZodN/ZoJHEjksEetZQeLjsAUPoPn4/tT3yZLpLnwsmR1335beSryRCh + G29dqBx8XatDmZ+O3W7PPuIKCp9VupxP6Qo1+MCIFZa4gsUEddcc1wyuz+9Nfh8U - w1K3AoGALMPx248bmEuDpoFE6eN2UN+KnPklcgnL1mES2S6pTrvMVjj9UqXz8tUw + IQKBgQC3YjitqxP1dxzZwTtwJ/c8R/lLWAasI6EBjUd7pFZuh/9igtV/urvpuqog - MrkH8MzLAk+bKpCCS2lk1W74elMokpwOO8fpE7CgvXnw+/+Sx00F2LQWd3CDTCLH + Zmk7gs37xciTlmcDqK9pBy1nBg5dUyJbPjd0TkNPi7+9cGZD68GfvCHfDykuHKV2 - oGN92BX+ro+XMH81GV2ZtXZ+i14S/w5BWtRbWSaeH1ziSmQHn6E= + zuSE7JqM0hDEC+qXRcw8lg0lODM5X/SsUqmbxj7yWrdqf3u4rg== -----END RSA PRIVATE KEY----- @@ -11931,62 +13281,62 @@ vector: segmentSize: 131072 total: 255 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:MDMF:w67agpynf3ekrgn77oodf4e7pe:tp2kckjjzwurldnfquf3qbnrydtcxyhs5ctpxgd5ryg65orobgga + expected: URI:MDMF:nn7eh7thqv5fcx66toeltgxtfe:gf643o6uocn3hvmapcu57cd5teoulsyrqlouysq72qegisvc4p7a format: kind: ssk params: format: mdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEowIBAAKCAQEA5RF9f33SaUdiHVu0Of8gxy2RpbFERzN1iFkaTTL3cJapemQ4 + MIIEogIBAAKCAQEAxWHP0UAVKpUHireN8BgAJjXIQiynm9HdoIfvGrkdnP44Tghe - ljCNiG8tgKApsFmKi6F+ZJVjqkZKmOEzvYwkKRN1C3/qsOs7p45egY8s70E8deOA + 9MzIU50xT+ysUmTrXar5oBRDx98mv8YuJ5dpUoHl6Q87uVQhS1Qgpdvwh9MWY1gA - X7ryuuRJAG5xaSZ+4VHpWM1WZnIBqqKyPZIuROwcjDwpsDbLU3+XQkyNhOD34i5y + NbOqNpveTvRvHGSLyxrysuUwwjerumpPWTVvhgLmxOryytbwsDb2RmS98yq9pX5d - 5revRI2vPjg63tmDHZ7CpmIay94ayBlIgXQlmmdF2KWBHYslnkSza6eJDe8DC+en + wsyuOf2RZ4yRn3y/QmrbAtui5iCvpzkmeqf+kuXJFYcbd9S2/pNXpLL+J6UzTovJ - IJcCfrZ0R5wEgYKH+YxW6ZyENTs2xApvdg368JuD9o76Bd8zKsgjpfSt9mWyjTQS + rySjXAGqu0amNNbkQxvL6Ofd+ZtOmZ2zGHQiFIus0AkbjaM60mDiFd5QwY39lXUK - WTgxodnddDiwu7md3LCqxvxeJjbFUXEfDzBwrwIDAQABAoIBAAOtEJFZ0LqPO1CS + V3VTejHWrS16wbq+Av033vq7jn7Z7nOGzGg+yQIDAQABAoIBAFXeUomG5m5q/Sf0 - 9sigJlkDC117vIB+AzHKPZpsxnXj60uReA7IhT3TqLOOV1QKOSeSmblE8nceBDc6 + 6LPdzRrSZPec86HPMCqpWHT4uZBV7GrOK3k2KaRui0ho/yKtMtPCEOz6Q+6M/w+J - afemlOxrmzzb82HW5XR1ou8tcaPh9Gd9eN/CMQA8eKAbiS4OFgplhyzZ0PUVzoJD + CQVCUpiJWFsGvIXIut9JjxZ403BTfbbkTtsN+WvebV1N65SfjU1jwNfg61Bi5buo - H10HswXVIUsKnFjhF1etFQEOUOPAfAtEedGTN2ReGTQrXcLBwd74D0Ha1TFEcTWX + ijKWE5lqY7ihOdTSo00V7Bf9tcE2P8NTXlM7rhZ47QvFkEwNoxHlSFJRMp+5GN4Y - KibVD0lYiE4poPPFiAF0KsHnHYidHJ0WjrrydGmBZb93+S2IETff+Vrgx5dWdnkD + CTjIjBUvBjda78Xl8eOLmYg2Ct+SbwW714K0hNwVzz6//k1oh9GtCtD+n/ST7h52 - g/7ICYw05+CgkWG7FMo4yTRrIFaU80Nrrct+EMDb07PVONIDmTRu4a6sLVYIcDsD + 13mg77iq9jAVqj9NmRWdcd31mX6nFXQ3mK98py6sOuF3NxiFMK6Ginx2rrux7c/G - nW7l48ECgYEA8uRPli1whnxBOxNV87Ko/K6Ji0pAsn2TgSttzaDqD6HV+wrOSVgH + Nmfkp70CgYEA8fV45HO0S9bC5Rk9TbFYhg52XhXue7Mj1s9FI38Ft+ZXeT7Fa63j - HBFcLBJ417w0rq0oowe39dSKSO0xt0O70bMe0XvBxQKk5iLS8NMszR//MB+qz29w + 6WfOhiGyJmRR4cWdAJr/RZ9xqNyCZLcUdh3pBvpYR9dUyVpTxUVv3MIFHWpkc433 - XDt71aC68p+wEpQqVC6iyPd3S9ZCZ+/1Lu1Qh/GFae0ocLz7FyAGEsECgYEA8W4z + g1ZEAPWzdHQG8yVx6/OyOzsUg/IsPGwc2sG1h4OWf6SC3bGVic3L+8sCgYEA0NYb - FpbCX5xFT1V2sbPHOBT08yv3qiMwQ2jlCqpsfv+aV70MRFFi01BrXUn8l7gL5dcZ + 7KpnDiwbZ1BlcV4MGIYL+Z7N85/jK7oXBJU0Icpv4OQdryu3JFSgDHp8bcvhMvyj - EWORFZKKXI39zLunghrPgWiFUT4DqnQwT9e7dwWhnxlo4HpGJPtvxBCxgiCPxY6Z + IqFXH2YkiOi/xlvWnzj9QAP7FViuIb0Oz9bWiaNZLn7QB+Bs3oqgiLS2LvhbSQJq - xpcELoyu1gRZFcxT9w+zdmQ7Yk0m0tqS40+1D28CgYEAxOXt2nkVef/qRUCEcdyH + Bx2LGIEGjnPWJfV3lytqCI69qbuOlvdlf2CmxTsCgYBazLXLdahJdZS6CNi6mT0R - /uZiW8cisU75L0IMbiAe/fMcari0x2ITyV4NUTDcQ06vilaW1aphJ2hXfYzCu6St + Qcgl0rEmdrmSWUIm6fopYyWceHP5zs3iv3P/XhHO2oLn6RLcMU5uwEEVD3tXdGUX - 8e15cyoWx2VAVcsvIsidzd89WD6jkirtc+dImMIGKr7m1fjEY5+2mKF7VL/o7ybn + Vm4mkjgi7aoBzgX11/L8s0rcGRsNSk+CWBM5EPuBTjF1ea3g0BkopSkzwuPa4O+L - pFX+7WUN2PPGz7Vy+qkcI0ECgYBYTIaQz2idkUjkIAy+J1NIVpnTyhPVfPMs5FNI + IHqRGk6WJBSAQa5Ogo50NQKBgCo9pJhSP1YWhdR35ozvwPKU6ocrH+1PQdvuYAmF - mFYACLnJNxIidmWfhX0O7H1ee+iWEhpP+stYSXUjLqdRVpyIAAg+exyvPvAWSlJV + RG4xTD/o5DgyV3D5zQW5IMH0ozB0+WpfyAeJ2Yn3yhKNMPQzysXQCFFhBpe8beqM - EUC14jBfQOrTlsTKx87ztWtGfWQ3y9TABgF4iOl0yrhOOaHH7U0kkroJVNBLM7ef + QgjFCZzl+Z4ePuckkyQTqWYGxjAWVOvrhd8G+hSGSaKT7ASfu2rPtH1Ieqb+k4EY - PUqqLwKBgHRcKRmJ32sGssOvJqfbgnR0+qm0TyXDbt0blFHrBwp61a/Gv4H9AviI + Q6NRAoGAUl7vKlquxaJmvieV9q34/2NCFUu8aBlLAK8w3sirAVM3UU/ISeEYdJfN - uRAiVpWasnQTKxrfnnnZ6sl2VQDKdh6z/6jDDbW0ps+NxhomXtUIgVl6iNuQfwFs + OCumo5DKFR5COKu6hkeFhqRI3ieukX9k3FMUiNemnquubGvTQUlkwEJjUv0IRfTB - PK9qwLv+lGhsJyRln5kiO3g4yJjl3rV6iU3CeaNO8CmA80nNg+n0 + +gWW9npE1Z6LUoQ6su4iVRid9Goq7nLj3lfFCT99Y9upLbYcGEI= -----END RSA PRIVATE KEY----- @@ -12012,62 +13362,62 @@ vector: segmentSize: 131072 total: 255 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:SSK:rcl5ruuw3bwcb6v4h7zhalhvxm:an63dzmzwznxhpwrw4umbnwcptpo6xglkpf37wxj6qfqwkj7io3a + expected: URI:SSK:irynodq3ps65ftyonzektbciby:asfulhlz36ydefcp6pjz2s4osrrs3eghnv7um6kqogaiqqcins3q format: kind: ssk params: format: sdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEpAIBAAKCAQEAuvOrIeIsSg11qCGYkmxbAO9lx3SjF61sjGRhQpHALaopUQAi + MIIEogIBAAKCAQEAtBL2bv+jmSUrBy9rEW+f3y7FS5qfLTP/jryKOd18rpIELuj9 - LNzBFdjJIoVYGNkI/xFrTfo3YwkWWvvsquvWrgnjXiLEc2hsAIq7t0tj6sbpsCG/ + U63C8bVHvJj5ppuDRy1BGA91tVLgr0bLE9GeXWxI0pNzILiTpADn1B7xjpAzLAJ5 - SEyq6v4cP7MXS+koR3bY5+GSOCVLxhRMW31AjkC452AmIwAI11GNbbdxiLqZcXgv + UuEPDoawcq6wIM0t5uNj7jrMGm5jW5C3Bq4vlYz/Jxfxz7UjbJtTyswhzRbxvYET - 72NCIEiiBNUGbtrvHN2D3L5lH9i0MNJzhw+XBIY2wV3GJ4TN+ox7+JlZDuQR8iPq + 7Q5OhIOuosSdgrlUb2UK/rR54hDmAen4eaPi4Z4N/FuUlsL9rAURL4bp5LBV0IHs - vgXzRvKH0t/0u/bL+eySBUQyYP4cYIZNWhsETuaMU5HfXxZzAQrcFA3Qf1T49TCq + gkXUhlgXMc+u8DgfSCCGaWTk6bH9rXDhMBbKGKVhY0uU2lX2B4r0JFql6LjAvzrC - kgMovvMOhALfc74PZU/KuUeYQRpqZrJZL4ygzwIDAQABAoIBABojGAFoOA3wXMsx + I7vrbquJXCKTOVRE2aCSvqb3+83slsfkt298MwIDAQABAoIBAD3G+aNX8XTDMxFK - Sz3pZT23upCMp4KXbe4g2JIwg2A/AoC/FogUIvCVeuVXJC5xJXdXrZtcyKKRci0t + e8VmEadcINSQrcYwvh5mYVd7vGAZePTs+qfAB46rvfoeUxuM44tCI8BQ4XV9Atcv - PHTW/RAe9MlD7hg+eJ8Ixl7FbQY1YhWMWkxW36xz82oadT7ZLZb0MDYXDNTJrhLW + BtF8SwPK9+rCKi+SExijNOVpXj6mCuAw0xWTX3qAy7z0YDlImtRIs4pRwp7Yux55 - 4qgF1mEs/kzm3d4V0qab0byNoZNeWbAkShqGPqKY6ctzUJxQ6QGRGM0dvhyJRez+ + NclFbc61Kf3r8YnsH0SNvz/mKzxz7NZyrWy+oN0M/VBr3IndorZ5ht34ovOr2zkV - wcqjuzdQJ96hTkotokysmhyrojR8FHvA2xVtsMdYU0GPUIg/8j6P3lXPdNpROtbz + We5rbMVpLtR/tYHuwtjH0J0aLs0TUCmzygTO2QCcAGI/XE4rAsCPr9G/avQx5jis - M7k7YEFPFF/ORFCniUmNVITG1CFWkVcP2kK3UeRghtx/7XDYalWmiDVXx+y9KKOM + DdvW0paMIYPAr1ioYN0iUwvXQVnEeEWINepKlKul4myCdM/OdH4NYOMt4McI9Mbe - 7Ie5UiECgYEAxHqj2zowAtrPJP5d0rzzIE9Lv+WtHpn75vUUbwp8Y//uxJw7VULR + WOBaR50CgYEA1uUGQgTiyuNXLRkhf2Bm035eoY2NAngefWadNlCJaR9FwGkJYeM+ - eNeZOfCvPnOSTUz+dEZUSofi2ryykJfF/XfEkASbNrrPNndCtkOAj+RACaANM5cw + DAABxnjrXLtMTboKuZN5CMSetIWTFA2HgLnLLUvAx2VPaVTaHQCA0Mdpwqxaq5uN - fhK50ahEPapsCew2SdMm52U9aHfgnsyOIdaxw8gBtS0Y+yhXJEzhji8CgYEA85Yr + Mhk6QPeDVEWqZnfqBpT6wBGvus9Y1CKaaoj74WAsbjwwwhyxcqjIyc8CgYEA1oTX - 9S+t5jBvNaDjBUTegj+FCuKAn7amH81uHXtYUeBbZdRa7IOjoYQLXLS69W3lH9wb + wUXNngPQRL4QBsdOJz2GlOWuBGee4ZW+GIVw3CjyI2S7+xWHLaAG8TsNqg6GHP4V - 140PdjrYrGfCkveLDY4Dfj8jwrpPx3AIBG7no6huKhtW2JlI2heUo7XxwPvd3VFI + MLp+XtTzGJprnK8nF8n5gNZIEUr3bPMBJBHWZzNb4EzPBl334YMORuE72EcTRWEa - A5sRKBZSE+8/8MBIBWp+oGRtilOGWTFuqh4PD2ECgYEAusrlsOx27J/dw4vY4xsk + 6q3ZogbepWm/piOG0oyx+yxcDtMpR7IZrp4mFF0CgYBZ7eoagrTuNwlqZBPynEMr - AZmhqITQu4EljYN+s7rCW8fb1iu59OsbfslqMT1zPepeMwN5/k1GobzinZY8JV9F + yryLWxNhrycDT4gHDNkUVvP3u30jq9dxaidUCZJlcjRSasLGOoLyOmY4IZYVVDwa - qh4NT+YxMi0UBvIHCITQWvxjLUNuiZe5UIK5CmvwxLebEyvwyOrn16HWadVeRVqv + kKYIRKVeTHVZHRtR+73soScPQtWG70e9aXVJbstU3vqaeyBCtOHiswQZZ2BDFmAM - 3dfhFQK3LOn9D/pgLnCxF50CgYEA52vtJ2y9Es1BWvoXtZHQtH4UsFqxSQwGmIBb + qVrPTFILp8C32w4fb6bnXQKBgDZmwgB1n0tvVCXavV26tYsmAzdHd/YOATDcNLUr - 9baSGnfFXeF64OnQNEt3YAR0+2gFH1fHO+rQncsav/F0cpysh5w8xVzHZOINmbVe + Qg/TInTvWuy17O4ZIymR/EkgHcrEdMNCyEFsZ6nZn2jA0n0p72hI70XTaSPsDGIF - aJfViy8iOu7ue6pmBI4SsdbScD7acsIeYQ6aJjPOlxHe9aQ4yKx80XWYfKsOIP9N + VAYf9DDRyb6nnfFGtxwqim6yt6Rkl9rj88kvTM9OHhgX8lz66Tf1a/MmgdV4ySKL - 3GHifQECgYA3DLtshPp1N356rWpJJuxzrTpuL0ROct0zWR+5rdBEIy7LTz+Iotwn + YMTRAoGAG3JKI3n4MM22AykwVmCkhPDoZk7/ilGkdE7NSrweNoHvePdO21NBiTfU - gORrsE/WG58z7VS0nu00dfwKsSaP6dK0+DnrAhDUixWQmAutQ46nj+uyrg4cDAX4 + NTIg6eKHJR5EufHTLNvW54L40jiGM6cSJAXUEh1BA9ODsPCsi8YVw7FRPyZBP506 - ZqringGYaZeXDbm0/JcmyRSPixLcUYrGkBPl6/jeiJbTmJGzau/xkQ== + AqEaxdVFEL/GB35RpJPJJIvExR8PqyDFxHmDZ3WEGGPPvXb+Llo= -----END RSA PRIVATE KEY----- @@ -12081,62 +13431,62 @@ vector: segmentSize: 131072 total: 255 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:MDMF:yls2bowrrs2wvyjdndkllchjjq:fcl6llxuj3bmzfwadvjdswvi7ul7y2ikwvp2svf5gblo6g7zvgca + expected: URI:MDMF:umz5bassl5ndo65j6syf4dekqy:jw2rkz5kts253bmupj4vdbxw65tdtahvmtmp2nzzcgglyduus3bq format: kind: ssk params: format: mdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEpAIBAAKCAQEAqMnCxUIeKVVqT6KEaMDpJykvC9R4eGTbsXJI8dO1X/ESsyAT + MIIEpAIBAAKCAQEA3JDtfIt+04LLAf9VfMIMYrbtFIzKMlpLx0K7B/JpCtHWhQvp - 2JK/Msh1B00wJvuMPs2zApZ57heb5k3mRg+IVqiYcBW1wnnsj1vijT2pgPa1ZhVB + ozW2ttJfwHv+5dB4ykoqerNpdhxJnOxxrCdyHxf6AH7mI5gA8lFsH/0UsOSXS6Hw - ix0/eN9126jlEZGqGjI4NnuxufDNO6eiUKCD3qbcyF6Q+TZmkfn03DWEaygmU+DD + 8fMJxtn4pH6VF+qmd6qsHYyb6IDb1C9EvxndZuBl2GOE30+WjHcaMJ0oCbzgrF2V - nMcTuxqL5tcYDkl3UbiJV9CjDmKUu6hJRh2xryuFcj6WpHJguXWu1R087Oj4SZqP + mUPTuc4dreqA6Mx8/SUIWRQxE6b2ozrW+eEH6VH3kRndHWtsqdrl9d7vUdqh/ye2 - VmTo52bbuKg3QNW1eeHfDWaYH550oFRLDl5ojgq4SXRWuFGA+cLnGu6bukf+bkHq + asady6qLCKCwdJ/R9/NxeoREpAm4pnQzSUGPxNHS8EQcgLk/AvrGH6EcP/rB+tZ7 - 68a62txhBtfDdfoLYiQ6270dNS20PbY/uRxzowIDAQABAoIBAFDqeH8MVV1HX3HR + /0P4fraqe781k2pPGrqXTD10zZZgtW664zxaDwIDAQABAoIBABePA07CN5Gv7q8P - 3VxCrwNhEPbA4wgEgfWtbh7QeXEHJwnMZPc8UoDL7J6VeHIXwYISJrEk5ksn8ksU + 7rmcoGYK09fWEeK+8kkeP4vhwIZ/U0Jyu0nLevCcF84fcGJrmftBYLgqYaFT9Cjm - KUKJC7lPldSV887JmIiZaiB/4RS8MPZBVmyUlushZWTqsPYdOMjaLmygG/Gh6SGi + uF2C+RWJIhLbewliOvem6r2f8o3SXLafXXT6WJj8vyoSuyoKzi0J9chSNHTpDpHj - GYRBjzZcFBfSjfmLBN0SUTqIRXUAwI11SEYmEJZl/5L/CNfmVbxTuGq7A1Usu6AK + Wpxuzs8mOLqcJp0TiykFr65xms0vRJGsZsqyA8JDPUBKRckdn6xtuaBgkYCBRQLj - RZEGrDaPqPip/obB/WKMt1e4NibVST4q+CpFewuOiaWL1gZPQL/lL62HdOAmU65i + iHgSRXCgI+TTL/TkJzYEkSFvA9TRPKYOzuiTjdRyZWlZCK5r/aVHYVv8BZXoRMZ6 - TuT6645wvjyTtmYH6u+nj4IpR7j8/YbK71ShjCB47uevWu+75lVGVhlSDuSIaKb1 + zjHqKNoBn0/JWBnxi3YNO6bTnM/I3IOSDoCW7pd2LQB65J48Hj2glnQJanDJ+M0q - TxV7hgECgYEA4EfVci3rSCQ1O4GbmaChfvQeg/6pe5ZbwQ1BfYz6LuQKSyiXrDge + D9h0wQECgYEA/ZLlcPEphcFk1/8nQf6IZsd0ZnB8TCYSIj+ahDyzlVLUqPlRwnMm - d+47+QeDlxDM+lpjWBjzTS1eD4Wq4Qy3EwW9HzChDcA41tmKZ5r9KLctbbgqfq6h + kY/Dk8nF3f9b3XTuD3AOYtMeDpzH1Ji0pCpuDrBJDj7bVMeqO3bvG9I4D82DYWrQ - xgFg9GxdqpopHnGe3p6IrbSbJyKkd5JyNwN0dTowhsh0nkSBE5++fSMCgYEAwKjM + eBsPYLn1sKdQ3E4hVt+82kha6dJp4RjkINiQ+1wM9ehBPQbf3px7gtECgYEA3q0u - 8AzqHSH/ObxACrdjQFM5q6pg+Scp11/ODYzPglwWVEceQsJ/tn+w2pz1hypJp19Z + sNTWMSRVZSa8AHk1bef2CwLYMPUWPK9h4feXcxEqWBDwmMKx1cGYqy/lYe6LAIr5 - c6gr9o3oOkfWPsSognyzHkL1AjQFvvuWyfPg4NpVRq2g16YYvIWaY6JrP9GMxgt2 + +44V6VfrNhE08dJisR729BMtZ3oIQfpOVPqvGM47NqZi0ckyA8aKfh4EOpzYQkH4 - fiKzn6dg9dUoHK+BdQbDGshRJyXrvd0xzG7N14ECgYEArpD6385B7XrRPCnbNK5E + pzh9fjwJFY5rq+qNVNBlJPwwjSa1C24Wbi1sht8CgYEAxfZUoaPk4sNk0ywjneX0 - RQ45mj9jJ2CWtiJdMR3DtS+lm25S76cWf/6cC27/y1s2UD5+SJnS9eUz6xz9LgG5 + 3yh/uym+IEToi0xUeUBagw0zcOeT6Na1GZa+/TXc/79IHNAYunyk/ooLQSUs7NB6 - lULIOzicgpl1JDVadt254jEBWP8ZhFTkcbus/VJDbYBkNN/26gu3Eo0anlFmdfM0 + 1l85pMYDgteXq8xlHh9v9KxdkBjFpNwa/GlDzCPhp5Q4EIX+iTAK4+7w6vKWLmGc - lwFHad8K2j24F1/2n5GcsMUCgYEAiu1ukzAM5qMsY9rfJ5skxC7/qE29jg1yu6+H + V/g618G5bJFxvQ8M32ITGsECgYAnbjlHXNj06L8qYzqFRvFcHegmuQE5Yhzm8BOA - a/f9b1iudWmvZZ7R761Wv95to2GYKUy1uZQs16dvLg+9bBfuF+KKW6kW+ta+ygCs + JQyvdomuAInqMwe0l0yGe7u9pLT+ip2LmvRsVoIzF8btT1jkjlwiikbO/P/7VuyK - tMbbg+mNkuED2l4Y+mExeuWVhzi51dpQQRcPBnLxlXR3b3AT32rX6IlJE/zhaVGH + Bb39wX8gxUPYbC0sF/ssK/qJun5c9TunuMwYD194brjIP4d5TlGqw/GA/Sqv9HWK - Zo8EeoECgYB2OFnIATVM4BUAj8snqsnFKkL9Rq7gwu2eHY0y3uOKySM0TmydfFKr + WwbNtwKBgQCHGgwM8CeTCarJGPf+Z5+VJyrjAtybE31yhlLpPd3OgCxxGcczItsA - etrv2vkeWPtMwpXtEt4A38/pAnuRoF7f8M09nx/w/W9wk+c23mNf+VBMoM/lX/wp + ipKu1boyc+nkLpOcpm7DvRK0RzrFu7XXcX03mdlxokuVuuxOSbcCUTTkWaBUyXhq - ZaRAXyXqOVPVEE4MKrozNaOCSlmjxMZxQSLB8VLwOHB0hcMkI9LW3g== + PUgqs6k7wjKRmpvttA1MC3clLHEYdZ8uDPVyusNKdeRizqxyY8ic7g== -----END RSA PRIVATE KEY----- @@ -12162,62 +13512,62 @@ vector: segmentSize: 131072 total: 255 - convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:SSK:jofr7syf5rhfw7vic7izxstipy:iav4tic7zeutmsedt55yib7njlrmiazmoxnqgvhekm4tx2eqd45a + expected: URI:SSK:m726xdiezljlbxgy6ud6622etm:wycmmsogqcaot6j2xh7fdfl7y3csgogszo5bc6pfaoivwnmlvsma format: kind: ssk params: format: sdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEpAIBAAKCAQEApsC64mN+RSE8c67iLI4Ld+IJjvceNF5gBI3EZ6VgvZJ1vvJi + MIIEogIBAAKCAQEAirky56Wj85ffM32v5KlYNEzn4iV1CfBQCeBhkRUBqdvIN+93 - OjAEc/9LPDWa2Cdj7yvH91aXaba9uzXtScmskZ9/A+qLE1GEEYSP3TheL6T4p4Bi + MdiwR8yeuJRn1A0xJsEA5j96+xIKBg/ID4PsMtskerfMU2LtQgD8LdtKUA25kx9w - tTO3v2YWu+RwSzc4WRqWKrmUmrpiUVWNL9O0ch+oRYZYJqcvtZGBoE0a4Ka1DjO3 + 9UZWqap/90DNf+yfjh6SolFr/riRtv0pdRLNz2e9pykBAfhnyX/ergYV2OMCeH+I - 3eCtCnl+vj1zybQv0bU5IizyNJQ17qaadoWkTBmE22t7/A04CUwK/lMaQEyJLqHJ + dwoUDB9Cqu9/6lLYkPx+GTiw2G58Sk06/YHrQmtfDKDUxbUmmnM0zSqb9JJg5Pve - QS2b9N9wjLkxUFMMLgWTjtQuA4w9vDSl73iVAK/GnzMvXOv34lY5kI0XW1ExoJ/l + wEXEGqXeLgr/BSizd8R3ChJVYEp76uh2vNooNJl7Jp0QPvyKbg4b4D3CePFoEoi1 - NNU2anyuS88U5r5hXTcfrJR5ZsIYxoiTx2gv6QIDAQABAoIBAAN7J0OZ3F1U1Opc + /Wub3M+Z421Q+YNb7fuX5AoX5y0AJpKuv0vn7QIDAQABAoIBABX6NL8fzhtttGIN - 0qGnuvdPF5A9miqxdCtwKrMXtZnrhGv+qxyIG1WxFQjeHRwJUXmhFjj0fK9zJkmR + N8RXct6oT3VTv1jRfnCuIG7yf8a4B96avMzEGltppryx4EgnReHwolKX/IURzM48 - 1gDp9gFpvRjvtOTLuTg05lxxxGyV8u9rO5RJDrtPBic4vPvi/JkGmC8u5dpnjO0h + ilKw8Qb2km8xtriAO/vguZMPQpTvs7aD/OqS1/B9z1Ot69CoPXfvzoXSNfSPK6Cs - /jMrBhuyS7zc0bsH1zQBBD3ckjSxmZohdnBjS2ATMtgg2mTjtp53FU6GanEaaGjc + t3hxf/MrqY91zs/P2auB61axJp1rT9bIQr9zVpcfXb5dr468l/ea+pISQ3hQDskZ - BQUlonz/gqw/p6O/dY/B/2jEwKvENRq5hLDAdnqn83tY85LFSuXaTZVGYFCdqgYK + 9AURZ7a5uuUSvqInlKI6NFHBAQevt2oC3tdcYn98scU1h4Sewgw6571Me5vZiomJ - YZVuTkpk833eKNUWVIEovqBQkcn1R9JRx9SsvZZK5aHg14+4lusmMbMaXnOTHv1P + cglaSJjf15FtFaS+7j8yM2Xpi2sUk9O+T71uNDsoB5lfuNF9VanMADCnXk0qDZ8W - YdhVKDECgYEAy4cNQZAK7fBXHoCMuHbXEc5KwMmfbm4Spt20mZJR/DxgA2CagKLs + lj0HtHkCgYEAu2XdRh1G6H7Rs0rnEGwFjrMwt0o7tilo51KV4pui26AR6BAYiPP3 - 0S7BP7X23R7Jx3vY8AvEI8BvZyYyszzy5tsnbFXiNQ1LMkY9rWQ4sbCeYaj2LkaS + TJ7JNebvZF+dTCIqqFK2gcb8Xa5c5b+9nxI5VFTbaoez4ad9f5kWSBCD6I7RKJib - WeSTXaByjbRTOTzZsq6XhQvGrmxtRAmrI2us/I1DbvIB6b4e5QbbOtkCgYEA0b6G + +U0ATHVG65n6k+k1Zi2ITyf/W0f7jZIUnB+26B9W4Wzfl8WS9rEuM6kCgYEAvYHF - XsFWFUtJGdjYC6spvX9d4eIc1Flr383ahPcSWQC7akaGn6HNnVWbiICJEJHrwPBw + AVuN00PUDREvFHhvp4SzrUm+W4A3o00k8hIPkLsyvsvrSs4BzzaHW8+35AkYPQzH - r25QVEeJY4MGxrqvAfRn9CvsvWPQs5IkyphSWQTZKd3cgf0JEb+vZG3PWAmjAJSv + cKXn7TKvjgy8yDdTmk4iLkKuPNBWShqWY7M7bc5OexIGuIFPkp+1WOq9mp00uwqF - sRgh51jd0qHZQT05qyDt7CXUVx1UdlZlBb6z05ECgYEAmHqOgN70gTx9WFnAk3Zd + KTgRduvaJ//xDxZiNsNMYsZ64vp7TGDSOFbBPKUCgYAI7JvyB9jln4x6/lkspghJ - PHbL5FFpg2ctzBvvcNqBV7K3z+/w8IyfVTxtBVlDIHgvfacYaQa3pH1IOQQSGdyA + uGzcfbOEREqToZIzvXeu/9t6crHIa93eDz3DzGCgJhGGm6XuaCn62jAQggo4gr4U - slnf2DcjqNFT089x59Rc8Sq8Dbhy70pp3LT1fsB08hr1+rzO8CIDXGbtK8IJvl5r + Ajkqs/PTCe1eFKzcU70E54xwmcSKK2JaJ/mYqokbFTUisBtz5z0zj9MQVMg9ALTs - +7ZwvCjtK1JeAoswRC910UkCgYEAlNqBZFgTrtMaUySo10cnPVxaFYgya6X2wAPJ + jnIWcc+gYp/vSWBrURrDKQKBgHLL7Z9I6r0T1Zyk0DRCUMDVrlJG3b1oCkwuKzdI - NJpgRBgX6imZO0tKsIFj+3E1VTQqO2iooGhKzDVk1OHVek5dC6cX65sMzbA8GmT6 + oY03GSJjPQFvkcEIcy62wdqtd7VjzFz842XY0mfmZ2WRvl82/ZWwZwQH4H27ZWa3 - hWmq75BYSrUw3HPm7ti6Mi0YfOOB8lSTh7yXuyc/bk/87qbz+XZKRFDorNac7csM + 6EQ4OWpsHQ4fpyhW/vACIyFKIes8EDZL9KhpbxnT/R76nDw4Skl7mm1s9svpyu69 - sRIRb3ECgYBxu7Etp/NBRo8VtRRLsqndCL5jLxON66cRq8MK6kzJFhzgdfp36ERo + /wjRAoGAesaYVXpU7npkNSmhZrlx4kUETa1lVvj4TmT4LM9Cp8gKpVPeFc6p6khb - NV8HKvg0f5c+44GPpi67zyYiMWPwTyE9+Lq24DRdSMCec61G2E6L8M/q3UtlK/3o + QBUSwuYYgtcvf5oGk09R+q/KWwe9ppMmj5rUfgBkEVZ/ayC+7GQWEdtZHg6TAppZ - I8hpC26neWSUf3hNur2yDbufdetBZefKxGhS+yF8XRnrtv4Y1CRQhA== + VsDT5FKUq/VvHFHciSCvQKosCyQzAn107htm3epmi5sP2i6HXZU= -----END RSA PRIVATE KEY----- @@ -12231,62 +13581,62 @@ vector: segmentSize: 131072 total: 255 - convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:MDMF:gsqslvviaskirtqbx4wegueyyi:2bcvqw7x3szvnmemjpthphinab3aqolfgjkxigw5ghfhuzfbdeoa + expected: URI:MDMF:3zidms5g7qzx5flfxcadclukpu:egrqrou64kwdseq7ezt66hkyjqbbjpalocp53yuiqjnh5ot5nmfq format: kind: ssk params: format: mdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEogIBAAKCAQEAstun1fppsTbRKmHPeu4GlX6VoCxCE9liDqmewpjDK7hAKJ7n + MIIEpAIBAAKCAQEA2Hz1ojFKwlGap0pdBpbwIR1nUr2lpVkCrPvXJv0hD8PEGa2c - /EPUeHP9mFJZ2v5cY5E6KcbCrLJYCkGPUS86BfcumyINoQBtSGo2SV5iCD2yUv+f + FDXahwSgTqempSiDmxvG/HHl4LRuvFQmEhdTsg0jEolie2h/33b02GQj0CfP7vw2 - L4AerAcLWPdU81sRd1TU69/SWgQklqFh4ie9eGS2AWf2yOWJqMyl4iTcKuu8wq/4 + AxQmtfVWxf8sIP5TPQ1tUaqYWKww36IObzAqA4FnKLHuy2YQf5c3f83jhLVlDOYP - /fA+iEaryk0oDvMRx8lQsQf4EgaSv/V2ZZVSbSlo1iA9TawUbZxsVtBPZ9q5ht1R + wxvSucWuCjrlBOfXT0/NWvmDbPvkiNWhMTt576+0XiXcsKeTGuhZfRuj6sFjIvAi - 9g/KW563S3pjAfCITWTYYH1rnBO9+Tw/+X48o1EmIrW1ObLWqYV6d9q2v66ZQZ8h + 4hCDepVXy6shquZO6MKqQ0tJUeXgrcpBWrvkr23pkQoi146x4D7mxYdVRzUgyzdU - wlxeYhP8Q0V5siazvnFPDzxPWvKMeAtbDTU6+QIDAQABAoIBABZvR/+mn/BLJHRw + iQrRYJ2XyIuj+T3POB4ymE3gs3ffQAOmY08t+QIDAQABAoIBAAH669NBNxFKcxfv - /WH5jljdHm6PbqBnwY1+SDw3hi+rNl0CBa5WYcXUIsii98No6XTRyB5qYIvh+Poc + 7yRlI1dzAA1kPmLiCstF9wlizz3bheYEB48YXZvJ2ZggLyVh44kUarat6Wyey3sa - XBo+VsRdy3pJDLWXxJ1zOSj9zkUjXAVeK/z80JwabBl2OLEnyKqTuPt3QT7qSx6b + nCOJUsk1Sg2vs7bA9X4RhdSB3y4wL9YPhZ3z5GInMMhyvx0ivmR3t9K2qDPHCKAC - 0pfYDUOXOl81x7ZOWHSUavBRWE6HunX8/3y7qKWV9h5DGyGJ2G/IJXnI/uuwamiX + PAND2HdVcTn3A+H/GIw8nmcWnSvLHNiOI6waEn2cvNvIBKaHxuRaT2bphJ0at9wm - YLKkeMKYczQHb4ke/e96QM7r2GcHqPisT4Vz7oD2kWuCsRTD5Y2s0wzxKKL+hJja + BkCyM/uS7d96LwZ6RJ8YSnF3uYCro6UgySA1yECalrDgHRb87v+EVOXrqO+Qij0U - vWl4JgzFZu5Fyxx4izoZuHHc76SKNass2zr8RidfDx7s2ozWZi4Q+wIDhE6MQw69 + ccEBi+TW/6zzrCUinG25uxAafi2UJGSNMOf2AsTGwqlj5cuRrySlLWCCJS3ZuOM1 - wlwa4SsCgYEA54jnAzM5IMm37nyEF5l8hMtNqIO1EpMg/J3T8rfGyG/wYFvubRmB + 1wLNKGECgYEA3VVnMVrcQuf6TYWfBTfT/+so9zbvNjUsS/HPdlZtrZHxMylAHsih - zZ1NnVQIuqkjlsgEcGTKEKBvGJyH+bw3ekWnvNCSs9jhu0uAUzqhHP9hkETmgqr4 + 0zeHdo8iDyxUKQYy+Li/1R8Cu8psoQ/iZD7eMl4f+aJtzuGywlBlsF2GQDfRvTX5 - E/VsQGRsMQPCstDN6JgT1rhOO0xde3epwjhk+rW0SNbTbQAz6O7mAGcCgYEAxcHU + dSnuoiGM9A7ph9O6muTPUdb89BlNlVvgw6noZgTd32oBKonGUNlO8rECgYEA+mVF - IAs8e+y4wj+X6RdeipGeK2EFFSb5M+GUtsQlSYIc/w7AZLN0sBbKhont+/PLXLiM + lqpLkLkTtjhugc5jwXhvoGLPKhU6GSYNcr29eO3JKUBRgkNJoX9bMLbw3EHAIjt5 - PEZgzfYCp6pRo7oBqs+IJO1xvt+jcOyRRJrwohxgs4IDKTTxfGGTJSFrBM7wJP4H + WKCL3bP6QjW6R1YZd7iBqXnIMDjdLJYu14bS8rVTw8MGWNvfsFgJVrY7eKf71dNC - Qcvclq87MG/AkYnCvzchVQz0B8J/BTcyfDpkTZ8CgYBSjKcApxSpMgJYDyDxYRHa + G7F2yFXKpUkmcm6x4AEfbuKwpUd0uabgCMH68ckCgYBNUX5BAYqcXMlVt237tqr6 - LroDaOH4O2i5aHQWx5sh/3cOg/hgAYYcDweLHlj2ZDOCINIkWGsKvoidl8GLMqX8 + Zb3jzm72MtEMnqZoonyh+6+Uvb5GgrP1QxqxUgMF5ehohF/d/zwUSUb9LxOPmCrv - /DSvxxVm9d6VbnfUNMUYl5zrWQVudRJ52zi7RJKmbxbNtlCTqxT3q0KJNdLmoGVw + 9f5M/hCRdiqB8Novg0Jiv+kcGePNA0PnqARS4wGIaIUwC8jOP0wlPMMUypoNqRD1 - D7dBA+PBTIaZCEd5tyNd5QKBgEwOSLvuNlve5gvnE1CVKUoXyQIb9S12aL9YUa6c + iS9EJEMVvsQ1hfefWqp3oQKBgQCpi9lK45S3Mhq+0AdDrdSuNDahi0ZrYGQuky2X - 704/GVHK5ZmVHxqeGVP29i1BTQQjAeQomRB9PfYn3fAfGIcN++lf3LAxKJXElfYR + /BJHx/rmC78lTRqWV/4PRlBhU8QdadgIwuzx+eQC4Q55LzufbTee4e9Dd72La0Xc - tNxUF6jSJs8RSpKwoDvWh2c5A0jm3fmjIvpc+GGfiSswFVMfK9We/reBSQLDgMog + elZsMYu+ilfJ41fbuEDajhpG4LgNWTbyOYAMtsq4kIeQBJQ88YWvN6AUygWnj+8y - VvU3AoGATSkjR5caom0oPfYqZ4FGs6hKQkX0DXrnVPiExh0aJEy3J0oyPHsvyjJC + /uZEwQKBgQDVQWWlaEbJofp/s1B7ncYdVQFfnr8PuVpRPDqVHc3OcalyTYaznWiq - 5U9a3yARLxS/mIgCX1B5Ti1Ti4YsnQq25fCS7qVVDpKV/cDI9O0y+UOHjqdTK3VM + 4lHor+uy5Vd3RLzrI4Lm6GeibnSSejYVZkh0UddDA7e2CRJD4ni1f2oAalFPZd7k - pzSD8utm8mb/la90IwST3gv2DZvblsG7ib1k3WwcQakfHa5xH58= + WW/9ijHdHpKKp4Jw52d7Z/tfrDNukBnCZ/t9z/iYXMBeFf/XYnMe9A== -----END RSA PRIVATE KEY----- @@ -12312,62 +13662,62 @@ vector: segmentSize: 131072 total: 255 - convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:SSK:k3rnsfs2yazedhiqpsdjyxdk2a:yufwbecazt46va7jsxrg6zdtfbznh2fmw46v24bltyjclbdy55ia + expected: URI:SSK:bjfs7kysomxsga74nyejdwojji:qhxtmljd5h5t5azmpoboidjf7ti5yerpokkmwtmo2dw4ppt42zuq format: kind: ssk params: format: sdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEpQIBAAKCAQEAsmqX0nYFM4DEpUyVDY/0/nftAicRAx0DR0TqlV+8+WbsDy7J + MIIEowIBAAKCAQEAwek+eTfmmunqMaNTfyfWdEnbjjeVm0QMhLYY8gC0q5Pid1v1 - jFRe3YpGPsG/SuhAykR87JshLKyJSvyAjIB4rS3RDjciqz+DhysNq7+0aGanMCYT + fjMuAspOREl88Dn//5VrVQRTgLG9UvAIbG1J052p3HP6AXKP+1GiLaXnHEycgvLj - lvUMZMr/O4kUQElw9GPwL42/p8YbeOYcwp6B/ZREXpNCmI/aSQLoNRxFBskh0QTD + U8uZEzW2JS9xjY8DPegTAC2WdAjcgk2iys4OUWzTTMkRlM8gJ+04GAMWldTbVicr - kKNVfycUkBRgw7XpJN2Zvz3IWq7Ou0ZHP6tNRXYRaOwdFM2/0mAkucPiMr751LeR + T1TWG+QmmhHIGSVWPBw+uTxsywDBvHns9asJKSCrnc8B56nHtnDe+QbiP0GDSK8i - L14e4jcC2zz6JVcz3o0UeGs8oRy8kcrpY1kJ348ERM+hLeESkYjOYYPexOc7paVI + L8HidT7nULimgpwj70QYLRMS8RqbaYzfEy1kx6+tpiVKbsVYg03lw9R1RWHtA4CI - k1AFKubkC49Vl0YpTl9wSdKq06wyF+YaQpsd7QIDAQABAoIBABh6mv9/fVmyZGiY + LW2JJOXAn9YIFqUJm/9MujpCSVptrBpnIMskGQIDAQABAoIBAAlYGhlWcwTwiYdK - kAfHFVJXom9N7F3cvG8qE7RwaQVf+2ne3bzQ854aQ2aHXydYI9GMoYY2B5BxULn2 + UaCilLrmVeToyKC32wA909O86nfhKLbWkNd7vmIb3xZ7JxY0E/dm5cpDQDCHAB1M - 2G1OkCAUne8BHhLYWUOxosPKfuZnFS+8PapTz0JB/tBMj7h9Sw/g6Vqg7GeIvQqy + xqR48LzjOldOpGkTnPIVTo1zaEC3cAxUXtV/zbjSAVy8iEo0GR8k0t49lCQzysfM - qcYDCOtBjTrbogK8E/M6AKGO0iKqub77ZK/iBQ7UZFqSZS+t7atPMb2oKWLGWtO+ + V3BSlSM3LV3Oy6DXiVBl5eqBouyQhMH6oKPhGR4OuycW0BPYtzNft9xzDeuHYFg/ - pDbdpDxwC9ZC+h+tQDlqMjP7ZLvYqIezwI/YqEM7oOPNL+KvCYO5X6BNJdKoNuMW + XODl5cbzmRMHfvtcROw95U1lKKHig+44oZdLp3ghrP/QfAxsMniYy3Sp2cMbHIRL - Po+dsY1PPixmaEeUIq9tv5tzqn2K0jbCtXbDkDpVXlNW7MCYTRCFGdejYYmizqmR + 2ovNPSEugMKN67PM2NnkeE3pWBaH5zUrSu9M5I5jaVXmeZovGuoLr3ss5Sfb2z5V - 2YwKeFkCgYEA2ckoTYRPbmtT5HE/7l/AJpsdKZfqQ6Zng952y9BQuJgdme4S/vNn + 8NwCBQ0CgYEA+kRQECGtYw8E8b5XHKyc1rlj2KPV6bF/cOUEtcU8r763YtkzHFdd - BMDxBg94Qy3Q3g463HLVSI7YJSo0R/Px6pVdxPcEjxjlj3PyCuDYCFbpGvvOZEbA + OI7ErgQI8I+tbNgAsBPLhEWAct9aH6HHep6PlwwA2XGJHKtVMY3FATN4U7C5BkIA - N73QLpbtisVfyg4BFwA1Whn9SXD+KyJl6JDm2VQBg8/NZ2baOO1Tp+kCgYEA0bj6 + IbvfSrIux3mBhfS12PN9x+KWvcysQmQYEUeibAOdH2priY/q4QYVLCUCgYEAxlpv - xTPf4wn2GhQBBGnr20ILoJuLt8+GanRz3Xkb7a5QCeqdUY7hl0++fgeDJ2hd1mks + BRaj5zntcxZ9AHjPgSGg2f0I4f1AV2r5PBzhKL5XNJLDhYRc1GT2m/LTpB6ie/3P - i/pjPR8mzdI6eoXBjbxhOxZNJLN7ijTpYo9k0o6rF+ijCbo5wHubU7SLEdV0eZyR + 0xzF/H9SR9+G3cHMQoS+2VrX64+WmwVVLUTKurMqEZl/ZzIp6g4Lbk/pMJ67jCGG - qPlG7QkF/665ZGEaB4FUVhyb3zRuY4DseU/eh2UCgYEAzmhRM06f/bXpF8yh2+mR + E9Y18GaUZTOuwQRKnOuMbNVORR5/h+S5cr+L2+UCgYAERuaXX/v2lWsgNoCGnOyR - 8sT2WbJqS92NpDSXAMoZhypce8Rg6pOD4sR+atEEmR72I073SHHpZNBFWMvsKvmw + PtnV+fbN55ql80QBVz2SQ1AfAFc/RL7zGH2D+82rTslH8ukQGUaBHC71x5tirwEZ - ITWZXpEDGCBviYtJLjg1Z4n/ehyHWxCXIv1aLp3K2sf/5j9plwQSjKevIAjgS79Z + t1v82Neq36XYN5VdI28adia4R2ziDn6yFOPcAu+JuSndgDEbZA3iPJ0W4UiQWeWP - OJcEw5tTqDqtoT+guW1s6OkCgYEArIF+bEVeLG9eKlc3+vxxT9nEnKg1Rc2CoAAH + ZgoAjo9A2jC8SRlafyAdgQKBgFywa8rD7qmhry0lqBotWkIslb7n+FuqfYOcMIV/ - 6i2bRmcyWOXN328qqn6ijyH4xKp5PUsnpEAh7v23umbpSSzKZ56DT8npTH4B6U3a + tVPVxmiB4K4m0T5LQ9ZSHcZGroUkcRZlDrvUP33onVxJMIsw/wIQ6m9gdO9SCaCS - hwKyCOvnWfQ2X2L57BUAT9ra5aFxfDLIMXhR2dmpQIXk4udoNLIxv98qa2/COUCr + 0e12xcTdpuRxU5bVI1BUNVMMCfYMwFvKsP/634N/KD14JOm5RLCi6OVxwASfxG0z - wqFqxm0CgYEA0Tv3TBFqDVP0eMKURY1EDwkjv751wtsCLcU9xqaWKIAa7OKzHDN5 + x0ZlAoGBAPEC1P9iODm4dxw7bCewFciF5XxT4TCSMHc9WE6okZuv3X8husTtZ1K6 - 1VxTylsyU/7A64C5a9CWyGkMYdkYlgYqPnAo0Em3fzAty1BQZJzvzv7LtP6rB0qt + OPLQqVDytIq/i98vp8YF5MMxuLgZGaOLpEPmHSc7w+00gXr2xxt33VWUQRzqcL4K - 8NEwbiFpUorowTfRyWfxRGhYfD+Z5r+NcLNTTYn4qbFsP8h49iBKSWQ= + RKEZQGP0jaVDX26z+qfM9ZCuOkiD+tr/vnhzcvZiaSBM6vnhdzFf -----END RSA PRIVATE KEY----- @@ -12381,62 +13731,62 @@ vector: segmentSize: 131072 total: 255 - convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:MDMF:6h7aerepvayokutkd2gg7vuase:ypguygm6yi2xwivz2l3d6orux7ik4dfpv6b6ta3tm5ydpntjb2pa + expected: URI:MDMF:lrzl2tr6irxtqw27psdcusdg7y:6koh7cij6mugtzogurcttf6ed7ul2km56hqu72hy33j2tsn6c54q format: kind: ssk params: format: mdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEogIBAAKCAQEAowwAgweNcKjekhbkOYjboZg7YQKqOiJyKujfAs7FrLB1cRJk + MIIEowIBAAKCAQEA6cNZgrmKTqX0wV/DLvTE6vn+qllrAJWP3WehDE6oWhbLxpFi - IdM2jvBv7E3PP6uw4VZVSOMxvx3BhNiU+oiR4ECzuDw7X0K25F6aEvWsbm6NQnlm + eyBNA4vtkIKpJAdmzP18k2iSJZZIv96WaYLkD9L3AbJrAhp4oBh8JHb34EpfuUjl - Vf6lF1MgRBZf3AD7xmoqDjhsxsBrYvhByIWGGZdx0G8Zw56OhRCOMkSa44eHhgyz + 61L5XbEJPqgMKWx1hn96HVCM5he4ZekYKT0tNKIwkIMN3Mz6bxIJR4yfQgPtfiqU - 9M1nobyVvS1lRRD9Em9c/16LkNz2zZQF/ebpo5jK7fuf7uYyc8JHmjB3J0OIMfKt + 364CJnZjFhHoQzzzGV2ObsiDnEzlketbz/x928canoDNwR934pRHrwBYYvx64VO9 - Ubdt9FGJWvuhMM/gXP4d40jBkwopPwE50TCLFFH4hEIi7nhdvjdgQxg01ZYA/yXZ + g97Ox/YcKqFHpXYInps87WlUbNBDXE26/e5mrOpAxPLFXmnbGKozRWHL6w4Kit6S - /GODUQbjGLAitDwIaqkWzLEOHeGweXYIPl6eHQIDAQABAoIBABFF8TvwbeSUj1fM + gDCCka9QSosvILbpjegPd4Nvxv9sB7qcscX+swIDAQABAoIBAF74XAnFmoCoYL8d - wwrxW8tH1GqXnK8h/RRcrVufykNcQmTjPOZ0eOA1yrWvHJizOL72pXxeTWPg5CKN + Sj1t+QCj70hDCrtSh//B5caLwE7VexVhpHp0XYWG2E3BH7mA/k1i4LU8oz99BnJZ - y2KrW0D1udR31RZne/a/qvT9P/JHFgIH2HadzqGk9dMgx7EIDaRclO4CvktkETxf + Go+kO0aIhYydcWcJ3R7hw7HG2Z64aJpsmOhZrfDYB3L6r/I2W6r4aGK3gn7KfUJ+ - 7qAuvSEy8STS0FjwEEs6kMX0jLAz/TSCGHNj0aiDU/cydz6TsDdQSV9AVUygZ70M + CDBc59w91nAnpj6h0k7Eq5tzcJJPImS8S2fePcM2zxcMvm0o6iHUJsKPzN5VJJpc - LQ8vF5e8+nogwMDVwUAFch1VVwWvddyR/hoo3/hCDnw47BZYOwwO75YwcVDhVCxf + GHTZq8/hVu1jyuzLEqCL00XHc3AQteiQoHE9gICeb2fn2jnW17ogWMCqXFq6qBL1 - oAxURnrlBXcAk2oOmJhRimw9fcT+WdClBHcq8e1ACHuK4JcGEqXnLKuSMBKd2nB4 + MgPXipuJ36Ngh9kX2OaeJs6KLiElxT8C/y1N0aetnLJlxwzV4Vh4kx4bnsrFBBK/ - SmYyzSUCgYEAyExi4Jh8zm3r1TJ9LA17v5oSK+JwZMSo6n62RTzUcI7/ri1MTM8p + dRYkS50CgYEA8HGT3kascanqYZAABSglY1ByOWCD7dJHgk2cVdCWcQlZ+rDmvmaj - Zk/lHK4q/51wsj4Vw20AWCfQPc3XnceBxAYY2ehnEfOWX7BqQpDUYQn1o19+qVeK + LIJWXHOTCWIgIhYpbnV9EZp+mZ9gmcsobqh+viAX0P3an+GtRdBikWNpsbUvX9OK - SpjLE4EMpTPxDvxkjkCflRVyLRomrX39kpXzE+6SZH+z7s5nM1MrdwcCgYEA0GOc + Me9APdVi0nDwmmfSRhzjQA0b+5gJ0SUEgbKXOl8T3g/yxa/lnkNvTc0CgYEA+OMf - vNnaWWb0+rS0x4DZddjNQaYHru3qN7B3+7Le+LMWnZQPyQ4FNdKSuABItS/Wwsc3 + dr04EJwKJRwO5uZZ8c0Pf2SEbGzAjQifZUp3HWg25hto25iPGNbU8OJ5HkyZqJbf - e4fo975YafXN6BRgwNmkpIFjazbJZsB8CLGB2qZFwP4L/HXt8KygX0aQqtyZAYit + 5kPpuChRlX3VKOz6s5v4sNjf1kE8bMnnRDk8waD5MJQziEtMZ5utGs78Z93etP7m - 3EN5jpSXMPCo8P5lVcpR2LuThaNq0F46Mpd09LsCgYA5N2DTaZvVWB8TEs4g5GUi + EWXfsbzTF8cNcQPCdfLc/ynXWRRx7eJ1YXLR/n8CgYAU+YNxr26Zl733ds1Zpc/l - MX/ZW2Dh2C+sdK/ajWreEGtHNRdjpZXc7Ru0mqgbxrynngaXga6kgBMDZKagIpqW + Iv5j3PSFSYOtbUHHBqQpBizQPqBSWbfASTppZDeeaO3uq0o/9YXMhFKo6gtOPzeu - BWvZ64Jt5VhiU0G3bCnO4opxtdi3xRLzBjyUgLu9AV5t+nk7DYjIjIzGB39e9euW + t4oe4cPSGmL48YHhBjWjAy4UL38Ld/OlOX68JiIxw2JpxcbFEP4N91bKks/Aa99B - kRET84WMAdLDd+CRD8QNxwKBgFLnnl6/qQ+yVzo6lEVerKUmyJoajKn6exkGuuVH + xSeGEwczpuaBsj9wl+dcvQKBgFJ1ZUpAvJ98IzxSRHmpneknyFerpNgLW+weDDlR - B1AYJ6IvWoxZaJc+HCLZ8hMrYhyBl0AtFPEjKBeXtABlwwxWShssYrovxLZ9U5s9 + 5479pRqtwBrpO5e+LYS1c+1e8ZXSjtHKdFfIO+dsbkAF67WwGj/1SovAx1U/u3h2 - y3SKe+vI5kndPPloJDFjaIChXLnwf4LG0WB5GyFcTUn7W6Ni52b7UTh0iDU3l52z + AjQgsg6vOzePwvucr0hvhV6gOpX60Zy4BNntNn7tOv3Tggzz7tY3NZrU1D49RiiI - BdOrAoGAQH/BnYcsQGkK2szMfGE/kH87CXKnxyCCu31Cbv8/YhgKsf7P8sl7fdmF + ExzfAoGBANEXl4L7qCTan3hhM8z2/RS6bVC+8Dp4E1Kyd1bxzpGmjuHrClf4aRiI - rub+WqTFnWLVrD26nm5ZPzS1y3xzHdIJSrgDQiUX5st3aPvi4wsHmJjkXdg01kHb + kvD8iSd5tI3weoS7Nv4QP5FtLNKlI7zgAfWrli8uvCW7FoPlkF7JbWKfDn9gwSy+ - iX9WyeGf1gdpw33l7i/LxA56xz/TruHUWbAAqIRgwVPrmjSJhH8= + 1WL/kD/fdr6kSg40dBbb2zqC/M90DVvLZKmoTOQWk+7D5EsBNyUP -----END RSA PRIVATE KEY----- @@ -12462,62 +13812,62 @@ vector: segmentSize: 131072 total: 255 - convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:SSK:wijpjyc3v4fjksxk4bvumczuqq:bqcwlgncua2dccyqskz7dv5cs3kajaf24i527ksuyq3u3oosvjna + expected: URI:SSK:37oycc3shngu67rijcu22qy7ge:rq65qf43dfsbraa6axxhtmm6xhzxeirrwyfjvqa3wmta7rbuw4ja format: kind: ssk params: format: sdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEpQIBAAKCAQEAtFKkGTyimfLhWurNlDMzBBTS1LI3DCmNUDZuf8qb2lu9WPXH + MIIEowIBAAKCAQEAmpUbuxF3CDJJ0hTgu2qnv/tyPJ/VzSwgewi0bI4e/g2dplol - 3FkqkaqjTXi2wNEPZkVQtLnrgrh58aIlFOpCpUvYmn4QI7+n0tqIMj/EsZ59S4/n + eHW3ss7agvO6XPbI6TWqBM1OhDrlvuGdiN1KDtHg4aaJSQJaq/0f9N5t/f/kYnT4 - T/ia21MBxJnV9X4Y8Q6KMrev5hqmXUPnaSaS3Fl3rqQ9Kuwg+KVjTHN5Lv5Mro1g + ipaX3kmkTrsFsPJJkFYYWSwGeFfGTYBUr39cfJELTdoFDTXAd4D6AnYkZHvUXGke - 6WcDsHwKlkLZxP9NROb1oXAP0ehUMhJWvhAUiuIikHVinYe6lP2te/3hSH5kzzxS + 8f+lk7pxR2UVx8udWyK3yO34tQmaBi9uWNACfzv9QMpaYEJVd/wadtxMb1OC2PR0 - sccG9JwpIjrKSiSg9yJoP9TxZYUbEK21E12I0cEvTytWAFOc45XWZgzE1p4+zwar + W6fmtBIFIREzkHsnpdrUp5jYw1yHKBUAgmdtGBQmOvRAyOeUwc9IoItzSlVvh+z6 - U/RKYNilJUWdNxiltoSlxME0VZPUdctCfWasYwIDAQABAoIBADnZfV2/Tyb2fYeu + q2It8DphEixauzfEzhCJoZgdasmhBoO3bdwAMQIDAQABAoIBAC8xZo3t/xEZiUAB - Zm+SEV7Q02Z7Gh/jwLsoC0EiHefqoI9Gomy1imubA7LZ9D9dkoQr3p1sO/r+9dgo + 77pIDX6nHXE0uukwl5n4Rlz95qhZL9AhpV7pUXPdgwiHsFXBYgUQxR9CLr5f3NQx - PZ89HE5tS7sckE73sH57r0/3l0GoZ+fy7bGBPyT3t0x8UeDlKFlFYd6tgVff2tl2 + vQ6TwJBVsvoxBaisd1IarS7s2Ve6T9dfLqHg5+yNPwRqRIqI7byLDFPtBOyon3n5 - 7GmWf20Dotq7RAheIqHCZV3kec8yKTqyjUeaEvNZBWczYqMXdOisj8N9j2FUrWRq + u+D4WRwOjAzwiqpFxsS1M56cwu+KBQtQ8/cqWbQeWeMWKHi3b89WhoB8QfLV7DS+ - PVu/z4yLc8oo2YoEgzSnOhTsmHpUumfxLcj32ZSr5Y4MZcmwCMVlGe0WoZHPoWU2 + jRes7EB9nnW5wtUXLEtnd6LRD/LW+RfC5QEqeeSnVVVAhGgD54R9wEgS5+TsC0Px - OumF8CqcFzIJXjHhLld4JZ/w8DdjjPaOP/wiAnD9j8LZuQ+oaL3HJxLEFOo4n0Ev + UQMaz+4FJjV54yJeB9WueHbpxgOJ9Fq/khINMug8y2KZvtnEUQ5zEPBzhN7AAie4 - +zyB6rkCgYEA8zNi3ZMNIyEK6+OEOvDKP4mKq2TRn2CBJGOHNkvM5i6wY897szKa + oWqxN6ECgYEAyFXYTEIfUtxoMGPuySPEntVOo9GRNmJAmVC75iDH1xlXZH//gGov - 1jNXoE9pHF1onQDg/g1y5lejpVMdCjB4p7JLozMh8D2JQuKM9Zmws8C0641XWcVn + hKA0fbGuU55cZE7TZulDfIarN820ButlprVmPBebSiR0/Zdkal7ogjv0StqziUkH - TFBdIu7gp+zchiXWiBou/SvKIorTWMkriM5KR88lFVLwx4WC+X02LBUCgYEAvdAb + tWB/+aPprCi4jKOVaR1+Li8RRZTgMjvDaZIlmaz2GOb72xFJ2wp6498CgYEAxYjK - vmzryZRAcYEeLfOoGr8UpCTDIIVrcIVh6e9CTQC0OAgDr3SwmypNtK1h8Rf5tLS7 + xHUkykqDTxkZATds7+aub//s2XfXPjFFPgf/7jhr7cVLGaSsIcajPGvPqfMqwqMn - O2MbOOwYvfAW33oL/Od/2om0CA35MyL90ETbh+zSPH+rSlLq2dbUKa5Y1VnTyUSD + 87Pon56yhbfOQor6fKKYXE0BwRX//7AOLn6mFQVpmoySstbDLxIA0qdwyYUbMuBm - sdzU3iAy/hAPePOysA01Lo8vAP4paztR74BW/JcCgYEA3HmoVkEats8czI688IYc + 9o/YREP6sT+cmAY4iuXiJGu4xo8AOYZV5IjPHe8CgYEApL0S9QKax4S/mKtUvMpQ - hA9X5FuI4hil2uxTxvhe8ApBpKqTdPgagLeY59815h4UWclTL13X3VR0Kcu4VuVs + 8VvvIv8+Lj51aJ3fJcpnCxanqtkmve6TzLgA8iuectySlVnMtZ+0Az6qpWTeWaJR - bHLpuTEAwn+28SjbK0hCdiLsoWLIXrzkEb4FQUcX6YSEwySIYWiDUscg/8GlKidt + INmijF/NLxbzrWVFCcOp5w5uQO+/G3GWiSwlkJ+dlBiYSe5q+tlp3YiO520ZP7Wt - zR9fHcx/zN4dJHQ4MZ++vaUCgYEAivXPSeLd3+6sGyym1odaG3KmfuD3BVkH5hGQ + Z67qhIiahrfK+8YnuZvQmnkCgYAXhBNvk+qPUpOTRQ+e/3Quky3NE5CkywmK097E - ND5YMJ2CUr7zS8FHBeG7j7mbSXD++1+Q7xJIPK0EFBGv/R2Rpy4n+Or1JSxtsxU7 + ZbtoJrtikQxBv0LmunkQZl1QhCxhA39sGczlw8TI+nrJnTX4xHqS8m/1BqN1UwY9 - 8fxnJ6Sl6WqiEUCQ9LgFDRq5qEAh/2gsbcs5AAFcs4k4epkWyTJyK8rhY32u/vUn + LsKi2gQabAXC2KJf5irG6TwaIYh9ZA2d6L38UoNzunjv+D2e+4MShuh2auvB7WYo - sAoqJLMCgYEAg6/I9L+HoMqN0P7hpek0uNVpEYaQ8OB4AXsnC8m+EJzzcAtmcw0Q + UMknbQKBgCaeMDyOJEYdMPS9Bw7JYAd/vNS9xOjE2Icme4HZUl+JdL1EDCQj7/vE - ligos8cK+vsDpcRCV5uMBcDU7hfwIq8cf3f4BEfFVsuPWtZbhbPVAThVCJrL/jdE + EldLMt1NT1/msmPH0oOYi9XtCiO2l5jGtHA+gVVNP+LuxsHsrcnFVix3mjpDLbxn - hIf4TMzMCuMyh1UQq5PmlEqobKpbLowktuvs9k9iWN8rpEhRim3vm7Y= + v8ZOIynGqz8Rif96TV/668SwVoUm4kG+EBLynIsPtAGJEYeNKIHB -----END RSA PRIVATE KEY----- @@ -12531,62 +13881,62 @@ vector: segmentSize: 131072 total: 255 - convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:MDMF:xhdmt6meav75nzygmnmk74rqfq:efuyehuxybs6jf7qn4qhi5up2mziuaiddv2rzg5v5mmtgzmiuwga + expected: URI:MDMF:67btu4c4e5b4dictm7lug7rgp4:6ir3a7vofnm3lkackuqxn7bngupp7dhrcezxofoasrwcozx3japa format: kind: ssk params: format: mdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEowIBAAKCAQEAsRUOmJdDBY2BE6PCQ11VpPYLOYBTeM7p2FWHaFfbBrhTCFmZ + MIIEowIBAAKCAQEAgsuk1pcCXme4e/8m2RTs86cJmsijUmOrLHmj6Tl+G4Wq4hJ0 - 8bIhttRLPbtbYxi7GNsjVGr3w0c+rufJnrsW3bOg8eOj+0sTpzrMVzwawSGf5t3L + SjqO0Q0aoFIhL8iLxKaSm+haRNASv9iwp1QJvc2yL1Pa6PUFzSdwLkW4AYGSL9nx - KgNAOjgSBaaGWX8IPWc9kUCbCJVAgDUL2dLFQOH/9YPlaxqtDgbjx7FXuASc1d4U + 9q3md7ctNyZhvf3PFM8xSnSCsI0R590M4QdWwS6CaTBB7c4Noc+g2pPhktcbPFZf - 2x9AdYOcFP31QaN+b9sF+x8y/V/H2LWG/q04OK+t+VyUnRWhY7IWRC5JEWvw5XMC + EZX6X3GrfjwRMGuXHZq4UNoWntddAcoaaMA/lwgiDJvwriwRe946YY3gXPOc7wca - LkDl6LmZaVvuKV+7u4/D5i9amhY6KQwajc8++0iO4piHixnLzS6zLGSMzxoKEUwy + ea+WEBQn8HehwPLIYGkPW4Edzyv6f69CbWoghgT6qzUMNuAQ5jcA9QGY5xlvNzGU - AlB4/DDZlVSivN42mP4qBNUtqYBC6brVVEKkdwIDAQABAoIBAAcEHMO8KOijnNVo + X2omztyxcdgslXZryV5VFuFBer67M+hB2Y+VDQIDAQABAoIBADWyFs0GF7HcEO/O - kjE74nSaijYcbBihdUk5lX4NDqpRshOznewIl1nFR+t80xGDELt0YafEZGd1TC0Y + 0wsBvTlWFOpXfj0/r7FFitYfhTcVTA8dlmI24hTOtWSl8vvj8AVegQfCfvSLG3dp - rx4kSLMLnR9x87S1WpfJ0AACa22/5YD7kRAe3tpi3FWJYOtTPPVWpzoF3RHWMAVx + JTS8mncyb/lgCpnipWwQycwlUSJFKFe+uMgVomz5ZXWjqzLNdOtNGCZB6LlEYNp4 - LjRsK53D41gYtYWsoeZBxmz9pspHCGZbMqekiV7CSln8jHuXGEGrf4Z7r6BVwaGJ + dGYZljMevekjJ53SHuSUEaxKU6vtTv+TldyJcCkde+V2STeK9R/aihABHQBVK3cn - FIO9oESJhXp3oGYu+omq/WCHYPUVMKdWqtizVMBbvcXuALG6yeEAeCGk5mqJOfA/ + JrWOcJxGJJ5ZUE0kxVjf8r9QZx7eUODT9Bn1FgmIlUJtcJvUgzQEibdCce/zUE+T - AzyDU17FI2D12fMDhqUaTYhzu9I5stIio4/6RmJrwLaxK8efCAWohfHiJeMTIdyZ + Ov+7z7UoAlynuEshFDeK8IEEDWSOqHqyvc0cg1GwRL7SGZo73CBHi7DYS0odNuUz - 8NvAEgkCgYEAx1PsUSXOowwG0l7wjmtYvPcl6xcnIsWMmYeTAv+JaMfWBvh1LPKi + tJArr30CgYEAuMrCS5otqSL4uHp04bPFSKepYB+/7QQgW/X8NL6Xa6gCRBdGN0q9 - 56mnBEm7viu7SS9vKjnoFGizn6pxDTqnYeDnzFTLHtZayqmzbtedhh+Pw9Bucxgz + E/GINWagSkihBw1lbjFBdEKeq3sz+c/UTiV0j75zU57oN9kDDM56tlKhAvwaz8/K - LeBiTDqd1tzBSn5d4LwtIAECQW6m9FnF318azVVb/2vQNgDtZhy0Oe0CgYEA4239 + gcm6u0eePioWSGhX8bWUsJrxNVq9/CbJEJ6wjI7zV0KZBoMCvhmmiOMCgYEAtTJF - arJgtB4tltS0P9c4kjDmC7qokaVDFZ7udHTsAd0VebteGe+kIIhDH9QUk83B6isC + FCQNuACFptHI5EhEo3qKq1tdsKyq/4VJEdEfMAgdwCceJo8HyVXNFXBGTyIdebJa - n13hYXkItBMixu5WWeoKlXWQ8rmm6989gsO5fiDczbQWHUaverSlfqt0dZdByrsm + cdVYUHVCROP+I65tc4lGcaDgmfQUho4+HawKA7S2BQSEewDGnDA8rHozq8zMEbNP - YtdWo1kvMTzpf7IGh/DRejj1XjuJWZ6LPCNfO3MCgYBiBFfJ06CYNtrH6h26uvjI + znPhiqh2iD/gGKkpRol14hp9CaSy3DPiHrsB/U8CgYEAihK16ldhFqeSwAR/oMT5 - +3Ou9hStmZ05Bhz5tXT5jIMnrFfagXowFxHlHujubAzNwUCV8CG6n33svuCW08fp + +7eKzs/qT+ZtZ0j9EUv3R/FZABeD13x4mpY19/Ceg+KQrvxLdXJIPd4pQGfmBhpL - brItnWjAwkGlNOviTq7MfIqyjLUzbawFHDjaKVzigm2eVyOM1pwOB9D3IhWBRP/z + v7gsx9q9wRVS3afAp6j/94r1040bW3sfDKr2Y0i37Cr2S1ProibS2sJqyDrtCaLR - ho3keNwjbv3VAIG829KYRQKBgQDcLighnAAzYOQSGmtHQz3pip2szVFVcAG6dNu2 + SSHJOLz3BZQ1UrBBNFlmHZECgYAtBwb/kE4QcaDE6dEAWa0k6ujW2GeZ5e7AfMDB - s1upkjiwWc2InpDvTfxuXAxv68vIwUsQrvr8OwlKDRymKyg+dG86s09ZLpOD1+Td + urQDXaD3BUGK13RZ5gaG01XFiHbGrTmonBnMNLd5IyceetQcJ/rnddEasPsAzQxG - LE/w5C/glnCydzR8P1fZgnSFQ6LWesl297NRAY7GxInqrpfUFDk5cttaF5mpwexa + l2ANt7Sb3pmFb5XrbllFi0CX6tazd1nXthhQOrjp9uWbez4Ul0hCHc8AvHruGb6R - lIQmMQKBgDAP5J0OntdDvvqwfnXUOqL/ODsPaNMqZjBQUW8q/dZC3xxyMW/CKtuA + YGuIJQKBgAwXJb3Tg+jZ8Eq5fvjWc5s2fxrX4Z1BTtY68CjY7rAaZHoereCB5V4V - Qji8dYENM5OJheY9muqI2zTCfdvEM7628D9NtIn7F0WDhVADdYYHggs83GbcC1uL + Cbdw2vEKYpYJFlFsS631wrF/7ZTHdkt0Cm+MVo67lQg2uSKJpGLXw5YcHIag1JhT - U2vuh66vlkvFOdiD4aZwnawXBDE55HYjuYUybjyx7dzaLy6fDfJ7 + sIvtzyOnufvRg/SA/sbqapMukycDhFXEHQrAi1gIRadkMJfsRZed -----END RSA PRIVATE KEY----- @@ -12612,62 +13962,62 @@ vector: segmentSize: 131072 total: 255 - convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:SSK:4h2egxe2omtmnmvmmqmhatse6i:54kptz4n2j6ig4q2ugywxtiddcfo3oaswyhzce7wq62ey4gxrm2q + expected: URI:SSK:ohn5yz4as4vgezj2r7ohcudtzu:onvxdfapx32vsz457rp3mqeap7q3hm2arhlje5x4ctndu2jfsv7a format: kind: ssk params: format: sdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEowIBAAKCAQEAu5KyHrCLGfxh+ne2HLxb2YpPxSdaasMWYwyGEGe56UCMCXZ6 + MIIEogIBAAKCAQEAmiJHD0uN13uoQUJJFUQATJKvFtFBXoHEe83NDAc+cUkNym9E - xpp9YaUkZNeTEyjNivUYtPPyCFl/sKMiaDKFe7PcEnxRzLxlt64lCCQ851AV89j9 + c0bZmuT+HH5GPAqkhH1PKcSXRq5/W4vZiK8safWJHzREdi+gI4pHIz0BILyd5dmx - Sd6Qe3LNK2WodP7PBv3QdWnBEjjme6WZQ1IZfwyYW1ycJ/LsELa4Q/jZebM6r9hT + /bAdUMXsQMZq/QSzaN3HdxK4lJ1JA6Ncct9K5w0QPC+EpL21mFqgvLmnKmEjUBV7 - aJpQlSGR/Ir4WERrybfooGSQCqNtXns4Dh96o19L00vvyb7U54zodnbAe4rVUzrl + 93G5vw+EFCkg5JqKqLJu17KBpTQYJ74DOWuGQ5kgx9zX9Kk5errD8EDSoVBRnfjb - V8/hhbzw1Rc2ttAjSeR57jUGqvlDhDFfLmYW2rRqb+Y5sjz8vqqMAGQ2DSxf+iLV + PSLZwoCRdibxem31dDmYp7niBjP7NKPdNSa4bpF+mQE/4T8PCvTNGiAqjUZkz+AD - Kh9O+3oys4QiYCBzEDfwxPMkBILR/jWXuLm3nwIDAQABAoIBAESZ/V0uEmHZpXf2 + OQ7di4pZPThcqold+3DLCQgg2cNJHMxzsllltQIDAQABAoIBABrFb2A5uEZAJSZI - intuBGXGqTAhGXeMjEaDkRC07xC5E75uP68dV5f7zxi2o0rRlIMq6vNbePzGxuWy + lBcMe5zRMXYeHGOE2JLEWSQIshDNJocNsm7vVGZx9a0PRbWyB4c3mKNhkQDm2BoX - dGYJfDpm6Kk2ILCxgr4wCck1f7TV3IGHrfNzXAJaVWF216qaettCvxgCKqPgfaNh + fU4fVvCEhC+WTXnVpdPmdZqqQuLjv+0nVaIBj+XyqqlJjVWrFlpVgwqshsDRXNgz - SHGPuFV4JMzdTRtrRB1ExpXNkLRqT2gxThxZPrf5sw2zicg8+fjN9JliPEWa9kQq + 7J/LJuBgxXweqMRQaxUuUJLXEDDs7wW6SR90mI2mwEWi0G59CWekRlg882KKhhTv - 2GiaNFDoi90kCedgakbGcNoVbKGdRMFS8jzJ5u4w8++TKBBwW1HByC8LutSyG/1u + ZTIqQ8J1VN5IVOFuWW6rRiGKLN0FYuX6nXYcTPneQi8uLGJ6lK3hP+UAiA+G8hAK - ng3oWBoh3i5RbeUcZCTKV8FN0GqZEKTwso83GIR6X5RjhmwcGAuQ495U0uQ7m1od + 9w/EMZXAdBa8eVqlpsSXB1cQTeZNbQxo0FZnOY3IqasLo4oLvgmHI3zmce2xgXyV - p4vE/SkCgYEA/9dQQKOTpe/+sOwRoKeUATa60lJUdhs1bqHJv3IDYfjoxLy1Nwn4 + IcjB0xUCgYEA2Qz1AeAdhXDKtrAIem9+1g+H3jguNsSnNL0cFuy9S7AKcH3BlAHF - OPsAlo3vSpTbEk4xeBq4Xc2VefCnVq9Xvmt/ro2UnHF8rmyPZlKlXUUokUO+3fL/ + 64j52VbKk5lNbmysR3cYmeh7CUqiuPW2PmOCVqbG9KKxttBSffFAOHB6UKblfzjI - gqg2JPYSuMJUhyXnbo8Mcdu3HR0/J2n6rCy/a+yuqkXDmFbqkxDX2aMCgYEAu7CG + THy7oge01FFGtt2GLrtoetpbdHWwGog2paMeF+uWarjz6/1USgP1b9MCgYEAtcsA - ja8gllxKgNx0LcAahkKhLWysxBkc7oXTlsx23tc0dfXKQ95Oz16pRJsEoMAvQCKM + qEpuih7rz/f+x3zskcqg1qT5oS505ny/m0Gx/H0WkFzhgh2XM9VbEK29npAivBMm - 1UekhV+zJyAAPyzQvBWL6uhzc5Y3DlIJpGRG5I6oX1wc+YYTQ9b2/1N77QbmS4mC + wx7bbIjrJnzJQFpHx8mJoZ/IsLXSKOMi83pun0EhpAjmePVPnl0+0JJH9h9RLJb6 - A8JOhFpFUj5upZX26M16+0G4DqFRJWgC4BIYAdUCgYBNR2iCXea3dOrl3ijk8jmO + esCLSVLViDIr4Uz1DHkKpHNseaZ6mJsr0xkz51cCgYAA+hz0ODUJz7sp3Vr8ahoR - tE1yQlQo6McXB9+86F+FNH57DtVeLrC/5XGkCHODf7s8qEnhEZEnJHZGQx8I3CYQ + DprW9jvHBVWXWC6TL9eeSpmRbg98AhIJAGHXh5t71JnToGuaGsAimThMj2hyGrEK - 6r/CphmBt/YFad1W9xfkOIOsfV4mBMSRXuYb/Ahjrq+Bsz1Y8/S6X7fMH414Blcl + UNpaV3/XxA+2ufNVG8vlNSRnzoiD7RaBuaIClbRLrF38Hr0m4rMSsn7s5Ea5p9lP - ss6Pdwq7fB884OQyUCAjKQKBgD999eaUKbfzvJhe+0ZGyDJG8/ND4iXsQOdHik5n + H4/YHbhcnJ6Edmx1tNTa/wKBgDCTlhFiEjeGG6zur33Ou8gZRPEWFD4lk8ci/nAW - GIdF0c9duHDBEXQBF83Hiwc+PD278lxsAfHEb/x6TNsSNAKMX2q1++hMFo/XnL9p + FeFJ64WXzApgrc6D7FmAk3KTQTTQSUNKM4fE7lDSd1Riy1tvVv+BGrddXlLenrBA - 1LmYsMihhoO6oWW/oIq7GR8TyHAhMkRHRPxs9SpfSFrnokEa0dGRZ8w7MhIvX1mh + vt5/IOYcGrmnkybV87r325LAu4gWr8etO4rUP4qtHVyOm8xBa76VuR6ohYnRrNwz - hVGJAoGBAJsL+KD+DQDZgeiKhal2Kvuv7xXjwTY71BfEHQRq0aE/ZKd+r74bfOKL + l9LxAoGACGlXnDUpI90vU+AJ03iIqDDtTklsTXEVtzz7cBb/3iEXtqQhUghhkagv - YPYl4tuoHhz+lHrVHX2S6pz748cUSceLaJcrswi2EMkUDYNCB5RfEAZXTLuUOIfV + IMrFHv91YW/KqmjWIpsEBSQdkkxzvf83C4xm1C4aUtKa8y3IHA6WwkMWvkn8EM3f - nLVBNj/I7/uu5ZH9ml0hECVcE/VUMERq0P+8OcBa6lAh5G2x8uXZ + s/4FM9Bo7/jtQKGqiLDj28Z5kfTa/CXORqG1kiNtqej2zXXHIMM= -----END RSA PRIVATE KEY----- @@ -12681,62 +14031,62 @@ vector: segmentSize: 131072 total: 255 - convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:MDMF:pi4occv6d6dflmwj4j3ta37wym:wjq4hq5z7uyszt5yzk7hbvunse3fbgfes2jinzcnbkw3aec2npaa + expected: URI:MDMF:nstlxuewinqw4ugsceyyzo25iy:niiwkb23khhxzlsze3hkphtqke5w34gb3xau2yl4kyevxyjnhlqq format: kind: ssk params: format: mdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEowIBAAKCAQEAshiWbCIuj7q3vh+IegmyYGUjLcGxZb14yXMVtP5E2CVmZ6rv + MIIEowIBAAKCAQEAoNBCT+MDYgWIm49IgY5NRDrTwPy7aG0OQ39aw8oSPplMVECX - hAng9qgwPQ0rLDH3jsqgln1zISIH9IoTkMJ3/pUTZTmF4eJbjBn7go7KHUaHLl+8 + RfZqDzP9Dpc3ngKthBz+7RW4FrS8GH1mResg9m6U89aNh2HURZGPTzQRGMju1REw - Bgf/rp1wvOW0r0Uo86sppTkRqUp0Ww4pkGJFk71QMlFCrq2KRaLzWIAViQ9m14t6 + U0y4n2ZSfo+t2GgTeU6qwYCsKhVNwz2pJNfkDp3bcwW1MsYr8KHm4iydt2QEapu5 - Kg+D1IX7Fj8BvH7SNzt98mIYpsAfg6A2ioarhIa6yEyLcFf7B8lx3MZmEzRWjjBJ + eyaalBQ6uPVn9IwWH3ObYYVdfaFDN+V1Ztkgc9t6dk9sLU5r4115k1PKPhvPBnh8 - 0fVx78yVobYbJ9l0iXhlNs2O0iqhwItpimvcXzTQVsp6bxs2MRS8awqD81V9a3CJ + Kz70AJauI7d3OFItJqmHYd+qhrDgXNlQ3pyweBRvHMURi8AqWQ1ZuHr76hB3WPhc - S0dysc0cQFJRLp28/BRhIYgnHa2fkhvPcQ9Y3wIDAQABAoIBAB+prmiYJSYJgKxN + BsQrS/dRpIupucBu9Gs12xPtvvlGykRk7CsRFwIDAQABAoIBABHNrKG6gLvf3/VU - B+MGgU+Q+5Ghe4wGhQhvrP7KK+wvrgalRcL4TKYdncHk6vWHBqe8z5Mhx4uu9LId + 4hKRxgUZPCs/76GKfUtEtLA7VVS/1P743aZ9ttUzDL+KRzqDkmEvcpudzXEaFj7h - sD/Oyy2YTGP1N5/Cshr07Zm3ECjnRpZQj+mUl3jwZcA3qIl2psK3fgZxYHn0Ej60 + 1ypDczVFHdF2/dkwn/cJu+NpYMEtMZ++FOsL8d6Xzec8EeOE9i72YholHCpWjHLi - BGC2j/8lq7Heb5gFo20g/NmRoAKHTgQj+Jzz8En+0OnlZM5WfhhCjqNlJZJD6XTK + hzDQg+uIV2y/A4X5AZFU40JD7TwJthat2H0i3ReHycp9s5mSkInTtqKDtE3su2gR - 7A3BRYmheLZ7zJk6ynxnSlfBQY2Qa5dQ8LDj7lBFI7BaaDobRXuj57Wnua7TvqTu + TB3RZ03BA3YddUjv5YlkFHfVKfWdf2XiR6kJt+tqPo4iNyDKp4G/NdNeppgRkfPZ - UtFWjdwvLp4wfwXzEXP2SaY2RYjNKCFVdFCgTPNijeJemwhqL7/q+TIpHQw33xxf + BnvTZjJIT8w1PQS6Q2vafodQa4DJYQo/EAdxsBvba+RMAxPwhx0ReN5gqt9g9W89 - dQIzaFUCgYEA2ojvmrgJ8LuPXdVQcsfNPE10ZCQVFUWdwgm9KGB94vyOtFKf6kaA + zSAV8OUCgYEAvNUtjEtvS4V5iK5TLFLgE0jH3mvHbn5wPiMwvomralNwxD/oZ4kf - UGR7I3IjC98mfu13/7ovJsuXGuqGujmnvJkiGcf9ADaAYBZJex73utNgCsk/wdpj + Mu3No/qGxz8JfLFn75FJC5JWJ3YU4LJhNrlNyYONKry0PPxEO6lkSZB4gk9fXO47 - 8ce2FnuSJYPVf44Uk5epSBVhm8PsJR8qmx97EJCGccHInYnltMw4j10CgYEA0KDe + 38rswz0Xz+cRhDlBAVPXF8WHkt7riqTuhvtCcNFPsXmB9hmtgnOyMl0CgYEA2gOx - wl4uMcbMhN8m/NGe6U+n51Pb3HKwlM6AZWMIMdTQTellxRjZmGW5k4xzLypPcg9K + of5J+X+EFcK+YtJpDnxoFU1NxgKqZrw5o4HCExfieWoT+1SGf1J6H06u2gEwqj7Y - VVLw4XGJ0UtK1IWTzAXLbXccPb247E1fw1daRC3Ns4RmNfkXHqReYdAkA+PEveJV + 7FCFp5wq+Afzlv/Rg7A/lTOIbnB7+dMvJ4+D321Q378ouj6F/0qLabtFJuxvq1FA - eZVT4++U5rALDDv7IbpDfemZgSAn8pd/GIJbUWsCgYAkD0BqKUAKpwhLFW3G4s0s + MSJ3qxWLrlb6xPN9qiuzMHtg3jiqAbNQx+fywgMCgYEAlsmwRoCSTf82rnNuDU3c - zCMOex23ettDL1Q1G2bqU35ApvmYMLXvjgT7nlPGG7ZAb3LDkbdCEYoHePduNyFE + iumqWK0+IriqjqPxL6WlkREyUjQqNEsl87g7Zv8OExr+S2kq8v3UE352d6puP4OR - b4g+9M78gAHC2Sqa1EtQWpyYawjINf8T4D5di1pcMlrCR3GBwR6/tDd8+mE25uOi + 524PdKQs3Py0/KIBJpc8cxX/dSdGomHGxA06BSnK0wTUUv6ZLyMw9lWQzjJeamcL - 4RjvbMmib7VouV5b7O7QSQKBgQDB8/yD2Ei7z+SM1mSJf1tr7bjbrzNj41/UR5JI + 5hPL2WT7O2Ao7ElS6YHTwS0CgYA0YjzVQqd9ppETNXbPgeUyUNwleiyczlkpVEK5 - L2QL55vsAsKxFKQeMNvwlw7yVzRahmqFnkEAZaxJXeToZHJ9pxly39vqbjm/vUL8 + Md1y/wMwzzc75YRnpWaojRxgT3blATLYHUTwEAsXC7oQ5yjtbnToobg/aRGw5nhn - +HWbkrV8Yecf4D2lKAvyhS0mTJa4LPVnvfKqoi3ctObgbdbPiTl7kjM6PynENyFa + FgnGrpqHGIRts8Y4oC29WvzzrE3sqRo2dCSy2/tzCX05w5PHRrbIiGyvGIho3jAj - KL46lwKBgCLm9MbqjcKN4rxyL3/GXMOM/fzzByV4/R44Lqqokz24zSHqXNgGFkY6 + yGzBBwKBgBrogWqCWfccQPYbPla6ba4gZtLGZgdmzZo9IvpKjv9zzWnvSyAd0JiZ - I2CsFRYVhLYry55w3JvJZaAOhcVUWECZq0CW2u7OKU6p3yvKnhABZNj2HMosAO2N + dMazfQuBoPlntxBBlpu9waZaw5K0eT2j6OmczoOflow1YT/PseEm9K+JgzpHejfJ - Doxbw1b5rKoQ21WTphDyXJtkQ+D4t73SNLHKJcAXoynFz3iSJgU6 + Mj3ne9iBD8PfFtuKsMJKEpofYRiQL6e0lr7ogTAycog6N7vWVk6D -----END RSA PRIVATE KEY----- @@ -12762,62 +14112,62 @@ vector: segmentSize: 131072 total: 255 - convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:SSK:fftsqsnu3m2c7qh6go33cn6dlu:gejcmq4xor775rddgx22wyxyefnxii3hlwc4lfqaq2bsawmlyl2q + expected: URI:SSK:qrpf526nw3z773vbald5la25oq:db3krhancbpchrqxt4nc2vyvlwts6yoqnksnt5ayzwq5zlcmgfoa format: kind: ssk params: format: sdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEogIBAAKCAQEAsBZiSzZEuupoJeNzcwVKFI+PeUtyJSg9NywDAvxz8Dl/pWL5 + MIIEpQIBAAKCAQEAov9sWVeoF7MGLbhaZtLS+hM6QhOWvKBvrk+zs0y0tsjwWxdw - CZVluzfbzmcKFNBUQDYmDmHwdWJ3uBpP8iyUpWhpHrpTVB5btBDcpLlUP3XT0iyd + Ojc8PflqtV+xUiJh/4lHDhS4oYhNbjoPZjQNubB11cxOxOmhCObyYvM/DuYXev6s - LHBTUv6IglONpIAdBxWsA3Ck1ExqU64LbaqE6/NWXdoFoUD/8CJI/l8nishcp6Rt + xTGv8dARhZUlQU5D5JvqoO1cWKalZoYeCw1N4yovpuxXQWHvUrq3ch0M+MBs04c6 - CrnXscgLC9eKIjb2INyWuUK0Sx0Jt9lqnTuw+VO4hHodfvZG4p3asSlk09uGA944 + OM/nbyLyMCq2FHUKqVe8aVqwYxN7I5yIhOjmyzjIkMibDEmfrBZAzvOqJ0SMUjNL - 2YAin5USfNhZbfw8kkGsKuCsZZStlgkLM4yP3Uzl9FD81yt2AUrDK41dAI/Sqn56 + IXN11MVpln54wwrNz7UZQfFgHEMzEYb9oBzWSzJu1C9MaE3GSZasQz6TkdcrOFRP - tXY4pRgd3DUD2wOBazJUcnqq+s2e6Xos3bgiZwIDAQABAoIBACFuVoIWDw13tIdA + XozQXPSRp2IcFTPnQHg29ZB/UaZ5eWUrVfkhEQIDAQABAoIBAASOHdGLXRO4eZI6 - /CnBvtNRgDlUoBq63YhshDPcbzyUBg6F0GdH5HUbgVFaEblq5hv8y9PeN1Np+vXK + hjA8cQ/zDJw/HuXLmANnj86RdLVs/SaWi5jc5U6YE07ZS0PP2SxCgl1W3+gHvp43 - lRQS77PJs2+Ai5KEYv+4VdO2Ve7odWtJShvmRYOTzKIFr/Yj1p8CN9K9X6XoziUN + eimxh7aqQ0jDymm/W7Q7fAee46K/dGWIC30BS/j2hx7UEbP3A3e2kcKIj52cnp+0 - /aB4B91uKR8PZhM77nuOXtJgiXbZC9JsGiJBAFh66Fu092+CTfRrxQMGzo0mZGAO + XM+TQht4mNdR4Ihfu8f0lt7WCABFmPm/MTRvOKplxVd3rMA8oAnzLHOIRddYDHen - pcpFtb16lezmsMkk6ojc+/lPUNQw6ZEGDeZU/Ye5i7bbWD3h/mGUfszRw//ePz4+ + 4b1qw/QjlUzPqtH3jmDVSMu+QLzKKX0d+wPEeGvVzZojs+D6iuztjvzR/om0pUM7 - W7fAKWftY3I7XGAeURKgpgCL8eq32GUGu3q8+EQilK24YA3GcNtv05mUyl3kdYDD + TXJu0FvJ3XjpzxuJrIFlQXplX3bW8WBUM+aNX1K4BlTJQkRbMHJ1PnJmsVpDBQ+c - 9tc/Qd0CgYEA1p/xGn77EOgFELshE6ZADWdfb4gzjNUwZNa87093N2N6jKdFqo5f + yc8IbWECgYEAz4//F9IaIUpe8m16Ev4ou1WaOnDrVuitTJ03/r27v6XZMEeQSGBT - B2DdL0znQrkJu8X7FCTHpLQLa4WwJxK8mbhWuX2NrW0hEf4Ts62eCt2YnSXnM5FZ + xhiaPPxd0mFHVj2SSGAhp3ChgSVc7CojC3lpI/vKmyX/k6RYsaMlwTpD6CtBVCDl - ewgtMCVfK1Fe38e7pbB1eNeBxGzLZ+HNDou6i4YmimhZQxnqgBbUheMCgYEA0giQ + DuzIP947JtfVu6A7ucRGxSscHFQK+DxDRKFet9wBIXEKwBvkfJkH8TkCgYEAyQkU - 8Olwekgdl4N0E2hU0qIPXkDqaqCg03N6kpiOajEOhfs16o3MKZGLE256XGHeerDy + 9zFTbXBktUvgwmjIiEaAcHg6mgnTKeI/KaPh1OatvP30g11sZtXa+YuSfa6FUHWR - 0ANEQh2Nl/AMGZm3+Meb3y0Uwuuws6/KwM/JcNgaLtQN/YB8IPR98Xpy52LNbUl3 + aESSSVn5omwOd5sDTvS8NcwzBQV5XwHA94fL2clnrOtsdt4eR4grmhSF7N8hEKXs - kefZhgmglCtsKkkC9TyDUPGrhOPpwnvihDrwOK0CgYA7LhkWXEMwcznKVj8Vovbw + Pkbb5rNMrNfV6NiG2ilzho6+KAABgYGWAmtHppkCgYEAhZxQ/jl+Ho3sPqwgV9eJ - ezuWjnDgeXyBobCxMDFIRZbqJ8mO8PkFGNGElGkEPe+QJlRIRqgCI18uw8tByunU + ysWY6SPFKoXPALF32SCzmfOdsolupFh1tOAjcTyW/JUoQaiS4MoY/9rt699sSI37 - XT8UoKkrU/cVdgDKv6nfhyDo3CW3U3Hf+e4z42otkJ9fhzyXwGNz8cCnf/RKbbnU + TyiReNtdma/FLHovqfG2nQLvsaUegZRHPutHIG3ir+diK0xDBhsF32gXyViEUzUf - M/U5OcFw0rsiRIgjz6fETwKBgG3lY4laa6rf3vPvKSYaef94Ilhr2PwPrfbVvnXD + rC8gdMRHagqFfBK4a0hrJ2kCgYEAsqeogxjJvEsSpG7/GaXG+Bw3Tjv6UCQFDYar - 8whnQLUj+1MN/JxndgCl1spNvy4tNe9XNjjt469zP7GJd/Ro5QsCOJHA5sUuHwWB + fRTPv8UUhwzku81NZYINbJEVqS1r+hnRE+lEW31jNG102ePfJ86kZ+bFPGQl/UFw - 82HjANgtxmA1AT3xD5DxQ/wD+37KaRDj4jI7CVVd3wvFBhIxJniM6vTul9pWHfCY + vElo5m1u/iPlqykvnYAsx2wPrHaKSuI5NQsBp16V+FCDH7808DHAIcc+xAtlSzLf - MidFAoGAM4LVlcuT9++F4xNUt2m2Dt1IMAZkPxUu7jgCDkdzI5o8e46XxBLpcxnT + Hd06upkCgYEAilmq8p9cUry8Qk+h9BWGskzWNH/tmxDtPA+Q1jcB2axiukrol1J1 - 9hoAgbEud+dAP93Ef67IVYznUw3YIWE/TUGOHQ34XxU+1UWeyXOYVmcUVx/nAAO0 + tHwT2fQBsDr73oDUHBMZTYxRCHFOApff4k/bTBeCNv08xzhU0RXo+F3Npjj4b20n - vhurhFwi6VFFC7D/zfAIl+GL4pMQduZW+qTKpgT8Ct+K9XRxbhs= + aXOwLcX3b5ihtM0ebzioMdx6PAZSqQbs0lP7mDvCm/Jpl8+DLANJ9jI= -----END RSA PRIVATE KEY----- @@ -12831,62 +14181,62 @@ vector: segmentSize: 131072 total: 255 - convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:MDMF:zmzg6e3cbwab2iyqnn3icxpxvy:kmw57dgygnrprsa6xg34y7euvjkvmedmabizz6ijsudlzw2qlhha + expected: URI:MDMF:lndkjnbiakpgufkyoq3zqiafx4:g3lynxko4n7fhsnts5knddchlym2hsb5tcbbuyhanz7fzjfxq3cq format: kind: ssk params: format: mdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEowIBAAKCAQEAwyDXoB/+OawD+8wWjumK7pY01ABDOe0++G7yHQUWfJIGhOlW + MIIEpAIBAAKCAQEA8JxtwE2NOlHhy50hjDK70rexn83ald2xwqtK3828FbKQk+UY - GvW6miz1J1Qwic6FGm/Nuoqa8PoVKY3KqhsR55+AjkH/Ikc+8CqeQKGljmgcveM8 + +chzZkUHohC3hWMt9W4mQz6quedx7vtV45IXCDVk7q461Po+68Xb+Hu3fDTV0tke - PL2oSAILvFZF03mhSKKZY4oFLBFrWB7KB+agOwkLQpOwPaYbncbjGGhI2i5dIdFW + Jmncz/OvKriqT3eLGS2Lk7zLVcOX9wdh6XC6cD/qIQCs+cf5uENZKcBcu+bjLu8l - It5roL/xSvi1WTxNlJfjsdGPApdVtURP9EiR2fLOjpCFZFu9wO5J7a28jeRRN8ux + ViHCA91Rs4mXoqkwXjvRq2xfR9DsjXCwCzhEv6Q9fswGrnAEj3K3Aog/szTEu9Ik - 2N9FcPG3Nx1rmd8BkNip0O/AFzjTOTpNniqUBOjmqhz1PIHu+vhkizvmob9w2IjS + 0sSC3NtmljeynQyLfVqmPTKuTxv6utSRZ8+sP+14yB77WHnnhBVDIiKaIdCFL0Rf - JjORGEl8qgD6aTpTH7N4fv9IKGM4vwYpi8JzewIDAQABAoIBAEBVCKJDSgbrnpia + jg+RZp+jwRQI9tRBVf0eWa0jhQKp4kbms15wmQIDAQABAoIBAAF3njEgWKsvQDYq - relKOEL3BM3MlF15yaQQuAQ3VDWX00xovbm/wFjqb50a1bHpg9q2d8aDwhem6+k6 + B6AgV/BZUvz1OXwpf0KFMxfQOVpGSuhXyvopyIEBjKT4E1UmnhnkGk34jyqrh6fe - VVIGAL4zySedvKcphCecdXZrlPDBhJBaZdbE1MGA4yuh6f2SAUm4SggWTiQ8Tf7M + WEr0TYIKHQr3P7upxiNLXVmMUyJs2L0vkZpUKS+Q3KmJ1n84OEzT26Yv1vEutSEb - j+FQ+Qzdq3e0x4tbw4keNGssnrBHu4euq71xjGMeKY9+lltfWXsc057dl/sESHlI + rwqz8b7ta/EpgtApccNjRO7eWSvgHAerVuRAmtSAuk6SVPjk3nc8IyvVhAEx4Zp/ - 89vbdwUvrdvqiDpfUTT4nfc6shlIWDGYEFQAoO1TZcxfAldLJmTAduVxdnWiXYMX + wczllb1AMQyMx9WQQAoFreRobXBwN6IQucAXZEvV5BoIerp15B6fFKwqpToG3KmS - P0ozBfX90zD543PmCkqdHOnIMSEGqkZLS/lACcmcU9gUqOlroELgL16IH4si3LoH + D0K/2mq59UsjlOyaEFu9vy1CV0M8XmxkQCatCwPShH88/x6Wbkf1ktAfq67OPv+7 - QvFgBiECgYEA49+yfcdSxQdG0z1Dpob2dWeO3LMeiUE9AAWUGA3q2DRnfb3qsBr2 + QX7PFjECgYEA9VZBp9vOQ6Uc3sVIwN4oucBQ5hRZ/lmk/dyPNQD7HrXB+YY5uXDR - E/QK96Mgp0impMTJ/dxtCRi/nYSA/A2rKPE+lMjcowSvFcDkkouJ4ePgeM3xLXp+ + BE/JzsYmKg5XP4iEGAEl49px0OY9D+kXY2Vr9rqUjStfB3UHO+PKXw7z+/NNDBYt - fvjjlfEGi+DkeLz8o3LlIfjypXTMiOc3utmQ0GGzcuOrKdqYj7ECo+sCgYEA2zZ1 + 99DQ+vkjCKT2OJfJ/HI4lptJK9QWHBdN7VMl3Fzqn/PHV7A18nu7YhECgYEA+xGW - rJn+Mwm29LtTht2skYVPqA95Os2u+6J5B1NE6rmqaoq3V8IWG96dXBRZt9NRKHB9 + 78+JvGRoMAD3YtzbitoQim6TIEuKmICmtNFf+r0GJ/sY6LEjTm1Gy6kNarp6jRPN - 6B3YppkYY4N4WaAQ2dI98b/PM7BqNfiFeUaEr2ppAxB9qy7X6uNRrQa/LvjsLkb2 + 0KoxLbHwJpOH+1X5FGcWoI+CWum5jbEdUOvQUb9+JPm0CtiGF9ESrRbAtM7Tg9OO - unoOgHLIjW/w+xwWUE2eN3qO1Suq33pUTkrL2rECgYEAzsdZIwXSt/Pocxtu3hgu + zLQLwx10agEZ2hFzAY7U7cIDW06ZsHIXJ/WqHgkCgYAMh4o022nuVHlj+ylbCD2G - YU89tkvb89T9U528Sy+l4dd76gCCjJeKoYScxyaCJQqqHW5tlS2Gy/BnQLrSiOam + NwcqqPFrpwJhIKmDqHgqulecubkq+lMCaFzDHaWHUlIsYXl1jGF2AIr9gzStIlda - YJq5nS2/+TXw4x6My+ZPkmnEchr/NbOoQfP8IT38IMZMzLtBzdge0HslRLr+N2UJ + cSyRXjgF+agRxm1HJrwIHMhjHqrZqixQ0q5Jkv2yDFKy0zWymdbAAlA7V8qFRr9p - j0aKQG3H9wNdeLdiJVINAU0CgYAhXJpwGEedkN7tRA0kO1xmETncQ+6ZSnBVD5cH + Fm0BkxE8eAO/O7WVm7IXMQKBgQCUKHMbnSs4oz/gZBGYo6BitgBg0JO90RY+nFzE - zF5ysqsC5/WbP4iJ2UltmBNHbLuvQd+HkfNE94vEqV+JlFi8LckLn7tzDGg9qoL1 + A3JSMs25NjIizrV5CH9om6AxRU4ghnlEE8rlnkWLXjA2nytXYOY3ZbiVEavP857L - wAu1fqZYtwvJH6nwr4Pgp2Q1S+D18greunC2j8GB9QVh0hZ7RjTMELToMGsi88Uc + K/1I4Gn+Q+R9Kf0nfNc6kVyy1gJ3npZ8MhtmzrDuBSxORVHKr5DzpTP5485KE2ma - 3TlFIQKBgC8x9Gnshu0Y98p7AE8QSYYptliF6+RE79yPL7VqL8AO7G/oThBWwIXh + yRvUkQKBgQDuY5L4hsSoel7/1gDI7Y7LFDO/3YDHUjdezsy0ZcbAk68bC74ZZYPq - w6d2JkweOUISA+KFh7syr1eZVOwDdYZ5FFuUoorwgmzO9MkYyqsO9Ug74lGpxKIM + TPa6D/RU1MzZKhcQCTlx2jpW01z7oqwnTDo4DnprLTzPq2Fz+3UP543/Zh99T/yi - uM56Ry9B7WVwQPUvl+lvfs0BOeaS0BevKbKOLIA/2coprAThIl0L + 35EmfCANEuaWUWftgpA/3Ed5q4ymuzGBh7R7WfQ1ZeQ/vfuPpCSzpw== -----END RSA PRIVATE KEY----- @@ -12912,62 +14262,62 @@ vector: segmentSize: 131072 total: 255 - convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:SSK:o7snf35itqnyoxqkpoapqi3xia:un3axj3ncz5bzws5mdjvn5epx2kb73arykyspud556qyaognvkxa + expected: URI:SSK:amhnz7udmtjsa2duvirkppu7ay:cocikvbx7m6op3rjflwmx73qin2q4cvm25kqjble2nuf7ei4y2fq format: kind: ssk params: format: sdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEowIBAAKCAQEAu/5ks+NdjN33/76IH92yDttv2M/Aqcxs7E1/bs4Hi65FJUbD + MIIEpAIBAAKCAQEAvQZiPNC4SwC7jB2W8gNAM2Guvl6mLO6xiNi0LCvdAzncfRmi - eT1e3VwpHgDju9WsQ4+73nzPrH+i4X3CJa7laO0gCCIEmuq6ujeL2c7T4FbPaaJL + IUXVNDbkxqETHRPQzAXz5k9hVuo5+iBqcrIfkHjokPhDfXP0Y4QN8/e4Sz0f8E0R - vjsbbxzThYmD69Sxyf2MEojGYL5mCZrqV8U5IPQ+dDn0XOtvolfM3Lqi3i9LWRkR + 3XD3FPZWwSLhRG9HIMuHf1ciebYZRSDI2ku8ul3RDZXebfEZ2PhWlOMztCaE6nVf - dpFP5fqCxacw4ZgQFVFRShkWEWzNNK5bTbzMlNjqasBuCa9P5HtppZM8Xl/EkQ1Q + X7+IYBg+qSND924gIbQyIZlKbYZjeAomPzKfSWpZzbjcGLH0DqFjMjdxDIe/3pb4 - vJBNDBVeTqfNYw1+xomrzLxmErHIHj762GTZiyhcEc7zZ9sLQ8Q8Mme/j4szCZOk + tpVubg1ssWT8dzisY+K1e7ok/qwgggeqRhy1+jjMWqma5EXPQqBA8my1uFZ/YxLz - VgMykqNWI6y1cC379daCDybxOMJE1gzhyyxbHwIDAQABAoIBADyVnN2OQglSRYih + v2PMHqe+X/rnQ+qia+ul3F0/+XQj1hfLLHO0swIDAQABAoIBAAFIP8IS+QxWKzk3 - Xhwq5aW9GTv9pAD0tQuoZA+RDUR6KqV7Oya43PgoqcWWEs5na4cwbKKkhYb5cUQL + Wpei3DidAZwZUB0rs7MdDJnFf+pJs8CpEVK9Nvgxx61pzPCO35CrtEXMRENcUmOK - M8TSKvOYK7EDSYmlaPz3RrYwXf7X6ysHVzKcuNgjqZVI/n9DgfJvKDOW6Zum8Jpa + eOcb4UY8kNHYoG5kcOzKAw/51J1nni6HCnXv+vnxG5llN/qb6GPHB9ieLr4YZ6lE - 1vfnQuR4YiIxxSsm4simVAq0iSVh+WKehUFniv0rRZNONGBq3cnKYYiRWwGK/9m4 + GKbCEeK+iiTflZSiKJ1ZNqJabwvSnk2QW1+Hg0XBiM4MBlHjvj1IPJ9GFOsgt2Et - QHEyTG+IJ08Wj9ZN8TVGbfceL9S6U2pT8fwUjYBX3di30aNrQL12kLCdX0S7EsOa + uSMJT4wzQVAevIEfa0Dg7uB6ahDMz/RZ2PgUDseVMdwKYuiuuT3Uog3Quqa97pYR - ueGWEXid64tpAZJ5pcrOEdVo1KReUM1ghb8C7eOXJUgsW9+/9X/obdjvduOyeenr + Pnws1X9MXVHPm85XAOKGtwWV7y4Hgp6YsVT3SbJWLbHs/4EAyw/ExbAvuvy5j1df - JzhcG6ECgYEAxn/3oRuNY2qG6OcFsj+8TbclbRXI10iTXz1/38GngbgQrqd+s18w + TcB9F9ECgYEAwhyO/KDviIy8UP1ilcZ3+oHQP9Du5jY9/30ATRluOWrSnl5fIg81 - iMZKcC+DIjwAWoL3qt9hDZtX0bJK27aDF70nKrJLX/QLustMwuzjG/d1JqIIesYF + +y3KVlZtnpCwFAPivY2P/a7tsaarJY1+JhwtUomTR3T/m6DBQ+rTujQeDZVEQ243 - Qj3VANBqf/+htY417dn8RNZYgQmYuKZvUs/DftWRVa7EOsdEbphJ938CgYEA8nNT + ucyGvTDq9UiONkrP4SgIIEookMC9+rBjUzTsNZ4FWDxvAfYayn/DBBECgYEA+Uqn - /y1jd97MLYUPU4K19SigDe/z6cggyWc1Pn6meB4ynVuD7CUuCkiNYah77jJrPhi8 + Xc+55BCh645lLfl4HCbw2oHaZiv+WJOUUvtwAUpLTWf5xHLWzQVXukQ5gMgZcSnJ - By6+7aM1HrDvsytn1nYyWiHgftMgBNfgYGmEPJuqMFX1BAqNGVARWDmKK+4OSYd+ + tgHW3qKtgpPsqCIwtGU4Cjmaghci9r3TvZ49fUgA0z5ZIQv/wAkIOp2cf3uJ9uVx - w92WFYQiudBVKT2AarDzXJzClZqX9JxtZVtEbGECgYBMfWdI17smAhi2ir9xLoo7 + Pi+Xs5Wt0AD+uq+k1RucGmMhlFO3lxK8kR0+oIMCgYEAp8iXr7ZMVfOQM1FSDbRn - UEXFwU1BWCAh5SrvaEpJ/EnBY525NQcYzYBFtqlLed+RAUK6v5VAjwnKLnAWNkBR + sJjUsNSgK01neZdK01nP9MFpHIrmIEKVnm+OHeLHDfBywlo5ey8J73Vs78no1aTg - 13vOQiI1eW9Dra+ItYvWbQbhujKWTNQd8IGx7J39cN45ffFeFE/Xntk/8Bi/nrLr + DYD9jAJu061F4/eoFlS8fo7eC0+imcaDVI59SLsn3KzCgBtaZHx9yatQNQ7lJ/Of - MFBfAaEdaVkIZV0DWP+3tQKBgQCKcMwv040Or4vLGkWMHAEmghISo0eV4I7IMkS2 + ZySvqAjXBdX2/fMEZVTZ9IECgYAdtyxkHlLGQMVMUtj0tfv/PxUOttPVwgC7hjvz - 8L7BrAyeydjkiL5nZNJGR1yswOF3zcvgFhMzwpPceJAGsOxUC53o1ZtJD+kimtom + +EzNmpGHVJGNPTMllTFz3pYMJ84Akz6cF3QJbdLI8eEP2aN8nWQks+EbCK7+Qnpu - c1ns+b4OZ6bGrfev0oZ06DY7q21BEzuRQAApPRBPJeTa7aFcSrpL0b9SibnnFUNq + 6+HggSi4BYKSUd/WgD0e35K8D3nOmGL7SqkGmxzw4m16y10WmgftjUt/ZstHktAv - Mtk5QQKBgAamclnCCxuHZKIwv5FARZYz7kvo5FbnVDYUKnDbMyorG45O12AY+TTP + bBD/CQKBgQC2pgs1s9B4Gen0kW8l6nVLWCeQ8ualP1wzG9ZVtayiLKUY2pfGDoDU - IKapXaZVgmvHxo9jU4vEOEdSZWBoTUTfaqU63CTPt0qTSYT3yDrIS0D4XjmFE02z + t1Y+irOwSnXoF/KIixgBjXCSqwVphZjl91TD8TF14Z0KV1DRjRLcms721gaNwN/A - 8miML5FPzrbAmqBAuInPf8Kuygh3To6XIYlLelBtUHT4AvbPy42S + C7WY/9mu+FBjue6lFtCnsHbbONilHqXd/GqKpUscosOMJHWdc/r5TA== -----END RSA PRIVATE KEY----- @@ -12981,62 +14331,62 @@ vector: segmentSize: 131072 total: 255 - convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:MDMF:frektgvj5mvd3vvsiznx64lto4:ll45oloqbsk3aygknsm63trp2d5edvjjhj73isjecykyva5ypxyq + expected: URI:MDMF:hqicjls7azttyinzxclbtyduli:cc7zh7zdovyqe4bd6k4q45uz37qg36swsd6yfegamoszoqiewoiq format: kind: ssk params: format: mdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEowIBAAKCAQEAukmiKqDEglepQN2zig3FeA9AIEou03zyxqlSZtrLwJkQdv9+ + MIIEpAIBAAKCAQEAkAdhqn9YOrOyEP1wO6nmbagEawIgoX1+XA4cglBMRg2VaxK6 - 8Gu/7sBox8OP7eUnwIexMq67fBOEVg10hovmRaGPjJ55zFmaCg/Bc4TGHuoaPtkJ + /RLYYCIqGPkuAZyX5UTfkOJXCrzQmywCq9XvSScpX7L/JGX6sla16F+FXtPaQPkx - EROeR46qqeJ7K9vgmEsBPmsGkA7X3Ldf7WnmxJ8J48i7cRSa5EGWOW1avSL+FWgv + 5PTVjDvkyYxR0gr/RsVqzdpe1jjuwCIU4K0mgLbnaGnatNMHofBTeV3n3eW1fb4W - 7BD6g1V+r2eZ7lpaYDB4uOCm4ENF84mfcaQl0lclJBRacY2iXLe+KXvlk1lWUVJH + sf8fOC+vsYtVJgMqk64hPAMtUqR8IoMgTPDpy/oIZ0I9Os/abCvRUb6QQJDtJRr/ - hjAmyvB4eQxnD006QFnobMWaq0e2V8Vux4coCuNpSpipfMho0CqfYYKtF2hlIqo4 + b4guTqQmXKsuESwfC+oZQ6VUMglT/rXnndNV8ixrQtfSExFxfk28uvz1FKAmrUZ2 - yUZM2UmgzlFZKsir+V54Wi8MF5vccowQSbIeUQIDAQABAoIBACrptXKucDY6bWHk + ULTewSN4GF20x7NrsjzwL92dgHh38keK4IKWIQIDAQABAoIBACFGY00FsLeXLmt4 - 8Gv3+ipLERGfJSRQ3zhGXxYUhuVKHVHcT2ig2ajtJ/YEpc4+gKbIW1h6ifPuJwkP + cgaGwSLSb3rdefZ1TM0twW5l6MlCeCPNpv+y6+SB4CH256cdq4YffFs3v45Ogw9m - tm0cIyKdMg1JoHMjnOl+cajjyCPs97jMlFsbstV3FvdllcwnrZhHhvTTAMMEuFM+ + gpN6kJbhAlEGxKV/HgU3vT0bXG/FGCZsrBdObUvBxqC912Vkfwe1snAupDxv2NDw - 5tkxERjwLf4MCqnk/j1gonN+Lm3tEXptzSaSdnNfdkuWNDw8Bv5Yo+Jfu7Pb5aJD + zsv9lOil2R6pXgrquleyc0aV6Gy+PYtCQo72Rhn8ik2Qblp3VWDqsJ7UwiPV416D - GYbOD8o/zAfuPMxOETw6UJn2Uy5uRms1OUbxSrQroarndkwhvcjm89o8VDjMM+sV + 9Hz/9H7FsCW7VLgL52IWgBh3/X5HlGlbjvDI2fsz/idwzu6ZJyRYn6SLCTqQEFjK - qEj7aAWIIOAGnMlVeHILMBgfFJljAaTLeg7fuUPN2Z8UR9B2MVz12vKjsOane+WQ + TYd0xhYCDKwdKBbvS5Cu1j+v4rwM7TnvUn1RE9a5k718odC8wNP2/W0BocMbb1jl - 8Rq4SXECgYEAxBLyWw630e3VXvpPRxh+/SRdKPrY34aVm6alr86TAR498KOYMwe8 + X1p3KvECgYEAvHoNB7rUjXZjY/wioqR3o5ETVr9WMF6VJQlv1MFH6Lc+vvRmc8XR - bmvznO/azm6kmEIQXjuIt/5JP7/dqc+2SjQ0eAhbmQW7lSUjHuKBp6wh5DtJFR/Y + ViQjrE2BQIEzyQL+bOXe43GCUfg/5uMosj9l9ZDjL9/BSJUVmlzNXeXlXc+/5uPs - UVzW5spHxdOSr8/NwPpYItVMXZxV3HQHs+SOPagiwRpwelCqxUF/d60CgYEA8zj9 + Mp8JFjnCPy2eVm5mlmgujEKEcF9neNtSZI61uWa7RrGxrVuVkxNtfn0CgYEAw6DV - Qd/7sOvQKqTTJFlJRpKidFXWBon+5d4dX4EGNoUicSC5Y+x5JyZf97WrybAaFH+N + 3qOyFXJsGk1uT+KMzE+M84wOLJsxQPEmljX7LzFnbQhH8CcFqVOMRoLk1YACYroL - iAX8RlUIY1PHBtlFNNBtmGQBCObxgOX2LNnAFFI0iEMW3IzJQ4euga2+/Hy7dVTP + hj1x2x6iFe+r+kbIIkuDnZD/wezY07iTBhfqXckVfAX9n6w0mzSlOoGZWlkbL/Ia - KMQH2gU5p/jSa0lUf39VaUSdFGYCFWsAXSowpbUCgYAJNjaqroNWWo0mvC3TUkRN + p9kuBwWWr9FwHefn39mX/y0bzLoHBCMRZBkOk3UCgYEAgVBoQkZwcUKp/L7QcLDR - ElNKJJbh0Ynf2TF5lAP2DnysfJMe+qMQsQOuANrPzgTvnlL0iml+83RviU0ZuEeB + GRt/nkQW+YbbY5bu8JVQJh4b4d5DsOknsKeJBj4DEWPUSPVR5RtuarTFikH+bgar - Lvi0FvhutQU+GZOP1OZwgTbKaTqiwm9AS1NRXnmGwszmc6XgBiLz5/+BemHSTKU7 + NGkFJpArH/ywW4FWWhuUF/mU/mF8tAjrVOwCywoD+V7uRTToFAgU78zvmz4J+0TX - /2XrYaXYWqykInwTbmNVtQKBgDZALe01opRR5Pq+DQJ8j+WX63h7dOO8gAiRxId6 + agD0M+mFUoK2ek/c9xUcSe0CgYEAnh2e/wY7583FxjSTVon71x7tA+RNiIwe8Sh4 - 5gHfLFGDdRaetl9PJfTApvKzvv13fgArJZwid16AX1JdwBwJqYhmNfzgVlnj8UcL + UaxryycZOy0YR+iiUMuwc0VUg6OlSfqpWeTL45kM1MIUtIMFO4LhbDdIIIu4bNeg - wtZFh8YlAMJs/K99YiU2tfTndYC0TAjRwNaWd8fJrlWT468Und5/GXJlVm2kkk41 + LaqiyQ7ACLAm4Cmlk1Snv3QEaNvgd44tMUD+TLqdopmbDvDjnzAWBC+Hap9pEFTv - jOhJAoGBAJCGNXwTLn1tT4u4CSrm7aUXJpYzHYDt2VCZG0Ul0m11WP9UhrnZF4CL + t4HxIAkCgYAcPS0TVpKiTqOek0oBoLxT4VZfqp2E5OlI3cVPBP7qEJTIaxAn/T7O - SYZqJinP8HoEmFxj4wwAmEAZPKJgB2zBJw+EEJg1Gh3aZiW35b4n7cjTuCpXXydp + /U1/zJppJIf2svpZLWp+smkUV2VxWWl4k9so46KLBYVG5n5R4x+g+YbgRKioMRGm - nFQK2C9j1mvjkjN91aAwDHs/4TKZJrE9pxDzFSfGVYB8hLVgTDNB + cbzUaxovQsg4RLRUoKUzqE7fxPCxSm4aZkGjnDb7JvFsxhSeafnmKw== -----END RSA PRIVATE KEY----- @@ -13062,62 +14412,62 @@ vector: segmentSize: 131072 total: 255 - convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:SSK:mrqbm56ktfuwkaqvyjc2sxi23i:tuf7v3b6olgzfszpzaio2hvbwlvcvy52r6juxaxw3pfsmwkuytha + expected: URI:SSK:ljw6sfa5f5n2x4u5asgq2dvhoi:wmjel2jkzzwwbizyoqj44czwger4xrjus7r52ujhd5mvlnkt4cja format: kind: ssk params: format: sdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEowIBAAKCAQEAoZ1/YVjqmWSf6ka+6Kqz5zF9YcQ6hiF01IsTbMhCPVyGvn6v + MIIEowIBAAKCAQEApjugYoc3XSJhbpTGOoIpqlTOTKHVBBfMsvXXlYdFIHmQw+cu - Xi/foccPEfNPM2RT+UNR2bARpcj/mnoD2r1/Jf1TQv7VcoDEsCNye4prd5c2hCTP + 6HorcanqnJzGf3L0oOV2BAOxv1cee/G3c1xVyzqzEXR66esHHA7W1C+9z+H6zTc/ - DXhSAPAB9dHqvPs4EEon7srIpNq3TtlbWoDO7DKSDzBErGbxhu10guHoZBsYScaL + FNjEixwRa7IpToVm056kFmnsNnAW1Xy/OJsh7n8YMZ0dh7GGjb+q07FeldpYJvzm - +0R/KdiUQn4fHBVKTgJwOEYBaTwMpd7CxOnR2gnQKmEAYw/fXgYzNXAqni4AvOdu + 18gEovb9N/TZUab1MscTf/ne6scWGJxAN2njmWG/mhyoPWOha07WMPvqIRTc4Knj - 72R8eMrOqr4MegG9DX4Pl4l45Hq3D1LCyo5SFS3QcD9zs3QPBzkqBAMzPIZ3VH6z + 9YD967WM4Oycml9Mag8HOpbBZ1DH6boKKHqROe7v0SjeRdt9NCfRWJ983E+SOJMV - eToUunHePOfN6AHWSv7D4rSNJQPxGfOpzhsnFwIDAQABAoIBAEpV5wEfpMhpQCTB + OEtD08tj3rkweCbG4i5lEkL5ORI56Y+9zjXM8wIDAQABAoIBABQJcOIfcU1xFPRq - 5Y2e9qCgYstVNpX7TYF1drnSYqVWqaN1IbRw0KvYo1XeU8+PlhBQppU6JuPaT7b9 + y2AHC3WkBj/Xa+Ez6zERD/zOkscA0DHE3nMYMr9fH0/kV8rJ9PGl5u1B8r1hB2Qi - 6Ef0YUdX/bQTAppoIA/kPgQU5tla8/hT9eh8Lzu/KSeoJhBGfMMBWNy86QzqjOX3 + NR4bHZ5DA42RkDU85pz7ruphnMv/bacpxxlAraQk7HaiQXdc/hF4+EdZWicPqLjv - k81M8eAyYnwZ93xU3ULydWS+A+YG1KGluEjait2cmkgyGdu7l1tHhY9XVXUeJ3hX + 8e6lSFhCioyEZyhRfin81d7xbLi8KFd6+LualDFVrOjFmpAyXneQseUfc40kGC6q - 6OUfSUNYDo5Z5ZLialAws89GvJxA9FiIQvLx3l31+X72wpfn5GdOvnO0TiCipB1b + a1IIiNTleQeyFjIhvWRV6lxjZ3jbq/79s3UtO5uVcsh9r1xThTIQZTW9NdKAJdCl - I4VOENKDdYhKHUJIIbBE0sE2i9y24GsSEebeRx7e9xmPRVcC2HJjY2feVGYho27w + wdPg8OwuGASuyZkbE15ZTESCbUN2ERoC5ZTflO6qI3wcxSq4LrGA+uq1N2Ez/onl - 8i1l+gECgYEA3Qmvf1mJhb+G9/UC+9y3PGgzpks+1PF/xIjIBw30PyDiyjvK+2aS + x/bur1kCgYEA42DJos4fx6Q9hkUcuaVMdF07zgw1lQ08QVj5/VndgY9IEoHJ9DqX - bWXTne2ChRRETpMPnTSLOy1oE1Pn6g6Ud20VCB/M/c3qGFsiqV7CSKdj8vBViL1y + F1ZdMWzHKKdPanGWiVOlG3m1xaThxkFvhri+hFb+PDQAoAJ2uN2Pc5trRP/lxE0u - +b8xbS97jwvIv+xsVU8rSy7TI4FkAPcfZUttfzr7AQ52yFUjjkR8nQ0CgYEAuy2o + 9e6gY/bspLxYWiCZwtzZ10Qy6m5KSRO3REC++zNVCd/E3F7GvyqEGwkCgYEAuyhy - fmBY+PA78DyGzlcZrgsMeL8gaCZP4KcmNpOPika6tbUsIn/0gDcEWX5TunBjTfBC + zyi1lJP6zkFNKtgOhPwBitd9CS2EshSqtCViKS70GjyZWqxzyq7V2SBhKHT8j14/ - Hd0vh0ZSOhMHlPqOdVPh6hi7YNI1z57/LLZh3JVLJ9zcAsVQIdjS/XVVhq0vVl7y + +vRHvg7dK/lZelwE5qXfMODkcrzdt1Pjpw29xLTTesPRx+dhDZXYOwsd4Edk+t/D - U1qQ17OEdCNGHrAit793iZZROOQWlvak40v+87MCgYAR4PeuEFr8U4qiQdI09xxn + igH2M8q4V4iA4Xp2CwrVbwid60B/p0WV6MpFGxsCgYBfSQg2ubqHp0RBKGVJRwQr - KXKMD+gMJ2CTUBEF6Q4JkSpm+0Em5pwPdz4PtydohkQkKucHazmb1sdlUNMgbn95 + H4cYafVqaQl/ORJKIYa57Jl/Z/SB7Ku0k/Sp6bPsTXDyYnd7RRpD0VVjZh1XP8TE - zXv3BUN6gA5gW/bIxl5mrAt8mg4BGnnTU7C2yTFwV56sT35PxDCXSzlO1Od24IZM + 6FaujuYrxH8ejunBvteG0vK5D6PyB4ZOeZmtSqUQw/0ih9bn2jVQCLxtkZp/1UtP - ljZMJUQqSLY47BINLuL5fQKBgCMnDAHP7mWyGE+hzl9qFDSPdqQmoNtudonmWlLd + xvJBwtk4MhYFY5JWOjLyQQKBgC54x7E6qYPADsnCGzglN827iWKBSVHLFKTnTs+2 - m5OIfQArKkLAbRa3PmXgR7E38i5s9L3PEGIDXuXxNPdRpvd57W+dfXNNhzWa0ql/ + bJ5PQ1t0apvCMGpGaWElkhpqmf+7ZmWY3GuL4000+AvS54Ch9T58yRzYWrFXyjJD - Bxn6H8c4v0j17Xqt0dIv+wPz+nPqGPB2jcU0vadiCIUy5xJDLxvz0wUwMN3hLE5T + zjgWsmBMWT2q7UVjTLK0evGiqKdGgpY1EH4huw45Hc9fCgqJ4R9V42hztn7BX4zT - s2npAoGBAL3K0N5zwUQ6WbhwUUOsySK/yRKaO/25gB32Ty0IptCRGcYFy0AlUAPg + FL+7AoGBAJfX4rjrlxziDdVLEdJZQmJYOOVVzXKLJ2MF4zsimeNa9FjEiAgiEhBD - qk8LSRMSMiT5+pQR/Y1CfoL3SP1Uj4rIazdbyJX3kCuA0IsxbruDTJ5EPPHZxroO + /rPgnX4ktS6EmhBV+Y6jibMRNWsadW3Ax8rtH0mNuTK9x07RBdPg3aagGJdNBjty - zuLxkMtaP3moi6ZN6bjlB/SoTzNgTXqewSjD9URejrUpI2qGIxbk + Dt09KtA9KOPgwTHhKP438g7Me6ymoZ4e0/1vrvEh+Buejulsahmj -----END RSA PRIVATE KEY----- @@ -13131,62 +14481,62 @@ vector: segmentSize: 131072 total: 255 - convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:MDMF:6vmdxm2fxaycmkip2duwcjjvkm:76m7xsaovohn77xmdijxemu4mjjyf7hnwh3kjtkzd2bmtwfq64wq + expected: URI:MDMF:xznz7n64qdxjwcqxpo3mufyxca:l2yaiaxgcyx5wch6k2f7x7dindcddjjnx5odka6rbng3gsdkzp5a format: kind: ssk params: format: mdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEowIBAAKCAQEAsZ62I1ouovh2L6+f6dWQdENE3E4h+ZL2VGOZ7MEJ+nQ8jlJ4 + MIIEpAIBAAKCAQEAkCrFEf2l8ZyPTx29aApnYaAK4d8BpZ7kA3dmGshLdNhl3e9V - c27iOwYxHA3OivMfjOdqbOi6ROlmnymqNnJlBTRO77k8uSX4J+mC1+tqeTP31boz + duskK4G9nuBSvk/SVq5v2cky6jx/LaAr2+3cXdpCvFJzqWpvsiGP5e8jkR03ED5A - AOnJdSIb3uG+0ela8e3+vNPP9X/SBkDt6vMUaTpinKZvCwOSI6KdubtuvZSDZEug + qH82iwztFpNu8Bb2DwhMr8LFXB18ijPYBB6GPFYsWFXjddbdkMDY4yG2OtofQsUP - WZuYuA4E89pNw8GR2EQkoEyFpBaR2a8dv0xJRYMDzPBLeluFqhH8iTUWir/KcOS3 + TwQiaRRZOkvdzSxLKZmsao9wrbE0oJanlWk4B09KL1htIRDv8SCuTGRHqFZ/ve4M - bESyRJWvPQHZk4WuKYF9VX0KV1w4GPOWmziEI0XoXS4Lf73yUfcT8YgUf/6T0Efy + 5h+Lp9ykUhxvS1IMymyExm8grH9jLz+kNO0Wgj5UTCa6l9iD9maNW4+2YKeYSako - QhpAPJTKQhTeqOH4Enjv4FEWWvw9KFBe78/VrwIDAQABAoIBAD9f4R3xBfXZEBZI + QcwEKEFln+7cBQQ1AQNt0+vPkLgXcoiWWGRIkwIDAQABAoIBAA1w52G82nazHYtc - pBajQDTzcYTnjeDGKoUGuruKTvyhb5/aVibdt/OWbHxVgs36HFZClasBSMDgxGBi + nvbvGAy6jU6Ev6m6LXut60eY9VawrEKrreCsnfkucJ2PlYynyC5hDLhAhsOJng6E - 1dwyac/3D3kiT0PCg+39t9VBpo5TWAjWtG8Ne3eDMY2PX233RJ9QqxUFwEwYjL/1 + w0IPJFSzD4tw9oFM6pqwKrM0f1uSs+UK9h/03a12KDlKx/TmIc0XUtWvh5NbLmwm - d85eZ/h5wAijq7gy0HhNg9hqw4L5e1boleRd058yKIhgOc+QWA7j1IyWe3/Z/ZMu + LEAqEQ2EDUCzypqxzIN9RLCxyRnQbhY1B5Vsq/8anmZPcDIRuoAa8Ed8CcJlbLAQ - +yADc8OJEIaIntls7mJNb6CXuQikxFWPtz5O7H42qFpsU5yPSDmjaw7MyM/7S26L + qmpOjzvbDEplAs61HaIvVBM7H8tzZ3h7HY6zipPDhP3fJ+F8o5gGOZzLGYz39pHB - l4I/YXYRsNXYqj+l2KPXSce8619P8TzTsGRRBiSKdRrbwQTKyHRjkp1l+DpV5dDZ + lNrvrk4g5CFj6O9h2BBN+sW3hSE+KFl7jSt1p9GGewO2hnZsjU2NAbX5Rz98TNPb - QP8OC90CgYEA5vFnJBcPdApn+xL6BCTwqsu92FF+p7pGW2n2rYZTE9hN3vQs903/ + TJSkNIUCgYEAto9/b8ZzJ4OOHYzWoVgOMkhNuE0hNcvx9n5lFrxWNvx79Q4CgrQT - wjLyrp2Ax33LM3eUkfl6XxOTfOHnZdNVpumMT8ARc7uL/+dsgydqNj9vnvepgiN4 + Bk6e07o7udcapjj1eWPj+PfVFALSmrO1lbmAg3kYMzVdZrzOjgYKWCPFNjp/Np/L - fRhQPXftsfGnIgdLPuoowDppljvQ/bnsQ8lztBHwlstS7XXgUz62DuUCgYEAxOQ5 + i5iJYJz6lY5rGrzcNSjDwRl4W/iVualEoYFfH/cYt16gUn5I7whELqcCgYEAyils - 1PRujR72L4LsiUYSG1mTu1echK3MDwh1l7eVwUk5SBTZUaTQpTKDid79cHcMUzPt + uOIgEBzvkB/tq4GV0OhGcSIMBPES4ydhjZhWO9PvZANfklPM/0MfWWC7as1LSQGV - cp1FlBgxi6xg5V6C/mkIW0tQ0zD89uhQOJKstr50LaB3dVEgFYng7nNa6GyeXq2X + jNVxmhrJeZzlqgaG1e+WDclo5rBn6UikjYxNKOwM1G3z3ekPhzdvbaf+nH3F6PR+ - P+GsmZUAMsR5sFoG35Daz3zwrNabmPDxozZzdQMCgYEAmE0TTAW5Nzm1oSq+nwUN + aBTwzT3hIIWFd9+CDeeMXqoNadJZLAk6GQn4YDUCgYA0Fo9qyfmTPaLv5X5bvK8Y - glWi+YmlEVATHi4fdAhluWyoziQRk3Zo+NVInkdYqjcXTvXJkQsJ3LG4Tl9cjxZ0 + Q68BNeiS2+Tmyrt3GDeVKscHbX7j4hNHimkgyhM+fBRbdwb7IrgqEjRWqFOE1l+q - IgNbeSydVcmVZkpkkYnozaXAIwIJU724tCbYo/D3XKaVJifRQ8iA32SmRWFlTi7S + H6p+WK/B9Kj4pkhdF3YeHd6oEVq4sDE4XEZeLYwF3gPLNjWyaTYpQ2Ym/69gsN4n - 1VGBcHt0Qr4MDnXyXnO49NECgYAOSEfxrLGARyiwlZy28IBLv5m500clUL4msQRm + Iq2MhkkkELi3sNaIdRhXIwKBgQCnVeinlIzjqX/mdXc+WlIPDOSZ6ou2X3G50rQe - twiD9t3S3sBM7dm8wgdMrwJPcDNSrcehssrjTUX6zcxRlyOFdPUIOlRonXscJgn7 + BzWB6iiiSWSHc5QgyoedbMNVYT7q3EPUwix5WajhYCx+M07SsLEtEkUhhm1MnROQ - sJgawYIH9UX1GqdrKI9KfM+xYH+0en8oQSSWF3rmM95n7n/lI8rblkKXJxIua/v7 + Es0fjVwFTknoqmxvCUTTqJXJJRZ3gEFNl9/Gk2zQhZT3p2s4ZSw8g1f8+t9S4wRT - TO3fJQKBgDPDQ3U7S1LTZsPlRwaq9abgnTSbu/Qt828RaCI1Cct5iflekF1TEjwu + C3yq7QKBgQCNp7VRwjdpSliGZzPCcKjPAy3fN8ap1XGRtw7I4AKCpXqS0Q5ee58B - F3tZnslI5Su5H1BukxVyvS35SqKJ7vzMlHAhYp1T+c7l3GTAun6AvA3tEX/5heL5 + yKWnt+B8SZYzZUyDmOr2FwGC2xUG5gM9WLQ0S6bDJvrh1qHfPCtFZWH4zeUWmuPi - JmityktNTsG3zrnPPiYmRxk9Tm1Ec5QPMwwYG+T8ZS+S8oYPpsRu + B9vtQibh5X25jiluU+wr/pyFCoHb4ULo2ddBVB4JalLsrhjxYq0utQ== -----END RSA PRIVATE KEY----- @@ -13199,6 +14549,156 @@ vector: required: 71 segmentSize: 131072 total: 255 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:CHK:dov2az3yktkdiinlcvvmhsjwfe:dddl24n4cj4ttibyu6j27ujcstssqkfzu6lp353iuezfvdu363ca:71:255:4194304 + format: + kind: chk + params: null + sample: + length: 4194304 + seed: 2NVWOXvtwcAFf0J/g0fDC+ypQbnCU7awZoCxcnhpkVM= + zfec: + required: 71 + segmentSize: 131072 + total: 255 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:SSK:67grzglua2vdwco27f6euk6ybm:cn6owpmbn2q43vfd2vdsq7ptpz73aibiboslvzbd4bdpl3frym5a + format: + kind: ssk + params: + format: sdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEowIBAAKCAQEAzetleq8DMBrg1fbfhl3aQZH6aWLrtcBK9/TrvK66KhxfXfdk + + FU1I35e6O92+oaOMNIkQzVrGNTcQ43qqDzuJEBLxLZLIH3DP/dN8c2mbKlqF903s + + 7vFLyTTCzB5UV5IL1OhoWLrrqkdqtvSRO/gnAD+IjBUuCvqqDW6vWlLQPTzIkgrm + + 4GobyH8MOHX304CX/zll0zUbJkcukeb/XogSOg86CtTXy/yCbfU7FUf9QdauGGmb + + ikUAID9L6sXeb7lqOlJPj16cDv3aGKNsQaN+0QZStXHElpcYSppPrqqOuH48cCH1 + + EB4oVJOplz+SIeOXRVtJScF8jAUHwYq8I/1OZQIDAQABAoIBAAoNr8MBSmw4fmOp + + OEPiJHwbuW3DGPOjCMX5sjdgZycT2DVz6kdT96EwUjUGtX5EyDKZwoqUH6XiI16d + + KDJdhhU/0itBxagrT4GlUqfRLx8uk5MoaHnzU++Qi4uAVj5+8I7w2ISvKOmuWkUb + + xFTONRvrqNSrWrfJ1zBCstL7xL5IjqpZRUF2xvtZu1QPAxKz6MxuGhXxUI6OVTTC + + F2/hZin7Qb+mRb6S9RJlfWhmMxmne1DNxKNh/miz8h69KDeMKh7+EhvIqOgtxJQp + + XjCI0rFVtx4ghmPKJdvbOizXnSBx+IbJWQ90IPkNsplIt45eaVrSb5Y9DCJxDWoY + + xdm6aEECgYEA7XsMax7QwWox1PDqu2s8YA9175s48hhc2alBS/yYsf8adCHvKXIL + + 5WvmZL9PApMFSCG6K4er9Z7nEdoF3kZBuYRm763k7qaO2Nj7JMDMX9TPZHTb5MYD + + TPiMvNCkqaV/S9ETPe1aeZU3k/aeX8RekYYtDgQ6DWVaZYg7IHZOvZECgYEA3fpG + + tZH2+vz7JzAZ07L+5CQEVX3hWCLi6PGOxqqz0cj2xKntz3XcKNWU65LWRNy/4P5c + + Ab8gfDHXX80e67QXadZcXWmCDLX2BmbJIneclxCp//JdqsVOdILZ6Zc7h5EATWWl + + 8YEh5fcgDX6nehKUwLv4aBNdzM06m1uem3Og6ZUCgYBgT94ad5XsSzhIhyh7uCL1 + + Rm/rLAWtUaoecGFWAuyei7pbzQNkyKcAdYEr7NaLUbr7pQoO62gXJknKWKS2n8G6 + + DnN80wacrxoR4fYA0txQJUuzDx27K39dMRRK40dUshTtV665F9DwrE6tCID0j/xW + + gpc1LwuoMSm3McfhA3otsQKBgQCOrE2EaJQRUEbxMiZ3fiX8ZvXuKSGMr6eex5vY + + L2GypfOOBhaW8I8YI+c63r8fta8SowpqCPmNOc/PgJyuLKub2C63z5fKKa4/AROo + + Nq8MHabWnmX73COIGY6MaCrYAKfsFzhomHI8R/FvGwf0GztHAcowwrnYZ9SShHnW + + OqSjXQKBgGdHwIpNiKsM52SevnkVcjN5Dz6rrHb3t9VNOby2LkdYnWdOG16NKPH4 + + IoQ62YPDmcJomTQ2AjEkM93ha1e1SZuQnnMv0IMbE80LF4qHuWXJ/gc24Ruy0xOi + + mQRQ1tKPZ02L1BTeEfbwSR//OA7trVsBmrck0a7Qjqb/ogpSnORH + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 4194304 + seed: 2NVWOXvtwcAFf0J/g0fDC+ypQbnCU7awZoCxcnhpkVM= + zfec: + required: 71 + segmentSize: 131072 + total: 255 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:MDMF:pe3cd5dbpwdrm7lmeepipeh26y:3mstlvoiza65p7zol4eenbiz6k4uvdgmmpnfxklaqdvwdz52oq6q + format: + kind: ssk + params: + format: mdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEowIBAAKCAQEArSoxW/rzJSR0lE73K7ZigVVYf+cxvOJfNpmSGPZs13+5Vm6p + + krmH0fUIzmEfCE05m2dN+6y0C9JzfsdWQyqsuSeZxCD70ND/UUNzGMrF4EVGoEH1 + + vxkyK95Rjbg4q0oeiN9uVua07ry6C9paxjTa/27sFn8DybxJTdofCHy3qaijfHiK + + t1814hXv1rU1hzpKScDqvU4Y9jG402I5bCdccofKIrs2YX+SSW7CpqQTCoGWPOIs + + bNJK6hBGXmmmPc4aleACYOzOUsdWZUi/80OOAsMMtYMkQPVbwazHHupKpabIsJ+r + + J9brBqe6QGiZnd7QdlnW9SmKB/PWhrHrXd4gaQIDAQABAoIBABR97zxmzJpDH6aj + + V52tJigO/PuZ1Ol40nKoJsFcfBHedATV8KxD115RxHqDxMPbO6t3xKM5U08o1vEU + + TtGA/dKlbI1op9QUv3oS5M50xIjfOdXiKF42cZj+ZKFEQTSH/2gMJMcU4ylzXQLl + + EqPtAlODAV5CJqUbaoNTgiOjeqqRb+yfbTZZAN2sEL45XSexlSM76Piv5hJt34Hk + + 4w/eXRHnr+L9WGvSMnxAHfLr5Yr1PDLyFqRyE+rHQ5vIoD3t7oWySO0+OgRTUdE8 + + w1gnM2fYHLpDZu0PohpHo/51e3N427IWUFQU41m1Py7EDb6vyNLb6s3PnFSp82RB + + oaRiMJkCgYEA3fAXRXqiNCX/pX7iq/vH7vaLP6skvUKgjRe4s47E46xnkiwqaWBM + + FRjTQ169Ojs/L00afUKkuox9UG1Cp/cra311KHiChNqBqba5Eos+sk3wEBpql0FK + + gwdeKBtt25mJqMjvKGMaM2MYpP9gKn85JFTo9KGVAFeyPqhLCoMg86UCgYEAx73Q + + y9aV2euWy+0Z8HsZeDKDSB+u1l4xCOIna4gkTTn2PP28xYGVx3pcvaZYWVWUZA8U + + U+1eUo+s7Qctj6lXuPX2aJI1IlpFxNNyKi620srfnDlirq94b8upZySUBYUD0VzB + + SDbbqlVKsTyijUxhQuCSFelo/pjC2p9OAnDAznUCgYEAszRINivdiWodUM5xzRkS + + yVt9+L0Cf2erKAI9e48OYCA3yQmsfUXqaSaQf9ehx8FLNbB2cSo8xPznuudeaS3l + + e3fj//e+u/OLuzP1oIma6HKSIw6RfuyTc9WhK5VqUWVaiFUm919+KnwbzC8AwY/U + + 3gdJyy4lmA83t+xAG47iLpkCgYA7ITUTctXvqi989RbNuxNiIsn8auyuJzoq4BA9 + + ZBMjDXqYuaDNczwszktwFTNoVs5UBKbG5akblc7iaFKTidUfOykT8dxq7ABlcRcF + + 58hVhJtHuzE8d1OW/NqMXya2r5bevq+1OhAzT4aKC2IvpCHS03pLpEphvEVKxQgp + + 7skVFQKBgCSmyb/MDbRe1X24pJ9yc6EHNH2XifO8B5/9vI5lTMnHzaeva9YM0MGH + + OrsTFL5TrTTMpb4nQiQafyOT75HJCCoeA8E6tnNyhcwMuBNOCIoD0CpA30HzKVi4 + + eB+5h1t3mPf0kWVhvNcdidGMcgyr8ekP8GYtE9GnNT16U3PX86Xu + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 4194304 + seed: 2NVWOXvtwcAFf0J/g0fDC+ypQbnCU7awZoCxcnhpkVM= + zfec: + required: 71 + segmentSize: 131072 + total: 255 - convergence: ZOyIygCyaOW6GjVnihtTFg== expected: URI:CHK:znxyqo2qglpkhjafnuesu4foge:trau477bj5u5wjfpy4oq5e3aziajgbpyfjdv4svyf27m7atpa34a:71:255:8388607 format: @@ -13212,62 +14712,62 @@ vector: segmentSize: 131072 total: 255 - convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:SSK:zf4xzfaaxdqql24hihzcosu3ry:6iyxlie44txks67hs3mir7k5oxruafv4jmnarmn4dzoqtu6zegza + expected: URI:SSK:5viwo3rrpmhruutemkyjhkna7y:c3pe6gxnospoucmcbujn3mgnlky3vs5dx2bbvqm4uwjreu4vd63q format: kind: ssk params: format: sdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEogIBAAKCAQEAwEUK2HGFp/yqwhrgRen19FcZa7QOshoBUlurIEf79PbKBxsR + MIIEogIBAAKCAQEAwRdN0XCfwI8rQ+l/QsT3MEwTmvLlKgs0mYInN/KH9uMT2KpI - D4IQJAwsAJY4A6Vy5QM4U4pqtJpopirj6dTYnY2v9cfdAINRL4D1yqwKR8NJQK7q + t1fjzryeOi6pPRgnrAFMRXGaUo5i+LkKVS3ULBH6UTlUvJmUVpQjToZFvbLAwg+b - KKmS3kPY1cnoYlUz3qmykxtGJACDTRqssA0qvkzKPbR86beaFg4bT8esQw1W7us1 + GJjM5lDYXeT4LimqlXHxOLCTwjoQljsM2whYgI7Yjhm5LesBgiCf55GU/lw2gup3 - 1F1c1hFrI+9d6UO6a0SgwlZCy+zPMFLIeZJvE7bStDQsZWDKfLOmdFt4/+ks3t+P + zfnpeqOtwmQfwzZoKIpEQ4ZnCzSLldlSUmR62haEBGiEfnZDVCIvdf+oC0Qqgua6 - Gj3ZZVl/oxxmAP9XdI0zIoQsZ/ze0PkQ8Nbn+sJWJIcWx6JQp6a7KyCHt0W/8pkt + L9AofDFZYRemNZUGIRumAPoJMk/hPQZo5/g8USz7JiJKA0DI39nILF6RLpgAyPxF - m6YuZofNbWuWRU2bYyXdUyFO4dxAJUZOMI5ulQIDAQABAoH/PnUOB9QmD0a5auMP + xX+X7bwcxn+ycSIeEi1lx2LxaJttyUXTg82nOQIDAQABAoIBAASx5dz9UY0TjhhB - 5Zgkzh+dP4GUi84M9DF/s8mga6qydpdvZchx9FTavzHo+kE7D3wgb6OAL18eh3y4 + IayEcIQ2nVVrqYHLsvQ2k3CLT54DqHRgs5LtqqbYtDoy7z+Cilhm0a1wlTGDr8lf - ns6NrXlwckzExzezCaNRP/SM8/ATzZ39ZcqxGClKZQFJ5V5AnmqJzBKFgzp/RHSv + am5mxl1p9H2sGLDbRR2TzYX3wtNZeNFfIsTG1liVR6WEzzoEHlcy5Ywc0wLqeYPF - G6CWxH39PJC2RWgHuvo7hqgC+RflJ3psa8x/4C5qAdMfSg09C0z9PJB0rxfKo92s + nMromYpKrt5JptSEfc1lsK2nPwmuNm4YycfqYa+pXKaO2QKvmYjGVZ3Di3ZBKl2f - kBIuTwhVjQrTL0tSntV/I0fFMRmu/ddikaEG5dtLpJEOd5ZPvqSXi5q4a1qMlZ1Y + pf2MRrf87aJj7admBk6PKG2dhs5FI0HLPjNmvk39EKpLFyJiKFWoDFwizIY200xB - pEXUwNSMLHdIpGHs/BdO7nKAbxnx1NQYzkkSDcrfPMT5b3F4eIju2aa3jsz4hOe+ + jFy86Ic0PGVz+jSsTWYH1+f2w1PcZ2Jjss2EF4iZrOXdGCm2xgXuf1R0FxFMs40M - 8gC5AoGBAOM0DPiIY3T/pFdGlu3tTjAatx/li6iWGSmMnRR1adQvFA0k1XBUDcTe + 714HU0ECgYEA+H8lLs106NG8OePE+1uxtPKGz8OxGWYZoxT223JMJzq7pxfrr3TK - peK6sc/dgGUnkbhulLEeAp0oT/jeWXGpQeOzMgJReDDjuBIMrpQkYPt2D+o/bKU2 + qRswldZ/LlzuZ9tcEyVATZqUUFcRD3kTn4/gQb8S7spEHTTdZm+fhAS+EyKBhVOw - e06l1/E8ByLOr5y9zDO6VBHMjLk6wvdAXDYwbwDJ4xnVwH93zCqvAoGBANijhK9f + DInkl6HBAO7O2wvp/Wy1yz9kcrerQpJLWxxKMVyOLO5nX3Ae7gj1PUkCgYEAxuvg - sYlfqnovz5zqagddHSFuw9ju2yQCu6pXu8SOR3vKSzQvmKkkfRhEqKbiWtvDMmLk + 7+MsoJUi2AymHp2Pbasmfq9cS6tFD2pPh3isFZFvU5hLqlT83BOZAI64jkb2z2/0 - aqMSQTxdI7QG8/9FTUSLtq3j2mdHRBUKAVP8l7r31YiJ3V1y4WnU8FkGpLA9fb3p + ryGLUtoA2zIRSKnSp+cKEZJxzQzRxKlRy+HupXvxqVRLwe439q304OOCclkNJPfv - FahoSLBSUqifKrRV4ke0GsfrzSx17megqfv7AoGBAImY2koh/2m58MNSYtGRKAsG + 888gOQKmyTfguP+KvBmMkiqAqZXz+snaHh2iynECgYB1/2oYn1c0duN6Wb3f3dq0 - AuV0VRIyZOa+29qqCP+Ry2jyZ7jxjq0t0fTv8APdN4cLYbr6bV6euCKJaXVk43Js + obWCUtp1xRXHat0Nt2iR+EHDRoiT+FGDm3WmsQQTb+2FQ5SlQrsWHqDuxWlEf6nh - eRT3T1AMGugw4Sc9OvVI2tsvcxAAfUHJLwBAe/kCy6eO2NfqMiMZsxRcdtUu+yhW + yuAiWCkVWtadR80aJ0cH2XiofWojdWnTimcR2a1cVAnF2hJyVHy+1otMLgsUwYMm - eAHxbyhhHAJna39HBATNAoGACnjdER0vF9ToCMAG6S7rsS9vGQ6hqPri3PrE15cm + 8HgKmHiqvUo493S4c2iAgQKBgEIhCm9VS3G7ApFmaxdEc/kWa76z13AEaPn98qBr - HHpEOletCvjCCGsbIPEwteB7Q+RLqzwfa4KWZLSb5Tfw04YmFgoq6nz5McTgJaQ2 + unGVHrhgqc7fYAxdq4Cm8a3C46wEYQiTkzig5qX4GAza///3a755u8FaIKZLT7kC - LDkpnIAecls3uCy1eMgyVhtcGqjeSy/ZPCrOWLeiB0Sqa807AvxRzxg28s9AlwHN + zA5RjP4o2uKGqi4kmILmv2f6OMdwcWHRGro5Km88V0XJFjsAF15EKO+3vRtDXXKj - NcECgYEAsjgNzgVFMXftH2jzUZ1wsh+sWvcQXQg+vP4jktu4g2gkVY+OUVrM3PeM + kYnRAoGANztb3ObR6BF8O5x4SWcwwQL2BGQ62zWv4EWESwjlKnRzxcbuuuo4zLjP - TRAEALeoYwBsKQilrSl02BqbuHC/KlhWRK5V1pD6/3DF2l+ZbqtcD+s+INC7qtEJ + ++3ZcCv6IXd73pFgq9GNziTJs6xS1Nfa1O1BcJmrnHVuLjfmrTP7UV8xciDFvN+y - 8oY2X93CoQ6CoaSl/xt+MxGgz9/swmPwdil7RtLfSO/b799G2Qo= + z876LSrjUehLzbPQbkLbhlX81cIPK0NjXLe0mea3+b1xvtTZlKc= -----END RSA PRIVATE KEY----- @@ -13281,62 +14781,62 @@ vector: segmentSize: 131072 total: 255 - convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:MDMF:52vqpwav2qgidi7kfi4scwkdte:7neinigvwuvqtmkztw6rjcx442xtjq2oolli7gtewun7qatfseza + expected: URI:MDMF:prou2lgkp3l47jduvnely4pbtu:styymkrx5ja65ke2bv3w3oywyreq5mmdr4ujovm6p3pskoubodtq format: kind: ssk params: format: mdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEowIBAAKCAQEAnpjFdXwFGIs7CTQpJqStcYsUAgW1WKwcB2p6qWdyEOMiYz8n + MIIEogIBAAKCAQEA3jkvQeQAoJhf+j5aF4Oszl1IwFfW3SKPNisjSo+8Yfixq79K - nDtoy5TRDhIgY201ZbQ+cRkfbVXCP1be95R64h/jUO2edVZHD3SORUK3EobsALq7 + 6mPF7c5zZBioF2fcTpNmEX1hjpkc4BZ2NwFaxvstAtjahLiHDQkGh2tGhuRi/jo+ - jUtQ5wH4NVVZtW/r1cMIFD+iwPBqzRMAEku7MJf2ntWysmVu6R7/pCrLdemw5/V8 + x8lBS0Jtxh1FfdxvnfrJIk5fhY1/+3bFZcOQILIfROwHGpYM2K5MHOV9a87LBiTM - WUQytYOmDbqFH1CZlqpxGov6pxz3LBPBHPdwaM8xQtgHotc4NCCyoz4j9DaXaSNf + 3Q71jEbs4HGgHAcShOVcy+sCUMpnSv46vvuHKRMEl//JsydjK0rZQo34NX7GwBs5 - JoeTgYBM1SrNYwQuSvJLV8VXlTe6xjEkVz7dRp8krDrJuDXEcs0BITdbrb1PV8fh + 4wnpCzC3CZ5mH66CE2aYy1owI8LEQKdI+eiU/ka1yb54yS7hjJ/0I3wX0X2mJ+Qc - 4eDDVDbggcjznSDw0IFMnuhqkfRJNLNSudiSuQIDAQABAoIBAAJPyT8FZACf1Og1 + YMIG9YbIfQd0UKZeT0kNmWtN+3JJdkni3G7nUQIDAQABAoIBAGukJXj0OT2RMYRk - L61dxJ5tT8kYwrQsbAsqoOeTt6yp1uA59S6YihY/kM2C86BnYNoe5rMY0eWy1I4+ + qX0UYiM/2lqY8dIByH8DnD+kqiqGrYE00tQAakKLqydELj/QJk3FZj54jkXlcrA1 - Sqkyq5jcrKBLGl5s98Owp/s39fmp6Eo5bo7obGE1nOPQHurfWwFmYpmC9PEZgAEF + ESQJuvABgMcNRaPeQkSVJ5124B29CRp+GiTqHn+W+NdrHFsf6M0MSlscvXZSmTi0 - uCBMJMoYSPK3PC/P/S4eMs02h3ksPTCtCTWhouSpHRT2/fUxYi5+W8kXVI8gHgWE + Sl/Fv82mmjDnZ3WAU06t9t5UZ1FNXaTex8v0+AY5p3m7dvM1DWfJyTC3F9T6a3XK - UBi8XZFxEdAIITAmXR9zCG1s63Q31ssEtakuiwp/Qbnk32yrDdkjAPsEMNoT7kwo + xkupwTTUnZh/y5kbedaXS6Dc6+C3/BG+7MBR3Hix3DcbQFYU/S03tP8DzaQ2aWtb - BXAN1mGVOMlBLxcx64lPo/glbzgiJLQkbTvJnAKZbf0Uys9xwtZsRa9ZtJe6g6A4 + iHbxI1Z8q2QpPd0rOMOuquW0PxlJSQFe9+ZGKThhVfBoUhi4Wm5qjVibnhd5+pnh - 0NRIh1UCgYEA2hRWHC4N3JOlHEDQ41nrcK+Ubls5Yifruq5TcuFWplc4041KaXqK + VzkNozUCgYEA9r9eXVvEgOu4NhKw+k3L2Jdj4ibdfVyMUg60i4NXozEDx9FW8C4r - ttH8k+ziqGUjN3H3ouT/UH4GvY8IbBjGbESIXUOT9OHRG8PdS3zUCN+D0Euzbib6 + jxgdVYkdnMFKoToPn9dFBxX8CmSf0n0+C2Vj9c1Bjfm0COEFFHq5y1cpLjg7ULZF - cxt3jM+kcOjkspvAGTRABxqYhxsqsUEZ18OVIJnnIH+IEG3LgJCd94UCgYEAuiya + +F3utDLtnpgRhDOlVXl6E2Y7LD6L98nwOd2apyKgekYIwJh1RwZ5pKMCgYEA5o5m - P1yhmAvCEOorx6R5fmze3Qp2qt2/GVklxwR/rYMcw2Kf+772vd2k20Ge2R5OZbLU + A40Gt0bYW/0rvl86DVFKmhlxKC6mYfDoxosQkMgJzHS/aacef0M1G/wDVJw6TSMv - /eBUi+WUs3rEsr+x3IFT/Xuovp4FCcfi/zkUN+qYt+ArE2AvqkC1QOvUV+QJf2E2 + ggVDq7Bd1cnkiqwlZCurJ2LDlywtkpguO/4kQvdtYgO2mzhcC30QwhoYKhG8LRf2 - g9BO2chGFs1ASjOs+gxKRoSZUsymJa+sPvCFAqUCgYASawtGwAD9sx6Lv1GlEfAX + SP6BchNZ6evOMo97VidvCH96UNlexrqGj91/z3sCgYAAgSeqPTPLp6+6vJMMD/io - iUyw8VVsW9DF6Hk1x6BI1i7/dvxk4iua+yso1yXhcQFDaoWupUaG5s3s7oqYjpMb + uraDkdzGEthempUX6+7T8Je3YuAwoYeJRV1Z/WvIFEUYy0uY4hHMD+lyA/6nqYXk - i8I0lkOFuBiwDp+/A2DpCu+YBPy3feVDGXvEUbkirBi8mPjlaAtMTku5hWrao5Pq + 9BIeQIsvxSDvG7as8gtLNSRqaccFRTojZd3FFI2T02/Fu21NHXB4da8NShtzKECL - LCOJKFZj4UF9mbhJOG2O/QKBgHkyoBevjeMVhHjOeUG2aQFMjqkHLsl9IfK2fklZ + fb6BNPrrBRWjfyxONt8szQKBgBIXxKusqoVBewMlCATFhlG7OmaDbpzfpFD1Td8e - PGUQfaEUi2Gvp6FisPereGWPvSmnidDcQS3xfyR4P6S99mO+LZdO8UNmS5FadwP/ + 1Kr398TiuI66/aqxBH7wtPYz2GNrSnQio5/alFKNqHC1d623u5O4rW60mdLyPFaa - fJIKPvE1FdW/QEhtZ5Gj9NBiu2wZNQwKh8pu/nHJnJixm2IMri3KFKY6Y88U1eUD + 6A+VSTEy52ag8qA4LVN+Jr1ObP0A72PlDRV9rUWtKp5PIjetmooJLvkfRc/EnYC2 - XxOhAoGBAM8YFY6k9NdX4Uxoqq+NwUfQRK8ScrZ5xF2Ay9phslaeZ6ixh42mJLGD + uiv3AoGAU5U7ZTY/JnisBTemkVKlw8Vrxx9weT8VGO5VDW2HbGbOZkxORT+njc7S - 9Mve186qIEi52yNXP03bg1dyOJkzrRsCEXQcUXeFN79fzFcZRB8YujaRv5divmG+ + YmsnNnr69IbUVMCGERomyhicNe8c+ePulCkKoKtOqni5eCqmUA7tMLUl4rrvKT4Z - L4Jg0GEDiLjvU74WMikbeejKaddrsV6Pv8A/IwwFvboi71e5ocpn + DCZns5bcsoRDJXKyz36qFgSTQ2p04ycrm8HumkzH/x/cboQqFD8= -----END RSA PRIVATE KEY----- @@ -13362,62 +14862,62 @@ vector: segmentSize: 131072 total: 255 - convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:SSK:rout3byiktrg4ntuwwa5zdyu4i:wxf4dojlr33qrry7ltrde2idxz7v4labogocg52mj2vrusjwizaq + expected: URI:SSK:keqm4ealssgmiq3acynkwtilna:ro4e4c7ueczcitmppcr6cnlj6lk5cv6tqeb55e3yztg64qwy3wba format: kind: ssk params: format: sdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEowIBAAKCAQEAsbsTzJshdeA57Vow4v4l8sxmZcyIHCo4Zp6VXqDXrBHMSDYN + MIIEowIBAAKCAQEAqXifYzGbGnv/uvDk3WVV3V5T/jlDSMj7hYdrpZkRZNIadtl2 - +NL6trC2T4CU3wQNpmxTrdK1ackBWf6b9tfygEACzC1rNqUmlcjdpiC9zx+Qp900 + jdsE/pO+l1E5UE4Ov6VlLD7Xqg1BCnisJ+oInJI1iDuZwjvSRXAFl6wfc/F0pVXz - YBUc7BAQhhJRFIVcw4K9HJTL3UnrV4cyIuyKzLEQf5sBIa89FYREZF+XWsvcB7xK + kU51t1iyBXviZcx/Cc6P7JxEH2LOiRXhTUdOcTUx2lJKGf0QNVSUgTzfofmrOHiN - P5m9epQ937PGlHQIcoAlzFnq501JWWOjL9nALhI1bsDvuV/D8PzJeTAeo5RIK6BC + ZMOU4sZRTYa54LXhv7PSltyJQlxUvuLnFCXzYY/9PZTxzpGzP9JtpFbccX2rYyG8 - xb/DtTuNwuMXnvAh/tj2GFPJMtQ6bCkRgugPtpIv+MKeiJ2W51GlgXHGsAUfnYTJ + +mcT8FMQLk2IiHTK2AgfPDzJEQH4riMzxBCEjI19nT3c/spP1Hqy9UvZRanNsMKg - NU4y0SbCVo4959ACsXxTyAGFXd1rCf2JGr4IlwIDAQABAoIBABqx2iXzsQFm9dxo + SDTC0WQy7hIrmY6I5iraa2xuR/iGyjkYYejeIwIDAQABAoIBAATLnYr5J7DXe7CQ - HQqjGKkQuVqV72Wnk8QaEp5dczdljvTTpaKXcc/J2AA19GZQ0goKoEDt8pCaf8j4 + DZHSylRaZBkqNe+h0Agv+bLHjgBqTiSif2nMPAZTwaBtYnOGpxfF8F1Fhz9fF6PL - HI3lXoeT1be8JKvW/2Yk/uGqbkfzWkNUTr8VvLvaJ9kzXBlEdKvq8aOe18X7ic0i + ICu4nHNb2rVH4rIxlH/9B3VdVK8ZdnvVMZX7gFgbTSFULt9hUEvDmOCu5rTO4tni - Qc4MEEbxW2SHMBLSos6eCLW4w99/mgPQlR+42Yib32/aMUfDMo3H+gDcAuf022GD + tpUWt2tOWpc3eB26M4dqjmvnw/gXlXE/kknYLbtks0cgLuFeXIax4TasHXItI7KO - CKOYDqQbgIFR+GDP+cmL+86h10OnyAsk/riNI3W/yCAW5OzRq6kZxBiogp7JJjcU + L6/NNtp5QARbC35v4c5MfLUsAKBptrx0SjIw+9B66r1Jsg2JXw5+rsTNHGKK8I0f - BDOzMfTdHf5QGHD+nA26FZcmD57BAl/Gf6SHu/WZIjcTXu2wtF+G+Smeqk5KISJ/ + CZJ0XX7TG8gy6RTXQxEi5FTE2XXJl2TRWRS7l9pbyM4vJ63Fn52Hdquh+sbUuyxc - EyubuxUCgYEA3ZCL3F4P/T3ct2TnYvbBx9CJcF9nH06uCwtGHh6XFFPDYFIk3wNf + jMxlf3kCgYEAvdyUEqOf8MoLtXPyShODkVUwwQzMxWxzYRbXL2ZFBysvPgKoekeP - 52YZ0ayPXZgi+b7Z7b26nMEHzCwY+PP5NuNtly5HK8jJ5db/MOaSQfbemvAMh4lz + 6Nq/a79aRb5mgKxMeh3RgDd+JQD+X+mIBS7dxWlcoheKam5pvw8uqF8sCzop62YY - j8hUehiOPcv95Tlqb03Z/i206Tsaweb6Yfg7MuEVKDVLXuowgWuwx5MCgYEAzVqA + 0qtvp4CkYBNdUWczsT3nMhjS9QJUQCyUIx1tTLpi+DAKJqHinUl62YsCgYEA5IGr - O7tIT4Yccb8JlIX6feE2Qy5ZYtPpQ3iPm86mwM9k73zMFQH7lKvr5GUkZ31aJFWg + uZXnD3BAf7RIu72UgSF5SKx7ucqATRmneFd4ho4zvaw2C+GzX/OuVW56EIzFojL2 - o3RKfI8UpF+denePJqSiYirHJ+t7tTtXpHa+pZsA8ZhEhgtY3Xzy3M97hpS3bBiU + TXxaWJHe2y0FlNFNXRvTOBHEv6Mlm5UVAt0ULliJ2O4DCO6rYxAtDrnu0wOAectz - 9p8hstrWVL02ZNEBDtF74uzHg48a1UxVG3u+FW0CgYBUAUoF82P8kEfvAML6MrSm + lw5/XCoKDAFGSMenTxIRrHZ1wFkqVxcsROg+MMkCgYBJzG15+UP4EnEOrOzmwkMH - Hdr+UC25IQu8BDpBkTeW7WtWSc7Q/2aNRZjkdpik09nu9v2JtjXa2RUrxExzl40V + wLdcsp79tjP67yfhcr0uFikcz2excBOODUkOlqh+J44sQczQQrrmPau4snQtz9Zh - 0oTqnRE++JIUIr/+um0ZtZARDpKxkNvP2BSvdj/4Di/liS9hpBLS3GGLTG2ItxqX + PWBSlau+DaxtxlEwRLR8GdJC4u7cYykO6jhSQXyjI6PIOncrU8aEAIYvWiJpd2p1 - qpZHZC+xXwOEqSZa64nLIQKBgFSnr19waHHoHofBsmhZBxenpR/y1oSISYw4AjO/ + Y8DSbDiABBxN++rb/G3WFQKBgQDQaay0whJStHE/iLFl+o1+EYfLTvYyCI5ow+NJ - 8DxiAwE7WEJ8y8LRUPCZxXUoVuXNquhXQ3Gv5lmQ1TGsYgYTLqH7cpiBWkEvEoVJ + EY6uOvjaID3TLHIsK9dvuCnA+oQvYgffuHG2oqT+htu2VggXyg8l7p7iouzkMF9P - MnTAvpXaKL19pgfAv7nJiunDGw5j39z/YvwBfQP38JmFE8ORFlpJNEKG1xABZMBs + k1CazMo9fyhpdzX+TnyqF8/JykHd1ECDIAftibJMLMVsEB17MuHHyOuxGiJR+KK5 - tcLNAoGBAM66r4drwUkNCV4OVpfjTvUFgIveuDwv7OHqcVCdzZPENMw47FrRSUhU + 3pEKcQKBgBYbOkjHTSnijJf9xkqvnk6M9bJu6LXvc2JjnixB2J2YxqlblJWw4L7n - rHS72Y47J/yVN0s4jGLfJJRjxCWGJwQAAEW4hrWfqK8XNnZi3tte1zMJo8Daq88m + 4Om6dx0vvhEx5E7XvNn/3ZyuouzGk5z6iKxCNFq3YzIOhe8VS6nozs9l5X385qUn - DC2aS9NUQRyYhMPDTVQ5uo6svKPbWSoz6mwERrEQSpmuGtG0/Um3 + oV6h6h8pvwzuimK0pm4i1V0eJSagW5kwNVKiYekaplU8We0O4yPU -----END RSA PRIVATE KEY----- @@ -13431,62 +14931,62 @@ vector: segmentSize: 131072 total: 255 - convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:MDMF:tocfx6ufmjum3z2phuylf5t63u:cyaf3kny6cmws6oflgrpl3lc3d27m4cbpizlwqcrjkzvcpcd32fa + expected: URI:MDMF:fojgru3tcvv25rlf54gmpurhqe:czcchpdw3g5tablqxfl7vezf6de33un56mrhpkswdaclo7ekmeea format: kind: ssk params: format: mdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEogIBAAKCAQEAlTYNp0fY74mV8IsKNY4J8PWFpAL34P06ClTUAJ5/jA4UsX5E + MIIEpAIBAAKCAQEA1YgD17SLc99hZfEdDFeozhbsXLe2bbF2oCY4mx5SzH75Dn36 - o5mfu5XHibCE2FwT9eTvlQMFH1Sq+OR08d//S/V4CGC48S8Tb3OtLTWTANQtNlfs + Pa5XwUYhgq7Ejv+nHhIn9w1byoY/pbaqs0WUxV9481OCX1BjCVQwS18ZfKvuZx2N - VEzdvnyAZwlD8BOHTF6rothwaabsdiZkzijKxFvjDMcBsvTdXbzdbO504dgqC/Sw + SQs0I/lEZVoLgdH2bnabZG6qxPGXrzYgCYKL2D0rKadCCRbDfIqs8EgosJ7wE8pi - R/hYo09e0qNnM9/VcHQwPK8HvJhPCJGlt3XOYfQX67pJyvBIM3ly7BWCqI0t7WbD + 57CkHQZePKyECdpBivXzOhAnBVOIIBh98LbEENZfzfuG3e3Q+LvsWQQ5PUOVobE1 - ngr09KOr0F+AwP0i9HQ07nEW1cwgkHEdKjMkTuDdrfvfyjiVL33OXn8l8kRbm+uR + cxvzWbHN6QLBmNqu3suf23gu13ZfwtXKZMxfsKVVOoF9qYDJ2SGrNfQdoMIU8MuK - 3RHQfoYKUMaR/rn4ibJv1coFuOtO+izCiGOfswIDAQABAoIBAEawPPerChMxU1+J + kFl/ur79Ld9Bu9NBxA0TwjGAUtMniQcz7FkwOwIDAQABAoIBABirV6JOmgPfjV2d - /2QvznXhW1bAMT7duMl8NpO0gyiO4y7TayE2fn4YD1gj0EvQE4TC2N33eE3Hhtgz + LxlzcS2qKVGG6f0fURCsicKmDLPSgYyikkwY/ct3Aj0aWtwYfiKzv0lEElRCEU1g - I1QTkpchy6PsbrGUY9jBLKHmZ6ZU3raIIOYvJD5CLXKi6RSrq8V6dEXJ1De5ZPz0 + XrVKdyccYhlejwPbAi4cO14h1Qx5wpfIKsADGtmDHVtGPWkYrEtTyZ0fSfxp2vfj - Y31nxegQwBglj6CAcP8foqcgsS7syIuseLSuRWEy6GkvxK7qaLc3L2FoyTqftL6+ + nWzr16M6Yee0iqUJK7mSPeuespDA/e0/zrJjMSE4hjblphHJsPS71Mx89zEJywJb - Rvaoc0oqU8XpTyx7Vs2NvB8RWNhb2T4NZvqP8IOAZkRQ4ueO4zU6vv4KHF8+zcml + dZBvMr07LL/YT/I+B58nEPu0VdS3ekJXGWW2wnBRLG6n7pX/X1cTZGO9DRw/mXIc - M0cTC50aCrX3zgMeUUVWm6k3vFK8uO3mdt7k3YNfNGXnwrejLaPnwBzh81kpD+85 + MDtpE8O8Wn7Fpzt1wN5DfSs+SX54YQmVHMpnS/jM5DXWQP8pgVUH8nHlqVDlZQjH - 0VeV+0ECgYEAxfDodtZGqyff4dhPoYFeKvDELANVw+ImuSAkaVHQGx+ZP6hsxXA1 + OWL/+x0CgYEA9cwUxl6MI/PFq6TtWnHl92/dIWipAoW17z9iIlXVS2RGDgOGXLan - fwSnF7jitopiq3uAaYPU1yiGlgVCDbWyNhZAn8eRbzX9Js5ynMZGqyXdqhmVDxVa + xpf/R+877+BxEcfmE11IGpu3BpBSD7gX8glqH/VfhXQjzrVYj/33s08KSPrf9J6a - g0XPQ56qMYBlllk2WprCYbdRxvnNmGarCaei3vZHGviqk4rOjPXoEd8CgYEAwPoX + /JJvgwLjrIO4L1X4geYTjDNHHpNLiX4Gg+OkXrtcQHN+CIXOM0n22y0CgYEA3mUQ - Vf8PU8cXjAypzgT3izgeg9s4EHDDpeIziaoOGkVXwvSAdsjM1R1exr66z/7cdn10 + /M5S2tyzDHK6cmyzf9WFZwyHhwYMdju7jOwErf8VGJCn2rrCDAdIlNWG+ru8fOz4 - U9YUx7AjVkQH87upS2yQr8tPC5yunNAmw5IceMNiQTvb/2I/MkUsMSyChaxvqOAI + F/++Xxq8fYr0caOzDghK2O+IF0fPaDxYwFuHBT4vRHjVjau8sYidhIeK4V/JLrBF - fKxY1DnwlmkDD+Tr/tp/JCD+ufqovu28y21Z9K0CgYArl4eKjFwZ23k5wqqe1d/I + uZ1NxA4kX3d7f+6Y8YWMpie16ple08b6yN9VOgcCgYEA5HoyqX5DeDvl97pUI2mS - MyfwzXc44XhHsuVx8FuVbZsRYuU5giG17G9kEQqUyts6CsPX+PmJvNoO9e97F3W8 + YWHrRF3cFIsj5eOeHdp5bR4lfGtMXywuYozxb/VyWnTfxa5yMHfaSVmLVR+cGB6A - 5Z+r0Iad6FTtE/A3yI7NqFQt3t5t6PT7Dge8S5gNuMoml1UaFRUT8gxndqIpmwq1 + q6ySqGhWxV+C1Wd+jkJ+GIAVSGdi/CjeWn7oBvkNl1PNRrr8SAsNCpqztjkm0wSB - 4J5E3hYAwZzHS317m7hVHwKBgFjoYCv8sSEWDuE1TF5gp3P6zQRO0YuxiFI63yfD + m+Fj7ebtRr/UXKm8VbKgM3UCgYEAgwUj1uxu38X6LjFBKrxjm8Jdj3JQPfoQSW+z - s2+jFwX5A962MLjXKT1DzmnZr9Tfg+LENRqzKfSqr0c55IudXyO+9ZISA9i3hcSA + dLhvoVqQQSKn4TL5s0BvQE/z76++whKRrwHaVAlaVtQQYwrAKFo2Tkv/70c9J/m1 - 4qE402HepEMLDraoa+3T5eaURXV2kjJubRaKAzAo/YIrJBdsrzsEAJfKxkgA3ASV + h83kY/BYxIwzs/0jc6w6sKNx7IkT60+qJEpKUGDMiPnJZntY26GEVTc783Rb64Rk - QuaZAoGARhyMG2qeoUzs335JXPJ3lGEtQFK2tSeJxGL/zT2lqLy9orHz4e1dI2h/ + pwb8HO0CgYAFAmDbWNKQynEFHE0brggHalBZiAoBa1hmOqwYto4/phUIFdqjt9Pm - rj2PNI6LJsKANOdGeGkNgyXzP5RnTPMQ10tK1eZRNYeGTB3u2/E7psDFJXuHxB84 + dmiA72AuK0YnKfMP481tM1IcCwzUIYaxbFMDUN2yOuzGVvQppa5TrePm2KM52fp6 - NYxoNA1n9G4jL/60pPerHUbNEXXr+v/lqdPFcPPa+MlPaFbrfUA= + 6vZZcaQ0TXRp9Mo9BroG0CkaTQ/XTm+zfPpoDWPE9cq7L39g8meABA== -----END RSA PRIVATE KEY----- @@ -13512,62 +15012,62 @@ vector: segmentSize: 131072 total: 256 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:SSK:fn4mrtlcnpcps4rnzolo3ogiau:qko67s2m6qeg3tdbljexw654dek2rx7s6hog5ahkqiyq5cejxmdq + expected: URI:SSK:pijpudpxiovhh4mwyatsoo6rhe:j4zu6olizd7znzfvxqxtjkdr4vwvmvjncqafvjqkfi6ppk47gopq format: kind: ssk params: format: sdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEpAIBAAKCAQEAxQnq3zmS38I+dYnHOhTmzY2yQ/IB37CvcBvEHvjnPngQtBAs + MIIEpAIBAAKCAQEAn64dg0fS5XbYmqhhNZVejtWlUnL5oqHiVo7tMYh17LRrq+lL - OwR+0cc1GYMNkOpry/b5W2HtFDf4JOKWF57z+aNal6CrV2QHfO7dcLOj35e1V4hA + rJ9ra9j+FATvN6rYY4FdkN614veLud1eys/hvt4bgJ0gWxOFCkCfvlak7U8CPw3T - Y5hTQhal8gl2qEPIyLg5sJrXebmUgqnkXHoQmnz7A2TJlSuXUyadI6VzFRLWg5Oy + Jmg0xA6bwJPI6QUBysodMZQTNn7ZkFOglSAxQKsssPw9FrOiH7LwYDG+KsbrFdcO - IV0Hef23zjXUm5n9IN51r/tyP2egkalDjlVd0rD1h50zyGOo5EXppktGyRkUy3CC + DN5q3zKELzVts4tlZSwuzKZe/DNiwgaLYousjbqInS6vxYJWfuXL4wP0igjWt9ON - PsqH8ZCbR0W6N4PCsWGrsu9rm0DTjwxfWpbnYJqBJSQWRmEZ4YnJRVm+MKXfBV9w + 17pfF2+/K/ys4VxjUoopbxRyp27R9HZq2dbmpv3DolTXDH8sQtUK0u3EvD3qemBt - 7V3xekGMzA+lwYF5Gc3i7bheiCrtST1RvlHtdQIDAQABAoIBAAb+o6JaAl9EH4B/ + BkYDyGBQ6uIU/Rc6RNXUwT8r64amQc1+NpG4mwIDAQABAoIBAEkpsgQQyKSyy5Qx - rB1hOZJJee8Ui8F7nba+nZc14cujaoBh5JgRwEjFKBroPpaK49nBQjfewZJKrFnu + SjkO84Bmi5U3cQH/QoF+g1eKut113U+rWS0C7mk/x0rM5/6NnRAamhBiutv/qFnF - 20IqZ+HQTTp9vydiiyuBtUW3ctVQpuTdFuASO75oXGq7sEUn5txNQesFjCmrj1yW + AEXU8g5OHjPTyptwWijUa1z+vhqtdM4HO5QBcwvR1bNrA0chMC0GZlHtEtCJVo42 - GF+6C5XYYvbLYKaVfhE7GS/3Qx8X1KLfUx/qhM4DvW+SEciW2zeuBVb0ilpuUmkX + gwFQ+sAyrgt5x1O7grEbf9/TatqWCF7pOF6fJMkYfODAWbvdoOoG3icQxDj82Ya2 - i7f9444eIZ2SvE/jdPL54Iym91wW5icXs9YNIgoKQoPB0OCUbAsiDIrIhvXemHOZ + UakJFH5ZDKKc4B0fV9CSiT7HCFI6ZGBjJfFkRphU+h0+qdgQB/qWSwBZ2VhK1rKQ - ltOPlccfbxxz1/B1fcP6t8aYvVHeCDv7aPgWSbPQ7J+HdQCimo2TshgpuY58OZbh + zMKjluwn0I+nrQUzmshKe5StHLwTezkWkaZ1iMvQaHl4fnw32d5c7SIh/hmy0s7j - 7rR/ECECgYEA1RsFum5dDpfqhU6IyQpR3CWVx9YVJRU/NRPD3LYYDRvoYv80TeQz + cqHoukECgYEA1TVlxFZKCPTJxJmHACU/tMq7HZqESDojplOlSv4apkWC6qJkJYUn - 6ISoq/JOhmSw5Ff9LTcPsX6dUVJkPqPYyBAg93D2bunFCEvw48IpgfXzFAr5v0ye + 5/VwYmiVbw2Owk4borWN1qaCI0EqbVCkVi1g/pk1jAkRdG0yz7IBVGlronAvttWk - Xg4dTfyGRhyCUyvv7hZsWyr1Wd+YB2OoTHjvMYD3rwGmpbywVpwC+WUCgYEA7LL/ + n8aw7sBXkQcaNybz1k2KVfREFHb9kQfZ3e6RpiKQ5YiMnrFVUku32ZMCgYEAv7pw - orl57GPpxiIzB90i9S7dZC8w7WGKwUZVHBIbXSW8HNOGrO8M8VjduDm8KKVM0Yue + nK76qG9/GiXRZRjJjUbtzxczwnG1jN0BqW40jK5Wjku8Om/TwceJlamgNwrCYwb/ - lZf9ut2nIzjKvtIUp9JDi7pKOOUHj2F5E7RU0hweoAGv7543ybGW32IT55bEvRbR + pe/a9lhYOT4f57sOmcHn86L76CKKY1BL93khF8AFFGmO1bHrs7PaJQOtr5aVKYRP - 8bOmEfq4e0cg6Aah+GVsPHi2JyrFMV1lzGBW6tECgYAVR217ACoqmuDADud5q54g + ehFNOwgJYCQWPF9yKoen1Lx+MJkwsKAAHwyvadkCgYBDDYcS51xjUrD9/pbBifVu - 7V/XZHkYCtcU5bRZBZXBOVgrCnCelnrYbOaqxLcylDtVkbOmIClg/9OVmzSHTLUI + I3ATkFvX50j870OFwUKaVjQlHKtITYdOYRdWK7QLeAUUwMHaOyT/g+BbvAve00TL - xROFobH5wT37ZhnXpDugzn5HMhFeGLh3i9FBSEXgGlipFWoPzA1lzRRStRDpK/pS + wXvGtmJrxxJRPmKDhWT7qifqr0OiSbB7e157x8wCVWx+Oebn1/0QqUCb+wwmB4US - KIE54DbbMr8BLaYt/8YMQQKBgQCHE79/BYnmtT37rBijLDd+5DfDrIqnbTraAWEg + UgxGZoqRVY97/SNrPVr1twKBgQCDjEJT3uLwyn9ky2neeaFgo7frDTpgQXCVk9Xe - m9Sx2472hGAe4GzqbmRZddlC+NJV4u+lPw+1TDjNiONq8kiHXR7e5njk7w7ZbC7E + EFVR6RROUbx2Q+AA5w2JeHcLDQDOvTCPBAEyYO83Z16wunGMIbUqPzujzH8zIRbe - Z+zf2tw/Q7c7b3c2yvnmkPn697dekV9OJ89mA0a0U2sb/m0AbCDQgbKxt17BRSOK + V2fTSdayaLKuAIN+KvqTxvBWt3TkpXl6gYCB7kOwiVIQXlSQxb7rgeD7K0BzD3TF - 9o+jgQKBgQCKs0ujjCu3DvNODuYt3OUtziFMlF0om6cNCR4KgVKs8h/3mY8m0J9z + 2QhKEQKBgQCNQVMGBWjXXo+pvyumYHBg8GxO9ZsfI5bzDyNgJqG65MCQDB47hrZg - tna58i1IvDq95yBFwttxSIXPh7y7RqDlvhu/2yRFtWEZw90nOmuIjzn61rFLuly6 + Ng0UwIF99v0tjfDWmzx/ZPIOJFhHYv8rSKyh8j/u9U1GbquH7bpsq7ICNl4DZ54u - FCHWEBOCD4jqKMLb6qmH7fQh5+n8X+/kyIn9D4kKFAFKBd22JIkD6Q== + AoI31FyAYBP3wOIBPLe/SxA9MZhoIQVBh7d//CKn5+rnCZyRDqMCBw== -----END RSA PRIVATE KEY----- @@ -13581,62 +15081,62 @@ vector: segmentSize: 131072 total: 255 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:MDMF:xssuhuqvupvlwchl3sprk2f4o4:jziwrrjb34qffqrmdxihkgnvs5wve3drgw723zynz66om2livyuq + expected: URI:MDMF:uqzfer3za2hbweacbjh5w4hvjm:ujlv7l7tyq25e5l2ctgjunsutsjx7nxv563675sshijvc7rpd5la format: kind: ssk params: format: mdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEogIBAAKCAQEAqRRBpklSNPInOpu675erg5aYjdN5XZXES4D1sSKtE65W9rcf + MIIEowIBAAKCAQEAyW1VZvVCO0ryuRbM14typdFJ5+iZtIiILWrHcR8W+AKXxOdk - DBYX9OATQHR2+CqtqAEriQc6/xYjyklDC5935ZKa/H9hY/AS0Q+oIIYNXofR47aU + vP1XFbyTzJiigYY2V4z6yIYqq27XNcLpPZq2RuxJ3bl0VSaeRC5DzTkaKtfwlL+W - 6h3jjNYg+Ynp6B2EfSkcCck71TpkQAht1xg4gpt5F+wWbQW3y6GVcw4VG1sr5cNx + R3qE4Et2YpgGusuG8w9MNOsBZQ+BbhvrOzU3aro5aCBt74C7cdok1GXut/+JPNMi - st2eIg7yJiHFvnpwRZOQiSZDRSJ7Ar5EP8eomyqb6y7UAgWs0E9217Jx8IYSbZZ3 + AkAP6ZWfVaUYwFzTEwmhwLFotyvIXrx8p9Y62XrtWNTKIvr2WUIbW67lyPkPcvU3 - fpuNpgR8B0GJztw2bcdtrDZL5Umfgnpv6Edh/omJTSo0zMbEcBnZEv0W+q79+q+S + OMs0iPRP8TnJpo5GU5pg75uuMpsugl83w9ixt1Kwziulo6uflBkXbeB40nbrudmH - 1PukFCXy1RdaqHGgvKyRb/pfI2xp2eWqwfVSrQIDAQABAoIBAAKSlX8iDXdTGegJ + ZBExZmvcyHs+MM0hPI1s5NHwG6/1qb6fg72oXwIDAQABAoIBABjLnDU54MbSwZFW - N4/EBy7iNeGujoxZ0Nn0lTMZRniOsLG3TpMgq5j6ZytAumYDQY+AA7llhpnGrKER + RK4N7PWLj2j8YZtvKTBKEjYTKS0riIpFH8oB96vl1F3dtjdykZLyeFah6XPEB7sG - eqIsznu2k9d0rrbWZSA7ieuDbqT6EbunwF325MqfSsgmjOG5v+rRmyTGTuaV1Ab/ + /NZICsMtSCSCtVbcE6R5+3+yVU3L2kI9WV8ALoY7091sMHvjHQAjtHJZMYlCwOCQ - ZIB0f2OervnUtJtQY3VcjjQTykbCFtmSF4NdiFXR9EOdEwH4UFg26f6idaoEhho9 + kELwGJvLQ9DVGSNf+fMYcuswhxPgKVv09VCjOw5T0Bmi/acM1diimO+YFx1HA8uQ - qzWt02fVN/hyU9IrnMMbHYc45SPEAW0/rVtFlxSYWz0l5/Ma+jnQzT4cQzQS1/M1 + gm1Ent/v2z11zDPZiAA+Z9tCxALdLOmGqyz/GWvfO4p4S26YD7JbF1bfpBk+2EXZ - JOUORANFt/K+q7Iyddbu5xVxClzedi2pwekJFlQXRcVPXZXGJ7X3KVvfbx3GKUHc + tHrVM/iK8BJdp6fOgjgOGCxungdU+YUfv5c65oZwQCM+qzyFzaIsICyXWptvX6HR - Hj2agaECgYEA4tMDj6xBF2RQoAJvQgpHCxYkarFPUDmM4+qjEUbQ09tyL9RXVzf3 + reF25nUCgYEAzw+xZ0D3AjFSAC/WqkpwaWwpZRGgAcGRQpIp3VYNooNZhVJ+jWhq - 1MlK90EJP89MakYOb/T6+MMqSG5/xvkbrJO7kBMfyDmA7IaVzcMDdmUTLuWtqWsg + UEIEKuF7DEQq/u/y48QVQz07mOuWpFNa8/t2D3kt2RHWU80kLVxvBnkbsvnInjcB - mpXhnBO/utGEc6jzHisxMwvFFP7lcHq03WQykNUKSLJ2WNvHYdOZ5/0CgYEAvtPH + cNQSVGT7zFGqdQJI/GxYpQN4m7PYcgZSkhQo0CuwP37TEX7Nf/DQYIsCgYEA+Qi9 - 3NwGlr0n22K4/LJdKTtvGBiA/1TJ+XstGN5BY33LjlzaElVjetJ2c2+HmZeq6Omt + mwkcIWRWI8Y1xjcDwUwbitmm3tD7AJWmgLyTtW7f03XLk/TBLBJPf/9AhKGAN2zL - GXqUYdXcbSUxPcL8p/ElDYAOQuOyGeAvZn2fub3mqFSWJZSGymCN5TGGv6p5NxCW + P431+xoab+tiUeWAlJqYkCqrPD7eagEut9HOI7Ttl1c2NN3E6o4yYu7IxD2O1BoZ - LzqV7tOCZ62Y2LpntGG1b+NCf/0FNItDyLlzXHECgYAGLYsmSaHIOlI72XUgTllb + wx5a7hxywIR7f2pWsmGhOcZtMMUp1wI0IrHrnf0CgYAf2oguHD5jpfa5dKKPe/gj - AvJg+Y1YeQjOWGCyosQjURHOHbF3Ta3xXL4u99WBqGrDZj8Ua46+YcpwCJpwV+6a + H7KWi8mTu5V/KkEqfayHTbGd4vz5ABEq250Mg7eMQYhjw8IX6/hhabAbbFK2YORj - B7gPF4ZBFNffGVdRMGOSwPQBzf2p5KIRs81eS+dn9jbuU4azpqeDZWmrxbmIE7+D + GFInOzskY6wXJD3mhIvH8SWjuO11+XxNQTK4rPhXjFCuw3U67+gLKqeJPHeVwwc8 - XCxIZ5UNH9c7WlkW4AWMHQKBgHcuR74dwRO2EcWIE+bm8x5EW28eJrrRVs+06YaF + 1cEZlT795aLO1DUE86T61QKBgH+jgTrTIn3i5VuUnb8oN159WaiDAco2JlAYY6yb - kSs1LsO8JAqdP+M+vPH9rx/zRK/w+cZW84NjESctumJLfIbbKfwThVSrZtmYVaJa + +sEFQOcq+tqsmc2y3NhnxXO1Kvg9ZLcAVdELgf1XEZ+UF6ES05sgo39PYcPHM2C3 - RT65Zuys35Wa/NA6m4SQeQsNymTkvBfFLE0b1m8wUazSRuC2wZ2evzK2cODPNceQ + wgX/F793zaqu99yYYS7f4Drkqi3/6rBdAJIGNrKBtKKLqD/pVi88in5yr40p7frS - Y4dRAoGAKO7vgYPdViIDpIUh3GW31Hq06VunMGgyEEq5ju2tdMrQuFQwKdo/ZrKq + YkcpAoGBAJ/hMr8iXNI7cOPDx38heBeB4AKSPFBDJhmxNfhk6NbZvnZgyUmbZqUJ - IePAVFWwA4YpW0I5vTR/p0OtD7iiCz6M/uVQ/D8I4LQMEzRamh6+BQUGHNWmR+Fq + T3ewhYYrOhAJpGd0sn6O0ReL/R3atMIPKxdMp9lbxItASmdxLtJSFUx2Q285ml/T - TCwwpPlW6D8IL6YiB8B0gNMTqJOdGw09DuAVOhbW0KMepvwkvq0= + sgmrKsiGM+UAVULBB5kVr57LSmCDzGkNqup+k6RDt2u7zWPbz0K5 -----END RSA PRIVATE KEY----- @@ -13662,62 +15162,62 @@ vector: segmentSize: 131072 total: 256 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:SSK:xyd3y6wxlwoca4jjp4q2hvmqt4:6p5fnb3dahfki334ykjopeodpcwat7j7hkbgob4e4oajyervreza + expected: URI:SSK:24fiitrvlsely7grnfn3ghrsgq:2kgylma3t7u24nj5tn3p4p7vfyokvyryp7mrymngl64yib5zmfxq format: kind: ssk params: format: sdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEogIBAAKCAQEAr6x6+RfkAriq3hA67zbuejAfXnMBbAMGf+K9AHsuYlJzw8Kb + MIIEpQIBAAKCAQEA4ofaPstvgkxaUAL20n5dlDVSnjxTiNPDctdvy7WoZ6fGx4wF - eoqtUFFvGMG+qB4zAI5zI0LwLzKda1li2tTzRzIlQ2ZEUKAoV9JD9TCMUr71s4mn + m9F48+0Pa6bnelrJgQg977Yv+Ne65eiryX1QS8IomGJq+8xSWYT2wfiOKd6RthPU - jM9hO4sS740VEPqwArB3pg+/dPsFAKSNW5C/o5reU3/WMOmhUXQ94td1TpGvixjJ + dp+XhuVOdJNGjfoYrcKDRiCDhAfnSF2rT0ECVDDbW6B1rMoQihbC6nxPMKAgroMn - hFtPQdfhnTUKdzLU3Au0WFwB26LpvPRDKmGjJ+pvjnmWtY4J0IyPdgRVXXe3VYhX + 0cJfq+Z3/9txo88BCGXYqNTPJ0tB4U5kQikoylT3KehNcqcu3wBHf3i1JwAAQugi - gM5N4e6IBY0hM7/HJNYiiZS3jNTXWZtPsij6vyb7vbjWTt8n1Or/Hj52JxBLcsrq + JmZPYb6KWV8w22MBuvijSW1l3GZeHrwbhBo2QfNMeEfHHuzuy7jmfXHvsMM3vqUV - Mq6zMgzKI7S19U5ZCvqiWEJtNXQPilXrzhtScQIDAQABAoIBAB7JAlTYEbJDY9gd + kB5o4Cvxs/zVAX5MkteyRZITudQ3UTV0D0SWkQIDAQABAoIBAAm6xFzFF8LUwl5E - 7oIAtZpuh/TAgSmJPzCWjqoArDB5RAW0exQmrLgUSTyEqVFjV2s7y2QMxTP0Mf9/ + ZP3bz+t2h4K6BuvODfYn8FUjYMUkTM0zo5FdOQuKXOAGo6jlhFkb6wPEOKGWASDb - rtHr+wUJPdv5ljOt2VwIgjW7z/9pLPwNPbIwnli1prgZmG09DS6vd4w/mrzYh3gl + J6G4Vai4q/iw5XUaLKv2ojM+QzuOI9uvqgT29tGkQK87ciDVTeu3T98aX2ZeK94M - HguDin3UdC0cTDAOpSE11mmD+e/uT21SxDFCXY/w2zfhzf+Bvk4iuZPOGveFwz8j + /ozMDIG5is+6f8NjobvknWDn5O4q0HsT2eeDbDf/FIp/dIRDjtp6BJaDy3Opql7L - oPrHG5ZuEtS64b0j8JnFseRpkWc+nVCqbP6DJHDy089/C0Auu/+ijk72jh1Kp1Ce + 56RWuBj9b3on5dC4zuzKJSM6p0wVsXyiQxI2U6qxDdH6fdzolpeyrpzjpM2lK9I7 - j932ieDY5dElfJWXJwOshM8TG0+VoZ5szEWBegGpw3syGIlGZOc8cm+kh31WBuIE + rhSTgm9E1D3nqgC1mCC+5MVuZNXuTc19Ma5pEP+Ntc7LonOKbFfe8ysC48G8Gn14 - iSuwQzUCgYEAuomJoOhyaicT1VS7WHY6u4/N5w9LRDulrkes3PLv1dbq94a3gX6v + oJS5Y1cCgYEA9wmAZuzANhyraO2rltuBvWSVny8YiYrVNAatPzNiGTtQEUCdTN3m - K1n/wvHtTrJrus5QxsXExJqXmDPI6WniKA15zAhnk+70txb1FwSEK7Mzg8hOKo+T + MJ4yprzslUtofLqADBmCTN7eUpYBVuTFSSKK7DJAaDHzIy4h7o12rS7ZD6tkj6D9 - U1SzPNfq/6TLlJ261/+G1wGFY73iCP1e8CbmuznTo3HObabFZCuJ58UCgYEA8RdV + H3HbmUJLylU/qzx7eHX25WJwEkU9cK/ufoTc799noJYlBhSltNLGIO8CgYEA6r/i - KJjpD35+uKCceJfqHxAF7Rq1WAn/UlGzWWP5KeSt+CBq8dgxjjfoRQKT70ArXU0p + ugpXuN5fZ2Nu3T+WPQNGEgwQdhrWD4J6HnjlEFfPuj+gxIe42vEU/1slDI4ujgHn - bf4LeJHpbHiuMe6oCWtk9y9KNxuGt3vSSMhWs3nZDP8+3XQEItsHPROACm9eDzfY + bFx2lN+LsmbkDsweOpAJV6KqLUALcqdIrInX2BEfhs9KgqfifGROZHf+FwTb9UxG - PBlcMqD0r85909n5C968qOjs5K7VO/nhDL+bvr0CgYBE+tVtLmgY/yhjbDj3Zokj + 4g+FeE8W7FDz4ENa6i0GvuLRery+zLAoymApwH8CgYEAgf4X+REf29meQSq/njSH - kPMYbdxseA41m4W+EwxDrH0pWaUEev917YsZ4PLbdjlGVEMkrj+sYGqMuyGhxyj9 + wteI/CjWKppJsoTI6XbqagiSC2IK5AXoOTElyiOkArOZmfixpKxPqo+kQaT5s3XS - nLYckEMVPnk6N4AcqeviaRs0sV7OeFeHqju51TKupJcv9wAAHhsT6RkVoEM1BdUU + cregjsWqqqmOHbcK1/LMvjjms54m3oWCbOeG/NCr/R560GqVNkAs2WvBOXwB5qhN - w53xQFoWB+DJRbGa8ErH7QKBgFsjze6d96UC1dbn6J7yFvCNNyBOM3XHubyd5CYL + QXo8oGTYrOIVPWvj/pDi/TUCgYEAjO9a/XqMI+9Ns9KckrRETKkUfm1DzMRb07/v - 1BqRN28Qmj041GsGGYlVEyWj5YDM9bd+DUoUJuD5sihwJxgAgFetienRPxlH9tPK + 9S97xo4Rpq3gpV0efEPU6WIdIiaSiKtX91Sj1MlJI3hmXwPo+hvToAuGw9f5h4Ir - 4HPSwUnXiCVhgVrH4DGnmITZWv53xwfZMnB1RmrbrdeTlEF3f2x/OWat7TBSI1CV + PXscXRoapWL6RuroLOpDrknkAInoTKLYw4uyBALnrkUDxZZqlMEnlZ6zSU7b9iOk - csQlAoGAHsFeiygmDgSaCvxlWBqMhG2Tcx/kIdzV9iMUsFLzsOrkLKjCrFZhdzjg + uat2JZ0CgYEA8sEwTSjLOBpLUxnFSm2oVA2lsn3cNr40aHFfqA7Uc4C/Q/znSfV7 - fliN/lhVltkttVgaWeoE9oKheEwo0CbTRog2S1xhQsdG/vaAUNcTTSpfGUQiSj6A + MfmYuykvFAemOzdHqsveZakeOwJxe9gQQ2Ar73kiHtUt/vVQxCoAFnumW9SBvjIs - 7YUrhzG5vWMqW4P7pBYtf7C0PpgQExEBdvMkQOCbuTKitpCudAc= + 3/8J8VeUUgbLada2ki0hVIWWYmIG5NmF3+QB1c+/Jaerh1qeM0wUnl0= -----END RSA PRIVATE KEY----- @@ -13731,62 +15231,62 @@ vector: segmentSize: 131072 total: 255 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:MDMF:24djav5cdiafx3bkazhgzjt44q:4d24tlgaq6dnutv666bsnu64tzzx2u7kfroa5pmxq6vf5rr4pi4a + expected: URI:MDMF:gdvwak2g3obbqzs25y62axubiq:c4j6xu4nazb4axrd3xpouuhhjso4oedqyulo7w643rimve53rkba format: kind: ssk params: format: mdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEpAIBAAKCAQEAt9YOBJoP70y45U6H3ghYRMJbiG2bRIW8FoYH58oaqA7N6En8 + MIIEowIBAAKCAQEAsc4qFPbdyhiuLD7dSe70KFACIY7EvzKOzcWH3X4Dx0LZ45TH - xtvQX0Goq9wH4WHzNa4twtZvLeI2PU1wCJ0Fc5rkbUlSxYBzzHQaZa/fPIQJfomh + 92euaqW27FGBM2T+z2SiYaTAyPEDeqoOiulRZxZTk2HFv+VVUmjNROlbh4gQ8GwG - VdGUWPO65BOSEublp2ljpar0iBBMU/W5GuSsXVQTxCNayiGRkiYTYhSPOyDkRiAU + /WwCq7eYTWW/N59N/5cSlyO81KqF2BTtVEjdt4Hqx+5D8nkwLoDDd4hHy2EJAL64 - me0vLylHHft+5wXJSiusndp22ZEgR7qKCd1dG8U1F/BFXGXPv1EKkm20n5lRFFzL + +TqdS9030116IVSHYuMdW5bn83UQak3+ihLX3AB1xASfv5h1Nwqk9xiJ8PevugC4 - ISFvli+fXiCwnjYwoo+f2h5eecojRZZsDYepPnUthc+tM513l5Hj7yFlPfpGfLOZ + /gbcwJEqAzxMEXE2jBXrzLaHs4qeCl8wgzg7R/pHlQMOYBzUCDna2ovT7gHHC2Bs - As2dwnVH/djIdSPIwIujGF4HwCZPwfiH2oIzLQIDAQABAoIBACnR9WbpGEJnKPGr + zmBj7uhHm02uu7dYK0S3WtzYDtbbA7qZrGMLvwIDAQABAoIBACTdxtru7sCrCl4R - Q3/IVLIxp6p5yrZUGQVjsLUzVgyQr5lOCYXAeB7PXDxaZeWJB4+Y49qctvaQbSfV + MMfWHFjJcg+sLv4nyPVAajHSIY1svonR/P4+yKrDLmDka2IRJEYzKvoM844Wbu69 - b6zZ8aVKoXfWFBEPZ2hlqiKjV2yYGePSEeRotK9mpMehRxvrMGe88xj7Mr0oPgDk + mONTijXSKsUJxjtKHT1HjpwluH0rCLwY4gAkp48cM5+Eo7ewN7dxhwDAf7QmoTbS - l5stVaO2jneSVmNArzG43TR0+l9n+FObmDfUcoUZvE7CWsOFmwXO8Max6Wm9ZheB + 6/yIWTRl4xzOOddqKkPSHfVIU/6GiR0taBb44yz7VCp7iMSqa1VHSHcXBxU+lspp - 8T/bfaahbsXfYVRUjgTFDhKBguePPyPr+7kxInJloZs/3oiIShRzRPlYZK4YiuWd + f53a8mZ8+nupsLGTsne70JGe0ipq9YqE370Rz4J0Onj1EtzOgQJN6fQdZoD1M/Jq - TJjg/xPxaxnAsmvr+Pz8Knib6tqiErwFdjv5SnLy+44Tv+uZbuRkAjnKE4Nads0Q + ldCIs7D94XZ2J8Xp1WXkiSE4dRwNyfJiANFvYbXt9UOCx1XXGm2tVYTUhFbLDzlW - M5uk+WkCgYEA/7tVqv7yCL+5/izvuCEDPKGG2Ln7G1Viwlt723HccumMQFxEt3Qk + R9YFYNECgYEA0L0QDQHCJexuuH4AS4BmGcLj1tSrHA7BViSrnhPrQTjaWeqWdiZh - QAbM3vESp6GyzL7yNDGkaDegzf4QlUiKwlgRroUM6rC2SBRtqykbR4m+eqrO1bDE + Kv01tWEeIdODCmSIJPVMS6aywoII2bgQAFvzOX3tAl//Kc8wdFSiz7WASyatVW3r - Phn+k9AL/BGgGyz8qPWby8tN7jPKa9i6InljOry6U+wiStaYaIsN+PUCgYEAuAdq + uv7dtTGSsBWN7fXYqvjr0eB/mvyqVBlYzk8qgSpaWEhkkudn/IG18jECgYEA2hAl - budrdeBt38o9ov/jMbe5m4uITEu64HWWZowS+FoIRNgRjB4qtZCMmqqUVLOUy845 + hl2LDt0D8En7LxiKIIB+0s7cFMJfkjNRHyXcEzbmTNHhwTQ6puYE5TwbSRw4JMiB - 2eVFDGz8DUqo1s1Vfvop6seQLu8I6OVsLkyhB3FRK1nVKxtRKpqYpk+7OXQuJz1+ + oDIvIAMgA51Hwk5aC/aGoo6z60wvHTzb0XpMlSWvytuhN5fRH6xXY8TYM9I39ROI - H2XRkrPxXECXe1h5LlLRmNb928let/8ili4cTlkCgYEAh0EnkCcDEAmHb52ItBQR + xU0qvGbKgQQulwgEzRVGc9RAk6BJ+9dN5DH48O8CgYEAvqTuk/KXL6vRNA9glZSf - yDGORnYnD0/byfvkyC2ycLyBR1EFrxmoSozOMmPCgBKPpKahJ2XSFKTHUeu8DZiu + q8ej8AIshWO0kMjNNYNbyiXyx0zKPv6uoGTDOPWKX7qeZE+NSLQBCtclTSEWlELX - exdlUq5gJIiOABV943b8TJvXuL06Y974C/hnovn4PLt9uKHUh/BPFDxU3VVbDCs2 + 2nwgmNG6NgEXO0hQKO9kA/DxS7H3fZ73PcKpG2Q7ZTdKeZugWAcg2n8ADL3XkxfT - VyFokBpdWiGcCYTyWuig3TkCgYA1Hc7Wm+0kZNbR1SndNkZ5PzJPdwKsIt+ZkdcL + VBpZ576W5Sq5MLLI9oZBdwECgYBMQk9NMRN7bDF/a+/q5XMQsL8pa+wtWlhf4ZBi - WjrPfA0O8d5+tuZU6ZfrvHh7yimUeb2w6r/3Si2mGHqLJVEcCVC390nighPsROvo + CzRuh8l8Xf3MOj60tUZLAH0uUS8VNgWXB1XRpSYh/XPl8M6u1lT5LlyfUfI8EFdz - oS2JXGe1P4SLoKLYzS5qMnEzsBjyMomIvnazBUUQ/4O5klvHxxfAKa20FndEXFu7 + Z4i2tApJMAuuTGp24CdjnahaXw7wpxcyoKzsXCo/ej3s1YIQUntj8Htw5SJab04v - RSveYQKBgQD4mdczo8e6Nz+PBv7jlsd/mRjnJzs1Q1tSdhjL65IKJ673MmB4oLTq + GobLgwKBgGzt/D8Kilx8RmDCBSYnUhYGqpw2m09knujK1O61S5yLODcUVgzp1CL+ - mP6VE8W/bcy/6Nim5OrpfKEXzXt1GATO5UTgk21BafqZHXMQytcWcdD7LA0km8tj + eTJQGAZxtGH//P1FHSl1VE3nGCuwCv44a3xZ94+BooL0NHFyLbLSMbekz4Imnsva - giBvfNSc+ivMsbwG9dhlZjAHHJWVbNxfdDR356vgTsVLSZsRbCJLUg== + dgkZ0y5hxm3MzWWZex+5YaGOLTzHCZbv+ec2SLnpYT9h9BMlZE84 -----END RSA PRIVATE KEY----- @@ -13812,62 +15312,62 @@ vector: segmentSize: 131072 total: 256 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:SSK:bx2cqrsph5eo2vxvobyrow7rdu:t5ayg4kz22cax43uwt5auiu4dnmt4dllohkyerr3dxoxc4nahriq + expected: URI:SSK:vfgmoktxuv5e5ajwayx7qn4h2m:n2se3m5k26k474losb3nhwevksf7iewruqwcwvk52dyfe6ihieqa format: kind: ssk params: format: sdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEowIBAAKCAQEAt3knxNRwWNFiiJNOD1d1vq2pxaqVn7lxWYU0X8BUdUcONvW5 + MIIEowIBAAKCAQEAxsCouaG6eszb6f1uywMum/M3wvq8N1j5JlamB6q5AJp8WzG2 - i019ISlqUPU+Z2KOBh8Zbpr7k0zMktAqeei/CmTHUcM4zmxZGFOFTgvTL/kZu66Y + ABOiemaykaAf2p47/yA0V3JTYS4+lUye8jZhMaN2iNKk7Z4aSaHNc+uZEJQxVeVW - 6DvO20rBKtPzpFpyySeqUtstHKwtz73DZworSXljDdj6nUSySQB6JTs9wk28IsbV + CblpobGIgtSdVHf+bsZBWh/rS3Eo7msxtmlx8EZ9H3ww3gJMwuneIfy5n28R1+mF - //gWrcZNhjxJf5rsYy7cFkcKW5xK9Axm9LIKTilf0wNeRpMiFO5u2rzKTlQX+UCT + Mxq0iFO46PaCP85kbvFwjH9AUioImx8uP8L6eUNdx/L2IqAd/cQhT/3eVXF/n+fI - JojhpI7PnSMuRnMmA8D14BaJEIXw2Tb90diIqDuM1/DE1i2nLLchGGrfxkPJqDUv + 0rydpLT3F5cKSdKqta03A3mFD/BLjbwbx4dCFYaH9W5IRWx8ytx5sRcalBe8g06N - sVpAM1HD7ayStU1Dq6GvY30EICLseWoJunTiDQIDAQABAoIBADXASOZhVoiuzy8z + kXA2WQeAqCpMVTs9KmIvYIU4QeblU2d9SXchNwIDAQABAoIBAEYeWChNb229cmRl - +KqF9Pjn59UBJNSmf466d52VuyigqIlxc+pbyUzt1TfioWWoefNRKSI+RXXiCgz4 + vb2vnLT2JJkMPnTEVfn5nc+conIdDnxZ2FzEkJDgRGVt+W72XjJO2Uh0lAf3+apQ - 73jHtzBUVhCeIQZYt8FotqUm0bg8Qk252RIwc2nLfMwPTFHaLcbA2CVuEMlVqBY0 + gs7u8nFBuyLgNcGDAsExbTtVRgX8Uj98jlMV77dU29VUT0EqqD/Kf+nc0vUlsgwT - ggqt8ACWj25/IuzwM0sv2JkPwggqPV+XAf3RtdsaJYG6i+N/pGOypaRodxcR1iWj + E1HId6MOKzx9YvwgEZa+TVjuQUqGlIhu/inRXtJryNKXoK3p9X2vCxr9bGcj2urt - 1WJq2vfzJeS20redrsEnP9K7SfGh4AV9Juky4H9W1YbON4NHLCyorRvU8lFPf+6Q + 3PWIj1EhcDagoBFporOFcvknjEI6FdYFxHUMHABDricPmsPlyWgR0P0T0BfALDSq - MVss0UCIxJpVzk/Y1BuYBOOmgCeJyoNeANMjb6RSTaADD3UGaqHKjDzoqblgasrR + 9n21DNn5/dJ7SUt2oCsHGOjVLw6aNZ3hVrpiFnkWYo4HsUBrDr3ChUj38SC/1tHc - o/kddAECgYEAuQG/UOdy1hgpoT6upA0/7wb5zpGJc4hmZUwvz/GONUzaNzjMGwOr + iM6oRrkCgYEA7RmTZvzXxZCMNBwFQWsTrlfl8HEdyKZNhT9VR3F1zES34Db1FeVI - SQIBtXVNd25ntGqWpsuUlk0HsJJHVwB/2ACQyJS+YG3qcR8Dz1Z0A4l0gWy+w+RI + Z4VyQC+oBIVef4A8HNEmdA00SRDrxEGVuxBMwQZu15d0HlZT1NMutYZQq7E7h3z5 - 7XERE6p1U/ui9gTr2QyKWZ6Y7eIAl7spiwApmdDeJNtr9UfbpVJlcGMCgYEA/eDB + 2YIgVRQ9lG07/jDaEwjfcNLo/yCYFzfhx2O7xCKKzdtlYIwU5AlWxSMCgYEA1piK - 88BdT1jP3WazAumrGQyKQvibFtTMVY5XiUaM07AN6EjELU8PcWjRZ451xJ9W0fc8 + jJ1kmM5zhb+jxF+lHbUd2W/d4ZqQUyQ98Hetk5uV2aoBbCLiJaTiO6r/ZQafgX1s - Gsb72FNgcWuphERC4qeagAHuKCAF74oywYfKU+AjItgR4231UBL0IEK2xnv9LDWF + VWx7dLC3sV9qUKI++ymkhV5vUmgg5eQkg31hXg5SIUNVWypN/jOg33e5P/TdUDJ3 - BBGOKC8mQAqYkvnZDyx3eHXBV/zmRR+vDF6nls8CgYEAkVjuFYHAlrMlAaldS0Wd + yycy4YuwptaXL9T9Qp0Js+RyK9toC9zSJ2ShZt0CgYBKUDLYG7WRca3QA1xOVb5U - lQzF9aQheMMQr0TLy3LbZsSaLAhTUmXvi8wny4f89HeowfV7pk8KzYp3ICHMKm4a + ba5fP0UDh8RSWDhlbRVr0boEJ5WHqFaaQ8Q8g/NYf2jP86Rjr9Yql5zkrc4HtDq2 - AnlvRiaV6uxv46+aLqqdOqoi/guRVBVltiW+ZNTmmLR5sw7qu/s+NmqDe2CzVoGU + 5/P2qAqDvi+h9pLN6OcB9DhCqAktfSleWB/EKtTmOZqNIEipoKVP2ns2w8OHu3cj - gb/+7vlJjWtVxb5OsfOp/kECgYAN4jy5F8wCitjTQsqHXj/9HrJw9yeEGB8UjrQ6 + pInMfrscrIBI038UviyZGwKBgBLdK+i6eTpZg5wxQXMkuT4ISsxvYgDP9nnoiK1X - zaDl8rrP+SrBT5GIojLRdvj5x7z3vo2K6VbcfbLIgRrEIPeHbaMFXRWpHBc3AlfE + x+Fe3uhYYnGgC2MlwGFgYbz+vQzD+r7zn1KdqjgkXBMkgAbSHU1ABOcokiPDT1Zj - PajS7W7+eNKBnYHM3zx6hyt3r1ApGsQrdMpRaEKvPeUaJI+6RLRD4iywoyP0o8bu + sihzd9LGuX0fFeYPocejHZy6qK3BEfjAxF9BSVERMg8ZWP3/EfhHT6X9ToMkcTDX - 5j5EAQKBgGa98kvhw21u84umuGP8H1ASAg2C3rGO7uPiq8VKgIoYbSwkAIeYcR/q + TrzdAoGBAOahVk7AQlyZ2doxTPLA8+gcJevkLoEGmAHSKIZwvyWoAitfAlBNBEnV - Ewybhf8Yc8Y+PtLMQ1HlaJhUp2bw3QxV3aqYZuZ6rZNhiDLIcmSeWYxPWhwG46eE + zfGePpIJF39DbRrqyTbYTVBrKp3/Z9KWM+QORxqWlwFd//lw3bTLMGL7o0hX/nAN - UWbY02wv+ckMPp7j1+JLrXPdfUdeGoIEwxsVTO2vYnZHH+GtWkyB + X5HWJ0r2CwMbRMFl0Jgwml4ONor4WcbM4GGdHKX0m1h1c82EuGFc -----END RSA PRIVATE KEY----- @@ -13881,62 +15381,62 @@ vector: segmentSize: 131072 total: 255 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:MDMF:oc54ehxm4xmvhrkeukpq3mft4a:blz57s35sxrodbg6ylpxtlc5icgssdxbdzb2r5m3guuwpvpux52a + expected: URI:MDMF:wosfoa5glh4le3cvced4iz75au:gmw5hlx2n5vmryjujpt6bfxntgfgbdcqwykgxaq7sqocsgbjtkiq format: kind: ssk params: format: mdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEpAIBAAKCAQEA24Hnx8PM86/KeJ0h8Q3H9g+OP7JdSYhZHYkbXTZfKAUihhs8 + MIIEowIBAAKCAQEA3Z+dHpI1JFhLpqYkOBcmoiEbOS7m+DJBxmWsBpy3owBVcKHW - 1PY+wjUtaqyJW4FO8jxqQi8QoliLLT/4Tj8Is3D7friw5qlYoxU83umEut1y7GXC + k0kxM9+bq6oyiUBP4MI7OPcH5A+1dKaUaTBUKRGaFvJfPlsO1PHV5wyD4KdfOIeM - j2kgDi2SkBCcLcn39Z+Eb//d9854TDeV9xDyiQ7GK28n6h0bcEzEWCYL6XdNE6W8 + ZrcZ01glAIlezAdINKLXUgnOE9cY969/q7hcMLUVRuCQoF23kSI3pEQDQxC5+lIQ - 4rUPKfVNFqtstbrXkLNr2BZ9anB6Hv5kvxyCSyEtu/uOhFePnBRBFX3MYHuxL3lU + /GjyNAL3mCgeYGsVeG4Ey+cNgrX3SaBjKMW6UJRNHnt5oszDPNi86ErsthDwmg6m - y9CUdbtIIek9ybFgmIjnB9SBJ3M1Lcxa/dBHFy352X4fbpMjPoA5Lj9UBOXy0WKG + lgj0qqXdOYYIc8JHq2e+3ERsota5ucBHnfS1UHpcFxgMkteymrzGmKd9Cv9alAtS - jCJgulkSJsdqth9KY6pzYJEmh8xwF3rwz7Hg0wIDAQABAoIBACMZ8uaO+Qc+5THE + dRyHlngdt4VK3Jw4EIlaLRPnVqT5xY4huxQPSwIDAQABAoIBAB07kwl3xpuzK9Nh - btkNSxyeADFPZHuNwjJm6mlNeIn9yDeJw4CKoB6OQmT8kjp/wxAZeSR8QjyzzA3A + AdEGOLnU/RbHWX7ufh+RxKWgoVZWUm7HYhrOYjeR7KIxknXpLkAazp3+c6OA8PHg - XQSmL84CEzWAc0lvay0pCELdNMxs/SOwYhxswyOBRh6jiVYJJg2xJIyEbgpiifom + kR3o2okKQdV6BdcfQq8S8SCHVZPZ7+TweDQKPdVTQJo4BHGMGlmbCyTOl0ilrCzr - KWUI4L/qDOaFL+zQGsMqg3tVGjKLBWtvlKN3d7gzood4SE56Vr0hUGeZ9sEXZrJ6 + kL8RU9O25wYQ0/LbOb3ikg1QTU/Yk5SXqbquyK3c1Amz79NxuzeMTDEjCpRlTbmn - kcIDND5O94FNtIK53y/+lb7mMtKUVoraFLF1lK7UlT3Lh9T6iTax6OHjbHtQs/6L + Sq2PW2k7jNoULTuCsw0nEl6wtHADPx39xMYWgfXoK4caiTY+JbG3Ru3TJYHHytHF - KSLXumtKNaLH6feU0NaEzAn07Xmr4B/6KW/5dOZzXrpstVegtECTO9QrLZEIFLyO + MTCvJWFX3ykjVPtl054syo/z0krqcacBmagocA2lG4Yael3AcxbOox7Z7zwAZjbm - x8JjqgECgYEA4J1R0iZYbcQtZHrS21pXWv6B9HNfMAwMNyZH8KAnH+ngg6FWEeDJ + 2rxAeBkCgYEA98QXfpX/xuwezaDuABm+5cQqIXUjL2QG9cO/7yvdQFs2QNHDztqr - D+B8PcKLCBUqMqSkDRVty85px8drmsyvW+bZfyqrSAGKP3s+D5GuGk6kYDHSAo62 + /WpGxlJCOAK955HNWJ6AVQQaNsl5LZ2HaxiRNwSEiv+FKLUo6KknPDE7eppDu1en - I2D5/4BtC7a9jMHY2beH6RpzesN4R6ttpQhGvKAjuseiTnYqIUi00cMCgYEA+i3m + mpKrQa1W6+cG2Ufsg5BHr5VQr/0KbeIQ+izulsEfqxMq9PLhEYpnNkMCgYEA5P0c - euGWM5Rnf/u6eoXs70ulNxPzJA98sbqJy1yXCgNfZhdkdygXPwDTq9ZemLJTOcY5 + T7ptCa3eGJEDq/Q1zH0BMJMgczbljT559psTmHVcckW6R1x/SN6p9wH5vzUDqlVy - vK61feU8S+xBT1YPvewKF0IWgLPag/tFzTrd2EnK2Qtlc0MqkidBYAMAOFCqTxo6 + qeD9Kz1aG4YvWJXWh9JgSqgkeJVvfRm8SUR8CDunKshQ8fSbqsFLDXaModIy6sUS - 4ujbOkPY9T5Kc9b0lIUGyCicR37BouJ//y5hM7ECgYAjFA6iLkDjK58XMSNbBHne + dEgUpC1c/ufkrPeeGI2CZJy2Uwnuqa0jUtGm5lkCgYEAySTsAfuapA7LTxroPTKp - CR9MiPQVsdv6hOz5RFm33zOj+v9RHXTpGNruXkKOSZfkftfr/yu9h4f3nkpMy6ib + lPU1UuY7A11MfTdG2c+dloK2P9dMBOHoIRqnjJf5ZGltbNMkh15eRybGdVYJR6wM - Rqsy8/v569umXF3t2oeBLkT3jPBKW/VQAyYn4+ujx69Em0V9gu8j1XCxfHN9ZeVi + 5TgTpDvJsuKQYyT3qjKxRJ+fbwBQHoah7c5GtFIaL/flyn4mmASI/hXVZJqkXeLa - v68kaDIMSn8rl8Kungc3NwKBgQCBDsym91iUoyoBS8qXCh+AEnXYQ+JZ5+Nbi+8p + 74+MvtzYbdVo2WVYrRnUgusCgYAdG9vcertfrp18C/smgb3RB9b94MYQP1tA8D86 - iUohUDwWXlrlXTkgtzx6mMuT2eo1E50VSMs3dtn0EJxgYPUd9HYAKYeSPTWsgCMy + zQ3ZpJmi4SBD8AsyLTP39WVVHB0iKwiPdc1ZEMyCkTU1kp6Z13FsLCGuvnhUs8/O - C/wFZ4vNC6P6IdwEKVwAO4wRgQtaYx2dkKIHHJj/anLd7zWcqEMnXkvAVhNuA4ok + lIkb1tFyS9KWX1zmgPnUdUx9SaY1V+X3qC4PjMC0mq/kGPoc7ugzeARpW+rd4OeL - Cbj7AQKBgQC7q4JbGdNjqxuy5ftE9u31bkhgES5+MTrW59ov6tJZK4qru4RTeqwr + oKERyQKBgE4p6k5qG2JtFc15uYeRX9Dg+zcGDbKDJIHK/NlF740a9Ffwao7ul69z - QmvudO0s4evjufev7eL+kb/TxGcM+o8y0mp5VJKRQyO6+FNdpr4loJUg2XPtHyjq + qx5HycLgIIk+uVamRu6Lm/8if1E7bwf1Phr1GV5qLm5QeVQXhAjCFL3d7ieNrbQr - 6YYfMIpJWNPnt0f4zip0jOd5J8Iob6mJe8C0DLHsAJvfmnZFyf09YA== + 66YNSC7CyUg8oj5r9CPGrFrAQnSqT/OH8Q6i4RTMMaX6YxK7tHch -----END RSA PRIVATE KEY----- @@ -13962,62 +15462,62 @@ vector: segmentSize: 131072 total: 256 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:SSK:avkuy26pwv2j7cjc56sapn3fim:buvkvpbbxr5qmqiurhuidhipro6jimma4mjzliv23zfpozxuilka + expected: URI:SSK:rems4oug5xkgjc24vouktsjheq:jkytq32l7ih2xsaosgrrjhsblmyjnxn23zacvefyg6lnh2s2fmxa format: kind: ssk params: format: sdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEowIBAAKCAQEAuWyYTFaC3zq/JrBkVOANkU2EAkKOxOtATyRTx8qJCvFt07gN + MIIEogIBAAKCAQEAqt7c+hoHCRc82m10tbn6HkGoPCP8ezDEw0jxvH6b1ads4+Fn - ZbKwcj/VXeawmris8SjQFofHftC2eCoaLdIi5D7sHlErjELitbPm1OruM2N3I3tO + tzavlbpSY6oXnSvISGLfOuxnDX0iBJ25m9qJaeZufP/uc94UPxxPIjyKrC/87ixo - rLDyZwccYDqLhHVWhL7EfQBJcvD4DmAYNEUa9dfv4SHEX+wke5jrxLfb6gob2XR3 + Fi8Fh4KMd3ZbJbIoJi5MSnONEoecIx5lgPf9ckRsq0UqsHHRacWiA1M1c5ZTRw7u - o5b0fBqoGCm8cUDrFodZh4g2G4hMYP1nSPyIUFV6ElFdnWqMTZB0kfMYelT1EVF0 + LZuaB0udxBFuvsPSB2vejvqLMqmiLUYn22yrsbFPlufUw6hh3QLbjiW/qQB3KflG - 440xKjWZkTEXmNtIeoAYbnLTkwvt8vA74FOq8NDUgyyDx61ZjfaQkH7mQ7Oqhnyc + 70W/6Gog2aLErsu45RG+uxoM28HHuzinYmnoSAUWmA6D7P+gEKtyS508u/3EWH+l - LPX2ivaTZA92b6QnxxlNE3sV/QCOwgzTgMW/0QIDAQABAoIBAAG4Gqjhh/TZIvbR + O5BC4s4CJRDDaoVjShWwcICwotnQGF8yCjwlRwIDAQABAoIBAAeTD+D8Wgx9SGLM - PZrmWWXam8HYG2ICwt3A+thgPblI4AFtpE0oNRfYFOq6FfLXSb4yKEy/LUe1GG4A + ZayP8nZN5cseCJleEj2FPxtifL5OCNsvM52hDHaK3GUeS06vJqviyiTUCZtvonGU - iO3aFAn89dw5mS9jmt2/qWEZvQPjtRHyhZoXCWZQY/BV9p9vpZHVQXXdu/CZgJlE + RlU81548q0WE7jPvsnO7vwb0VRdd8eHi+7g5XGQ4lXPO6NUfySeBd5CjVQEZxted - hZDtf5ieLAqQsTUI99UgB7aTFFJFCe8B2DHravbY/n+2irdBw5FzFujGH65tlDQb + siqIk7vG1w8JPA5h9UJcPZrVIPJJ3eF+wtJkNh5Vv1CUyR4yER0ZDulSJkrccxqW - YSEKryMu/5JY51hABvY0+Oh4JvxqzRu+8wYwD7li0JgUWhgbMt414+xGBXH9ioE9 + ELMkpLn38wc8iFsa2yDWJy/h/e9j5ElhXedqYC1Ii/WKp2jPebd+rZ94/GXu7/yC - Tk7iZGb4tqkc990ZboT8dSEK4XQ7Kmcc+8+DUV3BS9NrxrhiISLxx35bPThDMYMT + E8xABg1p6MwVkVgykYwYrlS56fJ3hTz+NdGpVA1e6DklkD9KvyL4JEm7Wmk86Og1 - asRhyAECgYEA18pr3/otUnSsRKh6EMYi2Qn06si/vu5h5W0mtf/PdRdaCDoPETXA + A7cQJ7kCgYEAwSC34k+SpF3Gzci1Bb3DJFSwvX46ASHr9jRQBItP8NGgw7leIxbo - 9jFR+vO7P0/sumfd48+mOeqanwT11AbytBr9Xq06Ry/B+hpXzMwiG70FNn8Wia22 + 8ebQo1W/9peZiZsBGGXSDbL66+ZJ9BD16yF5hoKzE3w3AAEmYqqyl6PQfGMpatuI - sDNKRE5GCDbxsQgp9CcW8HUJsAM5UO8cQ9syKNKRCec5JmxbqCWSO9ECgYEA2/mj + wzl4AmF+jQ7dMbGM4UDXrGZUbM6WgCcDcat9LEQafct53bcHPxK6oW8CgYEA4n85 - 3cBulGHs88D7e+8q/oavqv9wzHhnEiNBBvNpkDKNetzZ49rJkcxu4KdNuQLkt1m/ + iuvaH6HON4lik1yDMwlwgZ//pt1YpYWg25Sb7SbhzLCXhUP0r0XYdCrXROos0AtG - FBnRMzPKzlrAAnEvol4IVK0S9ZsQ3d9PlFZpyghncnuvL7zVl1M0VVEfOLC0A82u + 59c4uMY1bG2lXXpKEv+r6yHkmVSshag8gle2LBPewYmeULxoa0FsPHYW/Wwc8SEK - zEHapqCt66kMb7Dwk/+9dTGYpi3vhvojnzJQRAECgYEAmuSfnjvzwFYjOX09cUDn + ogkZzuTmPYXJbqVGqSqx5rdV5hEqzGBDf2giHakCgYBD+BjfbDPm5x4lpIKZL6zz - zqbI+KZ0jFaMSqSYvtcKUOAcLf+OxSmygoVQdTPyWjXClOLtcRKiHLx7lF15H2KF + J19AgaE2btLVxpl2z/Tlg1F6MM4BuXloUVySb4Zs6fPeaxAanxMrQRdwWI8kd6el - YCZnbEgnpuVu9VlnYIe+i+6YCVAcG2Nn2P5X9sPAnTDjN9HGW4ybeKpp87+8qo2X + BhX4Eh2mOOw+cykoRn0uQzgH3vpfoj3iv2IOLHPWfym36I31ZNXC1gzWcmqjVZev - 2lVCoe7TUSp56UyqVf3yA6ECgYBSWFgsSb3bU/EUqlg546UPlLGr7GV4VVYYJxRP + tLQMFTfhl/Ae6OCDATtvvwKBgGe8XV6DJyPVt903zy4u8OgvKpgz76M9PZyR11q6 - ms0YiqQFqyjxr9Qm/QVAmcBxkpC1xiXOS3/RkADKUJRyFZbETDkIIaXoRP0CYXbz + da/oXwKg3sTqmuar1rdd57pohp3CjHci25fFMDK5BUQK/mI1N0g5/bk8Tsfohc4s - y4lcdNrsszo4P5MhS6dajLyIRzWL+vIFSl2kZJ/WiPi70tusO17bwQ4onyd8OqUd + 3gLSFvQNU7UmlayKCkimzWDEY30M5RHRmUBpFgqXe+pxSCuyokhJL85vjmqMrF1u - EgOUAQKBgEKtBZ806szsVIa0eFC6PHxt9b+N7YgeGoK8x9bCHNS11nXTibLImecX + FDIBAoGACVQqNIjcyMXkw8U2b2PhIIlUjTOp/l4rZh5xXdvzqnTPGCDZnVSKO3++ - bhKCf/xmibabYO7o+V0O2RyKZxldRWds5Xj68fCFHER5Ax/9sF70/To2UouE1AHB + XNitZKlJUNWzJdUxIE7V+WgDTcNu8RNZVfBJetNnaLGol4hPtBE8QgD+qs6rOg0e - rL4c5j+3y3pguBLfsjsWFCJxgAh5aETMnhPaGDSyy67D0PwFlezH + rNFOU9pBTSeDNM4oovChqmOj6zMkO8qeUlOmlLg7ELkHGLfwxak= -----END RSA PRIVATE KEY----- @@ -14031,62 +15531,62 @@ vector: segmentSize: 131072 total: 255 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:MDMF:6p3cogoniw7gnwoyg2574ephkm:ag54qqj5o2bhyiy2zc2gu3vk2flf3kutphtstpphjgzzceyscpcq + expected: URI:MDMF:ynzckg6zuaoi4ttwllb2b2o23q:sbrcnxnuqnnefcqeg5vglctxu565gl73fubzmapouivrxjqz33zq format: kind: ssk params: format: mdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEowIBAAKCAQEAvjficy3D4RnknUUEgM1Z76aRdQRHkHjLDa0mTJVFGVCUcI/l + MIIEowIBAAKCAQEAmCloFzFxnh0SodP3jlmVdWPwHgx4mCHU/qAxxWYASfX1h910 - oapqe4YlcJvrjHHq4OCEDoTaSYG5ypsc2YIerM8rNwI+nYC1AHwg5Y7ou/l11TdW + uMZTsJZ+O8AsIFfkQtCy16C6VneL68JZ/Aq4qFGgajfWziGNrFbPoe5gxpl+06vM - 32xS1wtbFaSLXfmTWsnt6ssWQjMjyxZ2ZcN2wiClfM3eDCp7lb29lVEd7NF5gKWA + ViGLjYE4Pn4qM5tgIe9U1xby62nbYEQVhe/ApFD/DMhCfIhhpBp+4vUtSzXQaxjy - qHfy3ESMLJLkLSkqyWoUA8olXPJ67WVKP+jTuqzTHEZ42ItsDJ0eLq4jXqHJYQJU + FFyusFTNPHD2fD7U5CFB00vStQmvmSSXeKcdT0ciQmxJh1dSgPVMbKFO9tM9TeNq - yAbCExmW+rFKw4Ae5j8Vh618p7joa1j3BuecNdbthSbRN79XknubWC/7MkMOjQGy + HwQzOR/YZn4Y7zxqsHJ8wQS83YBYqpvDfvK1ILrieboW8EJmZsiM7Tq3NCcOBOwa - TdemtMi5Opb/uUjE8pRzCenLGNx9IF3mOGoB4wIDAQABAoIBAE1TxpjwF9sgfZF5 + n4dYjNqgrfkcQ+R0eMqyhTBoH2U9dPe3AMZ3kwIDAQABAoIBAEsb9VOthmYD99SF - hzUdRdxoqGUbkkQm9tTeeN1VKTv7R/ziYoVwE82XYQ0ANadogAVfABAu7dZICFFW + 6ycLNWly4W4Tvdtqp9bggHDuPqpDjOV5/UnQLDN4tesMmzuD5xrMJdumbRSNgjXo - 8Uly3il+JqE8Jlw9AFfsHit0BySzarV8w7IcBSkqkqKfu5A+byrPQArc+HV8+KYM + A78UE76SPFryIUgy69nsKCXIo2ClGCOoI/9II7i/1mGSqYY75iIaH4jkvRhTcoR8 - waDo7xRH1T6BKi1j782VzsYura2hXsLKx6XNTBmjqf+fetZrZjDubdGQEACmadDw + Vxt8E12I1b0bhSYvs/LrWULyv17l+GBW3QhLZRYpAVpXipySK2hvzNS13z6SPo7+ - 29uZ9cs41bC1KPCpB20QzL/x54kczYchkpUm21Yq9LKSqtWoXQQjF+OkYkoL4ShG + wwyYg5Si5+WhY7DJP1yGku7ihLGw6MbHv8Fqr+L7bceeCAZ2BAoAP+UlqQo1UumM - I+GEOuHU5i6qk94sre1TRjZLN0H2u9hhNr7i1r2hPIBWx1gFNItm64tEEjYxUJzM + d7BDpicjhbQycscf0cNVpq77uPtp6ULQjkur/GOEU9CkKr94uThABqVID+oQ59TY - 83wzAwECgYEA6V+wuHs6WCMIBe8Y9UH0OSpVsK6b9TBgrNQHN/Lbs6zgGtzTm7mm + NuevEJECgYEAvEF1f151xSw4jZAsVw/ZV6F3xslnc8cJjsXAD3mb9bbdVUxKU0Ms - r6D6doreYovS3z9yCJnmQZ+J03p+cnfiAghdH8BwevjkHKiS+bOqVmO3CBb+KC2S + VQBwIRpx+FXM/gv8dHPEITM9U4DM3DjAVZEKIPxmCQwiaf8NQyCMVtoD+2VOpVbX - f4JjHcLEtXp6Oh86QiYwA9HmNFfEHuZFxje3JiF+JWNF7Ldmyz/gLsECgYEA0KkU + GSu4E43uDVg46ECdyrTz2J+cc6so87sWbX4MMXI1z7kJOKhvg2cDm+cCgYEAzurk - gZp8ECeHjdaIHI/cyTjzTpjKKfDRRuOW8sxymdNaNiDxIL/wUtFfQocrtxvVIo4d + DvXW2k9XvvEv5dYdKRqgUpjNqNqFij0nQ9W9JHvz4tR5eWgSXMgfHyeJKMrr3Cbh - Q0GUqyS6OgV19YXCVPB8Am+LgwiddNZc8ZYdu4dySF4azJXdKoa8E6PlQHq14gY5 + q1IgIyJtF+esFJZ3cPdMzf7ZwEZPF/ivngnTgdlOQ69lO10pVALmO2Od82HBYjxS - NzsbcjjsTLdv2iD5Er/AO+rnPeldiYdpEGPYfaMCgYBiN9GqcsJlYaj4xl4cqntc + m7mvRoK88E6j0PODjmcJqLXM5MN6eHKyvwlPMXUCgYBdE/RXNEoAYfvYKmdx6Fkq - q8KQr4wXrxqg4kN/eoiYoANZiuLMQWAzvm5rAZsCopJHPu6BTDQqHjjldkJNbsMB + laAV/jCTMt7L44QxYow08eP/L4g0IKtDn8LQ6zVcdnezSBPbM/3N+Hqi1bT0UW7v - 0/9NY7JzLtjibtgcm07vONxJXVPuGO/1Fi0c02Hydu+GEqp0OJowoWBfWyjBUGzB + H5YldwWwBXric4OIJAifTI3Zd15qK0SQomgR6wO/P1Zrpr8doVhLS6dcHU1TLLZL - NaWxOJtcpOFC9RUgKWvygQKBgBMLu+Fwlm5rDUZ3FIl24DJFzn+YFqvpXVDZKUgU + Dp5SuEhY2wDvLYBtNLq5EwKBgQCfXBSs2PXSSQ1BR6wmDVOEFrenJXwvMa1rnFGj - PUmpLwzNyPSyUF9e2REbgXP/SF8VFbqz27wbaBwvr2qvwOM76DXYtKVLPgQSJP2w + UvhLIxPgfNfZgyexQYeGjQJ74lzovyFKuwN5S8hNguXrLT9sR2pltIOsK/o6chN/ - NBqP7HCKlmuiKkPddIFebmiKStvMsaBG9uRgKcF+5OjGJbX+Zq+Ra3YNPQp2n7Jt + Wf4FoYE/a9RBdiygQWNkFgLOMVmo+OB+gvHVElfFlCtigEmv4Pd1ch8NiOfH4D5+ - Sq99AoGBAMS9ZDgnYPu4H8MFXRuwtk9Ti7IjzSA8zBh1MpQFhdXMzg64s/8Moq5c + FwNhCQKBgCJIoZu9Cef70VcRWjdsNxbI6Da6zulD8gIbsw2yyR+c/qjQMULIf0GB - eQHnT/j6bqVL3OevmJGqhwPPIuizWvskG3eEH1LC4+PPk2L+Ap9m5upIlzXLJsWw + SFEFdQNNn6eWRgwK0hyQBi1UGpR+kLEf11T4bepWD6KNbVwEUuve2TRTCWf3GjKp - Irq0H0KV01DDGhJVIB0E0DG2q4bFnweXqQJoq1NxGxPrChcbo0Bb + MTMJv/OAU6uulvOsoBFhHnhgOj+IV9M8GgeDyVPXYL4+IOjTfxCb -----END RSA PRIVATE KEY----- @@ -14112,62 +15612,62 @@ vector: segmentSize: 131072 total: 256 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:SSK:cgswz2wpgsmmxpsk5y6w7fcypm:orvbm4ilcz4cedxaxsdvlztgevn6vy7edrl7xhmeoubdovgohknq + expected: URI:SSK:n73xrpurczd5aoyiwcvpnjnmtq:zq6mwbtqsg3kutvuhwrlfhmiu5s4qmtbkccv67p226zowcylkaea format: kind: ssk params: format: sdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEowIBAAKCAQEAtdjLVkFxYw9RpV2VkySp0nQabpd9//h3KNSIWi0S2WqJlfCp + MIIEowIBAAKCAQEAsIX7cTKvU5wt4iW8FKUMN5i+uZCAfZrq/KdefeHFnYJStQTl - 1Ek42VQlqj0cZ6FoAgPiYCv3FIjem9T0LGYmp+UOv3Gu93GXgh6eIC/b6LZlYlRU + ov8IrO7aArYUh8gK/tPVCz2Yl6JV9A1Y/8qyzBqRVAT4vIDNwqyVwlhUJQc1pwUr - L0PzDGi/rFhtI62YLluaLVeabJEI9SQThKRuDbtZSlpNG/WcWxoG1TL0hA83DPlP + xiSsFroLCR4ljJxZkp2CqJSyrYHjVwAs0Sy6L9v16gV8wOAGS/Z879nb2UB2RZzF - Ud3aeah6d2q97BFv0KlHTdq7MX8pjUUuxU5+JZlu9XE2TGOv9Wa+IUF18mTvRoLd + JrRAlAy5P54DvzLA2t5VMzV4TeZgnz/YlYHLBl6e1+Xj3JU9KB0YoyCbSPV8kgKn - rjq1R7nw5z0+Y90R1sLQ08NgOSjb+ytnl8LKXLtSI7v0t1Al8jJCsEq1HMnP8cLW + PxmKKCyVxfFSpqqBWN0M3s3yVZ42PffvErzcXjNzx5LFj/dCRBmU9aZnwxHpD1SK - 5Dj7Iy3QQkM7RudqQSaFzwHcNqweQknj6C18XQIDAQABAoIBABOXJ84YE3NnkBUv + d6EVSEZI6Yq6Y5PhYW6bqTUImHN6+yobbPIkKwIDAQABAoIBADBicSK8dVEyGGOJ - mtCU+j1I8saynV65Js/dmFRBV4QtXRCBMoVDNNOnpBMjWERgeB2sy3OEPlnA+7Jw + 200/ViNxEyoS4R1Mlsds6toPRdbgD2J9tqHgTNT13TzsAqGbI+RoVNdxaT966Btu - Dl9wyyilSEx0STQxJvBcBLmqTkJTfA1MJnGdHA8HLeG5o7BcoEYblOWI8thQBlYF + gywNt8d5KseAW1tz5LJNEvmDs4C4wqyGntJ/X8oU8YxsvncVrfmhgdxKcdVcKl/A - LUZ0lxf1n1ifl9UI8G+Zbd5ZX0w7+dnezSrOcXcNqLwmwtZn/mz/WOC857q1Gej9 + 9Qfavif7HyMnoOPPI/qzU9h8eyXHb/A+HBAUsofDpwcATFTCfgDDAQcY8k8yAD7r - xx0+hHIN/nKPkltvbrm7T/wfo4gs3wu3KCz7ewAUAtdtjWi0P5ibLNaRVcANuVjf + xI4VshbyVlIZrYgdVLKjkHNf3uuUISQ6Lam/pMyatutyQmGyaL/NSG4PbSYSLlg+ - /CmHvJ/C+Pl5PdPUqDt5Dg/aQJvfHXicCJE7Jv91spVv/dsJ7R00jpiyqP5j4a0p + q1tEmHqDeGWsLupb2qD9Xsk/bpDD5XbZpernC6SVc1oBSNb9AV8w1qGT9JYBVcHe - zqmzpuUCgYEA9Tk4BvlRQv4hpOEjCthRwsMaqETWcSN01gXA/eJNB7Xw4rOa1DZH + g9rEx5UCgYEA6gqEZVjgo0kbt+F018JjtDeC5Vd13nkZ5jVxYngRZhx2rIu5bysr - 8m6QKKJsX0jsaL93ZrDI/6BcC4DYoptJOQ5vIJyKIKCYhld1jEZKNiQMsKmslGlF + hYfnW0iEWNcY2x0PJE0Z+mO78g8w/pLZFYl0FIgGFSqIHMHrEGG0oexos5ndDl5R - M9OsHc/GsmBg8VgZWfsXdthuYFmm9MvFhm9MRDQ7E/JriZgI+iS1NbcCgYEAvdaV + BlToPxA7Z2tbsNK2YwlKgM+0JeMiecJXEdQ1uPmFKN30CNFLzHi53N8CgYEAwRXv - bqAXDf6x4Z6pfX4T++qZg/iNDZOyY1vrmPkjEuWUorezzqUxkw/DOEtSdczPAc+L + SoDEYN5YZJ/rAPLFedrmm/ima6YCBT3gkVnaZDwX4ii3hgSpr6VHktukf+dIgi9O - DUEE4pg1y/FAFvIbPSDX3DHsba3UobzA1GkOF9K8mZrB/R89Nd7OgnDBGjDlNozY + /bLId1DmGqFLLfaifjLBD2DO9UCzWP0WOz8POq2XJg7Kdsc9UUq4dfH7/rGdk4L+ - KKjXilAc8oIiTUKjBsrwZLrw4MODc6r2EadxPosCgYAfi/fgNcy1cJoFaw0mBQQn + zvWWx9ZZWNaACi/JXUpeAUyDIVWHn1KLWXXL1jUCgYEA1HxZ+d24jec5WDhEqgNe - qQ/R2+E2dtg9/EmCn81HE6nkkDR33m/NCVo0UAjfDTOUmiUTKeUBtbCBrlawPIfj + HGft2qUOab8PSZgp6lnSih+7iyqMYCcUq3ZZEeKD7ljTw1PdxHqP5GoaYEl0lRzk - 9i5npJvEbMSSa8fsftJnOqYDSCCyiwRjEXUP9L3cDrgJ9Ep2n+251UgFzyLCVUCY + JQ6XqnBY/WyRCXLyJPxgUEbgRHekYIA3FgWOmnr1RA8Pvzl/x+jOkKaDC4btbRiJ - 9dJ657k97K7W6Z8mBvjk6wKBgGw6URkvldU5tkntva042sXNOtY9NpVd9d6lggzF + jrFZWSiJwjHJdxv2spzFOocCgYB/V04HnsDk+f7l7in46COg5+NrPiPTnxp6BoMS - RJS6ZGHcH1uZXEj+PIr0jj9wkzfyDdFxlwpkQo9Rq/so7hSMi+QSZjslVksbJEg0 + mWXU8WT2/M98jZqzgpefnUfyKsDBSx4XZ0+akToQmguQ9rXX8PUuhTQ4v0EJEXEW - 2H8GetWLoDrhu3Dh5JQDGmQHKjZOV9HeaHuHLumm/U1Ux0LRIfobhcZuUJv6BK2N + BdKvakjjCqIwj9o6wMLC1qLRKKa54IzYRVP52731PxIWpclxw1gYFzPsShI12ySY - 64b/AoGBAIl7UTiqbxoQmB8iOtHAI5ZNJrLmRek9tPnnOzaICgrCxq7Z0/B8ATaA + DX4veQKBgEXU69EPIoJcr6VAkK8nBNR8IC3KyywyYXSPd2ClRxFn7voo7anlwbFx - a4xPwTtapnohHaZHvqgr6o34huCx4ZEFvYpsuo6vTVD3qUz6Rcnmy5PORW5WId3A + pgsC0o6FkIswGmo7sdDbdpWH/Th1w1dsZVX2ZsIvG9JATj7/OJ3trt8YxTRz73SE - 4jv+MQ/KAzgrgm1o0eWTauO6XL71xdVoLSEYeNpXa9yeRiC8e2vx + 0c4pi85sMLZXs4mKyxpP3U9rdIHi2+2FWzDoHenktW+pGsxRhc99 -----END RSA PRIVATE KEY----- @@ -14181,62 +15681,62 @@ vector: segmentSize: 131072 total: 255 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:MDMF:jw3kbrabv5oefmhvjlnii53v6u:b2432ea4hpbslu5dy6yy4ibnrvvol27p226qgm7p2udhx2bxyiwa + expected: URI:MDMF:bhlcqlyropnz2rikjv4f6fcaua:tfd4e5zhktjok5wdltqic35h4yya4ohubpnymwu6w74qxopby7wa format: kind: ssk params: format: mdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEoQIBAAKCAQEA0riPyzItZ74hqew/pA5c/1cgr2LIU6nzdT4jRLzgFsLqm5ZP + MIIEowIBAAKCAQEAtt5BBPkmlQ3RzinuPNFTEEcVwo4ApkrWk+zeeAv45dR3LNpu - BHVnB5qj8NhU8VkMI3U9ERQe3q6OKYka8BCCB7rpuva5X26i7Cs59Nmpn3VL2UcK + um2wFXqHUc3t4dukG8bV7n2zIbvfJ2rjWzCqBXAPgzHqm+ks4fjtsazKTu00vU8G - DCSdpn7RrHqaxZRx7ERbc2T9hEWi7fAAaUhAF+2+cYocBedEcgp3/waJtmsFyp/b + W2+zyhDvFiiNJpXl9C4E4Pb6emD5vAtUyc43v1ubBgmPUB2gu7ZBhZHdQY2jL1aF - HAYZxLzjCDO6JDsclkh/Xvj3BoBemlKqET65QZl8RPGUn3/jRlRkR3uZBqzSX+iI + EQIbtV1tzjx4eV5MpLiv/NnaEz6WxRdd0nJ2kfecxgvTZWdXxn5HnKEgPqmF4Bw8 - BGIXiKSAnirI+ec+UbbiSjdyJATWG+Trn+6oiZuF1E1Vb4AjSns+MHPY0sA2fsJk + +6sn45Og1yh84TzqYAzemdi2eFozKLztgUH2803XdPzrf2dJ6a2QUOOxYtQCiTdj - L57IpqeXJaRC9Kc4ofBfORC/wFjT3AAXobnluQIDAQABAoIBABJkBLq7e55/B9Gc + 8EM46IWvl761+XEnHxyjvNCYUwVSRWO2QYTrrQIDAQABAoIBAAdk8UgcMv0v893c - epZvIXswh7v+31R78/FS1cGpSVZ7Nv4SwX02YOJXQwEhZFJ3DtnmYMjFjIcrTWF/ + QC/hXvR3i1+0kj4nJIoSt+Qux7+zWaZMptGPAeG8dKBQLWBGm1osLhZYqtegWyOi - I5B5pFuX2s/UOiwDzCjYAfQmbgkqc874Bf620GKIVXTb83eUg9fWxHN/CCg76qMh + 5NKZIybZIydw60WmphP1FtdqXzvVx54oBd/IooJ3MNO6jAqVYRkAi131Xqd4KGD8 - C+wkX+GmwHUI1HbIbx8T3lKt56V5q46pshFGW3fEw06Z7waW+qFAYSOvbaTlcayL + LE+EsShhseEKnerlZ8xvUDLwdPvjhsZZ+ia7u4mHTr3JGHCVwotiIjKt6OZUvmR+ - JenivLYnlL0uQ8rq3dJLlaKj2yXv2WXv1GL2fxcdI00/twBcsU7CXf5GC0q8MFyQ + CJ9G7QYbScTp2MgEfDYZwq7PnzapZHSuaIE+bLlZs8AO6IhVBfJLORTSxudU9SJu - ent/R5yyDl4wpN/WWnI6fe+OUI1YydooHz0/9xgdJpYEL01rmz1wP1ptt8SHNB+Z + 2E/t10ubaI5j2BvOcv7WIca0SrWGmb0q54n0UUog5Qdc7vNnI9GYkVygAYLdixkQ - ZBPvBR0CgYEA1+jN4N4r+Ke6SWnNARI1LwnB3e228wlEht/b/PynImhz9zv57VAb + IzuAtVkCgYEA0iyiXyNtrUloxrEKrDRjSG+Bvh94uUxm7MW4uZoIWvj5h251Wr/O - NmGw3GTpQS0vd/FnI6dY96dvtBQzDzMUqGIaQphs5f0phGLQ9bmLJCJEzefenCup + s/WIEEwgS+Qr+CUWBjzgRRlrRzHB13B3+qcgkOZsJ2UKsikcnZLUQKetFK3MaTjR - s+fsMJYW60zJUncZ6Nea24iJtN5eew3vF565NEKYGK5E1p0xloepDRUCgYEA+dkg + 44/EzlOrY+2gc4nmf0Ihxh1Sn3IB50x2xVW9Aa0SoiabPYBanoIbHLMCgYEA3r12 - Qjwh3UF6GTGs3Ot8q44ovCMetbSce0WQ09D37L38/E/8V9MKGTWhk2Zkp9FMmQAo + d2SbXFTh4rWmqq6lUP54rQEcxfInxnkMnSNmBXjj0bM0i2auc0O601724H+HGybA - VUdBXA0DacarPFGbIpZvu3Ln2Wx9FCAu75Q/XvFoPPEv+37LiQQZ+7tFTWiLL1Bg + KHtF9lNLNuRSsFVgcQtPSUCsJooliGY4rlXHoOcXkVYf2ElthYbRUgqLcEC3NDZR - 6JRZnvIWSeJEDfx/ahZ+xwjRjdPmglb2tfIPRxUCfwKHt+HquJkxXf1+P+jDTdw/ + Z5eKC/bYuS3bHFAJVYsflr1/J+/WqtVWu9DOxh8CgYEAz3leRFqd710zQEkOxxXk - QQZYwswWT7dE6E8OpubAUpuTGFqvlaINgwUSKamZ3fSJ36uLSn+cdrKlifOjpZpT + GGJzCnLY4trIE93PT/D9ZIi5Evd4g8Aq1b2Ats3fZ+tzeD9r8XZw0eWY4Cv/NaSB - i/s7zgrj7Jigj9JRWlASFrxS+0jZOiPhk+L930bin3lX6/XOkQIBl5uG/5RxlVux + 2/7ViBTfGTiGiX9KD0cdnkGn+2ziB9EeaOzIlAFGhJvUM5oi3ucynfbeVCXgOStj - gHocTav8XtIlBW++Jz0CgYEA3cJLMJ7gy9qG/f+qV7eoQzj9jOd7JXp2fa+kOW07 + Z8QOk7P9W/KOdvTY//Zhuz8CgYBWx1/pQiQZQ+TBi94EL4iu1oWzeXR5Vk/SzoRw - 8OQ8vNJdvrHxP6jrjcIPSyipXQ/XvMFvEL34LpWIfRRNpuhxqaX2hXQWnJtoLXue + kEMGLMQthfEZwoaC18do5F2wt16u4FkLLIPkZS0vlKL2mjy5rhtUwcKQPVBEJPc+ - t617gMPue8Hx894xFc8FVwyYpVkpeqXZ2gszn0Z2cxePG+F1i0GXhdPzv/JiLeH4 + TKM69+3BrNk5TdpCpHTWzs7mjAAUcnkir/KTmLd05f2wuSn5zvseonNw3ss2wWlK - j3UCgYA/7sJn1Mf8FdIm4ufxQajMASUgYQ1+vtC5LlD+95lM6nNUfBziX1g51YN3 + QR7eJwKBgCzDobkvlnchBuDD+TPuhU2IiNLkLTCoLB8Ucifp1+Te7gB6nVwWPkqG - ZJEfm+tpL+vbwCIsCl+GSIsyZKf27nuatJkbdyssLdyxYmy/McT+LF1pbfgrtuye + FPafuOHUkE5HbZnQJr9YKbKdaRJRF4uq7Bm0pgvYiZrrLalAzuaR4XlxkHHGhQcL - xAUZtFLdwZoG/wJd9DLImkMlsJHGP5HEKq+fuh2fl5Rdra9cVQ== + +/IKsYJQc/APe9+VNpT85NTulKtBx+1tLmSAq22opfBAfFbu0JrV -----END RSA PRIVATE KEY----- @@ -14262,62 +15762,62 @@ vector: segmentSize: 131072 total: 256 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:SSK:vl335scvu2r23lujmtoojgpxxm:llq67pxqynexgqco2d4yfhtikmyxoqzd77jlykvr2oq4yhkiknlq + expected: URI:SSK:mz4aqmo7whthklijqffcfixkoi:bghtfcymccbxrij7odre2xmjc4boyyzxxqy6r5rjgo35k4xmpi6q format: kind: ssk params: format: sdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEpAIBAAKCAQEAwupALUN9JNh1i0NmSU2d4jqgDdhlaSUHLjXypU9Q4zUVXWKy + MIIEpAIBAAKCAQEAqJwmyXuOk1WqyoiVygVY90sEyxvUgibDOxmwpbkztfTykTVo - EvREz44AWplN5HI33XUsEWlEsSOgyBshqt6Ja09L0z9aoyWgW35HYbZO1BqEtbUW + pD71+dXfqPuHZoPX2LCLZaHdXdQdxV/zmsL9D/DFmZs+DeIL9DEEzgo9ehOKThpo - woGv7OY/FJIouFxCi6KbqoHTu8jywECV3iS9lHya5PaLcapEzpjJnVmNA9g1DU3e + /2NTxdFhIe1f7oWsZBStDcIvgZkVeqNryHpt7i+jSt7j/LR3MyLNQ4hVdAoSB7Gq - 1mJCJP4IPb2o26Reuqq1F1G8vkgQFnF/0absKzUc3zQU/4p8F2jwpGr54Zggg/Dh + CzvpclFr5BGs9icArp1xcLu1BM8+Rjk0WVmRbVqUdW3S4ksagQl+X78betPl2fb/ - AtwkPBFVOZJNYb1tUFpYT/+axpYn3syMce5wwzIkfaziwoLXEbO1mVLkl91waXhC + QiOWPeNhKvHD978C+iwEX/mrTvTL3p7a8kBrxO9NIt+t9aEjf+pSGxUZi5YCOl4i - L8H9U+1f8s1qR596VmeU8nQDUOgKXYw1wggX1wIDAQABAoIBABb9bkhod3BLH8In + kREf39CLrgCeTXdvnYJEQHVSAM4RaPX46Ce17QIDAQABAoIBAARrg4DIomYuWrnQ - Vv86am7un0ZCyeNW/LvUnSQmcNH7xuNW6s4VhbA9fYkyH9/cIP67/VCoa/PA0gwI + dszC8yAYcVm5swpuZbPI6p6NilN8xlcUJVgY5m3UM3bEkToYvrHJfv39DkaFZvpj - NzZiPS8tETJ/fH9Vxs5D3MOHr1CROCn+jAqxJUD5/2K7wpXMPAUgTuATpBe4IfnP + l4k5D1U5pJRwQ2ItyM5v8oZMMmxe0sNVYec//VQ0Nu2iwV8JVgmRmS/BJWmqT6vV - JF4pUzsaX2K2OchUXv1HRDCNCXb0alvDjTpH3HQfwQ1a2F6y2d/RONEFGsTjaepC + WN/6haM20HsH+MYJHQ7UHLlme4b9KZzBv87rD/UOfOU+oB6V3ydvMoxdQggm02UC - PKP+R0vuL+FNuOVEpV5SXhdyuUfzzb/rnSq3BCPWsrvSkPVTeyHfz0DjA0k5k9vs + b2f/Mz9k2e1Vn5C/79Q+V3PjPCp30QZNX1MQB8bIAXhMhXK6Dm5MeMcTpJPHsKkJ - mprc7eQYZCoB0bJQUtSgt0b1w6jG3M7gksj4WbR9Esj4ychDvwUSYBgQaD01MSXC + GW10zDfnn9+OsaHc9exFt2otx5l+55L0/xxoIx7qX9/gkQaIf+EsYmvBhQsgsgXi - H8cBK0ECgYEAxGtgUKYIhxpU23ntk/EPr0Jq9IJ8RiJx+/yHXcAwd+RwRO47MXEv + 9z0WnsECgYEA6+q0Da/bPCjwWQddvEAlGhmJDR7e+UWn0PLFPtzmIvCbv2FCbEtj - Ut/2ipB3xRSPJgigfopW0bNFOFubPWEttoDbcZLRXq+2btRv3eOWxjmu3V2re1m4 + MpPCoGoChco91gRnr51OpeA+eoiZ08FW+czbN6Z0L5wnJCIAMO42HarZzGKstCAX - PigcCBF+pPB+0K3+KJC3b1KKJ3NZq0K07cuoVXStHbe0xz9b3Fxe4HECgYEA/goN + iB+HsETXwkSSdqbAU76KDMsRI6mjQFVXxIKns5iCvTCvbC3X094taUUCgYEAtvak - lJ6HvW02WZHO9v42WOC7dYdTDX5KB7neP0U6d+mXvlM2aj8FbmaMjE1IbVf2C4Nq + Hspo5BrSVCsnXoARI3tlwRHRDTHDqFvSquqg1+XunerPxjLYUpdr3ez91+UKXZmZ - cgZ1EP/LbnOuLXsEJA3bud/hEJCWCHkomnAnjeO3VI7Ab9EcjxP/icYwOqsPwFQr + M5CpmdGbU6oX5smz29MtxVdo7LwAjRrUiXLe167R4p+hXUpF/PGTQZd6K5OHtQWs - jlfd48CmV1yxUfX2bcgZ4r9EZyrv45MvTgiRoMcCgYEAwhsO0oRR9xl5dG947fOS + ZpqcmjnHCYHEDL5g35r2o7VeDDkP3vXUXaem4IkCgYEAubUemSuWU4wSbrKaueZw - uXLceYedwj12ATyclXRBMaopnKUFECY3SyIS5PpBshxQHpj16jdR9ue/ZyN4NV8c + jlQNi3OCqAyJ5rRESpDO5DAtGgCwrdjGNHkWGvp4E+M4u/Dpwdb9oxubcw92r7ch - qreKpSEfEQB6O+pqJpArxvboLNDNjxep8Jr4oKyRR/R2jkjz2yiwbi3WY4glvA4u + BTCaW/s+uH+eXBYbumi51q64FeiS9JPSkkfnovz+LqGV/aqT+RgjSaDMVBtkM+86 - +LSDRPgJwE10NFcTs6ABDrECgYEAyhXpM77gsFm+kIYjI+yaAx3TQe1Crks2TOsY + UKlc48YpHE5nuKt5mwDpFFUCgYAiqCmCY5jmzGXW5623as7UR8WIgtV0iF6lf28y - 1zAVEOrr4WWEtgQoJ+jACaQ453K/sez6snZcjgdOJzEy788aPiwgDL8B5RF/qIHp + TOtWugkvBJGC25K6YlBeY0vaH1qNTFEGwXo+1sNzX57TapWVKDVdUidf4GTCVbi4 - QOHTNVZeso9Umh65H0CDWXAlUaZew1qxw2w2gUTxjjGYhWCqhi5WGUCaA4/ugRTG + qHepp0W+hbpNL4p+VUwteoH7yyBDm+WCMftEA3m+RURbnZw//tyFOg+shQqKk0o8 - 3saGQmUCgYAGup3rh1KO5kxEYpTIi13Fg72SPaApsG8AMc1jUncfuX6BfpymuDqC + y0sC8QKBgQDTLI8FBom/UNpBTOwISJzNIQbDv6ws1iidoMXhlnB9G/Q+7DHDI14J - o5Z2CRc8aruyyUsZ9WTD73h7A+R6WpUOd0Yz0kzV4SFhpmLvqpEFf/QoDncIFDL0 + ByAeiAPOSav0ogNxoMOdHu6KXXMACriTuFa8s/h4Sno4vXETLqJrukjs265TQXZU - 8NPDU8gzMEl+uTHt22+U9kI7Gvv0mF5bXgjESZXM8BVqdVtsi3QK3w== + 1EB0u+QnZVYXUmEGIfjr8fhlOfEkUGh8DhEW+U2QyM+wlXChr3rr2w== -----END RSA PRIVATE KEY----- @@ -14331,62 +15831,62 @@ vector: segmentSize: 131072 total: 255 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:MDMF:ctfka7yb44bh6lhv5wnpigqyne:hruni3sq537s3lkckc76w7xaxj432tyvlzd7jgpzncqvpzbqlukq + expected: URI:MDMF:far7xpcwexy5c6rsnh2utklpfy:fun6nzy7kxfr2zsrxmkwy3rslurixuq5dgsvzw5tg7cl7mnffzma format: kind: ssk params: format: mdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEowIBAAKCAQEA05yQCFDrBVQeigp0LeWhVUJHfMzJyukKpIKHsnOzqTuME/Wo + MIIEowIBAAKCAQEAyK1QDzzJ+fA4Qk2K9m3DVSqHCiWtCNPWu4JQtn4Rz3ligo3C - jAc4OBPGR5TUaNPolhsb6KJnYWnaOh+S28IF0tWVtNU8tonKfytImkdC42ogGeDO + 1Duq335a1RVYMayuOma1miTKAS1rp01Xi37q/o5cMEL9Y9aRrehvomYoezDQKfP9 - G4QM1opk5eouiXGFz89EEobdkrw5hSMXOTFjQaVkcZjOeXP3fBKQ2G3zTixXmT+m + pO4GQJ+WHqFCDeCj2Y2/d8fCOzmc0A94yNAJNf7+Y1ymqLefS2aoL15ElG6amQT8 - OLXM7RxFmNnx0ar0I0qMfCE9zlbnBV2PLQrJ4/dA9ictoPFgibKhLEm985qZutkR + rHKi667pK5rcHA4q0GK5osaJRQd6TR7jmzcZeNh4fq0HkNb+8fCYq2ZXOwtUM9Hr - Hwlfc2R3nPPD3hSZ7mn9lZINJn7w/FkOClX+m2XUj8b3FZ+YxodA5N01egU6QXQI + U9E23/zpqZCvHNqXVv9p3H+psPW1ZaN9yIA8RCtGuWgkt7oRJ3K0ovzhe+oNOq9A - H9vznaU9rD+lRw0TiiiF6A0KeKlQ+Jn4xynl2QIDAQABAoIBACSi9hLig5YkFrd6 + zaMFMTTvjURAvLC6FloZ2atK7AKgm70fIBngVwIDAQABAoIBABSpP2pqNFNe57Pv - kNvDZne+5maWhBdj2opZ6Ql926ygmSN5hCleNJ8M2WbaPx45Fgsq/V19BJ4KeBRZ + 3uRTVb6hg8jIK0IS6XNhzeSUI0pMsZdGeC44vHWJQVnZ+jwXDtMlewIVUpT/c6uE - FBGFGYIDpYwt4PmPiKYUxdikHtIFtTIVyEleRuS8CDUAIvd71pmAfn4gqGr3uJOy + e4R5u0EdMCGp7API9jPFECVUxks0seH35IAEH7GsnJyntrDOFaCTBwkSkI1fd6U2 - 3Bn0UYVzj5zVQmYnrEDoa/h0rMurNBHTeiXaSecPdz7rgVWyyHg6dxsjf6iOthNF + SpXGUYR1LgTV57TMPwLY3W0PFBmCAQF3TU0RQFK36cOB+ufGEKds2U3xJZWQV9n6 - 2Wc97iorM+uRqwewXy4rKjW5zL+MnHr0NUDZW/vlUfOdE2VzUV4niz5mYfYL2PzH + gVt6Izd8D8s8bbPU1bUH0Q4TZvsl9D9yabAkSmh/3a2mta3l2Po0rhtjfNXTTib6 - fBI7vzxlu1WyIybp2gKMOUS1UbS9me/duJla2wtGLYC1gMUh6OnHFOMk7OSUrsX6 + yZU5IU/aYO2fWBWXK4GzqjE1daWuc9zXQQQeRRh5OVU+LVJQxAbjEsTlA+wqOj7J - uZfWxNsCgYEA7Re5mTG2MnC0uv04DMUrUlKADoaA3O3kV3GP0Ql5TLXY4XWr/vmK + aCgYIdUCgYEA7Rxx7iM2PBjPm6xKApLNWAyk+V/nd/ccnJoWx4uZJeiZrp257wlx - sW4Ygo8othOxGvwcGaDTtOYCykxhYjap8H2ZCM+JFvnit1jrL7iU4VJ9shQRrWZT + XBnSQAFE0HHvQ32RROnF/dHaBC+98GUH7RQLCYcISnsphya2PdOZotkQRXG/GB0p - de01aC81n6HoZPPc5TEh+nGdcM76dZdmwIaW4GCkBIWNGoWptf6Pd7sCgYEA5Hyj + 3pt+YJAHygnSiKxtyWY1Q+6PzA8WA6crkypq1NYrc9+4x/MHmWKm1u0CgYEA2KnY - TZqLUwYus2cUbNKOfyYWky09UCsV6w4xbdaxJUa5u68mE3hLN7syr82JHwm5AjZn + GO/ESa/dGlaY0z9Y5iIiwetnD/zJGkb76wsQfB+vjfqqGgSFisudzW3O2mdDz7Sd - ItseFNYRzLMzeV22hP4LkxeyBaF/RBq0NXuZh68dDHbehfsJvOHSAMI7IbGMFSG8 + cjTtvdplDpVLx6r10jqyWU/2HvJpLVfpmu+x5jp/RQpqNnkXLyfrTX8K6ZvcKLX5 - zfZUtLonqbFUoWVK4R3akxoZ+fJOu3t3jWrWrXsCgYEAvr4oliPVVe0wqYMQtc1m + dja4z6Fee5c9VhY3/PRHE2Ovdi4uoLdKjYSHR9MCgYEAj/WLPphmX0p5Ef0i2jkj - lftDhOwW/ibxXpxBPMZnbRybmH9n2WD/gNF3LIpqEVn0USZkoQWvbMjjk8cxTad2 + L2hN6ZJOyMlht7reRbz9+MQmOpxMvVKwXsjWnEGo9B2YtRNR1dNRgG+evJf37DKL - vsD8/oag3vg4upLx21mfhUstTrgwpJU/Lg+huOjKNlw2sAk1PLpzgJ4pMNmDzFj6 + A2f944T2hbINXp8kWplUWEkN1fvfl9ZtC1jA/AO2lvYruwtlhLfncx0udShbp1Ah - 1IczGN8G9ZBQPfcs2vsqhwMCgYAeay1+hmWoDvmmrsF8X2fTK6nzvCEejC3l1kTk + 5rIENsDplOqqF8v4OypoPWkCgYB1jBA11z+DSuqGM51OXvv4P2TkGLcdsWPZ4dEj - X6HD2a+eegnyq6Av8j8kQpPPywaTcdS3Qj61/W3vN6hRrxU+jWfTFGOB9mcwFPIK + QCl9biNswCYxX2qkVrwSjBTB4Wyk77TMFXM2oZpaQx2OAm7D1ByW4A8D0zjE5QFU - 8MKW2sxePXEQm0RHnjTMHw+qQ63nnk85iGLskJ/5Kn+e4RJf+A6CaQYuTYEH2r8m + kd7OrcYGyxO84g12BA5hSR++hlT3sWLag+3YmBAOtYsNfZh6oH0/Q8IaOAwMHeVQ - 16NvAwKBgCRgNzvhRnzbZe5qYjkACHhej4tS/2yIHE2kKhRUedQuvt9O6jjYyKj2 + yiorCQKBgGFHKWecHohrZozSwg7DJoaQdWw6h9RWMA/oOXM+80CYdbbP7GksWV1o - 22Vo7FkwJO7AvGp5cxf0jREGQL/TPQJpPuKHohiTzHNCmEaNX8y4NrGy5upXFYdi + Ay9q1NuZGfe2z3xXYP3F2wGjIWehqGsYc53zXu3mGGYhvQJzxyRCwAYp1Vsn8R5s - AXYQYyHDC2U/RGCBtNeiuC5bXKb2eTx7a5e7Wje/aeQwGhBqIl7I + XWsmcm5jw+o0N5vCjq5vDqlpjrnib9Gdbz8/8hK6wXZlUWyeudSC -----END RSA PRIVATE KEY----- @@ -14412,62 +15912,62 @@ vector: segmentSize: 131072 total: 256 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:SSK:7fbakosfeg63ckbiuhgz2gbujy:zgs7fojm4gu37syyzdzcdfigwf74qkfz5z5oz3lxaqy7dyc7tz2a + expected: URI:SSK:vj2bo3rs7lkwz5lla46j5mfd7a:zfqfc377sixrjzajsk6kf27ouoz5fpdgszpg37k35q3ccoie2mxq format: kind: ssk params: format: sdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEowIBAAKCAQEAld5wn4Q8j2GiySAr3mW/Xl+94oWndB4kV5VCNzB0bamvqWmy + MIIEpQIBAAKCAQEA1Ct7KgFNZeSkYGbFtOwIHbMXiIr9zYZLKsHNkg15XGBnG/ii - yv9KzLsEj3IDYQlKk2iGG6+QGjCG78rz0kfaE2YwzRwVYA5cchSzy6+ASqNRiVkw + 13p6Sv1XuqugHcTCk+xXjWL1GjB4xxD4Bxo3Vz30T6EpDVd+6FNXB2YO97NJVrn1 - v5YDszsTchZrIv7Ni2PQzafnGPaKzhqWqLPF4DJlSXK3rU0idXjqm86pL9u2cavQ + DNUYgQ0UihnEo2WThNGcRRmthyYBPYGll7TgbgNMdhazOtX124mJM1qA+OlRm6dJ - sEkm43UsZuDenTS0MZvXfIgqTqrtaqxp4qbhGBSTTBUyBQx8VXC/zH+NfqPWXGyZ + fF7rAvfYUfXSmKKk1pWxhX0Fk2PhMGQi7HSMR42dgy9ZmpM6I5v+CRflPI9Q1Lwl - AQ9jW/FqDFbjG5348Q16Au78WSSzLMKtgviguJonQULt3okwzgo994jt46gcCqRP + gDjpZA/nTRhiALhHMEHmOC3aCSznM7Glf56MOvuvP6L+QUnK4DYDs2Bv4oVy1EVY - SJE+TeIkdz4oq9MQ683Z7AqQhyN/5L4n/l5ShQIDAQABAoIBABAkdHcKFEfRWWpW + dZFQ+2nh/OWjW9igPOhYzq/y3e0XCvV3rGXWiQIDAQABAoIBAAG8aHOS3dIkReMk - d8MtrG4q29YRVVcRhBKW9hnhszi4pT4XL3XkB5eDsVsOCcUi7hBwmrlSsK/ReEdN + 7QNqnaV+yPeel/V4iWDo2QiiAHqlrmQSMroBEBhtbHNyVHzUiGsIiR+96mzQNZr+ - 0fNdX+TlBe6hzr+Y7GYxSqhuz9+6NacYn0KTkvR0MYUBWyrazSLtbmkoY6DxtUO7 + 6v3wVE6zGRIyKrTuy3LOS8Jw2VUrSF4cn9eNCXhKLRu8KOfxytGl1IRicZ95rwTq - 42xqaK7cXsKJg7U78LE8g/CiUuDfnEnqWe2LxEgtfaMEEJgvyp0FutW6dJebtlxW + CRdDHvBOm33FoxwAy7e8y65dcMLt7mmzY3V7eaUruJx8fRob3dA7POIml10NbM2z - heQqpZr+0aKGgXrUMwIhoIJy6Wv4DiBjnuAwWavRpMPCvFAaJMSGHf/TTQR5a3+X + 1TM65uqrVQgihMlQg3cArL95K/R84FZs23V1+EaobpC7gpvFn9J13pgXhT/pBP7s - a/yZCGQ71TV0vkz9CNnxwX1XauJWkKSzs+zCbugpI4MDR9jL6FWuj48KCc/9Lk0e + V+9tXPmITVDaaoO5Bw4Z5EqCQCEEzqeN6v0agl4b57k2eoE/mE8p+sZ2gyPUH0Wj - Lvtlp+ECgYEAzV6CHuRp4ISmqhlhZLRouaOeeGMKpDgh+4RPBILCQ5UfpSEivsyE + llFujMkCgYEA4X7MtD80nEA+dg6uRhMnN9eLVmVLHXpQpuMdcdg/5EXPUCOUz4Xk - eI6ng7yvz0TPRg6Nbmkh8VttnlNK9PXfv0FDYprlzYB+h2ZG98tI7qdK9C+LSbnz + I3N3Mx3EVWOc2vw8gvY3ZbS7vpMEEzCzyoYbEPa4fQ24UQ4aI1qL8gFuHsA5ZnUI - vUa6IJZ4ez2g3xqrWr2eLjFsXhyND+ZOu4E/g0iMM3S6AJH/0Kts1O0CgYEAutEi + FcvnTyZsJgoT8rRTm3+lsqmkzAhcHVMH8W/Q3uJhtCtJGNEzo2yyiPcCgYEA8N8z - z0i/CnYY+CXoFK64qAVBX/1WauwzpiOMpFfb2OQ+DjbWTsL4J4XGWKw130SxWJ7A + /s8gU8HA6hFQw2drjJCglJnrzK8jGq26bnnmLykXPnTuOwQLH2gKY4PRqPEOtgvp - 6T/dtgfK9t1LUfTprSs+A5t8cPeIG5Bp9HTiUq3TAm0lIubGV+US/cB7muBKgv4/ + yEMXSGXktXxMNvdfY7FNDZvmdBBVR/wAPFbz2yZgmdAQdZHpmRAJPW4kJlXSTmg8 - ppdET3InhnLQ6CwxtyrXQpki3tRvJPAn1achGPkCgYBTchRDAyJ2JNAni3qEVb27 + 6QgW8bsYV4RNL5f94IOk7QinchiF3jGaK9GWPH8CgYEAmWNLlAC6pN7+ngf2fCxj - uFzao7ueMGS2cvM8bPkMRtp92THp/uXQqn4sTA3PlTD3UVBsTXGKRVEMJOHvGLya + LRUt7yMQKYkee6daTCqxq3HhR74sZ83IFmVg3CCPgRY1iLCz6NHbdQ+v9j7DMtqa - VKVRuoincI946rjpVINE3VraTzs0cMc14Dgep6U6xjbIkGiRzTwpntFeiBFVJYpW + MlVu6+coL8i5bEmPdiUNtR1L7xcK9Kr/SPRe7/RO9ME+OIZ5qPj3mcTUGQZGwpvM - K9UnveGhwssVEj20hwMInQKBgH0/bVkPapV26/KiZ2BGa6KqM1RJosB4r3/5YXdl + d2t8RWDw3UHkg0ErQyuZdpkCgYEAzKtsG/zVpDXDfWCfNpp/GV6fBAXSBgdfFcE+ - OA3HqBsbhL61VG4a8AnPGycfBM9nT+qRWPGLc/XiE3dU/b2Nujvs6JdMPUJNpduw + 47ayr6oDtS9Yak8iQFqAUVTl5t6FuIxg5qiTdRIXh1qJzD7gD+7M4V5yMHbccCEh - 6XOI+mksB7PIiL2w5PSfMb96FDqSftYPoEqrO/iVzZ1607H71OnfhVNjlUhsgihp + 9iOQa8utU6UnBy+nxUaKA2e+UdCktbj+4KfeDyMCKQMjLujAcXCKyFqNJXbO8SFG - rnTxAoGBAJXyhoFN1dtbc5lulsertBsbslrksINDM5qjiqrLsMl0L+0HPieysfOl + tisNtHMCgYEAiaub4VuVuJGuJKYSdlSL3G6NGZaXYqkeyx3AiBZw+qBoECNIjVmr - qLJU5sV6jOeAYu9z2PtgUpdppg9/il/5xkCgYjFsG443TJrKZ8vO++dTLz/i7sq0 + afsUnv5ocfpBc+7zQGb/hmEpAeqxMwza0oMfJSt68kLm0ljL/kUxpRR6MSv1yPMQ - bDljZVkezRSHQQgIdRuJ/wQlDWUk1tQ5dPGLRAWyF4OfnjOBMht1 + DaI4cX1tEURu5Wqs+iXtKiASxFNCXcD+tSL6937wef2BDte/QFWEyus= -----END RSA PRIVATE KEY----- @@ -14481,62 +15981,62 @@ vector: segmentSize: 131072 total: 255 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:MDMF:66alqxuv5jdgccbh4r7petivoe:qkkfvjzii5pg7urohmys26hwgxxz7kxjrk5n6yrgepmxhl5wxxaq + expected: URI:MDMF:avc56kylzakxlqqhpo52hd73oe:c7x52qxnbsjgdc5fgxlrc2as3rie6x6ro266qnu62dbsncn3l24a format: kind: ssk params: format: mdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEowIBAAKCAQEApNQTfBcsEhSdsh9LXtHm+mZu9aA0TaKl42P9Hy493i03qHGr + MIIEowIBAAKCAQEAn2qG7SUv6kD+8MZ13j8G9Us+w8B3/j1QC4SdgV7qoQ8ziAFx - agkcSqVEF9TF96F90RJ0/dWNpFkGfZTVIR3bKjDQb7uM4rWOXQaPqPj0xaTi646j + mJe440QDfjFQNhPtoa17RRzzBeZfaMu83oBscl9oqBVf+WuxQJlHNh8zVNPZEBtw - httNayNarHQvLP4ZZ9oTvCjavrW42aDh4VLiTXxS5SIZ9F7tRVA1KEz7CckN59Qq + VkA82X7RHzNjH97YU3DV/DRypdMPCmuIUW++TI+t2c+W7KsKPAOJx6rrJHK+3RgC - 5kai1Y9ZHyuasGW48rTfYRQYefak3jeeuGIL96Mq7qJqttuOO6KcWu8QMA42GxOT + n2VuyQHhZkLqqNc3ZexwoKFGMHIB9XtHSCSAMakD7o7ITcqp9W55xMeLPWLtEvnr - wDjatLT1eCkr9yQfQjv9ziIKPS7t6qXdi8yP05vFMFRN1EgiSxh22ruxYc0Q7/u6 + 0k2uSWjJ4AmKNf1NKLQ5eLcIzBi04SiOAUm3p1gtW+qPfIChbV72RB5ZWL1miiaS - NOnDpMxjgUkVAg+Xj0TU3T2RQe4LDFHmVPaF+QIDAQABAoIBAEMd0ClRTjK2jlf/ + XpE+F424wm4pZNf4EwvRKKcX8QEcBMH/m2sVywIDAQABAoIBABv/E9oS9VLA/mTf - gjNECWeg2kHOUD3kouPqzSErNSoJA4blckUlHI4QqZ+ClnH1IkRF3bmWgayQS6JL + nbSdwgWTJN8w7oHWV7fmHtkpB7CoYEbq5f3D64LyH2DqnSkaH9oMgwEUv/NRzYC0 - PlXT0HBnnBhDKGUQRL4Ac/L8HL92GqiMVm4NUoLzzHI4hRUvCq1NEYgmopvRZ0nG + gyNaT3FYoyMdueCuUo9DO/fby+KCX/UNtJFZL7aMqII/vpFKzBf/UX084sOPiO0u - xvN3Sor+uspujl8BYGA+/sZAQmCDA7AOr0G9ZMVNDSDt0cUiw6z4APbo3t9MYG+I + DF8s7jE47HG7nMhMk3wNrQVFVY1tWGlL1VUgt4ICm8Zz4+o+F+P7/Zp8fZB9iGNj - 8OHj/nhd4cN9qvV7/CUZK521BlcK3R9XbRamDNeSkD3yi0Zw+dL7+0Mg4NWZq/4c + Zq9z1q5VMDj2wdNEt+vvLcWrvBnphmqYj2axk5VWi0LfMMVycWGtsFV7JHurNYBa - cKD64OMRgBHsqNu5ln2Lkk3moKvu1/Bv0HORLwzp9FfP/LT5h+eDnYV7GIP0vT1c + iEJa9Ew1LxZc5Vq0s6RBtNjT5rDqrlXZwZi3nyUHDls21auke/ivK4PhQWz6ThZP - /6wo7jkCgYEA4zrRNr4k/6rcOv3uUZUMfNf2cX3oolERjIVmvA1NIbZUh/KcE/Rm + /u88Z3ECgYEA0aeS+9HC2wsBW7SH87N0Be/J7K7YSiAnWRE2gyi9Lc9pcB9zsZHC - Hzw5oGt5/TKEQ8zXBSJYq3fMbnHEEcskk8MtUx51jGaDB9N3x6uUDhN+aimjHysE + NkJykJeiocrDduAm/EwjwSgP56u9ESpIh9RK4XdoSUmlYlcxmCekWNZVyTNoWIY9 - kaJGRC591mzFY6gy4fjE4+91itX6Pr+k0xIEiDU1Oplm4gIj2gUHdgsCgYEAubKm + 3wDOCeEK44bPPl5/dAWqlG3/t3BP3UddoZoosJXNNMtwb8aq8yJ1TZsCgYEAwqfw - D69KiYcLRg5oo7HDLC69JPiyucAvNqlsNkX2bMZNQQa8bgVyjymRiWLRL6dc0bnE + 0mxNAdk/GzT6yAYf8Kf+eDCYhzlnnG0v3h3Eq6wm5enuQ4TqRplem3m8bs97hYsM - 8mcge2QhYKrGGGqehK+F8b2ouz+nJQD6Qbi9kEtEhcwl+swovai+ZjscivIeTGqv + ofso6HjsmAQtm7JMM5gduBERIx2cH1uIYRhJhxgQOwq8yRTukzRGEzi+Nyfmf/jZ - uLfXG1GP2vlG7WN92WlIdni2uI4KBGSyk18HCosCgYAK9pmFhKMQWtQJXJsVAJX7 + Ui8aPwRxtx0h6g9C8LhdORVxISR7Mj1mD0fw85ECgYAnxGBl8ZjDUagVS/4JpL1a - qAfR7fs9aZ/pIb6VMCcai0uEy6XQKKiMtUEqhkT6fGd5RfbR3phcnYkVgxOssBpx + LuyfP175WHX+N/yeDkkr+k6mnOCmCt7KyfnPIWQQylQfJU9fxdV2WvIBYJsBOYL+ - rqcPLZcKUR/dTsymq5aXH0WoJZ4jMNYlmKi/PWcA43qallDuKiyFutX2/t/2CxUO + eK8nay3V0OlU6PMYSFStISKugljFidkMhquORih5leWTj/se98AuXVsG4X/UmifR - wf3J/Jc23pPiL6w/JqL3hQKBgA3V/L+AbQpQMIvYuP0xWnxpQxiFGzPx2NK2zuRA + cltLe26sF/agzQ86BQw5BwKBgDc36tHWVRYEKamvIsDhM+hRz5cKugoKF8FBHAYX - VDsIj2r/6Hw+FaoLC9fzr+hgDO9nawAwpN/stvvv3XCmSQdT2KQJYJALDxYXu424 + TbYhVLt929AdgVPbqAHUy8ZnZzPf2QqOM/GWdA8/iCyVrJYqPav8c28RtDsU/SAG - CQ++O+3IJzBHk+WFtCID132WyqEg9dTKhdF4Q0Kqfhlj51WSnZ6OIfcgRijLo+6N + Ar2m7tvA1QL5xB/QAVzsiNEeqX19+zAcGobr3NJEGl3KTIP62L8bvQbY0XXUAwKs - DwY/AoGBANXP4ot40pSCFd8xZfi6pLzABjxvNRnYUjWrx+ZxCAQkTlBH2vow4T3Y + tsZxAoGBANGJEzWLUx7m4ZLKvapAU1FPx0pqqZCyzrU02mg831Eeht9K3s7B4WBb - ILMmoCr9vTqxP2Z5DqrNbvB7FlHbhOJarwEOnJUqsfT6+/637RoO+YdU8ZMQjd0o + HMNM5wdVuN0eT+hUEmazU6Hwg8JvPMCrWVLgj9SraBgyTWLEMAbKY0TYYjaOwhdy - 9AECN7nyZnANwUd4FhIGKa0VU9oEBhNsMWbE3aoQFrlkBr8Y2AY9 + 5B4F4j+30c9o7l7jr2x3I/cZETNM0avvB+1sd8AgKk9px2oRYqEv -----END RSA PRIVATE KEY----- @@ -14549,6 +16049,156 @@ vector: required: 101 segmentSize: 131072 total: 255 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:CHK:oakc4ghiol3loekapj7br572kq:exzvg47pa2b5v6tojc4wguux4pmerisstqacowbzwtbqybsxtxcq:101:256:4194304 + format: + kind: chk + params: null + sample: + length: 4194304 + seed: 2NVWOXvtwcAFf0J/g0fDC+ypQbnCU7awZoCxcnhpkVM= + zfec: + required: 101 + segmentSize: 131072 + total: 256 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:SSK:obw4p237di4oqr2qlt6gkodahi:4f7ql2uzoornpccbjnmaucgnsiie4mhoqq7lnir423nvfqfj75oq + format: + kind: ssk + params: + format: sdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEogIBAAKCAQEAnP0nyUESoXrvtjUGHtMh6BCdCVa4SwMDktzM9p7CDXVjr1jz + + EH6I2Fq1EmhJJ1TbaXod3s4SRuzRw5OGyAaX2PIzkntzpdHKai4VEeRGJTGUAh1+ + + UjNoUHSTgoiPf+VuBfl8uHUWe1uIKPEEqPDddvzGR6iFWTfp+eQHz6w4piAeLlth + + CA8+Ti97iuoqFk7Q07aPSY/v80dhuyNL1nMwHSJWePYBODTNGc7RsUa6He/K5dK/ + + dptSZGFtHPZ8PsI+C4KqsKtGoyIcbefUQ1QlFJcxQbq8pr1FtCHZXGXzGcza8od0 + + uziCBzU0iVHyqrkCgDGcjSci+z82a+HEEtAEzQIDAQABAoIBADqIi3KXAC7USxd5 + + SrhoiW5g72RhgKJ7U7RI/mT/yaPB/rKM7EfcngJpQ7VCy+/NzGdWAFgoJplqSEXv + + NiRTjP93QvJddD/B6oJPf0yl+993TlPBkm1svHqvFKbpavPJZA33OWD/SywgczKs + + tsuUz0ZDtlxWga3D0sn5E06DzLVnkyGqDW55st/lYx4sTJ3luRd6tF+y0vz9v871 + + 6N/t/j5LcvaOjuLTkOmvoCG2cOsHQGj7rlXKIkxiHaIbFONEYuhK0+txs3XfuWsY + + +2yfqQiavT/1UyfqZu7J1GXS8HrhtBxR3a8854uHUiaDP8ncGUEI9vJ6m3uM6rMz + + UlBhP4MCgYEAtn/vyTqGtzTIfh48aUfiGFOtshZ5MRBVtcG86GIoGSK1wN5Xwoku + + Vtwmz73HPbctd8WXF32OpKPwoobM1L816sb3yoDSTLjC+nl2vXlPOjY9eVFZEaht + + jQhF5V25CR9sb2RwhnriXz0rfW/BoADvYm3lh5a7l87enIQI3vcovs8CgYEA3DcA + + U+S8zq3o+PSk9ywiS5rJ5W9mfGzVg0v5UvCNTMBbFfkkh1206Oolr7TlAKBbgM52 + + Z8XVzosNjU02ULZFA8UM/g1/t4kFf1LmQ43rZm/TO4Bm6l5sOsx2+Qr8s0V0ekON + + 6F4dex4n2WQx0ODikEQkcwGKVwOejTz+ydySyaMCgYAJwTbg66btvf2FeDpEalo0 + + cKyVG0xpCfV63JsrVKvOBCPw5jGMrWZzsBrG+d7fdp4Qi9gyojxwom6nUUs7h+jq + + 3q25/j6/aRTK7JkjMYvBkcqhZG69WeJZKnsJ8oOEcFCMd7LoDUNyUcO0VbfkxIgH + + G9ar86udRqpxdUFAIbfk0wKBgDqZDCZGxJL+pfKxLsBy5wFVRAogVZYgY8RXUBXo + + 2sCkotg6/qRipAQiYjraGOHMyeyBg/JjK1yVldqWxDBAACdbpPRpZSXSeTsDNTCe + + sBgHA59esIQG8ifHRpVIfiu5/J+YIEfH23JqeNIZHkRlwwP+jfBoZYZ3+RW+OFJA + + tnKfAoGAWrOXeeycShbIKjWS1XljZ3+Nxd7ZARX5pf7FvP+MwqVpW7zEgcG3Rc3I + + gkh1In8t7bzKhUVMLDmJIZnpMFNUHjIWAAshIQHivs6d1EnErqCHkTEofQnnIzgG + + HzUTEC2ppRiXZLg/5l+opBbah2afagIMbODMgc7KEmVfdTjkDwk= + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 4194304 + seed: 2NVWOXvtwcAFf0J/g0fDC+ypQbnCU7awZoCxcnhpkVM= + zfec: + required: 101 + segmentSize: 131072 + total: 255 +- convergence: YWFhYWFhYWFhYWFhYWFhYQ== + expected: URI:MDMF:ytzxo5777my4naxdqn6c2zwmpa:shrbjb27qpwlmcjimpuhnysfondpd54j44c73h76eudklkc2kb5a + format: + kind: ssk + params: + format: mdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEogIBAAKCAQEAwrgs9CQIUW4BZ228HTF0MJ0xGU6AhvNcNwmSJ+Of5tPj2Yx5 + + g/MNsMJYyOxxnMoODLPAgSX/J1EqU/+/RqwmvcXOw3OBmLdxnMk+VYPBZxbup3xw + + fYCygPHGR/qcQvJKQ76XvkiFaiD2iRTtX0aEG7iZYeLgcQD6PQvgMKsdX03nKmyn + + l1zWTeCpUBaR0RUJK1kwh4wO95Ma6PLAkQles39N/A4I5pBlVpN5j5olIHyG8pLW + + hvbNU1yV+5JY1OCtpoAxoKzehQx2vvKeCwlqh5gViu+FN5j1/7trA2dz7RNAEGu9 + + X5lw3tvqZ3IxhX1/4Brswvs9BYvFfPyFzb8sVwIDAQABAoIBAAZJSlCHK9glufQF + + OEWTAO/sj83fa9vUDRBM85Y1qPz299U7zdTH6spNljfFzT4JzVS+XSduOvYPV9Nr + + H1lgYv07A2TFilznl8p2FaFzRdaEPzpSlKSOUyvT0BonN5at/svtz8eZqDNFkwSP + + gVPBCGc+IlvTAegYng3WDYdUtJxDpUnO+BDg2PoZ6lzRlzyNbxiPBA7221yyBk6U + + b4WZUtHl5mfCcfMby7vaWAZtwtFiYXDeu7bOZwBtYyjpI1T5l4yUVw2dF97eAsWd + + wwbSYbVs3bspN66VFfaZQJrZEeX2gYDizQo0iROySC7hcyXQrNmnufrsgGIwhrXl + + AKch9mECgYEA+sUKHB1nR+Zl7QzB8iS+aIK/XUgCYVavkg7eTreUzVe8/CPAEWGW + + yQnvDlWKWSnge+LtEHEWYmNedUhMW8zGx+b8/3aGMFr9VeWDrFKGe5yNnGms5pkm + + RJrpJxC1CXzullHoIQ9Gw4XQStaSqK/ShJowh9q00QznV0owbKMXW7cCgYEAxsfc + + eKvlwgpyddL3WNSXhggqOjwd8cBIku8GoVyfzrgsPYHPoZSUZCO8iZ/Gg04tbEAe + + oRLLYQ/5i5kcbFD5tbWji895Lwds87Gw5TAIAc7f4Za339Vqpl6dqOOe2pi/q7F2 + + LjLeaOfWPFsnCg0qZ3IzADiZ0cY3KqQMh/td9GECgYAaeYj6tOP9hEaIg0tKjDSK + + Bhu79mlB64v3qJgxyVHtZ/Ds0b1qWFo5+VGCuuczSKeJjMiobrgFRSZozWw6WOE3 + + o5xcQCAkpMaQNf3zyHaoQDv3InT9l3eh0JUC6dGjIcxylE0kiF9ZLxxxejvbkUxx + + cXHkNePXGjymS4/XOFSz+wKBgDVvKkvh4Xw8tLIJiOX/F9A2x6sp197RknC1AjJE + + JM074uCR0Y+c0hrtJFRWd9V6IWm0/sbLt5Ia6jjlaqePSODYt+LwXaIPu/DyNhwV + + wkFCLBqHGlx4ERgx3O22alBWuUddB+i5UeIfWA6XbjIcgeaW4zDPBkJGpzO2L4wq + + PQJhAoGAKGFp1XoDgr9k3y4yfUn8AsLauTsZn0JZk8lvYuZQqMxaV5F7riF6jYkQ + + SlkF8LATvS/0QUpLOuF3QiSafKlXpAodQoTG+vBkaR708U5qN6m7fFyH/S7seaKL + + m23LxcRu4bvsRAsPC29MnLdhECtTSr6SBrfqG0fY7bm2Bnpolls= + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 4194304 + seed: 2NVWOXvtwcAFf0J/g0fDC+ypQbnCU7awZoCxcnhpkVM= + zfec: + required: 101 + segmentSize: 131072 + total: 255 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== expected: URI:CHK:asmjgajvwpwvylxdcf5rm2gyd4:imwtdi4eqq3wm4bvk6npho3ta2fewceqzl4f7wyyabg5i4s6h3mq:101:256:8388607 format: @@ -14562,62 +16212,62 @@ vector: segmentSize: 131072 total: 256 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:SSK:4vpgtviu6is27ulopk2utss46a:egw2djhxc4opssnuzrystenfu6ieyww4zds5dhcmwhey3nsqcz3a + expected: URI:SSK:vyjriuuydnhdjssca562b4qkue:en4dcdgixll72bqyhfbfyzp2xoood2ti6x6djtvaritkvgg7ktcq format: kind: ssk params: format: sdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEogIBAAKCAQEAqwfCwchGfDSxmoZpPVtXwHIqlxTVVI6MjsT+LWHorlGlkB46 + MIIEpAIBAAKCAQEArKoEUAfL/7SQTQnIgVEBkZ7WYFHMKO2DHSDnHT3rn4xn6D29 - 0cELOP1/SMstSw3NP9KXo3o5R8CN2B4vm7PBbGNuOulmoo4vb3ZBeIUPNAiHM2Na + u224k2GNtOQ/WiwhGHyCXKAY9fhBReTkmMeR7NZZCTgafZhI8LlVgw4NTTZhHPxX - Vwhe1yqCfadAtmktpGQPLMMgZpftRspnKLOnPIe+TUgHZR/E7sh8uaHGyK0TZVuh + HPaLl9AtWWqPqtSl5NB9pckMoFm/E/dWOlfVmPFAV0VtcWxKis6B6maFKDxjifTD - WS/mnXPdJWs14ATz8Gfzqf1t7AfK8En0dYhDqK25YsxZNObaPgA0CoRV+ZZ/H3lc + cMu7T01bG0/WDDYZrok18CHbc9ci6YsxwheOo1O3BL9Id3ZzS7MkLjguT5pS7b/A - ZxcP0pYfJOO1wszLmdxjpXqyff5oOHEidQvzf3qpUf/B6OAna1DR9oYsGyWUkXLR + O7+ll2v2QLdgwR8hXLR/OqYdQA6HyGi38DQ0YaIMwzKgqE9US6OYCpiD8SU69dcU - myGFmKsZ2LN7jY+OjpZFz03PhbYdA1lZbPQAywIDAQABAoIBABsEJ9HqgU/UwbVS + y1IncnfkVYPivmfABsaxQNndd3OSWc6qoJnjwwIDAQABAoIBAEr6Fx5TjGuw0lqJ - GW+qwnV7fe94SAE1CfjyHzXWAHurbrFm77hQe9BYeBxsk86w8xoWJFlGJk7c7iBp + rYYuEXLVGP1Bd9ir7pv2/jUN/uPM+g/4w4uApT5mhbzvwmzbLdHuu0MSiFRDJcD+ - wZrLzfhWyy5yeKxMNNaw4cNYPsA5FKTxGG/PuljXYIjZZvx+0snj+w2sE63WaTxv + mJ+ZRc4k9AvTT3mLZ90UdcQPlYoaW4hVMVTT4KEfVpn18oX5ikI2oOEdUTzOS/GV - Hn/XGR9o+zjunCnE10aM1D7n0iRtXaVYjFbjvwpY7NsNeznnwb4jnTSFbAnlXfZs + HV3/ZzLfTBO5g8FVh3cIHpUVSKxD+h8mbCTiFlgL6kDWEiUbl+LdON4c+uatBzUU - zylUtrhQh9hgcPEyvFeXn8TAjPa0ygC6nh3pYrSklJFr0Gy1kaCR58di3GFxBeH4 + LNtviE4166nWK9ruVTqGVL9TQ6kTEFUwZNKsVbAQdpyV509SWP29LtWevXyFBcWU - M94PbbCcz0ZbU9QvAt+y4yhT5YO4v2aotAAGWgA/xAmm9UvTIxU91iKbaKCfcwys + e+RCZyPZG0cStomWPrmbktfgSV1ZsjgNdajQqUYzc4tsZRnQiRK2+0qcwc07r+kD - rFnLezkCgYEAxIQR6HAUonoYxohY2DJp5c+D7lND9PIr6RIUX/nkHWgsVP0i8ZzP + ZsU3rTUCgYEA3HKc6riwJKRIAaSK1gIl9sqvTPtvU8zTLW24Yj/2wwFpNj0uGYnX - 8uimRjV8oIYEWpRRSg6cWkjHGnx577oMMTtajLIcz7aCVsbFWlBmHuyGdy/bMo6K + EorA0yt5PY7bW55TXCgRIk8HpuYdQiH0atitjEd/0oVzpXW5hFDhLYh8kGQv2aly - gXJpzc+vhvOJ5mKK1aVC3mgK4DF2Gs8ExrP2456jZheLsLTMdjuyQF0CgYEA3szT + /t+y06ZIpw2QF32S+PsIfDztUuNv4qLivtu78LJHxEZPIW3UEdf1Hp8CgYEAyIKd - BAaOab+I+Lmuyv6I9EgCzs7/bK2PjnR1W3zwPQr8fOF/AOV5PqJZYEaPXo59ddKE + YAOZ01wfgFhP6gezFhlHgn5ThMpYnNYF4QDwfwK6iQ7CagzyqW8/lfggYooE/QCN - Oy8z8eq63ggUQ/TWcDpdh31z3gHiR8oqLr6PbqyX7XDsADO+xubQrFnm5VOzgfMx + hqRGWsNh1/bndt7gXvF3Kvwhhx5FbHZZdfU4YNna2BVtYxxXUEgxqDNCDpe7Q/2w - JgGIcreb+Re9M7PCBuUG8kX+PIWuSwCe2R/eU0cCgYAMxVRwmZANuwePJ182tZgC + 5iOg8oXMLxt6fI8iObLNa6EInHI5rH9fbOlPvF0CgYEAmk90Pe2oBw4kBVpbgPCi - MkktnMWmznIiFGW0kwXLD3EKGOVDdGBjNdFQcLtnpy3zQP5DZM2uZFpkE0DNXnba + CH3adeWvCRbgX/Vk0wl5PwmWz0vGIERXk3gi/+53gLqmHBzYtzKow75UWeTMaEWC - YDQTPqP2r7KqtwIuS1lHmzFl33tMPs0remb71AphJ8SHb1H8bl/5GiPSzAQT2+5A + ZORln0NRW1jlGdYtVUyUQx4+K4il4hP2FikacYL9akpZKchSAA0g5G51pcbkw91H - h4N86VtPECqo0icTa++6lQKBgE3eNk3s4K8y4vNTKjUGOuVtmZWgIQNhsY+vQikE + IViI1zTEfcTFkV3iy9bCk3sCgYAHoJhV960ZUi7Mlg9sKqDQXWPP/fg1W/Ek/is5 - hI5BHbejtBijGvn6EdSlNIxuroiUV+S7faMqT780AakylBPLQk8NWIaaD/TZQl7t + FO0RF8x6vDn/CMEOWvIDRW4N8YwhB61aitM2TqphKb5CUlYcpnjPBMpNtoQTjSj4 - +QFMTxkMY186to2btAjYrustkspzLZVD6eV2KIwpcNX2GHUCbKgWMGIEssLB58Ko + CLz9Siw9/gqsM37KygRBjrmbjoAMJRFen8pWj2pl/FibdmJp6XhQ+M44DUxOWIYL - 8bIXAoGAEvhd4Khh96bm2noW1DuxzQ5E104h+63qhL+1E45dHuv4x/JeofhxM6nz + wZBL7QKBgQDKYr2S186YX9X3Vfypoq6s5Jsvq//smGYzcar1xIfpLKx7+sY+Grym - CxdSOWA4v7gPh54BRRYF/PGiPYqSDqHabcq2pTD/QewpUVEmEaFD238TJT0hclYL + yVE9Af5tsvYQuzSmZb76E4ro8uoq0+5q09C0me8ZUhtDyotl9aBpK11eldCikWAr - nDLDFCIU+RB6qFYHw9RI8HaokJ9e/E3rW6HCBsDCKhvO3XFLEfI= + rCMfdanQQ7BakB0tj4GBiTYbH46DwS0lEZ8hcxQdBrL6Pi2TU4ydYA== -----END RSA PRIVATE KEY----- @@ -14631,62 +16281,62 @@ vector: segmentSize: 131072 total: 255 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:MDMF:ulidi3bj255xykqakea4fg5w2q:js7l227a44aarx3irvmy5jjpxnz4isszwu6q4hxoudolw2deqdzq + expected: URI:MDMF:sdheilfncok34xxpgh52hgiyli:zq4duuz3gntecfnj6o5mu4rqkl3edfzg3ohpqdn4jjy37okcefta format: kind: ssk params: format: mdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEowIBAAKCAQEAr4OybIo4VeHz0z3325Le6DBTa2GcEqamyxhUtOErMlObQakg + MIIEpAIBAAKCAQEAtNBH7DSJbQ2f1i93iJVTEOt55EDlz+BTI+E5bB++KTHobKLn - TFZuxf5jI9fs+ShHGwFiMZkV6BjVZSL2AmXaY4SSepMvjgybLe0XozfknblqI/yq + RJFvxTX3wfwrqs8KpCNh/YNAgwOuHlzmEvE8rs6eWXWo2Ik/YvIeBaTFynYjYaoK - hAft2bILttUb/H5+QZ4JIKFwHKV1DEq5GTO/oacJ1NdartrwuYEqzjjZNyTFITEP + acMBhGCMHyfc/1vQ80+PnRYMCjJjNvtfxIJKWcT4EmGxO/gtFYZ0OJVpHJMTiaQe - 6nKQPoORqLb9vvszUsEJjWImxleILngOrvDCsDn02SRAYwOj2lVC1QmrVzYDeMSn + XPl378CS/bTx84LzZNGkQ9VJ38Qq3ywx/zMm2jYtABggNv+ygMRdYC3lFgDY5duk - yxUinuw39fwwQ/MHJOeUtRh38ecCQd2svdNzgcIGknsQGUkz8ojdGGmq3Hwohvdd + DrbrninZG3ghBRo6LFxYbWxBd14XrnaoT6feip3lxkSMg1BKNJxFiZ67CGp9cLj+ - +Rt1aqkaW0ci+TllQsFBxtlSCnQ/M9h34ztUgwIDAQABAoIBABgou52fJQQFVyej + EbAADOVwFN7u4g6vuRmfGJFpRfYsTmnLeEQCYQIDAQABAoIBABcHbBGmg1ZTZOcn - pwNtYwt443Kre+1BTUI1ditztxt1ULCoFA8N8q+ERadAaJkfRzJbbWXAWbiZ+n2y + toa71g+Snjy3E04NmSk5t1GRHWwrwhmMCf6Os3ifrgWT84/WvNk49HMQc3f8UQHm - Y0SPOpFqRTNkIS6fY5jdwtwvrGNdi1OqytnjsYS+skgXa4PE8aIcm8sHDcSTrdnk + /RquhPcSs9JbDP0/RcZ5Zd98JADsWQdIW/kqcBgHH/Gb5ybS7+L9YCI2u6PU1RQc - SzhB3EXnFT5b6lqZPnt6YScDwjqJtdCOnDI7W8iuYkgaHIRU79C4bmRDAZTE3NNJ + og7qNUQHtTzKGoOz0TIrpMPMK84Kqp3O1RQERX7iLef5d6N30FRguXByX+ixLXNz - 8Z5wutXoQJIwJh8uNFluS0En1ZsB1Htnq1Yq0TRGqXvtSYOie8r/WxMJ6TNCKT84 + elAeTaTzexLdRwVBsCcYAEaCerqNznHbSKIpHsIApYYp3ZzPls5qRmFX81jTngwp - a75tXl2fQbCZyDElRRFgp2b2PF/Wg4bxbdPmgZv/TSDAvXzFroLub3WKV8YNKIiY + 2i9WV44/B4GWH/ttHHlIrpiUijhPSYvvEf0dwKtELxP1h1BgHXL3hy2lJyGUHFHf - Baj0d8kCgYEA0T06jOCKWYrX/zHaOEYf7k1t3jUi9MRjKOgorgaW42h+MJseVL00 + 9sQ4hp0CgYEA0CygU/QTwP06IFUlyH7O2LIGcDcqeTS6TPuaxGpzATMqWDVhqNLw - XwYfNOQoWJJkrg7ZtfpLYfqTzmNYYlsEyIqzCYHbLd6eWjPFy4yys5mmy/IhGVRO + 49FTKzfLHqnUSB86c38215CW14dXU2DqQDG8zeqhdFeLnelcQw4Zp6BnUp9YyVuZ - zw2c18/Yfh/T97XN2GbEcZgoXL2CdBcrV62QxM0mIVb+AdPCDhHzC68CgYEA1r0L + S8OScGFyt6k1CmDRiVYyaH8YGQqWBkORSFRcULIVGS13hFUiXTNrLEMCgYEA3lp8 - WTazWDckS05pice07SvpitieYWCVWFFTzqQHFcbWSWFj7cRHUnX6KC2UJC0frAu5 + cuIQBerGla2zRaSwD+WUxSxa0U9lEvV7Ov0KEX6mf21veDPaoCqD1O/Wo04CZQ0d - 4s7h5E+OFRo8zqZiPhs1XVutxhycKMSTIJfamsKEE+TmW19QbkwkNz1z/g17Oq4A + 8GLKm7OL2A7DqpCs4BAew+fWteoHUsujii00kfxh41WcjkN/qzj4K6sJDkuA1wnf - uGqECWYD9UOzFPHp3N5LEUfb9fmr6i1qqT5WFW0CgYBFQXrrvjaMxMQRl7KfBbbz + usa5QExLz7C9yFOsXVTSlimBCvlzGMcAr83ufosCgYBhzKVh66ggIZdeO0Jt6A07 - 7XT8I6JaWdZoZ89vKocu5hs+g1lauvVmrmQN4abpCiuA4TF2Zk4lNAdQPNm4VGAU + Rp+5tmEQ4lGn+whhwHTZGnWJTULdMoSTMvM0uZiGhljBrVIjkp9sNHR5Ow8uj7hd - 8LOp5e1iFVlcid5iLUPI5oaq4o3KEHm1VtAfLpB9zpMeXnKvufQzlSVm7OMNAc46 + gkBmKRXC96ITBOAgbI5m7ve7nDr1FkB1lKLGgzGG0Uqm3odyUvmJmDP1B8Elnjax - yxwrx6tjRaP1ft2wQoiryQKBgQCNVTMo7qWvg+txXRR9SGG+T86QQe5L7QOecziT + 2VgpXRCGbJLaq7hiOtbdywKBgQCogaaCYbOG6G9qi2Kqyq3qvi/KZVzF9wdAIO0s - osW/AXV8KotYrHy8u0WAOC9ud/yGgdlAfCWU3P+IyBIJeNzkP4gp//Mplx74fhjP + vQreSz7enw6054ctjkquGrxssfe6oQApZpTo/l5idH3wSwfYHh9Sk+XxotO9+TmM - tOJ+RVQku90Zemw3jAmyCdJT/Y+DmY6D0idBAFHOlVZCjM39PpltsDwHcuJBaM7w + w8ltQPjmEcE9RwX/uoLIhSutu6Z+UKtOnr9RbQCe+vA4WsDrUhbtWbLDoFuLUXTB - wURrKQKBgB7OW4hFkhHRiHpFM3St9M9Y0t0gPbop6Ejls6dVkbn+wbzGL8h3dyig + xqyRFQKBgQDA0pWlsy6Al/7cokmPuxIWX6Un+o0DQb0Hia+eXlFvEcuQUQ0W2rja - vq9/ZecoyZGJyVgcSeWfxeS6Ph2AWjTquvf4RHBAxe6vBtRa8In1qxZR/6ZSDWFB + JIgZvmKVu1V4yI8M9NRfC+E6QDsKNAsDTKBR0T5xWwhSeofP8+BUqEpbsTwgup+L - rHLHBFDuCzacIIzMafKVSUxNphPMhDPLTLjkQzKx97GG+Ue+EX4N + DIoa/pu9VGGp17UcNUhNy9pQ38jHHw9TBtQ57HINmKyFNQzaYJchYw== -----END RSA PRIVATE KEY----- @@ -14712,62 +16362,62 @@ vector: segmentSize: 131072 total: 256 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:SSK:lcniehz5whmmv2jrqfrk66end4:bduw4q4vdi2vp4aytu77gz5uljlvacu325esisujxhj3rv664b5q + expected: URI:SSK:vivq5nfnxznyjdxt47o56uef4m:t74laopuqbdodc46r2xm36sffbonespuirkipsudey2qyza4cdda format: kind: ssk params: format: sdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEpQIBAAKCAQEA9dLuspnfLBRf7mbfi85V/CuXdufMNH+QrTCJYuYOBmJiwsu7 + MIIEowIBAAKCAQEA3IYI6nlVJ1Jjks7Ovoh6Bn7aWEoZm+fZD2ArQQGdi7RWHGDD - bkkgTn/uGw5DEN+0M68HrAgklC9bUyDM3QBW+DlwyQ8c/eX79waws41BXbhEf2ha + hG6T1e3m8cTx8uJdVDYT9FfMH/Q6ux9buLsk+CYii6ftGeXiie5wiIZRCN8hTFbg - z6cHQO6ZSwunNpkKXxCkAtrvwchGAUJGVQzS420t2aOygQCo3OJDKg1cg+l7+J45 + CcDv9SA6sF4CRo5V2MSnNN1tTu+WHVE0G5T9n1YSE4kPvufq9tBaWda0dqEAXMXc - 1BKpL64qb857jv+368KI4VLyfK3gpNJLF+tgWfjrZTsD+NS4wrPPId0mPkcSxOMh + zv5nWMCyJHB+zrR8GbIyRFgQGFW5xmXzXgBIrNFAoEArrinzjje3AJTRGG18i25J - ea3F/fyQL0uEVlsts5E0xa127+b0PpJhgM2y+wDBg8Mkpxjne3yaFzrJMn1mvMG9 + HVAYGXFbhkZQIl1D+hBDq8Cncux8z9VkTPk9t0P4MH3DAYQcxyA3oSu7XykYwNml - JeyViBL35HDDNVUVYa95gFUUlHvb7hfFJoDWZQIDAQABAoIBAAIxkB/pJw43u5mZ + cMimZpdBAzKxozLD2Kw61jAJMTuJrfXRAMFw7QIDAQABAoIBACzebrj6h/twtYbc - SfR7JsqMpE5e31crIm3TZSUAgujM8q8Hlu5QMUj5UOlxE8Z+rlzjtK8vIZmFpe9A + 4k79KMrii52UMiK5KT6KJDLdV8dhoXWzsIRlFVpdRfSiTFJNgYzNVgEMziEgScTi - PWgfdXisT6NACNIa1TDbw2zPeoQelOu3OdvdDrQmRDB4EPhgwWqqHisaLCMtAJxf + DAEJvutoovXEbKcs0YucArSck6dY8wb71CjX41r+PEK/VfoyIsBwvs6wUPnTILmF - 5sDzW1g5eLwWuTuDd4ntbAwxYcj20PDeBi9z9Y2ObDi77sgXVqYo8/fsG1wm5aoH + WLNV/MarNhFYfWYr/PBME14dI+nQ1oFNYm6XSZ1XHmWt+tyKt2quvtQUwCRoo+HV - RoOnN5cA0EhV3xlXbqWEhiso0jLRiNYS92qhJCFmM6mRLaKa8f0PtNfc00ARV/hA + D2zymB/238OcxOEK9w4cWC7Rt/x9+C/Sij6eNSrDsJ9UqCeQFzWwd3QAcu1zaeAf - PdOhavmxBfzT9FGtgTcD2SARMjJzNpeK4QmI8J9ICIMtWhOXoF8gpbN5VnAlce/+ + r2GDG0eFQ9Ylepg5dempFTzzPs6Wg0UJ4z9ZDd5vazGHCoxg05/JpfKZ0khBkdbQ - x94VNiUCgYEA+4zuseq3EIxpDj3jyzaz9z6N19sDaxPDckNp3ApxxGgJm68ouBpY + Z+8ANZECgYEA/fptrH82rIyYFFxJHtAQn3bO8o8lMwYxMVmvLVMw5EVSPMyqOoWK - T6CLEUnrm9sHxJLrhzoTmBweTWTIh/P63ZpI5QP7o5/Yv4HDTCquehsgK7ruZOe3 + idf9s9e/FifdjzhtefIng9tkiL1eAWom++KrCQZfp9UOa0/Qo7jrWQunZd4HtLlv - KcfPScwnWbQON8wrqWuoEPCALqZuMx3OhWC0m0d3GSQ2M9P4CSpl148CgYEA+iwR + C5uGHPxqivLMorCqAbdx1v9CKMNpdeRojOFYhI3qm86BNXpCDxv9iKkCgYEA3kdu - roCsiUpCdiRmBqY9vTwp7jQ9nfrFQ4OobNDJQGZvw2i490BtX5rIN6KA7HsAlK8W + OVDtiW0zBGuBTJM+QWt39+tE7MWkPoCHGWaz11pksI3Wa3ShwNUUior8+a9zs7u3 - vv8rS+36xE4QP5zmKVXxo/O+EpfxiY7+AdppiEwAFXi1rzwOdSrLiTymH3ZV5TAD + TqR0VIYQ1BIYft1gEjS3rGzrJDD/MFT0oCYJ3Zfk4oFu8qHQbjGGVO3VV6r4aXB4 - D14xRJ/h1ErsU23kGJuTFCF9mQ2NcxoCL+MemMsCgYEA4niYse+qTyjKsHrB6kPe + 5RSEBxQ3pFwKnxQ0o/tJRfK6WOi+m7o/t86C/KUCgYEA0QkxA2yg31vIP9nFBOtT - tLtJwsu4gR+y991/og55LKWp+NMy6sU1OsNEURVnHNOOY8kOaZm86FZwZadV7yiW + AyySH+nZQCm0i125ZKC3+OllSk1ZPllzMQjo8wB7cgzVum9DC79W7pvAHxtdJ+Tq - dAqilCUI2eBgqNHv/VPz75UaWqSaWphPTDtLAZre1qEHp+6WZJq7Hj0Yemd2kWjF + uR5Sj1cDm+srtv82RcqJSfzhhmI8DW8iCney5mCKgFpeOvkUs9z8gWwOU+aiAjpA - dUmCcMZfkHAMqI6vIbldJTsCgYEA4bMciKjCAGKkr12LRnh4vt8mnSc4Z+y0R4Li + ItPGOzNjCWHpzs4VWMI85iECgYBDU+mcLNo2dUAtx457rmH+GNpW2wmemmMcl3vU - UrnSt20za8JxDXBsvJIyDC9pzO/zyDBmfw5LC4e6c5xSAHIXHDfTd60RUEkQup/s + gtpYkcXMALqBA+v259P3/w+PZcirGWH1zTR7Ybx5MB4BV3bBLPyxmrBC3yB8+E68 - /dME3thiQvzSPTQbfw2K71duMHhcahb0y8qY/GaaISMvLt23qZPCD6lfXNPjR3Kx + r6jvWRH4VfJQRhlHN3MUCJJFosDp1yqXYPZ42nPcMhD5jHpBbV0Nde9h/OW3b9vb - gm6PTh0CgYEAx3uCHVPRKnQiFS37mq0ILMNX/uzp/1u1RPg2X12G9udV6IJRC0MN + Bg+BDQKBgGkKUXDFHqCT5w5I6MHReATnuENJmRTUGGCZa/tfN2GDDZA2/HRV3Rr+ - kBYgKNYfkaNyY31Uq1WwKbaAdE94MujHIBXm0dIFHjhmUdKNY4JhGRACSXl26leG + pk48N7BZH8gst8hUwI+2iTuWnm1zenuKkiLbF3J9kzpyj4ujagXOWpGgiE+r2hPP - ypPnzLYGZj1SnzY+d4DlW2FmgmX67hhU/EYzxi9BEDLlr6CyKqcViTE= + ltoNSOSWFBnJEiLw4CbUInwfmwagHvThaHN97eoLDbGuVOFviM7f -----END RSA PRIVATE KEY----- @@ -14781,62 +16431,62 @@ vector: segmentSize: 131072 total: 255 - convergence: YWFhYWFhYWFhYWFhYWFhYQ== - expected: URI:MDMF:w5b5dztozqfftxg2q523vnagtu:27vxh6qxrkchh6f3i3aetesnifpsyjsx7frmglalgq5gnsms5qdq + expected: URI:MDMF:f75y3bpbubssf334zpkgvuvbgi:nsiebimpy75qg57jqp7266brgjsrbloegjm6eepbnpb2bxltuf3q format: kind: ssk params: format: mdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEoQIBAAKCAQEAuKXnDH1YYffYnNa6yrvT6KSnpWMIACmTxFfdUhRzaGIH5G3D + MIIEpAIBAAKCAQEAqsL1Nb01oNkFMGk4oZbitKIILdWtFu4Awk06LC6cLbZSJm/q - EouUzJRyNw+8DWTwukve7fMj1Tu837qYG4qlP+4hrT6Jt66B95NZqEUcMEicQV5K + drIYi/43UWHFpww0UI8E4BZiG1+AUmap2fsYK0vrsQQmiORhxwM7F6TeWvfa+3L7 - j/yRkjwhVbEeLAQh5yPOYqr0rB27lMFNSB96fNafH4S+N1EQ9/aqSumAgoMVApQS + 1XXAnjSFnJXMNrozxYAprhapt4zKLS1xzDWXlqH9hS4dbEercfdui0XeC4xGpGpB - xhaMylwrOroezo9eJ4jy6z6i54CuI0Ay/ojmNyQrpJSWRXn2YYRs+eKeKi2IIav0 + Rw7CSY7JJf/RG6XCbeI2T1lwkUhNeajy/xsWihhTdG4zUsrvFzu3eJfH9kgYs/M7 - dCM6SaD7wubreznrNKUETgP7jFFWAK/c+v/OAh5ktx89UZB9HHtM+sxW4H0n893M + eUK+cvF0K6YgoFwzg3MdbQXMSuGUbNz1VJjWZK8hDrbNbY6vufP0sqmmRWnd2a+S - MLzrwgKcshiPO9plim0oHtoYh9OeYPVEGLUimwIDAQABAoH/SCCGIFrWK26lLp6y + gSAXl53o/y626ILVy02rCG7Q5jtNLCgZpOW/HQIDAQABAoIBABbIfD73R7h2PCMB - WH8GR9oJopEMjwOutQOdcHKMojmo25IEoTnk5gUWmGuNCa5kWmFIs6pGVQUAwmQh + ZvToVMcU91JmN/nfN2q1MxXCAkR3Fu7Z78Z2bKABAxBwoxZuomw9KMFdOym7zDsy - BqEh69cZUJMdOKLyIcNtQk28tR+n6eDrP1NpibXzT4XgQ1FZ7PAPrnsZGCKFI5Ze + R2c2ATuFnaS4kQuirQkIfVHiRWiNuUHjTYZld5WkHE+QDPcgUNgBCY6Yp3w95Juy - mc2yCxHLFoTDyNfhyPqRirGrgU8IGRtgXPj9LmoKSxOfZnU5+4OHwbpNpYI8E9Q1 + fIRqghcu9cxXIsXXqiBCixU59S5FCRsuoCtVvKrZrD+9NzwC+mOPkNf4uHnsYP/g - LH0+G8bsvMw73LPSMb3HCWK8uilsLv6hmPMI02Y1MbG3OTg/dGbKH2LShegPsrwu + 5ZjAqJNQWbRqaslq8R7vJ6lLzVgjGBLFlaJKepv5cOMMpxxlj12j3gg2DWNSHbZe - D7OqVYVjQf2JmzxnBDPs/3yNDb5mm3ybJ5CdAEEs2HdAy1usyHoq5n4yKYp9VtKZ + s3Hv9NyWhje5sI+96Stowc5oiuzfbiDvNsD023Mz+9rfheZdxd4l2ELrU9BVdkA4 - YeShAoGBAPGTBc8l/mlXV+ciFY5hTtS4NTyOce+jZMhlbCuN41myCou5jyiqu0UK + HxJtOWECgYEA3/evZTGPlmpPCoy4URtj7j0fpxYVxCPlEHjpdghEywqA8EMz0MZa - SCR3OTkXYL7Puv1apxFPsYmMceTEE5M9KzBX1Ks0c2flHxCIoWX1LGrFcMnzz9lS + 2JHxWZKmeA79wWCbb9qvupzMiys41bK4qXsWnjg7ESNTD+HhJtpNXNkpSs+zZcC9 - DOP3rr23ShA/8A6Ho7ag1LWq/OTREOqO/ImFIxAU6XlLPjg+WGiXAoGBAMOspBbm + pLjABEEsfvWs6/laclbrqI3IhXKQRCjBtfAga562PkqG/v9aypm0cSMCgYEAwy8y - 4XoYYms+vmEgPu61HKO/+qBjT2bNBn/AyFKrcyO4iA/BTt0gCSfMe+TI0YXgO63n + cmT1N5DVq5X1Fd1LPpFvSIDjoZNsSYaqP40FkWnIMSoUswHY8XWW/0LTXz8b9bgg - W9KDEKPkybyvopOuIimo73RqeydwkRxT85YJ6HLUjbHWpwRsuMxWikXbpSELQayo + CijCavAGFdcK26TlAo+k/zQ0MwinjH348VuDQPcp8P+LAjeEqUwYngXqBpi+uw8F - qSlw4KbFd5K2BbgPh8knYk5zlo5aCR/O3LKdAoGBAMXZnxlgSbSm93RywurgoXqw + CEUIvX/lV98y627XfEHjv/nk4H7bsgUIWjYpsr8CgYEAn6syzc/RcAiGJR1BYgFG - /9D/7SrSTJmgD27Af6KXofF74VbyNfw+hoVvK+upTPAaHFCh7VDNT1+TKjitqkae + 8td8s1/ZUKXObjnlJpKqiJ4KYj9mt1ZR+cfB6nvUVg9J9Qzsg4fCdCXI5QaBVEg/ - A4BNfv1VMOu3iLC25lEl8uHjoROV3vZjL/GJipEQy9TxOL/9sUTDBlNfnk4dOFiT + wgPQkifAZG1skAwWud4z/ReMipscaFRKXx6fNelI0ZJQH0L7qjwxcU7zP7/2/cCY - ERvkcaobJnjT+jqAPVIzAoGAbbsTCgTPzTh/eMTm3nDG2faW6P1v/yGyFWREkL/7 + qR5x3oedoTb8mtptXbbKn0UCgYEAuoC4tXIeli/A26n4fCHuKiURrrfpypRxnngc - luCu4QlKxAsTvs2IVNlHYTV8yibFUPC9fYAihMZ4m2ejNE4iuloSbqaICcYGmmw5 + 6Yi4z0/CyKerC7kyMNbpp5OVIafN8ac0hkCYNVKQngHTEDmp0h6rzGd3kWQtpSMh - 3ZoQ0NSB4YkOgFy4BV9Ci4pxP+agHcM3mhXC5cM3Gv8Yle+fph5/8p6/f3TeSQgo + 4o5NBqCl5PBpRX8DNjnONAD2s8L0TQ13A4Xjah9xZ9uQbkKFiOf01ZXUy2asSphg - m8ECgYACL5kmkEe0X1SEa+B82EWLqm5lO0xRjkY0jcHi4O2dejKiYi7EdjJ0ubUD + eMLUaUUCgYA/J5+yMOFTta88jkSCFcdlSH5WqwSkU7qvkrHwO/rEPOITpP+UbkKr - Yp/DKgifrFTrnEiMyh3ZUzz8xrVoDamjEgdZUBC7ddVjKfKojodJLSOVcHdfY6OC + fWj3UwDW1MPx4HFrhuDpCaUX3dq9iAyXe/6FemPHvMR6YMC1CqimT/XoJtUSruIY - DOQGE1X6dZnP7YEQTUzQYcrrWeXIwxHxHXgI51lLI2Qye81CAg== + SCXiDkDZcPwqOzGy/ZJJj6cXRXbzwMxnxbPSt/Se1RHJeY7aTobIhA== -----END RSA PRIVATE KEY----- @@ -14862,62 +16512,62 @@ vector: segmentSize: 131072 total: 256 - convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:SSK:hjxb3zuhd4rvidmlcdb3h7jns4:jsvjjjob6nua665fucn6afl2qs2gbna4fvyfh4z7pqumetn2hgjq + expected: URI:SSK:trfwvi5t2nxr6ag7n37ziyiaui:gpom7gv2rn7pfsoqr2bdyr36au4sac46d4cbxn626jsc2ls2tboa format: kind: ssk params: format: sdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEowIBAAKCAQEAlUw6OvHhmBZC8sDriUJEApeHB+1vi3HhBYHVGwaiwLGzsfuP + MIIEoQIBAAKCAQEA24TZYXTTI5iKyFpzzyXUWZgltXj+oYogvqnh1Q7z/Yn2qxno - Wa67PruHDGpwdzbbBM9JGaoz4fz7u7x8DK7PO/9tUEZ4y4Z6iEGZKHd4OPio+Wlg + 7r+lssZobVMFPbLaKVHZwoY+zzzNu39/Ahif12f/c89Jk133m6xV0QNnwiuP7q7/ - RLobgVW9CHiFkijUJwfwSvPDVKhxpVTggFb16+fDoZ+/9UP/rZrkjhfGaO/YKmDu + g4hXjAiRe99xnjf0lU1AsuIFrErhoCHthj+Q4ejI4kl6vorfOJRLMdeeWEDhP+bM - 3Mfb3mf58KSYYWoOujJfcjntIpxTB6F6fogmVOTpyBvAQMlDzaW6wpKAK/kw0c4b + O0r6ElNwg289MaedCJUl4xMHeclfTVxtZ5qJp16U6H6iey/YF/6vN4tN18mjYygY - vErw2ZC/6tJmB3SCK6fNY5dhIQDMd+UWqP4A32E7O9LZla0bgC+nzHfHNxvfx6C2 + vOk1pOLRHuWjVy6K4iOyJpPDcXzARLazAQQy/5SFMk+g02EIvEGqjpoQY9F5tFCa - X4alu/ax5slhwbyHLzipDENpnkLb8OHqZ8ZzCwIDAQABAoIBADdJVCLx4ZWdYMte + 0bullE/52NzDoi/5lS2iV9E64nK342s2giEwpwIDAQABAoH/IJWXrt+od6As+ZBz - b5qTpHXFQSbJYU4lLKwKaS0p5ukupRmay3ntf796WEdbvywWb0K3tB1B7xaXxWy/ + oEv9OU9cSZOsOE5IjgSpgPa3QOs5siwmZ0oLTn4lAhVQsdfaikecC0PiDuD3qN1D - HrzfmzRfoU5h2meb9BIzIJFgtG98fa5mvFSXCop5gpf5cZUvc2jEwtIutL3L5tHP + /Quqrk7BnG8ofLd2CaWLF3tp62iL9OIFC4ExfZfIAJsqQlIL/B17fRIPxYNX+m0O - vZcpHMZwO/zFGKOtu6fBPTP1T+8ZmYnHj0cwtSEBcBn/W8e3UjUu/GoHiPjfzyr0 + 1N6aou3q46wEjEQQ3lUk1EEtP+wbIKfC+9el4jmImdroBiV+hQYgzYCKvC7ZysBF - xGb1WOjyvwp98pOWZ0+0lauOlPuIKmCnFNJgiByyHUR5zI/NFSLmzV2eKikvy9y8 + wVPWZ9w+BecTwqwFelkvNfN9FDaN0TawGuTRYg2SsDbTo8NfddmoluqOryGEkfJj - WqMvpBNd4qnKLK0d6QQQvWog/T9vTbh7n3z5oNB3Ex74AbnZWBqqzRfPtgVnqjPe + xUWtpt9GGeERBXytKY4oyEMLN+xglSvUkTdgREHyKvJdIJUI0+nVD2FfXeoB7YfR - gU0U1YECgYEAwfDS57s7qCX6gu676D0ITnMozYdl5DVSircTctHAGmoeaQtKfNgf + uxLhAoGBAPDXyXTSgs85qUSn0EwUlvLkqXX4CSr3kPudAErRe2vnB6GzIWLoixxd - v4n3RwuiXaMqLzVgdJ8FVoiedfDOMGZo7PY0mfgkKPpwcCqbRLqzn+dzJ41ToYXM + coFvLgSCYTzlQ3TXfHpykSeNmadavsDFuaCES95OwZOl9xYHLJzdFwVv/0jv9Flo - CF4QuBKcz6w+t6J95WflpKm+8qSWEIeY/p+FE44rO1tRS8DqQVuQgsECgYEAxRJb + yzOyajThxBLV5k/vnoR5igK10I7YsT2tTHoPI/DA/+gq7VoA/IuZAoGBAOlVgyPy - FAuDGx0rc53nqBp5zWT4x/eidQtHm+s0w+za82OMTqlgdSZNWyU6rdBbate9sn/r + L7Ntu5IVokLFIWJI2dmBvW8fBUjo7+lH+XBkDzdczxlr6ZJMRFksgP8Gk0tXCah8 - 7wSonaWolmIeRjcnOyAUW2eH8FVLPyalrADlIdNRGEClsdybjHDIefQ/SgbYKm+z + c/yzMEZB43Y71MHeR5Vog4mDJZpsdcdM9j8RTteEpssVlyQ29EwqMQduECAXWglG - A4JUPtTOUeGjg4CiHXvp48w1K2mFxdzcACPTxMsCgYEAmozRZX2dgtgRFDovYFkS + 2iSWgSoqgeNyR7/hfmUYFiomQSGgp/xtp0Y/AoGAaBD1rZLgjuYda9sPODCVYPLI - v4Gh6HeXyQ59IrHWO8/O6L3cUhV/XJHWawsFFYa98yTNvyUoIod+94CT1qT5izRx + /n5kh7pdXTtjyvBlYiR7ubULMg/FPEZsmd0Oh0hG9+cglLYfxVEHw4193UBquCU3 - NTTWokROfKFm7NvnNBQchLcq20ASf0tiVuCvLiEW+Z/nsus4rJHpPRlQY4ipVa7Z + plJD7hUds8y8zTngXw9xSRoxtrRoYtHTK81l8t+yt2jRkay6VAeoSK+DJJYhT8M4 - Sz/QCs9mwDx7QoUPqNnRBYECgYALAXdsqyfrP7nJfywMy026FsV+BWphNvwMzRnp + Dm3IW9kpOoqB8KgId9ECgYATNxSOoEInX5EDzb5IC13dbyxpihKklQRlZbFkH6Y+ - RzUDGrAfRH5KjJUNXgrk4hn44YuKiHJYqt3vz+yWWWxvZ20ddDEu2Z1R4rGNGU9v + CC9smrr/V/CrOJakVVLmLY9xs+A6vMz8cXE3R/PIZ9L0iC6S8kFq0J8HIYlteTwK - R62EMhT5UcLvJ+7X7QSFKwrNy4wO8qYAsCqcR64uDHfhRDHJi74IJsNhZUc/QZJX + I42/l8/4h3Wj6NajcxIIj5rKWcHzY59RRgerBkceCOo5tgMnph0lKXNRpp5O3mTZ - v6h3+QKBgGeexmjqVZZr7vibx/TooEizGqrx/Kz7vYFN2wgFrZo+Yi6Nrj0/dEv3 + FwKBgQDBC9od6eTL5KYL/PU0S3im4HA0Hy4530WHS5TEJVsuv0Jnc6vNayimMnGV - 1p7umm1+MDh/acoQ42zW2wwhXn0yxTPWpWB5pZyiz0A7kJhwTXx7XWC6n9rnjuJE + jhauS2BivD4GSe2wdI0MUtGUW8XN8qf1jNBY07vzEn9Ag1k9Rd3F/K5blYU+B27W - ELMWbLucVThn6GS0MChS9/pBEXdbHf2odboaFo0Pd0Z7hNWk2NNV + XZMNa+DYA2bZN0rDnX3Y6otwPRuWHS/Bnt34oORhcle8AD4kNQ== -----END RSA PRIVATE KEY----- @@ -14931,62 +16581,62 @@ vector: segmentSize: 131072 total: 255 - convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:MDMF:peebvtagia7nkbfophcnzbo2k4:tkxiwgqxp5fzpmb4oc4dbgqldwlcqocr37m4qwtzfuesfhpnxf3q + expected: URI:MDMF:civy35latqkoitmqun3f4vlg2a:5fxqze6lzmdznzaxvpk6dmopps4ixesnjkp6m2vpjeh2l5imgpea format: kind: ssk params: format: mdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEogIBAAKCAQEAnoh69ybyw3zQDi1FamZlbGjgvziqR5QUdo8xVXarkbk5LmNQ + MIIEogIBAAKCAQEAtzVd7X2GXo1jANrdYD8jCHkl1Zt7fO/TDWahvNbBHQYcm6Uv - T0nXF3VUh5O3PHl4k9siXMTp2UmWxWaLiiHLD/r/t9dfXkiR51dx6YJcC7OkPYLM + UI6l3ZLTW/nRLJ/ZvB9ymauAQ3cAs5C1Ok3S66YhlYqbvCmRHYbvBiKIun0nElxG - 3Fvh8JeTWwOITYIbHCqWa5YMYQqhauI2njE9fABTUBNPaA4PZpa8nhKW2QGHCTht + IP5Ox0y++xM0MCWctRKi7WhidQqywivNBZglERI5L4/O1/TipcxEFedLTQY9UVl4 - 7vYQylqpRxM6MQEjBHw/8ElnRJymIluQ8to/eYgwZqBtC6EZoU3rhluFi9MfEYE2 + dEXrw2zYw76PMlReg88Q9jJcHQRcAJqwaRvkOTZ2qjtLnYk8tJfBEqHG44pKVzvL - Ua5xD4XkLEqASQBg9kRBSOrEhNgF++cehF1TiDAL4J0ydWSUAQIUOCkqS5HeWbQi + sLFFZLNgl9cCq/xHLPpCfpwpm98KR8uFfYyudX2fG7TY4i9mCS0SgcKBJiNAV6Aa - 8V0wuCqUGth/HP8VGdh7YJMTkW7wR5lFAEXkfQIDAQABAoIBAAtwLHMVn9Fj+Xz0 + IkQHrMs1PQJe4/ychz73SkjKRvoOreIVcESHYQIDAQABAoIBABiRSThq2/0raAKK - XjxJjArQ3Fpfo8WLVRiixzvz1ngqpYoHx10ZJkg+gm1Pxpo5522/k1CfMoIncZXn + FuQMa2H8PujTSf6xuUNDhz5HrQs7kdQEVWEv08mv4fRkPlrFz8CUlf1J3HAPkfJC - iqzaOFT8VqP0iaB7Wu+WmxTuf2amvPRlMhO6G2io/wxDinuRJhSXrAeyKU19H11f + Xi8ElxtfAoNnXCViDJHhUYWo1V1uoXHqmkPb3kwG/Fg2Vcn8DTTR3DPKSuOnjNuR - WfJ6+gUu3tP5uLJ4xTqxKIW9MJ4ShYlB8YWJDZR9gm+XrvenzWi9e472PpdTr+rA + XJOauKO+phj854+ZiNgTWXD3fGdGS9G8WVmGWy1yrJaL7TgjjrfSBFOAN0yyy1qO - jTmeTrPnPWvx6GUR484fHi4pCFn8Mxs7yA3Gts1x0FKD/dn6+TwbTfSNO4dyqTjq + k6WUFy/tF74UiAT10vMnkfzMMtaAOt7BL5siGOfndpH40hM1231lf39oWp+DDxOu - xz0o4gGCohQdy1w9pXBtUfQCzUh+vIbDxreJz1owexLvXpYWpfL3XqkfUcWfxZxE + /yt7MysTvHIWB4L76k/s1XehAtEPCjrwF2ucqHlh4Dv4lCLIJTy6hbtd9VnDE/ZM - FBgZBBECgYEA04hmJ6YdsKmjZO1N/7uRP9nmz8v70Jxb6BJ2PtOdraxuIem0eDw8 + RGSBZI0CgYEA7oQCFLKAUdcmpsmjF8Nb+CsuwhmbOLbfXdaz+gG1pfqMxf4kw7NG - pp7I6rfwCbd8VHm8K5OJM7ap8IuyClLbONWdjWcb9Tm5cjIggdiMcFNNR+KQCzaN + Af5Z6HgnheXRg6CX0QPv7qPe/V66Fm1WhD79Tja0OWJ742e9h+P6WoBVj92xKzVC - 2UMMo15pKYYEzi32nIDbHY36zY75VvY1GnBwqYZMGEkjFQvsrzLcmakCgYEAv9vp + +0CHh6HyQLVBfkulMVNu/EBchrKlW9V2y2ndE9Y+gl1DRQaCI1g4DrUCgYEAxKN2 - pdepdQX+OJ+2avoJseyEsVRBKRn8S+lghGGxAcA4ONC2VumigA6R30BtPhvvpm5b + XeI3a4oYtT53nGW0zQwALWBfVNWznjx88Ywlof31GFF8gRHgXYtiPzeN+mOjQMhG - 3CAR4WHQIn6bGADf0bBj5AwUsyh1rdNAEPCRIQNQNFGmbJ+q3dIVsAOIOR15FZyh + 6d1q/V5PJguMw+2cQIPw/+CWQA7s4dwRqk3Jc6Zr4V40Gpdn7vUnGbTAw3b7QLJR - 6jO8F3EeaNW/nofLd0kL6RsqNkVQ9buD2kCrQLUCgYAhRjJziDDhaj3WkXGUiae2 + BW/Fp3/331v1LPbJziWvIJe51SZQOPDIUcq+lX0CgYAgiFrsTciY4RrBhyE6vYfO - eItTIo4w6XeXkNfi2BzUhewpD38g7rDHsPB/44ExthgrnZ6Y6DNL3C7tNLxD3Xa4 + 2rz+9pUocDEZUI6t3AvVvs3yt452Lv6uiO3kencRmV4xcPckKEBSsYFZ19DT/Eff - gPmwlYiTUYo3SWVNp4en36KnbR8ldGZpx59ET4SRUJCO8jH5ulc9Vekezp+wKzh9 + s+PDBk0gwqEZTG4amers6zJAdEGVHiers4qI4nrzfoWXX2QBzVqHB5RXPwi09PHG - OTSvpf1wUIjhNaf8gy6qSQKBgG/4gFvxiUxquvuA+o1kb9QPHUIA0iaSq9QB1/pq + HwNrkD5oc6YYRSH9BixnPQKBgCPFYLj/d/l2K7x82qF21wceEcIvb+gs3/n/IvOF - qUtES4udA02l/NiPqEKK7zaYRzzym1nUvZqz4yy+hvVzTSyrrSCijFIjAsr3xyQb + /SqU2ktMN4v7Rod93aeGYauVCJO2W0Ab6WSiDV/sZfUWeoA6AFNr9ak9jdYghI4o - whdqP5zJKj+qz6W1vkKDTTVIJiqex0BQAS4NLpowZSy4Q97SGslouTIDMkS3szPl + jGcfdSyQSIY12NBdhHlmqlJkiJxiU97bUGeCHgNh5R4C3v9DV7JkJ4gg6iMRmj9W - Gg0hAoGANhX3+SQxBZAwqkgoPwheBe75VIeQYQQ9mruxNJ6pya+PI+GRWxzrVF1z + 1Ii5AoGAUzrIstKLKfJBPXp4UP5LTglZSGCGqDj62/iBAdIZ4xz8703g/riq0h/d - 6L1hBwHq2CHNiZ6R5ImXIIezNU0umqreojMn7rqhc04NvZqdrSkwPA5lqFnfD/ou + uAZipYubDV8U0WipfEhjUw2NTY74FzFRnl3Ya31Ms1gOXwuc+EagS/M+ohmb6eK3 - 4BknZX2gObawZA54qHCgvKTFHlk0F5bavfqeqvuFu3miJCTQ7uA= + HVaVQohUL//L0C+jqlxfjl6sQxUF0IizsR5XPm9VkFSJqEVYb/A= -----END RSA PRIVATE KEY----- @@ -15012,62 +16662,62 @@ vector: segmentSize: 131072 total: 256 - convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:SSK:lhvxjaof6loeiw4crze6xczaze:uib2dujnn3ockx42o5oj6kpdaar777nfyvht5z74zwil2okawuhq + expected: URI:SSK:vwfthgrx5rep4ixdssp2psxjja:vs6adgwkfwnganksm6w42saotjlebku2jni4y23uuczi3q23i6iq format: kind: ssk params: format: sdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEpAIBAAKCAQEAub+SzMzecLZrHoxX9QwQIDv9chKraN/m05DP+qOX3ls/5Ev0 + MIIEpAIBAAKCAQEA8TipW7unNnPRZ6aycrHuDFicLcuI+QXAG9F393JiO87SnmZm - 3ASAhk8uVPonUQ05U26aOzgeAkvjMIq2wvf7b7il5ZXuE9gcif+vTpy/uhyPwhty + zjI2FWGFDV3XpTb8+6QA44n9TTWiAT62b9kljU7s9Wf4dBfaJlYIrC5HIS209i19 - hvAxX2mdJxLfiNkYsXOuU4/bSQEmvQN5eKHdoz4Pk4qNjXO97JM9XszzWC0XSM/h + tpIoQgdh31TYqXy+1o8oJDhTBD6mUJ3Lotj8oAKqUaVTKqJnfNSzPMdChYHYvE1c - PayiaMgFeFRh4DRtO1f5V9IpCtsMZhyP+w8RuhFh2FAzO9qzLgKQQi5DiHQBYWc+ + /Tr4c7gpxXMXEMTD311HtYZd3YnZ4dZryWfyrdf5/eQfrUFeY2MuPdl1ag57+2yN - KVcIoTLp9gNpijppFIdhwmjgcZ/OBGKogHqi6rfGvaDq2RcK9QlKQ7h9BFfXGSRc + 4mXynTGHmENgG0isd2OKgyBmlHhNPq1lNsxUrQda9ht0dcwYPvLL5qNnVc2z3e5+ - HZCG9xRETu9H3p/rSe+RwWeUvmHes+KmmfAUhQIDAQABAoIBABp3W3lm74Lr2xN9 + RH/2lBzIh9QTjpo+pdMxqlQR6od4jfZdrFZSDwIDAQABAoIBAFnzVaAZ+VAiXy+G - N8MottuA8LniQx4sWP1oMtopmSgLpGzpDbiTw6Rff+CHzDZWRgbHSZ6KfmwhV/vA + J1wCwrCC6HZhRCoMPWeCNHinBD+mL78Wk3aHnchaTam+2TfIKg1SSmyPG9BLVCaf - qA3bu63Nh4XQ+R1Gu7pF/jqbRw5Dp5AmzQjBDKflqoi4vbUICeau7vXlF3+tdFGW + sptBv7GSgWU/yJPOAzCxe1lthmO2bhkwvIS0uuNoalRECOkm1ekfiAn9oONf01gT - PyabGbN60klZgpXXGgatbB8n4Lx9YeeD4/I36dqdqGDrGA9+8Vb8UPVWo4ytTYtC + h4ip3oZyh/2bJ0iqN+oCTPY4nbM3j3Hx7Q7NI0AJ+rutEhM6Ina7RjE/VmDSmM1a - V9gzBduz3L+oT/FH5MyrWkb8sPhICYyZ21/9gOH8/dlLZA8mYxIvBBXumE+H7Wyw + ICGCXmae7Zg20bM8rg75BZ8Jf/td+5lvtfaEK/070X4AGPri1lAJ3vdlHjQbvJJw - 1HDI6hFVJgneqQRisBvJOahKdvqbsJvqxXk58Y6dYPkISYsvqXNazMZztNoYH1jp + atSA9Ky1Qpe1t+rPjMBEAdUxp3bzinMGRwt/tSj3JNXKfyp6JMO5zM/dOknT4C99 - bYkZFcECgYEAyDbMhjH2abUEY00Xsfmk0bro+GSmZvks2ZZh8W7UnaTZmykmbVpY + WkM2O6UCgYEA83SKCn/lZhGBZD4thyRHRCcyVeO3GxlIO9mjt1f7JXiay3hOlqOu - nrc5bgqFulfAbI5Uic0dn15hvV5+HiaUov2ipQqCwGPhJKpVbdMhV4rNTVG+ktbL + x/sT/w/Hdn+qSD8F3l+IFYP8X0hToN8oslu140qPS0RPuOyIFigm7jw++VqnZ85i - 17hI6rZNQ/1UWp8A2TsrAFA54aLdyuFpO3Wj3h+KG9hahMqk9hXlN70CgYEA7YDw + OhZ/KuHsfnlxPmc+vS4Qh3MkDiqG3xfLVH9jUmCXDhQG5UhOMgOsgBUCgYEA/aan - dEOyYaU/aWeCZ1hAFNeDPMdRdOM3YFwKley0crE3UznacgCn6QX/K3ltbj3/XFUX + lBQA2jyoXiwnlE9P0HF6ZSbxHZaUQrw8xDuufjQcBR3mmAg7p8Yb/EOnQKTjage3 - M4O7f4kqxTrhGOBRcIoWn/YqteZe2Qlj4k1cET+Wfg3U/u/S//VhpIqegjIXaTAU + EDzNJnqPxtYjubxKtBu/7rjvaMkvavfH6JAN/Rl3fkibOcKbkoSa7mGmuUE6P2UR - q0s/O70yrDACSNxIYR7pf4J9Q+I4ACBg4MBkmGkCgYEAlDDbWUiJ1twBD270Zusc + 6tV9vd1ITN359AE4VdUEapzanyuAjIhc8ehxLpMCgYAsMOM9tKl3NYY/I+ovta4Z - r4/k+FWnRPiR1cuVWxppjPWDi3D93FrO1UtQ2r43FSH2b2M593U2w8scFQpn1vE/ + +ONyI7uA973c30yQYy/7RUET3eql/WAkfLbMfZi/Mb0/D/GIw953yVVuFjrX4KoK - exS42efZt4U2E+lvqgZn22AFbYFfyVfrMRRaBEBDGFvdn+WovyEoRucasIPYHl6R + dgs2Drqj9uphrs2k9/TZGaZ0rLfmZ9f8o0jCB/BdpL2hjiwdOtdVPtk0mROSO0d1 - gU0lqTc3Bj0xYrCLQQobyxECgYEAuufo0yZfYDbCY3nhBuFNdNlxX0hgU0No1f05 + NwpYUaAZthjqVY2cFn6hYQKBgQC/hTJLML9kCSDn2lcYOLp/HO/ZqImuWaAgs5j+ - G2lvTH8oUefKgEMB4QEmIZlqxAIoTwprus+lo6VXsmU2tfP6Qz14tqPsUsAbzmN0 + YkHisN3nTyhp6u2ARKmk1EBZIydDTAgBjqcoQqqE6/OVroKJc9p8Gc9LQ302O1kK - ZqiIls5a6ZKLF6G2hFYgZHPub/lpsQ70hSUvexzWnukdMyegEkZYbU9Mszp45aiV + VJr7XFtJUvFBr5tgChghnkIQ5xtf+qSIuCJ1Vbvdrk2o27L5vBnVlhHM1T/+3Iex - dOoTgFkCgYBJrgooAWpL7k0AT8M8ApxAu9tvKwfP+HXj/V1fM2pmtrfPNGjSK3A4 + cFzlWQKBgQCsutY68bH2Wx5E2iiO/Eay9XQRky8HRvXAZDZSG0Bfn7dTcF5EddWk - pyG2m+TbWO469WJ5oJTikZk+qiFgv4nIm2BDQfEtnZ9Hh3ypHEtlFlhn7rWsRHvv + vhRSjEpU/6wrHcVJb1clP99uIpzgK4QG7t6mZNZ/StqCHTtyNisaungfz/B3k+li - 7LtGzb4ohtIzgQCK+l8gcZ9h+VhfeLp0tp0jBUD272eeL0j6gwBVow== + HMntS+IYxlYIsxBNLIPJLqNPBqAlQb9SJkoO1noTR/+4OQlSb5KNpA== -----END RSA PRIVATE KEY----- @@ -15081,62 +16731,62 @@ vector: segmentSize: 131072 total: 255 - convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:MDMF:i2cep5mifmrncv4xwen22ppu7m:yfgy5mvr2mmz7t4dflek6ixp6yckvcj3t5zktvmazwfvvekfevka + expected: URI:MDMF:2o3r63lhfs3qvi7r54gawvyuca:cl62d7ozemosy26dkj3xuvdjrzuf554mbqyawhvi4jj53ddqrisq format: kind: ssk params: format: mdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEowIBAAKCAQEAteX517k4p0pP+zpTv2q3u8n2VrjajlE7eITVGQiSh/y5Fvs3 + MIIEpAIBAAKCAQEAjNzgfrhm9Cs93A9LJ9yUmKkTL4JBhDUXXT/O09oWWn3wpq61 - KWpAPATaxkafcCjDraQv+FoH2QEdw6cVRNw5J/xlnofkNGIOgSVGMM3udfYV3ywH + Ouf2NcMGwMC+1tsNWA/QQYwU66JE1s+iow3k5a2jd+Ov+pRZJa8R1HO21iIERvjF - /+usPlfOpJ5PQJ0csbP15wOKhdeLCklvmA7PvzoyFV/80sWUnriM1FZqkfVWF6VZ + qJOvcxa+PBL37dZcJP+ubnhK6O3sIdgvZRy8NOM0iscXoDgJZWJnWlIIcQIVy2w7 - eL4iBCz1z+c2rbAky5JRSXGhpX0Bw+G9f8Vch+JeJGE7p+IhH9E1bmqSj4PtZwe0 + 32596JhdXHH1eAd4WH5NtjCQQId5/29OPPUlo3CscH18gymseagJ1Al7ixSRnkp4 - o4hcZVaZs1Doiah98Thz1/3hFvRIH2D8YO7nbvdlsQMSE90rReVnSjCsUO8mwumY + AnyHxk1I1aelpzRIVSJZPWrfFp1eIm3KPha2SPvcHGugGk+faTuJ7HWFHN8pLQcy - /k+AGMmofUpfRvdrrBGgZhqWsPEuCx9pkW3qpwIDAQABAoIBAA/sqdJWf1y590WV + qTXCSMjw4tsHKEbDvb7A3LCnAtTi5ovuYHt06wIDAQABAoIBAD/J7me0PfsocdTr - xiYwaBRzKnNOLKgf+XZkHqnZ48Yu/F9EMACasjPu8t4/6Y5uqy7k/GQgMaawX9Q0 + oA8nFqujNSr4g47JNBFoSdMqGaFVEtuIlk1cqeRisvYq0sEdZYeRca+dLgQe8amN - qPqF0yUqhhT6baKeYQmyYzI7nSBLon+OwcaIceRlWIx15ZdRCeWOzTzjxPlRtPT6 + UYshSZyw6yvpkdGZyF0GUL6ywANsWB+DnI7ggj1N+UvfEyNDRWsD1gv0sYeV5q7U - F+B2j3580Eypwh8LuCarHn0qcZsEeyyfRaJy9Ydpszqx6q1BwTOppF1OiQcjQvX8 + 5XGWd6xDj5Gg4xQNDEQ7Ma53I5d4vD9QgQpIHBrpTuTlz3nowkGIzXnpVQ+54Oae - 5p3uTTFPHS4aT/U5971WNGBhy1PyRJYoVpOzYJWqGfxtX6JmrWgZf9+DYQuEXUmp + lPb8oBmLBagIHVzWY7Bh64oFh/kr6o9db/QkGIz0h1/UdMhj6HU0+kOUG/pSWHbk - XHfz2ds0cE8vtcFfjwaylpnEXA3TiSO7055BnQQlEcsDqmgCErh7y8dQ4ZDR7Qrh + 5fRoHL7mbXgNzKZaGPOZbATzgXcha5uyWDs0r8WekKrNsUvm7E5vOE9UPdLCnM0O - zgiHtY0CgYEA4EsQ2NPYrlFFa2B0+jpfsq5g1+zDCYS2HeYtt35nz1KKMMiCdPJt + 7YHe1+ECgYEAuwDpwl+IzdAtjPNQUtsOBKuDD6/VempGKX0oyTwf4SvY8VmU0Juv - Ene4cwo0dcKawlPNOQspRqIJnctgxVMYQw2OWIm3mn5QCL9YRXj9P0mkMMSGpsVo + AhHIq3UyPdJpBLY0Be0UIvbQWNXdmRDnriADLTq7g2QYDwVOI/bmRxYUI85FXjIC - 3LDkXhLLbig+VhF9eSJdu7bNm6MTViI8aYclYVJ7c2oVq7Q3sIkWjzUCgYEAz5yv + RT2u8nQWAzFfOhURsF23PUwB/rdjtBSvAdfNSletKp8abFApFonhuNMCgYEAwNXS - LE4Kr/CaGT4GQ7cusYsS6LXbZhpLVh+ce351K/T33j18NfEMhF6ctbGM4va8kgOM + BQ9nw20GeqZnYZOASnD4a+iW45coNSkSGETGO1J917xWtXzvGAAAqlSv/kNXYYqr - XrmjARpBG9RVmQn5set0Yw1Xzu716JctGGJL7dZ4dnaOzf/YKw3ZSipYpWBIGLfN + RA8yWuYc9nLBx7623cpjt9CH2vXefodDLlUB4QSKEuP4f9/ttsw/L5xmJVdQ6pTL - 6buhTrB3ERwYrRwNlmMwArPXI7x+wL9R3+FsQesCgYEAvs081KdKseezHUgd2uwj + MNpU3XgtNl8qsOVAokOORpbY5f8leuuloylaxIkCgYAhSyRTKtccbXfupFMkrUNt - krYi7iycMhGydzbjdzBSER0PL7ayu9erD8XGpB5vSCo3Ss7NSxSClXKsqY5kkRhC + qWuIG3ISfWFIebQNP9sdJ8VUEvLfwRgDck8b152+S/vOjvHsLC1tnCuz5T+yxMO6 - EHCMwibNiOChJv/XkKn/DYKQ6WeVgHN45Bya+KgWZGxZsxAH5C9m+5Pjzt1oSqKv + yJBIOTCxT9zIr9UdqhONjGzBgzPudVDaKwU+vVQ99UhS+vVPRSAela21P8lMgnI2 - L7pnAyaOnD0HmFyj70p/ZW0CgYAroYUa7YfHc+wes+9DGeNBQrYFm/pw2cPNZLVR + DcnK9pkqAXGe3xaxoJLDaQKBgQCoASJLelKC9xfv/86OOr5JHQeyrB/aBbXoKvIy - KsFbLI9O8GMDPxZfVzbd5GN0a2Az23ULjz3XhHn8bEJU+Ei2gIIkMvCqN4QMjoDW + 5qh2wrYVIWfCEykUFdx+ie4TboRQ3Um9sCfE/js5lF20MzqLHWunmCzk3dWNEze6 - qAnHARSt6LqYRlVarv1kXcPldXeRYkdvAJSk4ecT/HCfKM8eNNgpKTxkcT++KDb/ + xCEw9I1/S9MTRfuLiYN7bZ2o5tv+pMgqte2+TpfFiUBegj2/oW/xnDc4mwUChQ/4 - svM6YwKBgD9PdZuHZUWelB+ELEpvpqjCYWaO6NF6C5tWLPD7uTSuP12mYz6qUkG9 + iW0lUQKBgQColYu/R9ympb0+hMYMtTDrJRuREuyNCB6429vX5OG/RIxfduxCvVmI - zuf3yvGkeiEYmajzrL/29MHEjphWHoFchlUAkv9Bmg+HjLNpC5iPyIB9kR03tC7Q + SASInZIbWFSqH+zt8l+L2BEi7fzpGSk3I5/9o0obRRM+9HnJD0c44P7Bg5IIsFv4 - kKedJkJQtfWNINQrcDvEEfpz1+xxfyKZ02DGBITjGAd5ssW3yEcD + aIg0X4KaSZdNaLndO5ieyffsY+dEH0pSg3GRSrSn9inqiogQZB50pA== -----END RSA PRIVATE KEY----- @@ -15162,62 +16812,62 @@ vector: segmentSize: 131072 total: 256 - convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:SSK:sghr4ibrl7tsy7ijarale5whtq:23bvdxndpvbygxjdrjjxbm2x5dncjnicywugvtnwopehqpl5qwma + expected: URI:SSK:7adguvusvvbmctuzlf4g3arfv4:tkh7d36kwo7aexp5kcxl5qlhcahkaox56wgnaoz3ezeciurh5qua format: kind: ssk params: format: sdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEowIBAAKCAQEAszz2tUQEFK1L+sKzY01w/0UqsfpX/QD9GZiIg6BzCMkSq5Ui + MIIEogIBAAKCAQEAmvahwGh2jLEIK2xDDGU7d3aleXiC23PzGCDVbngFhe61hfAG - ONrE6SUa0aLswwEJAqxVLi8nO7zjBy2XPMKyrw4KovNodoO+7LhpQobpUz6yFyBu + ViA5s4fvMo7cgz/GGc1acIPKaLiZESPViz2P1QPNNMCFvTgKkYdRzE9JWks4Vzx8 - qu8LLQJzXZWVjxHngTmsLT6a3CWxXVJeMC6AoSXxyzvLeb9D5Z8+98N3H7fk0BPr + cv3V8x6fwgzPeuGij0y/20AosukVws+9QyAvEXleO0BtDnddPhPPfebe0JOr64BE - DKLQ13/njPUGdKeLZ8PEuIp02cUs7/0f/cw4n3ZA4FGCpkS5q/TIo/k/imsiOY2p + 1ky8r4VE3wIMKdxigyqWvCuIrSolLDpiCaj5Puw5KwMN17+KMuyIkARse0PFGboK - le50s1S1+3DAhoL3G4O+TgI5+0jsgyTH83f3tfyxpbM8tll3VZQkAmQsjJdnI5YL + PB+n99MyxpMZZWeDesrOF56HT0Wl18AdEfqDs/M3OqSobMy4SCmc91CjkXEnw2dz - 0Bb8BhHYWJYLReOJzCQxpOK+UvoiOYBGyn/g/wIDAQABAoIBAEDGxo6KD0N9wdjV + GVMJbyutQn4/5vrLVZb/Y8I/y/gzZ6YQOCNtZwIDAQABAoIBAEPv9KIiKjcsNeSz - Zsl7olvPHngF9qisI8yNUMDpSsmhCYtTMXQEtGdiDog27oQnKp95sqsnRXGUeSQN + pgF9MEEDpzBGATis8NqXKnsv61v4d2StAlon7qQi6F9F+q8f+n29ZfUGEmsu4wx8 - +Ptvje4wD+4GM/mo8WZR21C8uzRnkytCgFxsWcihexoWRl/XY6hTNIOBfawUPz5v + pVZSOwisjf6emQOH2jpLFTV5XTNU3vJ/9h+D4ZSgzHGKpDu/SEGC6Gn7CtzFC2FJ - 1zRoifozYWhGqunMEvi4jaQzUyj1zJOfEdVmPsXdTz1q25HBZl+UmffdS4MVJW4T + KjSPm5MRnppjeGxrMFnS3ZjY6r6OEFe7wRVjH2wOYQqHOqPOUpqWZJ1bktz3DzkP - E4WXgFrqO9G2gnoOE8sbo7RY7NL685EKM8n5kBJN64S+F45lNzO/B5OmWETXTAZl + tK1ZQ+rxu27kpFbxeqLohOLW3JTrgphxPAgq8VltL0Z1wC9Umd7FjKb0irkcFgNm - DavcXRM61jPkOt3ZAT7OxJMPLF8uGEZtUGAFt28KLaTfUvnb18hwQzvWsfVKcugn + 5ZtQJdHwXHuEyuYGO+zI601vw9XsBnMpl6NOv9nvhX9OFEDaft7RnU/2GuX900y0 - XnQB0okCgYEAx1oBhxaWO2B2qldDfv/w+yhT4XzWZ4eDjJ7MT9yQw+jVepemN29T + dG6zWUECgYEAvB00OoqIKnXkBHRJDGlL3degn6oYZlrQOjzQpX80HBA6/hmC4yy/ - igHySEZZgHhdtdNdle3yg4TQoUyhs8Uhc+vSuMvOP4vaMTrYB+e7zOtgRleE2A7Q + oll+pPlppmtuHlzniAQXcUsEtoaP5F/AXXDALKn9Ebk19VZUUBKWrMBE3jcV4ULT - W5b+4y/LSa2VdSPvkF40JJjIDBJs2d7ql8wBUe8uDeo/S2YnhKRr47UCgYEA5ivJ + AdQhzfMExZC63TNKmn9v2naKzg1HtwX9Ms0ze+jKVxLDIGWxg6fy7M8CgYEA0uLT - 5IuNR3Vp9qeU9lsIS9G4T0Ut99n7lNjPhNzg/TW1VdJa/ADtCY/hgUjaoR8A4/PQ + G3/xSrQbhYzBk+pBIIzPAZ9HDySJfrtje+qqmO40k9dfIrA482SlPUN1m0vQvMNq - f7JCXrd+FX9EKvXALYo4ofRoJB8sUIps7/B5vRba9qQdev6s4jOutscqHaIePLOY + yvuPhiX1gSvnEFbVSpNUsezIU8j3JDtzxmuTljYqURE7ZSpSXe7WYCgEAZeBQY2n - W4uWeW2CNgjz2sJT/Ura50ZQWxCrGY0IxMryymMCgYEAjqxkG4KW0rgfNZpuvB4B + Ncp6jYRptOn7O/E3wyVptrdY5RZc8Y4XwD+TC+kCgYBSZuiODEkBcIrleJrXGPjm - Ij+iiOcHq+DYzXN5Vk7NbOjeoHaYh2Qtrb/m3sM6my+KIe+8Mumxf1820bo+oKKd + wKHXzwbJL1avbBxpooMNF/7/d+Vh5iQ71cAoPCkPgVfHbSLu7fvm4Nm7qs41V8xI - ZpGIpql2WxSEfGdY5Y98YRS0OqO4d8liZaqTkZVLMNgC92tYsUI6n1aZFcq6DNP1 + Ii/MYNo+fUcppRthyALAwahpPvASsNcFogr80Etyz6dLZkBz1QcGR48eG6sifTkg - od5ns3QyydK0qgnajpv+e2ECgYAlZYfv8hyKN7F3udKiFDhM2U4w0vSdCHWvwWo2 + m8rFqH+aDNn0wxczeMps+QKBgHEJE47R7UvVbksPP1NBZNdFok+ESFpdgzViy9hH - FA0aFtfXkc3mk9/vZckl0Eh1VSw33S1LEhNmgDmkFIFm2XbX71U0OxmQhOAWVedN + 2FlQlO4Jqvy06FHNyKQl3Iv4/1GujTdvz2ZgQk+ScK/ZW0o13lfgSyBdv9qz40Kf - NK0S49u/pvqDOU2tkugYGlPlbKmtAyEF/q/8GqbFUL8OE/TBeqAGY446vYKPLDL4 + tuP09Imvat626J9gvZec20jfJHE2tEGo3jesmdxW7ksa6IC5NQizDfr9GaSAPUrW - hmDcMQKBgCQPtno05prufVeyuCGn6zXDNZkrN90lOL9oRh2g8xu+Zv8wLQZmyY9o + yMLBAoGAHI8TJFiri+YF0HonCiJvQzSb1dX4CQ8zWoLb8bgRN204Opx2jF6fmGER - qpGKQvLx6QpKCL2zmL4p25zXPobKK/KusqOyVhW9sI/ry3b+4I/IiM9XjMsY9FeL + B0RkYizKEjWwpbTAaTiYvMsjOn7eClmsyj2V/03cQ7mppZbFo+TmxhunYrfiXKpe - hkoE2IISx/cKvrxjFmccnffDZD1u8drriVKra2TRM/81P+tlhInT + AiQTdpujt6/dkc7b/LlDVjZ1eNRCBy/ChJW85mfpogC794xyiWg= -----END RSA PRIVATE KEY----- @@ -15231,62 +16881,62 @@ vector: segmentSize: 131072 total: 255 - convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:MDMF:cce6edzhxmwz5gilmwmfg3vu3m:3qjixjwr2n752zvevufhatb3a2ugcuey5w7u7fvykmi5lbsjqgcq + expected: URI:MDMF:vd6qxch4fcvs2szi2saiklzoky:btq6xrsirtyy6ugjjgpaguaajd6zopupixy5e3ngo7odya3i33iq format: kind: ssk params: format: mdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEowIBAAKCAQEA7EAXUuSzMzjd8py8jXmJJz66vnbGPrwSU22sl3gfRg87Ap4T + MIIEpAIBAAKCAQEAseah1KDj5lbm5ULH1Tkdtp2aNSeWxxYzQ7ifXjfJGDx5yh7j - k/PwAMIwbCpbZ1T4Nwi8RsMg8+JvZ/FVxJp9zi3C4rk8lKlJ0BGECKjyjQ57L+yE + 9aGPRxmB4tmdABQUfNY58aoaYoJo/JWOV5OAq68RdGGHx3o3Pa4gjpDMdIq52biJ - SNfUMB7G769llAR7UoYZYuJ3JgEF21UuRL9RH2JV+MPuEA/FhNUfaG424TmRZ75O + lNnAGh20TJcd8lirD7RRtEoBpzQ/mWDBKiWPxm24imUUlaRzEs4BdzjaK9mASuu+ - yTOBztsbzxIgnrTf9JBpyn4iVL1L/mR8/8eT8UlLHgP40hpnhS+frA9vH1J4v1AD + etKDvoLy8ajTGtomcU4gKAwbLYoMJwsG5hGicJlnVJ53PTjfsH0egpLW6vUfglEH - VVOW2DtYmyS2eAuQJXMAy6XATrd9hWaGy0X6D8q0fXwGb6VuU+zeTBP5T6YwawGL + tgaNaGfPtPvj5uJMxmm1vrXUufLok1dUxHaFPHzX4j16v8FUCZWoP5QEKPASh4Vq - amAvqHFmiIYwdghLJp7UbFwmXgh9+oMvmk4/BwIDAQABAoIBAEhp/LwzzZnNvHo5 + 5/BnFdWuQP46C/Wbwbn0Q3F2WqwnCWVuvXe68wIDAQABAoIBAB73PaEEtlagNsWe - ALJ8pkWZPLRUw79G9ncMDvL+ptdao8PRoD6hbtdMrnr5ILszmEGGM++cr+URawR5 + N6lyLS1dxntNHk4eG6NEjhz4ydyZnjtj4Bsf2ZAvLPAfH9hlJmHKakCZ8sjF2V9p - PMjeceFYtXu5O5B1s2JLfCULZA1IewndfU62mRuG04N0R7ZvCT3qTK26rLrBZYIt + 6uJsjt+TdA7VcSx0Jgxq9EjMhIIeqZXvrKcHtgv1sq4IOdK2w2PS58vhe+MuUYmx - QdlgqwTrp15w7++MZgapLM9duQSfgcvw6mXpPWxJVdEl01icjgEU/aX/DMpu+uJK + kT30Vs/bxlz0lj4r50nVKOUnNLaf9PwSSR3SoLAARLsvjpCSjs2QjxUDcnTmDp8U - L3UfJOlMX0RNBxLWZIF09Ur94T6zc3HzzDB35kNNG/cxibI3qDh2KSBgaJudiZZP + DAffOA9lGz8Ch8m3BR561h8h0+muLAXxJ6qx0cZ0+FqXof/FWo9MFMJy41+IkO6O - m+P/uX8+0xIt19Wi7ooXvNlgmueZraJWYLYlpKUOpxJlBd2CBAKooE0O/VWhidpB + YNRRD3bPQmbegvAx/yiKiVjfo1J5Z8bEgYNM8nKX5IUbdVd3sLaobEFXX0CAOzE6 - OEr+X8ECgYEA7N+yemfO3iJfThXruqfRqg7dCcmOWdwevS3AVxlYaXqyGXShD3C6 + 6JA84cECgYEA8XRtvIkAgcp3D6bDCTymxwBX8t2ltCqYezjQif2Tfb3Nnoem5wxE - +BQPnX678+uIlp+aMcONAEdtY7SJVnQjlu4HKYiL64pVjYmrpG2Bk2igHxvNGR0v + CTJypmX0BqVYlYmb5bZ2BAi5A0ijRTwpZx9mnXB3HKmH6SX9Bnqlq8EiZMLeTYgd - LUZgPG4pllGuOLx06PQUyipjTZGKYjPr6pqtOXMk2Y/jo4dadSXcIXUCgYEA/1OB + I0hHdd2P3y9vbu6MuZX2KYbXKun70Qb54lGfi7D3FWQ7e4V/IWkoYEECgYEAvJ4d - tOR0zqB4G0bQ597UCZQZ9E4N0r/V2Yd6OUecnDxCDq8Keg0XXTHLZ56NvlR0hcn6 + CMDNiqQOHZXHLEJgUID3JlrIFEMrp7pc191pjg7hZ0pOoQzonGzfzkxxrtWGsA1O - jrylKgxtuU5s37Zn5DaGOd4mQbOvqPnhMJ511A9bSWo8TthkDbUT1VaOeVEaQig6 + AyIxI5Wutoe/VPi4dpkM5oqNiYACUhwTCNmwoO64K95IUwyCsFJ/LXhhHVpu04GJ - bESqRxOTZbiJg80skWx70FLXIRx0/mPt3rIkswsCgYEAjOqXdxKCksvH/uAzmJt1 + THB56kQ5HbyOoAZimlCrw2LxwOnicYQwaHrpDjMCgYANBDbKPCR/2rdSa64F+HQR - s8Gb5dKuiO7WqpypLCe73SRNB6/GkTTzRdpJX9yhW/7nBxRz2t8G5v+XKBWjDneR + NE6JdDNzo/w2YFi1p6rk02+bRTrVJ88fI84UdFiUZyOAZDu4RX7VNtcqeyb6G4Ur - JJz+TcsZ0ko9kzIvlmY/C77WYyta3HHsOvb/EXRH8VEuYDpdIqjyJUMKSH8o4Dsb + 3wB8KkzxiZ4fDoI2cDQwLyg4gFzVlyni9gmMLBaOdJMwSsHhW1k64d8FnDmMCjE8 - Qjo6i07gwT1Eo2hGfCLFznECgYAIfDv5SQZgv5B+R5I1woAFeXiLV/S5pkpzGj+D + ZyQPtsmLKK0gOpEg7vdTQQKBgQC3xtsFP05FqmkyfFAvGJFdfvrQfR17WKM9bsCt - m8+mmZIQbtzIRZsbK8Z4wRow0xm0QIwlJjvO8+7Jk8Omg6dcPDulvK5EzLXvxa4o + d0c0qd0HVghctQYj+5TpHeSac+QivyPmu7bjNCGiKYvMD/czXxaJvi//7CDWvhHx - MXv0+jWscO1kKWjZ08S++Etv2LQosrGOW5HVHt9tJ/7Z9H2gr5xFxhsELK/urF+B + yqFlfJMn8xHHEWZ4xDi0JhmBjy5ymEEdoG25SzXXenQBCZejQbzJyCtDSt9euWyt - YSY7FQKBgDDzEpSvjNAFVEpgMyPR0bKHBNSAQ8lcNZ7Q4UaVuiGluwvs0G4yLeak + MCzJrwKBgQDUR4StEeumfmLz6yGPxIucP6hxYBJPfS3i9dDV08rJkw+2Ubzk+oGV - s6dufCJu4O6fKuxwIq7u4VLFMdG8XP4iNszJ0KJaxkfVUNdkdGun40gv29znm9qD + AU5NGUlSKdw/pbBRyXgICdWwB2I1NNv1qHiGoiaavK+/OmI89MiXhZ19OvrUc9P2 - cFKxkFQa+ko9Rtt0LbU032IEaCvOpJbk6G7bJZlF1w7lPSjJ6gl9 + ZX+BDrxdjMWbm1uZxap0yMTAc5G3Y42M0HHFgM+fU2JZF/CDqDsW4A== -----END RSA PRIVATE KEY----- @@ -15312,62 +16962,62 @@ vector: segmentSize: 131072 total: 256 - convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:SSK:nur52u6a4vs42cm2bezo5gvbdi:odirpm7xdlhvhneedjqnve3daaq2vunkdj3to5frmsytxxhcuqua + expected: URI:SSK:2nkvk3zukvkwkomg7lcmwi4k2a:ulzy6ok2oaftysynb5uqactvbqa57ohasakq2qe3bd5j4jiyz5vq format: kind: ssk params: format: sdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEpAIBAAKCAQEA1iFUfknFQ8V+AOW83RO72R3QhTSFqJmldpQydQHnzIT728GU + MIIEpAIBAAKCAQEArb0j+SDFBcKDajB2eSNb95aBayyfNNwnuw/flNkdxKlwDFEd - dsE84C2tiHnk1eYpSjf5SXTPfi2wzXDpsW1EePR6nsfThVG5hkxryhEhBeCj5mak + k3JU+xAfc3+CdmMVlugQtXFNIAADiorcu/zCs1mh0Y1nLrrQGKLdiARPkUMavbgz - +mgaCiCarHxOzz4fedoC0KWns8BfTwCZv2vd3W1QiL2i7l1EwqpmsyvoWV4HHLKp + Ubv/GAl20OMER3fg87SguoHJNvkeHOSSsd3wiczXkB4H8tUZJffGe4Syoxv5CLhY - nNT0uwVgPxEsrvCYKlPItnDyh+Xdr4Pbtg0uaoAWyFFkVKAF7YVbC2SEUGE7Af/k + /9H1sd/Ducqjdf26RTbFDL/jLTPjmiOroZAv7p/yF3mVaXsQPgN1TOSJs71BmFKv - 0N3Q4Efmzj/vz+aB+8Pa4uflZ5ekf16o8US2K3TxOToIqawEMn6bfMJ5z2fi0pqa + SlLHwKjiFycMNbOoEGHL8iEUzWBU0HIEVm9ISQY91kOIb7SGN/PkOPQNWjyq5yeC - 4thFPlkFBtewMhezmzQT4qF7RVSMaC6P0Q0h9QIDAQABAoIBABDjGwdEqR6BpkDK + XLTL5YQ2qIy3J2CjOTUR1h5gEuK5S1+ig1j05QIDAQABAoIBAA6tC3zf58S0yaUO - 3Xyv8DocvFOtAzd7Oo3h/SK2LkI2YKiBmUROVA83+v4O4umtl6cHSA0vfaetUcq5 + svNIqVwguo3zFv/AGRsUHC7WqE0UgwKHV5g88DDFC+MVwk99zzUQJVkuWPV7CtGJ - 82wvOl2xpjP8fWV/vvpk74FFnY2ZnENw5+TprdgLnzcoLIzykMfq1hr/XXzzGHEi + KVw33bqIt8KbzzuDTFDIcS4sLwx2PqwIA03EM6g0JHVAt/vRhI8RkwIuNHEQWhrW - En4Cs0Ihu161WfKjf2c8yhGqTk4x06jtEwicL/UZo+mB9+mtXq5vzI+/5pJisDiG + tA2SUd9SDmN+Je29UoKCi6Gjc/OTJXcc8hTHIoAcoMX23pNQWDjNEKXRrXs6F7Gs - zgYAw/p7IEk8BG1Gv+99+BVQuNKSz+M8V/Bt+DVVb9Iobz6ehUCWN12rQO7S8tgt + Q2yfauPXf2cM71cZCeV9hkqedjIln0R30SdtAS7I2IeRNTsB25w/S3zE7xZryZa9 - VcJrXK0DxM0HrQvnSmmhbTqtdb7kUJvB4EeSS9qlv71tLlpqNFTvQdh+pdL4w4zm + +Jg++rW8hlbmJpm/00iIaJWhHIxbPO7tyQIOkr4PCFMvtHLUKnzCjmL3Gsd/BW+m - HbjOCacCgYEA+py0erJ+hyzEi8H3YZiau+3AWnncIkOr1q3LSD9gYWMxQjVpAqzJ + +QAZ14ECgYEA7O1dRawkbzapYPUNed54lBWtIlaC/J5vpAgnelqbxDH9BIRhxQnV - pqeHPRmemjG0NbzzOMv9jDdq5Bmev0Jggk7k2pWhcZ8dz7VrAuKMa+dpGmktaqnA + T7Up49feR5a+2x0QSgjsg2rUxF0t6vP3BAHfe9D0zGFU6YOa4HrQ5fH2/wXx2g9T - QcHFBTDHyuUjcm5XRTkcf3EzeXp/By3SqXhTRoscihVZWTN4C3w6hwcCgYEA2rvW + aOBCtwOL/C/DUc3aMD4EOHLXxismQXXWxPk/+EBtLUA0XIRLZSnuj0cCgYEAu7mU - 25htboHdnoR8nHi235bryy3GAajJPY3kVYKSpkiCk6uu0ZSpBCZF286ETdw5PoZk + XH7no73Xo8IqdJbUhfKfZSsIN1klCcEnnCZEuIi77x8ZvPM8pHyrJomFew5jsvZ+ - 2ldyzFvo9u/0dK4G+7t1oHDOwYrqO6h8G/P3nEuA3WKvt+Z+s6jG6vbs6uEmkmpu + EPGTvd77u01/79wLgKE94s8GvYnggWE1ItScBx3A/8PnQODSdEMsr4Tdm6Qx47oZ - LUxKM8M1663IlgRQ/TOqJ98o5enKrvmwUdBW9CMCgYEAoJWNKBn77Y4IGy2c4JKy + 5tgM1u7kYBwao3L9aguP9dChBn6e93mOqQuYqHMCgYEAkmkgcXSuUzeRNgRZHo13 - g70ixlbTcbk/AP64BYFmtsCirbQfp7EkPX+XrtUdxdwXh1+d0kUUIKbZ/XNVP2S/ + L/OxOP4DFf8GeHQ9iSPDDFvjwk3YaT3pXsdSKqV0jALA0IDGVynqlk+HSg1W2dGH - BoCbMF005+N3bMLo4R5dsD7GIEBI89H1+ay6HEtXmnEdN5Pwo9CmrBrTSwHtJ6J7 + PSe3Jjl7fW1MXr1gEQZ0XxTGkNPon9tGrRGgyJ3dfKs7ZSrzgUphq0x0wNZbXqpm - HFCXu9oj3W80o23RfDqMHj0CgYA8maAMVO20mRw6Z8BSZYtc5OZM81CRcx7WA/LH + XPS2LkAJ96Osd9udB9gAvvMCgYAvtyIAyLj0I8L1+tpzvArU6TCetGtoNh519kSt - 0hYpJZuvp/gWLpapBKWEIXI8VBA0B233pBS1E522lIJotTJQGf6bxcUyj/cXMjW4 + KgT5qreqNguCvYjCfnW6W+YzuxqYWJL+l4joEA+IMlC8lP/PeCyUw+6AqtUHzb+F - VN48GhsIuueuDpj503/Q5zp6VIioNf5yZFmGf8X3lr0k+uspS2AQDd653A0Ab0Lv + 1Oi73lI6MH9NPFgB+TkYe/sgHoIX0ivXQz8wOpSN4VbcCNRk6f9zic4EKpcZbCpY - V8ZPewKBgQCjv8GtGEA+dbyhaXM4Rr+GVprQftbs65u8cj51kvILVZnUGL4NlLgt + yXvKBQKBgQDHDB8/e4ZaiWY5VQmCk1GRZmNmuyII+vC0ZynlSUP6HqXn/Std/6Ze - f4991qQ1OBYyktmxsl2MkcZ3bVc64ehIgjp52yN2tyye8jUoZioCXq814Ru6OTd+ + EEfnVCZHEvRkNwRYAWcaY54uvqj+yeLYFeIX8vi+rs68uZ4jKI3Hz8vn7rgzPulP - hQPw1PP69yKBxosPoMuLTgxIuo3N6uv4bBiGgKut2hPIncCzzc9png== + Aa6fShcdGh8xhhdl3bM62/E87ACcr97QSvUsOuhIYwr098ivyyStCw== -----END RSA PRIVATE KEY----- @@ -15381,62 +17031,62 @@ vector: segmentSize: 131072 total: 255 - convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:MDMF:k5beixrivinifsorf3gsa2nwve:ylto5ecf56zgcbozhrhvfiftldxm7jz6tlectqkorp6cmqlef6sq + expected: URI:MDMF:jnnfwffwn6qx6qtfhvk742p6g4:gzqrivbdeczwpgkwkjna2taooajonyb3h7aijcyhv7ptaxtycroa format: kind: ssk params: format: mdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEowIBAAKCAQEArCBOrgZyPxL1uQMs9Fu0Oynkc2Pt/KO145f2D8q/1RlwaHTZ + MIIEpAIBAAKCAQEAj/NZYu1pNcdeSCnt3N/BDg1jmU7jlbD1pgKaSXLAblkSAhD8 - LgSWzRXdMjlNWOz2Yf+tp1ARRwSr0tLH8J+50eSKKqtyrZCJyurl0A+xOpOXWdS2 + CE1VLCsWfNw+dbTJtvStGncEiw8gK/fQwSoQjxxP91qxD/6RpNAA7RamHol2bMiQ - sPEApM0EqU3wHtQHVbQNZdnQl3Tfvr7lMPudnlP4JVJ9BALU+fk4Hd/V1g26dlU5 + 0gtjT8H32LmS1TBYfszI3AD07+cFGAqbV+X6lobQKRK8U2eWD3x8V1pD9RHLJr6h - Fefny8pBn0ET6Gxwxzsv9/EEzP5Zdh9RsNNPETs2YvjXKiXtwHSrLluKrub/5XC4 + nMPQ/epF1pxGf22+g7aauEEWPOStTUH3l8sf33Wg3WN2SkxDqSRyZXiETpta9Rdv - ZcWc+3bvcPIgk0DJC2+e++0H/vB4mtGrFW9JfkD4ZsbZFPN0hU6g25OKq5MRjNix + oMjcsWHqGOTy9y94oG2B4AJHWrR1mefFN3Uu0ApfpThoUcVyI7k1XN/7y3HhEZru - 3Mnr6APKdW4d3KNwdWkXZbwlrIBZJ3gSSx9g+wIDAQABAoIBAFEQ/j10B8axFU3H + GJPaFafHjH5DZeHC6fxmJkC6vXSMLClNcfY/RQIDAQABAoIBAAmf2ddbK7hghM16 - sxp7Pk1HE7NM8z8zk2zXmyog4WxqCMkJj2fe/W8lxwHqfwVMxVWuZ4kParO5/XrT + 3jAxFqGHpCPdKiq5OudXT7T+8t493s4cEBnG/92U4OtGt3dbt8vfdo5pLDjW8U33 - jxtC/u9d5bzm9qHMGzmYnBf77AqcjIHgbxKyzwzPCkz6ygaa8cFphY3coiNTBjX+ + QvITS6mh2TfezK1W1iqIjLNNWxx6ENyrmUEt6T0tKRLIr4hI8/XAX+KTvsyma8kE - Dk+dkcSJ46sgSITlGI2K1OUtELc5BMZkfHHgaSx7WMV49IZakzMzeJcRzu7notFh + doftNLCpQVQpsEU8TQRqjI6zlo0VQo0sPKZsvSfN6IOvPciLu039H7oqHxknHRy6 - zIxH+bMjSNy8w0Pb4lQZuATUJh9gsQKDI+XSWQTcYoW9/P4mEn41u/g5ZQuyLTNj + ACQIkCvAEH2U0OY3+ABE2y7Mn4/TW1eFsh/lp++FngmGgQcMIYZUR3TnOWx1wzwl - /0obuVSzvcaVizeI5v0CN4FeAS+dOhA+rj3m18guuqDoW4m7KDLM1QL5C3S7N2Qp + IiPfFo1jVAXa70Up4yHe2Nz7zjNh/VCJCTMA66MbPczOjIbUtc/GFcCvDp3qCnCj - DPQzbGECgYEA7Xbvydy0cOiVDXBWkqmXzA/eyuD6DH74hFIJFxIoI75shplvjLgs + nRjMDOkCgYEAw1kF9qQNgFlLTagCh+uVug1hvkZVPeDUd9MXtIn2TvjbdSkHilsn - uj5Q9RWukwxadlGiuRJla5urMlt9qhO6kx7pXYYWBdMWiY+2sx9dEf3z7NL64S7N + wrKguLRSgo/OKRj2J63xhvfgb91cMxsXb1+OHG+YNU2qhFhTq4k489rHID/iVskN - IXtSsaKaS+EWZHBfcfzhQMVg/A/9Eg5FIwUnIy+81lnEKcNuCjYqPDMCgYEAuY/E + Kr2GaAMGXQb+9t/2CqHCjkur5m5VxZSIM5OgRR/nLutM1NpHWG929r0CgYEAvKUV - KR7JPmpHOXOPCiJKWrhEhtdp1zTFeV5fERVM6tZbd7Qf4Ad0FRHfZtGP7kRKSEtU + yqv6gYMnolZBcC9LhB7nZ8yIBtlmYjXRHUuh1Dotx6UItlA41k6GNhFiZXL4xPbf - yWQC5VvWGDMgVwNnwAPpURYd5PN5yINwrBqsDpeQlj+3+06irdMvPII6wzs7KC5E + F69QzvV9Wuir9iW6b7tP2JUb1gnfgB+KH1F5PoI7hriUF88VIHWFeSg1ezwIMQvd - JdfwDX4dFcKXNgV75VEp2nfd+zmIWN2TFmuCgBkCgYAoJZuUvUOkcy3//6YjVZjc + 9IlJhlQTk2kvADo8jwCZahVGDwsQzsf/n2xw1ykCgYBau7axYHGE8/SuFSNXzmy1 - XzKDilW8FxtdA6GVzPQMVv1yJC6/08N8GV0GkovZQJVqu5KPR5TuBHuFIAK25m78 + BhIoNrLREuSc40dXa90jwSLtwCjocn59SEquf9LzIag4Hof21iwg7HEqhD6W3jZ8 - wJUjwq+mfHGrACkbT4okqJK8z06rE4aKypbIgX0kpwFqKbV5SA+tK7Gh6/IVQ2Rc + XH29Z3fjCjfxULVML2hsm2lx6TpP5QJgn7cWCJGkE+PI9y1ossmTHkKxvP3Jz7uT - 71oWkNOUScjoZqoL/+xUEQKBgA0FiLd2AJtPq/XdJSGJ7HvXSH/J6BSBEIaG19cE + eTYv5SmT+WauVtRclylCYQKBgQCsBZZTlHQA+gqAXEub82TXfB7kZnx8Um6sjAq+ - DqTALCUHT+FRxJSh73JwrFAFHM1b8/Q5/3YG7sw98jwI8iPoYlwdWDWz3Ez05Fg/ + viM6Fjt83J+PMKRDuKNmVn/1ptv3MG/Ld1EnCHFhHt8AvPK/xH1RMNeLXMF0Yk5f - eul/O1c/23JYP1RBaKQvY15F7s3QCVo6gA8CVZosUJ4q3lnmSzCYjsxNakMKMYM2 + tLntKHEDrvlMpMfNK52lF+d9EwcdQocJ4M8tMSoQuE/l0zU56f/73p5eRWb0SShu - Qi8pAoGBAOVd3g8BqGeJ3J/CrChPAau9uxAG3RBpehsqK3EHInoLb8bIo4Mm+ezr + xkI30QKBgQCw8SH8qMlso796eW6V4QKOD/zdfNTrZWfAK4BaFR5l4LuX7G25xZNL - 4EMIKjl/JgYOKM8h7QnazNQ0pYyg7k8bzSw6TXEssDK0G/ALGDouqFSDYdOyRUAA + rNe7CbDlFzzp4ukkW6kcJrxR/4RLJA6nRDR86kvzl4A4fwrGKCv1vY5y27ioQmWM - qe/ZiZGQqcMWp3UKxnQtGIsJEnXZ5nZqsJJUSVWNgIAAtZI0W3lr + WAEYfQ+aorfZ9lbO1+QEZc7kjXlbyt2/SiZxOqzm4CutbetMlaG7nQ== -----END RSA PRIVATE KEY----- @@ -15462,62 +17112,62 @@ vector: segmentSize: 131072 total: 256 - convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:SSK:r7gqup4re736yv5q6zeupiemri:wg76o4jefkm4rcq3hkcsf6ecfdro5qpg3rwotqhdpc3cicppdxlq + expected: URI:SSK:7u3yuuf5t6ixafggoo6rozlkca:c4vxsjiots2kqxh2xx2f7j6e2rgxtszskkvv2geby4f3flm4zcdq format: kind: ssk params: format: sdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEpAIBAAKCAQEAsHdxSt7u3z9iynZlqa9Mbx+Et6o/RxZxFWwF/u/LNuZJ+bgf + MIIEpAIBAAKCAQEAr1qCo/bz9q/oRev0Upfgc7gVB3b5/2e8jkhQ7hPyNL9xPzZF - X5kaU7KFqThr3YXvHGkTtsQ4AKVdKr7JqajQiU4fIyC0DVkzj7EvGqNpGcAMNcv2 + bih8Lbn1bX+1ZAR/cm8/Vtn58TTQTrJH66TCbkBYHryvodl2yA3gLExnfxoESpWu - 8iPXuUhvWujV43ThvI/AbGbLfxDeZAN3158Xrgh1FwEl1hReGNvwgKpMSoy0UxAv + tNRF2VOab7/RMrmseicirS706Nxk99aMk0Mc0kJ/3DnEmDJUKjfq+/zjCPUmrfv0 - CMUO18BotQPXtThkKDC56USLoGISzaTKI3tDjQI9O9hYVv4n/XIP8qkjHkTtLLwZ + 9xEiHAKHPN6EncJxQbPqWW1y7QPoEKoK/vBYusYrXa7OdDRmK6cmKWgtmNCoEvGS - TJTLdSVnXSzlDfwXJcKsPW2HIrWLOs15kow4681Lf4dv6JHeyLfruva4VUdxq2yV + IujkkeP3lCiqrVcofGGC00VizESa0q4Iyq8Fuz1LaDHNT5HO6EYPevNjW/hy89Z/ - nWCCE+p2+2+hEzvnu3MGAgk5bvM8OG0+kMT2xQIDAQABAoIBAAZqsCGGleg4yWpu + NZG4tOvodZsCsGqD5XlwyIrmjxrETiS2J49oPwIDAQABAoIBAB5GtuK/i1zu/2A/ - yoy/HYKnQlUdDp7StlXO1IX8FBHg5weQANYMqWwp4ESHvcZeSH7/jtJfyWCJTX1l + PraaAYeJY5wf10dZbm10oACTUhD4cwGyiadc9x/gCTeoQrXrruOfwKRqy2RxtXu4 - 1svsTKRIMAd/44Qzvj/8nlJJPGO4ZnPuANgLt8cFQbABf5uShGMt//4rxme5Bjkx + /YD6uBVYJ77a3kpIJWGiP6/2WzJPWClWkc0oD41YCYS87k5fT5/hrPOQ9XlVQFuo - S2zkBwFSoz1XqEJOrHWaHnad8AHAPhCu4yIQasv2YoshkcKx4WvgLt3n6034t5PH + YCo2/r1w/OmV0dNjcTO+5uQuRDbx75xtgYgICfHMOkFnwqJWRsNPtvq1oMA9VVrI - VIuqQ9SpzFUYR0a6OiU5kRURyKohBK5fX0hg5hwS1Rl2OTRb7ZhRcYrpCXSEoLPs + zu3GyKfKQXWMrJvhmE3s+HLpv3j3whh7W5yGw5Dum8bPxh7Atc+15mYr5mC9eoQh - tq3L0K+yGBmcawzwVYoM0/JaxxDwgzWRGBOjEXs8ujcx7C9wftJnjUDj2nxO2IEd + Wdu1dlY7lHX2uHJgcPwhLz53jQiqlLgZ+8nw8ZNvSeKqmqeR6nhrDaUb4idQBzBI - 5Ahyi1kCgYEA4rUVdaGKtmz+76qh4Ymn4T3Yji35+a2VGFoUb6GNCD1XYNz9jy0T + aWiv670CgYEA5e5J8QQuiF76HZqDkAN19yvMue8n7vckKyoHuhtTL+JOFdPkcob8 - LmbWUt1bqFEeaC9DTf0lwrLc4UITmeH3UjIB8uTaM/w861NynqbQgrKUb59JyWao + nb+WZkxyce9MjrRzSnXmvn1+HcY/g0+hw4qzpcQFJbyjaPREGKH2WQmYuJTHTJ5f - t2nk90tJc3YhDUfifvYEldRs4WakWZCQa6h3zB9z0eJdhA+arlqGTO0CgYEAx0SE + owfvfSJqK4p1tNqxAtCHHWJfOA6v9ulAhHZBF+2BoZgl1xZxOrGLXAUCgYEAwzwf - sbEcuq/dDaIEaDVA2BfITeIBOMWCoHdsVOwIMWHYEZ6W/04DDTLFWapr3Uzt7XHX + slrlisyPa04r8TfgMmwt+Pc/4g5UFC5WK66rWKMCzs8p1uLLhg39QwnrmaRxa/O1 - ixIcaucL35pjY6fdNNyCY2wwkCJfCLU94BpdYvgHzA27nUjM7jcndkGajMuk4rqJ + t7NKEVz1uOwikxmjJZCvnUgHrYVeCp0uVfzS6s2hY/iMh+36Ocy4L1/cBXEHXbdw - veoriuOhM0v2t+Gz4WK2XkyQTByj9I25AdkQbjkCgYEAt+ANxrm+SxX+dB8ea1J+ + 2TAkQwNYueexeO8dvBkuXll67PQaHsSfaRxfanMCgYEAsdIDpT3SruylClgA/1Nt - EodZ9H+/501txzGQr7YFMHCoRU0Ybx8tFo6cONuHMu6QTgo/eargDJmL4zv3r/EB + 2+YnwnROseS4OBmdODUBtLqUIRVqS5hRrb4JlrvwlmS3FHZB44gjF5b9/hDf9bGU - 6u3afMo3XMCyHGAzcBB2v/rdv+cfLrYQE6tU5Wpv6bEfP6lVQIqDz45avTrGBErn + LSILpVtfj7u/tN+T+mjnmBxv2/BT4dFprS/p6yC+c0X1mhS3aLHUjMkTUsspEw95 - iBo9CBdelhYWqT0KxW1wzkUCgYEAlEchMpRnm2eH50AbZWvTH7m6vHGjlRor1Lpo + Mfgyh0rLQinkud9FWlsMp/ECgYB5IpnsGPfpeejWxIcBQRELWBHiMs7hXNCQQPvY - 61xj0FNNk/bdx4bGcIjKH6nX7+nx1lFzIbJNYSMiS7Y3pQ1hZpd7kv4LuQVKkFFF + WKUZ9vKsDN/B47Ax+gYVDVewWdbCC1HJrCWdxlb0KRd+u959VVuRM/sHkAN8hHAW - hMA5o46LRsUlSanFjLGP9Mhmd8SFoo1KN/7LfeNarbAmG7igwONSbyMr8OcS/cSD + jCr14yZrF/Fh+adTK5FwW4LxoWLXpBURvQwSxEXN+1MjXQHPDrS1d8GMuhxm0Mqz - 2aMrPckCgYBM/1kCd5r245+iSPzO3k+003ECaLhQOEntBrHvis22OqBbEWZFujMg + 9hXBvwKBgQCpF+uROS92UluZoWsjLuc0Tz5YP5ykYzXM8e5FBD9dzs8DEudMyhe6 - nD5QrFohNIWiH8LH0JgAQkP/DvOIEYJ6f4LdIvDWqxZMEGbce9O0jzSm6mAjLV9z + vsSmuxaSPNNEAKG26wO1+774W2JFJkRLeJOpeyX5nhEl1tKxi3ACe+cMQvd2/bHA - 5x8FCU01dNT4dBxkDhjYOgdjsXIOc2P6ofoXk4Se471uOeKdSTWAeQ== + blKtALR2ZvXcoU6QC8XVABQ8S3h6+wMk4i1DjJ58UgHZcoHrozlBiA== -----END RSA PRIVATE KEY----- @@ -15531,62 +17181,62 @@ vector: segmentSize: 131072 total: 255 - convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:MDMF:qig5fstexhz5jhoq5g4nysljvi:uyagmadjjoys6diivtrztzcfqicwiwswchfwfcmppu3fmshdpsza + expected: URI:MDMF:kwets5gntkkdhed4hwdsfv3ujm:ws5ltuz5fytzncl3xkfkqc2f2zrrdnxarr22rckmkoruglgyin4a format: kind: ssk params: format: mdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEogIBAAKCAQEAwGlYMMfNp7uC+5rxGzNItvC0Y3vLkOGGpZjuR3WzjZCG1AS9 + MIIEpAIBAAKCAQEApvGy6mRElXz50jCzj5EW5hrMZo8fXQHgZghl+yaQ0NJoVagZ - IJJVqEKCFet0AHLcUJT6sfozrnus5YCWwBul0pc4ieQC+xzP3OIev3e/93scTaG+ + TOjtkuzVYnOlF/9QxOt+yAfuc+XjPn3DJGbnnIUi/8qOUI6CwQY3uUKBK8cSRMTw - 6kp//CGUHOxGkwMjZsc4tPKitKeTwoXijVaXvEVViwRzm7K8OOBeOmiOuhqp6Bu7 + A/LihRrwM0R4Qws/QiBQhD5KmanfcMlu309Pg3enAeAVpt86I8mgMxE6f26iXgFj - iyMmfCT7Po/IlbLfXmpk7PIAuyxRPRBuCQWgmP7Zm5UmsWudJ87P4P243ZrMOVTv + IaLFx+5el+PvPYqdOkcEDcL+HXVYDVEawzifkXGq1JtWWKnDhM3SsdmS32mTh2/6 - uJn2qROPhUOTFg3Lxv/i3HMO0fOSPMJkYBpJZmOvsF7cnFEAmfSH9blLrGB5lfwC + RmocrD7o6LSdOhgbhNqWdOvRt/ZLZif4Mm7TqG2nHFJ8OYCkpHK4m+JpNOiKmM5W - yv+VlKLhxAiGgGxhQo82ltgDuN71nPYOrpmfEQIDAQABAoIBAAKx8kDdqmWbI85E + BFdipmKRo5jhGZWG7w8rWAqsNY6GMbcTXwgNoQIDAQABAoIBAAHqoq2Q8N4f+Qy+ - wqcodUNo7l5QQ4VqhhV5K/yotNMEVBuLNPLfhdv69+g/cYY+yNOUOjygHIiPdcUL + ESOn7GHAI0JWqIskbT9yn3wYg19YWQkJtN+miWqvRBxdHEM4I8Tc+L/CYo0LUbZr - Se4T/Q/zKEEBmZT9N6alOeKUvNkMmqFpcduFhcX19CWH3dDmy+B4/o3Fsi4XKx1M + EnFRqp1IBIC4AjX/ytW9NOjQMAQxBP9L3P8Im+vgBSurgK9xWryvOwlnnyrgMb/d - vSPqbREfd5kmxVYU5cJjFykyVRR2r275abi1AglbHzscScBm2fFhDD3x2tAkHj7O + WiPfaNfnKOBLQqhfpe7Y/tkzPI8PsPWrhXL1CZJ7o3zWjyn/l7TNvqB+i7AgcxUx - kCIN776u2Mwz5r91oZKFoTaX6OzwXqEXWP4K6RAsjIqNPXiizP3kHCxyZ7YVGimI + cgIV4XxY/HKOHHDz2OsPxJk7G5/iLZbl0Toks1G28NP26rOP49UPBq1eycfnZ6Q2 - g0wySSidet4ebvEr9qkONjkaTh7BfLIWbgiDST+LVPbWQ6YJm2KPMbdK2aBqwNw1 + 8oywX+ivnK2AYZle1cMynCgIOdZNE+e7eb+bB539LjUtjmDSculRFvUn5AK7TV92 - MiQ9uXkCgYEA+wCUX8p2vEGer7tlwosrvsES6foF/10tLBAD87OJXw1iXIrpwhQR + gZfCrMkCgYEAw2cjlJhfPg2d+bMGcRPDGLEfincev2r0rbok7beEra5OqNaH1CD9 - XJxSLYlF2jWZo5XNnoyxfsetTRzgTzbbWpMklV8bmQ7edGfCdhKWnWA2B1dsQHfe + +ohgZd/WgrNbHsF0aC3kh+6p/kUrunbIYy6ZcAQKiQ0vnOie/TuLcPHWocxSTrLh - JTBhYkIQbnbYR10xlzfYUHNCsFEjF2GMvSezfuLSRLJwl88ryBcY69kCgYEAxD4d + xm1qt/Y2KBeSaR2MyhQVChJSWosRJYj4e/ErfCr8aLiMLRgA3dGr9SkCgYEA2rc+ - BIdrVZhQkswwbWirDEayrGN0qmwT/rD2C5Lv21UUY54vSyoQiyu92mWgrRDSKAPo + vTulxOHBKAnlBS7A2wnfaCdyYL/Pk2/Jec4yaSd+xsYgcIuH7p2EjO6xflU1oyZk - Owne4ydqHhkb36+Ss7dqUAEoXqddlWdTQ5K2+f2CLq6FJgfSE61SOzEm/+0LImjj + U7b6YvbpU/rwkJksdFiDrMrN7EPJFynYK4g0CRPl4P59Rc+gvJqDsVG2Vh2/ijeg - tVUOeZlhjqoVpgfQ6snCpYuUxMzOp35PJVGXYfkCgYBzh6fDk3glXHrC3hmPeulO + 0ybnNZFz4sKyKRVqI0Wu2ewTcDuduW4i5uemK7kCgYB4rcsoq44uycQmAa3ZykW0 - qqWfBlK+YE/LaS+4exmuo4VznQjNKNl47AazKOz67BLkha4X3SBRf2zYAoOIUnKS + izeakYT43TptzMef1LZpeXx1A8Fxfkq9HtrCMCLQJ6r/7KRS7vz0Aq8ULW4bQ97w - dQmwqw8T2xEvORb7q8ChfUhBBs8vuTyJl4QrascPYSpZZp7NwImTNgorB52ERIU4 + ekgjCSvkhrNAKd5/MPYmdAWFeaXfmtSbctn08WdzDVPL/YcFCrAPv08DQl39m4Ez - B08KBzLLJerHJTc8qMzyuQKBgCZuC3yxkEFo3I6C0hD66FQ1HBRKPbSKCbhcqzJF + MrgTgIzQtCFGfEuUsziLOQKBgQDI3+noZMMACxObEVNdKi6IPg4Im8op36Dm2ZGi - Chenp6CCf7x2dlrqq/ky4a5ClwUjDr1RB4bwVwWh4SWC2nW7O2SDdYZjvB3f6BxZ + pGWaPGLsbwVWOGB1IAigY41y6RGlMVqNpI1cnUd5EQ0m0PeKN81fwrfUGgGzm4Pl - hN+b13yQzJ5P9cHItUvGKl7/6qhIZh9Ckt0ZPlOT1z12VmFENYv5s55+hRGj2Jf0 + n2ejOrozpagqmOIYtpTjI5giiZnkeOjlZWKOyXM0vfphT0C2+oX3siG8P5TBvMyj - THLxAoGAKJjuqags6RfYV10+jm1n5mVyQBJKXj9jkWdMh4oV7a/Kx8LS/oNON+9g + Y/gzSQKBgQC15p03OtcgZJpBwQ8bEGYVLu/1YiEQS2TdIKib+XOLwUfgWzr8KHsk - jdL1CsvLH8qG4fKyysqzT5jEeSgnstpSCbkA5JderOyKzZmVcSaJCamy0GlER/tf + 1rv9A3s7OxBZu4MD+iA0WyhcNSzexf6wEOvada9LHQl7eRmsW5OKDJYWyh5wo6PW - Lm8Svjy3AMykDrX6Id/y4hkgPfPVAvGc/1fPxpn2TEsGLj9wzxQ= + CQLCE0HGlEZeBF6vexE+oD2/9T12/x/q0ylExTmXpWlkiydG2Vw55Q== -----END RSA PRIVATE KEY----- @@ -15612,62 +17262,62 @@ vector: segmentSize: 131072 total: 256 - convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:SSK:fqqrn6iq2w2o4fvbhce43gkqda:jupxaorkcxvuotuo4dghhecoi7drygxsmqu57bgpuadq6eshmcsa + expected: URI:SSK:m34dbj7wfsaq4efuz3viwht42u:66nxvmks4d4ebbj2c3behcobuk7w7y2i4yz43ethbflgqpgf6nha format: kind: ssk params: format: sdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEogIBAAKCAQEAntK4Mo12RhDyHAp4wqsbNWIYTUl5n6g1D5dz3CoTHzCL0bN+ + MIIEowIBAAKCAQEA79z0LJE/B4nEn+ehBk24vaDlLdj7N9ieX2njI0PrBfe1hiPk - I4HY9ORmvKKpfugBEJqSbslhHhvwe6ueUijA/oWUcZAAzJ7JskBqiSjaZ4fcTm9+ + YOhLDWXh+112jXJ9wLBdv5UrJxeFw4hxBADycqPSkC9BVte646lCANS47G5sHibX - ov1FN+6QI4EJyNQrsWGdXSZ7JDpgSkEgFdEhU40aUL/J5NDt+dSCag5UUXcsZvrj + vpmVoeuIV2GBA5OnWUV/5Z57JrNim3dcqJXtifmj8HmxuHOIBWxmKdl82jf9AXcX - ImrCd3uYGEvT6SGo4pbT1F5hC16pWU+GENc4B5sN8bfV7gSL/+2WainijX3yznhb + kk+F588b7YRjUNPNpv4PROeLpFu6xv7RzdyZJ/ByrSCU/WpW20OKCqNmBWRp0uV2 - qdB1jq6cUy4NXelqBzav2+dRzAVc/QeAzy6ixEUdrs8Fdy8tsSvBFh1KVTgcvnjP + HS5Ii5NCql/Cl21O0ITD+XmKefD0gz+ek0XHWkRMb394Y6igeS+ZCJadMqD7FzBE - t4h81NorBMU6nGE3pMpSH5dCj4zYO7w5FuBN6wIDAQABAoIBAAmlyTSg7TdSXjKf + Jm+n4JOCmJjicPRHPFzUWRsTOTAzTDZnzy6/MwIDAQABAoIBAGuQhQlVa1QIkp34 - 3DgPBscWX12Kgg3VvOtmis4r7B9v5n5lhdsITzKJEUiSJPOlijMFALIkH3chwVjx + 5Cus//s943h/dQ0Svdbg58ShSQyIjKVmhBx3H20XMtOkEq2U2dLm5GutS8hAkrJg - 0tswyJBtctf3JGVJm/zs9svAJIw3b4WANlQWSHceUbkmZH7DjKfowxNefAp2VVUN + hfn7KL6DO8KABoeYv80nUpuHyZPxYtfUqGxneIQ/2QkChzYg6WutsJC61NRCnqZE - YbKRkpGtt5SCIfR5UItlgkqI9kxAdV+ml/SW4HJhfx+2qFxfHP613tUaYYHev7qQ + TU+myHrW8g89q5ahbK6t8VS0HPrIQZOuqmYiGM/seTLbsQVkppli6ZfCASMV1Wqo - eH0rSmCtrwWUYC1aCM+zCau1p3WWO5aYOVccaHvDtEJsS8YV+LJXc4YIrCvgPHBC + 0A3q1RiHlLY5UCMYUxdPDv1MrhkRqMZ1HScHtQuqRrpijXcfeHYQwU50CFgPs2nJ - AQZLB/Z/FwG+JRJdajWhz8Wtr1+MtQ1Gi9vQb7tXeS/TE/hVXWY94nwbbXCUinzy + kk3Tm41jAFzYoBqe3E7p2R5Wlc7rbC43yalQ73JKRneS8IdAu75SWWcRH5+QASRa - k7OIOoECgYEAttcm1vSIggzZpS0jV4+PiEDrN5GBUIwPDyZNn8tca4sy1lvkG/dV + ojT2kS0CgYEA+AVYqwznAuhEp5x6w9uamy+/jwEOK1usM7MOo7PrPCu47z52t3Vb - il19L/DsmLz/eJKTo/Uaw4X8/oZflE1MAw3XniNmgeVS92i+EOgOtkaJnwjrGBhF + M9lgo8eY/PVn2jlgSMz48iZIE6fdr43xdtpxsZQZVu4bpXJ40B3YdNmob1easivx - Nfv3Bpvn+GrEzVHMuxgRfq1qohaSMGxHre3XEorssgTaDbHo432aICsCgYEA3l9o + 4fPjZZ/OTb46veFN1GNsndyQ7IP49NkiyOStJUey3LDxJQiIP56lxJ0CgYEA95Rr - jd8e0zWVkd4/Lkf8JMInPCBRWo+gyBfbke0CcrexWgk60t4KgKeoUn2n1ORBcLpU + 5S17yepS3dwkcr3H+zN7/UK1CnCzzxNKp7iTqK1YYXeOjNZb5s6qrtwAZx9GmRWi - fBK9kRk1T/t04fyh2nJSBh1STlrOaeuV0FeG7LblhKOhQSDDedvqn1+OcarpVh0G + U3gEWTOcKaAQGPjHA+ohYU/qDpYbY+EtfHVDJqc+hMy648vyWfeyZuVghJxtSPxp - CQZVpb+Rth438LJL+HlTydaFS+1olApNuu/l6UECgYArANN7vyvUGp2eAc3MLFG/ + jDJT/ACxnD6fqDzUXRtViBfaSia1VKtCipr7Ag8CgYAI7mhbAIPxHtwaDRB+rRHM - 5DTubuSRQz/PelzLdpMYIDcmv5oZEcUms/JbsjiTe/BCNYdQCrfuwLbOTmBwivWT + NNP5GligRxTUZ8ZHLttxt0FZnC46PQejvlg0jaN8uHmc6iQFexwb3DUMQCdDgyEG - yk+qO/1CE+O9mP8LDulW6aQ4qWpR0nOEzOw+u7CFducuu0yBvJlwx+zKjrB3fyAk + 3qbpdiPTdY+ZTZ38IJcC3jOqjsULVXnIYTf3GOIc+pSy8cITu+DVbnPpkHcOmiMe - wknRbKda/1Uh33Q8/S+g3QKBgGUk/lhxYQLuf36emRxC78QEb3YguQA5DgeVGnDw + iN2TUhmmyNhmNQBzCgt+IQKBgDO5coFC62XX6tAnOgYu2CUHMJRM533y5d4Rbbt0 - XcmyFb//LLtW9W35VE1ZDCqAO+e7SMw6dfD2h1I+7LYRg8jpcLeJRLORCAwTdMwT + uIS5EonqbIHIFxM1gjteA0eIJTu+ZVeC74WjXrDjm/lboFiVBbxK8d9yRO6tEM+7 - 07H9qr2+84y9C0x1I+2juBWpiIJ7pxAZyoEedndgnU8kuftlrB/FLFIRxRx450wc + v/fHYSxliXYmGc/qC/+rVGrgM4TYF0UPDrTLgE/gVYLUkpmRKGFyekyboa66yQAk - 6/VBAoGAYpZTjnJhuxY/RSFJPaRqY38VMrRx/+vYQw1r7bI1NSJJtnPnRdncHbj8 + OOTJAoGBAIm61KjWaZZlE5u4SwzYijI261G9M6pXbSChEjket+TOd2wHMRK5bBnx - eVO+4mL7J7EKP+L9VTIOs5DR9lOMZiPgiATK3KF+rNOfJp+f4Dm17wJrxBapI6aq + G0wESE41BeycrL44V4YWzJFCB2qB1efwnbNSeWkvxOsXMFok5IZxFOzKjPflcALA - 50Tt/b6IwVXIg0O8JAhUfFAzEf9H8XXBjVE1thSyWROSin4CimI= + fhejJzLC8m3nOklf7hudeDOYUKDezMnrr/ZW5VFHgMAw3rqq6pF+ -----END RSA PRIVATE KEY----- @@ -15681,62 +17331,62 @@ vector: segmentSize: 131072 total: 255 - convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:MDMF:yopwhqkdoqdvko3rkwvvvrxjwi:5ymwi5mcfinhtle4m74nco6yw3qfjjzxfy22ix7vk55ybfzkha5a + expected: URI:MDMF:36rzjuc6qwedeiybhkflvevttu:as44hfkawdkbcvicdc3nfynzgjtmznyihu5773wyll4rt64bbtvq format: kind: ssk params: format: mdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEpAIBAAKCAQEA45joc7rbhjHPGrvTZjJm94Qjec7SGCQpWsc5NIUz/dzbuA5E + MIIEowIBAAKCAQEAxts3c6ig0ClkvLx4Cl5tAT6xpfuw0NrZJ2RhFFBKz/ddOw+X - NLOOQ1qpVym7Nxz4UvDbX5Hc7jWaaVQYClq9WVYxNla90VGBC5eyZQL1p7cORmBB + QWp6zneMjN3j/vPZzJLq3ZECGqFdsl1o3qYSAtZFDLnyh2cewAiYttv9EyZLHeIF - +eSrbPvHpYUMC4WPR3e5CeeH+dKSmkS2QTdPh2Jmk4wS1CF1Anf4fBd1XSPIql86 + hSKqa8IegYtem3pNYzp7W8K4yxv8KIq3cO837BmeDaRNUVv88sO03AWFyb+4B+z9 - X0D/NagRVaFVNrutjBJ9rMVr9utrTpLH76FMpfZHl+pDSoJMzHUrbjLHK7nwcH2K + mCJsHsHrfegoXCAKKSAILPTfHB0wbLEl/zh6u06yIcSRzSSGtbcun5qoCH5AGsf6 - EZY+UHJ8PEG+zDy6rsRFPt2pG+vpGIP8TyR/X0uTqasPMxMLkUsAAIQvT7RlZjtz + wOWDJfZdJXF35tQ8RZnV8edYMmH7KeNQkpiYiP6eGN1fYtLJJak10nvy51460lm4 - KF8IbnNxPJVyP8Kx5NSeH1sfpq2N7iHA2W38AwIDAQABAoIBAD7rmOfVsrbIsl7L + t59VbMG5UVmsgSe56xNm808p5nQJ+XvclRcCFwIDAQABAoIBAAnCOUrs6Wi9wfVR - qkfIi1rGNyCHouF1rdEg3pm8cYEvO7cIiqafNSc9uy8TpGQ6KBSV7a/gHVnli8iu + 1HGXJ7H0xy1g7XuaZHy+kYBjM6tYCtRlU/lrBc17EsPY9yGMYFGzRgu13DdoAtvy - rO6/4zT9dSF2nYdupuRTjcgLJ2q0Wsft+I9jPlkkyi7iN1BAHjo9yPQKBDd2lXz8 + zJIx7oFYIuRYfb1FA2kjeDHFGFAnxBjv03CF/nyyS+KfRHch0zQS6ySE06JtZFGt - nf4tklj1RTJpORNYJIcIL7PusE2M/3je3eu8nKdyaPvV/ysNqQeOPU9EpzoO+737 + rl9ARhYekwySqxQFN85xaenq6swMjTpslHpkcNKLLBuCIXbQT/351JqW90p2VYVT - gE1rMtsHk4kE7Bb8DnNBzPEmCDaajdtTuHpY1Y1pavrM4FLp7GSJk7Qa6o4YMKB1 + 9n9fQ5teggKaTQYcDkYR7uPMUJRDQRczLP8xU9naqOki7Q+SAnmxbWOO3jyJIoQj - 5/TF8pUbCh4V0BJQgbb+vAbLVjM8o/CPlipviaLymq04GxDwkSq73+3JnltMGM2W + mLqWzxV4l0Yts14WYDiw7SpNVdcN7qVcfVsn60Do6ekxnJ7E/d0GbRsrN/VdksDr - KIJsHUECgYEA+K3nkONzJeTTVcDiXtEeAVz2cePvwr6dB96GubCUU5JL2fvNiOgp + RcjJv1ECgYEA5gBkKiDf3kvZT8R55g0kTuFgEPPLVcKxR3D4QfUitioMHKTe7dRA - lmHRyov/a8v2UmkDAsSAshd5RCDn3zkFNNzz6fLqxninYnx6/ZNgMk0q475POxjh + xs4N/0yzgurAx5FzJtMmcWFd7ypj1+UaNgR3Q1q+8n7mdpxEp1dnN5XTEa8sEAQx - o9EYwILnBs5t86tHYDElrwvPR6/zVFthi+mCtFweLICorB8pugLy1TMCgYEA6kwg + EF4MYV9RT8xnVYzmXtWKRq3UYhYM/rYcYlRs/SYaRRvD0Kd0oZxk4ecCgYEA3VWR - E1y9XkY/PExVAS1XFHi08byK2hkpXygqmY0U0zwKwXmEOc5hHRBKPwQDWoRvB/JD + pITCHne6zQwgj3hFyUsHCk59KeCbyiBP4BKYLWaFKfSj+bgyYATvuZtSGDBJUpyZ - Dvj3XkxCxbXnEZGLzSrjJUUEOiRm0oEnMr/AsWeBCKMoT0+CiXiUxfuaNIkI2dH2 + cFXw0IlLTgMGqGQvAMySXH3aDK21aGglkDxpDpIBywxW767wmvA/stP/T7IPLvtp - jPXX8E+QGoSYuw4RJSTRaMOj3iOVFljGfKuhnfECgYEAvqle3Mh2dXw+yAWtybKd + 7ZC85Zt6v1LmAy8/xfZ9Rvu0qilGTzlFbuQXOFECgYEAgc44uZoCHphaFe3CCjbi - RcBHt0RihDZu4SSsuNv4rSaCf2u+xxPxJrpzBc9Wkwh7H+4hf9K3NVQoBqMQBCaM + he4mZIri+AzANpyoT7lElOCYI1ZdRoZi5JCIT8x/B2Tr1fXdsky6xoR4GjGnVcJE - pl4tqJY1iNviwfDcv2RqIcbmdlxoFNBb16SuTJNQm/hTdrpAbDDiSpZMYxM1Bd1W + D7ZnhMjjOUKrWMeK65KlezaAf9uIF6X19tHNVOsRneKzcxHpNh54Qrl6Qr1FKj+n - KdZr/uqNu+Mc73KpJFO0aN8CgYAikv88vDe5nLYiKMV2egFapQFWltMKoiHnx96Z + N0uEkz581wH7enf3l/oG6YMCgYAfz2uGNJpdnKGZVLPdStDk1EanwY4VlbVuQGSa - cCc9kKOpr0vi1+Ce0FOUfvwbtGVKD+bzY6vlP22vDUu+3PJ7YTPJwSiBh/OgZqyp + dLGwXLqoxANJIaMDz9HQYDVVSqNPHziiP4fDwOe0x5SOYQ+sUrp6VpAfIFwhLE6x - IYDG7RYudx0wrvP9Y0zY9mroC7zBn+k5HeIytRr3vs9m8wl2qLs6MXySAEA03v7T + wyzqLivZzeU0v3TPH9ZX0kYwYwvxmaqovROZAFaM5tIuBP1qazmoGQbnKdV0D2we - UOR1kQKBgQCw2euWQNd/s7v/GlldPB1RJcd+so2cjgoLRMh3ar9Uh2Y3ySKwLSRc + OuPncQKBgFGxH9re7e6aoCSxRnPwZgD2dbI/Ddpw7qmuRc1vSrwfQ0I+eNyIQzhv - NmH5g3dl1K+VYJU1maB4ECWB/Gmh2bnfFqfUHoPUEfssB/nL5sEWeE8l8H/osT/7 + sT/Bkw0J2SY3IjLdFgPbqkKgXX045kT5acVJFns7dQG4lh+sayiUjM9dhiG8Dhfk - MfTZYIW4CBmWU8dTQR3kcPm3+leOfiZf6HbavnH0j+xbMskw+/otjA== + U3gbMspIoeGx+zH7RX9NwDaUP8M7oeyZKBRZV0QX85hxwv0wNF2z -----END RSA PRIVATE KEY----- @@ -15762,62 +17412,62 @@ vector: segmentSize: 131072 total: 256 - convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:SSK:ybpdoqejf3ioc2vfgaq4xo7qci:3lwqfnsmzeb34lpdq4fkj5yyahmxt3b2yvhsxsfqavtrffllzjca + expected: URI:SSK:xd45qxekarwrt6evpagw3yq6qa:56q3eead3qjwxvxxrzwwoleounmeneuw2o6mdcho2jqaghv7unra format: kind: ssk params: format: sdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEowIBAAKCAQEAlthpYlADDHI3YZuTCHjeaIW6ZfYUeQyi3XPo5bSiASDWLFb0 + MIIEpAIBAAKCAQEAw0dwdT9MuyMSXaX2etcE8p/PwUd76I0p+mTR/0uiS/L5t1Dn - F0bSgprF1Jxp0etSgj8eOozOWKak4Mx5Cl5lJKon+IiT6oU4JeZQ4ulqTUxvxikv + wW6xgdXqMKKw1OAdfJ8ANrgLkoTpnl8gBlalR1OKG6r/yLhVFj1qoHfIq/iDdyH+ - ta5qD+jCTfua66Ntua08E2FEzqf6zEKwDDsLKps8KNdJA03BNrxDzKg+lMgSA1kg + O+9P7deS0Huowz6WsOkN6HQ4vG4Kswj/SxHcbWuSmffb3otKkNr8zwRdfYDcCrv5 - tAWRfZPaiKUP8T4yLI6LtfzJwm4p5KPkBSKJ2y+m28sbsPtaUXN2B1zhHYK/y0lp + 8PTE45SYFCU1x/ZFLCXb7y8pyPQ64h/M2MCNE3NvnxntkWMtxPDdc7577jPM+9Pg - qZo5UrYr7nOVrAxoSMOUsPTA3+BPBQ9MfujmV4GOHDDAUsSYo6DZnnK+hC3SiUjq + A4Yo0Oy+H65rN3x6Gpgtm9p0A9HuD++XFIs/vfAdnHBn9gdKegtUKxxlcQZuAm7L - vQTengWFVhtScOVY/onivy1aa6pf2e5lOloq4QIDAQABAoIBABc1BEGT0w7pzjYT + Oj4JpeckQj1DBcOz4V+uRnJVmjXz8GqPUjt4CwIDAQABAoIBADHXTv2t7VCqL+rV - cUjZukiVClsAGPY783KOj+4xyXAoVSBNNbsUXPQAS8oB+7sdW9Gd/vs9s8v/wgNe + DCboMGwYm/cR3q9JMFinWO3XCRJnbpL6RFybexTPW1mUfJuoo/4FuxE8Qj/gSpxQ - ceDJKlZnDJniqUm+D9KvibzfRvN4JwTITAbE5pxmpZN0eh52mOhQC/C6A5P8/qDI + XSUhAkhFKy/KokRBv/3QKSHR0NcLs8o5U+FKQFVNOzyZA0bp954prZIR79UwJOzz - 7Gd1Tf/SvNarsxfxsSAdUXD9vA23iFLnVBYRzLMajDApRbdUvn+VTLNee0emNaaj + M+2eHmuTFDTWDTksxhhYvEO+MnpwchZEUyUWGYwQu+bCIrGNI4SzfJhlRM7wBUy6 - dpFfe45Q4ChjOeyZIurGUKm/Qf8yGtfym+asoQ5wQE+E0Q3c+a/T7k9MsHpPEkFk + 9iZ7e+hKsj3dwZVKxXNq2m7hhxZ2h9EqfRCq2LJjFiSto+zM4crDgQslxzntHdtO - bt8H2TAEpptbyL7HMWAQNsBKiKiOq+uQTrxd+2oVrclOkKfXQBLthWunQKQGEFI9 + bKn5McyWtR2FbzDhz4/uPc2E91+JMjA9jIMZ24egXs9OSejN8rQvw04DojiCKMHp - DQabWuECgYEAwEDC3QMhPnrtJqYcm+U/owQnmozf2sHopxctYtEQRu8E5RzvP04j + cDjagDkCgYEA3mF3bHymDPkc9EiDkr0vx7cR0O3+Caw17vVlQ5OtMhCFLhW7f2th - fbOA3g8jx7hDc/dMMcoFFCl4XmxN2Vf8dJFeY8Jn4IgUB0G7MrA0US69+giiR38u + 2m6e8yZfQA0B0srQM5rPgSE+9U/HvLwSQzpZJewfOevS1hohTvUaf+9CRfRTHxKv - fo/Z6xnYmeRwWjbA0G9zRxh2Q5DNErgfognWtPd5JB8oBUOE6/MS0ccCgYEAyNzM + nsimR7SIfi3bcHUltPDiGKkqlmLNofN2zoB1A8SqblwnLHdfRP/kQI0CgYEA4M0W - Y075Pjw4oQ+wuaJgc2xrO/W12ady9cgEn/R57lhADjjqj/MMjNObo2kXsTvXhHJa + 7FoSWO7a/26MW0ibxhxXl8sfcgzIhDQa9P353g2HDIc2AQK4//xHQ/nXEcL1jOvZ - SQnRepjbY3wxxd/tXXz0+vSABIYe+FS6iOTyH2uT8phWPRyTdRgjPxORC9UcNrmt + I3IgjqjfIjW51HIwbjEz976QceIuJKv8xw2VBk265tnE9171KdpKpv9T7oooQVBg - ayltx8uKc/ejCTDAXjRWeqpV/YuCclhS4NhMHhcCgYEAtLxtNO9bUmyoA/yHyrtD + kW7F8leKTXIFoFH0Ug3xe//sHEFffddg7sux8PcCgYEA2xX/B/yNwz4xzmEabxi/ - DxK51J55WCORf3vXjB10yuqrVGTWOlJQJT0aeigLgBenOh8Tf38nNSQjZ8kzio8f + 1+x+Ou4dVv44bGGLEFaPTUGFU+/JNzFdyEsvgbGOKZYm87sn/49HW5qbYiblSwWm - 48pBzVEW7Mug4I2X2fgyxttFeAij3skewZakzFN5AHv0b6snqwwLeJvzmmNHl0CH + oGD9ryS/Ztr0bkZ0BkvnfZ8EFdMtiPFp3+8iEobD6jvXcyWWrnqa5VzUPjC9Eg7A - ZIMRWQGJ3j54FjK6hEL4v0MCgYB7V7rasMA1C13q6WuoUqHPvyAKbdQBl+XsL6tH + P5XCsqGwnuVfGqnITDwmbYkCgYAJUddacx2BnF1t64pcGnWC0Bf9jgk+tDL38CUR - XiURy3dqRGEljCaEw9yq9/noh8rMNjxi5XiRRBRUfwtBa0SjR/xXrpR+Mq4J6F2z + 9RmP0CXCKjTd89vxmObndYsqDFgbwIdfBdM9ttiRVYLfwOArIVUTN05LumHJWWwA - eoOD79el/Q3X8RhLq0rZjPZgwbjVkid/yqzzJ3YZyFOBbUJzlsgUA01SqLyt4rbi + YJrGCSDvgyW8T66ATrF7nOhA4m9qzcdDIEGKm4B7V3fOPrePU06oomKKhVdNI0m8 - A3CaeQKBgAYBnzLghXOOzISsAIxU9ei8mT7YBC4Hj2p+9HlA35HdATtY+Ulz2WK3 + fKti3wKBgQCRlS4RDAQXu+LbCwvGfiR5LRN/UzS2TwetqoRkrE99V9LvO24aFNJN - cghLqGrfdZccG8iYSnno19sMwGmT6UyV5Gv+nG5prJvBjiOAeUwseHFIDVxzHD9s + HEtBxhnpo29SXStmRTiV6Dsyp9LjQxFsVlSsiHN+f5SwuKKNMVxZlRAxLx5uCE/v - PMMImnOXcb/EFYndL7VhlJxxYCjSHj0ZivOvFov18OmMu9wKECuD + O6qsKi7zb3B/eTNLSKk+KjMM5PSpfgKnb30I6hHxUWH0cKULheK4UQ== -----END RSA PRIVATE KEY----- @@ -15831,62 +17481,62 @@ vector: segmentSize: 131072 total: 255 - convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:MDMF:heohri2cx7hxspitv4rolwkmie:5sjg4nzwgaaixlznaespichb5spo6plws26d43ooagu7vjss7quq + expected: URI:MDMF:xo543uh52xe7gpramuim6dsxt4:5zxemp7hunykp752nywtbfgqiacrxy4qzptesaaijjlhvwshrjja format: kind: ssk params: format: mdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEowIBAAKCAQEAmc2Qc56mQzopYjf7p5IuN3YoTLceZr9jwKRjVo2q3EfpoLXy + MIIEowIBAAKCAQEAlS1aW4nTOvDv3ObiFiccAmhSCUcQxJDTMW5o/ohlr4nofwm2 - +vxV/QO1Yxf8u+HajZXbueR8aWQLpsji8tomuCU84cTS1HC2YXqAFeoAyrYXamc9 + z9N3XJzlQ1z/O9zBUVbhPKvtqNxT7IhobM/YTzycvoutknDoNaSE3bkreh5++xhR - h4MLx+lPUCBsTkHYg5OfdyD2QPWe8hqOFjq0GD7bIKn3hJBLndnOh3tlI7QyM09e + F0hn0Q2he+KPY1KdxtktwTaROLl1rd/6qC1OR7VHAJF7qjhhsyTqw6aZFXjV2Eq2 - eU/Mbo8dsJ3uon/a3ubLxeHG/GPj1uMX5V700XZIZEcAjJgqXwP+Hd+8geDXvn78 + lh2PYARfOsH2UxAY3ZP1hmXiCI5wNbsxXzhm8le07sJVQSo+kFbQ6alroy9DYQlK - DYXXX7KbhOvuUHfp+iCyIsmiOz0qmS39PWs/redpJneEdSE7rabfrdxXXLCguudx + bnIaPy+WOMVzYOyZRiG5ZW6A4sNdSTAbJ/3dMf8zRumDCrrGpv+K+gsaThcaOVSJ - WjPe2CVOAOJokNtwu3ccYuaF+Fq5N/yIj3gZnQIDAQABAoIBAAoFC9OUctVvXRHS + jf414vtlzpih0Ot5etOhQ2UE6acx8+L2T3uTcQIDAQABAoIBACju8+9QR1zSBg9o - ftkIW21ui6qPxXHBJzD+JKCXYxmtr6kyIU27kaiFjNQTVHoy+Qd/S0y9d2NwSpgH + bztC6gWjGHehP3Ggh8L1l+vYA4cCYYCSas5mKUeJacNtPj/v2D/4hf9+8cy3AHRU - f01968bUWjaFGY0QeLk5/00uLPYHzde3ORlybpqL9whLzHJ+tKnBvMJIifJqbfvs + Dctl0OYVLLGAZFVdk+o8RZUNnWd5/L/rsTyhSpNrmRcEWPIZFmAc8dgln/2frjHS - wfPtyBzKP4FNwVvIAL0cWumVnt2n/j7OF3ER1XZuKZjq8EPj7ekcGA5CP7eBBkqA + 1tXU8Ljufhgi03sm03AzvhOHoFPqk8WfslmsnpD36qmbGbV7AU1A0oLa5M73TkHu - sFXjUETxWwUADj/jEuiBGW7zKlcDdVJ1y4xyVWRao/p7gEY3vAnVh6TsG+mCd2di + rY4u8q2vzDBRfnJ8hwkcyHzA+43uHf5uGeYieSZBxP+m0GzUChnYBkNGKmuHGqOC - N4r1uLkhLV79a9Y4w9KRGybMQEh8pS9JGfV8PeRxQywLSL+Ja1E1U/+1TjVW+79G + RPx5YAMfu/oECq6cRvuTLTGiHhDOi15ukh2HVwjGDZaj1qHMXXQJ/ghVcpZcDUNv - MOb6FusCgYEAyxXF6FAGpZhLPLPVZ3w6JrtKZ3uO1oRUbuhQPBlOY2EjzUbflP93 + 4H7vpI0CgYEAten7o2hbutKmuh6e/fr3TJrbD0jWajYAf0/Luo4rmF/VOskeUbiT - KwAImwahjlW/DeLg2iX1zTKHVDTYBKyIW4Mod6a6xQQy4I1kTinkLyczNzkhnUU8 + OJ98NeDomiLDBAiRK1kaaxoBadMVN1lNQ5UiRW0srHWNsfQr/998+ZzlJCIWmQ0Z - iTcUadPXH61kUcuReMfaoz9nDSixDu0z8/7M1aa8U6qGR4Ae+bs4oV8CgYEAweCR + B4GOpUdRo2GwhgsWS3xIM+GS5iqTc9QyciPAMjqM1esCqkliMMhjXGsCgYEA0e5J - NiyRKXWaEej+ioR88F3kVYTbWadqGU+mgVez+uxfUI1BVJcIbRftv/dWkRiRLKWt + uQE21G4RpV95hLpep8kBa1HwrIoFqOev8636/F+nMVR3X8RHy1Z02tEK1uXxKr0g - LpJWGRC3S6655aPQuLuHrzrdvJbsfoUrg2oneh//gUY9Lg0rmO4jEjvKklSNeYpv + 1n4KYWTURYRlvPehDunPhR6Wc3GAWOJ2gvcZSgPjKjzlaRPXDQDTGSJidd7/ClKh - h4L07T9kqnnZGemOytuBhVx66GyjTTeN65PSOoMCgYA63hvZBGF43NVqSiKg9bSR + XzG0zdSFmvpA+tRSbbtGQQjSyhSvRN2Fip+JBpMCgYAD44mel6eGWeR4jBkIAupw - h5bAumMkMYWcBIFFenxreDv9g/7JXOf5MfBMp7Zq4NYZu1s8QOaoTW5G7W50pGJ+ + d8sBC6SRxq/CCPmo9ksWSc4sIIqGYrS6/CXSnQk76kxS9L/ttkzrRzYKhhmpAj61 - TF2NmWnoNBhfWPzrX19Cf9Vru4bP5MLwb2PebUadaxB6WUzYuu3Yhkdj3Bi+3+lA + mCWQaGIRGb46tKaQJL3uNB1t5VCoWvBTCcD75YdoP7lfVDNYz8JXYZYbV4OpcTrW - X+qWP9e1VOfJkAzqjOeUdwKBgEkt24HIRq6Qfiwedt2P7pzHw+TntefcQjb1kpKl + 187PBBNoq0p2S3VO56nAGwKBgQCVDZ+Cn/4SLmSRCoz5VGpIr0s2q+M6XnVOS9J+ - qQCgccW026DzNTIAYzQfRuSTklB45Kp8f9UMMzN06yQbti/UUP26SXHiwbdryqXa + LhV6g1/ugo6PjIl9MlGd27bahkEJm2dpY+xy4mhlQ3AJD7lnIVOarPEd3oTGl2SV - zrXRGB8ShQs522fpEwHR4b9j/NaQg1JyAsL+N6AFSAX423YEbpoI8zeBsg32VzJB + 8GQgTUpJfxtT1CZosSExQ1ytXDuxVKIHOP+q9S43r1/buE0eZE2pd15S5QTc3Hwo - ZIvDAoGBAIfor/nuKBi66QYvXeyPbRWicxCTAV9Bsah2G2xTCyO5c0Ac3Fj4xkXE + xMVByQKBgFfnZcaXdyC4PtOahvs9UoV/15FtKjw9luS1zFrr+llA7aGeDRkoSUNH - mSrzrHGZZaGRlGlFqZYDy7qfaXh8pkY1gwpc1uMhqYvNTptPaCpC23WX4YgfY1fr + oBSvpotaM8xUaHbUN2jTY+GCZe+d4fgEqofeDE8dVqIoFfwvKsH6EHRxwhGjamlN - KuRpZE59kXCogaZYzJIvN7YtSVPOmwsW/h+FBmWkL7eB3+7XE19z + HRur7hNPvC4QcDiUK+zF4jT4qnJHXo1M+7tlwbOXEMUk4E23AvDC -----END RSA PRIVATE KEY----- @@ -15899,6 +17549,156 @@ vector: required: 101 segmentSize: 131072 total: 255 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:CHK:tnevhyl46kwtokjchy7pymrdga:7w2z4jd2hfnnng4n2ziz64hi6yh4rqfodxmlpt5w5ksknlfv4xha:101:256:4194304 + format: + kind: chk + params: null + sample: + length: 4194304 + seed: 2NVWOXvtwcAFf0J/g0fDC+ypQbnCU7awZoCxcnhpkVM= + zfec: + required: 101 + segmentSize: 131072 + total: 256 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:SSK:isisbwovtqib45rwg5g2mfdi7y:muylv2hhzo5wugdclu6us7ngvlbecv3uxpzrqgj3lnwsgxodz6wa + format: + kind: ssk + params: + format: sdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEogIBAAKCAQEAsPiJlX/DPcGvoJPp1ChAI6HlQO6qRMiGHZ6tqA6fto5b5yK3 + + bB2UzU+bG94CgObFMqUf479s+1VRWbSybfsRwXLx6SXVmvSHYFawC8ZRs3YIEP7i + + Fl6U1vJqtxdxOLLXUHfd4VQZuRah3fAx+ZHqpO5/pWRW9JJSGqJ83C01MIwI4WmI + + YryKu9oIJIi1bv9VkbfhMNx5fM6MSHQltm8fuBWBLzntql2eckvHUryRj/glAV9p + + VLU28bgLVMeViLmZFNVUlmjczJFSVAML7v37n7/xPQABAPpSnhlN775p5yN+cB1y + + 9JsPpL45IF7ihNe0jHQEKI+sQ5bnWdvtznWTiwIDAQABAoH/RLqS1rYjjrZ/7ZJz + + H+7cU46uYkM5jpK+hD8wRDj9KlF0Xe2ChtZh2EI2EiETIumKXLWviu6fXOcMmB3W + + xRqQAJGWZyb6X1SkCWiKqXV1qPIitNF/a+LOQRqd+ERrPHU7CiVoIusLEJYqMGXx + + 6HAMpJocCQiKGe6XOu/ke+K4UF3ZTBL4ar6yZuIYyiiGlpgNAwroDyg+hO+snyXA + + catMRT0h9oAFp0864cFTHanxjgSsbZjkyZtV2AFb1u2rjjd6E6NgUmJ+KKjK4QKR + + qIMNOzlEupAzEkh8jObtlLY+9ZAM8eRGNOa8RNimZq8ntl7Hhl9y/uw539M/Xdjf + + ejG1AoGBAMy1Vza5NKvnAFOaaORF+S1gpT7GaDfvtXjaokZbdEBDVy83KlrGDJNl + + XR5HbqZoeKGA/mbzHTWZrnZ2y2BzoGbc0aM6rtFsJ3rqabRlbmdyQFK6mjLmZGzC + + 5EMPG+pWXt95d7woxtqdbcMutz6ZtWn2uquXz4kM9R2SoWPvucKlAoGBAN1QBciq + + k+a0r4z1DEs20Yz8P1dCZdO4FnicI8Vk0IKwRXbJDYPpTwAH5ePNXlx1jDynLs7B + + 4J9csTWfhFwR5oeVzR5GWwmbx9cw6g1n+q/rpOaYqtQ+h0Jia5kXCKNcEio982kb + + CxtGDUgJj3iNi483a/wCCcKQXIIAkbxmcBZvAoGAQYA82cvFKMQPfLDJo1Eoe/aS + + qVV+/3b6ECOVDQIyXmWtvfPe35DDcV5bv1aH90MyZisKPBLKY946zrkQNlqJFqDN + + i3c5fNUohNIA5LIX843BOzduI59IvuxVcYeiHQdp8APD5jb9+fGpr2yBQcyZGcDS + + 1hkLVQUKYV4Luhh4zekCgYEA0VkjP4DsS25cKbCcIoIGk6EBod9zR2V6DDlXNSB6 + + hUWNUCI7oK6QRm0yL91TB49CSxWyl26atuUN1LXClP1x3ov77kmLUHmF/q+Ml4Xm + + g4cbA+8imYdUl51WPwik6TLtE/xqRuCIDxKi+aPhjZ4HiEBa65ZZ+Sxp9afoNBmK + + qg0CgYEAwZynW9qj3nH19nqqHPvOgGhYShotmLRznf6mmPqapQInlv+42ltxVsCB + + bKYgyM5KRZSFYVjKH5y1SFAByY32wqIZSx7Z7IbuSQNeoAt+DLlPfusuSnQRk03u + + PU4IolenY9gbyniTLxkcKjmRDQa6L9C6/ADI4FhUSTWlbcw3228= + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 4194304 + seed: 2NVWOXvtwcAFf0J/g0fDC+ypQbnCU7awZoCxcnhpkVM= + zfec: + required: 101 + segmentSize: 131072 + total: 255 +- convergence: ZOyIygCyaOW6GjVnihtTFg== + expected: URI:MDMF:2vhlj3tpmx3nhv4zq65aoss3um:dh3ql3v2mlnxizs3med6mmd72dxzzgh2varaq5zgs5e5vmob6hcq + format: + kind: ssk + params: + format: mdmf + key: '-----BEGIN RSA PRIVATE KEY----- + + MIIEpAIBAAKCAQEA03uKqCbjzcK4tMQi1f7mAQl2t++ihbWQI2m12rsf312o7QOp + + td+qmKsQNGokH80lc8wdlkgGMTUtn4J869+i9uE3EDT+JhOEkRS9tPYT+sOaMoVt + + EVhioMHc55FruqXOTKv+BxkF8DFnBF6jFciQafH6RE2D8L+ASwjXOghgqLcZeIQp + + BRDe03niKh/Es8R8R+NunUd65qn0ztXTHITDBbxVUAN0Mx8rYy7cYi7drGLA56N2 + + KqY+kvWOXoDYzuzY/ILz8fQWHdNUOp2WbVWQ2MnDm0GSBUgy8y3CpWxmSifdg5mi + + nunNG1RsEWHV56N3SOCz2+zYiNaTyC2BzIOmLQIDAQABAoIBAAx4SrsS3U3YVHx8 + + MDL+vmCDRq0eg9kZ1UTxSm16U9fKNa1mu4641d3+ADecUUL6yAw66z0IRC00ndsQ + + 86gTiLrBRDZAGa2fFasEVmdMN0NgXerqs9KuVn/KI+oXngrkafSwE4t8Qz21fAXl + + mqeB1a9u5ZCPeDSC5jVbxm0VP2B98qElxzdBxYZnxxmnisYa66/ghkDgmwcmN/mV + + /r00j2ZXz/15CKd0JKNmztt2g+iXNDKtUSmJQZL78oQiYzFDL5nm0wQNM8XBBE1c + + arMSlZ9ZpOZ5NJXp33XsLbjF+cWn2oCb8fwEHkrVYP4Km4rebSCEZpx5bqQfIOrP + + +NtyF+ECgYEA61MofwHzJkeIxcKUal3aKmgD3XqDbRgn3cZfkqdG3Osm+YwO/mmu + + MBW9RzrMuBTtqxckYNta04UbsunfFiaQm3xCrmmtwxRzn+re+jp410yxeYj0fQuh + + +vRkez8LHYn2BKzfQbWgX1QdrUzoR24oJezwWQWgbkuMGq7ORqnNR2kCgYEA5hAh + + zne7xz3iChd5vzsbBmkfcT6Tp+QcQqLt9d9wEupMJ9ppOQzk4rBXwFQvjcTUOs9n + + AIo/qcB7+XAchCp3WCBdGC77Rz9hBosCjah/coNJ2PVM0CMF8VhxMbI7C7IQlKPd + + bZEmTCwQOR6kKKTD28CqWHA7NIryj5Ux2gi/NCUCgYEAx5Jy2aOxrlkkaXMnoz2M + + 9EHaZU6tfyvpQ3AlRZ6PvnO/Tgu1+5VsoGMPbwUy8TruhRbPR0VAtfpBD27AP2zd + + Xr/3XStKrhL+LDVofRZxvUXRjZzUm+ftq4LwZIWGy7pg5n4lqPh71dzkfkCnDU0i + + x2c2PolDEccIPujZD5yZ92ECgYB6+jSYATjHEDU738CckCOqEZdVGXYkULMqi51X + + yNBHzCZZR07nyBSxeEHv9RBWX9hyd1s/1qahPtsGQv97Rpf065fXzYVUWHSs4rHC + + t0cpFzTqXHVq7M3IbNZVEkitv8lNKyq53tTx8rvZTJ/Deg+X8C0eiR+cvolaZw32 + + 1qYeYQKBgQCtjRniQKjBS9A+cs7/5XOPCKfOqYOQSxlMrnto5sX/CO+PTvT2m4jn + + YTgXKQ3SerpTVqNITtghSPBPV/eVJfLDuG8c0ZzWNOjCaZ5M91rMuAgJ3ug3LupF + + XUrl6Ks6B5OzXgGqkk8SCnsaKq+3JDQdvN72xOQdaS9Gi5HhZfJ4Ww== + + -----END RSA PRIVATE KEY----- + + ' + mutable: null + sample: + length: 4194304 + seed: 2NVWOXvtwcAFf0J/g0fDC+ypQbnCU7awZoCxcnhpkVM= + zfec: + required: 101 + segmentSize: 131072 + total: 255 - convergence: ZOyIygCyaOW6GjVnihtTFg== expected: URI:CHK:tg5w4m6hi5ezeapuix3lrrbbtq:7kerk5plqpsd2upnole3ifxoegxdux7prazbvzeu2kyilc6idk7a:101:256:8388607 format: @@ -15912,62 +17712,62 @@ vector: segmentSize: 131072 total: 256 - convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:SSK:xyg3ycxyqwlggew4yuakicf7pq:diw2ygma3sejm6gfrilmr74seleyrsvujixkwgr5qrq4stdfsqra + expected: URI:SSK:3y6p62da5ek5bl263mhgva45ke:hvu3gjbdhqkaauzqy7wt2aiz4t47qcqfm4h3ur34gcrxq4skwxkq format: kind: ssk params: format: sdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEpQIBAAKCAQEAqwHXEgjR1fSRtrW9og7WwbSP4FyK+Ce9qM15D7eNVlVw3iuJ + MIIEogIBAAKCAQEAzclQ4Lb4GgTvkDXTIih0g1v4ys/bce595lQxSX2nNjeSbKjr - XOR2CoXrv3N1BQDz6dHggdsb2KAQxtLp6uaZJtBneVKDc1wVOINfmDDdU6n//LyL + o3hZRVJFxFCe3ylEFcsAtMflktITGzje/mWgrmn38bxYRASOBVdoLFXPiO8aCgT0 - cTEO6LejOipcOUypo1w1wMS8UVVhVjuRSHQwN2HR0j649uPyfS72HP7+QkzlrDe4 + Excm4smNf++TQp6G3oQtey+xf4EwOtuBw8R4Ehm+CqFZrhB+qfMY726eUW4XtmZY - qYB1T8q7w0X+EvdvqnrzStHfU009uKsWXLKNh20pUKxKirYY7epi5Yw2dcNuX+XI + zmpmM8pTWISgoZEqm1tKrV+qLfArtvrG84+VhSkzTg4u+oJKOnUINWfSPfQZd9nY - 3o/fLO6pJVQRwu6lQHIywJdua/3PlyRv+DJq+y7yNk1W+cId4OMCBzm004ugjDGw + rzNMe3K9OuM5JqYbdIVbnUS5jXFs0egNui5IwEgUtsom1nAWiljgo4o5NepXa60H - VOPe4BkcYpa4hOMNU/+0haKf9xvL8XD8l8khEQIDAQABAoIBAAamHQq9V0xIdLBs + XTFs0gpWzoZAO1A8J7JuuLN7PLzBJhk4ZwLTeQIDAQABAoIBAAmJxFiIDoeZNTfi - D04ZYdn492Wfr4MPx4DkUU5OQpNuYcOvoWkAVIMa+yQv3OdHhtRK5d3iqyngalqS + UMkPZjgc9BMFb5rQJrB9dEPfYbfU+1HTQgnDhyK8CZ0L7hMypNPcQwn+Dm1f1JAh - qYfptJ9016sYzVXbSRNwDzPThY161QJKgejyIXxkpHbu3fRX4dohTBE9TP2kElNo + Uo+oyvnukjYXiGFNsz3640qTxyDmETdH676zRuOB20/D3VfcBG0NpBSGvUPHS4Kc - IEFXDCdhGSeBqw8lZICfxm+w7emds8Q5v339j9hiLDIyNSIjTZEidLADPOdZwQ1Q + 4D7ATV7sZ8czG5aib9aFfKFDZ63nNsAYWFJ0SqwwVembFkC9vOsxenPKmM5O2d8i - cGk2BItVLsf5tamKlLpik2gfrVc5DtSkbAncY4eI6nvWfZd8Bn/K0LeqxUQ4XJk4 + 7G4wXRtzTMjhsccmQJfCoe4xAVuFCKhNlPB2LegglHfFowXcTc3LD8dninnZBfsp - KSpNfb3kUwsLxBvb3kjAQUif1tSPlEBOnqN3YbSd2VKTCTHe2SLljOHfPPJWoxtZ + APwbb+2g/tWzj7+qn28ESxjYG84kLRvg5g1/RMgaYqLRti3vgERVnjG7Luf4hCxM - A6z8FAECgYEA1D6+956RMQ3YLPCwaZJ/+IHFeg6JfN1krFxLew1nGOScUP4nELkV + fATU90ECgYEA2x1HqjMhSlkmjZNEjsQ8Z5sMv/+rD747IWGFumCf0PVl23WmQujC - b8Wm8rUjOZX0BGyC26IAf03WGUwmN2HbNxxD6HtHnC24GS2a9tdN7NTh1GloStUL + OW+bGHnY8A1TBKlrkW4V3HJfn3TGIaRkTRGzYsvtYGnIrgREJeNiQOkETydJq281 - RIBdP3FwhtvwKCM3+CNzAA10D6KJVeMf2dLr6PgXSbxxS+T6IKdp6gECgYEAzkLC + Dw8yvBquPAlruuCWucxL2lLpTJvDJYQ8WAKATXR/0XpSnynBGz9jq7kCgYEA8G2q - djJZfQBULYJiaOcMguhB9JxmB85kAAek0VU0TDNtKKEJGUr6icpI2/TgJOHhRC41 + 1zPbvS6Gj+7yMHNH1TiiFub3lZEevUkDGhHdFg/cROZCuMTK+DFzB4uGqZilXUw8 - gq0tXkv7QvqZZX1Owc9n+B3OA2WuBu15G9l1QoWv5MLhPUI9sqXgt06uZbktQLz6 + OASKuTw4lGUSNLNEkBuuZAUnnZ+LpGt7Cln1avnVwzSOK639zfYRaWMo8Rc3Yy2X - YMQ4QMo5qGE6ALl+R+MDUNdpkscCNmiaxfy6lxECgYEA0SYnvxEVmFZBMT/ZR59i + puWu7VjHn5/GojACdeThr4xU5gkhBXv6Xy4BxcECgYBFWEXm+pmNkxtdcP8gg8Bu - 1brjo3yRxWbRXvvwMYkqkCAvXaylSFhqpGMMOd1/oa5/8KARb2c7wDcuhG1Ct46J + NabaWMrFh7nk/Z059/x8QD3FL723rTxSuxyFqYJbrovYjNnLQ+DNTLEwoN9XpFRO - m8wRqxVYorF22fDT5OyT0I6TH2Ljr+IyoUUxHmSl827mQFc8PxyHpYScWw/a77TJ + A80W9l0gxznIwPbkWsssqdJATrnE9MQBCRlQaM09mOmsUgnBsYNMDDNjmGQxSmFi - 3Td423EmWbYFmzk/tk/jEgECgYEAoV2JVX8+k5TWRmRjKT7ZgvDB6OUSzbiic4OH + pR//41/UZvchAjDoM66SmQKBgE1t5CEeUFwiya888r5rweyHKpxZkc6XR+EJzHfu - Zl4KdDMni0mxHKCUMYiYR7zkPvaYjga4xmtFuygmgtgbelL2cpoY9PwcWHwMEk9n + 3NaoEPYXedFrfzpjInqBksK3qDndvV8FB3AUVtxjmHNkcGZAo+8OQe3fXed7vcpd - GGqgWlLMsWPlY0+XhVRQ4hgkSGD/Dk7KczoP6GBNi3XFMxvrt8HarjxY1APtrzNX + ok3rW85b9JVYmW5lGsJn7t2F7o6ANmDHg4homRFtMVk2QPSa25vfg8/5jKrpfH5+ - It39/IECgYEAkrucWapwRmbr3v5IHcisK4Dp9ygNkEKph3ZqG7TLaqHZMiQ8l4Yz + oI+BAoGAW8ezqR6768jFvKx1Fd9m2vTk29TF7CSHSCgjp8qBtryIQralfdDqdFSW - SCNFbNGEOzixLefEev8XAUVCaSvWe02PKBUru0a1jxiXtvxUFZT1QW9ndWDS9FWn + 1riWoPU5uW4HWNzuQuZOCiqnwTCsc6tHxsxwgWCbtv0nliJ/VxsGQBfM7lyNHyfq - Z6go15bSALqAfdS+/uKpvVTejkFh1Nsx1nOX/iJjEg4pog4IouvZKH8= + tr4pVuZ8uXd9v2JEe6sI2muFjGmTu76ieglgamb/KlIwb61+EWQ= -----END RSA PRIVATE KEY----- @@ -15981,62 +17781,62 @@ vector: segmentSize: 131072 total: 255 - convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:MDMF:wkhxfy2hw3xjgktv2wk5q7caya:xn722v2lrbpll6lumqphdircizjuafl67euotd7tjaixxn5lj7oq + expected: URI:MDMF:qvcv4vtu3lbnjijwf26wiojbbe:snscnddmlq6642bhlhlaqtw7actaqggvwv4gpezgjqf7sxs3kena format: kind: ssk params: format: mdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEowIBAAKCAQEAoJgqinR7T31kTqPuSts1f1Ku7JuzlceOe+fIIzPMGRjn1k8v + MIIEogIBAAKCAQEAq1Nn0JufzDRSFksuIYmhr4sLG2RjLhZD8Lf1E10JhSWrfb7Q - y5gfPC3l+2H+C7AlmFAHOIjflnqyV1hP0jIssrrchpMqCkn9GAcCHeTNJcGgaxb/ + muqmRmSv2f/2ell0cXxDtM/7OCjxrgZ92uspBZ70io/T90EjQ9D0Nxlaz+CpW9m5 - H0Nq9bT3PmWewXUNabcQBtBWX3u8BZ7BtOiQKPnE8V7nBQdpWOeH+KBStHJ0MwEU + 1qXhCdYzGjaV6M0rsOnU5lWXPFjdTz9f81VD6paPSGLJsoO7jSpu20n7DL67O66C - ModphtbjN8FfwgmXzxEUEoEqelgwQfcwfx5u51Fr2HQdaLuuuM3XZ/QUOYsBONbn + OMdKof2Pr4/pN6sfOyn09VKAzOAZoP7sGrre5xJ8T4LnBYGJ6nS2wyIU8dlou0Q6 - GgI9N9IHJGIy35R2Ws/9+0qSc93XAbY1/C1TYnCAzNVxFpGjb0EQoG9wTWQMz95u + PqOx+7OIsDxYgaMPtA0nR3n84WpnsKGnQBkrPGqPClhfhF/7w44WU075CIvuq/Gz - oNPG25pblbyGIlsxhcPY+m61nTlbkqrsdc0L2wIDAQABAoIBABybjrR0VIUP/r7d + B04e7D49UlEEwOOiYG/r4NpYIHrhmulv5mOinwIDAQABAoIBADeJJdHdYINVQnav - h/TwwMJqHbwLbn3HezPKUcYnk3uDCsWL/KUld6b2PCpARguZ+NB9rROemknJmJxj + kBiXAK5iqAsNE4lQ9l0FhI/uTLO4bkqom/5bqeKPqOFFs6Qdcz2GRnxKHukpfI4o - oHB+vKSoEeGtNId5r6rIkNF3cS4BJIz/HzpX/aVAc+y7GIE400daM3IrSb+foJpV + 1IsuR3HnAOYZkWBI4SGOjlt+AI36CWwYu8D0rGn/4TjSEO4R8+O5KKYxgICzXane - sgcCiK+r8q/WqoukStlqAThCgwketAMVwe+0UMfYCbpxTU4zi9hr0hqauoJwSe3n + pT+/l/BnNbMFMtSHFzi/VIgJBzQt54OAe2sKwCSB5JUBBgJVce0idFwqvoVv7jpe - ZMS19x1cw1s7sSt+naA9i86ICZ3kPaqJJW9kOfxFd9tP3rigs5K0ye4mgbiFUGns + mxZpevJlwUjJPw8ieDOPoS7UwzEMKHkbH/iJ+mGWV3JeLH76DG6BDGIA6+oLB9X1 - +pKHbPxtWna++F+QUR9pH9UyD8W0dlu+klVyzUJTIXTrWRe4wDmwfaU6wAKZWidm + QE/JXXB54t4jybw9vIT7toC0A6lF24FoXqCzoAHX8RgJjCeNvpA0Q103FktgErp0 - 56JfijkCgYEA10AMC1JO84Ebtfh4vXLna9JYHT/B9qjyiz/lLDwTyKv6JEp1E/ng + awKCMKECgYEAxO6Hy8RIK7rcmSpdPbTPOd/X9uMfJ5wuhGHgt3TFqlNYHTiQY35v - 8ASMUjO3411yOAlM1bl3kJnuCRYsHACWJPtsmRRUgQdR9TCyuYWAgLLPhV3BDy2S + S58Rh5v3wsDbFczg9BMbYYY//PuCoAeNI2O+iDR4El9u1T8aMRCePIm8Z5v4SFUk - vej414UwhoA4LVrNeJmnVke5LJy4xNXBamBlFkcaEwlVY+4SJp+nkZkCgYEAvv9D + /Y7bAO2Sg9y95IF5s8EL8d5yo6S3ZZUKhZ48zfTN+oDG2fAiOa8/VUsCgYEA3ra2 - jxDfoWdZJTue/hbY4xDe7qJ8mqsPZKUQ6p1Y9WKbHs/xrHyNpJoR6c8B3qcZghxN + m7VyTxDAKRjq0BqnNUrg/UStv6pHA0N+iyiHoE9F3xVHKHhBP4X0aEps9bfrErYL - L1+83tfbz3aKViUONhtkY44Q4Madjr9XhprsHkO8J0yIha9ls9oyF516sKlvMNxn + b/cQvRAS/Cb+FaMfeWTSwYzMPok5xyoUtsNwjeLwPyVp+TPdOzJVuQmj6ydaGtpD - dHF6bDBy7DNQnOhvuI3aXLJsQeOO9Rf+BnEQmZMCgYEAkCcSadqbiTQjzMA0jBuR + KuJbbNr9vjci5fduB9DFUXrv7rrcTSTJjldf130CgYAUaxLjWq+M8SvsKYtPWY7e - pIHgBOaYDYqjtGH8Jp2tWizifr9mnRQxckx5dOux8RC515FS5acpzato4Kj6rV4v + 1kmjDHtvdO8RxMAy5UWVWlzZcsLtve82LQD5SX+PzsUoZnywcca1/uBlj4JEq2PD - L2E6H2KgHTE70ArnBpvDrW0S0WwySOnqZkjJrfxCvTDNboJrLKMqj/vEpX3nt9q3 + 1pSrtJz6crCgJZHGoo11g2Zoa7B7d3CFZalpWDiHuXxq083ViF9/rWu/cdWeD6zu - h3g6+qpvaeRMTXo4qakuXbECgYA76nn4FHQC/xfBDV4IGYS6Xp2AwOpT3tu6V+nh + m7B8PjSZE38Km65Awt3TLwKBgG+oWSr9sD6VnlG8bVVCV5xvWxd/TEDwhMPNHe90 - n7C7cc51sQgAcyY//7Ek5rKQdV0UKuqvtNncEl07TNWCxqcZpCgu7u8uhEAC+tVr + tXKY6+XpTBCtIcFQTnXPAou61r89x8QtsRWormv+vJpqewgolUV2apvbvrzsixAK - PYhayibpMSIWxfoinI1gSR+m8dAWxN2TctHTxLMYk9RzFJuPirh4oeRCGy/KhVdE + Mi7gnSR7hILtDrh0BuhLPgRSaWlXDh+89qs/q8Gm8Pcsstx2PccZBJvC0VpX3Dlh - EA4EDwKBgEUx6Qm4bfLbIvKHmL05IWXuSdwsAVG0bNPEZJf9ht+ykNcMOlvF/UWx + 8uodAoGAfD6xgNtZdqc876KrfiGli6IyjQgkKyVdSe41qqlxC6C1i0ICex22WveZ - Z7fk0jwFutfud7KkjBRvuSuDTrexFNsaIW5RqWU9iZf8Ep0eoRNAXcZk6sHCq+ds + pgjbN7JwBRfLht23vnrqIVr/1P6neDQ6nh9aKNUi6wsldg6nqgwfxXx8ChAEFZRR - nDpIWHZq2Ir3SfIPd0GxWzasl75XnMN4rt1M37G+lC0AB6PVw9vD + RZu9RLIfWHGagPiKF0FIK07/q1PRtfAiQsSIZfo4yA4Lqojauns= -----END RSA PRIVATE KEY----- @@ -16062,62 +17862,62 @@ vector: segmentSize: 131072 total: 256 - convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:SSK:66qkzmv6xs4vajgz322wfjf24y:7xee2z66ue32dln2z2zmffcza66rrxqdiptul7gm2uaf4p6x37ra + expected: URI:SSK:366siuvcfpanoq5xpsby6aeu7m:udkk4awrzcrxtmhjldtezkscedaax4bdkaq3rczqgozdizy5hjha format: kind: ssk params: format: sdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEowIBAAKCAQEA1Mh1wFJg8dsZJAxPH+vCXEjFTKZfZjLz3+hN20jAdmr/xtgx + MIIEpAIBAAKCAQEAl7MOMMbOn8dHhuStb5vOd3YhKX9ItY8jaFt2aUy4n6vhXSNF - kSup1k8EvFXZ9gc7Vfu7JVb719r6b6Jeeibc1x9rKY+wf2ok4J+aMKANE/vUPU45 + /KbIN2wQTDZdqHjMfnVIU82TIgmOzfgy3H8y5MStXKI8+zKediZVFW1SX+tVOJkr - CN6BLdLQaeshbruToHcZqykwu9zhgjXmGWzQRiuKx8/3IOnx8IOefRoC5Hsge/yw + gHg5Yk7t/ahxXFnlNKt/hmU+Wpy65keIhZlDW3DXN8brN/abCN8gkoFHFZtl4FxW - ky+azSi//fzyThu+3wFoRXbOvCCcw+Rd+j7g37xUbbaXfIUs1UiRuovGLdoYsv9A + DhhBk08IBQdSM/m+QQJ4LuPcwlJamgRxL4f5lng2zRpnDXjtv+NMwUp/D+9hN/V9 - A0MTzuJWnMjqr+4W7rYJ9xU4vkibkRxntDO0qlvwxIzLamUINnPSyCKWdytNuL1D + 3IUy/n6+G4iJQOdZDId7PmZB5XreXgDpuOo5Qh6E5iPWEYzpOpc4441F6huwCN+7 - GD4PPoPWL/PJx3FVI5YAbyeevkPiuYx5m6dpjwIDAQABAoIBACYeXT3dZCGfswLK + hOcnbNJv7GjRdl1wVwpoIUVgI7suKk9pDDMr+QIDAQABAoIBAEGGYhbHiPCTD15A - s7gPt6tpm/LN0URRN3AywRPaFiSAqZ6ZJ1QO4ueSE1Kb/KZ/CCmwpYecbBRw1bF/ + 4HlY/3GyYNif1jQ2Q8EL4LXTIdw2Tf4BAnYDRHBMCS4iPYpLw2jMGBW6slb9ceWd - AHbYlHJzXfK9m4xP2xhkby5r1bvxPsXWyA/nMHQhkpWO+lfIgbta6r3HbMQS31FA + 07pSZxVRruBYY6bNUo0OOaorsm0kJYdxAc1YINFJ7pqma3DMk6iQe2D90lUpZcGa - z55ZaHxRm5SNFIQQdPe11IQrzz3X3yo23qLoGoNFTt5puu3d4ihP/JGe5zJPyVvp + HGo4rVOOBihdj7R4nLbUSil+FcpKzuCf/7SBIlhjMeLH9rEtlDJ6az8B+UKilJJP - MVMb8RVXwcfBGnkzPff2mBN41C+/oQ7iESzCSZeNRPDft/74Ma3xKAlMN1/rBR76 + nACArs0Q/l4kCyKa9tkjzqU4BGEWB77uTzxwFSBvguiELXhfYRSxTqlCkkQWR6TQ - kJEwDCqBpPmF/zDJ+ZXF6Up40CTBXCYa6Sf19Sas5IlFwEoIOF2BxjpW1swUW/wc + jgp3rkMXIDeYaowYOC7HQLClLghH7HMlYkV0lAP7sgZUmdFwGKRpUy95fxFtGI/k - zvBT1wECgYEA9t4szXjoU0z0BYOSngwB64K3c5Q3L8nPr1tjZv5allpEizhKz+qn + KCVE+xECgYEAzuDttHfAcTSgP6vzW7Z6QFksB+XLMUspVVQDgf4Bllg899lw3ndV - NOrkb8hVBFrqmpTho5i+GI0oVn+ol7k0xaUjmC4h4crNx0ZtyYruWwxL+rt3OWdv + +55dDCsljhCZw4o+y8MktgtAWLZte7znUtRLHsbD6l6DPUnDcVXdRcFzqKeqXlAZ - zudC4KfkdhLfd0+XO76HXpJFpfV0j88JLP3ULwBGF37MUauBfpwbUo8CgYEA3KeA + ATWrti5rEqQu9l1AzduaY7eWJCtXP5mywlb+3xOdJyyYdPP7UI1SnncCgYEAu7gS - 6cTXfM43fp53sCRMN1TfQZ4ave/+iLK59SKGMCGa6TI33j1/1Uj1HL6HDNHyieRa + bAJ+h675WhK9bCcBMpeCP0su80Ovz3Bu/UzUa5VW0LPnFt6AMCGlJXu2q4jFo0ba - ExqP1Sf/9iVVEqg45QBgb3xNCoWpcnxtBFKIuvQCpk1mRsG5nLOPHp8OuGGPRWuI + j2uIe1qV8SIMWS+G+XxmueqeXfptoTIeHkBYOL/1YlCONkbodHQx33UW7RVTc5MU - 7gx3qSWx71VoqDWlRzFSUFW/2QSXK964TmNR+QECgYEAjv8/IH4qxSXMK+183kPC + HiTzjBqrL46CrtrN91sA8K7pmeuPAk9apIvs9Q8CgYEAqwmtnSHQmgefYWThW3bf - UPNU5IQ0O2BBByh+ucgYHQOItMQUwb8Av+xYClAWvwES6BvZX/Q4GOybMw+bTtef + VeojjBgBSSzR7Hj8OXHukAU9ytAcD+Fr1g7U8OWPNAgniFH4nvAkntlohq+0jrPc - M+Vmat4+DhZ1gDrRmW76ho7m7APvGbdK0qSu3ociFSr1ep0F0zuYGjXMVkeKD0sz + ME/SF4zPlyoyqO4eRsptmWlaHRsZsMXaFnTwFTwFTDEvnoH0vP2NhFnZKOgoRy3k - 23XklJ0p/K4cGCqqRfaS9Q0CgYAL887Z2t3JVupOo4rcMbsnLCPDzCqqqztgcD3+ + a+YO7BHEQQoOtcqtgaiFoPsCgYEAtQN+4CBXmscjM7Q2bIAAK6Tlt9rr3zA57DJj - d1ZJeSiJBT1dfntUNFWCrxdlrGG08nemnUO5SidlT/RhxFcAoJqYr2UE8uSQ3QiS + FGZtv4A2QvH3uJm9yqvm8Aonz6kHy7abMwlihnCHfgpzFd06roFDHawcIktGQ9Zs - uV3KsrkKBRtLLec+A8P25qrHdhFqsz6Blo9MzEvtKPU4V1+SkathyqNPwB3oNHJL + LIenirGwEanUOIqPxRv2q5/hB6U035HIKHlBUKy2vhkR80KSsh9S/MPuBrqbIIMc - XLnuAQKBgGsjwA8kCUwoygz/4kXk/vjtt42a+4g6pjRjNeSOZ+mnS5ijXdZj0faI + yOcVDAkCgYA1t45yzJwsMy98fVQgJl+Cb50fWmiaDCVlytBt/n7Otxuica96EbqL - 90C1LN6JIQ6Bg9n77dSCnCn2LuDf/sh3wbvxgyStq89UbMVrTs/e70Xkacvt7yn6 + BiDuf5yicNlAjByHeKhv7FlykLqMkxDBiCNyxwZZKErZdn9+6hV5/9zCGx7JeSJr - HIB77Wk165vZHdPGfej0NADL20eYMbjGAFqfSjxmqGUhVBvlYdV2 + uSVEoab5zdhSqEkxNJBBLo8HuGHrjRKyhLchJFMiXuo4maJ1zKu85A== -----END RSA PRIVATE KEY----- @@ -16131,62 +17931,62 @@ vector: segmentSize: 131072 total: 255 - convergence: ZOyIygCyaOW6GjVnihtTFg== - expected: URI:MDMF:r34bsxt6bqcmhegwkufjcnkhz4:d6oj6l4dw65yazscqi2dgtlvetptoryzsj6ix2kd76uby3a4d4va + expected: URI:MDMF:buw2ujk4pqecvd224j4tibki74:vi3aaz2qzffsyfvafcencavush7x65mrdwg6stxpqidsckfhhy4q format: kind: ssk params: format: mdmf key: '-----BEGIN RSA PRIVATE KEY----- - MIIEogIBAAKCAQEAsid0CP/N0kwJXdAvA/ipizC0or3iReAfMWLBLJco9ehKX47b + MIIEpAIBAAKCAQEA1TmUXQyj6HXUw5Q8Nh+LeL3oFFs2JcXW0qvLbUloNT9G42sR - Phfc9nlrcCfBFeeRUi2G53uaw8xZeLuBKt7VkBeaKHAmGQ9DWKKiKWDPXdm3E1e1 + 3GPeGQ28QLFRzhtawf7p4hdfQSkMccba/MOwik4vZTq6kzuit0juIt75+p5j6U3P - dcnmq+xLcgSmVV1rvetbER+MBMorzCeTk054UBPpFIepcyZCCBdVJ92MQjSG4YeR + LUnMzjEr0cIXLOhQGwWT1aukQGzbxSndz+J8XtnehVEggmhQ2cGDnniM+Ub96FkR - 162YckojPEtM2T8w3hIrUzHBC/mEmBDczzIXw1libCi6vchtA+g/DOzqgKjg57a2 + S+Z/cHWTzL7njyeYZ4MeUwWsO4F9vg/fYd/EFTsdEbqk1IV8yqnKuuYObNTCtlqW - dBSPMlD98Rew0NcGWe2ZZCgD6vvtuHkgTn1tD3m6wPRb2KLwsMqC+by/aqluyjN7 + sJi1D2wFiFBVxi8nfRzy6Gif6B8Hf2+O1WQvmqdC/bUNl0wpouXdNRYttPH43BQs - ppfT0fTxkjpgA7b2mTppjKATU36R3fT9lUap/wIDAQABAoIBABM7wIKQNgT0LIwH + 5O5CTbbNgACHskcDqsdqRYuzEwLQfDgKKqnqFwIDAQABAoIBAAkyx1eBInMIubLI - mRZ8AWHY5PIvlH7SuXYNdql9EEZWyQtpUttwnBcH+MS8+1LaveJPWUx4bbb3GC8O + z14nNzi5lbWHGhSJN9x1dqo3D/pu7iuW9svuXPyoIRr361By9hPZqN26r2M3TVTG - nzKk/WLMXtpAG/zXRwAKCX50yIPAqMvadj68iL4a3pz1SjfWbswBiBgLKISuhu1I + VyGvWXkczVlWiZvRsG2HTmYNbJlwHRRJss73RmyguI1IegAuvAHvjwSZN5714Pc4 - KzMwujsloHjqRIL9Uwz95+BL7Nk2FbGEFYxRlmliiLfA6U2KEkHZbo4DpmVGyizU + ZhLAH0z5ErQwVh56E2yJbhiAhrGAxPjz5VecgpFtegv6gYTSN790qr4LotMkGiae - EGx3cTm/M4p7gC9sSa7l/LWhjGMfw09jMwx7Fv9NX+esqWmrQzHCjsEtYjuf4U14 + q93EBXTxrzp58ReYOTl7OTu4a791XLtJq8ap23Lmu9s55QJ86vvBX1767WArgHcq - HoyfrneIGDhO3IVHOIMG5En9BCT/YLW9wnioGWlGbBKjnJcQYMNPTZrlQhoeVJW3 + 7rAf3Ti7YfHRKjw+zAK+CeLa+as4f7HD+B508QhJjhPNqQU6xdeOkuQkDX+vjq0m - Wmesy9ECgYEA4Exzq30VU9MMs6hwIXfHfekPj7evCeflZLyjlRJP0lHR+iMgc0rg + CvksTR0CgYEA9pRSPULncSkthLy1PVmNAPRfZl1iCa3Zo/UZsddWMrSS3rMKABtE - FZ3HB/k5Dt4xySKIfJ7Htt28E29ep1/AnzWPlKCCZABM3USi/YEvocWgY61X3Slg + EKyzJWzqLj00w6sQezz0YY6BJwYmHoJkDLDPHoSkxircuEgN+rJNLw7uTtOcSlX9 - bjYdRqviMOXk9QSEONRLgYn+QPSJeYQL0GqPw6eQ7bIVR1EPQqiwl70CgYEAy1Vn + yMonaIUKw3wK5BbPmCIAv456AQw3hicqc1qWcWwBXFWvO+iyiBsxuUUCgYEA3V8I - x4952UVai+8SibopC/GSc/jUKIzX0Rcw4h6UigzCMVkn7SESILic08JOye3pUN9C + pRXIkxltc4LG/tytQFOPChf5fJhLlnR9UPopWK81SiViHmeT/jlvywk8thkvUuQw - 7i0Sooy5B8CJ1xzUz08xD9h1bWsaFNFgOuV9P+U+GqNV4sJTNZJDzT3cvzgaW7mI + FOo1pXcidkcfvHWYWtbGgaRHmu27o30bNa1Jus1DgLq399VFvXYWQBauUOIfY/1Q - 2iTSHsGjRxhJhR7VohS1qZot6fPkBPVks1xxFmsCgYAcHZk2OtSskDz8XDXKDDCm + xtxhHcPFGxSi0SYUA2px7rBb3KOhqkHEylgGlasCgYB98qbLGdhj6beRXF5q1sn6 - eMtpkXXQgRABI6BBtGzrCTSP7U1JBm62ZvOm7TeYxINrGfgP3vtb0cmcig5MXrVP + Gdh8zegcr4tCfxg/yZEC109Jp0PNaB/tMHlU/XvkYGkKJN+HQ0xEZGi9yRtBbDK0 - f7BCyifuDxeTeOIRctscpSAovnbQEzqyNfhPfoY46OhdSjakxP+9+iUz0TNWVxYA + dL9mhDQx8ITLMCrLybU4+zRoWRg0tBWsMO3OKl6kGUDq3mfs+jlNnvXcgSP/RxQc - BwuEVAHXucXvDZsjGPAh5QKBgFcwSg3yYedeq9LxMtvH7a3naks8WY0Bx9EqxpVP + 1cGQb62GP1IBlMtUUCemzQKBgQCgv+/hIS5jUyWdqaujStAsVAEczUgH5/eLq8+M - U5ZWnjaW6l3uHl3Vi7npyesgjzlUYtjKjwEQoo7GatTI0iAK7xjCUqgWktp2ZXMb + S/xWP/SsgPT9Ky3WgBLkFzMU8Ljisn0P0vtdymMmDIPJMIOQA0JmxcqRgGyvTZvC - 0LdDT3wQqdVQSmngTB6H9k4wemz2g842l7sEgUUNDwl8DVMw2izdpe553D6cExAu + oLFXitKn2e7Zcu+Pov6JT28JoQo2a66KmWGUYaLyBUwuID6MNHHDaCFs2Q3+OoAS - BXf9AoGAK3aklO/J503M7YwUz5dTLReQgX+JCZWtaiZ0UG0/87fsEM0vZ31+llP6 + h1VQvQKBgQD0DA0nokwoMxZ6ClV+B+G6NSmo6JbKOtnoqqBiM5rqw0ME3h1B26hO - bZD47qNvhl69jx8jRc0N6Ol4q6vkfXN2qC+Ocbdhv1XjVK5O7451i/qPavbL7r0J + 5A41/AUKxjrFbcqE6Cm1WGQmR5vJDmKEEhF3SQXuEYm6Ji+l+awbOdYq6GvS5kCH - 1KlxBElQFKsG4qyWQHFh3BS2BER/pFkqHvyoZ+AinoUKakljXEg= + pU+imOa8uwgmK2TlYYj9LaV+mRLqqIvJ6396y66IJeTlcLLoNTasXA== -----END RSA PRIVATE KEY----- From 58f20ff9c7b271bfe1839cf3738d648c25e14ed9 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 20 Jan 2023 19:50:57 -0500 Subject: [PATCH 1400/2309] advertise all the names --- integration/vectors/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/integration/vectors/__init__.py b/integration/vectors/__init__.py index 9fadfbb8d..31c32d0aa 100644 --- a/integration/vectors/__init__.py +++ b/integration/vectors/__init__.py @@ -7,6 +7,7 @@ __all__ = [ "Sample", "SeedParam", "encode_bytes", + "save_capabilities", "capabilities", ] From d77d8d09fcd081894763fe18a81ac3fc3aaf52c9 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 20 Jan 2023 19:58:06 -0500 Subject: [PATCH 1401/2309] Give the integration tests a little longer to finish. --- .circleci/run-tests.sh | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.circleci/run-tests.sh b/.circleci/run-tests.sh index 854013c32..6d7a881fe 100755 --- a/.circleci/run-tests.sh +++ b/.circleci/run-tests.sh @@ -45,14 +45,15 @@ fi # A prefix for the test command that ensure it will exit after no more than a # certain amount of time. Ideally, we would only enforce a "silent" period -# timeout but there isn't obviously a ready-made tool for that. The test -# suite only takes about 5 - 6 minutes on CircleCI right now. 15 minutes -# seems like a moderately safe window. +# timeout but there isn't obviously a ready-made tool for that. The unit test +# suite only takes about 5 - 6 minutes on CircleCI right now. The integration +# tests are a bit longer than that. 45 minutes seems like a moderately safe +# window. # # This is primarily aimed at catching hangs on the PyPy job which runs for # about 21 minutes and then gets killed by CircleCI in a way that fails the # job and bypasses our "allowed failure" logic. -TIMEOUT="timeout --kill-after 1m 25m" +TIMEOUT="timeout --kill-after 1m 45m" # Run the test suite as a non-root user. This is the expected usage some # small areas of the test suite assume non-root privileges (such as unreadable From 96f90cdf10d9772935e4551a0ee7bbd9d0645dab Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 20 Jan 2023 21:09:28 -0500 Subject: [PATCH 1402/2309] news fragment --- newsfragments/3967.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3967.minor diff --git a/newsfragments/3967.minor b/newsfragments/3967.minor new file mode 100644 index 000000000..e69de29bb From 96d783534a986e2bca9ef5a00e99240aa2232c88 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 23 Jan 2023 08:37:31 -0500 Subject: [PATCH 1403/2309] Bump mach-nix and pypi-deps-db The newer pypi-deps-db has the pycddl release we want. The newer mach-nix is required to be compatible with fixes in that pypi-deps-db. --- nix/sources.json | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/nix/sources.json b/nix/sources.json index 950151416..18aa18e3f 100644 --- a/nix/sources.json +++ b/nix/sources.json @@ -1,14 +1,14 @@ { "mach-nix": { - "branch": "master", + "branch": "switch-to-nix-pypi-fetcher-2", "description": "Create highly reproducible python environments", "homepage": "", - "owner": "davhau", + "owner": "PrivateStorageio", "repo": "mach-nix", - "rev": "bdc97ba6b2ecd045a467b008cff4ae337b6a7a6b", - "sha256": "12b3jc0g0ak6s93g3ifvdpwxbyqx276k1kl66bpwz8a67qjbcbwf", + "rev": "f6d1a1841d8778c199326f95d0703c16bee2f8c4", + "sha256": "0krc4yhnpbzc4yhja9frnmym2vqm5zyacjnqb3fq9z9gav8vs9ls", "type": "tarball", - "url": "https://github.com/davhau/mach-nix/archive/bdc97ba6b2ecd045a467b008cff4ae337b6a7a6b.tar.gz", + "url": "https://github.com/PrivateStorageio/mach-nix/archive/f6d1a1841d8778c199326f95d0703c16bee2f8c4.tar.gz", "url_template": "https://github.com///archive/.tar.gz" }, "niv": { @@ -53,10 +53,10 @@ "homepage": "", "owner": "DavHau", "repo": "pypi-deps-db", - "rev": "5fe7d2d1c85cd86d64f4f079eef3f1ff5653bcd6", - "sha256": "0pc6mj7rzvmhh303rvj5wf4hrksm4h2rf4fsvqs0ljjdmgxrqm3f", + "rev": "5440c9c76f6431f300fb6a1ecae762a5444de5f6", + "sha256": "08r3iiaxzw9v2gq15y1m9bwajshyyz9280g6aia7mkgnjs9hnd1n", "type": "tarball", - "url": "https://github.com/DavHau/pypi-deps-db/archive/5fe7d2d1c85cd86d64f4f079eef3f1ff5653bcd6.tar.gz", + "url": "https://github.com/DavHau/pypi-deps-db/archive/5440c9c76f6431f300fb6a1ecae762a5444de5f6.tar.gz", "url_template": "https://github.com///archive/.tar.gz" } } From 55139bb3f9e91ef2959853a7988624623b774114 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 23 Jan 2023 08:50:04 -0500 Subject: [PATCH 1404/2309] We can demand >= 0.4 now --- setup.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/setup.py b/setup.py index a9b42d522..f867e901d 100644 --- a/setup.py +++ b/setup.py @@ -139,11 +139,10 @@ install_requires = [ "werkzeug != 2.2.0", "treq", "cbor2", - # Ideally we want 0.4+ to be able to pass in mmap(), but it's not strictly - # necessary yet until we fix the workaround to - # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3963 in - # allmydata.storage.http_server. - "pycddl", + + # 0.4 adds the ability to pass in mmap() values which greatly reduces the + # amount of copying involved. + "pycddl >= 0.4", # for pid-file support "psutil", From e2eac5855c3c35f3b5903922213eda58fd2d707f Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 23 Jan 2023 08:52:00 -0500 Subject: [PATCH 1405/2309] Remove handling for older versions of pycddl >=0.4 is now a hard-requirement --- src/allmydata/storage/http_server.py | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 387353d24..094b29c04 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -11,7 +11,6 @@ import binascii from tempfile import TemporaryFile from os import SEEK_END, SEEK_SET import mmap -from importlib.metadata import version as get_package_version, PackageNotFoundError from cryptography.x509 import Certificate as CryptoCertificate from zope.interface import implementer @@ -60,20 +59,6 @@ from ..util.base32 import rfc3548_alphabet from allmydata.interfaces import BadWriteEnablerError -# Until we figure out Nix (https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3963), -# need to support old pycddl which can only take bytes: -from distutils.version import LooseVersion - -try: - PYCDDL_BYTES_ONLY = LooseVersion(get_package_version("pycddl")) < LooseVersion( - "0.4" - ) -except PackageNotFoundError: - # This can happen when building PyInstaller distribution. We'll just assume - # you installed a modern pycddl, cause why wouldn't you? - PYCDDL_BYTES_ONLY = False - - class ClientSecretsException(Exception): """The client did not send the appropriate secrets.""" @@ -572,7 +557,7 @@ class HTTPServer(object): fd = request.content.fileno() except (ValueError, OSError): fd = -1 - if fd >= 0 and not PYCDDL_BYTES_ONLY: + if fd >= 0: # It's a file, so we can use mmap() to save memory. message = mmap.mmap(fd, 0, access=mmap.ACCESS_READ) else: From 1f3993b689d51a43244d2bcd17494eda1c66de0f Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 23 Jan 2023 11:37:13 -0500 Subject: [PATCH 1406/2309] Don't block on CDDL validation. --- src/allmydata/storage/http_server.py | 39 ++++++++++++++++++++-------- 1 file changed, 28 insertions(+), 11 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 53c4db5c5..982f15383 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -21,6 +21,8 @@ from twisted.internet.interfaces import ( IStreamServerEndpoint, IPullProducer, ) +from twisted.internet import reactor +from twisted.internet.task import deferToThread from twisted.internet.address import IPv4Address, IPv6Address from twisted.internet.defer import Deferred from twisted.internet.ssl import CertificateOptions, Certificate, PrivateCertificate @@ -56,6 +58,7 @@ from .common import si_a2b from .immutable import BucketWriter, ConflictingWriteError from ..util.hashutil import timing_safe_compare from ..util.base32 import rfc3548_alphabet +from ..util.deferredutil import async_to_deferred from allmydata.interfaces import BadWriteEnablerError @@ -529,7 +532,7 @@ class HTTPServer(object): # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3861 raise _HTTPError(http.NOT_ACCEPTABLE) - def _read_encoded( + async def _read_encoded( self, request, schema: Schema, max_size: int = 1024 * 1024 ) -> Any: """ @@ -543,7 +546,8 @@ class HTTPServer(object): # Make sure it's not too large: request.content.seek(SEEK_END, 0) - if request.content.tell() > max_size: + size = request.content.tell() + if size > max_size: raise _HTTPError(http.REQUEST_ENTITY_TOO_LARGE) request.content.seek(SEEK_SET, 0) @@ -562,12 +566,21 @@ class HTTPServer(object): message = mmap.mmap(fd, 0, access=mmap.ACCESS_READ) else: message = request.content.read() - schema.validate_cbor(message) + + # Pycddl will release the GIL when validating larger documents, so + # let's take advantage of multiple CPUs: + if size > 10_000: + await deferToThread(reactor, schema.validate_cbor(message)) + else: + schema.validate_cbor(message) # The CBOR parser will allocate more memory, but at least we can feed # it the file-like object, so that if it's large it won't be make two # copies. request.content.seek(SEEK_SET, 0) + # Typically deserialization to Python will not release the GIL, and + # indeed as of Jan 2023 cbor2 didn't have any code to release the GIL + # in the decode path. As such, running it in a different thread has no benefit. return cbor2.load(request.content) ##### Generic APIs ##### @@ -585,10 +598,11 @@ class HTTPServer(object): "/storage/v1/immutable/", methods=["POST"], ) - def allocate_buckets(self, request, authorization, storage_index): + @async_to_deferred + async def allocate_buckets(self, request, authorization, storage_index): """Allocate buckets.""" upload_secret = authorization[Secrets.UPLOAD] - info = self._read_encoded(request, _SCHEMAS["allocate_buckets"]) + info = await self._read_encoded(request, _SCHEMAS["allocate_buckets"]) # We do NOT validate the upload secret for existing bucket uploads. # Another upload may be happening in parallel, with a different upload @@ -745,7 +759,8 @@ class HTTPServer(object): "/storage/v1/immutable///corrupt", methods=["POST"], ) - def advise_corrupt_share_immutable( + @async_to_deferred + async def advise_corrupt_share_immutable( self, request, authorization, storage_index, share_number ): """Indicate that given share is corrupt, with a text reason.""" @@ -754,7 +769,7 @@ class HTTPServer(object): except KeyError: raise _HTTPError(http.NOT_FOUND) - info = self._read_encoded(request, _SCHEMAS["advise_corrupt_share"]) + info = await self._read_encoded(request, _SCHEMAS["advise_corrupt_share"]) bucket.advise_corrupt_share(info["reason"].encode("utf-8")) return b"" @@ -766,9 +781,10 @@ class HTTPServer(object): "/storage/v1/mutable//read-test-write", methods=["POST"], ) - def mutable_read_test_write(self, request, authorization, storage_index): + @async_to_deferred + async def mutable_read_test_write(self, request, authorization, storage_index): """Read/test/write combined operation for mutables.""" - rtw_request = self._read_encoded( + rtw_request = await self._read_encoded( request, _SCHEMAS["mutable_read_test_write"], max_size=2**48 ) secrets = ( @@ -840,7 +856,8 @@ class HTTPServer(object): "/storage/v1/mutable///corrupt", methods=["POST"], ) - def advise_corrupt_share_mutable( + @async_to_deferred + async def advise_corrupt_share_mutable( self, request, authorization, storage_index, share_number ): """Indicate that given share is corrupt, with a text reason.""" @@ -849,7 +866,7 @@ class HTTPServer(object): }: raise _HTTPError(http.NOT_FOUND) - info = self._read_encoded(request, _SCHEMAS["advise_corrupt_share"]) + info = await self._read_encoded(request, _SCHEMAS["advise_corrupt_share"]) self._storage_server.advise_corrupt_share( b"mutable", storage_index, share_number, info["reason"].encode("utf-8") ) From 80938b76a59670845741d602f9d0eb696009ab84 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 23 Jan 2023 11:39:00 -0500 Subject: [PATCH 1407/2309] News fragment. --- newsfragments/3968.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3968.minor diff --git a/newsfragments/3968.minor b/newsfragments/3968.minor new file mode 100644 index 000000000..e69de29bb From ba793e2c166e34de902417114b1a9792cb70ce37 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 24 Jan 2023 11:10:50 -0500 Subject: [PATCH 1408/2309] Make it actually work. --- src/allmydata/storage/http_server.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 1fe92915e..e1c15efd6 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -23,7 +23,7 @@ from twisted.internet.interfaces import ( IPullProducer, ) from twisted.internet import reactor -from twisted.internet.task import deferToThread +from twisted.internet.threads import deferToThread from twisted.internet.address import IPv4Address, IPv6Address from twisted.internet.defer import Deferred from twisted.internet.ssl import CertificateOptions, Certificate, PrivateCertificate @@ -639,7 +639,7 @@ class HTTPServer(object): storage_index, share_number, upload_secret, bucket ) - return self._send_encoded( + return await self._send_encoded( request, {"already-have": set(already_got), "allocated": set(sharenum_to_bucket)}, ) @@ -826,7 +826,9 @@ class HTTPServer(object): ) except BadWriteEnablerError: raise _HTTPError(http.UNAUTHORIZED) - return self._send_encoded(request, {"success": success, "data": read_data}) + return await self._send_encoded( + request, {"success": success, "data": read_data} + ) @_authorized_route( _app, From d5f5d394dd450043ac934dfe03236ae78ea14a64 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 24 Jan 2023 13:15:40 -0500 Subject: [PATCH 1409/2309] Test changes in max default segment size (both directions). --- src/allmydata/immutable/downloader/node.py | 4 +- src/allmydata/test/test_system.py | 45 +++++++++++++++++++++- 2 files changed, 47 insertions(+), 2 deletions(-) diff --git a/src/allmydata/immutable/downloader/node.py b/src/allmydata/immutable/downloader/node.py index 10ce0e5c7..02153444a 100644 --- a/src/allmydata/immutable/downloader/node.py +++ b/src/allmydata/immutable/downloader/node.py @@ -49,6 +49,8 @@ class DownloadNode(object): """Internal class which manages downloads and holds state. External callers use CiphertextFileNode instead.""" + default_max_segment_size = DEFAULT_MAX_SEGMENT_SIZE + # Share._node points to me def __init__(self, verifycap, storage_broker, secret_holder, terminator, history, download_status): @@ -76,7 +78,7 @@ class DownloadNode(object): # .guessed_segment_size, .guessed_num_segments, and # .ciphertext_hash_tree (with a dummy, to let us guess which hashes # we'll need) - self._build_guessed_tables(DEFAULT_MAX_SEGMENT_SIZE) + self._build_guessed_tables(self.default_max_segment_size) # filled in when we parse a valid UEB self.have_UEB = False diff --git a/src/allmydata/test/test_system.py b/src/allmydata/test/test_system.py index 10a64c1fe..235565020 100644 --- a/src/allmydata/test/test_system.py +++ b/src/allmydata/test/test_system.py @@ -28,13 +28,16 @@ from allmydata.storage.server import si_a2b from allmydata.immutable import offloaded, upload from allmydata.immutable.literal import LiteralFileNode from allmydata.immutable.filenode import ImmutableFileNode +from allmydata.immutable.downloader.node import DownloadNode from allmydata.util import idlib, mathutil from allmydata.util import log, base32 from allmydata.util.encodingutil import quote_output, unicode_to_argv from allmydata.util.fileutil import abspath_expanduser_unicode from allmydata.util.consumer import MemoryConsumer, download_to_data +from allmydata.util.deferredutil import async_to_deferred from allmydata.interfaces import IDirectoryNode, IFileNode, \ - NoSuchChildError, NoSharesError, SDMF_VERSION, MDMF_VERSION + NoSuchChildError, NoSharesError, SDMF_VERSION, MDMF_VERSION, \ + DEFAULT_MAX_SEGMENT_SIZE from allmydata.monitor import Monitor from allmydata.mutable.common import NotWriteableError from allmydata.mutable import layout as mutable_layout @@ -1811,6 +1814,46 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase): d.addCallback(_got_lit_filenode) return d + @async_to_deferred + async def test_upload_download_immutable_different_default_max_segment_size(self): + """ + Tahoe-LAFS used to have a default max segment size of 128KB, and is now + 1MB. Test that an upload created when 128KB was the default can be + downloaded with 1MB as the default (i.e. old uploader, new downloader), + and vice versa, (new uploader, old downloader). + """ + await self.set_up_nodes(2) + + # Just 1 share: + for c in self.clients: + c.encoding_params["k"] = 1 + c.encoding_params["happy"] = 1 + c.encoding_params["n"] = 1 + + await self._upload_download_different_max_segment(128 * 1024, 1024 * 1024) + + await self._upload_download_different_max_segment(1024 * 1024, 128 * 1024) + + + async def _upload_download_different_max_segment( + self, upload_segment_size, download_segment_size + ): + """Upload with one max segment size, download with another.""" + data = b"123456789" * 1_000_000 + + uploader = self.clients[0].getServiceNamed("uploader") + uploadable = upload.Data(data, convergence=None) + assert uploadable.max_segment_size == None + uploadable.max_segment_size = upload_segment_size + results = await uploader.upload(uploadable) + + assert DownloadNode.default_max_segment_size == DEFAULT_MAX_SEGMENT_SIZE + self.patch(DownloadNode, "default_max_segment_size", download_segment_size) + uri = results.get_uri() + node = self.clients[1].create_node_from_uri(uri) + mc = await node.read(MemoryConsumer(), 0, None) + self.assertEqual(b"".join(mc.chunks), data) + class Connections(SystemTestMixin, unittest.TestCase): FORCE_FOOLSCAP_FOR_STORAGE = True From 6ccccde9e09ec34164d65bb8ea0a7a0c5e0e5adc Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 24 Jan 2023 13:15:55 -0500 Subject: [PATCH 1410/2309] Increase the max default segment size. --- src/allmydata/interfaces.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/interfaces.py b/src/allmydata/interfaces.py index f055a01e2..8f6a0c37a 100644 --- a/src/allmydata/interfaces.py +++ b/src/allmydata/interfaces.py @@ -41,7 +41,7 @@ URI = StringConstraint(300) # kind of arbitrary MAX_BUCKETS = 256 # per peer -- zfec offers at most 256 shares per file -DEFAULT_MAX_SEGMENT_SIZE = 128*1024 +DEFAULT_MAX_SEGMENT_SIZE = 1024*1024 ShareData = StringConstraint(None) URIExtensionData = StringConstraint(1000) From 8bbce2bd1349ca362b7f26b1ed4c7bf0edd59fee Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 24 Jan 2023 13:17:46 -0500 Subject: [PATCH 1411/2309] News file. --- newsfragments/3946.bugfix | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/3946.bugfix diff --git a/newsfragments/3946.bugfix b/newsfragments/3946.bugfix new file mode 100644 index 000000000..c17a098e7 --- /dev/null +++ b/newsfragments/3946.bugfix @@ -0,0 +1 @@ +Downloads of large immutables should now finish much faster. \ No newline at end of file From cf4d7675352ad6e85136af90a9c5be39c0e00628 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 24 Jan 2023 13:28:24 -0500 Subject: [PATCH 1412/2309] Fix whitespace. --- src/allmydata/immutable/downloader/node.py | 2 +- src/allmydata/test/test_system.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/allmydata/immutable/downloader/node.py b/src/allmydata/immutable/downloader/node.py index 02153444a..ec4f751df 100644 --- a/src/allmydata/immutable/downloader/node.py +++ b/src/allmydata/immutable/downloader/node.py @@ -50,7 +50,7 @@ class DownloadNode(object): callers use CiphertextFileNode instead.""" default_max_segment_size = DEFAULT_MAX_SEGMENT_SIZE - + # Share._node points to me def __init__(self, verifycap, storage_broker, secret_holder, terminator, history, download_status): diff --git a/src/allmydata/test/test_system.py b/src/allmydata/test/test_system.py index 235565020..9849c6b30 100644 --- a/src/allmydata/test/test_system.py +++ b/src/allmydata/test/test_system.py @@ -1831,7 +1831,6 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase): c.encoding_params["n"] = 1 await self._upload_download_different_max_segment(128 * 1024, 1024 * 1024) - await self._upload_download_different_max_segment(1024 * 1024, 128 * 1024) From 921a2083dcefdb5f431cdac195fc9ac510605349 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 25 Jan 2023 15:47:35 -0500 Subject: [PATCH 1413/2309] Make sure (immutable) test vectors are run with the segment size that was used to generate them. --- integration/test_vectors.py | 3 ++- integration/util.py | 16 +++++++++++++++- src/allmydata/client.py | 8 ++++++++ 3 files changed, 25 insertions(+), 2 deletions(-) diff --git a/integration/test_vectors.py b/integration/test_vectors.py index 3e0790786..1f8747004 100644 --- a/integration/test_vectors.py +++ b/integration/test_vectors.py @@ -36,7 +36,8 @@ async def test_capability(reactor, request, alice, case, expected): computed value. """ # rewrite alice's config to match params and convergence - await reconfigure(reactor, request, alice, (1, case.params.required, case.params.total), case.convergence) + await reconfigure( + reactor, request, alice, (1, case.params.required, case.params.total), case.convergence, case.segment_size) # upload data in the correct format actual = upload(alice, case.fmt, case.data) diff --git a/integration/util.py b/integration/util.py index c7ed31a09..a6543c34d 100644 --- a/integration/util.py +++ b/integration/util.py @@ -46,6 +46,7 @@ from allmydata.util.configutil import ( write_config, ) from allmydata import client +from allmydata.interfaces import DEFAULT_MAX_SEGMENT_SIZE import pytest_twisted @@ -729,7 +730,10 @@ def upload(alice: TahoeProcess, fmt: CHK | SSK, data: bytes) -> str: return cli(*argv).decode("utf-8").strip() -async def reconfigure(reactor, request, node: TahoeProcess, params: tuple[int, int, int], convergence: None | bytes) -> None: +async def reconfigure(reactor, request, node: TahoeProcess, + params: tuple[int, int, int], + convergence: None | bytes, + max_segment_size: None | int = None) -> None: """ Reconfigure a Tahoe-LAFS node with different ZFEC parameters and convergence secret. @@ -769,6 +773,16 @@ async def reconfigure(reactor, request, node: TahoeProcess, params: tuple[int, i changed = True config.write_private_config("convergence", base32.b2a(convergence)) + if max_segment_size is not None: + cur_segment_size = int(config.get_config("client", "shares._max_immutable_segment_size_for_testing", DEFAULT_MAX_SEGMENT_SIZE)) + if cur_segment_size != max_segment_size: + changed = True + config.set_config( + "client", + "shares._max_immutable_segment_size_for_testing", + str(max_segment_size) + ) + if changed: # restart the node print(f"Restarting {node.node_dir} for ZFEC reconfiguration") diff --git a/src/allmydata/client.py b/src/allmydata/client.py index 73672f30a..c06961c8c 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -88,6 +88,7 @@ _client_config = configutil.ValidConfiguration( "shares.happy", "shares.needed", "shares.total", + "shares._max_immutable_segment_size_for_testing", "storage.plugins", ), "storage": ( @@ -896,6 +897,13 @@ class _Client(node.Node, pollmixin.PollMixin): DEP["k"] = int(self.config.get_config("client", "shares.needed", DEP["k"])) DEP["n"] = int(self.config.get_config("client", "shares.total", DEP["n"])) DEP["happy"] = int(self.config.get_config("client", "shares.happy", DEP["happy"])) + # At the moment this is only used for testing, thus the janky config + # attribute name. + DEP["max_segment_size"] = int(self.config.get_config( + "client", + "shares._max_immutable_segment_size_for_testing", + DEP["max_segment_size"]) + ) # for the CLI to authenticate to local JSON endpoints self._create_auth_token() From 669296d5d68a7705698bcc2fe15fa56491948fdc Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 26 Jan 2023 11:44:53 -0500 Subject: [PATCH 1414/2309] News file. --- newsfragments/3935.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3935.minor diff --git a/newsfragments/3935.minor b/newsfragments/3935.minor new file mode 100644 index 000000000..e69de29bb From c37e330efd35af20cdba267ff4a82d3df4d19efb Mon Sep 17 00:00:00 2001 From: Chris Wood Date: Fri, 27 Jan 2023 10:02:59 -0500 Subject: [PATCH 1415/2309] Add charset_normalizer.md__mypyc to hidden imports Fixes: https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3966 Overrides: https://github.com/tahoe-lafs/tahoe-lafs/pull/1248 Ref.: https://github.com/pyinstaller/pyinstaller-hooks-contrib/issues/534 --- pyinstaller.spec | 1 + 1 file changed, 1 insertion(+) diff --git a/pyinstaller.spec b/pyinstaller.spec index eece50757..01b1ac4ac 100644 --- a/pyinstaller.spec +++ b/pyinstaller.spec @@ -36,6 +36,7 @@ hidden_imports = [ 'allmydata.stats', 'base64', 'cffi', + 'charset_normalizer.md__mypyc', 'collections', 'commands', 'Crypto', From 87dad9bd2bf96eedfd671e17fe003932358d7157 Mon Sep 17 00:00:00 2001 From: Chris Wood Date: Fri, 27 Jan 2023 10:07:50 -0500 Subject: [PATCH 1416/2309] Remove "charset_normalizer < 3" constraint --- setup.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/setup.py b/setup.py index f867e901d..1974145cb 100644 --- a/setup.py +++ b/setup.py @@ -147,13 +147,6 @@ install_requires = [ # for pid-file support "psutil", "filelock", - - # treq needs requests, requests needs charset_normalizer, - # charset_normalizer breaks PyInstaller - # (https://github.com/Ousret/charset_normalizer/issues/253). So work around - # this by using a lower version number. Once upstream issue is fixed, or - # requests drops charset_normalizer, this can go away. - "charset_normalizer < 3", ] setup_requires = [ From a292f52de1aa44e34e9b72ab57d4ac608b6b5da3 Mon Sep 17 00:00:00 2001 From: Chris Wood Date: Fri, 27 Jan 2023 11:47:50 -0500 Subject: [PATCH 1417/2309] Try debugging CI/ubuntu-20.04 integration tests.. Does restoring the "charset_normalizer < 3" pin make the tests pass? --- setup.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/setup.py b/setup.py index 1974145cb..f867e901d 100644 --- a/setup.py +++ b/setup.py @@ -147,6 +147,13 @@ install_requires = [ # for pid-file support "psutil", "filelock", + + # treq needs requests, requests needs charset_normalizer, + # charset_normalizer breaks PyInstaller + # (https://github.com/Ousret/charset_normalizer/issues/253). So work around + # this by using a lower version number. Once upstream issue is fixed, or + # requests drops charset_normalizer, this can go away. + "charset_normalizer < 3", ] setup_requires = [ From e046627d3178846bdc581ce5f38de07f52f0cdca Mon Sep 17 00:00:00 2001 From: Chris Wood Date: Fri, 27 Jan 2023 11:59:24 -0500 Subject: [PATCH 1418/2309] Try debugging CI/ubuntu-20.04 integration tests... Does removing the `charset_normalizer.md__mypyc` hidden import make the tests pass? --- pyinstaller.spec | 1 - 1 file changed, 1 deletion(-) diff --git a/pyinstaller.spec b/pyinstaller.spec index 01b1ac4ac..eece50757 100644 --- a/pyinstaller.spec +++ b/pyinstaller.spec @@ -36,7 +36,6 @@ hidden_imports = [ 'allmydata.stats', 'base64', 'cffi', - 'charset_normalizer.md__mypyc', 'collections', 'commands', 'Crypto', From 15c7916e0812e6baa2a931cd54b18f3382a8456e Mon Sep 17 00:00:00 2001 From: Chris Wood Date: Fri, 27 Jan 2023 12:46:30 -0500 Subject: [PATCH 1419/2309] Revert previous two commits (e046627, a292f52) --- pyinstaller.spec | 1 + setup.py | 7 ------- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/pyinstaller.spec b/pyinstaller.spec index eece50757..01b1ac4ac 100644 --- a/pyinstaller.spec +++ b/pyinstaller.spec @@ -36,6 +36,7 @@ hidden_imports = [ 'allmydata.stats', 'base64', 'cffi', + 'charset_normalizer.md__mypyc', 'collections', 'commands', 'Crypto', diff --git a/setup.py b/setup.py index f867e901d..1974145cb 100644 --- a/setup.py +++ b/setup.py @@ -147,13 +147,6 @@ install_requires = [ # for pid-file support "psutil", "filelock", - - # treq needs requests, requests needs charset_normalizer, - # charset_normalizer breaks PyInstaller - # (https://github.com/Ousret/charset_normalizer/issues/253). So work around - # this by using a lower version number. Once upstream issue is fixed, or - # requests drops charset_normalizer, this can go away. - "charset_normalizer < 3", ] setup_requires = [ From ff964b2310382b92c9d0392fa94705d3b64f2dec Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Sat, 28 Jan 2023 08:53:53 -0500 Subject: [PATCH 1420/2309] news fragment --- newsfragments/3969.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3969.minor diff --git a/newsfragments/3969.minor b/newsfragments/3969.minor new file mode 100644 index 000000000..e69de29bb From 230ce346c578c7a0969661b60f931397d4081c36 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Sat, 28 Jan 2023 08:54:00 -0500 Subject: [PATCH 1421/2309] circleci env var notes --- .circleci/circleci.txt | 78 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 .circleci/circleci.txt diff --git a/.circleci/circleci.txt b/.circleci/circleci.txt new file mode 100644 index 000000000..c7adf9ec1 --- /dev/null +++ b/.circleci/circleci.txt @@ -0,0 +1,78 @@ +# A master build looks like this: + +# BASH_ENV=/tmp/.bash_env-63d018969ca480003a031e62-0-build +# CI=true +# CIRCLECI=true +# CIRCLE_BRANCH=master +# CIRCLE_BUILD_NUM=76545 +# CIRCLE_BUILD_URL=https://circleci.com/gh/tahoe-lafs/tahoe-lafs/76545 +# CIRCLE_JOB=NixOS 21.11 +# CIRCLE_NODE_INDEX=0 +# CIRCLE_NODE_TOTAL=1 +# CIRCLE_PROJECT_REPONAME=tahoe-lafs +# CIRCLE_PROJECT_USERNAME=tahoe-lafs +# CIRCLE_REPOSITORY_URL=git@github.com:tahoe-lafs/tahoe-lafs.git +# CIRCLE_SHA1=ed0bda2d7456f4a2cd60870072e1fe79864a49a1 +# CIRCLE_SHELL_ENV=/tmp/.bash_env-63d018969ca480003a031e62-0-build +# CIRCLE_USERNAME=alice +# CIRCLE_WORKFLOW_ID=6d9bb71c-be3a-4659-bf27-60954180619b +# CIRCLE_WORKFLOW_JOB_ID=0793c975-7b9f-489f-909b-8349b72d2785 +# CIRCLE_WORKFLOW_WORKSPACE_ID=6d9bb71c-be3a-4659-bf27-60954180619b +# CIRCLE_WORKING_DIRECTORY=~/project + +# A build of an in-repo PR looks like this: + +# BASH_ENV=/tmp/.bash_env-63d1971a0298086d8841287e-0-build +# CI=true +# CIRCLECI=true +# CIRCLE_BRANCH=3946-less-chatty-downloads +# CIRCLE_BUILD_NUM=76612 +# CIRCLE_BUILD_URL=https://circleci.com/gh/tahoe-lafs/tahoe-lafs/76612 +# CIRCLE_JOB=NixOS 21.11 +# CIRCLE_NODE_INDEX=0 +# CIRCLE_NODE_TOTAL=1 +# CIRCLE_PROJECT_REPONAME=tahoe-lafs +# CIRCLE_PROJECT_USERNAME=tahoe-lafs +# CIRCLE_PULL_REQUEST=https://github.com/tahoe-lafs/tahoe-lafs/pull/1251 +# CIRCLE_PULL_REQUESTS=https://github.com/tahoe-lafs/tahoe-lafs/pull/1251 +# CIRCLE_REPOSITORY_URL=git@github.com:tahoe-lafs/tahoe-lafs.git +# CIRCLE_SHA1=921a2083dcefdb5f431cdac195fc9ac510605349 +# CIRCLE_SHELL_ENV=/tmp/.bash_env-63d1971a0298086d8841287e-0-build +# CIRCLE_USERNAME=bob +# CIRCLE_WORKFLOW_ID=5e32c12e-be37-4868-9fa8-6a6929fec2f1 +# CIRCLE_WORKFLOW_JOB_ID=316ca408-81b4-4c96-bbdd-644e4c3e01e5 +# CIRCLE_WORKFLOW_WORKSPACE_ID=5e32c12e-be37-4868-9fa8-6a6929fec2f1 +# CIRCLE_WORKING_DIRECTORY=~/project +# CI_PULL_REQUEST=https://github.com/tahoe-lafs/tahoe-lafs/pull/1251 + +# A build of a PR from a fork looks like this: + +# BASH_ENV=/tmp/.bash_env-63d40f7b2e89cd3de10e0db9-0-build +# CI=true +# CIRCLECI=true +# CIRCLE_BRANCH=pull/1252 +# CIRCLE_BUILD_NUM=76678 +# CIRCLE_BUILD_URL=https://circleci.com/gh/tahoe-lafs/tahoe-lafs/76678 +# CIRCLE_JOB=NixOS 21.05 +# CIRCLE_NODE_INDEX=0 +# CIRCLE_NODE_TOTAL=1 +# CIRCLE_PROJECT_REPONAME=tahoe-lafs +# CIRCLE_PROJECT_USERNAME=tahoe-lafs +# CIRCLE_PR_NUMBER=1252 +# CIRCLE_PR_REPONAME=tahoe-lafs +# CIRCLE_PR_USERNAME=carol +# CIRCLE_PULL_REQUEST=https://github.com/tahoe-lafs/tahoe-lafs/pull/1252 +# CIRCLE_PULL_REQUESTS=https://github.com/tahoe-lafs/tahoe-lafs/pull/1252 +# CIRCLE_REPOSITORY_URL=git@github.com:tahoe-lafs/tahoe-lafs.git +# CIRCLE_SHA1=15c7916e0812e6baa2a931cd54b18f3382a8456e +# CIRCLE_SHELL_ENV=/tmp/.bash_env-63d40f7b2e89cd3de10e0db9-0-build +# CIRCLE_USERNAME= +# CIRCLE_WORKFLOW_ID=19c917c8-3a38-4b20-ac10-3265259fa03e +# CIRCLE_WORKFLOW_JOB_ID=58e95215-eccf-4664-a231-1dba7fd2d323 +# CIRCLE_WORKFLOW_WORKSPACE_ID=19c917c8-3a38-4b20-ac10-3265259fa03e +# CIRCLE_WORKING_DIRECTORY=~/project +# CI_PULL_REQUEST=https://github.com/tahoe-lafs/tahoe-lafs/pull/1252 + +# A build of a PR from a fork where the owner has enabled CircleCI looks +# the same as a build of an in-repo PR, except it runs on th owner's +# CircleCI namespace. From 3d58194c3ae73ae2048df952418bbdaaf82c6156 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Sat, 28 Jan 2023 08:56:48 -0500 Subject: [PATCH 1422/2309] Complexify the upstream-vs-forked detection --- .circleci/lib.sh | 95 ++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 83 insertions(+), 12 deletions(-) diff --git a/.circleci/lib.sh b/.circleci/lib.sh index 7717cdb18..b5c900371 100644 --- a/.circleci/lib.sh +++ b/.circleci/lib.sh @@ -1,21 +1,20 @@ # Run a command, enabling cache writes to cachix if possible. The command is # accepted as a variable number of positional arguments (like argv). function cache_if_able() { - # The `cachix watch-exec ...` does our cache population. When it sees - # something added to the store (I guess) it pushes it to the named cache. - # - # We can only *push* to it if we have a CACHIX_AUTH_TOKEN, though. - # in-repo jobs will get this from CircleCI configuration but jobs from - # forks may not. - echo "Building PR from user/org: ${CIRCLE_PROJECT_USERNAME}" - if [ -v CACHIX_AUTH_TOKEN ]; then + # Dump some info about our build environment. + describe_build + + if is_cache_writeable; then + # If the cache is available we'll use it. This lets fork owners set + # up their own caching if they want. echo "Cachix credentials present; will attempt to write to cache." + + # The `cachix watch-exec ...` does our cache population. When it sees + # something added to the store (I guess) it pushes it to the named + # cache. cachix watch-exec "${CACHIX_NAME}" -- "$@" else - # If we're building a from a forked repository then we're allowed to - # not have the credentials (but it's also fine if the owner of the - # fork supplied their own). - if [ "${CIRCLE_PROJECT_USERNAME}" == "tahoe-lafs" ]; then + if is_cache_required; then echo "Required credentials (CACHIX_AUTH_TOKEN) are missing." return 1 else @@ -24,3 +23,75 @@ function cache_if_able() { fi fi } + +function is_cache_writeable() { + # We can only *push* to the cache if we have a CACHIX_AUTH_TOKEN. in-repo + # jobs will get this from CircleCI configuration but jobs from forks may + # not. + [ -v CACHIX_AUTH_TOKEN ] +} + +function is_cache_required() { + # If we're building in tahoe-lafs/tahoe-lafs then we must use the cache. + # If we're building anything from a fork then we're allowed to not have + # the credentials. + is_upstream +} + +# Return success if the origin of this build is the tahoe-lafs/tahoe-lafs +# repository itself (and so we expect to have cache credentials available), +# failure otherwise. +# +# See circleci.txt for notes about how this determination is made. +function is_upstream() { + # CIRCLE_PROJECT_USERNAME is set to the org the build is happening for. + # If a PR targets a fork of the repo then this is set to something other + # than "tahoe-lafs". + [ "$CIRCLE_PROJECT_USERNAME" == "tahoe-lafs" ] && + + # CIRCLE_BRANCH is set to the real branch name for in-repo PRs and + # "pull/NNNN" for pull requests from forks. + # + # CIRCLE_PULL_REQUEST is set to the full URL of the PR page which ends + # with that same "pull/NNNN" for PRs from forks. + ! endswith "/$CIRCLE_BRANCH" "$CIRCLE_PULL_REQUEST" +} + +# Return success if $2 ends with $1, failure otherwise. +function endswith() { + suffix=$1 + shift + + haystack=$1 + shift + + case "$haystack" in + *${suffix}) + return 0 + ;; + + *) + return 1 + ;; + esac +} + +function describe_build() { + echo "Building PR for user/org: ${CIRCLE_PROJECT_USERNAME}" + echo "Building branch: ${CIRCLE_BRANCH}" + if is_upstream; then + echo "Upstream build." + else + echo "Non-upstream build." + fi + if is_cache_required; then + echo "Cache is required." + else + echo "Cache not required." + fi + if is_cache_writeable; then + echo "Cache is writeable." + else + echo "Cache not writeable." + fi +} From 4ea4286a7f52429addb4dbd1eb0467659fb2b66f Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Sat, 28 Jan 2023 09:21:34 -0500 Subject: [PATCH 1423/2309] Use CIRCLE_PULL_REQUESTS in case there are multiple which, of course, there never are, except for during testing of this branch --- .circleci/lib.sh | 28 +++++++++++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/.circleci/lib.sh b/.circleci/lib.sh index b5c900371..c692b5f88 100644 --- a/.circleci/lib.sh +++ b/.circleci/lib.sh @@ -52,9 +52,31 @@ function is_upstream() { # CIRCLE_BRANCH is set to the real branch name for in-repo PRs and # "pull/NNNN" for pull requests from forks. # - # CIRCLE_PULL_REQUEST is set to the full URL of the PR page which ends - # with that same "pull/NNNN" for PRs from forks. - ! endswith "/$CIRCLE_BRANCH" "$CIRCLE_PULL_REQUEST" + # CIRCLE_PULL_REQUESTS is set to a comma-separated list of the full + # URLs of the PR pages which share an underlying branch, with one of + # them ended with that same "pull/NNNN" for PRs from forks. + ! any_element_endswith "/$CIRCLE_BRANCH" "," "$CIRCLE_PULL_REQUESTS" +} + +# Return success if splitting $3 on $2 results in an array with any element +# that ends with $1, failure otherwise. +function any_element_endswith() { + suffix=$1 + shift + + sep=$1 + shift + + haystack=$1 + shift + + IFS="${sep}" read -r -a elements <<< "$haystack" + for elem in "${elements[@]}"; do + if endswith "$suffix" "$elem"; then + return 0 + fi + done + return 1 } # Return success if $2 ends with $1, failure otherwise. From cad81c9bdda0ad5d1bfe066d0ae8fccdcb851615 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Sat, 28 Jan 2023 16:21:45 -0500 Subject: [PATCH 1424/2309] Twiddle the news fragment to pass codechecks --- newsfragments/3966.bugfix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/newsfragments/3966.bugfix b/newsfragments/3966.bugfix index ead94c47c..384dcf797 100644 --- a/newsfragments/3966.bugfix +++ b/newsfragments/3966.bugfix @@ -1 +1 @@ -Fix incompatibility with newer versions of the transitive charset_normalizer dependency when using PyInstaller. \ No newline at end of file +Fix incompatibility with transitive dependency charset_normalizer >= 3 when using PyInstaller. From 9553901ca191551581e9404a81ede1d8e6bbd36f Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 31 Jan 2023 14:22:57 -0500 Subject: [PATCH 1425/2309] Add caveats. --- benchmarks/upload_download.py | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/benchmarks/upload_download.py b/benchmarks/upload_download.py index bd1b08e7a..aa5f506bc 100644 --- a/benchmarks/upload_download.py +++ b/benchmarks/upload_download.py @@ -3,13 +3,25 @@ First attempt at benchmarking uploads and downloads. To run: -$ pytest benchmarks/upload_download.py -s -v -Wignore +$ pytest benchmarks/upload_download.py -s -v -Wignore TODO Parameterization (pytest?) -- Foolscap vs not foolscap -- Number of nodes -- Data size -- Number of needed/happy/total shares. + + - Foolscap vs not foolscap + + - Number of nodes + + - Data size + + - Number of needed/happy/total shares. + +CAVEATS: The goal here isn't a realistic benchmark, or a benchmark that will be +measured over time, or is expected to be maintainable over time. This is just +a quick and easy way to measure the speed of certain operations, compare HTTP +and Foolscap, and see the short-term impact of changes. + +Eventually this will be replaced by a real benchmark suite that can be run over +time to measure something more meaningful. """ from time import time, process_time From b477c59e155f7aa8c47db9cd115db5d997bc110f Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 31 Jan 2023 15:53:16 -0500 Subject: [PATCH 1426/2309] Actually have a working run-in-thread code path --- src/allmydata/storage/http_server.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 7490e54b7..6b94c227a 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -545,11 +545,11 @@ class HTTPServer(object): raise _HTTPError(http.UNSUPPORTED_MEDIA_TYPE) # Make sure it's not too large: - request.content.seek(SEEK_END, 0) + request.content.seek(0, SEEK_END) size = request.content.tell() if size > max_size: raise _HTTPError(http.REQUEST_ENTITY_TOO_LARGE) - request.content.seek(SEEK_SET, 0) + request.content.seek(0, SEEK_SET) # We don't want to load the whole message into memory, cause it might # be quite large. The CDDL validator takes a read-only bytes-like @@ -570,7 +570,7 @@ class HTTPServer(object): # Pycddl will release the GIL when validating larger documents, so # let's take advantage of multiple CPUs: if size > 10_000: - await deferToThread(reactor, schema.validate_cbor(message)) + await deferToThread(schema.validate_cbor, message) else: schema.validate_cbor(message) From c371a1f6b317a52fc5f1cb2b4c7207c4b3668a48 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 31 Jan 2023 15:53:28 -0500 Subject: [PATCH 1427/2309] Add benchmark for parallel uploads. --- benchmarks/upload_download.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/benchmarks/upload_download.py b/benchmarks/upload_download.py index 3e761d4a1..42f288e68 100644 --- a/benchmarks/upload_download.py +++ b/benchmarks/upload_download.py @@ -38,6 +38,7 @@ from tempfile import mkdtemp import os from twisted.trial.unittest import TestCase +from twisted.internet.defer import gatherResults from allmydata.util.deferredutil import async_to_deferred from allmydata.util.consumer import MemoryConsumer @@ -112,3 +113,13 @@ class ImmutableBenchmarks(SystemTestMixin, TestCase): with timeit("download"): data = await result.download_best_version() self.assertEqual(data, DATA) + + @async_to_deferred + async def test_upload_mutable_in_parallel(self): + # To test larger files, change this: + DATA = b"Some data to upload\n" * 1_000_000 + with timeit(" upload"): + await gatherResults([ + self.clients[0].create_mutable_file(MutableData(DATA)) + for _ in range(20) + ]) From 7f3af6a8ede45dcabc24af0903b41e582c5a5ca6 Mon Sep 17 00:00:00 2001 From: dlee Date: Fri, 3 Feb 2023 16:30:07 -0600 Subject: [PATCH 1428/2309] typechecks made more strict using more flags --- mypy.ini | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/mypy.ini b/mypy.ini index 01cbb57a8..c6d8affe3 100644 --- a/mypy.ini +++ b/mypy.ini @@ -1,3 +1,9 @@ [mypy] ignore_missing_imports = True plugins=mypy_zope:plugin +show_column_numbers = True +pretty = True +show_error_codes = True +warn_unused_configs =True +warn_redundant_casts = True +strict_equality = True \ No newline at end of file From e2e33933a88db59e98e749ab59d7115f3e169aa6 Mon Sep 17 00:00:00 2001 From: dlee Date: Fri, 3 Feb 2023 16:48:06 -0600 Subject: [PATCH 1429/2309] Forgot to push newsfragment --- newsfragments/3971.minor | 1 + src/allmydata/crypto/rsa.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 newsfragments/3971.minor diff --git a/newsfragments/3971.minor b/newsfragments/3971.minor new file mode 100644 index 000000000..a6cbb6a89 --- /dev/null +++ b/newsfragments/3971.minor @@ -0,0 +1 @@ +Changes made to mypy.ini to make mypy more 'strict' and prevent future regressions. \ No newline at end of file diff --git a/src/allmydata/crypto/rsa.py b/src/allmydata/crypto/rsa.py index e579a3d2a..dca908467 100644 --- a/src/allmydata/crypto/rsa.py +++ b/src/allmydata/crypto/rsa.py @@ -127,7 +127,7 @@ def der_string_from_signing_key(private_key: PrivateKey) -> bytes: :returns: bytes representing `private_key` """ _validate_private_key(private_key) - return private_key.private_bytes( # type: ignore[attr-defined] + return private_key.private_bytes( encoding=Encoding.DER, format=PrivateFormat.PKCS8, encryption_algorithm=NoEncryption(), From 80db4a9de4b3c95ec231d1a55dc7f48f96a014f5 Mon Sep 17 00:00:00 2001 From: Daniel Date: Fri, 3 Feb 2023 21:25:24 -0600 Subject: [PATCH 1430/2309] Delete rsa.py --- src/allmydata/crypto/rsa.py | 225 ------------------------------------ 1 file changed, 225 deletions(-) delete mode 100644 src/allmydata/crypto/rsa.py diff --git a/src/allmydata/crypto/rsa.py b/src/allmydata/crypto/rsa.py deleted file mode 100644 index dca908467..000000000 --- a/src/allmydata/crypto/rsa.py +++ /dev/null @@ -1,225 +0,0 @@ -""" -Helper functions for cryptography-related operations inside Tahoe -using RSA public-key encryption and decryption. - -In cases where these functions happen to use and return objects that -are documented in the `cryptography` library, code outside this module -should only use functions from allmydata.crypto.rsa and not rely on -features of any objects that `cryptography` documents. - -That is, the public and private keys are opaque objects; DO NOT depend -on any of their methods. -""" - -from __future__ import annotations - -from typing_extensions import TypeAlias -from typing import Callable - -from functools import partial - -from cryptography.exceptions import InvalidSignature -from cryptography.hazmat.backends import default_backend -from cryptography.hazmat.primitives import hashes -from cryptography.hazmat.primitives.asymmetric import rsa, padding -from cryptography.hazmat.primitives.serialization import load_der_private_key, load_der_public_key, \ - Encoding, PrivateFormat, PublicFormat, NoEncryption - -from allmydata.crypto.error import BadSignature - -PublicKey: TypeAlias = rsa.RSAPublicKey -PrivateKey: TypeAlias = rsa.RSAPrivateKey - -# This is the value that was used by `pycryptopp`, and we must continue to use it for -# both backwards compatibility and interoperability. -# -# The docs for `cryptography` suggest to use the constant defined at -# `cryptography.hazmat.primitives.asymmetric.padding.PSS.MAX_LENGTH`, but this causes old -# signatures to fail to validate. -RSA_PSS_SALT_LENGTH = 32 - -RSA_PADDING = padding.PSS( - mgf=padding.MGF1(hashes.SHA256()), - salt_length=RSA_PSS_SALT_LENGTH, -) - - - -def create_signing_keypair(key_size: int) -> tuple[PrivateKey, PublicKey]: - """ - Create a new RSA signing (private) keypair from scratch. Can be used with - `sign_data` function. - - :param key_size: length of key in bits - - :returns: 2-tuple of (private_key, public_key) - """ - priv_key = rsa.generate_private_key( - public_exponent=65537, - key_size=key_size, - backend=default_backend() - ) - return priv_key, priv_key.public_key() - - -def create_signing_keypair_from_string(private_key_der: bytes) -> tuple[PrivateKey, PublicKey]: - """ - Create an RSA signing (private) key from previously serialized - private key bytes. - - :param private_key_der: blob as returned from `der_string_from_signing_keypair` - - :returns: 2-tuple of (private_key, public_key) - """ - _load = partial( - load_der_private_key, - private_key_der, - password=None, - backend=default_backend(), - ) - - def load_with_validation() -> PrivateKey: - k = _load() - assert isinstance(k, PrivateKey) - return k - - def load_without_validation() -> PrivateKey: - k = _load(unsafe_skip_rsa_key_validation=True) - assert isinstance(k, PrivateKey) - return k - - # Load it once without the potentially expensive OpenSSL validation - # checks. These have superlinear complexity. We *will* run them just - # below - but first we'll apply our own constant-time checks. - load: Callable[[], PrivateKey] = load_without_validation - try: - unsafe_priv_key = load() - except TypeError: - # cryptography<39 does not support this parameter, so just load the - # key with validation... - unsafe_priv_key = load_with_validation() - # But avoid *reloading* it since that will run the expensive - # validation *again*. - load = lambda: unsafe_priv_key - - if not isinstance(unsafe_priv_key, rsa.RSAPrivateKey): - raise ValueError( - "Private Key did not decode to an RSA key" - ) - if unsafe_priv_key.key_size != 2048: - raise ValueError( - "Private Key must be 2048 bits" - ) - - # Now re-load it with OpenSSL's validation applied. - safe_priv_key = load() - - return safe_priv_key, safe_priv_key.public_key() - - -def der_string_from_signing_key(private_key: PrivateKey) -> bytes: - """ - Serializes a given RSA private key to a DER string - - :param private_key: a private key object as returned from - `create_signing_keypair` or `create_signing_keypair_from_string` - - :returns: bytes representing `private_key` - """ - _validate_private_key(private_key) - return private_key.private_bytes( - encoding=Encoding.DER, - format=PrivateFormat.PKCS8, - encryption_algorithm=NoEncryption(), - ) - - -def der_string_from_verifying_key(public_key: PublicKey) -> bytes: - """ - Serializes a given RSA public key to a DER string. - - :param public_key: a public key object as returned from - `create_signing_keypair` or `create_signing_keypair_from_string` - - :returns: bytes representing `public_key` - """ - _validate_public_key(public_key) - return public_key.public_bytes( - encoding=Encoding.DER, - format=PublicFormat.SubjectPublicKeyInfo, - ) - - -def create_verifying_key_from_string(public_key_der: bytes) -> PublicKey: - """ - Create an RSA verifying key from a previously serialized public key - - :param bytes public_key_der: a blob as returned by `der_string_from_verifying_key` - - :returns: a public key object suitable for use with other - functions in this module - """ - pub_key = load_der_public_key( - public_key_der, - backend=default_backend(), - ) - assert isinstance(pub_key, PublicKey) - return pub_key - - -def sign_data(private_key: PrivateKey, data: bytes) -> bytes: - """ - :param private_key: the private part of a keypair returned from - `create_signing_keypair_from_string` or `create_signing_keypair` - - :param data: the bytes to sign - - :returns: bytes which are a signature of the bytes given as `data`. - """ - _validate_private_key(private_key) - return private_key.sign( - data, - RSA_PADDING, - hashes.SHA256(), - ) - -def verify_signature(public_key: PublicKey, alleged_signature: bytes, data: bytes) -> None: - """ - :param public_key: a verifying key, returned from `create_verifying_key_from_string` or `create_verifying_key_from_private_key` - - :param bytes alleged_signature: the bytes of the alleged signature - - :param bytes data: the data which was allegedly signed - """ - _validate_public_key(public_key) - try: - public_key.verify( - alleged_signature, - data, - RSA_PADDING, - hashes.SHA256(), - ) - except InvalidSignature: - raise BadSignature() - - -def _validate_public_key(public_key: PublicKey) -> None: - """ - Internal helper. Checks that `public_key` is a valid cryptography - object - """ - if not isinstance(public_key, rsa.RSAPublicKey): - raise ValueError( - f"public_key must be an RSAPublicKey not {type(public_key)}" - ) - - -def _validate_private_key(private_key: PrivateKey) -> None: - """ - Internal helper. Checks that `public_key` is a valid cryptography - object - """ - if not isinstance(private_key, rsa.RSAPrivateKey): - raise ValueError( - f"private_key must be an RSAPrivateKey not {type(private_key)}" - ) From 31c5b78e6ab9d17efffd647767fbf8224ced77f6 Mon Sep 17 00:00:00 2001 From: dlee Date: Fri, 3 Feb 2023 21:35:55 -0600 Subject: [PATCH 1431/2309] Add back rsa.py accidentally removed file on website --- src/allmydata/crypto/rsa.py | 225 ++++++++++++++++++++++++++++++++++++ 1 file changed, 225 insertions(+) create mode 100644 src/allmydata/crypto/rsa.py diff --git a/src/allmydata/crypto/rsa.py b/src/allmydata/crypto/rsa.py new file mode 100644 index 000000000..e579a3d2a --- /dev/null +++ b/src/allmydata/crypto/rsa.py @@ -0,0 +1,225 @@ +""" +Helper functions for cryptography-related operations inside Tahoe +using RSA public-key encryption and decryption. + +In cases where these functions happen to use and return objects that +are documented in the `cryptography` library, code outside this module +should only use functions from allmydata.crypto.rsa and not rely on +features of any objects that `cryptography` documents. + +That is, the public and private keys are opaque objects; DO NOT depend +on any of their methods. +""" + +from __future__ import annotations + +from typing_extensions import TypeAlias +from typing import Callable + +from functools import partial + +from cryptography.exceptions import InvalidSignature +from cryptography.hazmat.backends import default_backend +from cryptography.hazmat.primitives import hashes +from cryptography.hazmat.primitives.asymmetric import rsa, padding +from cryptography.hazmat.primitives.serialization import load_der_private_key, load_der_public_key, \ + Encoding, PrivateFormat, PublicFormat, NoEncryption + +from allmydata.crypto.error import BadSignature + +PublicKey: TypeAlias = rsa.RSAPublicKey +PrivateKey: TypeAlias = rsa.RSAPrivateKey + +# This is the value that was used by `pycryptopp`, and we must continue to use it for +# both backwards compatibility and interoperability. +# +# The docs for `cryptography` suggest to use the constant defined at +# `cryptography.hazmat.primitives.asymmetric.padding.PSS.MAX_LENGTH`, but this causes old +# signatures to fail to validate. +RSA_PSS_SALT_LENGTH = 32 + +RSA_PADDING = padding.PSS( + mgf=padding.MGF1(hashes.SHA256()), + salt_length=RSA_PSS_SALT_LENGTH, +) + + + +def create_signing_keypair(key_size: int) -> tuple[PrivateKey, PublicKey]: + """ + Create a new RSA signing (private) keypair from scratch. Can be used with + `sign_data` function. + + :param key_size: length of key in bits + + :returns: 2-tuple of (private_key, public_key) + """ + priv_key = rsa.generate_private_key( + public_exponent=65537, + key_size=key_size, + backend=default_backend() + ) + return priv_key, priv_key.public_key() + + +def create_signing_keypair_from_string(private_key_der: bytes) -> tuple[PrivateKey, PublicKey]: + """ + Create an RSA signing (private) key from previously serialized + private key bytes. + + :param private_key_der: blob as returned from `der_string_from_signing_keypair` + + :returns: 2-tuple of (private_key, public_key) + """ + _load = partial( + load_der_private_key, + private_key_der, + password=None, + backend=default_backend(), + ) + + def load_with_validation() -> PrivateKey: + k = _load() + assert isinstance(k, PrivateKey) + return k + + def load_without_validation() -> PrivateKey: + k = _load(unsafe_skip_rsa_key_validation=True) + assert isinstance(k, PrivateKey) + return k + + # Load it once without the potentially expensive OpenSSL validation + # checks. These have superlinear complexity. We *will* run them just + # below - but first we'll apply our own constant-time checks. + load: Callable[[], PrivateKey] = load_without_validation + try: + unsafe_priv_key = load() + except TypeError: + # cryptography<39 does not support this parameter, so just load the + # key with validation... + unsafe_priv_key = load_with_validation() + # But avoid *reloading* it since that will run the expensive + # validation *again*. + load = lambda: unsafe_priv_key + + if not isinstance(unsafe_priv_key, rsa.RSAPrivateKey): + raise ValueError( + "Private Key did not decode to an RSA key" + ) + if unsafe_priv_key.key_size != 2048: + raise ValueError( + "Private Key must be 2048 bits" + ) + + # Now re-load it with OpenSSL's validation applied. + safe_priv_key = load() + + return safe_priv_key, safe_priv_key.public_key() + + +def der_string_from_signing_key(private_key: PrivateKey) -> bytes: + """ + Serializes a given RSA private key to a DER string + + :param private_key: a private key object as returned from + `create_signing_keypair` or `create_signing_keypair_from_string` + + :returns: bytes representing `private_key` + """ + _validate_private_key(private_key) + return private_key.private_bytes( # type: ignore[attr-defined] + encoding=Encoding.DER, + format=PrivateFormat.PKCS8, + encryption_algorithm=NoEncryption(), + ) + + +def der_string_from_verifying_key(public_key: PublicKey) -> bytes: + """ + Serializes a given RSA public key to a DER string. + + :param public_key: a public key object as returned from + `create_signing_keypair` or `create_signing_keypair_from_string` + + :returns: bytes representing `public_key` + """ + _validate_public_key(public_key) + return public_key.public_bytes( + encoding=Encoding.DER, + format=PublicFormat.SubjectPublicKeyInfo, + ) + + +def create_verifying_key_from_string(public_key_der: bytes) -> PublicKey: + """ + Create an RSA verifying key from a previously serialized public key + + :param bytes public_key_der: a blob as returned by `der_string_from_verifying_key` + + :returns: a public key object suitable for use with other + functions in this module + """ + pub_key = load_der_public_key( + public_key_der, + backend=default_backend(), + ) + assert isinstance(pub_key, PublicKey) + return pub_key + + +def sign_data(private_key: PrivateKey, data: bytes) -> bytes: + """ + :param private_key: the private part of a keypair returned from + `create_signing_keypair_from_string` or `create_signing_keypair` + + :param data: the bytes to sign + + :returns: bytes which are a signature of the bytes given as `data`. + """ + _validate_private_key(private_key) + return private_key.sign( + data, + RSA_PADDING, + hashes.SHA256(), + ) + +def verify_signature(public_key: PublicKey, alleged_signature: bytes, data: bytes) -> None: + """ + :param public_key: a verifying key, returned from `create_verifying_key_from_string` or `create_verifying_key_from_private_key` + + :param bytes alleged_signature: the bytes of the alleged signature + + :param bytes data: the data which was allegedly signed + """ + _validate_public_key(public_key) + try: + public_key.verify( + alleged_signature, + data, + RSA_PADDING, + hashes.SHA256(), + ) + except InvalidSignature: + raise BadSignature() + + +def _validate_public_key(public_key: PublicKey) -> None: + """ + Internal helper. Checks that `public_key` is a valid cryptography + object + """ + if not isinstance(public_key, rsa.RSAPublicKey): + raise ValueError( + f"public_key must be an RSAPublicKey not {type(public_key)}" + ) + + +def _validate_private_key(private_key: PrivateKey) -> None: + """ + Internal helper. Checks that `public_key` is a valid cryptography + object + """ + if not isinstance(private_key, rsa.RSAPrivateKey): + raise ValueError( + f"private_key must be an RSAPrivateKey not {type(private_key)}" + ) From b221954946c2b2e93d8b36124c17c3a21ee4c0ed Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 6 Feb 2023 12:09:55 -0500 Subject: [PATCH 1432/2309] A working thread pool. --- src/allmydata/test/test_util.py | 34 +++++++++++++++++++++ src/allmydata/util/cputhreadpool.py | 47 +++++++++++++++++++++++++++++ 2 files changed, 81 insertions(+) create mode 100644 src/allmydata/util/cputhreadpool.py diff --git a/src/allmydata/test/test_util.py b/src/allmydata/test/test_util.py index 9a0af1e06..f4e7d21e0 100644 --- a/src/allmydata/test/test_util.py +++ b/src/allmydata/test/test_util.py @@ -15,8 +15,10 @@ import six import os, time, sys import yaml import json +from threading import current_thread from twisted.trial import unittest +from twisted.internet import reactor from foolscap.api import Violation, RemoteException from allmydata.util import idlib, mathutil @@ -26,6 +28,8 @@ from allmydata.util import pollmixin from allmydata.util import yamlutil from allmydata.util import rrefutil from allmydata.util.fileutil import EncryptedTemporaryFile +from allmydata.util.cputhreadpool import defer_to_thread +from allmydata.util.deferredutil import async_to_deferred from allmydata.test.common_util import ReallyEqualMixin from .no_network import fireNow, LocalWrapper @@ -588,3 +592,33 @@ class RrefUtilTests(unittest.TestCase): ) self.assertEqual(result.version, "Default") self.assertIdentical(result, rref) + + +class CPUThreadPool(unittest.TestCase): + """Tests for cputhreadpool.""" + + @async_to_deferred + async def test_runs_in_thread(self): + """The given function runs in a thread.""" + def f(*args, **kwargs): + time.sleep(0.1) + return current_thread(), args, kwargs + + this_thread = current_thread().ident + result = defer_to_thread(reactor, f, 1, 3, key=4, value=5) + + # Callbacks run in the correct thread: + callback_thread_ident = [] + def passthrough(result): + callback_thread_ident.append(current_thread().ident) + return result + + result.addCallback(passthrough) + + # The task ran in a different thread: + thread, args, kwargs = await result + self.assertEqual(callback_thread_ident[0], this_thread) + self.assertNotEqual(thread.ident, this_thread) + self.assertEqual(args, (1, 3)) + self.assertEqual(kwargs, {"key": 4, "value": 5}) + diff --git a/src/allmydata/util/cputhreadpool.py b/src/allmydata/util/cputhreadpool.py new file mode 100644 index 000000000..a7f9d8bd6 --- /dev/null +++ b/src/allmydata/util/cputhreadpool.py @@ -0,0 +1,47 @@ +""" +A global thread pool for CPU-intensive tasks. + +Motivation: + +* Certain tasks are blocking on CPU, and so should be run in a thread. +* The Twisted thread pool is used for operations that don't necessarily block + on CPU, like DNS lookups. CPU processing should not block DNS lookups! +* The number of threads should be fixed, and tied to the number of available + CPUs. + +As a first pass, this uses ``os.cpu_count()`` to determine the max number of +threads. This may create too many threads, as it doesn't cover things like +scheduler affinity or cgroups, but that's not the end of the world. +""" + +import os +from typing import TypeVar, Callable, cast +from functools import partial + +from twisted.python.threadpool import ThreadPool +from twisted.internet.defer import Deferred +from twisted.internet.threads import deferToThreadPool +from twisted.internet.interfaces import IReactorFromThreads + + +_CPU_THREAD_POOL = ThreadPool(minthreads=0, maxthreads=os.cpu_count(), name="TahoeCPU") +# Daemon threads allow shutdown to happen: +_CPU_THREAD_POOL.threadFactory = partial(_CPU_THREAD_POOL.threadFactory, daemon=True) +_CPU_THREAD_POOL.start() + + +# Eventually type annotations should use PEP 612, but that requires Python +# 3.10. +R = TypeVar("R") + + +def defer_to_thread( + reactor: IReactorFromThreads, f: Callable[..., R], *args, **kwargs +) -> Deferred[R]: + """Run the function in a thread, return the result as a ``Deferred``.""" + # deferToThreadPool has no type annotations... + result = deferToThreadPool(reactor, _CPU_THREAD_POOL, f, *args, **kwargs) + return cast(Deferred[R], result) + + +__all__ = ["defer_to_thread"] From 5909f451e336bba6a15606e073e9eb9b0d74581d Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 6 Feb 2023 13:54:47 -0500 Subject: [PATCH 1433/2309] Use the CPU thread pool for CBOR validation. --- src/allmydata/protocol_switch.py | 2 +- src/allmydata/storage/http_server.py | 8 +++--- src/allmydata/test/test_storage_http.py | 35 +++++++++++++++++++++---- 3 files changed, 35 insertions(+), 10 deletions(-) diff --git a/src/allmydata/protocol_switch.py b/src/allmydata/protocol_switch.py index b0af84c33..208efec6c 100644 --- a/src/allmydata/protocol_switch.py +++ b/src/allmydata/protocol_switch.py @@ -89,7 +89,7 @@ class _FoolscapOrHttps(Protocol, metaclass=_PretendToBeNegotiation): certificate=cls.tub.myCertificate.original, ) - http_storage_server = HTTPServer(storage_server, swissnum) + http_storage_server = HTTPServer(reactor, storage_server, swissnum) cls.https_factory = TLSMemoryBIOFactory( certificate_options, False, diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 6b94c227a..0d2280e2c 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -21,8 +21,6 @@ from twisted.internet.interfaces import ( IStreamServerEndpoint, IPullProducer, ) -from twisted.internet import reactor -from twisted.internet.threads import deferToThread from twisted.internet.address import IPv4Address, IPv6Address from twisted.internet.defer import Deferred from twisted.internet.ssl import CertificateOptions, Certificate, PrivateCertificate @@ -59,6 +57,7 @@ from .immutable import BucketWriter, ConflictingWriteError from ..util.hashutil import timing_safe_compare from ..util.base32 import rfc3548_alphabet from ..util.deferredutil import async_to_deferred +from ..util.cputhreadpool import defer_to_thread from allmydata.interfaces import BadWriteEnablerError @@ -489,8 +488,9 @@ class HTTPServer(object): return str(failure.value).encode("utf-8") def __init__( - self, storage_server, swissnum + self, reactor, storage_server, swissnum ): # type: (StorageServer, bytes) -> None + self._reactor = reactor self._storage_server = storage_server self._swissnum = swissnum # Maps storage index to StorageIndexUploads: @@ -570,7 +570,7 @@ class HTTPServer(object): # Pycddl will release the GIL when validating larger documents, so # let's take advantage of multiple CPUs: if size > 10_000: - await deferToThread(schema.validate_cbor, message) + await defer_to_thread(self._reactor, schema.validate_cbor, message) else: schema.validate_cbor(message) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 55754b29b..beb36e87a 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -18,10 +18,12 @@ sadly, an internal implementation detail of Twisted being leaked to tests... For definitely synchronous calls, you can just use ``result_of()``. """ +import time from base64 import b64encode from contextlib import contextmanager from os import urandom from typing import Union, Callable, Tuple, Iterable +from queue import Queue from cbor2 import dumps from pycddl import ValidationError as CDDLValidationError from hypothesis import assume, given, strategies as st @@ -31,13 +33,14 @@ from klein import Klein from hyperlink import DecodedURL from collections_extended import RangeMap from twisted.internet.task import Clock, Cooperator -from twisted.internet.interfaces import IReactorTime +from twisted.internet.interfaces import IReactorTime, IReactorFromThreads from twisted.internet.defer import CancelledError, Deferred from twisted.web import http from twisted.web.http_headers import Headers from werkzeug import routing from werkzeug.exceptions import NotFound as WNotFound from testtools.matchers import Equals +from zope.interface import implementer from .common import SyncTestCase from ..storage.http_common import get_content_type, CBOR_MIME_TYPE @@ -449,6 +452,23 @@ class CustomHTTPServerTests(SyncTestCase): self.assertEqual(len(self._http_server.clock.getDelayedCalls()), 0) +@implementer(IReactorFromThreads) +class Reactor(Clock): + """Fake reactor.""" + def __init__(self): + Clock.__init__(self) + self._queue = Queue() + + def callFromThread(self, f, *args, **kwargs): + self._queue.put((f, args, kwargs)) + + def advance(self, *args, **kwargs): + Clock.advance(self, *args, **kwargs) + while not self._queue.empty(): + f, args, kwargs = self._queue.get() + f(*args, **kwargs) + + class HttpTestFixture(Fixture): """ Setup HTTP tests' infrastructure, the storage server and corresponding @@ -460,7 +480,7 @@ class HttpTestFixture(Fixture): lambda pool: self.addCleanup(pool.closeCachedConnections) ) self.addCleanup(StorageClient.stop_test_mode) - self.clock = Clock() + self.clock = Reactor() self.tempdir = self.useFixture(TempDir()) # The global Cooperator used by Twisted (a) used by pull producers in # twisted.web, (b) is driven by a real reactor. We want to push time @@ -475,7 +495,7 @@ class HttpTestFixture(Fixture): self.storage_server = StorageServer( self.tempdir.path, b"\x00" * 20, clock=self.clock ) - self.http_server = HTTPServer(self.storage_server, SWISSNUM_FOR_TEST) + self.http_server = HTTPServer(self.clock, self.storage_server, SWISSNUM_FOR_TEST) self.treq = StubTreq(self.http_server.get_resource()) self.client = StorageClient( DecodedURL.from_text("http://127.0.0.1"), @@ -501,13 +521,18 @@ class HttpTestFixture(Fixture): # OK, no result yet, probably async HTTP endpoint handler, so advance # time, flush treq, and try again: - for i in range(100): + for i in range(10_000): self.clock.advance(0.001) - self.treq.flush() + self.treq.flush() + if result: + break + time.sleep(0.001) + if result: return result[0] if error: error[0].raiseException() + raise RuntimeError( "We expected given Deferred to have result already, but it wasn't. " + "This is probably a test design issue." From 4576d10915b909d0612da427a78291c2e99434aa Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 6 Feb 2023 13:56:59 -0500 Subject: [PATCH 1434/2309] Add an explanation. --- src/allmydata/test/test_storage_http.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index beb36e87a..eb5bcd4db 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -454,7 +454,11 @@ class CustomHTTPServerTests(SyncTestCase): @implementer(IReactorFromThreads) class Reactor(Clock): - """Fake reactor.""" + """ + Fake reactor that supports time APIs and callFromThread. + + Advancing the clock also runs any callbacks scheduled via callFromThread. + """ def __init__(self): Clock.__init__(self) self._queue = Queue() @@ -526,6 +530,13 @@ class HttpTestFixture(Fixture): self.treq.flush() if result: break + # By putting the sleep at the end, tests that are completely + # synchronous and don't use threads will have already broken out of + # the loop, and so will finish without any sleeps. This allows them + # to run as quickly as possible. + # + # However, some tests do talk to APIs that use a thread pool on the + # backend, so we need to allow actual time to pass for those. time.sleep(0.001) if result: From c4114e032e00819ffa93aad5ae92debd1604141b Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 6 Feb 2023 15:33:08 -0500 Subject: [PATCH 1435/2309] Fix type signature. --- src/allmydata/storage/http_server.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 0d2280e2c..c6c3ab615 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -24,6 +24,7 @@ from twisted.internet.interfaces import ( from twisted.internet.address import IPv4Address, IPv6Address from twisted.internet.defer import Deferred from twisted.internet.ssl import CertificateOptions, Certificate, PrivateCertificate +from twisted.internet.interfaces import IReactorFromThreads from twisted.web.server import Site, Request from twisted.protocols.tls import TLSMemoryBIOFactory from twisted.python.filepath import FilePath @@ -488,8 +489,11 @@ class HTTPServer(object): return str(failure.value).encode("utf-8") def __init__( - self, reactor, storage_server, swissnum - ): # type: (StorageServer, bytes) -> None + self, + reactor: IReactorFromThreads, + storage_server: StorageServer, + swissnum: bytes, + ): self._reactor = reactor self._storage_server = storage_server self._swissnum = swissnum From eb26c97ef70edeafe3f6777759dcca7d6a08c98c Mon Sep 17 00:00:00 2001 From: dlee Date: Mon, 6 Feb 2023 15:29:53 -0600 Subject: [PATCH 1436/2309] implicit_optional flag added and errors related to flag fixed --- mypy.ini | 1 + src/allmydata/web/common.py | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/mypy.ini b/mypy.ini index c6d8affe3..e6e7d16ff 100644 --- a/mypy.ini +++ b/mypy.ini @@ -5,5 +5,6 @@ show_column_numbers = True pretty = True show_error_codes = True warn_unused_configs =True +no_implicit_optional = True warn_redundant_casts = True strict_equality = True \ No newline at end of file diff --git a/src/allmydata/web/common.py b/src/allmydata/web/common.py index c49354217..3d85b1c4d 100644 --- a/src/allmydata/web/common.py +++ b/src/allmydata/web/common.py @@ -707,12 +707,12 @@ def url_for_string(req, url_string): T = TypeVar("T") @overload -def get_arg(req: IRequest, argname: str | bytes, default: T = None, *, multiple: Literal[False] = False) -> T | bytes: ... +def get_arg(req: IRequest, argname: str | bytes, default: Optional[T] = None, *, multiple: Literal[False] = False) -> T | bytes: ... @overload -def get_arg(req: IRequest, argname: str | bytes, default: T = None, *, multiple: Literal[True]) -> T | tuple[bytes, ...]: ... +def get_arg(req: IRequest, argname: str | bytes, default: Optional[T] = None, *, multiple: Literal[True]) -> T | tuple[bytes, ...]: ... -def get_arg(req: IRequest, argname: str | bytes, default: T = None, *, multiple: bool = False) -> None | T | bytes | tuple[bytes, ...]: +def get_arg(req: IRequest, argname: str | bytes, default: Optional[T] = None, *, multiple: bool = False) -> None | T | bytes | tuple[bytes, ...]: """Extract an argument from either the query args (req.args) or the form body fields (req.fields). If multiple=False, this returns a single value (or the default, which defaults to None), and the query args take From b0f4e463eb11aa4d3564027a9c6d232236fa272e Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 6 Feb 2023 17:48:32 -0500 Subject: [PATCH 1437/2309] Work with newer i2pd. --- integration/test_i2p.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/integration/test_i2p.py b/integration/test_i2p.py index 15f9d73cf..2deb01fab 100644 --- a/integration/test_i2p.py +++ b/integration/test_i2p.py @@ -55,9 +55,12 @@ def i2p_network(reactor, temp_dir, request): proto, which("docker"), ( - "docker", "run", "-p", "7656:7656", "purplei2p/i2pd:release-2.43.0", + "docker", "run", "-p", "7656:7656", "purplei2p/i2pd:release-2.45.1", # Bad URL for reseeds, so it can't talk to other routers. "--reseed.urls", "http://localhost:1/", + # Make sure we see the "ephemeral keys message" + "--log=stdout", + "--loglevel=info" ), ) From f4255cdaa340c217546bd9c24538292d864f65d8 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 7 Feb 2023 09:03:45 -0500 Subject: [PATCH 1438/2309] More accurate names. --- integration/util.py | 4 ++-- misc/simulators/storage-overhead.py | 4 ++-- src/allmydata/client.py | 4 ++-- src/allmydata/immutable/downloader/node.py | 4 ++-- src/allmydata/immutable/upload.py | 4 ++-- src/allmydata/interfaces.py | 2 +- src/allmydata/mutable/publish.py | 6 +++--- src/allmydata/test/mutable/test_update.py | 8 ++++---- src/allmydata/test/test_system.py | 4 ++-- 9 files changed, 20 insertions(+), 20 deletions(-) diff --git a/integration/util.py b/integration/util.py index a6543c34d..02cda655d 100644 --- a/integration/util.py +++ b/integration/util.py @@ -46,7 +46,7 @@ from allmydata.util.configutil import ( write_config, ) from allmydata import client -from allmydata.interfaces import DEFAULT_MAX_SEGMENT_SIZE +from allmydata.interfaces import DEFAULT_IMMUTABLE_MAX_SEGMENT_SIZE import pytest_twisted @@ -774,7 +774,7 @@ async def reconfigure(reactor, request, node: TahoeProcess, config.write_private_config("convergence", base32.b2a(convergence)) if max_segment_size is not None: - cur_segment_size = int(config.get_config("client", "shares._max_immutable_segment_size_for_testing", DEFAULT_MAX_SEGMENT_SIZE)) + cur_segment_size = int(config.get_config("client", "shares._max_immutable_segment_size_for_testing", DEFAULT_IMMUTABLE_MAX_SEGMENT_SIZE)) if cur_segment_size != max_segment_size: changed = True config.set_config( diff --git a/misc/simulators/storage-overhead.py b/misc/simulators/storage-overhead.py index 5a741834e..9959f5575 100644 --- a/misc/simulators/storage-overhead.py +++ b/misc/simulators/storage-overhead.py @@ -5,7 +5,7 @@ from __future__ import print_function import sys, math from allmydata import uri, storage from allmydata.immutable import upload -from allmydata.interfaces import DEFAULT_MAX_SEGMENT_SIZE +from allmydata.interfaces import DEFAULT_IMMUTABLE_MAX_SEGMENT_SIZE from allmydata.util import mathutil def roundup(size, blocksize=4096): @@ -26,7 +26,7 @@ class BigFakeString(object): def tell(self): return self.fp -def calc(filesize, params=(3,7,10), segsize=DEFAULT_MAX_SEGMENT_SIZE): +def calc(filesize, params=(3,7,10), segsize=DEFAULT_IMMUTABLE_MAX_SEGMENT_SIZE): num_shares = params[2] if filesize <= upload.Uploader.URI_LIT_SIZE_THRESHOLD: urisize = len(uri.LiteralFileURI("A"*filesize).to_string()) diff --git a/src/allmydata/client.py b/src/allmydata/client.py index c06961c8c..dfd1c3c81 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -50,7 +50,7 @@ from allmydata.interfaces import ( IStatsProducer, SDMF_VERSION, MDMF_VERSION, - DEFAULT_MAX_SEGMENT_SIZE, + DEFAULT_IMMUTABLE_MAX_SEGMENT_SIZE, IFoolscapStoragePlugin, IAnnounceableStorageServer, ) @@ -607,7 +607,7 @@ class _Client(node.Node, pollmixin.PollMixin): DEFAULT_ENCODING_PARAMETERS = {"k": 3, "happy": 7, "n": 10, - "max_segment_size": DEFAULT_MAX_SEGMENT_SIZE, + "max_segment_size": DEFAULT_IMMUTABLE_MAX_SEGMENT_SIZE, } def __init__(self, config, main_tub, i2p_provider, tor_provider, introducer_clients, diff --git a/src/allmydata/immutable/downloader/node.py b/src/allmydata/immutable/downloader/node.py index ec4f751df..a1ef4b485 100644 --- a/src/allmydata/immutable/downloader/node.py +++ b/src/allmydata/immutable/downloader/node.py @@ -19,7 +19,7 @@ from foolscap.api import eventually from allmydata import uri from allmydata.codec import CRSDecoder from allmydata.util import base32, log, hashutil, mathutil, observer -from allmydata.interfaces import DEFAULT_MAX_SEGMENT_SIZE +from allmydata.interfaces import DEFAULT_IMMUTABLE_MAX_SEGMENT_SIZE from allmydata.hashtree import IncompleteHashTree, BadHashError, \ NotEnoughHashesError @@ -49,7 +49,7 @@ class DownloadNode(object): """Internal class which manages downloads and holds state. External callers use CiphertextFileNode instead.""" - default_max_segment_size = DEFAULT_MAX_SEGMENT_SIZE + default_max_segment_size = DEFAULT_IMMUTABLE_MAX_SEGMENT_SIZE # Share._node points to me def __init__(self, verifycap, storage_broker, secret_holder, diff --git a/src/allmydata/immutable/upload.py b/src/allmydata/immutable/upload.py index 6b9b48f6a..06482688f 100644 --- a/src/allmydata/immutable/upload.py +++ b/src/allmydata/immutable/upload.py @@ -48,7 +48,7 @@ from allmydata.util.rrefutil import add_version_to_remote_reference from allmydata.interfaces import IUploadable, IUploader, IUploadResults, \ IEncryptedUploadable, RIEncryptedUploadable, IUploadStatus, \ NoServersError, InsufficientVersionError, UploadUnhappinessError, \ - DEFAULT_MAX_SEGMENT_SIZE, IPeerSelector + DEFAULT_IMMUTABLE_MAX_SEGMENT_SIZE, IPeerSelector from allmydata.immutable import layout from io import BytesIO @@ -1692,7 +1692,7 @@ class AssistedUploader(object): class BaseUploadable(object): # this is overridden by max_segment_size - default_max_segment_size = DEFAULT_MAX_SEGMENT_SIZE + default_max_segment_size = DEFAULT_IMMUTABLE_MAX_SEGMENT_SIZE default_params_set = False max_segment_size = None diff --git a/src/allmydata/interfaces.py b/src/allmydata/interfaces.py index 8f6a0c37a..493a4a31f 100644 --- a/src/allmydata/interfaces.py +++ b/src/allmydata/interfaces.py @@ -41,7 +41,7 @@ URI = StringConstraint(300) # kind of arbitrary MAX_BUCKETS = 256 # per peer -- zfec offers at most 256 shares per file -DEFAULT_MAX_SEGMENT_SIZE = 1024*1024 +DEFAULT_IMMUTABLE_MAX_SEGMENT_SIZE = 1024*1024 ShareData = StringConstraint(None) URIExtensionData = StringConstraint(1000) diff --git a/src/allmydata/mutable/publish.py b/src/allmydata/mutable/publish.py index a7bca6cba..5f689dbc0 100644 --- a/src/allmydata/mutable/publish.py +++ b/src/allmydata/mutable/publish.py @@ -36,7 +36,7 @@ from allmydata.mutable.layout import get_version_from_checkstring,\ SDMFSlotWriteProxy KiB = 1024 -DEFAULT_MAX_SEGMENT_SIZE = 128 * KiB +DEFAULT_MUTABLE_MAX_SEGMENT_SIZE = 128 * KiB PUSHING_BLOCKS_STATE = 0 PUSHING_EVERYTHING_ELSE_STATE = 1 DONE_STATE = 2 @@ -367,7 +367,7 @@ class Publish(object): self.data = newdata self.datalength = newdata.get_size() - #if self.datalength >= DEFAULT_MAX_SEGMENT_SIZE: + #if self.datalength >= DEFAULT_MUTABLE_MAX_SEGMENT_SIZE: # self._version = MDMF_VERSION #else: # self._version = SDMF_VERSION @@ -551,7 +551,7 @@ class Publish(object): def setup_encoding_parameters(self, offset=0): if self._version == MDMF_VERSION: - segment_size = DEFAULT_MAX_SEGMENT_SIZE # 128 KiB by default + segment_size = DEFAULT_MUTABLE_MAX_SEGMENT_SIZE # 128 KiB by default else: segment_size = self.datalength # SDMF is only one segment # this must be a multiple of self.required_shares diff --git a/src/allmydata/test/mutable/test_update.py b/src/allmydata/test/mutable/test_update.py index c3ba1e9f7..1c91590bd 100644 --- a/src/allmydata/test/mutable/test_update.py +++ b/src/allmydata/test/mutable/test_update.py @@ -20,7 +20,7 @@ from testtools.matchers import ( from twisted.internet import defer from allmydata.interfaces import MDMF_VERSION from allmydata.mutable.filenode import MutableFileNode -from allmydata.mutable.publish import MutableData, DEFAULT_MAX_SEGMENT_SIZE +from allmydata.mutable.publish import MutableData, DEFAULT_MUTABLE_MAX_SEGMENT_SIZE from ..no_network import GridTestMixin from .. import common_util as testutil @@ -180,7 +180,7 @@ class Update(GridTestMixin, AsyncTestCase, testutil.ShouldFailMixin): # long -- this is 7 segments in the default segment size. So we # need to add 2 segments worth of data to push it over a # power-of-two boundary. - segment = b"a" * DEFAULT_MAX_SEGMENT_SIZE + segment = b"a" * DEFAULT_MUTABLE_MAX_SEGMENT_SIZE new_data = self.data + (segment * 2) d0 = self.do_upload_mdmf() def _run(ign): @@ -232,9 +232,9 @@ class Update(GridTestMixin, AsyncTestCase, testutil.ShouldFailMixin): return d0 def test_multiple_segment_replace(self): - replace_offset = 2 * DEFAULT_MAX_SEGMENT_SIZE + replace_offset = 2 * DEFAULT_MUTABLE_MAX_SEGMENT_SIZE new_data = self.data[:replace_offset] - new_segment = b"a" * DEFAULT_MAX_SEGMENT_SIZE + new_segment = b"a" * DEFAULT_MUTABLE_MAX_SEGMENT_SIZE new_data += 2 * new_segment new_data += b"replaced" rest_offset = len(new_data) diff --git a/src/allmydata/test/test_system.py b/src/allmydata/test/test_system.py index 9849c6b30..1b8a8dbfe 100644 --- a/src/allmydata/test/test_system.py +++ b/src/allmydata/test/test_system.py @@ -37,7 +37,7 @@ from allmydata.util.consumer import MemoryConsumer, download_to_data from allmydata.util.deferredutil import async_to_deferred from allmydata.interfaces import IDirectoryNode, IFileNode, \ NoSuchChildError, NoSharesError, SDMF_VERSION, MDMF_VERSION, \ - DEFAULT_MAX_SEGMENT_SIZE + DEFAULT_IMMUTABLE_MAX_SEGMENT_SIZE from allmydata.monitor import Monitor from allmydata.mutable.common import NotWriteableError from allmydata.mutable import layout as mutable_layout @@ -1846,7 +1846,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase): uploadable.max_segment_size = upload_segment_size results = await uploader.upload(uploadable) - assert DownloadNode.default_max_segment_size == DEFAULT_MAX_SEGMENT_SIZE + assert DownloadNode.default_max_segment_size == DEFAULT_IMMUTABLE_MAX_SEGMENT_SIZE self.patch(DownloadNode, "default_max_segment_size", download_segment_size) uri = results.get_uri() node = self.clients[1].create_node_from_uri(uri) From 51d44ba676e7097813ac9a6b2a66390d26fe1f3b Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 7 Feb 2023 09:04:20 -0500 Subject: [PATCH 1439/2309] Document. --- src/allmydata/interfaces.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/allmydata/interfaces.py b/src/allmydata/interfaces.py index 493a4a31f..6f0b0a00e 100644 --- a/src/allmydata/interfaces.py +++ b/src/allmydata/interfaces.py @@ -41,6 +41,7 @@ URI = StringConstraint(300) # kind of arbitrary MAX_BUCKETS = 256 # per peer -- zfec offers at most 256 shares per file +# The default size for segments of new CHK ("immutable") uploads. DEFAULT_IMMUTABLE_MAX_SEGMENT_SIZE = 1024*1024 ShareData = StringConstraint(None) From ea052b3c802d98ae3bc9cc3c8b4e34e521b06ac7 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 7 Feb 2023 09:08:06 -0500 Subject: [PATCH 1440/2309] Pass in missing argument. --- integration/test_vectors.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/integration/test_vectors.py b/integration/test_vectors.py index 1f8747004..6e7b5746a 100644 --- a/integration/test_vectors.py +++ b/integration/test_vectors.py @@ -111,7 +111,8 @@ async def generate( request, alice, (happy, case.params.required, case.params.total), - case.convergence + case.convergence, + case.segment_size ) # Give the format a chance to make an RSA key if it needs it. From 3bc3cf39d0024389278167550ed859bfc696c923 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 7 Feb 2023 09:44:51 -0500 Subject: [PATCH 1441/2309] Test using an integration test. --- integration/test_get_put.py | 50 +++++++++++++++++++++++++++++-- src/allmydata/test/test_system.py | 44 +-------------------------- 2 files changed, 49 insertions(+), 45 deletions(-) diff --git a/integration/test_get_put.py b/integration/test_get_put.py index bbdc363ea..fd29d51fe 100644 --- a/integration/test_get_put.py +++ b/integration/test_get_put.py @@ -3,11 +3,13 @@ Integration tests for getting and putting files, including reading from stdin and stdout. """ -from subprocess import Popen, PIPE +from subprocess import Popen, PIPE, check_output import pytest +from pytest_twisted import ensureDeferred +from twisted.internet import reactor -from .util import run_in_thread, cli +from .util import run_in_thread, cli, reconfigure DATA = b"abc123 this is not utf-8 decodable \xff\x00\x33 \x11" try: @@ -62,3 +64,47 @@ def test_get_to_stdout(alice, get_put_alias, tmpdir): ) assert p.stdout.read() == DATA assert p.wait() == 0 + + +@ensureDeferred +async def test_upload_download_immutable_different_default_max_segment_size(alice, get_put_alias, tmpdir, request): + """ + Tahoe-LAFS used to have a default max segment size of 128KB, and is now + 1MB. Test that an upload created when 128KB was the default can be + downloaded with 1MB as the default (i.e. old uploader, new downloader), and + vice versa, (new uploader, old downloader). + """ + tempfile = tmpdir.join("file") + large_data = DATA * 100_000 + assert len(large_data) > 2 * 1024 * 1024 + with tempfile.open("wb") as f: + f.write(large_data) + + async def set_segment_size(segment_size): + await reconfigure( + reactor, + request, + alice, + (1, 1, 1), + None, + max_segment_size=segment_size + ) + + # 1. Upload file 1 with default segment size set to 1MB + await set_segment_size(1024 * 1024) + cli(alice, "put", str(tempfile), "getput:seg1024kb") + + # 2. Download file 1 with default segment size set to 128KB + await set_segment_size(128 * 1024) + assert large_data == check_output( + ["tahoe", "--node-directory", alice.node_dir, "get", "getput:seg1024kb", "-"] + ) + + # 3. Upload file 2 with default segment size set to 128KB + cli(alice, "put", str(tempfile), "getput:seg128kb") + + # 4. Download file 2 with default segment size set to 1MB + await set_segment_size(1024 * 1024) + assert large_data == check_output( + ["tahoe", "--node-directory", alice.node_dir, "get", "getput:seg128kb", "-"] + ) diff --git a/src/allmydata/test/test_system.py b/src/allmydata/test/test_system.py index 1b8a8dbfe..10a64c1fe 100644 --- a/src/allmydata/test/test_system.py +++ b/src/allmydata/test/test_system.py @@ -28,16 +28,13 @@ from allmydata.storage.server import si_a2b from allmydata.immutable import offloaded, upload from allmydata.immutable.literal import LiteralFileNode from allmydata.immutable.filenode import ImmutableFileNode -from allmydata.immutable.downloader.node import DownloadNode from allmydata.util import idlib, mathutil from allmydata.util import log, base32 from allmydata.util.encodingutil import quote_output, unicode_to_argv from allmydata.util.fileutil import abspath_expanduser_unicode from allmydata.util.consumer import MemoryConsumer, download_to_data -from allmydata.util.deferredutil import async_to_deferred from allmydata.interfaces import IDirectoryNode, IFileNode, \ - NoSuchChildError, NoSharesError, SDMF_VERSION, MDMF_VERSION, \ - DEFAULT_IMMUTABLE_MAX_SEGMENT_SIZE + NoSuchChildError, NoSharesError, SDMF_VERSION, MDMF_VERSION from allmydata.monitor import Monitor from allmydata.mutable.common import NotWriteableError from allmydata.mutable import layout as mutable_layout @@ -1814,45 +1811,6 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase): d.addCallback(_got_lit_filenode) return d - @async_to_deferred - async def test_upload_download_immutable_different_default_max_segment_size(self): - """ - Tahoe-LAFS used to have a default max segment size of 128KB, and is now - 1MB. Test that an upload created when 128KB was the default can be - downloaded with 1MB as the default (i.e. old uploader, new downloader), - and vice versa, (new uploader, old downloader). - """ - await self.set_up_nodes(2) - - # Just 1 share: - for c in self.clients: - c.encoding_params["k"] = 1 - c.encoding_params["happy"] = 1 - c.encoding_params["n"] = 1 - - await self._upload_download_different_max_segment(128 * 1024, 1024 * 1024) - await self._upload_download_different_max_segment(1024 * 1024, 128 * 1024) - - - async def _upload_download_different_max_segment( - self, upload_segment_size, download_segment_size - ): - """Upload with one max segment size, download with another.""" - data = b"123456789" * 1_000_000 - - uploader = self.clients[0].getServiceNamed("uploader") - uploadable = upload.Data(data, convergence=None) - assert uploadable.max_segment_size == None - uploadable.max_segment_size = upload_segment_size - results = await uploader.upload(uploadable) - - assert DownloadNode.default_max_segment_size == DEFAULT_IMMUTABLE_MAX_SEGMENT_SIZE - self.patch(DownloadNode, "default_max_segment_size", download_segment_size) - uri = results.get_uri() - node = self.clients[1].create_node_from_uri(uri) - mc = await node.read(MemoryConsumer(), 0, None) - self.assertEqual(b"".join(mc.chunks), data) - class Connections(SystemTestMixin, unittest.TestCase): FORCE_FOOLSCAP_FOR_STORAGE = True From 4192908801122efb632998b2898f8c51ebf54612 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 13 Feb 2023 08:59:04 -0500 Subject: [PATCH 1442/2309] Add a note with some frequency scaling info --- benchmarks/upload_download.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/benchmarks/upload_download.py b/benchmarks/upload_download.py index 42f288e68..0ca034af9 100644 --- a/benchmarks/upload_download.py +++ b/benchmarks/upload_download.py @@ -13,6 +13,15 @@ To reset: $ tc qdisc del dev lo root netem +Frequency scaling can spoil the results. +To see the range of frequency scaling on a Linux system: + +$ cat /sys/devices/system/cpu/cpu*/cpufreq/scaling_available_frequencies + +And to pin the CPU frequency to the lower bound found in these files: + +$ sudo cpupower frequency-set -f + TODO Parameterization (pytest?) - Foolscap vs not foolscap From 0e4d39229ace8ec82f2a4598dbcee619fa4fa408 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 13 Feb 2023 13:27:36 -0500 Subject: [PATCH 1443/2309] Remove the Docker image definitions And since the "Docker Compose" definition depends on them, remove it as well. These have been unmaintained for a long time and their goals are unknown to the current development team. --- Dockerfile | 10 ---------- Dockerfile.dev | 25 ----------------------- docker-compose.yml | 49 ---------------------------------------------- 3 files changed, 84 deletions(-) delete mode 100644 Dockerfile delete mode 100644 Dockerfile.dev delete mode 100644 docker-compose.yml diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 842093fdb..000000000 --- a/Dockerfile +++ /dev/null @@ -1,10 +0,0 @@ -FROM python:2.7 - -ADD . /tahoe-lafs -RUN \ - cd /tahoe-lafs && \ - git pull --depth=100 && \ - pip install . && \ - rm -rf ~/.cache/ - -WORKDIR /root diff --git a/Dockerfile.dev b/Dockerfile.dev deleted file mode 100644 index b0fd24b5e..000000000 --- a/Dockerfile.dev +++ /dev/null @@ -1,25 +0,0 @@ -FROM debian:9 -LABEL maintainer "gordon@leastauthority.com" -RUN apt-get update -RUN DEBIAN_FRONTEND=noninteractive apt-get -yq upgrade -RUN DEBIAN_FRONTEND=noninteractive apt-get -yq install build-essential python-dev libffi-dev libssl-dev python-virtualenv git -RUN \ - git clone https://github.com/tahoe-lafs/tahoe-lafs.git /root/tahoe-lafs; \ - cd /root/tahoe-lafs; \ - virtualenv --python=python2.7 venv; \ - ./venv/bin/pip install --upgrade setuptools; \ - ./venv/bin/pip install --editable .; \ - ./venv/bin/tahoe --version; -RUN \ - cd /root; \ - mkdir /root/.tahoe-client; \ - mkdir /root/.tahoe-introducer; \ - mkdir /root/.tahoe-server; -RUN /root/tahoe-lafs/venv/bin/tahoe create-introducer --location=tcp:introducer:3458 --port=tcp:3458 /root/.tahoe-introducer -RUN /root/tahoe-lafs/venv/bin/tahoe start /root/.tahoe-introducer -RUN /root/tahoe-lafs/venv/bin/tahoe create-node --location=tcp:server:3457 --port=tcp:3457 --introducer=$(cat /root/.tahoe-introducer/private/introducer.furl) /root/.tahoe-server -RUN /root/tahoe-lafs/venv/bin/tahoe create-client --webport=3456 --introducer=$(cat /root/.tahoe-introducer/private/introducer.furl) --basedir=/root/.tahoe-client --shares-needed=1 --shares-happy=1 --shares-total=1 -VOLUME ["/root/.tahoe-client", "/root/.tahoe-server", "/root/.tahoe-introducer"] -EXPOSE 3456 3457 3458 -ENTRYPOINT ["/root/tahoe-lafs/venv/bin/tahoe"] -CMD [] diff --git a/docker-compose.yml b/docker-compose.yml deleted file mode 100644 index 1d23be71a..000000000 --- a/docker-compose.yml +++ /dev/null @@ -1,49 +0,0 @@ -version: '2' -services: - client: - build: - context: . - dockerfile: ./Dockerfile.dev - volumes: - - ./misc:/root/tahoe-lafs/misc - - ./integration:/root/tahoe-lafs/integration - - ./src:/root/tahoe-lafs/static - - ./setup.cfg:/root/tahoe-lafs/setup.cfg - - ./setup.py:/root/tahoe-lafs/setup.py - ports: - - "127.0.0.1:3456:3456" - depends_on: - - "introducer" - - "server" - entrypoint: /root/tahoe-lafs/venv/bin/tahoe - command: ["run", "/root/.tahoe-client"] - server: - build: - context: . - dockerfile: ./Dockerfile.dev - volumes: - - ./misc:/root/tahoe-lafs/misc - - ./integration:/root/tahoe-lafs/integration - - ./src:/root/tahoe-lafs/static - - ./setup.cfg:/root/tahoe-lafs/setup.cfg - - ./setup.py:/root/tahoe-lafs/setup.py - ports: - - "127.0.0.1:3457:3457" - depends_on: - - "introducer" - entrypoint: /root/tahoe-lafs/venv/bin/tahoe - command: ["run", "/root/.tahoe-server"] - introducer: - build: - context: . - dockerfile: ./Dockerfile.dev - volumes: - - ./misc:/root/tahoe-lafs/misc - - ./integration:/root/tahoe-lafs/integration - - ./src:/root/tahoe-lafs/static - - ./setup.cfg:/root/tahoe-lafs/setup.cfg - - ./setup.py:/root/tahoe-lafs/setup.py - ports: - - "127.0.0.1:3458:3458" - entrypoint: /root/tahoe-lafs/venv/bin/tahoe - command: ["run", "/root/.tahoe-introducer"] From b8cbf143cd97db92b50a022f5a23cda056a2b956 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 13 Feb 2023 13:28:26 -0500 Subject: [PATCH 1444/2309] news fragment --- newsfragments/3974.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3974.minor diff --git a/newsfragments/3974.minor b/newsfragments/3974.minor new file mode 100644 index 000000000..e69de29bb From 19e58f19ca9a31876a1eaf20014c98ec9be31035 Mon Sep 17 00:00:00 2001 From: dlee Date: Tue, 14 Feb 2023 11:21:57 -0600 Subject: [PATCH 1445/2309] Fixes truthy conditional --- newsfragments/3975.minor | 1 + src/allmydata/web/status.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 newsfragments/3975.minor diff --git a/newsfragments/3975.minor b/newsfragments/3975.minor new file mode 100644 index 000000000..08fba6dd6 --- /dev/null +++ b/newsfragments/3975.minor @@ -0,0 +1 @@ +Fixes truthy conditional in status.py \ No newline at end of file diff --git a/src/allmydata/web/status.py b/src/allmydata/web/status.py index 65647f491..a413348c4 100644 --- a/src/allmydata/web/status.py +++ b/src/allmydata/web/status.py @@ -550,7 +550,7 @@ class DownloadStatusElement(Element): length = r_ev["length"] bytes_returned = r_ev["bytes_returned"] decrypt_time = "" - if bytes: + if bytes_returned is not None: decrypt_time = self._rate_and_time(bytes_returned, r_ev["decrypt_time"]) speed, rtt = "","" if r_ev["finish_time"] is not None: From b7cadfc53a65dd294fade8614517d693f8b3b677 Mon Sep 17 00:00:00 2001 From: dlee Date: Tue, 14 Feb 2023 11:38:35 -0600 Subject: [PATCH 1446/2309] Fixes bad practice of naming variable a built-in type --- newsfragments/3976.minor | 1 + src/allmydata/web/status.py | 20 ++++++++++---------- 2 files changed, 11 insertions(+), 10 deletions(-) create mode 100644 newsfragments/3976.minor diff --git a/newsfragments/3976.minor b/newsfragments/3976.minor new file mode 100644 index 000000000..4d6245e73 --- /dev/null +++ b/newsfragments/3976.minor @@ -0,0 +1 @@ +Fixes variable name same as built-in type. \ No newline at end of file diff --git a/src/allmydata/web/status.py b/src/allmydata/web/status.py index a413348c4..1bcf0979f 100644 --- a/src/allmydata/web/status.py +++ b/src/allmydata/web/status.py @@ -1616,30 +1616,30 @@ class StatisticsElement(Element): @renderer def uploads(self, req, tag): files = self._stats["counters"].get("uploader.files_uploaded", 0) - bytes = self._stats["counters"].get("uploader.bytes_uploaded", 0) + bytes_uploaded = self._stats["counters"].get("uploader.bytes_uploaded", 0) return tag(("%s files / %s bytes (%s)" % - (files, bytes, abbreviate_size(bytes)))) + (files, bytes_uploaded, abbreviate_size(bytes_uploaded)))) @renderer def downloads(self, req, tag): files = self._stats["counters"].get("downloader.files_downloaded", 0) - bytes = self._stats["counters"].get("downloader.bytes_downloaded", 0) + bytes_uploaded = self._stats["counters"].get("downloader.bytes_downloaded", 0) return tag("%s files / %s bytes (%s)" % - (files, bytes, abbreviate_size(bytes))) + (files, bytes_uploaded, abbreviate_size(bytes_uploaded))) @renderer def publishes(self, req, tag): files = self._stats["counters"].get("mutable.files_published", 0) - bytes = self._stats["counters"].get("mutable.bytes_published", 0) - return tag("%s files / %s bytes (%s)" % (files, bytes, - abbreviate_size(bytes))) + bytes_uploaded = self._stats["counters"].get("mutable.bytes_published", 0) + return tag("%s files / %s bytes (%s)" % (files, bytes_uploaded, + abbreviate_size(bytes_uploaded))) @renderer def retrieves(self, req, tag): files = self._stats["counters"].get("mutable.files_retrieved", 0) - bytes = self._stats["counters"].get("mutable.bytes_retrieved", 0) - return tag("%s files / %s bytes (%s)" % (files, bytes, - abbreviate_size(bytes))) + bytes_uploaded = self._stats["counters"].get("mutable.bytes_retrieved", 0) + return tag("%s files / %s bytes (%s)" % (files, bytes_uploaded, + abbreviate_size(bytes_uploaded))) @renderer def raw(self, req, tag): From a7ddcbf868c53feed28442f6b977e6a9f3bfb812 Mon Sep 17 00:00:00 2001 From: dlee Date: Tue, 14 Feb 2023 12:50:37 -0600 Subject: [PATCH 1447/2309] Changes requested --- src/allmydata/web/status.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/web/status.py b/src/allmydata/web/status.py index 1bcf0979f..4a902a98b 100644 --- a/src/allmydata/web/status.py +++ b/src/allmydata/web/status.py @@ -550,7 +550,7 @@ class DownloadStatusElement(Element): length = r_ev["length"] bytes_returned = r_ev["bytes_returned"] decrypt_time = "" - if bytes_returned is not None: + if bytes_returned: decrypt_time = self._rate_and_time(bytes_returned, r_ev["decrypt_time"]) speed, rtt = "","" if r_ev["finish_time"] is not None: From badba97ff20a961153ddd86490c9646e042df172 Mon Sep 17 00:00:00 2001 From: dlee Date: Fri, 17 Feb 2023 16:20:29 -0600 Subject: [PATCH 1448/2309] Type annotations added for wormholetesting.py --- src/allmydata/test/cli/wormholetesting.py | 35 +++++++++++------------ 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/src/allmydata/test/cli/wormholetesting.py b/src/allmydata/test/cli/wormholetesting.py index 744f9d75a..7cf9d7eff 100644 --- a/src/allmydata/test/cli/wormholetesting.py +++ b/src/allmydata/test/cli/wormholetesting.py @@ -32,8 +32,7 @@ For example:: from __future__ import annotations -from typing import Iterator, Optional, List, Tuple -from collections.abc import Awaitable +from typing import Iterator, Optional, List, Tuple, Any, TextIO from inspect import getargspec from itertools import count from sys import stderr @@ -66,18 +65,18 @@ class MemoryWormholeServer(object): def create( self, - appid, - relay_url, - reactor, - versions={}, - delegate=None, - journal=None, - tor=None, - timing=None, - stderr=stderr, - _eventual_queue=None, - _enable_dilate=False, - ): + appid: str, + relay_url: str, + reactor: Any, + versions: Any={}, + delegate: Optional[Any]=None, + journal: Optional[Any]=None, + tor: Optional[Any]=None, + timing: Optional[Any]=None, + stderr: TextIO=stderr, + _eventual_queue: Optional[Any]=None, + _enable_dilate: bool=False, + )-> _MemoryWormhole: """ Create a wormhole. It will be able to connect to other wormholes created by this instance (and constrained by the normal appid/relay_url @@ -184,7 +183,7 @@ class _WormholeApp(object): return code - def wait_for_wormhole(self, code: WormholeCode) -> Awaitable[_MemoryWormhole]: + def wait_for_wormhole(self, code: WormholeCode) -> Deferred[_MemoryWormhole]: """ Return a ``Deferred`` which fires with the next wormhole to be associated with the given code. This is used to let the first end of a wormhole @@ -262,7 +261,7 @@ class _MemoryWormhole(object): return d return succeed(self._code) - def get_welcome(self): + def get_welcome(self) -> Deferred[str]: return succeed("welcome") def send_message(self, payload: WormholeMessage) -> None: @@ -276,8 +275,8 @@ class _MemoryWormhole(object): ) d = self._view.wormhole_by_code(self._code, exclude=self) - def got_wormhole(wormhole): - msg = wormhole._payload.get() + def got_wormhole(wormhole: _MemoryWormhole) -> Deferred[_MemoryWormhole]: + msg: Deferred[_MemoryWormhole] = wormhole._payload.get() return msg d.addCallback(got_wormhole) From 86dbcb21ce4f27e779b3d8febc633c1cbf3fd97e Mon Sep 17 00:00:00 2001 From: dlee Date: Fri, 17 Feb 2023 16:24:32 -0600 Subject: [PATCH 1449/2309] Refactored verify function to update deprecated getargspec function with getfullargspec and maintained strictness --- src/allmydata/test/cli/wormholetesting.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/src/allmydata/test/cli/wormholetesting.py b/src/allmydata/test/cli/wormholetesting.py index 7cf9d7eff..9ce199545 100644 --- a/src/allmydata/test/cli/wormholetesting.py +++ b/src/allmydata/test/cli/wormholetesting.py @@ -33,7 +33,7 @@ For example:: from __future__ import annotations from typing import Iterator, Optional, List, Tuple, Any, TextIO -from inspect import getargspec +from inspect import getfullargspec from itertools import count from sys import stderr @@ -133,18 +133,24 @@ class TestingHelper(object): return wormhole -def _verify(): +def _verify() -> None: """ Roughly confirm that the in-memory wormhole creation function matches the interface of the real implementation. """ # Poor man's interface verification. - a = getargspec(create) - b = getargspec(MemoryWormholeServer.create) + a = getfullargspec(create) + b = getfullargspec(MemoryWormholeServer.create) + # I know it has a `self` argument at the beginning. That's okay. b = b._replace(args=b.args[1:]) - assert a == b, "{} != {}".format(a, b) + + # Just compare the same information to check function signature + assert a.varkw == b.varkw + assert a.args == b.args + assert a.varargs == b.varargs + assert a.kwonlydefaults == b.kwonlydefaults _verify() From be9d76e2b8cffda206afe066fe00be2db8dd6759 Mon Sep 17 00:00:00 2001 From: dlee Date: Fri, 17 Feb 2023 16:24:52 -0600 Subject: [PATCH 1450/2309] Added newsfragment --- newsfragments/3970.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3970.minor diff --git a/newsfragments/3970.minor b/newsfragments/3970.minor new file mode 100644 index 000000000..e69de29bb From e3ad50a08498a1d74363cbb711c3c55bf7210960 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 20 Feb 2023 11:54:50 -0500 Subject: [PATCH 1451/2309] Just skip usage of reconfigure() on Windows. --- integration/test_get_put.py | 5 +++++ integration/util.py | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/integration/test_get_put.py b/integration/test_get_put.py index fd29d51fe..76c6ee600 100644 --- a/integration/test_get_put.py +++ b/integration/test_get_put.py @@ -4,6 +4,7 @@ and stdout. """ from subprocess import Popen, PIPE, check_output +import sys import pytest from pytest_twisted import ensureDeferred @@ -66,6 +67,10 @@ def test_get_to_stdout(alice, get_put_alias, tmpdir): assert p.wait() == 0 +@pytest.mark.skipif( + sys.platform.startswith("win"), + reason="reconfigure() has issues on Windows" +) @ensureDeferred async def test_upload_download_immutable_different_default_max_segment_size(alice, get_put_alias, tmpdir, request): """ diff --git a/integration/util.py b/integration/util.py index 02cda655d..884785deb 100644 --- a/integration/util.py +++ b/integration/util.py @@ -738,6 +738,8 @@ async def reconfigure(reactor, request, node: TahoeProcess, Reconfigure a Tahoe-LAFS node with different ZFEC parameters and convergence secret. + TODO This appears to have issues on Windows. + If the current configuration is different from the specified configuration, the node will be restarted so it takes effect. @@ -753,6 +755,9 @@ async def reconfigure(reactor, request, node: TahoeProcess, :return: ``None`` after the node configuration has been rewritten, the node has been restarted, and the node is ready to provide service. """ + # TODO reconfigure() seems to have issues on Windows. If you need to use it + # there, delete this assert and try to figure out what's going on... + assert not sys.platform.startswith("win") happy, needed, total = params config = node.get_config() From b14b2d0409cab0a828b76a026b1d9ee40dbc4008 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 20 Feb 2023 12:01:14 -0500 Subject: [PATCH 1452/2309] Use a nicer shutdown mechanism. --- src/allmydata/util/cputhreadpool.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/src/allmydata/util/cputhreadpool.py b/src/allmydata/util/cputhreadpool.py index a7f9d8bd6..a7407804a 100644 --- a/src/allmydata/util/cputhreadpool.py +++ b/src/allmydata/util/cputhreadpool.py @@ -17,6 +17,7 @@ scheduler affinity or cgroups, but that's not the end of the world. import os from typing import TypeVar, Callable, cast from functools import partial +import threading from twisted.python.threadpool import ThreadPool from twisted.internet.defer import Deferred @@ -25,8 +26,19 @@ from twisted.internet.interfaces import IReactorFromThreads _CPU_THREAD_POOL = ThreadPool(minthreads=0, maxthreads=os.cpu_count(), name="TahoeCPU") -# Daemon threads allow shutdown to happen: -_CPU_THREAD_POOL.threadFactory = partial(_CPU_THREAD_POOL.threadFactory, daemon=True) +if hasattr(threading, "_register_atexit"): + # This is a private API present in Python 3.8 or later, specifically + # designed for thread pool shutdown. Since it's private, it might go away + # at any point, so if it doesn't exist we still have a solution. + threading._register_atexit(_CPU_THREAD_POOL.stop) +else: + # Daemon threads allow shutdown to happen without any explicit stopping of + # threads. There are some bugs in old Python versions related to daemon + # threads (fixed in subsequent CPython patch releases), but Python's own + # thread pools use daemon threads in those versions so we're no worse off. + _CPU_THREAD_POOL.threadFactory = partial( + _CPU_THREAD_POOL.threadFactory, daemon=True + ) _CPU_THREAD_POOL.start() From 31024ceb4c04291152e38360344fcbdce30a1d02 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 20 Feb 2023 12:44:03 -0500 Subject: [PATCH 1453/2309] reconfigure() is only an issue if it changes something... --- integration/util.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/integration/util.py b/integration/util.py index 884785deb..d0444cfc8 100644 --- a/integration/util.py +++ b/integration/util.py @@ -755,9 +755,6 @@ async def reconfigure(reactor, request, node: TahoeProcess, :return: ``None`` after the node configuration has been rewritten, the node has been restarted, and the node is ready to provide service. """ - # TODO reconfigure() seems to have issues on Windows. If you need to use it - # there, delete this assert and try to figure out what's going on... - assert not sys.platform.startswith("win") happy, needed, total = params config = node.get_config() @@ -789,6 +786,11 @@ async def reconfigure(reactor, request, node: TahoeProcess, ) if changed: + # TODO reconfigure() seems to have issues on Windows. If you need to + # use it there, delete this assert and try to figure out what's going + # on... + assert not sys.platform.startswith("win") + # restart the node print(f"Restarting {node.node_dir} for ZFEC reconfiguration") await node.restart_async(reactor, request) From 5b14561ec0c1827744c6d4b5127de7bececabb4d Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 20 Feb 2023 12:02:34 -0700 Subject: [PATCH 1454/2309] use attrs directly --- src/allmydata/cli/grid_manager.py | 3 ++- src/allmydata/grid_manager.py | 8 ++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/allmydata/cli/grid_manager.py b/src/allmydata/cli/grid_manager.py index 220f091cd..9e4e911bf 100644 --- a/src/allmydata/cli/grid_manager.py +++ b/src/allmydata/cli/grid_manager.py @@ -9,6 +9,7 @@ from datetime import ( ) import click +import attr from twisted.python.filepath import ( FilePath, @@ -196,7 +197,7 @@ def sign(ctx, name, expiry_days): "No storage-server called '{}' exists".format(name) ) - certificate_data = json.dumps(certificate.asdict(), indent=4) + certificate_data = json.dumps(attr.asdict(certificate), indent=4) click.echo(certificate_data) if fp is not None: next_serial = 0 diff --git a/src/allmydata/grid_manager.py b/src/allmydata/grid_manager.py index ff5ce735d..09b9ed001 100644 --- a/src/allmydata/grid_manager.py +++ b/src/allmydata/grid_manager.py @@ -19,7 +19,10 @@ from allmydata.util import ( dictutil, ) -from attrs import define, asdict, Factory +from attrs import ( + define, + Factory, +) @define @@ -41,9 +44,6 @@ class SignedCertificate(object): signature=data["signature"].encode("ascii") ) - def asdict(self): - return asdict(self) - @define class _GridManagerStorageServer(object): From a6cf06cc6d3b691d0971790582332cfbc3026139 Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 20 Feb 2023 12:09:51 -0700 Subject: [PATCH 1455/2309] http needs access to GridManager --- src/allmydata/grid_manager.py | 8 ++++++-- src/allmydata/storage_client.py | 33 +++++++++++++++++++++++++++------ 2 files changed, 33 insertions(+), 8 deletions(-) diff --git a/src/allmydata/grid_manager.py b/src/allmydata/grid_manager.py index 09b9ed001..062a5d9d0 100644 --- a/src/allmydata/grid_manager.py +++ b/src/allmydata/grid_manager.py @@ -6,7 +6,11 @@ import sys from datetime import ( datetime, ) -from typing import Optional, Union +from typing import ( + Optional, + Union, + List, +) from twisted.python.filepath import FilePath @@ -99,7 +103,7 @@ def create_grid_manager(): ) -def _load_certificates_for(config_path: Optional[FilePath], name: str, gm_key=Optional[ed25519.Ed25519PublicKey]): +def _load_certificates_for(config_path: Optional[FilePath], name: str, gm_key=Optional[ed25519.Ed25519PublicKey]) -> List[_GridManagerCertificate]: """ Load any existing certificates for the given storage-server. diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 8a8aa1f2e..6fab1707b 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -293,17 +293,22 @@ class StorageFarmBroker(service.MultiService): by the given announcement. """ assert isinstance(server_id, bytes) - if len(server["ann"].get(ANONYMOUS_STORAGE_NURLS, [])) > 0: - s = HTTPNativeStorageServer(server_id, server["ann"]) - s.on_status_changed(lambda _: self._got_connection()) - return s - handler_overrides = server.get("connections", {}) gm_verifier = create_grid_manager_verifier( self.storage_client_config.grid_manager_keys, server["ann"].get("grid-manager-certificates", []), "pub-{}".format(str(server_id, "ascii")), # server_id is v0- not pub-v0-key .. for reasons? ) + if len(server["ann"].get(ANONYMOUS_STORAGE_NURLS, [])) > 0: + s = HTTPNativeStorageServer( + server_id, + server["ann"], + grid_manager_verifier=gm_verifier, + ) + s.on_status_changed(lambda _: self._got_connection()) + return s + + handler_overrides = server.get("connections", {}) s = NativeStorageServer( server_id, server["ann"], @@ -1013,13 +1018,14 @@ class HTTPNativeStorageServer(service.MultiService): "connected". """ - def __init__(self, server_id: bytes, announcement, reactor=reactor): + def __init__(self, server_id: bytes, announcement, reactor=reactor, grid_manager_verifier=None): service.MultiService.__init__(self) assert isinstance(server_id, bytes) self._server_id = server_id self.announcement = announcement self._on_status_changed = ObserverList() self._reactor = reactor + self._grid_manager_verifier = grid_manager_verifier furl = announcement["anonymous-storage-FURL"].encode("utf-8") ( self._nickname, @@ -1069,6 +1075,21 @@ class HTTPNativeStorageServer(service.MultiService): """ return self._on_status_changed.subscribe(status_changed) + def upload_permitted(self): + """ + If our client is configured with Grid Manager public-keys, we will + only upload to storage servers that have a currently-valid + certificate signed by at least one of the Grid Managers we + accept. + + :return: True if we should use this server for uploads, False + otherwise. + """ + # if we have no Grid Manager keys configured, choice is easy + if self._grid_manager_verifier is None: + return True + return self._grid_manager_verifier() + # Special methods used by copy.copy() and copy.deepcopy(). When those are # used in allmydata.immutable.filenode to copy CheckResults during # repair, we want it to treat the IServer instances as singletons, and From bdf4c49a3467d84b076c415cde2f976819a74653 Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 20 Feb 2023 22:57:15 -0700 Subject: [PATCH 1456/2309] fine, move the if statement --- src/allmydata/grid_manager.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/allmydata/grid_manager.py b/src/allmydata/grid_manager.py index 062a5d9d0..fb70a6d02 100644 --- a/src/allmydata/grid_manager.py +++ b/src/allmydata/grid_manager.py @@ -103,7 +103,7 @@ def create_grid_manager(): ) -def _load_certificates_for(config_path: Optional[FilePath], name: str, gm_key=Optional[ed25519.Ed25519PublicKey]) -> List[_GridManagerCertificate]: +def _load_certificates_for(config_path: FilePath, name: str, gm_key=Optional[ed25519.Ed25519PublicKey]) -> List[_GridManagerCertificate]: """ Load any existing certificates for the given storage-server. @@ -119,8 +119,6 @@ def _load_certificates_for(config_path: Optional[FilePath], name: str, gm_key=Op :raises: ed25519.BadSignature if any certificate signature fails to verify """ - if config_path is None: - return [] cert_index = 0 cert_path = config_path.child('{}.cert.{}'.format(name, cert_index)) certificates = [] @@ -199,7 +197,7 @@ def load_grid_manager(config_path: Optional[FilePath]): storage_servers[name] = _GridManagerStorageServer( name, ed25519.verifying_key_from_string(srv_config['public_key'].encode('ascii')), - _load_certificates_for(config_path, name, public_key), + [] if config_path is None else _load_certificates_for(config_path, name, public_key), ) return _GridManager(private_key_bytes, storage_servers) From 7b98edac041312b6565a355d8cbc73c52c4a8f40 Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 20 Feb 2023 23:56:19 -0700 Subject: [PATCH 1457/2309] add more words --- docs/managed-grid.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/managed-grid.rst b/docs/managed-grid.rst index df12399d3..aac2bf20a 100644 --- a/docs/managed-grid.rst +++ b/docs/managed-grid.rst @@ -55,6 +55,10 @@ certificate, or no certificate) because clients check the ciphertext and re-assembled plaintext against the keys in the capability; "grid-manager" certificates only control uploads. +Clients make use of this functionality by configuring one or more Grid Manager public keys. +This tells the client to only upload to storage-servers that have a currently-valid certificate from any of the Grid Managers their client allows. +In case none are configured, the default behavior (of using any storage server) prevails. + Grid Manager Data Storage ------------------------- From 8e20fa0fbf033ac9559326e805d644293597e1e0 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 21 Feb 2023 00:01:13 -0700 Subject: [PATCH 1458/2309] whitespace --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 5a99447ad..98177bd41 100644 --- a/setup.py +++ b/setup.py @@ -146,7 +146,7 @@ install_requires = [ # Command-line parsing "click >= 7.0", - + # for pid-file support "psutil", "filelock", From affe0cb37b2c361c7e4a5fe8e9f98f5b366da02b Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 21 Feb 2023 00:02:53 -0700 Subject: [PATCH 1459/2309] fine? --- src/allmydata/util/configutil.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/allmydata/util/configutil.py b/src/allmydata/util/configutil.py index b84d96319..b48e2c034 100644 --- a/src/allmydata/util/configutil.py +++ b/src/allmydata/util/configutil.py @@ -74,10 +74,7 @@ def write_config(tahoe_cfg, config): :return: ``None`` """ tmp = tahoe_cfg.temporarySibling() - try: - tahoe_cfg.parent().makedirs() - except OSError: - pass + tahoe_cfg.parent().makedirs(ignoreExistingDirectory=True) # FilePath.open can only open files in binary mode which does not work # with ConfigParser.write. with open(tmp.path, "wt") as fp: From 43d29986a66b98d358cbb6ef3315ec1258b056c2 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 21 Feb 2023 00:04:03 -0700 Subject: [PATCH 1460/2309] Update src/allmydata/cli/grid_manager.py Co-authored-by: Jean-Paul Calderone --- src/allmydata/cli/grid_manager.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/allmydata/cli/grid_manager.py b/src/allmydata/cli/grid_manager.py index 220f091cd..af66fa1c6 100644 --- a/src/allmydata/cli/grid_manager.py +++ b/src/allmydata/cli/grid_manager.py @@ -205,13 +205,10 @@ def sign(ctx, name, expiry_days): fname = "{}.cert.{}".format(name, next_serial) try: f = fp.child(fname).create() + except FileExistsError: + f = None except OSError as e: - if e.errno == 17: # file exists - f = None - else: - raise click.ClickException( - "{}: {}".format(fname, e) - ) + raise click.ClickException(f"{fname}: {e}") next_serial += 1 with f: f.write(certificate_data.encode("ascii")) From 032b852bab1babd3cfd3cb9acc975560d74f7315 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 21 Feb 2023 01:51:13 -0700 Subject: [PATCH 1461/2309] define -> frozen --- src/allmydata/grid_manager.py | 3 ++- src/allmydata/test/test_grid_manager.py | 7 +++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/allmydata/grid_manager.py b/src/allmydata/grid_manager.py index fb70a6d02..7f1c6eda5 100644 --- a/src/allmydata/grid_manager.py +++ b/src/allmydata/grid_manager.py @@ -25,11 +25,12 @@ from allmydata.util import ( from attrs import ( define, + frozen, Factory, ) -@define +@frozen class SignedCertificate(object): """ A signed certificate. diff --git a/src/allmydata/test/test_grid_manager.py b/src/allmydata/test/test_grid_manager.py index 0f8df69d6..c282d1237 100644 --- a/src/allmydata/test/test_grid_manager.py +++ b/src/allmydata/test/test_grid_manager.py @@ -434,11 +434,14 @@ class GridManagerInvalidVerifier(SyncTestCase): An incorrect signature is rejected """ # make signature invalid - self.cert0.signature = invalid_signature.encode("ascii") + invalid_cert = SignedCertificate( + self.cert0.certificate, + invalid_signature.encode("ascii"), + ) verify = create_grid_manager_verifier( [self.gm._public_key], - [self.cert0], + [invalid_cert], ed25519.string_from_verifying_key(self.pub0), bad_cert = lambda key, cert: None, ) From 38669cc3cec58fa9d66b48f063d905382f11b4ec Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 21 Feb 2023 02:15:20 -0700 Subject: [PATCH 1462/2309] define -> frozen --- src/allmydata/grid_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/grid_manager.py b/src/allmydata/grid_manager.py index 7f1c6eda5..6e71c9902 100644 --- a/src/allmydata/grid_manager.py +++ b/src/allmydata/grid_manager.py @@ -50,7 +50,7 @@ class SignedCertificate(object): ) -@define +@frozen class _GridManagerStorageServer(object): """ A Grid Manager's notion of a storage server From 154f1ce14329be643cf222ab791d85a5e0cd0486 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 21 Feb 2023 09:38:54 -0500 Subject: [PATCH 1463/2309] No need for sleep. --- src/allmydata/test/test_util.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/allmydata/test/test_util.py b/src/allmydata/test/test_util.py index f4e7d21e0..310bfdfd2 100644 --- a/src/allmydata/test/test_util.py +++ b/src/allmydata/test/test_util.py @@ -601,7 +601,6 @@ class CPUThreadPool(unittest.TestCase): async def test_runs_in_thread(self): """The given function runs in a thread.""" def f(*args, **kwargs): - time.sleep(0.1) return current_thread(), args, kwargs this_thread = current_thread().ident From 2811c80dc39fb75c5c969b29ede0c2a1e714fddf Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 21 Feb 2023 09:57:21 -0500 Subject: [PATCH 1464/2309] Fix timeout in parallel benchmark. --- benchmarks/upload_download.py | 4 ++++ src/allmydata/test/common_system.py | 6 +++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/benchmarks/upload_download.py b/benchmarks/upload_download.py index 0ca034af9..3dfa63336 100644 --- a/benchmarks/upload_download.py +++ b/benchmarks/upload_download.py @@ -74,6 +74,10 @@ class ImmutableBenchmarks(SystemTestMixin, TestCase): # To use Foolscap, change to True: FORCE_FOOLSCAP_FOR_STORAGE = False + # Don't reduce HTTP connection timeouts, that messes up the more aggressive + # benchmarks: + REDUCE_HTTP_CLIENT_TIMEOUT = False + @async_to_deferred async def setUp(self): SystemTestMixin.setUp(self) diff --git a/src/allmydata/test/common_system.py b/src/allmydata/test/common_system.py index 01966824a..39b2b43d1 100644 --- a/src/allmydata/test/common_system.py +++ b/src/allmydata/test/common_system.py @@ -681,6 +681,9 @@ class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin): # test code. FORCE_FOOLSCAP_FOR_STORAGE : Optional[bool] = None + # If True, reduce the timeout on connections: + REDUCE_HTTP_CLIENT_TIMEOUT : bool = True + def setUp(self): self._http_client_pools = [] http_client.StorageClient.start_test_mode(self._got_new_http_connection_pool) @@ -707,7 +710,8 @@ class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin): d.addTimeout(1, reactor) return d - pool.getConnection = getConnectionWithTimeout + if self.REDUCE_HTTP_CLIENT_TIMEOUT: + pool.getConnection = getConnectionWithTimeout def close_idle_http_connections(self): """Close all HTTP client connections that are just hanging around.""" From 95bb7afba7c0c883ff7c9099dc9cd839daef3c6c Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 21 Feb 2023 10:42:06 -0500 Subject: [PATCH 1465/2309] Sketch of happy eyeballs. --- src/allmydata/storage_client.py | 32 ++++++++++++++++++++++++++------ 1 file changed, 26 insertions(+), 6 deletions(-) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 8e9ad3656..8191014e8 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -47,7 +47,7 @@ from zope.interface import ( ) from twisted.python.failure import Failure from twisted.web import http -from twisted.internet.task import LoopingCall +from twisted.internet.task import LoopingCall, deferLater from twisted.internet import defer, reactor from twisted.application import service from twisted.plugin import ( @@ -935,6 +935,21 @@ class NativeStorageServer(service.MultiService): self._reconnector.reset() +async def _pick_a_http_server(reactor, nurls: list[DecodedURL]) -> DecodedURL: + """Pick the first server we successfully talk to.""" + while True: + try: + _, index = await defer.DeferredList([ + StorageClientGeneral( + StorageClient.from_nurl(nurl, reactor) + ).get_version() for nurl in nurls + ], consumeErrors=True, fireOnOneCallback=True) + return nurls[index] + except Exception as e: + log.err(e, "Failed to connect to any of the HTTP NURLs for server") + await deferLater(reactor, 1, lambda: None) + + @implementer(IServer) class HTTPNativeStorageServer(service.MultiService): """ @@ -962,10 +977,8 @@ class HTTPNativeStorageServer(service.MultiService): ) = _parse_announcement(server_id, furl, announcement) # TODO need some way to do equivalent of Happy Eyeballs for multiple NURLs? # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3935 - nurl = DecodedURL.from_text(announcement[ANONYMOUS_STORAGE_NURLS][0]) - self._istorage_server = _HTTPStorageServer.from_http_client( - StorageClient.from_nurl(nurl, reactor) - ) + self._nurls = [DecodedURL.from_text(u) for u in announcement[ANONYMOUS_STORAGE_NURLS]] + self._istorage_server = None self._connection_status = connection_status.ConnectionStatus.unstarted() self._version = None @@ -1033,7 +1046,14 @@ class HTTPNativeStorageServer(service.MultiService): version = self.get_version() return _available_space_from_version(version) - def start_connecting(self, trigger_cb): + @async_to_deferred + async def start_connecting(self, trigger_cb): + # The problem with this scheme is that while picking the HTTP server to + # talk to, we don't have connection status updates... + nurl = await _pick_a_http_server(reactor, self._nurls) + self._istorage_server = _HTTPStorageServer.from_http_client( + StorageClient.from_nurl(nurl, reactor) + ) self._lc = LoopingCall(self._connect) self._lc.start(1, True) From 2ac6580c269268e891a1f79f23a81f0d56917512 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 21 Feb 2023 10:56:38 -0500 Subject: [PATCH 1466/2309] Welcome to the world of tomorrow. --- src/allmydata/test/test_storage_client.py | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/src/allmydata/test/test_storage_client.py b/src/allmydata/test/test_storage_client.py index 1a84f35ec..0657a814e 100644 --- a/src/allmydata/test/test_storage_client.py +++ b/src/allmydata/test/test_storage_client.py @@ -1,16 +1,8 @@ """ -Ported from Python 3. +Tests for allmydata.storage_client. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - -from six import ensure_text +from __future__ import annotations from json import ( loads, @@ -475,7 +467,7 @@ class StoragePluginWebPresence(AsyncTestCase): # config validation policy). "tub.port": tubport_endpoint, "tub.location": tubport_location, - "web.port": ensure_text(webport_endpoint), + "web.port": str(webport_endpoint), }, storage_plugin=self.storage_plugin, basedir=self.basedir, From 32768e310ae5316cc2e6fc0f0f969368c1ff3eee Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 21 Feb 2023 11:30:47 -0500 Subject: [PATCH 1467/2309] Unit test for _pick_a_http_server. --- src/allmydata/storage_client.py | 48 ++++++++++------- src/allmydata/test/test_storage_client.py | 64 ++++++++++++++++++++++- 2 files changed, 93 insertions(+), 19 deletions(-) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 8191014e8..436335431 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -33,7 +33,7 @@ Ported to Python 3. from __future__ import annotations from six import ensure_text -from typing import Union +from typing import Union, Callable, Any import re, time, hashlib from os import urandom from configparser import NoSectionError @@ -935,19 +935,25 @@ class NativeStorageServer(service.MultiService): self._reconnector.reset() -async def _pick_a_http_server(reactor, nurls: list[DecodedURL]) -> DecodedURL: - """Pick the first server we successfully talk to.""" +async def _pick_a_http_server( + reactor, + nurls: list[DecodedURL], + request: Callable[[DecodedURL, Any], defer.Deferred[Any]] +) -> DecodedURL: + """Pick the first server we successfully send a request to.""" while True: - try: - _, index = await defer.DeferredList([ - StorageClientGeneral( - StorageClient.from_nurl(nurl, reactor) - ).get_version() for nurl in nurls - ], consumeErrors=True, fireOnOneCallback=True) + result = await defer.DeferredList([ + request(reactor, nurl) for nurl in nurls + ], consumeErrors=True, fireOnOneCallback=True) + # Apparently DeferredList is an awful awful API. If everything fails, + # you get back a list of (False, Failure), if it succeeds, you get a + # tuple of (value, index). + if isinstance(result, list): + await deferLater(reactor, 1, lambda: None) + else: + assert isinstance(result, tuple) + _, index = result return nurls[index] - except Exception as e: - log.err(e, "Failed to connect to any of the HTTP NURLs for server") - await deferLater(reactor, 1, lambda: None) @implementer(IServer) @@ -975,9 +981,10 @@ class HTTPNativeStorageServer(service.MultiService): self._short_description, self._long_description ) = _parse_announcement(server_id, furl, announcement) - # TODO need some way to do equivalent of Happy Eyeballs for multiple NURLs? - # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3935 - self._nurls = [DecodedURL.from_text(u) for u in announcement[ANONYMOUS_STORAGE_NURLS]] + self._nurls = [ + DecodedURL.from_text(u) + for u in announcement[ANONYMOUS_STORAGE_NURLS] + ] self._istorage_server = None self._connection_status = connection_status.ConnectionStatus.unstarted() @@ -1048,9 +1055,14 @@ class HTTPNativeStorageServer(service.MultiService): @async_to_deferred async def start_connecting(self, trigger_cb): - # The problem with this scheme is that while picking the HTTP server to - # talk to, we don't have connection status updates... - nurl = await _pick_a_http_server(reactor, self._nurls) + # TODO file a bug: The problem with this scheme is that while picking + # the HTTP server to talk to, we don't have connection status + # updates... + def request(reactor, nurl: DecodedURL): + return StorageClientGeneral( + StorageClient.from_nurl(nurl, reactor) + ).get_version() + nurl = await _pick_a_http_server(reactor, self._nurls, request) self._istorage_server = _HTTPStorageServer.from_http_client( StorageClient.from_nurl(nurl, reactor) ) diff --git a/src/allmydata/test/test_storage_client.py b/src/allmydata/test/test_storage_client.py index 0657a814e..38ef8c1d3 100644 --- a/src/allmydata/test/test_storage_client.py +++ b/src/allmydata/test/test_storage_client.py @@ -7,8 +7,10 @@ from __future__ import annotations from json import ( loads, ) - import hashlib +from typing import Union, Any + +from hyperlink import DecodedURL from fixtures import ( TempDir, ) @@ -52,6 +54,7 @@ from twisted.internet.defer import ( from twisted.python.filepath import ( FilePath, ) +from twisted.internet.task import Clock from foolscap.api import ( Tub, @@ -80,12 +83,14 @@ from allmydata.webish import ( WebishServer, ) from allmydata.util import base32, yamlutil +from allmydata.util.deferredutil import async_to_deferred from allmydata.storage_client import ( IFoolscapStorageServer, NativeStorageServer, StorageFarmBroker, _FoolscapStorage, _NullStorage, + _pick_a_http_server ) from ..storage.server import ( StorageServer, @@ -731,3 +736,60 @@ storage: yield done self.assertTrue(done.called) + + +class PickHTTPServerTests(unittest.SynchronousTestCase): + """Tests for ``_pick_a_http_server``.""" + + def loop_until_result(self, url_to_results: dict[DecodedURL, list[tuple[float, Union[Exception, Any]]]]) -> Deferred[DecodedURL]: + """ + Given mapping of URLs to list of (delay, result), return the URL of the + first selected server. + """ + clock = Clock() + + def request(reactor, url): + delay, value = url_to_results[url].pop(0) + result = Deferred() + def add_result_value(): + if isinstance(value, Exception): + result.errback(value) + else: + result.callback(value) + reactor.callLater(delay, add_result_value) + return result + + d = async_to_deferred(_pick_a_http_server)( + clock, list(url_to_results.keys()), request + ) + for i in range(1000): + clock.advance(0.1) + return d + + def test_first_successful_connect_is_picked(self): + """ + Given multiple good URLs, the first one that connects is chosen. + """ + earliest_url = DecodedURL.from_text("http://a") + latest_url = DecodedURL.from_text("http://b") + d = self.loop_until_result({ + latest_url: [(2, None)], + earliest_url: [(1, None)] + }) + self.assertEqual(self.successResultOf(d), earliest_url) + + def test_failures_are_retried(self): + """ + If the initial requests all fail, ``_pick_a_http_server`` keeps trying + until success. + """ + eventually_good_url = DecodedURL.from_text("http://good") + bad_url = DecodedURL.from_text("http://bad") + d = self.loop_until_result({ + eventually_good_url: [ + (1, ZeroDivisionError()), (0.1, ZeroDivisionError()), (1, None) + ], + bad_url: [(0.1, RuntimeError()), (0.1, RuntimeError()), (0.1, RuntimeError())] + }) + self.flushLoggedErrors(ZeroDivisionError, RuntimeError) + self.assertEqual(self.successResultOf(d), eventually_good_url) From 1b6d5e1bda302ba464f20175896dcc2a12390e96 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 21 Feb 2023 11:56:46 -0700 Subject: [PATCH 1468/2309] Revert "use attrs directly" This reverts commit 5b14561ec0c1827744c6d4b5127de7bececabb4d. --- src/allmydata/cli/grid_manager.py | 3 +-- src/allmydata/grid_manager.py | 4 ++++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/allmydata/cli/grid_manager.py b/src/allmydata/cli/grid_manager.py index 9e4e911bf..220f091cd 100644 --- a/src/allmydata/cli/grid_manager.py +++ b/src/allmydata/cli/grid_manager.py @@ -9,7 +9,6 @@ from datetime import ( ) import click -import attr from twisted.python.filepath import ( FilePath, @@ -197,7 +196,7 @@ def sign(ctx, name, expiry_days): "No storage-server called '{}' exists".format(name) ) - certificate_data = json.dumps(attr.asdict(certificate), indent=4) + certificate_data = json.dumps(certificate.asdict(), indent=4) click.echo(certificate_data) if fp is not None: next_serial = 0 diff --git a/src/allmydata/grid_manager.py b/src/allmydata/grid_manager.py index 6e71c9902..31c342ed6 100644 --- a/src/allmydata/grid_manager.py +++ b/src/allmydata/grid_manager.py @@ -26,6 +26,7 @@ from allmydata.util import ( from attrs import ( define, frozen, + asdict, Factory, ) @@ -49,6 +50,9 @@ class SignedCertificate(object): signature=data["signature"].encode("ascii") ) + def asdict(self): + return asdict(self) + @frozen class _GridManagerStorageServer(object): From 82045b4298ea8eb4516b9e0639f6c856d21509f1 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 21 Feb 2023 14:19:44 -0700 Subject: [PATCH 1469/2309] store signature as raw bytes, not base32 --- src/allmydata/cli/grid_manager.py | 2 +- src/allmydata/grid_manager.py | 19 +++++++++++++------ src/allmydata/test/test_grid_manager.py | 2 +- 3 files changed, 15 insertions(+), 8 deletions(-) diff --git a/src/allmydata/cli/grid_manager.py b/src/allmydata/cli/grid_manager.py index 220f091cd..433e30434 100644 --- a/src/allmydata/cli/grid_manager.py +++ b/src/allmydata/cli/grid_manager.py @@ -196,7 +196,7 @@ def sign(ctx, name, expiry_days): "No storage-server called '{}' exists".format(name) ) - certificate_data = json.dumps(certificate.asdict(), indent=4) + certificate_data = json.dumps(certificate.marshal(), indent=4) click.echo(certificate_data) if fp is not None: next_serial = 0 diff --git a/src/allmydata/grid_manager.py b/src/allmydata/grid_manager.py index 31c342ed6..a17a1fab5 100644 --- a/src/allmydata/grid_manager.py +++ b/src/allmydata/grid_manager.py @@ -39,7 +39,8 @@ class SignedCertificate(object): # A JSON-encoded, UTF-8-encoded certificate. certificate : bytes - # The signature in base32. + # The signature (although the signature is in base32 in "public", + # this contains the decoded raw bytes -- not base32) signature : bytes @classmethod @@ -47,11 +48,17 @@ class SignedCertificate(object): data = json.load(file_like) return cls( certificate=data["certificate"].encode("utf-8"), - signature=data["signature"].encode("ascii") + signature=base32.a2b(data["signature"].encode("ascii")), ) - def asdict(self): - return asdict(self) + def marshal(self): + """ + :return dict: a json-able dict + """ + return dict( + certificate=self.certificate, + signature=base32.b2a(self.signature), + ) @frozen @@ -261,7 +268,7 @@ class _GridManager(object): sig = ed25519.sign_data(self._private_key, cert_data) certificate = SignedCertificate( certificate=cert_data, - signature=base32.b2a(sig), + signature=sig, ) vk = ed25519.verifying_key_from_signing_key(self._private_key) ed25519.verify_signature(vk, sig, cert_data) @@ -388,7 +395,7 @@ def validate_grid_manager_certificate(gm_key, alleged_cert): try: ed25519.verify_signature( gm_key, - base32.a2b(alleged_cert.signature), + alleged_cert.signature, alleged_cert.certificate, ) except ed25519.BadSignature: diff --git a/src/allmydata/test/test_grid_manager.py b/src/allmydata/test/test_grid_manager.py index c282d1237..95395f12e 100644 --- a/src/allmydata/test/test_grid_manager.py +++ b/src/allmydata/test/test_grid_manager.py @@ -175,7 +175,7 @@ class GridManagerVerifier(SyncTestCase): self.assertEqual( ed25519.verify_signature( gm_key, - base32.a2b(cert0.signature), + cert0.signature, cert0.certificate, ), None From d91bfcb1d2f95b8eb8714f679786df657d03426f Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 21 Feb 2023 14:38:43 -0700 Subject: [PATCH 1470/2309] clarify --- src/allmydata/grid_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/grid_manager.py b/src/allmydata/grid_manager.py index a17a1fab5..319d87002 100644 --- a/src/allmydata/grid_manager.py +++ b/src/allmydata/grid_manager.py @@ -69,7 +69,7 @@ class _GridManagerStorageServer(object): name : str public_key : ed25519.Ed25519PublicKey - certificates : list = Factory(list) + certificates : list = Factory(list) # SignedCertificates def add_certificate(self, certificate): """ From 00ef4661a64062ee0e0cd5878c9201797bd0cc92 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 21 Feb 2023 16:06:24 -0700 Subject: [PATCH 1471/2309] ISO dates, not seconds --- src/allmydata/cli/grid_manager.py | 3 ++- src/allmydata/grid_manager.py | 22 +++++++++++++++------- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/src/allmydata/cli/grid_manager.py b/src/allmydata/cli/grid_manager.py index 433e30434..63094dfcd 100644 --- a/src/allmydata/cli/grid_manager.py +++ b/src/allmydata/cli/grid_manager.py @@ -24,6 +24,7 @@ from allmydata.grid_manager import ( create_grid_manager, save_grid_manager, load_grid_manager, + current_datetime_with_zone, ) from allmydata.util import jsonbytes as json @@ -167,7 +168,7 @@ def list(ctx): name, str(ctx.obj.grid_manager.storage_servers[name].public_key_string(), "utf-8"))) for cert in ctx.obj.grid_manager.storage_servers[name].certificates: - delta = datetime.utcnow() - cert.expires + delta = current_datetime_with_zone() - cert.expires click.echo("{} cert {}: ".format(blank_name, cert.index), nl=False) if delta.total_seconds() < 0: click.echo("valid until {} ({})".format(cert.expires, abbreviate_time(delta))) diff --git a/src/allmydata/grid_manager.py b/src/allmydata/grid_manager.py index 319d87002..7d1d12fcd 100644 --- a/src/allmydata/grid_manager.py +++ b/src/allmydata/grid_manager.py @@ -5,6 +5,7 @@ Functions and classes relating to the Grid Manager internal state import sys from datetime import ( datetime, + timezone, ) from typing import ( Optional, @@ -115,6 +116,14 @@ def create_grid_manager(): ) +def current_datetime_with_zone(): + """ + :returns: a timezone-aware datetime object representing the + current timestamp in UTC + """ + return datetime.now(timezone.utc) + + def _load_certificates_for(config_path: FilePath, name: str, gm_key=Optional[ed25519.Ed25519PublicKey]) -> List[_GridManagerCertificate]: """ Load any existing certificates for the given storage-server. @@ -150,7 +159,7 @@ def _load_certificates_for(config_path: FilePath, name: str, gm_key=Optional[ed2 _GridManagerCertificate( filename=cert_path.path, index=cert_index, - expires=datetime.utcfromtimestamp(cert_data['expires']), + expires=datetime.fromisoformat(cert_data['expires']), public_key=ed25519.verifying_key_from_string(cert_data['public_key'].encode('ascii')), ) ) @@ -257,10 +266,9 @@ class _GridManager(object): raise KeyError( "No storage server named '{}'".format(name) ) - expiration = datetime.utcnow() + expiry - epoch_offset = (expiration - datetime(1970, 1, 1)).total_seconds() + expiration = current_datetime_with_zone() + expiry cert_info = { - "expires": epoch_offset, + "expires": expiration.isoformat(), "public_key": srv.public_key_string(), "version": 1, } @@ -421,7 +429,7 @@ def create_grid_manager_verifier(keys, certs, public_key, now_fn=None, bad_cert= certificates for. :param callable now_fn: a callable which returns the current UTC - timestamp (or datetime.utcnow if None). + timestamp (or current_datetime_with_zone() if None). :param callable bad_cert: a two-argument callable which is invoked when a certificate verification fails. The first argument is @@ -436,7 +444,7 @@ def create_grid_manager_verifier(keys, certs, public_key, now_fn=None, bad_cert= expired) in `certs` signed by one of the keys in `keys`. """ - now_fn = datetime.utcnow if now_fn is None else now_fn + now_fn = current_datetime_with_zone if now_fn is None else now_fn valid_certs = [] # if we have zero grid-manager keys then everything is valid @@ -479,7 +487,7 @@ def create_grid_manager_verifier(keys, certs, public_key, now_fn=None, bad_cert= """ now = now_fn() for cert in valid_certs: - expires = datetime.utcfromtimestamp(cert['expires']) + expires = datetime.fromisoformat(cert["expires"]) if cert['public_key'].encode("ascii") == public_key: if expires > now: # not-expired From 6ee5c7588059da0c7100c4e48b23858dfb58672b Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 21 Feb 2023 16:13:11 -0700 Subject: [PATCH 1472/2309] comment -> ticket --- src/allmydata/client.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/allmydata/client.py b/src/allmydata/client.py index 2eee9ef9c..b98d7e0d8 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -859,11 +859,9 @@ class _Client(node.Node, pollmixin.PollMixin): grid_manager_certificates = self.config.get_grid_manager_certificates() announcement[u"grid-manager-certificates"] = grid_manager_certificates - # XXX we should probably verify that the certificates are - # valid and not expired, as that could be confusing for the - # storage-server operator -- but then we need the public key - # of the Grid Manager (should that go in the config too, - # then? How to handle multiple grid-managers?) + # Note: certificates are not verified for validity here, but + # that may be useful. See: + # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3977 for ic in self.introducer_clients: ic.publish("storage", announcement, self._node_private_key) From 70459cfbf1f3c6dd070d83f1d5cdb3e5ebfded44 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 21 Feb 2023 16:47:58 -0700 Subject: [PATCH 1473/2309] use possibly-overriden stderr --- src/allmydata/scripts/admin.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/scripts/admin.py b/src/allmydata/scripts/admin.py index 6a24614c0..eceb9a55a 100644 --- a/src/allmydata/scripts/admin.py +++ b/src/allmydata/scripts/admin.py @@ -198,7 +198,7 @@ def add_grid_manager_cert(options): options['name'], cert_path.path, ) - print(msg, file=options.parent.parent.stderr) + print(msg, file=options.stderr) return 1 config.set_config("storage", "grid_management", "True") @@ -210,7 +210,7 @@ def add_grid_manager_cert(options): cert_count = len(config.enumerate_section("grid_manager_certificates")) print("There are now {} certificates".format(cert_count), - file=options.parent.parent.stderr) + file=options.stderr) return 0 From fb10e13c6842ea4e290a3b1c8d82a0b6d2a7e246 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 21 Feb 2023 18:16:45 -0700 Subject: [PATCH 1474/2309] might want to use these at parse time --- src/allmydata/scripts/runner.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/allmydata/scripts/runner.py b/src/allmydata/scripts/runner.py index 973c18d95..d9fbc1b0a 100644 --- a/src/allmydata/scripts/runner.py +++ b/src/allmydata/scripts/runner.py @@ -165,6 +165,8 @@ def parse_or_exit(config, argv, stdout, stderr): :return: ``config``, after using it to parse the argument list. """ try: + config.stdout = stdout + config.stderr = stderr parse_options(argv[1:], config=config) except usage.error as e: # `parse_options` may have the side-effect of initializing a From 69a480dc082ec91dc0ab4590387fd720ab904267 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 21 Feb 2023 18:44:09 -0700 Subject: [PATCH 1475/2309] rewrite test --- src/allmydata/test/test_grid_manager.py | 35 +++++++++++++++---------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/src/allmydata/test/test_grid_manager.py b/src/allmydata/test/test_grid_manager.py index 95395f12e..384c5636a 100644 --- a/src/allmydata/test/test_grid_manager.py +++ b/src/allmydata/test/test_grid_manager.py @@ -74,26 +74,33 @@ class GridManagerUtilities(SyncTestCase): """ An error is reported loading invalid certificate version """ - cert_path = self.mktemp() + gm_path = FilePath(self.mktemp()) + gm_path.makedirs() + config = { + "grid_manager_config_version": 0, + "private_key": "priv-v0-ub7knkkmkptqbsax4tznymwzc4nk5lynskwjsiubmnhcpd7lvlqa", + "storage_servers": { + "radia": { + "public_key": "pub-v0-cbq6hcf3pxcz6ouoafrbktmkixkeuywpcpbcomzd3lqbkq4nmfga" + } + } + } + with gm_path.child("config.json").open("wb") as f: + f.write(json.dumps_bytes(config)) + fake_cert = { "certificate": "{\"expires\":1601687822,\"public_key\":\"pub-v0-cbq6hcf3pxcz6ouoafrbktmkixkeuywpcpbcomzd3lqbkq4nmfga\",\"version\":22}", "signature": "fvjd3uvvupf2v6tnvkwjd473u3m3inyqkwiclhp7balmchkmn3px5pei3qyfjnhymq4cjcwvbpqmcwwnwswdtrfkpnlaxuih2zbdmda" } - with open(cert_path, "wb") as f: + with gm_path.child("radia.cert.0").open("wb") as f: f.write(json.dumps_bytes(fake_cert)) - config_data = ( - "[grid_managers]\n" - "fluffy = pub-v0-vqimc4s5eflwajttsofisp5st566dbq36xnpp4siz57ufdavpvlq\n" - "[grid_manager_certificates]\n" - "ding = {}\n".format(cert_path) + + with self.assertRaises(ValueError) as ctx: + load_grid_manager(gm_path) + self.assertIn( + "22", + str(ctx.exception), ) - config = config_from_string("/foo", "portnum", config_data, client_valid_config()) - self.assertEqual( - {"fluffy": "pub-v0-vqimc4s5eflwajttsofisp5st566dbq36xnpp4siz57ufdavpvlq"}, - config.enumerate_section("grid_managers") - ) - certs = config.get_grid_manager_certificates() - self.assertEqual([fake_cert], certs) def test_load_certificates_unknown_key(self): """ From 0ae7da7352918101a1ee559dcc0ef72f57d4ecc8 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 21 Feb 2023 23:20:28 -0700 Subject: [PATCH 1476/2309] prop up the fragile scaffolding --- src/allmydata/test/cli/test_admin.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/allmydata/test/cli/test_admin.py b/src/allmydata/test/cli/test_admin.py index 9bf12471f..0570467a5 100644 --- a/src/allmydata/test/cli/test_admin.py +++ b/src/allmydata/test/cli/test_admin.py @@ -200,6 +200,8 @@ class AddCertificateCommand(SyncTestCase): "--filename", "-", ] ) + self.tahoe.subOptions.subOptions.stdin = self.tahoe.stdin + self.tahoe.subOptions.subOptions.stderr = self.tahoe.stderr rc = add_grid_manager_cert(self.tahoe.subOptions.subOptions) self.assertEqual(rc, 0) @@ -227,6 +229,8 @@ class AddCertificateCommand(SyncTestCase): "--filename", "-", ] ) + self.tahoe.subOptions.subOptions.stdin = self.tahoe.stdin + self.tahoe.subOptions.subOptions.stderr = self.tahoe.stderr rc = add_grid_manager_cert(self.tahoe.subOptions.subOptions) self.assertEqual(rc, 0) @@ -239,6 +243,8 @@ class AddCertificateCommand(SyncTestCase): "--filename", "-", ] ) + self.tahoe.subOptions.subOptions.stdin = self.tahoe.stdin + self.tahoe.subOptions.subOptions.stderr = self.tahoe.stderr rc = add_grid_manager_cert(self.tahoe.subOptions.subOptions) self.assertEqual(rc, 1) self.assertIn( From 6aff94dd8fd78e159d0d013674b9d73a3ee72aff Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 22 Feb 2023 00:15:32 -0700 Subject: [PATCH 1477/2309] flake8, more frozen --- src/allmydata/cli/grid_manager.py | 1 - src/allmydata/grid_manager.py | 4 +--- src/allmydata/test/test_grid_manager.py | 1 - 3 files changed, 1 insertion(+), 5 deletions(-) diff --git a/src/allmydata/cli/grid_manager.py b/src/allmydata/cli/grid_manager.py index 32234be0d..3110a072e 100644 --- a/src/allmydata/cli/grid_manager.py +++ b/src/allmydata/cli/grid_manager.py @@ -4,7 +4,6 @@ A CLI for configuring a grid manager. from typing import Optional from datetime import ( - datetime, timedelta, ) diff --git a/src/allmydata/grid_manager.py b/src/allmydata/grid_manager.py index 7d1d12fcd..e264734b2 100644 --- a/src/allmydata/grid_manager.py +++ b/src/allmydata/grid_manager.py @@ -25,9 +25,7 @@ from allmydata.util import ( ) from attrs import ( - define, frozen, - asdict, Factory, ) @@ -93,7 +91,7 @@ class _GridManagerStorageServer(object): } -@define +@frozen class _GridManagerCertificate(object): """ Represents a single certificate for a single storage-server diff --git a/src/allmydata/test/test_grid_manager.py b/src/allmydata/test/test_grid_manager.py index 384c5636a..78280a168 100644 --- a/src/allmydata/test/test_grid_manager.py +++ b/src/allmydata/test/test_grid_manager.py @@ -22,7 +22,6 @@ from allmydata.crypto import ( ed25519, ) from allmydata.util import ( - base32, jsonbytes as json, ) from allmydata.grid_manager import ( From 74e77685a35905bc9bdd0609ac73309a9da7f10d Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 23 Feb 2023 10:07:57 -0500 Subject: [PATCH 1478/2309] Get rid of DeferredList. --- src/allmydata/storage_client.py | 35 ++++++++++++++++++++++----------- 1 file changed, 24 insertions(+), 11 deletions(-) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 436335431..0d2443d64 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -942,18 +942,31 @@ async def _pick_a_http_server( ) -> DecodedURL: """Pick the first server we successfully send a request to.""" while True: - result = await defer.DeferredList([ - request(reactor, nurl) for nurl in nurls - ], consumeErrors=True, fireOnOneCallback=True) - # Apparently DeferredList is an awful awful API. If everything fails, - # you get back a list of (False, Failure), if it succeeds, you get a - # tuple of (value, index). - if isinstance(result, list): - await deferLater(reactor, 1, lambda: None) + result : defer.Deferred[Union[DecodedURL, None]] = defer.Deferred() + + def succeeded(nurl: DecodedURL, result=result): + # Only need the first successful NURL: + if result.called: + return + result.callback(nurl) + + def failed(failure, failures=[], result=result): + log.err(failure) + failures.append(None) + if len(failures) == len(nurls): + # All our potential NURLs failed... + result.callback(None) + + for index, nurl in enumerate(nurls): + request(reactor, nurl).addCallback( + lambda _, nurl=nurl: nurl).addCallbacks(succeeded, failed) + + first_nurl = await result + if first_nurl is None: + # Failed to connect to any of the NURLs: + await deferLater(reactor, 1, lambda: None) else: - assert isinstance(result, tuple) - _, index = result - return nurls[index] + return first_nurl @implementer(IServer) From f41f4a5e0cc1fde83e7f184b6dd9c08775994d4b Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 23 Feb 2023 10:10:25 -0500 Subject: [PATCH 1479/2309] Correct type. --- src/allmydata/storage_client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 0d2443d64..ac2a3cf5e 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -938,7 +938,7 @@ class NativeStorageServer(service.MultiService): async def _pick_a_http_server( reactor, nurls: list[DecodedURL], - request: Callable[[DecodedURL, Any], defer.Deferred[Any]] + request: Callable[[Any, DecodedURL], defer.Deferred[Any]] ) -> DecodedURL: """Pick the first server we successfully send a request to.""" while True: @@ -951,7 +951,7 @@ async def _pick_a_http_server( result.callback(nurl) def failed(failure, failures=[], result=result): - log.err(failure) + log.err(failure, "Failed to connect to NURL") failures.append(None) if len(failures) == len(nurls): # All our potential NURLs failed... From 99de5fa54c95331086d4ad92a6cc46b1823ccec7 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 23 Feb 2023 10:12:25 -0500 Subject: [PATCH 1480/2309] Link to follow-up ticket. --- src/allmydata/storage_client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index ac2a3cf5e..420be8461 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -1068,9 +1068,9 @@ class HTTPNativeStorageServer(service.MultiService): @async_to_deferred async def start_connecting(self, trigger_cb): - # TODO file a bug: The problem with this scheme is that while picking + # TODO The problem with this scheme is that while picking # the HTTP server to talk to, we don't have connection status - # updates... + # updates... https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3978 def request(reactor, nurl: DecodedURL): return StorageClientGeneral( StorageClient.from_nurl(nurl, reactor) From b6e20dfa812be98ffb569aa023f1081cf83938af Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 23 Feb 2023 13:26:30 -0500 Subject: [PATCH 1481/2309] Slightly longer timeout. --- src/allmydata/storage_client.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 420be8461..93c890c56 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -963,8 +963,9 @@ async def _pick_a_http_server( first_nurl = await result if first_nurl is None: - # Failed to connect to any of the NURLs: - await deferLater(reactor, 1, lambda: None) + # Failed to connect to any of the NURLs, try again in a few + # seconds: + await deferLater(reactor, 5, lambda: None) else: return first_nurl From b95a1d2b79a465a70a65e08511221cfe80bc9dc0 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 23 Feb 2023 13:27:41 -0500 Subject: [PATCH 1482/2309] Nicer type annotations. --- src/allmydata/storage_client.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 93c890c56..c2468c679 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -33,7 +33,7 @@ Ported to Python 3. from __future__ import annotations from six import ensure_text -from typing import Union, Callable, Any +from typing import Union, Callable, Any, Optional import re, time, hashlib from os import urandom from configparser import NoSectionError @@ -942,7 +942,7 @@ async def _pick_a_http_server( ) -> DecodedURL: """Pick the first server we successfully send a request to.""" while True: - result : defer.Deferred[Union[DecodedURL, None]] = defer.Deferred() + result : defer.Deferred[Optional[DecodedURL]] = defer.Deferred() def succeeded(nurl: DecodedURL, result=result): # Only need the first successful NURL: @@ -1300,7 +1300,7 @@ class _HTTPBucketWriter(object): return self.finished -def _ignore_404(failure: Failure) -> Union[Failure, None]: +def _ignore_404(failure: Failure) -> Optional[Failure]: """ Useful for advise_corrupt_share(), since it swallows unknown share numbers in Foolscap. From a6a2eb1c93d479bd936836a125a0d6ce5f4575a3 Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 23 Feb 2023 15:37:46 -0700 Subject: [PATCH 1483/2309] export it too Co-authored-by: Jean-Paul Calderone --- src/allmydata/util/jsonbytes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/util/jsonbytes.py b/src/allmydata/util/jsonbytes.py index 1cc8c8ef0..4a1813275 100644 --- a/src/allmydata/util/jsonbytes.py +++ b/src/allmydata/util/jsonbytes.py @@ -116,4 +116,4 @@ loads = json.loads load = json.load -__all__ = ["dumps", "loads"] +__all__ = ["dumps", "loads", "load"] From 96e1e9ffac0f2b60d1a4db14761939ad96d9467e Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 23 Feb 2023 19:45:01 -0500 Subject: [PATCH 1484/2309] Move where choosing a NURL happens. --- src/allmydata/storage_client.py | 36 ++++++++++++++++++++------------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index c2468c679..53131c88a 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -1067,19 +1067,7 @@ class HTTPNativeStorageServer(service.MultiService): version = self.get_version() return _available_space_from_version(version) - @async_to_deferred - async def start_connecting(self, trigger_cb): - # TODO The problem with this scheme is that while picking - # the HTTP server to talk to, we don't have connection status - # updates... https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3978 - def request(reactor, nurl: DecodedURL): - return StorageClientGeneral( - StorageClient.from_nurl(nurl, reactor) - ).get_version() - nurl = await _pick_a_http_server(reactor, self._nurls, request) - self._istorage_server = _HTTPStorageServer.from_http_client( - StorageClient.from_nurl(nurl, reactor) - ) + def start_connecting(self, trigger_cb): self._lc = LoopingCall(self._connect) self._lc.start(1, True) @@ -1113,7 +1101,24 @@ class HTTPNativeStorageServer(service.MultiService): def try_to_connect(self): self._connect() - def _connect(self): + @async_to_deferred + async def _connect(self): + if self._istorage_server is None: + # We haven't selected a server yet, so let's do so. + + # TODO The problem with this scheme is that while picking + # the HTTP server to talk to, we don't have connection status + # updates... https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3978 + def request(reactor, nurl: DecodedURL): + return StorageClientGeneral( + StorageClient.from_nurl(nurl, reactor) + ).get_version() + + nurl = await _pick_a_http_server(reactor, self._nurls, request) + self._istorage_server = _HTTPStorageServer.from_http_client( + StorageClient.from_nurl(nurl, reactor) + ) + result = self._istorage_server.get_version() def remove_connecting_deferred(result): @@ -1127,6 +1132,9 @@ class HTTPNativeStorageServer(service.MultiService): self._failed_to_connect ) + # TODO Make sure LoopingCall waits for the above timeout for looping again: + #return self._connecting_deferred + def stopService(self): if self._connecting_deferred is not None: self._connecting_deferred.cancel() From e09d19463dfffc9f6b68ff99593fb96f2bdf5233 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 24 Feb 2023 09:53:28 -0500 Subject: [PATCH 1485/2309] Logging errors breaks some tests. --- src/allmydata/storage_client.py | 5 ++++- src/allmydata/test/test_storage_client.py | 1 - 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 53131c88a..a2726fe09 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -951,7 +951,10 @@ async def _pick_a_http_server( result.callback(nurl) def failed(failure, failures=[], result=result): - log.err(failure, "Failed to connect to NURL") + # Logging errors breaks a bunch of tests, and it's not a _bug_ to + # have a failed connection, it's often expected and transient. More + # of a warning, really? + log.msg("Failed to connect to NURL: {}".format(failure)) failures.append(None) if len(failures) == len(nurls): # All our potential NURLs failed... diff --git a/src/allmydata/test/test_storage_client.py b/src/allmydata/test/test_storage_client.py index 38ef8c1d3..d7420b62f 100644 --- a/src/allmydata/test/test_storage_client.py +++ b/src/allmydata/test/test_storage_client.py @@ -791,5 +791,4 @@ class PickHTTPServerTests(unittest.SynchronousTestCase): ], bad_url: [(0.1, RuntimeError()), (0.1, RuntimeError()), (0.1, RuntimeError())] }) - self.flushLoggedErrors(ZeroDivisionError, RuntimeError) self.assertEqual(self.successResultOf(d), eventually_good_url) From 2f6632ecb97927d881933998594b0535008a4604 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 24 Feb 2023 10:58:22 -0500 Subject: [PATCH 1486/2309] Improve type checking. --- src/allmydata/util/cputhreadpool.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/allmydata/util/cputhreadpool.py b/src/allmydata/util/cputhreadpool.py index a7407804a..225232e04 100644 --- a/src/allmydata/util/cputhreadpool.py +++ b/src/allmydata/util/cputhreadpool.py @@ -18,6 +18,7 @@ import os from typing import TypeVar, Callable, cast from functools import partial import threading +from typing_extensions import ParamSpec from twisted.python.threadpool import ThreadPool from twisted.internet.defer import Deferred @@ -30,25 +31,24 @@ if hasattr(threading, "_register_atexit"): # This is a private API present in Python 3.8 or later, specifically # designed for thread pool shutdown. Since it's private, it might go away # at any point, so if it doesn't exist we still have a solution. - threading._register_atexit(_CPU_THREAD_POOL.stop) + threading._register_atexit(_CPU_THREAD_POOL.stop) # type: ignore else: # Daemon threads allow shutdown to happen without any explicit stopping of # threads. There are some bugs in old Python versions related to daemon # threads (fixed in subsequent CPython patch releases), but Python's own # thread pools use daemon threads in those versions so we're no worse off. - _CPU_THREAD_POOL.threadFactory = partial( + _CPU_THREAD_POOL.threadFactory = partial( # type: ignore _CPU_THREAD_POOL.threadFactory, daemon=True ) _CPU_THREAD_POOL.start() -# Eventually type annotations should use PEP 612, but that requires Python -# 3.10. +P = ParamSpec("P") R = TypeVar("R") def defer_to_thread( - reactor: IReactorFromThreads, f: Callable[..., R], *args, **kwargs + reactor: IReactorFromThreads, f: Callable[P, R], *args: P.args, **kwargs: P.kwargs ) -> Deferred[R]: """Run the function in a thread, return the result as a ``Deferred``.""" # deferToThreadPool has no type annotations... From 5640b6b5e740f937e9da42d2bbad30a994d4491a Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 24 Feb 2023 10:59:26 -0500 Subject: [PATCH 1487/2309] Apparently tests can be async now. --- src/allmydata/test/test_util.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/allmydata/test/test_util.py b/src/allmydata/test/test_util.py index 310bfdfd2..336edf3e2 100644 --- a/src/allmydata/test/test_util.py +++ b/src/allmydata/test/test_util.py @@ -29,7 +29,6 @@ from allmydata.util import yamlutil from allmydata.util import rrefutil from allmydata.util.fileutil import EncryptedTemporaryFile from allmydata.util.cputhreadpool import defer_to_thread -from allmydata.util.deferredutil import async_to_deferred from allmydata.test.common_util import ReallyEqualMixin from .no_network import fireNow, LocalWrapper @@ -597,7 +596,6 @@ class RrefUtilTests(unittest.TestCase): class CPUThreadPool(unittest.TestCase): """Tests for cputhreadpool.""" - @async_to_deferred async def test_runs_in_thread(self): """The given function runs in a thread.""" def f(*args, **kwargs): From 3d0b17bc1c197a73b212b6b5eac0ad3b3ee43297 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 27 Feb 2023 11:37:18 -0500 Subject: [PATCH 1488/2309] Make cancellation more likely to happen. --- src/allmydata/storage_client.py | 88 ++++++++++++++--------- src/allmydata/test/test_storage_client.py | 28 ++++---- 2 files changed, 72 insertions(+), 44 deletions(-) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index a2726fe09..549062d63 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -47,7 +47,7 @@ from zope.interface import ( ) from twisted.python.failure import Failure from twisted.web import http -from twisted.internet.task import LoopingCall, deferLater +from twisted.internet.task import LoopingCall from twisted.internet import defer, reactor from twisted.application import service from twisted.plugin import ( @@ -935,42 +935,52 @@ class NativeStorageServer(service.MultiService): self._reconnector.reset() -async def _pick_a_http_server( +def _pick_a_http_server( reactor, nurls: list[DecodedURL], request: Callable[[Any, DecodedURL], defer.Deferred[Any]] -) -> DecodedURL: - """Pick the first server we successfully send a request to.""" - while True: - result : defer.Deferred[Optional[DecodedURL]] = defer.Deferred() +) -> defer.Deferred[Optional[DecodedURL]]: + """Pick the first server we successfully send a request to. - def succeeded(nurl: DecodedURL, result=result): - # Only need the first successful NURL: - if result.called: - return - result.callback(nurl) + Fires with ``None`` if no server was found, or with the ``DecodedURL`` of + the first successfully-connected server. + """ - def failed(failure, failures=[], result=result): - # Logging errors breaks a bunch of tests, and it's not a _bug_ to - # have a failed connection, it's often expected and transient. More - # of a warning, really? - log.msg("Failed to connect to NURL: {}".format(failure)) - failures.append(None) - if len(failures) == len(nurls): - # All our potential NURLs failed... - result.callback(None) + to_cancel : list[defer.Deferred] = [] - for index, nurl in enumerate(nurls): - request(reactor, nurl).addCallback( - lambda _, nurl=nurl: nurl).addCallbacks(succeeded, failed) + def cancel(result: Optional[defer.Deferred]): + for d in to_cancel: + if not d.called: + d.cancel() + if result is not None: + result.errback(defer.CancelledError()) - first_nurl = await result - if first_nurl is None: - # Failed to connect to any of the NURLs, try again in a few - # seconds: - await deferLater(reactor, 5, lambda: None) - else: - return first_nurl + result : defer.Deferred[Optional[DecodedURL]] = defer.Deferred(canceller=cancel) + + def succeeded(nurl: DecodedURL, result=result): + # Only need the first successful NURL: + if result.called: + return + result.callback(nurl) + # No point in continuing other requests if we're connected: + cancel(None) + + def failed(failure, failures=[], result=result): + # Logging errors breaks a bunch of tests, and it's not a _bug_ to + # have a failed connection, it's often expected and transient. More + # of a warning, really? + log.msg("Failed to connect to NURL: {}".format(failure)) + failures.append(None) + if len(failures) == len(nurls): + # All our potential NURLs failed... + result.callback(None) + + for index, nurl in enumerate(nurls): + d = request(reactor, nurl) + to_cancel.append(d) + d.addCallback(lambda _, nurl=nurl: nurl).addCallbacks(succeeded, failed) + + return result @implementer(IServer) @@ -1117,8 +1127,22 @@ class HTTPNativeStorageServer(service.MultiService): StorageClient.from_nurl(nurl, reactor) ).get_version() - nurl = await _pick_a_http_server(reactor, self._nurls, request) - self._istorage_server = _HTTPStorageServer.from_http_client( + # LoopingCall.stop() doesn't cancel Deferreds, unfortunately: + # https://github.com/twisted/twisted/issues/11814 Thus we want + # store the Deferred so it gets cancelled. + picking = _pick_a_http_server(reactor, self._nurls, request) + self._connecting_deferred = picking + try: + nurl = await picking + finally: + self._connecting_deferred = None + + if nurl is None: + # We failed to find a server to connect to. Perhaps the next + # iteration of the loop will succeed. + return + else: + self._istorage_server = _HTTPStorageServer.from_http_client( StorageClient.from_nurl(nurl, reactor) ) diff --git a/src/allmydata/test/test_storage_client.py b/src/allmydata/test/test_storage_client.py index d7420b62f..a51e44a82 100644 --- a/src/allmydata/test/test_storage_client.py +++ b/src/allmydata/test/test_storage_client.py @@ -83,7 +83,6 @@ from allmydata.webish import ( WebishServer, ) from allmydata.util import base32, yamlutil -from allmydata.util.deferredutil import async_to_deferred from allmydata.storage_client import ( IFoolscapStorageServer, NativeStorageServer, @@ -741,7 +740,7 @@ storage: class PickHTTPServerTests(unittest.SynchronousTestCase): """Tests for ``_pick_a_http_server``.""" - def loop_until_result(self, url_to_results: dict[DecodedURL, list[tuple[float, Union[Exception, Any]]]]) -> Deferred[DecodedURL]: + def loop_until_result(self, url_to_results: dict[DecodedURL, list[tuple[float, Union[Exception, Any]]]]) -> tuple[int, DecodedURL]: """ Given mapping of URLs to list of (delay, result), return the URL of the first selected server. @@ -759,12 +758,15 @@ class PickHTTPServerTests(unittest.SynchronousTestCase): reactor.callLater(delay, add_result_value) return result - d = async_to_deferred(_pick_a_http_server)( - clock, list(url_to_results.keys()), request - ) - for i in range(1000): - clock.advance(0.1) - return d + iterations = 0 + while True: + iterations += 1 + d = _pick_a_http_server(clock, list(url_to_results.keys()), request) + for i in range(100): + clock.advance(0.1) + result = self.successResultOf(d) + if result is not None: + return iterations, result def test_first_successful_connect_is_picked(self): """ @@ -772,11 +774,12 @@ class PickHTTPServerTests(unittest.SynchronousTestCase): """ earliest_url = DecodedURL.from_text("http://a") latest_url = DecodedURL.from_text("http://b") - d = self.loop_until_result({ + iterations, result = self.loop_until_result({ latest_url: [(2, None)], earliest_url: [(1, None)] }) - self.assertEqual(self.successResultOf(d), earliest_url) + self.assertEqual(iterations, 1) + self.assertEqual(result, earliest_url) def test_failures_are_retried(self): """ @@ -785,10 +788,11 @@ class PickHTTPServerTests(unittest.SynchronousTestCase): """ eventually_good_url = DecodedURL.from_text("http://good") bad_url = DecodedURL.from_text("http://bad") - d = self.loop_until_result({ + iterations, result = self.loop_until_result({ eventually_good_url: [ (1, ZeroDivisionError()), (0.1, ZeroDivisionError()), (1, None) ], bad_url: [(0.1, RuntimeError()), (0.1, RuntimeError()), (0.1, RuntimeError())] }) - self.assertEqual(self.successResultOf(d), eventually_good_url) + self.assertEqual(iterations, 3) + self.assertEqual(result, eventually_good_url) From af51b022284f2b7284a806de7beb2223f5ad9961 Mon Sep 17 00:00:00 2001 From: dlee Date: Mon, 27 Feb 2023 15:05:52 -0600 Subject: [PATCH 1489/2309] Revert wait_for_wormhole function return type back to Awaitable for forward compatibility when we move to async def --- src/allmydata/test/cli/wormholetesting.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/allmydata/test/cli/wormholetesting.py b/src/allmydata/test/cli/wormholetesting.py index 9ce199545..b30b92fe1 100644 --- a/src/allmydata/test/cli/wormholetesting.py +++ b/src/allmydata/test/cli/wormholetesting.py @@ -38,7 +38,7 @@ from itertools import count from sys import stderr from attrs import frozen, define, field, Factory -from twisted.internet.defer import Deferred, DeferredQueue, succeed +from twisted.internet.defer import Deferred, DeferredQueue, succeed, Awaitable from wormhole._interfaces import IWormhole from wormhole.wormhole import create from zope.interface import implementer @@ -189,7 +189,7 @@ class _WormholeApp(object): return code - def wait_for_wormhole(self, code: WormholeCode) -> Deferred[_MemoryWormhole]: + def wait_for_wormhole(self, code: WormholeCode) -> Awaitable[_MemoryWormhole]: """ Return a ``Deferred`` which fires with the next wormhole to be associated with the given code. This is used to let the first end of a wormhole @@ -281,8 +281,8 @@ class _MemoryWormhole(object): ) d = self._view.wormhole_by_code(self._code, exclude=self) - def got_wormhole(wormhole: _MemoryWormhole) -> Deferred[_MemoryWormhole]: - msg: Deferred[_MemoryWormhole] = wormhole._payload.get() + def got_wormhole(wormhole: _MemoryWormhole) -> Deferred[WormholeMessage]: + msg: Deferred[WormholeMessage] = wormhole._payload.get() return msg d.addCallback(got_wormhole) From 582876197a724dd8c24b06160345d122832e03b6 Mon Sep 17 00:00:00 2001 From: dlee Date: Mon, 27 Feb 2023 15:14:58 -0600 Subject: [PATCH 1490/2309] Added default check to verify to ensure strictness --- src/allmydata/test/cli/wormholetesting.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/allmydata/test/cli/wormholetesting.py b/src/allmydata/test/cli/wormholetesting.py index b30b92fe1..d4e53a342 100644 --- a/src/allmydata/test/cli/wormholetesting.py +++ b/src/allmydata/test/cli/wormholetesting.py @@ -151,6 +151,7 @@ def _verify() -> None: assert a.args == b.args assert a.varargs == b.varargs assert a.kwonlydefaults == b.kwonlydefaults + assert a.defaults == b.defaults _verify() From 1587a71bbafcafdbbf83736f017e5f3ca8c84dd0 Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 27 Feb 2023 17:26:06 -0700 Subject: [PATCH 1491/2309] spelling Co-authored-by: Jean-Paul Calderone --- src/allmydata/util/configutil.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/util/configutil.py b/src/allmydata/util/configutil.py index b48e2c034..9e18e3716 100644 --- a/src/allmydata/util/configutil.py +++ b/src/allmydata/util/configutil.py @@ -167,7 +167,7 @@ class ValidConfiguration(object): def is_valid_item(self, section_name, item_name): """ - :return: True if the given section name, ite name pair is valid, False + :return: True if the given section name, item_name pair is valid, False otherwise. """ valid_items = self._static_valid_sections.get(section_name, ()) From 9f63441af65f030cabbd99c9b88eb65e1abcb990 Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 27 Feb 2023 17:31:39 -0700 Subject: [PATCH 1492/2309] types Co-authored-by: Jean-Paul Calderone --- src/allmydata/scripts/admin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/scripts/admin.py b/src/allmydata/scripts/admin.py index eceb9a55a..a59892617 100644 --- a/src/allmydata/scripts/admin.py +++ b/src/allmydata/scripts/admin.py @@ -113,7 +113,7 @@ class AddGridManagerCertOptions(BaseOptions): def getSynopsis(self): return "Usage: tahoe [global-options] admin add-grid-manager-cert [options]" - def postOptions(self): + def postOptions(self) -> None: if self['name'] is None: raise usage.UsageError( "Must provide --name option" From d55a4a1e651fea8867a0cf3955251d81ff226f3e Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 27 Feb 2023 17:32:16 -0700 Subject: [PATCH 1493/2309] whitespace Co-authored-by: Jean-Paul Calderone --- src/allmydata/scripts/admin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/scripts/admin.py b/src/allmydata/scripts/admin.py index a59892617..41921e2ee 100644 --- a/src/allmydata/scripts/admin.py +++ b/src/allmydata/scripts/admin.py @@ -123,7 +123,7 @@ class AddGridManagerCertOptions(BaseOptions): "Must provide --filename option" ) - data : Union [bytes, str] + data: str if self['filename'] == '-': print("reading certificate from stdin", file=self.parent.parent.stderr) data = self.parent.parent.stdin.read() From aed5061a96876bc2efc943c5f5ad60217e1ad9b1 Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 27 Feb 2023 17:33:53 -0700 Subject: [PATCH 1494/2309] wording Co-authored-by: Jean-Paul Calderone --- docs/managed-grid.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/managed-grid.rst b/docs/managed-grid.rst index aac2bf20a..1c5acaa20 100644 --- a/docs/managed-grid.rst +++ b/docs/managed-grid.rst @@ -224,7 +224,8 @@ Enrolling a Client: Config -------------------------- You may instruct a Tahoe client to use only storage servers from given -Grid Managers. If there are no such keys, any servers are used. If +Grid Managers. If there are no such keys, any servers are used +(but see https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3979). If there are one or more keys, the client will only upload to a storage server that has a valid certificate (from any of the keys). From 7c3f6cb4c7fcba278dea93d62a3ddea381835a7f Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 28 Feb 2023 07:55:43 -0500 Subject: [PATCH 1495/2309] Fix inverted assertion --- src/allmydata/test/test_storage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index 2234b5af2..434b42c0d 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -2870,7 +2870,7 @@ class MDMFProxies(AsyncTestCase, ShouldFailMixin): mr = MDMFSlotReadProxy(self.storage_server, b"si1", 0) d = mr.is_sdmf() d.addCallback(lambda issdmf: - self.assertFalse(issdmf)) + self.assertTrue(issdmf)) return d From b28ac6118b3e3329d627c23086db90a674803b3f Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 28 Feb 2023 10:43:49 -0700 Subject: [PATCH 1496/2309] different way to say 'all items okay' --- src/allmydata/client.py | 5 ++--- src/allmydata/util/configutil.py | 5 +---- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/src/allmydata/client.py b/src/allmydata/client.py index b98d7e0d8..d9521d999 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -76,7 +76,8 @@ def _is_valid_section(section_name): """ return ( section_name.startswith("storageserver.plugins.") or - section_name.startswith("storageclient.plugins.") + section_name.startswith("storageclient.plugins.") or + section_name in ("grid_managers", "grid_manager_certificates") ) @@ -93,8 +94,6 @@ _client_config = configutil.ValidConfiguration( "shares.total", "storage.plugins", ), - "grid_managers": None, # means "any options valid" - "grid_manager_certificates": None, "storage": ( "debug_discard", "enabled", diff --git a/src/allmydata/util/configutil.py b/src/allmydata/util/configutil.py index 9e18e3716..a82cb04a4 100644 --- a/src/allmydata/util/configutil.py +++ b/src/allmydata/util/configutil.py @@ -170,11 +170,8 @@ class ValidConfiguration(object): :return: True if the given section name, item_name pair is valid, False otherwise. """ - valid_items = self._static_valid_sections.get(section_name, ()) - if valid_items is None: - return True return ( - item_name in valid_items or + item_name in self._static_valid_sections.get(section_name, ()) or self._is_valid_item(section_name, item_name) ) From 5672a28350a453628c69b4d892e4079982756691 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 28 Feb 2023 10:43:55 -0700 Subject: [PATCH 1497/2309] more-specific error --- src/allmydata/util/configutil.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/util/configutil.py b/src/allmydata/util/configutil.py index a82cb04a4..74b0c714a 100644 --- a/src/allmydata/util/configutil.py +++ b/src/allmydata/util/configutil.py @@ -84,7 +84,7 @@ def write_config(tahoe_cfg, config): if platform.isWindows(): try: tahoe_cfg.remove() - except OSError: + except FileNotFoundError: pass tmp.moveTo(tahoe_cfg) From 8e7f2cd3ea79e0e59aa5c4b40e2ffa2745bd4433 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 28 Feb 2023 10:47:17 -0700 Subject: [PATCH 1498/2309] unused --- src/allmydata/scripts/admin.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/allmydata/scripts/admin.py b/src/allmydata/scripts/admin.py index 41921e2ee..579505399 100644 --- a/src/allmydata/scripts/admin.py +++ b/src/allmydata/scripts/admin.py @@ -12,8 +12,6 @@ if PY2: from six import ensure_binary -from typing import Union - from twisted.python import usage from twisted.python.filepath import ( FilePath, From 450eed78688142b78151725a10da4d569f53106f Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 8 Mar 2023 11:31:58 -0500 Subject: [PATCH 1499/2309] Test writing at an offset. --- src/allmydata/test/test_system.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/src/allmydata/test/test_system.py b/src/allmydata/test/test_system.py index 10a64c1fe..e68f367cd 100644 --- a/src/allmydata/test/test_system.py +++ b/src/allmydata/test/test_system.py @@ -33,6 +33,7 @@ from allmydata.util import log, base32 from allmydata.util.encodingutil import quote_output, unicode_to_argv from allmydata.util.fileutil import abspath_expanduser_unicode from allmydata.util.consumer import MemoryConsumer, download_to_data +from allmydata.util.deferredutil import async_to_deferred from allmydata.interfaces import IDirectoryNode, IFileNode, \ NoSuchChildError, NoSharesError, SDMF_VERSION, MDMF_VERSION from allmydata.monitor import Monitor @@ -657,7 +658,23 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase): self.failUnlessEqual(res, NEWERDATA) d.addCallback(_check_download_5) - def _corrupt_shares(res): + @async_to_deferred + async def _check_write_at_offset(newnode): + log.msg("writing at offset") + start = b"abcdef" + expected = b"abXYef" + uri = self._mutable_node_1.get_uri() + newnode = self.clients[0].create_node_from_uri(uri) + await newnode.overwrite(MutableData(start)) + version = await newnode.get_mutable_version() + await version.update(MutableData(b"XY"), 2) + result = await newnode.download_best_version() + self.assertEqual(result, expected) + # Revert to previous version + await newnode.overwrite(MutableData(NEWERDATA)) + d.addCallback(_check_write_at_offset) + + def _corrupt_shares(_res): # run around and flip bits in all but k of the shares, to test # the hash checks shares = self._find_all_shares(self.basedir) From 5dc108dfe87b53966bc252afbf1a7c6c77e9c7df Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 8 Mar 2023 11:38:31 -0500 Subject: [PATCH 1500/2309] Test large immutable upload and download. --- integration/test_get_put.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/integration/test_get_put.py b/integration/test_get_put.py index 76c6ee600..65020429e 100644 --- a/integration/test_get_put.py +++ b/integration/test_get_put.py @@ -53,17 +53,20 @@ def test_put_from_stdin(alice, get_put_alias, tmpdir): def test_get_to_stdout(alice, get_put_alias, tmpdir): """ It's possible to upload a file, and then download it to stdout. + + We test with large file, this time. """ tempfile = tmpdir.join("file") + large_data = DATA * 1_000_000 with tempfile.open("wb") as f: - f.write(DATA) + f.write(large_data) cli(alice, "put", str(tempfile), "getput:tostdout") p = Popen( ["tahoe", "--node-directory", alice.node_dir, "get", "getput:tostdout", "-"], stdout=PIPE ) - assert p.stdout.read() == DATA + assert p.stdout.read() == large_data assert p.wait() == 0 From 7bdfed6434c4129bfe766a72b8d6b38fe3d1349e Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 8 Mar 2023 11:55:30 -0500 Subject: [PATCH 1501/2309] News fragment. --- newsfragments/3959.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3959.minor diff --git a/newsfragments/3959.minor b/newsfragments/3959.minor new file mode 100644 index 000000000..e69de29bb From 9663db522c2380f162bf6965b6700ee4d6a5f2a1 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 8 Mar 2023 14:03:32 -0500 Subject: [PATCH 1502/2309] Make the client respect the force_foolscap flag, and default to Foolscap-only for now. --- newsfragments/3936.minor | 0 src/allmydata/storage_client.py | 4 +++- 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 newsfragments/3936.minor diff --git a/newsfragments/3936.minor b/newsfragments/3936.minor new file mode 100644 index 000000000..e69de29bb diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 6fab1707b..fa0d2de49 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -299,7 +299,9 @@ class StorageFarmBroker(service.MultiService): "pub-{}".format(str(server_id, "ascii")), # server_id is v0- not pub-v0-key .. for reasons? ) - if len(server["ann"].get(ANONYMOUS_STORAGE_NURLS, [])) > 0: + if not self.node_config.get_config( + "storage", "force_foolscap", default=True, boolean=True, + ) and len(server["ann"].get(ANONYMOUS_STORAGE_NURLS, [])) > 0: s = HTTPNativeStorageServer( server_id, server["ann"], From e9c3a227a17f3c23efbc9ac8ee0907101c0e8f62 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 8 Mar 2023 14:17:25 -0500 Subject: [PATCH 1503/2309] File follow-up ticket. --- src/allmydata/storage_client.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 388f8b4b8..686ee8e59 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -1248,7 +1248,8 @@ class HTTPNativeStorageServer(service.MultiService): ) # TODO Make sure LoopingCall waits for the above timeout for looping again: - #return self._connecting_deferred + # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3981 + #return self._connecting_deferred or maye await it def stopService(self): if self._connecting_deferred is not None: From 75da037d673a3db8db50a9472758f89960591f2d Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 8 Mar 2023 14:25:04 -0500 Subject: [PATCH 1504/2309] Add race() implementation from https://github.com/twisted/twisted/pull/11818 --- src/allmydata/test/test_deferredutil.py | 160 ++++++++++++++++++++++-- src/allmydata/util/deferredutil.py | 100 ++++++++++++++- 2 files changed, 247 insertions(+), 13 deletions(-) diff --git a/src/allmydata/test/test_deferredutil.py b/src/allmydata/test/test_deferredutil.py index a37dfdd6f..47121b4cb 100644 --- a/src/allmydata/test/test_deferredutil.py +++ b/src/allmydata/test/test_deferredutil.py @@ -1,23 +1,16 @@ """ Tests for allmydata.util.deferredutil. - -Ported to Python 3. """ -from __future__ import unicode_literals -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from future.utils import PY2 -if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - from twisted.trial import unittest from twisted.internet import defer, reactor +from twisted.internet.defer import Deferred from twisted.python.failure import Failure +from hypothesis.strategies import integers +from hypothesis import given from allmydata.util import deferredutil +from allmydata.util.deferredutil import race, MultiFailure class DeferredUtilTests(unittest.TestCase, deferredutil.WaitForDelayedCallsMixin): @@ -157,3 +150,148 @@ class AsyncToDeferred(unittest.TestCase): result = f(1, 0) self.assertIsInstance(self.failureResultOf(result).value, ZeroDivisionError) + + + +def _setupRaceState(numDeferreds: int) -> tuple[list[int], list[Deferred[object]]]: + """ + Create a list of Deferreds and a corresponding list of integers + tracking how many times each Deferred has been cancelled. Without + additional steps the Deferreds will never fire. + """ + cancelledState = [0] * numDeferreds + + ds: list[Deferred[object]] = [] + for n in range(numDeferreds): + + def cancel(d: Deferred, n: int = n) -> None: + cancelledState[n] += 1 + + ds.append(Deferred(canceller=cancel)) + + return cancelledState, ds + + +class RaceTests(unittest.SynchronousTestCase): + """ + Tests for L{race}. + """ + + @given( + beforeWinner=integers(min_value=0, max_value=3), + afterWinner=integers(min_value=0, max_value=3), + ) + def test_success(self, beforeWinner: int, afterWinner: int) -> None: + """ + When one of the L{Deferred}s passed to L{race} fires successfully, + the L{Deferred} return by L{race} fires with the index of that + L{Deferred} and its result and cancels the rest of the L{Deferred}s. + @param beforeWinner: A randomly selected number of Deferreds to + appear before the "winning" Deferred in the list passed in. + @param beforeWinner: A randomly selected number of Deferreds to + appear after the "winning" Deferred in the list passed in. + """ + cancelledState, ds = _setupRaceState(beforeWinner + 1 + afterWinner) + + raceResult = race(ds) + expected = object() + ds[beforeWinner].callback(expected) + + # The result should be the index and result of the only Deferred that + # fired. + self.assertEqual( + self.successResultOf(raceResult), + (beforeWinner, expected), + ) + # All Deferreds except the winner should have been cancelled once. + expectedCancelledState = [1] * beforeWinner + [0] + [1] * afterWinner + self.assertEqual( + cancelledState, + expectedCancelledState, + ) + + @given( + beforeWinner=integers(min_value=0, max_value=3), + afterWinner=integers(min_value=0, max_value=3), + ) + def test_failure(self, beforeWinner: int, afterWinner: int) -> None: + """ + When all of the L{Deferred}s passed to L{race} fire with failures, + the L{Deferred} return by L{race} fires with L{MultiFailure} wrapping + all of their failures. + @param beforeWinner: A randomly selected number of Deferreds to + appear before the "winning" Deferred in the list passed in. + @param beforeWinner: A randomly selected number of Deferreds to + appear after the "winning" Deferred in the list passed in. + """ + cancelledState, ds = _setupRaceState(beforeWinner + 1 + afterWinner) + + failure = Failure(Exception("The test demands failures.")) + raceResult = race(ds) + for d in ds: + d.errback(failure) + + actualFailure = self.failureResultOf(raceResult, MultiFailure) + self.assertEqual( + actualFailure.value.failures, + [failure] * len(ds), + ) + self.assertEqual( + cancelledState, + [0] * len(ds), + ) + + @given( + beforeWinner=integers(min_value=0, max_value=3), + afterWinner=integers(min_value=0, max_value=3), + ) + def test_resultAfterCancel(self, beforeWinner: int, afterWinner: int) -> None: + """ + If one of the Deferreds fires after it was cancelled its result + goes nowhere. In particular, it does not cause any errors to be + logged. + """ + # Ensure we have a Deferred to win and at least one other Deferred + # that can ignore cancellation. + ds: list[Deferred[None]] = [ + Deferred() for n in range(beforeWinner + 2 + afterWinner) + ] + + raceResult = race(ds) + ds[beforeWinner].callback(None) + ds[beforeWinner + 1].callback(None) + + self.successResultOf(raceResult) + self.assertEqual(len(self.flushLoggedErrors()), 0) + + def test_resultFromCancel(self) -> None: + """ + If one of the input Deferreds has a cancel function that fires it + with success, nothing bad happens. + """ + winner: Deferred[object] = Deferred() + ds: list[Deferred[object]] = [ + winner, + Deferred(canceller=lambda d: d.callback(object())), + ] + expected = object() + raceResult = race(ds) + winner.callback(expected) + + self.assertEqual(self.successResultOf(raceResult), (0, expected)) + + @given( + numDeferreds=integers(min_value=1, max_value=3), + ) + def test_cancel(self, numDeferreds: int) -> None: + """ + If the result of L{race} is cancelled then all of the L{Deferred}s + passed in are cancelled. + """ + cancelledState, ds = _setupRaceState(numDeferreds) + + raceResult = race(ds) + raceResult.cancel() + + self.assertEqual(cancelledState, [1] * numDeferreds) + self.failureResultOf(raceResult, MultiFailure) diff --git a/src/allmydata/util/deferredutil.py b/src/allmydata/util/deferredutil.py index 782663e8b..83de411ce 100644 --- a/src/allmydata/util/deferredutil.py +++ b/src/allmydata/util/deferredutil.py @@ -1,15 +1,18 @@ """ Utilities for working with Twisted Deferreds. - -Ported to Python 3. """ +from __future__ import annotations + import time from functools import wraps from typing import ( Callable, Any, + Sequence, + TypeVar, + Optional, ) from foolscap.api import eventually @@ -17,6 +20,7 @@ from eliot.twisted import ( inline_callbacks, ) from twisted.internet import defer, reactor, error +from twisted.internet.defer import Deferred from twisted.python.failure import Failure from allmydata.util import log @@ -234,3 +238,95 @@ def async_to_deferred(f): return defer.Deferred.fromCoroutine(f(*args, **kwargs)) return not_async + + +class MultiFailure(Exception): + """ + More than one failure occurred. + """ + + def __init__(self, failures: Sequence[Failure]) -> None: + super(MultiFailure, self).__init__() + self.failures = failures + + +_T = TypeVar("_T") + +# Eventually this should be in Twisted upstream: +# https://github.com/twisted/twisted/pull/11818 +def race(ds: Sequence[Deferred[_T]]) -> Deferred[tuple[int, _T]]: + """ + Select the first available result from the sequence of Deferreds and + cancel the rest. + @return: A cancellable L{Deferred} that fires with the index and output of + the element of C{ds} to have a success result first, or that fires + with L{MultiFailure} holding a list of their failures if they all + fail. + """ + # Keep track of the Deferred for the action which completed first. When + # it completes, all of the other Deferreds will get cancelled but this one + # shouldn't be. Even though it "completed" it isn't really done - the + # caller will still be using it for something. If we cancelled it, + # cancellation could propagate down to them. + winner: Optional[Deferred] = None + + # The cancellation function for the Deferred this function returns. + def cancel(result: Deferred) -> None: + # If it is cancelled then we cancel all of the Deferreds for the + # individual actions because there is no longer the possibility of + # delivering any of their results anywhere. We don't have to fire + # `result` because the Deferred will do that for us. + for d in to_cancel: + d.cancel() + + # The Deferred that this function will return. It will fire with the + # index and output of the action that completes first, or None if all of + # the actions fail. If it is cancelled, all of the actions will be + # cancelled. + final_result: Deferred[tuple[int, _T]] = Deferred(canceller=cancel) + + # A callback for an individual action. + def succeeded(this_output: _T, this_index: int) -> None: + # If it is the first action to succeed then it becomes the "winner", + # its index/output become the externally visible result, and the rest + # of the action Deferreds get cancelled. If it is not the first + # action to succeed (because some action did not support + # cancellation), just ignore the result. It is uncommon for this + # callback to be entered twice. The only way it can happen is if one + # of the input Deferreds has a cancellation function that fires the + # Deferred with a success result. + nonlocal winner + if winner is None: + # This is the first success. Act on it. + winner = to_cancel[this_index] + + # Cancel the rest. + for d in to_cancel: + if d is not winner: + d.cancel() + + # Fire our Deferred + final_result.callback((this_index, this_output)) + + # Keep track of how many actions have failed. If they all fail we need to + # deliver failure notification on our externally visible result. + failure_state = [] + + def failed(failure: Failure, this_index: int) -> None: + failure_state.append((this_index, failure)) + if len(failure_state) == len(to_cancel): + # Every operation failed. + failure_state.sort() + failures = [f for (ignored, f) in failure_state] + final_result.errback(MultiFailure(failures)) + + # Copy the sequence of Deferreds so we know it doesn't get mutated out + # from under us. + to_cancel = list(ds) + for index, d in enumerate(ds): + # Propagate the position of this action as well as the argument to f + # to the success callback so we can cancel the right Deferreds and + # propagate the result outwards. + d.addCallbacks(succeeded, failed, callbackArgs=(index,), errbackArgs=(index,)) + + return final_result From 0093edcd938b6f18692094a2442bcf236f42da39 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 8 Mar 2023 14:36:37 -0500 Subject: [PATCH 1505/2309] Refactor to use race(). --- src/allmydata/storage_client.py | 41 +++++++++------------------------ 1 file changed, 11 insertions(+), 30 deletions(-) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 686ee8e59..169d34e32 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -82,7 +82,7 @@ from allmydata.util.observer import ObserverList from allmydata.util.rrefutil import add_version_to_remote_reference from allmydata.util.hashutil import permute_server_hash from allmydata.util.dictutil import BytesKeyDict, UnicodeKeyDict -from allmydata.util.deferredutil import async_to_deferred +from allmydata.util.deferredutil import async_to_deferred, race from allmydata.storage.http_client import ( StorageClient, StorageClientImmutables, StorageClientGeneral, ClientException as HTTPClientException, StorageClientMutables, @@ -1017,42 +1017,23 @@ def _pick_a_http_server( Fires with ``None`` if no server was found, or with the ``DecodedURL`` of the first successfully-connected server. """ + queries = race([ + request(reactor, nurl).addCallback(lambda _, nurl=nurl: nurl) + for nurl in nurls + ]) - to_cancel : list[defer.Deferred] = [] - - def cancel(result: Optional[defer.Deferred]): - for d in to_cancel: - if not d.called: - d.cancel() - if result is not None: - result.errback(defer.CancelledError()) - - result : defer.Deferred[Optional[DecodedURL]] = defer.Deferred(canceller=cancel) - - def succeeded(nurl: DecodedURL, result=result): - # Only need the first successful NURL: - if result.called: - return - result.callback(nurl) - # No point in continuing other requests if we're connected: - cancel(None) - - def failed(failure, failures=[], result=result): + def failed(failure: Failure): # Logging errors breaks a bunch of tests, and it's not a _bug_ to # have a failed connection, it's often expected and transient. More # of a warning, really? log.msg("Failed to connect to NURL: {}".format(failure)) - failures.append(None) - if len(failures) == len(nurls): - # All our potential NURLs failed... - result.callback(None) + return None - for index, nurl in enumerate(nurls): - d = request(reactor, nurl) - to_cancel.append(d) - d.addCallback(lambda _, nurl=nurl: nurl).addCallbacks(succeeded, failed) + def succeeded(result: tuple[int, DecodedURL]): + _, nurl = result + return nurl - return result + return queries.addCallbacks(succeeded, failed) @implementer(IServer) From 4db65ea9369b80d92b4a6d4c0c8ae9c7a7b8916c Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 8 Mar 2023 14:53:43 -0500 Subject: [PATCH 1506/2309] Make tests test _pick_a_http_server more directly. --- src/allmydata/test/test_storage_client.py | 46 ++++++++++------------- 1 file changed, 19 insertions(+), 27 deletions(-) diff --git a/src/allmydata/test/test_storage_client.py b/src/allmydata/test/test_storage_client.py index a51e44a82..c919440d8 100644 --- a/src/allmydata/test/test_storage_client.py +++ b/src/allmydata/test/test_storage_client.py @@ -8,7 +8,7 @@ from json import ( loads, ) import hashlib -from typing import Union, Any +from typing import Union, Any, Optional from hyperlink import DecodedURL from fixtures import ( @@ -740,15 +740,15 @@ storage: class PickHTTPServerTests(unittest.SynchronousTestCase): """Tests for ``_pick_a_http_server``.""" - def loop_until_result(self, url_to_results: dict[DecodedURL, list[tuple[float, Union[Exception, Any]]]]) -> tuple[int, DecodedURL]: + def pick_result(self, url_to_results: dict[DecodedURL, tuple[float, Union[Exception, Any]]]) -> Optional[DecodedURL]: """ - Given mapping of URLs to list of (delay, result), return the URL of the - first selected server. + Given mapping of URLs to (delay, result), return the URL of the + first selected server, or None. """ clock = Clock() def request(reactor, url): - delay, value = url_to_results[url].pop(0) + delay, value = url_to_results[url] result = Deferred() def add_result_value(): if isinstance(value, Exception): @@ -758,15 +758,10 @@ class PickHTTPServerTests(unittest.SynchronousTestCase): reactor.callLater(delay, add_result_value) return result - iterations = 0 - while True: - iterations += 1 - d = _pick_a_http_server(clock, list(url_to_results.keys()), request) - for i in range(100): - clock.advance(0.1) - result = self.successResultOf(d) - if result is not None: - return iterations, result + d = _pick_a_http_server(clock, list(url_to_results.keys()), request) + for i in range(100): + clock.advance(0.1) + return self.successResultOf(d) def test_first_successful_connect_is_picked(self): """ @@ -774,25 +769,22 @@ class PickHTTPServerTests(unittest.SynchronousTestCase): """ earliest_url = DecodedURL.from_text("http://a") latest_url = DecodedURL.from_text("http://b") - iterations, result = self.loop_until_result({ - latest_url: [(2, None)], - earliest_url: [(1, None)] + bad_url = DecodedURL.from_text("http://bad") + result = self.pick_result({ + latest_url: (2, None), + earliest_url: (1, None), + bad_url: (0.5, RuntimeError()), }) - self.assertEqual(iterations, 1) self.assertEqual(result, earliest_url) def test_failures_are_retried(self): """ - If the initial requests all fail, ``_pick_a_http_server`` keeps trying - until success. + If the requests all fail, ``_pick_a_http_server`` returns ``None``. """ eventually_good_url = DecodedURL.from_text("http://good") bad_url = DecodedURL.from_text("http://bad") - iterations, result = self.loop_until_result({ - eventually_good_url: [ - (1, ZeroDivisionError()), (0.1, ZeroDivisionError()), (1, None) - ], - bad_url: [(0.1, RuntimeError()), (0.1, RuntimeError()), (0.1, RuntimeError())] + result = self.pick_result({ + eventually_good_url: (1, ZeroDivisionError()), + bad_url: (0.1, RuntimeError()) }) - self.assertEqual(iterations, 3) - self.assertEqual(result, eventually_good_url) + self.assertEqual(result, None) From 3702ad62335451d8148a70bc7e36fa93d3d52fa5 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 8 Mar 2023 14:54:53 -0500 Subject: [PATCH 1507/2309] Fix indentation. --- src/allmydata/storage_client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 169d34e32..4073d9b41 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -1212,8 +1212,8 @@ class HTTPNativeStorageServer(service.MultiService): return else: self._istorage_server = _HTTPStorageServer.from_http_client( - StorageClient.from_nurl(nurl, reactor) - ) + StorageClient.from_nurl(nurl, reactor) + ) result = self._istorage_server.get_version() From a61e41d5f9d1c4aae258c25055f13d807ea26720 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 8 Mar 2023 14:58:52 -0500 Subject: [PATCH 1508/2309] Document the motivation. --- src/allmydata/test/test_system.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/allmydata/test/test_system.py b/src/allmydata/test/test_system.py index e68f367cd..d11a6e866 100644 --- a/src/allmydata/test/test_system.py +++ b/src/allmydata/test/test_system.py @@ -658,6 +658,8 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase): self.failUnlessEqual(res, NEWERDATA) d.addCallback(_check_download_5) + # The previous checks upload a complete replacement. This uses a + # different API that is supposed to do a partial write at an offset. @async_to_deferred async def _check_write_at_offset(newnode): log.msg("writing at offset") From 8ccbd37d29906cef62d8db22573878534a783fdd Mon Sep 17 00:00:00 2001 From: dlee Date: Wed, 8 Mar 2023 15:16:03 -0600 Subject: [PATCH 1509/2309] Fix implicit re-export error by importing IWormhole from wormhole library directly --- src/allmydata/test/cli/test_invite.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/allmydata/test/cli/test_invite.py b/src/allmydata/test/cli/test_invite.py index 07756eeed..94d4395ff 100644 --- a/src/allmydata/test/cli/test_invite.py +++ b/src/allmydata/test/cli/test_invite.py @@ -19,7 +19,8 @@ from ...util.jsonbytes import dumps_bytes from ..common_util import run_cli from ..no_network import GridTestMixin from .common import CLITestMixin -from .wormholetesting import IWormhole, MemoryWormholeServer, TestingHelper, memory_server +from .wormholetesting import MemoryWormholeServer, TestingHelper, memory_server +from wormhole._interfaces import IWormhole # Logically: # JSONable = dict[str, Union[JSONable, None, int, float, str, list[JSONable]]] From 10b3eabed41baedd47e3b4f9ce55aec92699003a Mon Sep 17 00:00:00 2001 From: dlee Date: Wed, 8 Mar 2023 15:19:08 -0600 Subject: [PATCH 1510/2309] Apply per file flags corresponding to --strict to wormholetesting.py --- mypy.ini | 2 +- src/allmydata/test/cli/wormholetesting.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/mypy.ini b/mypy.ini index e6e7d16ff..27e9f6154 100644 --- a/mypy.ini +++ b/mypy.ini @@ -7,4 +7,4 @@ show_error_codes = True warn_unused_configs =True no_implicit_optional = True warn_redundant_casts = True -strict_equality = True \ No newline at end of file +strict_equality = True diff --git a/src/allmydata/test/cli/wormholetesting.py b/src/allmydata/test/cli/wormholetesting.py index d4e53a342..a0050a75b 100644 --- a/src/allmydata/test/cli/wormholetesting.py +++ b/src/allmydata/test/cli/wormholetesting.py @@ -29,6 +29,7 @@ For example:: import wormhole run(peerA(wormhole)) """ +# mypy: warn-unused-configs, disallow-any-generics, disallow-subclassing-any, disallow-untyped-calls, disallow-untyped-defs, disallow-incomplete-defs, check-untyped-defs, disallow-untyped-decorators, warn-redundant-casts, warn-unused-ignores, warn-return-any, no-implicit-reexport, strict-equality, strict-concatenate from __future__ import annotations From 4f47a18c6af89e92c81641c9bcc96bb30398c355 Mon Sep 17 00:00:00 2001 From: dlee Date: Wed, 8 Mar 2023 15:29:50 -0600 Subject: [PATCH 1511/2309] Comments added for inline mypy config. Individual flags used as --strict flag can only be used on a per-module basis. --- src/allmydata/test/cli/wormholetesting.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/allmydata/test/cli/wormholetesting.py b/src/allmydata/test/cli/wormholetesting.py index a0050a75b..6fb2b791c 100644 --- a/src/allmydata/test/cli/wormholetesting.py +++ b/src/allmydata/test/cli/wormholetesting.py @@ -30,6 +30,10 @@ For example:: run(peerA(wormhole)) """ # mypy: warn-unused-configs, disallow-any-generics, disallow-subclassing-any, disallow-untyped-calls, disallow-untyped-defs, disallow-incomplete-defs, check-untyped-defs, disallow-untyped-decorators, warn-redundant-casts, warn-unused-ignores, warn-return-any, no-implicit-reexport, strict-equality, strict-concatenate +# This inline mypy config applies a per-file mypy config for this file. +# It applies the '--strict' list of flags to this file. +# If you want to test using CLI run the command remove the inline config above and run: +# "mypy --follow-imports silent --strict src/allmydata/test/cli/wormholetesting.py" from __future__ import annotations From 5ca07c311c78c53df4eb1e0e657c6f8f9f75229d Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 8 Mar 2023 16:36:18 -0500 Subject: [PATCH 1512/2309] Set up 3.11 in metadata and GitHub Actions. --- .github/workflows/ci.yml | 5 +++-- newsfragments/3982.feature | 1 + setup.py | 6 ++++-- 3 files changed, 8 insertions(+), 4 deletions(-) create mode 100644 newsfragments/3982.feature diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4d67f09bd..e006d90ac 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -51,12 +51,13 @@ jobs: - "3.8" - "3.9" - "3.10" + - "3.11" include: # On macOS don't bother with 3.8, just to get faster builds. - os: macos-latest python-version: "3.9" - os: macos-latest - python-version: "3.10" + python-version: "3.11" # We only support PyPy on Linux at the moment. - os: ubuntu-latest python-version: "pypy-3.8" @@ -174,7 +175,7 @@ jobs: # 22.04 has some issue with Tor at the moment: # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3943 - os: ubuntu-20.04 - python-version: "3.9" + python-version: "3.11" force-foolscap: false steps: diff --git a/newsfragments/3982.feature b/newsfragments/3982.feature new file mode 100644 index 000000000..0d48fa476 --- /dev/null +++ b/newsfragments/3982.feature @@ -0,0 +1 @@ +Added support for Python 3.11. \ No newline at end of file diff --git a/setup.py b/setup.py index 98177bd41..c3928cf81 100644 --- a/setup.py +++ b/setup.py @@ -55,7 +55,9 @@ install_requires = [ # * foolscap >= 0.12.6 has an i2p.sam_endpoint() that takes kwargs # * foolscap 0.13.2 drops i2p support completely # * foolscap >= 21.7 is necessary for Python 3 with i2p support. + # * foolscap >= 23.3 is necessary for Python 3.11. "foolscap >= 21.7.0", + "foolscap >= 23.3.0; python_version > '3.10'", # * cryptography 2.6 introduced some ed25519 APIs we rely on. Note that # Twisted[conch] also depends on cryptography and Twisted[tls] @@ -380,8 +382,8 @@ setup(name="tahoe-lafs", # also set in __init__.py package_dir = {'':'src'}, packages=find_packages('src') + ['allmydata.test.plugins'], classifiers=trove_classifiers, - # We support Python 3.8 or later. 3.11 is not supported yet. - python_requires=">=3.8, <3.11", + # We support Python 3.8 or later + python_requires=">=3.8", install_requires=install_requires, extras_require={ # Duplicate the Twisted pywin32 dependency here. See From 708d54b5fdc7b8221ce88781623ff191f565a5b6 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 8 Mar 2023 16:42:41 -0500 Subject: [PATCH 1513/2309] Fix use of API removed in 3.11. --- src/allmydata/test/cli/wormholetesting.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/allmydata/test/cli/wormholetesting.py b/src/allmydata/test/cli/wormholetesting.py index 744f9d75a..4775ca5ef 100644 --- a/src/allmydata/test/cli/wormholetesting.py +++ b/src/allmydata/test/cli/wormholetesting.py @@ -34,7 +34,7 @@ from __future__ import annotations from typing import Iterator, Optional, List, Tuple from collections.abc import Awaitable -from inspect import getargspec +from inspect import getfullargspec from itertools import count from sys import stderr @@ -141,8 +141,8 @@ def _verify(): """ # Poor man's interface verification. - a = getargspec(create) - b = getargspec(MemoryWormholeServer.create) + a = getfullargspec(create) + b = getfullargspec(MemoryWormholeServer.create) # I know it has a `self` argument at the beginning. That's okay. b = b._replace(args=b.args[1:]) assert a == b, "{} != {}".format(a, b) From b43150ba85c7a3b5d1cb5952f9a235f9493c04d7 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 8 Mar 2023 16:48:08 -0500 Subject: [PATCH 1514/2309] Add future import. --- src/allmydata/test/test_deferredutil.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/allmydata/test/test_deferredutil.py b/src/allmydata/test/test_deferredutil.py index 47121b4cb..34358d0c8 100644 --- a/src/allmydata/test/test_deferredutil.py +++ b/src/allmydata/test/test_deferredutil.py @@ -2,6 +2,8 @@ Tests for allmydata.util.deferredutil. """ +from __future__ import annotations + from twisted.trial import unittest from twisted.internet import defer, reactor from twisted.internet.defer import Deferred From 8062808de2c65df6eed8b5f6b2282fc681a9590a Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 9 Mar 2023 09:46:46 -0500 Subject: [PATCH 1515/2309] Add restriction 3.12 or later. --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index c3928cf81..53fc42c62 100644 --- a/setup.py +++ b/setup.py @@ -382,8 +382,8 @@ setup(name="tahoe-lafs", # also set in __init__.py package_dir = {'':'src'}, packages=find_packages('src') + ['allmydata.test.plugins'], classifiers=trove_classifiers, - # We support Python 3.8 or later - python_requires=">=3.8", + # We support Python 3.8 or later, 3.12 is untested for now + python_requires=">=3.8, <3.12", install_requires=install_requires, extras_require={ # Duplicate the Twisted pywin32 dependency here. See From db445af1c40667ab986aac42fbe47709c379539e Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 9 Mar 2023 09:59:36 -0500 Subject: [PATCH 1516/2309] Separate flags for forcing foolscap between client and server. --- integration/util.py | 6 ++++++ src/allmydata/client.py | 1 + src/allmydata/storage_client.py | 2 +- src/allmydata/test/common_system.py | 1 + 4 files changed, 9 insertions(+), 1 deletion(-) diff --git a/integration/util.py b/integration/util.py index d0444cfc8..04c925abf 100644 --- a/integration/util.py +++ b/integration/util.py @@ -367,6 +367,12 @@ def _create_node(reactor, request, temp_dir, introducer_furl, flog_gatherer, nam 'force_foolscap', str(force_foolscap), ) + set_config( + config, + 'client', + 'force_foolscap', + str(force_foolscap), + ) write_config(FilePath(config_path), config) created_d.addCallback(created) diff --git a/src/allmydata/client.py b/src/allmydata/client.py index ce55bd77f..204c90dc5 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -94,6 +94,7 @@ _client_config = configutil.ValidConfiguration( "shares.total", "shares._max_immutable_segment_size_for_testing", "storage.plugins", + "force_foolscap", ), "storage": ( "debug_discard", diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index fa0d2de49..a53ce9f22 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -300,7 +300,7 @@ class StorageFarmBroker(service.MultiService): ) if not self.node_config.get_config( - "storage", "force_foolscap", default=True, boolean=True, + "client", "force_foolscap", default=True, boolean=True, ) and len(server["ann"].get(ANONYMOUS_STORAGE_NURLS, [])) > 0: s = HTTPNativeStorageServer( server_id, diff --git a/src/allmydata/test/common_system.py b/src/allmydata/test/common_system.py index 39b2b43d1..3491d413d 100644 --- a/src/allmydata/test/common_system.py +++ b/src/allmydata/test/common_system.py @@ -871,6 +871,7 @@ class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin): setnode("nickname", u"client %d \N{BLACK SMILING FACE}" % (which,)) setconf(config, which, "storage", "force_foolscap", str(force_foolscap)) + setconf(config, which, "client", "force_foolscap", str(force_foolscap)) tub_location_hint, tub_port_endpoint = self.port_assigner.assign(reactor) setnode("tub.port", tub_port_endpoint) From 56b6dd86c33d136e0f74a93c34e5d2eec186f368 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 9 Mar 2023 10:33:21 -0500 Subject: [PATCH 1517/2309] Add unit test for client foolscap config flag. --- src/allmydata/client.py | 2 +- src/allmydata/storage_client.py | 17 ++++++--- src/allmydata/test/test_storage_client.py | 42 +++++++++++++++++++++++ 3 files changed, 56 insertions(+), 5 deletions(-) diff --git a/src/allmydata/client.py b/src/allmydata/client.py index 204c90dc5..8ca295f22 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -466,7 +466,7 @@ def create_introducer_clients(config, main_tub, _introducer_factory=None): return introducer_clients -def create_storage_farm_broker(config, default_connection_handlers, foolscap_connection_handlers, tub_options, introducer_clients): +def create_storage_farm_broker(config: _Config, default_connection_handlers, foolscap_connection_handlers, tub_options, introducer_clients): """ Create a StorageFarmBroker object, for use by Uploader/Downloader (and everybody else who wants to use storage servers) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index a53ce9f22..0b884db3b 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -89,6 +89,7 @@ from allmydata.storage.http_client import ( ClientException as HTTPClientException, StorageClientMutables, ReadVector, TestWriteVectors, WriteVector, TestVector, ClientException ) +from .node import _Config ANONYMOUS_STORAGE_NURLS = "anonymous-storage-NURLs" @@ -199,7 +200,7 @@ class StorageFarmBroker(service.MultiService): self, permute_peers, tub_maker, - node_config, + node_config: _Config, storage_client_config=None, ): service.MultiService.__init__(self) @@ -274,6 +275,16 @@ class StorageFarmBroker(service.MultiService): in self.storage_client_config.storage_plugins.items() }) + @staticmethod + def _should_we_use_http(node_config: _Config, announcement: dict) -> bool: + """ + Given an announcement dictionary and config, return whether we should + connect to storage server over HTTP. + """ + return not node_config.get_config( + "client", "force_foolscap", default=True, boolean=True, + ) and len(announcement.get(ANONYMOUS_STORAGE_NURLS, [])) > 0 + @log_call( action_type=u"storage-client:broker:make-storage-server", include_args=["server_id"], @@ -299,9 +310,7 @@ class StorageFarmBroker(service.MultiService): "pub-{}".format(str(server_id, "ascii")), # server_id is v0- not pub-v0-key .. for reasons? ) - if not self.node_config.get_config( - "client", "force_foolscap", default=True, boolean=True, - ) and len(server["ann"].get(ANONYMOUS_STORAGE_NURLS, [])) > 0: + if self._should_we_use_http(self.node_config, server["ann"]): s = HTTPNativeStorageServer( server_id, server["ann"], diff --git a/src/allmydata/test/test_storage_client.py b/src/allmydata/test/test_storage_client.py index 1a84f35ec..109122da6 100644 --- a/src/allmydata/test/test_storage_client.py +++ b/src/allmydata/test/test_storage_client.py @@ -94,10 +94,13 @@ from allmydata.storage_client import ( StorageFarmBroker, _FoolscapStorage, _NullStorage, + ANONYMOUS_STORAGE_NURLS ) from ..storage.server import ( StorageServer, ) +from ..client import config_from_string + from allmydata.interfaces import ( IConnectionStatus, IStorageServer, @@ -739,3 +742,42 @@ storage: yield done self.assertTrue(done.called) + + def test_should_we_use_http_default(self): + """Default is to not use HTTP; this will change eventually""" + basedir = self.mktemp() + node_config = config_from_string(basedir, "", "") + announcement = {ANONYMOUS_STORAGE_NURLS: ["pb://..."]} + self.assertFalse( + StorageFarmBroker._should_we_use_http(node_config, announcement) + ) + self.assertFalse( + StorageFarmBroker._should_we_use_http(node_config, {}) + ) + + def test_should_we_use_http(self): + """ + If HTTP is allowed, it will only be used if the announcement includes + some NURLs. + """ + basedir = self.mktemp() + + no_nurls = {} + empty_nurls = {ANONYMOUS_STORAGE_NURLS: []} + has_nurls = {ANONYMOUS_STORAGE_NURLS: ["pb://.."]} + + for force_foolscap, announcement, expected_http_usage in [ + ("false", no_nurls, False), + ("false", empty_nurls, False), + ("false", has_nurls, True), + ("true", empty_nurls, False), + ("true", no_nurls, False), + ("true", has_nurls, False), + ]: + node_config = config_from_string( + basedir, "", f"[client]\nforce_foolscap = {force_foolscap}" + ) + self.assertEqual( + StorageFarmBroker._should_we_use_http(node_config, announcement), + expected_http_usage + ) From 5d7d387593da2fdf101bac07f6bcc18fab1b783f Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 9 Mar 2023 13:45:50 -0500 Subject: [PATCH 1518/2309] Pacify mypy. --- src/allmydata/client.py | 2 +- src/allmydata/storage_client.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/allmydata/client.py b/src/allmydata/client.py index 8ca295f22..2adf59660 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -60,7 +60,7 @@ from allmydata.interfaces import ( ) from allmydata.nodemaker import NodeMaker from allmydata.blacklist import Blacklist - +from allmydata.node import _Config KiB=1024 MiB=1024*KiB diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 0b884db3b..837cc06d3 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -34,7 +34,7 @@ from __future__ import annotations from six import ensure_text -from typing import Union +from typing import Union, Any from os import urandom import re import time @@ -219,9 +219,9 @@ class StorageFarmBroker(service.MultiService): # own Reconnector, and will give us a RemoteReference when we ask # them for it. self.servers = BytesKeyDict() - self._static_server_ids = set() # ignore announcements for these + self._static_server_ids : set[bytes] = set() # ignore announcements for these self.introducer_client = None - self._threshold_listeners = [] # tuples of (threshold, Deferred) + self._threshold_listeners : list[tuple[float,defer.Deferred[Any]]]= [] # tuples of (threshold, Deferred) self._connected_high_water_mark = 0 @log_call(action_type=u"storage-client:broker:set-static-servers") From ccf12897f2913de4415580dc322c8231e8c49042 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 10 Mar 2023 09:02:08 -0500 Subject: [PATCH 1519/2309] Add content limits to server. --- newsfragments/3965.minor | 0 src/allmydata/storage/http_server.py | 17 ++++++++++++++--- 2 files changed, 14 insertions(+), 3 deletions(-) create mode 100644 newsfragments/3965.minor diff --git a/newsfragments/3965.minor b/newsfragments/3965.minor new file mode 100644 index 000000000..e69de29bb diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index c6c3ab615..fd7fd1187 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -606,7 +606,10 @@ class HTTPServer(object): async def allocate_buckets(self, request, authorization, storage_index): """Allocate buckets.""" upload_secret = authorization[Secrets.UPLOAD] - info = await self._read_encoded(request, _SCHEMAS["allocate_buckets"]) + # It's just a list of up to ~256 shares, shouldn't use many bytes. + info = await self._read_encoded( + request, _SCHEMAS["allocate_buckets"], max_size=8192 + ) # We do NOT validate the upload secret for existing bucket uploads. # Another upload may be happening in parallel, with a different upload @@ -773,7 +776,11 @@ class HTTPServer(object): except KeyError: raise _HTTPError(http.NOT_FOUND) - info = await self._read_encoded(request, _SCHEMAS["advise_corrupt_share"]) + # The reason can be a string with explanation, so in theory it could be + # longish? + info = await self._read_encoded( + request, _SCHEMAS["advise_corrupt_share"], max_size=32768, + ) bucket.advise_corrupt_share(info["reason"].encode("utf-8")) return b"" @@ -872,7 +879,11 @@ class HTTPServer(object): }: raise _HTTPError(http.NOT_FOUND) - info = await self._read_encoded(request, _SCHEMAS["advise_corrupt_share"]) + # The reason can be a string with explanation, so in theory it could be + # longish? + info = await self._read_encoded( + request, _SCHEMAS["advise_corrupt_share"], max_size=32768 + ) self._storage_server.advise_corrupt_share( b"mutable", storage_index, share_number, info["reason"].encode("utf-8") ) From 7a387a054eb5c9b86ec20c6b95b0489a0034af13 Mon Sep 17 00:00:00 2001 From: "Fon E. Noel NFEBE" Date: Sat, 11 Mar 2023 23:58:54 +0100 Subject: [PATCH 1520/2309] Fix more inverted assertions Just like in 7c3f6cb4c7fcba278dea93d62a3ddea381835a7f This commit corrects some wronly inverted assertions inside `test/test_storage.py` Signed-off-by: Fon E. Noel NFEBE --- src/allmydata/test/test_storage.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index af4d549bf..655753d90 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -1816,7 +1816,7 @@ class MutableServer(SyncTestCase): prefix = si[:2] prefixdir = os.path.join(self.workdir("test_remove"), "shares", prefix) bucketdir = os.path.join(prefixdir, si) - self.assertThat(prefixdir, Contains(os.path.exists(prefixdir))) + self.assertTrue(os.path.exists(prefixdir), prefixdir) self.assertFalse(os.path.exists(bucketdir), bucketdir) def test_writev_without_renew_lease(self): @@ -2425,7 +2425,7 @@ class MDMFProxies(AsyncTestCase, ShouldFailMixin): # any point during the process, it should fail to write when we # tell it to write. def _check_failure(results): - self.assertThat(results, Equals(2)) + self.assertThat(results, HasLength(2)) res, d = results self.assertFalse(res) @@ -3191,7 +3191,7 @@ class MDMFProxies(AsyncTestCase, ShouldFailMixin): d.addCallback(lambda ignored: mr.get_verinfo()) def _check_verinfo(verinfo): - self.assertThat(verinfo) + self.assertTrue(verinfo) self.assertThat(verinfo, HasLength(9)) (seqnum, root_hash, From f9acb56e82602081c09f2db8f1eab9db24ee3ddb Mon Sep 17 00:00:00 2001 From: "Fon E. Noel NFEBE" Date: Sun, 12 Mar 2023 00:16:38 +0100 Subject: [PATCH 1521/2309] Fix wrong expected val in assertion This is a follow up to 7a387a054eb5c9b86ec20c6b95b0489a0034af13 Signed-off-by: Fon E. Noel NFEBE --- src/allmydata/test/test_storage.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index 655753d90..c1d6004e8 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -1547,8 +1547,8 @@ class MutableServer(SyncTestCase): }, [(0,12), (20,5)], ) - self.assertThat(answer, (False, - Equals({0: [b"000000000011", b"22222"], + self.assertThat(answer, Equals((False, + {0: [b"000000000011", b"22222"], 1: [b"", b""], 2: [b"", b""], }))) From 74ff8cd08041a1107b05771778310449bf4d99f8 Mon Sep 17 00:00:00 2001 From: dlee Date: Mon, 13 Mar 2023 11:04:52 -0500 Subject: [PATCH 1522/2309] Per-file configuration for wormholetesting.py moved from inline mypy configuration moved to mypy.ini file --- mypy.ini | 16 ++++++++++++++++ src/allmydata/test/cli/wormholetesting.py | 5 ----- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/mypy.ini b/mypy.ini index 27e9f6154..c391c5594 100644 --- a/mypy.ini +++ b/mypy.ini @@ -8,3 +8,19 @@ warn_unused_configs =True no_implicit_optional = True warn_redundant_casts = True strict_equality = True + +[mypy-allmydata.test.cli.wormholetesting] +warn_unused_configs = True +disallow_any_generics = True +disallow_subclassing_any = True +disallow_untyped_calls = True +disallow_untyped_defs = True +disallow_incomplete_defs = True +check_untyped_defs = True +disallow_untyped_decorators = True +warn_redundant_casts = True +warn_unused_ignores = True +warn_return_any = True +no_implicit_reexport = True +strict_equality = True +strict_concatenate = True diff --git a/src/allmydata/test/cli/wormholetesting.py b/src/allmydata/test/cli/wormholetesting.py index 6fb2b791c..d4e53a342 100644 --- a/src/allmydata/test/cli/wormholetesting.py +++ b/src/allmydata/test/cli/wormholetesting.py @@ -29,11 +29,6 @@ For example:: import wormhole run(peerA(wormhole)) """ -# mypy: warn-unused-configs, disallow-any-generics, disallow-subclassing-any, disallow-untyped-calls, disallow-untyped-defs, disallow-incomplete-defs, check-untyped-defs, disallow-untyped-decorators, warn-redundant-casts, warn-unused-ignores, warn-return-any, no-implicit-reexport, strict-equality, strict-concatenate -# This inline mypy config applies a per-file mypy config for this file. -# It applies the '--strict' list of flags to this file. -# If you want to test using CLI run the command remove the inline config above and run: -# "mypy --follow-imports silent --strict src/allmydata/test/cli/wormholetesting.py" from __future__ import annotations From 61c835c8a05c15b7eabe29b453633b7c4da022e8 Mon Sep 17 00:00:00 2001 From: dlee Date: Mon, 13 Mar 2023 11:17:01 -0500 Subject: [PATCH 1523/2309] Added missing space between return type --- src/allmydata/test/cli/wormholetesting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/cli/wormholetesting.py b/src/allmydata/test/cli/wormholetesting.py index d4e53a342..be94a7981 100644 --- a/src/allmydata/test/cli/wormholetesting.py +++ b/src/allmydata/test/cli/wormholetesting.py @@ -76,7 +76,7 @@ class MemoryWormholeServer(object): stderr: TextIO=stderr, _eventual_queue: Optional[Any]=None, _enable_dilate: bool=False, - )-> _MemoryWormhole: + ) -> _MemoryWormhole: """ Create a wormhole. It will be able to connect to other wormholes created by this instance (and constrained by the normal appid/relay_url From b58dd2bb3bed375258f611eb0af39f6c08f64684 Mon Sep 17 00:00:00 2001 From: dlee Date: Mon, 13 Mar 2023 12:27:53 -0500 Subject: [PATCH 1524/2309] Remove flags that are unused --- mypy.ini | 2 -- 1 file changed, 2 deletions(-) diff --git a/mypy.ini b/mypy.ini index c391c5594..7acc0ddc5 100644 --- a/mypy.ini +++ b/mypy.ini @@ -10,7 +10,6 @@ warn_redundant_casts = True strict_equality = True [mypy-allmydata.test.cli.wormholetesting] -warn_unused_configs = True disallow_any_generics = True disallow_subclassing_any = True disallow_untyped_calls = True @@ -18,7 +17,6 @@ disallow_untyped_defs = True disallow_incomplete_defs = True check_untyped_defs = True disallow_untyped_decorators = True -warn_redundant_casts = True warn_unused_ignores = True warn_return_any = True no_implicit_reexport = True From 041a634d27f1f2adfdc82471e60192aaecb1fbfc Mon Sep 17 00:00:00 2001 From: dlee Date: Mon, 13 Mar 2023 13:08:32 -0500 Subject: [PATCH 1525/2309] Fix private interface import to test_invite --- src/allmydata/test/cli/test_invite.py | 4 ++-- src/allmydata/test/cli/wormholetesting.py | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/allmydata/test/cli/test_invite.py b/src/allmydata/test/cli/test_invite.py index 94d4395ff..31992a54d 100644 --- a/src/allmydata/test/cli/test_invite.py +++ b/src/allmydata/test/cli/test_invite.py @@ -19,8 +19,8 @@ from ...util.jsonbytes import dumps_bytes from ..common_util import run_cli from ..no_network import GridTestMixin from .common import CLITestMixin -from .wormholetesting import MemoryWormholeServer, TestingHelper, memory_server -from wormhole._interfaces import IWormhole +from .wormholetesting import MemoryWormholeServer, TestingHelper, memory_server, IWormhole + # Logically: # JSONable = dict[str, Union[JSONable, None, int, float, str, list[JSONable]]] diff --git a/src/allmydata/test/cli/wormholetesting.py b/src/allmydata/test/cli/wormholetesting.py index be94a7981..9fbe8b63e 100644 --- a/src/allmydata/test/cli/wormholetesting.py +++ b/src/allmydata/test/cli/wormholetesting.py @@ -32,6 +32,8 @@ For example:: from __future__ import annotations +__all__ = ['IWormhole'] + from typing import Iterator, Optional, List, Tuple, Any, TextIO from inspect import getfullargspec from itertools import count @@ -76,7 +78,7 @@ class MemoryWormholeServer(object): stderr: TextIO=stderr, _eventual_queue: Optional[Any]=None, _enable_dilate: bool=False, - ) -> _MemoryWormhole: + )-> _MemoryWormhole: """ Create a wormhole. It will be able to connect to other wormholes created by this instance (and constrained by the normal appid/relay_url From 2bb96d8452c6fc4eaad990088c63314fd54e6aed Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 13 Mar 2023 15:19:07 -0400 Subject: [PATCH 1526/2309] There are new autobahn releases; remove the upper bound. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 53fc42c62..82ff45764 100644 --- a/setup.py +++ b/setup.py @@ -118,7 +118,7 @@ install_requires = [ "attrs >= 18.2.0", # WebSocket library for twisted and asyncio - "autobahn < 22.4.1", # remove this when 22.4.3 is released + "autobahn", # Support for Python 3 transition "future >= 0.18.2", From 568e1b53177f33e53dae4d54b03c606b693c3319 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 13 Mar 2023 15:19:43 -0400 Subject: [PATCH 1527/2309] Replace the mach-nix-based package with a nixpkgs-based package The built-in nixpkgs `buildPythonPackage` doesn't do metadata discovery so we have to duplicate a lot of the package metadata. However, mach-nix is unmaintained and incompatible with newer versions of nixpkgs. --- .circleci/config.yml | 12 +++-- default.nix | 73 +++++----------------------- nix/pycddl.nix | 22 +++++++++ nix/sources.json | 52 ++++++++------------ nix/tahoe-lafs.nix | 112 +++++++++++++++++++++++++++++++++++++++++++ tests.nix | 88 ---------------------------------- 6 files changed, 174 insertions(+), 185 deletions(-) create mode 100644 nix/pycddl.nix create mode 100644 nix/tahoe-lafs.nix delete mode 100644 tests.nix diff --git a/.circleci/config.yml b/.circleci/config.yml index 152d56810..d07383b84 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -70,14 +70,18 @@ workflows: - "oraclelinux-8": {} - - "nixos": - name: "NixOS 21.05" - nixpkgs: "21.05" - - "nixos": name: "NixOS 21.11" nixpkgs: "21.11" + - "nixos": + name: "NixOS 22.11" + nixpkgs: "22.11" + + - "nixos": + name: "NixOS unstable" + nixpkgs: "unstable" + # Eventually, test against PyPy 3.8 #- "pypy27-buster": # {} diff --git a/default.nix b/default.nix index e4f2dd4d4..59903b1e2 100644 --- a/default.nix +++ b/default.nix @@ -21,16 +21,13 @@ let sources = import nix/sources.nix; in { - pkgsVersion ? "nixpkgs-21.11" # a string which chooses a nixpkgs from the + pkgsVersion ? "nixpkgs-22.11" # a string which chooses a nixpkgs from the # niv-managed sources data , pkgs ? import sources.${pkgsVersion} { } # nixpkgs itself -, pypiData ? sources.pypi-deps-db # the pypi package database snapshot to use - # for dependency resolution - -, pythonVersion ? "python39" # a string choosing the python derivation from - # nixpkgs to target +, pythonVersion ? "python310" # a string choosing the python derivation from + # nixpkgs to target , extras ? [ "tor" "i2p" ] # a list of strings identifying tahoe-lafs extras, # the dependencies of which the resulting package @@ -39,64 +36,18 @@ in # including them is a lot smaller than the cost of # re-building the whole thing to add them. -, mach-nix ? import sources.mach-nix { # the mach-nix package to use to build - # the tahoe-lafs package - inherit pkgs pypiData; - python = pythonVersion; -} }: -# The project name, version, and most other metadata are automatically -# extracted from the source. Some requirements are not properly extracted -# and those cases are handled below. The version can only be extracted if -# `setup.py update_version` has been run (this is not at all ideal but it -# seems difficult to fix) - so for now just be sure to run that first. -mach-nix.buildPythonPackage rec { - # Define the location of the Tahoe-LAFS source to be packaged. Clean up all - # as many of the non-source files (eg the `.git` directory, `~` backup - # files, nix's own `result` symlink, etc) as possible to avoid needing to - # re-build when files that make no difference to the package have changed. - src = pkgs.lib.cleanSource ./.; - +with pkgs.${pythonVersion}.pkgs; +callPackage ./nix/tahoe-lafs.nix { # Select whichever package extras were requested. inherit extras; - # Define some extra requirements that mach-nix does not automatically detect - # from inspection of the source. We typically don't need to put version - # constraints on any of these requirements. The pypi-deps-db we're - # operating with makes dependency resolution deterministic so as long as it - # works once it will always work. It could be that in the future we update - # pypi-deps-db and an incompatibility arises - in which case it would make - # sense to apply some version constraints here. - requirementsExtra = '' - # mach-nix does not yet support pyproject.toml which means it misses any - # build-time requirements of our dependencies which are declared in such a - # file. Tell it about them here. - setuptools_rust + # Define the location of the Tahoe-LAFS source to be packaged. Clean up as + # many of the non-source files (eg the `.git` directory, `~` backup files, + # nix's own `result` symlink, etc) as possible to avoid needing to re-build + # when files that make no difference to the package have changed. + tahoe-lafs-src = pkgs.lib.cleanSource ./.; - # mach-nix does not yet parse environment markers (e.g. "python > '3.0'") - # correctly. It misses all of our requirements which have an environment marker. - # Duplicate them here. - foolscap - eliot - pyrsistent - collections-extended - ''; - - # Specify where mach-nix should find packages for our Python dependencies. - # There are some reasonable defaults so we only need to specify certain - # packages where the default configuration runs into some issue. - providers = { - }; - - # Define certain overrides to the way Python dependencies are built. - _ = { - # Remove a click-default-group patch for a test suite problem which no - # longer applies because the project apparently no longer has a test suite - # in its source distribution. - click-default-group.patches = []; - }; - - passthru.meta.mach-nix = { - inherit providers _; - }; + # pycddl isn't packaged in nixpkgs so supply our own package of it. + pycddl = callPackage ./nix/pycddl.nix { }; } diff --git a/nix/pycddl.nix b/nix/pycddl.nix new file mode 100644 index 000000000..0f6a0329e --- /dev/null +++ b/nix/pycddl.nix @@ -0,0 +1,22 @@ +{ lib, fetchPypi, buildPythonPackage, rustPlatform }: +buildPythonPackage rec { + pname = "pycddl"; + version = "0.4.0"; + format = "pyproject"; + + src = fetchPypi { + inherit pname version; + sha256 = "sha256-w0CGbPeiXyS74HqZXyiXhvaAMUaIj5onwjl9gWKAjqY="; + }; + + nativeBuildInputs = with rustPlatform; [ + maturinBuildHook + cargoSetupHook + ]; + + cargoDeps = rustPlatform.fetchCargoTarball { + inherit src; + name = "${pname}-${version}"; + hash = "sha256-g96eeaqN9taPED4u+UKUcoitf5aTGFrW2/TOHoHEVHs="; + }; +} diff --git a/nix/sources.json b/nix/sources.json index 18aa18e3f..bcac22174 100644 --- a/nix/sources.json +++ b/nix/sources.json @@ -1,16 +1,4 @@ { - "mach-nix": { - "branch": "switch-to-nix-pypi-fetcher-2", - "description": "Create highly reproducible python environments", - "homepage": "", - "owner": "PrivateStorageio", - "repo": "mach-nix", - "rev": "f6d1a1841d8778c199326f95d0703c16bee2f8c4", - "sha256": "0krc4yhnpbzc4yhja9frnmym2vqm5zyacjnqb3fq9z9gav8vs9ls", - "type": "tarball", - "url": "https://github.com/PrivateStorageio/mach-nix/archive/f6d1a1841d8778c199326f95d0703c16bee2f8c4.tar.gz", - "url_template": "https://github.com///archive/.tar.gz" - }, "niv": { "branch": "master", "description": "Easy dependency management for Nix projects", @@ -23,18 +11,6 @@ "url": "https://github.com/nmattia/niv/archive/5830a4dd348d77e39a0f3c4c762ff2663b602d4c.tar.gz", "url_template": "https://github.com///archive/.tar.gz" }, - "nixpkgs-21.05": { - "branch": "nixos-21.05", - "description": "Nix Packages collection", - "homepage": "", - "owner": "NixOS", - "repo": "nixpkgs", - "rev": "0fd9ee1aa36ce865ad273f4f07fdc093adeb5c00", - "sha256": "1mr2qgv5r2nmf6s3gqpcjj76zpsca6r61grzmqngwm0xlh958smx", - "type": "tarball", - "url": "https://github.com/NixOS/nixpkgs/archive/0fd9ee1aa36ce865ad273f4f07fdc093adeb5c00.tar.gz", - "url_template": "https://github.com///archive/.tar.gz" - }, "nixpkgs-21.11": { "branch": "nixos-21.11", "description": "Nix Packages collection", @@ -47,16 +23,28 @@ "url": "https://github.com/NixOS/nixpkgs/archive/838eefb4f93f2306d4614aafb9b2375f315d917f.tar.gz", "url_template": "https://github.com///archive/.tar.gz" }, - "pypi-deps-db": { - "branch": "master", - "description": "Probably the most complete python dependency database", + "nixpkgs-22.11": { + "branch": "nixos-22.11", + "description": "Nix Packages collection", "homepage": "", - "owner": "DavHau", - "repo": "pypi-deps-db", - "rev": "5440c9c76f6431f300fb6a1ecae762a5444de5f6", - "sha256": "08r3iiaxzw9v2gq15y1m9bwajshyyz9280g6aia7mkgnjs9hnd1n", + "owner": "nixos", + "repo": "nixpkgs", + "rev": "970402e6147c49603f4d06defe44d27fe51884ce", + "sha256": "1v0ljy7wqq14ad3gd1871fgvd4psr7dy14q724k0wwgxk7inbbwh", "type": "tarball", - "url": "https://github.com/DavHau/pypi-deps-db/archive/5440c9c76f6431f300fb6a1ecae762a5444de5f6.tar.gz", + "url": "https://github.com/nixos/nixpkgs/archive/970402e6147c49603f4d06defe44d27fe51884ce.tar.gz", + "url_template": "https://github.com///archive/.tar.gz" + }, + "nixpkgs-unstable": { + "branch": "master", + "description": "Nix Packages collection", + "homepage": "", + "owner": "nixos", + "repo": "nixpkgs", + "rev": "d0c9a536331227ab883b4f6964be638fa436d81f", + "sha256": "1gg6v5rk1p26ciygdg262zc5vqws753rvgcma5rim2s6gyfrjaq1", + "type": "tarball", + "url": "https://github.com/nixos/nixpkgs/archive/d0c9a536331227ab883b4f6964be638fa436d81f.tar.gz", "url_template": "https://github.com///archive/.tar.gz" } } diff --git a/nix/tahoe-lafs.nix b/nix/tahoe-lafs.nix new file mode 100644 index 000000000..af0e4bf4d --- /dev/null +++ b/nix/tahoe-lafs.nix @@ -0,0 +1,112 @@ +{ buildPythonPackage +, tahoe-lafs-src +, extras + +# always dependencies +, attrs +, autobahn +, cbor2 +, click +, collections-extended +, cryptography +, distro +, eliot +, filelock +, foolscap +, future +, klein +, magic-wormhole +, netifaces +, psutil +, pycddl +, pyrsistent +, pyutil +, six +, treq +, twisted +, werkzeug +, zfec +, zope_interface + +# tor extra dependencies +, txtorcon + +# i2p extra dependencies +, txi2p-tahoe + +# test dependencies +, beautifulsoup4 +, fixtures +, hypothesis +, mock +, paramiko +, prometheus-client +, pytest +, pytest-timeout +, pytest-twisted +, tenacity +, testtools +, towncrier +}: +let + pname = "tahoe-lafs"; + version = "1.18.0.post1"; + + pickExtraDependencies = deps: extras: builtins.foldl' (accum: extra: accum ++ deps.${extra}) [] extras; + + pythonExtraDependencies = { + tor = [ txtorcon ]; + i2p = [ txi2p-tahoe ]; + }; + + pythonPackageDependencies = [ + attrs + autobahn + cbor2 + click + collections-extended + cryptography + distro + eliot + filelock + foolscap + future + klein + magic-wormhole + netifaces + psutil + pycddl + pyrsistent + pyutil + six + treq + twisted + (twisted.passthru.optional-dependencies.tls) + (twisted.passthru.optional-dependencies.conch) + werkzeug + zfec + zope_interface + ] ++ pickExtraDependencies pythonExtraDependencies extras; + + pythonCheckDependencies = [ + beautifulsoup4 + fixtures + hypothesis + mock + paramiko + prometheus-client + pytest + pytest-timeout + pytest-twisted + tenacity + testtools + towncrier + ]; +in +buildPythonPackage { + inherit pname version; + src = tahoe-lafs-src; + buildInputs = pythonPackageDependencies; + checkInputs = pythonCheckDependencies; + checkPhase = "TAHOE_LAFS_HYPOTHESIS_PROFILE=ci python -m twisted.trial -j $NIX_BUILD_CORES allmydata"; +} diff --git a/tests.nix b/tests.nix deleted file mode 100644 index f8ed678f3..000000000 --- a/tests.nix +++ /dev/null @@ -1,88 +0,0 @@ -let - sources = import nix/sources.nix; -in -# See default.nix for documentation about parameters. -{ pkgsVersion ? "nixpkgs-21.11" -, pkgs ? import sources.${pkgsVersion} { } -, pypiData ? sources.pypi-deps-db -, pythonVersion ? "python39" -, mach-nix ? import sources.mach-nix { - inherit pkgs pypiData; - python = pythonVersion; - } -}@args: -let - # We would like to know the test requirements but mach-nix does not directly - # expose this information to us. However, it is perfectly capable of - # determining it if we ask right... This is probably not meant to be a - # public mach-nix API but we pinned mach-nix so we can deal with mach-nix - # upgrade breakage in our own time. - mach-lib = import "${sources.mach-nix}/mach_nix/nix/lib.nix" { - inherit pkgs; - lib = pkgs.lib; - }; - tests_require = (mach-lib.extract "python39" ./. "extras_require" ).extras_require.test; - - # Get the Tahoe-LAFS package itself. This does not include test - # requirements and we don't ask for test requirements so that we can just - # re-use the normal package if it is already built. - tahoe-lafs = import ./. args; - - # If we want to get tahoe-lafs into a Python environment with a bunch of - # *other* Python modules and let them interact in the usual way then we have - # to ask mach-nix for tahoe-lafs and those other Python modules in the same - # way - i.e., using `requirements`. The other tempting mechanism, - # `packagesExtra`, inserts an extra layer of Python environment and prevents - # normal interaction between Python modules (as well as usually producing - # file collisions in the packages that are both runtime and test - # dependencies). To get the tahoe-lafs we just built into the environment, - # put it into nixpkgs using an overlay and tell mach-nix to get tahoe-lafs - # from nixpkgs. - overridesPre = [(self: super: { inherit tahoe-lafs; })]; - providers = tahoe-lafs.meta.mach-nix.providers // { tahoe-lafs = "nixpkgs"; }; - - # Make the Python environment in which we can run the tests. - python-env = mach-nix.mkPython { - # Get the packaging fixes we already know we need from putting together - # the runtime package. - inherit (tahoe-lafs.meta.mach-nix) _; - # Share the runtime package's provider configuration - combined with our - # own that causes the right tahoe-lafs to be picked up. - inherit providers overridesPre; - requirements = '' - # Here we pull in the Tahoe-LAFS package itself. - tahoe-lafs - - # Unfortunately mach-nix misses all of the Python dependencies of the - # tahoe-lafs satisfied from nixpkgs. Drag them in here. This gives a - # bit of a pyrrhic flavor to the whole endeavor but maybe mach-nix will - # fix this soon. - # - # https://github.com/DavHau/mach-nix/issues/123 - # https://github.com/DavHau/mach-nix/pull/386 - ${tahoe-lafs.requirements} - - # And then all of the test-only dependencies. - ${builtins.concatStringsSep "\n" tests_require} - - # txi2p-tahoe is another dependency with an environment marker that - # mach-nix doesn't automatically pick up. - txi2p-tahoe - ''; - }; -in -# Make a derivation that runs the unit test suite. -pkgs.runCommand "tahoe-lafs-tests" { } '' - export TAHOE_LAFS_HYPOTHESIS_PROFILE=ci - ${python-env}/bin/python -m twisted.trial -j $NIX_BUILD_CORES allmydata - - # It's not cool to put the whole _trial_temp into $out because it has weird - # files in it we don't want in the store. Plus, even all of the less weird - # files are mostly just trash that's not meaningful if the test suite passes - # (which is the only way we get $out anyway). - # - # The build log itself is typically available from `nix-store --read-log` so - # we don't need to record that either. - echo "passed" >$out - -'' From f1be1ca1de497f4b3ebb5d5795803a5a331dcba9 Mon Sep 17 00:00:00 2001 From: dlee Date: Mon, 13 Mar 2023 14:53:25 -0500 Subject: [PATCH 1528/2309] Added more elements to export list in wormholetesting.py --- src/allmydata/test/cli/wormholetesting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/cli/wormholetesting.py b/src/allmydata/test/cli/wormholetesting.py index 9fbe8b63e..99e26e64b 100644 --- a/src/allmydata/test/cli/wormholetesting.py +++ b/src/allmydata/test/cli/wormholetesting.py @@ -32,7 +32,7 @@ For example:: from __future__ import annotations -__all__ = ['IWormhole'] +__all__ = ['MemoryWormholeServer', 'TestingHelper', 'memory_server', 'IWormhole'] from typing import Iterator, Optional, List, Tuple, Any, TextIO from inspect import getfullargspec From fa2ba64d4d7c7f4393ab2378b7a12dd5f1b6aa08 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 13 Mar 2023 16:02:57 -0400 Subject: [PATCH 1529/2309] Also supply the i2p extra dependency, txi2p --- default.nix | 3 ++- nix/tahoe-lafs.nix | 4 ++-- nix/txi2p.nix | 14 ++++++++++++++ 3 files changed, 18 insertions(+), 3 deletions(-) create mode 100644 nix/txi2p.nix diff --git a/default.nix b/default.nix index 59903b1e2..c5f49a372 100644 --- a/default.nix +++ b/default.nix @@ -48,6 +48,7 @@ callPackage ./nix/tahoe-lafs.nix { # when files that make no difference to the package have changed. tahoe-lafs-src = pkgs.lib.cleanSource ./.; - # pycddl isn't packaged in nixpkgs so supply our own package of it. + # Some dependencies aren't packaged in nixpkgs so supply our own packages. pycddl = callPackage ./nix/pycddl.nix { }; + txi2p = callPackage ./nix/txi2p.nix { }; } diff --git a/nix/tahoe-lafs.nix b/nix/tahoe-lafs.nix index af0e4bf4d..386e3adc9 100644 --- a/nix/tahoe-lafs.nix +++ b/nix/tahoe-lafs.nix @@ -32,7 +32,7 @@ , txtorcon # i2p extra dependencies -, txi2p-tahoe +, txi2p # test dependencies , beautifulsoup4 @@ -56,7 +56,7 @@ let pythonExtraDependencies = { tor = [ txtorcon ]; - i2p = [ txi2p-tahoe ]; + i2p = [ txi2p ]; }; pythonPackageDependencies = [ diff --git a/nix/txi2p.nix b/nix/txi2p.nix new file mode 100644 index 000000000..a3a5fea3a --- /dev/null +++ b/nix/txi2p.nix @@ -0,0 +1,14 @@ +{ fetchPypi, buildPythonPackage, parsley, twisted, unittestCheckHook }: +buildPythonPackage rec { + pname = "txi2p-tahoe"; + version = "0.3.7"; + + src = fetchPypi { + inherit pname version; + hash = "sha256-+Vs9zaFS+ACI14JNxEme93lnWmncdZyFAmnTH0yhOiY="; + }; + + propagatedBuildInputs = [ twisted parsley ]; + checkInputs = [ unittestCheckHook ]; + pythonImportsCheck = [ "parsley" "ometa"]; +} From 02904a363b0bf73119ad05bfc0a889231eb53f78 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 13 Mar 2023 16:19:07 -0400 Subject: [PATCH 1530/2309] Drop nixpkgs 21.11 - it is missing some stuff we need Not only some nixpkgs facilities but it also includes a rustc that's too old to build pycddl. --- .circleci/config.yml | 6 +----- nix/sources.json | 12 ------------ nix/tahoe-lafs.nix | 14 ++++++++++++-- nix/txi2p.nix | 7 ++++++- 4 files changed, 19 insertions(+), 20 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index d07383b84..82bb263f9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -70,13 +70,9 @@ workflows: - "oraclelinux-8": {} - - "nixos": - name: "NixOS 21.11" - nixpkgs: "21.11" - - "nixos": name: "NixOS 22.11" - nixpkgs: "22.11" + nixpkgs: "21.11" - "nixos": name: "NixOS unstable" diff --git a/nix/sources.json b/nix/sources.json index bcac22174..ddf05d39d 100644 --- a/nix/sources.json +++ b/nix/sources.json @@ -11,18 +11,6 @@ "url": "https://github.com/nmattia/niv/archive/5830a4dd348d77e39a0f3c4c762ff2663b602d4c.tar.gz", "url_template": "https://github.com///archive/.tar.gz" }, - "nixpkgs-21.11": { - "branch": "nixos-21.11", - "description": "Nix Packages collection", - "homepage": "", - "owner": "NixOS", - "repo": "nixpkgs", - "rev": "838eefb4f93f2306d4614aafb9b2375f315d917f", - "sha256": "1bm8cmh1wx4h8b4fhbs75hjci3gcrpi7k1m1pmiy3nc0gjim9vkg", - "type": "tarball", - "url": "https://github.com/NixOS/nixpkgs/archive/838eefb4f93f2306d4614aafb9b2375f315d917f.tar.gz", - "url_template": "https://github.com///archive/.tar.gz" - }, "nixpkgs-22.11": { "branch": "nixos-22.11", "description": "Nix Packages collection", diff --git a/nix/tahoe-lafs.nix b/nix/tahoe-lafs.nix index 386e3adc9..11698f611 100644 --- a/nix/tahoe-lafs.nix +++ b/nix/tahoe-lafs.nix @@ -34,6 +34,15 @@ # i2p extra dependencies , txi2p +# twisted extra dependencies - if there is overlap with our dependencies we +# have to skip them since we can't have a name in the argument set twice. +, appdirs +, bcrypt +, idna +, pyasn1 +, pyopenssl +, service-identity + # test dependencies , beautifulsoup4 , fixtures @@ -81,8 +90,9 @@ let six treq twisted - (twisted.passthru.optional-dependencies.tls) - (twisted.passthru.optional-dependencies.conch) + # Get the dependencies for the Twisted extras we depend on, too. + twisted.passthru.optional-dependencies.tls + twisted.passthru.optional-dependencies.conch werkzeug zfec zope_interface diff --git a/nix/txi2p.nix b/nix/txi2p.nix index a3a5fea3a..c6b28aad4 100644 --- a/nix/txi2p.nix +++ b/nix/txi2p.nix @@ -1,4 +1,9 @@ -{ fetchPypi, buildPythonPackage, parsley, twisted, unittestCheckHook }: +{ fetchPypi +, buildPythonPackage +, parsley +, twisted +, unittestCheckHook +}: buildPythonPackage rec { pname = "txi2p-tahoe"; version = "0.3.7"; From b73045d93c1144ea1ef2ce9fb5129ce221e36943 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 13 Mar 2023 16:21:33 -0400 Subject: [PATCH 1531/2309] fix ci configuration --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 82bb263f9..b7009c2af 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -72,7 +72,7 @@ workflows: - "nixos": name: "NixOS 22.11" - nixpkgs: "21.11" + nixpkgs: "22.11" - "nixos": name: "NixOS unstable" From edd8e99178a711ae2d1c6765288872f3a6cbb0b9 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 13 Mar 2023 16:36:53 -0400 Subject: [PATCH 1532/2309] no more pypi-deps-db or mach-nix --- default.nix | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/default.nix b/default.nix index c5f49a372..21a17bdef 100644 --- a/default.nix +++ b/default.nix @@ -1,8 +1,7 @@ let # sources.nix contains information about which versions of some of our - # dependencies we should use. since we use it to pin nixpkgs and the PyPI - # package database, roughly all the rest of our dependencies are *also* - # pinned - indirectly. + # dependencies we should use. since we use it to pin nixpkgs, all the rest + # of our dependencies are *also* pinned - indirectly. # # sources.nix is managed using a tool called `niv`. as an example, to # update to the most recent version of nixpkgs from the 21.11 maintenance @@ -10,11 +9,6 @@ let # # niv update nixpkgs-21.11 # - # or, to update the PyPI package database -- which is necessary to make any - # newly released packages visible -- you likewise run: - # - # niv update pypi-deps-db - # # niv also supports chosing a specific revision, following a different # branch, etc. find complete documentation for the tool at # https://github.com/nmattia/niv From 93cd2aa354dee1777bdce719338fc61ce1209b04 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 13 Mar 2023 16:43:22 -0400 Subject: [PATCH 1533/2309] re-enable nix-based test suite runs --- .circleci/config.yml | 2 +- nix/tahoe-lafs.nix | 10 +++++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index b7009c2af..8b6dc8347 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -438,7 +438,7 @@ jobs: cache_if_able nix-build \ --cores 8 \ --argstr pkgsVersion "nixpkgs-<>" \ - tests.nix + nix/tests/ typechecks: docker: diff --git a/nix/tahoe-lafs.nix b/nix/tahoe-lafs.nix index 11698f611..ebb66a65e 100644 --- a/nix/tahoe-lafs.nix +++ b/nix/tahoe-lafs.nix @@ -2,6 +2,9 @@ , tahoe-lafs-src , extras +# control how the test suite is run +, doCheck ? false + # always dependencies , attrs , autobahn @@ -117,6 +120,11 @@ buildPythonPackage { inherit pname version; src = tahoe-lafs-src; buildInputs = pythonPackageDependencies; + + inherit doCheck; checkInputs = pythonCheckDependencies; - checkPhase = "TAHOE_LAFS_HYPOTHESIS_PROFILE=ci python -m twisted.trial -j $NIX_BUILD_CORES allmydata"; + checkPhase = '' + export TAHOE_LAFS_HYPOTHESIS_PROFILE=ci + python -m twisted.trial -j $NIX_BUILD_CORES allmydata + ''; } From f59c6a3acfaa6f2e60455ff5a91cb67015ab4a5c Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 13 Mar 2023 17:01:57 -0400 Subject: [PATCH 1534/2309] Get our dependencies at runtime, too. --- nix/tahoe-lafs.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nix/tahoe-lafs.nix b/nix/tahoe-lafs.nix index ebb66a65e..6d743c5b2 100644 --- a/nix/tahoe-lafs.nix +++ b/nix/tahoe-lafs.nix @@ -119,7 +119,7 @@ in buildPythonPackage { inherit pname version; src = tahoe-lafs-src; - buildInputs = pythonPackageDependencies; + propagatedBuildInputs = pythonPackageDependencies; inherit doCheck; checkInputs = pythonCheckDependencies; From 17a2c32e1f7c5eaf75340417da5741445e50b628 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 13 Mar 2023 17:02:10 -0400 Subject: [PATCH 1535/2309] Avoid colliding with the "extra" package in nixpkgs :/ --- default.nix | 14 +++++++------- nix/tahoe-lafs.nix | 4 ++-- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/default.nix b/default.nix index 21a17bdef..87f398ca5 100644 --- a/default.nix +++ b/default.nix @@ -23,18 +23,18 @@ in , pythonVersion ? "python310" # a string choosing the python derivation from # nixpkgs to target -, extras ? [ "tor" "i2p" ] # a list of strings identifying tahoe-lafs extras, - # the dependencies of which the resulting package - # will also depend on. Include all of the runtime - # extras by default because the incremental cost of - # including them is a lot smaller than the cost of - # re-building the whole thing to add them. +, extrasNames ? [ "tor" "i2p" ] # a list of strings identifying tahoe-lafs extras, + # the dependencies of which the resulting + # package will also depend on. Include all of the + # runtime extras by default because the incremental + # cost of including them is a lot smaller than the + # cost of re-building the whole thing to add them. }: with pkgs.${pythonVersion}.pkgs; callPackage ./nix/tahoe-lafs.nix { # Select whichever package extras were requested. - inherit extras; + inherit extrasNames; # Define the location of the Tahoe-LAFS source to be packaged. Clean up as # many of the non-source files (eg the `.git` directory, `~` backup files, diff --git a/nix/tahoe-lafs.nix b/nix/tahoe-lafs.nix index 6d743c5b2..ec1c83f73 100644 --- a/nix/tahoe-lafs.nix +++ b/nix/tahoe-lafs.nix @@ -1,6 +1,6 @@ { buildPythonPackage , tahoe-lafs-src -, extras +, extrasNames # control how the test suite is run , doCheck ? false @@ -99,7 +99,7 @@ let werkzeug zfec zope_interface - ] ++ pickExtraDependencies pythonExtraDependencies extras; + ] ++ pickExtraDependencies pythonExtraDependencies extrasNames; pythonCheckDependencies = [ beautifulsoup4 From 1e0e5304d7d7c8f9300fd9338b76a3ebf4195dcc Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 13 Mar 2023 17:02:50 -0400 Subject: [PATCH 1536/2309] actually add the test expression --- nix/tests/default.nix | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 nix/tests/default.nix diff --git a/nix/tests/default.nix b/nix/tests/default.nix new file mode 100644 index 000000000..0990c9b2b --- /dev/null +++ b/nix/tests/default.nix @@ -0,0 +1,4 @@ +# Build the package with the test suite enabled. +(import ../../. {}).override { + doCheck = true; +} From 0d11c6c07655f97ebedb102608fd9cd5b13a9f84 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 13 Mar 2023 17:06:06 -0400 Subject: [PATCH 1537/2309] package metadata --- nix/tahoe-lafs.nix | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/nix/tahoe-lafs.nix b/nix/tahoe-lafs.nix index ec1c83f73..43f46092d 100644 --- a/nix/tahoe-lafs.nix +++ b/nix/tahoe-lafs.nix @@ -127,4 +127,11 @@ buildPythonPackage { export TAHOE_LAFS_HYPOTHESIS_PROFILE=ci python -m twisted.trial -j $NIX_BUILD_CORES allmydata ''; + + meta = with lib; { + homepage = "https://tahoe-lafs.org/"; + description = "secure, decentralized, fault-tolerant file store"; + # Also TGPPL + license = licenses.gpl2Plus; + }; } From 1b9936bd1b89e8b253aa003a7e129e72adbb100e Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 13 Mar 2023 17:10:29 -0400 Subject: [PATCH 1538/2309] get lib :/ --- nix/tahoe-lafs.nix | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nix/tahoe-lafs.nix b/nix/tahoe-lafs.nix index 43f46092d..62fba48b1 100644 --- a/nix/tahoe-lafs.nix +++ b/nix/tahoe-lafs.nix @@ -1,4 +1,5 @@ -{ buildPythonPackage +{ lib +, buildPythonPackage , tahoe-lafs-src , extrasNames From 1c926aeb869817c0a2aaa76786075b5459c396a2 Mon Sep 17 00:00:00 2001 From: dlee Date: Mon, 13 Mar 2023 16:23:28 -0500 Subject: [PATCH 1539/2309] Add space to return type --- src/allmydata/test/cli/wormholetesting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/cli/wormholetesting.py b/src/allmydata/test/cli/wormholetesting.py index 99e26e64b..eb6249a0d 100644 --- a/src/allmydata/test/cli/wormholetesting.py +++ b/src/allmydata/test/cli/wormholetesting.py @@ -78,7 +78,7 @@ class MemoryWormholeServer(object): stderr: TextIO=stderr, _eventual_queue: Optional[Any]=None, _enable_dilate: bool=False, - )-> _MemoryWormhole: + ) -> _MemoryWormhole: """ Create a wormhole. It will be able to connect to other wormholes created by this instance (and constrained by the normal appid/relay_url From 6e6fc2d3070172dc9537b3379c305c070f0f41ea Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 13 Mar 2023 19:02:54 -0400 Subject: [PATCH 1540/2309] The Nix test expression includes a package build, so just do that --- .circleci/config.yml | 20 ++------------------ 1 file changed, 2 insertions(+), 18 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 8b6dc8347..a7dba614b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -412,29 +412,13 @@ jobs: --run 'python setup.py update_version' - "run": - name: "Build" + name: "Test" command: | # CircleCI build environment looks like it has a zillion and a # half cores. Don't let Nix autodetect this high core count # because it blows up memory usage and fails the test run. Pick a # number of cores that suites the build environment we're paying - # for (the free one!). - # - # Also, let it run more than one job at a time because we have to - # build a couple simple little dependencies that don't take - # advantage of multiple cores and we get a little speedup by doing - # them in parallel. - source .circleci/lib.sh - cache_if_able nix-build \ - --cores 3 \ - --max-jobs 2 \ - --argstr pkgsVersion "nixpkgs-<>" - - - "run": - name: "Test" - command: | - # Let it go somewhat wild for the test suite itself - source .circleci/lib.sh + # for (the free one!).h cache_if_able nix-build \ --cores 8 \ --argstr pkgsVersion "nixpkgs-<>" \ From 99559638b93b2ac08d6740d26e2bfbe5dee43fc5 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 13 Mar 2023 19:03:26 -0400 Subject: [PATCH 1541/2309] remove some repetition in the package definition --- default.nix | 12 ++++++--- nix/tahoe-lafs.nix | 64 ++++------------------------------------------ 2 files changed, 13 insertions(+), 63 deletions(-) diff --git a/default.nix b/default.nix index 87f398ca5..3ecc88ec1 100644 --- a/default.nix +++ b/default.nix @@ -31,7 +31,13 @@ in # cost of re-building the whole thing to add them. }: -with pkgs.${pythonVersion}.pkgs; +with (pkgs.${pythonVersion}.override { + packageOverrides = self: super: { + # Some dependencies aren't packaged in nixpkgs so supply our own packages. + pycddl = self.callPackage ./nix/pycddl.nix { }; + txi2p = self.callPackage ./nix/txi2p.nix { }; + }; +}).pkgs; callPackage ./nix/tahoe-lafs.nix { # Select whichever package extras were requested. inherit extrasNames; @@ -42,7 +48,5 @@ callPackage ./nix/tahoe-lafs.nix { # when files that make no difference to the package have changed. tahoe-lafs-src = pkgs.lib.cleanSource ./.; - # Some dependencies aren't packaged in nixpkgs so supply our own packages. - pycddl = callPackage ./nix/pycddl.nix { }; - txi2p = callPackage ./nix/txi2p.nix { }; + doCheck = false; } diff --git a/nix/tahoe-lafs.nix b/nix/tahoe-lafs.nix index 62fba48b1..380260c70 100644 --- a/nix/tahoe-lafs.nix +++ b/nix/tahoe-lafs.nix @@ -1,65 +1,11 @@ { lib +, pythonPackages , buildPythonPackage , tahoe-lafs-src , extrasNames # control how the test suite is run -, doCheck ? false - -# always dependencies -, attrs -, autobahn -, cbor2 -, click -, collections-extended -, cryptography -, distro -, eliot -, filelock -, foolscap -, future -, klein -, magic-wormhole -, netifaces -, psutil -, pycddl -, pyrsistent -, pyutil -, six -, treq -, twisted -, werkzeug -, zfec -, zope_interface - -# tor extra dependencies -, txtorcon - -# i2p extra dependencies -, txi2p - -# twisted extra dependencies - if there is overlap with our dependencies we -# have to skip them since we can't have a name in the argument set twice. -, appdirs -, bcrypt -, idna -, pyasn1 -, pyopenssl -, service-identity - -# test dependencies -, beautifulsoup4 -, fixtures -, hypothesis -, mock -, paramiko -, prometheus-client -, pytest -, pytest-timeout -, pytest-twisted -, tenacity -, testtools -, towncrier +, doCheck }: let pname = "tahoe-lafs"; @@ -67,12 +13,12 @@ let pickExtraDependencies = deps: extras: builtins.foldl' (accum: extra: accum ++ deps.${extra}) [] extras; - pythonExtraDependencies = { + pythonExtraDependencies = with pythonPackages; { tor = [ txtorcon ]; i2p = [ txi2p ]; }; - pythonPackageDependencies = [ + pythonPackageDependencies = with pythonPackages; [ attrs autobahn cbor2 @@ -102,7 +48,7 @@ let zope_interface ] ++ pickExtraDependencies pythonExtraDependencies extrasNames; - pythonCheckDependencies = [ + pythonCheckDependencies = with pythonPackages; [ beautifulsoup4 fixtures hypothesis From d648592a871520415196c6a1ee3b68081392a3da Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 13 Mar 2023 19:43:16 -0400 Subject: [PATCH 1542/2309] get the helper ... --- .circleci/config.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index a7dba614b..b1e121337 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -418,7 +418,8 @@ jobs: # half cores. Don't let Nix autodetect this high core count # because it blows up memory usage and fails the test run. Pick a # number of cores that suites the build environment we're paying - # for (the free one!).h + # for (the free one!). + source .circleci/lib.sh cache_if_able nix-build \ --cores 8 \ --argstr pkgsVersion "nixpkgs-<>" \ From d7018905b9537befb47753da92ad3f975f9951de Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 14 Mar 2023 09:57:29 -0400 Subject: [PATCH 1543/2309] Switch away from using stdin, it's flaky on Windows. --- integration/test_get_put.py | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/integration/test_get_put.py b/integration/test_get_put.py index 65020429e..1b6c30072 100644 --- a/integration/test_get_put.py +++ b/integration/test_get_put.py @@ -3,7 +3,7 @@ Integration tests for getting and putting files, including reading from stdin and stdout. """ -from subprocess import Popen, PIPE, check_output +from subprocess import Popen, PIPE, check_output, check_call import sys import pytest @@ -53,23 +53,38 @@ def test_put_from_stdin(alice, get_put_alias, tmpdir): def test_get_to_stdout(alice, get_put_alias, tmpdir): """ It's possible to upload a file, and then download it to stdout. - - We test with large file, this time. """ tempfile = tmpdir.join("file") - large_data = DATA * 1_000_000 with tempfile.open("wb") as f: - f.write(large_data) + f.write(DATA) cli(alice, "put", str(tempfile), "getput:tostdout") p = Popen( ["tahoe", "--node-directory", alice.node_dir, "get", "getput:tostdout", "-"], stdout=PIPE ) - assert p.stdout.read() == large_data + assert p.stdout.read() == DATA assert p.wait() == 0 +def test_large_file(alice, get_put_alias, tmp_path): + """ + It's possible to upload and download a larger file. + + We avoid stdin/stdout since that's flaky on Windows. + """ + tempfile = tmp_path / "file" + with tempfile.open("wb") as f: + f.write(DATA * 1_000_000) + cli(alice, "put", str(tempfile), "getput:largefile") + + outfile = tmp_path / "out" + check_call( + ["tahoe", "--node-directory", alice.node_dir, "get", "getput:largefile", str(outfile)], + ) + assert outfile.read_bytes() == tempfile.read_bytes() + + @pytest.mark.skipif( sys.platform.startswith("win"), reason="reconfigure() has issues on Windows" From ea5928ce537a97b7fa850dc26553c12566e3dc5f Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 14 Mar 2023 10:19:27 -0400 Subject: [PATCH 1544/2309] news fragment --- newsfragments/3987.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3987.minor diff --git a/newsfragments/3987.minor b/newsfragments/3987.minor new file mode 100644 index 000000000..e69de29bb From ff50bfe5c4d8e05c02a3fa58d754779855988375 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 14 Mar 2023 10:19:49 -0400 Subject: [PATCH 1545/2309] Accept all the arguments default.nix accepts, too --- nix/tests/default.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nix/tests/default.nix b/nix/tests/default.nix index 0990c9b2b..2eb490718 100644 --- a/nix/tests/default.nix +++ b/nix/tests/default.nix @@ -1,4 +1,4 @@ # Build the package with the test suite enabled. -(import ../../. {}).override { +args@{...}: (import ../../. args).override { doCheck = true; } From 10414e80eda31e956631d9f336858a58cc5fdad9 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 14 Mar 2023 10:25:02 -0400 Subject: [PATCH 1546/2309] Remove some unnecessary hierarchy I thought `default.nix` was handled specially for the purposes of automatic parameter population but it isn't. Instead, you just need this `args@{...}` pattern. --- .circleci/config.yml | 2 +- nix/{tests/default.nix => tests.nix} | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) rename nix/{tests/default.nix => tests.nix} (60%) diff --git a/.circleci/config.yml b/.circleci/config.yml index b1e121337..ab0573a3f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -423,7 +423,7 @@ jobs: cache_if_able nix-build \ --cores 8 \ --argstr pkgsVersion "nixpkgs-<>" \ - nix/tests/ + nix/tests.nix typechecks: docker: diff --git a/nix/tests/default.nix b/nix/tests.nix similarity index 60% rename from nix/tests/default.nix rename to nix/tests.nix index 2eb490718..42ca9f882 100644 --- a/nix/tests/default.nix +++ b/nix/tests.nix @@ -1,4 +1,4 @@ # Build the package with the test suite enabled. -args@{...}: (import ../../. args).override { +args@{...}: (import ../. args).override { doCheck = true; } From f8ea650b922ba5622debdf6a49121388e2a32e2d Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 14 Mar 2023 12:02:32 -0400 Subject: [PATCH 1547/2309] Wait for current loop iteration to finish before moving on to next iteration. --- src/allmydata/storage_client.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 5dd906005..faa48710f 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -1239,9 +1239,13 @@ class HTTPNativeStorageServer(service.MultiService): self._failed_to_connect ) - # TODO Make sure LoopingCall waits for the above timeout for looping again: - # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3981 - #return self._connecting_deferred or maye await it + # We don't want to do another iteration of the loop until this + # iteration has finished, so wait here: + try: + if self._connecting_deferred is not None: + await self._connecting_deferred + except Exception as e: + log.msg(f"Failed to connect to a HTTP storage server: {e}", level=log.CURIOUS) def stopService(self): if self._connecting_deferred is not None: From dd07a39399709b59d7c1c1e2e0cb5d3ac4d6ff65 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 14 Mar 2023 13:01:10 -0400 Subject: [PATCH 1548/2309] Don't bother with persistent connections when testing NURLs. --- src/allmydata/storage/http_client.py | 10 ++++++---- src/allmydata/storage_client.py | 6 +++++- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 90bda7fc0..3edf5f835 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -311,18 +311,20 @@ class StorageClient(object): @classmethod def from_nurl( - cls, - nurl: DecodedURL, - reactor, + cls, nurl: DecodedURL, reactor, persistent=True, retryAutomatically=True ) -> StorageClient: """ Create a ``StorageClient`` for the given NURL. + + ``persistent`` and ``retryAutomatically`` arguments are passed to the + new HTTPConnectionPool. """ assert nurl.fragment == "v=1" assert nurl.scheme == "pb" swissnum = nurl.path[0].encode("ascii") certificate_hash = nurl.user.encode("ascii") - pool = HTTPConnectionPool(reactor) + pool = HTTPConnectionPool(reactor, persistent=persistent) + pool.retryAutomatically = retryAutomatically pool.maxPersistentPerHost = 20 if cls.TEST_MODE_REGISTER_HTTP_POOL is not None: diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index faa48710f..2888b10e7 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -1203,8 +1203,12 @@ class HTTPNativeStorageServer(service.MultiService): # the HTTP server to talk to, we don't have connection status # updates... https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3978 def request(reactor, nurl: DecodedURL): + # Since we're just using this one off to check if the NURL + # works, no need for persistent pool or other fanciness. return StorageClientGeneral( - StorageClient.from_nurl(nurl, reactor) + StorageClient.from_nurl( + nurl, reactor, persistent=False, retryAutomatically=False + ) ).get_version() # LoopingCall.stop() doesn't cancel Deferreds, unfortunately: From 24212b412cba52481695b85e057da1ac04a96d67 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 14 Mar 2023 13:01:45 -0400 Subject: [PATCH 1549/2309] Fix 3.11 runs. --- tox.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/tox.ini b/tox.ini index 3e2dacbb2..538cded80 100644 --- a/tox.ini +++ b/tox.ini @@ -10,6 +10,7 @@ python = 3.8: py38-coverage 3.9: py39-coverage 3.10: py310-coverage + 3.11: py311-coverage pypy-3.8: pypy38 pypy-3.9: pypy39 From 505032d0cac83111341304335c88d2ee95afdd9f Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 14 Mar 2023 20:38:46 -0400 Subject: [PATCH 1550/2309] a note about what this is and what's going on upstream --- nix/pycddl.nix | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/nix/pycddl.nix b/nix/pycddl.nix index 0f6a0329e..563936cbb 100644 --- a/nix/pycddl.nix +++ b/nix/pycddl.nix @@ -1,3 +1,10 @@ +# package https://gitlab.com/tahoe-lafs/pycddl +# +# also in the process of being pushed upstream +# https://github.com/NixOS/nixpkgs/pull/221220 +# +# we should switch to the upstream package when it is available from our +# minimum version of nixpkgs { lib, fetchPypi, buildPythonPackage, rustPlatform }: buildPythonPackage rec { pname = "pycddl"; From 324a5ba397bfe211311cf7db1da7f70adc14b85d Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 14 Mar 2023 20:40:08 -0400 Subject: [PATCH 1551/2309] give the reader a hint about the interpretation of `./.` --- default.nix | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/default.nix b/default.nix index 3ecc88ec1..b87a6730a 100644 --- a/default.nix +++ b/default.nix @@ -42,10 +42,11 @@ callPackage ./nix/tahoe-lafs.nix { # Select whichever package extras were requested. inherit extrasNames; - # Define the location of the Tahoe-LAFS source to be packaged. Clean up as - # many of the non-source files (eg the `.git` directory, `~` backup files, - # nix's own `result` symlink, etc) as possible to avoid needing to re-build - # when files that make no difference to the package have changed. + # Define the location of the Tahoe-LAFS source to be packaged (the same + # directory as contains this file). Clean up as many of the non-source + # files (eg the `.git` directory, `~` backup files, nix's own `result` + # symlink, etc) as possible to avoid needing to re-build when files that + # make no difference to the package have changed. tahoe-lafs-src = pkgs.lib.cleanSource ./.; doCheck = false; From aaaec9a69dbed5c0b97566d3be8bde79fd9765e2 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 15 Mar 2023 15:42:52 -0400 Subject: [PATCH 1552/2309] package update instructions --- nix/pycddl.nix | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/nix/pycddl.nix b/nix/pycddl.nix index 563936cbb..703f00595 100644 --- a/nix/pycddl.nix +++ b/nix/pycddl.nix @@ -4,7 +4,29 @@ # https://github.com/NixOS/nixpkgs/pull/221220 # # we should switch to the upstream package when it is available from our -# minimum version of nixpkgs +# minimum version of nixpkgs. +# +# if you need to update this package to a new pycddl release then +# +# 1. change value given to `buildPythonPackage` for `version` to match the new +# release +# +# 2. change the value given to `fetchPypi` for `sha256` to `lib.fakeHash` +# +# 3. run `nix-build` +# +# 4. there will be an error about a hash mismatch. change the value given to +# `fetchPypi` for `sha256` to the "actual" hash value report. +# +# 5. change the value given to `cargoDeps` for `hash` to lib.fakeHash`. +# +# 6. run `nix-build` +# +# 7. there will be an error about a hash mismatch. change the value given to +# `cargoDeps` for `hash` to the "actual" hash value report. +# +# 8. run `nix-build`. it should succeed. if it does not, seek assistance. +# { lib, fetchPypi, buildPythonPackage, rustPlatform }: buildPythonPackage rec { pname = "pycddl"; From 52f43cefea6b447cda47cd5e75df4937190f8bd6 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 15 Mar 2023 15:44:45 -0400 Subject: [PATCH 1553/2309] Add 3.11. --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 538cded80..382ba973e 100644 --- a/tox.ini +++ b/tox.ini @@ -18,7 +18,7 @@ python = twisted = 1 [tox] -envlist = typechecks,codechecks,py{38,39,310}-{coverage},pypy27,pypy38,pypy39,integration +envlist = typechecks,codechecks,py{38,39,310,311}-{coverage},pypy27,pypy38,pypy39,integration minversion = 2.4 [testenv] From 2a8867f6cf0ca52ef62361b049babafffc956705 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 15 Mar 2023 15:47:43 -0400 Subject: [PATCH 1554/2309] more packaging instructions --- nix/txi2p.nix | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/nix/txi2p.nix b/nix/txi2p.nix index c6b28aad4..3464b7b3d 100644 --- a/nix/txi2p.nix +++ b/nix/txi2p.nix @@ -1,3 +1,23 @@ +# package https://github.com/tahoe-lafs/txi2p +# +# if you need to update this package to a new txi2p release then +# +# 1. change value given to `buildPythonPackage` for `version` to match the new +# release +# +# 2. change the value given to `fetchPypi` for `sha256` to `lib.fakeHash` +# +# 3. run `nix-build` +# +# 4. there will be an error about a hash mismatch. change the value given to +# `fetchPypi` for `sha256` to the "actual" hash value report. +# +# 5. if there are new runtime dependencies then add them to the argument list +# at the top. if there are new test dependencies add them to the +# `checkInputs` list. +# +# 6. run `nix-build`. it should succeed. if it does not, seek assistance. +# { fetchPypi , buildPythonPackage , parsley From 58e0e3def7fc25a8ed15f7d2203adf8eed0625e4 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 16 Mar 2023 08:52:15 -0400 Subject: [PATCH 1555/2309] see if this fixes the AttributeError --- integration/grid.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/integration/grid.py b/integration/grid.py index 4e5d8a900..cec30b79b 100644 --- a/integration/grid.py +++ b/integration/grid.py @@ -191,7 +191,11 @@ def create_storage_server(reactor, request, temp_dir, introducer, flog_gatherer, ) storage = StorageServer( process=node_process, - protocol=node_process.transport._protocol, + # node_process is a TahoeProcess. its transport is an + # IProcessTransport. in practice, this means it is a + # twisted.internet._baseprocess.BaseProcess. BaseProcess records the + # process protocol as its proto attribute. + protocol=node_process.transport.proto, ) returnValue(storage) From c9dba4d0a4d3074e3de1ce77c05065e411b6f0b8 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 16 Mar 2023 09:09:25 -0400 Subject: [PATCH 1556/2309] Fix a couple other `_protocol` attributes --- integration/grid.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/integration/grid.py b/integration/grid.py index cec30b79b..9b347cf6f 100644 --- a/integration/grid.py +++ b/integration/grid.py @@ -231,7 +231,7 @@ class Client(object): reactor, self.process.node_dir, request, None, ) self.process = process - self.protocol = self.process.transport._protocol + self.protocol = self.process.transport.proto yield await_client_ready(self.process, minimum_number_of_servers=servers) @@ -253,7 +253,7 @@ def create_client(reactor, request, temp_dir, introducer, flog_gatherer, name, w returnValue( Client( process=node_process, - protocol=node_process.transport._protocol, + protocol=node_process.transport.proto, ) ) From e6832dd71ca1ad70ccf856f8d6f74751c82b5a0c Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 16 Mar 2023 09:37:54 -0400 Subject: [PATCH 1557/2309] another one --- integration/grid.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration/grid.py b/integration/grid.py index 9b347cf6f..8c7e7624b 100644 --- a/integration/grid.py +++ b/integration/grid.py @@ -174,7 +174,7 @@ class StorageServer(object): self.process = yield _run_node( reactor, self.process.node_dir, request, None, ) - self.protocol = self.process.transport._protocol + self.protocol = self.process.transport.proto yield await_client_ready(self.process) From a24e6bd7f94311b185072132b3372fd4a0afb723 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 16 Mar 2023 16:31:28 -0400 Subject: [PATCH 1558/2309] Try to rewrite test_get_put.py::test_large_file into system-style test. --- src/allmydata/test/test_system.py | 32 +++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/src/allmydata/test/test_system.py b/src/allmydata/test/test_system.py index d11a6e866..b7873f14f 100644 --- a/src/allmydata/test/test_system.py +++ b/src/allmydata/test/test_system.py @@ -15,11 +15,15 @@ from past.builtins import chr as byteschr, long from six import ensure_text import os, re, sys, time, json +from subprocess import check_call +from pathlib import Path +from tempfile import mkdtemp from bs4 import BeautifulSoup from twisted.trial import unittest from twisted.internet import defer +from twisted.internet.threads import deferToThread from allmydata import uri from allmydata.storage.mutable import MutableShareFile @@ -1830,6 +1834,34 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase): d.addCallback(_got_lit_filenode) return d + async def test_immutable_upload_download(self): + """ + A reproducer for issue 3988: upload a large file and then download it. + """ + DATA = b"abc123 this is not utf-8 decodable \xff\x00\x33 \x11" * 1_000_000 + await self.set_up_nodes() + + async def run(*args): + await deferToThread(check_call, ["tahoe", "--node-directory", self.getdir("client0")] + list(args)) + + for c in self.clients: + c.encoding_params['k'] = 2 + c.encoding_params['happy'] = 3 + c.encoding_params['n'] = 4 + + await run("create-alias", "getput") + + tmp_path = Path(mkdtemp()) + tempfile = tmp_path / "input" + + with tempfile.open("wb") as f: + f.write(DATA) + await run("put", str(tempfile), "getput:largefile") + + outfile = tmp_path / "out" + await run("get", "getput:largefile", str(outfile)) + self.assertEqual(outfile.read_bytes(), tempfile.read_bytes()) + class Connections(SystemTestMixin, unittest.TestCase): FORCE_FOOLSCAP_FOR_STORAGE = True From a3ebd21b25c29fe8a871ae53967e0ed3f29be5d4 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 17 Mar 2023 15:30:14 -0400 Subject: [PATCH 1559/2309] implement retry ourselves, don't depend on tenacity --- setup.py | 1 - src/allmydata/test/test_iputil.py | 55 ++++++++++++++++++++++++------- 2 files changed, 44 insertions(+), 12 deletions(-) diff --git a/setup.py b/setup.py index 82ff45764..152c49f0e 100644 --- a/setup.py +++ b/setup.py @@ -413,7 +413,6 @@ setup(name="tahoe-lafs", # also set in __init__.py "beautifulsoup4", "html5lib", "junitxml", - "tenacity", # Pin old version until # https://github.com/paramiko/paramiko/issues/1961 is fixed. "paramiko < 2.9", diff --git a/src/allmydata/test/test_iputil.py b/src/allmydata/test/test_iputil.py index 081c80ee3..c060fcc04 100644 --- a/src/allmydata/test/test_iputil.py +++ b/src/allmydata/test/test_iputil.py @@ -4,18 +4,14 @@ Tests for allmydata.util.iputil. Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals - -from future.utils import PY2, native_str -if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 +from __future__ import annotations import os, socket import gc +from functools import wraps +from typing import TypeVar, Callable +from typing_extensions import TypeAlias from testtools.matchers import ( MatchesAll, IsInstance, @@ -25,8 +21,6 @@ from testtools.matchers import ( from twisted.trial import unittest -from tenacity import retry, stop_after_attempt - from foolscap.api import Tub from allmydata.util import iputil, gcutil @@ -39,6 +33,45 @@ from .common import ( SyncTestCase, ) +T = TypeVar("T") + +TestFunction: TypeAlias = Callable[[], T] +Decorator: TypeAlias = Callable[[TestFunction[T]], TestFunction[T]] + +def retry(stop: Callable[[], bool]) -> Decorator[T]: + """ + Call a function until the predicate says to stop or the function stops + raising an exception. + + :param stop: A callable to call after the decorated function raises an + exception. The decorated function will be called again if ``stop`` + returns ``False``. + + :return: A decorator function. + """ + def decorate(f: TestFunction[T]) -> TestFunction[T]: + @wraps(f) + def decorator(self) -> T: + while True: + try: + return f(self) + except Exception: + if stop(): + raise + return decorator + return decorate + +def stop_after_attempt(limit: int) -> Callable[[], bool]: + """ + Stop after ``limit`` calls. + """ + counter = 0 + def check(): + nonlocal counter + counter += 1 + return counter < limit + return check + class ListenOnUsed(unittest.TestCase): """Tests for listenOnUnused.""" @@ -127,7 +160,7 @@ class GetLocalAddressesSyncTests(SyncTestCase): IsInstance(list), AllMatch( MatchesAll( - IsInstance(native_str), + IsInstance(str), MatchesPredicate( lambda addr: socket.inet_pton(socket.AF_INET, addr), "%r is not an IPv4 address.", From a9f34655686764e633be4a6ccb8f2d79b841a291 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 17 Mar 2023 15:31:07 -0400 Subject: [PATCH 1560/2309] news fragment --- newsfragments/3989.installation | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/3989.installation diff --git a/newsfragments/3989.installation b/newsfragments/3989.installation new file mode 100644 index 000000000..a2155b65c --- /dev/null +++ b/newsfragments/3989.installation @@ -0,0 +1 @@ +tenacity is no longer a dependency. From 5cf892b441daf77ad5efada4a785dfa4a0e2ecf6 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 17 Mar 2023 15:32:13 -0400 Subject: [PATCH 1561/2309] Also remove it from the Nix packaging --- nix/tahoe-lafs.nix | 1 - 1 file changed, 1 deletion(-) diff --git a/nix/tahoe-lafs.nix b/nix/tahoe-lafs.nix index 380260c70..5986db420 100644 --- a/nix/tahoe-lafs.nix +++ b/nix/tahoe-lafs.nix @@ -58,7 +58,6 @@ let pytest pytest-timeout pytest-twisted - tenacity testtools towncrier ]; From 6a4346587cf06f7603572796daf4851bd98a1415 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 17 Mar 2023 15:46:27 -0400 Subject: [PATCH 1562/2309] Fix the type annotations --- src/allmydata/test/test_iputil.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/src/allmydata/test/test_iputil.py b/src/allmydata/test/test_iputil.py index c060fcc04..26274830f 100644 --- a/src/allmydata/test/test_iputil.py +++ b/src/allmydata/test/test_iputil.py @@ -11,7 +11,6 @@ import gc from functools import wraps from typing import TypeVar, Callable -from typing_extensions import TypeAlias from testtools.matchers import ( MatchesAll, IsInstance, @@ -33,12 +32,10 @@ from .common import ( SyncTestCase, ) -T = TypeVar("T") +T = TypeVar("T", contravariant=True) +U = TypeVar("U", covariant=True) -TestFunction: TypeAlias = Callable[[], T] -Decorator: TypeAlias = Callable[[TestFunction[T]], TestFunction[T]] - -def retry(stop: Callable[[], bool]) -> Decorator[T]: +def retry(stop: Callable[[], bool]) -> Callable[[Callable[[T], U]], Callable[[T], U]]: """ Call a function until the predicate says to stop or the function stops raising an exception. @@ -49,9 +46,9 @@ def retry(stop: Callable[[], bool]) -> Decorator[T]: :return: A decorator function. """ - def decorate(f: TestFunction[T]) -> TestFunction[T]: + def decorate(f: Callable[[T], U]) -> Callable[[T], U]: @wraps(f) - def decorator(self) -> T: + def decorator(self: T) -> U: while True: try: return f(self) From 61d9d82c55644b3a786b3b05725438a3cff02a18 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 20 Mar 2023 15:02:35 -0400 Subject: [PATCH 1563/2309] Make await_client_ready() non-blocking. --- integration/conftest.py | 6 +++--- integration/test_get_put.py | 4 ---- integration/test_servers_of_happiness.py | 2 +- integration/test_tor.py | 4 ++-- integration/util.py | 11 +++++------ newsfragments/3988.minor | 0 6 files changed, 11 insertions(+), 16 deletions(-) create mode 100644 newsfragments/3988.minor diff --git a/integration/conftest.py b/integration/conftest.py index dc0107eea..33e7998c1 100644 --- a/integration/conftest.py +++ b/integration/conftest.py @@ -393,7 +393,7 @@ def alice( finalize=False, ) ) - await_client_ready(process) + pytest_twisted.blockon(await_client_ready(process)) # 1. Create a new RW directory cap: cli(process, "create-alias", "test") @@ -424,7 +424,7 @@ alice-key ssh-rsa {ssh_public_key} {rwcap} # 4. Restart the node with new SFTP config. pytest_twisted.blockon(process.restart_async(reactor, request)) - await_client_ready(process) + pytest_twisted.blockon(await_client_ready(process)) print(f"Alice pid: {process.transport.pid}") return process @@ -439,7 +439,7 @@ def bob(reactor, temp_dir, introducer_furl, flog_gatherer, storage_nodes, reques storage=False, ) ) - await_client_ready(process) + pytest_twisted.blockon(await_client_ready(process)) return process diff --git a/integration/test_get_put.py b/integration/test_get_put.py index 1b6c30072..927ec622b 100644 --- a/integration/test_get_put.py +++ b/integration/test_get_put.py @@ -85,10 +85,6 @@ def test_large_file(alice, get_put_alias, tmp_path): assert outfile.read_bytes() == tempfile.read_bytes() -@pytest.mark.skipif( - sys.platform.startswith("win"), - reason="reconfigure() has issues on Windows" -) @ensureDeferred async def test_upload_download_immutable_different_default_max_segment_size(alice, get_put_alias, tmpdir, request): """ diff --git a/integration/test_servers_of_happiness.py b/integration/test_servers_of_happiness.py index b9de0c075..c63642066 100644 --- a/integration/test_servers_of_happiness.py +++ b/integration/test_servers_of_happiness.py @@ -31,7 +31,7 @@ def test_upload_immutable(reactor, temp_dir, introducer_furl, flog_gatherer, sto happy=7, total=10, ) - util.await_client_ready(edna) + yield util.await_client_ready(edna) node_dir = join(temp_dir, 'edna') diff --git a/integration/test_tor.py b/integration/test_tor.py index c78fa8098..901858347 100644 --- a/integration/test_tor.py +++ b/integration/test_tor.py @@ -42,8 +42,8 @@ if PY2: def test_onion_service_storage(reactor, request, temp_dir, flog_gatherer, tor_network, tor_introducer_furl): carol = yield _create_anonymous_node(reactor, 'carol', 8008, request, temp_dir, flog_gatherer, tor_network, tor_introducer_furl) dave = yield _create_anonymous_node(reactor, 'dave', 8009, request, temp_dir, flog_gatherer, tor_network, tor_introducer_furl) - util.await_client_ready(carol, minimum_number_of_servers=2) - util.await_client_ready(dave, minimum_number_of_servers=2) + yield util.await_client_ready(carol, minimum_number_of_servers=2) + yield util.await_client_ready(dave, minimum_number_of_servers=2) # ensure both nodes are connected to "a grid" by uploading # something via carol, and retrieve it using dave. diff --git a/integration/util.py b/integration/util.py index 04c925abf..c2befe47b 100644 --- a/integration/util.py +++ b/integration/util.py @@ -570,6 +570,10 @@ def await_client_ready(tahoe, timeout=10, liveness=60*2, minimum_number_of_serve We will try for up to `timeout` seconds for the above conditions to be true. Otherwise, an exception is raised """ + return deferToThread(_await_client_ready_blocking, tahoe, timeout, liveness, minimum_number_of_servers) + + +def _await_client_ready_blocking(tahoe, timeout, liveness, minimum_number_of_servers): start = time.time() while (time.time() - start) < float(timeout): try: @@ -792,16 +796,11 @@ async def reconfigure(reactor, request, node: TahoeProcess, ) if changed: - # TODO reconfigure() seems to have issues on Windows. If you need to - # use it there, delete this assert and try to figure out what's going - # on... - assert not sys.platform.startswith("win") - # restart the node print(f"Restarting {node.node_dir} for ZFEC reconfiguration") await node.restart_async(reactor, request) print("Restarted. Waiting for ready state.") - await_client_ready(node) + await await_client_ready(node) print("Ready.") else: print("Config unchanged, not restarting.") diff --git a/newsfragments/3988.minor b/newsfragments/3988.minor new file mode 100644 index 000000000..e69de29bb From aba60d271956d5100375c14b507a720582459860 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 20 Mar 2023 15:08:22 -0400 Subject: [PATCH 1564/2309] Run blocking tests in a thread. --- integration/test_get_put.py | 2 ++ integration/test_web.py | 11 +++++++++++ 2 files changed, 13 insertions(+) diff --git a/integration/test_get_put.py b/integration/test_get_put.py index 927ec622b..6b87c9b62 100644 --- a/integration/test_get_put.py +++ b/integration/test_get_put.py @@ -50,6 +50,7 @@ def test_put_from_stdin(alice, get_put_alias, tmpdir): assert read_bytes(tempfile) == DATA +@run_in_thread def test_get_to_stdout(alice, get_put_alias, tmpdir): """ It's possible to upload a file, and then download it to stdout. @@ -67,6 +68,7 @@ def test_get_to_stdout(alice, get_put_alias, tmpdir): assert p.wait() == 0 +@run_in_thread def test_large_file(alice, get_put_alias, tmp_path): """ It's possible to upload and download a larger file. diff --git a/integration/test_web.py b/integration/test_web.py index 95a09a5f5..b3c4a8e5f 100644 --- a/integration/test_web.py +++ b/integration/test_web.py @@ -18,6 +18,7 @@ import allmydata.uri from allmydata.util import jsonbytes as json from . import util +from .util import run_in_thread import requests import html5lib @@ -25,6 +26,7 @@ from bs4 import BeautifulSoup from pytest_twisted import ensureDeferred +@run_in_thread def test_index(alice): """ we can download the index file @@ -32,6 +34,7 @@ def test_index(alice): util.web_get(alice, u"") +@run_in_thread def test_index_json(alice): """ we can download the index file as json @@ -41,6 +44,7 @@ def test_index_json(alice): json.loads(data) +@run_in_thread def test_upload_download(alice): """ upload a file, then download it via readcap @@ -70,6 +74,7 @@ def test_upload_download(alice): assert str(data, "utf-8") == FILE_CONTENTS +@run_in_thread def test_put(alice): """ use PUT to create a file @@ -89,6 +94,7 @@ def test_put(alice): assert cap.needed_shares == int(cfg.get_config("client", "shares.needed")) +@run_in_thread def test_helper_status(storage_nodes): """ successfully GET the /helper_status page @@ -101,6 +107,7 @@ def test_helper_status(storage_nodes): assert str(dom.h1.string) == u"Helper Status" +@run_in_thread def test_deep_stats(alice): """ create a directory, do deep-stats on it and prove the /operations/ @@ -417,6 +424,7 @@ async def test_directory_deep_check(reactor, request, alice): assert dom is not None, "Operation never completed" +@run_in_thread def test_storage_info(storage_nodes): """ retrieve and confirm /storage URI for one storage node @@ -428,6 +436,7 @@ def test_storage_info(storage_nodes): ) +@run_in_thread def test_storage_info_json(storage_nodes): """ retrieve and confirm /storage?t=json URI for one storage node @@ -442,6 +451,7 @@ def test_storage_info_json(storage_nodes): assert data[u"stats"][u"storage_server.reserved_space"] == 1000000000 +@run_in_thread def test_introducer_info(introducer): """ retrieve and confirm /introducer URI for the introducer @@ -460,6 +470,7 @@ def test_introducer_info(introducer): assert "subscription_summary" in data +@run_in_thread def test_mkdir_with_children(alice): """ create a directory using ?t=mkdir-with-children From ded5b20924537bbda2471581606cc9adbe7f87ce Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 20 Mar 2023 15:20:39 -0400 Subject: [PATCH 1565/2309] Lint fix. --- integration/test_get_put.py | 1 - 1 file changed, 1 deletion(-) diff --git a/integration/test_get_put.py b/integration/test_get_put.py index 6b87c9b62..f121d6284 100644 --- a/integration/test_get_put.py +++ b/integration/test_get_put.py @@ -4,7 +4,6 @@ and stdout. """ from subprocess import Popen, PIPE, check_output, check_call -import sys import pytest from pytest_twisted import ensureDeferred From cce5d3adff757d67fdde04da8cd10a86e5a61176 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 20 Mar 2023 15:24:10 -0400 Subject: [PATCH 1566/2309] Don't actually need this. --- src/allmydata/test/test_system.py | 28 ---------------------------- 1 file changed, 28 deletions(-) diff --git a/src/allmydata/test/test_system.py b/src/allmydata/test/test_system.py index b7873f14f..c997ac734 100644 --- a/src/allmydata/test/test_system.py +++ b/src/allmydata/test/test_system.py @@ -1834,34 +1834,6 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase): d.addCallback(_got_lit_filenode) return d - async def test_immutable_upload_download(self): - """ - A reproducer for issue 3988: upload a large file and then download it. - """ - DATA = b"abc123 this is not utf-8 decodable \xff\x00\x33 \x11" * 1_000_000 - await self.set_up_nodes() - - async def run(*args): - await deferToThread(check_call, ["tahoe", "--node-directory", self.getdir("client0")] + list(args)) - - for c in self.clients: - c.encoding_params['k'] = 2 - c.encoding_params['happy'] = 3 - c.encoding_params['n'] = 4 - - await run("create-alias", "getput") - - tmp_path = Path(mkdtemp()) - tempfile = tmp_path / "input" - - with tempfile.open("wb") as f: - f.write(DATA) - await run("put", str(tempfile), "getput:largefile") - - outfile = tmp_path / "out" - await run("get", "getput:largefile", str(outfile)) - self.assertEqual(outfile.read_bytes(), tempfile.read_bytes()) - class Connections(SystemTestMixin, unittest.TestCase): FORCE_FOOLSCAP_FOR_STORAGE = True From 815066c4de7db05c5df86ca1b9ab7dbeb08cfab2 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 20 Mar 2023 15:25:52 -0400 Subject: [PATCH 1567/2309] Just use the utility. --- integration/util.py | 54 ++++++++++++++++++++++----------------------- 1 file changed, 26 insertions(+), 28 deletions(-) diff --git a/integration/util.py b/integration/util.py index c2befe47b..05fef8fed 100644 --- a/integration/util.py +++ b/integration/util.py @@ -430,6 +430,31 @@ class FileShouldVanishException(Exception): ) +def run_in_thread(f): + """Decorator for integration tests that runs code in a thread. + + Because we're using pytest_twisted, tests that rely on the reactor are + expected to return a Deferred and use async APIs so the reactor can run. + + In the case of the integration test suite, it launches nodes in the + background using Twisted APIs. The nodes stdout and stderr is read via + Twisted code. If the reactor doesn't run, reads don't happen, and + eventually the buffers fill up, and the nodes block when they try to flush + logs. + + We can switch to Twisted APIs (treq instead of requests etc.), but + sometimes it's easier or expedient to just have a blocking test. So this + decorator allows you to run the test in a thread, and the reactor can keep + running in the main thread. + + See https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3597 for tracking bug. + """ + @wraps(f) + def test(*args, **kwargs): + return deferToThread(lambda: f(*args, **kwargs)) + return test + + def await_file_contents(path, contents, timeout=15, error_if=None): """ wait up to `timeout` seconds for the file at `path` (any path-like @@ -555,6 +580,7 @@ def web_post(tahoe, uri_fragment, **kwargs): return resp.content +@run_in_thread def await_client_ready(tahoe, timeout=10, liveness=60*2, minimum_number_of_servers=1): """ Uses the status API to wait for a client-type node (in `tahoe`, a @@ -570,10 +596,6 @@ def await_client_ready(tahoe, timeout=10, liveness=60*2, minimum_number_of_serve We will try for up to `timeout` seconds for the above conditions to be true. Otherwise, an exception is raised """ - return deferToThread(_await_client_ready_blocking, tahoe, timeout, liveness, minimum_number_of_servers) - - -def _await_client_ready_blocking(tahoe, timeout, liveness, minimum_number_of_servers): start = time.time() while (time.time() - start) < float(timeout): try: @@ -626,30 +648,6 @@ def generate_ssh_key(path): f.write(s.encode("ascii")) -def run_in_thread(f): - """Decorator for integration tests that runs code in a thread. - - Because we're using pytest_twisted, tests that rely on the reactor are - expected to return a Deferred and use async APIs so the reactor can run. - - In the case of the integration test suite, it launches nodes in the - background using Twisted APIs. The nodes stdout and stderr is read via - Twisted code. If the reactor doesn't run, reads don't happen, and - eventually the buffers fill up, and the nodes block when they try to flush - logs. - - We can switch to Twisted APIs (treq instead of requests etc.), but - sometimes it's easier or expedient to just have a blocking test. So this - decorator allows you to run the test in a thread, and the reactor can keep - running in the main thread. - - See https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3597 for tracking bug. - """ - @wraps(f) - def test(*args, **kwargs): - return deferToThread(lambda: f(*args, **kwargs)) - return test - @frozen class CHK: """ From 23b977a4b1626683116c71e08ed1a4f33381d5f0 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 20 Mar 2023 15:27:16 -0400 Subject: [PATCH 1568/2309] Undo unnecessary imports. --- src/allmydata/test/test_system.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/allmydata/test/test_system.py b/src/allmydata/test/test_system.py index c997ac734..d11a6e866 100644 --- a/src/allmydata/test/test_system.py +++ b/src/allmydata/test/test_system.py @@ -15,15 +15,11 @@ from past.builtins import chr as byteschr, long from six import ensure_text import os, re, sys, time, json -from subprocess import check_call -from pathlib import Path -from tempfile import mkdtemp from bs4 import BeautifulSoup from twisted.trial import unittest from twisted.internet import defer -from twisted.internet.threads import deferToThread from allmydata import uri from allmydata.storage.mutable import MutableShareFile From 900b4a3c989f0ef707b0ab39b72063aa3f991a7c Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 21 Mar 2023 08:55:41 -0400 Subject: [PATCH 1569/2309] Package a version of collections-extended compatible with Python 3.11 --- nix/collections-extended.nix | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 nix/collections-extended.nix diff --git a/nix/collections-extended.nix b/nix/collections-extended.nix new file mode 100644 index 000000000..05254fc1b --- /dev/null +++ b/nix/collections-extended.nix @@ -0,0 +1,12 @@ +# Package a version that's compatible with Python 3.11. This can go away once +# https://github.com/mlenzen/collections-extended/pull/199 is merged and +# included in a version of nixpkgs we depend on. +{ fetchFromGitHub, collections-extended }: +collections-extended.overrideAttrs (old: { + src = fetchFromGitHub { + owner = "mlenzen"; + repo = "collections-extended"; + rev = "8b93390636d58d28012b8e9d22334ee64ca37d73"; + hash = "sha256-e7RCpNsqyS1d3q0E+uaE4UOEQziueYsRkKEvy3gCHt0="; + }; +}) From 41d5538921e07d291e3f88f6ffaf60d8c2e68daa Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 21 Mar 2023 08:56:05 -0400 Subject: [PATCH 1570/2309] Fix `maturin build` when using PyPy for the pycddl package --- nix/pycddl.nix | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/nix/pycddl.nix b/nix/pycddl.nix index 703f00595..4c68830d4 100644 --- a/nix/pycddl.nix +++ b/nix/pycddl.nix @@ -27,7 +27,7 @@ # # 8. run `nix-build`. it should succeed. if it does not, seek assistance. # -{ lib, fetchPypi, buildPythonPackage, rustPlatform }: +{ lib, fetchPypi, python, buildPythonPackage, rustPlatform }: buildPythonPackage rec { pname = "pycddl"; version = "0.4.0"; @@ -38,6 +38,12 @@ buildPythonPackage rec { sha256 = "sha256-w0CGbPeiXyS74HqZXyiXhvaAMUaIj5onwjl9gWKAjqY="; }; + # Without this, when building for PyPy, `maturin build` seems to fail to + # find the interpreter at all and then fails early in the build process with + # an error saying "unsupported Python interpreter". We can easily point + # directly at the relevant interpreter, so do that. + maturinBuildFlags = [ "--interpreter" python.executable ]; + nativeBuildInputs = with rustPlatform; [ maturinBuildHook cargoSetupHook From dd8f6d408d27908de924ce400c7c717536095f0b Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 21 Mar 2023 08:56:50 -0400 Subject: [PATCH 1571/2309] Remove the non-unit test dependencies from the unit test inputs --- nix/tahoe-lafs.nix | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/nix/tahoe-lafs.nix b/nix/tahoe-lafs.nix index 5986db420..2e1c4aa39 100644 --- a/nix/tahoe-lafs.nix +++ b/nix/tahoe-lafs.nix @@ -48,19 +48,15 @@ let zope_interface ] ++ pickExtraDependencies pythonExtraDependencies extrasNames; - pythonCheckDependencies = with pythonPackages; [ + unitTestDependencies = with pythonPackages; [ beautifulsoup4 fixtures hypothesis mock - paramiko prometheus-client - pytest - pytest-timeout - pytest-twisted testtools - towncrier ]; + in buildPythonPackage { inherit pname version; @@ -68,7 +64,7 @@ buildPythonPackage { propagatedBuildInputs = pythonPackageDependencies; inherit doCheck; - checkInputs = pythonCheckDependencies; + checkInputs = unitTestDependencies; checkPhase = '' export TAHOE_LAFS_HYPOTHESIS_PROFILE=ci python -m twisted.trial -j $NIX_BUILD_CORES allmydata From 35b921b11d1fdf23d8039a7cf4b526eb56474995 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 21 Mar 2023 08:57:21 -0400 Subject: [PATCH 1572/2309] Put Python package overrides in one place, and add a lot more of them These packaging changes fix issues against CPython 3.11 or PyPy. --- default.nix | 6 +- nix/python-overrides.nix | 133 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 134 insertions(+), 5 deletions(-) create mode 100644 nix/python-overrides.nix diff --git a/default.nix b/default.nix index b87a6730a..d616f63b8 100644 --- a/default.nix +++ b/default.nix @@ -32,11 +32,7 @@ in }: with (pkgs.${pythonVersion}.override { - packageOverrides = self: super: { - # Some dependencies aren't packaged in nixpkgs so supply our own packages. - pycddl = self.callPackage ./nix/pycddl.nix { }; - txi2p = self.callPackage ./nix/txi2p.nix { }; - }; + packageOverrides = import ./nix/python-overrides.nix; }).pkgs; callPackage ./nix/tahoe-lafs.nix { # Select whichever package extras were requested. diff --git a/nix/python-overrides.nix b/nix/python-overrides.nix new file mode 100644 index 000000000..87c42ad58 --- /dev/null +++ b/nix/python-overrides.nix @@ -0,0 +1,133 @@ +# Override various Python packages to create a package set that works for +# Tahoe-LAFS on CPython and PyPy. +self: super: +let + + # Run a function on a derivation if and only if we're building for PyPy. + onPyPy = f: drv: if super.isPyPy then f drv else drv; + + # Disable a Python package's test suite. + dontCheck = drv: drv.overrideAttrs (old: { doInstallCheck = false; }); + + # Disable building a Python package's documentation. + dontBuildDocs = alsoDisable: drv: (drv.override ({ + sphinxHook = null; + } // alsoDisable)).overrideAttrs ({ outputs, ... }: { + outputs = builtins.filter (x: "doc" != x) outputs; + }); + +in { + # Some dependencies aren't packaged in nixpkgs so supply our own packages. + pycddl = self.callPackage ./pycddl.nix { }; + txi2p = self.callPackage ./txi2p.nix { }; + + # collections-extended is currently broken for Python 3.11 in nixpkgs but + # we know where a working version lives. + collections-extended = self.callPackage ./collections-extended.nix { + inherit (super) collections-extended; + }; + + # greenlet is incompatible with PyPy but PyPy has a builtin equivalent. + # Fixed in nixpkgs in a5f8184fb816a4fd5ae87136838c9981e0d22c67. + greenlet = onPyPy (drv: null) super.greenlet; + + # tornado and tk pull in a huge dependency trees for functionality we don't + # care about, also tkinter doesn't work on PyPy. + matplotlib = super.matplotlib.override { tornado = null; enableTk = false; }; + + tqdm = super.tqdm.override { + # ibid. + tkinter = null; + # pandas is only required by the part of the test suite covering + # integration with pandas that we don't care about. pandas is a huge + # dependency. + pandas = null; + }; + + # The treq test suite depends on httpbin. httpbin pulls in babel (flask -> + # jinja2 -> babel) and arrow (brotlipy -> construct -> arrow). babel fails + # its test suite and arrow segfaults. + treq = onPyPy dontCheck super.treq; + + # the six test suite fails on PyPy because it depends on dbm which the + # nixpkgs PyPy build appears to be missing. Maybe fixed in nixpkgs in + # a5f8184fb816a4fd5ae87136838c9981e0d22c67. + six = onPyPy dontCheck super.six; + + # Building the docs requires sphinx which brings in a dependency on babel, + # the test suite of which fails. + pyopenssl = onPyPy (dontBuildDocs { sphinx-rtd-theme = null; }) super.pyopenssl; + + # Likewise for beautifulsoup4. + beautifulsoup4 = onPyPy (dontBuildDocs {}) super.beautifulsoup4; + + # The autobahn test suite pulls in a vast number of dependencies for + # functionality we don't care about. It might be nice to *selectively* + # disable just some of it but this is easier. + autobahn = onPyPy dontCheck super.autobahn; + + # and python-dotenv tests pulls in a lot of dependencies, including jedi, + # which does not work on PyPy. + python-dotenv = onPyPy dontCheck super.python-dotenv; + + # Upstream package unaccountably includes a sqlalchemy dependency ... but + # the project has no such dependency. Fixed in nixpkgs in + # da10e809fff70fbe1d86303b133b779f09f56503. + aiocontextvars = super.aiocontextvars.override { sqlalchemy = null; }; + + # By default, the sphinx docs are built, which pulls in a lot of + # dependencies - including jedi, which does not work on PyPy. + hypothesis = + (let h = super.hypothesis; + in + if (h.override.__functionArgs.enableDocumentation or false) + then h.override { enableDocumentation = false; } + else h).overrideAttrs ({ nativeBuildInputs, ... }: { + # The nixpkgs expression is missing the tzdata check input. + nativeBuildInputs = nativeBuildInputs ++ [ super.tzdata ]; + }); + + # flaky's test suite depends on nose and nose appears to have Python 3 + # incompatibilities (it includes `print` statements, for example). + flaky = onPyPy dontCheck super.flaky; + + # Replace the deprecated way of running the test suite with the modern way. + # This also drops a bunch of unnecessary build-time dependencies, some of + # which are broken on PyPy. Fixed in nixpkgs in + # 5feb5054bb08ba779bd2560a44cf7d18ddf37fea. + zfec = (super.zfec.override { + setuptoolsTrial = null; + }).overrideAttrs (old: { + checkPhase = "trial zfec"; + }); + + # collections-extended is packaged with poetry-core. poetry-core test suite + # uses virtualenv and virtualenv test suite fails on PyPy. + poetry-core = onPyPy dontCheck super.poetry-core; + + # The test suite fails with some rather irrelevant (to us) string comparison + # failure on PyPy. Probably a PyPy bug but doesn't seem like we should + # care. + rich = onPyPy dontCheck super.rich; + + # The pyutil test suite fails in some ... test ... for some deprecation + # functionality we don't care about. + pyutil = onPyPy dontCheck super.pyutil; + + # testCall1 fails fairly inscrutibly on PyPy. Perhaps someone can fix that, + # or we could at least just skip that one test. Probably better to fix it + # since we actually depend directly and significantly on Foolscap. + foolscap = onPyPy dontCheck super.foolscap; + + # Fixed by nixpkgs PR https://github.com/NixOS/nixpkgs/pull/222246 + psutil = super.psutil.overrideAttrs ({ pytestFlagsArray, disabledTests, ...}: { + # Upstream already disables some tests but there are even more that have + # build impurities that come from build system hardware configuration. + # Skip them too. + pytestFlagsArray = [ "-v" ] ++ pytestFlagsArray; + disabledTests = disabledTests ++ [ "sensors_temperatures" ]; + }); + + # CircleCI build systems don't have enough memory to run this test suite. + lz4 = dontCheck super.lz4; +} From a173df4561fb3fad1898f32421553962f7659ff1 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 21 Mar 2023 09:29:12 -0400 Subject: [PATCH 1573/2309] news fragment --- newsfragments/3991.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3991.minor diff --git a/newsfragments/3991.minor b/newsfragments/3991.minor new file mode 100644 index 000000000..e69de29bb From a8832b11b6e365564d1b53f35bd885353d550841 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 20 Jan 2023 14:29:17 -0500 Subject: [PATCH 1574/2309] Start adapting language to narrow down possible interpretations --- docs/proposed/http-storage-node-protocol.rst | 113 ++++++++++++++----- 1 file changed, 84 insertions(+), 29 deletions(-) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index aee201cf5..397d64ec2 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -3,7 +3,7 @@ Storage Node Protocol ("Great Black Swamp", "GBS") ================================================== -The target audience for this document is Tahoe-LAFS developers. +The target audience for this document is developers working on Tahoe-LAFS or on an alternate implementation intended to be interoperable. After reading this document, one should expect to understand how Tahoe-LAFS clients interact over the network with Tahoe-LAFS storage nodes. @@ -64,6 +64,10 @@ Glossary lease renew secret a short secret string which storage servers required to be presented before allowing a particular lease to be renewed +The key words +"MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" +in this document are to be interpreted as described in RFC 2119. + Motivation ---------- @@ -119,8 +123,8 @@ An HTTP-based protocol can make use of TLS in largely the same way to provide th Provision of these properties *is* dependant on implementers following Great Black Swamp's rules for x509 certificate validation (rather than the standard "web" rules for validation). -Requirements ------------- +Design Requirements +------------------- Security ~~~~~~~~ @@ -189,6 +193,9 @@ Solutions An HTTP-based protocol, dubbed "Great Black Swamp" (or "GBS"), is described below. This protocol aims to satisfy the above requirements at a lower level of complexity than the current Foolscap-based protocol. +Summary (Non-normative) +!!!!!!!!!!!!!!!!!!!!!!! + Communication with the storage node will take place using TLS. The TLS version and configuration will be dictated by an ongoing understanding of best practices. The storage node will present an x509 certificate during the TLS handshake. @@ -240,7 +247,7 @@ When Bob's client issues HTTP requests to Alice's storage node it includes the * They are encoded with Base32 for a length of 32 bytes. SPKI information discussed here is 32 bytes (SHA256 digest). They would be encoded in Base32 for a length of 52 bytes. - `base64url`_ provides a more compact encoding of the information while remaining URL-compatible. + `unpadded base64url`_ provides a more compact encoding of the information while remaining URL-compatible. This would encode the SPKI information for a length of merely 43 bytes. SHA1, the current Foolscap hash function, @@ -332,12 +339,15 @@ Details about the interface are encoded in the HTTP message body. Message Encoding ~~~~~~~~~~~~~~~~ -The preferred encoding for HTTP message bodies is `CBOR`_. -A request may be submitted using an alternate encoding by declaring this in the ``Content-Type`` header. -A request may indicate its preference for an alternate encoding in the response using the ``Accept`` header. -These two headers are used in the typical way for an HTTP application. +Clients and servers MUST use the ``Content-Type`` and ``Accept`` header fields as specified in `RFC 9110`_ for message body negotiation. -The only other encoding support for which is currently recommended is JSON. +The encoding for HTTP message bodies SHOULD be `CBOR`_. +Clients submitting requests using this encoding MUST include a ``Content-Type: application/cbor`` request header field. +A request MAY be submitted using an alternate encoding by declaring this in the ``Content-Type`` header field. +A request MAY indicate its preference for an alternate encoding in the response using the ``Accept`` header field. +A request which includes no ``Accept`` header field MUST be interpreted in the same way as a request including a ``Accept: application/cbor`` header field. + +Clients and servers SHOULD support ``application/json`` request and response message body encoding. For HTTP messages carrying binary share data, this is expected to be a particularly poor encoding. However, @@ -350,10 +360,19 @@ Because of the simple types used throughout and the equivalence described in `RFC 7049`_ these examples should be representative regardless of which of these two encodings is chosen. -The one exception is sets. -For CBOR messages, any sequence that is semantically a set (i.e. no repeated values allowed, order doesn't matter, and elements are hashable in Python) should be sent as a set. -Tag 6.258 is used to indicate sets in CBOR; see `the CBOR registry `_ for more details. -Sets will be represented as JSON lists in examples because JSON doesn't support sets. +One exception to this rule is for sets. +For CBOR messages, +any sequence that is semantically a set (i.e. no repeated values allowed, order doesn't matter, and elements are hashable in Python) should be sent as a set. +Tag 6.258 is used to indicate sets in CBOR; +see `the CBOR registry `_ for more details. +The JSON encoding does not support sets. +Sets MUST be represented as arrays in JSON-encoded messages. + +Another exception to this rule is for bytes. +The CBOR encoding natively supports a bytes type while the JSON encoding does not. +Bytes MUST be represented as strings giving the `Base64`_ representation of the original bytes value. + +Clients and servers MAY support additional request and response message body encodings. HTTP Design ~~~~~~~~~~~ @@ -368,29 +387,49 @@ one branch contains all of the share data; another branch contains all of the lease data; etc. -An ``Authorization`` header in requests is required for all endpoints. -The standard HTTP authorization protocol is used. -The authentication *type* used is ``Tahoe-LAFS``. -The swissnum from the NURL used to locate the storage service is used as the *credentials*. -If credentials are not presented or the swissnum is not associated with a storage service then no storage processing is performed and the request receives an ``401 UNAUTHORIZED`` response. +Clients and servers MUST use the ``Authorization`` header field, +as specified in `RFC 9110`_, +for authorization of all requests to all endpoints specified here. +The authentication *type* MUST be ``Tahoe-LAFS``. +Clients MUST present the swissnum from the NURL used to locate the storage service as the *credentials*. -There are also, for some endpoints, secrets sent via ``X-Tahoe-Authorization`` headers. -If these are: +If credentials are not presented or the swissnum is not associated with a storage service then the server MUST issue a ``401 UNAUTHORIZED`` response and perform no other processing of the message. + +Requests to certain endpoints MUST include additional secrets in the ``X-Tahoe-Authorization`` headers field. +The endpoints which require these secrets are: + +* ``PUT /storage/v1/lease/:storage_index``: + The secrets included MUST be ``lease-renew-secret`` and ``lease-cancel-secret``. + +* ``POST /storage/v1/immutable/:storage_index``: + The secrets included MUST be ``lease-renew-secret``, ``lease-cancel-secret``, and ``upload-secret``. + +* ``PATCH /storage/v1/immutable/:storage_index/:share_number``: + The secrets included MUST be ``upload-secret``. + +* ``PUT /storage/v1/immutable/:storage_index/:share_number/abort``: + The secrets included MUST be ``upload-secret``. + +* ``POST /storage/v1/mutable/:storage_index/read-test-write``: + The secrets included MUST be ``lease-renew-secret``, ``lease-cancel-secret``, and ``write-enabler``. + +If these secrets are: 1. Missing. 2. The wrong length. 3. Not the expected kind of secret. 4. They are otherwise unparseable before they are actually semantically used. -the server will respond with ``400 BAD REQUEST``. +the server MUST respond with ``400 BAD REQUEST`` and perform no other processing of the message. 401 is not used because this isn't an authorization problem, this is a "you sent garbage and should know better" bug. -If authorization using the secret fails, then a ``401 UNAUTHORIZED`` response should be sent. +If authorization using the secret fails, +then the server MUST send a ``401 UNAUTHORIZED`` response and perform no other processing of the message. Encoding ~~~~~~~~ -* ``storage_index`` should be base32 encoded (RFC3548) in URLs. +* ``storage_index`` MUST be `Base32`_ encoded in URLs. General ~~~~~~~ @@ -398,11 +437,14 @@ General ``GET /storage/v1/version`` !!!!!!!!!!!!!!!!!!!!!!!!!!! -Retrieve information about the version of the storage server. -Information is returned as an encoded mapping. -For example:: +This endpoint allows clients to retrieve some basic metadata about a storage server from the storage service. +The response MUST represent a mapping from schema identifiers to the metadata. - { "http://allmydata.org/tahoe/protocols/storage/v1" : +The only schema identifier specified is ``"http://allmydata.org/tahoe/protocols/storage/v1"``. +The server MUST include an entry in the mapping with this key. +The value for the key MUST be another mapping with the following keys and value types:: + + { "http://allmydata.org/tahoe/protocols/storage/v1": { "maximum-immutable-share-size": 1234, "maximum-mutable-share-size": 1235, "available-space": 123456, @@ -414,6 +456,11 @@ For example:: "application-version": "1.13.0" } +The server SHOULD populate as many fields as possible with accurate information about itself. + +XXX Document every single field + + ``PUT /storage/v1/lease/:storage_index`` !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! @@ -926,10 +973,18 @@ otherwise it will read a byte which won't match `b""`:: 204 NO CONTENT +.. _Base64: https://www.rfc-editor.org/rfc/rfc4648#section-4 + +.. _Base32: https://www.rfc-editor.org/rfc/rfc4648#section-6 + +.. _RFC 4648: https://tools.ietf.org/html/rfc4648 + .. _RFC 7469: https://tools.ietf.org/html/rfc7469#section-2.4 .. _RFC 7049: https://tools.ietf.org/html/rfc7049#section-4 +.. _RFC 9110: https://tools.ietf.org/html/rfc9110 + .. _CBOR: http://cbor.io/ .. [#] @@ -974,7 +1029,7 @@ otherwise it will read a byte which won't match `b""`:: spki_encoded = urlsafe_b64encode(spki_sha256) assert spki_encoded == tub_id - Note we use `base64url`_ rather than the Foolscap- and Tahoe-LAFS-preferred Base32. + Note we use `unpadded base64url`_ rather than the Foolscap- and Tahoe-LAFS-preferred Base32. .. [#] https://www.cvedetails.com/cve/CVE-2017-5638/ @@ -985,6 +1040,6 @@ otherwise it will read a byte which won't match `b""`:: .. [#] https://efail.de/ -.. _base64url: https://tools.ietf.org/html/rfc7515#appendix-C +.. _unpadded base64url: https://tools.ietf.org/html/rfc7515#appendix-C .. _attacking SHA1: https://en.wikipedia.org/wiki/SHA-1#Attacks From 7b207383088ce1f866bc6442e07b5f675ceea4b7 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 23 Jan 2023 10:40:41 -0500 Subject: [PATCH 1575/2309] some more edits --- docs/proposed/http-storage-node-protocol.rst | 128 ++++++++++--------- 1 file changed, 70 insertions(+), 58 deletions(-) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index 397d64ec2..cff6dc67b 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -438,27 +438,28 @@ General !!!!!!!!!!!!!!!!!!!!!!!!!!! This endpoint allows clients to retrieve some basic metadata about a storage server from the storage service. -The response MUST represent a mapping from schema identifiers to the metadata. +The response MUST validate against this CDDL schema:: -The only schema identifier specified is ``"http://allmydata.org/tahoe/protocols/storage/v1"``. -The server MUST include an entry in the mapping with this key. -The value for the key MUST be another mapping with the following keys and value types:: + {'http://allmydata.org/tahoe/protocols/storage/v1' => { + 'maximum-immutable-share-size' => uint + 'maximum-mutable-share-size' => uint + 'available-space' => uint + 'tolerates-immutable-read-overrun' => bool + 'delete-mutable-shares-with-zero-length-writev' => bool + 'fills-holes-with-zero-bytes' => bool + 'prevents-read-past-end-of-share-data' => bool + } + 'application-version' => bstr + } - { "http://allmydata.org/tahoe/protocols/storage/v1": - { "maximum-immutable-share-size": 1234, - "maximum-mutable-share-size": 1235, - "available-space": 123456, - "tolerates-immutable-read-overrun": true, - "delete-mutable-shares-with-zero-length-writev": true, - "fills-holes-with-zero-bytes": true, - "prevents-read-past-end-of-share-data": true - }, - "application-version": "1.13.0" - } +The server SHOULD populate as many fields as possible with accurate information about its behavior. -The server SHOULD populate as many fields as possible with accurate information about itself. +For fields which relate to a specific API +the semantics are documented below in the section for that API. +For fields that are more general than a single API the semantics are as follows: -XXX Document every single field +* available-space: + The server SHOULD use this field to advertise the amount of space that it currently considers unused and is willing to allocate for client requests. ``PUT /storage/v1/lease/:storage_index`` @@ -518,21 +519,23 @@ Writing !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! Initialize an immutable storage index with some buckets. -The buckets may have share data written to them once. -A lease is also created for the shares. +The server MUST allow share data to be written to the buckets at most one time. +The server MAY create a lease for the buckets. Details of the buckets to create are encoded in the request body. For example:: {"share-numbers": [1, 7, ...], "allocated-size": 12345} -The request must include ``X-Tahoe-Authorization`` HTTP headers that set the various secrets—upload, lease renewal, lease cancellation—that will be later used to authorize various operations. +The server SHOULD accept a value for **allocated-size** that is less than or equal to the value for the server's version message's **maximum-immutable-share-size** value. + +The request MUST include ``X-Tahoe-Authorization`` HTTP headers that set the various secrets—upload, lease renewal, lease cancellation—that will be later used to authorize various operations. For example:: X-Tahoe-Authorization: lease-renew-secret X-Tahoe-Authorization: lease-cancel-secret X-Tahoe-Authorization: upload-secret -The response body includes encoded information about the created buckets. +The response body MUST include encoded information about the created buckets. For example:: {"already-have": [1, ...], "allocated": [7, ...]} @@ -589,26 +592,28 @@ Rejected designs for upload secrets: !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! Write data for the indicated share. -The share number must belong to the storage index. -The request body is the raw share data (i.e., ``application/octet-stream``). -*Content-Range* requests are required; for large transfers this allows partially complete uploads to be resumed. +The share number MUST belong to the storage index. +The request body MUST be the raw share data (i.e., ``application/octet-stream``). +The request MUST include a *Content-Range* header field; +for large transfers this allows partially complete uploads to be resumed. + For example, a 1MiB share can be divided in to eight separate 128KiB chunks. Each chunk can be uploaded in a separate request. Each request can include a *Content-Range* value indicating its placement within the complete share. If any one of these requests fails then at most 128KiB of upload work needs to be retried. -The server must recognize when all of the data has been received and mark the share as complete +The server MUST recognize when all of the data has been received and mark the share as complete (which it can do because it was informed of the size when the storage index was initialized). -The request must include a ``X-Tahoe-Authorization`` header that includes the upload secret:: +The request MUST include a ``X-Tahoe-Authorization`` header that includes the upload secret:: X-Tahoe-Authorization: upload-secret Responses: -* When a chunk that does not complete the share is successfully uploaded the response is ``OK``. - The response body indicates the range of share data that has yet to be uploaded. +* When a chunk that does not complete the share is successfully uploaded the response MUST be ``OK``. + The response body MUST indicate the range of share data that has yet to be uploaded. That is:: { "required": @@ -620,11 +625,12 @@ Responses: ] } -* When the chunk that completes the share is successfully uploaded the response is ``CREATED``. +* When the chunk that completes the share is successfully uploaded the response MUST be ``CREATED``. * If the *Content-Range* for a request covers part of the share that has already, and the data does not match already written data, - the response is ``CONFLICT``. - At this point the only thing to do is abort the upload and start from scratch (see below). + the response MUST be ``CONFLICT``. + In this case the client MUST abort the upload. + The client MAY then restart the upload from scratch. Discussion `````````` @@ -650,34 +656,32 @@ From RFC 7231:: This cancels an *in-progress* upload. -The request must include a ``X-Tahoe-Authorization`` header that includes the upload secret:: +The request MUST include a ``X-Tahoe-Authorization`` header that includes the upload secret:: X-Tahoe-Authorization: upload-secret -The response code: - -* When the upload is still in progress and therefore the abort has succeeded, - the response is ``OK``. - Future uploads can start from scratch with no pre-existing upload state stored on the server. -* If the uploaded has already finished, the response is 405 (Method Not Allowed) - and no change is made. +If there is an incomplete upload with a matching upload-secret then the server MUST consider the abort to have succeeded. +In this case the response MUST be ``OK``. +The server MUST respond to all future requests as if the operations related to this upload did not take place. +If there is no incomplete upload with a matching upload-secret then the server MUST respond with ``Method Not Allowed`` (405). +The server MUST make no client-visible changes to its state in this case. ``POST /storage/v1/immutable/:storage_index/:share_number/corrupt`` !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! -Advise the server the data read from the indicated share was corrupt. The -request body includes an human-meaningful text string with details about the -corruption. It also includes potentially important details about the share. +Advise the server the data read from the indicated share was corrupt. +The request body includes an human-meaningful text string with details about the corruption. +It also includes potentially important details about the share. For example:: {"reason": "expected hash abcd, got hash efgh"} -.. share-type, storage-index, and share-number are inferred from the URL - -The response code is OK (200) by default, or NOT FOUND (404) if the share -couldn't be found. +The report pertains to the immutable share with a **storage index** and **share number** given in the request path. +If the identified **storage index** and **share number** are known to the server then the response SHOULD be accepted and made available to server administrators. +In this case the response SHOULD be ``OK``. +If the response is not accepted then the response SHOULD be ``Not Found`` (404). Reading ~~~~~~~ @@ -685,26 +689,34 @@ Reading ``GET /storage/v1/immutable/:storage_index/shares`` !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! -Retrieve a list (semantically, a set) indicating all shares available for the -indicated storage index. For example:: +Retrieve a list (semantically, a set) indicating all shares available for the indicated storage index. +For example:: [1, 5] -An unknown storage index results in an empty list. +If the **storage index** in the request path is not known to the server then the response MUST include an empty list. ``GET /storage/v1/immutable/:storage_index/:share_number`` !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! Read a contiguous sequence of bytes from one share in one bucket. -The response body is the raw share data (i.e., ``application/octet-stream``). -The ``Range`` header may be used to request exactly one ``bytes`` range, in which case the response code will be 206 (partial content). -Interpretation and response behavior is as specified in RFC 7233 § 4.1. -Multiple ranges in a single request are *not* supported; open-ended ranges are also not supported. +The response body MUST be the raw share data (i.e., ``application/octet-stream``). +The ``Range`` header MAY be used to request exactly one ``bytes`` range, +in which case the response code MUST be ``Partial Content`` (206). +Interpretation and response behavior MUST be as specified in RFC 7233 § 4.1. +Multiple ranges in a single request are *not* supported; +open-ended ranges are also not supported. +Clients MUST NOT send requests using these features. -If the response reads beyond the end of the data, the response may be shorter than the requested range. -The resulting ``Content-Range`` header will be consistent with the returned data. +If the response reads beyond the end of the data, +the response MUST be shorter than the requested range. +It MUST contain all data in the share and then end. +The resulting ``Content-Range`` header MUST be consistent with the returned data. -If the response to a query is an empty range, the ``NO CONTENT`` (204) response code will be used. +The server MUST indicate this behavior by specifying **True** for **tolerates-immutable-read-overrun** in its version response. + +If the response to a query is an empty range, +the server MUST send a ``No Content`` (204) response. Discussion `````````` @@ -743,13 +755,13 @@ The first write operation on a mutable storage index creates it (that is, there is no separate "create this storage index" operation as there is for the immutable storage index type). -The request must include ``X-Tahoe-Authorization`` headers with write enabler and lease secrets:: +The request MUST include ``X-Tahoe-Authorization`` headers with write enabler and lease secrets:: X-Tahoe-Authorization: write-enabler X-Tahoe-Authorization: lease-cancel-secret X-Tahoe-Authorization: lease-renew-secret -The request body includes test, read, and write vectors for the operation. +The request body MUST include test, read, and write vectors for the operation. For example:: { From 98a3691891bfbe4af2871a58fba0e586400e90cf Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 25 Jan 2023 09:55:40 -0500 Subject: [PATCH 1576/2309] Add more CDDL to the spec; remove some server version flags from it --- docs/proposed/http-storage-node-protocol.rst | 95 ++++++++++++++++---- src/allmydata/storage/http_client.py | 16 +++- src/allmydata/storage/http_server.py | 21 ++++- 3 files changed, 109 insertions(+), 23 deletions(-) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index cff6dc67b..838f88426 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -444,10 +444,6 @@ The response MUST validate against this CDDL schema:: 'maximum-immutable-share-size' => uint 'maximum-mutable-share-size' => uint 'available-space' => uint - 'tolerates-immutable-read-overrun' => bool - 'delete-mutable-shares-with-zero-length-writev' => bool - 'fills-holes-with-zero-bytes' => bool - 'prevents-read-past-end-of-share-data' => bool } 'application-version' => bstr } @@ -522,11 +518,18 @@ Initialize an immutable storage index with some buckets. The server MUST allow share data to be written to the buckets at most one time. The server MAY create a lease for the buckets. Details of the buckets to create are encoded in the request body. +The request body MUST validate against this CDDL schema:: + + { + share-numbers: #6.258([0*256 uint]) + allocated-size: uint + } + For example:: {"share-numbers": [1, 7, ...], "allocated-size": 12345} -The server SHOULD accept a value for **allocated-size** that is less than or equal to the value for the server's version message's **maximum-immutable-share-size** value. +The server SHOULD accept a value for **allocated-size** that is less than or equal to the lesser of the values of the server's version message's **maximum-immutable-share-size** or **available-space** values. The request MUST include ``X-Tahoe-Authorization`` HTTP headers that set the various secrets—upload, lease renewal, lease cancellation—that will be later used to authorize various operations. For example:: @@ -536,6 +539,13 @@ For example:: X-Tahoe-Authorization: upload-secret The response body MUST include encoded information about the created buckets. +The response body MUST validate against this CDDL schema:: + + { + already-have: #6.258([0*256 uint]) + allocated: #6.258([0*256 uint]) + } + For example:: {"already-have": [1, ...], "allocated": [7, ...]} @@ -614,7 +624,13 @@ Responses: * When a chunk that does not complete the share is successfully uploaded the response MUST be ``OK``. The response body MUST indicate the range of share data that has yet to be uploaded. - That is:: + The response body MUST validate against this CDDL schema:: + + { + required: [0* {begin: uint, end: uint}] + } + + For example:: { "required": [ { "begin": @@ -673,6 +689,11 @@ The server MUST make no client-visible changes to its state in this case. Advise the server the data read from the indicated share was corrupt. The request body includes an human-meaningful text string with details about the corruption. It also includes potentially important details about the share. +The request body MUST validate against this CDDL schema:: + + { + reason: tstr + } For example:: @@ -690,6 +711,10 @@ Reading !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! Retrieve a list (semantically, a set) indicating all shares available for the indicated storage index. +The response body MUST validate against this CDDL schema:: + + #6.258([0*256 uint]) + For example:: [1, 5] @@ -710,11 +735,9 @@ Clients MUST NOT send requests using these features. If the response reads beyond the end of the data, the response MUST be shorter than the requested range. -It MUST contain all data in the share and then end. +It MUST contain all data up to the end of the share and then end. The resulting ``Content-Range`` header MUST be consistent with the returned data. -The server MUST indicate this behavior by specifying **True** for **tolerates-immutable-read-overrun** in its version response. - If the response to a query is an empty range, the server MUST send a ``No Content`` (204) response. @@ -762,6 +785,20 @@ The request MUST include ``X-Tahoe-Authorization`` headers with write enabler an X-Tahoe-Authorization: lease-renew-secret The request body MUST include test, read, and write vectors for the operation. +The request body MUST validate against this CDDL schema:: + + { + "test-write-vectors": { + 0*256 share_number : { + "test": [0*30 {"offset": uint, "size": uint, "specimen": bstr}] + "write": [* {"offset": uint, "data": bstr}] + "new-length": uint / null + } + } + "read-vector": [0*30 {"offset": uint, "size": uint}] + } + share_number = uint + For example:: { @@ -784,6 +821,14 @@ For example:: The response body contains a boolean indicating whether the tests all succeed (and writes were applied) and a mapping giving read data (pre-write). +The response body MUST validate against this CDDL schema:: + + { + "success": bool, + "data": {0*256 share_number: [0* bstr]} + } + share_number = uint + For example:: { @@ -795,7 +840,7 @@ For example:: } } -A test vector or read vector that read beyond the boundaries of existing data will return nothing for any bytes past the end. +A server MUST return nothing for any bytes beyond the end of existing data for a test vector or read vector that reads tries to read such data. As a result, if there is no data at all, an empty bytestring is returned no matter what the offset or length. Reading @@ -805,23 +850,34 @@ Reading !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! Retrieve a set indicating all shares available for the indicated storage index. -For example (this is shown as list, since it will be list for JSON, but will be set for CBOR):: +The response body MUST validate against this CDDL schema:: + + #6.258([0*256 uint]) + +For example:: [1, 5] ``GET /storage/v1/mutable/:storage_index/:share_number`` !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! -Read data from the indicated mutable shares, just like ``GET /storage/v1/immutable/:storage_index`` +Read data from the indicated mutable shares, just like ``GET /storage/v1/immutable/:storage_index``. -The ``Range`` header may be used to request exactly one ``bytes`` range, in which case the response code will be 206 (partial content). -Interpretation and response behavior is as specified in RFC 7233 § 4.1. -Multiple ranges in a single request are *not* supported; open-ended ranges are also not supported. +The response body MUST be the raw share data (i.e., ``application/octet-stream``). +The ``Range`` header MAY be used to request exactly one ``bytes`` range, +in which case the response code MUST be ``Partial Content`` (206). +Interpretation and response behavior MUST be specified in RFC 7233 § 4.1. +Multiple ranges in a single request are *not* supported; +open-ended ranges are also not supported. +Clients MUST NOT send requests using these features. -If the response reads beyond the end of the data, the response may be shorter than the requested range. -The resulting ``Content-Range`` header will be consistent with the returned data. +If the response reads beyond the end of the data, +the response MUST be shorter than the requested range. +It MUST contain all data up to the end of the share and then end. +The resulting ``Content-Range`` header MUST be consistent with the returned data. -If the response to a query is an empty range, the ``NO CONTENT`` (204) response code will be used. +If the response to a query is an empty range, +the server MUST send a ``No Content`` (204) response. ``POST /storage/v1/mutable/:storage_index/:share_number/corrupt`` @@ -833,6 +889,9 @@ Just like the immutable version. Sample Interactions ------------------- +This section contains examples of client/server interactions to help illuminate the above specification. +This section is non-normative. + Immutable Data ~~~~~~~~~~~~~~ diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 90bda7fc0..2f4f8398e 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -70,15 +70,14 @@ class ClientException(Exception): # indicates a set. _SCHEMAS = { "get_version": Schema( + # Note that the single-quoted (`'`) string keys in this schema + # represent *byte* strings - per the CDDL specification. Text strings + # are represented using strings with *double* quotes (`"`). """ response = {'http://allmydata.org/tahoe/protocols/storage/v1' => { 'maximum-immutable-share-size' => uint 'maximum-mutable-share-size' => uint 'available-space' => uint - 'tolerates-immutable-read-overrun' => bool - 'delete-mutable-shares-with-zero-length-writev' => bool - 'fills-holes-with-zero-bytes' => bool - 'prevents-read-past-end-of-share-data' => bool } 'application-version' => bstr } @@ -447,6 +446,15 @@ class StorageClientGeneral(object): decoded_response = yield self._client.decode_cbor( response, _SCHEMAS["get_version"] ) + # Add some features we know are true because the HTTP API + # specification requires them and because other parts of the storage + # client implementation assumes they will be present. + decoded_response[b"http://allmydata.org/tahoe/protocols/storage/v1"].update({ + b'tolerates-immutable-read-overrun': True, + b'delete-mutable-shares-with-zero-length-writev': True, + b'fills-holes-with-zero-bytes': True, + b'prevents-read-past-end-of-share-data': True, + }) returnValue(decoded_response) @inlineCallbacks diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index fd7fd1187..b7ca2d971 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -592,7 +592,26 @@ class HTTPServer(object): @_authorized_route(_app, set(), "/storage/v1/version", methods=["GET"]) def version(self, request, authorization): """Return version information.""" - return self._send_encoded(request, self._storage_server.get_version()) + return self._send_encoded(request, self._get_version()) + + def _get_version(self) -> dict[str, Any]: + """ + Get the HTTP version of the storage server's version response. + + This differs from the Foolscap version by omitting certain obsolete + fields. + """ + v = self._storage_server.get_version() + v1_identifier = b"http://allmydata.org/tahoe/protocols/storage/v1" + v1 = v[v1_identifier] + return { + v1_identifier: { + b"maximum-immutable-share-size": v1[b"maximum-immutable-share-size"], + b"maximum-mutable-share-size": v1[b"maximum-mutable-share-size"], + b"available-space": v1[b"available-space"], + }, + b"application-version": v[b"application-version"], + } ##### Immutable APIs ##### From 48a2d4d31d86bc39c2e1caa06f6e9c3e6baac741 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 17 Feb 2023 13:58:58 -0500 Subject: [PATCH 1577/2309] ``Authorization`` is the right header field --- src/allmydata/storage/http_common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/storage/http_common.py b/src/allmydata/storage/http_common.py index 123ce403b..e5f07898e 100644 --- a/src/allmydata/storage/http_common.py +++ b/src/allmydata/storage/http_common.py @@ -28,7 +28,7 @@ def get_content_type(headers: Headers) -> Optional[str]: def swissnum_auth_header(swissnum: bytes) -> bytes: - """Return value for ``Authentication`` header.""" + """Return value for ``Authorization`` header.""" return b"Tahoe-LAFS " + b64encode(swissnum).strip() From 8645462f4e1b44507e9dcde63c05ff3ef9f30453 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 17 Feb 2023 14:00:03 -0500 Subject: [PATCH 1578/2309] Base64 encode the swissnum Typically swissnums themselves are base32 encoded but there's no requirement that this is the case. Base64 encoding in the header ensures we can represent whatever the value was. --- docs/proposed/http-storage-node-protocol.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index 838f88426..7f678d271 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -391,7 +391,7 @@ Clients and servers MUST use the ``Authorization`` header field, as specified in `RFC 9110`_, for authorization of all requests to all endpoints specified here. The authentication *type* MUST be ``Tahoe-LAFS``. -Clients MUST present the swissnum from the NURL used to locate the storage service as the *credentials*. +Clients MUST present the `Base64`_-encoded representation of the swissnum from the NURL used to locate the storage service as the *credentials*. If credentials are not presented or the swissnum is not associated with a storage service then the server MUST issue a ``401 UNAUTHORIZED`` response and perform no other processing of the message. From 369d26f0f8c4975c7855e43ce9033caf893c50f0 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 10 Mar 2023 11:17:09 -0500 Subject: [PATCH 1579/2309] There is a limit to the size of the corruption report a server must accept --- docs/proposed/http-storage-node-protocol.rst | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index 7f678d271..4f5a53906 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -692,7 +692,7 @@ It also includes potentially important details about the share. The request body MUST validate against this CDDL schema:: { - reason: tstr + reason: tstr .size (1..32765) } For example:: @@ -704,6 +704,11 @@ If the identified **storage index** and **share number** are known to the server In this case the response SHOULD be ``OK``. If the response is not accepted then the response SHOULD be ``Not Found`` (404). +Discussion +`````````` + +The seemingly odd length limit on ``reason`` is chosen so that the *encoded* representation of the message is limited to 32768. + Reading ~~~~~~~ From b27946c3c6b590667ee54f43f61bc72e57780d6d Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 10 Mar 2023 11:17:23 -0500 Subject: [PATCH 1580/2309] trim overlong section marker --- docs/proposed/http-storage-node-protocol.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index 4f5a53906..6e5b85716 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -864,7 +864,7 @@ For example:: [1, 5] ``GET /storage/v1/mutable/:storage_index/:share_number`` -!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! Read data from the indicated mutable shares, just like ``GET /storage/v1/immutable/:storage_index``. From c3afab15ed43a729a8517e7ded6a6877b3c765f0 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 13 Mar 2023 09:22:42 -0400 Subject: [PATCH 1581/2309] correct version type annotation --- src/allmydata/storage/http_server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index b7ca2d971..5560d3a73 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -594,7 +594,7 @@ class HTTPServer(object): """Return version information.""" return self._send_encoded(request, self._get_version()) - def _get_version(self) -> dict[str, Any]: + def _get_version(self) -> dict[bytes, Any]: """ Get the HTTP version of the storage server's version response. From 7859ba733717dbc75b98554311cf7a59733ed5f7 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 13 Mar 2023 09:25:49 -0400 Subject: [PATCH 1582/2309] fix title level inconsistency --- docs/proposed/http-storage-node-protocol.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index 6e5b85716..c9bdf3013 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -194,7 +194,7 @@ An HTTP-based protocol, dubbed "Great Black Swamp" (or "GBS"), is described belo This protocol aims to satisfy the above requirements at a lower level of complexity than the current Foolscap-based protocol. Summary (Non-normative) -!!!!!!!!!!!!!!!!!!!!!!! +~~~~~~~~~~~~~~~~~~~~~~~ Communication with the storage node will take place using TLS. The TLS version and configuration will be dictated by an ongoing understanding of best practices. From 5facd06725d2f0c11e497c84e4d90e90bc37dd95 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 13 Mar 2023 10:42:30 -0400 Subject: [PATCH 1583/2309] adjust markup to clarify the encoding exceptions --- docs/proposed/http-storage-node-protocol.rst | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index c9bdf3013..f6d90526e 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -347,6 +347,8 @@ A request MAY be submitted using an alternate encoding by declaring this in the A request MAY indicate its preference for an alternate encoding in the response using the ``Accept`` header field. A request which includes no ``Accept`` header field MUST be interpreted in the same way as a request including a ``Accept: application/cbor`` header field. +Clients and servers MAY support additional request and response message body encodings. + Clients and servers SHOULD support ``application/json`` request and response message body encoding. For HTTP messages carrying binary share data, this is expected to be a particularly poor encoding. @@ -360,7 +362,11 @@ Because of the simple types used throughout and the equivalence described in `RFC 7049`_ these examples should be representative regardless of which of these two encodings is chosen. -One exception to this rule is for sets. +There are two exceptions to this rule. + +1. Sets +!!!!!!! + For CBOR messages, any sequence that is semantically a set (i.e. no repeated values allowed, order doesn't matter, and elements are hashable in Python) should be sent as a set. Tag 6.258 is used to indicate sets in CBOR; @@ -368,12 +374,12 @@ see `the CBOR registry Date: Mon, 13 Mar 2023 10:44:09 -0400 Subject: [PATCH 1584/2309] nail it down --- docs/proposed/http-storage-node-protocol.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index f6d90526e..f9f2cd868 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -436,6 +436,7 @@ Encoding ~~~~~~~~ * ``storage_index`` MUST be `Base32`_ encoded in URLs. +* ``share_number`` MUST be a decimal representation General ~~~~~~~ From 6dc6d6f39f35fe6f51002b46b90727381b142e04 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 13 Mar 2023 11:06:16 -0400 Subject: [PATCH 1585/2309] inline the actual base32 alphabet we use --- docs/proposed/http-storage-node-protocol.rst | 100 ++++++++++++++++++- 1 file changed, 96 insertions(+), 4 deletions(-) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index f9f2cd868..21e27d7dd 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -244,9 +244,9 @@ When Bob's client issues HTTP requests to Alice's storage node it includes the * .. note:: Foolscap TubIDs are 20 bytes (SHA1 digest of the certificate). - They are encoded with Base32 for a length of 32 bytes. + They are encoded with `Base32_` for a length of 32 bytes. SPKI information discussed here is 32 bytes (SHA256 digest). - They would be encoded in Base32 for a length of 52 bytes. + They would be encoded in `Base32`_ for a length of 52 bytes. `unpadded base64url`_ provides a more compact encoding of the information while remaining URL-compatible. This would encode the SPKI information for a length of merely 43 bytes. SHA1, @@ -336,6 +336,100 @@ and shares. A particular resource is addressed by the HTTP request path. Details about the interface are encoded in the HTTP message body. +String Encoding +~~~~~~~~~~~~~~~ + +.. _Base32: + +Where the specification refers to Base32 the meaning is *unpadded* Base32 encoding as specified by `RFC 4648`_ using a *lowercase variation* of the alphabet from Section 6. + +That is, the alphabet is: + +.. list-table:: Base32 Alphabet + :header-rows: 1 + + * - Value + - Encoding + - Value + - Encoding + - Value + - Encoding + - Value + - Encoding + + * - 0 + - a + - 9 + - j + - 18 + - s + - 27 + - 3 + * - 1 + - b + - 10 + - k + - 19 + - t + - 28 + - 4 + * - 2 + - c + - 11 + - l + - 20 + - u + - 29 + - 5 + * - 3 + - d + - 12 + - m + - 21 + - v + - 30 + - 6 + * - 4 + - e + - 13 + - n + - 22 + - w + - 31 + - 7 + * - 5 + - f + - 14 + - o + - 23 + - x + - + - + * - 6 + - g + - 15 + - p + - 24 + - y + - + - + * - 7 + - h + - 16 + - q + - 25 + - z + - + - + * - 8 + - i + - 17 + - r + - 26 + - 2 + - + - + Message Encoding ~~~~~~~~~~~~~~~~ @@ -1058,8 +1152,6 @@ otherwise it will read a byte which won't match `b""`:: .. _Base64: https://www.rfc-editor.org/rfc/rfc4648#section-4 -.. _Base32: https://www.rfc-editor.org/rfc/rfc4648#section-6 - .. _RFC 4648: https://tools.ietf.org/html/rfc4648 .. _RFC 7469: https://tools.ietf.org/html/rfc7469#section-2.4 From 6771ca8ce4caf34cceacd806c8c7c45eb80af315 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 13 Mar 2023 13:29:53 -0400 Subject: [PATCH 1586/2309] fix table markup --- docs/proposed/http-storage-node-protocol.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index 21e27d7dd..ebe39578c 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -346,7 +346,7 @@ Where the specification refers to Base32 the meaning is *unpadded* Base32 encodi That is, the alphabet is: .. list-table:: Base32 Alphabet - :header-rows: 1 + :header-rows: 1 * - Value - Encoding From fe0e159e52712c14557e0c188798f8286e33ca65 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 13 Mar 2023 13:30:32 -0400 Subject: [PATCH 1587/2309] Give base32 a section heading We don't have any other sections but ... :shrug: --- docs/proposed/http-storage-node-protocol.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index ebe39578c..f81b2bc79 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -341,6 +341,9 @@ String Encoding .. _Base32: +Base32 +!!!!!! + Where the specification refers to Base32 the meaning is *unpadded* Base32 encoding as specified by `RFC 4648`_ using a *lowercase variation* of the alphabet from Section 6. That is, the alphabet is: From 6a0a895ee88e34da3c798acc19c5800af3fda414 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 13 Mar 2023 13:37:01 -0400 Subject: [PATCH 1588/2309] Encode the reason limit in the implementation as well --- src/allmydata/storage/http_server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 5560d3a73..3ae16ae5c 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -273,7 +273,7 @@ _SCHEMAS = { "advise_corrupt_share": Schema( """ request = { - reason: tstr + reason: tstr .size (1..32765) } """ ), From e98967731952208896e78cf8e1b697b367c6f8a0 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 21 Mar 2023 11:20:25 -0400 Subject: [PATCH 1589/2309] Pass in a pool instead of pool options. --- src/allmydata/storage/http_client.py | 11 ++++------- src/allmydata/storage_client.py | 7 ++++--- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 3edf5f835..1d798fecc 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -311,21 +311,18 @@ class StorageClient(object): @classmethod def from_nurl( - cls, nurl: DecodedURL, reactor, persistent=True, retryAutomatically=True + cls, nurl: DecodedURL, reactor, pool: Optional[HTTPConnectionPool] = None ) -> StorageClient: """ Create a ``StorageClient`` for the given NURL. - - ``persistent`` and ``retryAutomatically`` arguments are passed to the - new HTTPConnectionPool. """ assert nurl.fragment == "v=1" assert nurl.scheme == "pb" swissnum = nurl.path[0].encode("ascii") certificate_hash = nurl.user.encode("ascii") - pool = HTTPConnectionPool(reactor, persistent=persistent) - pool.retryAutomatically = retryAutomatically - pool.maxPersistentPerHost = 20 + if pool is None: + pool = HTTPConnectionPool(reactor) + pool.maxPersistentPerHost = 20 if cls.TEST_MODE_REGISTER_HTTP_POOL is not None: cls.TEST_MODE_REGISTER_HTTP_POOL(pool) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 2888b10e7..19d6ef4a7 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -43,6 +43,7 @@ from configparser import NoSectionError import attr from hyperlink import DecodedURL +from twisted.web.client import HTTPConnectionPool from zope.interface import ( Attribute, Interface, @@ -1205,10 +1206,10 @@ class HTTPNativeStorageServer(service.MultiService): def request(reactor, nurl: DecodedURL): # Since we're just using this one off to check if the NURL # works, no need for persistent pool or other fanciness. + pool = HTTPConnectionPool(reactor, persistent=False) + pool.retryAutomatically = False return StorageClientGeneral( - StorageClient.from_nurl( - nurl, reactor, persistent=False, retryAutomatically=False - ) + StorageClient.from_nurl(nurl, reactor, pool) ).get_version() # LoopingCall.stop() doesn't cancel Deferreds, unfortunately: From b65bc9dca70048d6e1c41abbf47d93a4f4b30d3e Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 21 Mar 2023 11:22:43 -0400 Subject: [PATCH 1590/2309] Better explanation. --- src/allmydata/storage_client.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 19d6ef4a7..a52bb3f75 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -1038,7 +1038,10 @@ def _pick_a_http_server( # Logging errors breaks a bunch of tests, and it's not a _bug_ to # have a failed connection, it's often expected and transient. More # of a warning, really? - log.msg("Failed to connect to NURL: {}".format(failure)) + log.msg( + "Failed to connect to a storage server advertised by NURL: {}".format( + failure) + ) return None def succeeded(result: tuple[int, DecodedURL]): From 7ae8b50d14465a6b4fa3d7f5b35bfdbf5056f28a Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 21 Mar 2023 11:26:40 -0400 Subject: [PATCH 1591/2309] Async! --- src/allmydata/storage_client.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index a52bb3f75..dffa78bc4 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -1019,11 +1019,12 @@ class NativeStorageServer(service.MultiService): self._reconnector.reset() -def _pick_a_http_server( +@async_to_deferred +async def _pick_a_http_server( reactor, nurls: list[DecodedURL], request: Callable[[Any, DecodedURL], defer.Deferred[Any]] -) -> defer.Deferred[Optional[DecodedURL]]: +) -> Optional[DecodedURL]: """Pick the first server we successfully send a request to. Fires with ``None`` if no server was found, or with the ``DecodedURL`` of @@ -1034,22 +1035,19 @@ def _pick_a_http_server( for nurl in nurls ]) - def failed(failure: Failure): + try: + _, nurl = await queries + return nurl + except Exception as e: # Logging errors breaks a bunch of tests, and it's not a _bug_ to # have a failed connection, it's often expected and transient. More # of a warning, really? log.msg( "Failed to connect to a storage server advertised by NURL: {}".format( - failure) + e) ) return None - def succeeded(result: tuple[int, DecodedURL]): - _, nurl = result - return nurl - - return queries.addCallbacks(succeeded, failed) - @implementer(IServer) class HTTPNativeStorageServer(service.MultiService): From 14aeaea02223970c1cbc2afbb7499a22231fbb62 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 21 Mar 2023 11:29:19 -0400 Subject: [PATCH 1592/2309] Another todo. --- src/allmydata/storage_client.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index dffa78bc4..c88613803 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -1201,7 +1201,11 @@ class HTTPNativeStorageServer(service.MultiService): if self._istorage_server is None: # We haven't selected a server yet, so let's do so. - # TODO The problem with this scheme is that while picking + # TODO This is somewhat inefficient on startup: it takes two successful + # version() calls before we are live talking to a server, it could only + # be one. See https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3992 + + # TODO Another problem with this scheme is that while picking # the HTTP server to talk to, we don't have connection status # updates... https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3978 def request(reactor, nurl: DecodedURL): From 264269f409bb7e567c4632335ed1d8a3a20faef4 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 21 Mar 2023 11:29:50 -0400 Subject: [PATCH 1593/2309] Better test name. --- src/allmydata/test/test_storage_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/test_storage_client.py b/src/allmydata/test/test_storage_client.py index 8273966ce..91668e7ca 100644 --- a/src/allmydata/test/test_storage_client.py +++ b/src/allmydata/test/test_storage_client.py @@ -819,7 +819,7 @@ class PickHTTPServerTests(unittest.SynchronousTestCase): }) self.assertEqual(result, earliest_url) - def test_failures_are_retried(self): + def test_failures_are_turned_into_none(self): """ If the requests all fail, ``_pick_a_http_server`` returns ``None``. """ From 44f5057ed39cba4f853ad3aaf862244323b29858 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 22 Mar 2023 08:07:59 -0400 Subject: [PATCH 1594/2309] fix link markup --- docs/proposed/http-storage-node-protocol.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index f81b2bc79..493cf8f58 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -244,7 +244,7 @@ When Bob's client issues HTTP requests to Alice's storage node it includes the * .. note:: Foolscap TubIDs are 20 bytes (SHA1 digest of the certificate). - They are encoded with `Base32_` for a length of 32 bytes. + They are encoded with `Base32`_ for a length of 32 bytes. SPKI information discussed here is 32 bytes (SHA256 digest). They would be encoded in `Base32`_ for a length of 52 bytes. `unpadded base64url`_ provides a more compact encoding of the information while remaining URL-compatible. From 7c0b21916f376f139e2569242e443fea60c40723 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 22 Mar 2023 08:35:17 -0400 Subject: [PATCH 1595/2309] specify the unit of `available-space` --- docs/proposed/http-storage-node-protocol.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index 493cf8f58..3e74c94d6 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -560,6 +560,7 @@ For fields that are more general than a single API the semantics are as follows: * available-space: The server SHOULD use this field to advertise the amount of space that it currently considers unused and is willing to allocate for client requests. + The value is a number of bytes. ``PUT /storage/v1/lease/:storage_index`` From e7ed17af17c7c77daa24203954ff2ef2198875c6 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 22 Mar 2023 08:42:32 -0400 Subject: [PATCH 1596/2309] fix some editing errors about overreads and generally try to clarify --- docs/proposed/http-storage-node-protocol.rst | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index 3e74c94d6..5009a992e 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -950,8 +950,17 @@ For example:: } } -A server MUST return nothing for any bytes beyond the end of existing data for a test vector or read vector that reads tries to read such data. -As a result, if there is no data at all, an empty bytestring is returned no matter what the offset or length. +A client MAY send a test vector or read vector to bytes beyond the end of existing data. +In this case a server MUST behave as if the test or read vector referred to exactly as much data exists. + +For example, +consider the case where the server has 5 bytes of data for a particular share. +If a client sends a read vector with an ``offset`` of 1 and a ``size`` of 4 then the server MUST respond with all of the data except the first byte. +If a client sends a read vector with the same ``offset`` and a ``size`` of 5 (or any larger value) then the server MUST respond in the same way. + +Similarly, +if there is no data at all, +an empty byte string is returned no matter what the offset or length. Reading ~~~~~~~ From c49aa446552f3060b4f53bddd300e288be1eb21d Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 22 Mar 2023 09:04:15 -0400 Subject: [PATCH 1597/2309] Update the raw number and give a reference for interpretation --- docs/performance.rst | 5 +++-- docs/specifications/dirnodes.rst | 10 +++++----- src/allmydata/client.py | 2 -- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/docs/performance.rst b/docs/performance.rst index 6ddeb1fe8..a0487c72c 100644 --- a/docs/performance.rst +++ b/docs/performance.rst @@ -82,8 +82,9 @@ network: A memory footprint: N/K*A -notes: Tahoe-LAFS generates a new RSA keypair for each mutable file that it -publishes to a grid. This takes up to 1 or 2 seconds on a typical desktop PC. +notes: +Tahoe-LAFS generates a new RSA keypair for each mutable file that it publishes to a grid. +This takes around 100 milliseconds on a relatively high-end laptop from 2021. Part of the process of encrypting, encoding, and uploading a mutable file to a Tahoe-LAFS grid requires that the entire file be in memory at once. For larger diff --git a/docs/specifications/dirnodes.rst b/docs/specifications/dirnodes.rst index 88fcd0fa9..c53d28a26 100644 --- a/docs/specifications/dirnodes.rst +++ b/docs/specifications/dirnodes.rst @@ -267,7 +267,7 @@ How well does this design meet the goals? value, so there are no opportunities for staleness 9. monotonicity: VERY: the single point of access also protects against retrograde motion - + Confidentiality leaks in the storage servers @@ -332,8 +332,9 @@ MDMF design rules allow for efficient random-access reads from the middle of the file, which would give the index something useful to point at. The current SDMF design generates a new RSA public/private keypair for each -directory. This takes considerable time and CPU effort, generally one or two -seconds per directory. We have designed (but not yet built) a DSA-based +directory. This takes some time and CPU effort (around 100 milliseconds on a +relatively high-end 2021 laptop) per directory. +We have designed (but not yet built) a DSA-based mutable file scheme which will use shared parameters to reduce the directory-creation effort to a bare minimum (picking a random number instead of generating two random primes). @@ -363,7 +364,7 @@ single child, looking up a single child) would require pulling or pushing a lot of unrelated data, increasing network overhead (and necessitating test-and-set semantics for the modification side, which increases the chances that a user operation will fail, making it more challenging to provide -promises of atomicity to the user). +promises of atomicity to the user). It would also make it much more difficult to enable the delegation ("sharing") of specific directories. Since each aggregate "realm" provides @@ -469,4 +470,3 @@ Preventing delegation between communication parties is just as pointless as asking Bob to forget previously accessed files. However, there may be value to configuring the UI to ask Carol to not share files with Bob, or to removing all files from Bob's view at the same time his access is revoked. - diff --git a/src/allmydata/client.py b/src/allmydata/client.py index 2adf59660..8a10fe9e7 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -175,8 +175,6 @@ class KeyGenerator(object): """I return a Deferred that fires with a (verifyingkey, signingkey) pair. The returned key will be 2048 bit""" keysize = 2048 - # RSA key generation for a 2048 bit key takes between 0.8 and 3.2 - # secs signer, verifier = rsa.create_signing_keypair(keysize) return defer.succeed( (verifier, signer) ) From c1de2efd2d97d4bc79afb40fe0f9dfe6c450b01b Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 22 Mar 2023 09:04:31 -0400 Subject: [PATCH 1598/2309] news fragment --- newsfragments/3993.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3993.minor diff --git a/newsfragments/3993.minor b/newsfragments/3993.minor new file mode 100644 index 000000000..e69de29bb From 8d0869f6140f2c701aa856238c6a66243d301111 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 22 Mar 2023 09:30:52 -0400 Subject: [PATCH 1599/2309] Factor some shared pieces of CircleCI configuration out * Take DOCKERHUB_CONTEXT off of the single arbitrary job it was hung on and make it standalone. This isolates it from future changes to that particular job. * Take DOCKERHUB_AUTH out of `jobs` so it doesn't need a lot of extra boilerplate to pass schema validation. * Give the "nixos" job a Python version parameter so it can be instantiated multiple times to test multiple Python versions. Change the "NixOS unstable" instantiation to use Python 3.11 as a demonstration. * Move a lot of the implementation of the "nixos" job into a "nix" executor and a "nix-build" command that, together, do the generic setup required to do any nix-based builds. --- .circleci/config.yml | 215 +++++++++++++++++++++++++++---------------- 1 file changed, 135 insertions(+), 80 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ab0573a3f..d39e3de56 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -11,19 +11,30 @@ # version: 2.1 -# A template that can be shared between the two different image-building +# Every job that pushes a Docker image from Docker Hub must authenticate to +# it. Define a couple yaml anchors that can be used to supply a the necessary credentials. + +# First is a CircleCI job context which makes Docker Hub credentials available +# in the environment. +# +# Contexts are managed in the CircleCI web interface: +# +# https://app.circleci.com/settings/organization/github/tahoe-lafs/contexts +dockerhub-context-template: &DOCKERHUB_CONTEXT + context: "dockerhub-auth" + +# Next is a Docker executor template that gets the credentials from the +# environment and supplies them to the executor. +dockerhub-auth-template: &DOCKERHUB_AUTH + - auth: + username: $DOCKERHUB_USERNAME + password: $DOCKERHUB_PASSWORD + + # A template that can be shared between the two different image-building # workflows. .images: &IMAGES jobs: - # Every job that pushes a Docker image from Docker Hub needs to provide - # credentials. Use this first job to define a yaml anchor that can be - # used to supply a CircleCI job context which makes Docker Hub credentials - # available in the environment. - # - # Contexts are managed in the CircleCI web interface: - # - # https://app.circleci.com/settings/organization/github/tahoe-lafs/contexts - - "build-image-debian-11": &DOCKERHUB_CONTEXT + - "build-image-debian-11": <<: *DOCKERHUB_CONTEXT - "build-image-ubuntu-20-04": <<: *DOCKERHUB_CONTEXT @@ -71,12 +82,20 @@ workflows: {} - "nixos": - name: "NixOS 22.11" + name: "<>" nixpkgs: "22.11" + matrix: + parameters: + pythonVersion: + - "python310" - "nixos": - name: "NixOS unstable" + name: "<>" nixpkgs: "unstable" + matrix: + parameters: + pythonVersion: + - "python311" # Eventually, test against PyPy 3.8 #- "pypy27-buster": @@ -113,30 +132,7 @@ workflows: # Build as part of the workflow but only if requested. when: "<< pipeline.parameters.build-images >>" - jobs: - dockerhub-auth-template: - # This isn't a real job. It doesn't get scheduled as part of any - # workflow. Instead, it's just a place we can hang a yaml anchor to - # finish the Docker Hub authentication configuration. Workflow jobs using - # the DOCKERHUB_CONTEXT anchor will have access to the environment - # variables used here. These variables will allow the Docker Hub image - # pull to be authenticated and hopefully avoid hitting and rate limits. - docker: &DOCKERHUB_AUTH - - image: "null" - auth: - username: $DOCKERHUB_USERNAME - password: $DOCKERHUB_PASSWORD - - steps: - - run: - name: "CircleCI YAML schema conformity" - command: | - # This isn't a real command. We have to have something in this - # space, though, or the CircleCI yaml schema validator gets angry. - # Since this job is never scheduled this step is never run so the - # actual value here is irrelevant. - codechecks: docker: - <<: *DOCKERHUB_AUTH @@ -374,56 +370,32 @@ jobs: Reference the name of a niv-managed nixpkgs source (see `niv show` and nix/sources.json) type: "string" + pythonVersion: + description: >- + Reference the name of a Python package in nixpkgs to use. + type: "string" - docker: - # Run in a highly Nix-capable environment. - - <<: *DOCKERHUB_AUTH - image: "nixos/nix:2.10.3" - - environment: - # CACHIX_AUTH_TOKEN is manually set in the CircleCI web UI and - # allows us to push to CACHIX_NAME. We only need this set for - # `cachix use` in this step. - CACHIX_NAME: "tahoe-lafs-opensource" + executor: "nix" steps: - - "run": - # Get cachix for Nix-friendly caching. - name: "Install Basic Dependencies" - command: | - NIXPKGS="https://github.com/nixos/nixpkgs/archive/nixos-<>.tar.gz" - nix-env \ - --file $NIXPKGS \ - --install \ - -A cachix bash - # Activate it for "binary substitution". This sets up - # configuration tht lets Nix download something from the cache - # instead of building it locally, if possible. - cachix use "${CACHIX_NAME}" + - "nix-build": + nixpkgs: "<>" + pythonVersion: "<>" + buildSteps: + - "run": + name: "Unit Test" + command: | + # The dependencies are all built so we can allow more + # parallelism here. + source .circleci/lib.sh + cache_if_able nix-build \ + --cores 8 \ + --argstr pkgsVersion "nixpkgs-<>" \ + --argstr pythonVersion "<>" \ + nix/unittests.nix - - "checkout" - - "run": - # The Nix package doesn't know how to do this part, unfortunately. - name: "Generate version" - command: | - nix-shell \ - -p 'python3.withPackages (ps: [ ps.setuptools ])' \ - --run 'python setup.py update_version' - - "run": - name: "Test" - command: | - # CircleCI build environment looks like it has a zillion and a - # half cores. Don't let Nix autodetect this high core count - # because it blows up memory usage and fails the test run. Pick a - # number of cores that suites the build environment we're paying - # for (the free one!). - source .circleci/lib.sh - cache_if_able nix-build \ - --cores 8 \ - --argstr pkgsVersion "nixpkgs-<>" \ - nix/tests.nix typechecks: docker: @@ -527,7 +499,6 @@ jobs: # build-image-pypy27-buster: # <<: *BUILD_IMAGE - # environment: # DISTRO: "pypy" # TAG: "buster" @@ -535,3 +506,87 @@ jobs: # # setting up PyPy 3 in the image building toolchain. This value is just # # for constructing the right Docker image tag. # PYTHON_VERSION: "2" + +executors: + nix: + docker: + # Run in a highly Nix-capable environment. + - <<: *DOCKERHUB_AUTH + image: "nixos/nix:2.10.3" + environment: + # CACHIX_AUTH_TOKEN is manually set in the CircleCI web UI and + # allows us to push to CACHIX_NAME. We only need this set for + # `cachix use` in this step. + CACHIX_NAME: "tahoe-lafs-opensource" + +commands: + nix-build: + parameters: + nixpkgs: + description: >- + Reference the name of a niv-managed nixpkgs source (see `niv show` + and nix/sources.json) + type: "string" + pythonVersion: + description: >- + Reference the name of a Python package in nixpkgs to use. + type: "string" + buildSteps: + description: >- + The build steps to execute after setting up the build environment. + type: "steps" + + steps: + - "run": + # Get cachix for Nix-friendly caching. + name: "Install Basic Dependencies" + command: | + NIXPKGS="https://github.com/nixos/nixpkgs/archive/nixos-<>.tar.gz" + nix-env \ + --file $NIXPKGS \ + --install \ + -A cachix bash + # Activate it for "binary substitution". This sets up + # configuration tht lets Nix download something from the cache + # instead of building it locally, if possible. + cachix use "${CACHIX_NAME}" + + - "checkout" + + - "run": + # The Nix package doesn't know how to do this part, unfortunately. + name: "Generate version" + command: | + nix-shell \ + -p 'python3.withPackages (ps: [ ps.setuptools ])' \ + --run 'python setup.py update_version' + + - "run": + name: "Build Dependencies" + command: | + # CircleCI build environment looks like it has a zillion and a + # half cores. Don't let Nix autodetect this high core count + # because it blows up memory usage and fails the test run. Pick a + # number of cores that suits the build environment we're paying + # for (the free one!). + source .circleci/lib.sh + # nix-shell will build all of the dependencies of the target but + # not the target itself. + cache_if_able nix-shell \ + --run "" \ + --cores 3 \ + --argstr pkgsVersion "nixpkgs-<>" \ + --argstr pythonVersion "<>" \ + ./default.nix + + - "run": + name: "Build Package" + command: | + source .circleci/lib.sh + cache_if_able nix-build \ + --cores 4 \ + --argstr pkgsVersion "nixpkgs-<>" \ + --argstr pythonVersion "<>" \ + ./default.nix + + - steps: "<>" From bc424dc1d1abbed3edc44eb053908ea0ca58efd6 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 22 Mar 2023 09:36:31 -0400 Subject: [PATCH 1600/2309] news fragment --- newsfragments/3994.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3994.minor diff --git a/newsfragments/3994.minor b/newsfragments/3994.minor new file mode 100644 index 000000000..e69de29bb From 727d10af931bed91d3a9216d97f8687869210d63 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 22 Mar 2023 09:40:58 -0400 Subject: [PATCH 1601/2309] hit the right build target --- .circleci/config.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index d39e3de56..b64152a94 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -392,10 +392,7 @@ jobs: --cores 8 \ --argstr pkgsVersion "nixpkgs-<>" \ --argstr pythonVersion "<>" \ - nix/unittests.nix - - - + nix/tests.nix typechecks: docker: From 0da059b64486642864c0dbd211ccdb98909d79c1 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 24 Mar 2023 10:10:18 -0400 Subject: [PATCH 1602/2309] Update the connection status during the initial choice of NURLs. --- src/allmydata/storage_client.py | 50 ++++++++++++----------- src/allmydata/test/test_storage_client.py | 23 +++++++---- 2 files changed, 41 insertions(+), 32 deletions(-) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index c88613803..f71931c8b 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -1024,7 +1024,7 @@ async def _pick_a_http_server( reactor, nurls: list[DecodedURL], request: Callable[[Any, DecodedURL], defer.Deferred[Any]] -) -> Optional[DecodedURL]: +) -> DecodedURL: """Pick the first server we successfully send a request to. Fires with ``None`` if no server was found, or with the ``DecodedURL`` of @@ -1035,18 +1035,8 @@ async def _pick_a_http_server( for nurl in nurls ]) - try: - _, nurl = await queries - return nurl - except Exception as e: - # Logging errors breaks a bunch of tests, and it's not a _bug_ to - # have a failed connection, it's often expected and transient. More - # of a warning, really? - log.msg( - "Failed to connect to a storage server advertised by NURL: {}".format( - e) - ) - return None + _, nurl = await queries + return nurl @implementer(IServer) @@ -1223,19 +1213,31 @@ class HTTPNativeStorageServer(service.MultiService): picking = _pick_a_http_server(reactor, self._nurls, request) self._connecting_deferred = picking try: - nurl = await picking - finally: - self._connecting_deferred = None - - if nurl is None: - # We failed to find a server to connect to. Perhaps the next - # iteration of the loop will succeed. - return - else: - self._istorage_server = _HTTPStorageServer.from_http_client( - StorageClient.from_nurl(nurl, reactor) + try: + nurl = await picking + finally: + self._connecting_deferred = None + except Exception as e: + # Logging errors breaks a bunch of tests, and it's not a _bug_ to + # have a failed connection, it's often expected and transient. More + # of a warning, really? + log.msg( + "Failed to connect to a storage server advertised by NURL: {}".format(e) ) + # Update the connection status: + self._failed_to_connect(Failure(e)) + + # Since we failed to find a server to connect to, give up + # for now. Perhaps the next iteration of the loop will + # succeed. + return + + # iF we've gotten this far, we've found a working NURL. + self._istorage_server = _HTTPStorageServer.from_http_client( + StorageClient.from_nurl(nurl, reactor) + ) + result = self._istorage_server.get_version() def remove_connecting_deferred(result): diff --git a/src/allmydata/test/test_storage_client.py b/src/allmydata/test/test_storage_client.py index 91668e7ca..cf4a939e8 100644 --- a/src/allmydata/test/test_storage_client.py +++ b/src/allmydata/test/test_storage_client.py @@ -63,6 +63,8 @@ from foolscap.ipb import ( IConnectionHintHandler, ) +from allmydata.util.deferredutil import MultiFailure + from .no_network import LocalWrapper from .common import ( EMPTY_CLIENT_CONFIG, @@ -782,7 +784,7 @@ storage: class PickHTTPServerTests(unittest.SynchronousTestCase): """Tests for ``_pick_a_http_server``.""" - def pick_result(self, url_to_results: dict[DecodedURL, tuple[float, Union[Exception, Any]]]) -> Optional[DecodedURL]: + def pick_result(self, url_to_results: dict[DecodedURL, tuple[float, Union[Exception, Any]]]) -> Deferred[DecodedURL]: """ Given mapping of URLs to (delay, result), return the URL of the first selected server, or None. @@ -803,7 +805,7 @@ class PickHTTPServerTests(unittest.SynchronousTestCase): d = _pick_a_http_server(clock, list(url_to_results.keys()), request) for i in range(100): clock.advance(0.1) - return self.successResultOf(d) + return d def test_first_successful_connect_is_picked(self): """ @@ -817,16 +819,21 @@ class PickHTTPServerTests(unittest.SynchronousTestCase): earliest_url: (1, None), bad_url: (0.5, RuntimeError()), }) - self.assertEqual(result, earliest_url) + self.assertEqual(self.successResultOf(result), earliest_url) - def test_failures_are_turned_into_none(self): + def test_failures_include_all_reasons(self): """ - If the requests all fail, ``_pick_a_http_server`` returns ``None``. + If all the requests fail, ``_pick_a_http_server`` raises a + ``allmydata.util.deferredutil.MultiFailure``. """ eventually_good_url = DecodedURL.from_text("http://good") bad_url = DecodedURL.from_text("http://bad") + exception1 = RuntimeError() + exception2 = ZeroDivisionError() result = self.pick_result({ - eventually_good_url: (1, ZeroDivisionError()), - bad_url: (0.1, RuntimeError()) + eventually_good_url: (1, exception1), + bad_url: (0.1, exception2), }) - self.assertEqual(result, None) + exc = self.failureResultOf(result).value + self.assertIsInstance(exc, MultiFailure) + self.assertEqual({f.value for f in exc.failures}, {exception2, exception1}) From 6659350ff3279c9c4162f16f5a14f8aa4d10fee4 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 24 Mar 2023 10:18:15 -0400 Subject: [PATCH 1603/2309] Improve type annotations. --- src/allmydata/util/deferredutil.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/allmydata/util/deferredutil.py b/src/allmydata/util/deferredutil.py index 83de411ce..77451b132 100644 --- a/src/allmydata/util/deferredutil.py +++ b/src/allmydata/util/deferredutil.py @@ -14,6 +14,7 @@ from typing import ( TypeVar, Optional, ) +from typing_extensions import Awaitable, ParamSpec from foolscap.api import eventually from eliot.twisted import ( @@ -226,7 +227,11 @@ def until( break -def async_to_deferred(f): +P = ParamSpec("P") +R = TypeVar("R") + + +def async_to_deferred(f: Callable[P, Awaitable[R]]) -> Callable[P, Deferred[R]]: """ Wrap an async function to return a Deferred instead. @@ -234,8 +239,8 @@ def async_to_deferred(f): """ @wraps(f) - def not_async(*args, **kwargs): - return defer.Deferred.fromCoroutine(f(*args, **kwargs)) + def not_async(*args: P.args, **kwargs: P.kwargs) -> Deferred[R]: + return defer.Deferred.fromCoroutine(f(*args, **kwargs)) # type: ignore return not_async From f0e60a80afa1b970299a9ebf97da1f7aeb12d784 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 24 Mar 2023 10:22:52 -0400 Subject: [PATCH 1604/2309] Remove unneeded import. --- src/allmydata/test/test_storage_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/test_storage_client.py b/src/allmydata/test/test_storage_client.py index cf4a939e8..0671526ae 100644 --- a/src/allmydata/test/test_storage_client.py +++ b/src/allmydata/test/test_storage_client.py @@ -8,7 +8,7 @@ from json import ( loads, ) import hashlib -from typing import Union, Any, Optional +from typing import Union, Any from hyperlink import DecodedURL from fixtures import ( From 7715c4c6d01fda3da79245a3438360281b5abab9 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 24 Mar 2023 10:23:10 -0400 Subject: [PATCH 1605/2309] News fragment. --- newsfragments/3978.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3978.minor diff --git a/newsfragments/3978.minor b/newsfragments/3978.minor new file mode 100644 index 000000000..e69de29bb From 9baafea00ed8aab9703c6d5af53a2efbed3303b0 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 24 Mar 2023 11:06:58 -0400 Subject: [PATCH 1606/2309] Refactor: simplify code so there are fewer codepaths. --- src/allmydata/storage_client.py | 68 ++++++++++++--------------------- 1 file changed, 25 insertions(+), 43 deletions(-) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index f71931c8b..96f37e599 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -1188,16 +1188,16 @@ class HTTPNativeStorageServer(service.MultiService): @async_to_deferred async def _connect(self): - if self._istorage_server is None: + async def get_istorage_server() -> _HTTPStorageServer: + if self._istorage_server is not None: + return self._istorage_server + # We haven't selected a server yet, so let's do so. # TODO This is somewhat inefficient on startup: it takes two successful # version() calls before we are live talking to a server, it could only # be one. See https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3992 - # TODO Another problem with this scheme is that while picking - # the HTTP server to talk to, we don't have connection status - # updates... https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3978 def request(reactor, nurl: DecodedURL): # Since we're just using this one off to check if the NURL # works, no need for persistent pool or other fanciness. @@ -1213,51 +1213,33 @@ class HTTPNativeStorageServer(service.MultiService): picking = _pick_a_http_server(reactor, self._nurls, request) self._connecting_deferred = picking try: - try: - nurl = await picking - finally: - self._connecting_deferred = None - except Exception as e: - # Logging errors breaks a bunch of tests, and it's not a _bug_ to - # have a failed connection, it's often expected and transient. More - # of a warning, really? - log.msg( - "Failed to connect to a storage server advertised by NURL: {}".format(e) - ) + nurl = await picking + finally: + self._connecting_deferred = None - # Update the connection status: - self._failed_to_connect(Failure(e)) - - # Since we failed to find a server to connect to, give up - # for now. Perhaps the next iteration of the loop will - # succeed. - return - - # iF we've gotten this far, we've found a working NURL. + # If we've gotten this far, we've found a working NURL. self._istorage_server = _HTTPStorageServer.from_http_client( StorageClient.from_nurl(nurl, reactor) ) + return self._istorage_server - result = self._istorage_server.get_version() - - def remove_connecting_deferred(result): - self._connecting_deferred = None - return result - - # Set a short timeout since we're relying on this for server liveness. - self._connecting_deferred = result.addTimeout(5, self._reactor).addBoth( - remove_connecting_deferred).addCallbacks( - self._got_version, - self._failed_to_connect - ) - - # We don't want to do another iteration of the loop until this - # iteration has finished, so wait here: try: - if self._connecting_deferred is not None: - await self._connecting_deferred - except Exception as e: - log.msg(f"Failed to connect to a HTTP storage server: {e}", level=log.CURIOUS) + try: + storage_server = await get_istorage_server() + + # Get the version from the remote server. Set a short timeout since + # we're relying on this for server liveness. + self._connecting_deferred = storage_server.get_version().addTimeout( + 5, self._reactor) + # We don't want to do another iteration of the loop until this + # iteration has finished, so wait here: + version = await self._connecting_deferred + self._got_version(version) + except Exception as e: + log.msg(f"Failed to connect to a HTTP storage server: {e}", level=log.CURIOUS) + self._failed_to_connect(Failure(e)) + finally: + self._connecting_deferred = None def stopService(self): if self._connecting_deferred is not None: From 33d30b5c80bad6a647e1d8e6d6268555e4a72826 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 24 Mar 2023 11:20:31 -0400 Subject: [PATCH 1607/2309] Type annotations. --- src/allmydata/storage_client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 96f37e599..de756e322 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -1069,7 +1069,7 @@ class HTTPNativeStorageServer(service.MultiService): DecodedURL.from_text(u) for u in announcement[ANONYMOUS_STORAGE_NURLS] ] - self._istorage_server = None + self._istorage_server : Optional[_HTTPStorageServer] = None self._connection_status = connection_status.ConnectionStatus.unstarted() self._version = None @@ -1456,7 +1456,7 @@ class _HTTPStorageServer(object): _http_client = attr.ib(type=StorageClient) @staticmethod - def from_http_client(http_client): # type: (StorageClient) -> _HTTPStorageServer + def from_http_client(http_client: StorageClient) -> _HTTPStorageServer: """ Create an ``IStorageServer`` from a HTTP ``StorageClient``. """ From 1f29d5a23a6c772c35588f01b1c2a853691a4f5c Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 24 Mar 2023 11:38:15 -0400 Subject: [PATCH 1608/2309] News fragment. --- newsfragments/3996.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3996.minor diff --git a/newsfragments/3996.minor b/newsfragments/3996.minor new file mode 100644 index 000000000..e69de29bb From ce6b7aeb828e4cdd2f2056e18ae7872dd53d6787 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 24 Mar 2023 11:38:22 -0400 Subject: [PATCH 1609/2309] More modern pylint and flake8 and friends. --- setup.py | 2 +- tox.ini | 7 +++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/setup.py b/setup.py index 152c49f0e..6528b01ed 100644 --- a/setup.py +++ b/setup.py @@ -400,7 +400,7 @@ setup(name="tahoe-lafs", # also set in __init__.py # disagreeing on what is or is not a lint issue. We can bump # this version from time to time, but we will do it # intentionally. - "pyflakes == 2.2.0", + "pyflakes == 3.0.1", "coverage ~= 5.0", "mock", "tox ~= 3.0", diff --git a/tox.ini b/tox.ini index 382ba973e..447745784 100644 --- a/tox.ini +++ b/tox.ini @@ -100,10 +100,9 @@ commands = [testenv:codechecks] basepython = python3 deps = - # Newer versions of PyLint have buggy configuration - # (https://github.com/PyCQA/pylint/issues/4574), so stick to old version - # for now. - pylint < 2.5 + # Make sure we get a version of PyLint that respects config, and isn't too + # old. + pylint < 2.18, >2.14 # On macOS, git inside of towncrier needs $HOME. passenv = HOME setenv = From 56e3aaad03f1839f50fce1a526f1b969517cc538 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 24 Mar 2023 11:41:25 -0400 Subject: [PATCH 1610/2309] Lint fix and cleanup. --- src/allmydata/immutable/upload.py | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/src/allmydata/immutable/upload.py b/src/allmydata/immutable/upload.py index 1d70759ff..9d6842f44 100644 --- a/src/allmydata/immutable/upload.py +++ b/src/allmydata/immutable/upload.py @@ -2,22 +2,12 @@ Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals +from __future__ import annotations -from future.utils import PY2, native_str -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 +from future.utils import native_str from past.builtins import long, unicode from six import ensure_str -try: - from typing import List -except ImportError: - pass - import os, time, weakref, itertools import attr @@ -915,8 +905,8 @@ class _Accum(object): :ivar remaining: The number of bytes still expected. :ivar ciphertext: The bytes accumulated so far. """ - remaining = attr.ib(validator=attr.validators.instance_of(int)) # type: int - ciphertext = attr.ib(default=attr.Factory(list)) # type: List[bytes] + remaining : int = attr.ib(validator=attr.validators.instance_of(int)) + ciphertext : list[bytes] = attr.ib(default=attr.Factory(list)) def extend(self, size, # type: int From eb1cb84455883660301f51c7783497963c58007e Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 24 Mar 2023 11:42:38 -0400 Subject: [PATCH 1611/2309] Lint fix and cleanup. --- src/allmydata/introducer/server.py | 19 ++++--------------- 1 file changed, 4 insertions(+), 15 deletions(-) diff --git a/src/allmydata/introducer/server.py b/src/allmydata/introducer/server.py index f0638439a..98136157d 100644 --- a/src/allmydata/introducer/server.py +++ b/src/allmydata/introducer/server.py @@ -2,24 +2,13 @@ Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals +from __future__ import annotations - -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from past.builtins import long from six import ensure_text import time, os.path, textwrap - -try: - from typing import Any, Dict, Union -except ImportError: - pass +from typing import Any, Union from zope.interface import implementer from twisted.application import service @@ -161,11 +150,11 @@ class IntroducerService(service.MultiService, Referenceable): # v1 is the original protocol, added in 1.0 (but only advertised starting # in 1.3), removed in 1.12. v2 is the new signed protocol, added in 1.10 # TODO: reconcile bytes/str for keys - VERSION = { + VERSION : dict[Union[bytes, str], Any]= { #"http://allmydata.org/tahoe/protocols/introducer/v1": { }, b"http://allmydata.org/tahoe/protocols/introducer/v2": { }, b"application-version": allmydata.__full_version__.encode("utf-8"), - } # type: Dict[Union[bytes, str], Any] + } def __init__(self): service.MultiService.__init__(self) From 958c08d6f577fa97e3b5a146405b4329a14c6235 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 24 Mar 2023 11:44:14 -0400 Subject: [PATCH 1612/2309] Lint fix and cleanup. --- src/allmydata/node.py | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) diff --git a/src/allmydata/node.py b/src/allmydata/node.py index 34abb307f..58ee33ef5 100644 --- a/src/allmydata/node.py +++ b/src/allmydata/node.py @@ -4,14 +4,8 @@ a node for Tahoe-LAFS. Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals +from __future__ import annotations -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from six import ensure_str, ensure_text import json @@ -23,11 +17,7 @@ import errno from base64 import b32decode, b32encode from errno import ENOENT, EPERM from warnings import warn - -try: - from typing import Union -except ImportError: - pass +from typing import Union import attr @@ -281,8 +271,7 @@ def _error_about_old_config_files(basedir, generated_files): raise e -def ensure_text_and_abspath_expanduser_unicode(basedir): - # type: (Union[bytes, str]) -> str +def ensure_text_and_abspath_expanduser_unicode(basedir: Union[bytes, str]) -> str: return abspath_expanduser_unicode(ensure_text(basedir)) From 76ecdfb7bcd9e64bb191409c33a10fd5621a7102 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 24 Mar 2023 11:44:59 -0400 Subject: [PATCH 1613/2309] Fix lint. --- src/allmydata/scripts/admin.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/scripts/admin.py b/src/allmydata/scripts/admin.py index 579505399..02fd9a143 100644 --- a/src/allmydata/scripts/admin.py +++ b/src/allmydata/scripts/admin.py @@ -255,9 +255,9 @@ def do_admin(options): return f(so) -subCommands = [ +subCommands : SubCommands = [ ("admin", None, AdminCommand, "admin subcommands: use 'tahoe admin' for a list"), - ] # type: SubCommands + ] dispatch = { "admin": do_admin, From e1839ff30d629129b2aed9f0462a3f1bae1df9de Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 24 Mar 2023 11:45:56 -0400 Subject: [PATCH 1614/2309] Fix lints. --- src/allmydata/scripts/cli.py | 22 +++++----------------- 1 file changed, 5 insertions(+), 17 deletions(-) diff --git a/src/allmydata/scripts/cli.py b/src/allmydata/scripts/cli.py index 579b37906..6e1f28d11 100644 --- a/src/allmydata/scripts/cli.py +++ b/src/allmydata/scripts/cli.py @@ -1,22 +1,10 @@ """ Ported to Python 3. """ -from __future__ import unicode_literals -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - import os.path, re, fnmatch -try: - from allmydata.scripts.types_ import SubCommands, Parameters -except ImportError: - pass +from allmydata.scripts.types_ import SubCommands, Parameters from twisted.python import usage from allmydata.scripts.common import get_aliases, get_default_nodedir, \ @@ -29,14 +17,14 @@ NODEURL_RE=re.compile("http(s?)://([^:]*)(:([1-9][0-9]*))?") _default_nodedir = get_default_nodedir() class FileStoreOptions(BaseOptions): - optParameters = [ + optParameters : Parameters = [ ["node-url", "u", None, "Specify the URL of the Tahoe gateway node, such as " "'http://127.0.0.1:3456'. " "This overrides the URL found in the --node-directory ."], ["dir-cap", None, None, "Specify which dirnode URI should be used as the 'tahoe' alias."] - ] # type: Parameters + ] def postOptions(self): self["quiet"] = self.parent["quiet"] @@ -484,7 +472,7 @@ class DeepCheckOptions(FileStoreOptions): (which must be a directory), like 'tahoe check' but for multiple files. Optionally repair any problems found.""" -subCommands = [ +subCommands : SubCommands = [ ("mkdir", None, MakeDirectoryOptions, "Create a new directory."), ("add-alias", None, AddAliasOptions, "Add a new alias cap."), ("create-alias", None, CreateAliasOptions, "Create a new alias cap."), @@ -503,7 +491,7 @@ subCommands = [ ("check", None, CheckOptions, "Check a single file or directory."), ("deep-check", None, DeepCheckOptions, "Check all files/directories reachable from a starting point."), ("status", None, TahoeStatusCommand, "Various status information."), - ] # type: SubCommands + ] def mkdir(options): from allmydata.scripts import tahoe_mkdir From 0cd197d4d0b156124f73df7b9b16607c846208ee Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 24 Mar 2023 11:46:46 -0400 Subject: [PATCH 1615/2309] Update another instance of List. --- src/allmydata/immutable/upload.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/immutable/upload.py b/src/allmydata/immutable/upload.py index 9d6842f44..0421de4e0 100644 --- a/src/allmydata/immutable/upload.py +++ b/src/allmydata/immutable/upload.py @@ -910,7 +910,7 @@ class _Accum(object): def extend(self, size, # type: int - ciphertext, # type: List[bytes] + ciphertext, # type: list[bytes] ): """ Accumulate some more ciphertext. From ae29ea2b23cda231a953e9b1a8c92016c3ac0f53 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 24 Mar 2023 11:47:43 -0400 Subject: [PATCH 1616/2309] Fix lint, and some Python 3 cleanups. --- src/allmydata/scripts/common.py | 36 ++++++++------------------------- 1 file changed, 8 insertions(+), 28 deletions(-) diff --git a/src/allmydata/scripts/common.py b/src/allmydata/scripts/common.py index c9fc8e031..d6ca8556d 100644 --- a/src/allmydata/scripts/common.py +++ b/src/allmydata/scripts/common.py @@ -4,29 +4,13 @@ Ported to Python 3. """ -from __future__ import unicode_literals -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 -else: - from typing import Union - +from typing import Union, Optional import os, sys, textwrap import codecs from os.path import join import urllib.parse -try: - from typing import Optional - from .types_ import Parameters -except ImportError: - pass - from yaml import ( safe_dump, ) @@ -37,6 +21,8 @@ from allmydata.util.assertutil import precondition from allmydata.util.encodingutil import quote_output, \ quote_local_unicode_path, argv_to_abspath from allmydata.scripts.default_nodedir import _default_nodedir +from .types_ import Parameters + def get_default_nodedir(): return _default_nodedir @@ -59,7 +45,7 @@ class BaseOptions(usage.Options): def opt_version(self): raise usage.UsageError("--version not allowed on subcommands") - description = None # type: Optional[str] + description : Optional[str] = None description_unwrapped = None # type: Optional[str] def __str__(self): @@ -80,10 +66,10 @@ class BaseOptions(usage.Options): class BasedirOptions(BaseOptions): default_nodedir = _default_nodedir - optParameters = [ + optParameters : Parameters = [ ["basedir", "C", None, "Specify which Tahoe base directory should be used. [default: %s]" % quote_local_unicode_path(_default_nodedir)], - ] # type: Parameters + ] def parseArgs(self, basedir=None): # This finds the node-directory option correctly even if we are in a subcommand. @@ -283,9 +269,8 @@ def get_alias(aliases, path_unicode, default): quote_output(alias)) return uri.from_string_dirnode(aliases[alias]).to_string(), path[colon+1:] -def escape_path(path): - # type: (Union[str,bytes]) -> str - u""" +def escape_path(path: Union[str, bytes]) -> str: + """ Return path quoted to US-ASCII, valid URL characters. >>> path = u'/føö/bar/☃' @@ -302,9 +287,4 @@ def escape_path(path): ]), "ascii" ) - # Eventually (i.e. as part of Python 3 port) we want this to always return - # Unicode strings. However, to reduce diff sizes in the short term it'll - # return native string (i.e. bytes) on Python 2. - if PY2: - result = result.encode("ascii").__native__() return result From 29a66e51583f549c6d080dc0dd25cf2a77b7039a Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 24 Mar 2023 12:01:12 -0400 Subject: [PATCH 1617/2309] Fix lint. --- src/allmydata/scripts/debug.py | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/src/allmydata/scripts/debug.py b/src/allmydata/scripts/debug.py index 6201ce28f..b6eba842a 100644 --- a/src/allmydata/scripts/debug.py +++ b/src/allmydata/scripts/debug.py @@ -1,19 +1,8 @@ """ Ported to Python 3. """ -from __future__ import unicode_literals -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from future.utils import PY2, bchr -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - -try: - from allmydata.scripts.types_ import SubCommands -except ImportError: - pass +from future.utils import bchr import struct, time, os, sys @@ -31,6 +20,7 @@ from allmydata.mutable.common import NeedMoreDataError from allmydata.immutable.layout import ReadBucketProxy from allmydata.util import base32 from allmydata.util.encodingutil import quote_output +from allmydata.scripts.types_ import SubCommands class DumpOptions(BaseOptions): def getSynopsis(self): @@ -1076,9 +1066,9 @@ def do_debug(options): return f(so) -subCommands = [ +subCommands : SubCommands = [ ("debug", None, DebugCommand, "debug subcommands: use 'tahoe debug' for a list."), - ] # type: SubCommands + ] dispatch = { "debug": do_debug, From 0e6825709dbe28178e58d0d66820b139531a24b5 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 24 Mar 2023 12:03:04 -0400 Subject: [PATCH 1618/2309] Fix lints. --- src/allmydata/scripts/create_node.py | 48 ++++++++++------------------ src/allmydata/scripts/runner.py | 21 +++--------- 2 files changed, 21 insertions(+), 48 deletions(-) diff --git a/src/allmydata/scripts/create_node.py b/src/allmydata/scripts/create_node.py index 5d9da518b..7d15b95ec 100644 --- a/src/allmydata/scripts/create_node.py +++ b/src/allmydata/scripts/create_node.py @@ -1,25 +1,11 @@ -# Ported to Python 3 - -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division -from __future__ import unicode_literals - -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - import io import os -try: - from allmydata.scripts.types_ import ( - SubCommands, - Parameters, - Flags, - ) -except ImportError: - pass +from allmydata.scripts.types_ import ( + SubCommands, + Parameters, + Flags, +) from twisted.internet import reactor, defer from twisted.python.usage import UsageError @@ -48,7 +34,7 @@ def write_tac(basedir, nodetype): fileutil.write(os.path.join(basedir, "tahoe-%s.tac" % (nodetype,)), dummy_tac) -WHERE_OPTS = [ +WHERE_OPTS : Parameters = [ ("location", None, None, "Server location to advertise (e.g. tcp:example.org:12345)"), ("port", None, None, @@ -57,29 +43,29 @@ WHERE_OPTS = [ "Hostname to automatically set --location/--port when --listen=tcp"), ("listen", None, "tcp", "Comma-separated list of listener types (tcp,tor,i2p,none)."), -] # type: Parameters +] -TOR_OPTS = [ +TOR_OPTS : Parameters = [ ("tor-control-port", None, None, "Tor's control port endpoint descriptor string (e.g. tcp:127.0.0.1:9051 or unix:/var/run/tor/control)"), ("tor-executable", None, None, "The 'tor' executable to run (default is to search $PATH)."), -] # type: Parameters +] -TOR_FLAGS = [ +TOR_FLAGS : Flags = [ ("tor-launch", None, "Launch a tor instead of connecting to a tor control port."), -] # type: Flags +] -I2P_OPTS = [ +I2P_OPTS : Parameters = [ ("i2p-sam-port", None, None, "I2P's SAM API port endpoint descriptor string (e.g. tcp:127.0.0.1:7656)"), ("i2p-executable", None, None, "(future) The 'i2prouter' executable to run (default is to search $PATH)."), -] # type: Parameters +] -I2P_FLAGS = [ +I2P_FLAGS : Flags = [ ("i2p-launch", None, "(future) Launch an I2P router instead of connecting to a SAM API port."), -] # type: Flags +] def validate_where_options(o): if o['listen'] == "none": @@ -508,11 +494,11 @@ def create_introducer(config): defer.returnValue(0) -subCommands = [ +subCommands : SubCommands = [ ("create-node", None, CreateNodeOptions, "Create a node that acts as a client, server or both."), ("create-client", None, CreateClientOptions, "Create a client node (with storage initially disabled)."), ("create-introducer", None, CreateIntroducerOptions, "Create an introducer node."), -] # type: SubCommands +] dispatch = { "create-node": create_node, diff --git a/src/allmydata/scripts/runner.py b/src/allmydata/scripts/runner.py index d9fbc1b0a..18387cea5 100644 --- a/src/allmydata/scripts/runner.py +++ b/src/allmydata/scripts/runner.py @@ -1,28 +1,15 @@ -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division -from __future__ import unicode_literals - -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - import os, sys -from six.moves import StringIO +from io import StringIO from past.builtins import unicode import six -try: - from allmydata.scripts.types_ import SubCommands -except ImportError: - pass - from twisted.python import usage from twisted.internet import defer, task, threads from allmydata.scripts.common import get_default_nodedir from allmydata.scripts import debug, create_node, cli, \ admin, tahoe_run, tahoe_invite +from allmydata.scripts.types_ import SubCommands from allmydata.util.encodingutil import quote_local_unicode_path, argv_to_unicode from allmydata.util.eliotutil import ( opt_eliot_destination, @@ -47,9 +34,9 @@ if _default_nodedir: NODEDIR_HELP += " [default for most commands: " + quote_local_unicode_path(_default_nodedir) + "]" -process_control_commands = [ +process_control_commands : SubCommands = [ ("run", None, tahoe_run.RunOptions, "run a node without daemonizing"), -] # type: SubCommands +] class Options(usage.Options): From aea748a890e16b845b477f39eeb2d186fdd40ea9 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 24 Mar 2023 12:03:43 -0400 Subject: [PATCH 1619/2309] Fix lint. --- src/allmydata/scripts/tahoe_invite.py | 18 +++--------------- 1 file changed, 3 insertions(+), 15 deletions(-) diff --git a/src/allmydata/scripts/tahoe_invite.py b/src/allmydata/scripts/tahoe_invite.py index b62d6a463..b44efdeb9 100644 --- a/src/allmydata/scripts/tahoe_invite.py +++ b/src/allmydata/scripts/tahoe_invite.py @@ -1,19 +1,6 @@ """ Ported to Python 3. """ -from __future__ import unicode_literals -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - -try: - from allmydata.scripts.types_ import SubCommands -except ImportError: - pass from twisted.python import usage from twisted.internet import defer, reactor @@ -21,6 +8,7 @@ from twisted.internet import defer, reactor from allmydata.util.encodingutil import argv_to_abspath from allmydata.util import jsonbytes as json from allmydata.scripts.common import get_default_nodedir, get_introducer_furl +from allmydata.scripts.types_ import SubCommands from allmydata.client import read_config @@ -112,10 +100,10 @@ def invite(options): print("Completed successfully", file=out) -subCommands = [ +subCommands : SubCommands = [ ("invite", None, InviteOptions, "Invite a new node to this grid"), -] # type: SubCommands +] dispatch = { "invite": invite, From 494a977525f6db91c3a3f5be2a7b75dd91b0ce22 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 24 Mar 2023 12:06:01 -0400 Subject: [PATCH 1620/2309] Fix lint. --- src/allmydata/storage/http_server.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 3ae16ae5c..7437b3ec7 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -4,7 +4,7 @@ HTTP server for storage. from __future__ import annotations -from typing import Dict, List, Set, Tuple, Any, Callable, Union, cast +from typing import Any, Callable, Union, cast from functools import wraps from base64 import b64decode import binascii @@ -67,8 +67,8 @@ class ClientSecretsException(Exception): def _extract_secrets( - header_values, required_secrets -): # type: (List[str], Set[Secrets]) -> Dict[Secrets, bytes] + header_values: list[str], required_secrets: set[Secrets] +) -> dict[Secrets, bytes]: """ Given list of values of ``X-Tahoe-Authorization`` headers, and required secrets, return dictionary mapping secrets to decoded values. @@ -173,7 +173,7 @@ class UploadsInProgress(object): _uploads: dict[bytes, StorageIndexUploads] = Factory(dict) # Map BucketWriter to (storage index, share number) - _bucketwriters: dict[BucketWriter, Tuple[bytes, int]] = Factory(dict) + _bucketwriters: dict[BucketWriter, tuple[bytes, int]] = Factory(dict) def add_write_bucket( self, @@ -798,7 +798,9 @@ class HTTPServer(object): # The reason can be a string with explanation, so in theory it could be # longish? info = await self._read_encoded( - request, _SCHEMAS["advise_corrupt_share"], max_size=32768, + request, + _SCHEMAS["advise_corrupt_share"], + max_size=32768, ) bucket.advise_corrupt_share(info["reason"].encode("utf-8")) return b"" @@ -973,7 +975,7 @@ def listen_tls( endpoint: IStreamServerEndpoint, private_key_path: FilePath, cert_path: FilePath, -) -> Deferred[Tuple[DecodedURL, IListeningPort]]: +) -> Deferred[tuple[DecodedURL, IListeningPort]]: """ Start a HTTPS storage server on the given port, return the NURL and the listening port. From 3212311bbe3dd7f17bd8b7a7d74ad70fa503a7a1 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 24 Mar 2023 12:06:49 -0400 Subject: [PATCH 1621/2309] Fix lint. --- src/allmydata/storage/lease_schema.py | 17 ++--------------- 1 file changed, 2 insertions(+), 15 deletions(-) diff --git a/src/allmydata/storage/lease_schema.py b/src/allmydata/storage/lease_schema.py index 7e604388e..63d3d4ed8 100644 --- a/src/allmydata/storage/lease_schema.py +++ b/src/allmydata/storage/lease_schema.py @@ -2,19 +2,7 @@ Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals - -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - -try: - from typing import Union -except ImportError: - pass +from typing import Union import attr @@ -95,8 +83,7 @@ class HashedLeaseSerializer(object): cls._hash_secret, ) - def serialize(self, lease): - # type: (Union[LeaseInfo, HashedLeaseInfo]) -> bytes + def serialize(self, lease: Union[LeaseInfo, HashedLeaseInfo]) -> bytes: if isinstance(lease, LeaseInfo): # v2 of the immutable schema stores lease secrets hashed. If # we're given a LeaseInfo then it holds plaintext secrets. Hash From 8d84e8a19f66cc05421608e3f25378f96ddad68c Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 24 Mar 2023 12:08:04 -0400 Subject: [PATCH 1622/2309] Fix lint. --- src/allmydata/storage/server.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/allmydata/storage/server.py b/src/allmydata/storage/server.py index 2bf99d74c..6099636f8 100644 --- a/src/allmydata/storage/server.py +++ b/src/allmydata/storage/server.py @@ -2,8 +2,9 @@ Ported to Python 3. """ from __future__ import annotations + from future.utils import bytes_to_native_str -from typing import Dict, Tuple, Iterable +from typing import Iterable, Any import os, re @@ -823,7 +824,7 @@ class FoolscapStorageServer(Referenceable): # type: ignore # warner/foolscap#78 self._server = storage_server # Canaries and disconnect markers for BucketWriters created via Foolscap: - self._bucket_writer_disconnect_markers = {} # type: Dict[BucketWriter,Tuple[IRemoteReference, object]] + self._bucket_writer_disconnect_markers : dict[BucketWriter, tuple[IRemoteReference, Any]] = {} self._server.register_bucket_writer_close_handler(self._bucket_writer_closed) From 4b25a923567e53b74ee04bba78b0342b7e09fe4d Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 24 Mar 2023 13:49:44 -0400 Subject: [PATCH 1623/2309] Limit cryptography for now. --- setup.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 152c49f0e..b12b8f4a2 100644 --- a/setup.py +++ b/setup.py @@ -63,7 +63,11 @@ install_requires = [ # Twisted[conch] also depends on cryptography and Twisted[tls] # transitively depends on cryptography. So it's anyone's guess what # version of cryptography will *really* be installed. - "cryptography >= 2.6", + + # * cryptography 40 broke constants we need; should really be using them + # * via pyOpenSSL; will be fixed in + # * https://github.com/pyca/pyopenssl/issues/1201 + "cryptography >= 2.6, < 40", # * The SFTP frontend depends on Twisted 11.0.0 to fix the SSH server # rekeying bug From 74e3e27bea3309b33dc153114aad151baf7a4dd2 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 24 Mar 2023 15:06:27 -0400 Subject: [PATCH 1624/2309] Fix lint. --- src/allmydata/test/cli/test_create.py | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) diff --git a/src/allmydata/test/cli/test_create.py b/src/allmydata/test/cli/test_create.py index 609888fb3..1d1576082 100644 --- a/src/allmydata/test/cli/test_create.py +++ b/src/allmydata/test/cli/test_create.py @@ -1,21 +1,11 @@ """ Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals - -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 +from __future__ import annotations import os -try: - from typing import Any, List, Tuple -except ImportError: - pass +from typing import Any from twisted.trial import unittest from twisted.internet import defer, reactor @@ -356,8 +346,7 @@ class Config(unittest.TestCase): self.assertIn("is not empty", err) self.assertIn("To avoid clobbering anything, I am going to quit now", err) -def fake_config(testcase, module, result): - # type: (unittest.TestCase, Any, Any) -> List[Tuple] +def fake_config(testcase: unittest.TestCase, module: Any, result: Any) -> list[tuple]: """ Monkey-patch a fake configuration function into the given module. From 0c92fe554ddc8ce8b4c1b4efed943ff71158efab Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 24 Mar 2023 15:07:22 -0400 Subject: [PATCH 1625/2309] Fix lint. --- src/allmydata/test/eliotutil.py | 22 +++------------------- 1 file changed, 3 insertions(+), 19 deletions(-) diff --git a/src/allmydata/test/eliotutil.py b/src/allmydata/test/eliotutil.py index dd21f1e9d..bdc779f1d 100644 --- a/src/allmydata/test/eliotutil.py +++ b/src/allmydata/test/eliotutil.py @@ -3,18 +3,6 @@ Tools aimed at the interaction between tests and Eliot. Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals - -# Python 2 compatibility -# Can't use `builtins.str` because it's not JSON encodable: -# `exceptions.TypeError: is not JSON-encodeable` -from past.builtins import unicode as str -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, max, min # noqa: F401 from six import ensure_text @@ -23,11 +11,7 @@ __all__ = [ "EliotLoggedRunTest", ] -try: - from typing import Callable -except ImportError: - pass - +from typing import Callable from functools import ( partial, wraps, @@ -147,8 +131,8 @@ class EliotLoggedRunTest(object): def with_logging( - test_id, # type: str - test_method, # type: Callable + test_id: str, + test_method: Callable, ): """ Decorate a test method with additional log-related behaviors. From 1668b2fcf6c6c65250922853047059786096add6 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 24 Mar 2023 15:09:11 -0400 Subject: [PATCH 1626/2309] Fix lint. --- src/allmydata/test/no_network.py | 47 ++++++++++++-------------------- 1 file changed, 17 insertions(+), 30 deletions(-) diff --git a/src/allmydata/test/no_network.py b/src/allmydata/test/no_network.py index 66748e4b1..2346d96c1 100644 --- a/src/allmydata/test/no_network.py +++ b/src/allmydata/test/no_network.py @@ -1,34 +1,23 @@ """ -Ported to Python 3. +This contains a test harness that creates a full Tahoe grid in a single +process (actually in a single MultiService) which does not use the network. +It does not use an Introducer, and there are no foolscap Tubs. Each storage +server puts real shares on disk, but is accessed through loopback +RemoteReferences instead of over serialized SSL. It is not as complete as +the common.SystemTestMixin framework (which does use the network), but +should be considerably faster: on my laptop, it takes 50-80ms to start up, +whereas SystemTestMixin takes close to 2s. + +This should be useful for tests which want to examine and/or manipulate the +uploaded shares, checker/verifier/repairer tests, etc. The clients have no +Tubs, so it is not useful for tests that involve a Helper. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals -# This contains a test harness that creates a full Tahoe grid in a single -# process (actually in a single MultiService) which does not use the network. -# It does not use an Introducer, and there are no foolscap Tubs. Each storage -# server puts real shares on disk, but is accessed through loopback -# RemoteReferences instead of over serialized SSL. It is not as complete as -# the common.SystemTestMixin framework (which does use the network), but -# should be considerably faster: on my laptop, it takes 50-80ms to start up, -# whereas SystemTestMixin takes close to 2s. +from __future__ import annotations -# This should be useful for tests which want to examine and/or manipulate the -# uploaded shares, checker/verifier/repairer tests, etc. The clients have no -# Tubs, so it is not useful for tests that involve a Helper. - -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 -from past.builtins import unicode from six import ensure_text -try: - from typing import Dict, Callable -except ImportError: - pass +from typing import Callable import os from base64 import b32encode @@ -251,7 +240,7 @@ def create_no_network_client(basedir): :return: a Deferred yielding an instance of _Client subclass which does no actual networking but has the same API. """ - basedir = abspath_expanduser_unicode(unicode(basedir)) + basedir = abspath_expanduser_unicode(str(basedir)) fileutil.make_dirs(os.path.join(basedir, "private"), 0o700) from allmydata.client import read_config @@ -577,8 +566,7 @@ class GridTestMixin(object): pass return sorted(shares) - def copy_shares(self, uri): - # type: (bytes) -> Dict[bytes, bytes] + def copy_shares(self, uri: bytes) -> dict[bytes, bytes]: """ Read all of the share files for the given capability from the storage area of the storage servers created by ``set_up_grid``. @@ -630,8 +618,7 @@ class GridTestMixin(object): with open(i_sharefile, "wb") as f: f.write(corruptdata) - def corrupt_all_shares(self, uri, corruptor, debug=False): - # type: (bytes, Callable[[bytes, bool], bytes], bool) -> None + def corrupt_all_shares(self, uri: Callable, corruptor: Callable[[bytes, bool], bytes], debug: bool=False): """ Apply ``corruptor`` to the contents of all share files associated with a given capability and replace the share file contents with its result. From 9d45cd85c712c9ee857d79032e026b1b149bbf0f Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 24 Mar 2023 15:12:16 -0400 Subject: [PATCH 1627/2309] Fix lint. --- src/allmydata/test/test_download.py | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/src/allmydata/test/test_download.py b/src/allmydata/test/test_download.py index 85d89cde6..4d57fa828 100644 --- a/src/allmydata/test/test_download.py +++ b/src/allmydata/test/test_download.py @@ -1,23 +1,14 @@ """ Ported to Python 3. """ -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division -from __future__ import unicode_literals -from future.utils import PY2, bchr -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 +from future.utils import bchr # system-level upload+download roundtrip test, but using shares created from # a previous run. This asserts that the current code is capable of decoding # shares from a previous version. -try: - from typing import Any -except ImportError: - pass +from typing import Any import six import os @@ -1197,8 +1188,7 @@ class Corruption(_Base, unittest.TestCase): return d - def _corrupt_flip_all(self, ign, imm_uri, which): - # type: (Any, bytes, int) -> None + def _corrupt_flip_all(self, ign: Any, imm_uri: bytes, which: int) -> None: """ Flip the least significant bit at a given byte position in all share files for the given capability. From 0bdea026f0023b318389ed26b025bc3d5d1f5355 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 24 Mar 2023 15:13:20 -0400 Subject: [PATCH 1628/2309] Fix lint. --- src/allmydata/test/test_helper.py | 19 ++++--------------- 1 file changed, 4 insertions(+), 15 deletions(-) diff --git a/src/allmydata/test/test_helper.py b/src/allmydata/test/test_helper.py index 933a2b591..b280f95df 100644 --- a/src/allmydata/test/test_helper.py +++ b/src/allmydata/test/test_helper.py @@ -1,14 +1,7 @@ """ Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals - -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 +from __future__ import annotations import os from struct import ( @@ -17,13 +10,8 @@ from struct import ( from functools import ( partial, ) -import attr -try: - from typing import List - from allmydata.introducer.client import IntroducerClient -except ImportError: - pass +import attr from twisted.internet import defer from twisted.trial import unittest @@ -35,6 +23,7 @@ from eliot.twisted import ( inline_callbacks, ) +from allmydata.introducer.client import IntroducerClient from allmydata.crypto import aes from allmydata.storage.server import ( si_b2a, @@ -132,7 +121,7 @@ class FakeCHKCheckerAndUEBFetcher(object): )) class FakeClient(service.MultiService): - introducer_clients = [] # type: List[IntroducerClient] + introducer_clients : list[IntroducerClient] = [] DEFAULT_ENCODING_PARAMETERS = {"k":25, "happy": 75, "n": 100, From 0377f858c2aa4628a2cc86018fcdea01a3ccc00e Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 24 Mar 2023 15:14:23 -0400 Subject: [PATCH 1629/2309] Correct type. --- src/allmydata/test/no_network.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/no_network.py b/src/allmydata/test/no_network.py index 2346d96c1..ee1f48b17 100644 --- a/src/allmydata/test/no_network.py +++ b/src/allmydata/test/no_network.py @@ -618,7 +618,7 @@ class GridTestMixin(object): with open(i_sharefile, "wb") as f: f.write(corruptdata) - def corrupt_all_shares(self, uri: Callable, corruptor: Callable[[bytes, bool], bytes], debug: bool=False): + def corrupt_all_shares(self, uri: bytes, corruptor: Callable[[bytes, bool], bytes], debug: bool=False): """ Apply ``corruptor`` to the contents of all share files associated with a given capability and replace the share file contents with its result. From 0d92aecbf3b97237f611628404efb24d45a4196e Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 24 Mar 2023 15:14:59 -0400 Subject: [PATCH 1630/2309] Fix lint. --- src/allmydata/test/test_istorageserver.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index 9e7e7b6e1..ded9ac1ac 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -8,9 +8,9 @@ reused across tests, so each test should be careful to generate unique storage indexes. """ -from future.utils import bchr +from __future__ import annotations -from typing import Set +from future.utils import bchr from random import Random from unittest import SkipTest @@ -1041,7 +1041,7 @@ class IStorageServerMutableAPIsTestsMixin(object): class _SharedMixin(SystemTestMixin): """Base class for Foolscap and HTTP mixins.""" - SKIP_TESTS = set() # type: Set[str] + SKIP_TESTS : set[str] = set() def _get_istorage_server(self): native_server = next(iter(self.clients[0].storage_broker.get_known_servers())) From f5d9947368d799581e65609da2bd18dfb5352509 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 24 Mar 2023 15:15:51 -0400 Subject: [PATCH 1631/2309] Fix lint. --- src/allmydata/uri.py | 19 ++----------------- 1 file changed, 2 insertions(+), 17 deletions(-) diff --git a/src/allmydata/uri.py b/src/allmydata/uri.py index 5641771d3..fccf05db9 100644 --- a/src/allmydata/uri.py +++ b/src/allmydata/uri.py @@ -6,26 +6,11 @@ Ported to Python 3. Methods ending in to_string() are actually to_bytes(), possibly should be fixed in follow-up port. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals - -from future.utils import PY2 -if PY2: - # Don't import bytes or str, to prevent future's newbytes leaking and - # breaking code that only expects normal bytes. - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, dict, list, object, range, max, min # noqa: F401 - from past.builtins import unicode as str from past.builtins import unicode, long import re - -try: - from typing import Type -except ImportError: - pass +from typing import Type from zope.interface import implementer from twisted.python.components import registerAdapter @@ -707,7 +692,7 @@ class DirectoryURIVerifier(_DirectoryBaseURI): BASE_STRING=b'URI:DIR2-Verifier:' BASE_STRING_RE=re.compile(b'^'+BASE_STRING) - INNER_URI_CLASS=SSKVerifierURI # type: Type[IVerifierURI] + INNER_URI_CLASS : Type[IVerifierURI] = SSKVerifierURI def __init__(self, filenode_uri=None): if filenode_uri: From 63549c71efee7c952ea0a9d8b3a80e4a53fa7236 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 24 Mar 2023 15:18:46 -0400 Subject: [PATCH 1632/2309] Fix lints, remove some Python 2 junk. --- src/allmydata/util/base32.py | 60 +++++++++++------------------------- 1 file changed, 18 insertions(+), 42 deletions(-) diff --git a/src/allmydata/util/base32.py b/src/allmydata/util/base32.py index ab65beeac..19a3bbe26 100644 --- a/src/allmydata/util/base32.py +++ b/src/allmydata/util/base32.py @@ -3,30 +3,11 @@ Base32 encoding. Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals - -from future.utils import PY2 -if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - -if PY2: - def backwardscompat_bytes(b): - """ - Replace Future bytes with native Python 2 bytes, so % works - consistently until other modules are ported. - """ - return getattr(b, "__native__", lambda: b)() - import string - maketrans = string.maketrans -else: - def backwardscompat_bytes(b): - return b - maketrans = bytes.maketrans - from typing import Optional +def backwardscompat_bytes(b): + return b +maketrans = bytes.maketrans +from typing import Optional import base64 from allmydata.util.assertutil import precondition @@ -34,7 +15,7 @@ from allmydata.util.assertutil import precondition rfc3548_alphabet = b"abcdefghijklmnopqrstuvwxyz234567" # RFC3548 standard used by Gnutella, Content-Addressable Web, THEX, Bitzi, Web-Calculus... chars = rfc3548_alphabet -vals = backwardscompat_bytes(bytes(range(32))) +vals = bytes(range(32)) c2vtranstable = maketrans(chars, vals) v2ctranstable = maketrans(vals, chars) identitytranstable = maketrans(b'', b'') @@ -61,16 +42,16 @@ def get_trailing_chars_without_lsbs(N): d = {} return b''.join(_get_trailing_chars_without_lsbs(N, d=d)) -BASE32CHAR = backwardscompat_bytes(b'['+get_trailing_chars_without_lsbs(0)+b']') -BASE32CHAR_4bits = backwardscompat_bytes(b'['+get_trailing_chars_without_lsbs(1)+b']') -BASE32CHAR_3bits = backwardscompat_bytes(b'['+get_trailing_chars_without_lsbs(2)+b']') -BASE32CHAR_2bits = backwardscompat_bytes(b'['+get_trailing_chars_without_lsbs(3)+b']') -BASE32CHAR_1bits = backwardscompat_bytes(b'['+get_trailing_chars_without_lsbs(4)+b']') -BASE32STR_1byte = backwardscompat_bytes(BASE32CHAR+BASE32CHAR_3bits) -BASE32STR_2bytes = backwardscompat_bytes(BASE32CHAR+b'{3}'+BASE32CHAR_1bits) -BASE32STR_3bytes = backwardscompat_bytes(BASE32CHAR+b'{4}'+BASE32CHAR_4bits) -BASE32STR_4bytes = backwardscompat_bytes(BASE32CHAR+b'{6}'+BASE32CHAR_2bits) -BASE32STR_anybytes = backwardscompat_bytes(bytes(b'((?:%s{8})*') % (BASE32CHAR,) + bytes(b"(?:|%s|%s|%s|%s))") % (BASE32STR_1byte, BASE32STR_2bytes, BASE32STR_3bytes, BASE32STR_4bytes)) +BASE32CHAR = b'['+get_trailing_chars_without_lsbs(0)+b']' +BASE32CHAR_4bits = b'['+get_trailing_chars_without_lsbs(1)+b']' +BASE32CHAR_3bits = b'['+get_trailing_chars_without_lsbs(2)+b']' +BASE32CHAR_2bits = b'['+get_trailing_chars_without_lsbs(3)+b']' +BASE32CHAR_1bits = b'['+get_trailing_chars_without_lsbs(4)+b']' +BASE32STR_1byte = BASE32CHAR+BASE32CHAR_3bits +BASE32STR_2bytes = BASE32CHAR+b'{3}'+BASE32CHAR_1bits +BASE32STR_3bytes = BASE32CHAR+b'{4}'+BASE32CHAR_4bits +BASE32STR_4bytes = BASE32CHAR+b'{6}'+BASE32CHAR_2bits +BASE32STR_anybytes = bytes(b'((?:%s{8})*') % (BASE32CHAR,) + bytes(b"(?:|%s|%s|%s|%s))") % (BASE32STR_1byte, BASE32STR_2bytes, BASE32STR_3bytes, BASE32STR_4bytes) def b2a(os): # type: (bytes) -> bytes """ @@ -80,7 +61,7 @@ def b2a(os): # type: (bytes) -> bytes """ return base64.b32encode(os).rstrip(b"=").lower() -def b2a_or_none(os): # type: (Optional[bytes]) -> Optional[bytes] +def b2a_or_none(os: Optional[bytes]) -> Optional[bytes]: if os is not None: return b2a(os) return None @@ -100,8 +81,6 @@ NUM_OS_TO_NUM_QS=(0, 2, 4, 5, 7,) NUM_QS_TO_NUM_OS=(0, 1, 1, 2, 2, 3, 3, 4) NUM_QS_LEGIT=(1, 0, 1, 0, 1, 1, 0, 1,) NUM_QS_TO_NUM_BITS=tuple([_x*8 for _x in NUM_QS_TO_NUM_OS]) -if PY2: - del _x # A fast way to determine whether a given string *could* be base-32 encoded data, assuming that the # original data had 8K bits for a positive integer K. @@ -135,8 +114,6 @@ def a2b(cs): # type: (bytes) -> bytes """ @param cs the base-32 encoded data (as bytes) """ - # Workaround Future newbytes issues by converting to real bytes on Python 2: - cs = backwardscompat_bytes(cs) precondition(could_be_base32_encoded(cs), "cs is required to be possibly base32 encoded data.", cs=cs) precondition(isinstance(cs, bytes), cs) @@ -144,9 +121,8 @@ def a2b(cs): # type: (bytes) -> bytes # Add padding back, to make Python's base64 module happy: while (len(cs) * 5) % 8 != 0: cs += b"=" - # Let newbytes come through and still work on Python 2, where the base64 - # module gets confused by them. - return base64.b32decode(backwardscompat_bytes(cs)) + + return base64.b32decode(cs) __all__ = ["b2a", "a2b", "b2a_or_none", "BASE32CHAR_3bits", "BASE32CHAR_1bits", "BASE32CHAR", "BASE32STR_anybytes", "could_be_base32_encoded"] From 6ce53000f0382cf53b72006d1df48648a2e8f651 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 24 Mar 2023 15:19:39 -0400 Subject: [PATCH 1633/2309] Fix lint. --- src/allmydata/util/deferredutil.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/allmydata/util/deferredutil.py b/src/allmydata/util/deferredutil.py index 83de411ce..70ce8dade 100644 --- a/src/allmydata/util/deferredutil.py +++ b/src/allmydata/util/deferredutil.py @@ -208,10 +208,9 @@ class WaitForDelayedCallsMixin(PollMixin): @inline_callbacks def until( - action, # type: Callable[[], defer.Deferred[Any]] - condition, # type: Callable[[], bool] -): - # type: (...) -> defer.Deferred[None] + action: Callable[[], defer.Deferred[Any]], + condition: Callable[[], bool], +) -> defer.Deferred[None]: """ Run a Deferred-returning function until a condition is true. From 06dc32a6c0e625c7188321aa6b6f5a2b2d2c7e89 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 24 Mar 2023 15:20:11 -0400 Subject: [PATCH 1634/2309] Fix lint. --- src/allmydata/util/pollmixin.py | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/src/allmydata/util/pollmixin.py b/src/allmydata/util/pollmixin.py index 582bafe86..b23277565 100644 --- a/src/allmydata/util/pollmixin.py +++ b/src/allmydata/util/pollmixin.py @@ -4,22 +4,10 @@ Polling utility that returns Deferred. Ported to Python 3. """ -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division -from __future__ import unicode_literals - -from future.utils import PY2 -if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 +from __future__ import annotations import time -try: - from typing import List -except ImportError: - pass - from twisted.internet import task class TimeoutError(Exception): @@ -29,7 +17,7 @@ class PollComplete(Exception): pass class PollMixin(object): - _poll_should_ignore_these_errors = [] # type: List[Exception] + _poll_should_ignore_these_errors : list[Exception] = [] def poll(self, check_f, pollinterval=0.01, timeout=1000): # Return a Deferred, then call check_f periodically until it returns From ee75bcd26bb2336c275224988feffdc531c36d05 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 24 Mar 2023 15:20:48 -0400 Subject: [PATCH 1635/2309] Fix lint. --- src/allmydata/web/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/web/common.py b/src/allmydata/web/common.py index 3d85b1c4d..bd1e3838e 100644 --- a/src/allmydata/web/common.py +++ b/src/allmydata/web/common.py @@ -117,7 +117,7 @@ def boolean_of_arg(arg): # type: (bytes) -> bool return arg.lower() in (b"true", b"t", b"1", b"on") -def parse_replace_arg(replace): # type: (bytes) -> Union[bool,_OnlyFiles] +def parse_replace_arg(replace: bytes) -> Union[bool,_OnlyFiles]: assert isinstance(replace, bytes) if replace.lower() == b"only-files": return ONLY_FILES From 51c7ca8d2cee964c94c8bce689520eb25c8325ee Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 24 Mar 2023 15:22:21 -0400 Subject: [PATCH 1636/2309] Workaround for incompatibility. --- setup.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 6528b01ed..854a333f1 100644 --- a/setup.py +++ b/setup.py @@ -63,7 +63,11 @@ install_requires = [ # Twisted[conch] also depends on cryptography and Twisted[tls] # transitively depends on cryptography. So it's anyone's guess what # version of cryptography will *really* be installed. - "cryptography >= 2.6", + + # * cryptography 40 broke constants we need; should really be using them + # * via pyOpenSSL; will be fixed in + # * https://github.com/pyca/pyopenssl/issues/1201 + "cryptography >= 2.6, < 40", # * The SFTP frontend depends on Twisted 11.0.0 to fix the SSH server # rekeying bug From 796fc5bdc532d3809fcda2a50175ebacd9eb0504 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 24 Mar 2023 15:27:51 -0400 Subject: [PATCH 1637/2309] Fix lint. --- misc/checkers/check_load.py | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/misc/checkers/check_load.py b/misc/checkers/check_load.py index 21576ea3a..d509b89ae 100644 --- a/misc/checkers/check_load.py +++ b/misc/checkers/check_load.py @@ -33,20 +33,11 @@ a mean of 10kB and a max of 100MB, so filesize=min(int(1.0/random(.0002)),1e8) """ +from __future__ import annotations import os, sys, httplib, binascii import urllib, json, random, time, urlparse -try: - from typing import Dict -except ImportError: - pass - -# Python 2 compatibility -from future.utils import PY2 -if PY2: - from future.builtins import str # noqa: F401 - if sys.argv[1] == "--stats": statsfiles = sys.argv[2:] # gather stats every 10 seconds, do a moving-window average of the last @@ -54,9 +45,9 @@ if sys.argv[1] == "--stats": DELAY = 10 MAXSAMPLES = 6 totals = [] - last_stats = {} # type: Dict[str, float] + last_stats : dict[str, float] = {} while True: - stats = {} # type: Dict[str, float] + stats : dict[str, float] = {} for sf in statsfiles: for line in open(sf, "r").readlines(): name, str_value = line.split(":") From 226da2fb2afa5961f8580619002905e3aeec580d Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Sun, 26 Mar 2023 11:49:17 -0400 Subject: [PATCH 1638/2309] Add missing pyyaml dependency It worked without this because we got the pyyaml dependency transitively but we should declare it directly since it is a direct dependency. --- nix/tahoe-lafs.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/nix/tahoe-lafs.nix b/nix/tahoe-lafs.nix index 2e1c4aa39..bf3ea83d3 100644 --- a/nix/tahoe-lafs.nix +++ b/nix/tahoe-lafs.nix @@ -34,6 +34,7 @@ let magic-wormhole netifaces psutil + pyyaml pycddl pyrsistent pyutil From 6bf1f0846a9858e50988cf235562b5f92c11ebd5 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Sun, 26 Mar 2023 12:56:26 -0400 Subject: [PATCH 1639/2309] additional news fragment --- newsfragments/3997.installation | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/3997.installation diff --git a/newsfragments/3997.installation b/newsfragments/3997.installation new file mode 100644 index 000000000..186be0fc2 --- /dev/null +++ b/newsfragments/3997.installation @@ -0,0 +1 @@ +Tahoe-LAFS is incompatible with cryptography >= 40 and now declares a requirement on an older version. From 51f763ca9ec7b571fdc8079c6c3e498e78691de1 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Sun, 26 Mar 2023 20:04:46 -0400 Subject: [PATCH 1640/2309] fix word-o --- .circleci/config.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index b64152a94..a1a95e9df 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -12,7 +12,8 @@ version: 2.1 # Every job that pushes a Docker image from Docker Hub must authenticate to -# it. Define a couple yaml anchors that can be used to supply a the necessary credentials. +# it. Define a couple yaml anchors that can be used to supply the necessary +# credentials. # First is a CircleCI job context which makes Docker Hub credentials available # in the environment. From ca7d60097c2b228191f9854db16960ad4abc667e Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Sun, 26 Mar 2023 20:05:35 -0400 Subject: [PATCH 1641/2309] update stale explanation about CACHIX_NAME --- .circleci/config.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index a1a95e9df..d46e255af 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -512,9 +512,9 @@ executors: - <<: *DOCKERHUB_AUTH image: "nixos/nix:2.10.3" environment: - # CACHIX_AUTH_TOKEN is manually set in the CircleCI web UI and - # allows us to push to CACHIX_NAME. We only need this set for - # `cachix use` in this step. + # CACHIX_AUTH_TOKEN is manually set in the CircleCI web UI and allows us + # to push to CACHIX_NAME. CACHIX_NAME tells cachix which cache to push + # to. CACHIX_NAME: "tahoe-lafs-opensource" commands: From 4211fd8525bfe5e45c323a9389df1ec5f54aab8d Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 27 Mar 2023 13:41:30 -0400 Subject: [PATCH 1642/2309] Revert to old code. --- src/allmydata/storage/http_client.py | 46 ++++++++++++++++------------ 1 file changed, 27 insertions(+), 19 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index a61f94708..44ba64363 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -355,7 +355,6 @@ class StorageClient(object): ) return headers - @inlineCallbacks def request( self, method, @@ -378,26 +377,35 @@ class StorageClient(object): Default timeout is 60 seconds. """ - with start_action( - action_type="allmydata:storage:http-client:request", - method=method, - url=str(url), - ) as ctx: - headers = self._get_headers(headers) + headers = self._get_headers(headers) - # Add secrets: - for secret, value in [ - (Secrets.LEASE_RENEW, lease_renew_secret), - (Secrets.LEASE_CANCEL, lease_cancel_secret), - (Secrets.UPLOAD, upload_secret), - (Secrets.WRITE_ENABLER, write_enabler_secret), - ]: - if value is None: - continue - headers.addRawHeader( - "X-Tahoe-Authorization", - b"%s %s" % (secret.value.encode("ascii"), b64encode(value).strip()), + # Add secrets: + for secret, value in [ + (Secrets.LEASE_RENEW, lease_renew_secret), + (Secrets.LEASE_CANCEL, lease_cancel_secret), + (Secrets.UPLOAD, upload_secret), + (Secrets.WRITE_ENABLER, write_enabler_secret), + ]: + if value is None: + continue + headers.addRawHeader( + "X-Tahoe-Authorization", + b"%s %s" % (secret.value.encode("ascii"), b64encode(value).strip()), + ) + + # Note we can accept CBOR: + headers.addRawHeader("Accept", CBOR_MIME_TYPE) + + # If there's a request message, serialize it and set the Content-Type + # header: + if message_to_serialize is not None: + if "data" in kwargs: + raise TypeError( + "Can't use both `message_to_serialize` and `data` " + "as keyword arguments at the same time" ) + kwargs["data"] = dumps(message_to_serialize) + headers.addRawHeader("Content-Type", CBOR_MIME_TYPE) return self._treq.request( method, url, headers=headers, timeout=timeout, **kwargs From b65e8c72dffe637d3a78f733866c212d823503c3 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 27 Mar 2023 13:55:53 -0400 Subject: [PATCH 1643/2309] Skip the tor integration tests if any needed tor tools are missing --- integration/conftest.py | 7 +++---- newsfragments/4000.minor | 0 2 files changed, 3 insertions(+), 4 deletions(-) create mode 100644 newsfragments/4000.minor diff --git a/integration/conftest.py b/integration/conftest.py index 33e7998c1..5970e5ba4 100644 --- a/integration/conftest.py +++ b/integration/conftest.py @@ -451,10 +451,9 @@ def chutney(reactor, temp_dir): chutney_dir = join(temp_dir, 'chutney') mkdir(chutney_dir) - # TODO: - - # check for 'tor' binary explicitly and emit a "skip" if we can't - # find it + missing = [exe for exe in ["tor", "tor-gencert"] if not which(exe)] + if missing: + pytest.skip(f"Some command-line tools not found: {missing}") # XXX yuck! should add a setup.py to chutney so we can at least # "pip install " and/or depend on chutney in "pip diff --git a/newsfragments/4000.minor b/newsfragments/4000.minor new file mode 100644 index 000000000..e69de29bb From fbcef2d1ae7f0893e4e4dc55066baa0b01feff4e Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 27 Mar 2023 13:32:40 -0400 Subject: [PATCH 1644/2309] Safely customize the Tor introducer's configuration Previously we clobbered the whole generated configuration and potentially wiped out additional important fields. Now we modify the configuration by just changing the fields we need to change. --- integration/conftest.py | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/integration/conftest.py b/integration/conftest.py index 33e7998c1..54632be26 100644 --- a/integration/conftest.py +++ b/integration/conftest.py @@ -43,7 +43,7 @@ from .util import ( generate_ssh_key, block_with_timeout, ) - +from allmydata.node import read_config # pytest customization hooks @@ -275,13 +275,6 @@ def introducer_furl(introducer, temp_dir): include_result=False, ) def tor_introducer(reactor, temp_dir, flog_gatherer, request): - config = ''' -[node] -nickname = introducer_tor -web.port = 4561 -log_gatherer.furl = {log_furl} -'''.format(log_furl=flog_gatherer) - intro_dir = join(temp_dir, 'introducer_tor') print("making introducer", intro_dir) @@ -301,9 +294,11 @@ log_gatherer.furl = {log_furl} ) pytest_twisted.blockon(done_proto.done) - # over-write the config file with our stuff - with open(join(intro_dir, 'tahoe.cfg'), 'w') as f: - f.write(config) + # adjust a few settings + config = read_config(intro_dir, "tub.port") + config.set_config("node", "nickname", "introducer-tor") + config.set_config("node", "web.port", "4561") + config.set_config("node", "log_gatherer.furl", flog_gatherer) # "tahoe run" is consistent across Linux/macOS/Windows, unlike the old # "start" command. From 1c11f9e7d4957fcb1312418feb65c6a56847586e Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 27 Mar 2023 13:35:14 -0400 Subject: [PATCH 1645/2309] Add a little more debug info to the integration test suite output --- integration/conftest.py | 3 +++ integration/test_tor.py | 2 +- integration/util.py | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/integration/conftest.py b/integration/conftest.py index 54632be26..280d98f72 100644 --- a/integration/conftest.py +++ b/integration/conftest.py @@ -321,7 +321,9 @@ def tor_introducer(reactor, temp_dir, flog_gatherer, request): pass request.addfinalizer(cleanup) + print("Waiting for introducer to be ready...") pytest_twisted.blockon(protocol.magic_seen) + print("Introducer ready.") return transport @@ -332,6 +334,7 @@ def tor_introducer_furl(tor_introducer, temp_dir): print("Don't see {} yet".format(furl_fname)) sleep(.1) furl = open(furl_fname, 'r').read() + print(f"Found Tor introducer furl: {furl}") return furl diff --git a/integration/test_tor.py b/integration/test_tor.py index 901858347..8398cf9a4 100644 --- a/integration/test_tor.py +++ b/integration/test_tor.py @@ -93,7 +93,7 @@ def _create_anonymous_node(reactor, name, control_port, request, temp_dir, flog_ web_port = "tcp:{}:interface=localhost".format(control_port + 2000) if True: - print("creating", node_dir.path) + print(f"creating {node_dir.path} with introducer {introducer_furl}") node_dir.makedirs() proto = util._DumpOutputProtocol(None) reactor.spawnProcess( diff --git a/integration/util.py b/integration/util.py index 05fef8fed..08c07a059 100644 --- a/integration/util.py +++ b/integration/util.py @@ -607,7 +607,7 @@ def await_client_ready(tahoe, timeout=10, liveness=60*2, minimum_number_of_serve continue if len(js['servers']) < minimum_number_of_servers: - print("waiting because insufficient servers") + print(f"waiting because {js['servers']} is fewer than required ({minimum_number_of_servers})") time.sleep(1) continue server_times = [ From 1c99817e1b0d098a422a2d0ccdc4cb6a5cb3d489 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 27 Mar 2023 13:41:51 -0400 Subject: [PATCH 1646/2309] Safely customize the client node's configuration This is similar to the fix to the `tor_introducer` fixture. --- integration/test_tor.py | 39 +++++++++++---------------------------- 1 file changed, 11 insertions(+), 28 deletions(-) diff --git a/integration/test_tor.py b/integration/test_tor.py index 8398cf9a4..b116fe319 100644 --- a/integration/test_tor.py +++ b/integration/test_tor.py @@ -25,6 +25,7 @@ from twisted.python.filepath import ( from allmydata.test.common import ( write_introducer, ) +from allmydata.client import read_config # see "conftest.py" for the fixtures (e.g. "tor_network") @@ -103,10 +104,14 @@ def _create_anonymous_node(reactor, name, control_port, request, temp_dir, flog_ sys.executable, '-b', '-m', 'allmydata.scripts.runner', 'create-node', '--nickname', name, + '--webport', web_port, '--introducer', introducer_furl, '--hide-ip', '--tor-control-port', 'tcp:localhost:{}'.format(control_port), '--listen', 'tor', + '--shares-needed', '1', + '--shares-happy', '1', + '--shares-total', '2', node_dir.path, ) ) @@ -115,35 +120,13 @@ def _create_anonymous_node(reactor, name, control_port, request, temp_dir, flog_ # Which services should this client connect to? write_introducer(node_dir, "default", introducer_furl) - with node_dir.child('tahoe.cfg').open('w') as f: - node_config = ''' -[node] -nickname = %(name)s -web.port = %(web_port)s -web.static = public_html -log_gatherer.furl = %(log_furl)s -[tor] -control.port = tcp:localhost:%(control_port)d -onion.external_port = 3457 -onion.local_port = %(local_port)d -onion = true -onion.private_key_file = private/tor_onion.privkey - -[client] -shares.needed = 1 -shares.happy = 1 -shares.total = 2 - -''' % { - 'name': name, - 'web_port': web_port, - 'log_furl': flog_gatherer, - 'control_port': control_port, - 'local_port': control_port + 1000, -} - node_config = node_config.encode("utf-8") - f.write(node_config) + config = read_config(node_dir.path, "tub.port") + config.set_config("node", "log_gatherer.furl", flog_gatherer) + config.set_config("tor", "onion", "true") + config.set_config("tor", "onion.external_port", "3457") + config.set_config("tor", "onion.local_port", str(control_port + 1000)) + config.set_config("tor", "onion.private_key_file", "private/tor_onion.privkey") print("running") result = yield util._run_node(reactor, node_dir.path, request, None) From 8613e36bae8068aea67ee662296ab367e19c268c Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 27 Mar 2023 14:06:16 -0400 Subject: [PATCH 1647/2309] Propagate parent environment to children in the integration tests --- integration/conftest.py | 5 +++- integration/test_i2p.py | 29 ++++++++---------------- integration/test_servers_of_happiness.py | 12 +++------- integration/test_tor.py | 21 ++++++----------- 4 files changed, 23 insertions(+), 44 deletions(-) diff --git a/integration/conftest.py b/integration/conftest.py index 33e7998c1..b8db4e580 100644 --- a/integration/conftest.py +++ b/integration/conftest.py @@ -146,7 +146,8 @@ def flog_gatherer(reactor, temp_dir, flog_binary, request): '--location', 'tcp:localhost:3117', '--port', '3117', gather_dir, - ) + ), + env=environ, ) pytest_twisted.blockon(out_protocol.done) @@ -159,6 +160,7 @@ def flog_gatherer(reactor, temp_dir, flog_binary, request): join(gather_dir, 'gatherer.tac'), ), path=gather_dir, + env=environ, ) pytest_twisted.blockon(twistd_protocol.magic_seen) @@ -177,6 +179,7 @@ def flog_gatherer(reactor, temp_dir, flog_binary, request): ( 'flogtool', 'dump', join(temp_dir, 'flog_gather', flogs[0]) ), + env=environ, ) print("Waiting for flogtool to complete") try: diff --git a/integration/test_i2p.py b/integration/test_i2p.py index 2deb01fab..96619a93a 100644 --- a/integration/test_i2p.py +++ b/integration/test_i2p.py @@ -2,26 +2,11 @@ Integration tests for I2P support. """ -from __future__ import unicode_literals -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - import sys from os.path import join, exists -from os import mkdir +from os import mkdir, environ from time import sleep - -if PY2: - def which(path): - # This will result in skipping I2P tests on Python 2. Oh well. - return None -else: - from shutil import which +from shutil import which from eliot import log_call @@ -62,6 +47,7 @@ def i2p_network(reactor, temp_dir, request): "--log=stdout", "--loglevel=info" ), + env=environ, ) def cleanup(): @@ -170,7 +156,8 @@ def test_i2p_service_storage(reactor, request, temp_dir, flog_gatherer, i2p_netw sys.executable, '-b', '-m', 'allmydata.scripts.runner', '-d', join(temp_dir, 'carol_i2p'), 'put', gold_path, - ) + ), + env=environ, ) yield proto.done cap = proto.output.getvalue().strip().split()[-1] @@ -184,7 +171,8 @@ def test_i2p_service_storage(reactor, request, temp_dir, flog_gatherer, i2p_netw sys.executable, '-b', '-m', 'allmydata.scripts.runner', '-d', join(temp_dir, 'dave_i2p'), 'get', cap, - ) + ), + env=environ, ) yield proto.done @@ -211,7 +199,8 @@ def _create_anonymous_node(reactor, name, control_port, request, temp_dir, flog_ '--hide-ip', '--listen', 'i2p', node_dir.path, - ) + ), + env=environ, ) yield proto.done diff --git a/integration/test_servers_of_happiness.py b/integration/test_servers_of_happiness.py index c63642066..8363edb35 100644 --- a/integration/test_servers_of_happiness.py +++ b/integration/test_servers_of_happiness.py @@ -1,17 +1,10 @@ """ Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals - -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import sys from os.path import join +from os import environ from twisted.internet.error import ProcessTerminated @@ -45,7 +38,8 @@ def test_upload_immutable(reactor, temp_dir, introducer_furl, flog_gatherer, sto sys.executable, '-b', '-m', 'allmydata.scripts.runner', '-d', node_dir, 'put', __file__, - ] + ], + env=environ, ) try: yield proto.done diff --git a/integration/test_tor.py b/integration/test_tor.py index 901858347..f82dcd052 100644 --- a/integration/test_tor.py +++ b/integration/test_tor.py @@ -1,17 +1,10 @@ """ Ported to Python 3. """ -from __future__ import unicode_literals -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import sys from os.path import join +from os import environ import pytest import pytest_twisted @@ -35,9 +28,6 @@ from allmydata.test.common import ( if sys.platform.startswith('win'): pytest.skip('Skipping Tor tests on Windows', allow_module_level=True) -if PY2: - pytest.skip('Skipping Tor tests on Python 2 because dependencies are hard to come by', allow_module_level=True) - @pytest_twisted.inlineCallbacks def test_onion_service_storage(reactor, request, temp_dir, flog_gatherer, tor_network, tor_introducer_furl): carol = yield _create_anonymous_node(reactor, 'carol', 8008, request, temp_dir, flog_gatherer, tor_network, tor_introducer_furl) @@ -65,7 +55,8 @@ def test_onion_service_storage(reactor, request, temp_dir, flog_gatherer, tor_ne sys.executable, '-b', '-m', 'allmydata.scripts.runner', '-d', join(temp_dir, 'carol'), 'put', gold_path, - ) + ), + env=environ, ) yield proto.done cap = proto.output.getvalue().strip().split()[-1] @@ -79,7 +70,8 @@ def test_onion_service_storage(reactor, request, temp_dir, flog_gatherer, tor_ne sys.executable, '-b', '-m', 'allmydata.scripts.runner', '-d', join(temp_dir, 'dave'), 'get', cap, - ) + ), + env=environ, ) yield proto.done @@ -108,7 +100,8 @@ def _create_anonymous_node(reactor, name, control_port, request, temp_dir, flog_ '--tor-control-port', 'tcp:localhost:{}'.format(control_port), '--listen', 'tor', node_dir.path, - ) + ), + env=environ, ) yield proto.done From 92eeaef4bded2dae3d1e0d99b176bee6a5a3ebaf Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 27 Mar 2023 14:07:31 -0400 Subject: [PATCH 1648/2309] news fragment --- newsfragments/4001.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/4001.minor diff --git a/newsfragments/4001.minor b/newsfragments/4001.minor new file mode 100644 index 000000000..e69de29bb From 50c4ad81136a21096a6ce540938c70afc299fadd Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 27 Mar 2023 14:07:53 -0400 Subject: [PATCH 1649/2309] news fragment --- newsfragments/3999.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3999.minor diff --git a/newsfragments/3999.minor b/newsfragments/3999.minor new file mode 100644 index 000000000..e69de29bb From fb8c10c55fd13afc4d408f8dce47cd79a5e403f4 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 27 Mar 2023 11:24:32 -0400 Subject: [PATCH 1650/2309] Use an already-installed Chutney if there is one --- integration/conftest.py | 114 +++++++++++++++++++--------------------- 1 file changed, 53 insertions(+), 61 deletions(-) diff --git a/integration/conftest.py b/integration/conftest.py index 33e7998c1..d184e61c9 100644 --- a/integration/conftest.py +++ b/integration/conftest.py @@ -1,6 +1,9 @@ """ Ported to Python 3. """ + +from __future__ import annotations + import sys import shutil from time import sleep @@ -19,6 +22,7 @@ from eliot import ( log_call, ) +from twisted.python.filepath import FilePath from twisted.python.procutils import which from twisted.internet.defer import DeferredList from twisted.internet.error import ( @@ -104,7 +108,7 @@ def reactor(): @pytest.fixture(scope='session') @log_call(action_type=u"integration:temp_dir", include_args=[]) -def temp_dir(request): +def temp_dir(request) -> str: """ Invoke like 'py.test --keep-tempdir ...' to avoid deleting the temp-dir """ @@ -446,7 +450,23 @@ def bob(reactor, temp_dir, introducer_furl, flog_gatherer, storage_nodes, reques @pytest.fixture(scope='session') @pytest.mark.skipif(sys.platform.startswith('win'), 'Tor tests are unstable on Windows') -def chutney(reactor, temp_dir): +def chutney(reactor, temp_dir: str) -> tuple[str, dict[str, str]]: + # Try to find Chutney already installed in the environment. + try: + import chutney + except ImportError: + # Nope, we'll get our own in a moment. + pass + else: + # We already have one, just use it. + return ( + # from `checkout/lib/chutney/__init__.py` we want to get back to + # `checkout` because that's the parent of the directory with all + # of the network definitions. So, great-grand-parent. + FilePath(chutney.__file__).parent().parent().parent().path, + # There's nothing to add to the environment. + {}, + ) chutney_dir = join(temp_dir, 'chutney') mkdir(chutney_dir) @@ -489,83 +509,55 @@ def chutney(reactor, temp_dir): ) pytest_twisted.blockon(proto.done) - return chutney_dir + return (chutney_dir, {"PYTHONPATH": join(chutney_dir, "lib")}) @pytest.fixture(scope='session') @pytest.mark.skipif(sys.platform.startswith('win'), reason='Tor tests are unstable on Windows') def tor_network(reactor, temp_dir, chutney, request): + """ + Build a basic Tor network. - # this is the actual "chutney" script at the root of a chutney checkout - chutney_dir = chutney - chut = join(chutney_dir, 'chutney') + :param chutney: The root directory of a Chutney checkout and a dict of + additional environment variables to set so a Python process can use + it. + + :return: None + """ + chutney_root, chutney_env = chutney + basic_network = join(chutney_root, 'networks', 'basic') + + env = environ.copy() + env.update(chutney_env) + chutney_argv = (sys.executable, '-m', 'chutney.TorNet') + def chutney(argv): + proto = _DumpOutputProtocol(None) + reactor.spawnProcess( + proto, + sys.executable, + chutney_argv + argv, + path=join(chutney_root), + env=env, + ) + return proto.done # now, as per Chutney's README, we have to create the network # ./chutney configure networks/basic # ./chutney start networks/basic - - env = environ.copy() - env.update({"PYTHONPATH": join(chutney_dir, "lib")}) - proto = _DumpOutputProtocol(None) - reactor.spawnProcess( - proto, - sys.executable, - ( - sys.executable, '-m', 'chutney.TorNet', 'configure', - join(chutney_dir, 'networks', 'basic'), - ), - path=join(chutney_dir), - env=env, - ) - pytest_twisted.blockon(proto.done) - - proto = _DumpOutputProtocol(None) - reactor.spawnProcess( - proto, - sys.executable, - ( - sys.executable, '-m', 'chutney.TorNet', 'start', - join(chutney_dir, 'networks', 'basic'), - ), - path=join(chutney_dir), - env=env, - ) - pytest_twisted.blockon(proto.done) + pytest_twisted.blockon(chutney(("configure", basic_network))) + pytest_twisted.blockon(chutney(("start", basic_network))) # print some useful stuff - proto = _CollectOutputProtocol() - reactor.spawnProcess( - proto, - sys.executable, - ( - sys.executable, '-m', 'chutney.TorNet', 'status', - join(chutney_dir, 'networks', 'basic'), - ), - path=join(chutney_dir), - env=env, - ) try: - pytest_twisted.blockon(proto.done) - except ProcessTerminated: - print("Chutney.TorNet status failed (continuing):") - print(proto.output.getvalue()) + pytest_twisted.blockon(chutney(("status", basic_network))) + except ProcessTerminated as e: + print("Chutney.TorNet status failed (continuing)") def cleanup(): print("Tearing down Chutney Tor network") - proto = _CollectOutputProtocol() - reactor.spawnProcess( - proto, - sys.executable, - ( - sys.executable, '-m', 'chutney.TorNet', 'stop', - join(chutney_dir, 'networks', 'basic'), - ), - path=join(chutney_dir), - env=env, - ) try: - block_with_timeout(proto.done, reactor) + block_with_timeout(chutney(("stop", basic_network)), reactor) except ProcessTerminated: # If this doesn't exit cleanly, that's fine, that shouldn't fail # the test suite. From d3d94937be84c751475b9e4d3da4820fe4cd0cdf Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 27 Mar 2023 11:24:40 -0400 Subject: [PATCH 1651/2309] Nothing uses the return value of this fixture --- integration/conftest.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/integration/conftest.py b/integration/conftest.py index d184e61c9..a8db66ec4 100644 --- a/integration/conftest.py +++ b/integration/conftest.py @@ -564,5 +564,3 @@ def tor_network(reactor, temp_dir, chutney, request): pass request.addfinalizer(cleanup) - - return chut From 81193aaddcfe319f09283492030eb7e47ac792a1 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 27 Mar 2023 14:43:58 -0400 Subject: [PATCH 1652/2309] news fragment --- newsfragments/4002.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/4002.minor diff --git a/newsfragments/4002.minor b/newsfragments/4002.minor new file mode 100644 index 000000000..e69de29bb From 0995772b24168a47049c35ed35825fc69a660316 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 27 Mar 2023 14:54:27 -0400 Subject: [PATCH 1653/2309] Explain why we ignore type check. --- src/allmydata/util/deferredutil.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/allmydata/util/deferredutil.py b/src/allmydata/util/deferredutil.py index 89dc9704c..58ca7dde0 100644 --- a/src/allmydata/util/deferredutil.py +++ b/src/allmydata/util/deferredutil.py @@ -239,6 +239,11 @@ def async_to_deferred(f: Callable[P, Awaitable[R]]) -> Callable[P, Deferred[R]]: @wraps(f) def not_async(*args: P.args, **kwargs: P.kwargs) -> Deferred[R]: + # Twisted documents fromCoroutine as accepting either a Generator or a + # Coroutine. However, the standard for type annotations of async + # functions is to return an Awaitable: + # https://github.com/twisted/twisted/issues/11832 + # So, we ignore the type warning. return defer.Deferred.fromCoroutine(f(*args, **kwargs)) # type: ignore return not_async From 7838f25bf8b70c981b29d8357a1edd427c253f80 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 27 Mar 2023 14:54:36 -0400 Subject: [PATCH 1654/2309] Clean up with simpler idiom. --- src/allmydata/storage_client.py | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index de756e322..ee555819c 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -1224,20 +1224,19 @@ class HTTPNativeStorageServer(service.MultiService): return self._istorage_server try: - try: - storage_server = await get_istorage_server() + storage_server = await get_istorage_server() - # Get the version from the remote server. Set a short timeout since - # we're relying on this for server liveness. - self._connecting_deferred = storage_server.get_version().addTimeout( - 5, self._reactor) - # We don't want to do another iteration of the loop until this - # iteration has finished, so wait here: - version = await self._connecting_deferred - self._got_version(version) - except Exception as e: - log.msg(f"Failed to connect to a HTTP storage server: {e}", level=log.CURIOUS) - self._failed_to_connect(Failure(e)) + # Get the version from the remote server. Set a short timeout since + # we're relying on this for server liveness. + self._connecting_deferred = storage_server.get_version().addTimeout( + 5, self._reactor) + # We don't want to do another iteration of the loop until this + # iteration has finished, so wait here: + version = await self._connecting_deferred + self._got_version(version) + except Exception as e: + log.msg(f"Failed to connect to a HTTP storage server: {e}", level=log.CURIOUS) + self._failed_to_connect(Failure(e)) finally: self._connecting_deferred = None From 4232c7f142d114d9440ceca7ea8ebfa9a044664c Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 27 Mar 2023 14:55:10 -0400 Subject: [PATCH 1655/2309] remove unused binding --- integration/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration/conftest.py b/integration/conftest.py index a8db66ec4..9f27ad014 100644 --- a/integration/conftest.py +++ b/integration/conftest.py @@ -551,7 +551,7 @@ def tor_network(reactor, temp_dir, chutney, request): # print some useful stuff try: pytest_twisted.blockon(chutney(("status", basic_network))) - except ProcessTerminated as e: + except ProcessTerminated: print("Chutney.TorNet status failed (continuing)") def cleanup(): From bd7c61cc5cce30ee75c0803e71ce9c5d7cc1643a Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 27 Mar 2023 16:58:15 -0400 Subject: [PATCH 1656/2309] Split up the state management logic from the server pinging logic. --- src/allmydata/storage_client.py | 65 ++++++++++++++++++++++----------- 1 file changed, 44 insertions(+), 21 deletions(-) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index ee555819c..a40e98b03 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -1186,8 +1186,46 @@ class HTTPNativeStorageServer(service.MultiService): def try_to_connect(self): self._connect() + def _connect(self) -> defer.Deferred[object]: + """ + Try to connect to a working storage server. + + If called while a previous ``_connect()`` is already running, it will + just return the same ``Deferred``. + + ``LoopingCall.stop()`` doesn't cancel ``Deferred``s, unfortunately: + https://github.com/twisted/twisted/issues/11814. Thus we want to store + the ``Deferred`` so we can cancel it when necessary. + + We also want to return it so that loop iterations take it into account, + and a new iteration doesn't start while we're in the middle of the + previous one. + """ + # Conceivably try_to_connect() was called on this before, in which case + # we already are in the middle of connecting. So in that case just + # return whatever is in progress: + if self._connecting_deferred is not None: + return self._connecting_deferred + + def done(_): + self._connecting_deferred = None + + connecting = self._pick_server_and_get_version() + # Set a short timeout since we're relying on this for server liveness. + connecting = connecting.addTimeout(5, self._reactor).addCallbacks( + self._got_version, self._failed_to_connect + ).addBoth(done) + self._connecting_deferred = connecting + return connecting + @async_to_deferred - async def _connect(self): + async def _pick_server_and_get_version(self): + """ + Minimal implementation of connection logic: pick a server, get its + version. This doesn't deal with errors much, so as to minimize + statefulness. It does change ``self._istorage_server``, so possibly + more refactoring would be useful to remove even that much statefulness. + """ async def get_istorage_server() -> _HTTPStorageServer: if self._istorage_server is not None: return self._istorage_server @@ -1207,15 +1245,7 @@ class HTTPNativeStorageServer(service.MultiService): StorageClient.from_nurl(nurl, reactor, pool) ).get_version() - # LoopingCall.stop() doesn't cancel Deferreds, unfortunately: - # https://github.com/twisted/twisted/issues/11814 Thus we want - # store the Deferred so it gets cancelled. - picking = _pick_a_http_server(reactor, self._nurls, request) - self._connecting_deferred = picking - try: - nurl = await picking - finally: - self._connecting_deferred = None + nurl = await _pick_a_http_server(reactor, self._nurls, request) # If we've gotten this far, we've found a working NURL. self._istorage_server = _HTTPStorageServer.from_http_client( @@ -1226,19 +1256,12 @@ class HTTPNativeStorageServer(service.MultiService): try: storage_server = await get_istorage_server() - # Get the version from the remote server. Set a short timeout since - # we're relying on this for server liveness. - self._connecting_deferred = storage_server.get_version().addTimeout( - 5, self._reactor) - # We don't want to do another iteration of the loop until this - # iteration has finished, so wait here: - version = await self._connecting_deferred - self._got_version(version) + # Get the version from the remote server. + version = await storage_server.get_version() + return version except Exception as e: log.msg(f"Failed to connect to a HTTP storage server: {e}", level=log.CURIOUS) - self._failed_to_connect(Failure(e)) - finally: - self._connecting_deferred = None + raise def stopService(self): if self._connecting_deferred is not None: From 2f106aa02adc157ef730ce49012da7e7f3b05b54 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 28 Mar 2023 08:35:31 -0400 Subject: [PATCH 1657/2309] use foolscap.reconnector.ReconnectionInfo where one is required It's *right* there. Just use it! --- src/allmydata/test/test_connection_status.py | 22 +++++++------------- 1 file changed, 7 insertions(+), 15 deletions(-) diff --git a/src/allmydata/test/test_connection_status.py b/src/allmydata/test/test_connection_status.py index 2bd8bf6ab..f6b36d5ba 100644 --- a/src/allmydata/test/test_connection_status.py +++ b/src/allmydata/test/test_connection_status.py @@ -1,21 +1,13 @@ """ Tests for allmydata.util.connection_status. - -Port to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals - -from future.utils import PY2 -if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import mock from twisted.trial import unittest +from foolscap.reconnector import ReconnectionInfo + from ..util import connection_status class Status(unittest.TestCase): @@ -33,7 +25,7 @@ class Status(unittest.TestCase): ci.connectionHandlers = {"h1": "hand1"} ci.winningHint = "h1" ci.establishedAt = 120 - ri = mock.Mock() + ri = ReconnectionInfo() ri.state = "connected" ri.connectionInfo = ci rc = mock.Mock @@ -51,7 +43,7 @@ class Status(unittest.TestCase): ci.connectionHandlers = {"h1": "hand1"} ci.winningHint = "h1" ci.establishedAt = 120 - ri = mock.Mock() + ri = ReconnectionInfo() ri.state = "connected" ri.connectionInfo = ci rc = mock.Mock @@ -70,7 +62,7 @@ class Status(unittest.TestCase): ci.listenerStatus = ("listener1", "successful") ci.winningHint = None ci.establishedAt = 120 - ri = mock.Mock() + ri = ReconnectionInfo() ri.state = "connected" ri.connectionInfo = ci rc = mock.Mock @@ -87,7 +79,7 @@ class Status(unittest.TestCase): ci = mock.Mock() ci.connectorStatuses = {"h1": "st1", "h2": "st2"} ci.connectionHandlers = {"h1": "hand1"} - ri = mock.Mock() + ri = ReconnectionInfo() ri.state = "connecting" ri.connectionInfo = ci rc = mock.Mock @@ -104,7 +96,7 @@ class Status(unittest.TestCase): ci = mock.Mock() ci.connectorStatuses = {"h1": "st1", "h2": "st2"} ci.connectionHandlers = {"h1": "hand1"} - ri = mock.Mock() + ri = ReconnectionInfo() ri.state = "waiting" ri.lastAttempt = 10 ri.nextAttempt = 20 From e2c6cc49d5e97627f4979ff4329c8cf3360f010e Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 28 Mar 2023 08:37:22 -0400 Subject: [PATCH 1658/2309] use foolscap.info.ConnectionInfo where one is required It's *right* there. Just use it! --- src/allmydata/test/test_connection_status.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/allmydata/test/test_connection_status.py b/src/allmydata/test/test_connection_status.py index f6b36d5ba..ba57f8aee 100644 --- a/src/allmydata/test/test_connection_status.py +++ b/src/allmydata/test/test_connection_status.py @@ -7,6 +7,7 @@ import mock from twisted.trial import unittest from foolscap.reconnector import ReconnectionInfo +from foolscap.info import ConnectionInfo from ..util import connection_status @@ -20,7 +21,7 @@ class Status(unittest.TestCase): "h2": "st2"}) def test_reconnector_connected(self): - ci = mock.Mock() + ci = ConnectionInfo() ci.connectorStatuses = {"h1": "st1"} ci.connectionHandlers = {"h1": "hand1"} ci.winningHint = "h1" @@ -38,7 +39,7 @@ class Status(unittest.TestCase): self.assertEqual(cs.last_received_time, 123) def test_reconnector_connected_others(self): - ci = mock.Mock() + ci = ConnectionInfo() ci.connectorStatuses = {"h1": "st1", "h2": "st2"} ci.connectionHandlers = {"h1": "hand1"} ci.winningHint = "h1" @@ -56,7 +57,7 @@ class Status(unittest.TestCase): self.assertEqual(cs.last_received_time, 123) def test_reconnector_connected_listener(self): - ci = mock.Mock() + ci = ConnectionInfo() ci.connectorStatuses = {"h1": "st1", "h2": "st2"} ci.connectionHandlers = {"h1": "hand1"} ci.listenerStatus = ("listener1", "successful") @@ -76,7 +77,7 @@ class Status(unittest.TestCase): self.assertEqual(cs.last_received_time, 123) def test_reconnector_connecting(self): - ci = mock.Mock() + ci = ConnectionInfo() ci.connectorStatuses = {"h1": "st1", "h2": "st2"} ci.connectionHandlers = {"h1": "hand1"} ri = ReconnectionInfo() @@ -93,7 +94,7 @@ class Status(unittest.TestCase): self.assertEqual(cs.last_received_time, 123) def test_reconnector_waiting(self): - ci = mock.Mock() + ci = ConnectionInfo() ci.connectorStatuses = {"h1": "st1", "h2": "st2"} ci.connectionHandlers = {"h1": "hand1"} ri = ReconnectionInfo() From 6b7ea29d887150a8e041fbb0fe835b75ca76d565 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 28 Mar 2023 08:40:25 -0400 Subject: [PATCH 1659/2309] use foolscap.reconnector.Reconnector where one is required Unfortunately we need to touch a private attribute directly to shove our expected info into it. This isn't so bad though. Foolscap isn't moving much and we're not touching anything complex, just setting a simple model attribute. --- src/allmydata/test/test_connection_status.py | 23 ++++++++++---------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/src/allmydata/test/test_connection_status.py b/src/allmydata/test/test_connection_status.py index ba57f8aee..3456a61f0 100644 --- a/src/allmydata/test/test_connection_status.py +++ b/src/allmydata/test/test_connection_status.py @@ -6,11 +6,17 @@ import mock from twisted.trial import unittest -from foolscap.reconnector import ReconnectionInfo +from foolscap.reconnector import ReconnectionInfo, Reconnector from foolscap.info import ConnectionInfo from ..util import connection_status +def reconnector(info: ReconnectionInfo) -> Reconnector: + rc = Reconnector(None, None, (), {}) + rc._reconnectionInfo = info + return rc + + class Status(unittest.TestCase): def test_hint_statuses(self): ncs = connection_status._hint_statuses(["h2","h1"], @@ -29,8 +35,7 @@ class Status(unittest.TestCase): ri = ReconnectionInfo() ri.state = "connected" ri.connectionInfo = ci - rc = mock.Mock - rc.getReconnectionInfo = mock.Mock(return_value=ri) + rc = reconnector(ri) cs = connection_status.from_foolscap_reconnector(rc, 123) self.assertEqual(cs.connected, True) self.assertEqual(cs.summary, "Connected to h1 via hand1") @@ -47,8 +52,7 @@ class Status(unittest.TestCase): ri = ReconnectionInfo() ri.state = "connected" ri.connectionInfo = ci - rc = mock.Mock - rc.getReconnectionInfo = mock.Mock(return_value=ri) + rc = reconnector(ri) cs = connection_status.from_foolscap_reconnector(rc, 123) self.assertEqual(cs.connected, True) self.assertEqual(cs.summary, "Connected to h1 via hand1") @@ -66,8 +70,7 @@ class Status(unittest.TestCase): ri = ReconnectionInfo() ri.state = "connected" ri.connectionInfo = ci - rc = mock.Mock - rc.getReconnectionInfo = mock.Mock(return_value=ri) + rc = reconnector(ri) cs = connection_status.from_foolscap_reconnector(rc, 123) self.assertEqual(cs.connected, True) self.assertEqual(cs.summary, "Connected via listener (listener1)") @@ -83,8 +86,7 @@ class Status(unittest.TestCase): ri = ReconnectionInfo() ri.state = "connecting" ri.connectionInfo = ci - rc = mock.Mock - rc.getReconnectionInfo = mock.Mock(return_value=ri) + rc = reconnector(ri) cs = connection_status.from_foolscap_reconnector(rc, 123) self.assertEqual(cs.connected, False) self.assertEqual(cs.summary, "Trying to connect") @@ -102,8 +104,7 @@ class Status(unittest.TestCase): ri.lastAttempt = 10 ri.nextAttempt = 20 ri.connectionInfo = ci - rc = mock.Mock - rc.getReconnectionInfo = mock.Mock(return_value=ri) + rc = reconnector(ri) with mock.patch("time.time", return_value=12): cs = connection_status.from_foolscap_reconnector(rc, 5) self.assertEqual(cs.connected, False) From 32cd54501d0f92de067c39b4178814e438bb49a4 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 28 Mar 2023 08:52:31 -0400 Subject: [PATCH 1660/2309] Pass a time function instead of patching the global --- src/allmydata/test/test_connection_status.py | 3 +-- src/allmydata/util/connection_status.py | 5 +++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/allmydata/test/test_connection_status.py b/src/allmydata/test/test_connection_status.py index 3456a61f0..6e258294b 100644 --- a/src/allmydata/test/test_connection_status.py +++ b/src/allmydata/test/test_connection_status.py @@ -105,8 +105,7 @@ class Status(unittest.TestCase): ri.nextAttempt = 20 ri.connectionInfo = ci rc = reconnector(ri) - with mock.patch("time.time", return_value=12): - cs = connection_status.from_foolscap_reconnector(rc, 5) + cs = connection_status.from_foolscap_reconnector(rc, 5, time=lambda: 12) self.assertEqual(cs.connected, False) self.assertEqual(cs.summary, "Reconnecting in 8 seconds (last attempt 2s ago)") diff --git a/src/allmydata/util/connection_status.py b/src/allmydata/util/connection_status.py index 0e8595e81..e9c2c1388 100644 --- a/src/allmydata/util/connection_status.py +++ b/src/allmydata/util/connection_status.py @@ -16,6 +16,7 @@ if PY2: import time from zope.interface import implementer from ..interfaces import IConnectionStatus +from foolscap.reconnector import Reconnector @implementer(IConnectionStatus) class ConnectionStatus(object): @@ -50,7 +51,7 @@ def _hint_statuses(which, handlers, statuses): non_connected_statuses["%s%s" % (hint, handler_dsc)] = dsc return non_connected_statuses -def from_foolscap_reconnector(rc, last_received): +def from_foolscap_reconnector(rc: Reconnector, last_received: int, time=time.time) -> ConnectionStatus: ri = rc.getReconnectionInfo() # See foolscap/reconnector.py, ReconnectionInfo, for details about possible # states. The returned result is a native string, it seems, so convert to @@ -80,7 +81,7 @@ def from_foolscap_reconnector(rc, last_received): # ci describes the current in-progress attempt summary = "Trying to connect" elif state == "waiting": - now = time.time() + now = time() elapsed = now - ri.lastAttempt delay = ri.nextAttempt - now summary = "Reconnecting in %d seconds (last attempt %ds ago)" % \ From 9a8430c90fcbb879e3e8e1b32a44c5f7aea8a5c0 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 28 Mar 2023 08:52:44 -0400 Subject: [PATCH 1661/2309] Remove porting boilerplate --- src/allmydata/util/connection_status.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/src/allmydata/util/connection_status.py b/src/allmydata/util/connection_status.py index e9c2c1388..d7cf18e1b 100644 --- a/src/allmydata/util/connection_status.py +++ b/src/allmydata/util/connection_status.py @@ -1,18 +1,7 @@ """ Parse connection status from Foolscap. - -Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals - -from future.utils import PY2 -if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - import time from zope.interface import implementer from ..interfaces import IConnectionStatus From 8e63fe2fddc8ca7b20a8a819d56c839546bc30e3 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 28 Mar 2023 08:52:55 -0400 Subject: [PATCH 1662/2309] Remove the unused mock import --- src/allmydata/test/test_connection_status.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/allmydata/test/test_connection_status.py b/src/allmydata/test/test_connection_status.py index 6e258294b..6c2e170f3 100644 --- a/src/allmydata/test/test_connection_status.py +++ b/src/allmydata/test/test_connection_status.py @@ -2,8 +2,6 @@ Tests for allmydata.util.connection_status. """ -import mock - from twisted.trial import unittest from foolscap.reconnector import ReconnectionInfo, Reconnector From 6d4278b465a4eb2194845884d363d373e84433f3 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 28 Mar 2023 08:53:21 -0400 Subject: [PATCH 1663/2309] Factor some repetition out of the tests --- src/allmydata/test/test_connection_status.py | 86 ++++++++++---------- src/allmydata/util/connection_status.py | 2 +- 2 files changed, 42 insertions(+), 46 deletions(-) diff --git a/src/allmydata/test/test_connection_status.py b/src/allmydata/test/test_connection_status.py index 6c2e170f3..f415a6ebf 100644 --- a/src/allmydata/test/test_connection_status.py +++ b/src/allmydata/test/test_connection_status.py @@ -2,21 +2,43 @@ Tests for allmydata.util.connection_status. """ -from twisted.trial import unittest +from typing import Optional from foolscap.reconnector import ReconnectionInfo, Reconnector from foolscap.info import ConnectionInfo from ..util import connection_status +from .common import SyncTestCase def reconnector(info: ReconnectionInfo) -> Reconnector: - rc = Reconnector(None, None, (), {}) + rc = Reconnector(None, None, (), {}) # type: ignore[no-untyped-call] rc._reconnectionInfo = info return rc +def connection_info( + statuses: dict[str, str], + handlers: dict[str, str], + winningHint: Optional[str], + establishedAt: Optional[int], +) -> ConnectionInfo: + ci = ConnectionInfo() # type: ignore[no-untyped-call] + ci.connectorStatuses = statuses + ci.connectionHandlers = handlers + ci.winningHint = winningHint + ci.establishedAt = establishedAt + return ci -class Status(unittest.TestCase): - def test_hint_statuses(self): +def reconnection_info( + state: str, + connection_info: ConnectionInfo, +) -> ReconnectionInfo: + ri = ReconnectionInfo() # type: ignore[no-untyped-call] + ri.state = state + ri.connectionInfo = connection_info + return ri + +class Status(SyncTestCase): + def test_hint_statuses(self) -> None: ncs = connection_status._hint_statuses(["h2","h1"], {"h1": "hand1", "h4": "hand4"}, {"h1": "st1", "h2": "st2", @@ -24,15 +46,9 @@ class Status(unittest.TestCase): self.assertEqual(ncs, {"h1 via hand1": "st1", "h2": "st2"}) - def test_reconnector_connected(self): - ci = ConnectionInfo() - ci.connectorStatuses = {"h1": "st1"} - ci.connectionHandlers = {"h1": "hand1"} - ci.winningHint = "h1" - ci.establishedAt = 120 - ri = ReconnectionInfo() - ri.state = "connected" - ri.connectionInfo = ci + def test_reconnector_connected(self) -> None: + ci = connection_info({"h1": "st1"}, {"h1": "hand1"}, "h1", 120) + ri = reconnection_info("connected", ci) rc = reconnector(ri) cs = connection_status.from_foolscap_reconnector(rc, 123) self.assertEqual(cs.connected, True) @@ -41,15 +57,9 @@ class Status(unittest.TestCase): self.assertEqual(cs.last_connection_time, 120) self.assertEqual(cs.last_received_time, 123) - def test_reconnector_connected_others(self): - ci = ConnectionInfo() - ci.connectorStatuses = {"h1": "st1", "h2": "st2"} - ci.connectionHandlers = {"h1": "hand1"} - ci.winningHint = "h1" - ci.establishedAt = 120 - ri = ReconnectionInfo() - ri.state = "connected" - ri.connectionInfo = ci + def test_reconnector_connected_others(self) -> None: + ci = connection_info({"h1": "st1", "h2": "st2"}, {"h1": "hand1"}, "h1", 120) + ri = reconnection_info("connected", ci) rc = reconnector(ri) cs = connection_status.from_foolscap_reconnector(rc, 123) self.assertEqual(cs.connected, True) @@ -58,16 +68,10 @@ class Status(unittest.TestCase): self.assertEqual(cs.last_connection_time, 120) self.assertEqual(cs.last_received_time, 123) - def test_reconnector_connected_listener(self): - ci = ConnectionInfo() - ci.connectorStatuses = {"h1": "st1", "h2": "st2"} - ci.connectionHandlers = {"h1": "hand1"} + def test_reconnector_connected_listener(self) -> None: + ci = connection_info({"h1": "st1", "h2": "st2"}, {"h1": "hand1"}, None, 120) ci.listenerStatus = ("listener1", "successful") - ci.winningHint = None - ci.establishedAt = 120 - ri = ReconnectionInfo() - ri.state = "connected" - ri.connectionInfo = ci + ri = reconnection_info("connected", ci) rc = reconnector(ri) cs = connection_status.from_foolscap_reconnector(rc, 123) self.assertEqual(cs.connected, True) @@ -77,13 +81,9 @@ class Status(unittest.TestCase): self.assertEqual(cs.last_connection_time, 120) self.assertEqual(cs.last_received_time, 123) - def test_reconnector_connecting(self): - ci = ConnectionInfo() - ci.connectorStatuses = {"h1": "st1", "h2": "st2"} - ci.connectionHandlers = {"h1": "hand1"} - ri = ReconnectionInfo() - ri.state = "connecting" - ri.connectionInfo = ci + def test_reconnector_connecting(self) -> None: + ci = connection_info({"h1": "st1", "h2": "st2"}, {"h1": "hand1"}, None, None) + ri = reconnection_info("connecting", ci) rc = reconnector(ri) cs = connection_status.from_foolscap_reconnector(rc, 123) self.assertEqual(cs.connected, False) @@ -93,15 +93,11 @@ class Status(unittest.TestCase): self.assertEqual(cs.last_connection_time, None) self.assertEqual(cs.last_received_time, 123) - def test_reconnector_waiting(self): - ci = ConnectionInfo() - ci.connectorStatuses = {"h1": "st1", "h2": "st2"} - ci.connectionHandlers = {"h1": "hand1"} - ri = ReconnectionInfo() - ri.state = "waiting" + def test_reconnector_waiting(self) -> None: + ci = connection_info({"h1": "st1", "h2": "st2"}, {"h1": "hand1"}, None, None) + ri = reconnection_info("waiting", ci) ri.lastAttempt = 10 ri.nextAttempt = 20 - ri.connectionInfo = ci rc = reconnector(ri) cs = connection_status.from_foolscap_reconnector(rc, 5, time=lambda: 12) self.assertEqual(cs.connected, False) diff --git a/src/allmydata/util/connection_status.py b/src/allmydata/util/connection_status.py index d7cf18e1b..751eee4fe 100644 --- a/src/allmydata/util/connection_status.py +++ b/src/allmydata/util/connection_status.py @@ -31,7 +31,7 @@ class ConnectionStatus(object): last_received_time=None, ) -def _hint_statuses(which, handlers, statuses): +def _hint_statuses(which, handlers, statuses) -> dict[str, str]: non_connected_statuses = {} for hint in which: handler = handlers.get(hint) From 2e6a40294b7c30bb9e000eaa52e9bc00097504a3 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 28 Mar 2023 08:53:37 -0400 Subject: [PATCH 1664/2309] Crank the type checking ratchet --- mypy.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy.ini b/mypy.ini index 7acc0ddc5..482fd6dd8 100644 --- a/mypy.ini +++ b/mypy.ini @@ -9,7 +9,7 @@ no_implicit_optional = True warn_redundant_casts = True strict_equality = True -[mypy-allmydata.test.cli.wormholetesting] +[mypy-allmydata.test.cli.wormholetesting,allmydata.test.test_connection_status] disallow_any_generics = True disallow_subclassing_any = True disallow_untyped_calls = True From a839ace32ae144457bf2f0c1be6e97903f84e7d3 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 28 Mar 2023 08:53:54 -0400 Subject: [PATCH 1665/2309] news fragment --- newsfragments/4003.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/4003.minor diff --git a/newsfragments/4003.minor b/newsfragments/4003.minor new file mode 100644 index 000000000..e69de29bb From 3ea9e97606b8605befc8e2b2f1a6342cf47c0336 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 28 Mar 2023 09:01:03 -0400 Subject: [PATCH 1666/2309] Python 3.8 compatibility --- src/allmydata/test/test_connection_status.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/allmydata/test/test_connection_status.py b/src/allmydata/test/test_connection_status.py index f415a6ebf..da41f5a47 100644 --- a/src/allmydata/test/test_connection_status.py +++ b/src/allmydata/test/test_connection_status.py @@ -2,6 +2,8 @@ Tests for allmydata.util.connection_status. """ +from __future__ import annotations + from typing import Optional from foolscap.reconnector import ReconnectionInfo, Reconnector From 80d8e5b465bbc717fc76708f319973ce40fb2907 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 28 Mar 2023 11:11:44 -0400 Subject: [PATCH 1667/2309] The function should return a coroutine. --- src/allmydata/util/deferredutil.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/src/allmydata/util/deferredutil.py b/src/allmydata/util/deferredutil.py index 58ca7dde0..695915ceb 100644 --- a/src/allmydata/util/deferredutil.py +++ b/src/allmydata/util/deferredutil.py @@ -13,8 +13,9 @@ from typing import ( Sequence, TypeVar, Optional, + Coroutine, ) -from typing_extensions import Awaitable, ParamSpec +from typing_extensions import ParamSpec from foolscap.api import eventually from eliot.twisted import ( @@ -230,7 +231,7 @@ P = ParamSpec("P") R = TypeVar("R") -def async_to_deferred(f: Callable[P, Awaitable[R]]) -> Callable[P, Deferred[R]]: +def async_to_deferred(f: Callable[P, Coroutine[defer.Deferred[R], None, R]]) -> Callable[P, Deferred[R]]: """ Wrap an async function to return a Deferred instead. @@ -239,12 +240,7 @@ def async_to_deferred(f: Callable[P, Awaitable[R]]) -> Callable[P, Deferred[R]]: @wraps(f) def not_async(*args: P.args, **kwargs: P.kwargs) -> Deferred[R]: - # Twisted documents fromCoroutine as accepting either a Generator or a - # Coroutine. However, the standard for type annotations of async - # functions is to return an Awaitable: - # https://github.com/twisted/twisted/issues/11832 - # So, we ignore the type warning. - return defer.Deferred.fromCoroutine(f(*args, **kwargs)) # type: ignore + return defer.Deferred.fromCoroutine(f(*args, **kwargs)) return not_async From e8c72e6753db8287ef1dcfa824080e1191746c2a Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 28 Mar 2023 12:55:41 -0400 Subject: [PATCH 1668/2309] Not sure if per method logging is worth it, will start from assumption that HTTP logging is enough. --- src/allmydata/storage/http_client.py | 28 ++++++++++++---------------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 44ba64363..fcfc5bff3 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -746,22 +746,18 @@ class StorageClientImmutables(object): """ Return the set of shares for a given storage index. """ - with start_action( - action_type="allmydata:storage:http-client:immutable:list-shares", - storage_index=storage_index, - ) as ctx: - url = self._client.relative_url( - "/storage/v1/immutable/{}/shares".format(_encode_si(storage_index)) - ) - response = yield self._client.request( - "GET", - url, - ) - if response.code == http.OK: - body = yield self._client.decode_cbor(response, _SCHEMAS["list_shares"]) - returnValue(set(body)) - else: - raise ClientException(response.code) + url = self._client.relative_url( + "/storage/v1/immutable/{}/shares".format(_encode_si(storage_index)) + ) + response = yield self._client.request( + "GET", + url, + ) + if response.code == http.OK: + body = yield self._client.decode_cbor(response, _SCHEMAS["list_shares"]) + returnValue(set(body)) + else: + raise ClientException(response.code) def advise_corrupt_share( self, From d36adf33a41be87814da8ccdf9a10c21813d53db Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 28 Mar 2023 13:06:43 -0400 Subject: [PATCH 1669/2309] Refactor; failing tests for some reason. --- src/allmydata/storage/http_server.py | 42 +++++++++++++++------------- 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 4f970b5a7..517771c02 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -106,28 +106,31 @@ def _authorization_decorator(required_secrets): def decorator(f): @wraps(f) def route(self, request, *args, **kwargs): - if not timing_safe_compare( - request.requestHeaders.getRawHeaders("Authorization", [""])[0].encode( - "utf-8" - ), - swissnum_auth_header(self._swissnum), - ): - request.setResponseCode(http.UNAUTHORIZED) - return b"" - authorization = request.requestHeaders.getRawHeaders( - "X-Tahoe-Authorization", [] - ) - try: - secrets = _extract_secrets(authorization, required_secrets) - except ClientSecretsException: - request.setResponseCode(http.BAD_REQUEST) - return b"Missing required secrets" with start_action( - action_type="allmydata:storage:http-server:request", + action_type="allmydata:storage:http-server:handle_request", method=request.method, path=request.path, ) as ctx: try: + # Check Authorization header: + if not timing_safe_compare( + request.requestHeaders.getRawHeaders("Authorization", [""])[0].encode( + "utf-8" + ), + swissnum_auth_header(self._swissnum), + ): + raise _HTTPError(http.UNAUTHORIZED) + + # Check secrets: + authorization = request.requestHeaders.getRawHeaders( + "X-Tahoe-Authorization", [] + ) + try: + secrets = _extract_secrets(authorization, required_secrets) + except ClientSecretsException: + raise _HTTPError(http.BAD_REQUEST) + + # Run the business logic: result = f(self, request, secrets, *args, **kwargs) except _HTTPError as e: # This isn't an error necessarily for logging purposes, @@ -136,8 +139,9 @@ def _authorization_decorator(required_secrets): ctx.add_success_fields(response_code=e.code) ctx.finish() raise - ctx.add_success_fields(response_code=request.code) - return result + else: + ctx.add_success_fields(response_code=request.code) + return result return route From ecfa76ac3268d8f23fe374b8f1ae7507e01f0773 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 28 Mar 2023 13:22:08 -0400 Subject: [PATCH 1670/2309] Python 3.8 compatibility --- src/allmydata/util/connection_status.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/allmydata/util/connection_status.py b/src/allmydata/util/connection_status.py index 751eee4fe..0ccdcd672 100644 --- a/src/allmydata/util/connection_status.py +++ b/src/allmydata/util/connection_status.py @@ -2,6 +2,8 @@ Parse connection status from Foolscap. """ +from __future__ import annotations + import time from zope.interface import implementer from ..interfaces import IConnectionStatus From fdf8519ed5a66a94caf551cf5f853a3a327b4ff3 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 28 Mar 2023 16:29:52 -0400 Subject: [PATCH 1671/2309] Define a protocol for listener/transport providers --- src/allmydata/listeners.py | 66 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 src/allmydata/listeners.py diff --git a/src/allmydata/listeners.py b/src/allmydata/listeners.py new file mode 100644 index 000000000..00ac9df4d --- /dev/null +++ b/src/allmydata/listeners.py @@ -0,0 +1,66 @@ +""" +Define a protocol for listening on a transport such that Tahoe-LAFS can +communicate over it, manage configuration for it in its configuration file, +detect when it is possible to use it, etc. +""" + +from __future__ import annotations + +from typing import Any, Protocol + +from attrs import frozen + +from .interfaces import IAddressFamily + +@frozen +class ListenerConfig: + """ + :ivar tub_ports: Entries to merge into ``[node]tub.port``. + + :ivar tub_locations: Entries to merge into ``[node]tub.location``. + + :ivar node_config: Entries to merge into the overall Tahoe-LAFS + configuration. XXX Note: Sections currently merge by overwriting + existing items with overlapping keys. In the future it would be nice + to merge every item sensibly (or error). + """ + tub_ports: list[str] + tub_locations: list[str] + node_config: dict[str, dict[str, str]] + +class Listener(Protocol): + """ + An object which can listen on a transport and allow Tahoe-LAFS + communication to happen over it. + """ + def is_available(self) -> bool: + """ + Can this type of listener actually be used in this runtime + environment? + """ + + def can_hide_ip(self) -> bool: + """ + Can the transport supported by this type of listener conceal the + node's public internet address from peers? + """ + + def create_config(self, reactor: Any, cli_config: Any) -> ListenerConfig: + """ + Set up an instance of this listener according to the given + configuration parameters. + + This may also allocate ephemeral resources if necessary. + + :return: The created configuration which can be merged into the + overall *tahoe.cfg* configuration file. + """ + + def create(self, config: Any, reactor: Any) -> IAddressFamily: + """ + Instantiate this listener according to the given + previously-generated configuration. + + :return: A handle on the listener which can be used to integrate it + into the Tahoe-LAFS node. + """ From cbfbfe8b1e3075b5c131f93c3c9e5b02b03b2ac1 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 28 Mar 2023 16:30:55 -0400 Subject: [PATCH 1672/2309] top-of-file cleanups --- src/allmydata/util/i2p_provider.py | 10 +--------- src/allmydata/util/tor_provider.py | 10 +--------- 2 files changed, 2 insertions(+), 18 deletions(-) diff --git a/src/allmydata/util/i2p_provider.py b/src/allmydata/util/i2p_provider.py index 071245adf..08a67c271 100644 --- a/src/allmydata/util/i2p_provider.py +++ b/src/allmydata/util/i2p_provider.py @@ -1,14 +1,6 @@ # -*- coding: utf-8 -*- -""" -Ported to Python 3. -""" -from __future__ import absolute_import, print_function, with_statement -from __future__ import division -from __future__ import unicode_literals -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 +from __future__ import annotations import os diff --git a/src/allmydata/util/tor_provider.py b/src/allmydata/util/tor_provider.py index 4ca19c01c..6d8278aad 100644 --- a/src/allmydata/util/tor_provider.py +++ b/src/allmydata/util/tor_provider.py @@ -1,14 +1,6 @@ # -*- coding: utf-8 -*- -""" -Ported to Python 3. -""" -from __future__ import absolute_import, print_function, with_statement -from __future__ import division -from __future__ import unicode_literals -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 +from __future__ import annotations import os From ed237b0dba674a7bac023163542cd6d904e20d77 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 29 Mar 2023 09:26:13 -0400 Subject: [PATCH 1673/2309] improve the Listener protocol somewhat --- src/allmydata/listeners.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/allmydata/listeners.py b/src/allmydata/listeners.py index 00ac9df4d..34b447868 100644 --- a/src/allmydata/listeners.py +++ b/src/allmydata/listeners.py @@ -6,11 +6,13 @@ detect when it is possible to use it, etc. from __future__ import annotations -from typing import Any, Protocol +from typing import Any, Awaitable, Protocol, Sequence, Mapping, Optional +from typing_extensions import Literal -from attrs import frozen +from attrs import frozen, define from .interfaces import IAddressFamily +from .util.iputil import allocate_tcp_port @frozen class ListenerConfig: @@ -19,14 +21,12 @@ class ListenerConfig: :ivar tub_locations: Entries to merge into ``[node]tub.location``. - :ivar node_config: Entries to merge into the overall Tahoe-LAFS - configuration. XXX Note: Sections currently merge by overwriting - existing items with overlapping keys. In the future it would be nice - to merge every item sensibly (or error). + :ivar node_config: Entries to add into the overall Tahoe-LAFS + configuration beneath a section named after this listener. """ - tub_ports: list[str] - tub_locations: list[str] - node_config: dict[str, dict[str, str]] + tub_ports: Sequence[str] + tub_locations: Sequence[str] + node_config: Mapping[str, Sequence[tuple[str, str]]] class Listener(Protocol): """ @@ -45,7 +45,7 @@ class Listener(Protocol): node's public internet address from peers? """ - def create_config(self, reactor: Any, cli_config: Any) -> ListenerConfig: + async def create_config(self, reactor: Any, cli_config: Any) -> Optional[ListenerConfig]: """ Set up an instance of this listener according to the given configuration parameters. @@ -56,7 +56,7 @@ class Listener(Protocol): overall *tahoe.cfg* configuration file. """ - def create(self, config: Any, reactor: Any) -> IAddressFamily: + def create(self, reactor: Any, config: Any) -> IAddressFamily: """ Instantiate this listener according to the given previously-generated configuration. From e15970a484031cb2d4673c03f06b314a19c25b43 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 29 Mar 2023 09:26:59 -0400 Subject: [PATCH 1674/2309] Add a couple simple Listeners that we need --- src/allmydata/listeners.py | 51 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/src/allmydata/listeners.py b/src/allmydata/listeners.py index 34b447868..149721bbc 100644 --- a/src/allmydata/listeners.py +++ b/src/allmydata/listeners.py @@ -64,3 +64,54 @@ class Listener(Protocol): :return: A handle on the listener which can be used to integrate it into the Tahoe-LAFS node. """ + +class TCPProvider: + """ + Support plain TCP connections. + """ + def is_available(self) -> Literal[True]: + return True + + def can_hide_ip(self) -> Literal[False]: + return False + + async def create_config(self, reactor: Any, cli_config: Any) -> ListenerConfig: + tub_ports = [] + tub_locations = [] + if cli_config["port"]: # --port/--location are a pair + tub_ports.append(cli_config["port"]) + tub_locations.append(cli_config["location"]) + else: + assert "hostname" in cli_config + hostname = cli_config["hostname"] + new_port = allocate_tcp_port() + tub_ports.append(f"tcp:{new_port}") + tub_locations.append(f"tcp:{hostname}:{new_port}") + + return ListenerConfig(tub_ports, tub_locations, {}) + + def create(self, reactor: Any, config: Any) -> IAddressFamily: + raise NotImplementedError() + + +@define +class StaticProvider: + """ + A provider that uses all pre-computed values. + """ + _available: bool + _hide_ip: bool + _config: Any + _address: IAddressFamily + + def is_available(self) -> bool: + return self._available + + def can_hide_ip(self) -> bool: + return self._hide_ip + + async def create_config(self, reactor: Any, cli_config: Any) -> None: + return await self._config + + def create(self, reactor: Any, config: Any) -> IAddressFamily: + return self._address From c52eb695055014821bdb3c5916b1278524ba448e Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 29 Mar 2023 09:27:26 -0400 Subject: [PATCH 1675/2309] Make the I2P and Tor providers implement the Listener protocol --- src/allmydata/test/test_i2p_provider.py | 8 ++-- src/allmydata/test/test_tor_provider.py | 20 +++++----- src/allmydata/util/i2p_provider.py | 43 ++++++++++++++++------ src/allmydata/util/tor_provider.py | 49 ++++++++++++++++--------- 4 files changed, 76 insertions(+), 44 deletions(-) diff --git a/src/allmydata/test/test_i2p_provider.py b/src/allmydata/test/test_i2p_provider.py index 364a85c5b..072b17647 100644 --- a/src/allmydata/test/test_i2p_provider.py +++ b/src/allmydata/test/test_i2p_provider.py @@ -177,7 +177,7 @@ class CreateDest(unittest.TestCase): with mock.patch("allmydata.util.i2p_provider.clientFromString", return_value=ep) as cfs: d = i2p_provider.create_config(reactor, cli_config) - tahoe_config_i2p, i2p_port, i2p_location = self.successResultOf(d) + i2p_config = self.successResultOf(d) connect_to_i2p.assert_called_with(reactor, cli_config, txi2p) cfs.assert_called_with(reactor, "goodport") @@ -189,9 +189,9 @@ class CreateDest(unittest.TestCase): "dest.private_key_file": os.path.join("private", "i2p_dest.privkey"), } - self.assertEqual(tahoe_config_i2p, expected) - self.assertEqual(i2p_port, "listen:i2p") - self.assertEqual(i2p_location, "i2p:FOOBAR.b32.i2p:3457") + self.assertEqual(dict(i2p_config.node_config["i2p"]), expected) + self.assertEqual(i2p_config.tub_ports, ["listen:i2p"]) + self.assertEqual(i2p_config.tub_locations, ["i2p:FOOBAR.b32.i2p:3457"]) _None = object() class FakeConfig(dict): diff --git a/src/allmydata/test/test_tor_provider.py b/src/allmydata/test/test_tor_provider.py index 86d54803a..65495fa0c 100644 --- a/src/allmydata/test/test_tor_provider.py +++ b/src/allmydata/test/test_tor_provider.py @@ -199,7 +199,7 @@ class CreateOnion(unittest.TestCase): with mock.patch("allmydata.util.tor_provider.allocate_tcp_port", return_value=999999): d = tor_provider.create_config(reactor, cli_config) - tahoe_config_tor, tor_port, tor_location = self.successResultOf(d) + tor_config = self.successResultOf(d) launch_tor.assert_called_with(reactor, executable, os.path.abspath(private_dir), txtorcon) @@ -216,10 +216,10 @@ class CreateOnion(unittest.TestCase): } if executable: expected["tor.executable"] = executable - self.assertEqual(tahoe_config_tor, expected) - self.assertEqual(tor_port, "tcp:999999:interface=127.0.0.1") - self.assertEqual(tor_location, "tor:ONION.onion:3457") - fn = os.path.join(basedir, tahoe_config_tor["onion.private_key_file"]) + self.assertEqual(dict(tor_config.node_config["tor"]), expected) + self.assertEqual(tor_config.tub_ports, ["tcp:999999:interface=127.0.0.1"]) + self.assertEqual(tor_config.tub_locations, ["tor:ONION.onion:3457"]) + fn = os.path.join(basedir, dict(tor_config.node_config["tor"])["onion.private_key_file"]) with open(fn, "rb") as f: privkey = f.read() self.assertEqual(privkey, b"privkey") @@ -253,7 +253,7 @@ class CreateOnion(unittest.TestCase): with mock.patch("allmydata.util.tor_provider.allocate_tcp_port", return_value=999999): d = tor_provider.create_config(reactor, cli_config) - tahoe_config_tor, tor_port, tor_location = self.successResultOf(d) + tor_config = self.successResultOf(d) connect_to_tor.assert_called_with(reactor, cli_config, txtorcon) txtorcon.EphemeralHiddenService.assert_called_with("3457 127.0.0.1:999999") @@ -267,10 +267,10 @@ class CreateOnion(unittest.TestCase): "onion.private_key_file": os.path.join("private", "tor_onion.privkey"), } - self.assertEqual(tahoe_config_tor, expected) - self.assertEqual(tor_port, "tcp:999999:interface=127.0.0.1") - self.assertEqual(tor_location, "tor:ONION.onion:3457") - fn = os.path.join(basedir, tahoe_config_tor["onion.private_key_file"]) + self.assertEqual(dict(tor_config.node_config["tor"]), expected) + self.assertEqual(tor_config.tub_ports, ["tcp:999999:interface=127.0.0.1"]) + self.assertEqual(tor_config.tub_locations, ["tor:ONION.onion:3457"]) + fn = os.path.join(basedir, dict(tor_config.node_config["tor"])["onion.private_key_file"]) with open(fn, "rb") as f: privkey = f.read() self.assertEqual(privkey, b"privkey") diff --git a/src/allmydata/util/i2p_provider.py b/src/allmydata/util/i2p_provider.py index 08a67c271..4d997945f 100644 --- a/src/allmydata/util/i2p_provider.py +++ b/src/allmydata/util/i2p_provider.py @@ -2,6 +2,9 @@ from __future__ import annotations +from typing import Any +from typing_extensions import Literal + import os from zope.interface import ( @@ -13,11 +16,12 @@ from twisted.internet.endpoints import clientFromString from twisted.internet.error import ConnectionRefusedError, ConnectError from twisted.application import service +from ..listeners import ListenerConfig from ..interfaces import ( IAddressFamily, ) -def create(reactor, config): +def create(reactor: Any, config: Any) -> IAddressFamily: """ Create a new Provider service (this is an IService so must be hooked up to a parent or otherwise started). @@ -47,6 +51,21 @@ def _import_txi2p(): except ImportError: # pragma: no cover return None +def is_available() -> bool: + """ + Can this type of listener actually be used in this runtime + environment? + + If its dependencies are missing then it cannot be. + """ + return not (_import_i2p() is None or _import_txi2p() is None) + +def can_hide_ip() -> Literal[True]: + """ + Can the transport supported by this type of listener conceal the + node's public internet address from peers? + """ + return True def _try_to_connect(reactor, endpoint_desc, stdout, txi2p): # yields True or None @@ -89,29 +108,28 @@ def _connect_to_i2p(reactor, cli_config, txi2p): else: raise ValueError("unable to reach any default I2P SAM port") -@inlineCallbacks -def create_config(reactor, cli_config): +async def create_config(reactor: Any, cli_config: Any) -> ListenerConfig: txi2p = _import_txi2p() if not txi2p: raise ValueError("Cannot create I2P Destination without txi2p. " "Please 'pip install tahoe-lafs[i2p]' to fix this.") - tahoe_config_i2p = {} # written into tahoe.cfg:[i2p] + tahoe_config_i2p = [] # written into tahoe.cfg:[i2p] private_dir = os.path.abspath(os.path.join(cli_config["basedir"], "private")) stdout = cli_config.stdout if cli_config["i2p-launch"]: raise NotImplementedError("--i2p-launch is under development.") else: print("connecting to I2P (to allocate .i2p address)..", file=stdout) - sam_port = yield _connect_to_i2p(reactor, cli_config, txi2p) + sam_port = await _connect_to_i2p(reactor, cli_config, txi2p) print("I2P connection established", file=stdout) - tahoe_config_i2p["sam.port"] = sam_port + tahoe_config_i2p.append(("sam.port", sam_port)) external_port = 3457 # TODO: pick this randomly? there's no contention. privkeyfile = os.path.join(private_dir, "i2p_dest.privkey") sam_endpoint = clientFromString(reactor, sam_port) print("allocating .i2p address...", file=stdout) - dest = yield txi2p.generateDestination(reactor, privkeyfile, 'SAM', sam_endpoint) + dest = await txi2p.generateDestination(reactor, privkeyfile, 'SAM', sam_endpoint) print(".i2p address allocated", file=stdout) i2p_port = "listen:i2p" # means "see [i2p]", calls Provider.get_listener() i2p_location = "i2p:%s:%d" % (dest.host, external_port) @@ -124,10 +142,11 @@ def create_config(reactor, cli_config): # * "private_key_file" points to the on-disk copy of the private key # material (although we always write it to the same place) - tahoe_config_i2p["dest"] = "true" - tahoe_config_i2p["dest.port"] = str(external_port) - tahoe_config_i2p["dest.private_key_file"] = os.path.join("private", - "i2p_dest.privkey") + tahoe_config_i2p.extend([ + ("dest", "true"), + ("dest.port", str(external_port)), + ("dest.private_key_file", os.path.join("private", "i2p_dest.privkey")), + ]) # tahoe_config_i2p: this is a dictionary of keys/values to add to the # "[i2p]" section of tahoe.cfg, which tells the new node how to launch @@ -141,7 +160,7 @@ def create_config(reactor, cli_config): # at both create-node and startup time. The data directory is not # recorded in tahoe.cfg - returnValue((tahoe_config_i2p, i2p_port, i2p_location)) + return ListenerConfig([i2p_port], [i2p_location], {"i2p": tahoe_config_i2p}) @implementer(IAddressFamily) diff --git a/src/allmydata/util/tor_provider.py b/src/allmydata/util/tor_provider.py index 6d8278aad..a30eaebc9 100644 --- a/src/allmydata/util/tor_provider.py +++ b/src/allmydata/util/tor_provider.py @@ -2,6 +2,9 @@ from __future__ import annotations +from typing import Any +from typing_extensions import Literal + import os from zope.interface import ( @@ -18,6 +21,7 @@ from .iputil import allocate_tcp_port from ..interfaces import ( IAddressFamily, ) +from ..listeners import ListenerConfig def _import_tor(): try: @@ -33,7 +37,13 @@ def _import_txtorcon(): except ImportError: # pragma: no cover return None -def create(reactor, config, import_tor=None, import_txtorcon=None): +def can_hide_ip() -> Literal[True]: + return True + +def is_available() -> bool: + return not (_import_tor() is None or _import_txtorcon() is None) + +def create(reactor: Any, config: Any, import_tor=None, import_txtorcon=None) -> IAddressFamily: """ Create a new _Provider service (this is an IService so must be hooked up to a parent or otherwise started). @@ -146,30 +156,29 @@ def _connect_to_tor(reactor, cli_config, txtorcon): else: raise ValueError("unable to reach any default Tor control port") -@inlineCallbacks -def create_config(reactor, cli_config): +async def create_config(reactor: Any, cli_config: Any) -> ListenerConfig: txtorcon = _import_txtorcon() if not txtorcon: raise ValueError("Cannot create onion without txtorcon. " "Please 'pip install tahoe-lafs[tor]' to fix this.") - tahoe_config_tor = {} # written into tahoe.cfg:[tor] + tahoe_config_tor = [] # written into tahoe.cfg:[tor] private_dir = os.path.abspath(os.path.join(cli_config["basedir"], "private")) stdout = cli_config.stdout if cli_config["tor-launch"]: - tahoe_config_tor["launch"] = "true" + tahoe_config_tor.append(("launch", "true")) tor_executable = cli_config["tor-executable"] if tor_executable: - tahoe_config_tor["tor.executable"] = tor_executable + tahoe_config_tor.append(("tor.executable", tor_executable)) print("launching Tor (to allocate .onion address)..", file=stdout) - (_, tor_control_proto) = yield _launch_tor( + (_, tor_control_proto) = await _launch_tor( reactor, tor_executable, private_dir, txtorcon) print("Tor launched", file=stdout) else: print("connecting to Tor (to allocate .onion address)..", file=stdout) - (port, tor_control_proto) = yield _connect_to_tor( + (port, tor_control_proto) = await _connect_to_tor( reactor, cli_config, txtorcon) print("Tor connection established", file=stdout) - tahoe_config_tor["control.port"] = port + tahoe_config_tor.append(("control.port", port)) external_port = 3457 # TODO: pick this randomly? there's no contention. @@ -178,12 +187,12 @@ def create_config(reactor, cli_config): "%d 127.0.0.1:%d" % (external_port, local_port) ) print("allocating .onion address (takes ~40s)..", file=stdout) - yield ehs.add_to_tor(tor_control_proto) + await ehs.add_to_tor(tor_control_proto) print(".onion address allocated", file=stdout) tor_port = "tcp:%d:interface=127.0.0.1" % local_port tor_location = "tor:%s:%d" % (ehs.hostname, external_port) privkey = ehs.private_key - yield ehs.remove_from_tor(tor_control_proto) + await ehs.remove_from_tor(tor_control_proto) # in addition to the "how to launch/connect-to tor" keys above, we also # record information about the onion service into tahoe.cfg. @@ -195,12 +204,12 @@ def create_config(reactor, cli_config): # * "private_key_file" points to the on-disk copy of the private key # material (although we always write it to the same place) - tahoe_config_tor["onion"] = "true" - tahoe_config_tor["onion.local_port"] = str(local_port) - tahoe_config_tor["onion.external_port"] = str(external_port) - assert privkey - tahoe_config_tor["onion.private_key_file"] = os.path.join("private", - "tor_onion.privkey") + tahoe_config_tor.extend([ + ("onion", "true"), + ("onion.local_port", str(local_port)), + ("onion.external_port", str(external_port)), + ("onion.private_key_file", os.path.join("private", "tor_onion.privkey")), + ]) privkeyfile = os.path.join(private_dir, "tor_onion.privkey") with open(privkeyfile, "wb") as f: if isinstance(privkey, str): @@ -219,7 +228,11 @@ def create_config(reactor, cli_config): # at both create-node and startup time. The data directory is not # recorded in tahoe.cfg - returnValue((tahoe_config_tor, tor_port, tor_location)) + return ListenerConfig( + [tor_port], + [tor_location], + {"tor": tahoe_config_tor}, + ) @implementer(IAddressFamily) From 74ebda771a0b0e403dd6167081cb5516cc9c124e Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 29 Mar 2023 09:46:54 -0400 Subject: [PATCH 1676/2309] Make `tahoe create-node` use the new listener protocol --- src/allmydata/scripts/create_node.py | 135 ++++++++++++++++---------- src/allmydata/test/cli/test_create.py | 55 ++++------- 2 files changed, 103 insertions(+), 87 deletions(-) diff --git a/src/allmydata/scripts/create_node.py b/src/allmydata/scripts/create_node.py index 7d15b95ec..85d1c46cd 100644 --- a/src/allmydata/scripts/create_node.py +++ b/src/allmydata/scripts/create_node.py @@ -1,3 +1,8 @@ + +from __future__ import annotations + +from typing import Optional + import io import os @@ -21,7 +26,37 @@ from allmydata.scripts.common import ( from allmydata.scripts.default_nodedir import _default_nodedir from allmydata.util.assertutil import precondition from allmydata.util.encodingutil import listdir_unicode, argv_to_unicode, quote_local_unicode_path, get_io_encoding -from allmydata.util import fileutil, i2p_provider, iputil, tor_provider, jsonbytes as json + +i2p_provider: Listener +tor_provider: Listener + +from allmydata.util import fileutil, i2p_provider, tor_provider, jsonbytes as json + +from ..listeners import ListenerConfig, Listener, TCPProvider, StaticProvider + +def _get_listeners() -> dict[str, Listener]: + """ + Get all of the kinds of listeners we might be able to use. + """ + return { + "tor": tor_provider, + "i2p": i2p_provider, + "tcp": TCPProvider(), + "none": StaticProvider( + available=True, + hide_ip=False, + config=defer.succeed(None), + # This is supposed to be an IAddressFamily but we have none for + # this kind of provider. We could implement new client and server + # endpoint types that always fail and pass an IAddressFamily here + # that uses those. Nothing would ever even ask for them (at + # least, yet), let alone try to use them, so that's a lot of extra + # work for no practical result so I'm not doing it now. + address=None, # type: ignore[arg-type] + ), + } + +_LISTENERS = _get_listeners() dummy_tac = """ import sys @@ -98,8 +133,11 @@ def validate_where_options(o): if o['listen'] != "none" and o.get('join', None) is None: listeners = o['listen'].split(",") for l in listeners: - if l not in ["tcp", "tor", "i2p"]: - raise UsageError("--listen= must be none, or one/some of: tcp, tor, i2p") + if l not in _LISTENERS: + raise UsageError( + "--listen= must be one/some of: " + f"{', '.join(sorted(_LISTENERS))}", + ) if 'tcp' in listeners and not o['hostname']: raise UsageError("--listen=tcp requires --hostname=") if 'tcp' not in listeners and o['hostname']: @@ -108,7 +146,7 @@ def validate_where_options(o): def validate_tor_options(o): use_tor = "tor" in o["listen"].split(",") if use_tor or any((o["tor-launch"], o["tor-control-port"])): - if tor_provider._import_txtorcon() is None: + if not _LISTENERS["tor"].is_available(): raise UsageError( "Specifying any Tor options requires the 'txtorcon' module" ) @@ -123,7 +161,7 @@ def validate_tor_options(o): def validate_i2p_options(o): use_i2p = "i2p" in o["listen"].split(",") if use_i2p or any((o["i2p-launch"], o["i2p-sam-port"])): - if i2p_provider._import_txi2p() is None: + if not _LISTENERS["i2p"].is_available(): raise UsageError( "Specifying any I2P options requires the 'txi2p' module" ) @@ -145,7 +183,7 @@ class _CreateBaseOptions(BasedirOptions): def postOptions(self): super(_CreateBaseOptions, self).postOptions() if self['hide-ip']: - if tor_provider._import_txtorcon() is None and i2p_provider._import_txi2p() is None: + if not (_LISTENERS["tor"].is_available() or _LISTENERS["i2p"].is_available()): raise UsageError( "--hide-ip was specified but neither 'txtorcon' nor 'txi2p' " "are installed.\nTo do so:\n pip install tahoe-lafs[tor]\nor\n" @@ -218,8 +256,20 @@ class CreateIntroducerOptions(NoDefaultBasedirOptions): validate_i2p_options(self) -@defer.inlineCallbacks -def write_node_config(c, config): +def merge_config( + left: Optional[ListenerConfig], + right: Optional[ListenerConfig], +) -> Optional[ListenerConfig]: + if left is None or right is None: + return None + return ListenerConfig( + list(left.tub_ports) + list(right.tub_ports), + list(left.tub_locations) + list(right.tub_locations), + dict(list(left.node_config.items()) + list(right.node_config.items())), + ) + + +async def write_node_config(c, config): # this is shared between clients and introducers c.write("# -*- mode: conf; coding: {c.encoding} -*-\n".format(c=c)) c.write("\n") @@ -232,9 +282,10 @@ def write_node_config(c, config): if config["hide-ip"]: c.write("[connections]\n") - if tor_provider._import_txtorcon(): + if _LISTENERS["tor"].is_available(): c.write("tcp = tor\n") else: + # XXX What about i2p? c.write("tcp = disabled\n") c.write("\n") @@ -253,38 +304,23 @@ def write_node_config(c, config): c.write("web.port = %s\n" % (webport,)) c.write("web.static = public_html\n") - listeners = config['listen'].split(",") + listener_config = ListenerConfig([], [], {}) + for listener_name in config['listen'].split(","): + listener = _LISTENERS[listener_name] + listener_config = merge_config( + (await listener.create_config(reactor, config)), + listener_config, + ) - tor_config = {} - i2p_config = {} - tub_ports = [] - tub_locations = [] - if listeners == ["none"]: - c.write("tub.port = disabled\n") - c.write("tub.location = disabled\n") + if listener_config is None: + tub_ports = ["disabled"] + tub_locations = ["disabled"] else: - if "tor" in listeners: - (tor_config, tor_port, tor_location) = \ - yield tor_provider.create_config(reactor, config) - tub_ports.append(tor_port) - tub_locations.append(tor_location) - if "i2p" in listeners: - (i2p_config, i2p_port, i2p_location) = \ - yield i2p_provider.create_config(reactor, config) - tub_ports.append(i2p_port) - tub_locations.append(i2p_location) - if "tcp" in listeners: - if config["port"]: # --port/--location are a pair - tub_ports.append(config["port"]) - tub_locations.append(config["location"]) - else: - assert "hostname" in config - hostname = config["hostname"] - new_port = iputil.allocate_tcp_port() - tub_ports.append("tcp:%s" % new_port) - tub_locations.append("tcp:%s:%s" % (hostname, new_port)) - c.write("tub.port = %s\n" % ",".join(tub_ports)) - c.write("tub.location = %s\n" % ",".join(tub_locations)) + tub_ports = listener_config.tub_ports + tub_locations = listener_config.tub_locations + + c.write("tub.port = %s\n" % ",".join(tub_ports)) + c.write("tub.location = %s\n" % ",".join(tub_locations)) c.write("\n") c.write("#log_gatherer.furl =\n") @@ -294,17 +330,12 @@ def write_node_config(c, config): c.write("#ssh.authorized_keys_file = ~/.ssh/authorized_keys\n") c.write("\n") - if tor_config: - c.write("[tor]\n") - for key, value in list(tor_config.items()): - c.write("%s = %s\n" % (key, value)) - c.write("\n") - - if i2p_config: - c.write("[i2p]\n") - for key, value in list(i2p_config.items()): - c.write("%s = %s\n" % (key, value)) - c.write("\n") + if listener_config is not None: + for section, items in listener_config.node_config.items(): + c.write(f"[{section}]\n") + for k, v in items: + c.write(f"{k} = {v}\n") + c.write("\n") def write_client_config(c, config): @@ -445,7 +476,7 @@ def create_node(config): fileutil.make_dirs(os.path.join(basedir, "private"), 0o700) cfg_name = os.path.join(basedir, "tahoe.cfg") with io.open(cfg_name, "w", encoding='utf-8') as c: - yield write_node_config(c, config) + yield defer.Deferred.fromCoroutine(write_node_config(c, config)) write_client_config(c, config) print("Node created in %s" % quote_local_unicode_path(basedir), file=out) @@ -488,7 +519,7 @@ def create_introducer(config): fileutil.make_dirs(os.path.join(basedir, "private"), 0o700) cfg_name = os.path.join(basedir, "tahoe.cfg") with io.open(cfg_name, "w", encoding='utf-8') as c: - yield write_node_config(c, config) + yield defer.Deferred.fromCoroutine(write_node_config(c, config)) print("Introducer created in %s" % quote_local_unicode_path(basedir), file=out) defer.returnValue(0) diff --git a/src/allmydata/test/cli/test_create.py b/src/allmydata/test/cli/test_create.py index 1d1576082..d100f481f 100644 --- a/src/allmydata/test/cli/test_create.py +++ b/src/allmydata/test/cli/test_create.py @@ -17,6 +17,7 @@ from ..common import ( disable_modules, ) from ...scripts import create_node +from ...listeners import ListenerConfig, StaticProvider from ... import client def read_config(basedir): @@ -45,7 +46,7 @@ class Config(unittest.TestCase): e = self.assertRaises(usage.UsageError, parse_cli, verb, *args) self.assertIn("option %s not recognized" % (option,), str(e)) - def test_create_client_config(self): + async def test_create_client_config(self): d = self.mktemp() os.mkdir(d) fname = os.path.join(d, 'tahoe.cfg') @@ -59,7 +60,7 @@ class Config(unittest.TestCase): "shares-happy": "1", "shares-total": "1", } - create_node.write_node_config(f, opts) + await create_node.write_node_config(f, opts) create_node.write_client_config(f, opts) # should succeed, no exceptions @@ -245,7 +246,7 @@ class Config(unittest.TestCase): parse_cli, "create-node", "--listen=tcp,none", basedir) - self.assertEqual(str(e), "--listen= must be none, or one/some of: tcp, tor, i2p") + self.assertEqual(str(e), "--listen=tcp requires --hostname=") def test_node_listen_bad(self): basedir = self.mktemp() @@ -253,7 +254,7 @@ class Config(unittest.TestCase): parse_cli, "create-node", "--listen=XYZZY,tcp", basedir) - self.assertEqual(str(e), "--listen= must be none, or one/some of: tcp, tor, i2p") + self.assertEqual(str(e), "--listen= must be one/some of: i2p, none, tcp, tor") def test_node_listen_tor_hostname(self): e = self.assertRaises(usage.UsageError, @@ -287,24 +288,15 @@ class Config(unittest.TestCase): self.assertIn("To avoid clobbering anything, I am going to quit now", err) @defer.inlineCallbacks - def test_node_slow_tor(self): - basedir = self.mktemp() + def test_node_slow(self): d = defer.Deferred() - self.patch(tor_provider, "create_config", lambda *a, **kw: d) - d2 = run_cli("create-node", "--listen=tor", basedir) - d.callback(({}, "port", "location")) - rc, out, err = yield d2 - self.assertEqual(rc, 0) - self.assertIn("Node created", out) - self.assertEqual(err, "") + slow = StaticProvider(True, False, d, None) + create_node._LISTENERS["xxyzy"] = slow + self.addCleanup(lambda: create_node._LISTENERS.pop("xxyzy")) - @defer.inlineCallbacks - def test_node_slow_i2p(self): basedir = self.mktemp() - d = defer.Deferred() - self.patch(i2p_provider, "create_config", lambda *a, **kw: d) - d2 = run_cli("create-node", "--listen=i2p", basedir) - d.callback(({}, "port", "location")) + d2 = run_cli("create-node", "--listen=xxyzy", basedir) + d.callback(None) rc, out, err = yield d2 self.assertEqual(rc, 0) self.assertIn("Node created", out) @@ -369,10 +361,12 @@ def fake_config(testcase: unittest.TestCase, module: Any, result: Any) -> list[t class Tor(unittest.TestCase): def test_default(self): basedir = self.mktemp() - tor_config = {"abc": "def"} + tor_config = {"tor": [("abc", "def")]} tor_port = "ghi" tor_location = "jkl" - config_d = defer.succeed( (tor_config, tor_port, tor_location) ) + config_d = defer.succeed( + ListenerConfig([tor_port], [tor_location], tor_config) + ) calls = fake_config(self, tor_provider, config_d) rc, out, err = self.successResultOf( @@ -391,10 +385,7 @@ class Tor(unittest.TestCase): def test_launch(self): basedir = self.mktemp() - tor_config = {"abc": "def"} - tor_port = "ghi" - tor_location = "jkl" - config_d = defer.succeed( (tor_config, tor_port, tor_location) ) + config_d = defer.succeed(None) calls = fake_config(self, tor_provider, config_d) rc, out, err = self.successResultOf( @@ -410,10 +401,7 @@ class Tor(unittest.TestCase): def test_control_port(self): basedir = self.mktemp() - tor_config = {"abc": "def"} - tor_port = "ghi" - tor_location = "jkl" - config_d = defer.succeed( (tor_config, tor_port, tor_location) ) + config_d = defer.succeed(None) calls = fake_config(self, tor_provider, config_d) rc, out, err = self.successResultOf( @@ -451,10 +439,10 @@ class Tor(unittest.TestCase): class I2P(unittest.TestCase): def test_default(self): basedir = self.mktemp() - i2p_config = {"abc": "def"} + i2p_config = {"i2p": [("abc", "def")]} i2p_port = "ghi" i2p_location = "jkl" - dest_d = defer.succeed( (i2p_config, i2p_port, i2p_location) ) + dest_d = defer.succeed(ListenerConfig([i2p_port], [i2p_location], i2p_config)) calls = fake_config(self, i2p_provider, dest_d) rc, out, err = self.successResultOf( @@ -479,10 +467,7 @@ class I2P(unittest.TestCase): def test_sam_port(self): basedir = self.mktemp() - i2p_config = {"abc": "def"} - i2p_port = "ghi" - i2p_location = "jkl" - dest_d = defer.succeed( (i2p_config, i2p_port, i2p_location) ) + dest_d = defer.succeed(None) calls = fake_config(self, i2p_provider, dest_d) rc, out, err = self.successResultOf( From 00ecb65c01478e59f9dd543e849c085556c8b767 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 29 Mar 2023 09:47:25 -0400 Subject: [PATCH 1677/2309] remove unused import --- src/allmydata/listeners.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/listeners.py b/src/allmydata/listeners.py index 149721bbc..3c4e71b9c 100644 --- a/src/allmydata/listeners.py +++ b/src/allmydata/listeners.py @@ -6,7 +6,7 @@ detect when it is possible to use it, etc. from __future__ import annotations -from typing import Any, Awaitable, Protocol, Sequence, Mapping, Optional +from typing import Any, Protocol, Sequence, Mapping, Optional from typing_extensions import Literal from attrs import frozen, define From e8bcfea4f3c318a907926de7cc507549c291b059 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 29 Mar 2023 09:56:30 -0400 Subject: [PATCH 1678/2309] news fragment --- newsfragments/4004.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/4004.minor diff --git a/newsfragments/4004.minor b/newsfragments/4004.minor new file mode 100644 index 000000000..e69de29bb From 2f3091a065fa3cb49183807b81785d8bd121f807 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 29 Mar 2023 10:00:38 -0400 Subject: [PATCH 1679/2309] pass mypy strict on the new module --- mypy.ini | 2 +- src/allmydata/listeners.py | 8 +++++--- src/allmydata/util/iputil.py | 12 +++--------- 3 files changed, 9 insertions(+), 13 deletions(-) diff --git a/mypy.ini b/mypy.ini index 7acc0ddc5..22b8a52f9 100644 --- a/mypy.ini +++ b/mypy.ini @@ -9,7 +9,7 @@ no_implicit_optional = True warn_redundant_casts = True strict_equality = True -[mypy-allmydata.test.cli.wormholetesting] +[mypy-allmydata.test.cli.wormholetesting,allmydata.listeners] disallow_any_generics = True disallow_subclassing_any = True disallow_untyped_calls = True diff --git a/src/allmydata/listeners.py b/src/allmydata/listeners.py index 3c4e71b9c..667f984e5 100644 --- a/src/allmydata/listeners.py +++ b/src/allmydata/listeners.py @@ -6,7 +6,7 @@ detect when it is possible to use it, etc. from __future__ import annotations -from typing import Any, Protocol, Sequence, Mapping, Optional +from typing import Any, Protocol, Sequence, Mapping, Optional, Union, Awaitable from typing_extensions import Literal from attrs import frozen, define @@ -101,7 +101,7 @@ class StaticProvider: """ _available: bool _hide_ip: bool - _config: Any + _config: Union[Awaitable[ListenerConfig], ListenerConfig] _address: IAddressFamily def is_available(self) -> bool: @@ -110,7 +110,9 @@ class StaticProvider: def can_hide_ip(self) -> bool: return self._hide_ip - async def create_config(self, reactor: Any, cli_config: Any) -> None: + async def create_config(self, reactor: Any, cli_config: Any) -> ListenerConfig: + if isinstance(self._config, ListenerConfig): + return self._config return await self._config def create(self, reactor: Any, config: Any) -> IAddressFamily: diff --git a/src/allmydata/util/iputil.py b/src/allmydata/util/iputil.py index fd3e88c7f..e71e514e8 100644 --- a/src/allmydata/util/iputil.py +++ b/src/allmydata/util/iputil.py @@ -1,17 +1,10 @@ """ Utilities for getting IP addresses. - -Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals +from future.utils import native_str -from future.utils import PY2, native_str -if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 +from typing import Callable import os, socket @@ -39,6 +32,7 @@ from .gcutil import ( fcntl = requireModule("fcntl") +allocate_tcp_port: Callable[[], int] from foolscap.util import allocate_tcp_port # re-exported try: From b81fad2970ff3eeea87fa869318dcee1f6db6ce9 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 3 Apr 2023 10:37:49 -0400 Subject: [PATCH 1680/2309] Make sure tests have the same error testing infrastructure as the real thing. --- src/allmydata/storage/http_server.py | 28 ++++++++++++++----------- src/allmydata/test/test_storage_http.py | 2 ++ 2 files changed, 18 insertions(+), 12 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 517771c02..5ccb43c60 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -489,6 +489,21 @@ def read_range( return d +def _add_error_handling(app: Klein): + """Add exception handlers to a Klein app.""" + @app.handle_errors(_HTTPError) + def _http_error(_, request, failure): + """Handle ``_HTTPError`` exceptions.""" + request.setResponseCode(failure.value.code) + return b"" + + @app.handle_errors(CDDLValidationError) + def _cddl_validation_error(_, request, failure): + """Handle CDDL validation errors.""" + request.setResponseCode(http.BAD_REQUEST) + return str(failure.value).encode("utf-8") + + class HTTPServer(object): """ A HTTP interface to the storage server. @@ -496,18 +511,7 @@ class HTTPServer(object): _app = Klein() _app.url_map.converters["storage_index"] = StorageIndexConverter - - @_app.handle_errors(_HTTPError) - def _http_error(self, request, failure): - """Handle ``_HTTPError`` exceptions.""" - request.setResponseCode(failure.value.code) - return b"" - - @_app.handle_errors(CDDLValidationError) - def _cddl_validation_error(self, request, failure): - """Handle CDDL validation errors.""" - request.setResponseCode(http.BAD_REQUEST) - return str(failure.value).encode("utf-8") + _add_error_handling(_app) def __init__( self, diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index eb5bcd4db..19529cd0e 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -54,6 +54,7 @@ from ..storage.http_server import ( ClientSecretsException, _authorized_route, StorageIndexConverter, + _add_error_handling ) from ..storage.http_client import ( StorageClient, @@ -253,6 +254,7 @@ class TestApp(object): clock: IReactorTime _app = Klein() + _add_error_handling(_app) _swissnum = SWISSNUM_FOR_TEST # Match what the test client is using @_authorized_route(_app, {Secrets.UPLOAD}, "/upload_secret", methods=["GET"]) From 41939e2b286fedca55af7ad202739751c40d7f3d Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 3 Apr 2023 11:11:24 -0400 Subject: [PATCH 1681/2309] Add some type annotations. --- src/allmydata/storage/http_client.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index fcfc5bff3..6450050a9 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -341,7 +341,7 @@ class StorageClient(object): https_url = DecodedURL().replace(scheme="https", host=nurl.host, port=nurl.port) return cls(https_url, swissnum, treq_client, reactor) - def relative_url(self, path): + def relative_url(self, path: str) -> DecodedURL: """Get a URL relative to the base URL.""" return self._base_url.click(path) @@ -357,14 +357,14 @@ class StorageClient(object): def request( self, - method, - url, - lease_renew_secret=None, - lease_cancel_secret=None, - upload_secret=None, - write_enabler_secret=None, - headers=None, - message_to_serialize=None, + method: str, + url: DecodedURL, + lease_renew_secret: Optional[bytes]=None, + lease_cancel_secret: Optional[bytes]=None, + upload_secret: Optional[bytes]=None, + write_enabler_secret: Optional[bytes]=None, + headers: Optional[Headers]=None, + message_to_serialize: object=None, timeout: float = 60, **kwargs, ): From 3b3ea5409c7c31b5158c2ec5093d7021a927d2d3 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 3 Apr 2023 11:26:08 -0400 Subject: [PATCH 1682/2309] Type says we should only pass in DecodedURL. --- src/allmydata/test/test_storage_http.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 19529cd0e..ea93ad360 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -54,7 +54,7 @@ from ..storage.http_server import ( ClientSecretsException, _authorized_route, StorageIndexConverter, - _add_error_handling + _add_error_handling, ) from ..storage.http_client import ( StorageClient, @@ -348,7 +348,7 @@ class CustomHTTPServerTests(SyncTestCase): response = result_of( self.client.request( "GET", - "http://127.0.0.1/upload_secret", + DecodedURL.from_text("http://127.0.0.1/upload_secret"), ) ) self.assertEqual(response.code, 400) @@ -356,7 +356,9 @@ class CustomHTTPServerTests(SyncTestCase): # With secret, we're good. response = result_of( self.client.request( - "GET", "http://127.0.0.1/upload_secret", upload_secret=b"MAGIC" + "GET", + DecodedURL.from_text("http://127.0.0.1/upload_secret"), + upload_secret=b"MAGIC", ) ) self.assertEqual(response.code, 200) @@ -380,7 +382,7 @@ class CustomHTTPServerTests(SyncTestCase): response = result_of( self.client.request( "GET", - f"http://127.0.0.1/bytes/{length}", + DecodedURL.from_text(f"http://127.0.0.1/bytes/{length}"), ) ) @@ -401,7 +403,7 @@ class CustomHTTPServerTests(SyncTestCase): response = result_of( self.client.request( "GET", - f"http://127.0.0.1/bytes/{length}", + DecodedURL.from_text(f"http://127.0.0.1/bytes/{length}"), ) ) @@ -416,7 +418,7 @@ class CustomHTTPServerTests(SyncTestCase): response = result_of( self.client.request( "GET", - "http://127.0.0.1/slowly_never_finish_result", + DecodedURL.from_text("http://127.0.0.1/slowly_never_finish_result"), ) ) @@ -444,7 +446,7 @@ class CustomHTTPServerTests(SyncTestCase): response = result_of( self.client.request( "GET", - "http://127.0.0.1/die", + DecodedURL.from_text("http://127.0.0.1/die"), ) ) @@ -461,6 +463,7 @@ class Reactor(Clock): Advancing the clock also runs any callbacks scheduled via callFromThread. """ + def __init__(self): Clock.__init__(self) self._queue = Queue() @@ -501,7 +504,9 @@ class HttpTestFixture(Fixture): self.storage_server = StorageServer( self.tempdir.path, b"\x00" * 20, clock=self.clock ) - self.http_server = HTTPServer(self.clock, self.storage_server, SWISSNUM_FOR_TEST) + self.http_server = HTTPServer( + self.clock, self.storage_server, SWISSNUM_FOR_TEST + ) self.treq = StubTreq(self.http_server.get_resource()) self.client = StorageClient( DecodedURL.from_text("http://127.0.0.1"), From 57ec669e1e70dc0fd6be4b6ddfbe3fe0f32221de Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 3 Apr 2023 11:29:57 -0400 Subject: [PATCH 1683/2309] Add logging for request(). --- src/allmydata/storage/http_client.py | 58 ++++++++++++++++++++++------ 1 file changed, 47 insertions(+), 11 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 6450050a9..cbd4634b4 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -19,7 +19,7 @@ from collections_extended import RangeMap from werkzeug.datastructures import Range, ContentRange from twisted.web.http_headers import Headers from twisted.web import http -from twisted.web.iweb import IPolicyForHTTPS +from twisted.web.iweb import IPolicyForHTTPS, IResponse from twisted.internet.defer import inlineCallbacks, returnValue, fail, Deferred, succeed from twisted.internet.interfaces import ( IOpenSSLClientConnectionCreator, @@ -355,19 +355,20 @@ class StorageClient(object): ) return headers - def request( + @async_to_deferred + async def request( self, method: str, - url: DecodedURL, - lease_renew_secret: Optional[bytes]=None, - lease_cancel_secret: Optional[bytes]=None, - upload_secret: Optional[bytes]=None, - write_enabler_secret: Optional[bytes]=None, - headers: Optional[Headers]=None, - message_to_serialize: object=None, + url: str, + lease_renew_secret: Optional[bytes] = None, + lease_cancel_secret: Optional[bytes] = None, + upload_secret: Optional[bytes] = None, + write_enabler_secret: Optional[bytes] = None, + headers: Optional[Headers] = None, + message_to_serialize: object = None, timeout: float = 60, **kwargs, - ): + ) -> Deferred[IResponse]: """ Like ``treq.request()``, but with optional secrets that get translated into corresponding HTTP headers. @@ -377,6 +378,41 @@ class StorageClient(object): Default timeout is 60 seconds. """ + with start_action( + action_type="allmydata:storage:http-client:request", + method=method, + url=url.to_text(), + timeout=timeout, + ) as ctx: + response = await self._request( + method, + url, + lease_renew_secret, + lease_cancel_secret, + upload_secret, + write_enabler_secret, + headers, + message_to_serialize, + timeout, + **kwargs, + ) + ctx.add_success_fields(response_code=response.code) + return response + + async def _request( + self, + method: str, + url: str, + lease_renew_secret: Optional[bytes] = None, + lease_cancel_secret: Optional[bytes] = None, + upload_secret: Optional[bytes] = None, + write_enabler_secret: Optional[bytes] = None, + headers: Optional[Headers] = None, + message_to_serialize: object = None, + timeout: float = 60, + **kwargs, + ) -> IResponse: + """The implementation of request().""" headers = self._get_headers(headers) # Add secrets: @@ -407,7 +443,7 @@ class StorageClient(object): kwargs["data"] = dumps(message_to_serialize) headers.addRawHeader("Content-Type", CBOR_MIME_TYPE) - return self._treq.request( + return await self._treq.request( method, url, headers=headers, timeout=timeout, **kwargs ) From 5e3fa04a3a6e0ae2210fcdc49347e18c3e384ec2 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 3 Apr 2023 11:30:22 -0400 Subject: [PATCH 1684/2309] Reformat with black. --- src/allmydata/storage/http_client.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index cbd4634b4..cead33732 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -488,12 +488,14 @@ class StorageClientGeneral(object): # Add some features we know are true because the HTTP API # specification requires them and because other parts of the storage # client implementation assumes they will be present. - decoded_response[b"http://allmydata.org/tahoe/protocols/storage/v1"].update({ - b'tolerates-immutable-read-overrun': True, - b'delete-mutable-shares-with-zero-length-writev': True, - b'fills-holes-with-zero-bytes': True, - b'prevents-read-past-end-of-share-data': True, - }) + decoded_response[b"http://allmydata.org/tahoe/protocols/storage/v1"].update( + { + b"tolerates-immutable-read-overrun": True, + b"delete-mutable-shares-with-zero-length-writev": True, + b"fills-holes-with-zero-bytes": True, + b"prevents-read-past-end-of-share-data": True, + } + ) returnValue(decoded_response) @inlineCallbacks From e19aeb5aea2bd57bfb2de3bc6f7f87a9097723c6 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 3 Apr 2023 11:40:48 -0400 Subject: [PATCH 1685/2309] Correct the annotation. --- src/allmydata/storage/http_client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index cead33732..bd9e3fc39 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -359,7 +359,7 @@ class StorageClient(object): async def request( self, method: str, - url: str, + url: DecodedURL, lease_renew_secret: Optional[bytes] = None, lease_cancel_secret: Optional[bytes] = None, upload_secret: Optional[bytes] = None, @@ -402,7 +402,7 @@ class StorageClient(object): async def _request( self, method: str, - url: str, + url: DecodedURL, lease_renew_secret: Optional[bytes] = None, lease_cancel_secret: Optional[bytes] = None, upload_secret: Optional[bytes] = None, From 1de8e811b5d963695af4c02886e85ee1ede36619 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 4 Apr 2023 10:58:22 -0400 Subject: [PATCH 1686/2309] Tweaks. --- integration/test_tor.py | 6 ++++-- integration/util.py | 1 + 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/integration/test_tor.py b/integration/test_tor.py index b116fe319..d418f786b 100644 --- a/integration/test_tor.py +++ b/integration/test_tor.py @@ -43,8 +43,8 @@ if PY2: def test_onion_service_storage(reactor, request, temp_dir, flog_gatherer, tor_network, tor_introducer_furl): carol = yield _create_anonymous_node(reactor, 'carol', 8008, request, temp_dir, flog_gatherer, tor_network, tor_introducer_furl) dave = yield _create_anonymous_node(reactor, 'dave', 8009, request, temp_dir, flog_gatherer, tor_network, tor_introducer_furl) - yield util.await_client_ready(carol, minimum_number_of_servers=2) - yield util.await_client_ready(dave, minimum_number_of_servers=2) + yield util.await_client_ready(carol, minimum_number_of_servers=2, timeout=60) + yield util.await_client_ready(dave, minimum_number_of_servers=2, timeout=60) # ensure both nodes are connected to "a grid" by uploading # something via carol, and retrieve it using dave. @@ -125,6 +125,8 @@ def _create_anonymous_node(reactor, name, control_port, request, temp_dir, flog_ config.set_config("node", "log_gatherer.furl", flog_gatherer) config.set_config("tor", "onion", "true") config.set_config("tor", "onion.external_port", "3457") + config.set_config("tor", "control.port", f"tcp:port={control_port}:host=127.0.0.1") + #config.set_config("tor", "launch", "True") config.set_config("tor", "onion.local_port", str(control_port + 1000)) config.set_config("tor", "onion.private_key_file", "private/tor_onion.privkey") diff --git a/integration/util.py b/integration/util.py index 08c07a059..a11c02225 100644 --- a/integration/util.py +++ b/integration/util.py @@ -90,6 +90,7 @@ class _CollectOutputProtocol(ProcessProtocol): self.done.errback(reason) def outReceived(self, data): + print("OUT: {!r}".format(data)) self.output.write(data) def errReceived(self, data): From efa51d41dcdbf430510927b4ad6d41a9835b267a Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 4 Apr 2023 10:58:28 -0400 Subject: [PATCH 1687/2309] Newer chutney. --- integration/conftest.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/integration/conftest.py b/integration/conftest.py index 280d98f72..36e7eef0b 100644 --- a/integration/conftest.py +++ b/integration/conftest.py @@ -465,7 +465,7 @@ def chutney(reactor, temp_dir): 'git', ( 'git', 'clone', - 'https://git.torproject.org/chutney.git', + 'https://gitlab.torproject.org/tpo/core/chutney.git', chutney_dir, ), env=environ, @@ -481,7 +481,7 @@ def chutney(reactor, temp_dir): ( 'git', '-C', chutney_dir, 'reset', '--hard', - 'c825cba0bcd813c644c6ac069deeb7347d3200ee' + 'c4f6789ad2558dcbfeb7d024c6481d8112bfb6c2' ), env=environ, ) From 2be9e949f0c22c1daba57c2951bb5ff8eac9654d Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 5 Apr 2023 09:02:34 -0400 Subject: [PATCH 1688/2309] add Ubuntu 22.04 unit test job to CircleCI --- .circleci/config.yml | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index d46e255af..638b4fc3e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -39,6 +39,8 @@ dockerhub-auth-template: &DOCKERHUB_AUTH <<: *DOCKERHUB_CONTEXT - "build-image-ubuntu-20-04": <<: *DOCKERHUB_CONTEXT + - "build-image-ubuntu-22-04": + <<: *DOCKERHUB_CONTEXT - "build-image-fedora-35": <<: *DOCKERHUB_CONTEXT - "build-image-oraclelinux-8": @@ -78,6 +80,9 @@ workflows: - "ubuntu-20-04": {} + - "ubuntu-22-04": + {} + # Equivalent to RHEL 8; CentOS 8 is dead. - "oraclelinux-8": {} @@ -333,6 +338,16 @@ jobs: <<: *UTF_8_ENVIRONMENT TAHOE_LAFS_TOX_ENVIRONMENT: "py39" + ubuntu-22-04: + <<: *DEBIAN + docker: + - <<: *DOCKERHUB_AUTH + image: "tahoelafsci/ubuntu:22.04-py3.10" + user: "nobody" + environment: + <<: *UTF_8_ENVIRONMENT + TAHOE_LAFS_TOX_ENVIRONMENT: "py310" + oraclelinux-8: &RHEL_DERIV docker: - <<: *DOCKERHUB_AUTH @@ -479,6 +494,15 @@ jobs: PYTHON_VERSION: "3.9" + build-image-ubuntu-22-04: + <<: *BUILD_IMAGE + + environment: + DISTRO: "ubuntu" + TAG: "22.04" + PYTHON_VERSION: "3.10" + + build-image-oraclelinux-8: <<: *BUILD_IMAGE From 8557c66b39ed6ec806160eec857ba017ee2506de Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 5 Apr 2023 09:03:20 -0400 Subject: [PATCH 1689/2309] Remove the "ubuntu-latest" unit test job from GitHub Actions --- .github/workflows/ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e006d90ac..adcf6cc5d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -46,7 +46,6 @@ jobs: matrix: os: - windows-latest - - ubuntu-latest python-version: - "3.8" - "3.9" From 7ae7db678eae92fa41450240f91b56335aaff9dd Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 5 Apr 2023 09:03:51 -0400 Subject: [PATCH 1690/2309] add CPython 3.8 and CPython 3.9 unit test jobs to CircleCI --- .circleci/config.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 638b4fc3e..77c29734d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -93,6 +93,8 @@ workflows: matrix: parameters: pythonVersion: + - "python38" + - "python39" - "python310" - "nixos": From 4c542dfa9b5aa06b3514e95204f74c0860e07329 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 5 Apr 2023 09:37:16 -0400 Subject: [PATCH 1691/2309] news fragment --- newsfragments/4006.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/4006.minor diff --git a/newsfragments/4006.minor b/newsfragments/4006.minor new file mode 100644 index 000000000..e69de29bb From 812458699dc22a62f49419a2fd62bcf7510b08b2 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 5 Apr 2023 11:38:28 -0400 Subject: [PATCH 1692/2309] The tcp listening port needs to match the onion local port, or you get connection refused when you try to connect to the hidden service. --- integration/test_tor.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/integration/test_tor.py b/integration/test_tor.py index d418f786b..6f6f54c25 100644 --- a/integration/test_tor.py +++ b/integration/test_tor.py @@ -126,8 +126,6 @@ def _create_anonymous_node(reactor, name, control_port, request, temp_dir, flog_ config.set_config("tor", "onion", "true") config.set_config("tor", "onion.external_port", "3457") config.set_config("tor", "control.port", f"tcp:port={control_port}:host=127.0.0.1") - #config.set_config("tor", "launch", "True") - config.set_config("tor", "onion.local_port", str(control_port + 1000)) config.set_config("tor", "onion.private_key_file", "private/tor_onion.privkey") print("running") From 13e9f88309c15f5f4bfe71c0abe8bff5a2c2326b Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 7 Apr 2023 15:23:20 -0400 Subject: [PATCH 1693/2309] Add necessary config option to ensure it listens on Tor, and also give correct Tor control port. --- integration/conftest.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/integration/conftest.py b/integration/conftest.py index 621c0224c..f3cf9a9d8 100644 --- a/integration/conftest.py +++ b/integration/conftest.py @@ -294,7 +294,8 @@ def tor_introducer(reactor, temp_dir, flog_gatherer, request): request, ( 'create-introducer', - '--tor-control-port', 'tcp:localhost:8010', + '--tor-control-port', 'tcp:localhost:8007', + '--hide-ip', '--listen=tor', intro_dir, ), From 7b9432482724297c6d637aee20c2a6f5d94339ff Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 7 Apr 2023 15:23:51 -0400 Subject: [PATCH 1694/2309] More debugging. --- integration/util.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/integration/util.py b/integration/util.py index a11c02225..ac3fe2833 100644 --- a/integration/util.py +++ b/integration/util.py @@ -140,6 +140,7 @@ class _MagicTextProtocol(ProcessProtocol): self.exited.callback(None) def outReceived(self, data): + print("OUT", data) data = str(data, sys.stdout.encoding) sys.stdout.write(data) self._output.write(data) @@ -148,6 +149,7 @@ class _MagicTextProtocol(ProcessProtocol): self.magic_seen.callback(self) def errReceived(self, data): + print("ERR", data) data = str(data, sys.stderr.encoding) sys.stdout.write(data) From 4d4649f5c24ff89f1b538a09740eaffefea80dd2 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 10 Apr 2023 11:28:26 -0400 Subject: [PATCH 1695/2309] Apply suggestions from code review Co-authored-by: Jean-Paul Calderone --- src/allmydata/storage/http_client.py | 2 +- src/allmydata/storage/http_server.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index bd9e3fc39..ea142ed85 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -368,7 +368,7 @@ class StorageClient(object): message_to_serialize: object = None, timeout: float = 60, **kwargs, - ) -> Deferred[IResponse]: + ) -> IResponse: """ Like ``treq.request()``, but with optional secrets that get translated into corresponding HTTP headers. diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 5ccb43c60..8647274f8 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -107,7 +107,7 @@ def _authorization_decorator(required_secrets): @wraps(f) def route(self, request, *args, **kwargs): with start_action( - action_type="allmydata:storage:http-server:handle_request", + action_type="allmydata:storage:http-server:handle-request", method=request.method, path=request.path, ) as ctx: From cebf62176ee7ec064936c111aa28cb65f69d649b Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 10 Apr 2023 11:40:59 -0400 Subject: [PATCH 1696/2309] WIP add logging to decode_cbor. --- src/allmydata/storage/http_client.py | 46 +++++++++++++++------------- 1 file changed, 25 insertions(+), 21 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index ea142ed85..131f23846 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -447,24 +447,28 @@ class StorageClient(object): method, url, headers=headers, timeout=timeout, **kwargs ) - def decode_cbor(self, response, schema: Schema): + async def decode_cbor(self, response, schema: Schema) -> object: """Given HTTP response, return decoded CBOR body.""" - - def got_content(f: BinaryIO): - data = f.read() - schema.validate_cbor(data) - return loads(data) - - if response.code > 199 and response.code < 300: - content_type = get_content_type(response.headers) - if content_type == CBOR_MIME_TYPE: - return limited_content(response, self._clock).addCallback(got_content) + with start_action(action_type="allmydata:storage:http-client:decode-cbor"): + if response.code > 199 and response.code < 300: + content_type = get_content_type(response.headers) + if content_type == CBOR_MIME_TYPE: + f = await limited_content(response, self._clock) + data = f.read() + schema.validate_cbor(data) + return loads(data) + else: + raise ClientException( + -1, + "Server didn't send CBOR, content type is {}".format( + content_type + ), + ) else: - raise ClientException(-1, "Server didn't send CBOR") - else: - return treq.content(response).addCallback( - lambda data: fail(ClientException(response.code, response.phrase, data)) - ) + data = ( + await limited_content(response, self._clock, max_length=10_000) + ).read() + raise ClientException(response.code, response.phrase, data) @define(hash=True) @@ -475,14 +479,14 @@ class StorageClientGeneral(object): _client: StorageClient - @inlineCallbacks - def get_version(self): + @async_to_deferred + async def get_version(self): """ Return the version metadata for the server. """ url = self._client.relative_url("/storage/v1/version") - response = yield self._client.request("GET", url) - decoded_response = yield self._client.decode_cbor( + response = await self._client.request("GET", url) + decoded_response = await self._client.decode_cbor( response, _SCHEMAS["get_version"] ) # Add some features we know are true because the HTTP API @@ -496,7 +500,7 @@ class StorageClientGeneral(object): b"prevents-read-past-end-of-share-data": True, } ) - returnValue(decoded_response) + return decoded_response @inlineCallbacks def add_or_renew_lease( From 2a7616e0bebe29865767141255e36e9058db77e6 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 12 Apr 2023 16:43:46 -0400 Subject: [PATCH 1697/2309] Get tests passing again. --- src/allmydata/storage/http_client.py | 36 ++++++++++++------------- src/allmydata/test/test_storage_http.py | 3 ++- 2 files changed, 19 insertions(+), 20 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 131f23846..7fc68c902 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -651,8 +651,8 @@ class StorageClientImmutables(object): _client: StorageClient - @inlineCallbacks - def create( + @async_to_deferred + async def create( self, storage_index, share_numbers, @@ -679,7 +679,7 @@ class StorageClientImmutables(object): ) message = {"share-numbers": share_numbers, "allocated-size": allocated_size} - response = yield self._client.request( + response = await self._client.request( "POST", url, lease_renew_secret=lease_renew_secret, @@ -687,14 +687,12 @@ class StorageClientImmutables(object): upload_secret=upload_secret, message_to_serialize=message, ) - decoded_response = yield self._client.decode_cbor( + decoded_response = await self._client.decode_cbor( response, _SCHEMAS["allocate_buckets"] ) - returnValue( - ImmutableCreateResult( - already_have=decoded_response["already-have"], - allocated=decoded_response["allocated"], - ) + return ImmutableCreateResult( + already_have=decoded_response["already-have"], + allocated=decoded_response["allocated"], ) @inlineCallbacks @@ -720,8 +718,8 @@ class StorageClientImmutables(object): response.code, ) - @inlineCallbacks - def write_share_chunk( + @async_to_deferred + async def write_share_chunk( self, storage_index, share_number, upload_secret, offset, data ): # type: (bytes, int, bytes, int, bytes) -> Deferred[UploadProgress] """ @@ -741,7 +739,7 @@ class StorageClientImmutables(object): _encode_si(storage_index), share_number ) ) - response = yield self._client.request( + response = await self._client.request( "PATCH", url, upload_secret=upload_secret, @@ -765,13 +763,13 @@ class StorageClientImmutables(object): raise ClientException( response.code, ) - body = yield self._client.decode_cbor( + body = await self._client.decode_cbor( response, _SCHEMAS["immutable_write_share_chunk"] ) remaining = RangeMap() for chunk in body["required"]: remaining.set(True, chunk["begin"], chunk["end"]) - returnValue(UploadProgress(finished=finished, required=remaining)) + return UploadProgress(finished=finished, required=remaining) def read_share_chunk( self, storage_index, share_number, offset, length @@ -783,21 +781,21 @@ class StorageClientImmutables(object): self._client, "immutable", storage_index, share_number, offset, length ) - @inlineCallbacks - def list_shares(self, storage_index: bytes) -> Deferred[set[int]]: + @async_to_deferred + async def list_shares(self, storage_index: bytes) -> Deferred[set[int]]: """ Return the set of shares for a given storage index. """ url = self._client.relative_url( "/storage/v1/immutable/{}/shares".format(_encode_si(storage_index)) ) - response = yield self._client.request( + response = await self._client.request( "GET", url, ) if response.code == http.OK: - body = yield self._client.decode_cbor(response, _SCHEMAS["list_shares"]) - returnValue(set(body)) + body = await self._client.decode_cbor(response, _SCHEMAS["list_shares"]) + return set(body) else: raise ClientException(response.code) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index ea93ad360..eca2be1c1 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -34,7 +34,7 @@ from hyperlink import DecodedURL from collections_extended import RangeMap from twisted.internet.task import Clock, Cooperator from twisted.internet.interfaces import IReactorTime, IReactorFromThreads -from twisted.internet.defer import CancelledError, Deferred +from twisted.internet.defer import CancelledError, Deferred, ensureDeferred from twisted.web import http from twisted.web.http_headers import Headers from werkzeug import routing @@ -520,6 +520,7 @@ class HttpTestFixture(Fixture): Like ``result_of``, but supports fake reactor and ``treq`` testing infrastructure necessary to support asynchronous HTTP server endpoints. """ + d = ensureDeferred(d) result = [] error = [] d.addCallbacks(result.append, error.append) From 3997eaaf9048af3daca9cc291875f17b2a403218 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 12 Apr 2023 17:00:31 -0400 Subject: [PATCH 1698/2309] Fix type annotations. --- src/allmydata/storage/http_client.py | 49 ++++++++++++++++++++-------- 1 file changed, 36 insertions(+), 13 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 7fc68c902..b1877cd5f 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -5,7 +5,7 @@ HTTP client that talks to the HTTP storage server. from __future__ import annotations from eliot import start_action, register_exception_extractor -from typing import Union, Optional, Sequence, Mapping, BinaryIO +from typing import Union, Optional, Sequence, Mapping, BinaryIO, cast, TypedDict from base64 import b64encode from io import BytesIO from os import SEEK_END @@ -486,13 +486,17 @@ class StorageClientGeneral(object): """ url = self._client.relative_url("/storage/v1/version") response = await self._client.request("GET", url) - decoded_response = await self._client.decode_cbor( - response, _SCHEMAS["get_version"] + decoded_response = cast( + dict[bytes, object], + await self._client.decode_cbor(response, _SCHEMAS["get_version"]), ) # Add some features we know are true because the HTTP API # specification requires them and because other parts of the storage # client implementation assumes they will be present. - decoded_response[b"http://allmydata.org/tahoe/protocols/storage/v1"].update( + cast( + dict[bytes, object], + decoded_response[b"http://allmydata.org/tahoe/protocols/storage/v1"], + ).update( { b"tolerates-immutable-read-overrun": True, b"delete-mutable-shares-with-zero-length-writev": True, @@ -687,8 +691,9 @@ class StorageClientImmutables(object): upload_secret=upload_secret, message_to_serialize=message, ) - decoded_response = await self._client.decode_cbor( - response, _SCHEMAS["allocate_buckets"] + decoded_response = cast( + dict[str, set[int]], + await self._client.decode_cbor(response, _SCHEMAS["allocate_buckets"]), ) return ImmutableCreateResult( already_have=decoded_response["already-have"], @@ -763,8 +768,11 @@ class StorageClientImmutables(object): raise ClientException( response.code, ) - body = await self._client.decode_cbor( - response, _SCHEMAS["immutable_write_share_chunk"] + body = cast( + dict[str, list[dict[str, int]]], + await self._client.decode_cbor( + response, _SCHEMAS["immutable_write_share_chunk"] + ), ) remaining = RangeMap() for chunk in body["required"]: @@ -794,7 +802,10 @@ class StorageClientImmutables(object): url, ) if response.code == http.OK: - body = await self._client.decode_cbor(response, _SCHEMAS["list_shares"]) + body = cast( + set[int], + await self._client.decode_cbor(response, _SCHEMAS["list_shares"]), + ) return set(body) else: raise ClientException(response.code) @@ -865,6 +876,12 @@ class ReadTestWriteResult: reads: Mapping[int, Sequence[bytes]] +# Result type for mutable read/test/write HTTP response. +MUTABLE_RTW = TypedDict( + "MUTABLE_RTW", {"success": bool, "data": dict[int, list[bytes]]} +) + + @frozen class StorageClientMutables: """ @@ -911,8 +928,11 @@ class StorageClientMutables: message_to_serialize=message, ) if response.code == http.OK: - result = await self._client.decode_cbor( - response, _SCHEMAS["mutable_read_test_write"] + result = cast( + MUTABLE_RTW, + await self._client.decode_cbor( + response, _SCHEMAS["mutable_read_test_write"] + ), ) return ReadTestWriteResult(success=result["success"], reads=result["data"]) else: @@ -942,8 +962,11 @@ class StorageClientMutables: ) response = await self._client.request("GET", url) if response.code == http.OK: - return await self._client.decode_cbor( - response, _SCHEMAS["mutable_list_shares"] + return cast( + set[int], + await self._client.decode_cbor( + response, _SCHEMAS["mutable_list_shares"] + ), ) else: raise ClientException(response.code) From 8bda370b30ef187d321840ecea97c56a439e1280 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 12 Apr 2023 17:00:47 -0400 Subject: [PATCH 1699/2309] News fragment. --- newsfragments/4005.misc | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/4005.misc diff --git a/newsfragments/4005.misc b/newsfragments/4005.misc new file mode 100644 index 000000000..e69de29bb From 840ed0bf47561c7c9720c91deab3d5a5c56a7b35 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 12 Apr 2023 17:04:00 -0400 Subject: [PATCH 1700/2309] Unused imports. --- src/allmydata/storage/http_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index b1877cd5f..ef4005414 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -20,7 +20,7 @@ from werkzeug.datastructures import Range, ContentRange from twisted.web.http_headers import Headers from twisted.web import http from twisted.web.iweb import IPolicyForHTTPS, IResponse -from twisted.internet.defer import inlineCallbacks, returnValue, fail, Deferred, succeed +from twisted.internet.defer import inlineCallbacks, Deferred, succeed from twisted.internet.interfaces import ( IOpenSSLClientConnectionCreator, IReactorTime, From 33ab0ce0422ff6211bd614bb8429fbe2084b9e02 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 12 Apr 2023 17:10:33 -0400 Subject: [PATCH 1701/2309] Fix name. --- newsfragments/{4005.misc => 4005.minor} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename newsfragments/{4005.misc => 4005.minor} (100%) diff --git a/newsfragments/4005.misc b/newsfragments/4005.minor similarity index 100% rename from newsfragments/4005.misc rename to newsfragments/4005.minor From 507d1f8394cedc672715fee7bec1b5ef75cb6037 Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 12 Apr 2023 22:34:45 -0600 Subject: [PATCH 1702/2309] Fix some Chutney things (and a couple cleanups): wait for bootstrap, increase timeout --- integration/conftest.py | 44 ++++++++++++++++++++++------------------- integration/test_tor.py | 2 +- 2 files changed, 25 insertions(+), 21 deletions(-) diff --git a/integration/conftest.py b/integration/conftest.py index f3cf9a9d8..7a4234de7 100644 --- a/integration/conftest.py +++ b/integration/conftest.py @@ -206,13 +206,6 @@ def flog_gatherer(reactor, temp_dir, flog_binary, request): include_result=False, ) def introducer(reactor, temp_dir, flog_gatherer, request): - config = ''' -[node] -nickname = introducer0 -web.port = 4560 -log_gatherer.furl = {log_furl} -'''.format(log_furl=flog_gatherer) - intro_dir = join(temp_dir, 'introducer') print("making introducer", intro_dir) @@ -232,6 +225,10 @@ log_gatherer.furl = {log_furl} ) pytest_twisted.blockon(done_proto.done) + config = read_config(intro_dir, "tub.port") + config.set_config("node", "nickname", "introducer-tor") + config.set_config("node", "web.port", "4561") + config.set_config("node", "log_gatherer.furl", flog_gatherer) # over-write the config file with our stuff with open(join(intro_dir, 'tahoe.cfg'), 'w') as f: f.write(config) @@ -283,7 +280,8 @@ def introducer_furl(introducer, temp_dir): ) def tor_introducer(reactor, temp_dir, flog_gatherer, request): intro_dir = join(temp_dir, 'introducer_tor') - print("making introducer", intro_dir) + print("making Tor introducer in {}".format(intro_dir)) + print("(this can take tens of seconds to allocate Onion address)") if not exists(intro_dir): mkdir(intro_dir) @@ -342,7 +340,7 @@ def tor_introducer_furl(tor_introducer, temp_dir): print("Don't see {} yet".format(furl_fname)) sleep(.1) furl = open(furl_fname, 'r').read() - print(f"Found Tor introducer furl: {furl}") + print(f"Found Tor introducer furl: {furl} in {furl_fname}") return furl @@ -510,7 +508,13 @@ def chutney(reactor, temp_dir: str) -> tuple[str, dict[str, str]]: ) pytest_twisted.blockon(proto.done) - return (chutney_dir, {"PYTHONPATH": join(chutney_dir, "lib")}) + return ( + chutney_dir, + { + "PYTHONPATH": join(chutney_dir, "lib"), + "CHUTNEY_START_TIME": "200", # default is 60 + } + ) @pytest.fixture(scope='session') @@ -544,17 +548,9 @@ def tor_network(reactor, temp_dir, chutney, request): return proto.done # now, as per Chutney's README, we have to create the network - # ./chutney configure networks/basic - # ./chutney start networks/basic pytest_twisted.blockon(chutney(("configure", basic_network))) - pytest_twisted.blockon(chutney(("start", basic_network))) - - # print some useful stuff - try: - pytest_twisted.blockon(chutney(("status", basic_network))) - except ProcessTerminated: - print("Chutney.TorNet status failed (continuing)") + # ensure we will tear down the network right before we start it def cleanup(): print("Tearing down Chutney Tor network") try: @@ -563,5 +559,13 @@ def tor_network(reactor, temp_dir, chutney, request): # If this doesn't exit cleanly, that's fine, that shouldn't fail # the test suite. pass - request.addfinalizer(cleanup) + + pytest_twisted.blockon(chutney(("start", basic_network))) + pytest_twisted.blockon(chutney(("wait_for_bootstrap", basic_network))) + + # print some useful stuff + try: + pytest_twisted.blockon(chutney(("status", basic_network))) + except ProcessTerminated: + print("Chutney.TorNet status failed (continuing)") diff --git a/integration/test_tor.py b/integration/test_tor.py index fb9d8c086..c3041f6d3 100644 --- a/integration/test_tor.py +++ b/integration/test_tor.py @@ -61,7 +61,7 @@ def test_onion_service_storage(reactor, request, temp_dir, flog_gatherer, tor_ne ) yield proto.done cap = proto.output.getvalue().strip().split()[-1] - print("TEH CAP!", cap) + print("capability: {}".format(cap)) proto = util._CollectOutputProtocol(capture_stderr=False) reactor.spawnProcess( From 9472841c39120b408bfb7efab3b90b3fcb048a53 Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 12 Apr 2023 23:01:28 -0600 Subject: [PATCH 1703/2309] enable tor, i2p services --- src/allmydata/introducer/server.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/allmydata/introducer/server.py b/src/allmydata/introducer/server.py index 98136157d..e0ff138cc 100644 --- a/src/allmydata/introducer/server.py +++ b/src/allmydata/introducer/server.py @@ -83,6 +83,8 @@ def create_introducer(basedir=u"."): i2p_provider, tor_provider, ) + i2p_provider.setServiceParent(node) + tor_provider.setServiceParent(node) return defer.succeed(node) except Exception: return Failure() From 175473df407157db53276e0721fec324242e4bd2 Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 13 Apr 2023 00:37:32 -0600 Subject: [PATCH 1704/2309] longer timeouts, forget less --- integration/conftest.py | 2 +- integration/test_tor.py | 4 ++-- src/allmydata/introducer/server.py | 4 ---- 3 files changed, 3 insertions(+), 7 deletions(-) diff --git a/integration/conftest.py b/integration/conftest.py index 7a4234de7..e7e021016 100644 --- a/integration/conftest.py +++ b/integration/conftest.py @@ -512,7 +512,7 @@ def chutney(reactor, temp_dir: str) -> tuple[str, dict[str, str]]: chutney_dir, { "PYTHONPATH": join(chutney_dir, "lib"), - "CHUTNEY_START_TIME": "200", # default is 60 + "CHUTNEY_START_TIME": "600", # default is 60 } ) diff --git a/integration/test_tor.py b/integration/test_tor.py index c3041f6d3..10e326e46 100644 --- a/integration/test_tor.py +++ b/integration/test_tor.py @@ -33,8 +33,8 @@ if sys.platform.startswith('win'): def test_onion_service_storage(reactor, request, temp_dir, flog_gatherer, tor_network, tor_introducer_furl): carol = yield _create_anonymous_node(reactor, 'carol', 8008, request, temp_dir, flog_gatherer, tor_network, tor_introducer_furl) dave = yield _create_anonymous_node(reactor, 'dave', 8009, request, temp_dir, flog_gatherer, tor_network, tor_introducer_furl) - yield util.await_client_ready(carol, minimum_number_of_servers=2, timeout=60) - yield util.await_client_ready(dave, minimum_number_of_servers=2, timeout=60) + yield util.await_client_ready(carol, minimum_number_of_servers=2, timeout=600) + yield util.await_client_ready(dave, minimum_number_of_servers=2, timeout=600) # ensure both nodes are connected to "a grid" by uploading # something via carol, and retrieve it using dave. diff --git a/src/allmydata/introducer/server.py b/src/allmydata/introducer/server.py index e0ff138cc..5dad89ae8 100644 --- a/src/allmydata/introducer/server.py +++ b/src/allmydata/introducer/server.py @@ -68,10 +68,6 @@ def create_introducer(basedir=u"."): default_connection_handlers, foolscap_connection_handlers = create_connection_handlers(config, i2p_provider, tor_provider) tub_options = create_tub_options(config) - # we don't remember these because the Introducer doesn't make - # outbound connections. - i2p_provider = None - tor_provider = None main_tub = create_main_tub( config, tub_options, default_connection_handlers, foolscap_connection_handlers, i2p_provider, tor_provider, From cf0d3c09f8fd863fb7103077d9564cff9c405317 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 13 Apr 2023 09:20:40 -0400 Subject: [PATCH 1705/2309] News file. --- newsfragments/4009.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/4009.minor diff --git a/newsfragments/4009.minor b/newsfragments/4009.minor new file mode 100644 index 000000000..e69de29bb From 64dbeeab8f55e17e40097195957789792b3a5fc6 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 13 Apr 2023 09:33:18 -0400 Subject: [PATCH 1706/2309] Add logging to get_version(). --- src/allmydata/storage/http_client.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index ef4005414..610fbaddc 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -480,10 +480,17 @@ class StorageClientGeneral(object): _client: StorageClient @async_to_deferred - async def get_version(self): + async def get_version(self) -> dict[bytes, object]: """ Return the version metadata for the server. """ + with start_action( + action_type="allmydata:storage:http-client:get-version", + ): + return await self._get_version() + + async def _get_version(self) -> dict[bytes, object]: + """Implementation of get_version().""" url = self._client.relative_url("/storage/v1/version") response = await self._client.request("GET", url) decoded_response = cast( From af845a40c6dd1fe626771c63dff0c8dd7a6d86c8 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 13 Apr 2023 09:38:33 -0400 Subject: [PATCH 1707/2309] Fix type annotations, removing Deferred in particular. --- src/allmydata/storage/http_client.py | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index ef4005414..e21cfc5cc 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -658,13 +658,13 @@ class StorageClientImmutables(object): @async_to_deferred async def create( self, - storage_index, - share_numbers, - allocated_size, - upload_secret, - lease_renew_secret, - lease_cancel_secret, - ): # type: (bytes, set[int], int, bytes, bytes, bytes) -> Deferred[ImmutableCreateResult] + storage_index: bytes, + share_numbers: set[int], + allocated_size: int, + upload_secret: bytes, + lease_renew_secret: bytes, + lease_cancel_secret: bytes, + ) -> ImmutableCreateResult: """ Create a new storage index for an immutable. @@ -725,8 +725,13 @@ class StorageClientImmutables(object): @async_to_deferred async def write_share_chunk( - self, storage_index, share_number, upload_secret, offset, data - ): # type: (bytes, int, bytes, int, bytes) -> Deferred[UploadProgress] + self, + storage_index: bytes, + share_number: int, + upload_secret: bytes, + offset: int, + data: bytes, + ) -> UploadProgress: """ Upload a chunk of data for a specific share. @@ -790,7 +795,7 @@ class StorageClientImmutables(object): ) @async_to_deferred - async def list_shares(self, storage_index: bytes) -> Deferred[set[int]]: + async def list_shares(self, storage_index: bytes) -> set[int]: """ Return the set of shares for a given storage index. """ From e9a9ac7110a88e8410a80d0481040fb44f614be2 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 13 Apr 2023 09:44:52 -0400 Subject: [PATCH 1708/2309] Rip out codecov for now. --- .circleci/config.yml | 2 +- .circleci/populate-wheelhouse.sh | 2 +- .github/workflows/ci.yml | 2 +- newsfragments/4010.minor | 0 4 files changed, 3 insertions(+), 3 deletions(-) create mode 100644 newsfragments/4010.minor diff --git a/.circleci/config.yml b/.circleci/config.yml index 77c29734d..54b2706cd 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -260,7 +260,7 @@ jobs: name: "Submit coverage results" command: | if [ -n "${UPLOAD_COVERAGE}" ]; then - /tmp/venv/bin/codecov + echo "TODO: Need a new coverage solution, see https://tahoe-lafs.org/trac/tahoe-lafs/ticket/4011" fi docker: diff --git a/.circleci/populate-wheelhouse.sh b/.circleci/populate-wheelhouse.sh index 857171979..374ca0adb 100755 --- a/.circleci/populate-wheelhouse.sh +++ b/.circleci/populate-wheelhouse.sh @@ -9,7 +9,7 @@ BASIC_DEPS="pip wheel" # Python packages we need to support the test infrastructure. *Not* packages # Tahoe-LAFS itself (implementation or test suite) need. -TEST_DEPS="tox~=3.0 codecov" +TEST_DEPS="tox~=3.0" # Python packages we need to generate test reports for CI infrastructure. # *Not* packages Tahoe-LAFS itself (implement or test suite) need. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index adcf6cc5d..1bb7c9efb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -79,7 +79,7 @@ jobs: - name: Install Python packages run: | - pip install --upgrade codecov "tox<4" tox-gh-actions setuptools + pip install --upgrade "tox<4" tox-gh-actions setuptools pip list - name: Display tool versions diff --git a/newsfragments/4010.minor b/newsfragments/4010.minor new file mode 100644 index 000000000..e69de29bb From e0ca48b707b1c6ca35c05083ebcda0547c23b920 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 13 Apr 2023 09:54:36 -0400 Subject: [PATCH 1709/2309] Add logging to add_or_renew_lease(). --- src/allmydata/storage/common.py | 5 +++++ src/allmydata/storage/http_client.py | 21 ++++++++++++++++----- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/src/allmydata/storage/common.py b/src/allmydata/storage/common.py index 17a3f41b7..89e29b081 100644 --- a/src/allmydata/storage/common.py +++ b/src/allmydata/storage/common.py @@ -39,6 +39,11 @@ def si_b2a(storageindex): def si_a2b(ascii_storageindex): return base32.a2b(ascii_storageindex) +def si_to_human_readable(storageindex: bytes) -> str: + """Create human-readable string of storage index.""" + assert len(storageindex) == 16 + return str(base32.b2a(storageindex), "ascii") + def storage_index_to_dir(storageindex): """Convert storage index to directory path. diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 610fbaddc..6f3bb05e1 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -44,7 +44,7 @@ from .http_common import ( CBOR_MIME_TYPE, get_spki_hash, ) -from .common import si_b2a +from .common import si_b2a, si_to_human_readable from ..util.hashutil import timing_safe_compare from ..util.deferredutil import async_to_deferred @@ -513,20 +513,31 @@ class StorageClientGeneral(object): ) return decoded_response - @inlineCallbacks - def add_or_renew_lease( + @async_to_deferred + async def add_or_renew_lease( self, storage_index: bytes, renew_secret: bytes, cancel_secret: bytes - ) -> Deferred[None]: + ) -> None: """ Add or renew a lease. If the renewal secret matches an existing lease, it is renewed. Otherwise a new lease is added. """ + with start_action( + action_type="allmydata:storage:http-client:add-or-renew-lease", + storage_index=si_to_human_readable(storage_index), + ): + return await self._add_or_renew_lease( + storage_index, renew_secret, cancel_secret + ) + + async def _add_or_renew_lease( + self, storage_index: bytes, renew_secret: bytes, cancel_secret: bytes + ) -> None: url = self._client.relative_url( "/storage/v1/lease/{}".format(_encode_si(storage_index)) ) - response = yield self._client.request( + response = await self._client.request( "PUT", url, lease_renew_secret=renew_secret, From 4c2f241361b1f337fd0f667c555bdce86f818268 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 13 Apr 2023 10:28:29 -0400 Subject: [PATCH 1710/2309] Add logging for limited_content(). --- src/allmydata/storage/http_client.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 6f3bb05e1..ce69fdb76 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -4,13 +4,14 @@ HTTP client that talks to the HTTP storage server. from __future__ import annotations -from eliot import start_action, register_exception_extractor from typing import Union, Optional, Sequence, Mapping, BinaryIO, cast, TypedDict from base64 import b64encode from io import BytesIO from os import SEEK_END from attrs import define, asdict, frozen, field +from eliot import start_action, register_exception_extractor +from eliot.twisted import DeferredContext # TODO Make sure to import Python version? from cbor2 import loads, dumps @@ -160,9 +161,18 @@ def limited_content( trickle of data continues to arrive, it will continue to run. """ d = succeed(None) + + # Sadly, addTimeout() won't work because we need access to the IDelayedCall + # in order to reset it on each data chunk received. timeout = clock.callLater(60, d.cancel) collector = _LengthLimitedCollector(max_length, timeout) + with start_action( + action_type="allmydata:storage:http-client:limited-content", + max_length=max_length, + ).context() as action: + d = DeferredContext(d) + # Make really sure everything gets called in Deferred context, treq might # call collector directly... d.addCallback(lambda _: treq.collect(response, collector)) @@ -177,7 +187,8 @@ def limited_content( timeout.cancel() return f - return d.addCallbacks(done, failed) + result = d.addCallbacks(done, failed) + return result.addActionFinish() @define From 28ff24b3a71e83ba04eab003e3264a1ce0706c3e Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 13 Apr 2023 10:40:35 -0400 Subject: [PATCH 1711/2309] Add logging to immutable creation. --- src/allmydata/storage/http_client.py | 29 ++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 6bca822b5..403a2a3c6 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -707,6 +707,35 @@ class StorageClientImmutables(object): Result fires when creating the storage index succeeded, if creating the storage index failed the result will fire with an exception. """ + with start_action( + action_type="allmydata:storage:http-client:immutable:create", + storage_index=si_to_human_readable(storage_index), + share_numbers=share_numbers, + allocated_size=allocated_size, + ) as ctx: + result = await self._create( + storage_index, + share_numbers, + allocated_size, + upload_secret, + lease_renew_secret, + lease_cancel_secret, + ) + ctx.add_success_fields( + already_have=result.already_have, allocated=result.allocated + ) + return result + + async def _create( + self, + storage_index: bytes, + share_numbers: set[int], + allocated_size: int, + upload_secret: bytes, + lease_renew_secret: bytes, + lease_cancel_secret: bytes, + ) -> ImmutableCreateResult: + """Implementation of create().""" url = self._client.relative_url( "/storage/v1/immutable/" + _encode_si(storage_index) ) From 464b47619028e9d194e660ad461467acaf8986b4 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 13 Apr 2023 13:11:17 -0400 Subject: [PATCH 1712/2309] Work on 3.8. --- src/allmydata/storage/http_client.py | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index e21cfc5cc..fc165bedd 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -5,7 +5,7 @@ HTTP client that talks to the HTTP storage server. from __future__ import annotations from eliot import start_action, register_exception_extractor -from typing import Union, Optional, Sequence, Mapping, BinaryIO, cast, TypedDict +from typing import Union, Optional, Sequence, Mapping, BinaryIO, cast, TypedDict, Set from base64 import b64encode from io import BytesIO from os import SEEK_END @@ -487,14 +487,14 @@ class StorageClientGeneral(object): url = self._client.relative_url("/storage/v1/version") response = await self._client.request("GET", url) decoded_response = cast( - dict[bytes, object], + Mapping[bytes, object], await self._client.decode_cbor(response, _SCHEMAS["get_version"]), ) # Add some features we know are true because the HTTP API # specification requires them and because other parts of the storage # client implementation assumes they will be present. cast( - dict[bytes, object], + Mapping[bytes, object], decoded_response[b"http://allmydata.org/tahoe/protocols/storage/v1"], ).update( { @@ -692,7 +692,7 @@ class StorageClientImmutables(object): message_to_serialize=message, ) decoded_response = cast( - dict[str, set[int]], + Mapping[str, Set[int]], await self._client.decode_cbor(response, _SCHEMAS["allocate_buckets"]), ) return ImmutableCreateResult( @@ -774,7 +774,7 @@ class StorageClientImmutables(object): response.code, ) body = cast( - dict[str, list[dict[str, int]]], + Mapping[str, Sequence[Mapping[str, int]]], await self._client.decode_cbor( response, _SCHEMAS["immutable_write_share_chunk"] ), @@ -795,7 +795,7 @@ class StorageClientImmutables(object): ) @async_to_deferred - async def list_shares(self, storage_index: bytes) -> set[int]: + async def list_shares(self, storage_index: bytes) -> Set[int]: """ Return the set of shares for a given storage index. """ @@ -808,7 +808,7 @@ class StorageClientImmutables(object): ) if response.code == http.OK: body = cast( - set[int], + Set[int], await self._client.decode_cbor(response, _SCHEMAS["list_shares"]), ) return set(body) @@ -881,9 +881,10 @@ class ReadTestWriteResult: reads: Mapping[int, Sequence[bytes]] -# Result type for mutable read/test/write HTTP response. +# Result type for mutable read/test/write HTTP response. Can't just use +# dict[int,list[bytes]] because on Python 3.8 that will error out. MUTABLE_RTW = TypedDict( - "MUTABLE_RTW", {"success": bool, "data": dict[int, list[bytes]]} + "MUTABLE_RTW", {"success": bool, "data": Mapping[int, Sequence[bytes]]} ) @@ -958,7 +959,7 @@ class StorageClientMutables: ) @async_to_deferred - async def list_shares(self, storage_index: bytes) -> set[int]: + async def list_shares(self, storage_index: bytes) -> Set[int]: """ List the share numbers for a given storage index. """ @@ -968,7 +969,7 @@ class StorageClientMutables: response = await self._client.request("GET", url) if response.code == http.OK: return cast( - set[int], + Set[int], await self._client.decode_cbor( response, _SCHEMAS["mutable_list_shares"] ), From aca35a553dd48b3175a7cfbaf8f694a87311f404 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 13 Apr 2023 13:30:38 -0400 Subject: [PATCH 1713/2309] Add logging to more immutable methods. --- src/allmydata/storage/http_client.py | 41 +++++++++++++++++++++++++--- 1 file changed, 37 insertions(+), 4 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index d91104fe6..693448b24 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -759,17 +759,28 @@ class StorageClientImmutables(object): allocated=decoded_response["allocated"], ) - @inlineCallbacks - def abort_upload( + @async_to_deferred + async def abort_upload( self, storage_index: bytes, share_number: int, upload_secret: bytes - ) -> Deferred[None]: + ) -> None: """Abort the upload.""" + with start_action( + action_type="allmydata:storage:http-client:immutable:abort-upload", + storage_index=si_to_human_readable(storage_index), + share_number=share_number, + ): + return await self._abort_upload(storage_index, share_number, upload_secret) + + async def _abort_upload( + self, storage_index: bytes, share_number: int, upload_secret: bytes + ) -> None: + """Implementation of ``abort_upload()``.""" url = self._client.relative_url( "/storage/v1/immutable/{}/{}/abort".format( _encode_si(storage_index), share_number ) ) - response = yield self._client.request( + response = await self._client.request( "PUT", url, upload_secret=upload_secret, @@ -803,6 +814,28 @@ class StorageClientImmutables(object): whether the _complete_ share (i.e. all chunks, not just this one) has been uploaded. """ + with start_action( + action_type="allmydata:storage:http-client:immutable:write-share-chunk", + storage_index=si_to_human_readable(storage_index), + share_number=share_number, + offset=offset, + data_len=len(data), + ) as ctx: + result = await self._write_share_chunk( + storage_index, share_number, upload_secret, offset, data + ) + ctx.add_success_fields(finished=result.finished) + return result + + async def _write_share_chunk( + self, + storage_index: bytes, + share_number: int, + upload_secret: bytes, + offset: int, + data: bytes, + ) -> UploadProgress: + """Implementation of ``write_share_chunk()``.""" url = self._client.relative_url( "/storage/v1/immutable/{}/{}".format( _encode_si(storage_index), share_number From d8f176bb8f7bd02e13f98de60f94d059c7c3ee95 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 13 Apr 2023 13:49:19 -0400 Subject: [PATCH 1714/2309] Type check fixes. --- src/allmydata/storage/http_client.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 693448b24..8be2adbdc 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -5,7 +5,17 @@ HTTP client that talks to the HTTP storage server. from __future__ import annotations -from typing import Union, Optional, Sequence, Mapping, BinaryIO, cast, TypedDict, Set +from typing import ( + Union, + Optional, + Sequence, + Mapping, + BinaryIO, + cast, + TypedDict, + Set, + Dict, +) from base64 import b64encode from io import BytesIO from os import SEEK_END @@ -506,14 +516,14 @@ class StorageClientGeneral(object): url = self._client.relative_url("/storage/v1/version") response = await self._client.request("GET", url) decoded_response = cast( - Mapping[bytes, object], + Dict[bytes, object], await self._client.decode_cbor(response, _SCHEMAS["get_version"]), ) # Add some features we know are true because the HTTP API # specification requires them and because other parts of the storage # client implementation assumes they will be present. cast( - Mapping[bytes, object], + Dict[bytes, object], decoded_response[b"http://allmydata.org/tahoe/protocols/storage/v1"], ).update( { From 250efe7d24cd43a676a6a7116da35f6ab226401a Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 13 Apr 2023 16:42:02 -0600 Subject: [PATCH 1715/2309] leftover --- integration/conftest.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/integration/conftest.py b/integration/conftest.py index e7e021016..b54e18e26 100644 --- a/integration/conftest.py +++ b/integration/conftest.py @@ -229,9 +229,6 @@ def introducer(reactor, temp_dir, flog_gatherer, request): config.set_config("node", "nickname", "introducer-tor") config.set_config("node", "web.port", "4561") config.set_config("node", "log_gatherer.furl", flog_gatherer) - # over-write the config file with our stuff - with open(join(intro_dir, 'tahoe.cfg'), 'w') as f: - f.write(config) # "tahoe run" is consistent across Linux/macOS/Windows, unlike the old # "start" command. From 5dcbc00989c94e314a018a8a9d79027796e95ffe Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 14 Apr 2023 10:18:55 -0400 Subject: [PATCH 1716/2309] News fragment. --- newsfragments/4012.bugfix | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/4012.bugfix diff --git a/newsfragments/4012.bugfix b/newsfragments/4012.bugfix new file mode 100644 index 000000000..97dfe6aad --- /dev/null +++ b/newsfragments/4012.bugfix @@ -0,0 +1 @@ +The command-line tools now have a 60-second timeout on individual network reads/writes/connects; previously they could block forever in some situations. \ No newline at end of file From d7ee1637dfabc3654ea6be735fbcd2094d39c1f3 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 14 Apr 2023 10:22:06 -0400 Subject: [PATCH 1717/2309] Set a timeout. --- src/allmydata/scripts/common_http.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/scripts/common_http.py b/src/allmydata/scripts/common_http.py index 95099a2eb..7542a045f 100644 --- a/src/allmydata/scripts/common_http.py +++ b/src/allmydata/scripts/common_http.py @@ -62,9 +62,9 @@ def do_http(method, url, body=b""): assert body.read scheme, host, port, path = parse_url(url) if scheme == "http": - c = http_client.HTTPConnection(host, port) + c = http_client.HTTPConnection(host, port, timeout=60) elif scheme == "https": - c = http_client.HTTPSConnection(host, port) + c = http_client.HTTPSConnection(host, port, timeout=60) else: raise ValueError("unknown scheme '%s', need http or https" % scheme) c.putrequest(method, path) From 67702572a9d4ce03c6614207234ae909672d6df5 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 14 Apr 2023 10:22:14 -0400 Subject: [PATCH 1718/2309] Do a little modernization. --- src/allmydata/scripts/common_http.py | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/src/allmydata/scripts/common_http.py b/src/allmydata/scripts/common_http.py index 7542a045f..4da1345c9 100644 --- a/src/allmydata/scripts/common_http.py +++ b/src/allmydata/scripts/common_http.py @@ -1,18 +1,11 @@ """ Ported to Python 3. """ -from __future__ import unicode_literals -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import os from io import BytesIO -from six.moves import urllib, http_client +from http import client as http_client +import urllib import six import allmydata # for __full_version__ From 1823dd4c03b3d715ef896453542d0ac10e7f4aad Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 14 Apr 2023 10:24:00 -0400 Subject: [PATCH 1719/2309] Switch to a slightly larger block size. --- src/allmydata/scripts/common_http.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/allmydata/scripts/common_http.py b/src/allmydata/scripts/common_http.py index 4da1345c9..4c0319d3c 100644 --- a/src/allmydata/scripts/common_http.py +++ b/src/allmydata/scripts/common_http.py @@ -55,9 +55,9 @@ def do_http(method, url, body=b""): assert body.read scheme, host, port, path = parse_url(url) if scheme == "http": - c = http_client.HTTPConnection(host, port, timeout=60) + c = http_client.HTTPConnection(host, port, timeout=60, blocksize=65536) elif scheme == "https": - c = http_client.HTTPSConnection(host, port, timeout=60) + c = http_client.HTTPSConnection(host, port, timeout=60, blocksize=65536) else: raise ValueError("unknown scheme '%s', need http or https" % scheme) c.putrequest(method, path) @@ -78,7 +78,7 @@ def do_http(method, url, body=b""): return BadResponse(url, err) while True: - data = body.read(8192) + data = body.read(65536) if not data: break c.send(data) From 2916984114bdb9ef85df5a34aa439f45c5a14cab Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 14 Apr 2023 10:29:25 -0400 Subject: [PATCH 1720/2309] More modernization. --- src/allmydata/scripts/common_http.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/src/allmydata/scripts/common_http.py b/src/allmydata/scripts/common_http.py index 4c0319d3c..7d627a7ad 100644 --- a/src/allmydata/scripts/common_http.py +++ b/src/allmydata/scripts/common_http.py @@ -1,12 +1,11 @@ """ -Ported to Python 3. +Blocking HTTP client APIs. """ import os from io import BytesIO from http import client as http_client import urllib -import six import allmydata # for __full_version__ from allmydata.util.encodingutil import quote_output @@ -44,7 +43,7 @@ class BadResponse(object): def do_http(method, url, body=b""): if isinstance(body, bytes): body = BytesIO(body) - elif isinstance(body, six.text_type): + elif isinstance(body, str): raise TypeError("do_http body must be a bytestring, not unicode") else: # We must give a Content-Length header to twisted.web, otherwise it @@ -87,16 +86,14 @@ def do_http(method, url, body=b""): def format_http_success(resp): - # ensure_text() shouldn't be necessary when Python 2 is dropped. return quote_output( - "%s %s" % (resp.status, six.ensure_text(resp.reason)), + "%s %s" % (resp.status, resp.reason), quotemarks=False) def format_http_error(msg, resp): - # ensure_text() shouldn't be necessary when Python 2 is dropped. return quote_output( - "%s: %s %s\n%s" % (msg, resp.status, six.ensure_text(resp.reason), - six.ensure_text(resp.read())), + "%s: %s %s\n%s" % (msg, resp.status, resp.reason, + resp.read()), quotemarks=False) def check_http_error(resp, stderr): From e4e6831497de5b653b04439425f7ebb5bce1d4d3 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 14 Apr 2023 10:55:40 -0400 Subject: [PATCH 1721/2309] Add logging to the rest of the immutable API operations. --- src/allmydata/storage/http_client.py | 49 +++++++++++++++++++++------- 1 file changed, 38 insertions(+), 11 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 8be2adbdc..1f477d1aa 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -886,21 +886,41 @@ class StorageClientImmutables(object): remaining.set(True, chunk["begin"], chunk["end"]) return UploadProgress(finished=finished, required=remaining) - def read_share_chunk( - self, storage_index, share_number, offset, length - ): # type: (bytes, int, int, int) -> Deferred[bytes] + @async_to_deferred + async def read_share_chunk( + self, storage_index: bytes, share_number: int, offset: int, length: int + ) -> bytes: """ Download a chunk of data from a share. """ - return read_share_chunk( - self._client, "immutable", storage_index, share_number, offset, length - ) + with start_action( + action_type="allmydata:storage:http-client:immutable:read-share-chunk", + storage_index=si_to_human_readable(storage_index), + share_number=share_number, + offset=offset, + length=length, + ) as ctx: + result = await read_share_chunk( + self._client, "immutable", storage_index, share_number, offset, length + ) + ctx.add_success_fields(data_len=len(result)) + return result @async_to_deferred async def list_shares(self, storage_index: bytes) -> Set[int]: """ Return the set of shares for a given storage index. """ + with start_action( + action_type="allmydata:storage:http-client:immutable:list-shares", + storage_index=si_to_human_readable(storage_index), + ) as ctx: + result = await self._list_shares(storage_index) + ctx.add_success_fields(shares=result) + return result + + async def _list_shares(self, storage_index: bytes) -> Set[int]: + """Implementation of ``list_shares()``.""" url = self._client.relative_url( "/storage/v1/immutable/{}/shares".format(_encode_si(storage_index)) ) @@ -917,16 +937,23 @@ class StorageClientImmutables(object): else: raise ClientException(response.code) - def advise_corrupt_share( + @async_to_deferred + async def advise_corrupt_share( self, storage_index: bytes, share_number: int, reason: str, - ): + ) -> None: """Indicate a share has been corrupted, with a human-readable message.""" - return advise_corrupt_share( - self._client, "immutable", storage_index, share_number, reason - ) + with start_action( + action_type="allmydata:storage:http-client:immutable:advise-corrupt-share", + storage_index=si_to_human_readable(storage_index), + share_number=share_number, + reason=reason, + ): + await advise_corrupt_share( + self._client, "immutable", storage_index, share_number, reason + ) @frozen From 2e06990c5c89355dc280a62ff7ea6b3796d2fdd4 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 14 Apr 2023 11:04:53 -0400 Subject: [PATCH 1722/2309] Remove bad assertion. --- src/allmydata/storage/common.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/allmydata/storage/common.py b/src/allmydata/storage/common.py index 89e29b081..f6d986f85 100644 --- a/src/allmydata/storage/common.py +++ b/src/allmydata/storage/common.py @@ -41,7 +41,6 @@ def si_a2b(ascii_storageindex): def si_to_human_readable(storageindex: bytes) -> str: """Create human-readable string of storage index.""" - assert len(storageindex) == 16 return str(base32.b2a(storageindex), "ascii") def storage_index_to_dir(storageindex): From 3395ee8fc59762197a3f446fe946085ce26827b9 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 14 Apr 2023 11:05:03 -0400 Subject: [PATCH 1723/2309] Add logging for mutable operations. --- src/allmydata/storage/http_client.py | 70 ++++++++++++++++++++++++---- 1 file changed, 60 insertions(+), 10 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 1f477d1aa..ec8cd4ade 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -1044,6 +1044,29 @@ class StorageClientMutables: Given a mapping between share numbers and test/write vectors, the tests are done and if they are valid the writes are done. """ + with start_action( + action_type="allmydata:storage:http-client:mutable:read-test-write", + storage_index=si_to_human_readable(storage_index), + ): + return await self._read_test_write_chunks( + storage_index, + write_enabler_secret, + lease_renew_secret, + lease_cancel_secret, + testwrite_vectors, + read_vector, + ) + + async def _read_test_write_chunks( + self, + storage_index: bytes, + write_enabler_secret: bytes, + lease_renew_secret: bytes, + lease_cancel_secret: bytes, + testwrite_vectors: dict[int, TestWriteVectors], + read_vector: list[ReadVector], + ) -> ReadTestWriteResult: + """Implementation of ``read_test_write_chunks()``.""" url = self._client.relative_url( "/storage/v1/mutable/{}/read-test-write".format(_encode_si(storage_index)) ) @@ -1073,25 +1096,45 @@ class StorageClientMutables: else: raise ClientException(response.code, (await response.content())) - def read_share_chunk( + @async_to_deferred + async def read_share_chunk( self, storage_index: bytes, share_number: int, offset: int, length: int, - ) -> Deferred[bytes]: + ) -> bytes: """ Download a chunk of data from a share. """ - return read_share_chunk( - self._client, "mutable", storage_index, share_number, offset, length - ) + with start_action( + action_type="allmydata:storage:http-client:mutable:read-share-chunk", + storage_index=si_to_human_readable(storage_index), + share_number=share_number, + offset=offset, + length=length, + ) as ctx: + result = await read_share_chunk( + self._client, "mutable", storage_index, share_number, offset, length + ) + ctx.add_success_fields(data_len=len(result)) + return result @async_to_deferred async def list_shares(self, storage_index: bytes) -> Set[int]: """ List the share numbers for a given storage index. """ + with start_action( + action_type="allmydata:storage:http-client:mutable:list-shares", + storage_index=si_to_human_readable(storage_index), + ) as ctx: + result = await self._list_shares(storage_index) + ctx.add_success_fields(shares=result) + return result + + async def _list_shares(self, storage_index: bytes) -> Set[int]: + """Implementation of ``list_shares()``.""" url = self._client.relative_url( "/storage/v1/mutable/{}/shares".format(_encode_si(storage_index)) ) @@ -1106,13 +1149,20 @@ class StorageClientMutables: else: raise ClientException(response.code) - def advise_corrupt_share( + @async_to_deferred + async def advise_corrupt_share( self, storage_index: bytes, share_number: int, reason: str, - ): + ) -> None: """Indicate a share has been corrupted, with a human-readable message.""" - return advise_corrupt_share( - self._client, "mutable", storage_index, share_number, reason - ) + with start_action( + action_type="allmydata:storage:http-client:mutable:advise-corrupt-share", + storage_index=si_to_human_readable(storage_index), + share_number=share_number, + reason=reason, + ): + await advise_corrupt_share( + self._client, "mutable", storage_index, share_number, reason + ) From 2d81ddc297b336607bc8eac691913e7e3ac2f178 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 14 Apr 2023 11:15:47 -0400 Subject: [PATCH 1724/2309] Don't call str() on bytes. --- src/allmydata/scripts/common_http.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/scripts/common_http.py b/src/allmydata/scripts/common_http.py index 7d627a7ad..a2cae5a85 100644 --- a/src/allmydata/scripts/common_http.py +++ b/src/allmydata/scripts/common_http.py @@ -92,7 +92,7 @@ def format_http_success(resp): def format_http_error(msg, resp): return quote_output( - "%s: %s %s\n%s" % (msg, resp.status, resp.reason, + "%s: %s %s\n%r" % (msg, resp.status, resp.reason, resp.read()), quotemarks=False) From 76ce54ea53e4e802da612f1f2cbc53c88e9764da Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 14 Apr 2023 13:23:28 -0600 Subject: [PATCH 1725/2309] remove debugging --- integration/util.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/integration/util.py b/integration/util.py index ac3fe2833..887602906 100644 --- a/integration/util.py +++ b/integration/util.py @@ -90,11 +90,9 @@ class _CollectOutputProtocol(ProcessProtocol): self.done.errback(reason) def outReceived(self, data): - print("OUT: {!r}".format(data)) self.output.write(data) def errReceived(self, data): - print("ERR: {!r}".format(data)) if self.capture_stderr: self.output.write(data) From abfca04af5a8c7e43fa18351b1131ebcf741efd8 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 14 Apr 2023 13:24:22 -0600 Subject: [PATCH 1726/2309] turn off i2p tests for now --- integration/test_i2p.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration/test_i2p.py b/integration/test_i2p.py index 96619a93a..597623d9c 100644 --- a/integration/test_i2p.py +++ b/integration/test_i2p.py @@ -133,7 +133,7 @@ def i2p_introducer_furl(i2p_introducer, temp_dir): @pytest_twisted.inlineCallbacks -def test_i2p_service_storage(reactor, request, temp_dir, flog_gatherer, i2p_network, i2p_introducer_furl): +def __test_i2p_service_storage(reactor, request, temp_dir, flog_gatherer, i2p_network, i2p_introducer_furl): yield _create_anonymous_node(reactor, 'carol_i2p', 8008, request, temp_dir, flog_gatherer, i2p_network, i2p_introducer_furl) yield _create_anonymous_node(reactor, 'dave_i2p', 8009, request, temp_dir, flog_gatherer, i2p_network, i2p_introducer_furl) # ensure both nodes are connected to "a grid" by uploading From d3c39f8604fd924bbac424ce201533b4656416b3 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 14 Apr 2023 15:27:19 -0600 Subject: [PATCH 1727/2309] fix i2p introducer, different ports --- integration/conftest.py | 2 +- integration/test_i2p.py | 13 ++++--------- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/integration/conftest.py b/integration/conftest.py index b54e18e26..f65c84141 100644 --- a/integration/conftest.py +++ b/integration/conftest.py @@ -227,7 +227,7 @@ def introducer(reactor, temp_dir, flog_gatherer, request): config = read_config(intro_dir, "tub.port") config.set_config("node", "nickname", "introducer-tor") - config.set_config("node", "web.port", "4561") + config.set_config("node", "web.port", "4562") config.set_config("node", "log_gatherer.furl", flog_gatherer) # "tahoe run" is consistent across Linux/macOS/Windows, unlike the old diff --git a/integration/test_i2p.py b/integration/test_i2p.py index 597623d9c..df619c6eb 100644 --- a/integration/test_i2p.py +++ b/integration/test_i2p.py @@ -68,13 +68,6 @@ def i2p_network(reactor, temp_dir, request): include_result=False, ) def i2p_introducer(reactor, temp_dir, flog_gatherer, request): - config = ''' -[node] -nickname = introducer_i2p -web.port = 4561 -log_gatherer.furl = {log_furl} -'''.format(log_furl=flog_gatherer) - intro_dir = join(temp_dir, 'introducer_i2p') print("making introducer", intro_dir) @@ -94,8 +87,10 @@ log_gatherer.furl = {log_furl} pytest_twisted.blockon(done_proto.done) # over-write the config file with our stuff - with open(join(intro_dir, 'tahoe.cfg'), 'w') as f: - f.write(config) + config = read_config(intro_dir, "tub.port") + config.set_config("node", "nickname", "introducer_i2p") + config.set_config("node", "web.port", "4563") + config.set_config("node", "log_gatherer.furl", flog_gatherer) # "tahoe run" is consistent across Linux/macOS/Windows, unlike the old # "start" command. From 34cee7ff73c05d978354894f734d710d3a5c1a2c Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 14 Apr 2023 15:44:52 -0600 Subject: [PATCH 1728/2309] missing import --- integration/test_i2p.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/integration/test_i2p.py b/integration/test_i2p.py index df619c6eb..10abb7e30 100644 --- a/integration/test_i2p.py +++ b/integration/test_i2p.py @@ -23,6 +23,8 @@ from twisted.internet.error import ProcessExitedAlready from allmydata.test.common import ( write_introducer, ) +from allmydata.node import read_config + if which("docker") is None: pytest.skip('Skipping I2P tests since Docker is unavailable', allow_module_level=True) From 3ccb7c4d1c1f26991a3a467fe4a559738eac0638 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 14 Apr 2023 15:45:17 -0600 Subject: [PATCH 1729/2309] re-enable i2p tests --- integration/test_i2p.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration/test_i2p.py b/integration/test_i2p.py index 10abb7e30..2aa1a536f 100644 --- a/integration/test_i2p.py +++ b/integration/test_i2p.py @@ -130,7 +130,7 @@ def i2p_introducer_furl(i2p_introducer, temp_dir): @pytest_twisted.inlineCallbacks -def __test_i2p_service_storage(reactor, request, temp_dir, flog_gatherer, i2p_network, i2p_introducer_furl): +def test_i2p_service_storage(reactor, request, temp_dir, flog_gatherer, i2p_network, i2p_introducer_furl): yield _create_anonymous_node(reactor, 'carol_i2p', 8008, request, temp_dir, flog_gatherer, i2p_network, i2p_introducer_furl) yield _create_anonymous_node(reactor, 'dave_i2p', 8009, request, temp_dir, flog_gatherer, i2p_network, i2p_introducer_furl) # ensure both nodes are connected to "a grid" by uploading From 8b81bd7ebef79617d75ab1d5d5745d50a3e212b9 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 14 Apr 2023 16:33:52 -0600 Subject: [PATCH 1730/2309] remove more debug --- integration/util.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/integration/util.py b/integration/util.py index 887602906..177983e2e 100644 --- a/integration/util.py +++ b/integration/util.py @@ -138,7 +138,6 @@ class _MagicTextProtocol(ProcessProtocol): self.exited.callback(None) def outReceived(self, data): - print("OUT", data) data = str(data, sys.stdout.encoding) sys.stdout.write(data) self._output.write(data) @@ -147,7 +146,6 @@ class _MagicTextProtocol(ProcessProtocol): self.magic_seen.callback(self) def errReceived(self, data): - print("ERR", data) data = str(data, sys.stderr.encoding) sys.stdout.write(data) From 8652bb71ad0ece21f9f75a1c290ccdb4abed6e92 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 14 Apr 2023 17:05:57 -0600 Subject: [PATCH 1731/2309] skip i2p tests again? --- integration/test_i2p.py | 1 + 1 file changed, 1 insertion(+) diff --git a/integration/test_i2p.py b/integration/test_i2p.py index 2aa1a536f..42b848130 100644 --- a/integration/test_i2p.py +++ b/integration/test_i2p.py @@ -130,6 +130,7 @@ def i2p_introducer_furl(i2p_introducer, temp_dir): @pytest_twisted.inlineCallbacks +@pytest.skip("I2P tests are not functioning at all, for unknown reasons") def test_i2p_service_storage(reactor, request, temp_dir, flog_gatherer, i2p_network, i2p_introducer_furl): yield _create_anonymous_node(reactor, 'carol_i2p', 8008, request, temp_dir, flog_gatherer, i2p_network, i2p_introducer_furl) yield _create_anonymous_node(reactor, 'dave_i2p', 8009, request, temp_dir, flog_gatherer, i2p_network, i2p_introducer_furl) From b5f6fa8933c03d5de2069de47d67230ae3d641f2 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 14 Apr 2023 19:07:27 -0600 Subject: [PATCH 1732/2309] skip properly --- integration/test_i2p.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration/test_i2p.py b/integration/test_i2p.py index 42b848130..a94648593 100644 --- a/integration/test_i2p.py +++ b/integration/test_i2p.py @@ -130,7 +130,7 @@ def i2p_introducer_furl(i2p_introducer, temp_dir): @pytest_twisted.inlineCallbacks -@pytest.skip("I2P tests are not functioning at all, for unknown reasons") +@pytest.mark.skip("I2P tests are not functioning at all, for unknown reasons") def test_i2p_service_storage(reactor, request, temp_dir, flog_gatherer, i2p_network, i2p_introducer_furl): yield _create_anonymous_node(reactor, 'carol_i2p', 8008, request, temp_dir, flog_gatherer, i2p_network, i2p_introducer_furl) yield _create_anonymous_node(reactor, 'dave_i2p', 8009, request, temp_dir, flog_gatherer, i2p_network, i2p_introducer_furl) From bed2d33427c449da15403b758d7730f81cce9ac4 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 17 Apr 2023 10:01:26 -0400 Subject: [PATCH 1733/2309] Fix lint. --- src/allmydata/storage/http_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index ec8cd4ade..071b9bdb1 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -181,7 +181,7 @@ def limited_content( with start_action( action_type="allmydata:storage:http-client:limited-content", max_length=max_length, - ).context() as action: + ).context(): d = DeferredContext(d) # Make really sure everything gets called in Deferred context, treq might From cda97e4fa63deac934e3085e8bc0edb0a30b85da Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 17 Apr 2023 10:06:50 -0400 Subject: [PATCH 1734/2309] Remove pylint, replacing with faster alternative. --- tox.ini | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/tox.ini b/tox.ini index 447745784..982157bf1 100644 --- a/tox.ini +++ b/tox.ini @@ -100,9 +100,7 @@ commands = [testenv:codechecks] basepython = python3 deps = - # Make sure we get a version of PyLint that respects config, and isn't too - # old. - pylint < 2.18, >2.14 + ruff # On macOS, git inside of towncrier needs $HOME. passenv = HOME setenv = @@ -114,9 +112,10 @@ commands = python misc/coding_tools/check-umids.py {posargs:{env:DEFAULT_FILES}} python misc/coding_tools/check-debugging.py {posargs:{env:DEFAULT_FILES}} python misc/coding_tools/find-trailing-spaces.py -r {posargs:{env:DEFAULT_FILES}} - # PyLint has other useful checks, might want to enable them: - # http://pylint.pycqa.org/en/latest/technical_reference/features.html - pylint --disable=all --enable=cell-var-from-loop {posargs:{env:DEFAULT_FILES}} + # B023: Find loop variables that aren't bound in a loop, equivalent of pylint + # cell-var-from-loop. + # ruff could probably replace flake8 and perhaps above tools as well... + ruff check --select=B023 {posargs:{env:DEFAULT_FILES}} # If towncrier.check fails, you forgot to add a towncrier news # fragment explaining the change in this branch. Create one at From aafbb00333312e6fc77dcb9558a8b5c9cad75b9e Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 17 Apr 2023 10:10:09 -0400 Subject: [PATCH 1735/2309] Use ruff for trailing whitespace. --- misc/coding_tools/find-trailing-spaces.py | 44 ----------------------- newsfragments/4014.minor | 0 tox.ini | 4 +-- 3 files changed, 2 insertions(+), 46 deletions(-) delete mode 100644 misc/coding_tools/find-trailing-spaces.py create mode 100644 newsfragments/4014.minor diff --git a/misc/coding_tools/find-trailing-spaces.py b/misc/coding_tools/find-trailing-spaces.py deleted file mode 100644 index 19e7e3c28..000000000 --- a/misc/coding_tools/find-trailing-spaces.py +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/env python - -from __future__ import print_function - -import os, sys - -from twisted.python import usage - -class Options(usage.Options): - optFlags = [ - ("recursive", "r", "Search for .py files recursively"), - ] - def parseArgs(self, *starting_points): - self.starting_points = starting_points - -found = [False] - -def check(fn): - f = open(fn, "r") - for i,line in enumerate(f.readlines()): - if line == "\n": - continue - if line[-1] == "\n": - line = line[:-1] - if line.rstrip() != line: - # the %s:%d:%d: lets emacs' compile-mode jump to those locations - print("%s:%d:%d: trailing whitespace" % (fn, i+1, len(line)+1)) - found[0] = True - f.close() - -o = Options() -o.parseOptions() -if o['recursive']: - for starting_point in o.starting_points: - for root, dirs, files in os.walk(starting_point): - for fn in [f for f in files if f.endswith(".py")]: - fn = os.path.join(root, fn) - check(fn) -else: - for fn in o.starting_points: - check(fn) -if found[0]: - sys.exit(1) -sys.exit(0) diff --git a/newsfragments/4014.minor b/newsfragments/4014.minor new file mode 100644 index 000000000..e69de29bb diff --git a/tox.ini b/tox.ini index 982157bf1..a191d6078 100644 --- a/tox.ini +++ b/tox.ini @@ -111,11 +111,11 @@ commands = flake8 {posargs:{env:DEFAULT_FILES}} python misc/coding_tools/check-umids.py {posargs:{env:DEFAULT_FILES}} python misc/coding_tools/check-debugging.py {posargs:{env:DEFAULT_FILES}} - python misc/coding_tools/find-trailing-spaces.py -r {posargs:{env:DEFAULT_FILES}} # B023: Find loop variables that aren't bound in a loop, equivalent of pylint # cell-var-from-loop. + # W291,W293: Trailing whitespace. # ruff could probably replace flake8 and perhaps above tools as well... - ruff check --select=B023 {posargs:{env:DEFAULT_FILES}} + ruff check --select=B023,W291,W293 {posargs:{env:DEFAULT_FILES}} # If towncrier.check fails, you forgot to add a towncrier news # fragment explaining the change in this branch. Create one at From 7b33931df2982bd35e0cdb38d0a1e1d77d34ce47 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 17 Apr 2023 10:21:20 -0400 Subject: [PATCH 1736/2309] Replace flake8 with ruff. --- .gitignore | 2 ++ .ruff.toml | 12 ++++++++++++ setup.cfg | 3 +++ setup.py | 3 +-- tox.ini | 7 +------ 5 files changed, 19 insertions(+), 8 deletions(-) create mode 100644 .ruff.toml diff --git a/.gitignore b/.gitignore index 7c7fa2afd..0cf688c54 100644 --- a/.gitignore +++ b/.gitignore @@ -53,3 +53,5 @@ zope.interface-*.egg # This is the plaintext of the private environment needed for some CircleCI # operations. It's never supposed to be checked in. secret-env-plain + +.ruff_cache \ No newline at end of file diff --git a/.ruff.toml b/.ruff.toml new file mode 100644 index 000000000..75ff62c2d --- /dev/null +++ b/.ruff.toml @@ -0,0 +1,12 @@ +select = [ + # Pyflakes checks + "F", + # Prohibit tabs: + "W191", + # No trailing whitespace: + "W291", + "W293", + # Make sure we bind closure variables in a loop (equivalent to pylint + # cell-var-from-loop): + "B023", +] \ No newline at end of file diff --git a/setup.cfg b/setup.cfg index f4539279e..9415b3ab4 100644 --- a/setup.cfg +++ b/setup.cfg @@ -6,6 +6,9 @@ develop = update_version develop bdist_egg = update_version bdist_egg bdist_wheel = update_version bdist_wheel +# This has been replaced by ruff (see .ruff.toml), which has same checks as +# flake8 plus many more, and is also faster. However, we're keeping this config +# in case people still use flake8 in IDEs, etc.. [flake8] # Enforce all pyflakes constraints, and also prohibit tabs for indentation. # Reference: diff --git a/setup.py b/setup.py index 854a333f1..3358aa6c6 100644 --- a/setup.py +++ b/setup.py @@ -399,12 +399,11 @@ setup(name="tahoe-lafs", # also set in __init__.py "gpg", ], "test": [ - "flake8", # Pin a specific pyflakes so we don't have different folks # disagreeing on what is or is not a lint issue. We can bump # this version from time to time, but we will do it # intentionally. - "pyflakes == 3.0.1", + "ruff==0.0.261", "coverage ~= 5.0", "mock", "tox ~= 3.0", diff --git a/tox.ini b/tox.ini index a191d6078..2daf8dca3 100644 --- a/tox.ini +++ b/tox.ini @@ -108,14 +108,9 @@ setenv = # entire codebase, including various pieces of supporting code. DEFAULT_FILES=src integration static misc setup.py commands = - flake8 {posargs:{env:DEFAULT_FILES}} + ruff check {posargs:{env:DEFAULT_FILES}} python misc/coding_tools/check-umids.py {posargs:{env:DEFAULT_FILES}} python misc/coding_tools/check-debugging.py {posargs:{env:DEFAULT_FILES}} - # B023: Find loop variables that aren't bound in a loop, equivalent of pylint - # cell-var-from-loop. - # W291,W293: Trailing whitespace. - # ruff could probably replace flake8 and perhaps above tools as well... - ruff check --select=B023,W291,W293 {posargs:{env:DEFAULT_FILES}} # If towncrier.check fails, you forgot to add a towncrier news # fragment explaining the change in this branch. Create one at From 6517cd4a48338bc13d991299d361e8c5b65fed22 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 17 Apr 2023 10:22:27 -0400 Subject: [PATCH 1737/2309] Fix lint found by ruff. --- misc/checkers/check_load.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/misc/checkers/check_load.py b/misc/checkers/check_load.py index d509b89ae..01a9ed832 100644 --- a/misc/checkers/check_load.py +++ b/misc/checkers/check_load.py @@ -1,5 +1,3 @@ -from __future__ import print_function - """ this is a load-generating client program. It does all of its work through a given tahoe node (specified by URL), and performs random reads and writes From c05afb19dfd7ffb5f053aa2a52972403ccb4fa43 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 17 Apr 2023 10:33:31 -0400 Subject: [PATCH 1738/2309] Don't install code, it's not necessary. --- tox.ini | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tox.ini b/tox.ini index 2daf8dca3..99487bc83 100644 --- a/tox.ini +++ b/tox.ini @@ -99,8 +99,10 @@ commands = [testenv:codechecks] basepython = python3 +skip_install = true deps = ruff + towncrier # On macOS, git inside of towncrier needs $HOME. passenv = HOME setenv = From ce93a7b869ceb29b49d985125e80a80fa19dad98 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 17 Apr 2023 11:29:43 -0400 Subject: [PATCH 1739/2309] News fragment. --- newsfragments/4015.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/4015.minor diff --git a/newsfragments/4015.minor b/newsfragments/4015.minor new file mode 100644 index 000000000..e69de29bb From 5da5a82a8cc36399c7dbc228109afee2a17ff21e Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 17 Apr 2023 12:02:04 -0400 Subject: [PATCH 1740/2309] Get rid of default mutable arguments. --- .ruff.toml | 2 ++ src/allmydata/client.py | 6 +++--- src/allmydata/dirnode.py | 4 +++- src/allmydata/hashtree.py | 7 +++++-- src/allmydata/immutable/upload.py | 4 +++- src/allmydata/interfaces.py | 6 +++--- src/allmydata/node.py | 12 ++++++++---- src/allmydata/nodemaker.py | 5 +++-- src/allmydata/test/cli/wormholetesting.py | 3 ++- src/allmydata/test/no_network.py | 4 +++- src/allmydata/test/test_system.py | 21 +++++++++------------ src/allmydata/test/web/test_web.py | 12 +++++++++--- src/allmydata/util/dbutil.py | 4 +++- 13 files changed, 56 insertions(+), 34 deletions(-) diff --git a/.ruff.toml b/.ruff.toml index 75ff62c2d..516255d2a 100644 --- a/.ruff.toml +++ b/.ruff.toml @@ -9,4 +9,6 @@ select = [ # Make sure we bind closure variables in a loop (equivalent to pylint # cell-var-from-loop): "B023", + # Don't use mutable default arguments: + "B006", ] \ No newline at end of file diff --git a/src/allmydata/client.py b/src/allmydata/client.py index 8a10fe9e7..1d959cb98 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -7,7 +7,7 @@ import os import stat import time import weakref -from typing import Optional +from typing import Optional, Iterable from base64 import urlsafe_b64encode from functools import partial # On Python 2 this will be the backported package: @@ -189,7 +189,7 @@ class Terminator(service.Service): return service.Service.stopService(self) -def read_config(basedir, portnumfile, generated_files=[]): +def read_config(basedir, portnumfile, generated_files: Iterable=()): """ Read and validate configuration for a client-style Node. See :method:`allmydata.node.read_config` for parameter meanings (the @@ -1103,7 +1103,7 @@ class _Client(node.Node, pollmixin.PollMixin): # may get an opaque node if there were any problems. return self.nodemaker.create_from_cap(write_uri, read_uri, deep_immutable=deep_immutable, name=name) - def create_dirnode(self, initial_children={}, version=None): + def create_dirnode(self, initial_children=None, version=None): d = self.nodemaker.create_new_mutable_directory(initial_children, version=version) return d diff --git a/src/allmydata/dirnode.py b/src/allmydata/dirnode.py index fdf373b45..ccd045b05 100644 --- a/src/allmydata/dirnode.py +++ b/src/allmydata/dirnode.py @@ -678,8 +678,10 @@ class DirectoryNode(object): return d # XXX: Too many arguments? Worthwhile to break into mutable/immutable? - def create_subdirectory(self, namex, initial_children={}, overwrite=True, + def create_subdirectory(self, namex, initial_children=None, overwrite=True, mutable=True, mutable_version=None, metadata=None): + if initial_children is None: + initial_children = {} name = normalize(namex) if self.is_readonly(): return defer.fail(NotWriteableError()) diff --git a/src/allmydata/hashtree.py b/src/allmydata/hashtree.py index 17467459b..57bdbd9a1 100644 --- a/src/allmydata/hashtree.py +++ b/src/allmydata/hashtree.py @@ -332,7 +332,7 @@ class IncompleteHashTree(CompleteBinaryTreeMixin, list): name += " (leaf [%d] of %d)" % (leafnum, numleaves) return name - def set_hashes(self, hashes={}, leaves={}): + def set_hashes(self, hashes=None, leaves=None): """Add a bunch of hashes to the tree. I will validate these to the best of my ability. If I already have a @@ -382,7 +382,10 @@ class IncompleteHashTree(CompleteBinaryTreeMixin, list): corrupted or one of the received hashes was corrupted. If it raises NotEnoughHashesError, then the otherhashes dictionary was incomplete. """ - + if hashes is None: + hashes = {} + if leaves is None: + leaves = {} assert isinstance(hashes, dict) for h in hashes.values(): assert isinstance(h, bytes) diff --git a/src/allmydata/immutable/upload.py b/src/allmydata/immutable/upload.py index 0421de4e0..a331cc5db 100644 --- a/src/allmydata/immutable/upload.py +++ b/src/allmydata/immutable/upload.py @@ -1391,7 +1391,9 @@ class CHKUploader(object): def get_upload_status(self): return self._upload_status -def read_this_many_bytes(uploadable, size, prepend_data=[]): +def read_this_many_bytes(uploadable, size, prepend_data=None): + if prepend_data is None: + prepend_data = [] if size == 0: return defer.succeed([]) d = uploadable.read(size) diff --git a/src/allmydata/interfaces.py b/src/allmydata/interfaces.py index 467d0d450..201ab082e 100644 --- a/src/allmydata/interfaces.py +++ b/src/allmydata/interfaces.py @@ -1447,7 +1447,7 @@ class IDirectoryNode(IFilesystemNode): is a file, or if must_be_file is True and the child is a directory, I raise ChildOfWrongTypeError.""" - def create_subdirectory(name, initial_children={}, overwrite=True, + def create_subdirectory(name, initial_children=None, overwrite=True, mutable=True, mutable_version=None, metadata=None): """I create and attach a directory at the given name. The new directory can be empty, or it can be populated with children @@ -2586,7 +2586,7 @@ class IClient(Interface): @return: a Deferred that fires with an IMutableFileNode instance. """ - def create_dirnode(initial_children={}): + def create_dirnode(initial_children=None): """Create a new unattached dirnode, possibly with initial children. @param initial_children: dict with keys that are unicode child names, @@ -2641,7 +2641,7 @@ class INodeMaker(Interface): for use by unit tests, to create mutable files that are smaller than usual.""" - def create_new_mutable_directory(initial_children={}): + def create_new_mutable_directory(initial_children=None): """I create a new mutable directory, and return a Deferred that will fire with the IDirectoryNode instance when it is ready. If initial_children= is provided (a dict mapping unicode child name to diff --git a/src/allmydata/node.py b/src/allmydata/node.py index 58ee33ef5..6c3082b50 100644 --- a/src/allmydata/node.py +++ b/src/allmydata/node.py @@ -17,7 +17,7 @@ import errno from base64 import b32decode, b32encode from errno import ENOENT, EPERM from warnings import warn -from typing import Union +from typing import Union, Iterable import attr @@ -172,7 +172,7 @@ def create_node_dir(basedir, readme_text): f.write(readme_text) -def read_config(basedir, portnumfile, generated_files=[], _valid_config=None): +def read_config(basedir, portnumfile, generated_files: Iterable = (), _valid_config=None): """ Read and validate configuration. @@ -741,7 +741,7 @@ def create_connection_handlers(config, i2p_provider, tor_provider): def create_tub(tub_options, default_connection_handlers, foolscap_connection_handlers, - handler_overrides={}, force_foolscap=False, **kwargs): + handler_overrides=None, force_foolscap=False, **kwargs): """ Create a Tub with the right options and handlers. It will be ephemeral unless the caller provides certFile= in kwargs @@ -755,6 +755,8 @@ def create_tub(tub_options, default_connection_handlers, foolscap_connection_han :param bool force_foolscap: If True, only allow Foolscap, not just HTTPS storage protocol. """ + if handler_overrides is None: + handler_overrides = {} # We listen simultaneously for both Foolscap and HTTPS on the same port, # so we have to create a special Foolscap Tub for that to work: if force_foolscap: @@ -922,7 +924,7 @@ def tub_listen_on(i2p_provider, tor_provider, tub, tubport, location): def create_main_tub(config, tub_options, default_connection_handlers, foolscap_connection_handlers, i2p_provider, tor_provider, - handler_overrides={}, cert_filename="node.pem"): + handler_overrides=None, cert_filename="node.pem"): """ Creates a 'main' Foolscap Tub, typically for use as the top-level access point for a running Node. @@ -943,6 +945,8 @@ def create_main_tub(config, tub_options, :param tor_provider: None, or a _Provider instance if txtorcon + Tor are installed. """ + if handler_overrides is None: + handler_overrides = {} portlocation = _tub_portlocation( config, iputil.get_local_addresses_sync, diff --git a/src/allmydata/nodemaker.py b/src/allmydata/nodemaker.py index 1b7ea5f45..39663bda9 100644 --- a/src/allmydata/nodemaker.py +++ b/src/allmydata/nodemaker.py @@ -135,8 +135,9 @@ class NodeMaker(object): d.addCallback(lambda res: n) return d - def create_new_mutable_directory(self, initial_children={}, version=None): - # initial_children must have metadata (i.e. {} instead of None) + def create_new_mutable_directory(self, initial_children=None, version=None): + if initial_children is None: + initial_children = {} for (name, (node, metadata)) in initial_children.items(): precondition(isinstance(metadata, dict), "create_new_mutable_directory requires metadata to be a dict, not None", metadata) diff --git a/src/allmydata/test/cli/wormholetesting.py b/src/allmydata/test/cli/wormholetesting.py index d1a3bfd07..647798bc8 100644 --- a/src/allmydata/test/cli/wormholetesting.py +++ b/src/allmydata/test/cli/wormholetesting.py @@ -70,7 +70,8 @@ class MemoryWormholeServer(object): appid: str, relay_url: str, reactor: Any, - versions: Any={}, + # Unfortunately we need a mutable default to match the real API + versions: Any={}, # noqa: B006 delegate: Optional[Any]=None, journal: Optional[Any]=None, tor: Optional[Any]=None, diff --git a/src/allmydata/test/no_network.py b/src/allmydata/test/no_network.py index ee1f48b17..e3b57fb95 100644 --- a/src/allmydata/test/no_network.py +++ b/src/allmydata/test/no_network.py @@ -476,7 +476,7 @@ class GridTestMixin(object): ]) def set_up_grid(self, num_clients=1, num_servers=10, - client_config_hooks={}, oneshare=False): + client_config_hooks=None, oneshare=False): """ Create a Tahoe-LAFS storage grid. @@ -489,6 +489,8 @@ class GridTestMixin(object): :return: ``None`` """ + if client_config_hooks is None: + client_config_hooks = {} # self.basedir must be set port_assigner = SameProcessStreamEndpointAssigner() port_assigner.setUp() diff --git a/src/allmydata/test/test_system.py b/src/allmydata/test/test_system.py index d11a6e866..b3287bf3b 100644 --- a/src/allmydata/test/test_system.py +++ b/src/allmydata/test/test_system.py @@ -1,20 +1,13 @@ """ Ported to Python 3. """ -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division -from __future__ import unicode_literals - -from future.utils import PY2 -if PY2: - # Don't import bytes since it causes issues on (so far unported) modules on Python 2. - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, dict, list, object, range, max, min, str # noqa: F401 +from __future__ import annotations from past.builtins import chr as byteschr, long from six import ensure_text import os, re, sys, time, json +from typing import Optional from bs4 import BeautifulSoup @@ -56,10 +49,12 @@ from .common_util import run_cli_unicode class RunBinTahoeMixin(object): - def run_bintahoe(self, args, stdin=None, python_options=[], env=None): + def run_bintahoe(self, args, stdin=None, python_options:Optional[list[str]]=None, env=None): # test_runner.run_bintahoe has better unicode support but doesn't # support env yet and is also synchronous. If we could get rid of # this in favor of that, though, it would probably be an improvement. + if python_options is None: + python_options = [] command = sys.executable argv = python_options + ["-b", "-m", "allmydata.scripts.runner"] + args @@ -1088,7 +1083,9 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase): headers["content-type"] = "multipart/form-data; boundary=%s" % str(sepbase, "ascii") return self.POST2(urlpath, body, headers, use_helper) - def POST2(self, urlpath, body=b"", headers={}, use_helper=False): + def POST2(self, urlpath, body=b"", headers=None, use_helper=False): + if headers is None: + headers = {} if use_helper: url = self.helper_webish_url + urlpath else: @@ -1409,7 +1406,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase): rc,out,err = yield run_cli(verb, *args, nodeargs=nodeargs, **kwargs) defer.returnValue((out,err)) - def _check_ls(out_and_err, expected_children, unexpected_children=[]): + def _check_ls(out_and_err, expected_children, unexpected_children=()): (out, err) = out_and_err self.failUnlessEqual(err, "") for s in expected_children: diff --git a/src/allmydata/test/web/test_web.py b/src/allmydata/test/web/test_web.py index 4c828817a..08dce0ac0 100644 --- a/src/allmydata/test/web/test_web.py +++ b/src/allmydata/test/web/test_web.py @@ -565,7 +565,9 @@ class WebMixin(TimezoneMixin): returnValue(data) @inlineCallbacks - def HEAD(self, urlpath, return_response=False, headers={}): + def HEAD(self, urlpath, return_response=False, headers=None): + if headers is None: + headers = {} url = self.webish_url + urlpath response = yield treq.request("head", url, persistent=False, headers=headers) @@ -573,7 +575,9 @@ class WebMixin(TimezoneMixin): raise Error(response.code, response="") returnValue( ("", response.code, response.headers) ) - def PUT(self, urlpath, data, headers={}): + def PUT(self, urlpath, data, headers=None): + if headers is None: + headers = {} url = self.webish_url + urlpath return do_http("put", url, data=data, headers=headers) @@ -618,7 +622,9 @@ class WebMixin(TimezoneMixin): body, headers = self.build_form(**fields) return self.POST2(urlpath, body, headers) - def POST2(self, urlpath, body="", headers={}, followRedirect=False): + def POST2(self, urlpath, body="", headers=None, followRedirect=False): + if headers is None: + headers = {} url = self.webish_url + urlpath if isinstance(body, str): body = body.encode("utf-8") diff --git a/src/allmydata/util/dbutil.py b/src/allmydata/util/dbutil.py index 916382972..45e59cf00 100644 --- a/src/allmydata/util/dbutil.py +++ b/src/allmydata/util/dbutil.py @@ -25,7 +25,7 @@ class DBError(Exception): def get_db(dbfile, stderr=sys.stderr, - create_version=(None, None), updaters={}, just_create=False, dbname="db", + create_version=(None, None), updaters=None, just_create=False, dbname="db", ): """Open or create the given db file. The parent directory must exist. create_version=(SCHEMA, VERNUM), and SCHEMA must have a 'version' table. @@ -33,6 +33,8 @@ def get_db(dbfile, stderr=sys.stderr, to get from ver=1 to ver=2. Returns a (sqlite3,db) tuple, or raises DBError. """ + if updaters is None: + updaters = {} must_create = not os.path.exists(dbfile) try: db = sqlite3.connect(dbfile) From 2a4dcb7a27cd0fd0592a9002e13c644a07ea8420 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 17 Apr 2023 13:08:26 -0400 Subject: [PATCH 1741/2309] More checks that are probably useful (doesn't trigger anything at the moment). --- .ruff.toml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.ruff.toml b/.ruff.toml index 516255d2a..2dd6b59b5 100644 --- a/.ruff.toml +++ b/.ruff.toml @@ -9,6 +9,10 @@ select = [ # Make sure we bind closure variables in a loop (equivalent to pylint # cell-var-from-loop): "B023", + # Don't silence exceptions in finally by accident: + "B012", # Don't use mutable default arguments: "B006", + # Errors from PyLint: + "PLE", ] \ No newline at end of file From 1371ffe9dc7e6a6b0346daad1603f6414bbd1fc7 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 25 Apr 2023 08:14:26 -0400 Subject: [PATCH 1742/2309] Just have ruff in one place. --- setup.py | 5 ----- tox.ini | 4 +++- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/setup.py b/setup.py index 3358aa6c6..2418c6dbe 100644 --- a/setup.py +++ b/setup.py @@ -399,11 +399,6 @@ setup(name="tahoe-lafs", # also set in __init__.py "gpg", ], "test": [ - # Pin a specific pyflakes so we don't have different folks - # disagreeing on what is or is not a lint issue. We can bump - # this version from time to time, but we will do it - # intentionally. - "ruff==0.0.261", "coverage ~= 5.0", "mock", "tox ~= 3.0", diff --git a/tox.ini b/tox.ini index 99487bc83..5f18b6b95 100644 --- a/tox.ini +++ b/tox.ini @@ -101,7 +101,9 @@ commands = basepython = python3 skip_install = true deps = - ruff + # Pin a specific version so we get consistent outcomes; update this + # occasionally: + ruff == 0.0.263 towncrier # On macOS, git inside of towncrier needs $HOME. passenv = HOME From ebed5100b9cf2a208875eb2977da23364559881d Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 25 Apr 2023 08:16:12 -0400 Subject: [PATCH 1743/2309] Switch to longer timeout so it's unlikely to impact users. --- newsfragments/4012.bugfix | 2 +- src/allmydata/scripts/common_http.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/newsfragments/4012.bugfix b/newsfragments/4012.bugfix index 97dfe6aad..24d5bb49a 100644 --- a/newsfragments/4012.bugfix +++ b/newsfragments/4012.bugfix @@ -1 +1 @@ -The command-line tools now have a 60-second timeout on individual network reads/writes/connects; previously they could block forever in some situations. \ No newline at end of file +The command-line tools now have a 300-second timeout on individual network reads/writes/connects; previously they could block forever in some situations. \ No newline at end of file diff --git a/src/allmydata/scripts/common_http.py b/src/allmydata/scripts/common_http.py index a2cae5a85..46676b3f5 100644 --- a/src/allmydata/scripts/common_http.py +++ b/src/allmydata/scripts/common_http.py @@ -54,9 +54,9 @@ def do_http(method, url, body=b""): assert body.read scheme, host, port, path = parse_url(url) if scheme == "http": - c = http_client.HTTPConnection(host, port, timeout=60, blocksize=65536) + c = http_client.HTTPConnection(host, port, timeout=300, blocksize=65536) elif scheme == "https": - c = http_client.HTTPSConnection(host, port, timeout=60, blocksize=65536) + c = http_client.HTTPSConnection(host, port, timeout=300, blocksize=65536) else: raise ValueError("unknown scheme '%s', need http or https" % scheme) c.putrequest(method, path) From 558e3bf79785fef5a8c3525f323590b9c7c15e36 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 25 Apr 2023 08:46:57 -0400 Subject: [PATCH 1744/2309] Fix unnecessary conversion. --- src/allmydata/storage/http_client.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index fc165bedd..f786b8f30 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -807,11 +807,10 @@ class StorageClientImmutables(object): url, ) if response.code == http.OK: - body = cast( + return cast( Set[int], await self._client.decode_cbor(response, _SCHEMAS["list_shares"]), ) - return set(body) else: raise ClientException(response.code) From 3d2e4d0798b874c493dc6b86487b527e36aa3324 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 25 Apr 2023 09:26:58 -0400 Subject: [PATCH 1745/2309] note about port selection --- integration/conftest.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/integration/conftest.py b/integration/conftest.py index f65c84141..f670c6486 100644 --- a/integration/conftest.py +++ b/integration/conftest.py @@ -289,6 +289,8 @@ def tor_introducer(reactor, temp_dir, flog_gatherer, request): request, ( 'create-introducer', + # The control port should agree with the configuration of the + # Tor network we bootstrap with chutney. '--tor-control-port', 'tcp:localhost:8007', '--hide-ip', '--listen=tor', From c595eea33e78eac579b3f3b63163a0354348a0d6 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 25 Apr 2023 09:27:51 -0400 Subject: [PATCH 1746/2309] always set the "start time" timeout in both the "we installed it ourselves" and the "we found an existing installation" cases. --- integration/conftest.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/integration/conftest.py b/integration/conftest.py index f670c6486..eaf740190 100644 --- a/integration/conftest.py +++ b/integration/conftest.py @@ -511,7 +511,6 @@ def chutney(reactor, temp_dir: str) -> tuple[str, dict[str, str]]: chutney_dir, { "PYTHONPATH": join(chutney_dir, "lib"), - "CHUTNEY_START_TIME": "600", # default is 60 } ) @@ -534,6 +533,10 @@ def tor_network(reactor, temp_dir, chutney, request): env = environ.copy() env.update(chutney_env) + env.update({ + # default is 60, probably too short for reliable automated use. + "CHUTNEY_START_TIME": "600", + }) chutney_argv = (sys.executable, '-m', 'chutney.TorNet') def chutney(argv): proto = _DumpOutputProtocol(None) From ba387453cf95ff04a322d8ba4531d81a5f31d7b2 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 25 Apr 2023 09:30:53 -0400 Subject: [PATCH 1747/2309] it's a bug fix! it's user-facing! --- newsfragments/3999.bugfix | 1 + newsfragments/3999.minor | 0 2 files changed, 1 insertion(+) create mode 100644 newsfragments/3999.bugfix delete mode 100644 newsfragments/3999.minor diff --git a/newsfragments/3999.bugfix b/newsfragments/3999.bugfix new file mode 100644 index 000000000..a8a8396f4 --- /dev/null +++ b/newsfragments/3999.bugfix @@ -0,0 +1 @@ +A bug where Introducer nodes configured to listen on Tor or I2P would not actually do so has been fixed. \ No newline at end of file diff --git a/newsfragments/3999.minor b/newsfragments/3999.minor deleted file mode 100644 index e69de29bb..000000000 From 825bcf3f3b8ca5a924d8c3075db653f9e5bf3c99 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 25 Apr 2023 09:31:04 -0400 Subject: [PATCH 1748/2309] revert reformatting --- integration/conftest.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/integration/conftest.py b/integration/conftest.py index eaf740190..b0d8da90f 100644 --- a/integration/conftest.py +++ b/integration/conftest.py @@ -507,12 +507,7 @@ def chutney(reactor, temp_dir: str) -> tuple[str, dict[str, str]]: ) pytest_twisted.blockon(proto.done) - return ( - chutney_dir, - { - "PYTHONPATH": join(chutney_dir, "lib"), - } - ) + return (chutney_dir, {"PYTHONPATH": join(chutney_dir, "lib")}) @pytest.fixture(scope='session') From fbb5f4c359800e606cc3d29d39d80896292e4c40 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 25 Apr 2023 09:31:10 -0400 Subject: [PATCH 1749/2309] slightly clarified comment --- integration/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration/conftest.py b/integration/conftest.py index b0d8da90f..cb590ef6f 100644 --- a/integration/conftest.py +++ b/integration/conftest.py @@ -547,7 +547,7 @@ def tor_network(reactor, temp_dir, chutney, request): # now, as per Chutney's README, we have to create the network pytest_twisted.blockon(chutney(("configure", basic_network))) - # ensure we will tear down the network right before we start it + # before we start the network, ensure we will tear down at the end def cleanup(): print("Tearing down Chutney Tor network") try: From f9a1eedaeadb818315bef8158902a47c863bbc65 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 25 Apr 2023 12:31:37 -0400 Subject: [PATCH 1750/2309] Make timeout optional, enable it only for integration tests. --- integration/conftest.py | 6 ++++++ newsfragments/4012.bugfix | 1 - newsfragments/4012.minor | 0 src/allmydata/scripts/common_http.py | 11 +++++++++-- 4 files changed, 15 insertions(+), 3 deletions(-) delete mode 100644 newsfragments/4012.bugfix create mode 100644 newsfragments/4012.minor diff --git a/integration/conftest.py b/integration/conftest.py index 879649588..d76b2a9c7 100644 --- a/integration/conftest.py +++ b/integration/conftest.py @@ -4,6 +4,7 @@ Ported to Python 3. from __future__ import annotations +import os import sys import shutil from time import sleep @@ -49,6 +50,11 @@ from .util import ( ) +# No reason for HTTP requests to take longer than two minutes in the +# integration tests. See allmydata/scripts/common_http.py for usage. +os.environ["__TAHOE_CLI_HTTP_TIMEOUT"] = "120" + + # pytest customization hooks def pytest_addoption(parser): diff --git a/newsfragments/4012.bugfix b/newsfragments/4012.bugfix deleted file mode 100644 index 24d5bb49a..000000000 --- a/newsfragments/4012.bugfix +++ /dev/null @@ -1 +0,0 @@ -The command-line tools now have a 300-second timeout on individual network reads/writes/connects; previously they could block forever in some situations. \ No newline at end of file diff --git a/newsfragments/4012.minor b/newsfragments/4012.minor new file mode 100644 index 000000000..e69de29bb diff --git a/src/allmydata/scripts/common_http.py b/src/allmydata/scripts/common_http.py index 46676b3f5..f138b9c07 100644 --- a/src/allmydata/scripts/common_http.py +++ b/src/allmydata/scripts/common_http.py @@ -53,10 +53,17 @@ def do_http(method, url, body=b""): assert body.seek assert body.read scheme, host, port, path = parse_url(url) + + # For testing purposes, allow setting a timeout on HTTP requests. If this + # ever become a user-facing feature, this should probably be a CLI option? + timeout = os.environ.get("__TAHOE_CLI_HTTP_TIMEOUT", None) + if timeout is not None: + timeout = float(timeout) + if scheme == "http": - c = http_client.HTTPConnection(host, port, timeout=300, blocksize=65536) + c = http_client.HTTPConnection(host, port, timeout=timeout, blocksize=65536) elif scheme == "https": - c = http_client.HTTPSConnection(host, port, timeout=300, blocksize=65536) + c = http_client.HTTPSConnection(host, port, timeout=timeout, blocksize=65536) else: raise ValueError("unknown scheme '%s', need http or https" % scheme) c.putrequest(method, path) From c0e49064ce64eb2860dba6c2957a86485c1a1e41 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 27 Apr 2023 09:50:02 -0400 Subject: [PATCH 1751/2309] Attempt to get more information about client unready state --- integration/util.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/integration/util.py b/integration/util.py index 177983e2e..39e5dfa6d 100644 --- a/integration/util.py +++ b/integration/util.py @@ -604,19 +604,27 @@ def await_client_ready(tahoe, timeout=10, liveness=60*2, minimum_number_of_serve print("waiting because '{}'".format(e)) time.sleep(1) continue + servers = js['servers'] - if len(js['servers']) < minimum_number_of_servers: - print(f"waiting because {js['servers']} is fewer than required ({minimum_number_of_servers})") + if len(servers) < minimum_number_of_servers: + print(f"waiting because {servers} is fewer than required ({minimum_number_of_servers})") time.sleep(1) continue + + print( + f"Now: {time.ctime()}\n" + f"Server last-received-data: {[time.ctime(s['last_received_data']) for s in servers]}" + ) + server_times = [ server['last_received_data'] - for server in js['servers'] + for server in servers ] # if any times are null/None that server has never been # contacted (so it's down still, probably) - if any(t is None for t in server_times): - print("waiting because at least one server not contacted") + never_received_data = server_times.count(None) + if never_received_data > 0: + print(f"waiting because {never_received_data} server(s) not contacted") time.sleep(1) continue From 8f1d1cc1a0db48a1aa219b9a8814ef4201206098 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 27 Apr 2023 10:23:06 -0400 Subject: [PATCH 1752/2309] Include node name in the logging output from subprocesses. --- integration/conftest.py | 6 +++--- integration/test_i2p.py | 4 ++-- integration/util.py | 11 ++++++----- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/integration/conftest.py b/integration/conftest.py index d76b2a9c7..69d1934b0 100644 --- a/integration/conftest.py +++ b/integration/conftest.py @@ -161,7 +161,7 @@ def flog_gatherer(reactor, temp_dir, flog_binary, request): ) pytest_twisted.blockon(out_protocol.done) - twistd_protocol = _MagicTextProtocol("Gatherer waiting at") + twistd_protocol = _MagicTextProtocol("Gatherer waiting at", "gatherer") twistd_process = reactor.spawnProcess( twistd_protocol, which('twistd')[0], @@ -244,7 +244,7 @@ log_gatherer.furl = {log_furl} # "tahoe run" is consistent across Linux/macOS/Windows, unlike the old # "start" command. - protocol = _MagicTextProtocol('introducer running') + protocol = _MagicTextProtocol('introducer running', "introducer") transport = _tahoe_runner_optional_coverage( protocol, reactor, @@ -320,7 +320,7 @@ log_gatherer.furl = {log_furl} # "tahoe run" is consistent across Linux/macOS/Windows, unlike the old # "start" command. - protocol = _MagicTextProtocol('introducer running') + protocol = _MagicTextProtocol('introducer running', "tor_introducer") transport = _tahoe_runner_optional_coverage( protocol, reactor, diff --git a/integration/test_i2p.py b/integration/test_i2p.py index 96619a93a..4d4dbe620 100644 --- a/integration/test_i2p.py +++ b/integration/test_i2p.py @@ -35,7 +35,7 @@ if sys.platform.startswith('win'): @pytest.fixture def i2p_network(reactor, temp_dir, request): """Fixture to start up local i2pd.""" - proto = util._MagicTextProtocol("ephemeral keys") + proto = util._MagicTextProtocol("ephemeral keys", "i2pd") reactor.spawnProcess( proto, which("docker"), @@ -99,7 +99,7 @@ log_gatherer.furl = {log_furl} # "tahoe run" is consistent across Linux/macOS/Windows, unlike the old # "start" command. - protocol = util._MagicTextProtocol('introducer running') + protocol = util._MagicTextProtocol('introducer running', "introducer") transport = util._tahoe_runner_optional_coverage( protocol, reactor, diff --git a/integration/util.py b/integration/util.py index 05fef8fed..b1692a7a3 100644 --- a/integration/util.py +++ b/integration/util.py @@ -12,7 +12,7 @@ import sys import time import json from os import mkdir, environ -from os.path import exists, join +from os.path import exists, join, basename from io import StringIO, BytesIO from subprocess import check_output @@ -129,8 +129,9 @@ class _MagicTextProtocol(ProcessProtocol): and then .callback()s on self.done and .errback's if the process exits """ - def __init__(self, magic_text): + def __init__(self, magic_text: str, name: str) -> None: self.magic_seen = Deferred() + self.name = f"{name}: " self.exited = Deferred() self._magic_text = magic_text self._output = StringIO() @@ -140,7 +141,7 @@ class _MagicTextProtocol(ProcessProtocol): def outReceived(self, data): data = str(data, sys.stdout.encoding) - sys.stdout.write(data) + sys.stdout.write(self.name + data) self._output.write(data) if not self.magic_seen.called and self._magic_text in self._output.getvalue(): print("Saw '{}' in the logs".format(self._magic_text)) @@ -148,7 +149,7 @@ class _MagicTextProtocol(ProcessProtocol): def errReceived(self, data): data = str(data, sys.stderr.encoding) - sys.stdout.write(data) + sys.stdout.write(self.name + data) def _cleanup_process_async(transport: IProcessTransport, allow_missing: bool) -> None: @@ -282,7 +283,7 @@ def _run_node(reactor, node_dir, request, magic_text, finalize=True): """ if magic_text is None: magic_text = "client running" - protocol = _MagicTextProtocol(magic_text) + protocol = _MagicTextProtocol(magic_text, basename(node_dir)) # "tahoe run" is consistent across Linux/macOS/Windows, unlike the old # "start" command. From 86a513282f67556e1791faa7ab09ac4fc3f5b6b7 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 27 Apr 2023 10:36:39 -0400 Subject: [PATCH 1753/2309] Include Foolscap logging in node output in integration tests. --- integration/conftest.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/integration/conftest.py b/integration/conftest.py index 69d1934b0..cdc65f9e8 100644 --- a/integration/conftest.py +++ b/integration/conftest.py @@ -54,6 +54,10 @@ from .util import ( # integration tests. See allmydata/scripts/common_http.py for usage. os.environ["__TAHOE_CLI_HTTP_TIMEOUT"] = "120" +# Make Foolscap logging go into Twisted logging, so that integration test logs +# include extra information +# (https://github.com/warner/foolscap/blob/latest-release/doc/logging.rst): +os.environ["FLOGTOTWISTED"] = "1" # pytest customization hooks From 9faf742b411e5dddd11e098eee7421b7154e3b25 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 27 Apr 2023 10:36:59 -0400 Subject: [PATCH 1754/2309] News file. --- newsfragments/4018.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/4018.minor diff --git a/newsfragments/4018.minor b/newsfragments/4018.minor new file mode 100644 index 000000000..e69de29bb From 3d0c872f4c3751d48c5cf1f2392ec431b11235f8 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 27 Apr 2023 10:44:10 -0400 Subject: [PATCH 1755/2309] restrict CI jobs to the wheelhouse --- .circleci/setup-virtualenv.sh | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/.circleci/setup-virtualenv.sh b/.circleci/setup-virtualenv.sh index feccbbf23..7087c5120 100755 --- a/.circleci/setup-virtualenv.sh +++ b/.circleci/setup-virtualenv.sh @@ -26,12 +26,7 @@ shift || : # Tell pip where it can find any existing wheels. export PIP_FIND_LINKS="file://${WHEELHOUSE_PATH}" - -# It is tempting to also set PIP_NO_INDEX=1 but (a) that will cause problems -# between the time dependencies change and the images are re-built and (b) the -# upcoming-deprecations job wants to install some dependencies from github and -# it's awkward to get that done any earlier than the tox run. So, we don't -# set it. +export PIP_NO_INDEX="1" # Get everything else installed in it, too. "${BOOTSTRAP_VENV}"/bin/tox \ From f9269158baf5103e014bbd439944690f3138c1af Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 27 Apr 2023 10:46:58 -0400 Subject: [PATCH 1756/2309] news fragment --- newsfragments/4019.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/4019.minor diff --git a/newsfragments/4019.minor b/newsfragments/4019.minor new file mode 100644 index 000000000..e69de29bb From 4d5b9f2d0c88e411b0cb032a46b8b681d984703c Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 27 Apr 2023 10:48:46 -0400 Subject: [PATCH 1757/2309] match the version in the docker image it is maybe wrong that we pin a specific version here and also only include a specific version (probably some interpretation of "the most recent release") in the docker image... --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 5f18b6b95..6e56496d4 100644 --- a/tox.ini +++ b/tox.ini @@ -36,7 +36,7 @@ deps = # happening at the time. The versions selected here are just the current # versions at the time. Bumping them to keep up with future releases is # fine as long as those releases are known to actually work. - pip==22.0.3 + pip==22.3.1 setuptools==60.9.1 wheel==0.37.1 subunitreporter==22.2.0 From 58ccecff5414e7ceded6aaac3666e289aa54dd5b Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 27 Apr 2023 11:17:19 -0400 Subject: [PATCH 1758/2309] Take a step towards unifying dependency pins used by tox env and Docker image building --- .circleci/populate-wheelhouse.sh | 21 +++-------------- setup.py | 39 +++++++++++++++++++++++++++++--- tox.ini | 19 +--------------- 3 files changed, 40 insertions(+), 39 deletions(-) diff --git a/.circleci/populate-wheelhouse.sh b/.circleci/populate-wheelhouse.sh index 374ca0adb..f103a6af8 100755 --- a/.circleci/populate-wheelhouse.sh +++ b/.circleci/populate-wheelhouse.sh @@ -3,18 +3,6 @@ # https://vaneyckt.io/posts/safer_bash_scripts_with_set_euxo_pipefail/ set -euxo pipefail -# Basic Python packages that you just need to have around to do anything, -# practically speaking. -BASIC_DEPS="pip wheel" - -# Python packages we need to support the test infrastructure. *Not* packages -# Tahoe-LAFS itself (implementation or test suite) need. -TEST_DEPS="tox~=3.0" - -# Python packages we need to generate test reports for CI infrastructure. -# *Not* packages Tahoe-LAFS itself (implement or test suite) need. -REPORTING_DEPS="python-subunit junitxml subunitreporter" - # The filesystem location of the wheelhouse which we'll populate with wheels # for all of our dependencies. WHEELHOUSE_PATH="$1" @@ -41,15 +29,12 @@ export PIP_FIND_LINKS="file://${WHEELHOUSE_PATH}" LANG="en_US.UTF-8" "${PIP}" \ wheel \ --wheel-dir "${WHEELHOUSE_PATH}" \ - "${PROJECT_ROOT}"[test] \ - ${BASIC_DEPS} \ - ${TEST_DEPS} \ - ${REPORTING_DEPS} + "${PROJECT_ROOT}"[testenv] \ + "${PROJECT_ROOT}"[test] # Not strictly wheelhouse population but ... Note we omit basic deps here. # They're in the wheelhouse if Tahoe-LAFS wants to drag them in but it will # have to ask. "${PIP}" \ install \ - ${TEST_DEPS} \ - ${REPORTING_DEPS} + "${PROJECT_ROOT}"[testenv] diff --git a/setup.py b/setup.py index 2418c6dbe..6e16381e6 100644 --- a/setup.py +++ b/setup.py @@ -398,10 +398,44 @@ setup(name="tahoe-lafs", # also set in __init__.py "dulwich", "gpg", ], + + # Here are the dependencies required to set up a reproducible test + # environment. This could be for CI or local development. These + # are *not* library dependencies of the test suite itself. They are + # the tools we use to run the test suite at all. + "testenv": [ + # Pin all of these versions for the same reason you ever want to + # pin anything: to prevent new releases with regressions from + # introducing spurious failures into CI runs for whatever + # development work is happening at the time. The versions + # selected here are just the current versions at the time. + # Bumping them to keep up with future releases is fine as long + # as those releases are known to actually work. + + # XXX For the moment, unpinned so we use whatever is in the + # image. The images vary in what versions they have. :/ + "pip", # ==22.0.3", + "wheel", # ==0.37.1" + "setuptools", # ==60.9.1", + "tox", # ~=3.0", + "subunitreporter", # ==22.2.0", + "python-subunit", # ==1.4.2", + "junitxml", # ==0.7", + "coverage", # ~= 5.0", + + # As an exception, we don't pin certifi because it contains CA + # certificates which necessarily change over time. Pinning this + # is guaranteed to cause things to break eventually as old + # certificates expire and as new ones are used in the wild that + # aren't present in whatever version we pin. Hopefully there + # won't be functionality regressions in new releases of this + # package that cause us the kind of suffering we're trying to + # avoid with the above pins. + "certifi", + ], + "test": [ - "coverage ~= 5.0", "mock", - "tox ~= 3.0", "pytest", "pytest-twisted", "hypothesis >= 3.6.1", @@ -410,7 +444,6 @@ setup(name="tahoe-lafs", # also set in __init__.py "fixtures", "beautifulsoup4", "html5lib", - "junitxml", # Pin old version until # https://github.com/paramiko/paramiko/issues/1961 is fixed. "paramiko < 2.9", diff --git a/tox.ini b/tox.ini index 6e56496d4..3b7a96503 100644 --- a/tox.ini +++ b/tox.ini @@ -30,24 +30,7 @@ passenv = TAHOE_LAFS_* PIP_* SUBUNITREPORTER_* USERPROFILE HOMEDRIVE HOMEPATH # available to those systems. Installing it ahead of time (with pip) avoids # this problem. deps = - # Pin all of these versions for the same reason you ever want to pin - # anything: to prevent new releases with regressions from introducing - # spurious failures into CI runs for whatever development work is - # happening at the time. The versions selected here are just the current - # versions at the time. Bumping them to keep up with future releases is - # fine as long as those releases are known to actually work. - pip==22.3.1 - setuptools==60.9.1 - wheel==0.37.1 - subunitreporter==22.2.0 - # As an exception, we don't pin certifi because it contains CA - # certificates which necessarily change over time. Pinning this is - # guaranteed to cause things to break eventually as old certificates - # expire and as new ones are used in the wild that aren't present in - # whatever version we pin. Hopefully there won't be functionality - # regressions in new releases of this package that cause us the kind of - # suffering we're trying to avoid with the above pins. - certifi + .[testenv] # We add usedevelop=False because testing against a true installation gives # more useful results. From 66d3de059432a3e1d12b14b50b66ac2ea263929d Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 27 Apr 2023 11:31:26 -0400 Subject: [PATCH 1759/2309] narrowly pin these dependencies This will break because these are not the versions on all Docker CI images but we need to pin them to rebuild those images with the correct versions. Rebuilding the images might break CI for all other branches. But! It's broken already, so it's not like it's any worse. --- setup.py | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/setup.py b/setup.py index 6e16381e6..127d17328 100644 --- a/setup.py +++ b/setup.py @@ -411,17 +411,14 @@ setup(name="tahoe-lafs", # also set in __init__.py # selected here are just the current versions at the time. # Bumping them to keep up with future releases is fine as long # as those releases are known to actually work. - - # XXX For the moment, unpinned so we use whatever is in the - # image. The images vary in what versions they have. :/ - "pip", # ==22.0.3", - "wheel", # ==0.37.1" - "setuptools", # ==60.9.1", - "tox", # ~=3.0", - "subunitreporter", # ==22.2.0", - "python-subunit", # ==1.4.2", - "junitxml", # ==0.7", - "coverage", # ~= 5.0", + "pip==22.0.3", + "wheel==0.37.1" + "setuptools==60.9.1", + "tox~=3.0", + "subunitreporter==22.2.0", + "python-subunit==1.4.2", + "junitxml==0.7", + "coverage ~= 5.0", # As an exception, we don't pin certifi because it contains CA # certificates which necessarily change over time. Pinning this From 29961a08b2c4097ee527043216a41b17a7da048d Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 27 Apr 2023 11:40:49 -0400 Subject: [PATCH 1760/2309] typo in the requirements list... --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 127d17328..b1d54b43c 100644 --- a/setup.py +++ b/setup.py @@ -412,7 +412,7 @@ setup(name="tahoe-lafs", # also set in __init__.py # Bumping them to keep up with future releases is fine as long # as those releases are known to actually work. "pip==22.0.3", - "wheel==0.37.1" + "wheel==0.37.1", "setuptools==60.9.1", "tox~=3.0", "subunitreporter==22.2.0", From 0f200e422e3278091843f6384fa91cfe5bf1101c Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 27 Apr 2023 15:48:49 -0400 Subject: [PATCH 1761/2309] Give it more time. --- integration/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration/util.py b/integration/util.py index b1692a7a3..c58fc2e93 100644 --- a/integration/util.py +++ b/integration/util.py @@ -582,7 +582,7 @@ def web_post(tahoe, uri_fragment, **kwargs): @run_in_thread -def await_client_ready(tahoe, timeout=10, liveness=60*2, minimum_number_of_servers=1): +def await_client_ready(tahoe, timeout=30, liveness=60*2, minimum_number_of_servers=1): """ Uses the status API to wait for a client-type node (in `tahoe`, a `TahoeProcess` instance usually from a fixture e.g. `alice`) to be From f6e4e862a9d1bb8cc16d27b791e76aec09a298e6 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 28 Apr 2023 07:50:50 -0400 Subject: [PATCH 1762/2309] Require that the actual test run step do this part Keep this script to wheelhouse population. We might be giving up a tiny bit of performance here but let's make it work at all before we make it fast. --- .circleci/populate-wheelhouse.sh | 7 ------- 1 file changed, 7 deletions(-) diff --git a/.circleci/populate-wheelhouse.sh b/.circleci/populate-wheelhouse.sh index f103a6af8..239c8367b 100755 --- a/.circleci/populate-wheelhouse.sh +++ b/.circleci/populate-wheelhouse.sh @@ -31,10 +31,3 @@ LANG="en_US.UTF-8" "${PIP}" \ --wheel-dir "${WHEELHOUSE_PATH}" \ "${PROJECT_ROOT}"[testenv] \ "${PROJECT_ROOT}"[test] - -# Not strictly wheelhouse population but ... Note we omit basic deps here. -# They're in the wheelhouse if Tahoe-LAFS wants to drag them in but it will -# have to ask. -"${PIP}" \ - install \ - "${PROJECT_ROOT}"[testenv] From 70caa22370b9646096e83cb18057287a3946698f Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 28 Apr 2023 07:51:45 -0400 Subject: [PATCH 1763/2309] have to do certifi in tox.ini by the time setup.py is being processed it is too late for certifi to help --- setup.py | 10 ---------- tox.ini | 22 +++++++++++++++------- 2 files changed, 15 insertions(+), 17 deletions(-) diff --git a/setup.py b/setup.py index b1d54b43c..599eb5898 100644 --- a/setup.py +++ b/setup.py @@ -419,16 +419,6 @@ setup(name="tahoe-lafs", # also set in __init__.py "python-subunit==1.4.2", "junitxml==0.7", "coverage ~= 5.0", - - # As an exception, we don't pin certifi because it contains CA - # certificates which necessarily change over time. Pinning this - # is guaranteed to cause things to break eventually as old - # certificates expire and as new ones are used in the wild that - # aren't present in whatever version we pin. Hopefully there - # won't be functionality regressions in new releases of this - # package that cause us the kind of suffering we're trying to - # avoid with the above pins. - "certifi", ], "test": [ diff --git a/tox.ini b/tox.ini index 3b7a96503..609a78b13 100644 --- a/tox.ini +++ b/tox.ini @@ -23,14 +23,22 @@ minversion = 2.4 [testenv] passenv = TAHOE_LAFS_* PIP_* SUBUNITREPORTER_* USERPROFILE HOMEDRIVE HOMEPATH -# Get "certifi" to avoid bug #2913. Basically if a `setup_requires=...` causes -# a package to be installed (with setuptools) then it'll fail on certain -# platforms (travis's OX-X 10.12, Slackware 14.2) because PyPI's TLS -# requirements (TLS >= 1.2) are incompatible with the old TLS clients -# available to those systems. Installing it ahead of time (with pip) avoids -# this problem. deps = - .[testenv] + # We pull in certify *here* to avoid bug #2913. Basically if a + # `setup_requires=...` causes a package to be installed (with setuptools) + # then it'll fail on certain platforms (travis's OX-X 10.12, Slackware + # 14.2) because PyPI's TLS requirements (TLS >= 1.2) are incompatible with + # the old TLS clients available to those systems. Installing it ahead of + # time (with pip) avoids this problem. + # + # We don't pin an exact version of it because it contains CA certificates + # which necessarily change over time. Pinning this is guaranteed to cause + # things to break eventually as old certificates expire and as new ones + # are used in the wild that aren't present in whatever version we pin. + # Hopefully there won't be functionality regressions in new releases of + # this package that cause us the kind of suffering we're trying to avoid + # with the above pins. + certifi # We add usedevelop=False because testing against a true installation gives # more useful results. From 17706f582ee54532b7a117b3b97ffce3e0108b7a Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 28 Apr 2023 07:52:05 -0400 Subject: [PATCH 1764/2309] use tox testenv `extras` to request testenv too --- tox.ini | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/tox.ini b/tox.ini index 609a78b13..2edb15a0b 100644 --- a/tox.ini +++ b/tox.ini @@ -43,9 +43,14 @@ deps = # We add usedevelop=False because testing against a true installation gives # more useful results. usedevelop = False -# We use extras=test to get things like "mock" that are required for our unit -# tests. -extras = test + +extras = + # Get general testing environment dependencies so we can run the tests + # how we like. + testenv + + # And get all of the test suite's actual direct Python dependencies. + test setenv = # Define TEST_SUITE in the environment as an aid to constructing the From f48eb81d9d741857b6fe6cbabab0e04942238036 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 28 Apr 2023 07:57:51 -0400 Subject: [PATCH 1765/2309] restrict werkzeug more, at least for the moment --- setup.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 599eb5898..d823029e2 100644 --- a/setup.py +++ b/setup.py @@ -141,8 +141,10 @@ install_requires = [ # HTTP server and client "klein", + # 2.2.0 has a bug: https://github.com/pallets/werkzeug/issues/2465 - "werkzeug != 2.2.0", + # 2.3.x has an incompatibility with Klein: https://github.com/twisted/klein/pull/575 + "werkzeug != 2.2.0, != 2.3.0, != 2.3.1", "treq", "cbor2", From 44cd746ce480c3a1641ebfe55350d15698625323 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 28 Apr 2023 11:43:26 -0400 Subject: [PATCH 1766/2309] Limit klein version for now. --- setup.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 2418c6dbe..751c7ebc3 100644 --- a/setup.py +++ b/setup.py @@ -141,8 +141,10 @@ install_requires = [ # HTTP server and client "klein", - # 2.2.0 has a bug: https://github.com/pallets/werkzeug/issues/2465 - "werkzeug != 2.2.0", + # 2.2.0 has a bug: https://github.com/pallets/werkzeug/issues/2465 and 2.3 is + # incompatible with klein 21.8 and earlier; see + # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/4020 for the latter. + "werkzeug != 2.2.0,<2.3", "treq", "cbor2", From c15dd6c9f0fb9d43c9a17db523d6f726f67ee593 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 28 Apr 2023 11:43:48 -0400 Subject: [PATCH 1767/2309] This wasn't the issue. --- integration/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration/util.py b/integration/util.py index c58fc2e93..b1692a7a3 100644 --- a/integration/util.py +++ b/integration/util.py @@ -582,7 +582,7 @@ def web_post(tahoe, uri_fragment, **kwargs): @run_in_thread -def await_client_ready(tahoe, timeout=30, liveness=60*2, minimum_number_of_servers=1): +def await_client_ready(tahoe, timeout=10, liveness=60*2, minimum_number_of_servers=1): """ Uses the status API to wait for a client-type node (in `tahoe`, a `TahoeProcess` instance usually from a fixture e.g. `alice`) to be From f0b98aead562495f7f2019dd886b4e61c5fe68f9 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 28 Apr 2023 13:32:42 -0400 Subject: [PATCH 1768/2309] You don't need tox *inside* your test environment. You need tox to *manage* your test environment (this is the premise, at least). --- .circleci/setup-virtualenv.sh | 4 ++++ setup.py | 1 - 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.circleci/setup-virtualenv.sh b/.circleci/setup-virtualenv.sh index 7087c5120..7fc6dc528 100755 --- a/.circleci/setup-virtualenv.sh +++ b/.circleci/setup-virtualenv.sh @@ -28,6 +28,10 @@ shift || : export PIP_FIND_LINKS="file://${WHEELHOUSE_PATH}" export PIP_NO_INDEX="1" +# Get tox inside the bootstrap virtualenv since we use tox to manage the rest +# of the environment. +"${BOOTSTRAP_VENV}"/bin/pip install "tox~=3.0" + # Get everything else installed in it, too. "${BOOTSTRAP_VENV}"/bin/tox \ -c "${PROJECT_ROOT}"/tox.ini \ diff --git a/setup.py b/setup.py index d823029e2..bac93a4bb 100644 --- a/setup.py +++ b/setup.py @@ -416,7 +416,6 @@ setup(name="tahoe-lafs", # also set in __init__.py "pip==22.0.3", "wheel==0.37.1", "setuptools==60.9.1", - "tox~=3.0", "subunitreporter==22.2.0", "python-subunit==1.4.2", "junitxml==0.7", From d67016d1b99ed1750c01af0caa4c137a768f31ce Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 28 Apr 2023 13:39:49 -0400 Subject: [PATCH 1769/2309] Get the right version of tox in the wheelhouse --- .circleci/populate-wheelhouse.sh | 3 ++- .circleci/setup-virtualenv.sh | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.circleci/populate-wheelhouse.sh b/.circleci/populate-wheelhouse.sh index 239c8367b..f7ce361a8 100755 --- a/.circleci/populate-wheelhouse.sh +++ b/.circleci/populate-wheelhouse.sh @@ -30,4 +30,5 @@ LANG="en_US.UTF-8" "${PIP}" \ wheel \ --wheel-dir "${WHEELHOUSE_PATH}" \ "${PROJECT_ROOT}"[testenv] \ - "${PROJECT_ROOT}"[test] + "${PROJECT_ROOT}"[test] \ + "tox~=3.0" diff --git a/.circleci/setup-virtualenv.sh b/.circleci/setup-virtualenv.sh index 7fc6dc528..3f0074da3 100755 --- a/.circleci/setup-virtualenv.sh +++ b/.circleci/setup-virtualenv.sh @@ -30,7 +30,7 @@ export PIP_NO_INDEX="1" # Get tox inside the bootstrap virtualenv since we use tox to manage the rest # of the environment. -"${BOOTSTRAP_VENV}"/bin/pip install "tox~=3.0" +"${BOOTSTRAP_VENV}"/bin/pip install tox # Get everything else installed in it, too. "${BOOTSTRAP_VENV}"/bin/tox \ From a088b1d8125404b53f76cd7408d29df08cb9ab2a Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 28 Apr 2023 13:49:14 -0400 Subject: [PATCH 1770/2309] don't bother to make a wheel of tox, just install it --- .circleci/populate-wheelhouse.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.circleci/populate-wheelhouse.sh b/.circleci/populate-wheelhouse.sh index f7ce361a8..14d421652 100755 --- a/.circleci/populate-wheelhouse.sh +++ b/.circleci/populate-wheelhouse.sh @@ -30,5 +30,8 @@ LANG="en_US.UTF-8" "${PIP}" \ wheel \ --wheel-dir "${WHEELHOUSE_PATH}" \ "${PROJECT_ROOT}"[testenv] \ - "${PROJECT_ROOT}"[test] \ - "tox~=3.0" + "${PROJECT_ROOT}"[test] + +# Put tox right into the bootstrap environment because everyone is going to +# need to use it. +"${PIP}" install "tox~=3.0" From 29c0ca59748507f95035de4aff1faaa65995102b Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 28 Apr 2023 13:51:22 -0400 Subject: [PATCH 1771/2309] put the tox installation near other software installation --- .circleci/create-virtualenv.sh | 4 ++++ .circleci/populate-wheelhouse.sh | 4 ---- .circleci/setup-virtualenv.sh | 4 ---- 3 files changed, 4 insertions(+), 8 deletions(-) diff --git a/.circleci/create-virtualenv.sh b/.circleci/create-virtualenv.sh index 810ce5ae2..7327d0859 100755 --- a/.circleci/create-virtualenv.sh +++ b/.circleci/create-virtualenv.sh @@ -47,3 +47,7 @@ export PIP_FIND_LINKS="file://${WHEELHOUSE_PATH}" # above, it may still not be able to get us a compatible version unless we # explicitly ask for one. "${PIP}" install --upgrade setuptools==44.0.0 wheel + +# Just about every user of this image wants to use tox from the bootstrap +# virtualenv so go ahead and install it now. +"${PIP}" install "tox~=3.0" diff --git a/.circleci/populate-wheelhouse.sh b/.circleci/populate-wheelhouse.sh index 14d421652..239c8367b 100755 --- a/.circleci/populate-wheelhouse.sh +++ b/.circleci/populate-wheelhouse.sh @@ -31,7 +31,3 @@ LANG="en_US.UTF-8" "${PIP}" \ --wheel-dir "${WHEELHOUSE_PATH}" \ "${PROJECT_ROOT}"[testenv] \ "${PROJECT_ROOT}"[test] - -# Put tox right into the bootstrap environment because everyone is going to -# need to use it. -"${PIP}" install "tox~=3.0" diff --git a/.circleci/setup-virtualenv.sh b/.circleci/setup-virtualenv.sh index 3f0074da3..7087c5120 100755 --- a/.circleci/setup-virtualenv.sh +++ b/.circleci/setup-virtualenv.sh @@ -28,10 +28,6 @@ shift || : export PIP_FIND_LINKS="file://${WHEELHOUSE_PATH}" export PIP_NO_INDEX="1" -# Get tox inside the bootstrap virtualenv since we use tox to manage the rest -# of the environment. -"${BOOTSTRAP_VENV}"/bin/pip install tox - # Get everything else installed in it, too. "${BOOTSTRAP_VENV}"/bin/tox \ -c "${PROJECT_ROOT}"/tox.ini \ From 04ef5a02b2a27e7dc224b11f51615369d99f7e74 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 28 Apr 2023 14:12:01 -0400 Subject: [PATCH 1772/2309] eh ... these things moved into the tox-managed venv not intentional but not sure what a _good_ fix is, so try this. --- .circleci/run-tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/run-tests.sh b/.circleci/run-tests.sh index 6d7a881fe..b1e45af9b 100755 --- a/.circleci/run-tests.sh +++ b/.circleci/run-tests.sh @@ -93,5 +93,5 @@ if [ -n "${ARTIFACTS}" ]; then # Create a junitxml results area. mkdir -p "$(dirname "${JUNITXML}")" - "${BOOTSTRAP_VENV}"/bin/subunit2junitxml < "${SUBUNIT2}" > "${JUNITXML}" || "${alternative}" + "${BOOTSTRAP_VENV}"/.tox/"${TAHOE_LAFS_TOX_ENVIRONMENT}"/bin/subunit2junitxml < "${SUBUNIT2}" > "${JUNITXML}" || "${alternative}" fi From fa034781b46f2d3ae5351a9c57c7d55825fdebfe Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 28 Apr 2023 14:29:21 -0400 Subject: [PATCH 1773/2309] Perhaps this is the correct way to locate the tox-managed venv --- .circleci/run-tests.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.circleci/run-tests.sh b/.circleci/run-tests.sh index b1e45af9b..d897cc729 100755 --- a/.circleci/run-tests.sh +++ b/.circleci/run-tests.sh @@ -79,9 +79,10 @@ else alternative="false" fi +WORKDIR=/tmp/tahoe-lafs.tox ${TIMEOUT} ${BOOTSTRAP_VENV}/bin/tox \ -c ${PROJECT_ROOT}/tox.ini \ - --workdir /tmp/tahoe-lafs.tox \ + --workdir "${WORKDIR}" \ -e "${TAHOE_LAFS_TOX_ENVIRONMENT}" \ ${TAHOE_LAFS_TOX_ARGS} || "${alternative}" @@ -93,5 +94,6 @@ if [ -n "${ARTIFACTS}" ]; then # Create a junitxml results area. mkdir -p "$(dirname "${JUNITXML}")" - "${BOOTSTRAP_VENV}"/.tox/"${TAHOE_LAFS_TOX_ENVIRONMENT}"/bin/subunit2junitxml < "${SUBUNIT2}" > "${JUNITXML}" || "${alternative}" + + "${WORKDIR}/${TAHOE_LAFS_TOX_ENVIRONMENT}/bin/subunit2junitxml" < "${SUBUNIT2}" > "${JUNITXML}" || "${alternative}" fi From 3c660aff5d23849cdc6cdd788a455a14f2bb7881 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 1 May 2023 09:19:01 -0400 Subject: [PATCH 1774/2309] a comment about the other test extra --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index bac93a4bb..49004b1ff 100644 --- a/setup.py +++ b/setup.py @@ -422,6 +422,7 @@ setup(name="tahoe-lafs", # also set in __init__.py "coverage ~= 5.0", ], + # Here are the library dependencies of the test suite. "test": [ "mock", "pytest", From 0af84c9ac1fdf07cb644d381b693dfcafcdf594e Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 1 May 2023 09:28:46 -0400 Subject: [PATCH 1775/2309] news fragment --- newsfragments/4020.installation | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/4020.installation diff --git a/newsfragments/4020.installation b/newsfragments/4020.installation new file mode 100644 index 000000000..8badf4b3c --- /dev/null +++ b/newsfragments/4020.installation @@ -0,0 +1 @@ +werkzeug 2.3.0 and werkzeug 2.3.1 are now blacklisted by the package metadata due to incompatibilities with klein. From 5f196050753b70c9224f1a10427d13649553b66a Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 1 May 2023 11:41:51 -0400 Subject: [PATCH 1776/2309] During testing, ensure we're not getting text/html unexpectedly. --- src/allmydata/storage/http_client.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index f786b8f30..7314adf38 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -443,11 +443,20 @@ class StorageClient(object): kwargs["data"] = dumps(message_to_serialize) headers.addRawHeader("Content-Type", CBOR_MIME_TYPE) - return await self._treq.request( + response = await self._treq.request( method, url, headers=headers, timeout=timeout, **kwargs ) - async def decode_cbor(self, response, schema: Schema) -> object: + if self.TEST_MODE_REGISTER_HTTP_POOL is not None: + if response.code != 404: + # We're doing API queries, HTML is never correct except in 404, but + # it's the default for Twisted's web server so make sure nothing + # unexpected happened. + assert get_content_type(response.headers) != "text/html" + + return response + + async def decode_cbor(self, response: IResponse, schema: Schema) -> object: """Given HTTP response, return decoded CBOR body.""" with start_action(action_type="allmydata:storage:http-client:decode-cbor"): if response.code > 199 and response.code < 300: From fbd6dbda47fec79f016ff5ba4609f1b03203a248 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 1 May 2023 11:42:02 -0400 Subject: [PATCH 1777/2309] text/html is a bad default content type. --- src/allmydata/storage/http_server.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 8647274f8..e0040d377 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -106,6 +106,9 @@ def _authorization_decorator(required_secrets): def decorator(f): @wraps(f) def route(self, request, *args, **kwargs): + # Don't set text/html content type by default: + request.defaultContentType = None + with start_action( action_type="allmydata:storage:http-server:handle-request", method=request.method, From 2292d64fcddc5d585551a0310a6b3076eb68caf3 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 1 May 2023 11:49:09 -0400 Subject: [PATCH 1778/2309] Set a better content type for data downloads. --- src/allmydata/storage/http_client.py | 6 ++++++ src/allmydata/storage/http_server.py | 4 +++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 7314adf38..64962e7b6 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -596,6 +596,12 @@ def read_share_chunk( if response.code == http.NO_CONTENT: return b"" + content_type = get_content_type(response.headers) + if content_type != "application/octet-stream": + raise ValueError( + f"Content-type was wrong: {content_type}, should be application/octet-stream" + ) + if response.code == http.PARTIAL_CONTENT: content_range = parse_content_range_header( response.headers.getRawHeaders("content-range")[0] or "" diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index e0040d377..0791c3389 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -778,6 +778,7 @@ class HTTPServer(object): ) def read_share_chunk(self, request, authorization, storage_index, share_number): """Read a chunk for an already uploaded immutable.""" + request.setHeader("content-type", "application/octet-stream") try: bucket = self._storage_server.get_buckets(storage_index)[share_number] except KeyError: @@ -883,7 +884,8 @@ class HTTPServer(object): ) def read_mutable_chunk(self, request, authorization, storage_index, share_number): """Read a chunk from a mutable.""" - + request.setHeader("content-type", "application/octet-stream") + try: share_length = self._storage_server.get_mutable_share_length( storage_index, share_number From 5632e82e1338541d3c8f574070f402fabdc3523c Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 1 May 2023 11:49:29 -0400 Subject: [PATCH 1779/2309] News fragment. --- newsfragments/4016.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/4016.minor diff --git a/newsfragments/4016.minor b/newsfragments/4016.minor new file mode 100644 index 000000000..e69de29bb From 8c8e24a3b9c7655c197d24ece14e559511727610 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 1 May 2023 11:50:05 -0400 Subject: [PATCH 1780/2309] Black reformat. --- src/allmydata/storage/http_server.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 0791c3389..7d7398b1e 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -117,9 +117,9 @@ def _authorization_decorator(required_secrets): try: # Check Authorization header: if not timing_safe_compare( - request.requestHeaders.getRawHeaders("Authorization", [""])[0].encode( - "utf-8" - ), + request.requestHeaders.getRawHeaders("Authorization", [""])[ + 0 + ].encode("utf-8"), swissnum_auth_header(self._swissnum), ): raise _HTTPError(http.UNAUTHORIZED) @@ -494,6 +494,7 @@ def read_range( def _add_error_handling(app: Klein): """Add exception handlers to a Klein app.""" + @app.handle_errors(_HTTPError) def _http_error(_, request, failure): """Handle ``_HTTPError`` exceptions.""" @@ -885,7 +886,7 @@ class HTTPServer(object): def read_mutable_chunk(self, request, authorization, storage_index, share_number): """Read a chunk from a mutable.""" request.setHeader("content-type", "application/octet-stream") - + try: share_length = self._storage_server.get_mutable_share_length( storage_index, share_number From b21b15f3954ae328fbaafd7d53839b62c4329d4b Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 1 May 2023 11:56:59 -0400 Subject: [PATCH 1781/2309] Blocking newer werkzeug is a temporary measure. --- newsfragments/4020.installation | 1 - newsfragments/4020.minor | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) delete mode 100644 newsfragments/4020.installation create mode 100644 newsfragments/4020.minor diff --git a/newsfragments/4020.installation b/newsfragments/4020.installation deleted file mode 100644 index 8badf4b3c..000000000 --- a/newsfragments/4020.installation +++ /dev/null @@ -1 +0,0 @@ -werkzeug 2.3.0 and werkzeug 2.3.1 are now blacklisted by the package metadata due to incompatibilities with klein. diff --git a/newsfragments/4020.minor b/newsfragments/4020.minor new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/newsfragments/4020.minor @@ -0,0 +1 @@ + From 4ca056b51c3dcc9f13f424bf133bd9dc34de8d93 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 1 May 2023 11:57:35 -0400 Subject: [PATCH 1782/2309] Be more general, 2.3.2 just came out for example. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 49004b1ff..4f28b4438 100644 --- a/setup.py +++ b/setup.py @@ -144,7 +144,7 @@ install_requires = [ # 2.2.0 has a bug: https://github.com/pallets/werkzeug/issues/2465 # 2.3.x has an incompatibility with Klein: https://github.com/twisted/klein/pull/575 - "werkzeug != 2.2.0, != 2.3.0, != 2.3.1", + "werkzeug != 2.2.0, < 2.3", "treq", "cbor2", From 5c2f18dfec743c104410ef514120a8b97ccc1364 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 2 May 2023 12:03:14 -0400 Subject: [PATCH 1783/2309] Set a higher timeout. --- src/allmydata/test/test_system.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/allmydata/test/test_system.py b/src/allmydata/test/test_system.py index d11a6e866..72e91f9b4 100644 --- a/src/allmydata/test/test_system.py +++ b/src/allmydata/test/test_system.py @@ -1749,6 +1749,10 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase): return d + # In CI this test can be very slow, so give it a longer timeout: + test_filesystem.timeout = 360 + + def test_filesystem_with_cli_in_subprocess(self): # We do this in a separate test so that test_filesystem doesn't skip if we can't run bin/tahoe. From 8fa89bd98585bd64b2510a6ec50ab44a5090bd4b Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 2 May 2023 12:05:40 -0400 Subject: [PATCH 1784/2309] Run a little faster. --- src/allmydata/test/test_system.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/test_system.py b/src/allmydata/test/test_system.py index 72e91f9b4..a6bed7f87 100644 --- a/src/allmydata/test/test_system.py +++ b/src/allmydata/test/test_system.py @@ -787,7 +787,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase): def test_filesystem(self): self.data = LARGE_DATA - d = self.set_up_nodes() + d = self.set_up_nodes(4) def _new_happy_semantics(ign): for c in self.clients: c.encoding_params['happy'] = 1 From d4f2038fd1ae943944c9a90e3e154371bc76d57d Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 2 May 2023 12:11:23 -0400 Subject: [PATCH 1785/2309] Rearrange nodes so it's possible to create even fewer. --- src/allmydata/test/common_system.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/allmydata/test/common_system.py b/src/allmydata/test/common_system.py index 3491d413d..fa8d943e5 100644 --- a/src/allmydata/test/common_system.py +++ b/src/allmydata/test/common_system.py @@ -819,8 +819,8 @@ class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin): helper_furl = f.read() self.helper_furl = helper_furl - if self.numclients >= 4: - with open(os.path.join(basedirs[3], 'tahoe.cfg'), 'a+') as f: + if self.numclients >= 2: + with open(os.path.join(basedirs[1], 'tahoe.cfg'), 'a+') as f: f.write( "[client]\n" "helper.furl = {}\n".format(helper_furl) @@ -836,9 +836,9 @@ class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin): log.msg("CONNECTED") # now find out where the web port was self.webish_url = self.clients[0].getServiceNamed("webish").getURL() - if self.numclients >=4: + if self.numclients >=2: # and the helper-using webport - self.helper_webish_url = self.clients[3].getServiceNamed("webish").getURL() + self.helper_webish_url = self.clients[1].getServiceNamed("webish").getURL() def _generate_config(self, which, basedir, force_foolscap=False): config = {} @@ -854,10 +854,10 @@ class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin): ("node", "tub.location"): allclients, # client 0 runs a webserver and a helper - # client 3 runs a webserver but no helper - ("node", "web.port"): {0, 3}, + # client 1 runs a webserver but no helper + ("node", "web.port"): {0, 1}, ("node", "timeout.keepalive"): {0}, - ("node", "timeout.disconnect"): {3}, + ("node", "timeout.disconnect"): {1}, ("helper", "enabled"): {0}, } From 9f78fd5c7f2d3340a3c64f79483373292570b732 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 2 May 2023 12:11:31 -0400 Subject: [PATCH 1786/2309] Use even fewer nodes. --- src/allmydata/test/test_system.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/test_system.py b/src/allmydata/test/test_system.py index a6bed7f87..c2fe1339f 100644 --- a/src/allmydata/test/test_system.py +++ b/src/allmydata/test/test_system.py @@ -787,7 +787,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase): def test_filesystem(self): self.data = LARGE_DATA - d = self.set_up_nodes(4) + d = self.set_up_nodes(2) def _new_happy_semantics(ign): for c in self.clients: c.encoding_params['happy'] = 1 From 1ca30e1d2fc00c61b56ad6603984dd4d705bd549 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 2 May 2023 12:11:44 -0400 Subject: [PATCH 1787/2309] News entry. --- newsfragments/4022.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/4022.minor diff --git a/newsfragments/4022.minor b/newsfragments/4022.minor new file mode 100644 index 000000000..e69de29bb From 22715abc854b68eaa38fd664d1ab207c88582894 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 2 May 2023 12:17:55 -0400 Subject: [PATCH 1788/2309] This is fine. --- src/allmydata/test/test_system.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/test_system.py b/src/allmydata/test/test_system.py index c2fe1339f..58384e4d8 100644 --- a/src/allmydata/test/test_system.py +++ b/src/allmydata/test/test_system.py @@ -1750,7 +1750,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase): return d # In CI this test can be very slow, so give it a longer timeout: - test_filesystem.timeout = 360 + test_filesystem.timeout = 360 # type: ignore[attr-defined] def test_filesystem_with_cli_in_subprocess(self): From 63b082759dabe44dfbed45effd10e2cae26d037b Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 2 May 2023 12:59:00 -0400 Subject: [PATCH 1789/2309] Use a modern coverage.py. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index d01efdf83..9837b3bab 100644 --- a/setup.py +++ b/setup.py @@ -418,7 +418,7 @@ setup(name="tahoe-lafs", # also set in __init__.py "subunitreporter==22.2.0", "python-subunit==1.4.2", "junitxml==0.7", - "coverage ~= 5.0", + "coverage==7.2.5", ], # Here are the library dependencies of the test suite. From 5a5031f02046c77a66761b5988177081631a6afc Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 3 May 2023 16:46:57 -0400 Subject: [PATCH 1790/2309] Try with newer Python, 3.11 might make it faster. --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1bb7c9efb..fe911e34d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -169,12 +169,12 @@ jobs: python-version: "3.9" force-foolscap: false - os: windows-latest - python-version: "3.9" + python-version: "3.11" force-foolscap: false # 22.04 has some issue with Tor at the moment: # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3943 - os: ubuntu-20.04 - python-version: "3.11" + python-version: "3.10" force-foolscap: false steps: From dca19525b9d9f2cd6c6f63fdaba6945bb5f4759a Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 3 May 2023 16:58:47 -0400 Subject: [PATCH 1791/2309] =?UTF-8?q?=F0=9F=AA=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- integration/conftest.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/integration/conftest.py b/integration/conftest.py index 7360b891b..bf04e4424 100644 --- a/integration/conftest.py +++ b/integration/conftest.py @@ -50,9 +50,9 @@ from .util import ( ) from allmydata.node import read_config -# No reason for HTTP requests to take longer than two minutes in the +# No reason for HTTP requests to take longer than four minutes in the # integration tests. See allmydata/scripts/common_http.py for usage. -os.environ["__TAHOE_CLI_HTTP_TIMEOUT"] = "120" +os.environ["__TAHOE_CLI_HTTP_TIMEOUT"] = "240" # Make Foolscap logging go into Twisted logging, so that integration test logs # include extra information From 83a6a7de2835148baaa6aaa94a449c77389887a3 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 3 May 2023 17:20:29 -0400 Subject: [PATCH 1792/2309] Newer klein and werkzeug. --- setup.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 9837b3bab..0453fa63f 100644 --- a/setup.py +++ b/setup.py @@ -140,10 +140,10 @@ install_requires = [ "collections-extended >= 2.0.2", # HTTP server and client - "klein", + # Latest version is necessary to work with latest werkzeug: + "klein >= 23.5.0", # 2.2.0 has a bug: https://github.com/pallets/werkzeug/issues/2465 - # 2.3.x has an incompatibility with Klein: https://github.com/twisted/klein/pull/575 - "werkzeug != 2.2.0, < 2.3", + "werkzeug != 2.2.0", "treq", "cbor2", From c70930b47921facf49d5cc371bd575bd715f7669 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 3 May 2023 17:21:07 -0400 Subject: [PATCH 1793/2309] News fragment. --- newsfragments/4024.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/4024.minor diff --git a/newsfragments/4024.minor b/newsfragments/4024.minor new file mode 100644 index 000000000..e69de29bb From 19690c9c7bd319bf9a2d1931ef4c8b37e9cc3803 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 4 May 2023 13:05:06 -0400 Subject: [PATCH 1794/2309] Don't mix blocking and async APIs! --- integration/test_web.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/integration/test_web.py b/integration/test_web.py index b3c4a8e5f..c7d2275ae 100644 --- a/integration/test_web.py +++ b/integration/test_web.py @@ -14,6 +14,8 @@ from __future__ import annotations import time from urllib.parse import unquote as url_unquote, quote as url_quote +from twisted.internet.defer import maybeDeferred + import allmydata.uri from allmydata.util import jsonbytes as json @@ -24,7 +26,7 @@ import requests import html5lib from bs4 import BeautifulSoup -from pytest_twisted import ensureDeferred +import pytest_twisted @run_in_thread def test_index(alice): @@ -185,7 +187,7 @@ def test_deep_stats(alice): time.sleep(.5) -@util.run_in_thread +@run_in_thread def test_status(alice): """ confirm we get something sensible from /status and the various sub-types @@ -251,7 +253,7 @@ def test_status(alice): assert found_download, "Failed to find the file we downloaded in the status-page" -@ensureDeferred +@run_in_thread async def test_directory_deep_check(reactor, request, alice): """ use deep-check and confirm the result pages work @@ -262,7 +264,8 @@ async def test_directory_deep_check(reactor, request, alice): required = 2 total = 4 - await util.reconfigure(reactor, request, alice, (happy, required, total), convergence=None) + result = util.reconfigure(reactor, request, alice, (happy, required, total), convergence=None) + pytest_twisted.blockon(maybeDeferred(result)) # create a directory resp = requests.post( From f54a2d3d76a0508c713c42c420105f98e384a5df Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 4 May 2023 13:05:46 -0400 Subject: [PATCH 1795/2309] News file. --- newsfragments/4023.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/4023.minor diff --git a/newsfragments/4023.minor b/newsfragments/4023.minor new file mode 100644 index 000000000..e69de29bb From 3d6b3b3b74f2fc4878629e9838471f3d12a1eb6b Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 4 May 2023 13:10:23 -0400 Subject: [PATCH 1796/2309] Use modern Docker image. --- .readthedocs.yaml | 5 +++++ newsfragments/4026.minor | 0 2 files changed, 5 insertions(+) create mode 100644 newsfragments/4026.minor diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 65b390f26..665b53178 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -1,5 +1,10 @@ version: 2 +build: + os: ubuntu-22.04 + tools: + python: "3.10" + python: install: - requirements: docs/requirements.txt diff --git a/newsfragments/4026.minor b/newsfragments/4026.minor new file mode 100644 index 000000000..e69de29bb From bee295e4119448958f1ea2b07666345328eaaee9 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 4 May 2023 13:33:54 -0400 Subject: [PATCH 1797/2309] Actually run the test. --- integration/test_web.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/integration/test_web.py b/integration/test_web.py index c7d2275ae..1d9498264 100644 --- a/integration/test_web.py +++ b/integration/test_web.py @@ -14,7 +14,7 @@ from __future__ import annotations import time from urllib.parse import unquote as url_unquote, quote as url_quote -from twisted.internet.defer import maybeDeferred +from twisted.internet.defer import ensureDeferred import allmydata.uri from allmydata.util import jsonbytes as json @@ -254,7 +254,7 @@ def test_status(alice): @run_in_thread -async def test_directory_deep_check(reactor, request, alice): +def test_directory_deep_check(reactor, request, alice): """ use deep-check and confirm the result pages work """ @@ -265,7 +265,7 @@ async def test_directory_deep_check(reactor, request, alice): total = 4 result = util.reconfigure(reactor, request, alice, (happy, required, total), convergence=None) - pytest_twisted.blockon(maybeDeferred(result)) + pytest_twisted.blockon(ensureDeferred(result)) # create a directory resp = requests.post( From 3b52457d1c42e4de5bffa7486738a837d81ceb56 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 4 May 2023 13:54:04 -0400 Subject: [PATCH 1798/2309] Try a different way. --- integration/test_web.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/integration/test_web.py b/integration/test_web.py index 1d9498264..fd29504f8 100644 --- a/integration/test_web.py +++ b/integration/test_web.py @@ -14,7 +14,7 @@ from __future__ import annotations import time from urllib.parse import unquote as url_unquote, quote as url_quote -from twisted.internet.defer import ensureDeferred +from twisted.internet.threads import deferToThread import allmydata.uri from allmydata.util import jsonbytes as json @@ -253,8 +253,8 @@ def test_status(alice): assert found_download, "Failed to find the file we downloaded in the status-page" -@run_in_thread -def test_directory_deep_check(reactor, request, alice): +@pytest_twisted.ensureDeferred +async def test_directory_deep_check(reactor, request, alice): """ use deep-check and confirm the result pages work """ @@ -264,9 +264,11 @@ def test_directory_deep_check(reactor, request, alice): required = 2 total = 4 - result = util.reconfigure(reactor, request, alice, (happy, required, total), convergence=None) - pytest_twisted.blockon(ensureDeferred(result)) + await util.reconfigure(reactor, request, alice, (happy, required, total), convergence=None) + await deferToThread(_test_directory_deep_check_blocking, alice) + +def _test_directory_deep_check_blocking(alice): # create a directory resp = requests.post( util.node_url(alice.node_dir, u"uri"), From 049502e8c282eb507d46cdfcf24a945aceaa7e33 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 4 May 2023 16:15:30 -0400 Subject: [PATCH 1799/2309] Don't mix blocking and async code. --- integration/test_get_put.py | 33 +++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/integration/test_get_put.py b/integration/test_get_put.py index f121d6284..0aca6954a 100644 --- a/integration/test_get_put.py +++ b/integration/test_get_put.py @@ -6,8 +6,10 @@ and stdout. from subprocess import Popen, PIPE, check_output, check_call import pytest -from pytest_twisted import ensureDeferred +from pytest_twisted import blockon from twisted.internet import reactor +from twisted.internet.threads import blockingCallFromThread +from twisted.internet.defer import Deferred from .util import run_in_thread, cli, reconfigure @@ -86,8 +88,8 @@ def test_large_file(alice, get_put_alias, tmp_path): assert outfile.read_bytes() == tempfile.read_bytes() -@ensureDeferred -async def test_upload_download_immutable_different_default_max_segment_size(alice, get_put_alias, tmpdir, request): +@run_in_thread +def test_upload_download_immutable_different_default_max_segment_size(alice, get_put_alias, tmpdir, request): """ Tahoe-LAFS used to have a default max segment size of 128KB, and is now 1MB. Test that an upload created when 128KB was the default can be @@ -100,22 +102,25 @@ async def test_upload_download_immutable_different_default_max_segment_size(alic with tempfile.open("wb") as f: f.write(large_data) - async def set_segment_size(segment_size): - await reconfigure( + def set_segment_size(segment_size): + return blockingCallFromThread( reactor, - request, - alice, - (1, 1, 1), - None, - max_segment_size=segment_size - ) + lambda: Deferred.fromCoroutine(reconfigure( + reactor, + request, + alice, + (1, 1, 1), + None, + max_segment_size=segment_size + )) + ) # 1. Upload file 1 with default segment size set to 1MB - await set_segment_size(1024 * 1024) + set_segment_size(1024 * 1024) cli(alice, "put", str(tempfile), "getput:seg1024kb") # 2. Download file 1 with default segment size set to 128KB - await set_segment_size(128 * 1024) + set_segment_size(128 * 1024) assert large_data == check_output( ["tahoe", "--node-directory", alice.node_dir, "get", "getput:seg1024kb", "-"] ) @@ -124,7 +129,7 @@ async def test_upload_download_immutable_different_default_max_segment_size(alic cli(alice, "put", str(tempfile), "getput:seg128kb") # 4. Download file 2 with default segment size set to 1MB - await set_segment_size(1024 * 1024) + set_segment_size(1024 * 1024) assert large_data == check_output( ["tahoe", "--node-directory", alice.node_dir, "get", "getput:seg128kb", "-"] ) From ba638f9ff602b3f7dcf8290f58adbab653ba25bd Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 4 May 2023 17:08:54 -0400 Subject: [PATCH 1800/2309] Remove unnecessary import. --- integration/test_get_put.py | 1 - 1 file changed, 1 deletion(-) diff --git a/integration/test_get_put.py b/integration/test_get_put.py index 0aca6954a..e30a34f97 100644 --- a/integration/test_get_put.py +++ b/integration/test_get_put.py @@ -6,7 +6,6 @@ and stdout. from subprocess import Popen, PIPE, check_output, check_call import pytest -from pytest_twisted import blockon from twisted.internet import reactor from twisted.internet.threads import blockingCallFromThread from twisted.internet.defer import Deferred From 20f55933bdf4742da8b0e68f9cd14fa0522a6323 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 5 May 2023 11:02:24 -0400 Subject: [PATCH 1801/2309] Update past deprecated runner. --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fe911e34d..77221e552 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -248,7 +248,7 @@ jobs: fail-fast: false matrix: os: - - macos-10.15 + - macos-latest - windows-latest - ubuntu-latest python-version: From 2e22df60fe6b29706464d3d3fa4f8b2e7baf51d5 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 8 May 2023 13:33:34 -0400 Subject: [PATCH 1802/2309] Try with fewer persistent HTTP connections. --- src/allmydata/storage/http_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 64962e7b6..0e12df7ce 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -325,7 +325,7 @@ class StorageClient(object): certificate_hash = nurl.user.encode("ascii") if pool is None: pool = HTTPConnectionPool(reactor) - pool.maxPersistentPerHost = 20 + pool.maxPersistentPerHost = 10 if cls.TEST_MODE_REGISTER_HTTP_POOL is not None: cls.TEST_MODE_REGISTER_HTTP_POOL(pool) From 2bd76058e927c7811b722538abb8127fadde45be Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 8 May 2023 13:48:54 -0400 Subject: [PATCH 1803/2309] Specific version of macOS. --- .github/workflows/ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 77221e552..dc9854ae4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -53,9 +53,9 @@ jobs: - "3.11" include: # On macOS don't bother with 3.8, just to get faster builds. - - os: macos-latest + - os: macos-12 python-version: "3.9" - - os: macos-latest + - os: macos-12 python-version: "3.11" # We only support PyPy on Linux at the moment. - os: ubuntu-latest @@ -165,7 +165,7 @@ jobs: fail-fast: false matrix: include: - - os: macos-latest + - os: macos-12 python-version: "3.9" force-foolscap: false - os: windows-latest @@ -248,7 +248,7 @@ jobs: fail-fast: false matrix: os: - - macos-latest + - macos-12 - windows-latest - ubuntu-latest python-version: From fea2450c604a3d34a3d78760d1cbc3085bdd521b Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 9 May 2023 10:53:17 -0400 Subject: [PATCH 1804/2309] Test and fix for really bad authorization header. --- src/allmydata/storage/http_server.py | 10 +++++++--- src/allmydata/test/test_storage_http.py | 19 +++++++++++++++++++ 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 7d7398b1e..cc336d1c7 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -116,10 +116,14 @@ def _authorization_decorator(required_secrets): ) as ctx: try: # Check Authorization header: + try: + auth_header = request.requestHeaders.getRawHeaders( + "Authorization", [""] + )[0].encode("utf-8") + except UnicodeError: + raise _HTTPError(http.BAD_REQUEST) if not timing_safe_compare( - request.requestHeaders.getRawHeaders("Authorization", [""])[ - 0 - ].encode("utf-8"), + auth_header, swissnum_auth_header(self._swissnum), ): raise _HTTPError(http.UNAUTHORIZED) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index eca2be1c1..77fb825e6 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -257,6 +257,10 @@ class TestApp(object): _add_error_handling(_app) _swissnum = SWISSNUM_FOR_TEST # Match what the test client is using + @_authorized_route(_app, {}, "/noop", methods=["GET"]) + def noop(self, request, authorization): + return "noop" + @_authorized_route(_app, {Secrets.UPLOAD}, "/upload_secret", methods=["GET"]) def validate_upload_secret(self, request, authorization): if authorization == {Secrets.UPLOAD: b"MAGIC"}: @@ -339,6 +343,21 @@ class CustomHTTPServerTests(SyncTestCase): ) self._http_server.clock = self.client._clock + def test_bad_swissnum_in_client(self) -> None: + """ + If the swissnum is invalid, a BAD REQUEST response code is returned. + """ + headers = Headers() + headers.addRawHeader("Authorization", b"\x00\xFF\x00\xFF") + response = result_of( + self.client._treq.request( + "GET", + DecodedURL.from_text("http://127.0.0.1/noop"), + headers=headers, + ) + ) + self.assertEqual(response.code, 400) + def test_authorization_enforcement(self): """ The requirement for secrets is enforced by the ``_authorized_route`` From 40b930c02c1dd20bd567419ef58b4fa30b6dfe8b Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 9 May 2023 16:47:28 -0400 Subject: [PATCH 1805/2309] Another test. --- src/allmydata/test/test_storage_http.py | 30 +++++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 77fb825e6..df5dc300c 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -343,11 +343,12 @@ class CustomHTTPServerTests(SyncTestCase): ) self._http_server.clock = self.client._clock - def test_bad_swissnum_in_client(self) -> None: + def test_bad_swissnum_from_client(self) -> None: """ If the swissnum is invalid, a BAD REQUEST response code is returned. """ headers = Headers() + # The value is not UTF-8. headers.addRawHeader("Authorization", b"\x00\xFF\x00\xFF") response = result_of( self.client._treq.request( @@ -358,10 +359,35 @@ class CustomHTTPServerTests(SyncTestCase): ) self.assertEqual(response.code, 400) + def test_bad_secret(self) -> None: + """ + If the secret is invalid (not base64), a BAD REQUEST + response code is returned. + """ + bad_secret = b"upload-secret []<>" + headers = Headers() + headers.addRawHeader( + "X-Tahoe-Authorization", + bad_secret, + ) + response = result_of( + self.client.request( + "GET", + DecodedURL.from_text("http://127.0.0.1/upload_secret"), + headers=headers, + ) + ) + self.assertEqual(response.code, 400) + + # TODO test other garbage values + def test_authorization_enforcement(self): """ The requirement for secrets is enforced by the ``_authorized_route`` decorator; if they are not given, a 400 response code is returned. + + Note that this refers to ``X-Tahoe-Authorization``, not the + ``Authorization`` header used for the swissnum. """ # Without secret, get a 400 error. response = result_of( @@ -1474,7 +1500,7 @@ class SharedImmutableMutableTestsMixin: self.client.advise_corrupt_share(storage_index, 13, reason) ) - for (si, share_number) in [(storage_index, 11), (urandom(16), 13)]: + for si, share_number in [(storage_index, 11), (urandom(16), 13)]: with assert_fails_with_http_code(self, http.NOT_FOUND): self.http.result_of_with_flush( self.client.advise_corrupt_share(si, share_number, reason) From 1c9de671049fe133692ed78a0bc35d04b7b3974c Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 9 May 2023 16:47:32 -0400 Subject: [PATCH 1806/2309] Nicer error messages, useful for debugging. --- src/allmydata/storage/http_server.py | 31 ++++++++++++++++++++-------- 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index cc336d1c7..924ae5a43 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -4,7 +4,7 @@ HTTP server for storage. from __future__ import annotations -from typing import Any, Callable, Union, cast +from typing import Any, Callable, Union, cast, Optional from functools import wraps from base64 import b64decode import binascii @@ -75,7 +75,7 @@ def _extract_secrets( secrets, return dictionary mapping secrets to decoded values. If too few secrets were given, or too many, a ``ClientSecretsException`` is - raised. + raised; its text is sent in the HTTP response. """ string_key_to_enum = {e.value: e for e in Secrets} result = {} @@ -84,6 +84,10 @@ def _extract_secrets( string_key, string_value = header_value.strip().split(" ", 1) key = string_key_to_enum[string_key] value = b64decode(string_value) + if value == b"": + raise ClientSecretsException( + "Failed to decode secret {}".format(string_key) + ) if key in (Secrets.LEASE_CANCEL, Secrets.LEASE_RENEW) and len(value) != 32: raise ClientSecretsException("Lease secrets must be 32 bytes long") result[key] = value @@ -91,7 +95,9 @@ def _extract_secrets( raise ClientSecretsException("Bad header value(s): {}".format(header_values)) if result.keys() != required_secrets: raise ClientSecretsException( - "Expected {} secrets, got {}".format(required_secrets, result.keys()) + "Expected {} in X-Tahoe-Authorization headers, got {}".format( + [r.value for r in required_secrets], list(result.keys()) + ) ) return result @@ -121,12 +127,14 @@ def _authorization_decorator(required_secrets): "Authorization", [""] )[0].encode("utf-8") except UnicodeError: - raise _HTTPError(http.BAD_REQUEST) + raise _HTTPError(http.BAD_REQUEST, "Bad Authorization header") if not timing_safe_compare( auth_header, swissnum_auth_header(self._swissnum), ): - raise _HTTPError(http.UNAUTHORIZED) + raise _HTTPError( + http.UNAUTHORIZED, "Wrong Authorization header" + ) # Check secrets: authorization = request.requestHeaders.getRawHeaders( @@ -134,8 +142,8 @@ def _authorization_decorator(required_secrets): ) try: secrets = _extract_secrets(authorization, required_secrets) - except ClientSecretsException: - raise _HTTPError(http.BAD_REQUEST) + except ClientSecretsException as e: + raise _HTTPError(http.BAD_REQUEST, str(e)) # Run the business logic: result = f(self, request, secrets, *args, **kwargs) @@ -276,8 +284,10 @@ class _HTTPError(Exception): Raise from ``HTTPServer`` endpoint to return the given HTTP response code. """ - def __init__(self, code: int): + def __init__(self, code: int, body: Optional[str] = None): + Exception.__init__(self, (code, body)) self.code = code + self.body = body # CDDL schemas. @@ -503,7 +513,10 @@ def _add_error_handling(app: Klein): def _http_error(_, request, failure): """Handle ``_HTTPError`` exceptions.""" request.setResponseCode(failure.value.code) - return b"" + if failure.value.body is not None: + return failure.value.body + else: + return b"" @app.handle_errors(CDDLValidationError) def _cddl_validation_error(_, request, failure): From 36bffb7f60bcf646fb940cff5ffd49caa8514742 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 9 May 2023 16:47:54 -0400 Subject: [PATCH 1807/2309] News file. --- newsfragments/4027.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/4027.minor diff --git a/newsfragments/4027.minor b/newsfragments/4027.minor new file mode 100644 index 000000000..e69de29bb From c92c93e6d56975c47bd8b2e0dea3a8cfba81960b Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 10 May 2023 16:31:53 -0400 Subject: [PATCH 1808/2309] Clean up cached HTTP connections on shutdown. --- newsfragments/4028.minor | 0 src/allmydata/storage/http_client.py | 7 ++++++- src/allmydata/storage_client.py | 5 +++++ 3 files changed, 11 insertions(+), 1 deletion(-) create mode 100644 newsfragments/4028.minor diff --git a/newsfragments/4028.minor b/newsfragments/4028.minor new file mode 100644 index 000000000..e69de29bb diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 0e12df7ce..e2b45e30c 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -310,6 +310,7 @@ class StorageClient(object): _base_url: DecodedURL _swissnum: bytes _treq: Union[treq, StubTreq, HTTPClient] + _pool: HTTPConnectionPool _clock: IReactorTime @classmethod @@ -339,7 +340,7 @@ class StorageClient(object): ) https_url = DecodedURL().replace(scheme="https", host=nurl.host, port=nurl.port) - return cls(https_url, swissnum, treq_client, reactor) + return cls(https_url, swissnum, treq_client, pool, reactor) def relative_url(self, path: str) -> DecodedURL: """Get a URL relative to the base URL.""" @@ -479,6 +480,10 @@ class StorageClient(object): ).read() raise ClientException(response.code, response.phrase, data) + def shutdown(self) -> Deferred: + """Shutdown any connections.""" + return self._pool.closeCachedConnections() + @define(hash=True) class StorageClientGeneral(object): diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index a40e98b03..94aae43f6 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -1271,6 +1271,11 @@ class HTTPNativeStorageServer(service.MultiService): if self._lc.running: self._lc.stop() self._failed_to_connect("shut down") + + maybe_storage_server = self.get_storage_server() + if maybe_storage_server is not None: + result.addCallback(lambda _: maybe_storage_server._http_client.shutdown()) + return result From ba9946e6ea863b47f5b6a544cf6b70db49dd4bf6 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 10 May 2023 16:34:02 -0400 Subject: [PATCH 1809/2309] Fix tests. --- src/allmydata/storage/http_client.py | 5 +++-- src/allmydata/test/test_storage_http.py | 5 ++++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index e2b45e30c..9c4a5538c 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -310,7 +310,7 @@ class StorageClient(object): _base_url: DecodedURL _swissnum: bytes _treq: Union[treq, StubTreq, HTTPClient] - _pool: HTTPConnectionPool + _pool: Optional[HTTPConnectionPool] _clock: IReactorTime @classmethod @@ -482,7 +482,8 @@ class StorageClient(object): def shutdown(self) -> Deferred: """Shutdown any connections.""" - return self._pool.closeCachedConnections() + if self._pool is not None: + return self._pool.closeCachedConnections() @define(hash=True) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index eca2be1c1..64491f7ae 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -331,6 +331,7 @@ class CustomHTTPServerTests(SyncTestCase): DecodedURL.from_text("http://127.0.0.1"), SWISSNUM_FOR_TEST, treq=treq, + pool=None, # We're using a Treq private API to get the reactor, alas, but only # in a test, so not going to worry about it too much. This would be # fixed if https://github.com/twisted/treq/issues/226 were ever @@ -512,6 +513,7 @@ class HttpTestFixture(Fixture): DecodedURL.from_text("http://127.0.0.1"), SWISSNUM_FOR_TEST, treq=self.treq, + pool=None, clock=self.clock, ) @@ -624,6 +626,7 @@ class GenericHTTPAPITests(SyncTestCase): DecodedURL.from_text("http://127.0.0.1"), b"something wrong", treq=StubTreq(self.http.http_server.get_resource()), + pool=None, clock=self.http.clock, ) ) @@ -1455,7 +1458,7 @@ class SharedImmutableMutableTestsMixin: self.client.advise_corrupt_share(storage_index, 13, reason) ) - for (si, share_number) in [(storage_index, 11), (urandom(16), 13)]: + for si, share_number in [(storage_index, 11), (urandom(16), 13)]: with assert_fails_with_http_code(self, http.NOT_FOUND): self.http.result_of_with_flush( self.client.advise_corrupt_share(si, share_number, reason) From 2ec1c1e43e0bae315897e9d6bb0d7e2df2640cb0 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 10 May 2023 17:23:15 -0400 Subject: [PATCH 1810/2309] Shut down alice. --- integration/conftest.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/integration/conftest.py b/integration/conftest.py index bf04e4424..6892b33a7 100644 --- a/integration/conftest.py +++ b/integration/conftest.py @@ -401,9 +401,6 @@ def alice( reactor, request, temp_dir, introducer_furl, flog_gatherer, "alice", web_port="tcp:9980:interface=localhost", storage=False, - # We're going to kill this ourselves, so no need for finalizer to - # do it: - finalize=False, ) ) pytest_twisted.blockon(await_client_ready(process)) From f5acaea134b017a3e9a0b0fa537836b268ae06a3 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 11 May 2023 09:05:58 -0400 Subject: [PATCH 1811/2309] bump the version of klein in the nix-based builds --- nix/klein.nix | 9 +++++++++ nix/python-overrides.nix | 6 ++++++ 2 files changed, 15 insertions(+) create mode 100644 nix/klein.nix diff --git a/nix/klein.nix b/nix/klein.nix new file mode 100644 index 000000000..be4426465 --- /dev/null +++ b/nix/klein.nix @@ -0,0 +1,9 @@ +{ klein, fetchPypi }: +klein.overrideAttrs (old: rec { + pname = "klein"; + version = "23.5.0"; + src = fetchPypi { + inherit pname version; + sha256 = "sha256-kGkSt6tBDZp/NRICg5w81zoqwHe9AHHIYcMfDu92Aoc="; + }; +}) diff --git a/nix/python-overrides.nix b/nix/python-overrides.nix index 87c42ad58..032b427ae 100644 --- a/nix/python-overrides.nix +++ b/nix/python-overrides.nix @@ -21,6 +21,12 @@ in { pycddl = self.callPackage ./pycddl.nix { }; txi2p = self.callPackage ./txi2p.nix { }; + # Update the version of klein. + klein = self.callPackage ./klein.nix { + # Avoid infinite recursion. + inherit (super) klein; + }; + # collections-extended is currently broken for Python 3.11 in nixpkgs but # we know where a working version lives. collections-extended = self.callPackage ./collections-extended.nix { From f83b73b5f31132340bb043b4dfef8088ce4403bd Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 16 May 2023 10:44:34 -0400 Subject: [PATCH 1812/2309] Make Tor provider available at the right place to enable it for HTTP storage client connections. --- src/allmydata/client.py | 8 ++++---- src/allmydata/storage_client.py | 12 +++++++++++- src/allmydata/util/tor_provider.py | 10 +++------- 3 files changed, 18 insertions(+), 12 deletions(-) diff --git a/src/allmydata/client.py b/src/allmydata/client.py index 1d959cb98..cb1fb9fa4 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -10,7 +10,6 @@ import weakref from typing import Optional, Iterable from base64 import urlsafe_b64encode from functools import partial -# On Python 2 this will be the backported package: from configparser import NoSectionError from foolscap.furl import ( @@ -47,7 +46,7 @@ from allmydata.util.encodingutil import get_filesystem_encoding from allmydata.util.abbreviate import parse_abbreviated_size from allmydata.util.time_format import parse_duration, parse_date from allmydata.util.i2p_provider import create as create_i2p_provider -from allmydata.util.tor_provider import create as create_tor_provider +from allmydata.util.tor_provider import create as create_tor_provider, _Provider as TorProvider from allmydata.stats import StatsProvider from allmydata.history import History from allmydata.interfaces import ( @@ -268,7 +267,7 @@ def create_client_from_config(config, _client_factory=None, _introducer_factory= introducer_clients = create_introducer_clients(config, main_tub, _introducer_factory) storage_broker = create_storage_farm_broker( config, default_connection_handlers, foolscap_connection_handlers, - tub_options, introducer_clients + tub_options, introducer_clients, tor_provider ) client = _client_factory( @@ -464,7 +463,7 @@ def create_introducer_clients(config, main_tub, _introducer_factory=None): return introducer_clients -def create_storage_farm_broker(config: _Config, default_connection_handlers, foolscap_connection_handlers, tub_options, introducer_clients): +def create_storage_farm_broker(config: _Config, default_connection_handlers, foolscap_connection_handlers, tub_options, introducer_clients, tor_provider: Optional[TorProvider]): """ Create a StorageFarmBroker object, for use by Uploader/Downloader (and everybody else who wants to use storage servers) @@ -500,6 +499,7 @@ def create_storage_farm_broker(config: _Config, default_connection_handlers, foo tub_maker=tub_creator, node_config=config, storage_client_config=storage_client_config, + tor_provider=tor_provider, ) for ic in introducer_clients: sb.use_introducer(ic) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index a40e98b03..af14201fb 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -77,6 +77,7 @@ from allmydata.grid_manager import ( from allmydata.crypto import ( ed25519, ) +from allmydata.util.tor_provider import _Provider as TorProvider from allmydata.util import log, base32, connection_status from allmydata.util.assertutil import precondition from allmydata.util.observer import ObserverList @@ -202,6 +203,7 @@ class StorageFarmBroker(service.MultiService): tub_maker, node_config: _Config, storage_client_config=None, + tor_provider: Optional[TorProvider]=None, ): service.MultiService.__init__(self) assert permute_peers # False not implemented yet @@ -223,6 +225,7 @@ class StorageFarmBroker(service.MultiService): self.introducer_client = None self._threshold_listeners : list[tuple[float,defer.Deferred[Any]]]= [] # tuples of (threshold, Deferred) self._connected_high_water_mark = 0 + self._tor_provider = tor_provider @log_call(action_type=u"storage-client:broker:set-static-servers") def set_static_servers(self, servers): @@ -315,6 +318,7 @@ class StorageFarmBroker(service.MultiService): server_id, server["ann"], grid_manager_verifier=gm_verifier, + tor_provider=tor_provider ) s.on_status_changed(lambda _: self._got_connection()) return s @@ -1049,7 +1053,7 @@ class HTTPNativeStorageServer(service.MultiService): "connected". """ - def __init__(self, server_id: bytes, announcement, reactor=reactor, grid_manager_verifier=None): + def __init__(self, server_id: bytes, announcement, reactor=reactor, grid_manager_verifier=None, tor_provider: Optional[TorProvider]=None): service.MultiService.__init__(self) assert isinstance(server_id, bytes) self._server_id = server_id @@ -1057,6 +1061,8 @@ class HTTPNativeStorageServer(service.MultiService): self._on_status_changed = ObserverList() self._reactor = reactor self._grid_manager_verifier = grid_manager_verifier + self._tor_provider = tor_provider + furl = announcement["anonymous-storage-FURL"].encode("utf-8") ( self._nickname, @@ -1242,6 +1248,8 @@ class HTTPNativeStorageServer(service.MultiService): pool = HTTPConnectionPool(reactor, persistent=False) pool.retryAutomatically = False return StorageClientGeneral( + # TODO if Tor client connections are enabled, use an Agent + # created via tor. StorageClient.from_nurl(nurl, reactor, pool) ).get_version() @@ -1249,6 +1257,8 @@ class HTTPNativeStorageServer(service.MultiService): # If we've gotten this far, we've found a working NURL. self._istorage_server = _HTTPStorageServer.from_http_client( + # TODO if Tor client connections are enabled, use an Agent + # created via tor. StorageClient.from_nurl(nurl, reactor) ) return self._istorage_server diff --git a/src/allmydata/util/tor_provider.py b/src/allmydata/util/tor_provider.py index 4ca19c01c..57bb3a83d 100644 --- a/src/allmydata/util/tor_provider.py +++ b/src/allmydata/util/tor_provider.py @@ -2,14 +2,10 @@ """ Ported to Python 3. """ -from __future__ import absolute_import, print_function, with_statement -from __future__ import division -from __future__ import unicode_literals -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 +from __future__ import annotations +from typing import Optional import os from zope.interface import ( @@ -41,7 +37,7 @@ def _import_txtorcon(): except ImportError: # pragma: no cover return None -def create(reactor, config, import_tor=None, import_txtorcon=None): +def create(reactor, config, import_tor=None, import_txtorcon=None) -> Optional[_Provider]: """ Create a new _Provider service (this is an IService so must be hooked up to a parent or otherwise started). From 3cf03a5c339ab6309016657ab40d7809e1ea1de1 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 18 May 2023 09:28:58 -0400 Subject: [PATCH 1813/2309] More glue to connect Tor up to the HTTP-based storage client. --- src/allmydata/client.py | 1 + src/allmydata/storage/http_client.py | 25 +++++++++++++++++++++---- src/allmydata/storage_client.py | 23 +++++++++++++++++++++-- 3 files changed, 43 insertions(+), 6 deletions(-) diff --git a/src/allmydata/client.py b/src/allmydata/client.py index cb1fb9fa4..e85ed4fe2 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -499,6 +499,7 @@ def create_storage_farm_broker(config: _Config, default_connection_handlers, foo tub_maker=tub_creator, node_config=config, storage_client_config=storage_client_config, + default_connection_handlers=default_connection_handlers, tor_provider=tor_provider, ) for ic in introducer_clients: diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 549fc9719..57ac6706b 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -15,6 +15,7 @@ from typing import ( TypedDict, Set, Dict, + Callable, ) from base64 import b64encode from io import BytesIO @@ -31,7 +32,7 @@ from collections_extended import RangeMap from werkzeug.datastructures import Range, ContentRange from twisted.web.http_headers import Headers from twisted.web import http -from twisted.web.iweb import IPolicyForHTTPS, IResponse +from twisted.web.iweb import IPolicyForHTTPS, IResponse, IAgent from twisted.internet.defer import inlineCallbacks, Deferred, succeed from twisted.internet.interfaces import ( IOpenSSLClientConnectionCreator, @@ -336,7 +337,13 @@ class StorageClient(object): @classmethod def from_nurl( - cls, nurl: DecodedURL, reactor, pool: Optional[HTTPConnectionPool] = None + cls, + nurl: DecodedURL, + reactor, + pool: Optional[HTTPConnectionPool] = None, + agent_factory: Optional[ + Callable[[object, IPolicyForHTTPS, HTTPConnectionPool], IAgent] + ] = None, ) -> StorageClient: """ Create a ``StorageClient`` for the given NURL. @@ -352,11 +359,21 @@ class StorageClient(object): if cls.TEST_MODE_REGISTER_HTTP_POOL is not None: cls.TEST_MODE_REGISTER_HTTP_POOL(pool) + def default_agent_factory( + reactor: object, + tls_context_factory: IPolicyForHTTPS, + pool: HTTPConnectionPool, + ) -> IAgent: + return Agent(reactor, tls_context_factory, pool=pool) + + if agent_factory is None: + agent_factory = default_agent_factory + treq_client = HTTPClient( - Agent( + agent_factory( reactor, _StorageClientHTTPSPolicy(expected_spki_hash=certificate_hash), - pool=pool, + pool, ) ) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index af14201fb..535bc3ffa 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -51,6 +51,7 @@ from zope.interface import ( ) from twisted.python.failure import Failure from twisted.web import http +from twisted.web.iweb import IAgent, IPolicyForHTTPS from twisted.internet.task import LoopingCall from twisted.internet import defer, reactor from twisted.application import service @@ -203,9 +204,13 @@ class StorageFarmBroker(service.MultiService): tub_maker, node_config: _Config, storage_client_config=None, + default_connection_handlers=None, tor_provider: Optional[TorProvider]=None, ): service.MultiService.__init__(self) + if default_connection_handlers is None: + default_connection_handlers = {"tcp": "tcp"} + assert permute_peers # False not implemented yet self.permute_peers = permute_peers self._tub_maker = tub_maker @@ -226,6 +231,7 @@ class StorageFarmBroker(service.MultiService): self._threshold_listeners : list[tuple[float,defer.Deferred[Any]]]= [] # tuples of (threshold, Deferred) self._connected_high_water_mark = 0 self._tor_provider = tor_provider + self._default_connection_handlers = default_connection_handlers @log_call(action_type=u"storage-client:broker:set-static-servers") def set_static_servers(self, servers): @@ -318,7 +324,8 @@ class StorageFarmBroker(service.MultiService): server_id, server["ann"], grid_manager_verifier=gm_verifier, - tor_provider=tor_provider + default_connection_handlers=self._default_connection_handlers, + tor_provider=self._tor_provider ) s.on_status_changed(lambda _: self._got_connection()) return s @@ -1053,7 +1060,7 @@ class HTTPNativeStorageServer(service.MultiService): "connected". """ - def __init__(self, server_id: bytes, announcement, reactor=reactor, grid_manager_verifier=None, tor_provider: Optional[TorProvider]=None): + def __init__(self, server_id: bytes, announcement, default_connection_handlers: dict[str,str], reactor=reactor, grid_manager_verifier=None, tor_provider: Optional[TorProvider]=None): service.MultiService.__init__(self) assert isinstance(server_id, bytes) self._server_id = server_id @@ -1062,6 +1069,7 @@ class HTTPNativeStorageServer(service.MultiService): self._reactor = reactor self._grid_manager_verifier = grid_manager_verifier self._tor_provider = tor_provider + self._default_connection_handlers = default_connection_handlers furl = announcement["anonymous-storage-FURL"].encode("utf-8") ( @@ -1224,6 +1232,17 @@ class HTTPNativeStorageServer(service.MultiService): self._connecting_deferred = connecting return connecting + def _agent_factory(self) -> Optional[Callable[[object, IPolicyForHTTPS, HTTPConnectionPool],IAgent]]: + """Return a factory for ``twisted.web.iweb.IAgent``.""" + # TODO default_connection_handlers should really be an object, not a dict... + handler = self._default_connection_handlers["tcp"] + if handler == "tcp": + return None + if handler == "tor": + raise RuntimeError("TODO implement this next") + else: + raise RuntimeError(f"Unsupported tcp connection {handler}") + @async_to_deferred async def _pick_server_and_get_version(self): """ From ffecdf8c773126a4feba6556d83a1d68dd6a70d6 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 18 May 2023 10:18:46 -0400 Subject: [PATCH 1814/2309] Switch to non-deprecated API. --- src/allmydata/test/test_tor_provider.py | 8 ++++---- src/allmydata/util/tor_provider.py | 16 +++++++--------- 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/src/allmydata/test/test_tor_provider.py b/src/allmydata/test/test_tor_provider.py index 86d54803a..eebb1ceef 100644 --- a/src/allmydata/test/test_tor_provider.py +++ b/src/allmydata/test/test_tor_provider.py @@ -94,16 +94,16 @@ class LaunchTor(unittest.TestCase): reactor = object() private_dir = "private" txtorcon = mock.Mock() - tpp = mock.Mock - tpp.tor_protocol = mock.Mock() - txtorcon.launch_tor = mock.Mock(return_value=tpp) + tor = mock.Mock + tor.protocol = mock.Mock() + txtorcon.launch = mock.Mock(return_value=tor) with mock.patch("allmydata.util.tor_provider.allocate_tcp_port", return_value=999999): d = tor_provider._launch_tor(reactor, tor_executable, private_dir, txtorcon) tor_control_endpoint, tor_control_proto = self.successResultOf(d) - self.assertIs(tor_control_proto, tpp.tor_protocol) + self.assertIs(tor_control_proto, tor.protocol) def test_launch(self): return self._do_test_launch(None) diff --git a/src/allmydata/util/tor_provider.py b/src/allmydata/util/tor_provider.py index 57bb3a83d..4dab6b866 100644 --- a/src/allmydata/util/tor_provider.py +++ b/src/allmydata/util/tor_provider.py @@ -100,18 +100,16 @@ def _launch_tor(reactor, tor_executable, private_dir, txtorcon): # us against one Tor being on $PATH at create-node time, but then a # different Tor being present at node startup. OTOH, maybe we don't # need to worry about it. - tor_config = txtorcon.TorConfig() - tor_config.DataDirectory = data_directory(private_dir) # unix-domain control socket - tor_config.ControlPort = "unix:" + os.path.join(private_dir, "tor.control") - tor_control_endpoint_desc = tor_config.ControlPort + tor_control_endpoint_desc = "unix:" + os.path.join(private_dir, "tor.control") - tor_config.SOCKSPort = allocate_tcp_port() - - tpp = yield txtorcon.launch_tor( - tor_config, reactor, + tor = yield txtorcon.launch( + reactor, + control_port=tor_control_endpoint_desc, + data_directory=data_directory(private_dir), tor_binary=tor_executable, + socks_port=allocate_tcp_port(), # can be useful when debugging; mirror Tor's output to ours # stdout=sys.stdout, # stderr=sys.stderr, @@ -119,7 +117,7 @@ def _launch_tor(reactor, tor_executable, private_dir, txtorcon): # now tor is launched and ready to be spoken to # as a side effect, we've got an ITorControlProtocol ready to go - tor_control_proto = tpp.tor_protocol + tor_control_proto = tor.protocol # How/when to shut down the new process? for normal usage, the child # tor will exit when it notices its parent (us) quit. Unit tests will From 34accd694c565c9d9a1c99b2bdf4069ba644085f Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 18 May 2023 10:51:31 -0400 Subject: [PATCH 1815/2309] Refactor to return something more useful. --- src/allmydata/test/test_tor_provider.py | 34 ++++++++++++------------- src/allmydata/util/tor_provider.py | 17 ++++++++----- 2 files changed, 27 insertions(+), 24 deletions(-) diff --git a/src/allmydata/test/test_tor_provider.py b/src/allmydata/test/test_tor_provider.py index eebb1ceef..295b258f0 100644 --- a/src/allmydata/test/test_tor_provider.py +++ b/src/allmydata/test/test_tor_provider.py @@ -1,15 +1,8 @@ """ Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals - -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 +from collections import namedtuple import os from twisted.trial import unittest from twisted.internet import defer, error @@ -95,15 +88,14 @@ class LaunchTor(unittest.TestCase): private_dir = "private" txtorcon = mock.Mock() tor = mock.Mock - tor.protocol = mock.Mock() txtorcon.launch = mock.Mock(return_value=tor) with mock.patch("allmydata.util.tor_provider.allocate_tcp_port", return_value=999999): d = tor_provider._launch_tor(reactor, tor_executable, private_dir, txtorcon) - tor_control_endpoint, tor_control_proto = self.successResultOf(d) - self.assertIs(tor_control_proto, tor.protocol) + tor_control_endpoint, tor_result = self.successResultOf(d) + self.assertIs(tor_result, tor) def test_launch(self): return self._do_test_launch(None) @@ -161,6 +153,12 @@ class ConnectToTor(unittest.TestCase): return self._do_test_connect(None, False) +class FakeTor: + """Pretends to be a ``txtorcon.Tor`` instance.""" + def __init__(self): + self.protocol = object() + + class CreateOnion(unittest.TestCase): def test_no_txtorcon(self): with mock.patch("allmydata.util.tor_provider._import_txtorcon", @@ -171,6 +169,7 @@ class CreateOnion(unittest.TestCase): self.assertEqual(str(f.value), "Cannot create onion without txtorcon. " "Please 'pip install tahoe-lafs[tor]' to fix this.") + def _do_test_launch(self, executable): basedir = self.mktemp() os.mkdir(basedir) @@ -181,9 +180,9 @@ class CreateOnion(unittest.TestCase): if executable: args.append("--tor-executable=%s" % executable) cli_config = make_cli_config(basedir, *args) - protocol = object() + tor_instance = FakeTor() launch_tor = mock.Mock(return_value=defer.succeed(("control_endpoint", - protocol))) + tor_instance))) txtorcon = mock.Mock() ehs = mock.Mock() # This appears to be a native string in the real txtorcon object... @@ -204,8 +203,8 @@ class CreateOnion(unittest.TestCase): launch_tor.assert_called_with(reactor, executable, os.path.abspath(private_dir), txtorcon) txtorcon.EphemeralHiddenService.assert_called_with("3457 127.0.0.1:999999") - ehs.add_to_tor.assert_called_with(protocol) - ehs.remove_from_tor.assert_called_with(protocol) + ehs.add_to_tor.assert_called_with(tor_instance.protocol) + ehs.remove_from_tor.assert_called_with(tor_instance.protocol) expected = {"launch": "true", "onion": "true", @@ -587,13 +586,14 @@ class Provider_Service(unittest.TestCase): txtorcon = mock.Mock() with mock_txtorcon(txtorcon): p = tor_provider.create(reactor, cfg) + tor_instance = FakeTor() tor_state = mock.Mock() - tor_state.protocol = object() + tor_state.protocol = tor_instance.protocol ehs = mock.Mock() ehs.add_to_tor = mock.Mock(return_value=defer.succeed(None)) ehs.remove_from_tor = mock.Mock(return_value=defer.succeed(None)) txtorcon.EphemeralHiddenService = mock.Mock(return_value=ehs) - launch_tor = mock.Mock(return_value=defer.succeed((None,tor_state.protocol))) + launch_tor = mock.Mock(return_value=defer.succeed((None,tor_instance))) with mock.patch("allmydata.util.tor_provider._launch_tor", launch_tor): d = p.startService() diff --git a/src/allmydata/util/tor_provider.py b/src/allmydata/util/tor_provider.py index 4dab6b866..9daf302cf 100644 --- a/src/allmydata/util/tor_provider.py +++ b/src/allmydata/util/tor_provider.py @@ -23,6 +23,7 @@ from ..interfaces import ( IAddressFamily, ) + def _import_tor(): try: from foolscap.connections import tor @@ -94,6 +95,10 @@ def _try_to_connect(reactor, endpoint_desc, stdout, txtorcon): @inlineCallbacks def _launch_tor(reactor, tor_executable, private_dir, txtorcon): + """ + Launches Tor, returns a corresponding ``(control endpoint string, + txtorcon.Tor instance)`` tuple. + """ # TODO: handle default tor-executable # TODO: it might be a good idea to find exactly which Tor we used, # and record it's absolute path into tahoe.cfg . This would protect @@ -115,10 +120,6 @@ def _launch_tor(reactor, tor_executable, private_dir, txtorcon): # stderr=sys.stderr, ) - # now tor is launched and ready to be spoken to - # as a side effect, we've got an ITorControlProtocol ready to go - tor_control_proto = tor.protocol - # How/when to shut down the new process? for normal usage, the child # tor will exit when it notices its parent (us) quit. Unit tests will # mock out txtorcon.launch_tor(), so there will never be a real Tor @@ -128,7 +129,8 @@ def _launch_tor(reactor, tor_executable, private_dir, txtorcon): # (because it's a TorProcessProtocol) which returns a Deferred # that fires when Tor has actually exited. - returnValue((tor_control_endpoint_desc, tor_control_proto)) + returnValue((tor_control_endpoint_desc, tor)) + @inlineCallbacks def _connect_to_tor(reactor, cli_config, txtorcon): @@ -163,8 +165,9 @@ def create_config(reactor, cli_config): if tor_executable: tahoe_config_tor["tor.executable"] = tor_executable print("launching Tor (to allocate .onion address)..", file=stdout) - (_, tor_control_proto) = yield _launch_tor( + (_, tor) = yield _launch_tor( reactor, tor_executable, private_dir, txtorcon) + tor_control_proto = tor.protocol print("Tor launched", file=stdout) else: print("connecting to Tor (to allocate .onion address)..", file=stdout) @@ -288,7 +291,7 @@ class _Provider(service.MultiService): returnValue(tor_control_endpoint) def _get_launched_tor(self, reactor): - # this fires with a tuple of (control_endpoint, tor_protocol) + # this fires with a tuple of (control_endpoint, txtorcon.Tor instance) if not self._tor_launched: self._tor_launched = OneShotObserverList() private_dir = self._config.get_config_path("private") From 47991f23fa5cb40eb60283d1503c0afaceae1272 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 18 May 2023 11:05:38 -0400 Subject: [PATCH 1816/2309] More refactoring to make it easier to get a txtorcon.Tor instance. --- src/allmydata/test/test_tor_provider.py | 13 ++++++------- src/allmydata/util/tor_provider.py | 13 ++++++++----- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/src/allmydata/test/test_tor_provider.py b/src/allmydata/test/test_tor_provider.py index 295b258f0..bd40a9b4a 100644 --- a/src/allmydata/test/test_tor_provider.py +++ b/src/allmydata/test/test_tor_provider.py @@ -628,9 +628,8 @@ class Provider_Service(unittest.TestCase): txtorcon = mock.Mock() with mock_txtorcon(txtorcon): p = tor_provider.create(reactor, cfg) - tor_state = mock.Mock() - tor_state.protocol = object() - txtorcon.build_tor_connection = mock.Mock(return_value=tor_state) + tor_instance = FakeTor() + txtorcon.connect = mock.Mock(return_value=tor_instance) ehs = mock.Mock() ehs.add_to_tor = mock.Mock(return_value=defer.succeed(None)) ehs.remove_from_tor = mock.Mock(return_value=defer.succeed(None)) @@ -642,12 +641,12 @@ class Provider_Service(unittest.TestCase): yield flushEventualQueue() self.successResultOf(d) self.assertIs(p._onion_ehs, ehs) - self.assertIs(p._onion_tor_control_proto, tor_state.protocol) + self.assertIs(p._onion_tor_control_proto, tor_instance.protocol) cfs.assert_called_with(reactor, "ep_desc") - txtorcon.build_tor_connection.assert_called_with(tcep) + txtorcon.connect.assert_called_with(reactor, tcep) txtorcon.EphemeralHiddenService.assert_called_with("456 127.0.0.1:123", b"private key") - ehs.add_to_tor.assert_called_with(tor_state.protocol) + ehs.add_to_tor.assert_called_with(tor_instance.protocol) yield p.stopService() - ehs.remove_from_tor.assert_called_with(tor_state.protocol) + ehs.remove_from_tor.assert_called_with(tor_instance.protocol) diff --git a/src/allmydata/util/tor_provider.py b/src/allmydata/util/tor_provider.py index 9daf302cf..ad36b6986 100644 --- a/src/allmydata/util/tor_provider.py +++ b/src/allmydata/util/tor_provider.py @@ -322,17 +322,20 @@ class _Provider(service.MultiService): require("external_port") require("private_key_file") - @inlineCallbacks - def _start_onion(self, reactor): + def _get_tor_instance(self, reactor: object): + """Return a ``Deferred`` that fires with a ``txtorcon.Tor`` instance.""" # launch tor, if necessary if self._get_tor_config("launch", False, boolean=True): - (_, tor_control_proto) = yield self._get_launched_tor(reactor) + return self._get_launched_tor(reactor).addCallback(lambda t: t[1]) else: controlport = self._get_tor_config("control.port", None) tcep = clientFromString(reactor, controlport) - tor_state = yield self._txtorcon.build_tor_connection(tcep) - tor_control_proto = tor_state.protocol + return self._txtorcon.connect(reactor, tcep) + @inlineCallbacks + def _start_onion(self, reactor): + tor_instance = yield self._get_tor_instance(reactor) + tor_control_proto = tor_instance.protocol local_port = int(self._get_tor_config("onion.local_port")) external_port = int(self._get_tor_config("onion.external_port")) From 2e0e0467fb56f655418cb2131545eb8e2e286432 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 18 May 2023 11:14:51 -0400 Subject: [PATCH 1817/2309] Hook up HTTP storage client Tor support. --- src/allmydata/storage_client.py | 11 ++++++++--- src/allmydata/util/tor_provider.py | 4 ++-- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 535bc3ffa..f6b3972f5 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -1232,16 +1232,21 @@ class HTTPNativeStorageServer(service.MultiService): self._connecting_deferred = connecting return connecting - def _agent_factory(self) -> Optional[Callable[[object, IPolicyForHTTPS, HTTPConnectionPool],IAgent]]: + async def _agent_factory(self) -> Optional[Callable[[object, IPolicyForHTTPS, HTTPConnectionPool],IAgent]]: """Return a factory for ``twisted.web.iweb.IAgent``.""" # TODO default_connection_handlers should really be an object, not a dict... handler = self._default_connection_handlers["tcp"] if handler == "tcp": return None if handler == "tor": - raise RuntimeError("TODO implement this next") + tor_instance = await self._tor_provider.get_tor_instance(self._reactor) + + def agent_factory(reactor: object, tls_context_factory: IPolicyForHTTPS, pool: HTTPConnectionPool) -> IAgent: + assert reactor == self._reactor + return tor_instance.web_agent(pool=pool, tls_context_factory=tls_context_factory) + return agent_factory else: - raise RuntimeError(f"Unsupported tcp connection {handler}") + raise RuntimeError(f"Unsupported tcp connection handler: {handler}") @async_to_deferred async def _pick_server_and_get_version(self): diff --git a/src/allmydata/util/tor_provider.py b/src/allmydata/util/tor_provider.py index ad36b6986..aaf43db73 100644 --- a/src/allmydata/util/tor_provider.py +++ b/src/allmydata/util/tor_provider.py @@ -322,7 +322,7 @@ class _Provider(service.MultiService): require("external_port") require("private_key_file") - def _get_tor_instance(self, reactor: object): + def get_tor_instance(self, reactor: object): """Return a ``Deferred`` that fires with a ``txtorcon.Tor`` instance.""" # launch tor, if necessary if self._get_tor_config("launch", False, boolean=True): @@ -334,7 +334,7 @@ class _Provider(service.MultiService): @inlineCallbacks def _start_onion(self, reactor): - tor_instance = yield self._get_tor_instance(reactor) + tor_instance = yield self.get_tor_instance(reactor) tor_control_proto = tor_instance.protocol local_port = int(self._get_tor_config("onion.local_port")) external_port = int(self._get_tor_config("onion.external_port")) From 0ccee4e958f9a70148aff3bf6bc82a6fc78a3a1b Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 19 May 2023 13:59:18 -0400 Subject: [PATCH 1818/2309] Hook up the Tor-based Agent when necessary. --- src/allmydata/storage_client.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index f6b3972f5..15303265f 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -1266,24 +1266,22 @@ class HTTPNativeStorageServer(service.MultiService): # version() calls before we are live talking to a server, it could only # be one. See https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3992 + agent_factory = await self._agent_factory() + def request(reactor, nurl: DecodedURL): # Since we're just using this one off to check if the NURL # works, no need for persistent pool or other fanciness. pool = HTTPConnectionPool(reactor, persistent=False) pool.retryAutomatically = False return StorageClientGeneral( - # TODO if Tor client connections are enabled, use an Agent - # created via tor. - StorageClient.from_nurl(nurl, reactor, pool) + StorageClient.from_nurl(nurl, reactor, pool, agent_factory=agent_factory) ).get_version() nurl = await _pick_a_http_server(reactor, self._nurls, request) # If we've gotten this far, we've found a working NURL. self._istorage_server = _HTTPStorageServer.from_http_client( - # TODO if Tor client connections are enabled, use an Agent - # created via tor. - StorageClient.from_nurl(nurl, reactor) + StorageClient.from_nurl(nurl, reactor, agent_factory=agent_factory) ) return self._istorage_server From 83d8efbb62ce21cf10303b63d0f92badd0f3dfd3 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 19 May 2023 13:59:29 -0400 Subject: [PATCH 1819/2309] Require the appropriate version of txtorcon. --- setup.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/setup.py b/setup.py index 0453fa63f..ec4936645 100644 --- a/setup.py +++ b/setup.py @@ -164,10 +164,9 @@ setup_requires = [ ] tor_requires = [ - # This is exactly what `foolscap[tor]` means but pip resolves the pair of - # dependencies "foolscap[i2p] foolscap[tor]" to "foolscap[i2p]" so we lose - # this if we don't declare it ourselves! - "txtorcon >= 0.17.0", + # 23.5 added support for custom TLS contexts in web_agent(), which is + # needed for the HTTP storage client to run over Tor. + "txtorcon >= 23.5.0", ] i2p_requires = [ From a1e00ffc3f5eb359bf10ce05fb279fb3ac3739d9 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 19 May 2023 13:59:43 -0400 Subject: [PATCH 1820/2309] Add a test that triggers client-side HTTP storage client to use Tor. --- integration/test_tor.py | 51 +++++++++++++++++++++++++++++--------- integration/util.py | 54 +++++++++++++++++++++++------------------ 2 files changed, 71 insertions(+), 34 deletions(-) diff --git a/integration/test_tor.py b/integration/test_tor.py index 10e326e46..35e1581eb 100644 --- a/integration/test_tor.py +++ b/integration/test_tor.py @@ -19,6 +19,7 @@ from allmydata.test.common import ( write_introducer, ) from allmydata.client import read_config +from allmydata.util.deferredutil import async_to_deferred # see "conftest.py" for the fixtures (e.g. "tor_network") @@ -31,13 +32,26 @@ if sys.platform.startswith('win'): @pytest_twisted.inlineCallbacks def test_onion_service_storage(reactor, request, temp_dir, flog_gatherer, tor_network, tor_introducer_furl): + """ + Two nodes and an introducer all configured to use Tahoe. + + The two nodes can talk to the introducer and each other: we upload to one + node, read from the other. + """ carol = yield _create_anonymous_node(reactor, 'carol', 8008, request, temp_dir, flog_gatherer, tor_network, tor_introducer_furl) dave = yield _create_anonymous_node(reactor, 'dave', 8009, request, temp_dir, flog_gatherer, tor_network, tor_introducer_furl) yield util.await_client_ready(carol, minimum_number_of_servers=2, timeout=600) yield util.await_client_ready(dave, minimum_number_of_servers=2, timeout=600) + yield upload_to_one_download_from_the_other(reactor, temp_dir, carol, dave) + + +@async_to_deferred +async def upload_to_one_download_from_the_other(reactor, temp_dir, upload_to: util.TahoeProcess, download_from: util.TahoeProcess): + """ + Ensure both nodes are connected to "a grid" by uploading something via one + node, and retrieve it using the other. + """ - # ensure both nodes are connected to "a grid" by uploading - # something via carol, and retrieve it using dave. gold_path = join(temp_dir, "gold") with open(gold_path, "w") as f: f.write( @@ -54,12 +68,12 @@ def test_onion_service_storage(reactor, request, temp_dir, flog_gatherer, tor_ne sys.executable, ( sys.executable, '-b', '-m', 'allmydata.scripts.runner', - '-d', join(temp_dir, 'carol'), + '-d', upload_to.node_dir, 'put', gold_path, ), env=environ, ) - yield proto.done + await proto.done cap = proto.output.getvalue().strip().split()[-1] print("capability: {}".format(cap)) @@ -69,19 +83,18 @@ def test_onion_service_storage(reactor, request, temp_dir, flog_gatherer, tor_ne sys.executable, ( sys.executable, '-b', '-m', 'allmydata.scripts.runner', - '-d', join(temp_dir, 'dave'), + '-d', download_from.node_dir, 'get', cap, ), env=environ, ) - yield proto.done - - dave_got = proto.output.getvalue().strip() - assert dave_got == open(gold_path, 'rb').read().strip() + await proto.done + download_got = proto.output.getvalue().strip() + assert download_got == open(gold_path, 'rb').read().strip() @pytest_twisted.inlineCallbacks -def _create_anonymous_node(reactor, name, control_port, request, temp_dir, flog_gatherer, tor_network, introducer_furl): +def _create_anonymous_node(reactor, name, control_port, request, temp_dir, flog_gatherer, tor_network, introducer_furl) -> util.TahoeProcess: node_dir = FilePath(temp_dir).child(name) web_port = "tcp:{}:interface=localhost".format(control_port + 2000) @@ -113,9 +126,9 @@ def _create_anonymous_node(reactor, name, control_port, request, temp_dir, flog_ # Which services should this client connect to? write_introducer(node_dir, "default", introducer_furl) + util.basic_node_configuration(request, flog_gatherer, node_dir.path) config = read_config(node_dir.path, "tub.port") - config.set_config("node", "log_gatherer.furl", flog_gatherer) config.set_config("tor", "onion", "true") config.set_config("tor", "onion.external_port", "3457") config.set_config("tor", "control.port", f"tcp:port={control_port}:host=127.0.0.1") @@ -125,3 +138,19 @@ def _create_anonymous_node(reactor, name, control_port, request, temp_dir, flog_ result = yield util._run_node(reactor, node_dir.path, request, None) print("okay, launched") return result + + +@pytest_twisted.inlineCallbacks +def test_anonymous_client(reactor, alice, request, temp_dir, flog_gatherer, tor_network, introducer_furl): + """ + A normal node (alice) and a normal introducer are configured, and one node + (carol) which is configured to be anonymous by talking via Tor. + + Carol should be able to communicate with alice. + + TODO how to ensure that carol is actually using Tor? + """ + carol = yield _create_anonymous_node(reactor, 'carol', 8008, request, temp_dir, flog_gatherer, tor_network, introducer_furl) + yield util.await_client_ready(carol, minimum_number_of_servers=2, timeout=600) + + yield upload_to_one_download_from_the_other(reactor, temp_dir, alice, carol) diff --git a/integration/util.py b/integration/util.py index cbc701fbc..402c14932 100644 --- a/integration/util.py +++ b/integration/util.py @@ -311,6 +311,36 @@ def _run_node(reactor, node_dir, request, magic_text, finalize=True): return d +def basic_node_configuration(request, flog_gatherer, node_dir: str): + """ + Setup common configuration options for a node, given a ``pytest`` request + fixture. + """ + config_path = join(node_dir, 'tahoe.cfg') + config = get_config(config_path) + set_config( + config, + u'node', + u'log_gatherer.furl', + flog_gatherer, + ) + force_foolscap = request.config.getoption("force_foolscap") + assert force_foolscap in (True, False) + set_config( + config, + 'storage', + 'force_foolscap', + str(force_foolscap), + ) + set_config( + config, + 'client', + 'force_foolscap', + str(force_foolscap), + ) + write_config(FilePath(config_path), config) + + def _create_node(reactor, request, temp_dir, introducer_furl, flog_gatherer, name, web_port, storage=True, magic_text=None, @@ -351,29 +381,7 @@ def _create_node(reactor, request, temp_dir, introducer_furl, flog_gatherer, nam created_d = done_proto.done def created(_): - config_path = join(node_dir, 'tahoe.cfg') - config = get_config(config_path) - set_config( - config, - u'node', - u'log_gatherer.furl', - flog_gatherer, - ) - force_foolscap = request.config.getoption("force_foolscap") - assert force_foolscap in (True, False) - set_config( - config, - 'storage', - 'force_foolscap', - str(force_foolscap), - ) - set_config( - config, - 'client', - 'force_foolscap', - str(force_foolscap), - ) - write_config(FilePath(config_path), config) + basic_node_configuration(request, flog_gatherer, node_dir) created_d.addCallback(created) d = Deferred() From 1b54853d3f8f18f686bc9d7363a79afa0b6935e5 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 19 May 2023 14:01:08 -0400 Subject: [PATCH 1821/2309] News file. --- newsfragments/4029.bugfix | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 newsfragments/4029.bugfix diff --git a/newsfragments/4029.bugfix b/newsfragments/4029.bugfix new file mode 100644 index 000000000..3ce4670ec --- /dev/null +++ b/newsfragments/4029.bugfix @@ -0,0 +1,2 @@ +The (still off-by-default) HTTP storage client will now use Tor when Tor-based client-side anonymity was requested. +Previously it would use normal TCP connections and not be anonymous. \ No newline at end of file From f5520fdf74ea5d9f3fafd3605e93a7181f98f5ec Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 22 May 2023 11:42:13 -0400 Subject: [PATCH 1822/2309] Better name. --- integration/test_tor.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/integration/test_tor.py b/integration/test_tor.py index 35e1581eb..ec5cc1bc4 100644 --- a/integration/test_tor.py +++ b/integration/test_tor.py @@ -144,13 +144,13 @@ def _create_anonymous_node(reactor, name, control_port, request, temp_dir, flog_ def test_anonymous_client(reactor, alice, request, temp_dir, flog_gatherer, tor_network, introducer_furl): """ A normal node (alice) and a normal introducer are configured, and one node - (carol) which is configured to be anonymous by talking via Tor. + (anonymoose) which is configured to be anonymous by talking via Tor. - Carol should be able to communicate with alice. + Anonymoose should be able to communicate with alice. - TODO how to ensure that carol is actually using Tor? + TODO how to ensure that anonymoose is actually using Tor? """ - carol = yield _create_anonymous_node(reactor, 'carol', 8008, request, temp_dir, flog_gatherer, tor_network, introducer_furl) - yield util.await_client_ready(carol, minimum_number_of_servers=2, timeout=600) + anonymoose = yield _create_anonymous_node(reactor, 'anonymoose', 8008, request, temp_dir, flog_gatherer, tor_network, introducer_furl) + yield util.await_client_ready(anonymoose, minimum_number_of_servers=2, timeout=600) - yield upload_to_one_download_from_the_other(reactor, temp_dir, alice, carol) + yield upload_to_one_download_from_the_other(reactor, temp_dir, alice, anonymoose) From 2741fb2b46afa54888d5591d9b757f639d8aae4d Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 22 May 2023 12:51:40 -0400 Subject: [PATCH 1823/2309] Don't persist state unnecessarily (and this appears to cause test failures) --- integration/conftest.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/integration/conftest.py b/integration/conftest.py index 6892b33a7..b29b9fe36 100644 --- a/integration/conftest.py +++ b/integration/conftest.py @@ -279,7 +279,7 @@ def introducer_furl(introducer, temp_dir): return furl -@pytest.fixture(scope='session') +@pytest.fixture @log_call( action_type=u"integration:tor:introducer", include_args=["temp_dir", "flog_gatherer"], @@ -342,7 +342,7 @@ def tor_introducer(reactor, temp_dir, flog_gatherer, request): return transport -@pytest.fixture(scope='session') +@pytest.fixture def tor_introducer_furl(tor_introducer, temp_dir): furl_fname = join(temp_dir, 'introducer_tor', 'private', 'introducer.furl') while not exists(furl_fname): From 1ed440812a86ab91289ff1ff21d4f2dfe6e4cb15 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 22 May 2023 13:00:20 -0400 Subject: [PATCH 1824/2309] Add a safety check. --- src/allmydata/storage/http_client.py | 8 ++++++++ src/allmydata/storage_client.py | 9 +++++++-- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 12d6c5feb..670d84be3 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -341,6 +341,9 @@ class StorageClient(object): cls, nurl: DecodedURL, reactor, + # TODO default_connection_handlers should really be a class, not a dict + # of strings... + default_connection_handlers: dict[str, str], pool: Optional[HTTPConnectionPool] = None, agent_factory: Optional[ Callable[[object, IPolicyForHTTPS, HTTPConnectionPool], IAgent] @@ -349,6 +352,11 @@ class StorageClient(object): """ Create a ``StorageClient`` for the given NURL. """ + # Safety check: if we're using normal TCP connections, we better not be + # configured for Tor or I2P. + if agent_factory is None: + assert default_connection_handlers["tcp"] == "tcp" + assert nurl.fragment == "v=1" assert nurl.scheme == "pb" swissnum = nurl.path[0].encode("ascii") diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 7bf00ad93..326b96ab4 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -1274,14 +1274,19 @@ class HTTPNativeStorageServer(service.MultiService): pool = HTTPConnectionPool(reactor, persistent=False) pool.retryAutomatically = False return StorageClientGeneral( - StorageClient.from_nurl(nurl, reactor, pool, agent_factory=agent_factory) + StorageClient.from_nurl( + nurl, reactor, self._default_connection_handlers, + pool=pool, agent_factory=agent_factory) ).get_version() nurl = await _pick_a_http_server(reactor, self._nurls, request) # If we've gotten this far, we've found a working NURL. self._istorage_server = _HTTPStorageServer.from_http_client( - StorageClient.from_nurl(nurl, reactor, agent_factory=agent_factory) + StorageClient.from_nurl( + nurl, reactor, self._default_connection_handlers, + agent_factory=agent_factory + ) ) return self._istorage_server From 084499dd4b53e08ffcab1a87cb5542b54c44998c Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 22 May 2023 13:02:58 -0400 Subject: [PATCH 1825/2309] Fix lint. --- src/allmydata/test/test_tor_provider.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/allmydata/test/test_tor_provider.py b/src/allmydata/test/test_tor_provider.py index bd40a9b4a..20f947d55 100644 --- a/src/allmydata/test/test_tor_provider.py +++ b/src/allmydata/test/test_tor_provider.py @@ -2,7 +2,6 @@ Ported to Python 3. """ -from collections import namedtuple import os from twisted.trial import unittest from twisted.internet import defer, error From 71cb357f45db29561b239fdf53d4eca714502e1f Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 22 May 2023 13:03:46 -0400 Subject: [PATCH 1826/2309] Upstream code should make sure this doesn't happen. --- src/allmydata/storage_client.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 326b96ab4..59f4242f6 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -1239,6 +1239,7 @@ class HTTPNativeStorageServer(service.MultiService): if handler == "tcp": return None if handler == "tor": + assert self._tor_provider is not None tor_instance = await self._tor_provider.get_tor_instance(self._reactor) def agent_factory(reactor: object, tls_context_factory: IPolicyForHTTPS, pool: HTTPConnectionPool) -> IAgent: From d15ea8cb52b87426c82fd1c70581ced87c7d391e Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 23 May 2023 13:24:29 -0400 Subject: [PATCH 1827/2309] Shutdown more immediately. --- src/allmydata/storage_client.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 94aae43f6..8541bb40c 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -1274,7 +1274,8 @@ class HTTPNativeStorageServer(service.MultiService): maybe_storage_server = self.get_storage_server() if maybe_storage_server is not None: - result.addCallback(lambda _: maybe_storage_server._http_client.shutdown()) + client_shutting_down = maybe_storage_server._http_client.shutdown() + result.addCallback(lambda _: client_shutting_down) return result From 1e46e36ee2cde0872b6fd0237f3586c489af0352 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 23 May 2023 13:46:32 -0400 Subject: [PATCH 1828/2309] More direct approach. --- src/allmydata/storage_client.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 8541bb40c..200f693c0 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -1272,9 +1272,8 @@ class HTTPNativeStorageServer(service.MultiService): self._lc.stop() self._failed_to_connect("shut down") - maybe_storage_server = self.get_storage_server() - if maybe_storage_server is not None: - client_shutting_down = maybe_storage_server._http_client.shutdown() + if self._istorage_server is not None: + client_shutting_down = self._istorage_server._http_client.shutdown() result.addCallback(lambda _: client_shutting_down) return result From 652c179602c36f71b63be502b1ee6e709f66102c Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 23 May 2023 14:08:03 -0400 Subject: [PATCH 1829/2309] Remove comment. --- src/allmydata/test/test_storage_http.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index df5dc300c..a6e6205f1 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -379,8 +379,6 @@ class CustomHTTPServerTests(SyncTestCase): ) self.assertEqual(response.code, 400) - # TODO test other garbage values - def test_authorization_enforcement(self): """ The requirement for secrets is enforced by the ``_authorized_route`` From 0e28c8ed4a88e39e500a8c3902e26a7787db650d Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 24 May 2023 08:54:56 -0400 Subject: [PATCH 1830/2309] bump the nix package of txtorcon --- nix/python-overrides.nix | 5 ++++- nix/txtorcon.nix | 9 +++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) create mode 100644 nix/txtorcon.nix diff --git a/nix/python-overrides.nix b/nix/python-overrides.nix index 032b427ae..d1c995e66 100644 --- a/nix/python-overrides.nix +++ b/nix/python-overrides.nix @@ -21,11 +21,14 @@ in { pycddl = self.callPackage ./pycddl.nix { }; txi2p = self.callPackage ./txi2p.nix { }; - # Update the version of klein. + # Some packages are of somewhat too-old versions - update them. klein = self.callPackage ./klein.nix { # Avoid infinite recursion. inherit (super) klein; }; + txtorcon = self.callPackage ./txtorcon.nix { + inherit (super) txtorcon; + }; # collections-extended is currently broken for Python 3.11 in nixpkgs but # we know where a working version lives. diff --git a/nix/txtorcon.nix b/nix/txtorcon.nix new file mode 100644 index 000000000..552c03fd0 --- /dev/null +++ b/nix/txtorcon.nix @@ -0,0 +1,9 @@ +{ txtorcon, fetchPypi }: +txtorcon.overrideAttrs (old: rec { + pname = "txtorcon"; + version = "23.5.0"; + src = fetchPypi { + inherit pname version; + hash = "sha256-k/2Aqd1QX2mNCGT+k9uLapwRRLX+uRUwggtw7YmCZRw="; + }; +}) From 96670ded65445aa07628816e6b7523e7d03b315e Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 1 Jun 2023 17:27:21 -0400 Subject: [PATCH 1831/2309] Switch to using officially support constants, now part of pyOpenSSL's public API. The cryptography APIs we were previously using were not supported and aren't available in all releases. --- newsfragments/3998.minor | 0 setup.py | 7 +++---- src/allmydata/storage/http_client.py | 13 +++++-------- 3 files changed, 8 insertions(+), 12 deletions(-) create mode 100644 newsfragments/3998.minor diff --git a/newsfragments/3998.minor b/newsfragments/3998.minor new file mode 100644 index 000000000..e69de29bb diff --git a/setup.py b/setup.py index 0453fa63f..c40e2dd2e 100644 --- a/setup.py +++ b/setup.py @@ -63,11 +63,10 @@ install_requires = [ # Twisted[conch] also depends on cryptography and Twisted[tls] # transitively depends on cryptography. So it's anyone's guess what # version of cryptography will *really* be installed. + "cryptography >= 2.6", - # * cryptography 40 broke constants we need; should really be using them - # * via pyOpenSSL; will be fixed in - # * https://github.com/pyca/pyopenssl/issues/1201 - "cryptography >= 2.6, < 40", + # * Used for custom HTTPS validation + "pyOpenSSL >= 23.2.0", # * The SFTP frontend depends on Twisted 11.0.0 to fix the SSH server # rekeying bug diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 5464b2e25..65f079aeb 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -46,7 +46,6 @@ import treq from treq.client import HTTPClient from treq.testing import StubTreq from OpenSSL import SSL -from cryptography.hazmat.bindings.openssl.binding import Binding from werkzeug.http import parse_content_range_header from .http_common import ( @@ -60,8 +59,6 @@ from .common import si_b2a, si_to_human_readable from ..util.hashutil import timing_safe_compare from ..util.deferredutil import async_to_deferred -_OPENSSL = Binding().lib - def _encode_si(si): # type: (bytes) -> str """Encode the storage index into Unicode string.""" @@ -256,11 +253,11 @@ class _TLSContextFactory(CertificateOptions): # not the usual TLS concerns about invalid CAs or revoked # certificates. things_are_ok = ( - _OPENSSL.X509_V_OK, - _OPENSSL.X509_V_ERR_CERT_NOT_YET_VALID, - _OPENSSL.X509_V_ERR_CERT_HAS_EXPIRED, - _OPENSSL.X509_V_ERR_DEPTH_ZERO_SELF_SIGNED_CERT, - _OPENSSL.X509_V_ERR_SELF_SIGNED_CERT_IN_CHAIN, + SSL.X509VerificationCodes.OK, + SSL.X509VerificationCodes.ERR_CERT_NOT_YET_VALID, + SSL.X509VerificationCodes.ERR_CERT_HAS_EXPIRED, + SSL.X509VerificationCodes.ERR_DEPTH_ZERO_SELF_SIGNED_CERT, + SSL.X509VerificationCodes.ERR_SELF_SIGNED_CERT_IN_CHAIN, ) # TODO can we do this once instead of multiple times? if errno in things_are_ok and timing_safe_compare( From 01bc35f1297bfa5a6be774bc20b071ae7c76b639 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 5 Jun 2023 10:29:57 -0400 Subject: [PATCH 1832/2309] Try to update nix pyopenssl. --- nix/pyopenssl.nix | 9 +++++++++ nix/python-overrides.nix | 6 ++++++ 2 files changed, 15 insertions(+) create mode 100644 nix/pyopenssl.nix diff --git a/nix/pyopenssl.nix b/nix/pyopenssl.nix new file mode 100644 index 000000000..8afbf8bc1 --- /dev/null +++ b/nix/pyopenssl.nix @@ -0,0 +1,9 @@ +{ pyopenssl, fetchPypi }: +pyopenssl.overrideAttrs (old: rec { + pname = "pyopenssl"; + version = "23.2.0"; + src = fetchPypi { + inherit pname version; + sha256 = "sha256-1b4bkcpzhmablf592g21rq3l8apbhklp6wcwlvgfflm4algr6vr7"; + }; +}) diff --git a/nix/python-overrides.nix b/nix/python-overrides.nix index 032b427ae..74cd3a893 100644 --- a/nix/python-overrides.nix +++ b/nix/python-overrides.nix @@ -27,6 +27,12 @@ in { inherit (super) klein; }; + # Update the version of pyopenssl. + pyopenssl = self.callPackage ./pyopenssl.nix { + # Avoid infinite recursion. + inherit (super) pyopenssl; + }; + # collections-extended is currently broken for Python 3.11 in nixpkgs but # we know where a working version lives. collections-extended = self.callPackage ./collections-extended.nix { From 894cb46304c4a1d4035fa17c8962f4bac37cefa0 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 5 Jun 2023 11:27:23 -0400 Subject: [PATCH 1833/2309] Try merging the two overrides. --- nix/pyopenssl.nix | 3 +++ nix/python-overrides.nix | 4 ---- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/nix/pyopenssl.nix b/nix/pyopenssl.nix index 8afbf8bc1..0c294a888 100644 --- a/nix/pyopenssl.nix +++ b/nix/pyopenssl.nix @@ -6,4 +6,7 @@ pyopenssl.overrideAttrs (old: rec { inherit pname version; sha256 = "sha256-1b4bkcpzhmablf592g21rq3l8apbhklp6wcwlvgfflm4algr6vr7"; }; + # Building the docs requires sphinx which brings in a dependency on babel, + # the test suite of which fails. + dontBuildDocs = isPyPy; }) diff --git a/nix/python-overrides.nix b/nix/python-overrides.nix index 74cd3a893..423297ef1 100644 --- a/nix/python-overrides.nix +++ b/nix/python-overrides.nix @@ -66,10 +66,6 @@ in { # a5f8184fb816a4fd5ae87136838c9981e0d22c67. six = onPyPy dontCheck super.six; - # Building the docs requires sphinx which brings in a dependency on babel, - # the test suite of which fails. - pyopenssl = onPyPy (dontBuildDocs { sphinx-rtd-theme = null; }) super.pyopenssl; - # Likewise for beautifulsoup4. beautifulsoup4 = onPyPy (dontBuildDocs {}) super.beautifulsoup4; From 203fd84a887231dc6fc19501a177de1ef2e496ba Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 5 Jun 2023 11:30:11 -0400 Subject: [PATCH 1834/2309] Need to import it. --- nix/pyopenssl.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nix/pyopenssl.nix b/nix/pyopenssl.nix index 0c294a888..b94aa254c 100644 --- a/nix/pyopenssl.nix +++ b/nix/pyopenssl.nix @@ -1,4 +1,4 @@ -{ pyopenssl, fetchPypi }: +{ pyopenssl, fetchPypi, isPyPy }: pyopenssl.overrideAttrs (old: rec { pname = "pyopenssl"; version = "23.2.0"; From 6e6bae9bf6c008d05fac1e852248cec706de6cb8 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 5 Jun 2023 11:34:52 -0400 Subject: [PATCH 1835/2309] Some random other hash who knows --- nix/pyopenssl.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nix/pyopenssl.nix b/nix/pyopenssl.nix index b94aa254c..5006cf1d0 100644 --- a/nix/pyopenssl.nix +++ b/nix/pyopenssl.nix @@ -4,7 +4,7 @@ pyopenssl.overrideAttrs (old: rec { version = "23.2.0"; src = fetchPypi { inherit pname version; - sha256 = "sha256-1b4bkcpzhmablf592g21rq3l8apbhklp6wcwlvgfflm4algr6vr7"; + sha256 = "sha256-1qgarxcmlrrrlyjnsry47lz04z8bviy7rrlbbp9874kdj799rckc"; }; # Building the docs requires sphinx which brings in a dependency on babel, # the test suite of which fails. From 43e4e1b09a3da502bb0c3433948e03df9f9e05ab Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 5 Jun 2023 11:52:24 -0400 Subject: [PATCH 1836/2309] Get rid of prefix. --- nix/pyopenssl.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nix/pyopenssl.nix b/nix/pyopenssl.nix index 5006cf1d0..fbf377fa7 100644 --- a/nix/pyopenssl.nix +++ b/nix/pyopenssl.nix @@ -4,7 +4,7 @@ pyopenssl.overrideAttrs (old: rec { version = "23.2.0"; src = fetchPypi { inherit pname version; - sha256 = "sha256-1qgarxcmlrrrlyjnsry47lz04z8bviy7rrlbbp9874kdj799rckc"; + sha256 = "1qgarxcmlrrrlyjnsry47lz04z8bviy7rrlbbp9874kdj799rckc"; }; # Building the docs requires sphinx which brings in a dependency on babel, # the test suite of which fails. From 940600e0ed5a8edc78febd53aac8177bceb56500 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 5 Jun 2023 12:54:51 -0400 Subject: [PATCH 1837/2309] Link to ticket. --- src/allmydata/storage_client.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 1946b5da2..2ea154263 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -1234,7 +1234,10 @@ class HTTPNativeStorageServer(service.MultiService): async def _agent_factory(self) -> Optional[Callable[[object, IPolicyForHTTPS, HTTPConnectionPool],IAgent]]: """Return a factory for ``twisted.web.iweb.IAgent``.""" - # TODO default_connection_handlers should really be an object, not a dict... + # TODO default_connection_handlers should really be an object, not a + # dict, so we can ask "is this using Tor" without poking at a + # dictionary with arbitrary strings... See + # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/4032 handler = self._default_connection_handlers["tcp"] if handler == "tcp": return None From 5af0ead5b9b1232526511ab81ad1f59bf370a290 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 6 Jun 2023 10:58:16 -0400 Subject: [PATCH 1838/2309] Refactor HTTP client creation to be more centralized. --- src/allmydata/storage/http_client.py | 170 +++++++++++++++--------- src/allmydata/storage_client.py | 50 ++----- src/allmydata/test/common_system.py | 4 +- src/allmydata/test/test_storage_http.py | 12 +- 4 files changed, 128 insertions(+), 108 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 670d84be3..fe2545c03 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -16,6 +16,7 @@ from typing import ( Set, Dict, Callable, + ClassVar, ) from base64 import b64encode from io import BytesIO @@ -60,6 +61,15 @@ from .http_common import ( from .common import si_b2a, si_to_human_readable from ..util.hashutil import timing_safe_compare from ..util.deferredutil import async_to_deferred +from ..util.tor_provider import _Provider as TorProvider + +try: + from txtorcon import Tor # type: ignore +except ImportError: + + class Tor: + pass + _OPENSSL = Binding().lib @@ -302,18 +312,30 @@ class _StorageClientHTTPSPolicy: ) -@define(hash=True) -class StorageClient(object): +@define +class StorageClientFactory: """ - Low-level HTTP client that talks to the HTTP storage server. + Create ``StorageClient`` instances, using appropriate + ``twisted.web.iweb.IAgent`` for different connection methods: normal TCP, + Tor, and eventually I2P. + + There is some caching involved since there might be shared setup work, e.g. + connecting to the local Tor service only needs to happen once. """ - # If set, we're doing unit testing and we should call this with - # HTTPConnectionPool we create. - TEST_MODE_REGISTER_HTTP_POOL = None + _default_connection_handlers: dict[str, str] + _tor_provider: Optional[TorProvider] + # Cache the Tor instance created by the provider, if relevant. + _tor_instance: Optional[Tor] = None + + # If set, we're doing unit testing and we should call this with any + # HTTPConnectionPool that gets passed/created to ``create_agent()``. + TEST_MODE_REGISTER_HTTP_POOL = ClassVar[ + Optional[Callable[[HTTPConnectionPool], None]] + ] @classmethod - def start_test_mode(cls, callback): + def start_test_mode(cls, callback: Callable[[HTTPConnectionPool], None]) -> None: """Switch to testing mode. In testing mode we register the pool with test system using the given @@ -328,66 +350,84 @@ class StorageClient(object): """Stop testing mode.""" cls.TEST_MODE_REGISTER_HTTP_POOL = None - # The URL is a HTTPS URL ("https://..."). To construct from a NURL, use - # ``StorageClient.from_nurl()``. - _base_url: DecodedURL - _swissnum: bytes - _treq: Union[treq, StubTreq, HTTPClient] - _pool: Optional[HTTPConnectionPool] - _clock: IReactorTime - - @classmethod - def from_nurl( - cls, + async def _create_agent( + self, nurl: DecodedURL, - reactor, - # TODO default_connection_handlers should really be a class, not a dict - # of strings... - default_connection_handlers: dict[str, str], - pool: Optional[HTTPConnectionPool] = None, - agent_factory: Optional[ - Callable[[object, IPolicyForHTTPS, HTTPConnectionPool], IAgent] - ] = None, - ) -> StorageClient: - """ - Create a ``StorageClient`` for the given NURL. - """ - # Safety check: if we're using normal TCP connections, we better not be - # configured for Tor or I2P. - if agent_factory is None: - assert default_connection_handlers["tcp"] == "tcp" + reactor: object, + tls_context_factory: IPolicyForHTTPS, + pool: HTTPConnectionPool, + ) -> IAgent: + """Create a new ``IAgent``, possibly using Tor.""" + if self.TEST_MODE_REGISTER_HTTP_POOL is not None: + self.TEST_MODE_REGISTER_HTTP_POOL(pool) + # TODO default_connection_handlers should really be an object, not a + # dict, so we can ask "is this using Tor" without poking at a + # dictionary with arbitrary strings... See + # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/4032 + handler = self._default_connection_handlers["tcp"] + + if handler == "tcp": + return Agent(reactor, tls_context_factory, pool=pool) + if handler == "tor": # TODO or nurl.scheme == "pb+tor": + assert self._tor_provider is not None + if self._tor_instance is None: + self._tor_instance = await self._tor_provider.get_tor_instance(reactor) + return self._tor_instance.web_agent( + pool=pool, tls_context_factory=tls_context_factory + ) + else: + raise RuntimeError(f"Unsupported tcp connection handler: {handler}") + + async def create_storage_client( + self, + nurl: DecodedURL, + reactor: IReactorTime, + pool: Optional[HTTPConnectionPool] = None, + ) -> StorageClient: + """Create a new ``StorageClient`` for the given NURL.""" assert nurl.fragment == "v=1" - assert nurl.scheme == "pb" - swissnum = nurl.path[0].encode("ascii") - certificate_hash = nurl.user.encode("ascii") + assert nurl.scheme in ("pb", "pb+tor") if pool is None: pool = HTTPConnectionPool(reactor) pool.maxPersistentPerHost = 10 - if cls.TEST_MODE_REGISTER_HTTP_POOL is not None: - cls.TEST_MODE_REGISTER_HTTP_POOL(pool) - - def default_agent_factory( - reactor: object, - tls_context_factory: IPolicyForHTTPS, - pool: HTTPConnectionPool, - ) -> IAgent: - return Agent(reactor, tls_context_factory, pool=pool) - - if agent_factory is None: - agent_factory = default_agent_factory - - treq_client = HTTPClient( - agent_factory( - reactor, - _StorageClientHTTPSPolicy(expected_spki_hash=certificate_hash), - pool, - ) + certificate_hash = nurl.user.encode("ascii") + agent = await self._create_agent( + nurl, + reactor, + _StorageClientHTTPSPolicy(expected_spki_hash=certificate_hash), + pool, + ) + treq_client = HTTPClient(agent) + https_url = DecodedURL().replace(scheme="https", host=nurl.host, port=nurl.port) + swissnum = nurl.path[0].encode("ascii") + return StorageClient( + https_url, + swissnum, + treq_client, + pool, + reactor, + self.TEST_MODE_REGISTER_HTTP_POOL is not None, ) - https_url = DecodedURL().replace(scheme="https", host=nurl.host, port=nurl.port) - return cls(https_url, swissnum, treq_client, pool, reactor) + +@define(hash=True) +class StorageClient(object): + """ + Low-level HTTP client that talks to the HTTP storage server. + + Create using a ``StorageClientFactory`` instance. + """ + + # The URL should be a HTTPS URL ("https://...") + _base_url: DecodedURL + _swissnum: bytes + _treq: Union[treq, StubTreq, HTTPClient] + _pool: HTTPConnectionPool + _clock: IReactorTime + # Are we running unit tests? + _test_mode: bool def relative_url(self, path: str) -> DecodedURL: """Get a URL relative to the base URL.""" @@ -495,12 +535,11 @@ class StorageClient(object): method, url, headers=headers, timeout=timeout, **kwargs ) - if self.TEST_MODE_REGISTER_HTTP_POOL is not None: - if response.code != 404: - # We're doing API queries, HTML is never correct except in 404, but - # it's the default for Twisted's web server so make sure nothing - # unexpected happened. - assert get_content_type(response.headers) != "text/html" + if self._test_mode and response.code != 404: + # We're doing API queries, HTML is never correct except in 404, but + # it's the default for Twisted's web server so make sure nothing + # unexpected happened. + assert get_content_type(response.headers) != "text/html" return response @@ -529,8 +568,7 @@ class StorageClient(object): def shutdown(self) -> Deferred: """Shutdown any connections.""" - if self._pool is not None: - return self._pool.closeCachedConnections() + return self._pool.closeCachedConnections() @define(hash=True) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 2ea154263..6a965aaac 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -89,7 +89,8 @@ from allmydata.util.deferredutil import async_to_deferred, race from allmydata.storage.http_client import ( StorageClient, StorageClientImmutables, StorageClientGeneral, ClientException as HTTPClientException, StorageClientMutables, - ReadVector, TestWriteVectors, WriteVector, TestVector, ClientException + ReadVector, TestWriteVectors, WriteVector, TestVector, ClientException, + StorageClientFactory ) from .node import _Config @@ -1068,8 +1069,9 @@ class HTTPNativeStorageServer(service.MultiService): self._on_status_changed = ObserverList() self._reactor = reactor self._grid_manager_verifier = grid_manager_verifier - self._tor_provider = tor_provider - self._default_connection_handlers = default_connection_handlers + self._storage_client_factory = StorageClientFactory( + default_connection_handlers, tor_provider + ) furl = announcement["anonymous-storage-FURL"].encode("utf-8") ( @@ -1232,26 +1234,6 @@ class HTTPNativeStorageServer(service.MultiService): self._connecting_deferred = connecting return connecting - async def _agent_factory(self) -> Optional[Callable[[object, IPolicyForHTTPS, HTTPConnectionPool],IAgent]]: - """Return a factory for ``twisted.web.iweb.IAgent``.""" - # TODO default_connection_handlers should really be an object, not a - # dict, so we can ask "is this using Tor" without poking at a - # dictionary with arbitrary strings... See - # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/4032 - handler = self._default_connection_handlers["tcp"] - if handler == "tcp": - return None - if handler == "tor": - assert self._tor_provider is not None - tor_instance = await self._tor_provider.get_tor_instance(self._reactor) - - def agent_factory(reactor: object, tls_context_factory: IPolicyForHTTPS, pool: HTTPConnectionPool) -> IAgent: - assert reactor == self._reactor - return tor_instance.web_agent(pool=pool, tls_context_factory=tls_context_factory) - return agent_factory - else: - raise RuntimeError(f"Unsupported tcp connection handler: {handler}") - @async_to_deferred async def _pick_server_and_get_version(self): """ @@ -1270,28 +1252,24 @@ class HTTPNativeStorageServer(service.MultiService): # version() calls before we are live talking to a server, it could only # be one. See https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3992 - agent_factory = await self._agent_factory() - - def request(reactor, nurl: DecodedURL): + @async_to_deferred + async def request(reactor, nurl: DecodedURL): # Since we're just using this one off to check if the NURL # works, no need for persistent pool or other fanciness. pool = HTTPConnectionPool(reactor, persistent=False) pool.retryAutomatically = False - return StorageClientGeneral( - StorageClient.from_nurl( - nurl, reactor, self._default_connection_handlers, - pool=pool, agent_factory=agent_factory) - ).get_version() + storage_client = await self._storage_client_factory.create_storage_client( + nurl, reactor, pool + ) + return await StorageClientGeneral(storage_client).get_version() nurl = await _pick_a_http_server(reactor, self._nurls, request) # If we've gotten this far, we've found a working NURL. - self._istorage_server = _HTTPStorageServer.from_http_client( - StorageClient.from_nurl( - nurl, reactor, self._default_connection_handlers, - agent_factory=agent_factory - ) + storage_client = await self._storage_client_factory.create_storage_client( + nurl, reactor, None ) + self._istorage_server = _HTTPStorageServer.from_http_client(storage_client) return self._istorage_server try: diff --git a/src/allmydata/test/common_system.py b/src/allmydata/test/common_system.py index fa8d943e5..cfb6c9f04 100644 --- a/src/allmydata/test/common_system.py +++ b/src/allmydata/test/common_system.py @@ -686,8 +686,8 @@ class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin): def setUp(self): self._http_client_pools = [] - http_client.StorageClient.start_test_mode(self._got_new_http_connection_pool) - self.addCleanup(http_client.StorageClient.stop_test_mode) + http_client.StorageClientFactory.start_test_mode(self._got_new_http_connection_pool) + self.addCleanup(http_client.StorageClientFactory.stop_test_mode) self.port_assigner = SameProcessStreamEndpointAssigner() self.port_assigner.setUp() self.addCleanup(self.port_assigner.tearDown) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 1380ab7e7..233d82989 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -58,6 +58,7 @@ from ..storage.http_server import ( ) from ..storage.http_client import ( StorageClient, + StorageClientFactory, ClientException, StorageClientImmutables, ImmutableCreateResult, @@ -323,10 +324,10 @@ class CustomHTTPServerTests(SyncTestCase): def setUp(self): super(CustomHTTPServerTests, self).setUp() - StorageClient.start_test_mode( + StorageClientFactory.start_test_mode( lambda pool: self.addCleanup(pool.closeCachedConnections) ) - self.addCleanup(StorageClient.stop_test_mode) + self.addCleanup(StorageClientFactory.stop_test_mode) # Could be a fixture, but will only be used in this test class so not # going to bother: self._http_server = TestApp() @@ -341,6 +342,7 @@ class CustomHTTPServerTests(SyncTestCase): # fixed if https://github.com/twisted/treq/issues/226 were ever # fixed. clock=treq._agent._memoryReactor, + test_mode=True, ) self._http_server.clock = self.client._clock @@ -529,10 +531,10 @@ class HttpTestFixture(Fixture): """ def _setUp(self): - StorageClient.start_test_mode( + StorageClientFactory.start_test_mode( lambda pool: self.addCleanup(pool.closeCachedConnections) ) - self.addCleanup(StorageClient.stop_test_mode) + self.addCleanup(StorageClientFactory.stop_test_mode) self.clock = Reactor() self.tempdir = self.useFixture(TempDir()) # The global Cooperator used by Twisted (a) used by pull producers in @@ -558,6 +560,7 @@ class HttpTestFixture(Fixture): treq=self.treq, pool=None, clock=self.clock, + test_mode=True, ) def result_of_with_flush(self, d): @@ -671,6 +674,7 @@ class GenericHTTPAPITests(SyncTestCase): treq=StubTreq(self.http.http_server.get_resource()), pool=None, clock=self.http.clock, + test_mode=True, ) ) with assert_fails_with_http_code(self, http.UNAUTHORIZED): From 74a121da74af8928cb212e550869fc4c7d511cde Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 6 Jun 2023 11:47:36 -0400 Subject: [PATCH 1839/2309] Fix bug which meant object could not be created. --- src/allmydata/storage/http_client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index fe2545c03..8cf79843f 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -330,9 +330,9 @@ class StorageClientFactory: # If set, we're doing unit testing and we should call this with any # HTTPConnectionPool that gets passed/created to ``create_agent()``. - TEST_MODE_REGISTER_HTTP_POOL = ClassVar[ + TEST_MODE_REGISTER_HTTP_POOL: ClassVar[ Optional[Callable[[HTTPConnectionPool], None]] - ] + ] = None @classmethod def start_test_mode(cls, callback: Callable[[HTTPConnectionPool], None]) -> None: From e8744f91e5176d6adf2f1fc4c2065f7041162573 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 6 Jun 2023 12:06:51 -0400 Subject: [PATCH 1840/2309] Hook up HTTP storage for servers listening on .onion addresses --- src/allmydata/protocol_switch.py | 14 +++++++++++--- src/allmydata/storage/http_client.py | 2 +- src/allmydata/storage/http_server.py | 11 +++++++++-- 3 files changed, 21 insertions(+), 6 deletions(-) diff --git a/src/allmydata/protocol_switch.py b/src/allmydata/protocol_switch.py index 208efec6c..941b104be 100644 --- a/src/allmydata/protocol_switch.py +++ b/src/allmydata/protocol_switch.py @@ -102,8 +102,15 @@ class _FoolscapOrHttps(Protocol, metaclass=_PretendToBeNegotiation): for location_hint in chain.from_iterable( hints.split(",") for hints in cls.tub.locationHints ): - if location_hint.startswith("tcp:"): - _, hostname, port = location_hint.split(":") + if location_hint.startswith("tcp:") or location_hint.startswith("tor:"): + scheme, hostname, port = location_hint.split(":") + if scheme == "tcp": + subscheme = None + else: + subscheme = "tor" + # If we're listening on Tor, the hostname needs to have an + # .onion TLD. + assert hostname.endswith(".onion") port = int(port) storage_nurls.add( build_nurl( @@ -111,9 +118,10 @@ class _FoolscapOrHttps(Protocol, metaclass=_PretendToBeNegotiation): port, str(swissnum, "ascii"), cls.tub.myCertificate.original.to_cryptography(), + subscheme ) ) - # TODO this is probably where we'll have to support Tor and I2P? + # TODO this is where we'll have to support Tor and I2P as well. # See https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3888#comment:9 # for discussion (there will be separate tickets added for those at # some point.) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 8cf79843f..cd3143924 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -369,7 +369,7 @@ class StorageClientFactory: if handler == "tcp": return Agent(reactor, tls_context_factory, pool=pool) - if handler == "tor": # TODO or nurl.scheme == "pb+tor": + if handler == "tor" or nurl.scheme == "pb+tor": assert self._tor_provider is not None if self._tor_instance is None: self._tor_instance = await self._tor_provider.get_tor_instance(reactor) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 924ae5a43..028ebf1c7 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -995,13 +995,20 @@ class _TLSEndpointWrapper(object): def build_nurl( - hostname: str, port: int, swissnum: str, certificate: CryptoCertificate + hostname: str, + port: int, + swissnum: str, + certificate: CryptoCertificate, + subscheme: Optional[str] = None, ) -> DecodedURL: """ Construct a HTTPS NURL, given the hostname, port, server swissnum, and x509 certificate for the server. Clients can then connect to the server using this NURL. """ + scheme = "pb" + if subscheme is not None: + scheme = f"{scheme}+{subscheme}" return DecodedURL().replace( fragment="v=1", # how we know this NURL is HTTP-based (i.e. not Foolscap) host=hostname, @@ -1013,7 +1020,7 @@ def build_nurl( "ascii", ), ), - scheme="pb", + scheme=scheme, ) From 57a6721670da156bbf94b72c0b904f04633f6757 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 6 Jun 2023 12:07:13 -0400 Subject: [PATCH 1841/2309] News file. --- newsfragments/3910.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3910.minor diff --git a/newsfragments/3910.minor b/newsfragments/3910.minor new file mode 100644 index 000000000..e69de29bb From a977180baf0b138c8dd3824d95ae3f66bec1540d Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 6 Jun 2023 12:15:31 -0400 Subject: [PATCH 1842/2309] Fix lint --- src/allmydata/storage_client.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 6a965aaac..a614c17db 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -51,7 +51,6 @@ from zope.interface import ( ) from twisted.python.failure import Failure from twisted.web import http -from twisted.web.iweb import IAgent, IPolicyForHTTPS from twisted.internet.task import LoopingCall from twisted.internet import defer, reactor from twisted.application import service From 20d4175abcbe2948f6e219fd4dd564e7fea47ae3 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 6 Jun 2023 12:18:02 -0400 Subject: [PATCH 1843/2309] Fix typecheck complaint --- src/allmydata/storage/http_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index cd3143924..e7df3709d 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -67,7 +67,7 @@ try: from txtorcon import Tor # type: ignore except ImportError: - class Tor: + class Tor: # type: ignore[no-redef] pass From 4b495bbe854237b97fc87036268fdbd4265d4a6a Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 12 Jun 2023 09:54:45 -0400 Subject: [PATCH 1844/2309] Slightly improved logging. --- integration/util.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/integration/util.py b/integration/util.py index 402c14932..b9f784af9 100644 --- a/integration/util.py +++ b/integration/util.py @@ -140,7 +140,8 @@ class _MagicTextProtocol(ProcessProtocol): def outReceived(self, data): data = str(data, sys.stdout.encoding) - sys.stdout.write(self.name + data) + for line in data.splitlines(): + sys.stdout.write(self.name + line + "\n") self._output.write(data) if not self.magic_seen.called and self._magic_text in self._output.getvalue(): print("Saw '{}' in the logs".format(self._magic_text)) @@ -148,7 +149,8 @@ class _MagicTextProtocol(ProcessProtocol): def errReceived(self, data): data = str(data, sys.stderr.encoding) - sys.stdout.write(self.name + data) + for line in data.splitlines(): + sys.stdout.write(self.name + line + "\n") def _cleanup_process_async(transport: IProcessTransport, allow_missing: bool) -> None: From eba1ed02269836457e663ace391a035286aa2187 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 12 Jun 2023 12:00:05 -0400 Subject: [PATCH 1845/2309] More isolated test setup. --- integration/test_tor.py | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/integration/test_tor.py b/integration/test_tor.py index ec5cc1bc4..c14cb717e 100644 --- a/integration/test_tor.py +++ b/integration/test_tor.py @@ -38,8 +38,8 @@ def test_onion_service_storage(reactor, request, temp_dir, flog_gatherer, tor_ne The two nodes can talk to the introducer and each other: we upload to one node, read from the other. """ - carol = yield _create_anonymous_node(reactor, 'carol', 8008, request, temp_dir, flog_gatherer, tor_network, tor_introducer_furl) - dave = yield _create_anonymous_node(reactor, 'dave', 8009, request, temp_dir, flog_gatherer, tor_network, tor_introducer_furl) + carol = yield _create_anonymous_node(reactor, 'carol', 8008, request, temp_dir, flog_gatherer, tor_network, tor_introducer_furl, 2) + dave = yield _create_anonymous_node(reactor, 'dave', 8009, request, temp_dir, flog_gatherer, tor_network, tor_introducer_furl, 2) yield util.await_client_ready(carol, minimum_number_of_servers=2, timeout=600) yield util.await_client_ready(dave, minimum_number_of_servers=2, timeout=600) yield upload_to_one_download_from_the_other(reactor, temp_dir, carol, dave) @@ -94,7 +94,7 @@ async def upload_to_one_download_from_the_other(reactor, temp_dir, upload_to: ut @pytest_twisted.inlineCallbacks -def _create_anonymous_node(reactor, name, control_port, request, temp_dir, flog_gatherer, tor_network, introducer_furl) -> util.TahoeProcess: +def _create_anonymous_node(reactor, name, control_port, request, temp_dir, flog_gatherer, tor_network, introducer_furl, shares_total: int) -> util.TahoeProcess: node_dir = FilePath(temp_dir).child(name) web_port = "tcp:{}:interface=localhost".format(control_port + 2000) @@ -116,7 +116,7 @@ def _create_anonymous_node(reactor, name, control_port, request, temp_dir, flog_ '--listen', 'tor', '--shares-needed', '1', '--shares-happy', '1', - '--shares-total', '2', + '--shares-total', str(shares_total), node_dir.path, ), env=environ, @@ -141,16 +141,23 @@ def _create_anonymous_node(reactor, name, control_port, request, temp_dir, flog_ @pytest_twisted.inlineCallbacks -def test_anonymous_client(reactor, alice, request, temp_dir, flog_gatherer, tor_network, introducer_furl): +def test_anonymous_client(reactor, request, temp_dir, flog_gatherer, tor_network, introducer_furl): """ - A normal node (alice) and a normal introducer are configured, and one node + A normal node (normie) and a normal introducer are configured, and one node (anonymoose) which is configured to be anonymous by talking via Tor. - Anonymoose should be able to communicate with alice. + Anonymoose should be able to communicate with normie. TODO how to ensure that anonymoose is actually using Tor? """ - anonymoose = yield _create_anonymous_node(reactor, 'anonymoose', 8008, request, temp_dir, flog_gatherer, tor_network, introducer_furl) - yield util.await_client_ready(anonymoose, minimum_number_of_servers=2, timeout=600) + normie = yield util._create_node( + reactor, request, temp_dir, introducer_furl, flog_gatherer, "normie", + web_port="tcp:9989:interface=localhost", + storage=True, needed=1, happy=1, total=1, + ) + yield util.await_client_ready(normie) - yield upload_to_one_download_from_the_other(reactor, temp_dir, alice, anonymoose) + anonymoose = yield _create_anonymous_node(reactor, 'anonymoose', 8008, request, temp_dir, flog_gatherer, tor_network, introducer_furl, 1) + yield util.await_client_ready(anonymoose, minimum_number_of_servers=1, timeout=600) + + yield upload_to_one_download_from_the_other(reactor, temp_dir, normie, anonymoose) From 939f0ded25ab83b0f25cc97994d75fc1135d46a8 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 12 Jun 2023 12:00:12 -0400 Subject: [PATCH 1846/2309] It's OK if some nodes are down. --- integration/util.py | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/integration/util.py b/integration/util.py index b9f784af9..31d351bc1 100644 --- a/integration/util.py +++ b/integration/util.py @@ -631,16 +631,9 @@ def await_client_ready(tahoe, timeout=10, liveness=60*2, minimum_number_of_serve server['last_received_data'] for server in servers ] - # if any times are null/None that server has never been - # contacted (so it's down still, probably) - never_received_data = server_times.count(None) - if never_received_data > 0: - print(f"waiting because {never_received_data} server(s) not contacted") - time.sleep(1) - continue - - # check that all times are 'recent enough' - if any([time.time() - t > liveness for t in server_times]): + # check that all times are 'recent enough' (it's OK if _some_ servers + # are down, we just want to make sure a sufficient number are up) + if len([time.time() - t <= liveness for t in server_times if t is not None]) < minimum_number_of_servers: print("waiting because at least one server too old") time.sleep(1) continue From e8150015ad4a555c564fcc3a3702f0e9435cc41a Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 12 Jun 2023 12:34:22 -0400 Subject: [PATCH 1847/2309] Try newer Python in the hopes this will speed things up. --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dc9854ae4..1061657b9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -166,7 +166,7 @@ jobs: matrix: include: - os: macos-12 - python-version: "3.9" + python-version: "3.11" force-foolscap: false - os: windows-latest python-version: "3.11" From 7ff20a34e0d5a52092d88146122c5db626a33140 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 12 Jun 2023 13:22:45 -0400 Subject: [PATCH 1848/2309] Skip on macOS :( --- integration/test_tor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration/test_tor.py b/integration/test_tor.py index c14cb717e..af83e2ba1 100644 --- a/integration/test_tor.py +++ b/integration/test_tor.py @@ -139,7 +139,7 @@ def _create_anonymous_node(reactor, name, control_port, request, temp_dir, flog_ print("okay, launched") return result - +@pytest.mark.skipif(sys.platform.startswith('darwin'), reason='This test has issues on macOS') @pytest_twisted.inlineCallbacks def test_anonymous_client(reactor, request, temp_dir, flog_gatherer, tor_network, introducer_furl): """ From e5b6049329a7eb22e1f3bc56a849bead18ffaa59 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 13 Jun 2023 10:16:50 -0400 Subject: [PATCH 1849/2309] match the package name on pypi, case and all otherwise urls are misconstructed and stuff fails --- nix/pyopenssl.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nix/pyopenssl.nix b/nix/pyopenssl.nix index fbf377fa7..14155491c 100644 --- a/nix/pyopenssl.nix +++ b/nix/pyopenssl.nix @@ -1,6 +1,6 @@ { pyopenssl, fetchPypi, isPyPy }: pyopenssl.overrideAttrs (old: rec { - pname = "pyopenssl"; + pname = "pyOpenSSL"; version = "23.2.0"; src = fetchPypi { inherit pname version; From 608fbce9f9be25f5f770b618b210864f47b5f4b1 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 13 Jun 2023 10:18:56 -0400 Subject: [PATCH 1850/2309] match the source tarball hash --- nix/pyopenssl.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nix/pyopenssl.nix b/nix/pyopenssl.nix index 14155491c..e2a23f38c 100644 --- a/nix/pyopenssl.nix +++ b/nix/pyopenssl.nix @@ -4,7 +4,7 @@ pyopenssl.overrideAttrs (old: rec { version = "23.2.0"; src = fetchPypi { inherit pname version; - sha256 = "1qgarxcmlrrrlyjnsry47lz04z8bviy7rrlbbp9874kdj799rckc"; + sha256 = "J2+TH1WkUufeppxxc+mE6ypEB85BPJGKo0tV+C+bi6w="; }; # Building the docs requires sphinx which brings in a dependency on babel, # the test suite of which fails. From 8421d406e9b246b5ce14809a408841f9f7c2537e Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 13 Jun 2023 10:33:54 -0400 Subject: [PATCH 1851/2309] Fix the name metadata as well It was already computed for the derivation we're going to override. It won't be recomputed again as a result of `overrideAttrs` so we recompute it and include it in the override. --- nix/pyopenssl.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/nix/pyopenssl.nix b/nix/pyopenssl.nix index e2a23f38c..3428b0eb7 100644 --- a/nix/pyopenssl.nix +++ b/nix/pyopenssl.nix @@ -2,6 +2,7 @@ pyopenssl.overrideAttrs (old: rec { pname = "pyOpenSSL"; version = "23.2.0"; + name = "${pname}-${version}"; src = fetchPypi { inherit pname version; sha256 = "J2+TH1WkUufeppxxc+mE6ypEB85BPJGKo0tV+C+bi6w="; From 0b0e5c5c93243b79a8f6e70f5b9ed9ca7c8f2020 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 13 Jun 2023 10:34:36 -0400 Subject: [PATCH 1852/2309] Keep using our `dontBuildDocs` helper function It does the necessary overrides for stopping doc builds and excluding certain inputs and outputs. We can't just set `dontBuildDocs` in the derivation because that's not a setting recognized by the Nixpkgs Python build system. --- nix/pyopenssl.nix | 3 --- nix/python-overrides.nix | 8 ++++++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/nix/pyopenssl.nix b/nix/pyopenssl.nix index 3428b0eb7..b8966fad1 100644 --- a/nix/pyopenssl.nix +++ b/nix/pyopenssl.nix @@ -7,7 +7,4 @@ pyopenssl.overrideAttrs (old: rec { inherit pname version; sha256 = "J2+TH1WkUufeppxxc+mE6ypEB85BPJGKo0tV+C+bi6w="; }; - # Building the docs requires sphinx which brings in a dependency on babel, - # the test suite of which fails. - dontBuildDocs = isPyPy; }) diff --git a/nix/python-overrides.nix b/nix/python-overrides.nix index 423297ef1..4a332b3fc 100644 --- a/nix/python-overrides.nix +++ b/nix/python-overrides.nix @@ -29,8 +29,12 @@ in { # Update the version of pyopenssl. pyopenssl = self.callPackage ./pyopenssl.nix { - # Avoid infinite recursion. - inherit (super) pyopenssl; + pyopenssl = + # Building the docs requires sphinx which brings in a dependency on babel, + # the test suite of which fails. + onPyPy (dontBuildDocs { sphinx-rtd-theme = null; }) + # Avoid infinite recursion. + super.pyopenssl; }; # collections-extended is currently broken for Python 3.11 in nixpkgs but From 6bc232745afba2c1121ceac0a4d05dfef3eee459 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 14 Jun 2023 09:56:25 -0400 Subject: [PATCH 1853/2309] News fragment. --- newsfragments/4035.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/4035.minor diff --git a/newsfragments/4035.minor b/newsfragments/4035.minor new file mode 100644 index 000000000..e69de29bb From 5561e11cfd3804a080fc9e85ee52fa01a7a11a1d Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 14 Jun 2023 10:31:11 -0400 Subject: [PATCH 1854/2309] Upgrade versions, install dependencies since mypy might want them --- tox.ini | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/tox.ini b/tox.ini index 2edb15a0b..89dbda748 100644 --- a/tox.ini +++ b/tox.ini @@ -121,20 +121,18 @@ commands = [testenv:typechecks] basepython = python3 -skip_install = True deps = - mypy - mypy-zope + mypy==1.3.0 + # When 0.9.2 comes out it will work with 1.3, it's just unreleased at the moment... + git+https://github.com/shoobx/mypy-zope types-mock types-six types-PyYAML types-pkg_resources types-pyOpenSSL - git+https://github.com/warner/foolscap - # Twisted 21.2.0 introduces some type hints which we are not yet - # compatible with. - # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3622 - twisted<21.2.0 + foolscap + # Upgrade when new releases come out: + Twisted==22.10.0 commands = mypy src From b45ee20ba8bd9b582ad479b228be47390145d0de Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 14 Jun 2023 11:07:48 -0400 Subject: [PATCH 1855/2309] MyPy fixes for allmydata.storage. --- src/allmydata/storage/http_client.py | 23 +++++++++++------------ src/allmydata/storage/http_common.py | 2 +- src/allmydata/storage/http_server.py | 13 +++++++++---- src/allmydata/storage/lease.py | 4 +++- src/allmydata/storage/lease_schema.py | 2 +- src/allmydata/storage/server.py | 4 +++- 6 files changed, 28 insertions(+), 20 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 7f53a4378..59213417c 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -34,7 +34,7 @@ from werkzeug.datastructures import Range, ContentRange from twisted.web.http_headers import Headers from twisted.web import http from twisted.web.iweb import IPolicyForHTTPS, IResponse, IAgent -from twisted.internet.defer import inlineCallbacks, Deferred, succeed +from twisted.internet.defer import Deferred, succeed from twisted.internet.interfaces import ( IOpenSSLClientConnectionCreator, IReactorTime, @@ -70,7 +70,6 @@ except ImportError: pass - def _encode_si(si): # type: (bytes) -> str """Encode the storage index into Unicode string.""" return str(si_b2a(si), "ascii") @@ -179,24 +178,24 @@ def limited_content( This will time out if no data is received for 60 seconds; so long as a trickle of data continues to arrive, it will continue to run. """ - d = succeed(None) + result_deferred = succeed(None) # Sadly, addTimeout() won't work because we need access to the IDelayedCall # in order to reset it on each data chunk received. - timeout = clock.callLater(60, d.cancel) + timeout = clock.callLater(60, result_deferred.cancel) collector = _LengthLimitedCollector(max_length, timeout) with start_action( action_type="allmydata:storage:http-client:limited-content", max_length=max_length, ).context(): - d = DeferredContext(d) + d = DeferredContext(result_deferred) # Make really sure everything gets called in Deferred context, treq might # call collector directly... d.addCallback(lambda _: treq.collect(response, collector)) - def done(_): + def done(_: object) -> BytesIO: timeout.cancel() collector.f.seek(0) return collector.f @@ -659,15 +658,15 @@ class UploadProgress(object): required: RangeMap -@inlineCallbacks -def read_share_chunk( +@async_to_deferred +async def read_share_chunk( client: StorageClient, share_type: str, storage_index: bytes, share_number: int, offset: int, length: int, -) -> Deferred[bytes]: +) -> bytes: """ Download a chunk of data from a share. @@ -688,7 +687,7 @@ def read_share_chunk( # The default 60 second timeout is for getting the response, so it doesn't # include the time it takes to download the body... so we will will deal # with that later, via limited_content(). - response = yield client.request( + response = await client.request( "GET", url, headers=Headers( @@ -725,7 +724,7 @@ def read_share_chunk( raise ValueError("Server sent more than we asked for?!") # It might also send less than we asked for. That's (probably) OK, e.g. # if we went past the end of the file. - body = yield limited_content(response, client._clock, supposed_length) + body = await limited_content(response, client._clock, supposed_length) body.seek(0, SEEK_END) actual_length = body.tell() if actual_length != supposed_length: @@ -751,7 +750,7 @@ async def advise_corrupt_share( storage_index: bytes, share_number: int, reason: str, -): +) -> None: assert isinstance(reason, str) url = client.relative_url( "/storage/v1/{}/{}/{}/corrupt".format( diff --git a/src/allmydata/storage/http_common.py b/src/allmydata/storage/http_common.py index e5f07898e..f16a16785 100644 --- a/src/allmydata/storage/http_common.py +++ b/src/allmydata/storage/http_common.py @@ -22,7 +22,7 @@ def get_content_type(headers: Headers) -> Optional[str]: Returns ``None`` if no content-type was set. """ - values = headers.getRawHeaders("content-type") or [None] + values = headers.getRawHeaders("content-type", [None]) or [None] content_type = parse_options_header(values[0])[0] or None return content_type diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 028ebf1c7..c63a4ca08 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -386,13 +386,16 @@ class _ReadRangeProducer: a request. """ - request: Request + request: Optional[Request] read_data: ReadData - result: Deferred + result: Optional[Deferred[bytes]] start: int remaining: int def resumeProducing(self): + if self.result is None or self.request is None: + return + to_read = min(self.remaining, 65536) data = self.read_data(self.start, to_read) assert len(data) <= to_read @@ -441,7 +444,7 @@ class _ReadRangeProducer: def read_range( request: Request, read_data: ReadData, share_length: int -) -> Union[Deferred, bytes]: +) -> Union[Deferred[bytes], bytes]: """ Read an optional ``Range`` header, reads data appropriately via the given callable, writes the data to the request. @@ -478,6 +481,8 @@ def read_range( raise _HTTPError(http.REQUESTED_RANGE_NOT_SATISFIABLE) offset, end = range_header.ranges[0] + assert end is not None # should've exited in block above this if so + # If we're being ask to read beyond the length of the share, just read # less: end = min(end, share_length) @@ -496,7 +501,7 @@ def read_range( ContentRange("bytes", offset, end).to_header(), ) - d = Deferred() + d: Deferred[bytes] = Deferred() request.registerProducer( _ReadRangeProducer( request, read_data_with_error_handling, d, offset, end - offset diff --git a/src/allmydata/storage/lease.py b/src/allmydata/storage/lease.py index c056a7d28..c0d11abfd 100644 --- a/src/allmydata/storage/lease.py +++ b/src/allmydata/storage/lease.py @@ -173,7 +173,9 @@ class LeaseInfo(object): """ return attr.assoc( self, - _expiration_time=new_expire_time, + # MyPy is unhappy with this; long-term solution is likely switch to + # new @frozen attrs API, with type annotations. + _expiration_time=new_expire_time, # type: ignore[call-arg] ) def is_renew_secret(self, candidate_secret): diff --git a/src/allmydata/storage/lease_schema.py b/src/allmydata/storage/lease_schema.py index 63d3d4ed8..ba7dc991a 100644 --- a/src/allmydata/storage/lease_schema.py +++ b/src/allmydata/storage/lease_schema.py @@ -56,7 +56,7 @@ class HashedLeaseSerializer(object): """ Hash a lease secret for storage. """ - return blake2b(secret, digest_size=32, encoder=RawEncoder()) + return blake2b(secret, digest_size=32, encoder=RawEncoder) @classmethod def _hash_lease_info(cls, lease_info): diff --git a/src/allmydata/storage/server.py b/src/allmydata/storage/server.py index 6099636f8..d805df1c1 100644 --- a/src/allmydata/storage/server.py +++ b/src/allmydata/storage/server.py @@ -55,7 +55,9 @@ class StorageServer(service.MultiService): """ Implement the business logic for the storage server. """ - name = 'storage' + # The type in Twisted for services is wrong in 22.10... + # https://github.com/twisted/twisted/issues/10135 + name = 'storage' # type: ignore # only the tests change this to anything else LeaseCheckerClass = LeaseCheckingCrawler From 8493b42024c3e22438605b446a544747b27c8c6c Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 14 Jun 2023 14:02:24 -0400 Subject: [PATCH 1856/2309] Fix types. --- src/allmydata/test/cli/wormholetesting.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/allmydata/test/cli/wormholetesting.py b/src/allmydata/test/cli/wormholetesting.py index 647798bc8..91849901a 100644 --- a/src/allmydata/test/cli/wormholetesting.py +++ b/src/allmydata/test/cli/wormholetesting.py @@ -63,7 +63,7 @@ class MemoryWormholeServer(object): specific application id and relay URL combination. """ _apps: dict[ApplicationKey, _WormholeApp] = field(default=Factory(dict)) - _waiters: dict[ApplicationKey, Deferred] = field(default=Factory(dict)) + _waiters: dict[ApplicationKey, Deferred[IWormhole]] = field(default=Factory(dict)) def create( self, @@ -130,7 +130,7 @@ class TestingHelper(object): key = (relay_url, appid) if key in self._server._waiters: raise ValueError(f"There is already a waiter for {key}") - d = Deferred() + d : Deferred[IWormhole] = Deferred() self._server._waiters[key] = d wormhole = await d return wormhole @@ -166,7 +166,7 @@ class _WormholeApp(object): appid/relay_url scope. """ wormholes: dict[WormholeCode, IWormhole] = field(default=Factory(dict)) - _waiting: dict[WormholeCode, List[Deferred]] = field(default=Factory(dict)) + _waiting: dict[WormholeCode, List[Deferred[_MemoryWormhole]]] = field(default=Factory(dict)) _counter: Iterator[int] = field(default=Factory(count)) def allocate_code(self, wormhole: IWormhole, code: Optional[WormholeCode]) -> WormholeCode: @@ -192,13 +192,13 @@ class _WormholeApp(object): return code - def wait_for_wormhole(self, code: WormholeCode) -> Awaitable[_MemoryWormhole]: + def wait_for_wormhole(self, code: WormholeCode) -> Deferred[_MemoryWormhole]: """ Return a ``Deferred`` which fires with the next wormhole to be associated with the given code. This is used to let the first end of a wormhole rendezvous with the second end. """ - d = Deferred() + d : Deferred[_MemoryWormhole] = Deferred() self._waiting.setdefault(code, []).append(d) return d @@ -242,8 +242,8 @@ class _MemoryWormhole(object): _view: _WormholeServerView _code: Optional[WormholeCode] = None - _payload: DeferredQueue = field(default=Factory(DeferredQueue)) - _waiting_for_code: list[Deferred] = field(default=Factory(list)) + _payload: DeferredQueue[WormholeMessage] = field(default=Factory(DeferredQueue)) + _waiting_for_code: list[Deferred[WormholeCode]] = field(default=Factory(list)) def allocate_code(self) -> None: if self._code is not None: @@ -265,7 +265,7 @@ class _MemoryWormhole(object): def when_code(self) -> Deferred[WormholeCode]: if self._code is None: - d = Deferred() + d : Deferred[WormholeCode] = Deferred() self._waiting_for_code.append(d) return d return succeed(self._code) From 257aa289cdd5a7954261a03e93725394afbe8ca0 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 14 Jun 2023 14:04:45 -0400 Subject: [PATCH 1857/2309] Remote interfaces don't interact well with mypy. --- src/allmydata/introducer/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/introducer/client.py b/src/allmydata/introducer/client.py index 07f8a5f7a..a64596f0e 100644 --- a/src/allmydata/introducer/client.py +++ b/src/allmydata/introducer/client.py @@ -35,7 +35,7 @@ class InvalidCacheError(Exception): V2 = b"http://allmydata.org/tahoe/protocols/introducer/v2" -@implementer(RIIntroducerSubscriberClient_v2, IIntroducerClient) +@implementer(RIIntroducerSubscriberClient_v2, IIntroducerClient) # type: ignore[misc] class IntroducerClient(service.Service, Referenceable): def __init__(self, tub, introducer_furl, From 1fd81116cb9b7c29fdf5e3bb89430cfdb79b9f36 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 14 Jun 2023 14:08:30 -0400 Subject: [PATCH 1858/2309] Fix mypy complaint. --- src/allmydata/protocol_switch.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/allmydata/protocol_switch.py b/src/allmydata/protocol_switch.py index 941b104be..6a6bf8061 100644 --- a/src/allmydata/protocol_switch.py +++ b/src/allmydata/protocol_switch.py @@ -16,9 +16,10 @@ later in the configuration process. from __future__ import annotations from itertools import chain +from typing import cast from twisted.internet.protocol import Protocol -from twisted.internet.interfaces import IDelayedCall +from twisted.internet.interfaces import IDelayedCall, IReactorFromThreads from twisted.internet.ssl import CertificateOptions from twisted.web.server import Site from twisted.protocols.tls import TLSMemoryBIOFactory @@ -89,7 +90,7 @@ class _FoolscapOrHttps(Protocol, metaclass=_PretendToBeNegotiation): certificate=cls.tub.myCertificate.original, ) - http_storage_server = HTTPServer(reactor, storage_server, swissnum) + http_storage_server = HTTPServer(cast(IReactorFromThreads, reactor), storage_server, swissnum) cls.https_factory = TLSMemoryBIOFactory( certificate_options, False, From 3b5c6695d5fca060ac51575d8b81ac347dac1f18 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 14 Jun 2023 14:09:51 -0400 Subject: [PATCH 1859/2309] Pacify mypy. --- src/allmydata/testing/web.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/allmydata/testing/web.py b/src/allmydata/testing/web.py index 4f68b3774..95e92825b 100644 --- a/src/allmydata/testing/web.py +++ b/src/allmydata/testing/web.py @@ -276,6 +276,15 @@ class _SynchronousProducer(object): consumer.write(self.body) return succeed(None) + def stopProducing(self): + pass + + def pauseProducing(self): + pass + + def resumeProducing(self): + pass + def create_tahoe_treq_client(root=None): """ From cab24e4c7b1241c14e2f010e6a7d117755e732e4 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 14 Jun 2023 14:12:39 -0400 Subject: [PATCH 1860/2309] Another service name issue. --- src/allmydata/immutable/upload.py | 4 +++- src/allmydata/storage/server.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/allmydata/immutable/upload.py b/src/allmydata/immutable/upload.py index a331cc5db..36bd86fa6 100644 --- a/src/allmydata/immutable/upload.py +++ b/src/allmydata/immutable/upload.py @@ -1843,7 +1843,9 @@ class Uploader(service.MultiService, log.PrefixingLogMixin): """I am a service that allows file uploading. I am a service-child of the Client. """ - name = "uploader" + # The type in Twisted for services is wrong in 22.10... + # https://github.com/twisted/twisted/issues/10135 + name = "uploader" # type: ignore[assignment] URI_LIT_SIZE_THRESHOLD = 55 def __init__(self, helper_furl=None, stats_provider=None, history=None): diff --git a/src/allmydata/storage/server.py b/src/allmydata/storage/server.py index d805df1c1..858b87b1f 100644 --- a/src/allmydata/storage/server.py +++ b/src/allmydata/storage/server.py @@ -57,7 +57,7 @@ class StorageServer(service.MultiService): """ # The type in Twisted for services is wrong in 22.10... # https://github.com/twisted/twisted/issues/10135 - name = 'storage' # type: ignore + name = 'storage' # type: ignore[assignment] # only the tests change this to anything else LeaseCheckerClass = LeaseCheckingCrawler From 054c893539762449c377866fbc8c204536f9f2c5 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 14 Jun 2023 14:16:10 -0400 Subject: [PATCH 1861/2309] Pacify mypy --- src/allmydata/util/deferredutil.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/allmydata/util/deferredutil.py b/src/allmydata/util/deferredutil.py index 695915ceb..9e8d7bad4 100644 --- a/src/allmydata/util/deferredutil.py +++ b/src/allmydata/util/deferredutil.py @@ -14,6 +14,7 @@ from typing import ( TypeVar, Optional, Coroutine, + Generator ) from typing_extensions import ParamSpec @@ -212,7 +213,7 @@ class WaitForDelayedCallsMixin(PollMixin): def until( action: Callable[[], defer.Deferred[Any]], condition: Callable[[], bool], -) -> defer.Deferred[None]: +) -> Generator[Any, None, None]: """ Run a Deferred-returning function until a condition is true. From f42fb1e551e6b09ec4fa6812f06423f4afa8c828 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 14 Jun 2023 14:16:35 -0400 Subject: [PATCH 1862/2309] Unused import --- src/allmydata/test/cli/wormholetesting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/cli/wormholetesting.py b/src/allmydata/test/cli/wormholetesting.py index 91849901a..3bcad1ebf 100644 --- a/src/allmydata/test/cli/wormholetesting.py +++ b/src/allmydata/test/cli/wormholetesting.py @@ -40,7 +40,7 @@ from itertools import count from sys import stderr from attrs import frozen, define, field, Factory -from twisted.internet.defer import Deferred, DeferredQueue, succeed, Awaitable +from twisted.internet.defer import Deferred, DeferredQueue, succeed from wormhole._interfaces import IWormhole from wormhole.wormhole import create from zope.interface import implementer From ff1c1f700ee4d94ec5a3276383420f4690703a2d Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 14 Jun 2023 14:52:54 -0400 Subject: [PATCH 1863/2309] Remove unused methods. --- src/allmydata/interfaces.py | 41 ------------------------------------- 1 file changed, 41 deletions(-) diff --git a/src/allmydata/interfaces.py b/src/allmydata/interfaces.py index 201ab082e..1ebc23c75 100644 --- a/src/allmydata/interfaces.py +++ b/src/allmydata/interfaces.py @@ -493,47 +493,6 @@ class IStorageBroker(Interface): @return: unicode nickname, or None """ - # methods moved from IntroducerClient, need review - def get_all_connections(): - """Return a frozenset of (nodeid, service_name, rref) tuples, one for - each active connection we've established to a remote service. This is - mostly useful for unit tests that need to wait until a certain number - of connections have been made.""" - - def get_all_connectors(): - """Return a dict that maps from (nodeid, service_name) to a - RemoteServiceConnector instance for all services that we are actively - trying to connect to. Each RemoteServiceConnector has the following - public attributes:: - - service_name: the type of service provided, like 'storage' - last_connect_time: when we last established a connection - last_loss_time: when we last lost a connection - - version: the peer's version, from the most recent connection - oldest_supported: the peer's oldest supported version, same - - rref: the RemoteReference, if connected, otherwise None - - This method is intended for monitoring interfaces, such as a web page - that describes connecting and connected peers. - """ - - def get_all_peerids(): - """Return a frozenset of all peerids to whom we have a connection (to - one or more services) established. Mostly useful for unit tests.""" - - def get_all_connections_for(service_name): - """Return a frozenset of (nodeid, service_name, rref) tuples, one - for each active connection that provides the given SERVICE_NAME.""" - - def get_permuted_peers(service_name, key): - """Returns an ordered list of (peerid, rref) tuples, selecting from - the connections that provide SERVICE_NAME, using a hash-based - permutation keyed by KEY. This randomizes the service list in a - repeatable way, to distribute load over many peers. - """ - class IDisplayableServer(Interface): def get_nickname(): From 65775cd6bde8514fe5d3e0b809f7dd2a0dc50678 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 14 Jun 2023 14:54:24 -0400 Subject: [PATCH 1864/2309] Not used externally. --- src/allmydata/interfaces.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/src/allmydata/interfaces.py b/src/allmydata/interfaces.py index 1ebc23c75..0379e8633 100644 --- a/src/allmydata/interfaces.py +++ b/src/allmydata/interfaces.py @@ -510,16 +510,6 @@ class IServer(IDisplayableServer): def start_connecting(trigger_cb): pass - def get_rref(): - """Obsolete. Use ``get_storage_server`` instead. - - Once a server is connected, I return a RemoteReference. - Before a server is connected for the first time, I return None. - - Note that the rref I return will start producing DeadReferenceErrors - once the connection is lost. - """ - def upload_permitted(): """ :return: True if we should use this server for uploads, False From 11e01518388e6844f8ae086456f2cd6e81d08480 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 14 Jun 2023 16:02:04 -0400 Subject: [PATCH 1865/2309] Fix some type issues in storage_client.py --- src/allmydata/interfaces.py | 7 ++++++- src/allmydata/storage/http_client.py | 5 +++-- src/allmydata/storage_client.py | 10 ++++++---- 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/src/allmydata/interfaces.py b/src/allmydata/interfaces.py index 0379e8633..0f00c5417 100644 --- a/src/allmydata/interfaces.py +++ b/src/allmydata/interfaces.py @@ -17,11 +17,13 @@ if PY2: from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, pow, round, super, range, max, min # noqa: F401 from past.builtins import long +from typing import Dict from zope.interface import Interface, Attribute from twisted.plugin import ( IPlugin, ) +from twisted.internet.defer import Deferred from foolscap.api import StringConstraint, ListOf, TupleOf, SetOf, DictOf, \ ChoiceOf, IntegerConstraint, Any, RemoteInterface, Referenceable @@ -307,12 +309,15 @@ class RIStorageServer(RemoteInterface): store that on disk. """ +# The result of IStorageServer.get_version(): +VersionMessage = Dict[bytes, object] + class IStorageServer(Interface): """ An object capable of storing shares for a storage client. """ - def get_version(): + def get_version() -> Deferred[VersionMessage]: """ :see: ``RIStorageServer.get_version`` """ diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 59213417c..9b44d2a73 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -57,6 +57,7 @@ from .http_common import ( CBOR_MIME_TYPE, get_spki_hash, ) +from ..interfaces import VersionMessage from .common import si_b2a, si_to_human_readable from ..util.hashutil import timing_safe_compare from ..util.deferredutil import async_to_deferred @@ -576,7 +577,7 @@ class StorageClientGeneral(object): _client: StorageClient @async_to_deferred - async def get_version(self) -> dict[bytes, object]: + async def get_version(self) -> VersionMessage: """ Return the version metadata for the server. """ @@ -585,7 +586,7 @@ class StorageClientGeneral(object): ): return await self._get_version() - async def _get_version(self) -> dict[bytes, object]: + async def _get_version(self) -> VersionMessage: """Implementation of get_version().""" url = self._client.relative_url("/storage/v1/version") response = await self._client.request("GET", url) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index a614c17db..4efc845b4 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -33,7 +33,7 @@ Ported to Python 3. from __future__ import annotations from six import ensure_text -from typing import Union, Callable, Any, Optional +from typing import Union, Callable, Any, Optional, cast from os import urandom import re import time @@ -53,6 +53,7 @@ from twisted.python.failure import Failure from twisted.web import http from twisted.internet.task import LoopingCall from twisted.internet import defer, reactor +from twisted.internet.interfaces import IReactorTime from twisted.application import service from twisted.plugin import ( getPlugins, @@ -70,6 +71,7 @@ from allmydata.interfaces import ( IServer, IStorageServer, IFoolscapStoragePlugin, + VersionMessage ) from allmydata.grid_manager import ( create_grid_manager_verifier, @@ -1089,7 +1091,7 @@ class HTTPNativeStorageServer(service.MultiService): self._connection_status = connection_status.ConnectionStatus.unstarted() self._version = None self._last_connect_time = None - self._connecting_deferred = None + self._connecting_deferred : Optional[defer.Deferred[object]]= None def get_permutation_seed(self): return self._permutation_seed @@ -1266,7 +1268,7 @@ class HTTPNativeStorageServer(service.MultiService): # If we've gotten this far, we've found a working NURL. storage_client = await self._storage_client_factory.create_storage_client( - nurl, reactor, None + nurl, cast(IReactorTime, reactor), None ) self._istorage_server = _HTTPStorageServer.from_http_client(storage_client) return self._istorage_server @@ -1507,7 +1509,7 @@ class _HTTPStorageServer(object): """ return _HTTPStorageServer(http_client=http_client) - def get_version(self): + def get_version(self) -> defer.Deferred[VersionMessage]: return StorageClientGeneral(self._http_client).get_version() @defer.inlineCallbacks From 55d62d609b286f3778bb290f344ef03c652c1176 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 14 Jun 2023 16:50:02 -0400 Subject: [PATCH 1866/2309] Fix some mypy errors. --- src/allmydata/web/common.py | 8 ++++++-- src/allmydata/web/operations.py | 5 +++-- src/allmydata/webish.py | 4 +++- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/src/allmydata/web/common.py b/src/allmydata/web/common.py index bd1e3838e..a8de54c0d 100644 --- a/src/allmydata/web/common.py +++ b/src/allmydata/web/common.py @@ -86,6 +86,8 @@ from allmydata.util.encodingutil import ( ) from allmydata.util import abbreviate from allmydata.crypto.rsa import PrivateKey, PublicKey, create_signing_keypair_from_string +from ..webish import TahoeLAFSRequest + class WebError(Exception): def __init__(self, text, code=http.BAD_REQUEST): @@ -723,13 +725,15 @@ def get_arg(req: IRequest, argname: str | bytes, default: Optional[T] = None, *, :return: Either bytes or tuple of bytes. """ + # This is not... obvious to callers, let's say, but it does happen. + assert isinstance(req, TahoeLAFSRequest) if isinstance(argname, str): argname_bytes = argname.encode("utf-8") else: argname_bytes = argname - results = [] - if argname_bytes in req.args: + results : list[bytes] = [] + if req.args is not None and argname_bytes in req.args: results.extend(req.args[argname_bytes]) argname_unicode = str(argname_bytes, "utf-8") if req.fields and argname_unicode in req.fields: diff --git a/src/allmydata/web/operations.py b/src/allmydata/web/operations.py index aedf33f37..a564f8484 100644 --- a/src/allmydata/web/operations.py +++ b/src/allmydata/web/operations.py @@ -43,8 +43,9 @@ DAY = 24*HOUR class OphandleTable(resource.Resource, service.Service): """Renders /operations/%d.""" - - name = "operations" + # The type in Twisted for services is wrong in 22.10... + # https://github.com/twisted/twisted/issues/10135 + name = "operations" # type: ignore[assignment] UNCOLLECTED_HANDLE_LIFETIME = 4*DAY COLLECTED_HANDLE_LIFETIME = 1*DAY diff --git a/src/allmydata/webish.py b/src/allmydata/webish.py index 1b2b8192a..ec2582f80 100644 --- a/src/allmydata/webish.py +++ b/src/allmydata/webish.py @@ -242,7 +242,9 @@ class TahoeLAFSSite(Site, object): class WebishServer(service.MultiService): - name = "webish" + # The type in Twisted for services is wrong in 22.10... + # https://github.com/twisted/twisted/issues/10135 + name = "webish" # type: ignore[assignment] def __init__(self, client, webport, tempdir, nodeurl_path=None, staticdir=None, clock=None, now_fn=time.time): From af323d2bbb7753da7ec25c70ecb40f711c071c0b Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 14 Jun 2023 17:05:51 -0400 Subject: [PATCH 1867/2309] Get the code working again. --- src/allmydata/web/common.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/allmydata/web/common.py b/src/allmydata/web/common.py index a8de54c0d..0685d60a0 100644 --- a/src/allmydata/web/common.py +++ b/src/allmydata/web/common.py @@ -86,7 +86,6 @@ from allmydata.util.encodingutil import ( ) from allmydata.util import abbreviate from allmydata.crypto.rsa import PrivateKey, PublicKey, create_signing_keypair_from_string -from ..webish import TahoeLAFSRequest class WebError(Exception): @@ -725,8 +724,11 @@ def get_arg(req: IRequest, argname: str | bytes, default: Optional[T] = None, *, :return: Either bytes or tuple of bytes. """ - # This is not... obvious to callers, let's say, but it does happen. - assert isinstance(req, TahoeLAFSRequest) + # This is not... obvious to callers, let's say, but it does happen in + # pretty much all non-test real code. We have to import here to prevent + # circular import. + from ..webish import TahoeLAFSRequest + if isinstance(argname, str): argname_bytes = argname.encode("utf-8") else: @@ -736,7 +738,7 @@ def get_arg(req: IRequest, argname: str | bytes, default: Optional[T] = None, *, if req.args is not None and argname_bytes in req.args: results.extend(req.args[argname_bytes]) argname_unicode = str(argname_bytes, "utf-8") - if req.fields and argname_unicode in req.fields: + if isinstance(req, TahoeLAFSRequest) and req.fields and argname_unicode in req.fields: value = req.fields[argname_unicode].value if isinstance(value, str): value = value.encode("utf-8") From 44b752c87d97ba78fce3726e34d9ee0ee7b3fcdf Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 14 Jun 2023 17:43:39 -0400 Subject: [PATCH 1868/2309] Fix mypy issues --- src/allmydata/frontends/sftpd.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/src/allmydata/frontends/sftpd.py b/src/allmydata/frontends/sftpd.py index d2d614c77..14f17e12d 100644 --- a/src/allmydata/frontends/sftpd.py +++ b/src/allmydata/frontends/sftpd.py @@ -1925,7 +1925,11 @@ class FakeTransport(object): def loseConnection(self): logmsg("FakeTransport.loseConnection()", level=NOISY) - # getPeer and getHost can just raise errors, since we don't know what to return + def getHost(self): + pass + + def getPeer(self): + pass @implementer(ISession) @@ -1990,15 +1994,18 @@ class Dispatcher(object): def __init__(self, client): self._client = client - def requestAvatar(self, avatarID, mind, interface): + def requestAvatar(self, avatarId, mind, *interfaces): + [interface] = interfaces _assert(interface == IConchUser, interface=interface) - rootnode = self._client.create_node_from_uri(avatarID.rootcap) - handler = SFTPUserHandler(self._client, rootnode, avatarID.username) + rootnode = self._client.create_node_from_uri(avatarId.rootcap) + handler = SFTPUserHandler(self._client, rootnode, avatarId.username) return (interface, handler, handler.logout) class SFTPServer(service.MultiService): - name = "frontend:sftp" + # The type in Twisted for services is wrong in 22.10... + # https://github.com/twisted/twisted/issues/10135 + name = "frontend:sftp" # type: ignore[assignment] def __init__(self, client, accountfile, sftp_portstr, pubkey_file, privkey_file): From 27243ccfdfadd51db8a5d1fef260d6125b10f3bd Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 14 Jun 2023 17:45:15 -0400 Subject: [PATCH 1869/2309] Fix mypy issues --- src/allmydata/introducer/server.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/allmydata/introducer/server.py b/src/allmydata/introducer/server.py index 5dad89ae8..157a1b73c 100644 --- a/src/allmydata/introducer/server.py +++ b/src/allmydata/introducer/server.py @@ -142,9 +142,12 @@ def stringify_remote_address(rref): return str(remote) +# MyPy doesn't work well with remote interfaces... @implementer(RIIntroducerPublisherAndSubscriberService_v2) -class IntroducerService(service.MultiService, Referenceable): - name = "introducer" +class IntroducerService(service.MultiService, Referenceable): # type: ignore[misc] + # The type in Twisted for services is wrong in 22.10... + # https://github.com/twisted/twisted/issues/10135 + name = "introducer" # type: ignore[assignment] # v1 is the original protocol, added in 1.0 (but only advertised starting # in 1.3), removed in 1.12. v2 is the new signed protocol, added in 1.10 # TODO: reconcile bytes/str for keys From 9306f5edab0ed72e7acde86af6ff2d57fbe378b9 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 14 Jun 2023 17:48:13 -0400 Subject: [PATCH 1870/2309] Fix mypy issues --- src/allmydata/scripts/admin.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/allmydata/scripts/admin.py b/src/allmydata/scripts/admin.py index 02fd9a143..be700bcca 100644 --- a/src/allmydata/scripts/admin.py +++ b/src/allmydata/scripts/admin.py @@ -112,6 +112,9 @@ class AddGridManagerCertOptions(BaseOptions): return "Usage: tahoe [global-options] admin add-grid-manager-cert [options]" def postOptions(self) -> None: + assert self.parent is not None + assert self.parent.parent is not None + if self['name'] is None: raise usage.UsageError( "Must provide --name option" @@ -123,8 +126,8 @@ class AddGridManagerCertOptions(BaseOptions): data: str if self['filename'] == '-': - print("reading certificate from stdin", file=self.parent.parent.stderr) - data = self.parent.parent.stdin.read() + print("reading certificate from stdin", file=self.parent.parent.stderr) # type: ignore[attr-defined] + data = self.parent.parent.stdin.read() # type: ignore[attr-defined] if len(data) == 0: raise usage.UsageError( "Reading certificate from stdin failed" From ce1839f2033c1700caa6d7f9b1de5ef713b3e142 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 14 Jun 2023 17:51:54 -0400 Subject: [PATCH 1871/2309] Pacify mypy --- src/allmydata/test/test_storage_http.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 233d82989..2b4023bc5 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -514,8 +514,8 @@ class Reactor(Clock): Clock.__init__(self) self._queue = Queue() - def callFromThread(self, f, *args, **kwargs): - self._queue.put((f, args, kwargs)) + def callFromThread(self, callable, *args, **kwargs): + self._queue.put((callable, args, kwargs)) def advance(self, *args, **kwargs): Clock.advance(self, *args, **kwargs) From 0f8100b1e9646993a22175a77ca5a0ee1e707cce Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 14 Jun 2023 17:52:01 -0400 Subject: [PATCH 1872/2309] Fix whitespace --- src/allmydata/scripts/admin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/scripts/admin.py b/src/allmydata/scripts/admin.py index be700bcca..3acd52267 100644 --- a/src/allmydata/scripts/admin.py +++ b/src/allmydata/scripts/admin.py @@ -114,7 +114,7 @@ class AddGridManagerCertOptions(BaseOptions): def postOptions(self) -> None: assert self.parent is not None assert self.parent.parent is not None - + if self['name'] is None: raise usage.UsageError( "Must provide --name option" From 96afb0743ac2ad9eb4af3c9ce94fcb8d65ac49ba Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 14 Jun 2023 17:57:25 -0400 Subject: [PATCH 1873/2309] Pacify mypy --- src/allmydata/test/test_consumer.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/allmydata/test/test_consumer.py b/src/allmydata/test/test_consumer.py index 234fc2594..ee1908ba7 100644 --- a/src/allmydata/test/test_consumer.py +++ b/src/allmydata/test/test_consumer.py @@ -39,6 +39,12 @@ class Producer(object): self.consumer = consumer self.done = False + def stopProducing(self): + pass + + def pauseProducing(self): + pass + def resumeProducing(self): """Kick off streaming.""" self.iterate() From bf5213cb016923c757780779afc02ac2847d3347 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 14 Jun 2023 18:01:52 -0400 Subject: [PATCH 1874/2309] Pacify mypy --- src/allmydata/test/mutable/test_version.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/src/allmydata/test/mutable/test_version.py b/src/allmydata/test/mutable/test_version.py index 87050424b..c91c1d4f1 100644 --- a/src/allmydata/test/mutable/test_version.py +++ b/src/allmydata/test/mutable/test_version.py @@ -78,18 +78,21 @@ class Version(GridTestMixin, AsyncTestCase, testutil.ShouldFailMixin, \ fso.nodedirs = [os.path.dirname(abspath_expanduser_unicode(str(storedir))) for (i,ss,storedir) in self.iterate_servers()] - fso.stdout = StringIO() - fso.stderr = StringIO() + # This attribute isn't defined on FindSharesOptions but `find_shares()` + # definitely expects it... + fso.stdout = StringIO() # type: ignore[attr-defined] debug.find_shares(fso) - sharefiles = fso.stdout.getvalue().splitlines() + sharefiles = fso.stdout.getvalue().splitlines() # type: ignore[attr-defined] expected = self.nm.default_encoding_parameters["n"] self.assertThat(sharefiles, HasLength(expected)) + # This attribute isn't defined on DebugOptions but `dump_share()` + # definitely expects it... do = debug.DumpOptions() do["filename"] = sharefiles[0] - do.stdout = StringIO() + do.stdout = StringIO() # type: ignore[attr-defined] debug.dump_share(do) - output = do.stdout.getvalue() + output = do.stdout.getvalue() # type: ignore[attr-defined] lines = set(output.splitlines()) self.assertTrue("Mutable slot found:" in lines, output) self.assertTrue(" share_type: MDMF" in lines, output) @@ -104,10 +107,12 @@ class Version(GridTestMixin, AsyncTestCase, testutil.ShouldFailMixin, \ self.assertTrue(" verify-cap: %s" % vcap in lines, output) cso = debug.CatalogSharesOptions() cso.nodedirs = fso.nodedirs - cso.stdout = StringIO() - cso.stderr = StringIO() + # Definitely not options on CatalogSharesOptions, but the code does use + # stdout and stderr... + cso.stdout = StringIO() # type: ignore[attr-defined] + cso.stderr = StringIO() # type: ignore[attr-defined] debug.catalog_shares(cso) - shares = cso.stdout.getvalue().splitlines() + shares = cso.stdout.getvalue().splitlines() # type: ignore[attr-defined] oneshare = shares[0] # all shares should be MDMF self.failIf(oneshare.startswith("UNKNOWN"), oneshare) self.assertTrue(oneshare.startswith("MDMF"), oneshare) From 8d99ddc542825e4e893b92546fe0856bb283b701 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 15 Jun 2023 17:14:08 -0400 Subject: [PATCH 1875/2309] Pacify mypy --- src/allmydata/test/cli/test_invite.py | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/src/allmydata/test/cli/test_invite.py b/src/allmydata/test/cli/test_invite.py index 31992a54d..1302e5970 100644 --- a/src/allmydata/test/cli/test_invite.py +++ b/src/allmydata/test/cli/test_invite.py @@ -8,7 +8,7 @@ import json import os from functools import partial from os.path import join -from typing import Awaitable, Callable, Optional, Sequence, TypeVar, Union +from typing import Callable, Optional, Sequence, TypeVar, Union, Coroutine, Any, Tuple, cast, Generator from twisted.internet import defer from twisted.trial import unittest @@ -60,7 +60,7 @@ def make_simple_peer( server: MemoryWormholeServer, helper: TestingHelper, messages: Sequence[JSONable], -) -> Callable[[], Awaitable[IWormhole]]: +) -> Callable[[], Coroutine[defer.Deferred[IWormhole], Any, IWormhole]]: """ Make a wormhole peer that just sends the given messages. @@ -102,18 +102,24 @@ A = TypeVar("A") B = TypeVar("B") def concurrently( - client: Callable[[], Awaitable[A]], - server: Callable[[], Awaitable[B]], -) -> defer.Deferred[tuple[A, B]]: + client: Callable[[], Union[ + Coroutine[defer.Deferred[A], Any, A], + Generator[defer.Deferred[A], Any, A], + ]], + server: Callable[[], Union[ + Coroutine[defer.Deferred[B], Any, B], + Generator[defer.Deferred[B], Any, B], + ]], +) -> defer.Deferred[Tuple[A, B]]: """ Run two asynchronous functions concurrently and asynchronously return a tuple of both their results. """ - return defer.gatherResults([ + result = defer.gatherResults([ defer.Deferred.fromCoroutine(client()), defer.Deferred.fromCoroutine(server()), - ]) - + ]).addCallback(tuple) # type: ignore + return cast(defer.Deferred[Tuple[A, B]], result) class Join(GridTestMixin, CLITestMixin, unittest.TestCase): From db9597ee1995a5f20de3a58fe0c5660b693fd8ca Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 19 Jun 2023 16:07:31 -0600 Subject: [PATCH 1876/2309] add --allow-stdin-close option --- src/allmydata/scripts/tahoe_run.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/src/allmydata/scripts/tahoe_run.py b/src/allmydata/scripts/tahoe_run.py index aaf234b61..d54fe9af3 100644 --- a/src/allmydata/scripts/tahoe_run.py +++ b/src/allmydata/scripts/tahoe_run.py @@ -104,6 +104,11 @@ class RunOptions(BasedirOptions): " [default: %s]" % quote_local_unicode_path(_default_nodedir)), ] + optFlags = [ + ("allow-stdin-close", None, + 'Do not exit when stdin closes ("tahoe run" otherwise will exit).'), + ] + def parseArgs(self, basedir=None, *twistd_args): # This can't handle e.g. 'tahoe run --reactor=foo', since # '--reactor=foo' looks like an option to the tahoe subcommand, not to @@ -156,6 +161,7 @@ class DaemonizeTheRealService(Service, HookMixin): "running": None, } self.stderr = options.parent.stderr + self._close_on_stdin_close = False if options["allow-stdin-close"] else True def startService(self): @@ -202,7 +208,8 @@ class DaemonizeTheRealService(Service, HookMixin): srv.setServiceParent(self.parent) # exiting on stdin-closed facilitates cleanup when run # as a subprocess - on_stdin_close(reactor, reactor.stop) + if self._close_on_stdin_close: + on_stdin_close(reactor, reactor.stop) d.addCallback(created) d.addErrback(handle_config_error) d.addBoth(self._call_hook, 'running') @@ -213,11 +220,13 @@ class DaemonizeTheRealService(Service, HookMixin): class DaemonizeTahoeNodePlugin(object): tapname = "tahoenode" - def __init__(self, nodetype, basedir): + def __init__(self, nodetype, basedir, allow_stdin_close): self.nodetype = nodetype self.basedir = basedir + self.allow_stdin_close = allow_stdin_close def makeService(self, so): + so["allow-stdin-close"] = self.allow_stdin_close return DaemonizeTheRealService(self.nodetype, self.basedir, so) @@ -304,7 +313,9 @@ def run(reactor, config, runApp=twistd.runApp): print(config, file=err) print("tahoe %s: usage error from twistd: %s\n" % (config.subcommand_name, ue), file=err) return 1 - twistd_config.loadedPlugins = {"DaemonizeTahoeNode": DaemonizeTahoeNodePlugin(nodetype, basedir)} + twistd_config.loadedPlugins = { + "DaemonizeTahoeNode": DaemonizeTahoeNodePlugin(nodetype, basedir, config["allow-stdin-close"]) + } # our own pid-style file contains PID and process creation time pidfile = FilePath(get_pidfile(config['basedir'])) From a107e163351faa5f62dfc33781d5e12a5781a07e Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 19 Jun 2023 16:08:17 -0600 Subject: [PATCH 1877/2309] news --- newsfragments/4036.enhancement | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/4036.enhancement diff --git a/newsfragments/4036.enhancement b/newsfragments/4036.enhancement new file mode 100644 index 000000000..36c062718 --- /dev/null +++ b/newsfragments/4036.enhancement @@ -0,0 +1 @@ +tahoe run now accepts --allow-stdin-close to mean "keep running if stdin closes" \ No newline at end of file From 2fcb190c2f5a8550284eaa27a0ee0f8d2f2f068b Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 19 Jun 2023 17:53:57 -0600 Subject: [PATCH 1878/2309] add tests for both close-stdin cases --- src/allmydata/scripts/tahoe_run.py | 3 +- src/allmydata/test/cli/test_run.py | 95 ++++++++++++++++++++++++++++++ 2 files changed, 97 insertions(+), 1 deletion(-) diff --git a/src/allmydata/scripts/tahoe_run.py b/src/allmydata/scripts/tahoe_run.py index d54fe9af3..ff3ff9efd 100644 --- a/src/allmydata/scripts/tahoe_run.py +++ b/src/allmydata/scripts/tahoe_run.py @@ -205,7 +205,8 @@ class DaemonizeTheRealService(Service, HookMixin): d = service_factory() def created(srv): - srv.setServiceParent(self.parent) + if self.parent is not None: + srv.setServiceParent(self.parent) # exiting on stdin-closed facilitates cleanup when run # as a subprocess if self._close_on_stdin_close: diff --git a/src/allmydata/test/cli/test_run.py b/src/allmydata/test/cli/test_run.py index e84f52096..96640d45a 100644 --- a/src/allmydata/test/cli/test_run.py +++ b/src/allmydata/test/cli/test_run.py @@ -31,6 +31,12 @@ from twisted.python.filepath import ( from twisted.internet.testing import ( MemoryReactor, ) +from twisted.python.failure import ( + Failure, +) +from twisted.internet.error import ( + ConnectionDone, +) from twisted.internet.test.modulehelpers import ( AlternateReactor, ) @@ -147,6 +153,95 @@ class DaemonizeTheRealServiceTests(SyncTestCase): ) +class DaemonizeStopTests(SyncTestCase): + """ + Tests relating to stopping the daemon + """ + def setUp(self): + self.nodedir = FilePath(self.mktemp()) + self.nodedir.makedirs() + config = "" + self.nodedir.child("tahoe.cfg").setContent(config.encode("ascii")) + self.nodedir.child("tahoe-client.tac").touch() + + # arrange to know when reactor.stop() is called + self.reactor = MemoryReactor() + self.stop_calls = [] + + def record_stop(): + self.stop_calls.append(object()) + self.reactor.stop = record_stop + + super().setUp() + + def test_stop_on_stdin_close(self): + """ + We stop when stdin is closed. + """ + options = parse_options(["run", self.nodedir.path]) + stdout = options.stdout = StringIO() + stderr = options.stderr = StringIO() + stdin = options.stdin = StringIO() + run_options = options.subOptions + + with AlternateReactor(self.reactor): + service = DaemonizeTheRealService( + "client", + self.nodedir.path, + run_options, + ) + service.startService() + + # We happen to know that the service uses reactor.callWhenRunning + # to schedule all its work (though I couldn't tell you *why*). + # Make sure those scheduled calls happen. + waiting = self.reactor.whenRunningHooks[:] + del self.reactor.whenRunningHooks[:] + for f, a, k in waiting: + f(*a, **k) + + # there should be a single reader: our StandardIO process + # reader for stdin. Simulate it closing. + for r in self.reactor.getReaders(): + r.connectionLost(Failure(ConnectionDone())) + + self.assertEqual(len(self.stop_calls), 1) + + def test_allow_stdin_close(self): + """ + If --allow-stdin-close is specified then closing stdin doesn't + stop the process + """ + options = parse_options(["run", "--allow-stdin-close", self.nodedir.path]) + stdout = options.stdout = StringIO() + stderr = options.stderr = StringIO() + stdin = options.stdin = StringIO() + run_options = options.subOptions + + with AlternateReactor(self.reactor): + service = DaemonizeTheRealService( + "client", + self.nodedir.path, + run_options, + ) + service.startService() + + # We happen to know that the service uses reactor.callWhenRunning + # to schedule all its work (though I couldn't tell you *why*). + # Make sure those scheduled calls happen. + waiting = self.reactor.whenRunningHooks[:] + del self.reactor.whenRunningHooks[:] + for f, a, k in waiting: + f(*a, **k) + + # kind of cheating -- there are no readers, because we + # never instantiated a StandardIO in this case.. + for r in self.reactor.getReaders(): + r.connectionLost(Failure(ConnectionDone())) + + self.assertEqual(self.stop_calls, []) + + class RunTests(SyncTestCase): """ Tests for ``run``. From e765c8db6fb237e161b2c4830601a7486b88a4e9 Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 19 Jun 2023 17:55:30 -0600 Subject: [PATCH 1879/2309] move news --- newsfragments/{4036.enhancement => 4036.feature} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename newsfragments/{4036.enhancement => 4036.feature} (100%) diff --git a/newsfragments/4036.enhancement b/newsfragments/4036.feature similarity index 100% rename from newsfragments/4036.enhancement rename to newsfragments/4036.feature From 357c9b003f1b5866f9eb250b6c1b5313aacf86e3 Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 19 Jun 2023 17:55:36 -0600 Subject: [PATCH 1880/2309] flake8 --- src/allmydata/test/cli/test_run.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/allmydata/test/cli/test_run.py b/src/allmydata/test/cli/test_run.py index 96640d45a..731269d3d 100644 --- a/src/allmydata/test/cli/test_run.py +++ b/src/allmydata/test/cli/test_run.py @@ -179,9 +179,9 @@ class DaemonizeStopTests(SyncTestCase): We stop when stdin is closed. """ options = parse_options(["run", self.nodedir.path]) - stdout = options.stdout = StringIO() - stderr = options.stderr = StringIO() - stdin = options.stdin = StringIO() + options.stdout = StringIO() + options.stderr = StringIO() + options.stdin = StringIO() run_options = options.subOptions with AlternateReactor(self.reactor): @@ -213,9 +213,9 @@ class DaemonizeStopTests(SyncTestCase): stop the process """ options = parse_options(["run", "--allow-stdin-close", self.nodedir.path]) - stdout = options.stdout = StringIO() - stderr = options.stderr = StringIO() - stdin = options.stdin = StringIO() + options.stdout = StringIO() + options.stderr = StringIO() + options.stdin = StringIO() run_options = options.subOptions with AlternateReactor(self.reactor): From 02fba3b2b67e062fb8ca0f2aeb8177309002a451 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 20 Jun 2023 07:45:51 -0400 Subject: [PATCH 1881/2309] factor some duplication out of the tests --- src/allmydata/test/cli/test_run.py | 90 ++++++++++++++---------------- 1 file changed, 43 insertions(+), 47 deletions(-) diff --git a/src/allmydata/test/cli/test_run.py b/src/allmydata/test/cli/test_run.py index 731269d3d..30d5cd893 100644 --- a/src/allmydata/test/cli/test_run.py +++ b/src/allmydata/test/cli/test_run.py @@ -174,37 +174,55 @@ class DaemonizeStopTests(SyncTestCase): super().setUp() - def test_stop_on_stdin_close(self): + def _make_daemon(self, extra_argv: list[str]) -> DaemonizeTheRealService: """ - We stop when stdin is closed. + Create the daemonization service. + + :param extra_argv: Extra arguments to pass between ``run`` and the + node path. """ - options = parse_options(["run", self.nodedir.path]) + options = parse_options(["run"] + extra_argv + [self.nodedir.path]) options.stdout = StringIO() options.stderr = StringIO() options.stdin = StringIO() run_options = options.subOptions + return DaemonizeTheRealService( + "client", + self.nodedir.path, + run_options, + ) + def _run_daemon(self) -> None: + """ + Simulate starting up the reactor so the daemon plugin can do its + stuff. + """ + # We happen to know that the service uses reactor.callWhenRunning + # to schedule all its work (though I couldn't tell you *why*). + # Make sure those scheduled calls happen. + waiting = self.reactor.whenRunningHooks[:] + del self.reactor.whenRunningHooks[:] + for f, a, k in waiting: + f(*a, **k) + + def _close_stdin(self) -> None: + """ + Simulate closing the daemon plugin's stdin. + """ + # there should be a single reader: our StandardIO process + # reader for stdin. Simulate it closing. + for r in self.reactor.getReaders(): + r.connectionLost(Failure(ConnectionDone())) + + def test_stop_on_stdin_close(self): + """ + We stop when stdin is closed. + """ with AlternateReactor(self.reactor): - service = DaemonizeTheRealService( - "client", - self.nodedir.path, - run_options, - ) + service = self._make_daemon([]) service.startService() - - # We happen to know that the service uses reactor.callWhenRunning - # to schedule all its work (though I couldn't tell you *why*). - # Make sure those scheduled calls happen. - waiting = self.reactor.whenRunningHooks[:] - del self.reactor.whenRunningHooks[:] - for f, a, k in waiting: - f(*a, **k) - - # there should be a single reader: our StandardIO process - # reader for stdin. Simulate it closing. - for r in self.reactor.getReaders(): - r.connectionLost(Failure(ConnectionDone())) - + self._run_daemon() + self._close_stdin() self.assertEqual(len(self.stop_calls), 1) def test_allow_stdin_close(self): @@ -212,33 +230,11 @@ class DaemonizeStopTests(SyncTestCase): If --allow-stdin-close is specified then closing stdin doesn't stop the process """ - options = parse_options(["run", "--allow-stdin-close", self.nodedir.path]) - options.stdout = StringIO() - options.stderr = StringIO() - options.stdin = StringIO() - run_options = options.subOptions - with AlternateReactor(self.reactor): - service = DaemonizeTheRealService( - "client", - self.nodedir.path, - run_options, - ) + service = self._make_daemon(["--allow-stdin-close"]) service.startService() - - # We happen to know that the service uses reactor.callWhenRunning - # to schedule all its work (though I couldn't tell you *why*). - # Make sure those scheduled calls happen. - waiting = self.reactor.whenRunningHooks[:] - del self.reactor.whenRunningHooks[:] - for f, a, k in waiting: - f(*a, **k) - - # kind of cheating -- there are no readers, because we - # never instantiated a StandardIO in this case.. - for r in self.reactor.getReaders(): - r.connectionLost(Failure(ConnectionDone())) - + self._run_daemon() + self._close_stdin() self.assertEqual(self.stop_calls, []) From 7257851565c16ce200a0cfd64c2e92d3cc552783 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 20 Jun 2023 07:46:43 -0400 Subject: [PATCH 1882/2309] python 2/3 porting boilerplate cleanup --- src/allmydata/test/cli/test_run.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/src/allmydata/test/cli/test_run.py b/src/allmydata/test/cli/test_run.py index 30d5cd893..6254a6259 100644 --- a/src/allmydata/test/cli/test_run.py +++ b/src/allmydata/test/cli/test_run.py @@ -1,16 +1,6 @@ """ Tests for ``allmydata.scripts.tahoe_run``. - -Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals - -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import re from six.moves import ( From 592e77beca60c997d06ec62b51e054e5ae59b05f Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 20 Jun 2023 08:12:14 -0400 Subject: [PATCH 1883/2309] allow `list` as a generic container annotation --- src/allmydata/test/cli/test_run.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/allmydata/test/cli/test_run.py b/src/allmydata/test/cli/test_run.py index 6254a6259..2adcfea19 100644 --- a/src/allmydata/test/cli/test_run.py +++ b/src/allmydata/test/cli/test_run.py @@ -2,6 +2,8 @@ Tests for ``allmydata.scripts.tahoe_run``. """ +from __future__ import annotations + import re from six.moves import ( StringIO, From 122e0a73a979f379e6bc7a3f795847be8dc6db0b Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 22 Jun 2023 01:29:55 -0600 Subject: [PATCH 1884/2309] more-generic testing hook --- src/allmydata/storage/http_client.py | 9 ++------- src/allmydata/test/test_storage_http.py | 17 ++++++++++++++--- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index e7df3709d..8c0100656 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -427,7 +427,7 @@ class StorageClient(object): _pool: HTTPConnectionPool _clock: IReactorTime # Are we running unit tests? - _test_mode: bool + _analyze_response: Callable[[IResponse], None] = lambda _: None def relative_url(self, path: str) -> DecodedURL: """Get a URL relative to the base URL.""" @@ -534,12 +534,7 @@ class StorageClient(object): response = await self._treq.request( method, url, headers=headers, timeout=timeout, **kwargs ) - - if self._test_mode and response.code != 404: - # We're doing API queries, HTML is never correct except in 404, but - # it's the default for Twisted's web server so make sure nothing - # unexpected happened. - assert get_content_type(response.headers) != "text/html" + self._analyze_response(response) return response diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 233d82989..aaa858db4 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -316,6 +316,17 @@ def result_of(d): + "This is probably a test design issue." ) +def response_is_not_html(response): + """ + During tests, this is registered so we can ensure the web server + doesn't give us text/html. + + HTML is never correct except in 404, but it's the default for + Twisted's web server so we assert nothing unexpected happened. + """ + if response.code != 404: + assert get_content_type(response.headers) != "text/html" + class CustomHTTPServerTests(SyncTestCase): """ @@ -342,7 +353,7 @@ class CustomHTTPServerTests(SyncTestCase): # fixed if https://github.com/twisted/treq/issues/226 were ever # fixed. clock=treq._agent._memoryReactor, - test_mode=True, + analyze_response=response_is_not_html, ) self._http_server.clock = self.client._clock @@ -560,7 +571,7 @@ class HttpTestFixture(Fixture): treq=self.treq, pool=None, clock=self.clock, - test_mode=True, + analyze_response=response_is_not_html, ) def result_of_with_flush(self, d): @@ -674,7 +685,7 @@ class GenericHTTPAPITests(SyncTestCase): treq=StubTreq(self.http.http_server.get_resource()), pool=None, clock=self.http.clock, - test_mode=True, + analyze_response=response_is_not_html, ) ) with assert_fails_with_http_code(self, http.UNAUTHORIZED): From 75b9c59846bffadbeca2c5941931d335790a23bd Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 22 Jun 2023 01:54:53 -0600 Subject: [PATCH 1885/2309] refactor --- src/allmydata/storage/http_client.py | 7 ++++++- src/allmydata/storage/http_common.py | 13 +++++++++++++ src/allmydata/test/test_storage_http.py | 18 +++++------------- 3 files changed, 24 insertions(+), 14 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 8c0100656..f2165ffda 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -57,6 +57,7 @@ from .http_common import ( get_content_type, CBOR_MIME_TYPE, get_spki_hash, + response_is_not_html, ) from .common import si_b2a, si_to_human_readable from ..util.hashutil import timing_safe_compare @@ -402,13 +403,17 @@ class StorageClientFactory: treq_client = HTTPClient(agent) https_url = DecodedURL().replace(scheme="https", host=nurl.host, port=nurl.port) swissnum = nurl.path[0].encode("ascii") + response_check = lambda _: None + if self.TEST_MODE_REGISTER_HTTP_POOL is not None: + response_check = response_is_not_html + return StorageClient( https_url, swissnum, treq_client, pool, reactor, - self.TEST_MODE_REGISTER_HTTP_POOL is not None, + response_check, ) diff --git a/src/allmydata/storage/http_common.py b/src/allmydata/storage/http_common.py index e5f07898e..7ee137e1d 100644 --- a/src/allmydata/storage/http_common.py +++ b/src/allmydata/storage/http_common.py @@ -12,6 +12,7 @@ from cryptography.hazmat.primitives.serialization import Encoding, PublicFormat from werkzeug.http import parse_options_header from twisted.web.http_headers import Headers +from twisted.web.iweb import IResponse CBOR_MIME_TYPE = "application/cbor" @@ -27,6 +28,18 @@ def get_content_type(headers: Headers) -> Optional[str]: return content_type +def response_is_not_html(response: IResponse) -> None: + """ + During tests, this is registered so we can ensure the web server + doesn't give us text/html. + + HTML is never correct except in 404, but it's the default for + Twisted's web server so we assert nothing unexpected happened. + """ + if response.code != 404: + assert get_content_type(response.headers) != "text/html" + + def swissnum_auth_header(swissnum: bytes) -> bytes: """Return value for ``Authorization`` header.""" return b"Tahoe-LAFS " + b64encode(swissnum).strip() diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index aaa858db4..f660342ae 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -43,7 +43,11 @@ from testtools.matchers import Equals from zope.interface import implementer from .common import SyncTestCase -from ..storage.http_common import get_content_type, CBOR_MIME_TYPE +from ..storage.http_common import ( + get_content_type, + CBOR_MIME_TYPE, + response_is_not_html, +) from ..storage.common import si_b2a from ..storage.lease import LeaseInfo from ..storage.server import StorageServer @@ -316,18 +320,6 @@ def result_of(d): + "This is probably a test design issue." ) -def response_is_not_html(response): - """ - During tests, this is registered so we can ensure the web server - doesn't give us text/html. - - HTML is never correct except in 404, but it's the default for - Twisted's web server so we assert nothing unexpected happened. - """ - if response.code != 404: - assert get_content_type(response.headers) != "text/html" - - class CustomHTTPServerTests(SyncTestCase): """ Tests that use a custom HTTP server. From 992687a8b9a9390ac7534e3eb9d89519001d3bd5 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 26 Jun 2023 09:05:36 -0400 Subject: [PATCH 1886/2309] News fragment --- newsfragments/3622.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3622.minor diff --git a/newsfragments/3622.minor b/newsfragments/3622.minor new file mode 100644 index 000000000..e69de29bb From 5f9e784964ebbee5575cd6d57a2c65480ab99a15 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 26 Jun 2023 09:06:28 -0400 Subject: [PATCH 1887/2309] Better explanation --- src/allmydata/web/common.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/allmydata/web/common.py b/src/allmydata/web/common.py index 0685d60a0..1a0ba433b 100644 --- a/src/allmydata/web/common.py +++ b/src/allmydata/web/common.py @@ -724,9 +724,7 @@ def get_arg(req: IRequest, argname: str | bytes, default: Optional[T] = None, *, :return: Either bytes or tuple of bytes. """ - # This is not... obvious to callers, let's say, but it does happen in - # pretty much all non-test real code. We have to import here to prevent - # circular import. + # Need to import here to prevent circular import: from ..webish import TahoeLAFSRequest if isinstance(argname, str): @@ -739,6 +737,8 @@ def get_arg(req: IRequest, argname: str | bytes, default: Optional[T] = None, *, results.extend(req.args[argname_bytes]) argname_unicode = str(argname_bytes, "utf-8") if isinstance(req, TahoeLAFSRequest) and req.fields and argname_unicode in req.fields: + # In all but one or two unit tests, the request will be a + # TahoeLAFSRequest. value = req.fields[argname_unicode].value if isinstance(value, str): value = value.encode("utf-8") From a7f45ab35564682511860314ac05ea012f8e8606 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 26 Jun 2023 11:09:32 -0400 Subject: [PATCH 1888/2309] If this ever does get called, make the error less obscure. --- src/allmydata/frontends/sftpd.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/frontends/sftpd.py b/src/allmydata/frontends/sftpd.py index 14f17e12d..7ef9a8820 100644 --- a/src/allmydata/frontends/sftpd.py +++ b/src/allmydata/frontends/sftpd.py @@ -1926,10 +1926,10 @@ class FakeTransport(object): logmsg("FakeTransport.loseConnection()", level=NOISY) def getHost(self): - pass + raise NotImplementedError() def getPeer(self): - pass + raise NotImplementedError() @implementer(ISession) From a7100c749d400be0024c60eb5d93e92c45e32105 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 26 Jun 2023 11:20:46 -0400 Subject: [PATCH 1889/2309] Specific commit --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 89dbda748..2f245f2ed 100644 --- a/tox.ini +++ b/tox.ini @@ -124,7 +124,7 @@ basepython = python3 deps = mypy==1.3.0 # When 0.9.2 comes out it will work with 1.3, it's just unreleased at the moment... - git+https://github.com/shoobx/mypy-zope + git+https://github.com/shoobx/mypy-zope@f276030 types-mock types-six types-PyYAML From d8ca0176ab2341d42c3cd808bd9c1a166eec36f6 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 3 Jul 2023 11:05:29 -0400 Subject: [PATCH 1890/2309] Pass the correct arguments in. --- integration/conftest.py | 2 +- integration/test_tor.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/integration/conftest.py b/integration/conftest.py index 643295291..43f16d45b 100644 --- a/integration/conftest.py +++ b/integration/conftest.py @@ -214,7 +214,7 @@ def tor_introducer(reactor, temp_dir, flog_gatherer, request): config = read_config(intro_dir, "tub.port") config.set_config("node", "nickname", "introducer-tor") config.set_config("node", "web.port", "4561") - config.set_config("node", "log_gatherer.furl", flog_gatherer) + config.set_config("node", "log_gatherer.furl", flog_gatherer.furl) # "tahoe run" is consistent across Linux/macOS/Windows, unlike the old # "start" command. diff --git a/integration/test_tor.py b/integration/test_tor.py index 32572276a..4d0ce4f16 100644 --- a/integration/test_tor.py +++ b/integration/test_tor.py @@ -128,7 +128,7 @@ def _create_anonymous_node(reactor, name, control_port, request, temp_dir, flog_ # Which services should this client connect to? write_introducer(node_dir, "default", introducer_furl) - util.basic_node_configuration(request, flog_gatherer, node_dir.path) + util.basic_node_configuration(request, flog_gatherer.furl, node_dir.path) config = read_config(node_dir.path, "tub.port") config.set_config("tor", "onion", "true") From f4ed5cb0f347abb680f7ba143119877b7610a604 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 3 Jul 2023 11:30:35 -0400 Subject: [PATCH 1891/2309] Fix lint --- integration/conftest.py | 1 - 1 file changed, 1 deletion(-) diff --git a/integration/conftest.py b/integration/conftest.py index 43f16d45b..6de2e84af 100644 --- a/integration/conftest.py +++ b/integration/conftest.py @@ -30,7 +30,6 @@ import pytest import pytest_twisted from .util import ( - _CollectOutputProtocol, _MagicTextProtocol, _DumpOutputProtocol, _ProcessExitedProtocol, From 76f8ab617276e07f0cab382216d40c9dcf9b81c5 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 3 Jul 2023 13:07:56 -0400 Subject: [PATCH 1892/2309] Set the config the way we were in latest code. --- integration/grid.py | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/integration/grid.py b/integration/grid.py index ec8b1e0e0..794639b2f 100644 --- a/integration/grid.py +++ b/integration/grid.py @@ -43,6 +43,7 @@ from twisted.internet.protocol import ( ) from twisted.internet.error import ProcessTerminated +from allmydata.node import read_config from .util import ( _CollectOutputProtocol, _MagicTextProtocol, @@ -306,16 +307,6 @@ def create_introducer(reactor, request, temp_dir, flog_gatherer, port): """ Run a new Introducer and return an Introducer instance. """ - config = ( - '[node]\n' - 'nickname = introducer{port}\n' - 'web.port = {port}\n' - 'log_gatherer.furl = {log_furl}\n' - ).format( - port=port, - log_furl=flog_gatherer.furl, - ) - intro_dir = join(temp_dir, 'introducer{}'.format(port)) if not exists(intro_dir): @@ -334,9 +325,10 @@ def create_introducer(reactor, request, temp_dir, flog_gatherer, port): ) yield done_proto.done - # over-write the config file with our stuff - with open(join(intro_dir, 'tahoe.cfg'), 'w') as f: - f.write(config) + config = read_config(intro_dir, "tub.port") + config.set_config("node", "nickname", f"introducer-{port}") + config.set_config("node", "web.port", f"{port}") + config.set_config("node", "log_gatherer.furl", flog_gatherer.furl) # on windows, "tahoe start" means: run forever in the foreground, # but on linux it means daemonize. "tahoe run" is consistent From 92474375350022fc12521b060160beeba777e3db Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 4 Jul 2023 12:43:23 -0400 Subject: [PATCH 1893/2309] Add a flake with some packages and apps and an overlay --- default.nix | 29 +------ flake.lock | 80 +++++++++++++++++++ flake.nix | 162 +++++++++++++++++++++++++++++++++++++++ nix/overlay.nix | 10 +++ nix/python-overrides.nix | 42 +++++++--- nix/tahoe-lafs.nix | 74 ++++++++---------- nix/tests.nix | 4 - 7 files changed, 323 insertions(+), 78 deletions(-) create mode 100644 flake.lock create mode 100644 flake.nix create mode 100644 nix/overlay.nix delete mode 100644 nix/tests.nix diff --git a/default.nix b/default.nix index d616f63b8..196db4030 100644 --- a/default.nix +++ b/default.nix @@ -18,32 +18,11 @@ in pkgsVersion ? "nixpkgs-22.11" # a string which chooses a nixpkgs from the # niv-managed sources data -, pkgs ? import sources.${pkgsVersion} { } # nixpkgs itself +, pkgs ? import sources.${pkgsVersion} { # nixpkgs itself + overlays = [ (import ./nix/overlay.nix) ]; +} , pythonVersion ? "python310" # a string choosing the python derivation from # nixpkgs to target - -, extrasNames ? [ "tor" "i2p" ] # a list of strings identifying tahoe-lafs extras, - # the dependencies of which the resulting - # package will also depend on. Include all of the - # runtime extras by default because the incremental - # cost of including them is a lot smaller than the - # cost of re-building the whole thing to add them. - }: -with (pkgs.${pythonVersion}.override { - packageOverrides = import ./nix/python-overrides.nix; -}).pkgs; -callPackage ./nix/tahoe-lafs.nix { - # Select whichever package extras were requested. - inherit extrasNames; - - # Define the location of the Tahoe-LAFS source to be packaged (the same - # directory as contains this file). Clean up as many of the non-source - # files (eg the `.git` directory, `~` backup files, nix's own `result` - # symlink, etc) as possible to avoid needing to re-build when files that - # make no difference to the package have changed. - tahoe-lafs-src = pkgs.lib.cleanSource ./.; - - doCheck = false; -} +pkgs.${pythonVersion}.withPackages (ps: [ ps.tahoe-lafs ]) diff --git a/flake.lock b/flake.lock new file mode 100644 index 000000000..0fd3c90c9 --- /dev/null +++ b/flake.lock @@ -0,0 +1,80 @@ +{ + "nodes": { + "flake-utils": { + "inputs": { + "systems": "systems" + }, + "locked": { + "lastModified": 1687709756, + "narHash": "sha256-Y5wKlQSkgEK2weWdOu4J3riRd+kV/VCgHsqLNTTWQ/0=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "dbabf0ca0c0c4bce6ea5eaf65af5cb694d2082c7", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "nixpkgs-22_11": { + "locked": { + "lastModified": 1688392541, + "narHash": "sha256-lHrKvEkCPTUO+7tPfjIcb7Trk6k31rz18vkyqmkeJfY=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "ea4c80b39be4c09702b0cb3b42eab59e2ba4f24b", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-22.11", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs-unstable": { + "locked": { + "lastModified": 1688486599, + "narHash": "sha256-K8v2wCfHjA0LS6QeCZ/x+OU2hhINZG4qAAO6zvvqYhE=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "712caf8eb1c2ea0944d2f34f96570bca7193c1c8", + "type": "github" + }, + "original": { + "owner": "NixOS", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "flake-utils": "flake-utils", + "nixpkgs": [ + "nixpkgs-22_11" + ], + "nixpkgs-22_11": "nixpkgs-22_11", + "nixpkgs-unstable": "nixpkgs-unstable" + } + }, + "systems": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 000000000..89a4a6f14 --- /dev/null +++ b/flake.nix @@ -0,0 +1,162 @@ +{ + description = "Tahoe-LAFS, free and open decentralized data store"; + + inputs = { + # Two alternate nixpkgs pins. Ideally these could be selected easily from + # the command line but there seems to be no syntax/support for that. + # However, these at least cause certain revisions to be pinned in our lock + # file where you *can* dig them out - and the CI configuration does. + "nixpkgs-22_11" = { + url = github:NixOS/nixpkgs?ref=nixos-22.11; + }; + "nixpkgs-unstable" = { + url = github:NixOS/nixpkgs; + }; + + # Point the default nixpkgs at one of those. This avoids having getting a + # _third_ package set involved and gives a way to provide what should be a + # working experience by default (that is, if nixpkgs doesn't get + # overridden). + nixpkgs.follows = "nixpkgs-22_11"; + + # Also get flake-utils for simplified multi-system definitions. + flake-utils = { + url = github:numtide/flake-utils; + }; + }; + + outputs = { self, nixpkgs, flake-utils, ... }: + { + # Expose an overlay which adds our version of Tahoe-LAFS to the Python + # package sets we specify, as well as all of the correct versions of its + # dependencies. + # + # We will also use this to define some other outputs since it gives us + # the most succinct way to get a working Tahoe-LAFS package. + overlays.default = import ./nix/overlay.nix; + + } // (flake-utils.lib.eachDefaultSystem (system: let + + # First get the package set for this system architecture. + pkgs = import nixpkgs { + inherit system; + # And include our Tahoe-LAFS package in that package set. + overlays = [ self.overlays.default ]; + }; + + # Find out what Python versions we're working with. + pythonVersions = builtins.attrNames ( + pkgs.lib.attrsets.filterAttrs + # Match attribute names that look like a Python derivation - CPython + # or PyPy. We take care to avoid things like "python-foo" and + # "python3Full-unittest" though. We only want things like "pypy38" + # or "python311". + (name: _: null != builtins.match "(python|pypy)3[[:digit:]]{0,2}" name) + pkgs + ); + + # An element of pythonVersions which we'll use for the default package. + defaultPyVersion = "python310"; + + # Retrieve the actual Python package for each configured version. We + # already applied our overlay to pkgs so our packages will already be + # available. + pythons = builtins.map (pyVer: pkgs.${pyVer}) pythonVersions; + + # string -> string + # + # Construct the Tahoe-LAFS package name for the given Python runtime. + packageName = pyVersion: "${pyVersion}-tahoe-lafs"; + + # string -> string + # + # Construct the unit test application name for the given Python runtime. + unitTestName = pyVersion: "${pyVersion}-unittest"; + + # Create a derivation that includes a Python runtime, Tahoe-LAFS, and + # all of its dependencies. + makeRuntimeEnv = pyVersion: { + ${packageName pyVersion} = makeRuntimeEnv' pyVersion; + }; + + makeRuntimeEnv' = pyVersion: (pkgs.${pyVersion}.withPackages (ps: with ps; + [ tahoe-lafs ] ++ + tahoe-lafs.passthru.extras.i2p ++ + tahoe-lafs.passthru.extras.tor + )).overrideAttrs (old: { + name = packageName pyVersion; + }); + + # Create a derivation that includes a Python runtime, Tahoe-LAFS, and + # all of its dependencies. + makeTestEnv = pyVersion: { + ${packageName pyVersion} = (pkgs.${pyVersion}.withPackages (ps: with ps; + [ tahoe-lafs ] ++ + tahoe-lafs.passthru.extras.i2p ++ + tahoe-lafs.passthru.extras.tor ++ + tahoe-lafs.passthru.extras.unittest + )).overrideAttrs (old: { + name = packageName pyVersion; + }); + }; + in { + # Define the flake's package outputs. We'll define one version of the + # package for each version of Python we could find. We'll also point + # the flake's "default" package at one of these somewhat arbitrarily. + # The package consists of a Python environment with Tahoe-LAFS available + # to it. + packages = with pkgs.lib; + foldr mergeAttrs {} ([ + { default = self.packages.${system}.${packageName defaultPyVersion}; } + ] ++ (builtins.map makeRuntimeEnv pythonVersions)); + + # Define the flake's app outputs. We'll define a version of an app for + # running the test suite for each version of Python we could find. + # We'll also define a version of an app for running the "tahoe" + # command-line entrypoint for each version of Python we could find. + apps = + let + # We avoid writeShellApplication here because it has ghc as a + # dependency but ghc has Python as a dependency and our Python + # package override triggers a rebuild of ghc which takes a looong + # time. + writeScript = name: text: + let script = pkgs.writeShellScript name text; + in "${script}"; + + # A helper function to define the runtime entrypoint for a certain + # Python runtime. + makeTahoeApp = pyVersion: { + "tahoe-${pyVersion}" = { + type = "app"; + program = + writeScript "tahoe" + '' + ${makeRuntimeEnv' pyVersion}/bin/tahoe "$@" + ''; + }; + }; + + # A helper function to define the unit test entrypoint for a certain + # Python runtime. + makeUnitTestsApp = pyVersion: { + "${unitTestName pyVersion}" = { + type = "app"; + program = + writeScript "unit-tests" + '' + export TAHOE_LAFS_HYPOTHESIS_PROFILE=ci + ${makeTestEnv pyVersion}/bin/python -m twisted.trial "$@" + ''; + }; + }; + in + with pkgs.lib; + foldr mergeAttrs + { default = self.apps.${system}."tahoe-python3"; } + ( + (builtins.map makeUnitTestsApp pythonVersions) ++ + (builtins.map makeTahoeApp pythonVersions) + ); + })); +} diff --git a/nix/overlay.nix b/nix/overlay.nix new file mode 100644 index 000000000..41f0e3086 --- /dev/null +++ b/nix/overlay.nix @@ -0,0 +1,10 @@ +# This overlay adds Tahoe-LAFS and all of its properly-configured Python +# package dependencies to a Python package set. Downstream consumers can +# apply it to their own nixpkgs derivation to produce a Tahoe-LAFS package. +final: prev: { + # Add our overrides such that they will be applied to any Python derivation + # in nixpkgs. + pythonPackagesExtensions = prev.pythonPackagesExtensions ++ [ + (import ./python-overrides.nix) + ]; +} diff --git a/nix/python-overrides.nix b/nix/python-overrides.nix index 0ed415691..a333808be 100644 --- a/nix/python-overrides.nix +++ b/nix/python-overrides.nix @@ -9,14 +9,39 @@ let # Disable a Python package's test suite. dontCheck = drv: drv.overrideAttrs (old: { doInstallCheck = false; }); + # string -> any -> derivation -> derivation + # + # If the overrideable function for the given derivation accepts an argument + # with the given name, override it with the given value. + # + # Since we try to work with multiple versions of nixpkgs, sometimes we need + # to override a parameter that exists in one version but not others. This + # makes it a bit easier to do so. + overrideIfPresent = name: value: drv: + if (drv.override.__functionArgs ? ${name}) + then drv.override { "${name}" = value; } + else drv; + # Disable building a Python package's documentation. - dontBuildDocs = alsoDisable: drv: (drv.override ({ - sphinxHook = null; - } // alsoDisable)).overrideAttrs ({ outputs, ... }: { + dontBuildDocs = drv: ( + overrideIfPresent "sphinxHook" null ( + overrideIfPresent "sphinx-rtd-theme" null + drv + ) + ).overrideAttrs ({ outputs, ... }: { outputs = builtins.filter (x: "doc" != x) outputs; }); in { + tahoe-lafs = self.callPackage ./tahoe-lafs.nix { + # Define the location of the Tahoe-LAFS source to be packaged (the same + # directory as contains this file). Clean up as many of the non-source + # files (eg the `.git` directory, `~` backup files, nix's own `result` + # symlink, etc) as possible to avoid needing to re-build when files that + # make no difference to the package have changed. + tahoe-lafs-src = self.lib.cleanSource ../.; + }; + # Some dependencies aren't packaged in nixpkgs so supply our own packages. pycddl = self.callPackage ./pycddl.nix { }; txi2p = self.callPackage ./txi2p.nix { }; @@ -33,11 +58,10 @@ in { # Update the version of pyopenssl. pyopenssl = self.callPackage ./pyopenssl.nix { pyopenssl = - # Building the docs requires sphinx which brings in a dependency on babel, - # the test suite of which fails. - onPyPy (dontBuildDocs { sphinx-rtd-theme = null; }) - # Avoid infinite recursion. - super.pyopenssl; + # Building the docs requires sphinx which brings in a dependency on + # babel, the test suite of which fails. Avoid infinite recursion here + # by taking pyopenssl from the `super` package set. + onPyPy dontBuildDocs super.pyopenssl; }; # collections-extended is currently broken for Python 3.11 in nixpkgs but @@ -74,7 +98,7 @@ in { six = onPyPy dontCheck super.six; # Likewise for beautifulsoup4. - beautifulsoup4 = onPyPy (dontBuildDocs {}) super.beautifulsoup4; + beautifulsoup4 = onPyPy dontBuildDocs super.beautifulsoup4; # The autobahn test suite pulls in a vast number of dependencies for # functionality we don't care about. It might be nice to *selectively* diff --git a/nix/tahoe-lafs.nix b/nix/tahoe-lafs.nix index bf3ea83d3..f7037e1ae 100644 --- a/nix/tahoe-lafs.nix +++ b/nix/tahoe-lafs.nix @@ -1,24 +1,16 @@ +let + pname = "tahoe-lafs"; + version = "1.18.0.post1"; +in { lib , pythonPackages , buildPythonPackage , tahoe-lafs-src -, extrasNames - -# control how the test suite is run -, doCheck }: -let - pname = "tahoe-lafs"; - version = "1.18.0.post1"; - - pickExtraDependencies = deps: extras: builtins.foldl' (accum: extra: accum ++ deps.${extra}) [] extras; - - pythonExtraDependencies = with pythonPackages; { - tor = [ txtorcon ]; - i2p = [ txi2p ]; - }; - - pythonPackageDependencies = with pythonPackages; [ +buildPythonPackage rec { + inherit pname version; + src = tahoe-lafs-src; + propagatedBuildInputs = with pythonPackages; [ attrs autobahn cbor2 @@ -41,35 +33,37 @@ let six treq twisted - # Get the dependencies for the Twisted extras we depend on, too. - twisted.passthru.optional-dependencies.tls - twisted.passthru.optional-dependencies.conch werkzeug zfec zope_interface - ] ++ pickExtraDependencies pythonExtraDependencies extrasNames; + ] ++ + # Get the dependencies for the Twisted extras we depend on, too. + twisted.passthru.optional-dependencies.tls ++ + twisted.passthru.optional-dependencies.conch; - unitTestDependencies = with pythonPackages; [ - beautifulsoup4 - fixtures - hypothesis - mock - prometheus-client - testtools - ]; + # The test suite lives elsewhere. + doCheck = false; -in -buildPythonPackage { - inherit pname version; - src = tahoe-lafs-src; - propagatedBuildInputs = pythonPackageDependencies; - - inherit doCheck; - checkInputs = unitTestDependencies; - checkPhase = '' - export TAHOE_LAFS_HYPOTHESIS_PROFILE=ci - python -m twisted.trial -j $NIX_BUILD_CORES allmydata - ''; + passthru = { + extras = with pythonPackages; { + tor = [ txtorcon ]; + i2p = [ txi2p ]; + unittest = [ + beautifulsoup4 + fixtures + hypothesis + mock + prometheus-client + testtools + ]; + integrationtest = [ + pytest + pytest-twisted + paramiko + pytest-timeout + ]; + }; + }; meta = with lib; { homepage = "https://tahoe-lafs.org/"; diff --git a/nix/tests.nix b/nix/tests.nix deleted file mode 100644 index 42ca9f882..000000000 --- a/nix/tests.nix +++ /dev/null @@ -1,4 +0,0 @@ -# Build the package with the test suite enabled. -args@{...}: (import ../. args).override { - doCheck = true; -} From 452ba6af32de98046bcb964a692b2fd59e9a363f Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 4 Jul 2023 12:48:45 -0400 Subject: [PATCH 1894/2309] try to get ci to use the flake --- .circleci/config.yml | 47 ++++++++++++++++++++++---------------------- .circleci/lib.sh | 29 +++++++++++++++++++++++++++ 2 files changed, 52 insertions(+), 24 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 54b2706cd..59864675b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -403,14 +403,17 @@ jobs: - "run": name: "Unit Test" command: | - # The dependencies are all built so we can allow more - # parallelism here. source .circleci/lib.sh - cache_if_able nix-build \ - --cores 8 \ - --argstr pkgsVersion "nixpkgs-<>" \ - --argstr pythonVersion "<>" \ - nix/tests.nix + + # Translate the nixpkgs selection into a flake reference we + # can use to override the default nixpkgs input. + NIXPKGS=$(nixpkgs_flake_reference nixpkgs-<>) + + cache_if_able nix run \ + --override-input nixpkgs "$NIXPKGS" \ + .#<>-unittest -- \ + --jobs $UNITTEST_CORES \ + allmydata typechecks: docker: @@ -588,29 +591,25 @@ commands: - "run": name: "Build Dependencies" command: | - # CircleCI build environment looks like it has a zillion and a - # half cores. Don't let Nix autodetect this high core count - # because it blows up memory usage and fails the test run. Pick a - # number of cores that suits the build environment we're paying - # for (the free one!). source .circleci/lib.sh - # nix-shell will build all of the dependencies of the target but + + NIXPKGS=$(nixpkgs_flake_reference nixpkgs-<>) + + # `nix develop` will build all of the dependencies of the target but # not the target itself. - cache_if_able nix-shell \ - --run "" \ - --cores 3 \ - --argstr pkgsVersion "nixpkgs-<>" \ - --argstr pythonVersion "<>" \ - ./default.nix + cache_if_able nix develop \ + --command "true" \ + --cores $DEPENDENCY_CORES \ + --override-input nixpkgs "$NIXPKGS" \ + .#<>-tahoe-lafs - "run": name: "Build Package" command: | source .circleci/lib.sh - cache_if_able nix-build \ - --cores 4 \ - --argstr pkgsVersion "nixpkgs-<>" \ - --argstr pythonVersion "<>" \ - ./default.nix + NIXPKGS=$(nixpkgs_flake_reference nixpkgs-<>) + cache_if_able nix build \ + --override-input nixpkgs "$NIXPKGS" \ + .#<>-tahoe-lafs - steps: "<>" diff --git a/.circleci/lib.sh b/.circleci/lib.sh index c692b5f88..a53c33dce 100644 --- a/.circleci/lib.sh +++ b/.circleci/lib.sh @@ -1,3 +1,13 @@ +# CircleCI build environment looks like it has a zillion and a half cores. +# Don't let Nix autodetect this high core count because it blows up memory +# usage and fails the test run. Pick a number of cores that suits the build +# environment we're paying for (the free one!). +DEPENDENCY_CORES=3 + +# Once dependencies are built, we can allow some more concurrency for our own +# test suite. +UNITTEST_CORES=8 + # Run a command, enabling cache writes to cachix if possible. The command is # accepted as a variable number of positional arguments (like argv). function cache_if_able() { @@ -117,3 +127,22 @@ function describe_build() { echo "Cache not writeable." fi } + +# Inspect the flake input metadata for an input of a given name and return the +# revision at which that input is pinned. If the input does not exist then +# return garbage (probably "null"). +read_input_revision() { + input_name=$1 + shift + + nix flake metadata --json | jp --unquoted 'locks.nodes."'"$input_name"'".locked.rev' +} + +# Return a flake reference that refers to a certain revision of nixpkgs. The +# certain revision is the revision to which the specified input is pinned. +nixpkgs_flake_reference() { + input_name=$1 + shift + + echo "github:NixOS/nixpkgs?rev=$(read_input_revision $input_name)" +} From 2ddb0970a36415cb4b48ba01dfb74bff9b559d71 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 4 Jul 2023 12:50:48 -0400 Subject: [PATCH 1895/2309] access to "experimental" nix commands --- .circleci/config.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 59864675b..03af0f8a1 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -545,6 +545,9 @@ executors: # to push to CACHIX_NAME. CACHIX_NAME tells cachix which cache to push # to. CACHIX_NAME: "tahoe-lafs-opensource" + # Let us use features marked "experimental". For example, most/all of + # the `nix ` forms. + NIX_CONFIG: "experimental-features = nix-command flakes" commands: nix-build: From 95ee3994a944ae959ba8c89bf82a3a3b160a4ccb Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 4 Jul 2023 13:08:00 -0400 Subject: [PATCH 1896/2309] get a jmespath interpreter --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 03af0f8a1..f86bdfeee 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -575,7 +575,7 @@ commands: nix-env \ --file $NIXPKGS \ --install \ - -A cachix bash + -A cachix bash jp # Activate it for "binary substitution". This sets up # configuration tht lets Nix download something from the cache # instead of building it locally, if possible. From 3d62b9d0501586f3b4d53ce69552cc97ee74b744 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 4 Jul 2023 13:12:28 -0400 Subject: [PATCH 1897/2309] drop the separate build-dependencies step Since we don't run the unit tests as part of the package build now we don't need it --- .circleci/config.yml | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index f86bdfeee..4426b0827 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -591,21 +591,6 @@ commands: -p 'python3.withPackages (ps: [ ps.setuptools ])' \ --run 'python setup.py update_version' - - "run": - name: "Build Dependencies" - command: | - source .circleci/lib.sh - - NIXPKGS=$(nixpkgs_flake_reference nixpkgs-<>) - - # `nix develop` will build all of the dependencies of the target but - # not the target itself. - cache_if_able nix develop \ - --command "true" \ - --cores $DEPENDENCY_CORES \ - --override-input nixpkgs "$NIXPKGS" \ - .#<>-tahoe-lafs - - "run": name: "Build Package" command: | From 84eb1a3e3242fa401dd280c8b368ad31ed2e2f10 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 4 Jul 2023 13:13:16 -0400 Subject: [PATCH 1898/2309] still limit cores during build though --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 4426b0827..e503679ed 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -597,6 +597,7 @@ commands: source .circleci/lib.sh NIXPKGS=$(nixpkgs_flake_reference nixpkgs-<>) cache_if_able nix build \ + --cores "$DEPENDENCY_CORES" \ --override-input nixpkgs "$NIXPKGS" \ .#<>-tahoe-lafs From 4646c5ea38626896cb1cf1a984000145f30e0691 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 4 Jul 2023 13:22:37 -0400 Subject: [PATCH 1899/2309] identify the nixpkgs input correctly --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index e503679ed..f3965d242 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -89,7 +89,7 @@ workflows: - "nixos": name: "<>" - nixpkgs: "22.11" + nixpkgs: "22_11" matrix: parameters: pythonVersion: From f5dd14e0867d3dddbc521c03831853231645addb Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 4 Jul 2023 13:24:33 -0400 Subject: [PATCH 1900/2309] use a constant nixpkgs for build environment setup --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index f3965d242..f2d1e7a04 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -571,7 +571,7 @@ commands: # Get cachix for Nix-friendly caching. name: "Install Basic Dependencies" command: | - NIXPKGS="https://github.com/nixos/nixpkgs/archive/nixos-<>.tar.gz" + NIXPKGS="https://github.com/nixos/nixpkgs/archive/nixos-23.05.tar.gz" nix-env \ --file $NIXPKGS \ --install \ From 35254adebf9723d1936296137baf53266a45acc4 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 4 Jul 2023 13:28:24 -0400 Subject: [PATCH 1901/2309] try to make noise to avoid ci timeouts --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index f2d1e7a04..1c3c52d17 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -597,6 +597,7 @@ commands: source .circleci/lib.sh NIXPKGS=$(nixpkgs_flake_reference nixpkgs-<>) cache_if_able nix build \ + --verbose \ --cores "$DEPENDENCY_CORES" \ --override-input nixpkgs "$NIXPKGS" \ .#<>-tahoe-lafs From 4b932b86991b2cdefdb956f9ad782318bb067314 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 4 Jul 2023 13:43:18 -0400 Subject: [PATCH 1902/2309] turn off a lot of autobahn dependencies we don't need --- nix/python-overrides.nix | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/nix/python-overrides.nix b/nix/python-overrides.nix index a333808be..c6e25fd80 100644 --- a/nix/python-overrides.nix +++ b/nix/python-overrides.nix @@ -103,7 +103,22 @@ in { # The autobahn test suite pulls in a vast number of dependencies for # functionality we don't care about. It might be nice to *selectively* # disable just some of it but this is easier. - autobahn = onPyPy dontCheck super.autobahn; + autobahn = onPyPy dontCheck (super.autobahn.override { + base58 = null; + click = null; + ecdsa = null; + eth-abi = null; + jinja2 = null; + hkdf = null; + mnemonic = null; + py-ecc = null; + py-eth-sig-utils = null; + py-multihash = null; + rlp = null; + spake2 = null; + yapf = null; + eth-account = null; + }); # and python-dotenv tests pulls in a lot of dependencies, including jedi, # which does not work on PyPy. From d455e3c88fd32218a48e7ee13fc0c669adbd2ca7 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 4 Jul 2023 13:45:01 -0400 Subject: [PATCH 1903/2309] get more output from the build step --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 1c3c52d17..01cd999ca 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -598,6 +598,7 @@ commands: NIXPKGS=$(nixpkgs_flake_reference nixpkgs-<>) cache_if_able nix build \ --verbose \ + --print-build-logs \ --cores "$DEPENDENCY_CORES" \ --override-input nixpkgs "$NIXPKGS" \ .#<>-tahoe-lafs From 8a53175655faf7f6756fe958a9e23f8b16daa455 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 4 Jul 2023 13:50:33 -0400 Subject: [PATCH 1904/2309] compatibility with newer nixpkgs that includes the fix --- nix/python-overrides.nix | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/nix/python-overrides.nix b/nix/python-overrides.nix index c6e25fd80..9cdf63306 100644 --- a/nix/python-overrides.nix +++ b/nix/python-overrides.nix @@ -149,11 +149,10 @@ in { # This also drops a bunch of unnecessary build-time dependencies, some of # which are broken on PyPy. Fixed in nixpkgs in # 5feb5054bb08ba779bd2560a44cf7d18ddf37fea. - zfec = (super.zfec.override { - setuptoolsTrial = null; - }).overrideAttrs (old: { - checkPhase = "trial zfec"; - }); + zfec = (overrideIfPresent "setuptoolsTrial" null super.zfec).overrideAttrs ( + old: { + checkPhase = "trial zfec"; + }); # collections-extended is packaged with poetry-core. poetry-core test suite # uses virtualenv and virtualenv test suite fails on PyPy. From bf1db852197d22d307777a0005104d987ca92a48 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 4 Jul 2023 14:20:50 -0400 Subject: [PATCH 1905/2309] slightly regularize nixpkgs handling --- .circleci/config.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 01cd999ca..21d6a6114 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -89,7 +89,7 @@ workflows: - "nixos": name: "<>" - nixpkgs: "22_11" + nixpkgs: "nixpkgs-22_11" matrix: parameters: pythonVersion: @@ -99,7 +99,7 @@ workflows: - "nixos": name: "<>" - nixpkgs: "unstable" + nixpkgs: "nixpkgs-unstable" matrix: parameters: pythonVersion: @@ -385,8 +385,8 @@ jobs: parameters: nixpkgs: description: >- - Reference the name of a niv-managed nixpkgs source (see `niv show` - and nix/sources.json) + Reference the name of a flake-managed nixpkgs input (see `nix flake + metadata` and flake.nix) type: "string" pythonVersion: description: >- @@ -407,7 +407,7 @@ jobs: # Translate the nixpkgs selection into a flake reference we # can use to override the default nixpkgs input. - NIXPKGS=$(nixpkgs_flake_reference nixpkgs-<>) + NIXPKGS=$(nixpkgs_flake_reference <>) cache_if_able nix run \ --override-input nixpkgs "$NIXPKGS" \ @@ -595,7 +595,7 @@ commands: name: "Build Package" command: | source .circleci/lib.sh - NIXPKGS=$(nixpkgs_flake_reference nixpkgs-<>) + NIXPKGS=$(nixpkgs_flake_reference <>) cache_if_able nix build \ --verbose \ --print-build-logs \ From 3871a92890f68bf4e4fbc0a208b143e8093347d9 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 4 Jul 2023 14:36:27 -0400 Subject: [PATCH 1906/2309] fix another outdated comment --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 21d6a6114..d0942d504 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -554,8 +554,8 @@ commands: parameters: nixpkgs: description: >- - Reference the name of a niv-managed nixpkgs source (see `niv show` - and nix/sources.json) + Reference the name of a flake-managed nixpkgs input (see `nix flake + metadata` and flake.nix) type: "string" pythonVersion: description: >- From 9788e4c12f8f10f5ce7376e5db47b1901f073e70 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 4 Jul 2023 14:36:46 -0400 Subject: [PATCH 1907/2309] fix the test app definition --- flake.nix | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/flake.nix b/flake.nix index 89a4a6f14..e8c501bd1 100644 --- a/flake.nix +++ b/flake.nix @@ -90,15 +90,17 @@ # Create a derivation that includes a Python runtime, Tahoe-LAFS, and # all of its dependencies. makeTestEnv = pyVersion: { - ${packageName pyVersion} = (pkgs.${pyVersion}.withPackages (ps: with ps; - [ tahoe-lafs ] ++ - tahoe-lafs.passthru.extras.i2p ++ - tahoe-lafs.passthru.extras.tor ++ - tahoe-lafs.passthru.extras.unittest - )).overrideAttrs (old: { - name = packageName pyVersion; - }); + ${packageName pyVersion} = makeTestEnv' pyVersion; }; + + makeTestEnv' = pyVersion: (pkgs.${pyVersion}.withPackages (ps: with ps; + [ tahoe-lafs ] ++ + tahoe-lafs.passthru.extras.i2p ++ + tahoe-lafs.passthru.extras.tor ++ + tahoe-lafs.passthru.extras.unittest + )).overrideAttrs (old: { + name = packageName pyVersion; + }); in { # Define the flake's package outputs. We'll define one version of the # package for each version of Python we could find. We'll also point @@ -146,7 +148,7 @@ writeScript "unit-tests" '' export TAHOE_LAFS_HYPOTHESIS_PROFILE=ci - ${makeTestEnv pyVersion}/bin/python -m twisted.trial "$@" + ${makeTestEnv' pyVersion}/bin/python -m twisted.trial "$@" ''; }; }; From 72539ddfc7adf22bb3c5df256718a1a07277e3fa Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 4 Jul 2023 14:39:26 -0400 Subject: [PATCH 1908/2309] refactor the env builders a bit --- flake.nix | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/flake.nix b/flake.nix index e8c501bd1..8a29786c9 100644 --- a/flake.nix +++ b/flake.nix @@ -73,12 +73,14 @@ # Construct the unit test application name for the given Python runtime. unitTestName = pyVersion: "${pyVersion}-unittest"; + # (string -> a) -> (string -> b) -> string -> attrset a b + # + # Make a singleton attribute set from the result of two functions. + singletonOf = f: g: x: { ${f x} = g x; }; + # Create a derivation that includes a Python runtime, Tahoe-LAFS, and # all of its dependencies. - makeRuntimeEnv = pyVersion: { - ${packageName pyVersion} = makeRuntimeEnv' pyVersion; - }; - + makeRuntimeEnv = singletonOf packageName makeRuntimeEnv'; makeRuntimeEnv' = pyVersion: (pkgs.${pyVersion}.withPackages (ps: with ps; [ tahoe-lafs ] ++ tahoe-lafs.passthru.extras.i2p ++ @@ -89,11 +91,7 @@ # Create a derivation that includes a Python runtime, Tahoe-LAFS, and # all of its dependencies. - makeTestEnv = pyVersion: { - ${packageName pyVersion} = makeTestEnv' pyVersion; - }; - - makeTestEnv' = pyVersion: (pkgs.${pyVersion}.withPackages (ps: with ps; + makeTestEnv = pyVersion: (pkgs.${pyVersion}.withPackages (ps: with ps; [ tahoe-lafs ] ++ tahoe-lafs.passthru.extras.i2p ++ tahoe-lafs.passthru.extras.tor ++ @@ -148,7 +146,7 @@ writeScript "unit-tests" '' export TAHOE_LAFS_HYPOTHESIS_PROFILE=ci - ${makeTestEnv' pyVersion}/bin/python -m twisted.trial "$@" + ${makeTestEnv pyVersion}/bin/python -m twisted.trial "$@" ''; }; }; From ea50bb1c991097373790041a2525e5d8c47c63db Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 5 Jul 2023 10:19:25 -0400 Subject: [PATCH 1909/2309] News file. --- newsfragments/4038.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/4038.minor diff --git a/newsfragments/4038.minor b/newsfragments/4038.minor new file mode 100644 index 000000000..e69de29bb From bc78dbc25c487ff158ba74bcbea4f46f432b48ea Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 5 Jul 2023 10:21:40 -0400 Subject: [PATCH 1910/2309] Point to correct ticket --- src/allmydata/protocol_switch.py | 8 ++++---- src/allmydata/storage/http_client.py | 2 ++ 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/allmydata/protocol_switch.py b/src/allmydata/protocol_switch.py index 6a6bf8061..3b3268b79 100644 --- a/src/allmydata/protocol_switch.py +++ b/src/allmydata/protocol_switch.py @@ -112,6 +112,9 @@ class _FoolscapOrHttps(Protocol, metaclass=_PretendToBeNegotiation): # If we're listening on Tor, the hostname needs to have an # .onion TLD. assert hostname.endswith(".onion") + # The I2P scheme is yet not supported by the HTTP client, so we + # don't want generate a NURL that won't work. This will be + # fixed in https://tahoe-lafs.org/trac/tahoe-lafs/ticket/4037 port = int(port) storage_nurls.add( build_nurl( @@ -122,10 +125,7 @@ class _FoolscapOrHttps(Protocol, metaclass=_PretendToBeNegotiation): subscheme ) ) - # TODO this is where we'll have to support Tor and I2P as well. - # See https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3888#comment:9 - # for discussion (there will be separate tickets added for those at - # some point.) + return storage_nurls def __init__(self, *args, **kwargs): diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 9f5d6cce2..765e94319 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -375,6 +375,8 @@ class StorageClientFactory: pool=pool, tls_context_factory=tls_context_factory ) else: + # I2P support will be added here. See + # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/4037 raise RuntimeError(f"Unsupported tcp connection handler: {handler}") async def create_storage_client( From b07f279483741a5496cf5aa4c0a8d0dc8df4cc6a Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 5 Jul 2023 11:20:40 -0400 Subject: [PATCH 1911/2309] Also run Foolscap-only integration tests. --- .github/workflows/ci.yml | 18 ++++++++++-------- newsfragments/4040.minor | 0 2 files changed, 10 insertions(+), 8 deletions(-) create mode 100644 newsfragments/4040.minor diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1061657b9..d3862ffad 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -164,18 +164,20 @@ jobs: strategy: fail-fast: false matrix: - include: - - os: macos-12 - python-version: "3.11" - force-foolscap: false - - os: windows-latest - python-version: "3.11" - force-foolscap: false + os: # 22.04 has some issue with Tor at the moment: # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3943 + - ubuntu-20.04 + - macos-12 + - windows-latest + python-version: + - "3.11" + force-foolscap: + - false + include: - os: ubuntu-20.04 python-version: "3.10" - force-foolscap: false + force-foolscap: true steps: - name: Install Tor [Ubuntu] diff --git a/newsfragments/4040.minor b/newsfragments/4040.minor new file mode 100644 index 000000000..e69de29bb From 94e608f1363763e03df720e39828c9d281f05981 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 7 Jul 2023 10:02:52 -0400 Subject: [PATCH 1912/2309] more python package tweaks also point nixpkgs-unstable at HEAD of a PR with a cryptography upgrade I tried just overriding the upgrade into place but it results in infinite recursion, I suppose because cryptography is a dependency of some of the build tools and needs extra handling that I don't feel like figuring out for this short-term hack. someday the upgrade will land in nixpkgs master and we can switch back. --- flake.lock | 26 ++++++++++-- flake.nix | 11 +++++- nix/python-overrides.nix | 85 ++++++++++++---------------------------- nix/service-identity.nix | 61 ++++++++++++++++++++++++++++ nix/tahoe-lafs.nix | 8 +++- nix/twisted.patch | 12 ++++++ 6 files changed, 135 insertions(+), 68 deletions(-) create mode 100644 nix/service-identity.nix create mode 100644 nix/twisted.patch diff --git a/flake.lock b/flake.lock index 0fd3c90c9..165ff7230 100644 --- a/flake.lock +++ b/flake.lock @@ -34,17 +34,34 @@ "type": "github" } }, - "nixpkgs-unstable": { + "nixpkgs-23_05": { "locked": { - "lastModified": 1688486599, - "narHash": "sha256-K8v2wCfHjA0LS6QeCZ/x+OU2hhINZG4qAAO6zvvqYhE=", + "lastModified": 1688722168, + "narHash": "sha256-UDqeQd2neUXICpHAZSS965kGCJsfHkrOFS/vl80I7d8=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "712caf8eb1c2ea0944d2f34f96570bca7193c1c8", + "rev": "28d812a63a0f0d6c1170aac16f5219e506c44b79", "type": "github" }, "original": { "owner": "NixOS", + "ref": "release-23.05", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs-unstable": { + "locked": { + "lastModified": 1687349636, + "narHash": "sha256-wpWWNoKJ6z8Nt9egpeiKzsCgkkDO2SO4g6ab9SxgvpM=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "49b7c90c06e557e7473ef467f40d98e7c368d29f", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "pull/238965/head", "repo": "nixpkgs", "type": "github" } @@ -56,6 +73,7 @@ "nixpkgs-22_11" ], "nixpkgs-22_11": "nixpkgs-22_11", + "nixpkgs-23_05": "nixpkgs-23_05", "nixpkgs-unstable": "nixpkgs-unstable" } }, diff --git a/flake.nix b/flake.nix index 8a29786c9..83ee24bec 100644 --- a/flake.nix +++ b/flake.nix @@ -9,8 +9,11 @@ "nixpkgs-22_11" = { url = github:NixOS/nixpkgs?ref=nixos-22.11; }; + "nixpkgs-23_05" = { + url = github:NixOS/nixpkgs?ref=release-23.05; + }; "nixpkgs-unstable" = { - url = github:NixOS/nixpkgs; + url = github:NixOS/nixpkgs?ref=pull/238965/head; }; # Point the default nixpkgs at one of those. This avoids having getting a @@ -100,6 +103,8 @@ name = packageName pyVersion; }); in { + legacyPackages = pkgs; + # Define the flake's package outputs. We'll define one version of the # package for each version of Python we could find. We'll also point # the flake's "default" package at one of these somewhat arbitrarily. @@ -108,7 +113,9 @@ packages = with pkgs.lib; foldr mergeAttrs {} ([ { default = self.packages.${system}.${packageName defaultPyVersion}; } - ] ++ (builtins.map makeRuntimeEnv pythonVersions)); + ] ++ (builtins.map makeRuntimeEnv pythonVersions) + ++ (builtins.map (singletonOf unitTestName makeTestEnv) pythonVersions) + ); # Define the flake's app outputs. We'll define a version of an app for # running the test suite for each version of Python we could find. diff --git a/nix/python-overrides.nix b/nix/python-overrides.nix index 9cdf63306..de0ae028c 100644 --- a/nix/python-overrides.nix +++ b/nix/python-overrides.nix @@ -55,14 +55,22 @@ in { inherit (super) txtorcon; }; - # Update the version of pyopenssl. - pyopenssl = self.callPackage ./pyopenssl.nix { - pyopenssl = - # Building the docs requires sphinx which brings in a dependency on - # babel, the test suite of which fails. Avoid infinite recursion here - # by taking pyopenssl from the `super` package set. - onPyPy dontBuildDocs super.pyopenssl; - }; + # With our customized package set a Twisted unit test fails. Patch the + # Twisted test suite to skip that test. + twisted = super.twisted.overrideAttrs (old: { + patches = (old.patches or []) ++ [ ./twisted.patch ]; + }); + + # Update the version of pyopenssl - and since we're doing that anyway, we + # don't need the docs. Unfortunately this triggers a lot of rebuilding of + # dependent packages. + pyopenssl = dontBuildDocs (self.callPackage ./pyopenssl.nix { + inherit (super) pyopenssl; + }); + + # The cryptography that we get from nixpkgs to satisfy the pyopenssl upgrade + # that we did breaks service-identity ... so get a newer version that works. + service-identity = self.callPackage ./service-identity.nix { }; # collections-extended is currently broken for Python 3.11 in nixpkgs but # we know where a working version lives. @@ -76,16 +84,19 @@ in { # tornado and tk pull in a huge dependency trees for functionality we don't # care about, also tkinter doesn't work on PyPy. - matplotlib = super.matplotlib.override { tornado = null; enableTk = false; }; + matplotlib = onPyPy (matplotlib: matplotlib.override { + tornado = null; + enableTk = false; + }) super.matplotlib; - tqdm = super.tqdm.override { + tqdm = onPyPy (tqdm: tqdm.override { # ibid. tkinter = null; # pandas is only required by the part of the test suite covering # integration with pandas that we don't care about. pandas is a huge # dependency. pandas = null; - }; + }) super.tqdm; # The treq test suite depends on httpbin. httpbin pulls in babel (flask -> # jinja2 -> babel) and arrow (brotlipy -> construct -> arrow). babel fails @@ -103,57 +114,20 @@ in { # The autobahn test suite pulls in a vast number of dependencies for # functionality we don't care about. It might be nice to *selectively* # disable just some of it but this is easier. - autobahn = onPyPy dontCheck (super.autobahn.override { - base58 = null; - click = null; - ecdsa = null; - eth-abi = null; - jinja2 = null; - hkdf = null; - mnemonic = null; - py-ecc = null; - py-eth-sig-utils = null; - py-multihash = null; - rlp = null; - spake2 = null; - yapf = null; - eth-account = null; - }); + autobahn = dontCheck super.autobahn; # and python-dotenv tests pulls in a lot of dependencies, including jedi, # which does not work on PyPy. python-dotenv = onPyPy dontCheck super.python-dotenv; - # Upstream package unaccountably includes a sqlalchemy dependency ... but - # the project has no such dependency. Fixed in nixpkgs in - # da10e809fff70fbe1d86303b133b779f09f56503. - aiocontextvars = super.aiocontextvars.override { sqlalchemy = null; }; - # By default, the sphinx docs are built, which pulls in a lot of # dependencies - including jedi, which does not work on PyPy. - hypothesis = - (let h = super.hypothesis; - in - if (h.override.__functionArgs.enableDocumentation or false) - then h.override { enableDocumentation = false; } - else h).overrideAttrs ({ nativeBuildInputs, ... }: { - # The nixpkgs expression is missing the tzdata check input. - nativeBuildInputs = nativeBuildInputs ++ [ super.tzdata ]; - }); + hypothesis = onPyPy dontBuildDocs super.hypothesis; # flaky's test suite depends on nose and nose appears to have Python 3 # incompatibilities (it includes `print` statements, for example). flaky = onPyPy dontCheck super.flaky; - # Replace the deprecated way of running the test suite with the modern way. - # This also drops a bunch of unnecessary build-time dependencies, some of - # which are broken on PyPy. Fixed in nixpkgs in - # 5feb5054bb08ba779bd2560a44cf7d18ddf37fea. - zfec = (overrideIfPresent "setuptoolsTrial" null super.zfec).overrideAttrs ( - old: { - checkPhase = "trial zfec"; - }); - # collections-extended is packaged with poetry-core. poetry-core test suite # uses virtualenv and virtualenv test suite fails on PyPy. poetry-core = onPyPy dontCheck super.poetry-core; @@ -172,15 +146,6 @@ in { # since we actually depend directly and significantly on Foolscap. foolscap = onPyPy dontCheck super.foolscap; - # Fixed by nixpkgs PR https://github.com/NixOS/nixpkgs/pull/222246 - psutil = super.psutil.overrideAttrs ({ pytestFlagsArray, disabledTests, ...}: { - # Upstream already disables some tests but there are even more that have - # build impurities that come from build system hardware configuration. - # Skip them too. - pytestFlagsArray = [ "-v" ] ++ pytestFlagsArray; - disabledTests = disabledTests ++ [ "sensors_temperatures" ]; - }); - # CircleCI build systems don't have enough memory to run this test suite. - lz4 = dontCheck super.lz4; + lz4 = onPyPy dontCheck super.lz4; } diff --git a/nix/service-identity.nix b/nix/service-identity.nix new file mode 100644 index 000000000..fef68b16e --- /dev/null +++ b/nix/service-identity.nix @@ -0,0 +1,61 @@ +{ lib +, attrs +, buildPythonPackage +, cryptography +, fetchFromGitHub +, hatch-fancy-pypi-readme +, hatch-vcs +, hatchling +, idna +, pyasn1 +, pyasn1-modules +, pytestCheckHook +, pythonOlder +, setuptools +}: + +buildPythonPackage rec { + pname = "service-identity"; + version = "23.1.0"; + format = "pyproject"; + + disabled = pythonOlder "3.8"; + + src = fetchFromGitHub { + owner = "pyca"; + repo = pname; + rev = "refs/tags/${version}"; + hash = "sha256-PGDtsDgRwh7GuuM4OuExiy8L4i3Foo+OD0wMrndPkvo="; + }; + + nativeBuildInputs = [ + hatch-fancy-pypi-readme + hatch-vcs + hatchling + setuptools + ]; + + propagatedBuildInputs = [ + attrs + cryptography + idna + pyasn1 + pyasn1-modules + ]; + + nativeCheckInputs = [ + pytestCheckHook + ]; + + pythonImportsCheck = [ + "service_identity" + ]; + + meta = with lib; { + description = "Service identity verification for pyOpenSSL"; + homepage = "https://service-identity.readthedocs.io"; + changelog = "https://github.com/pyca/service-identity/releases/tag/${version}"; + license = licenses.mit; + maintainers = with maintainers; [ fab ]; + }; +} diff --git a/nix/tahoe-lafs.nix b/nix/tahoe-lafs.nix index f7037e1ae..5d86de4b2 100644 --- a/nix/tahoe-lafs.nix +++ b/nix/tahoe-lafs.nix @@ -46,8 +46,12 @@ buildPythonPackage rec { passthru = { extras = with pythonPackages; { - tor = [ txtorcon ]; - i2p = [ txi2p ]; + tor = [ + txtorcon + ]; + i2p = [ + txi2p + ]; unittest = [ beautifulsoup4 fixtures diff --git a/nix/twisted.patch b/nix/twisted.patch new file mode 100644 index 000000000..1b6846c8e --- /dev/null +++ b/nix/twisted.patch @@ -0,0 +1,12 @@ +diff --git a/src/twisted/internet/test/test_endpoints.py b/src/twisted/internet/test/test_endpoints.py +index c650fd8aa6..a1754fd533 100644 +--- a/src/twisted/internet/test/test_endpoints.py ++++ b/src/twisted/internet/test/test_endpoints.py +@@ -4214,6 +4214,7 @@ class WrapClientTLSParserTests(unittest.TestCase): + connectionCreator = connectionCreatorFromEndpoint(reactor, endpoint) + self.assertEqual(connectionCreator._hostname, "\xe9xample.example.com") + ++ @skipIf(True, "self.assertFalse(plainClient.transport.disconnecting) fails") + def test_tls(self): + """ + When passed a string endpoint description beginning with C{tls:}, From ddf4777153262a0fe4ccc27b5e992c744572379a Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 7 Jul 2023 10:37:05 -0400 Subject: [PATCH 1913/2309] link to the ticket --- nix/python-overrides.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/nix/python-overrides.nix b/nix/python-overrides.nix index de0ae028c..74a74b278 100644 --- a/nix/python-overrides.nix +++ b/nix/python-overrides.nix @@ -57,6 +57,7 @@ in { # With our customized package set a Twisted unit test fails. Patch the # Twisted test suite to skip that test. + # Filed upstream at https://github.com/twisted/twisted/issues/11892 twisted = super.twisted.overrideAttrs (old: { patches = (old.patches or []) ++ [ ./twisted.patch ]; }); From 2271ca4698a297c9b2edf4a3f3b1e54b8e3feb0f Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 7 Jul 2023 11:47:04 -0400 Subject: [PATCH 1914/2309] add missing test dependency --- nix/tahoe-lafs.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/nix/tahoe-lafs.nix b/nix/tahoe-lafs.nix index 5d86de4b2..dbc4f37f9 100644 --- a/nix/tahoe-lafs.nix +++ b/nix/tahoe-lafs.nix @@ -54,6 +54,7 @@ buildPythonPackage rec { ]; unittest = [ beautifulsoup4 + html5lib fixtures hypothesis mock From f7dd63407f3c58e64ea47b394414d224fb22d65d Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 7 Jul 2023 12:17:39 -0400 Subject: [PATCH 1915/2309] move all of the Python jobs to the nixpkgs unstable branch This is the only place we can get a compatible pyOpenSSL/cryptography at the moment --- .circleci/config.yml | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index d0942d504..e588cfa88 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -89,20 +89,13 @@ workflows: - "nixos": name: "<>" - nixpkgs: "nixpkgs-22_11" + nixpkgs: "nixpkgs-unstable" matrix: parameters: pythonVersion: - "python38" - "python39" - "python310" - - - "nixos": - name: "<>" - nixpkgs: "nixpkgs-unstable" - matrix: - parameters: - pythonVersion: - "python311" # Eventually, test against PyPy 3.8 From 0eb160f42cb7b433f3a2ea17006d6f93414249a5 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 7 Jul 2023 14:49:36 -0400 Subject: [PATCH 1916/2309] switch to the working version of nixpkgs by default --- flake.lock | 2 +- flake.nix | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/flake.lock b/flake.lock index 165ff7230..17270a7d0 100644 --- a/flake.lock +++ b/flake.lock @@ -70,7 +70,7 @@ "inputs": { "flake-utils": "flake-utils", "nixpkgs": [ - "nixpkgs-22_11" + "nixpkgs-unstable" ], "nixpkgs-22_11": "nixpkgs-22_11", "nixpkgs-23_05": "nixpkgs-23_05", diff --git a/flake.nix b/flake.nix index 83ee24bec..48dbfe7eb 100644 --- a/flake.nix +++ b/flake.nix @@ -20,7 +20,7 @@ # _third_ package set involved and gives a way to provide what should be a # working experience by default (that is, if nixpkgs doesn't get # overridden). - nixpkgs.follows = "nixpkgs-22_11"; + nixpkgs.follows = "nixpkgs-unstable"; # Also get flake-utils for simplified multi-system definitions. flake-utils = { From fd056026086f265236d3397c7fa9d4c27144e796 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 7 Jul 2023 16:00:00 -0400 Subject: [PATCH 1917/2309] Reduce the amount of test suite gymnastics with new WebishServer API Instead of forcing the test suite to try to discover the location of an unnamed temporary file, let it just assert that the file is created in the directory specified in the temporary file factory. --- src/allmydata/client.py | 7 ++- src/allmydata/test/web/test_web.py | 2 +- src/allmydata/test/web/test_webish.py | 89 +++++++++------------------ src/allmydata/webish.py | 40 ++++++++---- 4 files changed, 61 insertions(+), 77 deletions(-) diff --git a/src/allmydata/client.py b/src/allmydata/client.py index e85ed4fe2..fefd29657 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -1,5 +1,5 @@ """ -Ported to Python 3. +Functionality related to operating a Tahoe-LAFS node (client _or_ server). """ from __future__ import annotations @@ -7,6 +7,7 @@ import os import stat import time import weakref +import tempfile from typing import Optional, Iterable from base64 import urlsafe_b64encode from functools import partial @@ -1029,14 +1030,14 @@ class _Client(node.Node, pollmixin.PollMixin): def init_web(self, webport): self.log("init_web(webport=%s)", args=(webport,)) - from allmydata.webish import WebishServer + from allmydata.webish import WebishServer, anonymous_tempfile nodeurl_path = self.config.get_config_path("node.url") staticdir_config = self.config.get_config("node", "web.static", "public_html") staticdir = self.config.get_config_path(staticdir_config) ws = WebishServer( self, webport, - self._get_tempdir(), + anonymous_tempfile(self._get_tempdir()), nodeurl_path, staticdir, ) diff --git a/src/allmydata/test/web/test_web.py b/src/allmydata/test/web/test_web.py index 08dce0ac0..bd7ca3f7f 100644 --- a/src/allmydata/test/web/test_web.py +++ b/src/allmydata/test/web/test_web.py @@ -341,7 +341,7 @@ class WebMixin(TimezoneMixin): self.ws = webish.WebishServer( self.s, "0", - tempdir=tempdir.path, + webish.anonymous_tempfile(tempdir.path), staticdir=self.staticdir, clock=self.clock, now_fn=lambda:self.fakeTime, diff --git a/src/allmydata/test/web/test_webish.py b/src/allmydata/test/web/test_webish.py index 050f77d1c..d444df436 100644 --- a/src/allmydata/test/web/test_webish.py +++ b/src/allmydata/test/web/test_webish.py @@ -1,17 +1,8 @@ """ Tests for ``allmydata.webish``. - -Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals - -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 +import tempfile from uuid import ( uuid4, ) @@ -59,6 +50,7 @@ from ..common import ( from ...webish import ( TahoeLAFSRequest, TahoeLAFSSite, + anonymous_tempfile, ) @@ -183,8 +175,14 @@ class TahoeLAFSSiteTests(SyncTestCase): :return: ``None`` if the logging looks good. """ logPath = self.mktemp() + tempdir = self.mktemp() + FilePath(tempdir).makedirs() - site = TahoeLAFSSite(self.mktemp(), Resource(), logPath=logPath) + site = TahoeLAFSSite( + anonymous_tempfile(tempdir), + Resource(), + logPath=logPath, + ) site.startFactory() channel = DummyChannel() @@ -257,11 +255,17 @@ class TahoeLAFSSiteTests(SyncTestCase): Create and return a new ``TahoeLAFSRequest`` hooked up to a ``TahoeLAFSSite``. - :param bytes tempdir: The temporary directory to give to the site. + :param FilePath tempdir: The temporary directory to configure the site + to write large temporary request bodies to. The temporary files + will be named for ease of testing. :return TahoeLAFSRequest: The new request instance. """ - site = TahoeLAFSSite(tempdir.path, Resource(), logPath=self.mktemp()) + site = TahoeLAFSSite( + lambda: tempfile.NamedTemporaryFile(dir=tempdir.path), + Resource(), + logPath=self.mktemp(), + ) site.startFactory() channel = DummyChannel() @@ -275,6 +279,7 @@ class TahoeLAFSSiteTests(SyncTestCase): A request body smaller than 1 MiB is kept in memory. """ tempdir = FilePath(self.mktemp()) + tempdir.makedirs() request = self._create_request(tempdir) request.gotLength(request_body_size) self.assertThat( @@ -284,57 +289,21 @@ class TahoeLAFSSiteTests(SyncTestCase): def _large_request_test(self, request_body_size): """ - Assert that when a request with a body of of the given size is received - its content is written to the directory the ``TahoeLAFSSite`` is - configured with. + Assert that when a request with a body of the given size is + received its content is written a temporary file created by the given + tempfile factory. """ tempdir = FilePath(self.mktemp()) tempdir.makedirs() request = self._create_request(tempdir) - - # So. Bad news. The temporary file for the uploaded content is - # unnamed (and this isn't even necessarily a bad thing since it is how - # you get automatic on-process-exit cleanup behavior on POSIX). It's - # not visible by inspecting the filesystem. It has no name we can - # discover. Then how do we verify it is written to the right place? - # The question itself is meaningless if we try to be too precise. It - # *has* no filesystem location. However, it is still stored *on* some - # filesystem. We still want to make sure it is on the filesystem we - # specified because otherwise it might be on a filesystem that's too - # small or undesirable in some other way. - # - # I don't know of any way to ask a file descriptor which filesystem - # it's on, either, though. It might be the case that the [f]statvfs() - # result could be compared somehow to infer the filesystem but - # ... it's not clear what the failure modes might be there, across - # different filesystems and runtime environments. - # - # Another approach is to make the temp directory unwriteable and - # observe the failure when an attempt is made to create a file there. - # This is hardly a lovely solution but at least it's kind of simple. - # - # It would be nice if it worked consistently cross-platform but on - # Windows os.chmod is more or less broken. - if platform.isWindows(): - request.gotLength(request_body_size) - self.assertThat( - tempdir.children(), - HasLength(1), - ) - else: - tempdir.chmod(0o550) - with self.assertRaises(OSError) as ctx: - request.gotLength(request_body_size) - raise Exception( - "OSError not raised, instead tempdir.children() = {}".format( - tempdir.children(), - ), - ) - - self.assertThat( - ctx.exception.errno, - Equals(EACCES), - ) + request.gotLength(request_body_size) + # We can see the temporary file in the temporary directory we + # specified because _create_request makes a request that uses named + # temporary files instead of the usual anonymous temporary files. + self.assertThat( + tempdir.children(), + HasLength(1), + ) def test_unknown_request_size(self): """ diff --git a/src/allmydata/webish.py b/src/allmydata/webish.py index ec2582f80..2bae7f8a5 100644 --- a/src/allmydata/webish.py +++ b/src/allmydata/webish.py @@ -4,7 +4,7 @@ General web server-related utilities. from __future__ import annotations from six import ensure_str - +from typing import BinaryIO, Callable, Optional import re, time, tempfile from urllib.parse import parse_qsl, urlencode @@ -217,36 +217,50 @@ def censor(queryargs: bytes) -> bytes: return urlencode(result, safe="[]").encode("ascii") +def anonymous_tempfile(tempdir: bytes) -> BinaryIO: + """ + Create a no-argument callable for creating a new temporary file in the + given directory. + + :param tempdir: The directory in which temporary files with be created. + + :return: The callable. + """ + return lambda: tempfile.TemporaryFile(dir=tempdir) + + class TahoeLAFSSite(Site, object): """ The HTTP protocol factory used by Tahoe-LAFS. Among the behaviors provided: - * A configurable temporary directory where large request bodies can be - written so they don't stay in memory. + * A configurable temporary file factory for large request bodies to avoid + keeping them in memory. * A log formatter that writes some access logs but omits capability strings to help keep them secret. """ requestFactory = TahoeLAFSRequest - def __init__(self, tempdir, *args, **kwargs): + def __init__(self, make_tempfile: Callable[[], BinaryIO], *args, **kwargs): Site.__init__(self, *args, logFormatter=_logFormatter, **kwargs) - self._tempdir = tempdir + assert callable(make_tempfile) + with make_tempfile(): + pass + self._make_tempfile = make_tempfile - def getContentFile(self, length): + def getContentFile(self, length: Optional[int]) -> BinaryIO: if length is None or length >= 1024 * 1024: - return tempfile.TemporaryFile(dir=self._tempdir) + return self._make_tempfile() return BytesIO() - class WebishServer(service.MultiService): # The type in Twisted for services is wrong in 22.10... # https://github.com/twisted/twisted/issues/10135 name = "webish" # type: ignore[assignment] - def __init__(self, client, webport, tempdir, nodeurl_path=None, staticdir=None, + def __init__(self, client, webport, make_tempfile, nodeurl_path=None, staticdir=None, clock=None, now_fn=time.time): service.MultiService.__init__(self) # the 'data' argument to all render() methods default to the Client @@ -256,7 +270,7 @@ class WebishServer(service.MultiService): # time in a deterministic manner. self.root = root.Root(client, clock, now_fn) - self.buildServer(webport, tempdir, nodeurl_path, staticdir) + self.buildServer(webport, make_tempfile, nodeurl_path, staticdir) # If set, clock is a twisted.internet.task.Clock that the tests # use to test ophandle expiration. @@ -266,9 +280,9 @@ class WebishServer(service.MultiService): self.root.putChild(b"storage-plugins", StoragePlugins(client)) - def buildServer(self, webport, tempdir, nodeurl_path, staticdir): + def buildServer(self, webport, make_tempfile, nodeurl_path, staticdir): self.webport = webport - self.site = TahoeLAFSSite(tempdir, self.root) + self.site = TahoeLAFSSite(make_tempfile, self.root) self.staticdir = staticdir # so tests can check if staticdir: self.root.putChild(b"static", static.File(staticdir)) @@ -346,4 +360,4 @@ class IntroducerWebishServer(WebishServer): def __init__(self, introducer, webport, nodeurl_path=None, staticdir=None): service.MultiService.__init__(self) self.root = introweb.IntroducerRoot(introducer) - self.buildServer(webport, tempfile.tempdir, nodeurl_path, staticdir) + self.buildServer(webport, tempfile.TemporaryFile, nodeurl_path, staticdir) From 4b23b779e412797214736f6421d1776210d86738 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 7 Jul 2023 16:00:00 -0400 Subject: [PATCH 1918/2309] Reduce the amount of test suite gymnastics with new WebishServer API Instead of forcing the test suite to try to discover the location of an unnamed temporary file, let it just assert that the file is created in the directory specified in the temporary file factory. --- src/allmydata/client.py | 7 ++- src/allmydata/test/web/test_web.py | 2 +- src/allmydata/test/web/test_webish.py | 89 +++++++++------------------ src/allmydata/webish.py | 40 ++++++++---- 4 files changed, 61 insertions(+), 77 deletions(-) diff --git a/src/allmydata/client.py b/src/allmydata/client.py index e85ed4fe2..fefd29657 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -1,5 +1,5 @@ """ -Ported to Python 3. +Functionality related to operating a Tahoe-LAFS node (client _or_ server). """ from __future__ import annotations @@ -7,6 +7,7 @@ import os import stat import time import weakref +import tempfile from typing import Optional, Iterable from base64 import urlsafe_b64encode from functools import partial @@ -1029,14 +1030,14 @@ class _Client(node.Node, pollmixin.PollMixin): def init_web(self, webport): self.log("init_web(webport=%s)", args=(webport,)) - from allmydata.webish import WebishServer + from allmydata.webish import WebishServer, anonymous_tempfile nodeurl_path = self.config.get_config_path("node.url") staticdir_config = self.config.get_config("node", "web.static", "public_html") staticdir = self.config.get_config_path(staticdir_config) ws = WebishServer( self, webport, - self._get_tempdir(), + anonymous_tempfile(self._get_tempdir()), nodeurl_path, staticdir, ) diff --git a/src/allmydata/test/web/test_web.py b/src/allmydata/test/web/test_web.py index 08dce0ac0..bd7ca3f7f 100644 --- a/src/allmydata/test/web/test_web.py +++ b/src/allmydata/test/web/test_web.py @@ -341,7 +341,7 @@ class WebMixin(TimezoneMixin): self.ws = webish.WebishServer( self.s, "0", - tempdir=tempdir.path, + webish.anonymous_tempfile(tempdir.path), staticdir=self.staticdir, clock=self.clock, now_fn=lambda:self.fakeTime, diff --git a/src/allmydata/test/web/test_webish.py b/src/allmydata/test/web/test_webish.py index 050f77d1c..d444df436 100644 --- a/src/allmydata/test/web/test_webish.py +++ b/src/allmydata/test/web/test_webish.py @@ -1,17 +1,8 @@ """ Tests for ``allmydata.webish``. - -Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals - -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 +import tempfile from uuid import ( uuid4, ) @@ -59,6 +50,7 @@ from ..common import ( from ...webish import ( TahoeLAFSRequest, TahoeLAFSSite, + anonymous_tempfile, ) @@ -183,8 +175,14 @@ class TahoeLAFSSiteTests(SyncTestCase): :return: ``None`` if the logging looks good. """ logPath = self.mktemp() + tempdir = self.mktemp() + FilePath(tempdir).makedirs() - site = TahoeLAFSSite(self.mktemp(), Resource(), logPath=logPath) + site = TahoeLAFSSite( + anonymous_tempfile(tempdir), + Resource(), + logPath=logPath, + ) site.startFactory() channel = DummyChannel() @@ -257,11 +255,17 @@ class TahoeLAFSSiteTests(SyncTestCase): Create and return a new ``TahoeLAFSRequest`` hooked up to a ``TahoeLAFSSite``. - :param bytes tempdir: The temporary directory to give to the site. + :param FilePath tempdir: The temporary directory to configure the site + to write large temporary request bodies to. The temporary files + will be named for ease of testing. :return TahoeLAFSRequest: The new request instance. """ - site = TahoeLAFSSite(tempdir.path, Resource(), logPath=self.mktemp()) + site = TahoeLAFSSite( + lambda: tempfile.NamedTemporaryFile(dir=tempdir.path), + Resource(), + logPath=self.mktemp(), + ) site.startFactory() channel = DummyChannel() @@ -275,6 +279,7 @@ class TahoeLAFSSiteTests(SyncTestCase): A request body smaller than 1 MiB is kept in memory. """ tempdir = FilePath(self.mktemp()) + tempdir.makedirs() request = self._create_request(tempdir) request.gotLength(request_body_size) self.assertThat( @@ -284,57 +289,21 @@ class TahoeLAFSSiteTests(SyncTestCase): def _large_request_test(self, request_body_size): """ - Assert that when a request with a body of of the given size is received - its content is written to the directory the ``TahoeLAFSSite`` is - configured with. + Assert that when a request with a body of the given size is + received its content is written a temporary file created by the given + tempfile factory. """ tempdir = FilePath(self.mktemp()) tempdir.makedirs() request = self._create_request(tempdir) - - # So. Bad news. The temporary file for the uploaded content is - # unnamed (and this isn't even necessarily a bad thing since it is how - # you get automatic on-process-exit cleanup behavior on POSIX). It's - # not visible by inspecting the filesystem. It has no name we can - # discover. Then how do we verify it is written to the right place? - # The question itself is meaningless if we try to be too precise. It - # *has* no filesystem location. However, it is still stored *on* some - # filesystem. We still want to make sure it is on the filesystem we - # specified because otherwise it might be on a filesystem that's too - # small or undesirable in some other way. - # - # I don't know of any way to ask a file descriptor which filesystem - # it's on, either, though. It might be the case that the [f]statvfs() - # result could be compared somehow to infer the filesystem but - # ... it's not clear what the failure modes might be there, across - # different filesystems and runtime environments. - # - # Another approach is to make the temp directory unwriteable and - # observe the failure when an attempt is made to create a file there. - # This is hardly a lovely solution but at least it's kind of simple. - # - # It would be nice if it worked consistently cross-platform but on - # Windows os.chmod is more or less broken. - if platform.isWindows(): - request.gotLength(request_body_size) - self.assertThat( - tempdir.children(), - HasLength(1), - ) - else: - tempdir.chmod(0o550) - with self.assertRaises(OSError) as ctx: - request.gotLength(request_body_size) - raise Exception( - "OSError not raised, instead tempdir.children() = {}".format( - tempdir.children(), - ), - ) - - self.assertThat( - ctx.exception.errno, - Equals(EACCES), - ) + request.gotLength(request_body_size) + # We can see the temporary file in the temporary directory we + # specified because _create_request makes a request that uses named + # temporary files instead of the usual anonymous temporary files. + self.assertThat( + tempdir.children(), + HasLength(1), + ) def test_unknown_request_size(self): """ diff --git a/src/allmydata/webish.py b/src/allmydata/webish.py index ec2582f80..2bae7f8a5 100644 --- a/src/allmydata/webish.py +++ b/src/allmydata/webish.py @@ -4,7 +4,7 @@ General web server-related utilities. from __future__ import annotations from six import ensure_str - +from typing import BinaryIO, Callable, Optional import re, time, tempfile from urllib.parse import parse_qsl, urlencode @@ -217,36 +217,50 @@ def censor(queryargs: bytes) -> bytes: return urlencode(result, safe="[]").encode("ascii") +def anonymous_tempfile(tempdir: bytes) -> BinaryIO: + """ + Create a no-argument callable for creating a new temporary file in the + given directory. + + :param tempdir: The directory in which temporary files with be created. + + :return: The callable. + """ + return lambda: tempfile.TemporaryFile(dir=tempdir) + + class TahoeLAFSSite(Site, object): """ The HTTP protocol factory used by Tahoe-LAFS. Among the behaviors provided: - * A configurable temporary directory where large request bodies can be - written so they don't stay in memory. + * A configurable temporary file factory for large request bodies to avoid + keeping them in memory. * A log formatter that writes some access logs but omits capability strings to help keep them secret. """ requestFactory = TahoeLAFSRequest - def __init__(self, tempdir, *args, **kwargs): + def __init__(self, make_tempfile: Callable[[], BinaryIO], *args, **kwargs): Site.__init__(self, *args, logFormatter=_logFormatter, **kwargs) - self._tempdir = tempdir + assert callable(make_tempfile) + with make_tempfile(): + pass + self._make_tempfile = make_tempfile - def getContentFile(self, length): + def getContentFile(self, length: Optional[int]) -> BinaryIO: if length is None or length >= 1024 * 1024: - return tempfile.TemporaryFile(dir=self._tempdir) + return self._make_tempfile() return BytesIO() - class WebishServer(service.MultiService): # The type in Twisted for services is wrong in 22.10... # https://github.com/twisted/twisted/issues/10135 name = "webish" # type: ignore[assignment] - def __init__(self, client, webport, tempdir, nodeurl_path=None, staticdir=None, + def __init__(self, client, webport, make_tempfile, nodeurl_path=None, staticdir=None, clock=None, now_fn=time.time): service.MultiService.__init__(self) # the 'data' argument to all render() methods default to the Client @@ -256,7 +270,7 @@ class WebishServer(service.MultiService): # time in a deterministic manner. self.root = root.Root(client, clock, now_fn) - self.buildServer(webport, tempdir, nodeurl_path, staticdir) + self.buildServer(webport, make_tempfile, nodeurl_path, staticdir) # If set, clock is a twisted.internet.task.Clock that the tests # use to test ophandle expiration. @@ -266,9 +280,9 @@ class WebishServer(service.MultiService): self.root.putChild(b"storage-plugins", StoragePlugins(client)) - def buildServer(self, webport, tempdir, nodeurl_path, staticdir): + def buildServer(self, webport, make_tempfile, nodeurl_path, staticdir): self.webport = webport - self.site = TahoeLAFSSite(tempdir, self.root) + self.site = TahoeLAFSSite(make_tempfile, self.root) self.staticdir = staticdir # so tests can check if staticdir: self.root.putChild(b"static", static.File(staticdir)) @@ -346,4 +360,4 @@ class IntroducerWebishServer(WebishServer): def __init__(self, introducer, webport, nodeurl_path=None, staticdir=None): service.MultiService.__init__(self) self.root = introweb.IntroducerRoot(introducer) - self.buildServer(webport, tempfile.tempdir, nodeurl_path, staticdir) + self.buildServer(webport, tempfile.TemporaryFile, nodeurl_path, staticdir) From 3129898563ed42cc8266d7b3c5d3aad43c9d3f56 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 7 Jul 2023 16:04:54 -0400 Subject: [PATCH 1919/2309] news fragment --- newsfragments/4044.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/4044.minor diff --git a/newsfragments/4044.minor b/newsfragments/4044.minor new file mode 100644 index 000000000..e69de29bb From dfd34cfc0b8a4fccb5e39f2b825442b662a0a2fb Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 7 Jul 2023 16:26:31 -0400 Subject: [PATCH 1920/2309] suppress the new click mypy errors --- src/allmydata/cli/grid_manager.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/src/allmydata/cli/grid_manager.py b/src/allmydata/cli/grid_manager.py index 3110a072e..039f0f50f 100644 --- a/src/allmydata/cli/grid_manager.py +++ b/src/allmydata/cli/grid_manager.py @@ -28,7 +28,7 @@ from allmydata.grid_manager import ( from allmydata.util import jsonbytes as json -@click.group() +@click.group() # type: ignore[arg-type] @click.option( '--config', '-c', type=click.Path(), @@ -71,7 +71,7 @@ def grid_manager(ctx, config): ctx.obj = Config() -@grid_manager.command() +@grid_manager.command() # type: ignore[attr-defined] @click.pass_context def create(ctx): """ @@ -91,7 +91,7 @@ def create(ctx): ) -@grid_manager.command() +@grid_manager.command() # type: ignore[attr-defined] @click.pass_obj def public_identity(config): """ @@ -103,7 +103,7 @@ def public_identity(config): click.echo(config.grid_manager.public_identity()) -@grid_manager.command() +@grid_manager.command() # type: ignore[arg-type, attr-defined] @click.argument("name") @click.argument("public_key", type=click.STRING) @click.pass_context @@ -132,7 +132,7 @@ def add(ctx, name, public_key): return 0 -@grid_manager.command() +@grid_manager.command() # type: ignore[arg-type, attr-defined] @click.argument("name") @click.pass_context def remove(ctx, name): @@ -155,7 +155,8 @@ def remove(ctx, name): save_grid_manager(fp, ctx.obj.grid_manager, create=False) -@grid_manager.command() # noqa: F811 +@grid_manager.command() # type: ignore[attr-defined] + # noqa: F811 @click.pass_context def list(ctx): """ @@ -175,7 +176,7 @@ def list(ctx): click.echo("expired {} ({})".format(cert.expires, abbreviate_time(delta))) -@grid_manager.command() +@grid_manager.command() # type: ignore[arg-type, attr-defined] @click.argument("name") @click.argument( "expiry_days", From c06e3e12babdce4cf3731c718c89d29ad493be09 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 7 Jul 2023 16:32:15 -0400 Subject: [PATCH 1921/2309] try to work-around towncrier compatibility issue --- tox.ini | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tox.ini b/tox.ini index 2f245f2ed..127cd9178 100644 --- a/tox.ini +++ b/tox.ini @@ -100,6 +100,9 @@ deps = # Pin a specific version so we get consistent outcomes; update this # occasionally: ruff == 0.0.263 + # towncrier doesn't work with importlib_resources 6.0.0 + # https://github.com/twisted/towncrier/issues/528 + importlib_resources < 6.0.0 towncrier # On macOS, git inside of towncrier needs $HOME. passenv = HOME From e3e223c8d6863ff4ab8c2cdfb1243a823b5d23d9 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Sat, 8 Jul 2023 08:08:47 -0400 Subject: [PATCH 1922/2309] bump to a newer nixos image --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index e588cfa88..6b19bdfd5 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -532,7 +532,7 @@ executors: docker: # Run in a highly Nix-capable environment. - <<: *DOCKERHUB_AUTH - image: "nixos/nix:2.10.3" + image: "nixos/nix:2.16.1" environment: # CACHIX_AUTH_TOKEN is manually set in the CircleCI web UI and allows us # to push to CACHIX_NAME. CACHIX_NAME tells cachix which cache to push From 6710d625a2ea41506b8325dd55faa2368f182a44 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Sat, 8 Jul 2023 08:08:54 -0400 Subject: [PATCH 1923/2309] can we just run as an unprivileged user? --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 6b19bdfd5..bbb10d478 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -533,6 +533,7 @@ executors: # Run in a highly Nix-capable environment. - <<: *DOCKERHUB_AUTH image: "nixos/nix:2.16.1" + user: "nobody" environment: # CACHIX_AUTH_TOKEN is manually set in the CircleCI web UI and allows us # to push to CACHIX_NAME. CACHIX_NAME tells cachix which cache to push From 454ab223d1407f4076c4c0ae8e7509fb3e0ebb42 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Sat, 8 Jul 2023 08:10:52 -0400 Subject: [PATCH 1924/2309] nope, we can't just do that we lose permission to install stuff --- .circleci/config.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index bbb10d478..6b19bdfd5 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -533,7 +533,6 @@ executors: # Run in a highly Nix-capable environment. - <<: *DOCKERHUB_AUTH image: "nixos/nix:2.16.1" - user: "nobody" environment: # CACHIX_AUTH_TOKEN is manually set in the CircleCI web UI and allows us # to push to CACHIX_NAME. CACHIX_NAME tells cachix which cache to push From 5553019c4ec6c99eeec7c100b37f1da689fd7ea0 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Sat, 8 Jul 2023 08:14:27 -0400 Subject: [PATCH 1925/2309] switch to new `nix profile`-based installation --- .circleci/config.yml | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 6b19bdfd5..d8a70b2c4 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -564,15 +564,12 @@ commands: # Get cachix for Nix-friendly caching. name: "Install Basic Dependencies" command: | - NIXPKGS="https://github.com/nixos/nixpkgs/archive/nixos-23.05.tar.gz" - nix-env \ - --file $NIXPKGS \ - --install \ - -A cachix bash jp - # Activate it for "binary substitution". This sets up - # configuration tht lets Nix download something from the cache - # instead of building it locally, if possible. - cachix use "${CACHIX_NAME}" + NIXPKGS="nixpkgs/nixos-23.05" + nix profile install $NIXPKGS#cachix $NIXPKGS#bash $NIXPKGS#jp + # Activate it for "binary substitution". This sets up + # configuration tht lets Nix download something from the cache + # instead of building it locally, if possible. + cachix use "${CACHIX_NAME}" - "checkout" From c838967a54635f4feb9ac2209e7c5e8d2b374f62 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 11 Jul 2023 16:15:56 -0400 Subject: [PATCH 1926/2309] Improve the name and type annotation of the tempfile factory --- src/allmydata/client.py | 4 ++-- src/allmydata/test/web/test_web.py | 2 +- src/allmydata/test/web/test_webish.py | 4 ++-- src/allmydata/webish.py | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/allmydata/client.py b/src/allmydata/client.py index fefd29657..13909d5ca 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -1030,14 +1030,14 @@ class _Client(node.Node, pollmixin.PollMixin): def init_web(self, webport): self.log("init_web(webport=%s)", args=(webport,)) - from allmydata.webish import WebishServer, anonymous_tempfile + from allmydata.webish import WebishServer, anonymous_tempfile_factory nodeurl_path = self.config.get_config_path("node.url") staticdir_config = self.config.get_config("node", "web.static", "public_html") staticdir = self.config.get_config_path(staticdir_config) ws = WebishServer( self, webport, - anonymous_tempfile(self._get_tempdir()), + anonymous_tempfile_factory(self._get_tempdir()), nodeurl_path, staticdir, ) diff --git a/src/allmydata/test/web/test_web.py b/src/allmydata/test/web/test_web.py index bd7ca3f7f..42be0f50d 100644 --- a/src/allmydata/test/web/test_web.py +++ b/src/allmydata/test/web/test_web.py @@ -341,7 +341,7 @@ class WebMixin(TimezoneMixin): self.ws = webish.WebishServer( self.s, "0", - webish.anonymous_tempfile(tempdir.path), + webish.anonymous_tempfile_factory(tempdir.path), staticdir=self.staticdir, clock=self.clock, now_fn=lambda:self.fakeTime, diff --git a/src/allmydata/test/web/test_webish.py b/src/allmydata/test/web/test_webish.py index d444df436..a0206c7eb 100644 --- a/src/allmydata/test/web/test_webish.py +++ b/src/allmydata/test/web/test_webish.py @@ -50,7 +50,7 @@ from ..common import ( from ...webish import ( TahoeLAFSRequest, TahoeLAFSSite, - anonymous_tempfile, + anonymous_tempfile_factory, ) @@ -179,7 +179,7 @@ class TahoeLAFSSiteTests(SyncTestCase): FilePath(tempdir).makedirs() site = TahoeLAFSSite( - anonymous_tempfile(tempdir), + anonymous_tempfile_factory(tempdir), Resource(), logPath=logPath, ) diff --git a/src/allmydata/webish.py b/src/allmydata/webish.py index 2bae7f8a5..2a55b3446 100644 --- a/src/allmydata/webish.py +++ b/src/allmydata/webish.py @@ -217,7 +217,7 @@ def censor(queryargs: bytes) -> bytes: return urlencode(result, safe="[]").encode("ascii") -def anonymous_tempfile(tempdir: bytes) -> BinaryIO: +def anonymous_tempfile_factory(tempdir: bytes) -> Callable[[], BinaryIO]: """ Create a no-argument callable for creating a new temporary file in the given directory. From 79512a93e722f95e75ff63b07df0177c2781d464 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 11 Jul 2023 16:30:54 -0400 Subject: [PATCH 1927/2309] Adjust the temp factory return type BinaryIO is a subclass of IO[bytes] so it doesn't check out as the return type of a callable we pass around. Switch to the superclass instead. --- src/allmydata/webish.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/allmydata/webish.py b/src/allmydata/webish.py index 2a55b3446..ec5ad64c0 100644 --- a/src/allmydata/webish.py +++ b/src/allmydata/webish.py @@ -4,7 +4,7 @@ General web server-related utilities. from __future__ import annotations from six import ensure_str -from typing import BinaryIO, Callable, Optional +from typing import IO, Callable, Optional import re, time, tempfile from urllib.parse import parse_qsl, urlencode @@ -217,7 +217,7 @@ def censor(queryargs: bytes) -> bytes: return urlencode(result, safe="[]").encode("ascii") -def anonymous_tempfile_factory(tempdir: bytes) -> Callable[[], BinaryIO]: +def anonymous_tempfile_factory(tempdir: bytes) -> Callable[[], IO[bytes]]: """ Create a no-argument callable for creating a new temporary file in the given directory. @@ -243,14 +243,14 @@ class TahoeLAFSSite(Site, object): """ requestFactory = TahoeLAFSRequest - def __init__(self, make_tempfile: Callable[[], BinaryIO], *args, **kwargs): + def __init__(self, make_tempfile: Callable[[], IO[bytes]], *args, **kwargs): Site.__init__(self, *args, logFormatter=_logFormatter, **kwargs) assert callable(make_tempfile) with make_tempfile(): pass self._make_tempfile = make_tempfile - def getContentFile(self, length: Optional[int]) -> BinaryIO: + def getContentFile(self, length: Optional[int]) -> IO[bytes]: if length is None or length >= 1024 * 1024: return self._make_tempfile() return BytesIO() From eef52fa59fa64f166fc1484653abefac84a6dff7 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 11 Jul 2023 16:32:33 -0400 Subject: [PATCH 1928/2309] remove unused imports --- src/allmydata/client.py | 1 - src/allmydata/test/web/test_webish.py | 6 ------ 2 files changed, 7 deletions(-) diff --git a/src/allmydata/client.py b/src/allmydata/client.py index 13909d5ca..a1501f1ef 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -7,7 +7,6 @@ import os import stat import time import weakref -import tempfile from typing import Optional, Iterable from base64 import urlsafe_b64encode from functools import partial diff --git a/src/allmydata/test/web/test_webish.py b/src/allmydata/test/web/test_webish.py index a0206c7eb..523dfc878 100644 --- a/src/allmydata/test/web/test_webish.py +++ b/src/allmydata/test/web/test_webish.py @@ -6,9 +6,6 @@ import tempfile from uuid import ( uuid4, ) -from errno import ( - EACCES, -) from io import ( BytesIO, ) @@ -30,9 +27,6 @@ from testtools.matchers import ( HasLength, ) -from twisted.python.runtime import ( - platform, -) from twisted.python.filepath import ( FilePath, ) From 4c8a20c8767bf27881e8151f049dff856780bdb9 Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 12 Jul 2023 18:33:43 -0600 Subject: [PATCH 1929/2309] When finalizing a process, we can ignore the case where it isn't running --- integration/grid.py | 3 +-- integration/util.py | 45 +++++++++++++++++++++------------------------ 2 files changed, 22 insertions(+), 26 deletions(-) diff --git a/integration/grid.py b/integration/grid.py index 794639b2f..46fde576e 100644 --- a/integration/grid.py +++ b/integration/grid.py @@ -235,8 +235,7 @@ class Client(object): self.protocol = self.process.transport.proto yield await_client_ready(self.process, minimum_number_of_servers=servers) - - # XXX add stop / start / restart + # XXX add stop / start ? # ...maybe "reconfig" of some kind? diff --git a/integration/util.py b/integration/util.py index 6a3ec57f3..ff54b1831 100644 --- a/integration/util.py +++ b/integration/util.py @@ -177,38 +177,33 @@ class _MagicTextProtocol(ProcessProtocol): sys.stdout.write(self.name + line + "\n") -def _cleanup_process_async(transport: IProcessTransport, allow_missing: bool) -> None: +def _cleanup_process_async(transport: IProcessTransport) -> None: """ If the given process transport seems to still be associated with a running process, send a SIGTERM to that process. :param transport: The transport to use. - :param allow_missing: If ``True`` then it is not an error for the - transport to have no associated process. Otherwise, an exception will - be raised in that case. - :raise: ``ValueError`` if ``allow_missing`` is ``False`` and the transport has no process. """ if transport.pid is None: - if allow_missing: - print("Process already cleaned up and that's okay.") - return - else: - raise ValueError("Process is not running") + # in cases of "restart", we will have registered a finalizer + # that will kill the process -- but already explicitly killed + # it (and then ran again) due to the "restart". So, if the + # process is already killed, our job is done. + print("Process already cleaned up and that's okay.") + return print("signaling {} with TERM".format(transport.pid)) try: transport.signalProcess('TERM') except ProcessExitedAlready: # The transport object thought it still had a process but the real OS # process has already exited. That's fine. We accomplished what we - # wanted to. We don't care about ``allow_missing`` here because - # there's no way we could have known the real OS process already - # exited. + # wanted to. pass -def _cleanup_tahoe_process(tahoe_transport, exited, allow_missing=False): +def _cleanup_tahoe_process(tahoe_transport, exited): """ Terminate the given process with a kill signal (SIGTERM on POSIX, TerminateProcess on Windows). @@ -219,7 +214,7 @@ def _cleanup_tahoe_process(tahoe_transport, exited, allow_missing=False): :return: After the process has exited. """ from twisted.internet import reactor - _cleanup_process_async(tahoe_transport, allow_missing=allow_missing) + _cleanup_process_async(tahoe_transport) print(f"signaled, blocking on exit {exited}") block_with_timeout(exited, reactor) print("exited, goodbye") @@ -282,16 +277,20 @@ class TahoeProcess(object): ) def kill(self): - """Kill the process, block until it's done.""" + """ + Kill the process, block until it's done. + Does nothing if the process is already stopped (or never started). + """ print(f"TahoeProcess.kill({self.transport.pid} / {self.node_dir})") _cleanup_tahoe_process(self.transport, self.transport.exited) def kill_async(self): """ Kill the process, return a Deferred that fires when it's done. + Does nothing if the process is already stopped (or never started). """ print(f"TahoeProcess.kill_async({self.transport.pid} / {self.node_dir})") - _cleanup_process_async(self.transport, allow_missing=False) + _cleanup_process_async(self.transport) return self.transport.exited def restart_async(self, reactor: IReactorProcess, request: Any) -> Deferred: @@ -302,7 +301,7 @@ class TahoeProcess(object): handle requests. """ d = self.kill_async() - d.addCallback(lambda ignored: _run_node(reactor, self.node_dir, request, None, finalize=False)) + d.addCallback(lambda ignored: _run_node(reactor, self.node_dir, request, None)) def got_new_process(proc): # Grab the new transport since the one we had before is no longer # valid after the stop/start cycle. @@ -314,7 +313,7 @@ class TahoeProcess(object): return "".format(self._node_dir) -def _run_node(reactor, node_dir, request, magic_text, finalize=True): +def _run_node(reactor, node_dir, request, magic_text): """ Run a tahoe process from its node_dir. @@ -343,8 +342,7 @@ def _run_node(reactor, node_dir, request, magic_text, finalize=True): node_dir, ) - if finalize: - request.addfinalizer(tahoe_process.kill) + request.addfinalizer(tahoe_process.kill) d = protocol.magic_seen d.addCallback(lambda ignored: tahoe_process) @@ -386,8 +384,7 @@ def _create_node(reactor, request, temp_dir, introducer_furl, flog_gatherer, nam magic_text=None, needed=2, happy=3, - total=4, - finalize=True): + total=4): """ Helper to create a single node, run it and return the instance spawnProcess returned (ITransport) @@ -427,7 +424,7 @@ def _create_node(reactor, request, temp_dir, introducer_furl, flog_gatherer, nam d = Deferred() d.callback(None) d.addCallback(lambda _: created_d) - d.addCallback(lambda _: _run_node(reactor, node_dir, request, magic_text, finalize=finalize)) + d.addCallback(lambda _: _run_node(reactor, node_dir, request, magic_text)) return d From 0b9506dfada0c26d2fd305e9ed339bc5e2d6562c Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 13 Jul 2023 17:53:27 -0600 Subject: [PATCH 1930/2309] try new-enoug to avoid a type error --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 7ca2650d5..9ebdae550 100644 --- a/setup.py +++ b/setup.py @@ -151,7 +151,7 @@ install_requires = [ "pycddl >= 0.4", # Command-line parsing - "click >= 7.0", + "click >= 8.1.1", # for pid-file support "psutil", From a4801cc2ebe396dd29b284b2acc8fecd93ec405b Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 17 Jul 2023 17:10:45 -0600 Subject: [PATCH 1931/2309] CI uses tox less than 4 --- setup.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/setup.py b/setup.py index 9ebdae550..a2e870e8b 100644 --- a/setup.py +++ b/setup.py @@ -436,6 +436,8 @@ setup(name="tahoe-lafs", # also set in __init__.py "pytest-timeout", # Does our OpenMetrics endpoint adhere to the spec: "prometheus-client == 0.11.0", + # CI uses "tox<4", change here too if that becomes different + "tox < 4", ] + tor_requires + i2p_requires, "tor": tor_requires, "i2p": i2p_requires, From 6c5cb02ee5e667590d8383944054debada1f1f33 Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 17 Jul 2023 17:11:00 -0600 Subject: [PATCH 1932/2309] shush mypy --- src/allmydata/cli/grid_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/cli/grid_manager.py b/src/allmydata/cli/grid_manager.py index dfefeb576..d5a5d7e35 100644 --- a/src/allmydata/cli/grid_manager.py +++ b/src/allmydata/cli/grid_manager.py @@ -226,4 +226,4 @@ def _config_path_from_option(config: str) -> Optional[FilePath]: if __name__ == '__main__': - grid_manager() + grid_manager() # type: ignore From e6b3b658106359f6157d79dfe6b4974af4e49c93 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 19 Jul 2023 11:35:51 -0400 Subject: [PATCH 1933/2309] add some missing docstrings --- src/allmydata/test/cli/test_create.py | 19 +++++++++++++++++++ src/allmydata/util/i2p_provider.py | 5 +++++ 2 files changed, 24 insertions(+) diff --git a/src/allmydata/test/cli/test_create.py b/src/allmydata/test/cli/test_create.py index d100f481f..c6caf3395 100644 --- a/src/allmydata/test/cli/test_create.py +++ b/src/allmydata/test/cli/test_create.py @@ -47,6 +47,13 @@ class Config(unittest.TestCase): self.assertIn("option %s not recognized" % (option,), str(e)) async def test_create_client_config(self): + """ + ``create_node.write_client_config`` writes a configuration file + that can be parsed. + + TODO Maybe we should test that we can recover the given configuration + from the parse, too. + """ d = self.mktemp() os.mkdir(d) fname = os.path.join(d, 'tahoe.cfg') @@ -289,6 +296,10 @@ class Config(unittest.TestCase): @defer.inlineCallbacks def test_node_slow(self): + """ + A node can be created using a listener type that returns an + unfired Deferred from its ``create_config`` method. + """ d = defer.Deferred() slow = StaticProvider(True, False, d, None) create_node._LISTENERS["xxyzy"] = slow @@ -384,6 +395,10 @@ class Tor(unittest.TestCase): self.assertEqual(cfg.get("node", "tub.location"), "jkl") def test_launch(self): + """ + The ``--tor-launch`` command line option sets ``tor-launch`` to + ``True``. + """ basedir = self.mktemp() config_d = defer.succeed(None) @@ -400,6 +415,10 @@ class Tor(unittest.TestCase): self.assertEqual(args[1]["tor-control-port"], None) def test_control_port(self): + """ + The ``--tor-control-port`` command line parameter's value is + passed along as the ``tor-control-port`` value. + """ basedir = self.mktemp() config_d = defer.succeed(None) diff --git a/src/allmydata/util/i2p_provider.py b/src/allmydata/util/i2p_provider.py index 4d997945f..996237873 100644 --- a/src/allmydata/util/i2p_provider.py +++ b/src/allmydata/util/i2p_provider.py @@ -109,6 +109,11 @@ def _connect_to_i2p(reactor, cli_config, txi2p): raise ValueError("unable to reach any default I2P SAM port") async def create_config(reactor: Any, cli_config: Any) -> ListenerConfig: + """ + For a given set of command-line options, construct an I2P listener. + + This includes allocating a new I2P address. + """ txi2p = _import_txi2p() if not txi2p: raise ValueError("Cannot create I2P Destination without txi2p. " From c1c0b60862855c1966d8aee058e7398b6fd2d93c Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 19 Jul 2023 11:42:38 -0400 Subject: [PATCH 1934/2309] remove hard-coded tor/i2p in hide-ip support --- src/allmydata/scripts/create_node.py | 15 +++++++++++---- src/allmydata/util/dictutil.py | 17 +++++++++++++++++ 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/src/allmydata/scripts/create_node.py b/src/allmydata/scripts/create_node.py index 85d1c46cd..69215bcde 100644 --- a/src/allmydata/scripts/create_node.py +++ b/src/allmydata/scripts/create_node.py @@ -24,6 +24,7 @@ from allmydata.scripts.common import ( write_introducer, ) from allmydata.scripts.default_nodedir import _default_nodedir +from allmydata.util import dictutil from allmydata.util.assertutil import precondition from allmydata.util.encodingutil import listdir_unicode, argv_to_unicode, quote_local_unicode_path, get_io_encoding @@ -183,11 +184,17 @@ class _CreateBaseOptions(BasedirOptions): def postOptions(self): super(_CreateBaseOptions, self).postOptions() if self['hide-ip']: - if not (_LISTENERS["tor"].is_available() or _LISTENERS["i2p"].is_available()): + ip_hiders = dictutil.filter(lambda v: v.can_hide_ip(), _LISTENERS) + available = dictutil.filter(lambda v: v.is_available(), ip_hiders) + if not available: raise UsageError( - "--hide-ip was specified but neither 'txtorcon' nor 'txi2p' " - "are installed.\nTo do so:\n pip install tahoe-lafs[tor]\nor\n" - " pip install tahoe-lafs[i2p]" + "--hide-ip was specified but no IP-hiding listener is installed.\n" + "Try one of these:\n" + + "".join([ + f"\tpip install tahoe-lafs[{name}]\n" + for name + in ip_hiders + ]) ) class CreateClientOptions(_CreateBaseOptions): diff --git a/src/allmydata/util/dictutil.py b/src/allmydata/util/dictutil.py index 0a7df0a38..277fc30e9 100644 --- a/src/allmydata/util/dictutil.py +++ b/src/allmydata/util/dictutil.py @@ -2,6 +2,23 @@ Tools to mess with dicts. """ +from __future__ import annotations +from typing import Callable, TypeVar + +K = TypeVar("K") +V = TypeVar("V") + +def filter(pred: Callable[[V], bool], orig: dict[K, V]) -> dict[K, V]: + """ + Filter out key/value pairs that fail to match a predicate. + """ + return { + k: v + for (k, v) + in orig.items() + if pred(v) + } + class DictOfSets(dict): def add(self, key, value): if key in self: From 72c18579e245b2be317e3e83b317d2584f33c58b Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 19 Jul 2023 11:54:18 -0400 Subject: [PATCH 1935/2309] another docstring --- src/allmydata/scripts/create_node.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/allmydata/scripts/create_node.py b/src/allmydata/scripts/create_node.py index 69215bcde..0cd5c577b 100644 --- a/src/allmydata/scripts/create_node.py +++ b/src/allmydata/scripts/create_node.py @@ -267,6 +267,13 @@ def merge_config( left: Optional[ListenerConfig], right: Optional[ListenerConfig], ) -> Optional[ListenerConfig]: + """ + Merge two listener configurations into one configuration representing + both of them. + + If either is ``None`` then the result is ``None``. This supports the + "disable listeners" functionality. + """ if left is None or right is None: return None return ListenerConfig( From 911b54267be080ece9dc3f436a082834b1de3ae8 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 19 Jul 2023 11:54:22 -0400 Subject: [PATCH 1936/2309] StaticProviders don't need to change --- src/allmydata/listeners.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/listeners.py b/src/allmydata/listeners.py index 667f984e5..17f9ebd3b 100644 --- a/src/allmydata/listeners.py +++ b/src/allmydata/listeners.py @@ -94,7 +94,7 @@ class TCPProvider: raise NotImplementedError() -@define +@frozen class StaticProvider: """ A provider that uses all pre-computed values. From ee8155729dac54d1deaa5b5ab8ef835b5b667262 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 19 Jul 2023 12:03:18 -0400 Subject: [PATCH 1937/2309] clean up some type annotations --- src/allmydata/listeners.py | 8 ++++---- src/allmydata/util/tor_provider.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/allmydata/listeners.py b/src/allmydata/listeners.py index 17f9ebd3b..93eecf09f 100644 --- a/src/allmydata/listeners.py +++ b/src/allmydata/listeners.py @@ -9,7 +9,7 @@ from __future__ import annotations from typing import Any, Protocol, Sequence, Mapping, Optional, Union, Awaitable from typing_extensions import Literal -from attrs import frozen, define +from attrs import frozen from .interfaces import IAddressFamily from .util.iputil import allocate_tcp_port @@ -101,7 +101,7 @@ class StaticProvider: """ _available: bool _hide_ip: bool - _config: Union[Awaitable[ListenerConfig], ListenerConfig] + _config: Union[Awaitable[Optional[ListenerConfig]], Optional[ListenerConfig]] _address: IAddressFamily def is_available(self) -> bool: @@ -110,8 +110,8 @@ class StaticProvider: def can_hide_ip(self) -> bool: return self._hide_ip - async def create_config(self, reactor: Any, cli_config: Any) -> ListenerConfig: - if isinstance(self._config, ListenerConfig): + async def create_config(self, reactor: Any, cli_config: Any) -> Optional[ListenerConfig]: + if self._config is None or isinstance(self._config, ListenerConfig): return self._config return await self._config diff --git a/src/allmydata/util/tor_provider.py b/src/allmydata/util/tor_provider.py index f22371399..18a3281f4 100644 --- a/src/allmydata/util/tor_provider.py +++ b/src/allmydata/util/tor_provider.py @@ -42,7 +42,7 @@ def can_hide_ip() -> Literal[True]: def is_available() -> bool: return not (_import_tor() is None or _import_txtorcon() is None) -def create(reactor, config, import_tor=None, import_txtorcon=None) -> Optional[_Provider]: +def create(reactor, config, import_tor=None, import_txtorcon=None) -> _Provider: """ Create a new _Provider service (this is an IService so must be hooked up to a parent or otherwise started). From 40665d824d056fba0f929d636f7214d1eda8450b Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 19 Jul 2023 12:04:16 -0400 Subject: [PATCH 1938/2309] remove unused import --- src/allmydata/util/tor_provider.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/util/tor_provider.py b/src/allmydata/util/tor_provider.py index 18a3281f4..0e3090578 100644 --- a/src/allmydata/util/tor_provider.py +++ b/src/allmydata/util/tor_provider.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- from __future__ import annotations -from typing import Any, Optional +from typing import Any from typing_extensions import Literal import os From aa144fc62318ba9efbc4248a6757fc0c1ae8e032 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 19 Jul 2023 12:37:03 -0400 Subject: [PATCH 1939/2309] Make NURLs a set. --- newsfragments/4046.minor | 0 src/allmydata/client.py | 6 +++++- 2 files changed, 5 insertions(+), 1 deletion(-) create mode 100644 newsfragments/4046.minor diff --git a/newsfragments/4046.minor b/newsfragments/4046.minor new file mode 100644 index 000000000..e69de29bb diff --git a/src/allmydata/client.py b/src/allmydata/client.py index a1501f1ef..aff2d5815 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -837,7 +837,11 @@ class _Client(node.Node, pollmixin.PollMixin): if hasattr(self.tub.negotiationClass, "add_storage_server"): nurls = self.tub.negotiationClass.add_storage_server(ss, swissnum.encode("ascii")) self.storage_nurls = nurls - announcement[storage_client.ANONYMOUS_STORAGE_NURLS] = [n.to_text() for n in nurls] + # There is code in e.g. storage_client.py that checks if an + # announcement has changed. Since NURL order isn't meaningful, + # we don't want a change in the order to count as a change, so we + # send the NURLs as a set. CBOR supports sets, as does Foolscap. + announcement[storage_client.ANONYMOUS_STORAGE_NURLS] = {n.to_text() for n in nurls} announcement["anonymous-storage-FURL"] = furl enabled_storage_servers = self._enable_storage_servers( From 8e51643ed34bc1a8161daafbc055470aed87e9d5 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 19 Jul 2023 13:03:24 -0400 Subject: [PATCH 1940/2309] Test for upgrading from Foolscap to HTTP. --- src/allmydata/test/test_storage_client.py | 42 +++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/src/allmydata/test/test_storage_client.py b/src/allmydata/test/test_storage_client.py index 0671526ae..0df66a680 100644 --- a/src/allmydata/test/test_storage_client.py +++ b/src/allmydata/test/test_storage_client.py @@ -88,6 +88,7 @@ from allmydata.util import base32, yamlutil from allmydata.storage_client import ( IFoolscapStorageServer, NativeStorageServer, + HTTPNativeStorageServer, StorageFarmBroker, _FoolscapStorage, _NullStorage, @@ -625,6 +626,47 @@ storage: self.assertIdentical(s2, s) self.assertEqual(s2.get_permutation_seed(), permseed) + def test_upgrade_from_foolscap_to_http(self): + """ + When an announcement is initially Foolscap but then switches to HTTP, + HTTP is used, assuming HTTP is enabled. + """ + tub_maker = lambda _: new_tub() + config = config_from_string( + "/dev/null", "", "[client]\nforce_foolscap = False\n" + ) + broker = StorageFarmBroker(True, tub_maker, config) + broker.startService() + key_s = b'v0-1234-1' + + ones = str(base32.b2a(b"1"), "utf-8") + initial_announcement = { + "service-name": "storage", + "anonymous-storage-FURL": f"pb://{ones}@nowhere/fake2", + "permutation-seed-base32": "bbbbbbbbbbbbbbbbbbbbbbbb", + } + broker._got_announcement(key_s, initial_announcement) + initial_service = broker.servers[key_s] + self.assertIsInstance(initial_service, NativeStorageServer) + self.assertTrue(initial_service.running) + self.assertIdentical(initial_service.parent, broker) + + http_announcement = { + "service-name": "storage", + "anonymous-storage-FURL": f"pb://{ones}@nowhere/fake2", + "permutation-seed-base32": "bbbbbbbbbbbbbbbbbbbbbbbb", + ANONYMOUS_STORAGE_NURLS: [f"pb://{ones}@nowhere/fake2#v=1"], + } + broker._got_announcement(key_s, http_announcement) + self.assertFalse(initial_service.running) + self.assertEqual(initial_service.parent, None) + new_service = broker.servers[key_s] + self.assertIsInstance(new_service, HTTPNativeStorageServer) + self.assertTrue(new_service.running) + self.assertIdentical(new_service.parent, broker) + + return broker.stopService() + def test_static_permutation_seed_pubkey(self): broker = make_broker() server_id = b"v0-4uazse3xb6uu5qpkb7tel2bm6bpea4jhuigdhqcuvvse7hugtsia" From 0431c69cb84bf3021c61f324f1b44d188987d8d8 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 19 Jul 2023 13:03:43 -0400 Subject: [PATCH 1941/2309] News file. --- newsfragments/4047.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/4047.minor diff --git a/newsfragments/4047.minor b/newsfragments/4047.minor new file mode 100644 index 000000000..e69de29bb From 93f2a7a717f7025ad2f8895221b49a3837745732 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 19 Jul 2023 13:51:47 -0400 Subject: [PATCH 1942/2309] refer to non-duplicate ticket --- nix/python-overrides.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nix/python-overrides.nix b/nix/python-overrides.nix index 74a74b278..006c2682d 100644 --- a/nix/python-overrides.nix +++ b/nix/python-overrides.nix @@ -57,7 +57,7 @@ in { # With our customized package set a Twisted unit test fails. Patch the # Twisted test suite to skip that test. - # Filed upstream at https://github.com/twisted/twisted/issues/11892 + # Filed upstream at https://github.com/twisted/twisted/issues/11877 twisted = super.twisted.overrideAttrs (old: { patches = (old.patches or []) ++ [ ./twisted.patch ]; }); From b0397d3d089bf815fb2f5345fe77ff6feb16f8f5 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 19 Jul 2023 13:57:55 -0400 Subject: [PATCH 1943/2309] Replace default.nix with a compatibility shim This also means we drop our niv dependency --- default.nix | 40 ++++------- flake.lock | 17 +++++ flake.nix | 6 ++ nix/sources.json | 38 ----------- nix/sources.nix | 174 ----------------------------------------------- 5 files changed, 35 insertions(+), 240 deletions(-) delete mode 100644 nix/sources.json delete mode 100644 nix/sources.nix diff --git a/default.nix b/default.nix index 196db4030..1b9368b5f 100644 --- a/default.nix +++ b/default.nix @@ -1,28 +1,12 @@ -let - # sources.nix contains information about which versions of some of our - # dependencies we should use. since we use it to pin nixpkgs, all the rest - # of our dependencies are *also* pinned - indirectly. - # - # sources.nix is managed using a tool called `niv`. as an example, to - # update to the most recent version of nixpkgs from the 21.11 maintenance - # release, in the top-level tahoe-lafs checkout directory you run: - # - # niv update nixpkgs-21.11 - # - # niv also supports chosing a specific revision, following a different - # branch, etc. find complete documentation for the tool at - # https://github.com/nmattia/niv - sources = import nix/sources.nix; -in -{ - pkgsVersion ? "nixpkgs-22.11" # a string which chooses a nixpkgs from the - # niv-managed sources data - -, pkgs ? import sources.${pkgsVersion} { # nixpkgs itself - overlays = [ (import ./nix/overlay.nix) ]; -} - -, pythonVersion ? "python310" # a string choosing the python derivation from - # nixpkgs to target -}: -pkgs.${pythonVersion}.withPackages (ps: [ ps.tahoe-lafs ]) +# This is the flake-compat glue code. It loads the flake and gives us its +# outputs. +(import + ( + let lock = builtins.fromJSON (builtins.readFile ./flake.lock); in + fetchTarball { + url = "https://github.com/edolstra/flake-compat/archive/${lock.nodes.flake-compat.locked.rev}.tar.gz"; + sha256 = lock.nodes.flake-compat.locked.narHash; + } + ) + { src = ./.; } +).defaultNix.default diff --git a/flake.lock b/flake.lock index 17270a7d0..4d3764e4d 100644 --- a/flake.lock +++ b/flake.lock @@ -1,5 +1,21 @@ { "nodes": { + "flake-compat": { + "flake": false, + "locked": { + "lastModified": 1673956053, + "narHash": "sha256-4gtG9iQuiKITOjNQQeQIpoIB6b16fm+504Ch3sNKLd8=", + "owner": "edolstra", + "repo": "flake-compat", + "rev": "35bb57c0c8d8b62bbfd284272c928ceb64ddbde9", + "type": "github" + }, + "original": { + "owner": "edolstra", + "repo": "flake-compat", + "type": "github" + } + }, "flake-utils": { "inputs": { "systems": "systems" @@ -68,6 +84,7 @@ }, "root": { "inputs": { + "flake-compat": "flake-compat", "flake-utils": "flake-utils", "nixpkgs": [ "nixpkgs-unstable" diff --git a/flake.nix b/flake.nix index 48dbfe7eb..84a31da9c 100644 --- a/flake.nix +++ b/flake.nix @@ -26,6 +26,12 @@ flake-utils = { url = github:numtide/flake-utils; }; + + # And get a helper that lets us easily continue to provide a default.nix. + flake-compat = { + url = "github:edolstra/flake-compat"; + flake = false; + }; }; outputs = { self, nixpkgs, flake-utils, ... }: diff --git a/nix/sources.json b/nix/sources.json deleted file mode 100644 index ddf05d39d..000000000 --- a/nix/sources.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "niv": { - "branch": "master", - "description": "Easy dependency management for Nix projects", - "homepage": "https://github.com/nmattia/niv", - "owner": "nmattia", - "repo": "niv", - "rev": "5830a4dd348d77e39a0f3c4c762ff2663b602d4c", - "sha256": "1d3lsrqvci4qz2hwjrcnd8h5vfkg8aypq3sjd4g3izbc8frwz5sm", - "type": "tarball", - "url": "https://github.com/nmattia/niv/archive/5830a4dd348d77e39a0f3c4c762ff2663b602d4c.tar.gz", - "url_template": "https://github.com///archive/.tar.gz" - }, - "nixpkgs-22.11": { - "branch": "nixos-22.11", - "description": "Nix Packages collection", - "homepage": "", - "owner": "nixos", - "repo": "nixpkgs", - "rev": "970402e6147c49603f4d06defe44d27fe51884ce", - "sha256": "1v0ljy7wqq14ad3gd1871fgvd4psr7dy14q724k0wwgxk7inbbwh", - "type": "tarball", - "url": "https://github.com/nixos/nixpkgs/archive/970402e6147c49603f4d06defe44d27fe51884ce.tar.gz", - "url_template": "https://github.com///archive/.tar.gz" - }, - "nixpkgs-unstable": { - "branch": "master", - "description": "Nix Packages collection", - "homepage": "", - "owner": "nixos", - "repo": "nixpkgs", - "rev": "d0c9a536331227ab883b4f6964be638fa436d81f", - "sha256": "1gg6v5rk1p26ciygdg262zc5vqws753rvgcma5rim2s6gyfrjaq1", - "type": "tarball", - "url": "https://github.com/nixos/nixpkgs/archive/d0c9a536331227ab883b4f6964be638fa436d81f.tar.gz", - "url_template": "https://github.com///archive/.tar.gz" - } -} diff --git a/nix/sources.nix b/nix/sources.nix deleted file mode 100644 index 1938409dd..000000000 --- a/nix/sources.nix +++ /dev/null @@ -1,174 +0,0 @@ -# This file has been generated by Niv. - -let - - # - # The fetchers. fetch_ fetches specs of type . - # - - fetch_file = pkgs: name: spec: - let - name' = sanitizeName name + "-src"; - in - if spec.builtin or true then - builtins_fetchurl { inherit (spec) url sha256; name = name'; } - else - pkgs.fetchurl { inherit (spec) url sha256; name = name'; }; - - fetch_tarball = pkgs: name: spec: - let - name' = sanitizeName name + "-src"; - in - if spec.builtin or true then - builtins_fetchTarball { name = name'; inherit (spec) url sha256; } - else - pkgs.fetchzip { name = name'; inherit (spec) url sha256; }; - - fetch_git = name: spec: - let - ref = - if spec ? ref then spec.ref else - if spec ? branch then "refs/heads/${spec.branch}" else - if spec ? tag then "refs/tags/${spec.tag}" else - abort "In git source '${name}': Please specify `ref`, `tag` or `branch`!"; - in - builtins.fetchGit { url = spec.repo; inherit (spec) rev; inherit ref; }; - - fetch_local = spec: spec.path; - - fetch_builtin-tarball = name: throw - ''[${name}] The niv type "builtin-tarball" is deprecated. You should instead use `builtin = true`. - $ niv modify ${name} -a type=tarball -a builtin=true''; - - fetch_builtin-url = name: throw - ''[${name}] The niv type "builtin-url" will soon be deprecated. You should instead use `builtin = true`. - $ niv modify ${name} -a type=file -a builtin=true''; - - # - # Various helpers - # - - # https://github.com/NixOS/nixpkgs/pull/83241/files#diff-c6f540a4f3bfa4b0e8b6bafd4cd54e8bR695 - sanitizeName = name: - ( - concatMapStrings (s: if builtins.isList s then "-" else s) - ( - builtins.split "[^[:alnum:]+._?=-]+" - ((x: builtins.elemAt (builtins.match "\\.*(.*)" x) 0) name) - ) - ); - - # The set of packages used when specs are fetched using non-builtins. - mkPkgs = sources: system: - let - sourcesNixpkgs = - import (builtins_fetchTarball { inherit (sources.nixpkgs) url sha256; }) { inherit system; }; - hasNixpkgsPath = builtins.any (x: x.prefix == "nixpkgs") builtins.nixPath; - hasThisAsNixpkgsPath = == ./.; - in - if builtins.hasAttr "nixpkgs" sources - then sourcesNixpkgs - else if hasNixpkgsPath && ! hasThisAsNixpkgsPath then - import {} - else - abort - '' - Please specify either (through -I or NIX_PATH=nixpkgs=...) or - add a package called "nixpkgs" to your sources.json. - ''; - - # The actual fetching function. - fetch = pkgs: name: spec: - - if ! builtins.hasAttr "type" spec then - abort "ERROR: niv spec ${name} does not have a 'type' attribute" - else if spec.type == "file" then fetch_file pkgs name spec - else if spec.type == "tarball" then fetch_tarball pkgs name spec - else if spec.type == "git" then fetch_git name spec - else if spec.type == "local" then fetch_local spec - else if spec.type == "builtin-tarball" then fetch_builtin-tarball name - else if spec.type == "builtin-url" then fetch_builtin-url name - else - abort "ERROR: niv spec ${name} has unknown type ${builtins.toJSON spec.type}"; - - # If the environment variable NIV_OVERRIDE_${name} is set, then use - # the path directly as opposed to the fetched source. - replace = name: drv: - let - saneName = stringAsChars (c: if isNull (builtins.match "[a-zA-Z0-9]" c) then "_" else c) name; - ersatz = builtins.getEnv "NIV_OVERRIDE_${saneName}"; - in - if ersatz == "" then drv else - # this turns the string into an actual Nix path (for both absolute and - # relative paths) - if builtins.substring 0 1 ersatz == "/" then /. + ersatz else /. + builtins.getEnv "PWD" + "/${ersatz}"; - - # Ports of functions for older nix versions - - # a Nix version of mapAttrs if the built-in doesn't exist - mapAttrs = builtins.mapAttrs or ( - f: set: with builtins; - listToAttrs (map (attr: { name = attr; value = f attr set.${attr}; }) (attrNames set)) - ); - - # https://github.com/NixOS/nixpkgs/blob/0258808f5744ca980b9a1f24fe0b1e6f0fecee9c/lib/lists.nix#L295 - range = first: last: if first > last then [] else builtins.genList (n: first + n) (last - first + 1); - - # https://github.com/NixOS/nixpkgs/blob/0258808f5744ca980b9a1f24fe0b1e6f0fecee9c/lib/strings.nix#L257 - stringToCharacters = s: map (p: builtins.substring p 1 s) (range 0 (builtins.stringLength s - 1)); - - # https://github.com/NixOS/nixpkgs/blob/0258808f5744ca980b9a1f24fe0b1e6f0fecee9c/lib/strings.nix#L269 - stringAsChars = f: s: concatStrings (map f (stringToCharacters s)); - concatMapStrings = f: list: concatStrings (map f list); - concatStrings = builtins.concatStringsSep ""; - - # https://github.com/NixOS/nixpkgs/blob/8a9f58a375c401b96da862d969f66429def1d118/lib/attrsets.nix#L331 - optionalAttrs = cond: as: if cond then as else {}; - - # fetchTarball version that is compatible between all the versions of Nix - builtins_fetchTarball = { url, name ? null, sha256 }@attrs: - let - inherit (builtins) lessThan nixVersion fetchTarball; - in - if lessThan nixVersion "1.12" then - fetchTarball ({ inherit url; } // (optionalAttrs (!isNull name) { inherit name; })) - else - fetchTarball attrs; - - # fetchurl version that is compatible between all the versions of Nix - builtins_fetchurl = { url, name ? null, sha256 }@attrs: - let - inherit (builtins) lessThan nixVersion fetchurl; - in - if lessThan nixVersion "1.12" then - fetchurl ({ inherit url; } // (optionalAttrs (!isNull name) { inherit name; })) - else - fetchurl attrs; - - # Create the final "sources" from the config - mkSources = config: - mapAttrs ( - name: spec: - if builtins.hasAttr "outPath" spec - then abort - "The values in sources.json should not have an 'outPath' attribute" - else - spec // { outPath = replace name (fetch config.pkgs name spec); } - ) config.sources; - - # The "config" used by the fetchers - mkConfig = - { sourcesFile ? if builtins.pathExists ./sources.json then ./sources.json else null - , sources ? if isNull sourcesFile then {} else builtins.fromJSON (builtins.readFile sourcesFile) - , system ? builtins.currentSystem - , pkgs ? mkPkgs sources system - }: rec { - # The sources, i.e. the attribute set of spec name to spec - inherit sources; - - # The "pkgs" (evaluated nixpkgs) to use for e.g. non-builtin fetchers - inherit pkgs; - }; - -in -mkSources (mkConfig {}) // { __functor = _: settings: mkSources (mkConfig settings); } From d85f8d7caff9f942fc71a463621ed534bd94f040 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 19 Jul 2023 14:27:28 -0400 Subject: [PATCH 1944/2309] some more comments on the flake parts --- flake.nix | 79 ++++++++++++++++++++++++++++++++++++++----------------- 1 file changed, 55 insertions(+), 24 deletions(-) diff --git a/flake.nix b/flake.nix index 84a31da9c..a68600c72 100644 --- a/flake.nix +++ b/flake.nix @@ -46,33 +46,50 @@ } // (flake-utils.lib.eachDefaultSystem (system: let - # First get the package set for this system architecture. + # The package set for this system architecture. pkgs = import nixpkgs { inherit system; # And include our Tahoe-LAFS package in that package set. overlays = [ self.overlays.default ]; }; - # Find out what Python versions we're working with. - pythonVersions = builtins.attrNames ( - pkgs.lib.attrsets.filterAttrs + # pythonVersions :: [string] + # + # The version strings for the Python runtimes we'll work with. + pythonVersions = + let # Match attribute names that look like a Python derivation - CPython # or PyPy. We take care to avoid things like "python-foo" and # "python3Full-unittest" though. We only want things like "pypy38" # or "python311". - (name: _: null != builtins.match "(python|pypy)3[[:digit:]]{0,2}" name) - pkgs - ); + nameMatches = name: null != builtins.match "(python|pypy)3[[:digit:]]{0,2}" name; + # Sometimes an old version is left in the package set as an error + # saying something like "we remove this". Make sure we whatever we + # found by name evaluates without error, too. + notError = drv: (builtins.tryEval drv).success; + in + # Discover all of the Python runtime derivations by inspecting names + # and filtering out derivations with errors. + builtins.attrNames ( + pkgs.lib.attrsets.filterAttrs + (name: drv: nameMatches name && notError drv) + pkgs + ); + + # defaultPyVersion :: string + # # An element of pythonVersions which we'll use for the default package. - defaultPyVersion = "python310"; + defaultPyVersion = "python3"; + # pythons :: [derivation] + # # Retrieve the actual Python package for each configured version. We # already applied our overlay to pkgs so our packages will already be # available. pythons = builtins.map (pyVer: pkgs.${pyVer}) pythonVersions; - # string -> string + # packageName :: string -> string # # Construct the Tahoe-LAFS package name for the given Python runtime. packageName = pyVersion: "${pyVersion}-tahoe-lafs"; @@ -87,6 +104,8 @@ # Make a singleton attribute set from the result of two functions. singletonOf = f: g: x: { ${f x} = g x; }; + # makeRuntimeEnv :: string -> derivation + # # Create a derivation that includes a Python runtime, Tahoe-LAFS, and # all of its dependencies. makeRuntimeEnv = singletonOf packageName makeRuntimeEnv'; @@ -98,6 +117,8 @@ name = packageName pyVersion; }); + # makeTestEnv :: string -> derivation + # # Create a derivation that includes a Python runtime, Tahoe-LAFS, and # all of its dependencies. makeTestEnv = pyVersion: (pkgs.${pyVersion}.withPackages (ps: with ps; @@ -109,13 +130,14 @@ name = packageName pyVersion; }); in { + # A package set with out overlay on it. legacyPackages = pkgs; - # Define the flake's package outputs. We'll define one version of the - # package for each version of Python we could find. We'll also point - # the flake's "default" package at one of these somewhat arbitrarily. - # The package consists of a Python environment with Tahoe-LAFS available - # to it. + # The flake's package outputs. We'll define one version of the package + # for each version of Python we could find. We'll also point the + # flake's "default" package at the derivation corresponding to the + # default Python version we defined above. The package consists of a + # Python environment with Tahoe-LAFS available to it. packages = with pkgs.lib; foldr mergeAttrs {} ([ { default = self.packages.${system}.${packageName defaultPyVersion}; } @@ -123,22 +145,28 @@ ++ (builtins.map (singletonOf unitTestName makeTestEnv) pythonVersions) ); - # Define the flake's app outputs. We'll define a version of an app for - # running the test suite for each version of Python we could find. - # We'll also define a version of an app for running the "tahoe" - # command-line entrypoint for each version of Python we could find. + # The flake's app outputs. We'll define a version of an app for running + # the test suite for each version of Python we could find. We'll also + # define a version of an app for running the "tahoe" command-line + # entrypoint for each version of Python we could find. apps = let + # writeScript :: string -> string -> path + # + # Write a shell program to a file so it can be run later. + # # We avoid writeShellApplication here because it has ghc as a # dependency but ghc has Python as a dependency and our Python - # package override triggers a rebuild of ghc which takes a looong - # time. + # package override triggers a rebuild of ghc and many Haskell + # packages which takes a looong time. writeScript = name: text: let script = pkgs.writeShellScript name text; in "${script}"; - # A helper function to define the runtime entrypoint for a certain - # Python runtime. + # makeTahoeApp :: string -> attrset + # + # A helper function to define the Tahoe-LAFS runtime entrypoint for + # a certain Python runtime. makeTahoeApp = pyVersion: { "tahoe-${pyVersion}" = { type = "app"; @@ -150,8 +178,10 @@ }; }; - # A helper function to define the unit test entrypoint for a certain - # Python runtime. + # makeUnitTestsApp :: string -> attrset + # + # A helper function to define the Tahoe-LAFS unit test entrypoint + # for a certain Python runtime. makeUnitTestsApp = pyVersion: { "${unitTestName pyVersion}" = { type = "app"; @@ -165,6 +195,7 @@ }; in with pkgs.lib; + # Merge a default app definition with the rest of the apps. foldr mergeAttrs { default = self.apps.${system}."tahoe-python3"; } ( From ddfd95faff51cd121a3688e6d4e98e0ff559976d Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 19 Jul 2023 14:33:49 -0400 Subject: [PATCH 1945/2309] comment tweak --- .circleci/config.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index d8a70b2c4..ae7288509 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -564,10 +564,15 @@ commands: # Get cachix for Nix-friendly caching. name: "Install Basic Dependencies" command: | + # Get some build environment dependencies and let them float on a + # certain release branch. These aren't involved in the actual + # package build (only in CI environment setup) so the fact that + # they float shouldn't hurt reproducibility. NIXPKGS="nixpkgs/nixos-23.05" nix profile install $NIXPKGS#cachix $NIXPKGS#bash $NIXPKGS#jp - # Activate it for "binary substitution". This sets up - # configuration tht lets Nix download something from the cache + + # Activate our cachix cache for "binary substitution". This sets + # up configuration tht lets Nix download something from the cache # instead of building it locally, if possible. cachix use "${CACHIX_NAME}" From a3f50aa481b4c2187b0327e1998572d593800b7b Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 19 Jul 2023 17:29:50 -0400 Subject: [PATCH 1946/2309] bump to the newer nixpkgs branch --- flake.lock | 8 ++++---- flake.nix | 6 +++++- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/flake.lock b/flake.lock index 4d3764e4d..2b955d4d6 100644 --- a/flake.lock +++ b/flake.lock @@ -68,16 +68,16 @@ }, "nixpkgs-unstable": { "locked": { - "lastModified": 1687349636, - "narHash": "sha256-wpWWNoKJ6z8Nt9egpeiKzsCgkkDO2SO4g6ab9SxgvpM=", + "lastModified": 1689791806, + "narHash": "sha256-QpXjfiyBFwa7MV/J6nM5FoBreks9O7j9cAZxV22MR8A=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "49b7c90c06e557e7473ef467f40d98e7c368d29f", + "rev": "439ba0789ff84dddea64eb2d47a4a0d4887dbb1f", "type": "github" }, "original": { "owner": "NixOS", - "ref": "pull/238965/head", + "ref": "pull/244135/head", "repo": "nixpkgs", "type": "github" } diff --git a/flake.nix b/flake.nix index a68600c72..0dfa3d03a 100644 --- a/flake.nix +++ b/flake.nix @@ -12,8 +12,12 @@ "nixpkgs-23_05" = { url = github:NixOS/nixpkgs?ref=release-23.05; }; + + # We depend on a very new python-cryptography which is not yet available + # from any release branch of nixpkgs. However, it is contained in a PR + # currently up for review. Point our nixpkgs at that for now. "nixpkgs-unstable" = { - url = github:NixOS/nixpkgs?ref=pull/238965/head; + url = github:NixOS/nixpkgs?ref=pull/244135/head; }; # Point the default nixpkgs at one of those. This avoids having getting a From 44502b8620e8cf4d04bf6851a844cac6119cac13 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 19 Jul 2023 18:09:10 -0400 Subject: [PATCH 1947/2309] remove unused import --- src/allmydata/client.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/allmydata/client.py b/src/allmydata/client.py index 13909d5ca..a1501f1ef 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -7,7 +7,6 @@ import os import stat import time import weakref -import tempfile from typing import Optional, Iterable from base64 import urlsafe_b64encode from functools import partial From 08e364bbab2dad609d14e5a4c83e9d24d30baf32 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 20 Jul 2023 09:06:05 -0400 Subject: [PATCH 1948/2309] numpy is not supported on python38 anymore --- .circleci/config.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ae7288509..0c831af04 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -93,7 +93,6 @@ workflows: matrix: parameters: pythonVersion: - - "python38" - "python39" - "python310" - "python311" From 90e08314c21877989a3aba934f141a495dd94481 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 20 Jul 2023 10:58:10 -0400 Subject: [PATCH 1949/2309] try to shed root privileges We have root on CircleCI in the docker container. We can't currently shed them before we get inside the flake app because we can't run `nix build` as non-root inside the nix container. :/ https://github.com/nix-community/docker-nixpkgs/issues/62 --- flake.nix | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index 0dfa3d03a..53870eac2 100644 --- a/flake.nix +++ b/flake.nix @@ -193,7 +193,14 @@ writeScript "unit-tests" '' export TAHOE_LAFS_HYPOTHESIS_PROFILE=ci - ${makeTestEnv pyVersion}/bin/python -m twisted.trial "$@" + if [ $(id -u) = "0" ]; then + # The test suite assumes non-root permissions. Get rid + # of the root permissions we seem to have. + SUDO="sudo -u nobody" + else + SUDO="" + fi + $SUDO ${makeTestEnv pyVersion}/bin/python -m twisted.trial "$@" ''; }; }; From baadf1fad4fa5078845118890efa030aba813a06 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 20 Jul 2023 11:12:10 -0400 Subject: [PATCH 1950/2309] try su to get rid of root sudo fails because it isn't setuid root... I don't know why su would be, but maybe it is. --- flake.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index 53870eac2..329363d6f 100644 --- a/flake.nix +++ b/flake.nix @@ -196,7 +196,7 @@ if [ $(id -u) = "0" ]; then # The test suite assumes non-root permissions. Get rid # of the root permissions we seem to have. - SUDO="sudo -u nobody" + SUDO="${pkgs.su}/bin/su --shell /bin/sh - nobody --" else SUDO="" fi From 9585925627bbd66cc788d7a19b0dd0db425cc39f Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 20 Jul 2023 11:21:08 -0400 Subject: [PATCH 1951/2309] abandon user-switching effort su fails with "su: pam_start: error 26" --- flake.nix | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/flake.nix b/flake.nix index 329363d6f..0dfa3d03a 100644 --- a/flake.nix +++ b/flake.nix @@ -193,14 +193,7 @@ writeScript "unit-tests" '' export TAHOE_LAFS_HYPOTHESIS_PROFILE=ci - if [ $(id -u) = "0" ]; then - # The test suite assumes non-root permissions. Get rid - # of the root permissions we seem to have. - SUDO="${pkgs.su}/bin/su --shell /bin/sh - nobody --" - else - SUDO="" - fi - $SUDO ${makeTestEnv pyVersion}/bin/python -m twisted.trial "$@" + ${makeTestEnv pyVersion}/bin/python -m twisted.trial "$@" ''; }; }; From 2091b7ee86fabc5600d2afeb10fca73f9685ed51 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 20 Jul 2023 11:36:30 -0400 Subject: [PATCH 1952/2309] skip permission-related tests if the environment is not suitable posix superuser can do anything on the filesystem --- src/allmydata/test/cli/test_grid_manager.py | 3 ++- src/allmydata/test/test_client.py | 25 +++++++------------- src/allmydata/test/test_node.py | 26 ++++++--------------- 3 files changed, 17 insertions(+), 37 deletions(-) diff --git a/src/allmydata/test/cli/test_grid_manager.py b/src/allmydata/test/cli/test_grid_manager.py index 2ed738544..970315af7 100644 --- a/src/allmydata/test/cli/test_grid_manager.py +++ b/src/allmydata/test/cli/test_grid_manager.py @@ -222,7 +222,8 @@ class GridManagerCommandLine(TestCase): result.output, ) - @skipIf(not platform.isLinux(), "I only know how permissions work on linux") + @skipIf(platform.isWindows(), "We don't know how to set permissions on Windows.") + @skipIf(os.getuid() == 0, "cannot test as superuser which all of the permissions") def test_sign_bad_perms(self): """ Error reported if we can't create certificate file diff --git a/src/allmydata/test/test_client.py b/src/allmydata/test/test_client.py index 04d8e6160..848f98a13 100644 --- a/src/allmydata/test/test_client.py +++ b/src/allmydata/test/test_client.py @@ -1,17 +1,7 @@ -""" -Ported to Python 3. -""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals - -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 +from __future__ import annotations import os -import sys +from unittest import skipIf from functools import ( partial, ) @@ -42,6 +32,7 @@ from twisted.internet import defer from twisted.python.filepath import ( FilePath, ) +from twisted.python.runtime import platform from testtools.matchers import ( Equals, AfterPreprocessing, @@ -156,12 +147,12 @@ class Basic(testutil.ReallyEqualMixin, unittest.TestCase): yield client.create_client(basedir) self.assertIn("[client]helper.furl", str(ctx.exception)) + # if somebody knows a clever way to do this (cause + # EnvironmentError when reading a file that really exists), on + # windows, please fix this + @skipIf(platform.isWindows(), "We don't know how to set permissions on Windows.") + @skipIf(os.getuid() == 0, "cannot test as superuser which all of the permissions") def test_unreadable_config(self): - if sys.platform == "win32": - # if somebody knows a clever way to do this (cause - # EnvironmentError when reading a file that really exists), on - # windows, please fix this - raise unittest.SkipTest("can't make unreadable files on windows") basedir = "test_client.Basic.test_unreadable_config" os.mkdir(basedir) fn = os.path.join(basedir, "tahoe.cfg") diff --git a/src/allmydata/test/test_node.py b/src/allmydata/test/test_node.py index 0b371569a..7d6b09d12 100644 --- a/src/allmydata/test/test_node.py +++ b/src/allmydata/test/test_node.py @@ -1,14 +1,4 @@ -""" -Ported to Python 3. -""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals - -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 +from __future__ import annotations import base64 import os @@ -31,6 +21,7 @@ from unittest import skipIf from twisted.python.filepath import ( FilePath, ) +from twisted.python.runtime import platform from twisted.trial import unittest from twisted.internet import defer @@ -333,10 +324,8 @@ class TestCase(testutil.SignalMixin, unittest.TestCase): default = [("hello", "world")] self.assertEqual(config.items("nosuch", default), default) - @skipIf( - "win32" in sys.platform.lower() or "cygwin" in sys.platform.lower(), - "We don't know how to set permissions on Windows.", - ) + @skipIf(platform.isWindows(), "We don't know how to set permissions on Windows.") + @skipIf(os.getuid() == 0, "cannot test as superuser which all of the permissions") def test_private_config_unreadable(self): """ Asking for inaccessible private config is an error @@ -351,10 +340,8 @@ class TestCase(testutil.SignalMixin, unittest.TestCase): with self.assertRaises(Exception): config.get_or_create_private_config("foo") - @skipIf( - "win32" in sys.platform.lower() or "cygwin" in sys.platform.lower(), - "We don't know how to set permissions on Windows.", - ) + @skipIf(platform.isWindows(), "We don't know how to set permissions on Windows.") + @skipIf(os.getuid() == 0, "cannot test as superuser which all of the permissions") def test_private_config_unreadable_preexisting(self): """ error if reading private config data fails @@ -411,6 +398,7 @@ class TestCase(testutil.SignalMixin, unittest.TestCase): self.assertEqual(len(counter), 1) # don't call unless necessary self.assertEqual(value, "newer") + @skipIf(os.getuid() == 0, "cannot test as superuser which all of the permissions") def test_write_config_unwritable_file(self): """ Existing behavior merely logs any errors upon writing From 3e18301f86b00fe07f56c025610c3b5fa11d600f Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 20 Jul 2023 11:54:19 -0400 Subject: [PATCH 1953/2309] Try to get the version right --- flake.nix | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/flake.nix b/flake.nix index 0dfa3d03a..cc3eed5a1 100644 --- a/flake.nix +++ b/flake.nix @@ -190,10 +190,14 @@ "${unitTestName pyVersion}" = { type = "app"; program = - writeScript "unit-tests" - '' + let + py = makeTestEnv pyVersion; + in + writeScript "unit-tests" + '' + ${py} setup.py update_version export TAHOE_LAFS_HYPOTHESIS_PROFILE=ci - ${makeTestEnv pyVersion}/bin/python -m twisted.trial "$@" + ${py}/bin/python -m twisted.trial "$@" ''; }; }; From d82ade538ce4230f888ed2b0e28942a1a8ba11c7 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 20 Jul 2023 12:42:26 -0400 Subject: [PATCH 1954/2309] Use the working tree as the source of allmydata package --- flake.nix | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/flake.nix b/flake.nix index cc3eed5a1..17e142184 100644 --- a/flake.nix +++ b/flake.nix @@ -123,8 +123,9 @@ # makeTestEnv :: string -> derivation # - # Create a derivation that includes a Python runtime, Tahoe-LAFS, and - # all of its dependencies. + # Create a derivation that includes a Python runtime and all of the + # Tahoe-LAFS dependencies, but not Tahoe-LAFS itself, which we'll get + # from the working directory. makeTestEnv = pyVersion: (pkgs.${pyVersion}.withPackages (ps: with ps; [ tahoe-lafs ] ++ tahoe-lafs.passthru.extras.i2p ++ @@ -191,13 +192,14 @@ type = "app"; program = let - py = makeTestEnv pyVersion; + python = "${makeTestEnv pyVersion}/bin/python"; in writeScript "unit-tests" '' - ${py} setup.py update_version + ${python} setup.py update_version export TAHOE_LAFS_HYPOTHESIS_PROFILE=ci - ${py}/bin/python -m twisted.trial "$@" + export PYTHONPATH=$PWD/src + ${python} -m twisted.trial "$@" ''; }; }; From 2c40185ef682400d107a721177877e18040e2318 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 20 Jul 2023 14:00:55 -0400 Subject: [PATCH 1955/2309] slight simplification --- flake.nix | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/flake.nix b/flake.nix index 17e142184..791015988 100644 --- a/flake.nix +++ b/flake.nix @@ -164,9 +164,7 @@ # dependency but ghc has Python as a dependency and our Python # package override triggers a rebuild of ghc and many Haskell # packages which takes a looong time. - writeScript = name: text: - let script = pkgs.writeShellScript name text; - in "${script}"; + writeScript = name: text: "${pkgs.writeShellScript name text}"; # makeTahoeApp :: string -> attrset # From 57facc6335ae2787998890320569ce2c51a7e2bf Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 20 Jul 2023 14:19:12 -0400 Subject: [PATCH 1956/2309] narrow the type of cli_config a bit This has unfortunate interactions with the "stdout" attribute but I'm punting on that. --- src/allmydata/listeners.py | 7 ++++--- src/allmydata/util/i2p_provider.py | 7 +++++-- src/allmydata/util/tor_provider.py | 7 +++++-- 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/src/allmydata/listeners.py b/src/allmydata/listeners.py index 93eecf09f..6af19d1b9 100644 --- a/src/allmydata/listeners.py +++ b/src/allmydata/listeners.py @@ -10,6 +10,7 @@ from typing import Any, Protocol, Sequence, Mapping, Optional, Union, Awaitable from typing_extensions import Literal from attrs import frozen +from twisted.python.usage import Options from .interfaces import IAddressFamily from .util.iputil import allocate_tcp_port @@ -45,7 +46,7 @@ class Listener(Protocol): node's public internet address from peers? """ - async def create_config(self, reactor: Any, cli_config: Any) -> Optional[ListenerConfig]: + async def create_config(self, reactor: Any, cli_config: Options) -> Optional[ListenerConfig]: """ Set up an instance of this listener according to the given configuration parameters. @@ -75,7 +76,7 @@ class TCPProvider: def can_hide_ip(self) -> Literal[False]: return False - async def create_config(self, reactor: Any, cli_config: Any) -> ListenerConfig: + async def create_config(self, reactor: Any, cli_config: Options) -> ListenerConfig: tub_ports = [] tub_locations = [] if cli_config["port"]: # --port/--location are a pair @@ -110,7 +111,7 @@ class StaticProvider: def can_hide_ip(self) -> bool: return self._hide_ip - async def create_config(self, reactor: Any, cli_config: Any) -> Optional[ListenerConfig]: + async def create_config(self, reactor: Any, cli_config: Options) -> Optional[ListenerConfig]: if self._config is None or isinstance(self._config, ListenerConfig): return self._config return await self._config diff --git a/src/allmydata/util/i2p_provider.py b/src/allmydata/util/i2p_provider.py index 996237873..e9188c6ae 100644 --- a/src/allmydata/util/i2p_provider.py +++ b/src/allmydata/util/i2p_provider.py @@ -15,6 +15,7 @@ from twisted.internet.defer import inlineCallbacks, returnValue from twisted.internet.endpoints import clientFromString from twisted.internet.error import ConnectionRefusedError, ConnectError from twisted.application import service +from twisted.python.usage import Options from ..listeners import ListenerConfig from ..interfaces import ( @@ -108,7 +109,7 @@ def _connect_to_i2p(reactor, cli_config, txi2p): else: raise ValueError("unable to reach any default I2P SAM port") -async def create_config(reactor: Any, cli_config: Any) -> ListenerConfig: +async def create_config(reactor: Any, cli_config: Options) -> ListenerConfig: """ For a given set of command-line options, construct an I2P listener. @@ -120,7 +121,9 @@ async def create_config(reactor: Any, cli_config: Any) -> ListenerConfig: "Please 'pip install tahoe-lafs[i2p]' to fix this.") tahoe_config_i2p = [] # written into tahoe.cfg:[i2p] private_dir = os.path.abspath(os.path.join(cli_config["basedir"], "private")) - stdout = cli_config.stdout + # XXX We shouldn't carry stdout around by jamming it into the Options + # value. See https://tahoe-lafs.org/trac/tahoe-lafs/ticket/4048 + stdout = cli_config.stdout # type: ignore[attr-defined] if cli_config["i2p-launch"]: raise NotImplementedError("--i2p-launch is under development.") else: diff --git a/src/allmydata/util/tor_provider.py b/src/allmydata/util/tor_provider.py index 0e3090578..c40e65f42 100644 --- a/src/allmydata/util/tor_provider.py +++ b/src/allmydata/util/tor_provider.py @@ -13,6 +13,7 @@ from twisted.internet.defer import inlineCallbacks, returnValue from twisted.internet.endpoints import clientFromString, TCP4ServerEndpoint from twisted.internet.error import ConnectionRefusedError, ConnectError from twisted.application import service +from twisted.python.usage import Options from .observer import OneShotObserverList from .iputil import allocate_tcp_port @@ -154,14 +155,16 @@ def _connect_to_tor(reactor, cli_config, txtorcon): else: raise ValueError("unable to reach any default Tor control port") -async def create_config(reactor: Any, cli_config: Any) -> ListenerConfig: +async def create_config(reactor: Any, cli_config: Options) -> ListenerConfig: txtorcon = _import_txtorcon() if not txtorcon: raise ValueError("Cannot create onion without txtorcon. " "Please 'pip install tahoe-lafs[tor]' to fix this.") tahoe_config_tor = [] # written into tahoe.cfg:[tor] private_dir = os.path.abspath(os.path.join(cli_config["basedir"], "private")) - stdout = cli_config.stdout + # XXX We shouldn't carry stdout around by jamming it into the Options + # value. See https://tahoe-lafs.org/trac/tahoe-lafs/ticket/4048 + stdout = cli_config.stdout # type: ignore[attr-defined] if cli_config["tor-launch"]: tahoe_config_tor.append(("launch", "true")) tor_executable = cli_config["tor-executable"] From 024b5e428a84c3a78eda4b163bb47693224b6067 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 20 Jul 2023 14:23:31 -0400 Subject: [PATCH 1957/2309] narrow the type annotation for another Listener method param --- src/allmydata/listeners.py | 7 ++++--- src/allmydata/util/i2p_provider.py | 3 ++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/allmydata/listeners.py b/src/allmydata/listeners.py index 6af19d1b9..f97f699b4 100644 --- a/src/allmydata/listeners.py +++ b/src/allmydata/listeners.py @@ -14,6 +14,7 @@ from twisted.python.usage import Options from .interfaces import IAddressFamily from .util.iputil import allocate_tcp_port +from .node import _Config @frozen class ListenerConfig: @@ -57,7 +58,7 @@ class Listener(Protocol): overall *tahoe.cfg* configuration file. """ - def create(self, reactor: Any, config: Any) -> IAddressFamily: + def create(self, reactor: Any, config: _Config) -> IAddressFamily: """ Instantiate this listener according to the given previously-generated configuration. @@ -91,7 +92,7 @@ class TCPProvider: return ListenerConfig(tub_ports, tub_locations, {}) - def create(self, reactor: Any, config: Any) -> IAddressFamily: + def create(self, reactor: Any, config: _Config) -> IAddressFamily: raise NotImplementedError() @@ -116,5 +117,5 @@ class StaticProvider: return self._config return await self._config - def create(self, reactor: Any, config: Any) -> IAddressFamily: + def create(self, reactor: Any, config: _Config) -> IAddressFamily: return self._address diff --git a/src/allmydata/util/i2p_provider.py b/src/allmydata/util/i2p_provider.py index e9188c6ae..c480cd2f1 100644 --- a/src/allmydata/util/i2p_provider.py +++ b/src/allmydata/util/i2p_provider.py @@ -21,8 +21,9 @@ from ..listeners import ListenerConfig from ..interfaces import ( IAddressFamily, ) +from ..node import _Config -def create(reactor: Any, config: Any) -> IAddressFamily: +def create(reactor: Any, config: _Config) -> IAddressFamily: """ Create a new Provider service (this is an IService so must be hooked up to a parent or otherwise started). From 4713573621470dc0043587699b63ccf788bf1a40 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 20 Jul 2023 14:27:30 -0400 Subject: [PATCH 1958/2309] test for dictutil.filter --- src/allmydata/test/test_dictutil.py | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/src/allmydata/test/test_dictutil.py b/src/allmydata/test/test_dictutil.py index 7e26a6ed9..ce1c4a74c 100644 --- a/src/allmydata/test/test_dictutil.py +++ b/src/allmydata/test/test_dictutil.py @@ -1,17 +1,9 @@ """ Tests for allmydata.util.dictutil. - -Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals +from __future__ import annotations from future.utils import PY2, PY3 -if PY2: - # dict omitted to match dictutil.py. - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, list, object, range, str, max, min # noqa: F401 from unittest import skipIf @@ -168,3 +160,18 @@ class TypedKeyDictPython2(unittest.TestCase): # Demonstration of how bytes and unicode can be mixed: d = {u"abc": 1} self.assertEqual(d[b"abc"], 1) + + +class FilterTests(unittest.TestCase): + """ + Tests for ``dictutil.filter``. + """ + def test_filter(self) -> None: + """ + ``dictutil.filter`` returns a ``dict`` that contains the key/value + pairs for which the value is matched by the given predicate. + """ + self.assertEqual( + {1: 2}, + dictutil.filter(lambda v: v == 2, {1: 2, 2: 3}), + ) From da43acf52e25bad8c3158f9b6f1d56672d1ea5dd Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 20 Jul 2023 14:27:50 -0400 Subject: [PATCH 1959/2309] more accurate docstring for dictutil.filter --- src/allmydata/util/dictutil.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/util/dictutil.py b/src/allmydata/util/dictutil.py index 277fc30e9..58820993f 100644 --- a/src/allmydata/util/dictutil.py +++ b/src/allmydata/util/dictutil.py @@ -10,7 +10,7 @@ V = TypeVar("V") def filter(pred: Callable[[V], bool], orig: dict[K, V]) -> dict[K, V]: """ - Filter out key/value pairs that fail to match a predicate. + Filter out key/value pairs whose value fails to match a predicate. """ return { k: v From 02a696d73beec90e8d1d1071016440ee0d3e6d8a Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 21 Jul 2023 08:13:33 -0400 Subject: [PATCH 1960/2309] Make `merge_config` fail on overlapping configs This isn't expected to happen. If it does it would be nice to see it instead of silently continue working with some config dropped on the floor. --- src/allmydata/scripts/create_node.py | 7 ++++ src/allmydata/test/cli/test_create.py | 54 +++++++++++++++++++++++++++ 2 files changed, 61 insertions(+) diff --git a/src/allmydata/scripts/create_node.py b/src/allmydata/scripts/create_node.py index 0cd5c577b..4357abb49 100644 --- a/src/allmydata/scripts/create_node.py +++ b/src/allmydata/scripts/create_node.py @@ -273,9 +273,16 @@ def merge_config( If either is ``None`` then the result is ``None``. This supports the "disable listeners" functionality. + + :raise ValueError: If the keys in the node configs overlap. """ if left is None or right is None: return None + + overlap = set(left.node_config) & set(right.node_config) + if overlap: + raise ValueError(f"Node configs overlap: {overlap}") + return ListenerConfig( list(left.tub_ports) + list(right.tub_ports), list(left.tub_locations) + list(right.tub_locations), diff --git a/src/allmydata/test/cli/test_create.py b/src/allmydata/test/cli/test_create.py index c6caf3395..bc6aab760 100644 --- a/src/allmydata/test/cli/test_create.py +++ b/src/allmydata/test/cli/test_create.py @@ -25,6 +25,60 @@ def read_config(basedir): config = configutil.get_config(tahoe_cfg) return config +class MergeConfigTests(unittest.TestCase): + """ + Tests for ``create_node.merge_config``. + """ + def test_disable_left(self) -> None: + """ + If the left argument to ``create_node.merge_config`` is ``None`` + then the return value is ``None``. + """ + conf = ListenerConfig([], [], {}) + self.assertEqual(None, create_node.merge_config(None, conf)) + + def test_disable_right(self) -> None: + """ + If the right argument to ``create_node.merge_config`` is ``None`` + then the return value is ``None``. + """ + conf = ListenerConfig([], [], {}) + self.assertEqual(None, create_node.merge_config(conf, None)) + + def test_disable_both(self) -> None: + """ + If both arguments to ``create_node.merge_config`` are ``None`` + then the return value is ``None``. + """ + self.assertEqual(None, create_node.merge_config(None, None)) + + def test_overlapping_keys(self) -> None: + """ + If there are any keys in the ``node_config`` of the left and right + parameters that are shared then ``ValueError`` is raised. + """ + left = ListenerConfig([], [], {"foo": "bar"}) + right = ListenerConfig([], [], {"foo": "baz"}) + self.assertRaises(ValueError, lambda: create_node.merge_config(left, right)) + + def test_merge(self) -> None: + """ + ``create_node.merge_config`` returns a ``ListenerConfig`` that has + all of the ports, locations, and node config from each of the two + ``ListenerConfig`` values given. + """ + left = ListenerConfig(["left-port"], ["left-location"], {"left": "foo"}) + right = ListenerConfig(["right-port"], ["right-location"], {"right": "bar"}) + result = create_node.merge_config(left, right) + self.assertEqual( + ListenerConfig( + ["left-port", "right-port"], + ["left-location", "right-location"], + {"left": "foo", "right": "bar"}, + ), + result, + ) + class Config(unittest.TestCase): def test_client_unrecognized_options(self): tests = [ From 2d688df29924cea72c43535b49f11627a0843234 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 21 Jul 2023 08:19:27 -0400 Subject: [PATCH 1961/2309] get the node config types right --- src/allmydata/test/cli/test_create.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/src/allmydata/test/cli/test_create.py b/src/allmydata/test/cli/test_create.py index bc6aab760..406aebd48 100644 --- a/src/allmydata/test/cli/test_create.py +++ b/src/allmydata/test/cli/test_create.py @@ -57,8 +57,8 @@ class MergeConfigTests(unittest.TestCase): If there are any keys in the ``node_config`` of the left and right parameters that are shared then ``ValueError`` is raised. """ - left = ListenerConfig([], [], {"foo": "bar"}) - right = ListenerConfig([], [], {"foo": "baz"}) + left = ListenerConfig([], [], {"foo": [("b", "ar")]}) + right = ListenerConfig([], [], {"foo": [("ba", "z")]}) self.assertRaises(ValueError, lambda: create_node.merge_config(left, right)) def test_merge(self) -> None: @@ -67,14 +67,22 @@ class MergeConfigTests(unittest.TestCase): all of the ports, locations, and node config from each of the two ``ListenerConfig`` values given. """ - left = ListenerConfig(["left-port"], ["left-location"], {"left": "foo"}) - right = ListenerConfig(["right-port"], ["right-location"], {"right": "bar"}) + left = ListenerConfig( + ["left-port"], + ["left-location"], + {"left": [("f", "oo")]}, + ) + right = ListenerConfig( + ["right-port"], + ["right-location"], + {"right": [("ba", "r")]}, + ) result = create_node.merge_config(left, right) self.assertEqual( ListenerConfig( ["left-port", "right-port"], ["left-location", "right-location"], - {"left": "foo", "right": "bar"}, + {"left": [("f", "oo")], "right": [("ba", "r")]}, ), result, ) From feb9643dfe1022663d339c622e7a0e9e94c1ca90 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 20 Jul 2023 11:36:30 -0400 Subject: [PATCH 1962/2309] skip permission-related tests if the environment is not suitable posix superuser can do anything on the filesystem --- src/allmydata/test/cli/test_grid_manager.py | 3 ++- src/allmydata/test/test_client.py | 25 +++++++------------- src/allmydata/test/test_node.py | 26 ++++++--------------- 3 files changed, 17 insertions(+), 37 deletions(-) diff --git a/src/allmydata/test/cli/test_grid_manager.py b/src/allmydata/test/cli/test_grid_manager.py index 2ed738544..604cd6b7b 100644 --- a/src/allmydata/test/cli/test_grid_manager.py +++ b/src/allmydata/test/cli/test_grid_manager.py @@ -222,7 +222,8 @@ class GridManagerCommandLine(TestCase): result.output, ) - @skipIf(not platform.isLinux(), "I only know how permissions work on linux") + @skipIf(platform.isWindows(), "We don't know how to set permissions on Windows.") + @skipIf(os.getuid() == 0, "cannot test as superuser with all permissions") def test_sign_bad_perms(self): """ Error reported if we can't create certificate file diff --git a/src/allmydata/test/test_client.py b/src/allmydata/test/test_client.py index 04d8e6160..86c95a310 100644 --- a/src/allmydata/test/test_client.py +++ b/src/allmydata/test/test_client.py @@ -1,17 +1,7 @@ -""" -Ported to Python 3. -""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals - -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 +from __future__ import annotations import os -import sys +from unittest import skipIf from functools import ( partial, ) @@ -42,6 +32,7 @@ from twisted.internet import defer from twisted.python.filepath import ( FilePath, ) +from twisted.python.runtime import platform from testtools.matchers import ( Equals, AfterPreprocessing, @@ -156,12 +147,12 @@ class Basic(testutil.ReallyEqualMixin, unittest.TestCase): yield client.create_client(basedir) self.assertIn("[client]helper.furl", str(ctx.exception)) + # if somebody knows a clever way to do this (cause + # EnvironmentError when reading a file that really exists), on + # windows, please fix this + @skipIf(platform.isWindows(), "We don't know how to set permissions on Windows.") + @skipIf(os.getuid() == 0, "cannot test as superuser with all permissions") def test_unreadable_config(self): - if sys.platform == "win32": - # if somebody knows a clever way to do this (cause - # EnvironmentError when reading a file that really exists), on - # windows, please fix this - raise unittest.SkipTest("can't make unreadable files on windows") basedir = "test_client.Basic.test_unreadable_config" os.mkdir(basedir) fn = os.path.join(basedir, "tahoe.cfg") diff --git a/src/allmydata/test/test_node.py b/src/allmydata/test/test_node.py index 0b371569a..1469ec5b2 100644 --- a/src/allmydata/test/test_node.py +++ b/src/allmydata/test/test_node.py @@ -1,14 +1,4 @@ -""" -Ported to Python 3. -""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals - -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 +from __future__ import annotations import base64 import os @@ -31,6 +21,7 @@ from unittest import skipIf from twisted.python.filepath import ( FilePath, ) +from twisted.python.runtime import platform from twisted.trial import unittest from twisted.internet import defer @@ -333,10 +324,8 @@ class TestCase(testutil.SignalMixin, unittest.TestCase): default = [("hello", "world")] self.assertEqual(config.items("nosuch", default), default) - @skipIf( - "win32" in sys.platform.lower() or "cygwin" in sys.platform.lower(), - "We don't know how to set permissions on Windows.", - ) + @skipIf(platform.isWindows(), "We don't know how to set permissions on Windows.") + @skipIf(os.getuid() == 0, "cannot test as superuser with all permissions") def test_private_config_unreadable(self): """ Asking for inaccessible private config is an error @@ -351,10 +340,8 @@ class TestCase(testutil.SignalMixin, unittest.TestCase): with self.assertRaises(Exception): config.get_or_create_private_config("foo") - @skipIf( - "win32" in sys.platform.lower() or "cygwin" in sys.platform.lower(), - "We don't know how to set permissions on Windows.", - ) + @skipIf(platform.isWindows(), "We don't know how to set permissions on Windows.") + @skipIf(os.getuid() == 0, "cannot test as superuser with all permissions") def test_private_config_unreadable_preexisting(self): """ error if reading private config data fails @@ -411,6 +398,7 @@ class TestCase(testutil.SignalMixin, unittest.TestCase): self.assertEqual(len(counter), 1) # don't call unless necessary self.assertEqual(value, "newer") + @skipIf(os.getuid() == 0, "cannot test as superuser with all permissions") def test_write_config_unwritable_file(self): """ Existing behavior merely logs any errors upon writing From 60b361df2b711e83bb113cc5ce8da98a63870e56 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 21 Jul 2023 08:39:15 -0400 Subject: [PATCH 1963/2309] news fragment --- newsfragments/4049.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/4049.minor diff --git a/newsfragments/4049.minor b/newsfragments/4049.minor new file mode 100644 index 000000000..e69de29bb From 10941a02f8a7dbf04681f1e6e2274d70280cb670 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 21 Jul 2023 09:42:00 -0400 Subject: [PATCH 1964/2309] Go with the successfully-built release branch release-XX.YY is the source branch for NixOS Hydra (CI) runs nixos-XX.YY is updated after a successful release-XX.YY build --- flake.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index 791015988..2a3290ca2 100644 --- a/flake.nix +++ b/flake.nix @@ -10,7 +10,7 @@ url = github:NixOS/nixpkgs?ref=nixos-22.11; }; "nixpkgs-23_05" = { - url = github:NixOS/nixpkgs?ref=release-23.05; + url = github:NixOS/nixpkgs?ref=nixos-23.05; }; # We depend on a very new python-cryptography which is not yet available From d61029c8bbb47d3e73739d19683e30eab60235fb Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 21 Jul 2023 09:54:59 -0400 Subject: [PATCH 1965/2309] a few more words about the nixpkgs inputs --- flake.nix | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/flake.nix b/flake.nix index 2a3290ca2..5b3102c46 100644 --- a/flake.nix +++ b/flake.nix @@ -2,10 +2,16 @@ description = "Tahoe-LAFS, free and open decentralized data store"; inputs = { - # Two alternate nixpkgs pins. Ideally these could be selected easily from - # the command line but there seems to be no syntax/support for that. + # A couple possible nixpkgs pins. Ideally these could be selected easily + # from the command line but there seems to be no syntax/support for that. # However, these at least cause certain revisions to be pinned in our lock # file where you *can* dig them out - and the CI configuration does. + # + # These are really just examples for the time being since neither of these + # releases contains a package set that is completely compatible with our + # requirements. We could decide in the future that supporting multiple + # releases of NixOS at a time is worthwhile and then pins like these will + # help us test each of those releases. "nixpkgs-22_11" = { url = github:NixOS/nixpkgs?ref=nixos-22.11; }; From 7e972e4a53182ac64f4f8b80852662344082ba32 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 21 Jul 2023 10:24:09 -0400 Subject: [PATCH 1966/2309] further clarifying comments --- default.nix | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/default.nix b/default.nix index 1b9368b5f..62dfc8176 100644 --- a/default.nix +++ b/default.nix @@ -1,5 +1,6 @@ # This is the flake-compat glue code. It loads the flake and gives us its -# outputs. +# outputs. This gives us backwards compatibility with pre-flake consumers. +# All of the real action is in flake.nix. (import ( let lock = builtins.fromJSON (builtins.readFile ./flake.lock); in From 9abc3730a0b31f9fe701a2f1c776bfbcceafcd81 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 21 Jul 2023 10:59:37 -0400 Subject: [PATCH 1967/2309] Revert "suppress the new click mypy errors", Click 8.1.6 fixed the issue. This reverts commit dfd34cfc0b8a4fccb5e39f2b825442b662a0a2fb. --- newsfragments/4050.minor | 0 src/allmydata/cli/grid_manager.py | 15 +++++++-------- 2 files changed, 7 insertions(+), 8 deletions(-) create mode 100644 newsfragments/4050.minor diff --git a/newsfragments/4050.minor b/newsfragments/4050.minor new file mode 100644 index 000000000..e69de29bb diff --git a/src/allmydata/cli/grid_manager.py b/src/allmydata/cli/grid_manager.py index 039f0f50f..3110a072e 100644 --- a/src/allmydata/cli/grid_manager.py +++ b/src/allmydata/cli/grid_manager.py @@ -28,7 +28,7 @@ from allmydata.grid_manager import ( from allmydata.util import jsonbytes as json -@click.group() # type: ignore[arg-type] +@click.group() @click.option( '--config', '-c', type=click.Path(), @@ -71,7 +71,7 @@ def grid_manager(ctx, config): ctx.obj = Config() -@grid_manager.command() # type: ignore[attr-defined] +@grid_manager.command() @click.pass_context def create(ctx): """ @@ -91,7 +91,7 @@ def create(ctx): ) -@grid_manager.command() # type: ignore[attr-defined] +@grid_manager.command() @click.pass_obj def public_identity(config): """ @@ -103,7 +103,7 @@ def public_identity(config): click.echo(config.grid_manager.public_identity()) -@grid_manager.command() # type: ignore[arg-type, attr-defined] +@grid_manager.command() @click.argument("name") @click.argument("public_key", type=click.STRING) @click.pass_context @@ -132,7 +132,7 @@ def add(ctx, name, public_key): return 0 -@grid_manager.command() # type: ignore[arg-type, attr-defined] +@grid_manager.command() @click.argument("name") @click.pass_context def remove(ctx, name): @@ -155,8 +155,7 @@ def remove(ctx, name): save_grid_manager(fp, ctx.obj.grid_manager, create=False) -@grid_manager.command() # type: ignore[attr-defined] - # noqa: F811 +@grid_manager.command() # noqa: F811 @click.pass_context def list(ctx): """ @@ -176,7 +175,7 @@ def list(ctx): click.echo("expired {} ({})".format(cert.expires, abbreviate_time(delta))) -@grid_manager.command() # type: ignore[arg-type, attr-defined] +@grid_manager.command() @click.argument("name") @click.argument( "expiry_days", From 6cc517d1a7aab78858d6703c98a57c447ba78162 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 21 Jul 2023 11:04:51 -0400 Subject: [PATCH 1968/2309] Newer mypy. --- tox.ini | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tox.ini b/tox.ini index 127cd9178..b2997c05b 100644 --- a/tox.ini +++ b/tox.ini @@ -125,9 +125,8 @@ commands = [testenv:typechecks] basepython = python3 deps = - mypy==1.3.0 - # When 0.9.2 comes out it will work with 1.3, it's just unreleased at the moment... - git+https://github.com/shoobx/mypy-zope@f276030 + mypy==1.4.1 + mypy-zope types-mock types-six types-PyYAML From 4bbde0288e3fc10b7aca891a5c5b88897c6bb103 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 21 Jul 2023 11:10:54 -0400 Subject: [PATCH 1969/2309] Upgrade ruff. --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index b2997c05b..67a089b0c 100644 --- a/tox.ini +++ b/tox.ini @@ -99,7 +99,7 @@ skip_install = true deps = # Pin a specific version so we get consistent outcomes; update this # occasionally: - ruff == 0.0.263 + ruff == 0.0.278 # towncrier doesn't work with importlib_resources 6.0.0 # https://github.com/twisted/towncrier/issues/528 importlib_resources < 6.0.0 From f4949c699aaca13b29b5a07c6a7f17bff4fdf4aa Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 21 Jul 2023 11:17:26 -0400 Subject: [PATCH 1970/2309] relock with release/nixos-23.05 change --- flake.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/flake.lock b/flake.lock index 2b955d4d6..b7b74a0e4 100644 --- a/flake.lock +++ b/flake.lock @@ -52,16 +52,16 @@ }, "nixpkgs-23_05": { "locked": { - "lastModified": 1688722168, - "narHash": "sha256-UDqeQd2neUXICpHAZSS965kGCJsfHkrOFS/vl80I7d8=", + "lastModified": 1689885880, + "narHash": "sha256-2ikAcvHKkKh8J/eUrwMA+wy1poscC+oL1RkN1V3RmT8=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "28d812a63a0f0d6c1170aac16f5219e506c44b79", + "rev": "fa793b06f56896b7d1909e4b69977c7bf842b2f0", "type": "github" }, "original": { "owner": "NixOS", - "ref": "release-23.05", + "ref": "nixos-23.05", "repo": "nixpkgs", "type": "github" } From c350d8b7362eb2f4c89b366e05a6aa08819c30ec Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 21 Jul 2023 11:18:37 -0400 Subject: [PATCH 1971/2309] slightly reduce repetition by pulling out a mergeAttrs definition --- flake.nix | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/flake.nix b/flake.nix index 5b3102c46..2eb87334c 100644 --- a/flake.nix +++ b/flake.nix @@ -114,6 +114,12 @@ # Make a singleton attribute set from the result of two functions. singletonOf = f: g: x: { ${f x} = g x; }; + # [attrset] -> attrset + # + # Merge a list of attrset into a single attrset with overlap preferring + # rightmost values. + mergeAttrs = pkgs.lib.foldr pkgs.lib.mergeAttrs {}; + # makeRuntimeEnv :: string -> derivation # # Create a derivation that includes a Python runtime, Tahoe-LAFS, and @@ -149,11 +155,11 @@ # flake's "default" package at the derivation corresponding to the # default Python version we defined above. The package consists of a # Python environment with Tahoe-LAFS available to it. - packages = with pkgs.lib; - foldr mergeAttrs {} ([ - { default = self.packages.${system}.${packageName defaultPyVersion}; } - ] ++ (builtins.map makeRuntimeEnv pythonVersions) - ++ (builtins.map (singletonOf unitTestName makeTestEnv) pythonVersions) + packages = + mergeAttrs ( + [ { default = self.packages.${system}.${packageName defaultPyVersion}; } ] + ++ (builtins.map makeRuntimeEnv pythonVersions) + ++ (builtins.map (singletonOf unitTestName makeTestEnv) pythonVersions) ); # The flake's app outputs. We'll define a version of an app for running @@ -208,13 +214,11 @@ }; }; in - with pkgs.lib; # Merge a default app definition with the rest of the apps. - foldr mergeAttrs - { default = self.apps.${system}."tahoe-python3"; } - ( - (builtins.map makeUnitTestsApp pythonVersions) ++ - (builtins.map makeTahoeApp pythonVersions) - ); + mergeAttrs ( + [ { default = self.apps.${system}."tahoe-python3"; } ] + ++ (builtins.map makeUnitTestsApp pythonVersions) + ++ (builtins.map makeTahoeApp pythonVersions) + ); })); } From b4a6a90e9f56ff7187e9cf8a510797b1cdfd0926 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 21 Jul 2023 11:19:33 -0400 Subject: [PATCH 1972/2309] more clarifying comments --- flake.nix | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index 2eb87334c..44000c9ab 100644 --- a/flake.nix +++ b/flake.nix @@ -130,6 +130,9 @@ tahoe-lafs.passthru.extras.i2p ++ tahoe-lafs.passthru.extras.tor )).overrideAttrs (old: { + # By default, withPackages gives us a derivation with a fairly generic + # name (like "python-env"). Put our name in there for legibility. + # See the similar override in makeTestEnv. name = packageName pyVersion; }); @@ -144,10 +147,15 @@ tahoe-lafs.passthru.extras.tor ++ tahoe-lafs.passthru.extras.unittest )).overrideAttrs (old: { + # See the similar override in makeRuntimeEnv'. name = packageName pyVersion; }); in { - # A package set with out overlay on it. + # Include a package set with out overlay on it in our own output. This + # is mainly a development/debugging convenience as it will expose all of + # our Python package overrides beneath it. The magic name + # "legacyPackages" is copied from nixpkgs and has special support in the + # nix command line tool. legacyPackages = pkgs; # The flake's package outputs. We'll define one version of the package From 91866154d3d5b120e9ac132f861074cb188960ee Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 21 Jul 2023 11:29:06 -0400 Subject: [PATCH 1973/2309] expose our cache to anyone who wants it --- flake.nix | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/flake.nix b/flake.nix index 44000c9ab..bde792db3 100644 --- a/flake.nix +++ b/flake.nix @@ -1,6 +1,14 @@ { description = "Tahoe-LAFS, free and open decentralized data store"; + nixConfig = { + # Supply configuration for the build cache updated by our CI system. This + # should allow most users to avoid having to build a large number of + # packages (otherwise necessary due to our Python package overrides). + substituters = ["https://tahoe-lafs-opensource.cachix.org"]; + trusted-public-keys = ["tahoe-lafs-opensource.cachix.org-1:eIKCHOPJYceJ2gb74l6e0mayuSdXqiavxYeAio0LFGo="]; + }; + inputs = { # A couple possible nixpkgs pins. Ideally these could be selected easily # from the command line but there seems to be no syntax/support for that. From 22991fdd4cdff569d20541dc3b1019ec2b0cb0ec Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 21 Jul 2023 12:47:56 -0400 Subject: [PATCH 1974/2309] Set up Tor-related fixture dependencies, maybe even properly --- integration/conftest.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/integration/conftest.py b/integration/conftest.py index b29b9fe36..c94c05429 100644 --- a/integration/conftest.py +++ b/integration/conftest.py @@ -285,7 +285,7 @@ def introducer_furl(introducer, temp_dir): include_args=["temp_dir", "flog_gatherer"], include_result=False, ) -def tor_introducer(reactor, temp_dir, flog_gatherer, request): +def tor_introducer(reactor, temp_dir, flog_gatherer, request, tor_control_port): intro_dir = join(temp_dir, 'introducer_tor') print("making Tor introducer in {}".format(intro_dir)) print("(this can take tens of seconds to allocate Onion address)") @@ -299,9 +299,7 @@ def tor_introducer(reactor, temp_dir, flog_gatherer, request): request, ( 'create-introducer', - # The control port should agree with the configuration of the - # Tor network we bootstrap with chutney. - '--tor-control-port', 'tcp:localhost:8007', + '--tor-control-port', tor_control_port, '--hide-ip', '--listen=tor', intro_dir, @@ -516,6 +514,17 @@ def chutney(reactor, temp_dir: str) -> tuple[str, dict[str, str]]: return (chutney_dir, {"PYTHONPATH": join(chutney_dir, "lib")}) +@pytest.fixture(scope='session') +def tor_control_port(tor_network): + """ + Get an endpoint description for the Tor control port for the local Tor + network we run.. + """ + # We ignore tor_network because it can't tell us the control port. But + # asking for it forces the Tor network to be built before we run - so if + # we get the hard-coded control port value correct, there should be + # something listening at that address. + return 'tcp:localhost:8007' @pytest.fixture(scope='session') @pytest.mark.skipif(sys.platform.startswith('win'), From 8e7cc91434713b2c742c6316e73cbb20f3d9d68e Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 21 Jul 2023 12:48:18 -0400 Subject: [PATCH 1975/2309] news fragment --- newsfragments/4051.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/4051.minor diff --git a/newsfragments/4051.minor b/newsfragments/4051.minor new file mode 100644 index 000000000..e69de29bb From 45898ff8b8ae6218e52397d1d3c55ad9d71fed2e Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 24 Jul 2023 20:08:41 -0600 Subject: [PATCH 1976/2309] refactor: make sftp tests (etc) work with 'grid' refactoring --- integration/conftest.py | 67 ++++------------------ integration/grid.py | 97 ++++++++++++++++++++++++++++---- integration/test_get_put.py | 30 +++++----- integration/test_grid_manager.py | 4 +- integration/test_sftp.py | 17 +++--- integration/test_vectors.py | 16 +++--- integration/test_web.py | 40 ++++++------- integration/util.py | 3 +- 8 files changed, 151 insertions(+), 123 deletions(-) diff --git a/integration/conftest.py b/integration/conftest.py index 6de2e84af..837b54aa1 100644 --- a/integration/conftest.py +++ b/integration/conftest.py @@ -162,6 +162,10 @@ def flog_gatherer(reactor, temp_dir, flog_binary, request): @pytest.fixture(scope='session') @log_call(action_type=u"integration:grid", include_args=[]) def grid(reactor, request, temp_dir, flog_gatherer, port_allocator): + # XXX think: this creates an "empty" grid (introducer, no nodes); + # do we want to ensure it has some minimum storage-nodes at least? + # (that is, semantically does it make sense that 'a grid' is + # essentially empty, or not?) g = pytest_twisted.blockon( create_grid(reactor, request, temp_dir, flog_gatherer, port_allocator) ) @@ -271,64 +275,17 @@ def storage_nodes(grid): assert ok, "Storage node creation failed: {}".format(value) return grid.storage_servers -@pytest.fixture(scope="session") -def alice_sftp_client_key_path(temp_dir): - # The client SSH key path is typically going to be somewhere else (~/.ssh, - # typically), but for convenience sake for testing we'll put it inside node. - return join(temp_dir, "alice", "private", "ssh_client_rsa_key") @pytest.fixture(scope='session') @log_call(action_type=u"integration:alice", include_args=[], include_result=False) -def alice( - reactor, - temp_dir, - introducer_furl, - flog_gatherer, - storage_nodes, - alice_sftp_client_key_path, - request, -): - process = pytest_twisted.blockon( - _create_node( - reactor, request, temp_dir, introducer_furl, flog_gatherer, "alice", - web_port="tcp:9980:interface=localhost", - storage=False, - ) - ) - pytest_twisted.blockon(await_client_ready(process)) - - # 1. Create a new RW directory cap: - cli(process, "create-alias", "test") - rwcap = loads(cli(process, "list-aliases", "--json"))["test"]["readwrite"] - - # 2. Enable SFTP on the node: - host_ssh_key_path = join(process.node_dir, "private", "ssh_host_rsa_key") - accounts_path = join(process.node_dir, "private", "accounts") - with open(join(process.node_dir, "tahoe.cfg"), "a") as f: - f.write("""\ -[sftpd] -enabled = true -port = tcp:8022:interface=127.0.0.1 -host_pubkey_file = {ssh_key_path}.pub -host_privkey_file = {ssh_key_path} -accounts.file = {accounts_path} -""".format(ssh_key_path=host_ssh_key_path, accounts_path=accounts_path)) - generate_ssh_key(host_ssh_key_path) - - # 3. Add a SFTP access file with an SSH key for auth. - generate_ssh_key(alice_sftp_client_key_path) - # Pub key format is "ssh-rsa ". We want the key. - ssh_public_key = open(alice_sftp_client_key_path + ".pub").read().strip().split()[1] - with open(accounts_path, "w") as f: - f.write("""\ -alice-key ssh-rsa {ssh_public_key} {rwcap} -""".format(rwcap=rwcap, ssh_public_key=ssh_public_key)) - - # 4. Restart the node with new SFTP config. - pytest_twisted.blockon(process.restart_async(reactor, request)) - pytest_twisted.blockon(await_client_ready(process)) - print(f"Alice pid: {process.transport.pid}") - return process +def alice(reactor, request, grid, storage_nodes): + """ + :returns grid.Client: the associated instance for Alice + """ + alice = pytest_twisted.blockon(grid.add_client("alice")) + pytest_twisted.blockon(alice.add_sftp(reactor, request)) + print(f"Alice pid: {alice.process.transport.pid}") + return alice @pytest.fixture(scope='session') diff --git a/integration/grid.py b/integration/grid.py index 46fde576e..fe3befd3a 100644 --- a/integration/grid.py +++ b/integration/grid.py @@ -10,6 +10,7 @@ rely on 'the' global grid as provided by fixtures like 'alice' or from os import mkdir, listdir from os.path import join, exists +from json import loads from tempfile import mktemp from time import sleep @@ -26,6 +27,7 @@ from twisted.internet.defer import ( inlineCallbacks, returnValue, maybeDeferred, + Deferred, ) from twisted.internet.task import ( deferLater, @@ -54,19 +56,20 @@ from .util import ( _tahoe_runner_optional_coverage, TahoeProcess, await_client_ready, + generate_ssh_key, + cli, + reconfigure, ) import attr import pytest_twisted -# further directions: -# - "Grid" is unused, basically -- tie into the rest? -# - could make a Grid instance mandatory for create_* calls -# - could instead make create_* calls methods of Grid -# - Bring more 'util' or 'conftest' code into here -# - stop()/start()/restart() methods on StorageServer etc -# - more-complex stuff like config changes (which imply a restart too)? +# currently, we pass a "request" around a bunch but it seems to only +# be for addfinalizer() calls. +# - is "keeping" a request like that okay? What if it's a session-scoped one? +# (i.e. in Grid etc) +# - maybe limit to "a callback to hang your cleanup off of" (instead of request)? @attr.s @@ -170,6 +173,8 @@ class StorageServer(object): Note that self.process and self.protocol will be new instances after this. """ + # XXX per review comments, _can_ we make this "return a new + # instance" instead of mutating? self.process.transport.signalProcess('TERM') yield self.protocol.exited self.process = yield _run_node( @@ -213,6 +218,27 @@ class Client(object): protocol = attr.ib( validator=attr.validators.provides(IProcessProtocol) ) + request = attr.ib() # original request, for addfinalizer() + +## XXX convenience? or confusion? +# @property +# def node_dir(self): +# return self.process.node_dir + + @inlineCallbacks + def reconfigure_zfec(self, reactor, request, zfec_params, convergence=None, max_segment_size=None): + """ + Reconfigure the ZFEC parameters for this node + """ + # XXX this is a stop-gap to keep tests running "as is" + # -> we should fix the tests so that they create a new client + # in the grid with the required parameters, instead of + # re-configuring Alice (or whomever) + + rtn = yield Deferred.fromCoroutine( + reconfigure(reactor, self.request, self.process, zfec_params, convergence, max_segment_size) + ) + return rtn @inlineCallbacks def restart(self, reactor, request, servers=1): @@ -226,6 +252,8 @@ class Client(object): Note that self.process and self.protocol will be new instances after this. """ + # XXX similar to above, can we make this return a new instance + # instead of mutating? self.process.transport.signalProcess('TERM') yield self.protocol.exited process = yield _run_node( @@ -235,8 +263,55 @@ class Client(object): self.protocol = self.process.transport.proto yield await_client_ready(self.process, minimum_number_of_servers=servers) - # XXX add stop / start ? - # ...maybe "reconfig" of some kind? + @inlineCallbacks + def add_sftp(self, reactor, request): + """ + """ + # if other things need to add or change configuration, further + # refactoring could be useful here (i.e. move reconfigure + # parts to their own functions) + + # XXX why do we need an alias? + # 1. Create a new RW directory cap: + cli(self.process, "create-alias", "test") + rwcap = loads(cli(self.process, "list-aliases", "--json"))["test"]["readwrite"] + + # 2. Enable SFTP on the node: + host_ssh_key_path = join(self.process.node_dir, "private", "ssh_host_rsa_key") + sftp_client_key_path = join(self.process.node_dir, "private", "ssh_client_rsa_key") + accounts_path = join(self.process.node_dir, "private", "accounts") + with open(join(self.process.node_dir, "tahoe.cfg"), "a") as f: + f.write( + ("\n\n[sftpd]\n" + "enabled = true\n" + "port = tcp:8022:interface=127.0.0.1\n" + "host_pubkey_file = {ssh_key_path}.pub\n" + "host_privkey_file = {ssh_key_path}\n" + "accounts.file = {accounts_path}\n").format( + ssh_key_path=host_ssh_key_path, + accounts_path=accounts_path, + ) + ) + generate_ssh_key(host_ssh_key_path) + + # 3. Add a SFTP access file with an SSH key for auth. + generate_ssh_key(sftp_client_key_path) + # Pub key format is "ssh-rsa ". We want the key. + with open(sftp_client_key_path + ".pub") as pubkey_file: + ssh_public_key = pubkey_file.read().strip().split()[1] + with open(accounts_path, "w") as f: + f.write( + "alice-key ssh-rsa {ssh_public_key} {rwcap}\n".format( + rwcap=rwcap, + ssh_public_key=ssh_public_key, + ) + ) + + # 4. Restart the node with new SFTP config. + print("restarting for SFTP") + yield self.restart(reactor, request) + print("restart done") + # XXX i think this is broken because we're "waiting for ready" during first bootstrap? or something? @inlineCallbacks @@ -254,6 +329,7 @@ def create_client(reactor, request, temp_dir, introducer, flog_gatherer, name, w Client( process=node_process, protocol=node_process.transport.proto, + request=request, ) ) @@ -370,7 +446,7 @@ class Grid(object): Represents an entire Tahoe Grid setup A Grid includes an Introducer, Flog Gatherer and some number of - Storage Servers. + Storage Servers. Optionally includes Clients. """ _reactor = attr.ib() @@ -436,7 +512,6 @@ class Grid(object): returnValue(client) - # XXX THINK can we tie a whole *grid* to a single request? (I think # that's all that makes sense) @inlineCallbacks diff --git a/integration/test_get_put.py b/integration/test_get_put.py index e30a34f97..536185ef8 100644 --- a/integration/test_get_put.py +++ b/integration/test_get_put.py @@ -8,9 +8,8 @@ from subprocess import Popen, PIPE, check_output, check_call import pytest from twisted.internet import reactor from twisted.internet.threads import blockingCallFromThread -from twisted.internet.defer import Deferred -from .util import run_in_thread, cli, reconfigure +from .util import run_in_thread, cli DATA = b"abc123 this is not utf-8 decodable \xff\x00\x33 \x11" try: @@ -23,7 +22,7 @@ else: @pytest.fixture(scope="session") def get_put_alias(alice): - cli(alice, "create-alias", "getput") + cli(alice.process, "create-alias", "getput") def read_bytes(path): @@ -39,14 +38,14 @@ def test_put_from_stdin(alice, get_put_alias, tmpdir): """ tempfile = str(tmpdir.join("file")) p = Popen( - ["tahoe", "--node-directory", alice.node_dir, "put", "-", "getput:fromstdin"], + ["tahoe", "--node-directory", alice.process.node_dir, "put", "-", "getput:fromstdin"], stdin=PIPE ) p.stdin.write(DATA) p.stdin.close() assert p.wait() == 0 - cli(alice, "get", "getput:fromstdin", tempfile) + cli(alice.process, "get", "getput:fromstdin", tempfile) assert read_bytes(tempfile) == DATA @@ -58,10 +57,10 @@ def test_get_to_stdout(alice, get_put_alias, tmpdir): tempfile = tmpdir.join("file") with tempfile.open("wb") as f: f.write(DATA) - cli(alice, "put", str(tempfile), "getput:tostdout") + cli(alice.process, "put", str(tempfile), "getput:tostdout") p = Popen( - ["tahoe", "--node-directory", alice.node_dir, "get", "getput:tostdout", "-"], + ["tahoe", "--node-directory", alice.process.node_dir, "get", "getput:tostdout", "-"], stdout=PIPE ) assert p.stdout.read() == DATA @@ -78,11 +77,11 @@ def test_large_file(alice, get_put_alias, tmp_path): tempfile = tmp_path / "file" with tempfile.open("wb") as f: f.write(DATA * 1_000_000) - cli(alice, "put", str(tempfile), "getput:largefile") + cli(alice.process, "put", str(tempfile), "getput:largefile") outfile = tmp_path / "out" check_call( - ["tahoe", "--node-directory", alice.node_dir, "get", "getput:largefile", str(outfile)], + ["tahoe", "--node-directory", alice.process.node_dir, "get", "getput:largefile", str(outfile)], ) assert outfile.read_bytes() == tempfile.read_bytes() @@ -104,31 +103,30 @@ def test_upload_download_immutable_different_default_max_segment_size(alice, get def set_segment_size(segment_size): return blockingCallFromThread( reactor, - lambda: Deferred.fromCoroutine(reconfigure( + lambda: alice.reconfigure_zfec( reactor, request, - alice, (1, 1, 1), None, max_segment_size=segment_size - )) + ) ) # 1. Upload file 1 with default segment size set to 1MB set_segment_size(1024 * 1024) - cli(alice, "put", str(tempfile), "getput:seg1024kb") + cli(alice.process, "put", str(tempfile), "getput:seg1024kb") # 2. Download file 1 with default segment size set to 128KB set_segment_size(128 * 1024) assert large_data == check_output( - ["tahoe", "--node-directory", alice.node_dir, "get", "getput:seg1024kb", "-"] + ["tahoe", "--node-directory", alice.process.node_dir, "get", "getput:seg1024kb", "-"] ) # 3. Upload file 2 with default segment size set to 128KB - cli(alice, "put", str(tempfile), "getput:seg128kb") + cli(alice.process, "put", str(tempfile), "getput:seg128kb") # 4. Download file 2 with default segment size set to 1MB set_segment_size(1024 * 1024) assert large_data == check_output( - ["tahoe", "--node-directory", alice.node_dir, "get", "getput:seg128kb", "-"] + ["tahoe", "--node-directory", alice.process.node_dir, "get", "getput:seg128kb", "-"] ) diff --git a/integration/test_grid_manager.py b/integration/test_grid_manager.py index 1856ef435..437fe7455 100644 --- a/integration/test_grid_manager.py +++ b/integration/test_grid_manager.py @@ -173,7 +173,7 @@ def test_add_remove_client_file(reactor, request, temp_dir): @pytest_twisted.inlineCallbacks -def test_reject_storage_server(reactor, request, temp_dir, flog_gatherer, port_allocator): +def _test_reject_storage_server(reactor, request, temp_dir, flog_gatherer, port_allocator): """ A client with happines=2 fails to upload to a Grid when it is using Grid Manager and there is only 1 storage server with a valid @@ -252,7 +252,7 @@ def test_reject_storage_server(reactor, request, temp_dir, flog_gatherer, port_a @pytest_twisted.inlineCallbacks -def test_accept_storage_server(reactor, request, temp_dir, flog_gatherer, port_allocator): +def _test_accept_storage_server(reactor, request, temp_dir, flog_gatherer, port_allocator): """ Successfully upload to a Grid Manager enabled Grid. """ diff --git a/integration/test_sftp.py b/integration/test_sftp.py index 3fdbb56d7..01ddfdf8a 100644 --- a/integration/test_sftp.py +++ b/integration/test_sftp.py @@ -72,7 +72,7 @@ def test_bad_account_password_ssh_key(alice, tmpdir): another_key = os.path.join(str(tmpdir), "ssh_key") generate_ssh_key(another_key) - good_key = RSAKey(filename=os.path.join(alice.node_dir, "private", "ssh_client_rsa_key")) + good_key = RSAKey(filename=os.path.join(alice.process.node_dir, "private", "ssh_client_rsa_key")) bad_key = RSAKey(filename=another_key) # Wrong key: @@ -87,17 +87,16 @@ def test_bad_account_password_ssh_key(alice, tmpdir): "username": "someoneelse", "pkey": good_key, }) -def sftp_client_key(node): + +def sftp_client_key(client): + """ + :return RSAKey: the RSA client key associated with this grid.Client + """ + # XXX move to Client / grid.py? return RSAKey( - filename=os.path.join(node.node_dir, "private", "ssh_client_rsa_key"), + filename=os.path.join(client.process.node_dir, "private", "ssh_client_rsa_key"), ) -def test_sftp_client_key_exists(alice, alice_sftp_client_key_path): - """ - Weakly validate the sftp client key fixture by asserting that *something* - exists at the supposed key path. - """ - assert os.path.exists(alice_sftp_client_key_path) @run_in_thread def test_ssh_key_auth(alice): diff --git a/integration/test_vectors.py b/integration/test_vectors.py index 6e7b5746a..13a451d1c 100644 --- a/integration/test_vectors.py +++ b/integration/test_vectors.py @@ -15,7 +15,8 @@ from pytest_twisted import ensureDeferred from . import vectors from .vectors import parameters -from .util import reconfigure, upload, TahoeProcess +from .util import reconfigure, upload +from .grid import Client @mark.parametrize('convergence', parameters.CONVERGENCE_SECRETS) def test_convergence(convergence): @@ -36,11 +37,11 @@ async def test_capability(reactor, request, alice, case, expected): computed value. """ # rewrite alice's config to match params and convergence - await reconfigure( - reactor, request, alice, (1, case.params.required, case.params.total), case.convergence, case.segment_size) + await alice.reconfigure_zfec( + reactor, request, (1, case.params.required, case.params.total), case.convergence, case.segment_size) # upload data in the correct format - actual = upload(alice, case.fmt, case.data) + actual = upload(alice.process, case.fmt, case.data) # compare the resulting cap to the expected result assert actual == expected @@ -82,7 +83,7 @@ async def skiptest_generate(reactor, request, alice): async def generate( reactor, request, - alice: TahoeProcess, + alice: Client, cases: Iterator[vectors.Case], ) -> AsyncGenerator[[vectors.Case, str], None]: """ @@ -106,10 +107,9 @@ async def generate( # reliability of this generator, be happy if we can put shares anywhere happy = 1 for case in cases: - await reconfigure( + await alice.reconfigure_zfec( reactor, request, - alice, (happy, case.params.required, case.params.total), case.convergence, case.segment_size @@ -117,5 +117,5 @@ async def generate( # Give the format a chance to make an RSA key if it needs it. case = evolve(case, fmt=case.fmt.customize()) - cap = upload(alice, case.fmt, case.data) + cap = upload(alice.process, case.fmt, case.data) yield case, cap diff --git a/integration/test_web.py b/integration/test_web.py index b863a27fe..01f69bca0 100644 --- a/integration/test_web.py +++ b/integration/test_web.py @@ -33,7 +33,7 @@ def test_index(alice): """ we can download the index file """ - util.web_get(alice, u"") + util.web_get(alice.process, u"") @run_in_thread @@ -41,7 +41,7 @@ def test_index_json(alice): """ we can download the index file as json """ - data = util.web_get(alice, u"", params={u"t": u"json"}) + data = util.web_get(alice.process, u"", params={u"t": u"json"}) # it should be valid json json.loads(data) @@ -55,7 +55,7 @@ def test_upload_download(alice): FILE_CONTENTS = u"some contents" readcap = util.web_post( - alice, u"uri", + alice.process, u"uri", data={ u"t": u"upload", u"format": u"mdmf", @@ -67,7 +67,7 @@ def test_upload_download(alice): readcap = readcap.strip() data = util.web_get( - alice, u"uri", + alice.process, u"uri", params={ u"uri": readcap, u"filename": u"boom", @@ -85,11 +85,11 @@ def test_put(alice): FILE_CONTENTS = b"added via PUT" * 20 resp = requests.put( - util.node_url(alice.node_dir, u"uri"), + util.node_url(alice.process.node_dir, u"uri"), data=FILE_CONTENTS, ) cap = allmydata.uri.from_string(resp.text.strip().encode('ascii')) - cfg = alice.get_config() + cfg = alice.process.get_config() assert isinstance(cap, allmydata.uri.CHKFileURI) assert cap.size == len(FILE_CONTENTS) assert cap.total_shares == int(cfg.get_config("client", "shares.total")) @@ -116,7 +116,7 @@ def test_deep_stats(alice): URIs work """ resp = requests.post( - util.node_url(alice.node_dir, "uri"), + util.node_url(alice.process.node_dir, "uri"), params={ "format": "sdmf", "t": "mkdir", @@ -130,7 +130,7 @@ def test_deep_stats(alice): uri = url_unquote(resp.url) assert 'URI:DIR2:' in uri dircap = uri[uri.find("URI:DIR2:"):].rstrip('/') - dircap_uri = util.node_url(alice.node_dir, "uri/{}".format(url_quote(dircap))) + dircap_uri = util.node_url(alice.process.node_dir, "uri/{}".format(url_quote(dircap))) # POST a file into this directory FILE_CONTENTS = u"a file in a directory" @@ -176,7 +176,7 @@ def test_deep_stats(alice): while tries > 0: tries -= 1 resp = requests.get( - util.node_url(alice.node_dir, u"operations/something_random"), + util.node_url(alice.process.node_dir, u"operations/something_random"), ) d = json.loads(resp.content) if d['size-literal-files'] == len(FILE_CONTENTS): @@ -201,21 +201,21 @@ def test_status(alice): FILE_CONTENTS = u"all the Important Data of alice\n" * 1200 resp = requests.put( - util.node_url(alice.node_dir, u"uri"), + util.node_url(alice.process.node_dir, u"uri"), data=FILE_CONTENTS, ) cap = resp.text.strip() print("Uploaded data, cap={}".format(cap)) resp = requests.get( - util.node_url(alice.node_dir, u"uri/{}".format(url_quote(cap))), + util.node_url(alice.process.node_dir, u"uri/{}".format(url_quote(cap))), ) print("Downloaded {} bytes of data".format(len(resp.content))) assert str(resp.content, "ascii") == FILE_CONTENTS resp = requests.get( - util.node_url(alice.node_dir, "status"), + util.node_url(alice.process.node_dir, "status"), ) dom = html5lib.parse(resp.content) @@ -229,7 +229,7 @@ def test_status(alice): for href in hrefs: if href == u"/" or not href: continue - resp = requests.get(util.node_url(alice.node_dir, href)) + resp = requests.get(util.node_url(alice.process.node_dir, href)) if href.startswith(u"/status/up"): assert b"File Upload Status" in resp.content if b"Total Size: %d" % (len(FILE_CONTENTS),) in resp.content: @@ -241,7 +241,7 @@ def test_status(alice): # download the specialized event information resp = requests.get( - util.node_url(alice.node_dir, u"{}/event_json".format(href)), + util.node_url(alice.process.node_dir, u"{}/event_json".format(href)), ) js = json.loads(resp.content) # there's usually just one "read" operation, but this can handle many .. @@ -264,14 +264,14 @@ async def test_directory_deep_check(reactor, request, alice): required = 2 total = 4 - await util.reconfigure(reactor, request, alice, (happy, required, total), convergence=None) + await alice.reconfigure_zfec(reactor, request, (happy, required, total), convergence=None) await deferToThread(_test_directory_deep_check_blocking, alice) def _test_directory_deep_check_blocking(alice): # create a directory resp = requests.post( - util.node_url(alice.node_dir, u"uri"), + util.node_url(alice.process.node_dir, u"uri"), params={ u"t": u"mkdir", u"redirect_to_result": u"true", @@ -320,7 +320,7 @@ def _test_directory_deep_check_blocking(alice): print("Uploaded data1, cap={}".format(cap1)) resp = requests.get( - util.node_url(alice.node_dir, u"uri/{}".format(url_quote(cap0))), + util.node_url(alice.process.node_dir, u"uri/{}".format(url_quote(cap0))), params={u"t": u"info"}, ) @@ -484,14 +484,14 @@ def test_mkdir_with_children(alice): # create a file to put in our directory FILE_CONTENTS = u"some file contents\n" * 500 resp = requests.put( - util.node_url(alice.node_dir, u"uri"), + util.node_url(alice.process.node_dir, u"uri"), data=FILE_CONTENTS, ) filecap = resp.content.strip() # create a (sub) directory to put in our directory resp = requests.post( - util.node_url(alice.node_dir, u"uri"), + util.node_url(alice.process.node_dir, u"uri"), params={ u"t": u"mkdir", } @@ -534,7 +534,7 @@ def test_mkdir_with_children(alice): # create a new directory with one file and one sub-dir (all-at-once) resp = util.web_post( - alice, u"uri", + alice.process, u"uri", params={u"t": "mkdir-with-children"}, data=json.dumps(meta), ) diff --git a/integration/util.py b/integration/util.py index ff54b1831..b614a84bd 100644 --- a/integration/util.py +++ b/integration/util.py @@ -741,7 +741,6 @@ class SSK: def load(cls, params: dict) -> SSK: assert params.keys() == {"format", "mutable", "key"} return cls(params["format"], params["key"].encode("ascii")) - def customize(self) -> SSK: """ Return an SSK with a newly generated random RSA key. @@ -780,7 +779,7 @@ def upload(alice: TahoeProcess, fmt: CHK | SSK, data: bytes) -> str: f.write(data) f.flush() with fmt.to_argv() as fmt_argv: - argv = [alice, "put"] + fmt_argv + [f.name] + argv = [alice.process, "put"] + fmt_argv + [f.name] return cli(*argv).decode("utf-8").strip() From 6f9b9a3ac1123ca3eb9ecce85e98cc75dc6ccd89 Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 24 Jul 2023 20:12:01 -0600 Subject: [PATCH 1977/2309] only use original request --- integration/grid.py | 2 +- integration/test_get_put.py | 1 - integration/test_vectors.py | 3 +-- integration/test_web.py | 2 +- 4 files changed, 3 insertions(+), 5 deletions(-) diff --git a/integration/grid.py b/integration/grid.py index fe3befd3a..5ce4179ec 100644 --- a/integration/grid.py +++ b/integration/grid.py @@ -226,7 +226,7 @@ class Client(object): # return self.process.node_dir @inlineCallbacks - def reconfigure_zfec(self, reactor, request, zfec_params, convergence=None, max_segment_size=None): + def reconfigure_zfec(self, reactor, zfec_params, convergence=None, max_segment_size=None): """ Reconfigure the ZFEC parameters for this node """ diff --git a/integration/test_get_put.py b/integration/test_get_put.py index 536185ef8..2f6642493 100644 --- a/integration/test_get_put.py +++ b/integration/test_get_put.py @@ -105,7 +105,6 @@ def test_upload_download_immutable_different_default_max_segment_size(alice, get reactor, lambda: alice.reconfigure_zfec( reactor, - request, (1, 1, 1), None, max_segment_size=segment_size diff --git a/integration/test_vectors.py b/integration/test_vectors.py index 13a451d1c..bd5def8c5 100644 --- a/integration/test_vectors.py +++ b/integration/test_vectors.py @@ -38,7 +38,7 @@ async def test_capability(reactor, request, alice, case, expected): """ # rewrite alice's config to match params and convergence await alice.reconfigure_zfec( - reactor, request, (1, case.params.required, case.params.total), case.convergence, case.segment_size) + reactor, (1, case.params.required, case.params.total), case.convergence, case.segment_size) # upload data in the correct format actual = upload(alice.process, case.fmt, case.data) @@ -109,7 +109,6 @@ async def generate( for case in cases: await alice.reconfigure_zfec( reactor, - request, (happy, case.params.required, case.params.total), case.convergence, case.segment_size diff --git a/integration/test_web.py b/integration/test_web.py index 01f69bca0..08c6e6217 100644 --- a/integration/test_web.py +++ b/integration/test_web.py @@ -264,7 +264,7 @@ async def test_directory_deep_check(reactor, request, alice): required = 2 total = 4 - await alice.reconfigure_zfec(reactor, request, (happy, required, total), convergence=None) + await alice.reconfigure_zfec(reactor, (happy, required, total), convergence=None) await deferToThread(_test_directory_deep_check_blocking, alice) From bf2451bbcdbde50c72b90a20354ae4bc281b7b81 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 25 Jul 2023 15:31:18 -0400 Subject: [PATCH 1978/2309] Correct type. --- src/allmydata/test/test_storage_http.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 30f6a527d..52c47c6c3 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -172,7 +172,7 @@ class ExtractSecretsTests(SyncTestCase): ``ClientSecretsException``. """ with self.assertRaises(ClientSecretsException): - _extract_secrets(["FOO eA=="], {}) + _extract_secrets(["FOO eA=="], set()) def test_bad_secret_not_base64(self): """ From 46d10a6281d145f974bb5120120845e1eb6339e9 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 25 Jul 2023 15:31:30 -0400 Subject: [PATCH 1979/2309] Ensure and test (and necessary refactor) that lack of content-type is same as CBOR content-type, as per spec. --- src/allmydata/storage/http_server.py | 119 ++++++++++++------------ src/allmydata/test/test_storage_http.py | 46 +++++++++ 2 files changed, 108 insertions(+), 57 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index c63a4ca08..3ff98e933 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -530,6 +530,60 @@ def _add_error_handling(app: Klein): return str(failure.value).encode("utf-8") +async def read_encoded( + reactor, request, schema: Schema, max_size: int = 1024 * 1024 +) -> Any: + """ + Read encoded request body data, decoding it with CBOR by default. + + Somewhat arbitrarily, limit body size to 1MiB by default. + """ + content_type = get_content_type(request.requestHeaders) + if content_type is None: + content_type = CBOR_MIME_TYPE + if content_type != CBOR_MIME_TYPE: + raise _HTTPError(http.UNSUPPORTED_MEDIA_TYPE) + + # Make sure it's not too large: + request.content.seek(0, SEEK_END) + size = request.content.tell() + if size > max_size: + raise _HTTPError(http.REQUEST_ENTITY_TOO_LARGE) + request.content.seek(0, SEEK_SET) + + # We don't want to load the whole message into memory, cause it might + # be quite large. The CDDL validator takes a read-only bytes-like + # thing. Luckily, for large request bodies twisted.web will buffer the + # data in a file, so we can use mmap() to get a memory view. The CDDL + # validator will not make a copy, so it won't increase memory usage + # beyond that. + try: + fd = request.content.fileno() + except (ValueError, OSError): + fd = -1 + if fd >= 0: + # It's a file, so we can use mmap() to save memory. + message = mmap.mmap(fd, 0, access=mmap.ACCESS_READ) + else: + message = request.content.read() + + # Pycddl will release the GIL when validating larger documents, so + # let's take advantage of multiple CPUs: + if size > 10_000: + await defer_to_thread(reactor, schema.validate_cbor, message) + else: + schema.validate_cbor(message) + + # The CBOR parser will allocate more memory, but at least we can feed + # it the file-like object, so that if it's large it won't be make two + # copies. + request.content.seek(SEEK_SET, 0) + # Typically deserialization to Python will not release the GIL, and + # indeed as of Jan 2023 cbor2 didn't have any code to release the GIL + # in the decode path. As such, running it in a different thread has no benefit. + return cbor2.load(request.content) + + class HTTPServer(object): """ A HTTP interface to the storage server. @@ -587,56 +641,6 @@ class HTTPServer(object): # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3861 raise _HTTPError(http.NOT_ACCEPTABLE) - async def _read_encoded( - self, request, schema: Schema, max_size: int = 1024 * 1024 - ) -> Any: - """ - Read encoded request body data, decoding it with CBOR by default. - - Somewhat arbitrarily, limit body size to 1MiB by default. - """ - content_type = get_content_type(request.requestHeaders) - if content_type != CBOR_MIME_TYPE: - raise _HTTPError(http.UNSUPPORTED_MEDIA_TYPE) - - # Make sure it's not too large: - request.content.seek(0, SEEK_END) - size = request.content.tell() - if size > max_size: - raise _HTTPError(http.REQUEST_ENTITY_TOO_LARGE) - request.content.seek(0, SEEK_SET) - - # We don't want to load the whole message into memory, cause it might - # be quite large. The CDDL validator takes a read-only bytes-like - # thing. Luckily, for large request bodies twisted.web will buffer the - # data in a file, so we can use mmap() to get a memory view. The CDDL - # validator will not make a copy, so it won't increase memory usage - # beyond that. - try: - fd = request.content.fileno() - except (ValueError, OSError): - fd = -1 - if fd >= 0: - # It's a file, so we can use mmap() to save memory. - message = mmap.mmap(fd, 0, access=mmap.ACCESS_READ) - else: - message = request.content.read() - - # Pycddl will release the GIL when validating larger documents, so - # let's take advantage of multiple CPUs: - if size > 10_000: - await defer_to_thread(self._reactor, schema.validate_cbor, message) - else: - schema.validate_cbor(message) - - # The CBOR parser will allocate more memory, but at least we can feed - # it the file-like object, so that if it's large it won't be make two - # copies. - request.content.seek(SEEK_SET, 0) - # Typically deserialization to Python will not release the GIL, and - # indeed as of Jan 2023 cbor2 didn't have any code to release the GIL - # in the decode path. As such, running it in a different thread has no benefit. - return cbor2.load(request.content) ##### Generic APIs ##### @@ -677,8 +681,8 @@ class HTTPServer(object): """Allocate buckets.""" upload_secret = authorization[Secrets.UPLOAD] # It's just a list of up to ~256 shares, shouldn't use many bytes. - info = await self._read_encoded( - request, _SCHEMAS["allocate_buckets"], max_size=8192 + info = await read_encoded( + self._reactor, request, _SCHEMAS["allocate_buckets"], max_size=8192 ) # We do NOT validate the upload secret for existing bucket uploads. @@ -849,7 +853,8 @@ class HTTPServer(object): # The reason can be a string with explanation, so in theory it could be # longish? - info = await self._read_encoded( + info = await read_encoded( + self._reactor, request, _SCHEMAS["advise_corrupt_share"], max_size=32768, @@ -868,8 +873,8 @@ class HTTPServer(object): @async_to_deferred async def mutable_read_test_write(self, request, authorization, storage_index): """Read/test/write combined operation for mutables.""" - rtw_request = await self._read_encoded( - request, _SCHEMAS["mutable_read_test_write"], max_size=2**48 + rtw_request = await read_encoded( + self._reactor, request, _SCHEMAS["mutable_read_test_write"], max_size=2**48 ) secrets = ( authorization[Secrets.WRITE_ENABLER], @@ -955,8 +960,8 @@ class HTTPServer(object): # The reason can be a string with explanation, so in theory it could be # longish? - info = await self._read_encoded( - request, _SCHEMAS["advise_corrupt_share"], max_size=32768 + info = await read_encoded( + self._reactor, request, _SCHEMAS["advise_corrupt_share"], max_size=32768 ) self._storage_server.advise_corrupt_share( b"mutable", storage_index, share_number, info["reason"].encode("utf-8") diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 52c47c6c3..48ca072bc 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -42,6 +42,7 @@ from werkzeug.exceptions import NotFound as WNotFound from testtools.matchers import Equals from zope.interface import implementer +from ..util.deferredutil import async_to_deferred from .common import SyncTestCase from ..storage.http_common import ( get_content_type, @@ -59,6 +60,8 @@ from ..storage.http_server import ( _authorized_route, StorageIndexConverter, _add_error_handling, + read_encoded, + _SCHEMAS as SERVER_SCHEMAS, ) from ..storage.http_client import ( StorageClient, @@ -303,6 +306,14 @@ class TestApp(object): request.transport.loseConnection() return Deferred() + @_authorized_route(_app, set(), "/read_body", methods=["POST"]) + @async_to_deferred + async def read_body(self, request, authorization): + data = await read_encoded( + self.clock, request, SERVER_SCHEMAS["advise_corrupt_share"] + ) + return data["reason"] + def result_of(d): """ @@ -320,6 +331,7 @@ def result_of(d): + "This is probably a test design issue." ) + class CustomHTTPServerTests(SyncTestCase): """ Tests that use a custom HTTP server. @@ -504,6 +516,40 @@ class CustomHTTPServerTests(SyncTestCase): result_of(d) self.assertEqual(len(self._http_server.clock.getDelayedCalls()), 0) + def test_request_with_no_content_type_same_as_cbor(self): + """ + If no ``Content-Type`` header is set when sending a body, it is assumed + to be CBOR. + """ + response = result_of( + self.client.request( + "POST", + DecodedURL.from_text("http://127.0.0.1/read_body"), + data=dumps({"reason": "test"}), + ) + ) + self.assertEqual( + result_of(limited_content(response, self._http_server.clock, 100)).read(), + b"test", + ) + + def test_request_with_wrong_content(self): + """ + If a non-CBOR ``Content-Type`` header is set when sending a body, the + server complains appropriatly. + """ + headers = Headers() + headers.setRawHeaders("content-type", ["some/value"]) + response = result_of( + self.client.request( + "POST", + DecodedURL.from_text("http://127.0.0.1/read_body"), + data=dumps({"reason": "test"}), + headers=headers, + ) + ) + self.assertEqual(response.code, http.UNSUPPORTED_MEDIA_TYPE) + @implementer(IReactorFromThreads) class Reactor(Clock): From 7bac6996d18dc9d5f5b48686990a66a50a51021a Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 25 Jul 2023 15:32:14 -0400 Subject: [PATCH 1980/2309] Updates based on changing specs. --- docs/proposed/http-storage-node-protocol.rst | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index 5009a992e..fed5bb5dc 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -279,7 +279,7 @@ Such an announcement will resemble this:: { "anonymous-storage-FURL": "pb://...", # The old key - "gbs-anonymous-storage-url": "pb://...#v=1" # The new key + "anonymous-storage-NURLs": ["pb://...#v=1"] # The new keys } The transition process will proceed in three stages: @@ -320,12 +320,7 @@ The follow sequence of events is likely: Ideally, the client would not rely on an update from the introducer to give it the GBS NURL for the updated storage server. -Therefore, -when an updated client connects to a storage server using Foolscap, -it should request the server's version information. -If this information indicates that GBS is supported then the client should cache this GBS information. -On subsequent connection attempts, -it should make use of this GBS information. +In practice, we have decided not to implement this functionality. Server Details -------------- From 792af1c9189e0d9c93d69eb450c680eb8eaa7de1 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 25 Jul 2023 15:32:30 -0400 Subject: [PATCH 1981/2309] News fragment --- newsfragments/4042.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/4042.minor diff --git a/newsfragments/4042.minor b/newsfragments/4042.minor new file mode 100644 index 000000000..e69de29bb From 15df1a52ff596c2464c2bac8404fb3b8022fda7f Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 26 Jul 2023 13:35:59 -0400 Subject: [PATCH 1982/2309] Minimal HTTPS documentation. --- docs/architecture.rst | 14 ++++++++++++++ docs/configuration.rst | 16 ++++++++++++++++ newsfragments/4039.documentation | 1 + 3 files changed, 31 insertions(+) create mode 100644 newsfragments/4039.documentation diff --git a/docs/architecture.rst b/docs/architecture.rst index 4cfabd844..64e81ea99 100644 --- a/docs/architecture.rst +++ b/docs/architecture.rst @@ -57,6 +57,20 @@ The key-value store is implemented by a grid of Tahoe-LAFS storage servers -- user-space processes. Tahoe-LAFS storage clients communicate with the storage servers over TCP. +There are two supported protocols: + +* Foolscap, the only supported protocol in release before v1.19. +* HTTPS, new in v1.19. + +By default HTTPS is disabled (this will change in +https://tahoe-lafs.org/trac/tahoe-lafs/ticket/4041). When HTTPS is enabled on +the server, the server transparently listens for both Foolscap and HTTP on the +same port. Clients can use either; by default they will only use Foolscap, but +when configured appropriately they will use HTTPS when possible (this will +change in https://tahoe-lafs.org/trac/tahoe-lafs/ticket/4041). At this time the +only limitations of HTTPS is that I2P is not supported, so any usage of I2P only +uses Foolscap. + Storage servers hold data in the form of "shares". Shares are encoded pieces of files. There are a configurable number of shares for each file, 10 by default. Normally, each share is stored on a separate server, but in some diff --git a/docs/configuration.rst b/docs/configuration.rst index 08521ad44..feb29c0ca 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -679,6 +679,14 @@ Client Configuration location to prefer their local servers so that they can maintain access to all of their uploads without using the internet. +``force_foolscap = (boolean, optional)`` + + If this is ``True``, the client will only connect to storage servers via + Foolscap, regardless of whether they support HTTPS. If this is ``False``, + the client will prefer HTTPS when it is available on the server. The default + value is ``True`` (this will change in + https://tahoe-lafs.org/trac/tahoe-lafs/ticket/4041). + In addition, see :doc:`accepting-donations` for a convention for donating to storage server operators. @@ -796,6 +804,14 @@ Storage Server Configuration (i.e. ``BASEDIR/storage``), but it can be placed elsewhere. Relative paths will be interpreted relative to the node's base directory. +``force_foolscap = (boolean, optional)`` + + If this is ``True``, the node will expose the storage server via Foolscap + only, with no support for HTTPS. If this is ``False``, the server will + support both Foolscap and HTTPS on the same port. The default value is + ``True`` (this will change in + https://tahoe-lafs.org/trac/tahoe-lafs/ticket/4041). + In addition, see :doc:`accepting-donations` for a convention encouraging donations to storage server operators. diff --git a/newsfragments/4039.documentation b/newsfragments/4039.documentation new file mode 100644 index 000000000..33257443b --- /dev/null +++ b/newsfragments/4039.documentation @@ -0,0 +1 @@ +Document the ``force_foolscap`` configuration options for ``[storage]`` and ``[client]``. \ No newline at end of file From 411827a5c3c2ad499ac294e93c5f2b62a247ce28 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 26 Jul 2023 13:44:21 -0400 Subject: [PATCH 1983/2309] Update docs. --- docs/architecture.rst | 14 ++++++-------- docs/configuration.rst | 6 ++---- newsfragments/4041.feature | 1 + 3 files changed, 9 insertions(+), 12 deletions(-) create mode 100644 newsfragments/4041.feature diff --git a/docs/architecture.rst b/docs/architecture.rst index 64e81ea99..0e22741f1 100644 --- a/docs/architecture.rst +++ b/docs/architecture.rst @@ -62,14 +62,12 @@ There are two supported protocols: * Foolscap, the only supported protocol in release before v1.19. * HTTPS, new in v1.19. -By default HTTPS is disabled (this will change in -https://tahoe-lafs.org/trac/tahoe-lafs/ticket/4041). When HTTPS is enabled on -the server, the server transparently listens for both Foolscap and HTTP on the -same port. Clients can use either; by default they will only use Foolscap, but -when configured appropriately they will use HTTPS when possible (this will -change in https://tahoe-lafs.org/trac/tahoe-lafs/ticket/4041). At this time the -only limitations of HTTPS is that I2P is not supported, so any usage of I2P only -uses Foolscap. +By default HTTPS is enabled. When HTTPS is enabled on the server, the server +transparently listens for both Foolscap and HTTP on the same port. When it is +disabled, the server only supports Foolscap. Clients can use either; by default +they will use HTTPS when possible, falling back to I2p, but when configured +appropriately they will only use Foolscap. At this time the only limitations of +HTTPS is that I2P is not supported, so any usage of I2P only uses Foolscap. Storage servers hold data in the form of "shares". Shares are encoded pieces of files. There are a configurable number of shares for each file, 10 by diff --git a/docs/configuration.rst b/docs/configuration.rst index feb29c0ca..7f038192e 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -684,8 +684,7 @@ Client Configuration If this is ``True``, the client will only connect to storage servers via Foolscap, regardless of whether they support HTTPS. If this is ``False``, the client will prefer HTTPS when it is available on the server. The default - value is ``True`` (this will change in - https://tahoe-lafs.org/trac/tahoe-lafs/ticket/4041). + value is ``False``. In addition, see :doc:`accepting-donations` for a convention for donating to storage server operators. @@ -809,8 +808,7 @@ Storage Server Configuration If this is ``True``, the node will expose the storage server via Foolscap only, with no support for HTTPS. If this is ``False``, the server will support both Foolscap and HTTPS on the same port. The default value is - ``True`` (this will change in - https://tahoe-lafs.org/trac/tahoe-lafs/ticket/4041). + ``False``. In addition, see :doc:`accepting-donations` for a convention encouraging donations to storage server operators. diff --git a/newsfragments/4041.feature b/newsfragments/4041.feature new file mode 100644 index 000000000..ea4c65171 --- /dev/null +++ b/newsfragments/4041.feature @@ -0,0 +1 @@ +The storage server now supports a new, HTTPS-based protocol. \ No newline at end of file From 0e72f3c97a90b34c138af273d8b4e75e913d9368 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 26 Jul 2023 13:45:56 -0400 Subject: [PATCH 1984/2309] Disable forcing Foolscap on client and server. --- src/allmydata/node.py | 5 +---- src/allmydata/storage_client.py | 2 +- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/src/allmydata/node.py b/src/allmydata/node.py index 6c3082b50..33e8fd260 100644 --- a/src/allmydata/node.py +++ b/src/allmydata/node.py @@ -959,11 +959,8 @@ def create_main_tub(config, tub_options, tub_options, default_connection_handlers, foolscap_connection_handlers, - # TODO eventually we will want the default to be False, but for now we - # don't want to enable HTTP by default. - # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3934 force_foolscap=config.get_config( - "storage", "force_foolscap", default=True, boolean=True + "storage", "force_foolscap", default=False, boolean=True ), handler_overrides=handler_overrides, certFile=certfile, diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 4efc845b4..b58131837 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -293,7 +293,7 @@ class StorageFarmBroker(service.MultiService): connect to storage server over HTTP. """ return not node_config.get_config( - "client", "force_foolscap", default=True, boolean=True, + "client", "force_foolscap", default=False, boolean=True, ) and len(announcement.get(ANONYMOUS_STORAGE_NURLS, [])) > 0 @log_call( From aef6373915cbf3f55ac8e9d1309fbf250d35fa96 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 26 Jul 2023 14:05:15 -0400 Subject: [PATCH 1985/2309] Update unit tests to support HTTPS storage protocol on by default. --- src/allmydata/test/matchers.py | 12 ++++++++++++ src/allmydata/test/test_storage_client.py | 5 +++-- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/matchers.py b/src/allmydata/test/matchers.py index 3359a7ed5..cc8bf47be 100644 --- a/src/allmydata/test/matchers.py +++ b/src/allmydata/test/matchers.py @@ -13,6 +13,7 @@ if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import attr +from hyperlink import DecodedURL from testtools.matchers import ( Mismatch, @@ -95,6 +96,7 @@ def matches_storage_announcement(basedir, anonymous=True, options=None): } if anonymous: announcement[u"anonymous-storage-FURL"] = matches_furl() + announcement[u"anonymous-storage-NURLs"] = matches_nurls() if options: announcement[u"storage-options"] = MatchesListwise(options) return MatchesStructure( @@ -112,6 +114,16 @@ def matches_furl(): return AfterPreprocessing(decode_furl, Always()) +def matches_nurls(): + """ + Matches a sequence of NURLs. + """ + return AfterPreprocessing( + lambda nurls: [DecodedURL.from_text(u) for u in nurls], + Always() + ) + + def matches_base32(): """ Match any base32 encoded byte string. diff --git a/src/allmydata/test/test_storage_client.py b/src/allmydata/test/test_storage_client.py index 0671526ae..79bc475d0 100644 --- a/src/allmydata/test/test_storage_client.py +++ b/src/allmydata/test/test_storage_client.py @@ -742,13 +742,14 @@ storage: self.assertTrue(done.called) def test_should_we_use_http_default(self): - """Default is to not use HTTP; this will change eventually""" + """Default is to use HTTP; this will change eventually""" basedir = self.mktemp() node_config = config_from_string(basedir, "", "") announcement = {ANONYMOUS_STORAGE_NURLS: ["pb://..."]} - self.assertFalse( + self.assertTrue( StorageFarmBroker._should_we_use_http(node_config, announcement) ) + # Lacking NURLs, we can't use HTTP: self.assertFalse( StorageFarmBroker._should_we_use_http(node_config, {}) ) From 849f4ed2a57da1e2dd19b668dccba5967534224c Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 28 Jul 2023 11:14:09 -0400 Subject: [PATCH 1986/2309] More annotations. --- src/allmydata/storage/http_client.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 765e94319..79f6cfa89 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -72,7 +72,7 @@ except ImportError: pass -def _encode_si(si): # type: (bytes) -> str +def _encode_si(si: bytes) -> str: """Encode the storage index into Unicode string.""" return str(si_b2a(si), "ascii") @@ -80,7 +80,7 @@ def _encode_si(si): # type: (bytes) -> str class ClientException(Exception): """An unexpected response code from the server.""" - def __init__(self, code, *additional_args): + def __init__(self, code: int, *additional_args): Exception.__init__(self, code, *additional_args) self.code = code @@ -93,7 +93,7 @@ register_exception_extractor(ClientException, lambda e: {"response_code": e.code # Tags are of the form #6.nnn, where the number is documented at # https://www.iana.org/assignments/cbor-tags/cbor-tags.xhtml. Notably, #6.258 # indicates a set. -_SCHEMAS = { +_SCHEMAS : Mapping[str,Schema] = { "get_version": Schema( # Note that the single-quoted (`'`) string keys in this schema # represent *byte* strings - per the CDDL specification. Text strings @@ -155,7 +155,7 @@ class _LengthLimitedCollector: timeout_on_silence: IDelayedCall f: BytesIO = field(factory=BytesIO) - def __call__(self, data: bytes): + def __call__(self, data: bytes) -> None: self.timeout_on_silence.reset(60) self.remaining_length -= len(data) if self.remaining_length < 0: @@ -164,7 +164,7 @@ class _LengthLimitedCollector: def limited_content( - response, + response: IResponse, clock: IReactorTime, max_length: int = 30 * 1024 * 1024, ) -> Deferred[BinaryIO]: @@ -300,11 +300,11 @@ class _StorageClientHTTPSPolicy: expected_spki_hash: bytes # IPolicyForHTTPS - def creatorForNetloc(self, hostname, port): + def creatorForNetloc(self, hostname: str, port: int) -> _StorageClientHTTPSPolicy: return self # IOpenSSLClientConnectionCreator - def clientConnectionForTLS(self, tlsProtocol): + def clientConnectionForTLS(self, tlsProtocol: object) -> SSL.Connection: return SSL.Connection( _TLSContextFactory(self.expected_spki_hash).getContext(), None ) @@ -344,7 +344,7 @@ class StorageClientFactory: cls.TEST_MODE_REGISTER_HTTP_POOL = callback @classmethod - def stop_test_mode(cls): + def stop_test_mode(cls) -> None: """Stop testing mode.""" cls.TEST_MODE_REGISTER_HTTP_POOL = None @@ -437,7 +437,7 @@ class StorageClient(object): """Get a URL relative to the base URL.""" return self._base_url.click(path) - def _get_headers(self, headers): # type: (Optional[Headers]) -> Headers + def _get_headers(self, headers: Optional[Headers]) -> Headers: """Return the basic headers to be used by default.""" if headers is None: headers = Headers() @@ -565,7 +565,7 @@ class StorageClient(object): ).read() raise ClientException(response.code, response.phrase, data) - def shutdown(self) -> Deferred: + def shutdown(self) -> Deferred[object]: """Shutdown any connections.""" return self._pool.closeCachedConnections() From 2b7f3d1707b6e91ea8e517bcfa3a6cf892d1715e Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 28 Jul 2023 11:28:13 -0400 Subject: [PATCH 1987/2309] Add type annotations to `_authorization_decorator`. --- src/allmydata/storage/http_server.py | 44 +++++++++++++++++++------ src/allmydata/test/test_storage_http.py | 3 +- 2 files changed, 36 insertions(+), 11 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 3ff98e933..78ed1a974 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -4,7 +4,8 @@ HTTP server for storage. from __future__ import annotations -from typing import Any, Callable, Union, cast, Optional +from typing import Any, Callable, Union, cast, Optional, TypeVar, Sequence +from typing_extensions import ParamSpec, Concatenate from functools import wraps from base64 import b64decode import binascii @@ -27,6 +28,7 @@ from twisted.internet.defer import Deferred from twisted.internet.ssl import CertificateOptions, Certificate, PrivateCertificate from twisted.internet.interfaces import IReactorFromThreads from twisted.web.server import Site, Request +from twisted.web.iweb import IRequest from twisted.protocols.tls import TLSMemoryBIOFactory from twisted.python.filepath import FilePath @@ -68,7 +70,7 @@ class ClientSecretsException(Exception): def _extract_secrets( - header_values: list[str], required_secrets: set[Secrets] + header_values: Sequence[str], required_secrets: set[Secrets] ) -> dict[Secrets, bytes]: """ Given list of values of ``X-Tahoe-Authorization`` headers, and required @@ -102,18 +104,32 @@ def _extract_secrets( return result -def _authorization_decorator(required_secrets): +P = ParamSpec("P") +T = TypeVar("T") + + +def _authorization_decorator( + required_secrets: set[Secrets], +) -> Callable[ + [Callable[Concatenate[BaseApp, Request, dict[Secrets, bytes], P], T]], + Callable[Concatenate[BaseApp, Request, P], T], +]: """ 1. Check the ``Authorization`` header matches server swissnum. 2. Extract ``X-Tahoe-Authorization`` headers and pass them in. 3. Log the request and response. """ - def decorator(f): + def decorator( + f: Callable[Concatenate[BaseApp, Request, dict[Secrets, bytes], P], T] + ) -> Callable[Concatenate[BaseApp, Request, P], T]: @wraps(f) - def route(self, request, *args, **kwargs): - # Don't set text/html content type by default: - request.defaultContentType = None + def route( + self: BaseApp, request: Request, *args: P.args, **kwargs: P.kwargs + ) -> T: + # Don't set text/html content type by default. + # None is actually supported, see https://github.com/twisted/twisted/issues/11902 + request.defaultContentType = None # type: ignore[assignment] with start_action( action_type="allmydata:storage:http-server:handle-request", @@ -584,7 +600,13 @@ async def read_encoded( return cbor2.load(request.content) -class HTTPServer(object): +class BaseApp: + """Base class for ``HTTPServer`` and testing equivalent.""" + + _swissnum: bytes + + +class HTTPServer(BaseApp): """ A HTTP interface to the storage server. """ @@ -641,7 +663,6 @@ class HTTPServer(object): # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3861 raise _HTTPError(http.NOT_ACCEPTABLE) - ##### Generic APIs ##### @_authorized_route(_app, set(), "/storage/v1/version", methods=["GET"]) @@ -874,7 +895,10 @@ class HTTPServer(object): async def mutable_read_test_write(self, request, authorization, storage_index): """Read/test/write combined operation for mutables.""" rtw_request = await read_encoded( - self._reactor, request, _SCHEMAS["mutable_read_test_write"], max_size=2**48 + self._reactor, + request, + _SCHEMAS["mutable_read_test_write"], + max_size=2**48, ) secrets = ( authorization[Secrets.WRITE_ENABLER], diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 48ca072bc..1a334034d 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -62,6 +62,7 @@ from ..storage.http_server import ( _add_error_handling, read_encoded, _SCHEMAS as SERVER_SCHEMAS, + BaseApp, ) from ..storage.http_client import ( StorageClient, @@ -257,7 +258,7 @@ def gen_bytes(length: int) -> bytes: return result -class TestApp(object): +class TestApp(BaseApp): """HTTP API for testing purposes.""" clock: IReactorTime From 919e6b339d0eaf7019f231ad916b2f7ac25cef48 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 28 Jul 2023 12:58:22 -0400 Subject: [PATCH 1988/2309] Add type annotation to _authorized_route --- src/allmydata/storage/http_server.py | 77 ++++++++++++++++++------- src/allmydata/test/test_storage_http.py | 2 +- 2 files changed, 56 insertions(+), 23 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 78ed1a974..7ceb8328c 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -4,7 +4,7 @@ HTTP server for storage. from __future__ import annotations -from typing import Any, Callable, Union, cast, Optional, TypeVar, Sequence +from typing import Any, Callable, Union, cast, Optional, TypeVar, Sequence, Protocol from typing_extensions import ParamSpec, Concatenate from functools import wraps from base64 import b64decode @@ -16,7 +16,7 @@ import mmap from eliot import start_action from cryptography.x509 import Certificate as CryptoCertificate from zope.interface import implementer -from klein import Klein +from klein import Klein, KleinRenderable from twisted.web import http from twisted.internet.interfaces import ( IListeningPort, @@ -104,15 +104,23 @@ def _extract_secrets( return result +class BaseApp(Protocol): + """Protocol for ``HTTPServer`` and testing equivalent.""" + + _swissnum: bytes + + P = ParamSpec("P") T = TypeVar("T") +SecretsDict = dict[Secrets, bytes] +App = TypeVar("App", bound=BaseApp) def _authorization_decorator( required_secrets: set[Secrets], ) -> Callable[ - [Callable[Concatenate[BaseApp, Request, dict[Secrets, bytes], P], T]], - Callable[Concatenate[BaseApp, Request, P], T], + [Callable[Concatenate[App, Request, SecretsDict, P], T]], + Callable[Concatenate[App, Request, P], T], ]: """ 1. Check the ``Authorization`` header matches server swissnum. @@ -121,11 +129,14 @@ def _authorization_decorator( """ def decorator( - f: Callable[Concatenate[BaseApp, Request, dict[Secrets, bytes], P], T] - ) -> Callable[Concatenate[BaseApp, Request, P], T]: + f: Callable[Concatenate[App, Request, SecretsDict, P], T] + ) -> Callable[Concatenate[App, Request, P], T]: @wraps(f) def route( - self: BaseApp, request: Request, *args: P.args, **kwargs: P.kwargs + self: App, + request: Request, + *args: P.args, + **kwargs: P.kwargs, ) -> T: # Don't set text/html content type by default. # None is actually supported, see https://github.com/twisted/twisted/issues/11902 @@ -179,7 +190,22 @@ def _authorization_decorator( return decorator -def _authorized_route(app, required_secrets, *route_args, **route_kwargs): +def _authorized_route( + klein_app: Klein, + required_secrets: set[Secrets], + url: str, + *route_args: Any, + branch: bool = False, + **route_kwargs: Any, +) -> Callable[ + [ + Callable[ + Concatenate[App, Request, SecretsDict, P], + KleinRenderable, + ] + ], + Callable[..., KleinRenderable], +]: """ Like Klein's @route, but with additional support for checking the ``Authorization`` header as well as ``X-Tahoe-Authorization`` headers. The @@ -189,12 +215,23 @@ def _authorized_route(app, required_secrets, *route_args, **route_kwargs): :param required_secrets: Set of required ``Secret`` types. """ - def decorator(f): - @app.route(*route_args, **route_kwargs) + def decorator( + f: Callable[ + Concatenate[App, Request, SecretsDict, P], + KleinRenderable, + ] + ) -> Callable[..., KleinRenderable]: + @klein_app.route(url, *route_args, branch=branch, **route_kwargs) # type: ignore[arg-type] @_authorization_decorator(required_secrets) @wraps(f) - def handle_route(*args, **kwargs): - return f(*args, **kwargs) + def handle_route( + app: App, + request: Request, + secrets: SecretsDict, + *args: P.args, + **kwargs: P.kwargs, + ) -> KleinRenderable: + return f(app, request, secrets, *args, **kwargs) return handle_route @@ -367,7 +404,7 @@ class _ReadAllProducer: start: int = field(default=0) @classmethod - def produce_to(cls, request: Request, read_data: ReadData) -> Deferred: + def produce_to(cls, request: Request, read_data: ReadData) -> Deferred[bytes]: """ Create and register the producer, returning ``Deferred`` that should be returned from a HTTP server endpoint. @@ -600,12 +637,6 @@ async def read_encoded( return cbor2.load(request.content) -class BaseApp: - """Base class for ``HTTPServer`` and testing equivalent.""" - - _swissnum: bytes - - class HTTPServer(BaseApp): """ A HTTP interface to the storage server. @@ -637,7 +668,7 @@ class HTTPServer(BaseApp): """Return twisted.web ``Resource`` for this object.""" return self._app.resource() - def _send_encoded(self, request, data): + def _send_encoded(self, request: Request, data: object) -> Deferred[bytes]: """ Return encoded data suitable for writing as the HTTP body response, by default using CBOR. @@ -666,7 +697,7 @@ class HTTPServer(BaseApp): ##### Generic APIs ##### @_authorized_route(_app, set(), "/storage/v1/version", methods=["GET"]) - def version(self, request, authorization): + def version(self, request: Request, authorization: SecretsDict) -> KleinRenderable: """Return version information.""" return self._send_encoded(request, self._get_version()) @@ -698,7 +729,9 @@ class HTTPServer(BaseApp): methods=["POST"], ) @async_to_deferred - async def allocate_buckets(self, request, authorization, storage_index): + async def allocate_buckets( + self, request: Request, authorization: SecretsDict, storage_index: bytes + ) -> KleinRenderable: """Allocate buckets.""" upload_secret = authorization[Secrets.UPLOAD] # It's just a list of up to ~256 shares, shouldn't use many bytes. diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 1a334034d..e7b8059ee 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -266,7 +266,7 @@ class TestApp(BaseApp): _add_error_handling(_app) _swissnum = SWISSNUM_FOR_TEST # Match what the test client is using - @_authorized_route(_app, {}, "/noop", methods=["GET"]) + @_authorized_route(_app, set(), "/noop", methods=["GET"]) def noop(self, request, authorization): return "noop" From d669099a3515b4ff8c1524bc43f8fcd74782560a Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 28 Jul 2023 13:28:02 -0400 Subject: [PATCH 1989/2309] Add more type annotations. --- src/allmydata/storage/http_server.py | 69 ++++++++++++++++++++++------ 1 file changed, 55 insertions(+), 14 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 7ceb8328c..0cf3d25f4 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -770,7 +770,13 @@ class HTTPServer(BaseApp): "/storage/v1/immutable///abort", methods=["PUT"], ) - def abort_share_upload(self, request, authorization, storage_index, share_number): + def abort_share_upload( + self, + request: Request, + authorization: SecretsDict, + storage_index: bytes, + share_number: int, + ) -> KleinRenderable: """Abort an in-progress immutable share upload.""" try: bucket = self._uploads.get_write_bucket( @@ -801,7 +807,13 @@ class HTTPServer(BaseApp): "/storage/v1/immutable//", methods=["PATCH"], ) - def write_share_data(self, request, authorization, storage_index, share_number): + def write_share_data( + self, + request: Request, + authorization: SecretsDict, + storage_index: bytes, + share_number: int, + ) -> KleinRenderable: """Write data to an in-progress immutable upload.""" content_range = parse_content_range_header(request.getHeader("content-range")) if content_range is None or content_range.units != "bytes": @@ -811,14 +823,17 @@ class HTTPServer(BaseApp): bucket = self._uploads.get_write_bucket( storage_index, share_number, authorization[Secrets.UPLOAD] ) - offset = content_range.start - remaining = content_range.stop - content_range.start + offset = content_range.start or 0 + # We don't support an unspecified stop for the range: + assert content_range.stop is not None + # Missing body makes no sense: + assert request.content is not None + remaining = content_range.stop - offset finished = False while remaining > 0: data = request.content.read(min(remaining, 65536)) assert data, "uploaded data length doesn't match range" - try: finished = bucket.write(offset, data) except ConflictingWriteError: @@ -844,7 +859,9 @@ class HTTPServer(BaseApp): "/storage/v1/immutable//shares", methods=["GET"], ) - def list_shares(self, request, authorization, storage_index): + def list_shares( + self, request: Request, authorization: SecretsDict, storage_index: bytes + ) -> KleinRenderable: """ List shares for the given storage index. """ @@ -857,7 +874,13 @@ class HTTPServer(BaseApp): "/storage/v1/immutable//", methods=["GET"], ) - def read_share_chunk(self, request, authorization, storage_index, share_number): + def read_share_chunk( + self, + request: Request, + authorization: SecretsDict, + storage_index: bytes, + share_number: int, + ) -> KleinRenderable: """Read a chunk for an already uploaded immutable.""" request.setHeader("content-type", "application/octet-stream") try: @@ -874,7 +897,9 @@ class HTTPServer(BaseApp): "/storage/v1/lease/", methods=["PUT"], ) - def add_or_renew_lease(self, request, authorization, storage_index): + def add_or_renew_lease( + self, request: Request, authorization: SecretsDict, storage_index: bytes + ) -> KleinRenderable: """Update the lease for an immutable or mutable share.""" if not list(self._storage_server.get_shares(storage_index)): raise _HTTPError(http.NOT_FOUND) @@ -897,8 +922,12 @@ class HTTPServer(BaseApp): ) @async_to_deferred async def advise_corrupt_share_immutable( - self, request, authorization, storage_index, share_number - ): + self, + request: Request, + authorization: SecretsDict, + storage_index: bytes, + share_number: int, + ) -> KleinRenderable: """Indicate that given share is corrupt, with a text reason.""" try: bucket = self._storage_server.get_buckets(storage_index)[share_number] @@ -925,7 +954,9 @@ class HTTPServer(BaseApp): methods=["POST"], ) @async_to_deferred - async def mutable_read_test_write(self, request, authorization, storage_index): + async def mutable_read_test_write( + self, request: Request, authorization: SecretsDict, storage_index: bytes + ) -> KleinRenderable: """Read/test/write combined operation for mutables.""" rtw_request = await read_encoded( self._reactor, @@ -967,7 +998,13 @@ class HTTPServer(BaseApp): "/storage/v1/mutable//", methods=["GET"], ) - def read_mutable_chunk(self, request, authorization, storage_index, share_number): + def read_mutable_chunk( + self, + request: Request, + authorization: SecretsDict, + storage_index: bytes, + share_number: int, + ) -> KleinRenderable: """Read a chunk from a mutable.""" request.setHeader("content-type", "application/octet-stream") @@ -1007,8 +1044,12 @@ class HTTPServer(BaseApp): ) @async_to_deferred async def advise_corrupt_share_mutable( - self, request, authorization, storage_index, share_number - ): + self, + request: Request, + authorization: SecretsDict, + storage_index: bytes, + share_number: int, + ) -> KleinRenderable: """Indicate that given share is corrupt, with a text reason.""" if share_number not in { shnum for (shnum, _) in self._storage_server.get_shares(storage_index) From 0d0e32646fe305637d4cebedd8c9e4427db9fedd Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 28 Jul 2023 13:42:00 -0400 Subject: [PATCH 1990/2309] More type annotations. --- src/allmydata/storage/http_server.py | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 0cf3d25f4..ce07d8f2e 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -31,6 +31,7 @@ from twisted.web.server import Site, Request from twisted.web.iweb import IRequest from twisted.protocols.tls import TLSMemoryBIOFactory from twisted.python.filepath import FilePath +from twisted.python.failure import Failure from attrs import define, field, Factory from werkzeug.http import ( @@ -287,7 +288,7 @@ class UploadsInProgress(object): except (KeyError, IndexError): raise _HTTPError(http.NOT_FOUND) - def remove_write_bucket(self, bucket: BucketWriter): + def remove_write_bucket(self, bucket: BucketWriter) -> None: """Stop tracking the given ``BucketWriter``.""" try: storage_index, share_number = self._bucketwriters.pop(bucket) @@ -303,7 +304,7 @@ class UploadsInProgress(object): def validate_upload_secret( self, storage_index: bytes, share_number: int, upload_secret: bytes - ): + ) -> None: """ Raise an unauthorized-HTTP-response exception if the given storage_index+share_number have a different upload secret than the @@ -325,7 +326,7 @@ class StorageIndexConverter(BaseConverter): regex = "[" + str(rfc3548_alphabet, "ascii") + "]{26}" - def to_python(self, value): + def to_python(self, value: str) -> bytes: try: return si_a2b(value.encode("ascii")) except (AssertionError, binascii.Error, ValueError): @@ -413,7 +414,7 @@ class _ReadAllProducer: request.registerProducer(producer, False) return producer.result - def resumeProducing(self): + def resumeProducing(self) -> None: data = self.read_data(self.start, 65536) if not data: self.request.unregisterProducer() @@ -424,10 +425,10 @@ class _ReadAllProducer: self.request.write(data) self.start += len(data) - def pauseProducing(self): + def pauseProducing(self) -> None: pass - def stopProducing(self): + def stopProducing(self) -> None: pass @@ -445,7 +446,7 @@ class _ReadRangeProducer: start: int remaining: int - def resumeProducing(self): + def resumeProducing(self) -> None: if self.result is None or self.request is None: return @@ -482,10 +483,10 @@ class _ReadRangeProducer: if self.remaining == 0: self.stopProducing() - def pauseProducing(self): + def pauseProducing(self) -> None: pass - def stopProducing(self): + def stopProducing(self) -> None: if self.request is not None: self.request.unregisterProducer() self.request = None @@ -564,12 +565,13 @@ def read_range( return d -def _add_error_handling(app: Klein): +def _add_error_handling(app: Klein) -> None: """Add exception handlers to a Klein app.""" @app.handle_errors(_HTTPError) - def _http_error(_, request, failure): + def _http_error(self: Any, request: IRequest, failure: Failure) -> KleinRenderable: """Handle ``_HTTPError`` exceptions.""" + assert isinstance(failure.value, _HTTPError) request.setResponseCode(failure.value.code) if failure.value.body is not None: return failure.value.body @@ -577,7 +579,9 @@ def _add_error_handling(app: Klein): return b"" @app.handle_errors(CDDLValidationError) - def _cddl_validation_error(_, request, failure): + def _cddl_validation_error( + self: Any, request: IRequest, failure: Failure + ) -> KleinRenderable: """Handle CDDL validation errors.""" request.setResponseCode(http.BAD_REQUEST) return str(failure.value).encode("utf-8") From 00b7e7e17862335edd08b0b38b28f30f64b8f993 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 28 Jul 2023 13:48:43 -0400 Subject: [PATCH 1991/2309] More type annotations. --- src/allmydata/storage/http_server.py | 11 ++++++++--- src/allmydata/test/test_storage_https.py | 6 ++++-- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index ce07d8f2e..cf0e6dbb4 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -17,11 +17,13 @@ from eliot import start_action from cryptography.x509 import Certificate as CryptoCertificate from zope.interface import implementer from klein import Klein, KleinRenderable +from klein.resource import KleinResource from twisted.web import http from twisted.internet.interfaces import ( IListeningPort, IStreamServerEndpoint, IPullProducer, + IProtocolFactory, ) from twisted.internet.address import IPv4Address, IPv6Address from twisted.internet.defer import Deferred @@ -668,7 +670,7 @@ class HTTPServer(BaseApp): self._uploads.remove_write_bucket ) - def get_resource(self): + def get_resource(self) -> KleinResource: """Return twisted.web ``Resource`` for this object.""" return self._app.resource() @@ -1085,7 +1087,10 @@ class _TLSEndpointWrapper(object): @classmethod def from_paths( - cls, endpoint, private_key_path: FilePath, cert_path: FilePath + cls: type[_TLSEndpointWrapper], + endpoint: IStreamServerEndpoint, + private_key_path: FilePath, + cert_path: FilePath, ) -> "_TLSEndpointWrapper": """ Create an endpoint with the given private key and certificate paths on @@ -1100,7 +1105,7 @@ class _TLSEndpointWrapper(object): ) return cls(endpoint=endpoint, context_factory=certificate_options) - def listen(self, factory): + def listen(self, factory: IProtocolFactory) -> Deferred[IListeningPort]: return self.endpoint.listen( TLSMemoryBIOFactory(self.context_factory, False, factory) ) diff --git a/src/allmydata/test/test_storage_https.py b/src/allmydata/test/test_storage_https.py index a11b0eed5..0e0bbcc95 100644 --- a/src/allmydata/test/test_storage_https.py +++ b/src/allmydata/test/test_storage_https.py @@ -109,9 +109,11 @@ class PinningHTTPSValidation(AsyncTestCase): root.isLeaf = True listening_port = await endpoint.listen(Site(root)) try: - yield f"https://127.0.0.1:{listening_port.getHost().port}/" + yield f"https://127.0.0.1:{listening_port.getHost().port}/" # type: ignore[attr-defined] finally: - await listening_port.stopListening() + result = listening_port.stopListening() + if result is not None: + await result def request(self, url: str, expected_certificate: x509.Certificate): """ From f8e9631f532da65c46ef3d04039b34469b6ab11a Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 28 Jul 2023 13:49:21 -0400 Subject: [PATCH 1992/2309] News fragment. --- newsfragments/4052.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/4052.minor diff --git a/newsfragments/4052.minor b/newsfragments/4052.minor new file mode 100644 index 000000000..e69de29bb From 176fac7360f3797cb637d8fd9d90ba05c3fbe548 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 28 Jul 2023 14:20:05 -0400 Subject: [PATCH 1993/2309] Work in Python 3.8. --- src/allmydata/storage/http_server.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index cf0e6dbb4..66b0dd6de 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -4,7 +4,17 @@ HTTP server for storage. from __future__ import annotations -from typing import Any, Callable, Union, cast, Optional, TypeVar, Sequence, Protocol +from typing import ( + Any, + Callable, + Union, + cast, + Optional, + TypeVar, + Sequence, + Protocol, + Dict, +) from typing_extensions import ParamSpec, Concatenate from functools import wraps from base64 import b64decode @@ -115,7 +125,7 @@ class BaseApp(Protocol): P = ParamSpec("P") T = TypeVar("T") -SecretsDict = dict[Secrets, bytes] +SecretsDict = Dict[Secrets, bytes] App = TypeVar("App", bound=BaseApp) From 050ef6cca3d19b20f76b7d4bf80b2d82f30f2af6 Mon Sep 17 00:00:00 2001 From: meejah Date: Sat, 29 Jul 2023 04:04:05 -0600 Subject: [PATCH 1994/2309] tor-tests work; refactor ports --- integration/conftest.py | 49 ++++++++++++++++++++++++++++++++++++----- integration/test_i2p.py | 7 +++--- integration/test_tor.py | 15 ++++++------- integration/util.py | 2 +- 4 files changed, 55 insertions(+), 18 deletions(-) diff --git a/integration/conftest.py b/integration/conftest.py index 837b54aa1..92483da65 100644 --- a/integration/conftest.py +++ b/integration/conftest.py @@ -7,6 +7,7 @@ from __future__ import annotations import os import sys import shutil +from attr import define from time import sleep from os import mkdir, environ from os.path import join, exists @@ -189,7 +190,7 @@ def introducer_furl(introducer, temp_dir): include_args=["temp_dir", "flog_gatherer"], include_result=False, ) -def tor_introducer(reactor, temp_dir, flog_gatherer, request): +def tor_introducer(reactor, temp_dir, flog_gatherer, request, tor_network): intro_dir = join(temp_dir, 'introducer_tor') print("making Tor introducer in {}".format(intro_dir)) print("(this can take tens of seconds to allocate Onion address)") @@ -203,9 +204,7 @@ def tor_introducer(reactor, temp_dir, flog_gatherer, request): request, ( 'create-introducer', - # The control port should agree with the configuration of the - # Tor network we bootstrap with chutney. - '--tor-control-port', 'tcp:localhost:8007', + '--tor-control-port', tor_network.client_control_endpoint, '--hide-ip', '--listen=tor', intro_dir, @@ -306,6 +305,21 @@ def bob(reactor, temp_dir, introducer_furl, flog_gatherer, storage_nodes, reques @pytest.mark.skipif(sys.platform.startswith('win'), 'Tor tests are unstable on Windows') def chutney(reactor, temp_dir: str) -> tuple[str, dict[str, str]]: + """ + Instantiate the "networks/hs-v3" Chutney configuration for a local + Tor network. + + This provides a small, local Tor network that can run v3 Onion + Services. This has 10 tor processes: 3 authorities, 5 + exits+relays, a client (and one service-hosting node we don't use). + + We pin a Chutney revision, so things shouldn't change. Currently, + the ONLY node that exposes a valid SocksPort is "008c" (the + client) on 9008. + + The control ports start at 8000 (so the ControlPort for the one + client node is 8008). + """ # Try to find Chutney already installed in the environment. try: import chutney @@ -363,7 +377,24 @@ def chutney(reactor, temp_dir: str) -> tuple[str, dict[str, str]]: ) pytest_twisted.blockon(proto.done) - return (chutney_dir, {"PYTHONPATH": join(chutney_dir, "lib")}) + return chutney_dir, {"PYTHONPATH": join(chutney_dir, "lib")} + + +@define +class ChutneyTorNetwork: + """ + Represents a running Chutney (tor) network. Returned by the + "tor_network" fixture. + """ + dir: FilePath + environ: dict + client_control_port: int + + @property + def client_control_endpoint(self) -> str: + print("CONTROL", "tcp:localhost:{}".format(self.client_control_port)) + return "tcp:localhost:{}".format(self.client_control_port) + @pytest.fixture(scope='session') @@ -422,3 +453,11 @@ def tor_network(reactor, temp_dir, chutney, request): pytest_twisted.blockon(chutney(("status", basic_network))) except ProcessTerminated: print("Chutney.TorNet status failed (continuing)") + + # the "8008" comes from configuring "networks/basic" in chutney + # and then examining "net/nodes/008c/torrc" for ControlPort value + return ChutneyTorNetwork( + chutney_root, + chutney_env, + 8008, + ) diff --git a/integration/test_i2p.py b/integration/test_i2p.py index 2ee603573..ea3ddb62b 100644 --- a/integration/test_i2p.py +++ b/integration/test_i2p.py @@ -132,8 +132,8 @@ def i2p_introducer_furl(i2p_introducer, temp_dir): @pytest_twisted.inlineCallbacks @pytest.mark.skip("I2P tests are not functioning at all, for unknown reasons") def test_i2p_service_storage(reactor, request, temp_dir, flog_gatherer, i2p_network, i2p_introducer_furl): - yield _create_anonymous_node(reactor, 'carol_i2p', 8008, request, temp_dir, flog_gatherer, i2p_network, i2p_introducer_furl) - yield _create_anonymous_node(reactor, 'dave_i2p', 8009, request, temp_dir, flog_gatherer, i2p_network, i2p_introducer_furl) + yield _create_anonymous_node(reactor, 'carol_i2p', request, temp_dir, flog_gatherer, i2p_network, i2p_introducer_furl) + yield _create_anonymous_node(reactor, 'dave_i2p', request, temp_dir, flog_gatherer, i2p_network, i2p_introducer_furl) # ensure both nodes are connected to "a grid" by uploading # something via carol, and retrieve it using dave. gold_path = join(temp_dir, "gold") @@ -179,9 +179,8 @@ def test_i2p_service_storage(reactor, request, temp_dir, flog_gatherer, i2p_netw @pytest_twisted.inlineCallbacks -def _create_anonymous_node(reactor, name, control_port, request, temp_dir, flog_gatherer, i2p_network, introducer_furl): +def _create_anonymous_node(reactor, name, request, temp_dir, flog_gatherer, i2p_network, introducer_furl): node_dir = FilePath(temp_dir).child(name) - web_port = "tcp:{}:interface=localhost".format(control_port + 2000) print("creating", node_dir.path) node_dir.makedirs() diff --git a/integration/test_tor.py b/integration/test_tor.py index 4d0ce4f16..d7fed5790 100644 --- a/integration/test_tor.py +++ b/integration/test_tor.py @@ -38,8 +38,8 @@ def test_onion_service_storage(reactor, request, temp_dir, flog_gatherer, tor_ne The two nodes can talk to the introducer and each other: we upload to one node, read from the other. """ - carol = yield _create_anonymous_node(reactor, 'carol', 8008, request, temp_dir, flog_gatherer, tor_network, tor_introducer_furl, 2) - dave = yield _create_anonymous_node(reactor, 'dave', 8009, request, temp_dir, flog_gatherer, tor_network, tor_introducer_furl, 2) + carol = yield _create_anonymous_node(reactor, 'carol', 8100, request, temp_dir, flog_gatherer, tor_network, tor_introducer_furl, 2) + dave = yield _create_anonymous_node(reactor, 'dave', 8101, request, temp_dir, flog_gatherer, tor_network, tor_introducer_furl, 2) yield util.await_client_ready(carol, minimum_number_of_servers=2, timeout=600) yield util.await_client_ready(dave, minimum_number_of_servers=2, timeout=600) yield upload_to_one_download_from_the_other(reactor, temp_dir, carol, dave) @@ -94,9 +94,8 @@ async def upload_to_one_download_from_the_other(reactor, temp_dir, upload_to: ut @pytest_twisted.inlineCallbacks -def _create_anonymous_node(reactor, name, control_port, request, temp_dir, flog_gatherer, tor_network, introducer_furl, shares_total: int) -> util.TahoeProcess: +def _create_anonymous_node(reactor, name, web_port, request, temp_dir, flog_gatherer, tor_network, introducer_furl, shares_total: int) -> util.TahoeProcess: node_dir = FilePath(temp_dir).child(name) - web_port = "tcp:{}:interface=localhost".format(control_port + 2000) if node_dir.exists(): raise RuntimeError( "A node already exists in '{}'".format(node_dir) @@ -111,10 +110,10 @@ def _create_anonymous_node(reactor, name, control_port, request, temp_dir, flog_ sys.executable, '-b', '-m', 'allmydata.scripts.runner', 'create-node', '--nickname', name, - '--webport', web_port, + '--webport', str(web_port), '--introducer', introducer_furl, '--hide-ip', - '--tor-control-port', 'tcp:localhost:{}'.format(control_port), + '--tor-control-port', tor_network.client_control_endpoint, '--listen', 'tor', '--shares-needed', '1', '--shares-happy', '1', @@ -133,7 +132,7 @@ def _create_anonymous_node(reactor, name, control_port, request, temp_dir, flog_ config = read_config(node_dir.path, "tub.port") config.set_config("tor", "onion", "true") config.set_config("tor", "onion.external_port", "3457") - config.set_config("tor", "control.port", f"tcp:port={control_port}:host=127.0.0.1") + config.set_config("tor", "control.port", tor_network.client_control_endpoint) config.set_config("tor", "onion.private_key_file", "private/tor_onion.privkey") print("running") @@ -159,7 +158,7 @@ def test_anonymous_client(reactor, request, temp_dir, flog_gatherer, tor_network ) yield util.await_client_ready(normie) - anonymoose = yield _create_anonymous_node(reactor, 'anonymoose', 8008, request, temp_dir, flog_gatherer, tor_network, introducer_furl, 1) + anonymoose = yield _create_anonymous_node(reactor, 'anonymoose', 8102, request, temp_dir, flog_gatherer, tor_network, introducer_furl, 1) yield util.await_client_ready(anonymoose, minimum_number_of_servers=1, timeout=600) yield upload_to_one_download_from_the_other(reactor, temp_dir, normie, anonymoose) diff --git a/integration/util.py b/integration/util.py index b614a84bd..909def8ef 100644 --- a/integration/util.py +++ b/integration/util.py @@ -659,7 +659,7 @@ def await_client_ready(tahoe, timeout=10, liveness=60*2, minimum_number_of_serve print( f"Now: {time.ctime()}\n" - f"Server last-received-data: {[time.ctime(s['last_received_data']) for s in servers]}" + f"Server last-received-data: {[s['last_received_data'] for s in servers]}" ) server_times = [ From 01a87d85be5a11f40015d651ea1244ffb3a5a487 Mon Sep 17 00:00:00 2001 From: meejah Date: Sat, 29 Jul 2023 04:08:52 -0600 Subject: [PATCH 1995/2309] refactor: actually parallel --- integration/conftest.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/integration/conftest.py b/integration/conftest.py index 92483da65..aa85a38cd 100644 --- a/integration/conftest.py +++ b/integration/conftest.py @@ -266,8 +266,7 @@ def storage_nodes(grid): nodes_d = [] # start all 5 nodes in parallel for x in range(5): - #nodes_d.append(grid.add_storage_node()) - pytest_twisted.blockon(grid.add_storage_node()) + nodes_d.append(grid.add_storage_node()) nodes_status = pytest_twisted.blockon(DeferredList(nodes_d)) for ok, value in nodes_status: From e565b9e28c00138eed1cf3cfdb064c23ddad9ffc Mon Sep 17 00:00:00 2001 From: meejah Date: Sat, 29 Jul 2023 04:14:39 -0600 Subject: [PATCH 1996/2309] no, we can't --- integration/grid.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/integration/grid.py b/integration/grid.py index 5ce4179ec..064319f74 100644 --- a/integration/grid.py +++ b/integration/grid.py @@ -173,8 +173,6 @@ class StorageServer(object): Note that self.process and self.protocol will be new instances after this. """ - # XXX per review comments, _can_ we make this "return a new - # instance" instead of mutating? self.process.transport.signalProcess('TERM') yield self.protocol.exited self.process = yield _run_node( From c4ac548cba2c397774a5d2af3f09d1bf0a642dbc Mon Sep 17 00:00:00 2001 From: meejah Date: Sat, 29 Jul 2023 13:08:01 -0600 Subject: [PATCH 1997/2309] reactor from fixture --- integration/conftest.py | 2 +- integration/grid.py | 7 +------ 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/integration/conftest.py b/integration/conftest.py index d2024ce98..04dc400a2 100644 --- a/integration/conftest.py +++ b/integration/conftest.py @@ -117,7 +117,7 @@ def reactor(): @pytest.fixture(scope='session') @log_call(action_type=u"integration:port_allocator", include_result=False) def port_allocator(reactor): - return create_port_allocator(start_port=45000) + return create_port_allocator(reactor, start_port=45000) @pytest.fixture(scope='session') diff --git a/integration/grid.py b/integration/grid.py index 064319f74..343bd779f 100644 --- a/integration/grid.py +++ b/integration/grid.py @@ -529,7 +529,7 @@ def create_grid(reactor, request, temp_dir, flog_gatherer, port_allocator): returnValue(grid) -def create_port_allocator(start_port): +def create_port_allocator(reactor, start_port): """ Returns a new port-allocator .. which is a zero-argument function that returns Deferreds that fire with new, sequential ports @@ -546,11 +546,6 @@ def create_port_allocator(start_port): """ port = [start_port - 1] - # import stays here to not interfere with reactor selection -- but - # maybe this function should be arranged to be called once from a - # fixture (with the reactor)? - from twisted.internet import reactor - class NothingProtocol(Protocol): """ I do nothing. From fe96defa2b2e6f7934f97bf76f0b651b1c20b191 Mon Sep 17 00:00:00 2001 From: meejah Date: Sat, 29 Jul 2023 13:15:21 -0600 Subject: [PATCH 1998/2309] use existing port-allocator instead --- integration/conftest.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/integration/conftest.py b/integration/conftest.py index 04dc400a2..46f5a0a44 100644 --- a/integration/conftest.py +++ b/integration/conftest.py @@ -21,7 +21,7 @@ from eliot import ( from twisted.python.filepath import FilePath from twisted.python.procutils import which -from twisted.internet.defer import DeferredList +from twisted.internet.defer import DeferredList, succeed from twisted.internet.error import ( ProcessExitedAlready, ProcessTerminated, @@ -117,7 +117,16 @@ def reactor(): @pytest.fixture(scope='session') @log_call(action_type=u"integration:port_allocator", include_result=False) def port_allocator(reactor): - return create_port_allocator(reactor, start_port=45000) + from allmydata.util.iputil import allocate_tcp_port + + # these will appear basically random, which can make especially + # manual debugging harder but we're re-using code instead of + # writing our own...so, win? + def allocate(): + port = allocate_tcp_port() + return succeed(port) + return allocate + #return create_port_allocator(reactor, start_port=45000) @pytest.fixture(scope='session') From 7a8752c969d8dc64e3e68ba944f0bf98b4e33f48 Mon Sep 17 00:00:00 2001 From: meejah Date: Sat, 29 Jul 2023 13:18:23 -0600 Subject: [PATCH 1999/2309] docstring, remove duplicate port-allocator --- integration/conftest.py | 4 +--- integration/grid.py | 52 ++++++----------------------------------- 2 files changed, 8 insertions(+), 48 deletions(-) diff --git a/integration/conftest.py b/integration/conftest.py index 46f5a0a44..55a0bbbb5 100644 --- a/integration/conftest.py +++ b/integration/conftest.py @@ -47,6 +47,7 @@ from .grid import ( create_grid, ) from allmydata.node import read_config +from allmydata.util.iputil import allocate_tcp_port # No reason for HTTP requests to take longer than four minutes in the # integration tests. See allmydata/scripts/common_http.py for usage. @@ -117,8 +118,6 @@ def reactor(): @pytest.fixture(scope='session') @log_call(action_type=u"integration:port_allocator", include_result=False) def port_allocator(reactor): - from allmydata.util.iputil import allocate_tcp_port - # these will appear basically random, which can make especially # manual debugging harder but we're re-using code instead of # writing our own...so, win? @@ -126,7 +125,6 @@ def port_allocator(reactor): port = allocate_tcp_port() return succeed(port) return allocate - #return create_port_allocator(reactor, start_port=45000) @pytest.fixture(scope='session') diff --git a/integration/grid.py b/integration/grid.py index 343bd779f..79b5b45ad 100644 --- a/integration/grid.py +++ b/integration/grid.py @@ -510,11 +510,16 @@ class Grid(object): returnValue(client) -# XXX THINK can we tie a whole *grid* to a single request? (I think -# that's all that makes sense) +# A grid is now forever tied to its original 'request' which is where +# it must hang finalizers off of. The "main" one is a session-level +# fixture so it'll live the life of the tests but it could be +# per-function Grid too. @inlineCallbacks def create_grid(reactor, request, temp_dir, flog_gatherer, port_allocator): """ + Create a new grid. This will have one Introducer but zero + storage-servers or clients; those must be added by a test or + subsequent fixtures. """ intro_port = yield port_allocator() introducer = yield create_introducer(reactor, request, temp_dir, flog_gatherer, intro_port) @@ -527,46 +532,3 @@ def create_grid(reactor, request, temp_dir, flog_gatherer, port_allocator): flog_gatherer, ) returnValue(grid) - - -def create_port_allocator(reactor, start_port): - """ - Returns a new port-allocator .. which is a zero-argument function - that returns Deferreds that fire with new, sequential ports - starting at `start_port` skipping any that already appear to have - a listener. - - There can still be a race against other processes allocating ports - -- between the time when we check the status of the port and when - our subprocess starts up. This *could* be mitigated by instructing - the OS to not randomly-allocate ports in some range, and then - using that range here (explicitly, ourselves). - - NB once we're Python3-only this could be an async-generator - """ - port = [start_port - 1] - - class NothingProtocol(Protocol): - """ - I do nothing. - """ - - def port_generator(): - print("Checking port {}".format(port)) - port[0] += 1 - ep = TCP4ServerEndpoint(reactor, port[0], interface="localhost") - d = ep.listen(Factory.forProtocol(NothingProtocol)) - - def good(listening_port): - unlisten_d = maybeDeferred(listening_port.stopListening) - def return_port(_): - return port[0] - unlisten_d.addBoth(return_port) - return unlisten_d - - def try_again(fail): - return port_generator() - - d.addCallbacks(good, try_again) - return d - return port_generator From 67d5c82e103f49fb1d624e3ad6908de885c01842 Mon Sep 17 00:00:00 2001 From: meejah Date: Sat, 29 Jul 2023 13:34:12 -0600 Subject: [PATCH 2000/2309] codechecks / linter --- integration/conftest.py | 4 ---- integration/grid.py | 8 -------- integration/test_i2p.py | 9 ++++++--- integration/test_vectors.py | 2 +- 4 files changed, 7 insertions(+), 16 deletions(-) diff --git a/integration/conftest.py b/integration/conftest.py index 55a0bbbb5..a26d2043d 100644 --- a/integration/conftest.py +++ b/integration/conftest.py @@ -12,7 +12,6 @@ from time import sleep from os import mkdir, environ from os.path import join, exists from tempfile import mkdtemp -from json import loads from eliot import ( to_file, @@ -37,12 +36,9 @@ from .util import ( _create_node, _tahoe_runner_optional_coverage, await_client_ready, - cli, - generate_ssh_key, block_with_timeout, ) from .grid import ( - create_port_allocator, create_flog_gatherer, create_grid, ) diff --git a/integration/grid.py b/integration/grid.py index 79b5b45ad..94f8c3d7f 100644 --- a/integration/grid.py +++ b/integration/grid.py @@ -26,7 +26,6 @@ from twisted.python.procutils import which from twisted.internet.defer import ( inlineCallbacks, returnValue, - maybeDeferred, Deferred, ) from twisted.internet.task import ( @@ -36,13 +35,6 @@ from twisted.internet.interfaces import ( IProcessTransport, IProcessProtocol, ) -from twisted.internet.endpoints import ( - TCP4ServerEndpoint, -) -from twisted.internet.protocol import ( - Factory, - Protocol, -) from twisted.internet.error import ProcessTerminated from allmydata.node import read_config diff --git a/integration/test_i2p.py b/integration/test_i2p.py index ea3ddb62b..c99c469fa 100644 --- a/integration/test_i2p.py +++ b/integration/test_i2p.py @@ -24,6 +24,7 @@ from allmydata.test.common import ( write_introducer, ) from allmydata.node import read_config +from allmydata.util.iputil import allocate_tcp_port if which("docker") is None: @@ -132,8 +133,10 @@ def i2p_introducer_furl(i2p_introducer, temp_dir): @pytest_twisted.inlineCallbacks @pytest.mark.skip("I2P tests are not functioning at all, for unknown reasons") def test_i2p_service_storage(reactor, request, temp_dir, flog_gatherer, i2p_network, i2p_introducer_furl): - yield _create_anonymous_node(reactor, 'carol_i2p', request, temp_dir, flog_gatherer, i2p_network, i2p_introducer_furl) - yield _create_anonymous_node(reactor, 'dave_i2p', request, temp_dir, flog_gatherer, i2p_network, i2p_introducer_furl) + web_port0 = allocate_tcp_port() + web_port1 = allocate_tcp_port() + yield _create_anonymous_node(reactor, 'carol_i2p', web_port0, request, temp_dir, flog_gatherer, i2p_network, i2p_introducer_furl) + yield _create_anonymous_node(reactor, 'dave_i2p', web_port1, request, temp_dir, flog_gatherer, i2p_network, i2p_introducer_furl) # ensure both nodes are connected to "a grid" by uploading # something via carol, and retrieve it using dave. gold_path = join(temp_dir, "gold") @@ -179,7 +182,7 @@ def test_i2p_service_storage(reactor, request, temp_dir, flog_gatherer, i2p_netw @pytest_twisted.inlineCallbacks -def _create_anonymous_node(reactor, name, request, temp_dir, flog_gatherer, i2p_network, introducer_furl): +def _create_anonymous_node(reactor, name, web_port, request, temp_dir, flog_gatherer, i2p_network, introducer_furl): node_dir = FilePath(temp_dir).child(name) print("creating", node_dir.path) diff --git a/integration/test_vectors.py b/integration/test_vectors.py index bd5def8c5..1bcbcffa4 100644 --- a/integration/test_vectors.py +++ b/integration/test_vectors.py @@ -15,7 +15,7 @@ from pytest_twisted import ensureDeferred from . import vectors from .vectors import parameters -from .util import reconfigure, upload +from .util import upload from .grid import Client @mark.parametrize('convergence', parameters.CONVERGENCE_SECRETS) From 112770aeb31a7f95e59718c54ea59c69d842c1d7 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 31 Jul 2023 11:07:37 -0400 Subject: [PATCH 2001/2309] Don't hardcode tox --- setup.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/setup.py b/setup.py index a2e870e8b..86873ad53 100644 --- a/setup.py +++ b/setup.py @@ -435,9 +435,7 @@ setup(name="tahoe-lafs", # also set in __init__.py "paramiko < 2.9", "pytest-timeout", # Does our OpenMetrics endpoint adhere to the spec: - "prometheus-client == 0.11.0", - # CI uses "tox<4", change here too if that becomes different - "tox < 4", + "prometheus-client == 0.11.0" ] + tor_requires + i2p_requires, "tor": tor_requires, "i2p": i2p_requires, From ffe2e9773916e32a8a64e9eb0d0e1ae0939c7215 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 1 Aug 2023 10:54:46 -0400 Subject: [PATCH 2002/2309] Better phrasing --- docs/proposed/http-storage-node-protocol.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index fed5bb5dc..db400fb2b 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -278,8 +278,8 @@ This NURL will be announced alongside their existing Foolscap-based server's fUR Such an announcement will resemble this:: { - "anonymous-storage-FURL": "pb://...", # The old key - "anonymous-storage-NURLs": ["pb://...#v=1"] # The new keys + "anonymous-storage-FURL": "pb://...", # The old entry + "anonymous-storage-NURLs": ["pb://...#v=1"] # The new, additional entry } The transition process will proceed in three stages: From 341a32708b718caab6311ed2e517c548e493df35 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 1 Aug 2023 10:55:45 -0400 Subject: [PATCH 2003/2309] Docstring. --- src/allmydata/test/test_storage_http.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 48ca072bc..1f17621d7 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -309,6 +309,11 @@ class TestApp(object): @_authorized_route(_app, set(), "/read_body", methods=["POST"]) @async_to_deferred async def read_body(self, request, authorization): + """ + Accept an advise_corrupt_share message, return the reason. + + I.e. exercise codepaths used for reading CBOR from the body. + """ data = await read_encoded( self.clock, request, SERVER_SCHEMAS["advise_corrupt_share"] ) From 3b66afbdeac45ae3050f98b4103536e72a377d8e Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 1 Aug 2023 10:56:52 -0400 Subject: [PATCH 2004/2309] Be consistent with HTTPS --- docs/architecture.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/architecture.rst b/docs/architecture.rst index 64e81ea99..71ad67305 100644 --- a/docs/architecture.rst +++ b/docs/architecture.rst @@ -64,7 +64,7 @@ There are two supported protocols: By default HTTPS is disabled (this will change in https://tahoe-lafs.org/trac/tahoe-lafs/ticket/4041). When HTTPS is enabled on -the server, the server transparently listens for both Foolscap and HTTP on the +the server, the server transparently listens for both Foolscap and HTTPS on the same port. Clients can use either; by default they will only use Foolscap, but when configured appropriately they will use HTTPS when possible (this will change in https://tahoe-lafs.org/trac/tahoe-lafs/ticket/4041). At this time the From e545ab4a8022c52ee3a450ab501eedc470491d50 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 1 Aug 2023 15:31:38 -0400 Subject: [PATCH 2005/2309] More accurate type --- src/allmydata/storage/http_client.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 79f6cfa89..75b6eab22 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -41,6 +41,7 @@ from twisted.internet.interfaces import ( IDelayedCall, ) from twisted.internet.ssl import CertificateOptions +from twisted.protocols.tls import TLSMemoryBIOProtocol from twisted.web.client import Agent, HTTPConnectionPool from zope.interface import implementer from hyperlink import DecodedURL @@ -304,7 +305,7 @@ class _StorageClientHTTPSPolicy: return self # IOpenSSLClientConnectionCreator - def clientConnectionForTLS(self, tlsProtocol: object) -> SSL.Connection: + def clientConnectionForTLS(self, tlsProtocol: TLSMemoryBIOProtocol) -> SSL.Connection: return SSL.Connection( _TLSContextFactory(self.expected_spki_hash).getContext(), None ) From 009f063067a156ddab95bb3f554c43244cc05fc1 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 1 Aug 2023 15:34:40 -0400 Subject: [PATCH 2006/2309] Stricter type checking --- src/allmydata/storage/http_client.py | 14 ++++++++++---- src/allmydata/storage_client.py | 2 +- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 75b6eab22..b508c07fd 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -81,9 +81,13 @@ def _encode_si(si: bytes) -> str: class ClientException(Exception): """An unexpected response code from the server.""" - def __init__(self, code: int, *additional_args): - Exception.__init__(self, code, *additional_args) + def __init__( + self, code: int, message: Optional[str] = None, body: Optional[bytes] = None + ): + Exception.__init__(self, code, message, body) self.code = code + self.message = message + self.body = body register_exception_extractor(ClientException, lambda e: {"response_code": e.code}) @@ -94,7 +98,7 @@ register_exception_extractor(ClientException, lambda e: {"response_code": e.code # Tags are of the form #6.nnn, where the number is documented at # https://www.iana.org/assignments/cbor-tags/cbor-tags.xhtml. Notably, #6.258 # indicates a set. -_SCHEMAS : Mapping[str,Schema] = { +_SCHEMAS: Mapping[str, Schema] = { "get_version": Schema( # Note that the single-quoted (`'`) string keys in this schema # represent *byte* strings - per the CDDL specification. Text strings @@ -305,7 +309,9 @@ class _StorageClientHTTPSPolicy: return self # IOpenSSLClientConnectionCreator - def clientConnectionForTLS(self, tlsProtocol: TLSMemoryBIOProtocol) -> SSL.Connection: + def clientConnectionForTLS( + self, tlsProtocol: TLSMemoryBIOProtocol + ) -> SSL.Connection: return SSL.Connection( _TLSContextFactory(self.expected_spki_hash).getContext(), None ) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 4efc845b4..69ae2c22b 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -1428,7 +1428,7 @@ class _FakeRemoteReference(object): result = yield getattr(self.local_object, action)(*args, **kwargs) defer.returnValue(result) except HTTPClientException as e: - raise RemoteException(e.args) + raise RemoteException((e.code, e.message, e.body)) @attr.s From 14ebeba07d527a189a5cdf93c0f85392302b2ca1 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 1 Aug 2023 15:28:49 -0400 Subject: [PATCH 2007/2309] avoid re-computing the current time inside this loop It could lead to funny behavior if we cross a boundary at just the wrong time. Also the debug print could be misleading in such a case. --- integration/util.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/integration/util.py b/integration/util.py index 31d351bc1..756489120 100644 --- a/integration/util.py +++ b/integration/util.py @@ -622,8 +622,9 @@ def await_client_ready(tahoe, timeout=10, liveness=60*2, minimum_number_of_serve time.sleep(1) continue + now = time.time() print( - f"Now: {time.ctime()}\n" + f"Now: {time.ctime(now)}\n" f"Server last-received-data: {[time.ctime(s['last_received_data']) for s in servers]}" ) @@ -633,7 +634,7 @@ def await_client_ready(tahoe, timeout=10, liveness=60*2, minimum_number_of_serve ] # check that all times are 'recent enough' (it's OK if _some_ servers # are down, we just want to make sure a sufficient number are up) - if len([time.time() - t <= liveness for t in server_times if t is not None]) < minimum_number_of_servers: + if len([now - t <= liveness for t in server_times if t is not None]) < minimum_number_of_servers: print("waiting because at least one server too old") time.sleep(1) continue From a0b78a134e05e33d90e802011b8dfd428d8d358d Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 1 Aug 2023 15:49:30 -0400 Subject: [PATCH 2008/2309] Leave a hint about what successful "bootstrap" looks like --- integration/conftest.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/integration/conftest.py b/integration/conftest.py index c94c05429..9a7a47ec4 100644 --- a/integration/conftest.py +++ b/integration/conftest.py @@ -575,6 +575,32 @@ def tor_network(reactor, temp_dir, chutney, request): request.addfinalizer(cleanup) pytest_twisted.blockon(chutney(("start", basic_network))) + + # Wait for the nodes to "bootstrap" - ie, form a network among themselves. + # Successful bootstrap is reported with a message something like: + # + # Everything bootstrapped after 151 sec + # Bootstrap finished: 151 seconds + # Node status: + # test000a : 100, done , Done + # test001a : 100, done , Done + # test002a : 100, done , Done + # test003r : 100, done , Done + # test004r : 100, done , Done + # test005r : 100, done , Done + # test006r : 100, done , Done + # test007r : 100, done , Done + # test008c : 100, done , Done + # test009c : 100, done , Done + # Published dir info: + # test000a : 100, all nodes , desc md md_cons ns_cons , Dir info cached + # test001a : 100, all nodes , desc md md_cons ns_cons , Dir info cached + # test002a : 100, all nodes , desc md md_cons ns_cons , Dir info cached + # test003r : 100, all nodes , desc md md_cons ns_cons , Dir info cached + # test004r : 100, all nodes , desc md md_cons ns_cons , Dir info cached + # test005r : 100, all nodes , desc md md_cons ns_cons , Dir info cached + # test006r : 100, all nodes , desc md md_cons ns_cons , Dir info cached + # test007r : 100, all nodes , desc md md_cons ns_cons , Dir info cached pytest_twisted.blockon(chutney(("wait_for_bootstrap", basic_network))) # print some useful stuff From 871df0b1b4e5d5264c9cd244b02eac09a690510f Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 1 Aug 2023 15:49:44 -0400 Subject: [PATCH 2009/2309] Dump some more details about what we're waiting for --- integration/util.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/integration/util.py b/integration/util.py index 756489120..768741bd8 100644 --- a/integration/util.py +++ b/integration/util.py @@ -623,18 +623,22 @@ def await_client_ready(tahoe, timeout=10, liveness=60*2, minimum_number_of_serve continue now = time.time() - print( - f"Now: {time.ctime(now)}\n" - f"Server last-received-data: {[time.ctime(s['last_received_data']) for s in servers]}" - ) - server_times = [ server['last_received_data'] - for server in servers + for server + in servers + if server['last_received_data'] is not None ] + print( + f"Now: {time.ctime(now)}\n" + f"Liveness required: {liveness}\n" + f"Server last-received-data: {[time.ctime(s) for s in server_times]}\n" + f"Server ages: {[now - s for s in server_times]}\n" + ) + # check that all times are 'recent enough' (it's OK if _some_ servers # are down, we just want to make sure a sufficient number are up) - if len([now - t <= liveness for t in server_times if t is not None]) < minimum_number_of_servers: + if len([now - t <= liveness for t in server_times]) < minimum_number_of_servers: print("waiting because at least one server too old") time.sleep(1) continue From 9d670e54e29cb2c249100576949539cddc53b7f8 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 1 Aug 2023 15:56:02 -0400 Subject: [PATCH 2010/2309] Get the liveness filter condition right --- integration/util.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/integration/util.py b/integration/util.py index 768741bd8..7e10b4315 100644 --- a/integration/util.py +++ b/integration/util.py @@ -638,8 +638,12 @@ def await_client_ready(tahoe, timeout=10, liveness=60*2, minimum_number_of_serve # check that all times are 'recent enough' (it's OK if _some_ servers # are down, we just want to make sure a sufficient number are up) - if len([now - t <= liveness for t in server_times]) < minimum_number_of_servers: - print("waiting because at least one server too old") + alive = [t for t in server_times if now - t <= liveness] + if len(alive) < minimum_number_of_servers: + print( + f"waiting because we found {len(alive)} servers " + f"and want {minimum_number_of_servers}" + ) time.sleep(1) continue From b8ee7a4e98e0968d6bc2d8a60f8638a83b1e04ce Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 1 Aug 2023 15:56:20 -0400 Subject: [PATCH 2011/2309] news fragment --- newsfragments/4055.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/4055.minor diff --git a/newsfragments/4055.minor b/newsfragments/4055.minor new file mode 100644 index 000000000..e69de29bb From e3f30d8e58fa73dbd3a2af870cb1b2c1252eb184 Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 2 Aug 2023 14:48:36 -0600 Subject: [PATCH 2012/2309] fix comments about tor/chutney in integration config --- integration/conftest.py | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/integration/conftest.py b/integration/conftest.py index 171310570..89de83cdb 100644 --- a/integration/conftest.py +++ b/integration/conftest.py @@ -308,19 +308,10 @@ def bob(reactor, temp_dir, introducer_furl, flog_gatherer, storage_nodes, reques 'Tor tests are unstable on Windows') def chutney(reactor, temp_dir: str) -> tuple[str, dict[str, str]]: """ - Instantiate the "networks/hs-v3" Chutney configuration for a local - Tor network. + Install the Chutney software that is required to run a small local Tor grid. - This provides a small, local Tor network that can run v3 Onion - Services. This has 10 tor processes: 3 authorities, 5 - exits+relays, a client (and one service-hosting node we don't use). - - We pin a Chutney revision, so things shouldn't change. Currently, - the ONLY node that exposes a valid SocksPort is "008c" (the - client) on 9008. - - The control ports start at 8000 (so the ControlPort for the one - client node is 8008). + (Chutney lacks the normal "python stuff" so we can't just declare + it in Tox or similar dependencies) """ # Try to find Chutney already installed in the environment. try: @@ -404,6 +395,20 @@ def tor_network(reactor, temp_dir, chutney, request): """ Build a basic Tor network. + Instantiate the "networks/basic" Chutney configuration for a local + Tor network. + + This provides a small, local Tor network that can run v3 Onion + Services. It has 3 authorities, 5 relays and 2 clients. + + The 'chutney' fixture pins a Chutney git qrevision, so things + shouldn't change. This network has two clients which are the only + nodes with valid SocksPort configuration ("008c" and "009c" 9008 + and 9009) + + The control ports start at 8000 (so the ControlPort for the client + nodes are 8008 and 8009). + :param chutney: The root directory of a Chutney checkout and a dict of additional environment variables to set so a Python process can use it. From 8ec7f5485a1836c8a79f689a0b609d94aa0caf88 Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 2 Aug 2023 14:48:59 -0600 Subject: [PATCH 2013/2309] upload() needs the actual alice fixture --- integration/test_vectors.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration/test_vectors.py b/integration/test_vectors.py index 1bcbcffa4..f53ec1741 100644 --- a/integration/test_vectors.py +++ b/integration/test_vectors.py @@ -41,7 +41,7 @@ async def test_capability(reactor, request, alice, case, expected): reactor, (1, case.params.required, case.params.total), case.convergence, case.segment_size) # upload data in the correct format - actual = upload(alice.process, case.fmt, case.data) + actual = upload(alice, case.fmt, case.data) # compare the resulting cap to the expected result assert actual == expected From bd0bfa4ab7c3cc366503ef088b8497f60dd4388b Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 2 Aug 2023 14:49:36 -0600 Subject: [PATCH 2014/2309] define -> frozen Co-authored-by: Jean-Paul Calderone --- integration/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration/conftest.py b/integration/conftest.py index 171310570..36cda8f45 100644 --- a/integration/conftest.py +++ b/integration/conftest.py @@ -382,7 +382,7 @@ def chutney(reactor, temp_dir: str) -> tuple[str, dict[str, str]]: return chutney_dir, {"PYTHONPATH": join(chutney_dir, "lib")} -@define +@frozen class ChutneyTorNetwork: """ Represents a running Chutney (tor) network. Returned by the From 7127ae62a942a329d08c982a49883f3f2ed38ee5 Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 2 Aug 2023 14:50:04 -0600 Subject: [PATCH 2015/2309] fix types Co-authored-by: Jean-Paul Calderone --- integration/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration/conftest.py b/integration/conftest.py index 36cda8f45..52bffff61 100644 --- a/integration/conftest.py +++ b/integration/conftest.py @@ -389,7 +389,7 @@ class ChutneyTorNetwork: "tor_network" fixture. """ dir: FilePath - environ: dict + environ: Mapping[str, str] client_control_port: int @property From 3e2c784e7794de806280bf1c627ae4884c3ef508 Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 2 Aug 2023 14:58:04 -0600 Subject: [PATCH 2016/2309] likely to be more-right --- integration/grid.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/integration/grid.py b/integration/grid.py index 94f8c3d7f..00f0dd826 100644 --- a/integration/grid.py +++ b/integration/grid.py @@ -116,13 +116,14 @@ def create_flog_gatherer(reactor, request, temp_dir, flog_binary): flogs = [x for x in listdir(flog_dir) if x.endswith('.flog')] print("Dumping {} flogtool logfiles to '{}'".format(len(flogs), flog_file)) - reactor.spawnProcess( - flog_protocol, - flog_binary, - ( - 'flogtool', 'dump', join(temp_dir, 'flog_gather', flogs[0]) - ), - ) + for flog_path in flogs: + reactor.spawnProcess( + flog_protocol, + flog_binary, + ( + 'flogtool', 'dump', join(temp_dir, 'flog_gather', flog_path) + ), + ) print("Waiting for flogtool to complete") try: pytest_twisted.blockon(flog_protocol.done) From 63f4c6fcc6c421a96827558c569a08065a3dc2a6 Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 2 Aug 2023 14:58:55 -0600 Subject: [PATCH 2017/2309] import to top-level --- integration/grid.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration/grid.py b/integration/grid.py index 00f0dd826..b9af7ed5d 100644 --- a/integration/grid.py +++ b/integration/grid.py @@ -51,6 +51,7 @@ from .util import ( generate_ssh_key, cli, reconfigure, + _create_node, ) import attr @@ -181,7 +182,6 @@ def create_storage_server(reactor, request, temp_dir, introducer, flog_gatherer, """ Create a new storage server """ - from .util import _create_node node_process = yield _create_node( reactor, request, temp_dir, introducer.furl, flog_gatherer, name, web_port, storage=True, needed=needed, happy=happy, total=total, From f77b6c433778ed91b7c41abf6b2c1ddb3e5dc94a Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 2 Aug 2023 15:12:38 -0600 Subject: [PATCH 2018/2309] fix XXX comment + add docstring --- integration/conftest.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/integration/conftest.py b/integration/conftest.py index 89de83cdb..f7fb5f093 100644 --- a/integration/conftest.py +++ b/integration/conftest.py @@ -166,10 +166,12 @@ def flog_gatherer(reactor, temp_dir, flog_binary, request): @pytest.fixture(scope='session') @log_call(action_type=u"integration:grid", include_args=[]) def grid(reactor, request, temp_dir, flog_gatherer, port_allocator): - # XXX think: this creates an "empty" grid (introducer, no nodes); - # do we want to ensure it has some minimum storage-nodes at least? - # (that is, semantically does it make sense that 'a grid' is - # essentially empty, or not?) + """ + Provides a new Grid with a single Introducer and flog-gathering process. + + Notably does _not_ provide storage servers; use the storage_nodes + fixture if your tests need a Grid that can be used for puts / gets. + """ g = pytest_twisted.blockon( create_grid(reactor, request, temp_dir, flog_gatherer, port_allocator) ) From 8b175383af0ce7d4c835bb2a029797730fb4d646 Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 2 Aug 2023 15:15:33 -0600 Subject: [PATCH 2019/2309] flake8 --- integration/conftest.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/integration/conftest.py b/integration/conftest.py index be467bb34..313ff36c2 100644 --- a/integration/conftest.py +++ b/integration/conftest.py @@ -7,7 +7,7 @@ from __future__ import annotations import os import sys import shutil -from attr import define +from attr import frozen from time import sleep from os import mkdir, environ from os.path import join, exists @@ -28,6 +28,7 @@ from twisted.internet.error import ( import pytest import pytest_twisted +from typing import Mapping from .util import ( _MagicTextProtocol, From f663581ed32e2d0f1206074cca21151d227bf3bc Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 2 Aug 2023 16:27:18 -0600 Subject: [PATCH 2020/2309] temporarily remove new provides() usage --- integration/grid.py | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/integration/grid.py b/integration/grid.py index b9af7ed5d..c39b9cff9 100644 --- a/integration/grid.py +++ b/integration/grid.py @@ -31,10 +31,8 @@ from twisted.internet.defer import ( from twisted.internet.task import ( deferLater, ) -from twisted.internet.interfaces import ( - IProcessTransport, - IProcessProtocol, -) +from twisted.internet.protocol import ProcessProtocol # see ticket 4056 +from twisted.internet.process import Process # see ticket 4056 from twisted.internet.error import ProcessTerminated from allmydata.node import read_config @@ -71,11 +69,17 @@ class FlogGatherer(object): Flog Gatherer process. """ + # it would be best to use attr.validators.provides() here with the + # corresponding Twisted interface (IProcessTransport, + # IProcessProtocol) but that is deprecated; please replace with + # our own "provides" as part of + # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/4056#ticket + # insisting on a subclass is narrower than necessary process = attr.ib( - validator=attr.validators.provides(IProcessTransport) + validator=attr.validators.instance_of(Process) ) protocol = attr.ib( - validator=attr.validators.provides(IProcessProtocol) + validator=attr.validators.instance_of(ProcessProtocol) ) furl = attr.ib() @@ -155,7 +159,7 @@ class StorageServer(object): validator=attr.validators.instance_of(TahoeProcess) ) protocol = attr.ib( - validator=attr.validators.provides(IProcessProtocol) + validator=attr.validators.instance_of(ProcessProtocol) ) @inlineCallbacks @@ -207,7 +211,7 @@ class Client(object): validator=attr.validators.instance_of(TahoeProcess) ) protocol = attr.ib( - validator=attr.validators.provides(IProcessProtocol) + validator=attr.validators.instance_of(ProcessProtocol) ) request = attr.ib() # original request, for addfinalizer() @@ -335,7 +339,7 @@ class Introducer(object): validator=attr.validators.instance_of(TahoeProcess) ) protocol = attr.ib( - validator=attr.validators.provides(IProcessProtocol) + validator=attr.validators.instance_of(ProcessProtocol) ) furl = attr.ib() From d0208bc099a3c500a50fceae1fbe1785c0144725 Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 2 Aug 2023 16:52:29 -0600 Subject: [PATCH 2021/2309] different Process instance on different platforms --- integration/grid.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/integration/grid.py b/integration/grid.py index c39b9cff9..524da730f 100644 --- a/integration/grid.py +++ b/integration/grid.py @@ -32,7 +32,6 @@ from twisted.internet.task import ( deferLater, ) from twisted.internet.protocol import ProcessProtocol # see ticket 4056 -from twisted.internet.process import Process # see ticket 4056 from twisted.internet.error import ProcessTerminated from allmydata.node import read_config @@ -75,9 +74,7 @@ class FlogGatherer(object): # our own "provides" as part of # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/4056#ticket # insisting on a subclass is narrower than necessary - process = attr.ib( - validator=attr.validators.instance_of(Process) - ) + process = attr.ib() protocol = attr.ib( validator=attr.validators.instance_of(ProcessProtocol) ) From 4710e7b1772eaeaa8e1bc1138a4d36e6cf69ef87 Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 2 Aug 2023 17:07:09 -0600 Subject: [PATCH 2022/2309] provide our own provides() validator --- integration/grid.py | 28 ++++++++++------ src/allmydata/storage_client.py | 3 +- src/allmydata/util/attrs_provides.py | 50 ++++++++++++++++++++++++++++ 3 files changed, 69 insertions(+), 12 deletions(-) create mode 100644 src/allmydata/util/attrs_provides.py diff --git a/integration/grid.py b/integration/grid.py index 524da730f..03c3bb6e2 100644 --- a/integration/grid.py +++ b/integration/grid.py @@ -31,9 +31,15 @@ from twisted.internet.defer import ( from twisted.internet.task import ( deferLater, ) -from twisted.internet.protocol import ProcessProtocol # see ticket 4056 +from twisted.internet.interfaces import ( + IProcessTransport, + IProcessProtocol, +) from twisted.internet.error import ProcessTerminated +from allmydata.util.attrs_provides import ( + provides, +) from allmydata.node import read_config from .util import ( _CollectOutputProtocol, @@ -68,15 +74,15 @@ class FlogGatherer(object): Flog Gatherer process. """ - # it would be best to use attr.validators.provides() here with the - # corresponding Twisted interface (IProcessTransport, - # IProcessProtocol) but that is deprecated; please replace with - # our own "provides" as part of + # it would be best to use attr.validators.provides() here but that + # is deprecated; please replace with our own "provides" as part of # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/4056#ticket - # insisting on a subclass is narrower than necessary - process = attr.ib() + # for now, insisting on a subclass which is narrower than necessary + process = attr.ib( + validator=provides(IProcessTransport) + ) protocol = attr.ib( - validator=attr.validators.instance_of(ProcessProtocol) + validator=provides(IProcessProtocol) ) furl = attr.ib() @@ -156,7 +162,7 @@ class StorageServer(object): validator=attr.validators.instance_of(TahoeProcess) ) protocol = attr.ib( - validator=attr.validators.instance_of(ProcessProtocol) + validator=provides(IProcessProtocol) ) @inlineCallbacks @@ -208,7 +214,7 @@ class Client(object): validator=attr.validators.instance_of(TahoeProcess) ) protocol = attr.ib( - validator=attr.validators.instance_of(ProcessProtocol) + validator=provides(IProcessProtocol) ) request = attr.ib() # original request, for addfinalizer() @@ -336,7 +342,7 @@ class Introducer(object): validator=attr.validators.instance_of(TahoeProcess) ) protocol = attr.ib( - validator=attr.validators.instance_of(ProcessProtocol) + validator=provides(IProcessProtocol) ) furl = attr.ib() diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index d205edf08..8de3a9ca9 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -88,6 +88,7 @@ from allmydata.util.rrefutil import add_version_to_remote_reference from allmydata.util.hashutil import permute_server_hash from allmydata.util.dictutil import BytesKeyDict, UnicodeKeyDict from allmydata.util.deferredutil import async_to_deferred, race +from allmydata.util.attr_provides import provides from allmydata.storage.http_client import ( StorageClient, StorageClientImmutables, StorageClientGeneral, ClientException as HTTPClientException, StorageClientMutables, @@ -659,7 +660,7 @@ class _FoolscapStorage(object): permutation_seed = attr.ib() tubid = attr.ib() - storage_server = attr.ib(validator=attr.validators.provides(IStorageServer)) + storage_server = attr.ib(validator=provides(IStorageServer)) _furl = attr.ib() _short_description = attr.ib() diff --git a/src/allmydata/util/attrs_provides.py b/src/allmydata/util/attrs_provides.py new file mode 100644 index 000000000..4282c3d38 --- /dev/null +++ b/src/allmydata/util/attrs_provides.py @@ -0,0 +1,50 @@ +""" +Utilities related to attrs + +Handling for zope.interface is deprecated in attrs so we copy the +relevant support method here since we depend on zope.interface anyway +""" + +from attr._make import attrs, attrib + + +@attrs(repr=False, slots=True, hash=True) +class _ProvidesValidator: + interface = attrib() + + def __call__(self, inst, attr, value): + """ + We use a callable class to be able to change the ``__repr__``. + """ + if not self.interface.providedBy(value): + raise TypeError( + "'{name}' must provide {interface!r} which {value!r} " + "doesn't.".format( + name=attr.name, interface=self.interface, value=value + ), + attr, + self.interface, + value, + ) + + def __repr__(self): + return "".format( + interface=self.interface + ) + + +def provides(interface): + """ + A validator that raises a `TypeError` if the initializer is called + with an object that does not provide the requested *interface* (checks are + performed using ``interface.providedBy(value)`` (see `zope.interface + `_). + + :param interface: The interface to check for. + :type interface: ``zope.interface.Interface`` + + :raises TypeError: With a human readable error message, the attribute + (of type `attrs.Attribute`), the expected interface, and the + value it got. + """ + return _ProvidesValidator(interface) From cbf3eebc78aeb6a907a6e8a1d3dab84a4e5402ad Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 2 Aug 2023 17:08:28 -0600 Subject: [PATCH 2023/2309] news --- newsfragments/4056.bugfix | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 newsfragments/4056.bugfix diff --git a/newsfragments/4056.bugfix b/newsfragments/4056.bugfix new file mode 100644 index 000000000..1f94de0da --- /dev/null +++ b/newsfragments/4056.bugfix @@ -0,0 +1,3 @@ +Provide our own copy of attrs' "provides()" validor + +This validator is deprecated and slated for removal; that project's suggestion is to copy the code to our project. From 08b594b8be7437b53c7ad71999f130a6a11b54d8 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 7 Aug 2023 10:18:45 -0400 Subject: [PATCH 2024/2309] Declare the Windows orb for easier Windows environment setup --- .circleci/config.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 0c831af04..6b2d84d92 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -11,6 +11,10 @@ # version: 2.1 +orbs: + # Pull in CircleCI support for a Windows executor + windows: "circleci/windows@5.0.0" + # Every job that pushes a Docker image from Docker Hub must authenticate to # it. Define a couple yaml anchors that can be used to supply the necessary # credentials. From 814ba4c88b58c088b5445420ba35de3ed18f1477 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 7 Aug 2023 10:19:07 -0400 Subject: [PATCH 2025/2309] Add Windows executor holding the Windows test environment config --- .circleci/config.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 6b2d84d92..2375368ca 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -531,6 +531,12 @@ jobs: # PYTHON_VERSION: "2" executors: + windows: + # Choose a Windows environment that closest matches our testing + # requirements and goals. + # https://circleci.com/developer/orbs/orb/circleci/windows#executors-server-2022 + executor: "win/server-2022@2023.06.1" + nix: docker: # Run in a highly Nix-capable environment. From f8db7818128347d6e50dcbba93a59663597fe0d4 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 7 Aug 2023 10:19:25 -0400 Subject: [PATCH 2026/2309] Add a simple test job to see if the other pieces work --- .circleci/config.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 2375368ca..307d3ca69 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -116,6 +116,8 @@ workflows: - "another-locale": {} + - "windows-server-2022" + - "integration": # Run even the slow integration tests here. We need the `--` to # sneak past tox and get to pytest. @@ -137,6 +139,11 @@ workflows: when: "<< pipeline.parameters.build-images >>" jobs: + windows-server-2022: + steps: + - "run": | + Write-Host 'Hello, world.' + codechecks: docker: - <<: *DOCKERHUB_AUTH From d050faac92865a25c4b74f816f5b9796b586715f Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 7 Aug 2023 10:27:52 -0400 Subject: [PATCH 2027/2309] make the config match the schema --- .circleci/config.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 307d3ca69..07bdc7087 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -140,6 +140,7 @@ workflows: jobs: windows-server-2022: + executor: "windows" steps: - "run": | Write-Host 'Hello, world.' @@ -542,7 +543,8 @@ executors: # Choose a Windows environment that closest matches our testing # requirements and goals. # https://circleci.com/developer/orbs/orb/circleci/windows#executors-server-2022 - executor: "win/server-2022@2023.06.1" + machine: + image: "win/server-2022@2023.06.1" nix: docker: From a8d582237c89c7829c93cc4a744f10a63e17e177 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 7 Aug 2023 10:31:16 -0400 Subject: [PATCH 2028/2309] dump the useless orb --- .circleci/config.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 07bdc7087..b9866ad91 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -11,10 +11,6 @@ # version: 2.1 -orbs: - # Pull in CircleCI support for a Windows executor - windows: "circleci/windows@5.0.0" - # Every job that pushes a Docker image from Docker Hub must authenticate to # it. Define a couple yaml anchors that can be used to supply the necessary # credentials. @@ -544,7 +540,7 @@ executors: # requirements and goals. # https://circleci.com/developer/orbs/orb/circleci/windows#executors-server-2022 machine: - image: "win/server-2022@2023.06.1" + image: "windows-server-2022-gui:2023.06.1"" nix: docker: From f826914c589f7e0505a77d0cff3f6b24b2c2b669 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 7 Aug 2023 10:31:32 -0400 Subject: [PATCH 2029/2309] syntax --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index b9866ad91..6e46d89a5 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -540,7 +540,7 @@ executors: # requirements and goals. # https://circleci.com/developer/orbs/orb/circleci/windows#executors-server-2022 machine: - image: "windows-server-2022-gui:2023.06.1"" + image: "windows-server-2022-gui:2023.06.1" nix: docker: From 4abbadda47faed2d2279787fad6058a83756eb31 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 7 Aug 2023 10:32:31 -0400 Subject: [PATCH 2030/2309] Try a different tag https://circleci.com/developer/machine/image/windows-server-2022-gui says "2023.06.1" is a valid tag but real execution says "Job was rejected because resource class medium, image windows-server-2022-gui:2023.06.1 is not a valid resource class". Is it even complaining about the image tag? Or is it complaining about the resource class? I don't know. I didn't touch the resource class though. --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 6e46d89a5..e4c061da5 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -540,7 +540,7 @@ executors: # requirements and goals. # https://circleci.com/developer/orbs/orb/circleci/windows#executors-server-2022 machine: - image: "windows-server-2022-gui:2023.06.1" + image: "windows-server-2022-gui:current" nix: docker: From bd9d2e08ef4746516d46e168db20b60b7f859dc6 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 7 Aug 2023 10:35:11 -0400 Subject: [PATCH 2031/2309] eh? --- .circleci/config.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index e4c061da5..63bec2dde 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -136,7 +136,8 @@ workflows: jobs: windows-server-2022: - executor: "windows" + executor: + name: "windows" steps: - "run": | Write-Host 'Hello, world.' From 77c677ffc0ab957a0bb089d98f62c3eb9e78d7e3 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 7 Aug 2023 10:37:23 -0400 Subject: [PATCH 2032/2309] just like the example --- .circleci/config.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 63bec2dde..e1be57175 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -136,8 +136,10 @@ workflows: jobs: windows-server-2022: - executor: - name: "windows" + machine: + image: "windows-server-2022-gui:current" + shell: "powershell.exe -ExecutionPolicy Bypass" + resource_class: "windows.medium" steps: - "run": | Write-Host 'Hello, world.' From 422d4ee9ccfd16e3334a7a0ec743380cd89d1b3b Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 7 Aug 2023 10:37:55 -0400 Subject: [PATCH 2033/2309] previous rev started an environment, try to get a little closer to the one we want a more precise tag that won't shift around underneath us --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index e1be57175..363c0ee6e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -137,7 +137,7 @@ workflows: jobs: windows-server-2022: machine: - image: "windows-server-2022-gui:current" + image: "windows-server-2022-gui:2023.06.1" shell: "powershell.exe -ExecutionPolicy Bypass" resource_class: "windows.medium" steps: From 6400a396615905ebf8fa4ac564f7d966d9d3866e Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 7 Aug 2023 10:39:13 -0400 Subject: [PATCH 2034/2309] so ... can we use an executor? --- .circleci/config.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 363c0ee6e..368156bb3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -136,10 +136,7 @@ workflows: jobs: windows-server-2022: - machine: - image: "windows-server-2022-gui:2023.06.1" - shell: "powershell.exe -ExecutionPolicy Bypass" - resource_class: "windows.medium" + executor: "windows" steps: - "run": | Write-Host 'Hello, world.' @@ -543,7 +540,9 @@ executors: # requirements and goals. # https://circleci.com/developer/orbs/orb/circleci/windows#executors-server-2022 machine: - image: "windows-server-2022-gui:current" + image: "windows-server-2022-gui:2023.06.1" + shell: "powershell.exe -ExecutionPolicy Bypass" + resource_class: "windows.medium" nix: docker: From d369dc0f2cdeeca255f53f5418540d3783d9f35f Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 7 Aug 2023 10:43:08 -0400 Subject: [PATCH 2035/2309] try to install tox --- .circleci/config.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 368156bb3..f86b46636 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -138,8 +138,10 @@ jobs: windows-server-2022: executor: "windows" steps: - - "run": | - Write-Host 'Hello, world.' + - "checkout" + + - "run": + <<: *INSTALL_TOX codechecks: docker: From e6e38128bc1ce1a6630eb95eeadcd6433662cb8e Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 7 Aug 2023 10:43:58 -0400 Subject: [PATCH 2036/2309] yaml syntax --- .circleci/config.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index f86b46636..e70daec41 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -135,14 +135,6 @@ workflows: when: "<< pipeline.parameters.build-images >>" jobs: - windows-server-2022: - executor: "windows" - steps: - - "checkout" - - - "run": - <<: *INSTALL_TOX - codechecks: docker: - <<: *DOCKERHUB_AUTH @@ -161,6 +153,14 @@ jobs: command: | ~/.local/bin/tox -e codechecks + windows-server-2022: + executor: "windows" + steps: + - "checkout" + + - "run": + <<: *INSTALL_TOX + pyinstaller: docker: - <<: *DOCKERHUB_AUTH From 862bda9e631770a06ffa3d9c86b717d2e2b9e0a3 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 7 Aug 2023 10:47:36 -0400 Subject: [PATCH 2037/2309] attempt to do something useful --- .circleci/config.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index e70daec41..5856af47d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -161,6 +161,16 @@ jobs: - "run": <<: *INSTALL_TOX + - "run": + name: "Display tool versions" + command: | + python misc/build_helpers/show-tool-versions.py + + - "run": + name: "Run Unit Tests" + command: | + python -m tox + pyinstaller: docker: - <<: *DOCKERHUB_AUTH From e2fe1af3d9303ba17e0c60b3e5d2bba43e4340a1 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 7 Aug 2023 11:03:16 -0400 Subject: [PATCH 2038/2309] Configure Hypothesis for the Windows job --- .circleci/config.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 5856af47d..19d70adca 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -155,6 +155,9 @@ jobs: windows-server-2022: executor: "windows" + environment: + TAHOE_LAFS_HYPOTHESIS_PROFILE: "ci" + steps: - "checkout" From 3c8a11d46822a9e7508567b970579a2d66276bf3 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 7 Aug 2023 11:12:47 -0400 Subject: [PATCH 2039/2309] pick a Python to support on Windows --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 19d70adca..e9e5810e5 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -172,7 +172,7 @@ jobs: - "run": name: "Run Unit Tests" command: | - python -m tox + python -m tox -e py311-coverage pyinstaller: docker: From b440065952cfe9c98d65ab7728571a93aba50a6e Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 7 Aug 2023 12:59:44 -0400 Subject: [PATCH 2040/2309] avoid trying to call os.getuid on windows --- src/allmydata/test/cli/test_grid_manager.py | 6 ++++-- src/allmydata/test/common.py | 4 ++++ src/allmydata/test/test_client.py | 3 ++- src/allmydata/test/test_node.py | 7 ++++--- 4 files changed, 14 insertions(+), 6 deletions(-) diff --git a/src/allmydata/test/cli/test_grid_manager.py b/src/allmydata/test/cli/test_grid_manager.py index 604cd6b7b..b44b322d2 100644 --- a/src/allmydata/test/cli/test_grid_manager.py +++ b/src/allmydata/test/cli/test_grid_manager.py @@ -23,6 +23,9 @@ import click.testing from ..common_util import ( run_cli, ) +from ..common import ( + superuser, +) from twisted.internet.defer import ( inlineCallbacks, ) @@ -34,7 +37,6 @@ from twisted.python.runtime import ( ) from allmydata.util import jsonbytes as json - class GridManagerCommandLine(TestCase): """ Test the mechanics of the `grid-manager` command @@ -223,7 +225,7 @@ class GridManagerCommandLine(TestCase): ) @skipIf(platform.isWindows(), "We don't know how to set permissions on Windows.") - @skipIf(os.getuid() == 0, "cannot test as superuser with all permissions") + @skipIf(superuser, "cannot test as superuser with all permissions") def test_sign_bad_perms(self): """ Error reported if we can't create certificate file diff --git a/src/allmydata/test/common.py b/src/allmydata/test/common.py index db2921e86..d61bc28f1 100644 --- a/src/allmydata/test/common.py +++ b/src/allmydata/test/common.py @@ -117,6 +117,10 @@ from subprocess import ( PIPE, ) +# Is the process running as an OS user with elevated privileges (ie, root)? +# We only know how to determine this for POSIX systems. +superuser = getattr(os, "getuid", lambda: -1)() == 0 + EMPTY_CLIENT_CONFIG = config_from_string( "/dev/null", "tub.port", diff --git a/src/allmydata/test/test_client.py b/src/allmydata/test/test_client.py index 86c95a310..c0cce2809 100644 --- a/src/allmydata/test/test_client.py +++ b/src/allmydata/test/test_client.py @@ -77,6 +77,7 @@ from allmydata.scripts.common import ( from foolscap.api import flushEventualQueue import allmydata.test.common_util as testutil from .common import ( + superuser, EMPTY_CLIENT_CONFIG, SyncTestCase, AsyncBrokenTestCase, @@ -151,7 +152,7 @@ class Basic(testutil.ReallyEqualMixin, unittest.TestCase): # EnvironmentError when reading a file that really exists), on # windows, please fix this @skipIf(platform.isWindows(), "We don't know how to set permissions on Windows.") - @skipIf(os.getuid() == 0, "cannot test as superuser with all permissions") + @skipIf(superuser, "cannot test as superuser with all permissions") def test_unreadable_config(self): basedir = "test_client.Basic.test_unreadable_config" os.mkdir(basedir) diff --git a/src/allmydata/test/test_node.py b/src/allmydata/test/test_node.py index 1469ec5b2..90da877fb 100644 --- a/src/allmydata/test/test_node.py +++ b/src/allmydata/test/test_node.py @@ -62,6 +62,7 @@ from .common import ( ConstantAddresses, SameProcessStreamEndpointAssigner, UseNode, + superuser, ) def port_numbers(): @@ -325,7 +326,7 @@ class TestCase(testutil.SignalMixin, unittest.TestCase): self.assertEqual(config.items("nosuch", default), default) @skipIf(platform.isWindows(), "We don't know how to set permissions on Windows.") - @skipIf(os.getuid() == 0, "cannot test as superuser with all permissions") + @skipIf(superuser, "cannot test as superuser with all permissions") def test_private_config_unreadable(self): """ Asking for inaccessible private config is an error @@ -341,7 +342,7 @@ class TestCase(testutil.SignalMixin, unittest.TestCase): config.get_or_create_private_config("foo") @skipIf(platform.isWindows(), "We don't know how to set permissions on Windows.") - @skipIf(os.getuid() == 0, "cannot test as superuser with all permissions") + @skipIf(superuser, "cannot test as superuser with all permissions") def test_private_config_unreadable_preexisting(self): """ error if reading private config data fails @@ -398,7 +399,7 @@ class TestCase(testutil.SignalMixin, unittest.TestCase): self.assertEqual(len(counter), 1) # don't call unless necessary self.assertEqual(value, "newer") - @skipIf(os.getuid() == 0, "cannot test as superuser with all permissions") + @skipIf(superuser, "cannot test as superuser with all permissions") def test_write_config_unwritable_file(self): """ Existing behavior merely logs any errors upon writing From e92e7faeea00c1dacef75d36bf1815b5bdf296ad Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 7 Aug 2023 15:25:58 -0400 Subject: [PATCH 2041/2309] how much difference does this make --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index e9e5810e5..d204a5531 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -557,7 +557,7 @@ executors: machine: image: "windows-server-2022-gui:2023.06.1" shell: "powershell.exe -ExecutionPolicy Bypass" - resource_class: "windows.medium" + resource_class: "windows.large" nix: docker: From 9f5173e7302bb1258f64e03b5f8506587a816a4f Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 7 Aug 2023 15:58:33 -0400 Subject: [PATCH 2042/2309] attempt to report coverage results to coveralls --- .circleci/config.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index d204a5531..431b2c70a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -174,6 +174,12 @@ jobs: command: | python -m tox -e py311-coverage + - "run": + name: "Upload Coverage" + command: | + python -m pip install coveralls + python -m coveralls + pyinstaller: docker: - <<: *DOCKERHUB_AUTH From f649968ab5442dc1fa76554084cae99fca660a24 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 7 Aug 2023 16:21:27 -0400 Subject: [PATCH 2043/2309] configure the coveralls tool so it can upload the data --- .coveralls.yml | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 .coveralls.yml diff --git a/.coveralls.yml b/.coveralls.yml new file mode 100644 index 000000000..1486cf5b3 --- /dev/null +++ b/.coveralls.yml @@ -0,0 +1,2 @@ +service_name: "circleci" +repo_token: "JPf16rLB7T2yjgATIxFzTsEgMdN1UNq6o" From 371f82bb4da019df5581ed0c41e7d500b902ac3f Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 7 Aug 2023 16:21:50 -0400 Subject: [PATCH 2044/2309] avoid problems with trial and ENOSPC I don't know if these will show up in this environment ... just copy/pasted from the GitHub Actions config. --- .circleci/config.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 431b2c70a..2f580192d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -172,7 +172,13 @@ jobs: - "run": name: "Run Unit Tests" command: | - python -m tox -e py311-coverage + # On Windows, a non-blocking pipe might respond (when emulating + # Unix-y API) with ENOSPC to indicate buffer full. Trial doesn't + # handle this well, so it breaks test runs. To attempt to solve + # this, we pipe the output through passthrough.py that will + # hopefully be able to do the right thing by using Windows APIs. + python -m pip install twisted pywin32 + python -m tox -e py311-coverage | python misc/windows-enospc/passthrough.py - "run": name: "Upload Coverage" From 4c0b72b353615499f5eb3213d65bdc7644982b8c Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 7 Aug 2023 16:38:56 -0400 Subject: [PATCH 2045/2309] Delightfully, this deterministically breaks in the CircleCI env --- .circleci/config.yml | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 2f580192d..431b2c70a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -172,13 +172,7 @@ jobs: - "run": name: "Run Unit Tests" command: | - # On Windows, a non-blocking pipe might respond (when emulating - # Unix-y API) with ENOSPC to indicate buffer full. Trial doesn't - # handle this well, so it breaks test runs. To attempt to solve - # this, we pipe the output through passthrough.py that will - # hopefully be able to do the right thing by using Windows APIs. - python -m pip install twisted pywin32 - python -m tox -e py311-coverage | python misc/windows-enospc/passthrough.py + python -m tox -e py311-coverage - "run": name: "Upload Coverage" From e17c8f618ea9d9a2f9fd169d75f714401121c35e Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 7 Aug 2023 16:39:52 -0400 Subject: [PATCH 2046/2309] run a quick subset of the tests to more quickly test the following bits --- .circleci/config.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 431b2c70a..42e59075b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -171,6 +171,8 @@ jobs: - "run": name: "Run Unit Tests" + environment: + TEST_SUITE: "allmydata.test.test_uri" command: | python -m tox -e py311-coverage From b092dd57cf62b174f731e10c231d26235e72ec1f Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 7 Aug 2023 17:02:55 -0400 Subject: [PATCH 2047/2309] coveralls failed to find .coveralls.yml ... also tox.ini overrides TEST_SUITE :/ Set it in the right place --- .circleci/config.yml | 5 +++-- .coveralls.yml | 2 -- tox.ini | 2 +- 3 files changed, 4 insertions(+), 5 deletions(-) delete mode 100644 .coveralls.yml diff --git a/.circleci/config.yml b/.circleci/config.yml index 42e59075b..077896d92 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -171,15 +171,16 @@ jobs: - "run": name: "Run Unit Tests" - environment: - TEST_SUITE: "allmydata.test.test_uri" command: | python -m tox -e py311-coverage - "run": name: "Upload Coverage" + environment: + COVERALLS_REPO_TOKEN: "JPf16rLB7T2yjgATIxFzTsEgMdN1UNq6o" command: | python -m pip install coveralls + python -m coveralls debug python -m coveralls pyinstaller: diff --git a/.coveralls.yml b/.coveralls.yml deleted file mode 100644 index 1486cf5b3..000000000 --- a/.coveralls.yml +++ /dev/null @@ -1,2 +0,0 @@ -service_name: "circleci" -repo_token: "JPf16rLB7T2yjgATIxFzTsEgMdN1UNq6o" diff --git a/tox.ini b/tox.ini index 67a089b0c..18d7767a2 100644 --- a/tox.ini +++ b/tox.ini @@ -55,7 +55,7 @@ extras = setenv = # Define TEST_SUITE in the environment as an aid to constructing the # correct test command below. - TEST_SUITE = allmydata + TEST_SUITE = allmydata.test.test_uri commands = # As an aid to debugging, dump all of the Python packages and their From a261c1f2025966bdc944f8023185697ed1816895 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 7 Aug 2023 17:11:54 -0400 Subject: [PATCH 2048/2309] try to match the paths from circleci windows for coverage path rewriting --- .coveragerc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.coveragerc b/.coveragerc index d09554cad..32b803586 100644 --- a/.coveragerc +++ b/.coveragerc @@ -19,9 +19,11 @@ skip_covered = True source = # It looks like this in the checkout src/ -# It looks like this in the Windows build environment +# It looks like this in the GitHub Actions Windows build environment D:/a/tahoe-lafs/tahoe-lafs/.tox/py*-coverage/Lib/site-packages/ # Although sometimes it looks like this instead. Also it looks like this on macOS. .tox/py*-coverage/lib/python*/site-packages/ +# And on the CircleCI Windows build envronment... + .tox/py*-coverage/Lib/site-packages/ # On some Linux CI jobs it looks like this /tmp/tahoe-lafs.tox/py*-coverage/lib/python*/site-packages/ From 5c22bf95b41958d1fbcae5362fb6deea0eed98e9 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 7 Aug 2023 17:12:06 -0400 Subject: [PATCH 2049/2309] maybe we don't need the debug info now --- .circleci/config.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 077896d92..fe1f5f8cb 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -180,7 +180,6 @@ jobs: COVERALLS_REPO_TOKEN: "JPf16rLB7T2yjgATIxFzTsEgMdN1UNq6o" command: | python -m pip install coveralls - python -m coveralls debug python -m coveralls pyinstaller: From 7bc1f9300f3da31f5b283b0de20f30b4d9cdd05d Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 8 Aug 2023 08:24:24 -0400 Subject: [PATCH 2050/2309] try to get test results loaded into circleci --- .circleci/config.yml | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index fe1f5f8cb..ede1cdb8f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -170,9 +170,17 @@ jobs: python misc/build_helpers/show-tool-versions.py - "run": - name: "Run Unit Tests" + name: "Install Dependencies" command: | - python -m tox -e py311-coverage + python -m pip install .[testenv] .[test] + + - "run": + name: "Run Unit Tests" + environment: + SUBUNITREPORTER_OUTPUT_PATH: "test-results.subunit2" + PYTHONUNBUFFERED: "1" + command: | + python -b -m coverage run -m twisted.trial --reporter=subunitv2-file --rterrors allmydata - "run": name: "Upload Coverage" @@ -182,6 +190,14 @@ jobs: python -m pip install coveralls python -m coveralls + - "run": + name: "Convert Result Log" + command: | + Get-Content -Path test-results.subunit2 -Raw | subunit2junitxml | Out-File -FilePath test-results.xml + + - "store_artifacts": + path: "test-results.xml" + pyinstaller: docker: - <<: *DOCKERHUB_AUTH From 3f37f9aee5975453a5b0d9d4f8a978c5080a2a05 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 8 Aug 2023 08:44:15 -0400 Subject: [PATCH 2051/2309] try to force UTF-8 to make subunitreporter work --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ede1cdb8f..1946c9346 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -180,7 +180,7 @@ jobs: SUBUNITREPORTER_OUTPUT_PATH: "test-results.subunit2" PYTHONUNBUFFERED: "1" command: | - python -b -m coverage run -m twisted.trial --reporter=subunitv2-file --rterrors allmydata + python -X utf8 -b -m coverage run -m twisted.trial --reporter=subunitv2-file --rterrors allmydata - "run": name: "Upload Coverage" From b03cf0b37b5ca2dbe443aee2c142b8d6ddd4d295 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 8 Aug 2023 09:14:06 -0400 Subject: [PATCH 2052/2309] send the test results to the place circleci expects for processing --- .circleci/config.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 1946c9346..e8815842e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -195,6 +195,9 @@ jobs: command: | Get-Content -Path test-results.subunit2 -Raw | subunit2junitxml | Out-File -FilePath test-results.xml + - "store_test_results: + path: "test-results.xml" + - "store_artifacts": path: "test-results.xml" From 3a8480126b1813ad6eab3ac93df191176bc1a085 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 8 Aug 2023 09:19:20 -0400 Subject: [PATCH 2053/2309] syntax --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index e8815842e..845ac3662 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -195,7 +195,7 @@ jobs: command: | Get-Content -Path test-results.subunit2 -Raw | subunit2junitxml | Out-File -FilePath test-results.xml - - "store_test_results: + - "store_test_results": path: "test-results.xml" - "store_artifacts": From 286fb206d812f62bd7af08b3332bd3f927e948df Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 8 Aug 2023 09:59:23 -0400 Subject: [PATCH 2054/2309] try uncorrupting the xml file --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 845ac3662..74bf3019d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -193,7 +193,7 @@ jobs: - "run": name: "Convert Result Log" command: | - Get-Content -Path test-results.subunit2 -Raw | subunit2junitxml | Out-File -FilePath test-results.xml + subunit2junitxml test-results.xml - "store_test_results": path: "test-results.xml" From ab9db4964c0f4fa671d78240d1b015c891f39379 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 8 Aug 2023 10:28:16 -0400 Subject: [PATCH 2055/2309] another stab --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 74bf3019d..91d46414d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -180,7 +180,7 @@ jobs: SUBUNITREPORTER_OUTPUT_PATH: "test-results.subunit2" PYTHONUNBUFFERED: "1" command: | - python -X utf8 -b -m coverage run -m twisted.trial --reporter=subunitv2-file --rterrors allmydata + python -X utf8 -b -m coverage run -m twisted.trial --reporter=subunitv2-file --rterrors allmydata.test.test_uri - "run": name: "Upload Coverage" @@ -193,7 +193,7 @@ jobs: - "run": name: "Convert Result Log" command: | - subunit2junitxml test-results.xml + Start-Process subunit2junitxml -Wait -RedirectStandardInput test-results.subunit2 -RedirectStandardOutput test-results.xml - "store_test_results": path: "test-results.xml" From 2649027c74b53499f2bc0315186a14667bac72b8 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 8 Aug 2023 10:38:16 -0400 Subject: [PATCH 2056/2309] back to the complete test suite --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 91d46414d..d7e576298 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -180,7 +180,7 @@ jobs: SUBUNITREPORTER_OUTPUT_PATH: "test-results.subunit2" PYTHONUNBUFFERED: "1" command: | - python -X utf8 -b -m coverage run -m twisted.trial --reporter=subunitv2-file --rterrors allmydata.test.test_uri + python -X utf8 -b -m coverage run -m twisted.trial --reporter=subunitv2-file --rterrors allmydata - "run": name: "Upload Coverage" From 208531cddc5db10c419b6509c3223ed3a297b4e2 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 8 Aug 2023 10:38:33 -0400 Subject: [PATCH 2057/2309] should work without `-X utf8` now --- .circleci/config.yml | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index d7e576298..13936321a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -180,7 +180,7 @@ jobs: SUBUNITREPORTER_OUTPUT_PATH: "test-results.subunit2" PYTHONUNBUFFERED: "1" command: | - python -X utf8 -b -m coverage run -m twisted.trial --reporter=subunitv2-file --rterrors allmydata + python -b -m coverage run -m twisted.trial --reporter=subunitv2-file --rterrors allmydata - "run": name: "Upload Coverage" diff --git a/setup.py b/setup.py index 86873ad53..433721d2a 100644 --- a/setup.py +++ b/setup.py @@ -413,7 +413,7 @@ setup(name="tahoe-lafs", # also set in __init__.py "pip==22.0.3", "wheel==0.37.1", "setuptools==60.9.1", - "subunitreporter==22.2.0", + "subunitreporter==23.8.0", "python-subunit==1.4.2", "junitxml==0.7", "coverage==7.2.5", From 122655842cd3dd25af9a722639a338caca0fe188 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 8 Aug 2023 11:06:31 -0400 Subject: [PATCH 2058/2309] try to expose the other test run artifacts --- .circleci/config.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 13936321a..4e4e183a5 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -199,7 +199,10 @@ jobs: path: "test-results.xml" - "store_artifacts": - path: "test-results.xml" + path: "_trial_temp/test.log" + + - "store_artifacts": + path: "eliot.log" pyinstaller: docker: From 085a823dfdfa12a3911a6173ed8659724141af5f Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 8 Aug 2023 11:55:34 -0400 Subject: [PATCH 2059/2309] put back the full test suite for tox runs --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 18d7767a2..67a089b0c 100644 --- a/tox.ini +++ b/tox.ini @@ -55,7 +55,7 @@ extras = setenv = # Define TEST_SUITE in the environment as an aid to constructing the # correct test command below. - TEST_SUITE = allmydata.test.test_uri + TEST_SUITE = allmydata commands = # As an aid to debugging, dump all of the Python packages and their From 23628fffd84f6a358c22fccf1ca661ac10e9cfb7 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 8 Aug 2023 12:12:45 -0400 Subject: [PATCH 2060/2309] Try to parameterize the Python version for Windows tests And instantiate the job with two different Python versions --- .circleci/config.yml | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 4e4e183a5..4181025cc 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -112,7 +112,13 @@ workflows: - "another-locale": {} - - "windows-server-2022" + - "windows-server-2022": + name: "Windows Server 2022, Python <>" + matrix: + parameters: + pythonVersion: + - "3.9" + - "3.11" - "integration": # Run even the slow integration tests here. We need the `--` to @@ -154,6 +160,13 @@ jobs: ~/.local/bin/tox -e codechecks windows-server-2022: + parameters: + pythonVersion: + description: >- + An argument to pass to the `py` launcher to choose a Python version. + type: "string" + default: "" + executor: "windows" environment: TAHOE_LAFS_HYPOTHESIS_PROFILE: "ci" @@ -167,12 +180,12 @@ jobs: - "run": name: "Display tool versions" command: | - python misc/build_helpers/show-tool-versions.py + py -<> misc/build_helpers/show-tool-versions.py - "run": name: "Install Dependencies" command: | - python -m pip install .[testenv] .[test] + py -<> -m pip install .[testenv] .[test] - "run": name: "Run Unit Tests" @@ -180,15 +193,15 @@ jobs: SUBUNITREPORTER_OUTPUT_PATH: "test-results.subunit2" PYTHONUNBUFFERED: "1" command: | - python -b -m coverage run -m twisted.trial --reporter=subunitv2-file --rterrors allmydata + py -<> -b -m coverage run -m twisted.trial --reporter=subunitv2-file --rterrors allmydata - "run": name: "Upload Coverage" environment: COVERALLS_REPO_TOKEN: "JPf16rLB7T2yjgATIxFzTsEgMdN1UNq6o" command: | - python -m pip install coveralls - python -m coveralls + py -<> -m pip install coveralls + py -<> -m coveralls - "run": name: "Convert Result Log" From 67cc25df11d8c202de4ff9addd3b9c798e492009 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 8 Aug 2023 12:13:31 -0400 Subject: [PATCH 2061/2309] drop Windows unit tests from GitHub Actions This drops Python 3.8 and Python 3.10 Windows coverage. --- .github/workflows/ci.yml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d3862ffad..0f38b0291 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -44,13 +44,6 @@ jobs: strategy: fail-fast: false matrix: - os: - - windows-latest - python-version: - - "3.8" - - "3.9" - - "3.10" - - "3.11" include: # On macOS don't bother with 3.8, just to get faster builds. - os: macos-12 From 8c4d99f812de8254ce610bb9451fcdae4b4edcca Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 8 Aug 2023 12:22:18 -0400 Subject: [PATCH 2062/2309] try to do "parallel" coveralls reporting and finish it --- .circleci/config.yml | 34 +++++++++++++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 4181025cc..f7665cc29 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -24,6 +24,11 @@ version: 2.1 dockerhub-context-template: &DOCKERHUB_CONTEXT context: "dockerhub-auth" +# Required environment for using the coveralls tool to upload partial coverage +# reports and then finish the process. +coveralls-environment: &COVERALLS_ENVIRONMENT + COVERALLS_REPO_TOKEN: "JPf16rLB7T2yjgATIxFzTsEgMdN1UNq6o" + # Next is a Docker executor template that gets the credentials from the # environment and supplies them to the executor. dockerhub-auth-template: &DOCKERHUB_AUTH @@ -134,6 +139,11 @@ workflows: - "docs": {} + - "finish-coverage-report": + requires: + - "Windows Server 2022, Python 3.9" + - "Windows Server 2022, Python 3.11" + images: <<: *IMAGES @@ -141,6 +151,20 @@ workflows: when: "<< pipeline.parameters.build-images >>" jobs: + finish-coverage-report: + docker: + - <<: *DOCKERHUB_AUTH + image: "python:3-slim" + + steps: + - run: + name: "Indicate completion to coveralls.io" + environment: + <<: *COVERALLS_ENVIRONMENT + command: | + pip install coveralls==3.2.0 + python -m coveralls --finish + codechecks: docker: - <<: *DOCKERHUB_AUTH @@ -198,7 +222,15 @@ jobs: - "run": name: "Upload Coverage" environment: - COVERALLS_REPO_TOKEN: "JPf16rLB7T2yjgATIxFzTsEgMdN1UNq6o" + <<: *COVERALLS_ENVIRONMENT + # Mark the data as just one piece of many because we have more + # than one instance of this job (two on Windows now, some on other + # platforms later) which collects and reports coverage. This is + # necessary to cause Coveralls to merge multiple coverage results + # into a single report. Note the merge only happens when we + # "finish" a particular build, as identified by its "build_num" + # (aka "service_number"). + COVERALLS_PARALLEL: "true" command: | py -<> -m pip install coveralls py -<> -m coveralls From df05ed3c8f45ef880113c75ca2825e77e4deb0e4 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 8 Aug 2023 13:53:17 -0400 Subject: [PATCH 2063/2309] maybe ... not gonna use tox at all here? --- .circleci/config.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index f7665cc29..66e8c1553 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -198,9 +198,6 @@ jobs: steps: - "checkout" - - "run": - <<: *INSTALL_TOX - - "run": name: "Display tool versions" command: | From 72739b0606c20c5a1cc0223dd7c1415d103ba7cb Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 8 Aug 2023 13:57:17 -0400 Subject: [PATCH 2064/2309] try to be sure we can do the conversion --- .circleci/config.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 66e8c1553..85cdfb815 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -235,6 +235,10 @@ jobs: - "run": name: "Convert Result Log" command: | + # The Python for which we installed subunit is not necessarily on + # %PATH% so (possibly) re-install it with the default Python. + python -m pip install subunit2junitxml junitxml + Start-Process subunit2junitxml -Wait -RedirectStandardInput test-results.subunit2 -RedirectStandardOutput test-results.xml - "store_test_results": From 179f7b4bcb3f2021cfba2a0231ef0b5fc97fe6aa Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 8 Aug 2023 14:16:55 -0400 Subject: [PATCH 2065/2309] get the package name right ... sigh ... --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 85cdfb815..b5f081441 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -237,7 +237,7 @@ jobs: command: | # The Python for which we installed subunit is not necessarily on # %PATH% so (possibly) re-install it with the default Python. - python -m pip install subunit2junitxml junitxml + python -m pip install subunit junitxml Start-Process subunit2junitxml -Wait -RedirectStandardInput test-results.subunit2 -RedirectStandardOutput test-results.xml From fd6c7c880d9589e6b09a33630e441aa28f9b2736 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 8 Aug 2023 15:39:17 -0400 Subject: [PATCH 2066/2309] try to run the program a different way --- .circleci/config.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index b5f081441..c9cb657b6 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -238,8 +238,7 @@ jobs: # The Python for which we installed subunit is not necessarily on # %PATH% so (possibly) re-install it with the default Python. python -m pip install subunit junitxml - - Start-Process subunit2junitxml -Wait -RedirectStandardInput test-results.subunit2 -RedirectStandardOutput test-results.xml + subunit2junitxml --output-to=test-results.xml test-results.subunit2 - "store_test_results": path: "test-results.xml" From 1a7a552e0dd773f25f982c14e9ff61e39ce00de1 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 8 Aug 2023 16:05:22 -0400 Subject: [PATCH 2067/2309] where's my test output file --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index c9cb657b6..b164b42f1 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -237,6 +237,7 @@ jobs: command: | # The Python for which we installed subunit is not necessarily on # %PATH% so (possibly) re-install it with the default Python. + Set-PSDebug -Trace 2 python -m pip install subunit junitxml subunit2junitxml --output-to=test-results.xml test-results.subunit2 From 9c43a99c53deac29e28c8a408469257aa64f492b Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 8 Aug 2023 17:02:34 -0400 Subject: [PATCH 2068/2309] maybe python-subunit is less broken --- .circleci/config.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index b164b42f1..a13ce40d2 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -237,8 +237,7 @@ jobs: command: | # The Python for which we installed subunit is not necessarily on # %PATH% so (possibly) re-install it with the default Python. - Set-PSDebug -Trace 2 - python -m pip install subunit junitxml + python -m pip install python-subunit junitxml subunit2junitxml --output-to=test-results.xml test-results.subunit2 - "store_test_results": From 7a389bb3149cc6245d8373eacb38aac9976533dd Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 9 Aug 2023 09:23:43 -0400 Subject: [PATCH 2069/2309] it's cpython --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index a13ce40d2..a1092741e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -118,7 +118,7 @@ workflows: {} - "windows-server-2022": - name: "Windows Server 2022, Python <>" + name: "Windows Server 2022, CPython <>" matrix: parameters: pythonVersion: From d56ac6d6a2becf03d0fe3387340dfad19512d7f0 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 9 Aug 2023 09:24:37 -0400 Subject: [PATCH 2070/2309] note about error behavior of subunit2junitxml --- .circleci/config.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index a1092741e..12acea784 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -238,6 +238,11 @@ jobs: # The Python for which we installed subunit is not necessarily on # %PATH% so (possibly) re-install it with the default Python. python -m pip install python-subunit junitxml + + # subunit2junitxml exits with error if the result stream it is + # converting has test failures in it! So this step might fail. + # Since the step in which we actually _ran_ the tests won't fail + # even if there are test failures, this is a good thing for now. subunit2junitxml --output-to=test-results.xml test-results.subunit2 - "store_test_results": From e04340f30a2b45ddb293c01ba8852a547733303b Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 9 Aug 2023 09:28:42 -0400 Subject: [PATCH 2071/2309] supposedly this will work --- .circleci/config.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 12acea784..d4a2530c3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -141,8 +141,12 @@ workflows: - "finish-coverage-report": requires: - - "Windows Server 2022, Python 3.9" - - "Windows Server 2022, Python 3.11" + # Referencing the job by "alias" (as CircleCI calls the mapping + # key) instead of the value of its "name" property causes us to + # require every instance of the job from its matrix expansion. So + # this requirement is enough to require every Windows Server 2022 + # job. + - "windows-server-2022" images: <<: *IMAGES From d8df6d12d783ea6e0362cc62437b7355fa61af9a Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 9 Aug 2023 09:34:32 -0400 Subject: [PATCH 2072/2309] pin/upgrade coveralls --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index d4a2530c3..6e795c4b9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -166,7 +166,7 @@ jobs: environment: <<: *COVERALLS_ENVIRONMENT command: | - pip install coveralls==3.2.0 + pip install coveralls==3.3.1 python -m coveralls --finish codechecks: @@ -233,7 +233,7 @@ jobs: # (aka "service_number"). COVERALLS_PARALLEL: "true" command: | - py -<> -m pip install coveralls + py -<> -m pip install coveralls==3.3.1 py -<> -m coveralls - "run": From 7ebb3a2eadbd34cf0bd1a02cc2b2bf2e58551b2d Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 9 Aug 2023 09:34:40 -0400 Subject: [PATCH 2073/2309] some comments --- .circleci/config.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 6e795c4b9..1e45c5235 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -215,9 +215,18 @@ jobs: - "run": name: "Run Unit Tests" environment: + # Configure the results location for the subunitv2-file reporter + # from subunitreporter SUBUNITREPORTER_OUTPUT_PATH: "test-results.subunit2" + + # Try to get prompt output from the reporter to avoid no-output + # timeouts. PYTHONUNBUFFERED: "1" + command: | + # Run the test suite under coverage measurement using the + # parameterized version of Python, writing subunitv2-format + # results to the file given in the environment. py -<> -b -m coverage run -m twisted.trial --reporter=subunitv2-file --rterrors allmydata - "run": From 3e1f62fd7b3b6cf8d8ff226048cf36e122345f70 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 9 Aug 2023 09:36:32 -0400 Subject: [PATCH 2074/2309] cut down the test suite for faster testing, again --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 1e45c5235..dc73c18e7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -227,7 +227,7 @@ jobs: # Run the test suite under coverage measurement using the # parameterized version of Python, writing subunitv2-format # results to the file given in the environment. - py -<> -b -m coverage run -m twisted.trial --reporter=subunitv2-file --rterrors allmydata + py -<> -b -m coverage run -m twisted.trial --reporter=subunitv2-file --rterrors allmydata.test.test_uri - "run": name: "Upload Coverage" From e27c2e97411fcaba8d0bb105c1636fe1816bd406 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 9 Aug 2023 09:45:29 -0400 Subject: [PATCH 2075/2309] where's the coverage? --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index dc73c18e7..8dcb4c150 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -243,6 +243,7 @@ jobs: COVERALLS_PARALLEL: "true" command: | py -<> -m pip install coveralls==3.3.1 + py -<> -m coveralls debug py -<> -m coveralls - "run": From 38d6e5d8408e59c00f88d206639da241434dd4a3 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 9 Aug 2023 09:56:43 -0400 Subject: [PATCH 2076/2309] merge the "parallel" coverage files before invoking coveralls --- .circleci/config.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 8dcb4c150..487182ec3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -243,7 +243,15 @@ jobs: COVERALLS_PARALLEL: "true" command: | py -<> -m pip install coveralls==3.3.1 - py -<> -m coveralls debug + + # .coveragerc sets parallel = True so we don't have a `.coverage` + # file but a `.coverage.` file (or maybe more than + # one, but probably not). coveralls can't work with these so + # merge them before invoking it. + py -<> -m coverage combine + + # Now coveralls will be able to find the data, so have it do the + # upload. py -<> -m coveralls - "run": From 18c5f090518f694a684aae3948a89df0b7ffec29 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 9 Aug 2023 09:56:57 -0400 Subject: [PATCH 2077/2309] upload the coverage results to circleci too --- .circleci/config.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 487182ec3..55f881e4b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -276,6 +276,9 @@ jobs: - "store_artifacts": path: "eliot.log" + - "store_artifacts": + path: ".coverage" + pyinstaller: docker: - <<: *DOCKERHUB_AUTH From 571ded8680d09ab5a22b0a326f6b9944b843878f Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 9 Aug 2023 10:18:05 -0400 Subject: [PATCH 2078/2309] try to get the coveralls we already have instead of installing again --- .circleci/config.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 55f881e4b..865020c1c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -258,14 +258,15 @@ jobs: name: "Convert Result Log" command: | # The Python for which we installed subunit is not necessarily on - # %PATH% so (possibly) re-install it with the default Python. - python -m pip install python-subunit junitxml + # %PATH% so put it there. + $p = py -<> -c "import sys; print(sys.prefix)" + $env:PATH = "$env:PATH;$p\Scripts" # subunit2junitxml exits with error if the result stream it is # converting has test failures in it! So this step might fail. # Since the step in which we actually _ran_ the tests won't fail # even if there are test failures, this is a good thing for now. - subunit2junitxml --output-to=test-results.xml test-results.subunit2 + subunit2junitxml.exe --output-to=test-results.xml test-results.subunit2 - "store_test_results": path: "test-results.xml" From 4c16744199b87e7ba2700219e3a56992f851820c Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 9 Aug 2023 10:24:29 -0400 Subject: [PATCH 2079/2309] try to settle %PATH% once and for all at the start of the job --- .circleci/config.yml | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 865020c1c..12d3536e0 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -202,15 +202,26 @@ jobs: steps: - "checkout" + - "run": + name: "Fix $env:PATH" + command: | + # The Python for which we installed subunit is not necessarily on + # %PATH% so put it there. + # gets tools from packages we install. + $p = py -<> -c "import sys; print(sys.prefix)" + New-Item $Profile.CurrentUserAllHosts -Force + # $p gets "python" on PATH and $p\Scripts + Add-Content -Path $Profile.CurrentUserAllHosts -Value '$env:PATH = "$p;$p\Scripts;$env:PATH"' + - "run": name: "Display tool versions" command: | - py -<> misc/build_helpers/show-tool-versions.py + python misc/build_helpers/show-tool-versions.py - "run": name: "Install Dependencies" command: | - py -<> -m pip install .[testenv] .[test] + python -m pip install .[testenv] .[test] - "run": name: "Run Unit Tests" @@ -227,7 +238,7 @@ jobs: # Run the test suite under coverage measurement using the # parameterized version of Python, writing subunitv2-format # results to the file given in the environment. - py -<> -b -m coverage run -m twisted.trial --reporter=subunitv2-file --rterrors allmydata.test.test_uri + python -b -m coverage run -m twisted.trial --reporter=subunitv2-file --rterrors allmydata.test.test_uri - "run": name: "Upload Coverage" @@ -242,26 +253,21 @@ jobs: # (aka "service_number"). COVERALLS_PARALLEL: "true" command: | - py -<> -m pip install coveralls==3.3.1 + python -m pip install coveralls==3.3.1 # .coveragerc sets parallel = True so we don't have a `.coverage` # file but a `.coverage.` file (or maybe more than # one, but probably not). coveralls can't work with these so # merge them before invoking it. - py -<> -m coverage combine + python -m coverage combine # Now coveralls will be able to find the data, so have it do the # upload. - py -<> -m coveralls + python -m coveralls - "run": name: "Convert Result Log" command: | - # The Python for which we installed subunit is not necessarily on - # %PATH% so put it there. - $p = py -<> -c "import sys; print(sys.prefix)" - $env:PATH = "$env:PATH;$p\Scripts" - # subunit2junitxml exits with error if the result stream it is # converting has test failures in it! So this step might fail. # Since the step in which we actually _ran_ the tests won't fail From 139a329b38c9457b7875731dedc713f89a469324 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 9 Aug 2023 10:29:18 -0400 Subject: [PATCH 2080/2309] debug PATH setup --- .circleci/config.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 12d3536e0..9a1f8cd41 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -213,6 +213,11 @@ jobs: # $p gets "python" on PATH and $p\Scripts Add-Content -Path $Profile.CurrentUserAllHosts -Value '$env:PATH = "$p;$p\Scripts;$env:PATH"' + - "run": + name: "Reveal $env:PATH" + command: | + $env:PATH + - "run": name: "Display tool versions" command: | From 66177ae28e653d8134ae2d2da425a44b5cd5b931 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 9 Aug 2023 10:44:00 -0400 Subject: [PATCH 2081/2309] how about this impressive construction? previous version was constructing the value string wrong --- .circleci/config.yml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 9a1f8cd41..e530ddeb5 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -207,11 +207,15 @@ jobs: command: | # The Python for which we installed subunit is not necessarily on # %PATH% so put it there. - # gets tools from packages we install. $p = py -<> -c "import sys; print(sys.prefix)" + $q = py -<> -c "import sysconfig; print(sysconfig.get_path('scripts'))" + New-Item $Profile.CurrentUserAllHosts -Force - # $p gets "python" on PATH and $p\Scripts - Add-Content -Path $Profile.CurrentUserAllHosts -Value '$env:PATH = "$p;$p\Scripts;$env:PATH"' + # $p gets "python" on PATH and $q gets tools from packages we + # install. Note we carefully construct the string so that + # $env:PATH is not substituted now but $p and $q are. ` is the + # PowerShell string escape character. + Add-Content -Path $Profile.CurrentUserAllHosts -Value "`$env:PATH = `"$p;$q;`$env:PATH`"' - "run": name: "Reveal $env:PATH" From 0995b77020116d72abe3390a387231f124a1e599 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 9 Aug 2023 10:44:17 -0400 Subject: [PATCH 2082/2309] try stripping the interpreter-specific prefix from our paths --- .circleci/config.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index e530ddeb5..501e28b9a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -271,8 +271,10 @@ jobs: python -m coverage combine # Now coveralls will be able to find the data, so have it do the - # upload. - python -m coveralls + # upload. Also, have it strip the system config-specific prefix + # from all of the source paths. + $prefix = python -c "import sysconfig; print(sysconfig.get_path('purelib'))" + python -m coveralls --basedir $prefix - "run": name: "Convert Result Log" From f17939009466780d1b4d114cc4e886d160047f78 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 9 Aug 2023 10:52:10 -0400 Subject: [PATCH 2083/2309] fix quoting bug --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 501e28b9a..050871832 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -215,7 +215,7 @@ jobs: # install. Note we carefully construct the string so that # $env:PATH is not substituted now but $p and $q are. ` is the # PowerShell string escape character. - Add-Content -Path $Profile.CurrentUserAllHosts -Value "`$env:PATH = `"$p;$q;`$env:PATH`"' + Add-Content -Path $Profile.CurrentUserAllHosts -Value "`$env:PATH = `"$p;$q;`$env:PATH`" - "run": name: "Reveal $env:PATH" From e1269c836d39571aae4594be7e1cd9f419aa7fc1 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 9 Aug 2023 10:55:49 -0400 Subject: [PATCH 2084/2309] try again with closing quote --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 050871832..55be820a7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -215,7 +215,7 @@ jobs: # install. Note we carefully construct the string so that # $env:PATH is not substituted now but $p and $q are. ` is the # PowerShell string escape character. - Add-Content -Path $Profile.CurrentUserAllHosts -Value "`$env:PATH = `"$p;$q;`$env:PATH`" + Add-Content -Path $Profile.CurrentUserAllHosts -Value "`$env:PATH = `"$p;$q;`$env:PATH`"" - "run": name: "Reveal $env:PATH" From 89506a6f828f8da8e845b377fa4a8d4470ca076e Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 9 Aug 2023 11:03:33 -0400 Subject: [PATCH 2085/2309] back to the full test suite --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 55be820a7..b7f3ffd85 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -247,7 +247,7 @@ jobs: # Run the test suite under coverage measurement using the # parameterized version of Python, writing subunitv2-format # results to the file given in the environment. - python -b -m coverage run -m twisted.trial --reporter=subunitv2-file --rterrors allmydata.test.test_uri + python -b -m coverage run -m twisted.trial --reporter=subunitv2-file --rterrors allmydata - "run": name: "Upload Coverage" From e072fb60b85f23d801fed8086608bf26db4f0dd6 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 9 Aug 2023 11:20:28 -0400 Subject: [PATCH 2086/2309] fix the comment above PATH manipulation --- .circleci/config.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index b7f3ffd85..465a97a6b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -205,8 +205,11 @@ jobs: - "run": name: "Fix $env:PATH" command: | - # The Python for which we installed subunit is not necessarily on - # %PATH% so put it there. + # The Python this job is parameterized is not necessarily the one + # at the front of $env:PATH. Modify $env:PATH so that it is so we + # can just say "python" in the rest of the steps. Also get the + # related Scripts directory so tools from packages we install are + # also available. $p = py -<> -c "import sys; print(sys.prefix)" $q = py -<> -c "import sysconfig; print(sysconfig.get_path('scripts'))" From 35d731adf02fc3705d95884f1816b61bd6ac502c Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 9 Aug 2023 11:20:38 -0400 Subject: [PATCH 2087/2309] remove the debug step --- .circleci/config.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 465a97a6b..dbd275a76 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -220,11 +220,6 @@ jobs: # PowerShell string escape character. Add-Content -Path $Profile.CurrentUserAllHosts -Value "`$env:PATH = `"$p;$q;`$env:PATH`"" - - "run": - name: "Reveal $env:PATH" - command: | - $env:PATH - - "run": name: "Display tool versions" command: | From ce8a6d49c7c7e03edfe7f9893a958690b7e33089 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 9 Aug 2023 11:30:39 -0400 Subject: [PATCH 2088/2309] Attempt to cache packages downloaded with pip for Windows jobs --- .circleci/config.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index dbd275a76..629488137 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -198,10 +198,18 @@ jobs: executor: "windows" environment: TAHOE_LAFS_HYPOTHESIS_PROFILE: "ci" + # Tell pip where its download cache lives. This must agree with the + # "save_cache" step below or caching won't really work right. + PIP_CACHE_DIR: "pip-cache" steps: - "checkout" + - "restore_cache": + keys: + - "pip-packages-v1-{{ checksum \"setup.py\" }}" + - "pip-packages-v1-" + - "run": name: "Fix $env:PATH" command: | @@ -230,6 +238,12 @@ jobs: command: | python -m pip install .[testenv] .[test] + - "save_cache": + paths: + # Make sure this agrees with PIP_CACHE_DIR in the environment. + - "pip-cache" + key: "pip-packages-v1-{{ checksum \"setup.py\" }}" + - "run": name: "Run Unit Tests" environment: From fa72ac795166091304be631dc0911c2c9356e4cd Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 9 Aug 2023 12:00:56 -0400 Subject: [PATCH 2089/2309] a couple more comments about the windows job steps --- .circleci/config.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 629488137..d06d9a0a6 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -197,7 +197,11 @@ jobs: executor: "windows" environment: + # Tweak Hypothesis to make its behavior more suitable for the CI + # environment. This should improve reproducibility and lessen the + # effects of variable compute resources. TAHOE_LAFS_HYPOTHESIS_PROFILE: "ci" + # Tell pip where its download cache lives. This must agree with the # "save_cache" step below or caching won't really work right. PIP_CACHE_DIR: "pip-cache" @@ -205,6 +209,8 @@ jobs: steps: - "checkout" + # If possible, restore a pip download cache to save us from having to + # download all our Python dependencies from PyPI. - "restore_cache": keys: - "pip-packages-v1-{{ checksum \"setup.py\" }}" From c5cac7b5a7e7a3c5145c3893b6cdeab77bd64854 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 9 Aug 2023 12:01:02 -0400 Subject: [PATCH 2090/2309] get rid of the partial cache key CircleCI docs don't clearly explain what happens after a partial cache key match and reconstructing our cache is sufficiently cheap that it's probably not worth the complexity / uncertainty. --- .circleci/config.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index d06d9a0a6..7febae61a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -214,7 +214,6 @@ jobs: - "restore_cache": keys: - "pip-packages-v1-{{ checksum \"setup.py\" }}" - - "pip-packages-v1-" - "run": name: "Fix $env:PATH" From 4db44dc1787dc0c98b4e6513728df3ee7faf2287 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 9 Aug 2023 12:34:09 -0400 Subject: [PATCH 2091/2309] Attempt to cache all the wheels --- .circleci/config.yml | 33 ++++++++++++++++++++++++++++++--- 1 file changed, 30 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 7febae61a..e20587f75 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -206,6 +206,11 @@ jobs: # "save_cache" step below or caching won't really work right. PIP_CACHE_DIR: "pip-cache" + # And tell pip where it can find out cached wheelhouse for fast wheel + # installation, even for projects that don't distribute wheels. This + # must also agree with the "save_cache" step below. + PIP_FIND_LINKS: "wheelhouse" + steps: - "checkout" @@ -213,7 +218,11 @@ jobs: # download all our Python dependencies from PyPI. - "restore_cache": keys: - - "pip-packages-v1-{{ checksum \"setup.py\" }}" + # The download cache and/or the wheelhouse may contain Python + # version-specific binary packages so include the Python version + # in this key, as well as the canonical source of our + # dependencies. + - "pip-packages-v1-{{ parameters.pythonVersion }}-{{ checksum \"setup.py\" }}" - "run": name: "Fix $env:PATH" @@ -239,16 +248,34 @@ jobs: python misc/build_helpers/show-tool-versions.py - "run": - name: "Install Dependencies" + # It's faster to install a wheel than a source package. If we don't + # have a cached wheelhouse then build all of the wheels and dump + # them into a directory where they can become a cached wheelhouse. + # We would have built these wheels during installation anyway so it + # doesn't cost us anything extra and saves us effort next time. + name: "(Maybe) Build Wheels" command: | - python -m pip install .[testenv] .[test] + if ((Test-Path .\wheelhouse) -and (Test-Path .\wheelhouse\*)) { + echo "Found populated wheelhouse, skipping wheel building." + } else { + python -m pip wheel --wheel-dir $env:PIP_FIND_LINKS .[testenv] .[test] + } - "save_cache": paths: # Make sure this agrees with PIP_CACHE_DIR in the environment. - "pip-cache" + - "wheelhouse" key: "pip-packages-v1-{{ checksum \"setup.py\" }}" + - "run": + name: "Install Dependencies" + environment: + # By this point we should no longer need an index. + PIP_NO_INDEX: "1" + command: | + python -m pip install .[testenv] .[test] + - "run": name: "Run Unit Tests" environment: From 65d76c2e3c471efc192ddb1a0f6f67ddec9673b0 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 9 Aug 2023 12:53:07 -0400 Subject: [PATCH 2092/2309] must install `wheel` to build wheels with `pip wheel` it seems --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index e20587f75..4d2091e79 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -258,6 +258,7 @@ jobs: if ((Test-Path .\wheelhouse) -and (Test-Path .\wheelhouse\*)) { echo "Found populated wheelhouse, skipping wheel building." } else { + python -m pip install wheel python -m pip wheel --wheel-dir $env:PIP_FIND_LINKS .[testenv] .[test] } From a0389e83cc2e61bdade4d581a71f7aa0e33b7d1c Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 9 Aug 2023 12:54:02 -0400 Subject: [PATCH 2093/2309] use the *correct* templating system for this value --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 4d2091e79..9923a3fcc 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -222,7 +222,7 @@ jobs: # version-specific binary packages so include the Python version # in this key, as well as the canonical source of our # dependencies. - - "pip-packages-v1-{{ parameters.pythonVersion }}-{{ checksum \"setup.py\" }}" + - "pip-packages-v1-<< parameters.pythonVersion >>-{{ checksum \"setup.py\" }}" - "run": name: "Fix $env:PATH" From 14135ea3f0aa2d30c009ef9176170a732e28d5c9 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 9 Aug 2023 13:07:03 -0400 Subject: [PATCH 2094/2309] make sure the two mentions of the cache key agree --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 9923a3fcc..d892a0efd 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -222,7 +222,7 @@ jobs: # version-specific binary packages so include the Python version # in this key, as well as the canonical source of our # dependencies. - - "pip-packages-v1-<< parameters.pythonVersion >>-{{ checksum \"setup.py\" }}" + - &CACHE_KEY "pip-packages-v1-<< parameters.pythonVersion >>-{{ checksum \"setup.py\" }}" - "run": name: "Fix $env:PATH" @@ -267,7 +267,7 @@ jobs: # Make sure this agrees with PIP_CACHE_DIR in the environment. - "pip-cache" - "wheelhouse" - key: "pip-packages-v1-{{ checksum \"setup.py\" }}" + key: *CACHE_KEY - "run": name: "Install Dependencies" From a73b6d99c4564e0275c315f7c7627c92c21ea732 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 9 Aug 2023 14:10:58 -0400 Subject: [PATCH 2095/2309] we end up using the coveralls tool to clean up these paths and I stopped using tox so they don't look like this anymore --- .coveragerc | 2 -- 1 file changed, 2 deletions(-) diff --git a/.coveragerc b/.coveragerc index 32b803586..5b41f9ce3 100644 --- a/.coveragerc +++ b/.coveragerc @@ -23,7 +23,5 @@ source = D:/a/tahoe-lafs/tahoe-lafs/.tox/py*-coverage/Lib/site-packages/ # Although sometimes it looks like this instead. Also it looks like this on macOS. .tox/py*-coverage/lib/python*/site-packages/ -# And on the CircleCI Windows build envronment... - .tox/py*-coverage/Lib/site-packages/ # On some Linux CI jobs it looks like this /tmp/tahoe-lafs.tox/py*-coverage/lib/python*/site-packages/ From 27b97dc1d8b8f56887dc2508d91ec6239d25dafd Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 9 Aug 2023 15:01:07 -0400 Subject: [PATCH 2096/2309] bump it --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 86873ad53..433721d2a 100644 --- a/setup.py +++ b/setup.py @@ -413,7 +413,7 @@ setup(name="tahoe-lafs", # also set in __init__.py "pip==22.0.3", "wheel==0.37.1", "setuptools==60.9.1", - "subunitreporter==22.2.0", + "subunitreporter==23.8.0", "python-subunit==1.4.2", "junitxml==0.7", "coverage==7.2.5", From d93d6122f78f586306a129eb09d13c81b11a2b90 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 9 Aug 2023 15:01:16 -0400 Subject: [PATCH 2097/2309] news fragment --- newsfragments/4059.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/4059.minor diff --git a/newsfragments/4059.minor b/newsfragments/4059.minor new file mode 100644 index 000000000..e69de29bb From a95a6b88a92f50d1c33e37e1af6bf6d743d16dde Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 9 Aug 2023 17:04:26 -0400 Subject: [PATCH 2098/2309] note motivation for our choice of these python versions --- .circleci/config.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index d892a0efd..d327ecbc7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -121,6 +121,13 @@ workflows: name: "Windows Server 2022, CPython <>" matrix: parameters: + # Run the job for a number of CPython versions. These are the + # two versions installed on the version of the Windows VM image + # we specify (in the executor). This is handy since it means we + # don't have to do any Python installation work. We pin the + # Windows VM image so these shouldn't shuffle around beneath us + # but if we want to update that image or get different versions + # of Python, we probably have to do something here. pythonVersion: - "3.9" - "3.11" From c7f6b6484d033d0b184bb46afec2e134cc389346 Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 9 Aug 2023 15:15:20 -0600 Subject: [PATCH 2099/2309] spelling --- newsfragments/4056.bugfix | 2 +- src/allmydata/storage_client.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/newsfragments/4056.bugfix b/newsfragments/4056.bugfix index 1f94de0da..7e637b48c 100644 --- a/newsfragments/4056.bugfix +++ b/newsfragments/4056.bugfix @@ -1,3 +1,3 @@ -Provide our own copy of attrs' "provides()" validor +Provide our own copy of attrs' "provides()" validator This validator is deprecated and slated for removal; that project's suggestion is to copy the code to our project. diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 8de3a9ca9..c59db0817 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -88,7 +88,7 @@ from allmydata.util.rrefutil import add_version_to_remote_reference from allmydata.util.hashutil import permute_server_hash from allmydata.util.dictutil import BytesKeyDict, UnicodeKeyDict from allmydata.util.deferredutil import async_to_deferred, race -from allmydata.util.attr_provides import provides +from allmydata.util.attrs_provides import provides from allmydata.storage.http_client import ( StorageClient, StorageClientImmutables, StorageClientGeneral, ClientException as HTTPClientException, StorageClientMutables, From 9758569cffb9f62a5597a330d653cde7e8357169 Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 9 Aug 2023 15:16:07 -0600 Subject: [PATCH 2100/2309] obsolete comment --- integration/grid.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/integration/grid.py b/integration/grid.py index 03c3bb6e2..b97c22bf7 100644 --- a/integration/grid.py +++ b/integration/grid.py @@ -73,11 +73,6 @@ class FlogGatherer(object): """ Flog Gatherer process. """ - - # it would be best to use attr.validators.provides() here but that - # is deprecated; please replace with our own "provides" as part of - # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/4056#ticket - # for now, insisting on a subclass which is narrower than necessary process = attr.ib( validator=provides(IProcessTransport) ) From 295e816d4ee2cfc27130f96994e0742849390009 Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 9 Aug 2023 21:49:37 -0600 Subject: [PATCH 2101/2309] spell --- src/allmydata/storage_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 1f6b41b1c..9739091dc 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -761,7 +761,7 @@ class AnnouncementNotMatched(Exception): @attr.s(auto_exc=True) class MissingPlugin(Exception): """ - A particular plugin was request, but is missing + A particular plugin was requested but is missing """ plugin_name = attr.ib() From cf4fe0061cdddd254a850efc1f2949f0b49447f9 Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 9 Aug 2023 22:27:55 -0600 Subject: [PATCH 2102/2309] refactor where plugins are loaded; use this to error early for users --- src/allmydata/client.py | 5 +++ src/allmydata/node.py | 2 + src/allmydata/scripts/tahoe_run.py | 14 +++++++ src/allmydata/storage_client.py | 67 +++++++++++++++++++++--------- 4 files changed, 69 insertions(+), 19 deletions(-) diff --git a/src/allmydata/client.py b/src/allmydata/client.py index aff2d5815..cfc0977a1 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -483,6 +483,11 @@ def create_storage_farm_broker(config: _Config, default_connection_handlers, foo storage_client_config = storage_client.StorageClientConfig.from_node_config( config, ) + # ensure that we can at least load all plugins that the + # configuration mentions; doing this early (i.e. before creating + # storage-clients themselves) allows us to exit in case of a + # problem. + storage_client_config.get_configured_storage_plugins() def tub_creator(handler_overrides=None, **kwargs): return node.create_tub( diff --git a/src/allmydata/node.py b/src/allmydata/node.py index 6c3082b50..5b06cb963 100644 --- a/src/allmydata/node.py +++ b/src/allmydata/node.py @@ -30,10 +30,12 @@ from twisted.python.filepath import ( from twisted.python import log as twlog from twisted.application import service from twisted.python.failure import Failure +from twisted.plugin import getPlugins from foolscap.api import Tub import foolscap.logging.log +from allmydata.interfaces import IFoolscapStoragePlugin from allmydata.util import log from allmydata.util import fileutil, iputil from allmydata.util.fileutil import abspath_expanduser_unicode diff --git a/src/allmydata/scripts/tahoe_run.py b/src/allmydata/scripts/tahoe_run.py index ff3ff9efd..eba5ae329 100644 --- a/src/allmydata/scripts/tahoe_run.py +++ b/src/allmydata/scripts/tahoe_run.py @@ -42,6 +42,9 @@ from allmydata.util.pid import ( from allmydata.storage.crawler import ( MigratePickleFileError, ) +from allmydata.storage_client import ( + MissingPlugin, +) from allmydata.node import ( PortAssignmentRequired, PrivacyError, @@ -197,6 +200,17 @@ class DaemonizeTheRealService(Service, HookMixin): self.basedir, ) ) + elif reason.check(MissingPlugin): + self.stderr.write( + "Missing Plugin\n" + "The configuration requests a plugin:\n" + "\n {}\n\n" + "...which cannot be found.\n" + "This typically means that some software hasn't been installed or the plugin couldn't be instantiated.\n\n" + .format( + reason.value.plugin_name, + ) + ) else: self.stderr.write("\nUnknown error, here's the traceback:\n") reason.printTraceback(self.stderr) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 9739091dc..24abe2a18 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -187,6 +187,30 @@ class StorageClientConfig(object): grid_manager_keys, ) + def get_configured_storage_plugins(self): + """ + :returns Dict[str, IFoolscapStoragePlugin]: a dict mapping names + to instances for all available plugins + + :raises MissingPlugin: if the configuration asks for a plugin + for which there is no corresponding instance (e.g. it is + not installed). + """ + plugins = { + plugin.name: plugin + for plugin + in getPlugins(IFoolscapStoragePlugin) + } + + configured = dict() + for plugin_name in self.storage_plugins: + try: + plugin = plugins[plugin_name] + except KeyError: + raise MissingPlugin(plugin_name) + configured[plugin_name] = plugin + return configured + @implementer(IStorageBroker) class StorageFarmBroker(service.MultiService): @@ -765,10 +789,9 @@ class MissingPlugin(Exception): """ plugin_name = attr.ib() - nickname = attr.ib() def __str__(self): - return "Missing plugin '{}' for server '{}'".format(self.plugin_name, self.nickname) + return "Missing plugin '{}'".format(self.plugin_name) def _storage_from_foolscap_plugin(node_config, config, announcement, get_rref): @@ -782,26 +805,32 @@ def _storage_from_foolscap_plugin(node_config, config, announcement, get_rref): :param dict announcement: The storage announcement for the storage server we should build """ - plugins = { - plugin.name: plugin - for plugin - in getPlugins(IFoolscapStoragePlugin) - } storage_options = announcement.get(u"storage-options", []) - for plugin_name, plugin_config in list(config.storage_plugins.items()): + plugins = config.get_configured_storage_plugins() + + # for every storage-option that we have enabled locally (in order + # of preference), see if the announcement asks for such a thing. + # if it does, great: we return that storage-client + # otherwise we've run out of options... + + for options in storage_options: try: - plugin = plugins[plugin_name] + plugin = plugins[options[u"name"]] except KeyError: - raise MissingPlugin(plugin_name, announcement.get(u"nickname", "")) - for option in storage_options: - if plugin_name == option[u"name"]: - furl = option[u"storage-server-FURL"] - return furl, plugin.get_storage_client( - node_config, - option, - get_rref, - ) - plugin_names = ", ".join(sorted(list(config.storage_plugins.keys()))) + # we didn't configure this kind of plugin locally, so + # consider the next announced option + continue + + furl = options[u"storage-server-FURL"] + return furl, plugin.get_storage_client( + node_config, + options, + get_rref, + ) + + # none of the storage options in the announcement are configured + # locally; we can't make a storage-client. + plugin_names = ", ".join(sorted(plugins)) raise AnnouncementNotMatched(plugin_names) From d7cfb5dde9d11b52e8525d5666fbac355ba8eb1b Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 9 Aug 2023 23:21:28 -0600 Subject: [PATCH 2103/2309] show WebUI feedback when announcement-match fails --- src/allmydata/storage_client.py | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 24abe2a18..1b7b92acb 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -830,7 +830,7 @@ def _storage_from_foolscap_plugin(node_config, config, announcement, get_rref): # none of the storage options in the announcement are configured # locally; we can't make a storage-client. - plugin_names = ", ".join(sorted(plugins)) + plugin_names = ", ".join(sorted(option["name"] for option in storage_options)) raise AnnouncementNotMatched(plugin_names) @@ -872,6 +872,7 @@ def _make_storage_system( :return: An object enabling communication via Foolscap with the server which generated the announcement. """ + unmatched = None # Try to match the announcement against a plugin. try: furl, storage_server = _storage_from_foolscap_plugin( @@ -885,14 +886,10 @@ def _make_storage_system( get_rref, ) except AnnouncementNotMatched as e: - _log.error( - 'No plugin for storage-server "{nickname}" from plugins: {plugins}', - nickname=ann.get("nickname", ""), - plugins=e.args[0], - ) - except MissingPlugin as e: - _log.failure("Missing plugin") - return _NullStorage(''.format(e.args[0])) + # show a more-specific error to the user for this server + # (Note this will only be shown if the server _doesn't_ offer + # anonymous service, which will match below) + unmatched = _NullStorage('{}: missing plugin "{}"'.format(server_id.decode("utf8"), str(e))) else: return _FoolscapStorage.from_announcement( server_id, @@ -918,8 +915,10 @@ def _make_storage_system( storage_server, ) - # Nothing matched so we can't talk to this server. - return _null_storage + # Nothing matched so we can't talk to this server. If we have a + # specific reason in "unmatched", use it; otherwise the generic + # one + return unmatched or _null_storage @implementer(IServer) class NativeStorageServer(service.MultiService): From 09ea172b940c607a990e6cd5d4bbb9f98075795e Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 10 Aug 2023 12:06:29 -0600 Subject: [PATCH 2105/2309] reformat multiline strings; don't output "storage.plugins = None" --- src/allmydata/test/common.py | 42 ++++++++++++++++++++---------------- 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/src/allmydata/test/common.py b/src/allmydata/test/common.py index d61bc28f1..744c17efa 100644 --- a/src/allmydata/test/common.py +++ b/src/allmydata/test/common.py @@ -307,13 +307,18 @@ class UseNode(object): if self.plugin_config is None: plugin_config_section = "" else: - plugin_config_section = """ -[storageclient.plugins.{storage_plugin}] -{config} -""".format( - storage_plugin=self.storage_plugin, - config=format_config_items(self.plugin_config), -) + plugin_config_section = + "[storageclient.plugins.{storage_plugin}]\n" + "{config}\n" + .format( + storage_plugin=self.storage_plugin, + config=format_config_items(self.plugin_config), + ) + + if self.storage_plugin is None: + plugins = "" + else: + plugins = "storage.plugins = {}".format(self.storage_plugin) write_introducer( self.basedir, @@ -340,18 +345,17 @@ class UseNode(object): self.config = config_from_string( self.basedir.asTextMode().path, "tub.port", -""" -[node] -{node_config} - -[client] -storage.plugins = {storage_plugin} -{plugin_config_section} -""".format( - storage_plugin=self.storage_plugin, - node_config=format_config_items(node_config), - plugin_config_section=plugin_config_section, -) + "[node]\n" + "{node_config}\n" + "\n" + "[client]\n" + "{plugins}\n" + "{plugin_config_section}\n" + .format( + plugins=plugins, + node_config=format_config_items(node_config), + plugin_config_section=plugin_config_section, + ) ) def create_node(self): From b07d9e90cbcd557821a75a1fd7571e2c169dee73 Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 10 Aug 2023 12:07:03 -0600 Subject: [PATCH 2106/2309] correct test --- src/allmydata/test/test_storage_client.py | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/src/allmydata/test/test_storage_client.py b/src/allmydata/test/test_storage_client.py index e3b192a96..d719f227b 100644 --- a/src/allmydata/test/test_storage_client.py +++ b/src/allmydata/test/test_storage_client.py @@ -243,22 +243,18 @@ class UnrecognizedAnnouncement(unittest.TestCase): server.get_foolscap_write_enabler_seed() server.get_nickname() - def test_longname(self) -> None: + def test_missing_plugin(self) -> None: """ - ``NativeStorageServer.get_longname`` describes the missing plugin. + An exception is produced if the plugin is missing """ - server = self.native_storage_server( - StorageClientConfig( - storage_plugins={ - "nothing": {} - } + with self.assertRaises(MissingPlugin): + _ = self.native_storage_server( + StorageClientConfig( + storage_plugins={ + "nothing": {} + } + ) ) - ) - self.assertEqual( - server.get_longname(), - '', - ) - self.flushLoggedErrors(MissingPlugin) class PluginMatchedAnnouncement(SyncTestCase): From e3e5b4bc8d5deacad91a3c25243b9f71eec3a63d Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 10 Aug 2023 12:19:11 -0600 Subject: [PATCH 2107/2309] typo --- src/allmydata/test/common.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/src/allmydata/test/common.py b/src/allmydata/test/common.py index 744c17efa..1186bd540 100644 --- a/src/allmydata/test/common.py +++ b/src/allmydata/test/common.py @@ -307,13 +307,12 @@ class UseNode(object): if self.plugin_config is None: plugin_config_section = "" else: - plugin_config_section = - "[storageclient.plugins.{storage_plugin}]\n" - "{config}\n" - .format( - storage_plugin=self.storage_plugin, - config=format_config_items(self.plugin_config), - ) + plugin_config_section = ( + "[storageclient.plugins.{storage_plugin}]\n" + "{config}\n").format( + storage_plugin=self.storage_plugin, + config=format_config_items(self.plugin_config), + ) if self.storage_plugin is None: plugins = "" From 60e873bbe48f94af3079a1a60e0d5159b73e4c87 Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 10 Aug 2023 13:22:35 -0600 Subject: [PATCH 2108/2309] unused --- src/allmydata/node.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/allmydata/node.py b/src/allmydata/node.py index 5b06cb963..6c3082b50 100644 --- a/src/allmydata/node.py +++ b/src/allmydata/node.py @@ -30,12 +30,10 @@ from twisted.python.filepath import ( from twisted.python import log as twlog from twisted.application import service from twisted.python.failure import Failure -from twisted.plugin import getPlugins from foolscap.api import Tub import foolscap.logging.log -from allmydata.interfaces import IFoolscapStoragePlugin from allmydata.util import log from allmydata.util import fileutil, iputil from allmydata.util.fileutil import abspath_expanduser_unicode From 7322d8c0e60ecd33da155d7055c8917fc34aa83a Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 10 Aug 2023 14:28:55 -0600 Subject: [PATCH 2109/2309] better news --- newsfragments/3899.bugfix | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/newsfragments/3899.bugfix b/newsfragments/3899.bugfix index a55239c38..55d4fabd4 100644 --- a/newsfragments/3899.bugfix +++ b/newsfragments/3899.bugfix @@ -1 +1,4 @@ -Print a useful message when a storage-client cannot be matched to configuration +Provide better feedback from plugin configuration errors + +Local errors now print a useful message and exit. +Announcements that only contain invalid / unusable plugins now show a message in the Welcome page. From f51d49faa54c0fff3b2146bb6630e83da53484c5 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 11 Aug 2023 09:03:30 -0600 Subject: [PATCH 2110/2309] typing Co-authored-by: Jean-Paul Calderone --- src/allmydata/storage_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 1b7b92acb..7040ecd16 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -187,7 +187,7 @@ class StorageClientConfig(object): grid_manager_keys, ) - def get_configured_storage_plugins(self): + def get_configured_storage_plugins(self) -> dict[str, IFoolscapStoragePlugin]: """ :returns Dict[str, IFoolscapStoragePlugin]: a dict mapping names to instances for all available plugins From d81b64ba9e2dd8aa84a2812f702ef55cd1698f52 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 11 Aug 2023 09:05:16 -0600 Subject: [PATCH 2111/2309] docstring Co-authored-by: Jean-Paul Calderone --- src/allmydata/storage_client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 7040ecd16..a6a5336c6 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -189,8 +189,8 @@ class StorageClientConfig(object): def get_configured_storage_plugins(self) -> dict[str, IFoolscapStoragePlugin]: """ - :returns Dict[str, IFoolscapStoragePlugin]: a dict mapping names - to instances for all available plugins + :returns: a mapping from names to instances for all available + plugins :raises MissingPlugin: if the configuration asks for a plugin for which there is no corresponding instance (e.g. it is From c03076fe213382af5e754724f072fc50f9b61f49 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 11 Aug 2023 09:07:00 -0600 Subject: [PATCH 2112/2309] more robust comparison Co-authored-by: Jean-Paul Calderone --- src/allmydata/test/test_storage_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/test_storage_client.py b/src/allmydata/test/test_storage_client.py index d719f227b..328e90499 100644 --- a/src/allmydata/test/test_storage_client.py +++ b/src/allmydata/test/test_storage_client.py @@ -196,7 +196,7 @@ class UnrecognizedAnnouncement(unittest.TestCase): self._tub_maker, {}, node_config=EMPTY_CLIENT_CONFIG, - config=config or StorageClientConfig(), + config=config if config is not None else StorageClientConfig(), ) def test_no_exceptions(self): From a0769f59dce7b3d70f2e4833b0e4405d8ad8e472 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 11 Aug 2023 09:07:18 -0600 Subject: [PATCH 2113/2309] naming Co-authored-by: Jean-Paul Calderone --- src/allmydata/test/test_storage_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/test_storage_client.py b/src/allmydata/test/test_storage_client.py index 328e90499..5b2f80712 100644 --- a/src/allmydata/test/test_storage_client.py +++ b/src/allmydata/test/test_storage_client.py @@ -251,7 +251,7 @@ class UnrecognizedAnnouncement(unittest.TestCase): _ = self.native_storage_server( StorageClientConfig( storage_plugins={ - "nothing": {} + "missing-plugin-name": {} } ) ) From 2e76d554e2a1b6ecd090eedce64645a84e890710 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 11 Aug 2023 09:08:03 -0600 Subject: [PATCH 2114/2309] don't explicitly drop return Co-authored-by: Jean-Paul Calderone --- src/allmydata/test/test_storage_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/test_storage_client.py b/src/allmydata/test/test_storage_client.py index 5b2f80712..f8db402d0 100644 --- a/src/allmydata/test/test_storage_client.py +++ b/src/allmydata/test/test_storage_client.py @@ -248,7 +248,7 @@ class UnrecognizedAnnouncement(unittest.TestCase): An exception is produced if the plugin is missing """ with self.assertRaises(MissingPlugin): - _ = self.native_storage_server( + self.native_storage_server( StorageClientConfig( storage_plugins={ "missing-plugin-name": {} From 375ee54c80bff6cb4327eec54e753a086055c4e5 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 11 Aug 2023 09:08:19 -0600 Subject: [PATCH 2115/2309] typing Co-authored-by: Jean-Paul Calderone --- src/allmydata/test/test_storage_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/test_storage_client.py b/src/allmydata/test/test_storage_client.py index f8db402d0..97ce9fe68 100644 --- a/src/allmydata/test/test_storage_client.py +++ b/src/allmydata/test/test_storage_client.py @@ -186,7 +186,7 @@ class UnrecognizedAnnouncement(unittest.TestCase): def _tub_maker(self, overrides): return Service() - def native_storage_server(self, config=None): + def native_storage_server(self, config: Optional[StorageClientConfig] = None) -> NativeStorageServer: """ Make a ``NativeStorageServer`` out of an unrecognizable announcement. """ From c27b330984afdfc612f90707ccc1ffc2e2473042 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 11 Aug 2023 19:18:23 -0600 Subject: [PATCH 2116/2309] don't need fallback --- src/allmydata/storage_client.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 1b7b92acb..2a3b1dbad 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -772,8 +772,6 @@ class NonReconnector(object): def getReconnectionInfo(self): return ReconnectionInfo() -_null_storage = _NullStorage() - class AnnouncementNotMatched(Exception): """ @@ -915,10 +913,11 @@ def _make_storage_system( storage_server, ) - # Nothing matched so we can't talk to this server. If we have a - # specific reason in "unmatched", use it; otherwise the generic - # one - return unmatched or _null_storage + # Nothing matched so we can't talk to this server. (There should + # not be a way to get here without this local being valid) + assert unmatched is not None, "Expected unmatched plugin error" + return unmatched + @implementer(IServer) class NativeStorageServer(service.MultiService): From ffa589d6f827476ff7c7b98a7db52f34be4cf996 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 11 Aug 2023 21:19:02 -0600 Subject: [PATCH 2117/2309] import error --- src/allmydata/test/test_storage_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/test_storage_client.py b/src/allmydata/test/test_storage_client.py index 97ce9fe68..604884eba 100644 --- a/src/allmydata/test/test_storage_client.py +++ b/src/allmydata/test/test_storage_client.py @@ -8,7 +8,7 @@ from json import ( loads, ) import hashlib -from typing import Union, Any +from typing import Union, Any, Optional from hyperlink import DecodedURL from fixtures import ( From 356a1d0f792ae2c5ea65105f7e9ffb0eb1321aa0 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 11 Aug 2023 22:01:21 -0600 Subject: [PATCH 2118/2309] don't know why dict_keys are so confusing to mypy --- src/allmydata/storage_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index c95d72dbf..75e717037 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -209,7 +209,7 @@ class StorageClientConfig(object): except KeyError: raise MissingPlugin(plugin_name) configured[plugin_name] = plugin - return configured + return configured # type: ignore @implementer(IStorageBroker) From a5b95273d7b3b420be6bc57ec9c4cd56897425d5 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 11 Aug 2023 23:47:24 -0600 Subject: [PATCH 2119/2309] typing is .. good? --- src/allmydata/storage_client.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 75e717037..d35cd788b 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -33,7 +33,7 @@ Ported to Python 3. from __future__ import annotations from six import ensure_text -from typing import Union, Callable, Any, Optional, cast +from typing import Union, Callable, Any, Optional, cast, Dict from os import urandom import re import time @@ -202,14 +202,15 @@ class StorageClientConfig(object): in getPlugins(IFoolscapStoragePlugin) } - configured = dict() + # mypy doesn't like "str" in place of Any ... + configured: Dict[Any, IFoolscapStoragePlugin] = dict() for plugin_name in self.storage_plugins: try: plugin = plugins[plugin_name] except KeyError: raise MissingPlugin(plugin_name) configured[plugin_name] = plugin - return configured # type: ignore + return configured @implementer(IStorageBroker) From ad44958f0223c92b0133b4b65325ae540a54dd8a Mon Sep 17 00:00:00 2001 From: meejah Date: Sat, 12 Aug 2023 00:35:49 -0600 Subject: [PATCH 2120/2309] more kinds of whitespace --- src/allmydata/test/cli/test_run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/cli/test_run.py b/src/allmydata/test/cli/test_run.py index 2adcfea19..ae0f92131 100644 --- a/src/allmydata/test/cli/test_run.py +++ b/src/allmydata/test/cli/test_run.py @@ -264,7 +264,7 @@ class RunTests(SyncTestCase): self.assertThat(runs, Equals([])) self.assertThat(result_code, Equals(1)) - good_file_content_re = re.compile(r"\w[0-9]*\w[0-9]*\w") + good_file_content_re = re.compile(r"\s[0-9]*\s[0-9]*\s", re.M) @given(text()) def test_pidfile_contents(self, content): From 9b52313cda9ea89dd226b0432e00eaf3efeccf6c Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 16 Aug 2023 09:44:27 -0400 Subject: [PATCH 2121/2309] Better description. --- newsfragments/4041.feature | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/newsfragments/4041.feature b/newsfragments/4041.feature index ea4c65171..7d8df1a23 100644 --- a/newsfragments/4041.feature +++ b/newsfragments/4041.feature @@ -1 +1 @@ -The storage server now supports a new, HTTPS-based protocol. \ No newline at end of file +The storage server and client now support a new, HTTPS-based protocol. \ No newline at end of file From 4bffd567c3c8b067bc30c25eb68b5a4150c14b63 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 16 Aug 2023 09:45:10 -0400 Subject: [PATCH 2122/2309] Better docstring. --- src/allmydata/test/test_storage_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/test_storage_client.py b/src/allmydata/test/test_storage_client.py index 79bc475d0..514c4ef78 100644 --- a/src/allmydata/test/test_storage_client.py +++ b/src/allmydata/test/test_storage_client.py @@ -742,7 +742,7 @@ storage: self.assertTrue(done.called) def test_should_we_use_http_default(self): - """Default is to use HTTP; this will change eventually""" + """Default is to use HTTP.""" basedir = self.mktemp() node_config = config_from_string(basedir, "", "") announcement = {ANONYMOUS_STORAGE_NURLS: ["pb://..."]} From 152cadf54360c562961abbf5edebe2e69d72d794 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 17 Aug 2023 09:06:32 -0400 Subject: [PATCH 2123/2309] Less duplication, more accuracy in format. --- src/allmydata/test/test_storage_client.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/allmydata/test/test_storage_client.py b/src/allmydata/test/test_storage_client.py index d487408de..03c9fb9ae 100644 --- a/src/allmydata/test/test_storage_client.py +++ b/src/allmydata/test/test_storage_client.py @@ -671,12 +671,8 @@ storage: self.assertTrue(initial_service.running) self.assertIdentical(initial_service.parent, broker) - http_announcement = { - "service-name": "storage", - "anonymous-storage-FURL": f"pb://{ones}@nowhere/fake2", - "permutation-seed-base32": "bbbbbbbbbbbbbbbbbbbbbbbb", - ANONYMOUS_STORAGE_NURLS: [f"pb://{ones}@nowhere/fake2#v=1"], - } + http_announcement = initial_announcement.copy() + http_announcement[ANONYMOUS_STORAGE_NURLS] = {f"pb://{ones}@nowhere/fake2#v=1"} broker._got_announcement(key_s, http_announcement) self.assertFalse(initial_service.running) self.assertEqual(initial_service.parent, None) From 50ce8abf9fbff8504d5fd4a566c4b734b981ffa7 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 22 Aug 2023 08:50:27 -0400 Subject: [PATCH 2124/2309] adapt the existing case to a multi-case structure --- src/allmydata/storage/http_common.py | 15 +++-- .../test/data/spki-hash-test-vectors.yaml | 26 ++++++++ src/allmydata/test/test_storage_https.py | 60 ++++++++++--------- 3 files changed, 70 insertions(+), 31 deletions(-) create mode 100644 src/allmydata/test/data/spki-hash-test-vectors.yaml diff --git a/src/allmydata/storage/http_common.py b/src/allmydata/storage/http_common.py index 650d905e9..d59cab541 100644 --- a/src/allmydata/storage/http_common.py +++ b/src/allmydata/storage/http_common.py @@ -54,6 +54,15 @@ class Secrets(Enum): WRITE_ENABLER = "write-enabler" +def get_spki(certificate: Certificate) -> bytes: + """ + Get the bytes making up the DER encoded representation of the + `SubjectPublicKeyInfo` (RFC 7469) for the given certificate. + """ + return certificate.public_key().public_bytes( + Encoding.DER, PublicFormat.SubjectPublicKeyInfo + ) + def get_spki_hash(certificate: Certificate) -> bytes: """ Get the public key hash, as per RFC 7469: base64 of sha256 of the public @@ -61,7 +70,5 @@ def get_spki_hash(certificate: Certificate) -> bytes: We use the URL-safe base64 variant, since this is typically found in NURLs. """ - public_key_bytes = certificate.public_key().public_bytes( - Encoding.DER, PublicFormat.SubjectPublicKeyInfo - ) - return urlsafe_b64encode(sha256(public_key_bytes).digest()).strip().rstrip(b"=") + spki_bytes = get_spki(certificate) + return urlsafe_b64encode(sha256(spki_bytes).digest()).strip().rstrip(b"=") diff --git a/src/allmydata/test/data/spki-hash-test-vectors.yaml b/src/allmydata/test/data/spki-hash-test-vectors.yaml new file mode 100644 index 000000000..6be6a9b71 --- /dev/null +++ b/src/allmydata/test/data/spki-hash-test-vectors.yaml @@ -0,0 +1,26 @@ +vector: +- expected-hash: >- + JIj6ezHkdSBlHhrnezAgIC_mrVQHy4KAFyL-8ZNPGPM + expected-spki: >- + MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAv9vqtA8Toy9D6xLGq41iUafSiAXnuirWxML2ct/LAcGJzATg6JctmJxxZQL7vkmaFFPBF6Y39bOGbbECM2iQYn2Qemj5fl3IzKTnYLqzryGM0ZwwnNbPyetSe/sksAIYRLzn49d6l+AHR+DjGyvoLzIyGUTn41MTDafMNtPgWx1i+65lFW3GHYpEmugu4bjeUPizNja2LrqwvwFuYXwmKxbIMdioCoRvDGX9SI3/euFstuR4rbOEUDxniYRF5g6reP8UMF30zJzF5j0kyDg8Z5b1XpKFNZAeyRYxcs9wJCqVlP6BLPDnvNVpMXodnWLeTK+r6YWvGadGVufkYNC1PwIDAQAB + certificate: | + -----BEGIN CERTIFICATE----- + MIIDWTCCAkECFCf+I+3oEhTfqt+6ruH4qQ4Wst1DMA0GCSqGSIb3DQEBCwUAMGkx + CzAJBgNVBAYTAlpaMRAwDgYDVQQIDAdOb3doZXJlMRQwEgYDVQQHDAtFeGFtcGxl + dG93bjEcMBoGA1UECgwTRGVmYXVsdCBDb21wYW55IEx0ZDEUMBIGA1UEAwwLZXhh + bXBsZS5jb20wHhcNMjIwMzAyMTUyNTQ3WhcNMjMwMzAyMTUyNTQ3WjBpMQswCQYD + VQQGEwJaWjEQMA4GA1UECAwHTm93aGVyZTEUMBIGA1UEBwwLRXhhbXBsZXRvd24x + HDAaBgNVBAoME0RlZmF1bHQgQ29tcGFueSBMdGQxFDASBgNVBAMMC2V4YW1wbGUu + Y29tMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAv9vqtA8Toy9D6xLG + q41iUafSiAXnuirWxML2ct/LAcGJzATg6JctmJxxZQL7vkmaFFPBF6Y39bOGbbEC + M2iQYn2Qemj5fl3IzKTnYLqzryGM0ZwwnNbPyetSe/sksAIYRLzn49d6l+AHR+Dj + GyvoLzIyGUTn41MTDafMNtPgWx1i+65lFW3GHYpEmugu4bjeUPizNja2LrqwvwFu + YXwmKxbIMdioCoRvDGX9SI3/euFstuR4rbOEUDxniYRF5g6reP8UMF30zJzF5j0k + yDg8Z5b1XpKFNZAeyRYxcs9wJCqVlP6BLPDnvNVpMXodnWLeTK+r6YWvGadGVufk + YNC1PwIDAQABMA0GCSqGSIb3DQEBCwUAA4IBAQByrhn78GSS3dJ0pJ6czmhMX5wH + +fauCtt1+Wbn+ctTodTycS+pfULO4gG7wRzhl8KNoOqLmWMjyA2A3mon8kdkD+0C + i8McpoPaGS2wQcqC28Ud6kP9YO81YFyTl4nHVKQ0nmplT+eoLDTCIWMVxHHzxIgs + 2ybUluAc+THSjpGxB6kWSAJeg3N+f2OKr+07Yg9LiQ2b8y0eZarpiuuuXCzWeWrQ + PudP0aniyq/gbPhxq0tYF628IBvhDAnr/2kqEmVF2TDr2Sm/Y3PDBuPY6MeIxjnr + ox5zO3LrQmQw11OaIAs2/kviKAoKTFFxeyYcpS5RuKNDZfHQCXlLwt9bySxG + -----END CERTIFICATE----- diff --git a/src/allmydata/test/test_storage_https.py b/src/allmydata/test/test_storage_https.py index 0e0bbcc95..9e44b86b0 100644 --- a/src/allmydata/test/test_storage_https.py +++ b/src/allmydata/test/test_storage_https.py @@ -7,7 +7,9 @@ Protocol. """ from contextlib import asynccontextmanager +from base64 import b64decode +from yaml import safe_load from cryptography import x509 from twisted.internet.endpoints import serverFromString @@ -26,18 +28,27 @@ from .certs import ( private_key_to_file, cert_to_file, ) -from ..storage.http_common import get_spki_hash +from ..storage.http_common import get_spki, get_spki_hash from ..storage.http_client import _StorageClientHTTPSPolicy from ..storage.http_server import _TLSEndpointWrapper from ..util.deferredutil import async_to_deferred from .common_system import spin_until_cleanup_done +spki_test_vectors_path = FilePath(__file__).sibling("data").child("spki-hash-test-vectors.yaml") + class HTTPSNurlTests(SyncTestCase): """Tests for HTTPS NURLs.""" def test_spki_hash(self): - """The output of ``get_spki_hash()`` matches the semantics of RFC 7469. + """ + The output of ``get_spki_hash()`` matches the semantics of RFC + 7469. + + The test vector certificates were generated using the openssl command + line tool:: + + openssl req -x509 -newkey rsa:4096 -keyout key.pem -out cert.pem -days 365 The expected hash was generated using Appendix A instructions in the RFC:: @@ -45,32 +56,27 @@ class HTTPSNurlTests(SyncTestCase): openssl x509 -noout -in certificate.pem -pubkey | \ openssl asn1parse -noout -inform pem -out public.key openssl dgst -sha256 -binary public.key | openssl enc -base64 + + The expected SubjectPublicKeyInfo bytes were extracted from the implementation of `get_spki_hash` after its result matched the expected value generated by the command above. + """ - expected_hash = b"JIj6ezHkdSBlHhrnezAgIC_mrVQHy4KAFyL-8ZNPGPM" - certificate_text = b"""\ ------BEGIN CERTIFICATE----- -MIIDWTCCAkECFCf+I+3oEhTfqt+6ruH4qQ4Wst1DMA0GCSqGSIb3DQEBCwUAMGkx -CzAJBgNVBAYTAlpaMRAwDgYDVQQIDAdOb3doZXJlMRQwEgYDVQQHDAtFeGFtcGxl -dG93bjEcMBoGA1UECgwTRGVmYXVsdCBDb21wYW55IEx0ZDEUMBIGA1UEAwwLZXhh -bXBsZS5jb20wHhcNMjIwMzAyMTUyNTQ3WhcNMjMwMzAyMTUyNTQ3WjBpMQswCQYD -VQQGEwJaWjEQMA4GA1UECAwHTm93aGVyZTEUMBIGA1UEBwwLRXhhbXBsZXRvd24x -HDAaBgNVBAoME0RlZmF1bHQgQ29tcGFueSBMdGQxFDASBgNVBAMMC2V4YW1wbGUu -Y29tMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAv9vqtA8Toy9D6xLG -q41iUafSiAXnuirWxML2ct/LAcGJzATg6JctmJxxZQL7vkmaFFPBF6Y39bOGbbEC -M2iQYn2Qemj5fl3IzKTnYLqzryGM0ZwwnNbPyetSe/sksAIYRLzn49d6l+AHR+Dj -GyvoLzIyGUTn41MTDafMNtPgWx1i+65lFW3GHYpEmugu4bjeUPizNja2LrqwvwFu -YXwmKxbIMdioCoRvDGX9SI3/euFstuR4rbOEUDxniYRF5g6reP8UMF30zJzF5j0k -yDg8Z5b1XpKFNZAeyRYxcs9wJCqVlP6BLPDnvNVpMXodnWLeTK+r6YWvGadGVufk -YNC1PwIDAQABMA0GCSqGSIb3DQEBCwUAA4IBAQByrhn78GSS3dJ0pJ6czmhMX5wH -+fauCtt1+Wbn+ctTodTycS+pfULO4gG7wRzhl8KNoOqLmWMjyA2A3mon8kdkD+0C -i8McpoPaGS2wQcqC28Ud6kP9YO81YFyTl4nHVKQ0nmplT+eoLDTCIWMVxHHzxIgs -2ybUluAc+THSjpGxB6kWSAJeg3N+f2OKr+07Yg9LiQ2b8y0eZarpiuuuXCzWeWrQ -PudP0aniyq/gbPhxq0tYF628IBvhDAnr/2kqEmVF2TDr2Sm/Y3PDBuPY6MeIxjnr -ox5zO3LrQmQw11OaIAs2/kviKAoKTFFxeyYcpS5RuKNDZfHQCXlLwt9bySxG ------END CERTIFICATE----- -""" - certificate = x509.load_pem_x509_certificate(certificate_text) - self.assertEqual(get_spki_hash(certificate), expected_hash) + spki_cases = safe_load(spki_test_vectors_path.getContent())["vector"] + for n, case in enumerate(spki_cases): + certificate_text = case["certificate"].encode("ascii") + expected_spki = b64decode(case["expected-spki"]) + expected_hash = case["expected-hash"].encode("ascii") + + certificate = x509.load_pem_x509_certificate(certificate_text) + self.assertEqual( + expected_spki, + get_spki(certificate), + f"case {n} spki data mismatch", + ) + self.assertEqual( + expected_hash, + get_spki_hash(certificate), + f"case {n} spki hash mismatch", + ) class PinningHTTPSValidation(AsyncTestCase): From df491de4ee23e6c1a996a4753c0d910483a79d3f Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 22 Aug 2023 09:01:42 -0400 Subject: [PATCH 2125/2309] Add two more cases to the test vector --- .../test/data/spki-hash-test-vectors.yaml | 54 +++++++++++++++++++ src/allmydata/test/test_storage_https.py | 13 ++++- 2 files changed, 65 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/data/spki-hash-test-vectors.yaml b/src/allmydata/test/data/spki-hash-test-vectors.yaml index 6be6a9b71..33837a1ea 100644 --- a/src/allmydata/test/data/spki-hash-test-vectors.yaml +++ b/src/allmydata/test/data/spki-hash-test-vectors.yaml @@ -24,3 +24,57 @@ vector: PudP0aniyq/gbPhxq0tYF628IBvhDAnr/2kqEmVF2TDr2Sm/Y3PDBuPY6MeIxjnr ox5zO3LrQmQw11OaIAs2/kviKAoKTFFxeyYcpS5RuKNDZfHQCXlLwt9bySxG -----END CERTIFICATE----- + +- expected-hash: >- + jIvdTaNKVK_iyt2EOMb0PwF23vpY3yfsQwbr5V2Rt1k + expected-spki: >- + MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAxMjhLl8R6KX+/itDHCT/T7LQM1i9F6LHe3TW0KWY2FKC2Ov6sJi1pn4NM2qrlW3EUPhX4l0Ru0VE9ZJuwQB1nzFkZIP70Kr8MLmYBoDjWWXsxTiNG4Lj3ydMxBMq/LLSpgHYgb3+Hh+OQeByboW1nVWWm8+QjZNXHhMvRhJmYvyFi0VWoITe/L5R0ubMtGwZ5mal/z9OnvYcE+Jb4PUxiujDhhvAxr4acHscPDn8e4+HBswDSvIHwyxKkE/w6G0yiw736YUbGmxsThSqRqilujh3dAdIVJJxlxhHwrdUkdK/Eq96SOx/BB6M/M8n8KrRNgwuF25MsabRPphgT/l4M46ddyq4209skSnoa1uJdzfx7HQuWep2n0Nagu6WtcKtrzPI3/BKiOMzOcTNOI63VavCtn995CYY9aUoTpz/x/rlp/5TPM1KiaYMBaq+MneBtqlHyYEQUZP9l8QNtvMUO7nLYaYZhcs/QA+qmpJnxcK07njvmw6gh2oLXuvbUbohPVq/3dmRBdJh4tOZWtJsjFP0XYe41Hhw/sUSWXlJAPghLXBBbgAkkeyK5KatuvD7Lpfs/iuz17No1mo8MhLr3+EnzZ1JBuRo8Nksw4FX5ivZmJxt/HQ2UcQ9HZLejIZJbYBEpUu5hvaC0rOmWDWfftLAjD7DzDPu+u46ZNGa8ykCAwEAAQ== + certificate: | + -----BEGIN CERTIFICATE----- + MIIFazCCA1OgAwIBAgIUWcQFI0lueRJyK4txfA/Ydn0bPRIwDQYJKoZIhvcNAQEL + BQAwRTELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM + GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDAeFw0yMzA4MjIxMjUxNDFaFw0yNDA4 + MjExMjUxNDFaMEUxCzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEw + HwYDVQQKDBhJbnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQwggIiMA0GCSqGSIb3DQEB + AQUAA4ICDwAwggIKAoICAQDEyOEuXxHopf7+K0McJP9PstAzWL0Xosd7dNbQpZjY + UoLY6/qwmLWmfg0zaquVbcRQ+FfiXRG7RUT1km7BAHWfMWRkg/vQqvwwuZgGgONZ + ZezFOI0bguPfJ0zEEyr8stKmAdiBvf4eH45B4HJuhbWdVZabz5CNk1ceEy9GEmZi + /IWLRVaghN78vlHS5sy0bBnmZqX/P06e9hwT4lvg9TGK6MOGG8DGvhpwexw8Ofx7 + j4cGzANK8gfDLEqQT/DobTKLDvfphRsabGxOFKpGqKW6OHd0B0hUknGXGEfCt1SR + 0r8Sr3pI7H8EHoz8zyfwqtE2DC4XbkyxptE+mGBP+Xgzjp13KrjbT2yRKehrW4l3 + N/HsdC5Z6nafQ1qC7pa1wq2vM8jf8EqI4zM5xM04jrdVq8K2f33kJhj1pShOnP/H + +uWn/lM8zUqJpgwFqr4yd4G2qUfJgRBRk/2XxA228xQ7ucthphmFyz9AD6qakmfF + wrTueO+bDqCHagte69tRuiE9Wr/d2ZEF0mHi05la0myMU/Rdh7jUeHD+xRJZeUkA + +CEtcEFuACSR7Irkpq268Psul+z+K7PXs2jWajwyEuvf4SfNnUkG5Gjw2SzDgVfm + K9mYnG38dDZRxD0dkt6MhkltgESlS7mG9oLSs6ZYNZ9+0sCMPsPMM+767jpk0Zrz + KQIDAQABo1MwUTAdBgNVHQ4EFgQUl/JLslQ7ISm+9JR1dMaq2I54KAIwHwYDVR0j + BBgwFoAUl/JLslQ7ISm+9JR1dMaq2I54KAIwDwYDVR0TAQH/BAUwAwEB/zANBgkq + hkiG9w0BAQsFAAOCAgEAwcorbUP98LPyDmOdTe/Y9yLWSgD/xJV/L1oQpB8HhbXA + J3mEnlXtPMNFZULSdHxJycexeHe1tiDcFgatQv/YwURHW67s0TFHBXTvSitWz9tU + CL/t7pEIdKgzbUL2yQry7voWVUaXOf7//l/4P9x2/egn78L6+KuRek6umtIECsN0 + HoOiZzqTrXn2WNtnU1Br9m0cxFFzMzP/g2Rd9MUKjIDag7DLfvRCmTMK8825vTJI + L3nzGfWk5R+ZWO4BudfvQWpI7iMj2/7lRWxYvmS+SSJh+DFwYwV+4CaCPecXVI2x + cD/M3uKTLhUMWo1Ge0qQWhl/qwtJ6FNaxp86yiX8x8EHYB0bDZgH4xMQE0/6o0Vg + vKpy/IrEwnN8WM8yYLpm9kTe9H+jM/NEOxPMh4uid/FLmi7KN549UItAzUS3h7zP + gP4cpSW+3Dgj0l7C58RIWxwABIIJZMH/2wMT/PeNg2pqDjhkoPDg8rwsvaFn6T0u + 1A6pJFnVtWGUuyxJESVYBq4vNSLH68v/xkajxl62uWPDkpgAqWuj5TOUP0e/1Uj5 + wqF/jNlRhLMw10r0U40AYkzQjgN2Q4jasqUKsZyhDa8F8861BHsSvFPrASLy4UrZ + 9Tb4DMYXTNZOY6v1iQerRk4ujx/lTjlwuaX9FsirbkuLv/xF346uEl0jBYR7eMo= + -----END CERTIFICATE----- + +- expected-hash: >- + nG1UHCwz7nXHp2zMCiSfxRbCY29OK3RockkeOiw-t8A + expected-spki: >- + MCowBQYDK2VwAyEA6gbCgxeb9kkSDo4WbB76aTvBWnpyzColUKDxyDhPu94= + certificate: | + -----BEGIN CERTIFICATE----- + MIIBnzCCAVGgAwIBAgIUBM5d9fmVxhjKQod7TLp6Bb2vEd4wBQYDK2VwMEUxCzAJ + BgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRlcm5l + dCBXaWRnaXRzIFB0eSBMdGQwHhcNMjMwODIyMTI1NjE0WhcNMjQwODIxMTI1NjE0 + WjBFMQswCQYDVQQGEwJBVTETMBEGA1UECAwKU29tZS1TdGF0ZTEhMB8GA1UECgwY + SW50ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMCowBQYDK2VwAyEA6gbCgxeb9kkSDo4W + bB76aTvBWnpyzColUKDxyDhPu96jUzBRMB0GA1UdDgQWBBQC8cbPWjZilcD4FSU/ + J1sSNYwpAjAfBgNVHSMEGDAWgBQC8cbPWjZilcD4FSU/J1sSNYwpAjAPBgNVHRMB + Af8EBTADAQH/MAUGAytlcANBAGfmvq0a+Ip6nDBlj1tOpyJzcl1J+wj+4N72V23z + H1c75cXDrl9DMOqLwNVK9YD2wmaxPyEWO4tdth560Nir4QM= + -----END CERTIFICATE----- diff --git a/src/allmydata/test/test_storage_https.py b/src/allmydata/test/test_storage_https.py index 9e44b86b0..5ff193c3d 100644 --- a/src/allmydata/test/test_storage_https.py +++ b/src/allmydata/test/test_storage_https.py @@ -57,8 +57,13 @@ class HTTPSNurlTests(SyncTestCase): openssl asn1parse -noout -inform pem -out public.key openssl dgst -sha256 -binary public.key | openssl enc -base64 - The expected SubjectPublicKeyInfo bytes were extracted from the implementation of `get_spki_hash` after its result matched the expected value generated by the command above. + The OpenSSL base64-encoded output was then adjusted into the URL-safe + base64 variation: `+` and `/` were replaced with `-` and `_` and the + trailing `=` padding was removed. + The expected SubjectPublicKeyInfo bytes were extracted from the + implementation of `get_spki_hash` after its result matched the + expected value generated by the command above. """ spki_cases = safe_load(spki_test_vectors_path.getContent())["vector"] for n, case in enumerate(spki_cases): @@ -66,7 +71,11 @@ class HTTPSNurlTests(SyncTestCase): expected_spki = b64decode(case["expected-spki"]) expected_hash = case["expected-hash"].encode("ascii") - certificate = x509.load_pem_x509_certificate(certificate_text) + try: + certificate = x509.load_pem_x509_certificate(certificate_text) + except Exception as e: + self.fail(f"Loading case {n} certificate failed: {e}") + self.assertEqual( expected_spki, get_spki(certificate), From 83276ee3b7c81fe4ede215e1477e19742d452df7 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 22 Aug 2023 09:02:00 -0400 Subject: [PATCH 2126/2309] news fragment --- newsfragments/4061.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/4061.minor diff --git a/newsfragments/4061.minor b/newsfragments/4061.minor new file mode 100644 index 000000000..e69de29bb From 4df2d7704b86266164ed1e1ed38dc7f2495ecf2f Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 23 Aug 2023 07:04:29 -0400 Subject: [PATCH 2127/2309] get the test data included --- MANIFEST.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MANIFEST.in b/MANIFEST.in index 121a9778f..6cec1c847 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -4,7 +4,7 @@ include relnotes.txt include Dockerfile include tox.ini .appveyor.yml .travis.yml include .coveragerc -recursive-include src *.xhtml *.js *.png *.css *.svg *.txt +recursive-include src *.xhtml *.js *.png *.css *.svg *.txt *.yaml graft docs graft misc graft static From 45e201a2821dcef63474dc682bd3ce0eb8e501f7 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 5 Sep 2023 09:33:44 -0400 Subject: [PATCH 2128/2309] Apply suggestions from code review Co-authored-by: Jean-Paul Calderone --- src/allmydata/test/test_storage_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/test_storage_client.py b/src/allmydata/test/test_storage_client.py index 03c9fb9ae..13c6ccaea 100644 --- a/src/allmydata/test/test_storage_client.py +++ b/src/allmydata/test/test_storage_client.py @@ -657,6 +657,7 @@ storage: ) broker = StorageFarmBroker(True, tub_maker, config) broker.startService() + self.addCleanup(broker.stopService) key_s = b'v0-1234-1' ones = str(base32.b2a(b"1"), "utf-8") @@ -681,7 +682,6 @@ storage: self.assertTrue(new_service.running) self.assertIdentical(new_service.parent, broker) - return broker.stopService() def test_static_permutation_seed_pubkey(self): broker = make_broker() From 1931022ff0509d0711b872b465ba31593737eff2 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 5 Sep 2023 09:40:10 -0400 Subject: [PATCH 2129/2309] News fragment. --- newsfragments/4062.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/4062.minor diff --git a/newsfragments/4062.minor b/newsfragments/4062.minor new file mode 100644 index 000000000..e69de29bb From a8b68c217f8880f5493e4cb349293c3600d57d37 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 5 Sep 2023 09:40:22 -0400 Subject: [PATCH 2130/2309] Update to new mypy and Twisted for type checking. --- tox.ini | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index 67a089b0c..294f26284 100644 --- a/tox.ini +++ b/tox.ini @@ -125,7 +125,7 @@ commands = [testenv:typechecks] basepython = python3 deps = - mypy==1.4.1 + mypy==1.5.1 mypy-zope types-mock types-six @@ -134,7 +134,7 @@ deps = types-pyOpenSSL foolscap # Upgrade when new releases come out: - Twisted==22.10.0 + Twisted==23.8.0 commands = mypy src From e96d67f541c9a5ad8bbbb7fb392f8ae5bf9063cd Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 5 Sep 2023 09:44:48 -0400 Subject: [PATCH 2131/2309] Upgrade ruff. --- tox.ini | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 294f26284..027736c1a 100644 --- a/tox.ini +++ b/tox.ini @@ -99,9 +99,10 @@ skip_install = true deps = # Pin a specific version so we get consistent outcomes; update this # occasionally: - ruff == 0.0.278 + ruff == 0.0.287 # towncrier doesn't work with importlib_resources 6.0.0 # https://github.com/twisted/towncrier/issues/528 + # Will be fixed in first version of Towncrier that is larger than 2023.6. importlib_resources < 6.0.0 towncrier # On macOS, git inside of towncrier needs $HOME. From 7d7ca29e3d502c3e248bf49572501c8431c39aee Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 5 Sep 2023 10:14:10 -0400 Subject: [PATCH 2132/2309] Work around Hypothesis complaint about differing executors due to subclassing. See https://hypothesis.readthedocs.io/en/latest/settings.html#health-checks --- newsfragments/4063.minor | 0 src/allmydata/test/test_storage_http.py | 20 ++++++++++++++++++-- 2 files changed, 18 insertions(+), 2 deletions(-) create mode 100644 newsfragments/4063.minor diff --git a/newsfragments/4063.minor b/newsfragments/4063.minor new file mode 100644 index 000000000..e69de29bb diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index bc266e824..eaed5f07c 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -1673,10 +1673,12 @@ class SharedImmutableMutableTestsMixin: # semantically valid under HTTP. check_bad_range("bytes=0-") - @given(data_length=st.integers(min_value=1, max_value=300000)) - def test_read_with_no_range(self, data_length): + def _read_with_no_range_test(self, data_length): """ A read with no range returns the whole mutable/immutable. + + Actual test is defined in subclasses, to fix complaints from Hypothesis + about the method having different executors. """ storage_index, uploaded_data, _ = self.upload(1, data_length) response = self.http.result_of_with_flush( @@ -1770,6 +1772,13 @@ class ImmutableSharedTests(SharedImmutableMutableTestsMixin, SyncTestCase): def get_leases(self, storage_index): return self.http.storage_server.get_leases(storage_index) + @given(data_length=st.integers(min_value=1, max_value=300000)) + def test_read_with_no_range(self, data_length): + """ + A read with no range returns the whole immutable. + """ + return self._read_with_no_range_test(data_length) + class MutableSharedTests(SharedImmutableMutableTestsMixin, SyncTestCase): """Shared tests, running on mutables.""" @@ -1809,3 +1818,10 @@ class MutableSharedTests(SharedImmutableMutableTestsMixin, SyncTestCase): def get_leases(self, storage_index): return self.http.storage_server.get_slot_leases(storage_index) + + @given(data_length=st.integers(min_value=1, max_value=300000)) + def test_read_with_no_range(self, data_length): + """ + A read with no range returns the whole mutable. + """ + return self._read_with_no_range_test(data_length) From 9ee10af8846d2f762761d3990124a77d50cde2bc Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 5 Sep 2023 11:45:54 -0400 Subject: [PATCH 2133/2309] Start on benchmarking infrastructure: a framework for starting nodes. --- benchmarks/__init__.py | 8 ++ benchmarks/conftest.py | 107 ++++++++++++++++++++++++++ benchmarks/test_cli.py | 7 ++ benchmarks/upload_download.py | 138 ---------------------------------- integration/util.py | 2 +- newsfragments/4060.feature | 1 + 6 files changed, 124 insertions(+), 139 deletions(-) create mode 100644 benchmarks/__init__.py create mode 100644 benchmarks/conftest.py create mode 100644 benchmarks/test_cli.py delete mode 100644 benchmarks/upload_download.py create mode 100644 newsfragments/4060.feature diff --git a/benchmarks/__init__.py b/benchmarks/__init__.py new file mode 100644 index 000000000..57f0be071 --- /dev/null +++ b/benchmarks/__init__.py @@ -0,0 +1,8 @@ +"""pytest-based end-to-end benchmarks of Tahoe-LAFS. + +Usage: + +$ pytest benchmark --number-of-nodes=3 + +It's possible to pass --number-of-nodes multiple times. +""" diff --git a/benchmarks/conftest.py b/benchmarks/conftest.py new file mode 100644 index 000000000..8b4be8a23 --- /dev/null +++ b/benchmarks/conftest.py @@ -0,0 +1,107 @@ +""" +pytest infrastructure for benchmarks. + +The number of nodes is parameterized via a --number-of-nodes CLI option added +to pytest. +""" + +from os.path import abspath +from shutil import which, rmtree +from tempfile import mkdtemp +from pathlib import Path + +import pytest +import pytest_twisted + +from twisted.internet import reactor +from twisted.internet.defer import DeferredList, succeed + +from allmydata.util.iputil import allocate_tcp_port + +from integration.grid import Client, create_grid, create_flog_gatherer + + +def pytest_addoption(parser): + parser.addoption( + "--number-of-nodes", + action="append", + default=[], + type=int, + help="list of number_of_nodes to benchmark against", + ) + # Required to be compatible with integration.util code that we indirectly + # depend on, but also might be useful. + parser.addoption( + "--force-foolscap", + action="store_true", + default=False, + dest="force_foolscap", + help=( + "If set, force Foolscap only for the storage protocol. " + + "Otherwise HTTP will be used." + ), + ) + + +def pytest_generate_tests(metafunc): + # Make number_of_nodes accessible as a parameterized fixture: + if "number_of_nodes" in metafunc.fixturenames: + metafunc.parametrize( + "number_of_nodes", + metafunc.config.getoption("number_of_nodes"), + scope="session", + ) + + +def port_allocator(): + port = allocate_tcp_port() + return succeed(port) + + +@pytest.fixture(scope="session") +def grid(request): + """ + Provides a new Grid with a single Introducer and flog-gathering process. + + Notably does _not_ provide storage servers; use the storage_nodes + fixture if your tests need a Grid that can be used for puts / gets. + """ + tmp_path = mkdtemp(prefix="tahoe-benchmark") + request.addfinalizer(lambda: rmtree(tmp_path)) + flog_binary = which("flogtool") + flog_gatherer = pytest_twisted.blockon( + create_flog_gatherer(reactor, request, tmp_path, flog_binary) + ) + g = pytest_twisted.blockon( + create_grid(reactor, request, tmp_path, flog_gatherer, port_allocator) + ) + return g + + +@pytest.fixture(scope="session") +def storage_nodes(grid, number_of_nodes): + nodes_d = [] + for _ in range(number_of_nodes): + nodes_d.append(grid.add_storage_node()) + + nodes_status = pytest_twisted.blockon(DeferredList(nodes_d)) + for ok, value in nodes_status: + assert ok, "Storage node creation failed: {}".format(value) + return grid.storage_servers + + +@pytest.fixture(scope="session") +def client_node(request, grid, storage_nodes, number_of_nodes) -> Client: + """ + Create a grid client node with number of shares matching number of nodes. + """ + client_node = pytest_twisted.blockon( + grid.add_client( + "client_node", + needed=number_of_nodes, + happy=number_of_nodes, + total=number_of_nodes, + ) + ) + print(f"Client node pid: {client_node.process.transport.pid}") + return client_node diff --git a/benchmarks/test_cli.py b/benchmarks/test_cli.py new file mode 100644 index 000000000..16f10ea88 --- /dev/null +++ b/benchmarks/test_cli.py @@ -0,0 +1,7 @@ +"""Benchmarks for minimal `tahoe` CLI interactions.""" + +def test_cp_one_file(client_node): + """ + Upload a file with tahoe cp and then download it, measuring the latency of + both operations. + """ diff --git a/benchmarks/upload_download.py b/benchmarks/upload_download.py deleted file mode 100644 index 3dfa63336..000000000 --- a/benchmarks/upload_download.py +++ /dev/null @@ -1,138 +0,0 @@ -""" -First attempt at benchmarking uploads and downloads. - -To run: - -$ pytest benchmarks/upload_download.py -s -v -Wignore - -To add latency of e.g. 60ms on Linux: - -$ tc qdisc add dev lo root netem delay 30ms - -To reset: - -$ tc qdisc del dev lo root netem - -Frequency scaling can spoil the results. -To see the range of frequency scaling on a Linux system: - -$ cat /sys/devices/system/cpu/cpu*/cpufreq/scaling_available_frequencies - -And to pin the CPU frequency to the lower bound found in these files: - -$ sudo cpupower frequency-set -f - -TODO Parameterization (pytest?) - - - Foolscap vs not foolscap - - - Number of nodes - - - Data size - - - Number of needed/happy/total shares. - -CAVEATS: The goal here isn't a realistic benchmark, or a benchmark that will be -measured over time, or is expected to be maintainable over time. This is just -a quick and easy way to measure the speed of certain operations, compare HTTP -and Foolscap, and see the short-term impact of changes. - -Eventually this will be replaced by a real benchmark suite that can be run over -time to measure something more meaningful. -""" - -from time import time, process_time -from contextlib import contextmanager -from tempfile import mkdtemp -import os - -from twisted.trial.unittest import TestCase -from twisted.internet.defer import gatherResults - -from allmydata.util.deferredutil import async_to_deferred -from allmydata.util.consumer import MemoryConsumer -from allmydata.test.common_system import SystemTestMixin -from allmydata.immutable.upload import Data as UData -from allmydata.mutable.publish import MutableData - - -@contextmanager -def timeit(name): - start = time() - start_cpu = process_time() - try: - yield - finally: - print( - f"{name}: {time() - start:.3f} elapsed, {process_time() - start_cpu:.3f} CPU" - ) - - -class ImmutableBenchmarks(SystemTestMixin, TestCase): - """Benchmarks for immutables.""" - - # To use Foolscap, change to True: - FORCE_FOOLSCAP_FOR_STORAGE = False - - # Don't reduce HTTP connection timeouts, that messes up the more aggressive - # benchmarks: - REDUCE_HTTP_CLIENT_TIMEOUT = False - - @async_to_deferred - async def setUp(self): - SystemTestMixin.setUp(self) - self.basedir = os.path.join(mkdtemp(), "nodes") - - # 2 nodes - await self.set_up_nodes(2) - - # 1 share - for c in self.clients: - c.encoding_params["k"] = 1 - c.encoding_params["happy"] = 1 - c.encoding_params["n"] = 1 - - print() - - @async_to_deferred - async def test_upload_and_download_immutable(self): - # To test larger files, change this: - DATA = b"Some data to upload\n" * 10 - - for i in range(5): - # 1. Upload: - with timeit(" upload"): - uploader = self.clients[0].getServiceNamed("uploader") - results = await uploader.upload(UData(DATA, convergence=None)) - - # 2. Download: - with timeit("download"): - uri = results.get_uri() - node = self.clients[1].create_node_from_uri(uri) - mc = await node.read(MemoryConsumer(), 0, None) - self.assertEqual(b"".join(mc.chunks), DATA) - - @async_to_deferred - async def test_upload_and_download_mutable(self): - # To test larger files, change this: - DATA = b"Some data to upload\n" * 10 - - for i in range(5): - # 1. Upload: - with timeit(" upload"): - result = await self.clients[0].create_mutable_file(MutableData(DATA)) - - # 2. Download: - with timeit("download"): - data = await result.download_best_version() - self.assertEqual(data, DATA) - - @async_to_deferred - async def test_upload_mutable_in_parallel(self): - # To test larger files, change this: - DATA = b"Some data to upload\n" * 1_000_000 - with timeit(" upload"): - await gatherResults([ - self.clients[0].create_mutable_file(MutableData(DATA)) - for _ in range(20) - ]) diff --git a/integration/util.py b/integration/util.py index 85a2fc3ee..59be528dc 100644 --- a/integration/util.py +++ b/integration/util.py @@ -240,7 +240,7 @@ def _tahoe_runner_optional_coverage(proto, reactor, request, other_args): allmydata.scripts.runner` and `other_args`, optionally inserting a `--coverage` option if the `request` indicates we should. """ - if request.config.getoption('coverage'): + if request.config.getoption('coverage', False): args = [sys.executable, '-b', '-m', 'coverage', 'run', '-m', 'allmydata.scripts.runner', '--coverage'] else: args = [sys.executable, '-b', '-m', 'allmydata.scripts.runner'] diff --git a/newsfragments/4060.feature b/newsfragments/4060.feature new file mode 100644 index 000000000..5eea8134d --- /dev/null +++ b/newsfragments/4060.feature @@ -0,0 +1 @@ +Started work on a new end-to-end benchmarking framework. \ No newline at end of file From 31624019be682597a5edc1f1e2f9947b062e8ef3 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 5 Sep 2023 12:12:49 -0400 Subject: [PATCH 2134/2309] Do an upload and download. --- benchmarks/test_cli.py | 39 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 36 insertions(+), 3 deletions(-) diff --git a/benchmarks/test_cli.py b/benchmarks/test_cli.py index 16f10ea88..9f39038c7 100644 --- a/benchmarks/test_cli.py +++ b/benchmarks/test_cli.py @@ -1,7 +1,40 @@ """Benchmarks for minimal `tahoe` CLI interactions.""" -def test_cp_one_file(client_node): +from subprocess import Popen, PIPE + +import pytest + +from integration.util import cli + + +@pytest.fixture(scope="session") +def cli_alias(client_node): + cli(client_node.process, "create-alias", "cli") + + +def test_get_put_one_file(client_node, cli_alias, tmp_path): """ - Upload a file with tahoe cp and then download it, measuring the latency of - both operations. + Upload a file with ``tahoe put`` and then download it with ``tahoe get``, + measuring the latency of both operations. """ + file_size = 1000 # parameterize later on + file_path = tmp_path / "file" + DATA = b"0123456789" * (file_size // 10) + + with file_path.open("wb") as f: + f.write(DATA) + cli(client_node.process, "put", str(file_path), "cli:tostdout") + + p = Popen( + [ + "tahoe", + "--node-directory", + client_node.process.node_dir, + "get", + "cli:tostdout", + "-", + ], + stdout=PIPE, + ) + assert p.stdout.read() == DATA + assert p.wait() == 0 From c88c241f5c9e9af635d11e35a09354e68bf91674 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 5 Sep 2023 12:23:50 -0400 Subject: [PATCH 2135/2309] Smallest possible benchmark result tracking. --- benchmarks/conftest.py | 21 +++++++++++++++++++++ benchmarks/test_cli.py | 40 ++++++++++++++++++++++++---------------- 2 files changed, 45 insertions(+), 16 deletions(-) diff --git a/benchmarks/conftest.py b/benchmarks/conftest.py index 8b4be8a23..7f280d85b 100644 --- a/benchmarks/conftest.py +++ b/benchmarks/conftest.py @@ -9,6 +9,8 @@ from os.path import abspath from shutil import which, rmtree from tempfile import mkdtemp from pathlib import Path +from contextlib import contextmanager +from time import time import pytest import pytest_twisted @@ -105,3 +107,22 @@ def client_node(request, grid, storage_nodes, number_of_nodes) -> Client: ) print(f"Client node pid: {client_node.process.transport.pid}") return client_node + + +class Benchmarker: + """Keep track of benchmarking results.""" + + @contextmanager + def record(self, name, **parameters): + """Record the timing of running some code, if it succeeds.""" + start = time() + yield + elapsed = time() - start + # For now we just print the outcome: + parameters = " ".join(f"{k}={v}" for (k, v) in parameters.items()) + print(f"BENCHMARK RESULT: {name} {parameters} elapsed {elapsed} secs") + + +@pytest.fixture(scope="session") +def tahoe_benchmarker(): + return Benchmarker() diff --git a/benchmarks/test_cli.py b/benchmarks/test_cli.py index 9f39038c7..94eca4475 100644 --- a/benchmarks/test_cli.py +++ b/benchmarks/test_cli.py @@ -12,7 +12,9 @@ def cli_alias(client_node): cli(client_node.process, "create-alias", "cli") -def test_get_put_one_file(client_node, cli_alias, tmp_path): +def test_get_put_one_file( + client_node, cli_alias, tmp_path, tahoe_benchmarker, number_of_nodes +): """ Upload a file with ``tahoe put`` and then download it with ``tahoe get``, measuring the latency of both operations. @@ -20,21 +22,27 @@ def test_get_put_one_file(client_node, cli_alias, tmp_path): file_size = 1000 # parameterize later on file_path = tmp_path / "file" DATA = b"0123456789" * (file_size // 10) - with file_path.open("wb") as f: f.write(DATA) - cli(client_node.process, "put", str(file_path), "cli:tostdout") - p = Popen( - [ - "tahoe", - "--node-directory", - client_node.process.node_dir, - "get", - "cli:tostdout", - "-", - ], - stdout=PIPE, - ) - assert p.stdout.read() == DATA - assert p.wait() == 0 + with tahoe_benchmarker.record( + "cli-put-file", file_size=file_size, number_of_nodes=number_of_nodes + ): + cli(client_node.process, "put", str(file_path), "cli:tostdout") + + with tahoe_benchmarker.record( + "cli-get-file", file_size=file_size, number_of_nodes=number_of_nodes + ): + p = Popen( + [ + "tahoe", + "--node-directory", + client_node.process.node_dir, + "get", + "cli:tostdout", + "-", + ], + stdout=PIPE, + ) + assert p.stdout.read() == DATA + assert p.wait() == 0 From 496ffcdaa2f2f9a353e69fc991b574a0ab1f80bb Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 5 Sep 2023 12:25:18 -0400 Subject: [PATCH 2136/2309] Run codechecks on benchmarks too. --- benchmarks/conftest.py | 2 -- tox.ini | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/benchmarks/conftest.py b/benchmarks/conftest.py index 7f280d85b..381bd5670 100644 --- a/benchmarks/conftest.py +++ b/benchmarks/conftest.py @@ -5,10 +5,8 @@ The number of nodes is parameterized via a --number-of-nodes CLI option added to pytest. """ -from os.path import abspath from shutil import which, rmtree from tempfile import mkdtemp -from pathlib import Path from contextlib import contextmanager from time import time diff --git a/tox.ini b/tox.ini index 67a089b0c..a736a7af1 100644 --- a/tox.ini +++ b/tox.ini @@ -109,7 +109,7 @@ passenv = HOME setenv = # If no positional arguments are given, try to run the checks on the # entire codebase, including various pieces of supporting code. - DEFAULT_FILES=src integration static misc setup.py + DEFAULT_FILES=src integration benchmarks static misc setup.py commands = ruff check {posargs:{env:DEFAULT_FILES}} python misc/coding_tools/check-umids.py {posargs:{env:DEFAULT_FILES}} From 653f483d9fac8355de5d21f639df792d314b7e3f Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 7 Sep 2023 13:56:30 -0400 Subject: [PATCH 2137/2309] Measure a wider variety of file sizes. --- benchmarks/test_cli.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/benchmarks/test_cli.py b/benchmarks/test_cli.py index 94eca4475..c514cbd69 100644 --- a/benchmarks/test_cli.py +++ b/benchmarks/test_cli.py @@ -12,14 +12,14 @@ def cli_alias(client_node): cli(client_node.process, "create-alias", "cli") +@pytest.mark.parametrize("file_size", [1000, 100_000, 1_000_000, 10_000_000]) def test_get_put_one_file( - client_node, cli_alias, tmp_path, tahoe_benchmarker, number_of_nodes + file_size, client_node, cli_alias, tmp_path, tahoe_benchmarker, number_of_nodes ): """ Upload a file with ``tahoe put`` and then download it with ``tahoe get``, measuring the latency of both operations. """ - file_size = 1000 # parameterize later on file_path = tmp_path / "file" DATA = b"0123456789" * (file_size // 10) with file_path.open("wb") as f: From 6aa6c63b05c0b72a5ad45496e6a37e20315d36b3 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 7 Sep 2023 17:21:45 -0400 Subject: [PATCH 2138/2309] Make benchmarking results visible by default. --- benchmarks/conftest.py | 6 ++++-- benchmarks/test_cli.py | 12 +++++++++--- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/benchmarks/conftest.py b/benchmarks/conftest.py index 381bd5670..6ec01cfb8 100644 --- a/benchmarks/conftest.py +++ b/benchmarks/conftest.py @@ -5,6 +5,7 @@ The number of nodes is parameterized via a --number-of-nodes CLI option added to pytest. """ +import sys from shutil import which, rmtree from tempfile import mkdtemp from contextlib import contextmanager @@ -111,14 +112,15 @@ class Benchmarker: """Keep track of benchmarking results.""" @contextmanager - def record(self, name, **parameters): + def record(self, capsys: pytest.CaptureFixture[str], name, **parameters): """Record the timing of running some code, if it succeeds.""" start = time() yield elapsed = time() - start # For now we just print the outcome: parameters = " ".join(f"{k}={v}" for (k, v) in parameters.items()) - print(f"BENCHMARK RESULT: {name} {parameters} elapsed {elapsed} secs") + with capsys.disabled(): + print(f"\nBENCHMARK RESULT: {name} {parameters} elapsed {elapsed} secs\n") @pytest.fixture(scope="session") diff --git a/benchmarks/test_cli.py b/benchmarks/test_cli.py index c514cbd69..0c5f24c05 100644 --- a/benchmarks/test_cli.py +++ b/benchmarks/test_cli.py @@ -14,7 +14,13 @@ def cli_alias(client_node): @pytest.mark.parametrize("file_size", [1000, 100_000, 1_000_000, 10_000_000]) def test_get_put_one_file( - file_size, client_node, cli_alias, tmp_path, tahoe_benchmarker, number_of_nodes + file_size, + client_node, + cli_alias, + tmp_path, + tahoe_benchmarker, + number_of_nodes, + capsys, ): """ Upload a file with ``tahoe put`` and then download it with ``tahoe get``, @@ -26,12 +32,12 @@ def test_get_put_one_file( f.write(DATA) with tahoe_benchmarker.record( - "cli-put-file", file_size=file_size, number_of_nodes=number_of_nodes + capsys, "cli-put-file", file_size=file_size, number_of_nodes=number_of_nodes ): cli(client_node.process, "put", str(file_path), "cli:tostdout") with tahoe_benchmarker.record( - "cli-get-file", file_size=file_size, number_of_nodes=number_of_nodes + capsys, "cli-get-file", file_size=file_size, number_of_nodes=number_of_nodes ): p = Popen( [ From a497b8d86f2f5e335398dccfc894d4ae482f6334 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 7 Sep 2023 17:53:48 -0400 Subject: [PATCH 2139/2309] Also record CPU time of subprocesses. --- benchmarks/conftest.py | 21 ++++++++++++++++++--- setup.py | 3 ++- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/benchmarks/conftest.py b/benchmarks/conftest.py index 6ec01cfb8..2b60d8cdb 100644 --- a/benchmarks/conftest.py +++ b/benchmarks/conftest.py @@ -5,11 +5,12 @@ The number of nodes is parameterized via a --number-of-nodes CLI option added to pytest. """ -import sys +from resource import getrusage, RUSAGE_CHILDREN from shutil import which, rmtree from tempfile import mkdtemp from contextlib import contextmanager from time import time +from psutil import Process import pytest import pytest_twisted @@ -114,13 +115,27 @@ class Benchmarker: @contextmanager def record(self, capsys: pytest.CaptureFixture[str], name, **parameters): """Record the timing of running some code, if it succeeds.""" + process = Process() + + def get_children_cpu_time(): + cpu = 0 + for subprocess in process.children(): + usage = subprocess.cpu_times() + cpu += usage.system + usage.user + return cpu + + start_cpu = get_children_cpu_time() start = time() yield elapsed = time() - start - # For now we just print the outcome: + end_cpu = get_children_cpu_time() + elapsed_cpu = end_cpu - start_cpu + # FOR now we just print the outcome: parameters = " ".join(f"{k}={v}" for (k, v) in parameters.items()) with capsys.disabled(): - print(f"\nBENCHMARK RESULT: {name} {parameters} elapsed {elapsed} secs\n") + print( + f"\nBENCHMARK RESULT: {name} {parameters} elapsed={elapsed:.3} (secs) CPU={elapsed_cpu:.3} (secs)\n" + ) @pytest.fixture(scope="session") diff --git a/setup.py b/setup.py index 6f807a2d2..e8e7a1b83 100644 --- a/setup.py +++ b/setup.py @@ -435,7 +435,8 @@ setup(name="tahoe-lafs", # also set in __init__.py "paramiko < 2.9", "pytest-timeout", # Does our OpenMetrics endpoint adhere to the spec: - "prometheus-client == 0.11.0" + "prometheus-client == 0.11.0", + "psutil", ] + tor_requires + i2p_requires, "tor": tor_requires, "i2p": i2p_requires, From e21c2dd47074a7461a0c7513534a1ea3b41f4aeb Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 8 Sep 2023 10:26:17 -0400 Subject: [PATCH 2140/2309] Do multiple files sequentially, to reduce noise. --- benchmarks/test_cli.py | 58 +++++++++++++++++++++++++----------------- 1 file changed, 35 insertions(+), 23 deletions(-) diff --git a/benchmarks/test_cli.py b/benchmarks/test_cli.py index 0c5f24c05..dd5879495 100644 --- a/benchmarks/test_cli.py +++ b/benchmarks/test_cli.py @@ -7,48 +7,60 @@ import pytest from integration.util import cli -@pytest.fixture(scope="session") +@pytest.fixture(scope="module", autouse=True) def cli_alias(client_node): cli(client_node.process, "create-alias", "cli") @pytest.mark.parametrize("file_size", [1000, 100_000, 1_000_000, 10_000_000]) -def test_get_put_one_file( +def test_get_put_files_sequentially( file_size, client_node, - cli_alias, - tmp_path, tahoe_benchmarker, number_of_nodes, capsys, ): """ - Upload a file with ``tahoe put`` and then download it with ``tahoe get``, - measuring the latency of both operations. + Upload 5 files with ``tahoe put`` and then download them with ``tahoe + get``, measuring the latency of both operations. We do multiple uploads + and downloads to try to reduce noise. """ - file_path = tmp_path / "file" DATA = b"0123456789" * (file_size // 10) - with file_path.open("wb") as f: - f.write(DATA) with tahoe_benchmarker.record( capsys, "cli-put-file", file_size=file_size, number_of_nodes=number_of_nodes ): - cli(client_node.process, "put", str(file_path), "cli:tostdout") + for i in range(5): + p = Popen( + [ + "tahoe", + "--node-directory", + client_node.process.node_dir, + "put", + "-", + f"cli:get_put_files_sequentially{i}", + ], + stdin=PIPE, + ) + p.stdin.write(DATA) + p.stdin.write(str(i).encode("ascii")) + p.stdin.close() + assert p.wait() == 0 with tahoe_benchmarker.record( capsys, "cli-get-file", file_size=file_size, number_of_nodes=number_of_nodes ): - p = Popen( - [ - "tahoe", - "--node-directory", - client_node.process.node_dir, - "get", - "cli:tostdout", - "-", - ], - stdout=PIPE, - ) - assert p.stdout.read() == DATA - assert p.wait() == 0 + for i in range(5): + p = Popen( + [ + "tahoe", + "--node-directory", + client_node.process.node_dir, + "get", + f"cli:get_put_files_sequentially{i}", + "-", + ], + stdout=PIPE, + ) + assert p.stdout.read() == DATA + str(i).encode("ascii") + assert p.wait() == 0 From 6d626851bfbae700dd66fec2e66d320698325ff7 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 13 Sep 2023 09:26:30 -0400 Subject: [PATCH 2141/2309] news fragment --- newsfragments/4066.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/4066.minor diff --git a/newsfragments/4066.minor b/newsfragments/4066.minor new file mode 100644 index 000000000..e69de29bb From d6b38bc7a22cd5095fa086648d6a5bf9b833857f Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 13 Sep 2023 09:26:35 -0400 Subject: [PATCH 2142/2309] test vectors --- src/allmydata/test/test_hashtree.py | 32 +++++++++++++++-------------- src/allmydata/test/test_hashutil.py | 28 +++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 15 deletions(-) diff --git a/src/allmydata/test/test_hashtree.py b/src/allmydata/test/test_hashtree.py index 5abe2095e..d9be96bd5 100644 --- a/src/allmydata/test/test_hashtree.py +++ b/src/allmydata/test/test_hashtree.py @@ -1,32 +1,22 @@ """ Tests for allmydata.hashtree. - -Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals +from __future__ import annotations -from future.utils import PY2 -if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - - -from twisted.trial import unittest +from .common import SyncTestCase +from base64 import b32encode from allmydata.util.hashutil import tagged_hash from allmydata import hashtree - def make_tree(numleaves): leaves = [b"%d" % i for i in range(numleaves)] leaf_hashes = [tagged_hash(b"tag", leaf) for leaf in leaves] ht = hashtree.HashTree(leaf_hashes) return ht -class Complete(unittest.TestCase): +class Complete(SyncTestCase): def test_create(self): # try out various sizes, since we pad to a power of two ht = make_tree(6) @@ -40,6 +30,18 @@ class Complete(unittest.TestCase): self.failUnlessRaises(IndexError, ht.parent, 0) self.failUnlessRaises(IndexError, ht.needed_for, -1) + def test_well_known_tree(self): + self.assertEqual( + [b32encode(s).strip(b"=").lower() for s in make_tree(3)], + [b'vxuqudnucceja4pqkdqy5txapagxubm5moupzqywkbg2jrjkaola', + b'weycjri4jlcaunca2jyx2kr7sbtb7qdriog3f26g5jpc5awfeazq', + b'5ovy3g2wwjnxoqtja4licckxkbqjef4xsjtclk6gxnsl66kvow6a', + b'esd34nbzri75l3j2vwetpk3dvlvsxstkbaktomonrulpks3df3sq', + b'jkxbwa2tppyfax35o72tbjecxvaa4xphma6zbyfbkkku3ed2657a', + b'wfisavaqgab2raihe7dld2qjps4rtxyiubgfs5enziokey2msjwa', + b't3kza5vwx3tlowdemmgdyigp62ju57qduyfh7uulnfkc7mj2ncrq'], + ) + def test_needed_hashes(self): ht = make_tree(8) self.failUnlessEqual(ht.needed_hashes(0), set([8, 4, 2])) @@ -65,7 +67,7 @@ class Complete(unittest.TestCase): self.failUnless("\n 8:" in d) self.failUnless("\n 4:" in d) -class Incomplete(unittest.TestCase): +class Incomplete(SyncTestCase): def test_create(self): ht = hashtree.IncompleteHashTree(6) diff --git a/src/allmydata/test/test_hashutil.py b/src/allmydata/test/test_hashutil.py index 482e79c0b..1efa2d9db 100644 --- a/src/allmydata/test/test_hashutil.py +++ b/src/allmydata/test/test_hashutil.py @@ -44,6 +44,34 @@ class HashUtilTests(unittest.TestCase): self.failUnlessEqual(len(h2), 16) self.failUnlessEqual(h1, h2) + def test_well_known_tagged_hash(self): + self.assertEqual( + b"yra322btzoqjp4ts2jon5dztgnilcdg6jgztgk7joi6qpjkitg2q", + base32.b2a(hashutil.tagged_hash(b"tag", b"hello world")), + ) + self.assertEqual( + b"kfbsfssrv2bvtp3regne6j7gpdjcdjwncewriyfdtt764o5oa7ta", + base32.b2a(hashutil.tagged_hash(b"different", b"hello world")), + ) + self.assertEqual( + b"z34pzkgo36chbjz2qykonlxthc4zdqqquapw4bcaoogzvmmcr3zq", + base32.b2a(hashutil.tagged_hash(b"different", b"goodbye world")), + ) + + def test_well_known_tagged_pair_hash(self): + self.assertEqual( + b"wmto44q3shtezwggku2fxztfkwibvznkfu6clatnvfog527sb6dq", + base32.b2a(hashutil.tagged_pair_hash(b"tag", b"hello", b"world")), + ) + self.assertEqual( + b"lzn27njx246jhijpendqrxlk4yb23nznbcrihommbymg5e7quh4a", + base32.b2a(hashutil.tagged_pair_hash(b"different", b"hello", b"world")), + ) + self.assertEqual( + b"qnehpoypxxdhjheqq7dayloghtu42yr55uylc776zt23ii73o3oq", + base32.b2a(hashutil.tagged_pair_hash(b"different", b"goodbye", b"world")), + ) + def test_chk(self): h1 = hashutil.convergence_hash(3, 10, 1000, b"data", b"secret") h2 = hashutil.convergence_hasher(3, 10, 1000, b"secret") From 8f878275d430239f886c2ca4cab204a53f19eb38 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 13 Sep 2023 09:56:47 -0400 Subject: [PATCH 2143/2309] More accurate description --- benchmarks/test_cli.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/benchmarks/test_cli.py b/benchmarks/test_cli.py index dd5879495..42b4b45bf 100644 --- a/benchmarks/test_cli.py +++ b/benchmarks/test_cli.py @@ -28,7 +28,7 @@ def test_get_put_files_sequentially( DATA = b"0123456789" * (file_size // 10) with tahoe_benchmarker.record( - capsys, "cli-put-file", file_size=file_size, number_of_nodes=number_of_nodes + capsys, "cli-put-5-file-sequentially", file_size=file_size, number_of_nodes=number_of_nodes ): for i in range(5): p = Popen( @@ -48,7 +48,7 @@ def test_get_put_files_sequentially( assert p.wait() == 0 with tahoe_benchmarker.record( - capsys, "cli-get-file", file_size=file_size, number_of_nodes=number_of_nodes + capsys, "cli-get-5-files-sequentially", file_size=file_size, number_of_nodes=number_of_nodes ): for i in range(5): p = Popen( From a2f761a4ab17e53a45b8f1fa4773c4e0149709fe Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 25 Sep 2023 10:55:21 -0400 Subject: [PATCH 2144/2309] Get CPU usage from cgroup v2. --- benchmarks/__init__.py | 8 ++++++-- benchmarks/conftest.py | 31 ++++++++++++++++++++----------- 2 files changed, 26 insertions(+), 13 deletions(-) diff --git a/benchmarks/__init__.py b/benchmarks/__init__.py index 57f0be071..bcefa8d3a 100644 --- a/benchmarks/__init__.py +++ b/benchmarks/__init__.py @@ -1,8 +1,12 @@ -"""pytest-based end-to-end benchmarks of Tahoe-LAFS. +""" +pytest-based end-to-end benchmarks of Tahoe-LAFS. Usage: -$ pytest benchmark --number-of-nodes=3 +$ systemd-run --user --scope pytest benchmark --number-of-nodes=3 It's possible to pass --number-of-nodes multiple times. + +The systemd-run makes sure the tests run in their own cgroup so we get CPU +accounting correct. """ diff --git a/benchmarks/conftest.py b/benchmarks/conftest.py index 2b60d8cdb..1d522108b 100644 --- a/benchmarks/conftest.py +++ b/benchmarks/conftest.py @@ -5,6 +5,7 @@ The number of nodes is parameterized via a --number-of-nodes CLI option added to pytest. """ +import os from resource import getrusage, RUSAGE_CHILDREN from shutil import which, rmtree from tempfile import mkdtemp @@ -108,6 +109,23 @@ def client_node(request, grid, storage_nodes, number_of_nodes) -> Client: print(f"Client node pid: {client_node.process.transport.pid}") return client_node +def get_cpu_time_for_cgroup(): + """ + Get how many CPU seconds have been used in current cgroup so far. + + Assumes we're running in a v2 cgroup. + """ + with open("/proc/self/cgroup") as f: + cgroup = f.read().strip().split(":")[-1] + assert cgroup.startswith("/") + cgroup = cgroup[1:] + cpu_stat = os.path.join("/sys/fs/cgroup", cgroup, "cpu.stat") + with open(cpu_stat) as f: + for line in f.read().splitlines(): + if line.startswith("usage_usec"): + return int(line.split()[1]) / 1_000_000 + raise ValueError("Failed to find usage_usec") + class Benchmarker: """Keep track of benchmarking results.""" @@ -115,20 +133,11 @@ class Benchmarker: @contextmanager def record(self, capsys: pytest.CaptureFixture[str], name, **parameters): """Record the timing of running some code, if it succeeds.""" - process = Process() - - def get_children_cpu_time(): - cpu = 0 - for subprocess in process.children(): - usage = subprocess.cpu_times() - cpu += usage.system + usage.user - return cpu - - start_cpu = get_children_cpu_time() + start_cpu = get_cpu_time_for_cgroup() start = time() yield elapsed = time() - start - end_cpu = get_children_cpu_time() + end_cpu = get_cpu_time_for_cgroup() elapsed_cpu = end_cpu - start_cpu # FOR now we just print the outcome: parameters = " ".join(f"{k}={v}" for (k, v) in parameters.items()) From 130160e5a0adbda0f141259b8bd66bb57a2d19b1 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 25 Sep 2023 14:28:27 -0400 Subject: [PATCH 2145/2309] Drop psutil --- setup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.py b/setup.py index e8e7a1b83..c011d7389 100644 --- a/setup.py +++ b/setup.py @@ -436,7 +436,6 @@ setup(name="tahoe-lafs", # also set in __init__.py "pytest-timeout", # Does our OpenMetrics endpoint adhere to the spec: "prometheus-client == 0.11.0", - "psutil", ] + tor_requires + i2p_requires, "tor": tor_requires, "i2p": i2p_requires, From 173c3361edda7db3a9fcd94d1758aa22223bada4 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 25 Sep 2023 14:31:46 -0400 Subject: [PATCH 2146/2309] News fragment --- newsfragments/4065.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/4065.minor diff --git a/newsfragments/4065.minor b/newsfragments/4065.minor new file mode 100644 index 000000000..e69de29bb From 541c4b1e16096bebe0e4c28f20cd7e169d3b29b3 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 25 Sep 2023 14:31:53 -0400 Subject: [PATCH 2147/2309] Lints --- benchmarks/conftest.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/benchmarks/conftest.py b/benchmarks/conftest.py index 1d522108b..926978a29 100644 --- a/benchmarks/conftest.py +++ b/benchmarks/conftest.py @@ -6,12 +6,10 @@ to pytest. """ import os -from resource import getrusage, RUSAGE_CHILDREN from shutil import which, rmtree from tempfile import mkdtemp from contextlib import contextmanager from time import time -from psutil import Process import pytest import pytest_twisted From 1743d51bbf6d870da54bac2e390ca29bf26150a4 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 25 Sep 2023 15:40:52 -0400 Subject: [PATCH 2148/2309] Run key generation in a thread. --- src/allmydata/client.py | 25 ++++++++++++++++++------- src/allmydata/test/mutable/util.py | 2 +- src/allmydata/test/no_network.py | 2 ++ 3 files changed, 21 insertions(+), 8 deletions(-) diff --git a/src/allmydata/client.py b/src/allmydata/client.py index cfc0977a1..78c8484da 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -22,6 +22,7 @@ from zope.interface import implementer from twisted.plugin import ( getPlugins, ) +from twisted.internet.interfaces import IReactorFromThreads from twisted.internet import reactor, defer from twisted.application import service from twisted.application.internet import TimerService @@ -47,6 +48,7 @@ from allmydata.util.abbreviate import parse_abbreviated_size from allmydata.util.time_format import parse_duration, parse_date from allmydata.util.i2p_provider import create as create_i2p_provider from allmydata.util.tor_provider import create as create_tor_provider, _Provider as TorProvider +from allmydata.util.cputhreadpool import defer_to_thread from allmydata.stats import StatsProvider from allmydata.history import History from allmydata.interfaces import ( @@ -170,12 +172,19 @@ class KeyGenerator(object): """I create RSA keys for mutable files. Each call to generate() returns a single keypair.""" - def generate(self): - """I return a Deferred that fires with a (verifyingkey, signingkey) - pair. The returned key will be 2048 bit""" + def __init__(self, reactor: IReactorFromThreads): + self._reactor = reactor + + def generate(self) -> defer.Deferred[tuple[rsa.PublicKey, rsa.PrivateKey]]: + """ + I return a Deferred that fires with a (verifyingkey, signingkey) + pair. The returned key will be 2048 bit. + """ keysize = 2048 - signer, verifier = rsa.create_signing_keypair(keysize) - return defer.succeed( (verifier, signer) ) + return defer_to_thread( + self._reactor, rsa.create_signing_keypair, keysize + ).addCallback(lambda t: (t[1], t[0])) + class Terminator(service.Service): def __init__(self): @@ -622,11 +631,13 @@ class _Client(node.Node, pollmixin.PollMixin): } def __init__(self, config, main_tub, i2p_provider, tor_provider, introducer_clients, - storage_farm_broker): + storage_farm_broker, reactor=None): """ Use :func:`allmydata.client.create_client` to instantiate one of these. """ node.Node.__init__(self, config, main_tub, i2p_provider, tor_provider) + if reactor is None: + from twisted.internet import reactor self.started_timestamp = time.time() self.logSource = "Client" @@ -638,7 +649,7 @@ class _Client(node.Node, pollmixin.PollMixin): self.init_stats_provider() self.init_secrets() self.init_node_key() - self._key_generator = KeyGenerator() + self._key_generator = KeyGenerator(reactor) key_gen_furl = config.get_config("client", "key_generator.furl", None) if key_gen_furl: log.msg("[client]key_generator.furl= is now ignored, see #2783") diff --git a/src/allmydata/test/mutable/util.py b/src/allmydata/test/mutable/util.py index bed350652..ab2f64f36 100644 --- a/src/allmydata/test/mutable/util.py +++ b/src/allmydata/test/mutable/util.py @@ -317,7 +317,7 @@ def make_nodemaker_with_storage_broker(storage_broker): :param StorageFarmBroker peers: The storage broker to use. """ sh = client.SecretHolder(b"lease secret", b"convergence secret") - keygen = client.KeyGenerator() + keygen = client.KeyGenerator(reactor) nodemaker = NodeMaker(storage_broker, sh, None, None, None, {"k": 3, "n": 10}, SDMF_VERSION, keygen) diff --git a/src/allmydata/test/no_network.py b/src/allmydata/test/no_network.py index e3b57fb95..20e4057e2 100644 --- a/src/allmydata/test/no_network.py +++ b/src/allmydata/test/no_network.py @@ -246,6 +246,7 @@ def create_no_network_client(basedir): from allmydata.client import read_config config = read_config(basedir, u'client.port') storage_broker = NoNetworkStorageBroker() + from twisted.internet import reactor client = _NoNetworkClient( config, main_tub=None, @@ -253,6 +254,7 @@ def create_no_network_client(basedir): tor_provider=None, introducer_clients=[], storage_farm_broker=storage_broker, + reactor=reactor, ) # this is a (pre-existing) reference-cycle and also a bad idea, see: # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/2949 From ccdc2ff513480ab256d916d080e08e1cbf83f7fb Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 25 Sep 2023 15:41:41 -0400 Subject: [PATCH 2149/2309] News fragment --- newsfragments/4068.feature | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/4068.feature diff --git a/newsfragments/4068.feature b/newsfragments/4068.feature new file mode 100644 index 000000000..6c5530cfd --- /dev/null +++ b/newsfragments/4068.feature @@ -0,0 +1 @@ +Some operations now run in threads, improving the responsiveness of Tahoe nodes. \ No newline at end of file From 08e8dd308ff750b2f46643496f80a26bc773f152 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 16 Oct 2023 09:52:02 -0400 Subject: [PATCH 2150/2309] Detect blocked threads. --- src/allmydata/test/blocking.py | 40 ++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 src/allmydata/test/blocking.py diff --git a/src/allmydata/test/blocking.py b/src/allmydata/test/blocking.py new file mode 100644 index 000000000..ce7274106 --- /dev/null +++ b/src/allmydata/test/blocking.py @@ -0,0 +1,40 @@ +import sys +import traceback +import signal +import threading + +from twisted.internet import reactor + + +def print_stacks(): + print("Uh oh, something is blocking the event loop!") + current_thread = threading.get_ident() + for thread_id, frame in sys._current_frames().items(): + if thread_id == current_thread: + traceback.print_stack(frame) + break + + +def catch_blocking_in_event_loop(test=None): + """ + Print tracebacks if the event loop is blocked for more than a short amount + of time. + """ + signal.signal(signal.SIGALRM, lambda *args: print_stacks()) + + current_scheduled = [None] + + def cancel_and_rerun(): + signal.setitimer(signal.ITIMER_REAL, 0) + signal.setitimer(signal.ITIMER_REAL, 0.15) + current_scheduled[0] = reactor.callLater(0.1, cancel_and_rerun) + + cancel_and_rerun() + + def cleanup(): + signal.signal(signal.SIGALRM, signal.SIG_DFL) + signal.setitimer(signal.ITIMER_REAL, 0) + current_scheduled[0].cancel() + + if test is not None: + test.addCleanup(cleanup) From d3ca02fa3f386afea76d992efee1f85a1547e5c8 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 16 Oct 2023 10:07:36 -0400 Subject: [PATCH 2151/2309] Run blocking operations in thread pool --- src/allmydata/codec.py | 9 +++++---- src/allmydata/mutable/publish.py | 9 ++++++--- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/src/allmydata/codec.py b/src/allmydata/codec.py index 19345959e..a63f0a8c0 100644 --- a/src/allmydata/codec.py +++ b/src/allmydata/codec.py @@ -13,9 +13,10 @@ if PY2: from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from zope.interface import implementer -from twisted.internet import defer +from twisted.internet import defer, reactor from allmydata.util import mathutil from allmydata.util.assertutil import precondition +from allmydata.util.cputhreadpool import defer_to_thread from allmydata.interfaces import ICodecEncoder, ICodecDecoder import zfec @@ -53,9 +54,9 @@ class CRSEncoder(object): for inshare in inshares: assert len(inshare) == self.share_size, (len(inshare), self.share_size, self.data_size, self.required_shares) - shares = self.encoder.encode(inshares, desired_share_ids) - - return defer.succeed((shares, desired_share_ids)) + d = defer_to_thread(reactor, self.encoder.encode, inshares, desired_share_ids) + d.addCallback(lambda shares: (shares, desired_share_ids)) + return d def encode_proposal(self, data, desired_share_ids=None): raise NotImplementedError() diff --git a/src/allmydata/mutable/publish.py b/src/allmydata/mutable/publish.py index 3dcbe2dc5..e262ab967 100644 --- a/src/allmydata/mutable/publish.py +++ b/src/allmydata/mutable/publish.py @@ -14,7 +14,7 @@ import os, time from io import BytesIO from itertools import count from zope.interface import implementer -from twisted.internet import defer +from twisted.internet import defer, reactor from twisted.python import failure from allmydata.crypto import aes @@ -23,6 +23,8 @@ from allmydata.interfaces import IPublishStatus, SDMF_VERSION, MDMF_VERSION, \ IMutableUploadable from allmydata.util import base32, hashutil, mathutil, log from allmydata.util.dictutil import DictOfSets +from allmydata.util.deferredutil import async_to_deferred +from allmydata.util.cputhreadpool import defer_to_thread from allmydata import hashtree, codec from allmydata.storage.server import si_b2a from foolscap.api import eventually, fireEventually @@ -762,7 +764,8 @@ class Publish(object): return d - def _push_segment(self, encoded_and_salt, segnum): + @async_to_deferred + async def _push_segment(self, encoded_and_salt, segnum): """ I push (data, salt) as segment number segnum. """ @@ -776,7 +779,7 @@ class Publish(object): hashed = salt + sharedata else: hashed = sharedata - block_hash = hashutil.block_hash(hashed) + block_hash = await defer_to_thread(reactor, hashutil.block_hash, hashed) self.blockhashes[shareid][segnum] = block_hash # find the writer for this share writers = self.writers[shareid] From 2ccdd183c136d9d12411e0790f0317da7b26474b Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 16 Oct 2023 10:10:00 -0400 Subject: [PATCH 2152/2309] Just always run in thread --- src/allmydata/storage/http_server.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 66b0dd6de..7e6682207 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -638,10 +638,7 @@ async def read_encoded( # Pycddl will release the GIL when validating larger documents, so # let's take advantage of multiple CPUs: - if size > 10_000: - await defer_to_thread(reactor, schema.validate_cbor, message) - else: - schema.validate_cbor(message) + await defer_to_thread(reactor, schema.validate_cbor, message) # The CBOR parser will allocate more memory, but at least we can feed # it the file-like object, so that if it's large it won't be make two From b60e53b3fbed63aefb6264807414abe79be43a1b Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 16 Oct 2023 10:38:05 -0400 Subject: [PATCH 2153/2309] Run blocking code in a thread --- src/allmydata/codec.py | 9 ++++++--- src/allmydata/mutable/retrieve.py | 10 ++++++---- src/allmydata/storage/http_client.py | 19 ++++++++++++++----- 3 files changed, 26 insertions(+), 12 deletions(-) diff --git a/src/allmydata/codec.py b/src/allmydata/codec.py index a63f0a8c0..51dc74a8a 100644 --- a/src/allmydata/codec.py +++ b/src/allmydata/codec.py @@ -83,9 +83,12 @@ class CRSDecoder(object): len(some_shares), len(their_shareids)) precondition(len(some_shares) == self.required_shares, len(some_shares), self.required_shares) - data = self.decoder.decode(some_shares, - [int(s) for s in their_shareids]) - return defer.succeed(data) + return defer_to_thread( + reactor, + self.decoder.decode, + some_shares, + [int(s) for s in their_shareids] + ) def parse_params(serializedparams): pieces = serializedparams.split(b"-") diff --git a/src/allmydata/mutable/retrieve.py b/src/allmydata/mutable/retrieve.py index 64573a49a..93d0a410f 100644 --- a/src/allmydata/mutable/retrieve.py +++ b/src/allmydata/mutable/retrieve.py @@ -7,7 +7,7 @@ import time from itertools import count from zope.interface import implementer -from twisted.internet import defer +from twisted.internet import defer, reactor from twisted.python import failure from twisted.internet.interfaces import IPushProducer, IConsumer from foolscap.api import eventually, fireEventually, DeadReferenceError, \ @@ -20,6 +20,7 @@ from allmydata.interfaces import IRetrieveStatus, NotEnoughSharesError, \ from allmydata.util.assertutil import _assert, precondition from allmydata.util import hashutil, log, mathutil, deferredutil from allmydata.util.dictutil import DictOfSets +from allmydata.util.cputhreadpool import defer_to_thread from allmydata import hashtree, codec from allmydata.storage.server import si_b2a @@ -734,7 +735,8 @@ class Retrieve(object): return None - def _validate_block(self, results, segnum, reader, server, started): + @deferredutil.async_to_deferred + async def _validate_block(self, results, segnum, reader, server, started): """ I validate a block from one share on a remote server. """ @@ -767,9 +769,9 @@ class Retrieve(object): "block hash tree failure: %s" % e) if self._version == MDMF_VERSION: - blockhash = hashutil.block_hash(salt + block) + blockhash = await defer_to_thread(reactor, hashutil.block_hash, salt + block) else: - blockhash = hashutil.block_hash(block) + blockhash = await defer_to_thread(reactor, hashutil.block_hash, block) # If this works without an error, then validation is # successful. try: diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index b508c07fd..41c97bfb6 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -39,6 +39,7 @@ from twisted.internet.interfaces import ( IOpenSSLClientConnectionCreator, IReactorTime, IDelayedCall, + IReactorFromThreads, ) from twisted.internet.ssl import CertificateOptions from twisted.protocols.tls import TLSMemoryBIOProtocol @@ -64,6 +65,7 @@ from .common import si_b2a, si_to_human_readable from ..util.hashutil import timing_safe_compare from ..util.deferredutil import async_to_deferred from ..util.tor_provider import _Provider as TorProvider +from ..util.cputhreadpool import defer_to_thread try: from txtorcon import Tor # type: ignore @@ -436,7 +438,7 @@ class StorageClient(object): _swissnum: bytes _treq: Union[treq, StubTreq, HTTPClient] _pool: HTTPConnectionPool - _clock: IReactorTime + _clock: Union[IReactorTime, IReactorFromThreads] # Are we running unit tests? _analyze_response: Callable[[IResponse], None] = lambda _: None @@ -539,7 +541,9 @@ class StorageClient(object): "Can't use both `message_to_serialize` and `data` " "as keyword arguments at the same time" ) - kwargs["data"] = dumps(message_to_serialize) + kwargs["data"] = await defer_to_thread( + self._clock, dumps, message_to_serialize + ) headers.addRawHeader("Content-Type", CBOR_MIME_TYPE) response = await self._treq.request( @@ -557,8 +561,12 @@ class StorageClient(object): if content_type == CBOR_MIME_TYPE: f = await limited_content(response, self._clock) data = f.read() - schema.validate_cbor(data) - return loads(data) + + def validate_and_decode(): + schema.validate_cbor(data) + return loads(data) + + return await defer_to_thread(self._clock, validate_and_decode) else: raise ClientException( -1, @@ -1232,7 +1240,8 @@ class StorageClientMutables: return cast( Set[int], await self._client.decode_cbor( - response, _SCHEMAS["mutable_list_shares"] + response, + _SCHEMAS["mutable_list_shares"], ), ) else: From cb83b08923355d93b36c5f955fcf4dfab45fc500 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 16 Oct 2023 11:16:10 -0400 Subject: [PATCH 2154/2309] Decouple from reactor --- src/allmydata/client.py | 11 +++------ src/allmydata/codec.py | 4 +--- src/allmydata/mutable/publish.py | 2 +- src/allmydata/mutable/retrieve.py | 4 ++-- src/allmydata/storage/http_client.py | 6 ++--- src/allmydata/storage/http_server.py | 2 +- src/allmydata/test/mutable/util.py | 2 +- src/allmydata/test/no_network.py | 4 +--- src/allmydata/test/test_storage_http.py | 7 ++++++ src/allmydata/test/test_util.py | 22 +++++++++++++++-- src/allmydata/util/cputhreadpool.py | 32 +++++++++++++++++++++---- 11 files changed, 67 insertions(+), 29 deletions(-) diff --git a/src/allmydata/client.py b/src/allmydata/client.py index 78c8484da..9a1a75ebc 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -172,9 +172,6 @@ class KeyGenerator(object): """I create RSA keys for mutable files. Each call to generate() returns a single keypair.""" - def __init__(self, reactor: IReactorFromThreads): - self._reactor = reactor - def generate(self) -> defer.Deferred[tuple[rsa.PublicKey, rsa.PrivateKey]]: """ I return a Deferred that fires with a (verifyingkey, signingkey) @@ -182,7 +179,7 @@ class KeyGenerator(object): """ keysize = 2048 return defer_to_thread( - self._reactor, rsa.create_signing_keypair, keysize + rsa.create_signing_keypair, keysize ).addCallback(lambda t: (t[1], t[0])) @@ -631,13 +628,11 @@ class _Client(node.Node, pollmixin.PollMixin): } def __init__(self, config, main_tub, i2p_provider, tor_provider, introducer_clients, - storage_farm_broker, reactor=None): + storage_farm_broker): """ Use :func:`allmydata.client.create_client` to instantiate one of these. """ node.Node.__init__(self, config, main_tub, i2p_provider, tor_provider) - if reactor is None: - from twisted.internet import reactor self.started_timestamp = time.time() self.logSource = "Client" @@ -649,7 +644,7 @@ class _Client(node.Node, pollmixin.PollMixin): self.init_stats_provider() self.init_secrets() self.init_node_key() - self._key_generator = KeyGenerator(reactor) + self._key_generator = KeyGenerator() key_gen_furl = config.get_config("client", "key_generator.furl", None) if key_gen_furl: log.msg("[client]key_generator.furl= is now ignored, see #2783") diff --git a/src/allmydata/codec.py b/src/allmydata/codec.py index 51dc74a8a..402af9204 100644 --- a/src/allmydata/codec.py +++ b/src/allmydata/codec.py @@ -13,7 +13,6 @@ if PY2: from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from zope.interface import implementer -from twisted.internet import defer, reactor from allmydata.util import mathutil from allmydata.util.assertutil import precondition from allmydata.util.cputhreadpool import defer_to_thread @@ -54,7 +53,7 @@ class CRSEncoder(object): for inshare in inshares: assert len(inshare) == self.share_size, (len(inshare), self.share_size, self.data_size, self.required_shares) - d = defer_to_thread(reactor, self.encoder.encode, inshares, desired_share_ids) + d = defer_to_thread(self.encoder.encode, inshares, desired_share_ids) d.addCallback(lambda shares: (shares, desired_share_ids)) return d @@ -84,7 +83,6 @@ class CRSDecoder(object): precondition(len(some_shares) == self.required_shares, len(some_shares), self.required_shares) return defer_to_thread( - reactor, self.decoder.decode, some_shares, [int(s) for s in their_shareids] diff --git a/src/allmydata/mutable/publish.py b/src/allmydata/mutable/publish.py index e262ab967..bc87d5e0c 100644 --- a/src/allmydata/mutable/publish.py +++ b/src/allmydata/mutable/publish.py @@ -779,7 +779,7 @@ class Publish(object): hashed = salt + sharedata else: hashed = sharedata - block_hash = await defer_to_thread(reactor, hashutil.block_hash, hashed) + block_hash = await defer_to_thread(hashutil.block_hash, hashed) self.blockhashes[shareid][segnum] = block_hash # find the writer for this share writers = self.writers[shareid] diff --git a/src/allmydata/mutable/retrieve.py b/src/allmydata/mutable/retrieve.py index 93d0a410f..22e846aa5 100644 --- a/src/allmydata/mutable/retrieve.py +++ b/src/allmydata/mutable/retrieve.py @@ -769,9 +769,9 @@ class Retrieve(object): "block hash tree failure: %s" % e) if self._version == MDMF_VERSION: - blockhash = await defer_to_thread(reactor, hashutil.block_hash, salt + block) + blockhash = await defer_to_thread(hashutil.block_hash, salt + block) else: - blockhash = await defer_to_thread(reactor, hashutil.block_hash, block) + blockhash = await defer_to_thread(hashutil.block_hash, block) # If this works without an error, then validation is # successful. try: diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 41c97bfb6..f3aef8b88 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -541,9 +541,7 @@ class StorageClient(object): "Can't use both `message_to_serialize` and `data` " "as keyword arguments at the same time" ) - kwargs["data"] = await defer_to_thread( - self._clock, dumps, message_to_serialize - ) + kwargs["data"] = await defer_to_thread(dumps, message_to_serialize) headers.addRawHeader("Content-Type", CBOR_MIME_TYPE) response = await self._treq.request( @@ -566,7 +564,7 @@ class StorageClient(object): schema.validate_cbor(data) return loads(data) - return await defer_to_thread(self._clock, validate_and_decode) + return await defer_to_thread(validate_and_decode) else: raise ClientException( -1, diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 7e6682207..5b4e02288 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -638,7 +638,7 @@ async def read_encoded( # Pycddl will release the GIL when validating larger documents, so # let's take advantage of multiple CPUs: - await defer_to_thread(reactor, schema.validate_cbor, message) + await defer_to_thread(schema.validate_cbor, message) # The CBOR parser will allocate more memory, but at least we can feed # it the file-like object, so that if it's large it won't be make two diff --git a/src/allmydata/test/mutable/util.py b/src/allmydata/test/mutable/util.py index ab2f64f36..bed350652 100644 --- a/src/allmydata/test/mutable/util.py +++ b/src/allmydata/test/mutable/util.py @@ -317,7 +317,7 @@ def make_nodemaker_with_storage_broker(storage_broker): :param StorageFarmBroker peers: The storage broker to use. """ sh = client.SecretHolder(b"lease secret", b"convergence secret") - keygen = client.KeyGenerator(reactor) + keygen = client.KeyGenerator() nodemaker = NodeMaker(storage_broker, sh, None, None, None, {"k": 3, "n": 10}, SDMF_VERSION, keygen) diff --git a/src/allmydata/test/no_network.py b/src/allmydata/test/no_network.py index 20e4057e2..dbf994ee0 100644 --- a/src/allmydata/test/no_network.py +++ b/src/allmydata/test/no_network.py @@ -246,15 +246,13 @@ def create_no_network_client(basedir): from allmydata.client import read_config config = read_config(basedir, u'client.port') storage_broker = NoNetworkStorageBroker() - from twisted.internet import reactor client = _NoNetworkClient( config, main_tub=None, i2p_provider=None, tor_provider=None, introducer_clients=[], - storage_farm_broker=storage_broker, - reactor=reactor, + storage_farm_broker=storage_broker ) # this is a (pre-existing) reference-cycle and also a bad idea, see: # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/2949 diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index eaed5f07c..b866f027a 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -43,6 +43,7 @@ from testtools.matchers import Equals from zope.interface import implementer from ..util.deferredutil import async_to_deferred +from ..util.cputhreadpool import disable_thread_pool_for_test from .common import SyncTestCase from ..storage.http_common import ( get_content_type, @@ -345,6 +346,7 @@ class CustomHTTPServerTests(SyncTestCase): def setUp(self): super(CustomHTTPServerTests, self).setUp() + disable_thread_pool_for_test(self) StorageClientFactory.start_test_mode( lambda pool: self.addCleanup(pool.closeCachedConnections) ) @@ -701,6 +703,7 @@ class GenericHTTPAPITests(SyncTestCase): def setUp(self): super(GenericHTTPAPITests, self).setUp() + disable_thread_pool_for_test(self) self.http = self.useFixture(HttpTestFixture()) def test_missing_authentication(self) -> None: @@ -808,6 +811,7 @@ class ImmutableHTTPAPITests(SyncTestCase): def setUp(self): super(ImmutableHTTPAPITests, self).setUp() + disable_thread_pool_for_test(self) self.http = self.useFixture(HttpTestFixture()) self.imm_client = StorageClientImmutables(self.http.client) self.general_client = StorageClientGeneral(self.http.client) @@ -1317,6 +1321,7 @@ class MutableHTTPAPIsTests(SyncTestCase): def setUp(self): super(MutableHTTPAPIsTests, self).setUp() + disable_thread_pool_for_test(self) self.http = self.useFixture(HttpTestFixture()) self.mut_client = StorageClientMutables(self.http.client) @@ -1734,6 +1739,7 @@ class ImmutableSharedTests(SharedImmutableMutableTestsMixin, SyncTestCase): def setUp(self): super(ImmutableSharedTests, self).setUp() + disable_thread_pool_for_test(self) self.http = self.useFixture(HttpTestFixture()) self.client = self.clientFactory(self.http.client) self.general_client = StorageClientGeneral(self.http.client) @@ -1788,6 +1794,7 @@ class MutableSharedTests(SharedImmutableMutableTestsMixin, SyncTestCase): def setUp(self): super(MutableSharedTests, self).setUp() + disable_thread_pool_for_test(self) self.http = self.useFixture(HttpTestFixture()) self.client = self.clientFactory(self.http.client) self.general_client = StorageClientGeneral(self.http.client) diff --git a/src/allmydata/test/test_util.py b/src/allmydata/test/test_util.py index 111f817a8..2b3f33474 100644 --- a/src/allmydata/test/test_util.py +++ b/src/allmydata/test/test_util.py @@ -28,7 +28,7 @@ from allmydata.util import pollmixin from allmydata.util import yamlutil from allmydata.util import rrefutil from allmydata.util.fileutil import EncryptedTemporaryFile -from allmydata.util.cputhreadpool import defer_to_thread +from allmydata.util.cputhreadpool import defer_to_thread, disable_thread_pool_for_test from allmydata.test.common_util import ReallyEqualMixin from .no_network import fireNow, LocalWrapper @@ -613,7 +613,7 @@ class CPUThreadPool(unittest.TestCase): return current_thread(), args, kwargs this_thread = current_thread().ident - result = defer_to_thread(reactor, f, 1, 3, key=4, value=5) + result = defer_to_thread(f, 1, 3, key=4, value=5) # Callbacks run in the correct thread: callback_thread_ident = [] @@ -630,3 +630,21 @@ class CPUThreadPool(unittest.TestCase): self.assertEqual(args, (1, 3)) self.assertEqual(kwargs, {"key": 4, "value": 5}) + def test_when_disabled_runs_in_same_thread(self): + """ + If the CPU thread pool is disabled, the given function runs in the + current thread. + """ + disable_thread_pool_for_test(self) + def f(*args, **kwargs): + return current_thread().ident, args, kwargs + + this_thread = current_thread().ident + result = defer_to_thread(f, 1, 3, key=4, value=5) + l = [] + result.addCallback(l.append) + thread, args, kwargs = l[0] + + self.assertEqual(thread, this_thread) + self.assertEqual(args, (1, 3)) + self.assertEqual(kwargs, {"key": 4, "value": 5}) diff --git a/src/allmydata/util/cputhreadpool.py b/src/allmydata/util/cputhreadpool.py index 225232e04..33799e150 100644 --- a/src/allmydata/util/cputhreadpool.py +++ b/src/allmydata/util/cputhreadpool.py @@ -19,12 +19,13 @@ from typing import TypeVar, Callable, cast from functools import partial import threading from typing_extensions import ParamSpec +from unittest import TestCase from twisted.python.threadpool import ThreadPool -from twisted.internet.defer import Deferred +from twisted.internet.defer import Deferred, maybeDeferred from twisted.internet.threads import deferToThreadPool from twisted.internet.interfaces import IReactorFromThreads - +from twisted.internet import reactor _CPU_THREAD_POOL = ThreadPool(minthreads=0, maxthreads=os.cpu_count(), name="TahoeCPU") if hasattr(threading, "_register_atexit"): @@ -46,14 +47,37 @@ _CPU_THREAD_POOL.start() P = ParamSpec("P") R = TypeVar("R") +# Is running in a thread pool disabled? Should only be true in synchronous unit +# tests. +_DISABLED = False + def defer_to_thread( - reactor: IReactorFromThreads, f: Callable[P, R], *args: P.args, **kwargs: P.kwargs + f: Callable[P, R], *args: P.args, **kwargs: P.kwargs ) -> Deferred[R]: """Run the function in a thread, return the result as a ``Deferred``.""" + if _DISABLED: + return maybeDeferred(f, *args, **kwargs) + # deferToThreadPool has no type annotations... result = deferToThreadPool(reactor, _CPU_THREAD_POOL, f, *args, **kwargs) return cast(Deferred[R], result) -__all__ = ["defer_to_thread"] +def disable_thread_pool_for_test(test: TestCase) -> None: + """ + For the duration of the test, calls to C{defer_to_thread} will actually run + synchronously. + """ + global _DISABLED + + def restore(): + global _DISABLED + _DISABLED = False + + test.addCleanup(restore) + + _DISABLED = True + + +__all__ = ["defer_to_thread", "disable_thread_pool_for_test"] From daec717903d2b63a4e8e57203c90797cbc01e8cb Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 16 Oct 2023 11:27:08 -0400 Subject: [PATCH 2155/2309] Lints --- src/allmydata/client.py | 1 - src/allmydata/mutable/publish.py | 2 +- src/allmydata/mutable/retrieve.py | 2 +- src/allmydata/storage/http_client.py | 3 +-- src/allmydata/test/test_util.py | 1 - src/allmydata/util/cputhreadpool.py | 1 - 6 files changed, 3 insertions(+), 7 deletions(-) diff --git a/src/allmydata/client.py b/src/allmydata/client.py index 9a1a75ebc..803d2946e 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -22,7 +22,6 @@ from zope.interface import implementer from twisted.plugin import ( getPlugins, ) -from twisted.internet.interfaces import IReactorFromThreads from twisted.internet import reactor, defer from twisted.application import service from twisted.application.internet import TimerService diff --git a/src/allmydata/mutable/publish.py b/src/allmydata/mutable/publish.py index bc87d5e0c..3565d0d73 100644 --- a/src/allmydata/mutable/publish.py +++ b/src/allmydata/mutable/publish.py @@ -14,7 +14,7 @@ import os, time from io import BytesIO from itertools import count from zope.interface import implementer -from twisted.internet import defer, reactor +from twisted.internet import defer from twisted.python import failure from allmydata.crypto import aes diff --git a/src/allmydata/mutable/retrieve.py b/src/allmydata/mutable/retrieve.py index 22e846aa5..955ff39ed 100644 --- a/src/allmydata/mutable/retrieve.py +++ b/src/allmydata/mutable/retrieve.py @@ -7,7 +7,7 @@ import time from itertools import count from zope.interface import implementer -from twisted.internet import defer, reactor +from twisted.internet import defer from twisted.python import failure from twisted.internet.interfaces import IPushProducer, IConsumer from foolscap.api import eventually, fireEventually, DeadReferenceError, \ diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index f3aef8b88..7f6fecb98 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -39,7 +39,6 @@ from twisted.internet.interfaces import ( IOpenSSLClientConnectionCreator, IReactorTime, IDelayedCall, - IReactorFromThreads, ) from twisted.internet.ssl import CertificateOptions from twisted.protocols.tls import TLSMemoryBIOProtocol @@ -438,7 +437,7 @@ class StorageClient(object): _swissnum: bytes _treq: Union[treq, StubTreq, HTTPClient] _pool: HTTPConnectionPool - _clock: Union[IReactorTime, IReactorFromThreads] + _clock: IReactorTime # Are we running unit tests? _analyze_response: Callable[[IResponse], None] = lambda _: None diff --git a/src/allmydata/test/test_util.py b/src/allmydata/test/test_util.py index 2b3f33474..3b77b55a4 100644 --- a/src/allmydata/test/test_util.py +++ b/src/allmydata/test/test_util.py @@ -18,7 +18,6 @@ import json from threading import current_thread from twisted.trial import unittest -from twisted.internet import reactor from foolscap.api import Violation, RemoteException from allmydata.util import idlib, mathutil diff --git a/src/allmydata/util/cputhreadpool.py b/src/allmydata/util/cputhreadpool.py index 33799e150..e8b439eee 100644 --- a/src/allmydata/util/cputhreadpool.py +++ b/src/allmydata/util/cputhreadpool.py @@ -24,7 +24,6 @@ from unittest import TestCase from twisted.python.threadpool import ThreadPool from twisted.internet.defer import Deferred, maybeDeferred from twisted.internet.threads import deferToThreadPool -from twisted.internet.interfaces import IReactorFromThreads from twisted.internet import reactor _CPU_THREAD_POOL = ThreadPool(minthreads=0, maxthreads=os.cpu_count(), name="TahoeCPU") From 6e93b12cb569e9ca7d0e3b967f6f18423c91d13b Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 16 Oct 2023 11:31:48 -0400 Subject: [PATCH 2156/2309] Run in thread pool --- src/allmydata/mutable/publish.py | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/src/allmydata/mutable/publish.py b/src/allmydata/mutable/publish.py index 3565d0d73..9374b0d61 100644 --- a/src/allmydata/mutable/publish.py +++ b/src/allmydata/mutable/publish.py @@ -708,7 +708,8 @@ class Publish(object): writer.put_salt(salt) - def _encode_segment(self, segnum): + @async_to_deferred + async def _encode_segment(self, segnum): """ I encrypt and encode the segment segnum. """ @@ -730,11 +731,16 @@ class Publish(object): salt = os.urandom(16) - key = hashutil.ssk_readkey_data_hash(salt, self.readkey) + key = await defer_to_thread(hashutil.ssk_readkey_data_hash, salt, self.readkey) self._status.set_status("Encrypting") - encryptor = aes.create_encryptor(key) - crypttext = aes.encrypt_data(encryptor, data) - assert len(crypttext) == len(data) + + def encrypt(): + encryptor = aes.create_encryptor(key) + crypttext = aes.encrypt_data(encryptor, data) + assert len(crypttext) == len(data) + return crypttext + + crypttext = await defer_to_thread(encrypt) now = time.time() self._status.accumulate_encrypt_time(now - started) @@ -755,14 +761,11 @@ class Publish(object): piece = piece + b"\x00"*(piece_size - len(piece)) # padding crypttext_pieces[i] = piece assert len(piece) == piece_size - d = fec.encode(crypttext_pieces) - def _done_encoding(res): - elapsed = time.time() - started - self._status.accumulate_encode_time(elapsed) - return (res, salt) - d.addCallback(_done_encoding) - return d + res = await fec.encode(crypttext_pieces) + elapsed = time.time() - started + self._status.accumulate_encode_time(elapsed) + return (res, salt) @async_to_deferred async def _push_segment(self, encoded_and_salt, segnum): From 07a1288ab9c442e98b53465a3b1364d1ce855eaf Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 16 Oct 2023 11:41:07 -0400 Subject: [PATCH 2157/2309] Run in thread --- src/allmydata/mutable/retrieve.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/allmydata/mutable/retrieve.py b/src/allmydata/mutable/retrieve.py index 955ff39ed..41c87ac59 100644 --- a/src/allmydata/mutable/retrieve.py +++ b/src/allmydata/mutable/retrieve.py @@ -895,8 +895,8 @@ class Retrieve(object): d.addCallback(_process) return d - - def _decrypt_segment(self, segment_and_salt): + @deferredutil.async_to_deferred + async def _decrypt_segment(self, segment_and_salt): """ I take a single segment and its salt, and decrypt it. I return the plaintext of the segment that is in my argument. @@ -905,9 +905,14 @@ class Retrieve(object): self._set_current_status("decrypting") self.log("decrypting segment %d" % self._current_segment) started = time.time() - key = hashutil.ssk_readkey_data_hash(salt, self._node.get_readkey()) - decryptor = aes.create_decryptor(key) - plaintext = aes.decrypt_data(decryptor, segment) + readkey = self._node.get_readkey() + + def decrypt(): + key = hashutil.ssk_readkey_data_hash(salt, readkey) + decryptor = aes.create_decryptor(key) + return aes.decrypt_data(decryptor, segment) + + plaintext = await defer_to_thread(decrypt) self._status.accumulate_decrypt_time(time.time() - started) return plaintext From 72041c0a8c3af9d792d467cb5cb46f6a0f3c0d11 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 16 Oct 2023 11:44:29 -0400 Subject: [PATCH 2158/2309] More reasonable defaults --- src/allmydata/test/blocking.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/allmydata/test/blocking.py b/src/allmydata/test/blocking.py index ce7274106..6b6c05e5a 100644 --- a/src/allmydata/test/blocking.py +++ b/src/allmydata/test/blocking.py @@ -11,7 +11,7 @@ def print_stacks(): current_thread = threading.get_ident() for thread_id, frame in sys._current_frames().items(): if thread_id == current_thread: - traceback.print_stack(frame) + traceback.print_stack(frame, limit=10) break @@ -26,8 +26,8 @@ def catch_blocking_in_event_loop(test=None): def cancel_and_rerun(): signal.setitimer(signal.ITIMER_REAL, 0) - signal.setitimer(signal.ITIMER_REAL, 0.15) - current_scheduled[0] = reactor.callLater(0.1, cancel_and_rerun) + signal.setitimer(signal.ITIMER_REAL, 0.015) + current_scheduled[0] = reactor.callLater(0.01, cancel_and_rerun) cancel_and_rerun() From 040bb538517381c591c5dfb5675610629e600868 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 16 Oct 2023 11:45:40 -0400 Subject: [PATCH 2159/2309] Make it optional --- src/allmydata/test/common_system.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/allmydata/test/common_system.py b/src/allmydata/test/common_system.py index cfb6c9f04..83ae87883 100644 --- a/src/allmydata/test/common_system.py +++ b/src/allmydata/test/common_system.py @@ -685,6 +685,10 @@ class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin): REDUCE_HTTP_CLIENT_TIMEOUT : bool = True def setUp(self): + if os.getenv("TAHOE_DEBUG_BLOCKING") == "1": + from .blocking import catch_blocking_in_event_loop + catch_blocking_in_event_loop(self) + self._http_client_pools = [] http_client.StorageClientFactory.start_test_mode(self._got_new_http_connection_pool) self.addCleanup(http_client.StorageClientFactory.stop_test_mode) From 5f750ae40bc632b24ee25ef350ceec1741d331da Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 16 Oct 2023 12:41:28 -0400 Subject: [PATCH 2160/2309] News file --- newsfragments/4070.misc | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/4070.misc diff --git a/newsfragments/4070.misc b/newsfragments/4070.misc new file mode 100644 index 000000000..e69de29bb From d80cd454748c0958b35aa4b5c56923dc813f6496 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 16 Oct 2023 12:43:00 -0400 Subject: [PATCH 2161/2309] Tests only ran on Python 2 --- src/allmydata/test/test_windows.py | 227 ----------------------------- 1 file changed, 227 deletions(-) delete mode 100644 src/allmydata/test/test_windows.py diff --git a/src/allmydata/test/test_windows.py b/src/allmydata/test/test_windows.py deleted file mode 100644 index bae56bfed..000000000 --- a/src/allmydata/test/test_windows.py +++ /dev/null @@ -1,227 +0,0 @@ -# -*- coding: utf-8 -*- -# Tahoe-LAFS -- secure, distributed storage grid -# -# Copyright © 2020 The Tahoe-LAFS Software Foundation -# -# This file is part of Tahoe-LAFS. -# -# See the docs/about.rst file for licensing information. - -""" -Tests for the ``allmydata.windows``. -""" - -from __future__ import division -from __future__ import absolute_import -from __future__ import print_function -from __future__ import unicode_literals - -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - -from sys import ( - executable, -) -from json import ( - load, -) -from textwrap import ( - dedent, -) -from twisted.python.filepath import ( - FilePath, -) -from twisted.python.runtime import ( - platform, -) - -from testtools import ( - skipUnless, -) - -from testtools.matchers import ( - MatchesAll, - AllMatch, - IsInstance, - Equals, -) - -from hypothesis import ( - HealthCheck, - settings, - given, - note, -) - -from hypothesis.strategies import ( - lists, - text, - characters, -) - -from .common import ( - PIPE, - Popen, - SyncTestCase, -) - -slow_settings = settings( - suppress_health_check=[HealthCheck.too_slow], - deadline=None, - - # Reduce the number of examples required to consider the test a success. - # The default is 100. Launching a process is expensive so we'll try to do - # it as few times as we can get away with. To maintain good coverage, - # we'll try to pass as much data to each process as we can so we're still - # covering a good portion of the space. - max_examples=10, -) - -@skipUnless(platform.isWindows(), "get_argv is Windows-only") -@skipUnless(PY2, "Not used on Python 3.") -class GetArgvTests(SyncTestCase): - """ - Tests for ``get_argv``. - """ - def test_get_argv_return_type(self): - """ - ``get_argv`` returns a list of unicode strings - """ - # Hide the ``allmydata.windows.fixups.get_argv`` import here so it - # doesn't cause failures on non-Windows platforms. - from ..windows.fixups import ( - get_argv, - ) - argv = get_argv() - - # We don't know what this process's command line was so we just make - # structural assertions here. - self.assertThat( - argv, - MatchesAll( - IsInstance(list), - AllMatch(IsInstance(str)), - ), - ) - - # This test runs a child process. This is unavoidably slow and variable. - # Disable the two time-based Hypothesis health checks. - @slow_settings - @given( - lists( - text( - alphabet=characters( - blacklist_categories=('Cs',), - # Windows CommandLine is a null-terminated string, - # analogous to POSIX exec* arguments. So exclude nul from - # our generated arguments. - blacklist_characters=('\x00',), - ), - min_size=10, - max_size=20, - ), - min_size=10, - max_size=20, - ), - ) - def test_argv_values(self, argv): - """ - ``get_argv`` returns a list representing the result of tokenizing the - "command line" argument string provided to Windows processes. - """ - working_path = FilePath(self.mktemp()) - working_path.makedirs() - save_argv_path = working_path.child("script.py") - saved_argv_path = working_path.child("data.json") - with open(save_argv_path.path, "wt") as f: - # A simple program to save argv to a file. Using the file saves - # us having to figure out how to reliably get non-ASCII back over - # stdio which may pose an independent set of challenges. At least - # file I/O is relatively simple and well-understood. - f.write(dedent( - """ - from allmydata.windows.fixups import ( - get_argv, - ) - import json - with open({!r}, "wt") as f: - f.write(json.dumps(get_argv())) - """.format(saved_argv_path.path)), - ) - argv = [executable.decode("utf-8"), save_argv_path.path] + argv - p = Popen(argv, stdin=PIPE, stdout=PIPE, stderr=PIPE) - p.stdin.close() - stdout = p.stdout.read() - stderr = p.stderr.read() - returncode = p.wait() - - note("stdout: {!r}".format(stdout)) - note("stderr: {!r}".format(stderr)) - - self.assertThat( - returncode, - Equals(0), - ) - with open(saved_argv_path.path, "rt") as f: - saved_argv = load(f) - - self.assertThat( - saved_argv, - Equals(argv), - ) - - -@skipUnless(platform.isWindows(), "intended for Windows-only codepaths") -@skipUnless(PY2, "Not used on Python 3.") -class UnicodeOutputTests(SyncTestCase): - """ - Tests for writing unicode to stdout and stderr. - """ - @slow_settings - @given(characters(), characters()) - def test_write_non_ascii(self, stdout_char, stderr_char): - """ - Non-ASCII unicode characters can be written to stdout and stderr with - automatic UTF-8 encoding. - """ - working_path = FilePath(self.mktemp()) - working_path.makedirs() - script = working_path.child("script.py") - script.setContent(dedent( - """ - from future.utils import PY2 - if PY2: - from future.builtins import chr - - from allmydata.windows.fixups import initialize - initialize() - - # XXX A shortcoming of the monkey-patch approach is that you'd - # better not import stdout or stderr before you call initialize. - from sys import argv, stdout, stderr - - stdout.write(chr(int(argv[1]))) - stdout.close() - stderr.write(chr(int(argv[2]))) - stderr.close() - """ - )) - p = Popen([ - executable, - script.path, - str(ord(stdout_char)), - str(ord(stderr_char)), - ], stdout=PIPE, stderr=PIPE) - stdout = p.stdout.read().decode("utf-8").replace("\r\n", "\n") - stderr = p.stderr.read().decode("utf-8").replace("\r\n", "\n") - returncode = p.wait() - - self.assertThat( - (stdout, stderr, returncode), - Equals(( - stdout_char, - stderr_char, - 0, - )), - ) From 949b90447a545280fad06405b38bcae4a0913272 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 16 Oct 2023 12:45:17 -0400 Subject: [PATCH 2162/2309] Pacify mypy --- src/allmydata/test/test_auth.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/src/allmydata/test/test_auth.py b/src/allmydata/test/test_auth.py index bfe717f79..696ad1709 100644 --- a/src/allmydata/test/test_auth.py +++ b/src/allmydata/test/test_auth.py @@ -1,14 +1,8 @@ """ Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals -from future.utils import PY2 -if PY2: - from future.builtins import str, open # noqa: F401 +from typing import Literal from hypothesis import ( given, @@ -85,6 +79,9 @@ LINE_SEPARATORS = ( '\x0d', # carriage return ) +SURROGATES: Literal["Cs"] = "Cs" + + class AccountFileParserTests(unittest.TestCase): """ Tests for ``load_account_file`` and its helper functions. @@ -96,7 +93,7 @@ class AccountFileParserTests(unittest.TestCase): # They're not necessary to represent any non-surrogate code # point in unicode. They're also not legal individually but # only in pairs. - 'Cs', + SURROGATES, ), # Exclude all our line separators too. blacklist_characters=("\n", "\r"), From 5a6f20b9bb7609bd40c26b199ce4d734bcd07bce Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 16 Oct 2023 12:45:57 -0400 Subject: [PATCH 2163/2309] Correct name --- newsfragments/{4070.misc => 4070.minor} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename newsfragments/{4070.misc => 4070.minor} (100%) diff --git a/newsfragments/4070.misc b/newsfragments/4070.minor similarity index 100% rename from newsfragments/4070.misc rename to newsfragments/4070.minor From 7f53f40d76bd840bc5f567d10a490984ad477962 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 16 Oct 2023 13:25:30 -0400 Subject: [PATCH 2164/2309] Don't assume the result is immediately available --- src/allmydata/test/cli/test_check.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/cli/test_check.py b/src/allmydata/test/cli/test_check.py index 472105ca1..d539f3ebe 100644 --- a/src/allmydata/test/cli/test_check.py +++ b/src/allmydata/test/cli/test_check.py @@ -424,7 +424,7 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase): def _stash_uri(n): self.uriList.append(n.get_uri()) d.addCallback(_stash_uri) - d = c0.create_dirnode() + d.addCallback(lambda _: c0.create_dirnode()) d.addCallback(_stash_uri) d.addCallback(lambda ign: self.do_cli("check", self.uriList[0], self.uriList[1])) From 5d896e8035659fd8864e85b074836c0b6efa9e55 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 19 Oct 2023 11:09:54 -0400 Subject: [PATCH 2165/2309] Just do whole thing in one thread job --- src/allmydata/mutable/publish.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/allmydata/mutable/publish.py b/src/allmydata/mutable/publish.py index 9374b0d61..e57652467 100644 --- a/src/allmydata/mutable/publish.py +++ b/src/allmydata/mutable/publish.py @@ -729,18 +729,17 @@ class Publish(object): assert len(data) == segsize, len(data) - salt = os.urandom(16) - - key = await defer_to_thread(hashutil.ssk_readkey_data_hash, salt, self.readkey) self._status.set_status("Encrypting") - def encrypt(): + def encrypt(readkey): + salt = os.urandom(16) + key = hashutil.ssk_readkey_data_hash(salt, readkey) encryptor = aes.create_encryptor(key) crypttext = aes.encrypt_data(encryptor, data) assert len(crypttext) == len(data) - return crypttext + return salt, crypttext - crypttext = await defer_to_thread(encrypt) + salt, crypttext = await defer_to_thread(encrypt, self.readkey) now = time.time() self._status.accumulate_encrypt_time(now - started) From bab97cf319848ae6cdea4c4b3c5e5c71f3b5b6fc Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 19 Oct 2023 11:12:44 -0400 Subject: [PATCH 2166/2309] Document constraints. --- src/allmydata/storage/http_client.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 7f6fecb98..a98b097dd 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -474,7 +474,8 @@ class StorageClient(object): into corresponding HTTP headers. If ``message_to_serialize`` is set, it will be serialized (by default - with CBOR) and set as the request body. + with CBOR) and set as the request body. It should not be mutated + during execution of this function! Default timeout is 60 seconds. """ From 303e45b1e50ad6e43b8d19c72d6931b93e6ba8c5 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 19 Oct 2023 11:15:06 -0400 Subject: [PATCH 2167/2309] Expand docs --- src/allmydata/util/cputhreadpool.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/allmydata/util/cputhreadpool.py b/src/allmydata/util/cputhreadpool.py index e8b439eee..5c93e9e30 100644 --- a/src/allmydata/util/cputhreadpool.py +++ b/src/allmydata/util/cputhreadpool.py @@ -54,7 +54,12 @@ _DISABLED = False def defer_to_thread( f: Callable[P, R], *args: P.args, **kwargs: P.kwargs ) -> Deferred[R]: - """Run the function in a thread, return the result as a ``Deferred``.""" + """ + Run the function in a thread, return the result as a ``Deferred``. + + However, if ``disable_thread_pool_for_test()`` was called the function will + be called synchronously inside the current thread. + """ if _DISABLED: return maybeDeferred(f, *args, **kwargs) @@ -65,8 +70,8 @@ def defer_to_thread( def disable_thread_pool_for_test(test: TestCase) -> None: """ - For the duration of the test, calls to C{defer_to_thread} will actually run - synchronously. + For the duration of the test, calls to ``defer_to_thread()`` will actually + run synchronously, which is useful for synchronous unit tests. """ global _DISABLED From 20cfe70d483510a5efe5a8231b3b1f8d951e291e Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 19 Oct 2023 13:49:41 -0400 Subject: [PATCH 2168/2309] Switch defer_to_thread() API to hopefully be harder to screw up. --- src/allmydata/client.py | 9 ++++++--- src/allmydata/codec.py | 14 ++++++++------ src/allmydata/test/test_util.py | 19 +++---------------- src/allmydata/util/cputhreadpool.py | 18 +++++++++--------- 4 files changed, 26 insertions(+), 34 deletions(-) diff --git a/src/allmydata/client.py b/src/allmydata/client.py index 803d2946e..dd3c912de 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -48,6 +48,7 @@ from allmydata.util.time_format import parse_duration, parse_date from allmydata.util.i2p_provider import create as create_i2p_provider from allmydata.util.tor_provider import create as create_tor_provider, _Provider as TorProvider from allmydata.util.cputhreadpool import defer_to_thread +from allmydata.util.deferredutil import async_to_deferred from allmydata.stats import StatsProvider from allmydata.history import History from allmydata.interfaces import ( @@ -171,15 +172,17 @@ class KeyGenerator(object): """I create RSA keys for mutable files. Each call to generate() returns a single keypair.""" - def generate(self) -> defer.Deferred[tuple[rsa.PublicKey, rsa.PrivateKey]]: + @async_to_deferred + async def generate(self) -> tuple[rsa.PublicKey, rsa.PrivateKey]: """ I return a Deferred that fires with a (verifyingkey, signingkey) pair. The returned key will be 2048 bit. """ keysize = 2048 - return defer_to_thread( + private, public = await defer_to_thread( rsa.create_signing_keypair, keysize - ).addCallback(lambda t: (t[1], t[0])) + ) + return public, private class Terminator(service.Service): diff --git a/src/allmydata/codec.py b/src/allmydata/codec.py index 402af9204..af375a117 100644 --- a/src/allmydata/codec.py +++ b/src/allmydata/codec.py @@ -16,6 +16,7 @@ from zope.interface import implementer from allmydata.util import mathutil from allmydata.util.assertutil import precondition from allmydata.util.cputhreadpool import defer_to_thread +from allmydata.util.deferredutil import async_to_deferred from allmydata.interfaces import ICodecEncoder, ICodecDecoder import zfec @@ -45,7 +46,8 @@ class CRSEncoder(object): def get_block_size(self): return self.share_size - def encode(self, inshares, desired_share_ids=None): + @async_to_deferred + async def encode(self, inshares, desired_share_ids=None): precondition(desired_share_ids is None or len(desired_share_ids) <= self.max_shares, desired_share_ids, self.max_shares) if desired_share_ids is None: @@ -53,9 +55,8 @@ class CRSEncoder(object): for inshare in inshares: assert len(inshare) == self.share_size, (len(inshare), self.share_size, self.data_size, self.required_shares) - d = defer_to_thread(self.encoder.encode, inshares, desired_share_ids) - d.addCallback(lambda shares: (shares, desired_share_ids)) - return d + shares = await defer_to_thread(self.encoder.encode, inshares, desired_share_ids) + return (shares, desired_share_ids) def encode_proposal(self, data, desired_share_ids=None): raise NotImplementedError() @@ -77,12 +78,13 @@ class CRSDecoder(object): def get_needed_shares(self): return self.required_shares - def decode(self, some_shares, their_shareids): + @async_to_deferred + async def decode(self, some_shares, their_shareids): precondition(len(some_shares) == len(their_shareids), len(some_shares), len(their_shareids)) precondition(len(some_shares) == self.required_shares, len(some_shares), self.required_shares) - return defer_to_thread( + return await defer_to_thread( self.decoder.decode, some_shares, [int(s) for s in their_shareids] diff --git a/src/allmydata/test/test_util.py b/src/allmydata/test/test_util.py index 3b77b55a4..d3a36a756 100644 --- a/src/allmydata/test/test_util.py +++ b/src/allmydata/test/test_util.py @@ -612,24 +612,14 @@ class CPUThreadPool(unittest.TestCase): return current_thread(), args, kwargs this_thread = current_thread().ident - result = defer_to_thread(f, 1, 3, key=4, value=5) - - # Callbacks run in the correct thread: - callback_thread_ident = [] - def passthrough(result): - callback_thread_ident.append(current_thread().ident) - return result - - result.addCallback(passthrough) + thread, args, kwargs = await defer_to_thread(f, 1, 3, key=4, value=5) # The task ran in a different thread: - thread, args, kwargs = await result - self.assertEqual(callback_thread_ident[0], this_thread) self.assertNotEqual(thread.ident, this_thread) self.assertEqual(args, (1, 3)) self.assertEqual(kwargs, {"key": 4, "value": 5}) - def test_when_disabled_runs_in_same_thread(self): + async def test_when_disabled_runs_in_same_thread(self): """ If the CPU thread pool is disabled, the given function runs in the current thread. @@ -639,10 +629,7 @@ class CPUThreadPool(unittest.TestCase): return current_thread().ident, args, kwargs this_thread = current_thread().ident - result = defer_to_thread(f, 1, 3, key=4, value=5) - l = [] - result.addCallback(l.append) - thread, args, kwargs = l[0] + thread, args, kwargs = await defer_to_thread(f, 1, 3, key=4, value=5) self.assertEqual(thread, this_thread) self.assertEqual(args, (1, 3)) diff --git a/src/allmydata/util/cputhreadpool.py b/src/allmydata/util/cputhreadpool.py index 5c93e9e30..032a3a823 100644 --- a/src/allmydata/util/cputhreadpool.py +++ b/src/allmydata/util/cputhreadpool.py @@ -15,14 +15,13 @@ scheduler affinity or cgroups, but that's not the end of the world. """ import os -from typing import TypeVar, Callable, cast +from typing import TypeVar, Callable from functools import partial import threading from typing_extensions import ParamSpec from unittest import TestCase from twisted.python.threadpool import ThreadPool -from twisted.internet.defer import Deferred, maybeDeferred from twisted.internet.threads import deferToThreadPool from twisted.internet import reactor @@ -51,21 +50,22 @@ R = TypeVar("R") _DISABLED = False -def defer_to_thread( - f: Callable[P, R], *args: P.args, **kwargs: P.kwargs -) -> Deferred[R]: +async def defer_to_thread(f: Callable[P, R], *args: P.args, **kwargs: P.kwargs) -> R: """ - Run the function in a thread, return the result as a ``Deferred``. + Run the function in a thread, return the result. However, if ``disable_thread_pool_for_test()`` was called the function will be called synchronously inside the current thread. + + To reduce chances of synchronous tests being misleading as a result, this + is an async function on presumption that will encourage immediate ``await``ing. """ if _DISABLED: - return maybeDeferred(f, *args, **kwargs) + return f(*args, **kwargs) # deferToThreadPool has no type annotations... - result = deferToThreadPool(reactor, _CPU_THREAD_POOL, f, *args, **kwargs) - return cast(Deferred[R], result) + result = await deferToThreadPool(reactor, _CPU_THREAD_POOL, f, *args, **kwargs) + return result def disable_thread_pool_for_test(test: TestCase) -> None: From 40d649b3b202114bc0ce46fccb194b662b4991e3 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 23 Oct 2023 09:44:11 -0400 Subject: [PATCH 2169/2309] Make another slowish operation non-blocking --- src/allmydata/mutable/retrieve.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/src/allmydata/mutable/retrieve.py b/src/allmydata/mutable/retrieve.py index 41c87ac59..54ada2ca9 100644 --- a/src/allmydata/mutable/retrieve.py +++ b/src/allmydata/mutable/retrieve.py @@ -928,12 +928,20 @@ class Retrieve(object): reason, ) - - def _try_to_validate_privkey(self, enc_privkey, reader, server): + @deferredutil.async_to_deferred + async def _try_to_validate_privkey(self, enc_privkey, reader, server): node_writekey = self._node.get_writekey() - alleged_privkey_s = decrypt_privkey(node_writekey, enc_privkey) - alleged_writekey = hashutil.ssk_writekey_hash(alleged_privkey_s) - if alleged_writekey != node_writekey: + + def get_privkey(): + alleged_privkey_s = decrypt_privkey(node_writekey, enc_privkey) + alleged_writekey = hashutil.ssk_writekey_hash(alleged_privkey_s) + if alleged_writekey != node_writekey: + return None + privkey, _ = rsa.create_signing_keypair_from_string(alleged_privkey_s) + return privkey + + privkey = await defer_to_thread(get_privkey) + if privkey is None: self.log("invalid privkey from %s shnum %d" % (reader, reader.shnum), level=log.WEIRD, umid="YIw4tA") @@ -950,7 +958,6 @@ class Retrieve(object): # it's good self.log("got valid privkey from shnum %d on reader %s" % (reader.shnum, reader)) - privkey, _ = rsa.create_signing_keypair_from_string(alleged_privkey_s) self._node._populate_encprivkey(enc_privkey) self._node._populate_privkey(privkey) self._need_privkey = False From 101453cd56acba66817b0c8e7f25bd6c8889c53c Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 23 Oct 2023 09:57:32 -0400 Subject: [PATCH 2170/2309] Make operation non-blocking (assuming GIL is released) --- src/allmydata/mutable/filenode.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/allmydata/mutable/filenode.py b/src/allmydata/mutable/filenode.py index 00b31c52b..ede74d249 100644 --- a/src/allmydata/mutable/filenode.py +++ b/src/allmydata/mutable/filenode.py @@ -14,6 +14,7 @@ from allmydata.interfaces import IMutableFileNode, ICheckable, ICheckResults, \ IMutableFileVersion, IWriteable from allmydata.util import hashutil, log, consumer, deferredutil, mathutil from allmydata.util.assertutil import precondition +from allmydata.util.cputhreadpool import defer_to_thread from allmydata.uri import WriteableSSKFileURI, ReadonlySSKFileURI, \ WriteableMDMFFileURI, ReadonlyMDMFFileURI from allmydata.monitor import Monitor @@ -128,7 +129,8 @@ class MutableFileNode(object): return self - def create_with_keys(self, keypair, contents, + @deferredutil.async_to_deferred + async def create_with_keys(self, keypair, contents, version=SDMF_VERSION): """Call this to create a brand-new mutable file. It will create the shares, find homes for them, and upload the initial contents (created @@ -137,8 +139,8 @@ class MutableFileNode(object): use) when it completes. """ self._pubkey, self._privkey = keypair - self._writekey, self._encprivkey, self._fingerprint = derive_mutable_keys( - keypair, + self._writekey, self._encprivkey, self._fingerprint = await defer_to_thread( + derive_mutable_keys, keypair ) if version == MDMF_VERSION: self._uri = WriteableMDMFFileURI(self._writekey, self._fingerprint) @@ -149,7 +151,7 @@ class MutableFileNode(object): self._readkey = self._uri.readkey self._storage_index = self._uri.storage_index initial_contents = self._get_initial_contents(contents) - return self._upload(initial_contents, None) + return await self._upload(initial_contents, None) def _get_initial_contents(self, contents): if contents is None: From c852e2904883ae3a57bd3c2b4a680fae3e06ae4c Mon Sep 17 00:00:00 2001 From: Brian Warner Date: Wed, 25 Oct 2023 11:42:54 -0700 Subject: [PATCH 2171/2309] docs: describe BTC/development with OpenCollective Add OpenCollective transfer address to docs/donations.rst Also update the Aspiration sections with overall details, and the second phase we did in 2020. --- docs/donations.rst | 27 ++++++++++++++---------- docs/expenses.rst | 51 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+), 11 deletions(-) diff --git a/docs/donations.rst b/docs/donations.rst index a38e280ac..bc1d55c73 100644 --- a/docs/donations.rst +++ b/docs/donations.rst @@ -73,10 +73,15 @@ key on this list. ~$1020 1DskmM8uCvmvTKjPbeDgfmVsGifZCmxouG -* Aspiration contract (first phase, 2019) - $300k-$350k +* Aspiration contract + $300k-$350k (first phase, 2019) + $800k (second phase, 2020) 1gDXYQNH4kCJ8Dk7kgiztfjNUaA1KJcHv +* OpenCollective development work (2023) + ~$260k + 1KZYr8UU2XjuEdSPzn2pF8eRPZZvffByDf + Historical Donation Addresses ============================= @@ -104,17 +109,17 @@ This document is signed by the Tahoe-LAFS Release-Signing Key (GPG keyid (https://github.com/tahoe-lafs/tahoe-lafs.git) as `docs/donations.rst`. Both actions require access to secrets held closely by Tahoe developers. -signed: Brian Warner, 27-Dec-2018 +signed: Brian Warner, 25-Oct-2023 -----BEGIN PGP SIGNATURE----- -iQEzBAEBCAAdFiEE405i0G0Oac/KQXn/veDTHWhmanoFAlwlrdsACgkQveDTHWhm -anqEqQf/SdxMvI0+YbsZe+Gr/+lNWrNtfxAkjgLUZYRPmElZG6UKkNuPghXfsYRM -71nRbgbn05jrke7AGlulxNplTxYP/5LQVf5K1nvTE7yPI/LBMudIpAbM3wPiLKSD -qecrVZiqiIBPHWScyya91qirTHtJTJj39cs/N9937hD+Pm65paHWHDZhMkhStGH7 -05WtvD0G+fFuAgs04VDBz/XVQlPbngkmdKjIL06jpIAgzC3H9UGFcqe55HKY66jK -W769TiRuGLLS07cOPqg8t2hPpE4wv9Gs02hfg1Jc656scsFuEkh5eMMj/MXcFsED -8vwn16kjJk1fkeg+UofnXsHeHIJalQ== -=/E+V +iQEzBAEBCAAdFiEE405i0G0Oac/KQXn/veDTHWhmanoFAmU5YZMACgkQveDTHWhm +anqt+ggAo2kulNmjrWA5VhqE8i6ckkxQMRVY4y0LAfiI0ho/505ZBZvpoh/Ze31x +ZJj4DczHmZM+m3L+fZyubT4ldagYEojtwkYmxHAQz2DIV4PrdjsUQWyvkNcTBZWu +y5mR5ATk3EYRa19xGEosWK1OzW2kgRbpAbznuWsdxxw9vNENBrolGRsyJqRQHCiV +/4UkrGiOegaJSFMKy2dCyDF3ExD6wT9+fdqC5xDJZjhD+SUDJnD4oWLYLroj//v1 +sy4J+/ElNU9oaC0jDb9fx1ECk+u6B+YiaYlW/MrZNqzKCM/76yZ8sA2+ynsOHGtL +bPFpLJjX6gBwHkMqvkWhsJEojxkFVQ== +=gxlb -----END PGP SIGNATURE----- diff --git a/docs/expenses.rst b/docs/expenses.rst index fbb4293ef..b11acce74 100644 --- a/docs/expenses.rst +++ b/docs/expenses.rst @@ -131,3 +131,54 @@ developer summit. * acdfc299c35eed3bb27f7463ad8cdfcdcd4dcfd5184f290f87530c2be999de3e 1.41401086 (@$714.16) = $1009.83, plus 0.000133 tx-fee + +Aspiration Contract +------------------- + +In December 2018, we entered into an agreement with a non-profit named +Aspiration (https://aspirationtech.org/) to fund contractors for development +work. They handle payroll, taxes, and oversight, in exchange for an 8% +management fee. The first phase of work will extend through most of 2019. + +* Recipient: Aspiration +* Address: 1gDXYQNH4kCJ8Dk7kgiztfjNUaA1KJcHv + +These txids record the transfers from the primary 1Pxi address to the +Aspiration-specific 1gDXY subaddress. In some cases, leftover funds +were swept back into the main 1Pxi address after the transfers were +complete. + +First phase, transfers performed 28-Dec-2018 - 31-Dec-2018, total 89 +BTC, about $350K. + +* 95c68d488bd92e8c164195370aaa516dff05aa4d8c543d3fb8cfafae2b811e7a + 1.0 BTC plus 0.00002705 tx-fee +* c0a5b8e3a63c56c4365d4c3ded0821bc1170f6351502849168bc34e30a0582d7 + 89.0 BTC plus 0.00000633 tx-fee +* 421cff5f398509aaf48951520738e0e63dfddf1157920c15bdc72c34e24cf1cf + return 0.00005245 BTC to 1Pxi, less 0.00000211 tx-fee + +In November 2020, we funded a second phase of the work: 51.38094 BTC, +about $800K. + +* 7558cbf3b24e8d835809d2d6f01a8ba229190102efdf36280d0639abaa488721 + 1.0 BTC plus 0.00230766 tx-fee +* 9c78ae6bb7db62cbd6be82fd52d50a2f015285b562f05de0ebfb0e5afc6fd285 + 56.0 BTC plus 0.00057400 tx-fee +* fbee4332e8c7ffbc9c1bcaee773f063550e589e58d350d14f6daaa473966c368 + returning 5.61906 BTC to 1Pxi, less 0.00012000 tx-fee + + +Open Collective +--------------- + +In August 2023, we started working with Open Collective to fund a +grant covering development work performed over the last year. + +* Recipient: Open Collective (US) +* Address: 1KZYr8UU2XjuEdSPzn2pF8eRPZZvffByDf + +The first phase transferred 7.5 BTC (about $260K). + +* (txid) + (amount) From 134bcd7dd087c11c4b4c7e2cb0f5a23f8e64c0a0 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 15 Nov 2023 10:54:27 -0500 Subject: [PATCH 2172/2309] We're not support Python 2 anymore, and future breaks on 3.12 (for now) when standard_library is patched. --- src/allmydata/__init__.py | 25 ++++--------------------- 1 file changed, 4 insertions(+), 21 deletions(-) diff --git a/src/allmydata/__init__.py b/src/allmydata/__init__.py index 333394fc5..8fc7064ca 100644 --- a/src/allmydata/__init__.py +++ b/src/allmydata/__init__.py @@ -3,16 +3,6 @@ Decentralized storage grid. community web site: U{https://tahoe-lafs.org/} """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals - -from future.utils import PY2, PY3 -if PY2: - # Don't import future str() so we don't break Foolscap serialization on Python 2. - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, max, min # noqa: F401 - from past.builtins import unicode as str __all__ = [ "__version__", @@ -52,12 +42,6 @@ __appname__ = "tahoe-lafs" # https://tahoe-lafs.org/trac/tahoe-lafs/wiki/Versioning __full_version__ = __appname__ + '/' + str(__version__) - -# Install Python 3 module locations in Python 2: -from future import standard_library -standard_library.install_aliases() - - # Monkey-patch 3rd party libraries: from ._monkeypatch import patch patch() @@ -72,8 +56,7 @@ del patch # # Also note that BytesWarnings only happen if Python is run with -b option, so # in practice this should only affect tests. -if PY3: - import warnings - # Error on BytesWarnings, to catch things like str(b""), but only for - # allmydata code. - warnings.filterwarnings("error", category=BytesWarning, module=".*allmydata.*") +import warnings +# Error on BytesWarnings, to catch things like str(b""), but only for +# allmydata code. +warnings.filterwarnings("error", category=BytesWarning, module=".*allmydata.*") From 1b700be578df4b3f735f62f62f8cbfeae6de878c Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 15 Nov 2023 10:54:56 -0500 Subject: [PATCH 2173/2309] Allow 3.12 --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index c011d7389..fe2712a2d 100644 --- a/setup.py +++ b/setup.py @@ -385,8 +385,8 @@ setup(name="tahoe-lafs", # also set in __init__.py package_dir = {'':'src'}, packages=find_packages('src') + ['allmydata.test.plugins'], classifiers=trove_classifiers, - # We support Python 3.8 or later, 3.12 is untested for now - python_requires=">=3.8, <3.12", + # We support Python 3.8 or later, 3.13 is untested for now + python_requires=">=3.8, <3.13", install_requires=install_requires, extras_require={ # Duplicate the Twisted pywin32 dependency here. See From 22192b5e09c5e3133ce2dbc83e193227ffda2fc6 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 15 Nov 2023 11:00:58 -0500 Subject: [PATCH 2174/2309] Get rid of eliot patching that's not relevant to Python 3 --- setup.py | 2 +- src/allmydata/util/_eliot_updates.py | 195 --------------------------- src/allmydata/util/eliotutil.py | 11 +- 3 files changed, 7 insertions(+), 201 deletions(-) delete mode 100644 src/allmydata/util/_eliot_updates.py diff --git a/setup.py b/setup.py index fe2712a2d..e8633062f 100644 --- a/setup.py +++ b/setup.py @@ -113,7 +113,7 @@ install_requires = [ "magic-wormhole >= 0.10.2", # We want a new enough version to support custom JSON encoders. - "eliot >= 1.13.0", + "eliot >= 1.14.0", "pyrsistent", diff --git a/src/allmydata/util/_eliot_updates.py b/src/allmydata/util/_eliot_updates.py deleted file mode 100644 index 81db566a4..000000000 --- a/src/allmydata/util/_eliot_updates.py +++ /dev/null @@ -1,195 +0,0 @@ -""" -Bring in some Eliot updates from newer versions of Eliot than we can -depend on in Python 2. The implementations are copied from Eliot 1.14 and -only changed enough to add Python 2 compatibility. - -Every API in this module (except ``eliot_json_encoder``) should be obsolete as -soon as we depend on Eliot 1.14 or newer. - -When that happens: - -* replace ``capture_logging`` - with ``partial(eliot.testing.capture_logging, encoder_=eliot_json_encoder)`` -* replace ``validateLogging`` - with ``partial(eliot.testing.validateLogging, encoder_=eliot_json_encoder)`` -* replace ``MemoryLogger`` - with ``partial(eliot.MemoryLogger, encoder=eliot_json_encoder)`` - -Ported to Python 3. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals - -from future.utils import PY2 -if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - -import json as pyjson -from functools import wraps, partial - -from eliot import ( - MemoryLogger as _MemoryLogger, -) - -from eliot.testing import ( - check_for_errors, - swap_logger, -) - -from .jsonbytes import AnyBytesJSONEncoder - -# There are currently a number of log messages that include non-UTF-8 bytes. -# Allow these, at least for now. Later when the whole test suite has been -# converted to our SyncTestCase or AsyncTestCase it will be easier to turn -# this off and then attribute log failures to specific codepaths so they can -# be fixed (and then not regressed later) because those instances will result -# in test failures instead of only garbage being written to the eliot log. -eliot_json_encoder = AnyBytesJSONEncoder - -class _CustomEncoderMemoryLogger(_MemoryLogger): - """ - Override message validation from the Eliot-supplied ``MemoryLogger`` to - use our chosen JSON encoder. - - This is only necessary on Python 2 where we use an old version of Eliot - that does not parameterize the encoder. - """ - def __init__(self, encoder=eliot_json_encoder): - """ - @param encoder: A JSONEncoder subclass to use when encoding JSON. - """ - self._encoder = encoder - super(_CustomEncoderMemoryLogger, self).__init__() - - def _validate_message(self, dictionary, serializer): - """Validate an individual message. - - As a side-effect, the message is replaced with its serialized contents. - - @param dictionary: A message C{dict} to be validated. Might be mutated - by the serializer! - - @param serializer: C{None} or a serializer. - - @raises TypeError: If a field name is not unicode, or the dictionary - fails to serialize to JSON. - - @raises eliot.ValidationError: If serializer was given and validation - failed. - """ - if serializer is not None: - serializer.validate(dictionary) - for key in dictionary: - if not isinstance(key, str): - if isinstance(key, bytes): - key.decode("utf-8") - else: - raise TypeError(dictionary, "%r is not unicode" % (key,)) - if serializer is not None: - serializer.serialize(dictionary) - - try: - pyjson.dumps(dictionary, cls=self._encoder) - except Exception as e: - raise TypeError("Message %s doesn't encode to JSON: %s" % (dictionary, e)) - -if PY2: - MemoryLogger = partial(_CustomEncoderMemoryLogger, encoder=eliot_json_encoder) -else: - MemoryLogger = partial(_MemoryLogger, encoder=eliot_json_encoder) - -def validateLogging( - assertion, *assertionArgs, **assertionKwargs -): - """ - Decorator factory for L{unittest.TestCase} methods to add logging - validation. - - 1. The decorated test method gets a C{logger} keyword argument, a - L{MemoryLogger}. - 2. All messages logged to this logger will be validated at the end of - the test. - 3. Any unflushed logged tracebacks will cause the test to fail. - - For example: - - from unittest import TestCase - from eliot.testing import assertContainsFields, validateLogging - - class MyTests(TestCase): - def assertFooLogging(self, logger): - assertContainsFields(self, logger.messages[0], {"key": 123}) - - - @param assertion: A callable that will be called with the - L{unittest.TestCase} instance, the logger and C{assertionArgs} and - C{assertionKwargs} once the actual test has run, allowing for extra - logging-related assertions on the effects of the test. Use L{None} if you - want the cleanup assertions registered but no custom assertions. - - @param assertionArgs: Additional positional arguments to pass to - C{assertion}. - - @param assertionKwargs: Additional keyword arguments to pass to - C{assertion}. - - @param encoder_: C{json.JSONEncoder} subclass to use when validating JSON. - """ - encoder_ = assertionKwargs.pop("encoder_", eliot_json_encoder) - def decorator(function): - @wraps(function) - def wrapper(self, *args, **kwargs): - skipped = False - - kwargs["logger"] = logger = MemoryLogger(encoder=encoder_) - self.addCleanup(check_for_errors, logger) - # TestCase runs cleanups in reverse order, and we want this to - # run *before* tracebacks are checked: - if assertion is not None: - self.addCleanup( - lambda: skipped - or assertion(self, logger, *assertionArgs, **assertionKwargs) - ) - try: - return function(self, *args, **kwargs) - except self.skipException: - skipped = True - raise - - return wrapper - - return decorator - -# PEP 8 variant: -validate_logging = validateLogging - -def capture_logging( - assertion, *assertionArgs, **assertionKwargs -): - """ - Capture and validate all logging that doesn't specify a L{Logger}. - - See L{validate_logging} for details on the rest of its behavior. - """ - encoder_ = assertionKwargs.pop("encoder_", eliot_json_encoder) - def decorator(function): - @validate_logging( - assertion, *assertionArgs, encoder_=encoder_, **assertionKwargs - ) - @wraps(function) - def wrapper(self, *args, **kwargs): - logger = kwargs["logger"] - previous_logger = swap_logger(logger) - - def cleanup(): - swap_logger(previous_logger) - - self.addCleanup(cleanup) - return function(self, *args, **kwargs) - - return wrapper - - return decorator diff --git a/src/allmydata/util/eliotutil.py b/src/allmydata/util/eliotutil.py index 789ef38ff..b574f0def 100644 --- a/src/allmydata/util/eliotutil.py +++ b/src/allmydata/util/eliotutil.py @@ -61,6 +61,12 @@ from eliot import ( write_traceback, start_action, ) +from eliot.testing import ( + MemoryLogger, + capture_logging, +) +from eliot.json import EliotJSONEncoder as eliot_json_encoder + from eliot._validation import ( ValidationError, ) @@ -87,11 +93,6 @@ from twisted.internet.defer import ( ) from twisted.application.service import Service -from ._eliot_updates import ( - MemoryLogger, - eliot_json_encoder, - capture_logging, -) def validateInstanceOf(t): """ From 4b75b7c6e88d11a0463b8123bdf107f8795e4916 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 15 Nov 2023 11:14:48 -0500 Subject: [PATCH 2175/2309] Work with newer versions of Eliot --- src/allmydata/test/__init__.py | 4 ++-- src/allmydata/test/eliotutil.py | 5 +++-- src/allmydata/util/eliotutil.py | 30 ++++++++++++------------------ 3 files changed, 17 insertions(+), 22 deletions(-) diff --git a/src/allmydata/test/__init__.py b/src/allmydata/test/__init__.py index ad245ca77..bca4a4ebf 100644 --- a/src/allmydata/test/__init__.py +++ b/src/allmydata/test/__init__.py @@ -125,5 +125,5 @@ if sys.platform == "win32": initialize() from eliot import to_file -from allmydata.util.eliotutil import eliot_json_encoder -to_file(open("eliot.log", "wb"), encoder=eliot_json_encoder) +from allmydata.util.eliotutil import BytesEliotJSONEncoder +to_file(open("eliot.log", "wb"), encoder=BytesEliotJSONEncoder) diff --git a/src/allmydata/test/eliotutil.py b/src/allmydata/test/eliotutil.py index bdc779f1d..8a2f63203 100644 --- a/src/allmydata/test/eliotutil.py +++ b/src/allmydata/test/eliotutil.py @@ -29,6 +29,7 @@ from eliot import ( ILogger, ) from eliot.testing import ( + MemoryLogger, swap_logger, check_for_errors, ) @@ -38,7 +39,7 @@ from twisted.python.monkey import ( ) from ..util.eliotutil import ( - MemoryLogger, + BytesEliotJSONEncoder ) _NAME = Field.for_types( @@ -146,7 +147,7 @@ def with_logging( """ @wraps(test_method) def run_with_logging(*args, **kwargs): - validating_logger = MemoryLogger() + validating_logger = MemoryLogger(encoder=BytesEliotJSONEncoder) original = swap_logger(None) try: swap_logger(_TwoLoggers(original, validating_logger)) diff --git a/src/allmydata/util/eliotutil.py b/src/allmydata/util/eliotutil.py index b574f0def..ec417991c 100644 --- a/src/allmydata/util/eliotutil.py +++ b/src/allmydata/util/eliotutil.py @@ -3,17 +3,6 @@ Tools aimed at the interaction between Tahoe-LAFS implementation and Eliot. Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals - -from __future__ import ( - unicode_literals, - print_function, - absolute_import, - division, -) __all__ = [ "MemoryLogger", @@ -26,11 +15,6 @@ __all__ = [ "capture_logging", ] -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 -from six import ensure_text - from sys import ( stdout, ) @@ -42,6 +26,7 @@ from logging import ( ) from json import loads +from six import ensure_text from zope.interface import ( implementer, ) @@ -65,7 +50,7 @@ from eliot.testing import ( MemoryLogger, capture_logging, ) -from eliot.json import EliotJSONEncoder as eliot_json_encoder +from eliot.json import EliotJSONEncoder from eliot._validation import ( ValidationError, @@ -94,6 +79,15 @@ from twisted.internet.defer import ( from twisted.application.service import Service +class BytesEliotJSONEncoder(EliotJSONEncoder): + """Support encoding bytes.""" + + def default(self, o): + if isinstance(o, bytes): + return o.decode("utf-8", "backslashreplace") + return EliotJSONEncoder.default(self, o) + + def validateInstanceOf(t): """ Return an Eliot validator that requires values to be instances of ``t``. @@ -310,7 +304,7 @@ class _DestinationParser(object): rotateLength=rotate_length, maxRotatedFiles=max_rotated_files, ) - return lambda reactor: FileDestination(get_file(), eliot_json_encoder) + return lambda reactor: FileDestination(get_file(), encoder=BytesEliotJSONEncoder) _parse_destination_description = _DestinationParser().parse From ffe0979ad5533aa29e99c029230da577234e9620 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 15 Nov 2023 12:14:47 -0500 Subject: [PATCH 2176/2309] fail* methods have been removed in 3.12 (deprecated since 3.1!) --- src/allmydata/test/common.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/allmydata/test/common.py b/src/allmydata/test/common.py index 1186bd540..d627dce6a 100644 --- a/src/allmydata/test/common.py +++ b/src/allmydata/test/common.py @@ -1352,6 +1352,14 @@ class _TestCaseMixin(object): def assertRaises(self, *a, **kw): return self._dummyCase.assertRaises(*a, **kw) + def failUnless(self, *args, **kwargs): + """Backwards compatibility method.""" + self.assertTrue(*args, **kwargs) + + def failIf(self, *args, **kwargs): + """Backwards compatibility method.""" + self.assertFalse(*args, **kwargs) + class SyncTestCase(_TestCaseMixin, TestCase): """ From def7014c792663a14d29d7755328c868f012ef30 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 15 Nov 2023 12:15:06 -0500 Subject: [PATCH 2177/2309] json dumping doesn't like keys that are bytes (not sure how this ever worked) --- src/allmydata/client.py | 4 ++++ src/allmydata/storage_client.py | 1 - 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/allmydata/client.py b/src/allmydata/client.py index dd3c912de..03bf609e9 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -12,6 +12,7 @@ from base64 import urlsafe_b64encode from functools import partial from configparser import NoSectionError +from six import ensure_text from foolscap.furl import ( decode_furl, ) @@ -989,6 +990,9 @@ class _Client(node.Node, pollmixin.PollMixin): static_servers = servers_yaml.get("storage", {}) log.msg("found %d static servers in private/servers.yaml" % len(static_servers)) + static_servers = { + ensure_text(key): value for (key, value) in static_servers.items() + } self.storage_broker.set_static_servers(static_servers) except EnvironmentError: pass diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index ae7ea7ca1..2cbff82c6 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -273,7 +273,6 @@ class StorageFarmBroker(service.MultiService): # doesn't really matter but it makes the logging behavior more # predictable and easier to test (and at least one test does depend on # this sorted order). - servers = {ensure_text(key): value for (key, value) in servers.items()} for (server_id, server) in sorted(servers.items()): try: storage_server = self._make_storage_server( From ffafb5d877025169242f777fef837419e5acefb6 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 15 Nov 2023 12:15:28 -0500 Subject: [PATCH 2178/2309] Update to work with newer Eliot --- src/allmydata/test/test_client.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/allmydata/test/test_client.py b/src/allmydata/test/test_client.py index c0cce2809..f00e08662 100644 --- a/src/allmydata/test/test_client.py +++ b/src/allmydata/test/test_client.py @@ -67,7 +67,7 @@ from allmydata.util import ( configutil, jsonbytes as json, ) -from allmydata.util.eliotutil import capture_logging +from allmydata.util.eliotutil import capture_logging, BytesEliotJSONEncoder from allmydata.util.fileutil import abspath_expanduser_unicode from allmydata.interfaces import IFilesystemNode, IFileNode, \ IImmutableFileNode, IMutableFileNode, IDirectoryNode @@ -850,6 +850,7 @@ class StorageClients(SyncTestCase): actionType=u"storage-client:broker:set-static-servers", succeeded=True, ), + encoder_=BytesEliotJSONEncoder ) def test_static_servers(self, logger): """ @@ -884,6 +885,7 @@ class StorageClients(SyncTestCase): actionType=u"storage-client:broker:make-storage-server", succeeded=False, ), + encoder_=BytesEliotJSONEncoder ) def test_invalid_static_server(self, logger): """ From 5889b80332a627f17d250bd2c25b82661842f4cc Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 15 Nov 2023 12:15:34 -0500 Subject: [PATCH 2179/2309] assert_ has been removed in Python 3.12 --- src/allmydata/test/web/common.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/web/common.py b/src/allmydata/test/web/common.py index 43a13a902..fbf7b015f 100644 --- a/src/allmydata/test/web/common.py +++ b/src/allmydata/test/web/common.py @@ -23,7 +23,7 @@ def assert_soup_has_favicon(testcase, soup): ``BeautifulSoup`` object ``soup`` contains the tahoe favicon link. """ links = soup.find_all(u'link', rel=u'shortcut icon') - testcase.assert_( + testcase.assertTrue( any(t[u'href'] == u'/icon.png' for t in links), soup) @@ -92,6 +92,6 @@ def assert_soup_has_text(testcase, soup, text): ``BeautifulSoup`` object ``soup`` contains the passed in ``text`` anywhere as a text node. """ - testcase.assert_( + testcase.assertTrue( soup.find_all(string=re.compile(re.escape(text))), soup) From 3d753fef939ffa3e50cb25e482bfa701e3118ceb Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 15 Nov 2023 12:51:32 -0500 Subject: [PATCH 2180/2309] Make test less fragile --- src/allmydata/test/test_eliotutil.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/allmydata/test/test_eliotutil.py b/src/allmydata/test/test_eliotutil.py index cabe599b3..a170468a2 100644 --- a/src/allmydata/test/test_eliotutil.py +++ b/src/allmydata/test/test_eliotutil.py @@ -64,7 +64,7 @@ from twisted.internet.task import deferLater from twisted.internet import reactor from ..util.eliotutil import ( - eliot_json_encoder, + BytesEliotJSONEncoder, log_call_deferred, _parse_destination_description, _EliotLogging, @@ -188,8 +188,8 @@ class ParseDestinationDescriptionTests(SyncTestCase): """ reactor = object() self.assertThat( - _parse_destination_description("file:-")(reactor), - Equals(FileDestination(stdout, encoder=eliot_json_encoder)), + _parse_destination_description("file:-")(reactor).file, + Equals(stdout), ) From 984b74ee341ec5d5d8a1e125ef68c58b67d3b25b Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 15 Nov 2023 12:51:43 -0500 Subject: [PATCH 2181/2309] Remove usage of deprecated API --- src/allmydata/test/test_crypto.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/allmydata/test/test_crypto.py b/src/allmydata/test/test_crypto.py index 052ddfbd7..b7c84b447 100644 --- a/src/allmydata/test/test_crypto.py +++ b/src/allmydata/test/test_crypto.py @@ -507,7 +507,7 @@ class TestUtil(unittest.TestCase): """ remove a simple prefix properly """ - self.assertEquals( + self.assertEqual( remove_prefix(b"foobar", b"foo"), b"bar" ) @@ -523,7 +523,7 @@ class TestUtil(unittest.TestCase): """ removing a zero-length prefix does nothing """ - self.assertEquals( + self.assertEqual( remove_prefix(b"foobar", b""), b"foobar", ) @@ -532,7 +532,7 @@ class TestUtil(unittest.TestCase): """ removing a prefix which is the whole string is empty """ - self.assertEquals( + self.assertEqual( remove_prefix(b"foobar", b"foobar"), b"", ) From 16aa5fece221f210dc269fe1a16873cc5957c339 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 15 Nov 2023 12:53:51 -0500 Subject: [PATCH 2182/2309] More backwards compatibility --- src/allmydata/test/common.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/allmydata/test/common.py b/src/allmydata/test/common.py index d627dce6a..bd0feda10 100644 --- a/src/allmydata/test/common.py +++ b/src/allmydata/test/common.py @@ -1360,6 +1360,18 @@ class _TestCaseMixin(object): """Backwards compatibility method.""" self.assertFalse(*args, **kwargs) + def failIfEqual(self, *args, **kwargs): + """Backwards compatibility method.""" + self.assertNotEqual(*args, **kwargs) + + def failUnlessEqual(self, *args, **kwargs): + """Backwards compatibility method.""" + self.assertEqual(*args, **kwargs) + + def failUnlessReallyEqual(self, *args, **kwargs): + """Backwards compatibility method.""" + self.assertReallyEqual(*args, **kwargs) + class SyncTestCase(_TestCaseMixin, TestCase): """ From 0bd402e68066df59b7cf297a0d43f30ae386fbc4 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 15 Nov 2023 13:03:43 -0500 Subject: [PATCH 2183/2309] Handle logging of sets --- src/allmydata/test/test_eliotutil.py | 15 +++++++++++++++ src/allmydata/util/eliotutil.py | 2 ++ 2 files changed, 17 insertions(+) diff --git a/src/allmydata/test/test_eliotutil.py b/src/allmydata/test/test_eliotutil.py index a170468a2..a6ed31b81 100644 --- a/src/allmydata/test/test_eliotutil.py +++ b/src/allmydata/test/test_eliotutil.py @@ -88,6 +88,21 @@ def passes(): return AfterPreprocessing(run, Equals(True)) +class BytesEliotJSONEncoderTests(TestCase): + """Tests for ``BytesEliotJSONEncoder``.""" + + def test_encoding_bytes(self): + """``BytesEliotJSONEncoder`` can encode bytes.""" + encoder = BytesEliotJSONEncoder() + self.assertEqual(encoder.default(b"xxx"), "xxx") + self.assertEqual(encoder.default(bytes([12])), "\x0c") + + def test_encoding_sets(self): + """``BytesEliotJSONEncoder`` can encode sets.""" + encoder = BytesEliotJSONEncoder() + self.assertIn(encoder.default({1, 2}), ([1, 2], [2, 1])) + + class EliotLoggedTestTests(TestCase): """ Tests for the automatic log-related provided by ``AsyncTestCase``. diff --git a/src/allmydata/util/eliotutil.py b/src/allmydata/util/eliotutil.py index ec417991c..db05c6873 100644 --- a/src/allmydata/util/eliotutil.py +++ b/src/allmydata/util/eliotutil.py @@ -85,6 +85,8 @@ class BytesEliotJSONEncoder(EliotJSONEncoder): def default(self, o): if isinstance(o, bytes): return o.decode("utf-8", "backslashreplace") + if isinstance(o, set): + return list(o) return EliotJSONEncoder.default(self, o) From b28b769a6c93e0fe8b5a10d3fdc7ea519747e06e Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 15 Nov 2023 13:09:17 -0500 Subject: [PATCH 2184/2309] Add some coverage for 3.12 --- .github/workflows/ci.yml | 6 ++++-- tox.ini | 3 ++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0f38b0291..e3f1e83df 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -47,7 +47,7 @@ jobs: include: # On macOS don't bother with 3.8, just to get faster builds. - os: macos-12 - python-version: "3.9" + python-version: "3.12" - os: macos-12 python-version: "3.11" # We only support PyPy on Linux at the moment. @@ -55,6 +55,8 @@ jobs: python-version: "pypy-3.8" - os: ubuntu-latest python-version: "pypy-3.9" + - os: ubuntu-latest + python-version: "3.12" steps: # See https://github.com/actions/checkout. A fetch-depth of 0 @@ -169,7 +171,7 @@ jobs: - false include: - os: ubuntu-20.04 - python-version: "3.10" + python-version: "3.12" force-foolscap: true steps: diff --git a/tox.ini b/tox.ini index 11daa75fe..a23fc1a6a 100644 --- a/tox.ini +++ b/tox.ini @@ -11,6 +11,7 @@ python = 3.9: py39-coverage 3.10: py310-coverage 3.11: py311-coverage + 3.12: py312-coverage pypy-3.8: pypy38 pypy-3.9: pypy39 @@ -18,7 +19,7 @@ python = twisted = 1 [tox] -envlist = typechecks,codechecks,py{38,39,310,311}-{coverage},pypy27,pypy38,pypy39,integration +envlist = typechecks,codechecks,py{38,39,310,311,312}-{coverage},pypy27,pypy38,pypy39,integration minversion = 2.4 [testenv] From bdaf9b5e47af6263063e7264c7a94750cef31b8b Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 15 Nov 2023 13:10:20 -0500 Subject: [PATCH 2185/2309] Fix lints --- src/allmydata/storage_client.py | 1 - src/allmydata/test/test_eliotutil.py | 1 - 2 files changed, 2 deletions(-) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 2cbff82c6..b714d7757 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -32,7 +32,6 @@ Ported to Python 3. from __future__ import annotations -from six import ensure_text from typing import Union, Callable, Any, Optional, cast, Dict from os import urandom import re diff --git a/src/allmydata/test/test_eliotutil.py b/src/allmydata/test/test_eliotutil.py index a6ed31b81..2bee11216 100644 --- a/src/allmydata/test/test_eliotutil.py +++ b/src/allmydata/test/test_eliotutil.py @@ -47,7 +47,6 @@ from eliot import ( Message, MessageType, fields, - FileDestination, MemoryLogger, ) from eliot.twisted import DeferredContext From 3ddfb92484ddabe2bc1bb79fdb87d6fd0680e171 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 15 Nov 2023 13:11:03 -0500 Subject: [PATCH 2186/2309] News fragment --- newsfragments/3072.feature | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/3072.feature diff --git a/newsfragments/3072.feature b/newsfragments/3072.feature new file mode 100644 index 000000000..79ce6d56d --- /dev/null +++ b/newsfragments/3072.feature @@ -0,0 +1 @@ +Added support for Python 3.12, and work with Eliot 1.15 \ No newline at end of file From 83fa028925739c8806cdc4b3759abb556656cc6b Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 16 Nov 2023 13:53:51 -0500 Subject: [PATCH 2187/2309] Use the existing Tahoe JSON encoder. --- src/allmydata/test/__init__.py | 4 ++-- src/allmydata/test/eliotutil.py | 6 +++--- src/allmydata/test/test_client.py | 6 +++--- src/allmydata/test/test_eliotutil.py | 16 ---------------- src/allmydata/test/web/test_logs.py | 2 +- src/allmydata/util/eliotutil.py | 14 ++------------ src/allmydata/util/jsonbytes.py | 6 ++++++ 7 files changed, 17 insertions(+), 37 deletions(-) diff --git a/src/allmydata/test/__init__.py b/src/allmydata/test/__init__.py index bca4a4ebf..893aa15ce 100644 --- a/src/allmydata/test/__init__.py +++ b/src/allmydata/test/__init__.py @@ -125,5 +125,5 @@ if sys.platform == "win32": initialize() from eliot import to_file -from allmydata.util.eliotutil import BytesEliotJSONEncoder -to_file(open("eliot.log", "wb"), encoder=BytesEliotJSONEncoder) +from allmydata.util.jsonbytes import AnyBytesJSONEncoder +to_file(open("eliot.log", "wb"), encoder=AnyBytesJSONEncoder) diff --git a/src/allmydata/test/eliotutil.py b/src/allmydata/test/eliotutil.py index 8a2f63203..b1351abf0 100644 --- a/src/allmydata/test/eliotutil.py +++ b/src/allmydata/test/eliotutil.py @@ -38,8 +38,8 @@ from twisted.python.monkey import ( MonkeyPatcher, ) -from ..util.eliotutil import ( - BytesEliotJSONEncoder +from ..util.jsonbytes import ( + AnyBytesJSONEncoder ) _NAME = Field.for_types( @@ -147,7 +147,7 @@ def with_logging( """ @wraps(test_method) def run_with_logging(*args, **kwargs): - validating_logger = MemoryLogger(encoder=BytesEliotJSONEncoder) + validating_logger = MemoryLogger(encoder=AnyBytesJSONEncoder) original = swap_logger(None) try: swap_logger(_TwoLoggers(original, validating_logger)) diff --git a/src/allmydata/test/test_client.py b/src/allmydata/test/test_client.py index f00e08662..57748d5fa 100644 --- a/src/allmydata/test/test_client.py +++ b/src/allmydata/test/test_client.py @@ -67,7 +67,7 @@ from allmydata.util import ( configutil, jsonbytes as json, ) -from allmydata.util.eliotutil import capture_logging, BytesEliotJSONEncoder +from allmydata.util.eliotutil import capture_logging from allmydata.util.fileutil import abspath_expanduser_unicode from allmydata.interfaces import IFilesystemNode, IFileNode, \ IImmutableFileNode, IMutableFileNode, IDirectoryNode @@ -850,7 +850,7 @@ class StorageClients(SyncTestCase): actionType=u"storage-client:broker:set-static-servers", succeeded=True, ), - encoder_=BytesEliotJSONEncoder + encoder_=json.AnyBytesJSONEncoder ) def test_static_servers(self, logger): """ @@ -885,7 +885,7 @@ class StorageClients(SyncTestCase): actionType=u"storage-client:broker:make-storage-server", succeeded=False, ), - encoder_=BytesEliotJSONEncoder + encoder_=json.AnyBytesJSONEncoder ) def test_invalid_static_server(self, logger): """ diff --git a/src/allmydata/test/test_eliotutil.py b/src/allmydata/test/test_eliotutil.py index 2bee11216..52d709e4c 100644 --- a/src/allmydata/test/test_eliotutil.py +++ b/src/allmydata/test/test_eliotutil.py @@ -63,7 +63,6 @@ from twisted.internet.task import deferLater from twisted.internet import reactor from ..util.eliotutil import ( - BytesEliotJSONEncoder, log_call_deferred, _parse_destination_description, _EliotLogging, @@ -87,21 +86,6 @@ def passes(): return AfterPreprocessing(run, Equals(True)) -class BytesEliotJSONEncoderTests(TestCase): - """Tests for ``BytesEliotJSONEncoder``.""" - - def test_encoding_bytes(self): - """``BytesEliotJSONEncoder`` can encode bytes.""" - encoder = BytesEliotJSONEncoder() - self.assertEqual(encoder.default(b"xxx"), "xxx") - self.assertEqual(encoder.default(bytes([12])), "\x0c") - - def test_encoding_sets(self): - """``BytesEliotJSONEncoder`` can encode sets.""" - encoder = BytesEliotJSONEncoder() - self.assertIn(encoder.default({1, 2}), ([1, 2], [2, 1])) - - class EliotLoggedTestTests(TestCase): """ Tests for the automatic log-related provided by ``AsyncTestCase``. diff --git a/src/allmydata/test/web/test_logs.py b/src/allmydata/test/web/test_logs.py index 81ec357c0..a8b479a97 100644 --- a/src/allmydata/test/web/test_logs.py +++ b/src/allmydata/test/web/test_logs.py @@ -117,7 +117,7 @@ class TestStreamingLogs(AsyncTestCase): proto.transport.loseConnection() yield proto.is_closed - self.assertThat(len(messages), Equals(3)) + self.assertThat(len(messages), Equals(3), messages) self.assertThat(messages[0]["action_type"], Equals("test:cli:some-exciting-action")) self.assertThat(messages[0]["arguments"], Equals(["hello", "good-\\xff-day", 123, {"a": 35}, [None]])) diff --git a/src/allmydata/util/eliotutil.py b/src/allmydata/util/eliotutil.py index db05c6873..94d34f96f 100644 --- a/src/allmydata/util/eliotutil.py +++ b/src/allmydata/util/eliotutil.py @@ -50,7 +50,6 @@ from eliot.testing import ( MemoryLogger, capture_logging, ) -from eliot.json import EliotJSONEncoder from eliot._validation import ( ValidationError, @@ -78,16 +77,7 @@ from twisted.internet.defer import ( ) from twisted.application.service import Service - -class BytesEliotJSONEncoder(EliotJSONEncoder): - """Support encoding bytes.""" - - def default(self, o): - if isinstance(o, bytes): - return o.decode("utf-8", "backslashreplace") - if isinstance(o, set): - return list(o) - return EliotJSONEncoder.default(self, o) +from .jsonbytes import AnyBytesJSONEncoder def validateInstanceOf(t): @@ -306,7 +296,7 @@ class _DestinationParser(object): rotateLength=rotate_length, maxRotatedFiles=max_rotated_files, ) - return lambda reactor: FileDestination(get_file(), encoder=BytesEliotJSONEncoder) + return lambda reactor: FileDestination(get_file(), encoder=AnyBytesJSONEncoder) _parse_destination_description = _DestinationParser().parse diff --git a/src/allmydata/util/jsonbytes.py b/src/allmydata/util/jsonbytes.py index 4a1813275..ea95bb5b8 100644 --- a/src/allmydata/util/jsonbytes.py +++ b/src/allmydata/util/jsonbytes.py @@ -61,6 +61,9 @@ class UTF8BytesJSONEncoder(json.JSONEncoder): """ A JSON encoder than can also encode UTF-8 encoded strings. """ + def default(self, o): + return bytes_to_unicode(False, o) + def encode(self, o, **kwargs): return json.JSONEncoder.encode( self, bytes_to_unicode(False, o), **kwargs) @@ -77,6 +80,9 @@ class AnyBytesJSONEncoder(json.JSONEncoder): Bytes are decoded to strings using UTF-8, if that fails to decode then the bytes are quoted. """ + def default(self, o): + return bytes_to_unicode(True, o) + def encode(self, o, **kwargs): return json.JSONEncoder.encode( self, bytes_to_unicode(True, o), **kwargs) From 7b680b36ef1c600d97ef3fad4d5fd385d087c001 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 16 Nov 2023 17:54:24 -0500 Subject: [PATCH 2188/2309] Make setuptools a dependency --- pyproject.toml | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 pyproject.toml diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 000000000..fed528d4a --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,3 @@ +[build-system] +requires = ["setuptools"] +build-backend = "setuptools.build_meta" From c2b1f3168455d1231c3c669d1ff428b762679dac Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 16 Nov 2023 17:54:34 -0500 Subject: [PATCH 2189/2309] Work with tox 4 --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index a23fc1a6a..1c4604091 100644 --- a/tox.ini +++ b/tox.ini @@ -23,7 +23,7 @@ envlist = typechecks,codechecks,py{38,39,310,311,312}-{coverage},pypy27,pypy38,p minversion = 2.4 [testenv] -passenv = TAHOE_LAFS_* PIP_* SUBUNITREPORTER_* USERPROFILE HOMEDRIVE HOMEPATH +passenv = TAHOE_LAFS_*, PIP_*, SUBUNITREPORTER_*, USERPROFILE, HOMEDRIVE, HOMEPATH deps = # We pull in certify *here* to avoid bug #2913. Basically if a # `setup_requires=...` causes a package to be installed (with setuptools) From e276b3a9bedff0ac42df4bb4d3428d5572516834 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 17 Nov 2023 10:08:00 -0500 Subject: [PATCH 2190/2309] Increase setuptools and pip versions --- setup.py | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/setup.py b/setup.py index e8633062f..3172175e7 100644 --- a/setup.py +++ b/setup.py @@ -32,10 +32,6 @@ VERSION_PY_FILENAME = 'src/allmydata/_version.py' version = read_version_py(VERSION_PY_FILENAME) install_requires = [ - # we don't need much out of setuptools but the version checking stuff - # needs pkg_resources and PEP 440 version specifiers. - "setuptools >= 28.8.0", - "zfec >= 1.1.0", # zope.interface >= 3.6.0 is required for Twisted >= 12.1.0. @@ -158,10 +154,6 @@ install_requires = [ "filelock", ] -setup_requires = [ - 'setuptools >= 28.8.0', # for PEP-440 style versions -] - tor_requires = [ # 23.5 added support for custom TLS contexts in web_agent(), which is # needed for the HTTP storage client to run over Tor. @@ -410,9 +402,9 @@ setup(name="tahoe-lafs", # also set in __init__.py # selected here are just the current versions at the time. # Bumping them to keep up with future releases is fine as long # as those releases are known to actually work. - "pip==22.0.3", - "wheel==0.37.1", - "setuptools==60.9.1", + "pip==23.3.1", + "wheel==0.41.3", + "setuptools==68.2.2", "subunitreporter==23.8.0", "python-subunit==1.4.2", "junitxml==0.7", @@ -448,7 +440,6 @@ setup(name="tahoe-lafs", # also set in __init__.py "allmydata": ["ported-modules.txt"], }, include_package_data=True, - setup_requires=setup_requires, entry_points={ 'console_scripts': [ 'tahoe = allmydata.scripts.runner:run', From 0525b737630f200097f8e802f9ff403f76ad56fe Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 17 Nov 2023 10:14:44 -0500 Subject: [PATCH 2191/2309] Await coroutines! --- src/allmydata/test/test_multi_introducers.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/src/allmydata/test/test_multi_introducers.py b/src/allmydata/test/test_multi_introducers.py index a385abe54..2b0879530 100644 --- a/src/allmydata/test/test_multi_introducers.py +++ b/src/allmydata/test/test_multi_introducers.py @@ -28,14 +28,14 @@ INTRODUCERS_CFG_FURLS_COMMENTED="""introducers: class MultiIntroTests(unittest.TestCase): - def setUp(self): + async def setUp(self): # setup tahoe.cfg and basedir/private/introducers # create a custom tahoe.cfg self.basedir = os.path.dirname(self.mktemp()) c = open(os.path.join(self.basedir, "tahoe.cfg"), "w") config = {'hide-ip':False, 'listen': 'tcp', 'port': None, 'location': None, 'hostname': 'example.net'} - write_node_config(c, config) + await write_node_config(c, config) c.write("[storage]\n") c.write("enabled = false\n") c.close() @@ -63,8 +63,7 @@ class MultiIntroTests(unittest.TestCase): # assertions self.failUnlessEqual(ic_count, len(connections["introducers"])) - @defer.inlineCallbacks - def test_read_introducer_furl_from_tahoecfg(self): + async def test_read_introducer_furl_from_tahoecfg(self): """ The deprecated [client]introducer.furl item is still read and respected. """ @@ -72,7 +71,7 @@ class MultiIntroTests(unittest.TestCase): c = open(os.path.join(self.basedir, "tahoe.cfg"), "w") config = {'hide-ip':False, 'listen': 'tcp', 'port': None, 'location': None, 'hostname': 'example.net'} - write_node_config(c, config) + await write_node_config(c, config) fake_furl = "furl1" c.write("[client]\n") c.write("introducer.furl = %s\n" % fake_furl) @@ -139,14 +138,14 @@ introducers: """ class NoDefault(unittest.TestCase): - def setUp(self): + async def setUp(self): # setup tahoe.cfg and basedir/private/introducers # create a custom tahoe.cfg self.basedir = os.path.dirname(self.mktemp()) c = open(os.path.join(self.basedir, "tahoe.cfg"), "w") config = {'hide-ip':False, 'listen': 'tcp', 'port': None, 'location': None, 'hostname': 'example.net'} - write_node_config(c, config) + await write_node_config(c, config) c.write("[storage]\n") c.write("enabled = false\n") c.close() From fbbf1cf98a3943e660f6c57812837c2078cea67d Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 17 Nov 2023 10:46:41 -0500 Subject: [PATCH 2192/2309] Stop using pkg_resources --- misc/build_helpers/show-tool-versions.py | 24 +++++------------------- 1 file changed, 5 insertions(+), 19 deletions(-) diff --git a/misc/build_helpers/show-tool-versions.py b/misc/build_helpers/show-tool-versions.py index f70183ae1..4a85207f5 100644 --- a/misc/build_helpers/show-tool-versions.py +++ b/misc/build_helpers/show-tool-versions.py @@ -1,8 +1,7 @@ #! /usr/bin/env python -from __future__ import print_function - import locale, os, platform, subprocess, sys, traceback +from importlib.metadata import version, PackageNotFoundError def foldlines(s, numlines=None): @@ -72,17 +71,10 @@ def print_as_ver(): traceback.print_exc(file=sys.stderr) sys.stderr.flush() - def print_setuptools_ver(): try: - import pkg_resources - out = str(pkg_resources.require("setuptools")) - print("setuptools:", foldlines(out)) - except (ImportError, EnvironmentError): - sys.stderr.write("\nGot exception using 'pkg_resources' to get the version of setuptools. Exception follows\n") - traceback.print_exc(file=sys.stderr) - sys.stderr.flush() - except pkg_resources.DistributionNotFound: + print("setuptools:", version("setuptools")) + except PackageNotFoundError: print('setuptools: DistributionNotFound') @@ -91,14 +83,8 @@ def print_py_pkg_ver(pkgname, modulename=None): modulename = pkgname print() try: - import pkg_resources - out = str(pkg_resources.require(pkgname)) - print(pkgname + ': ' + foldlines(out)) - except (ImportError, EnvironmentError): - sys.stderr.write("\nGot exception using 'pkg_resources' to get the version of %s. Exception follows.\n" % (pkgname,)) - traceback.print_exc(file=sys.stderr) - sys.stderr.flush() - except pkg_resources.DistributionNotFound: + print(pkgname + ': ' + version(pkgname)) + except PackageNotFoundError: print(pkgname + ': DistributionNotFound') try: __import__(modulename) From 4daa584449ce3ec7975d3227e41b4d26f82c1a11 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 17 Nov 2023 10:54:33 -0500 Subject: [PATCH 2193/2309] Get rid of pkg_resources --- src/allmydata/web/root.py | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/src/allmydata/web/root.py b/src/allmydata/web/root.py index f1a8569d6..642912279 100644 --- a/src/allmydata/web/root.py +++ b/src/allmydata/web/root.py @@ -1,21 +1,13 @@ """ Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals - -from future.utils import PY2, PY3 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - import os import time from urllib.parse import quote as urlquote +from importlib.resources import files as resource_files, as_file +from contextlib import ExitStack from hyperlink import DecodedURL, URL -from pkg_resources import resource_filename from twisted.web import ( http, resource, @@ -251,15 +243,23 @@ class Root(MultiFormatResource): self.putChild(b"named", FileHandler(client)) self.putChild(b"status", status.Status(client.get_history())) self.putChild(b"statistics", status.Statistics(client.stats_provider)) - static_dir = resource_filename("allmydata.web", "static") - for filen in os.listdir(static_dir): - child_path = filen - if PY3: - child_path = filen.encode("utf-8") - self.putChild(child_path, static.File(os.path.join(static_dir, filen))) + + # Package resources may be on the filesystem, or they may be in a zip + # or something, so we need to do a bit more work to serve them as + # static files. + self._temporary_file_manager = ExitStack() + static_dir = resource_files("allmydata.web") / "static" + for child in static_dir.iterdir(): + child_path = child.name.encode("utf-8") + self.putChild(child_path, static.File( + self._temporary_file_manager.enter_context(as_file(child)) + )) self.putChild(b"report_incident", IncidentReporter()) + def __del__(self): + self._temporary_file_manager.close() + @exception_to_child def getChild(self, path, request): if not path: From e68d0fccb1970ced11edc23bb0d893e17e061f2d Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 17 Nov 2023 10:59:40 -0500 Subject: [PATCH 2194/2309] Tahoe shouldn't need to depend on setuptools --- src/allmydata/_auto_deps.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/allmydata/_auto_deps.py b/src/allmydata/_auto_deps.py index 521b17a45..424a4bf4c 100644 --- a/src/allmydata/_auto_deps.py +++ b/src/allmydata/_auto_deps.py @@ -39,7 +39,6 @@ package_imports = [ ('pycparser', 'pycparser'), ('PyYAML', 'yaml'), ('magic-wormhole', 'wormhole'), - ('setuptools', 'setuptools'), ('eliot', 'eliot'), ('attrs', 'attr'), ('autobahn', 'autobahn'), From 95341a2c07ae8847ab22e62b2f59ff264bea96c1 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 17 Nov 2023 11:07:40 -0500 Subject: [PATCH 2195/2309] Generalize static file setup. --- src/allmydata/web/common.py | 28 ++++++++++++++++++++++++++++ src/allmydata/web/introweb.py | 18 +++--------------- src/allmydata/web/root.py | 18 ++---------------- 3 files changed, 33 insertions(+), 31 deletions(-) diff --git a/src/allmydata/web/common.py b/src/allmydata/web/common.py index 1a0ba433b..2a3a9ea1c 100644 --- a/src/allmydata/web/common.py +++ b/src/allmydata/web/common.py @@ -4,6 +4,8 @@ Ported to Python 3. from __future__ import annotations from six import ensure_str +from importlib.resources import files as resource_files, as_file +from contextlib import ExitStack from typing import Optional, Union, TypeVar, overload from typing_extensions import Literal @@ -29,6 +31,7 @@ from twisted.web import ( http, resource, template, + static, ) from twisted.web.iweb import ( IRequest, @@ -852,3 +855,28 @@ def get_keypair(request: IRequest) -> tuple[PublicKey, PrivateKey] | None: return None privkey, pubkey = create_signing_keypair_from_string(urlsafe_b64decode(privkey_der)) return pubkey, privkey + + +class StaticFiles: + """ + Serve static files includes as resources. + + Package resources may be on the filesystem, or they may be in a zip + or something, so we need to do a bit more work to serve them as + static files. + """ + + def __init__(self): + self._temporary_file_manager = ExitStack() + + @classmethod + def add_static_children(cls, root: IResource): + """Add static files from C{allmydata.web} to the given resource.""" + self = cls() + static_dir = resource_files("allmydata.web") / "static" + for child in static_dir.iterdir(): + child_path = child.name.encode("utf-8") + root.putChild(child_path, static.File( + self._temporary_file_manager.enter_context(as_file(child)) + )) + root.__static_files_cleanup = self diff --git a/src/allmydata/web/introweb.py b/src/allmydata/web/introweb.py index 621a15a5c..880ff66e5 100644 --- a/src/allmydata/web/introweb.py +++ b/src/allmydata/web/introweb.py @@ -1,26 +1,16 @@ """ Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals - -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - -import time, os -from pkg_resources import resource_filename +import time from twisted.web.template import Element, XMLFile, renderElement, renderer from twisted.python.filepath import FilePath -from twisted.web import static import allmydata from allmydata.util import idlib, jsonbytes as json from allmydata.web.common import ( render_time, MultiFormatResource, SlotsSequenceElement, + StaticFiles ) @@ -38,9 +28,7 @@ class IntroducerRoot(MultiFormatResource): self.introducer_service = introducer_node.getServiceNamed("introducer") # necessary as a root Resource self.putChild(b"", self) - static_dir = resource_filename("allmydata.web", "static") - for filen in os.listdir(static_dir): - self.putChild(filen.encode("utf-8"), static.File(os.path.join(static_dir, filen))) + StaticFiles.add_static_children(self) def _create_element(self): """ diff --git a/src/allmydata/web/root.py b/src/allmydata/web/root.py index 642912279..d8ef360c9 100644 --- a/src/allmydata/web/root.py +++ b/src/allmydata/web/root.py @@ -4,8 +4,6 @@ Ported to Python 3. import os import time from urllib.parse import quote as urlquote -from importlib.resources import files as resource_files, as_file -from contextlib import ExitStack from hyperlink import DecodedURL, URL from twisted.web import ( @@ -46,6 +44,7 @@ from allmydata.web.common import ( render_time_delta, render_time, render_time_attr, + StaticFiles, ) from allmydata.web.private import ( create_private_tree, @@ -243,22 +242,9 @@ class Root(MultiFormatResource): self.putChild(b"named", FileHandler(client)) self.putChild(b"status", status.Status(client.get_history())) self.putChild(b"statistics", status.Statistics(client.stats_provider)) - - # Package resources may be on the filesystem, or they may be in a zip - # or something, so we need to do a bit more work to serve them as - # static files. - self._temporary_file_manager = ExitStack() - static_dir = resource_files("allmydata.web") / "static" - for child in static_dir.iterdir(): - child_path = child.name.encode("utf-8") - self.putChild(child_path, static.File( - self._temporary_file_manager.enter_context(as_file(child)) - )) - self.putChild(b"report_incident", IncidentReporter()) - def __del__(self): - self._temporary_file_manager.close() + StaticFiles.add_static_children(self) @exception_to_child def getChild(self, path, request): From 7d43eb76d9123a420917c2ea28caa3d43a8ce187 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 17 Nov 2023 11:08:18 -0500 Subject: [PATCH 2196/2309] Shouldn't need setuptools for runtime anymore --- setup.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/setup.py b/setup.py index c011d7389..97a32fa56 100644 --- a/setup.py +++ b/setup.py @@ -32,10 +32,6 @@ VERSION_PY_FILENAME = 'src/allmydata/_version.py' version = read_version_py(VERSION_PY_FILENAME) install_requires = [ - # we don't need much out of setuptools but the version checking stuff - # needs pkg_resources and PEP 440 version specifiers. - "setuptools >= 28.8.0", - "zfec >= 1.1.0", # zope.interface >= 3.6.0 is required for Twisted >= 12.1.0. @@ -412,7 +408,6 @@ setup(name="tahoe-lafs", # also set in __init__.py # as those releases are known to actually work. "pip==22.0.3", "wheel==0.37.1", - "setuptools==60.9.1", "subunitreporter==23.8.0", "python-subunit==1.4.2", "junitxml==0.7", From fea816c0b741f28fea9297f95c10e9c16796de2f Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 17 Nov 2023 11:09:02 -0500 Subject: [PATCH 2197/2309] News fragment --- newsfragments/4074.misc | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/4074.misc diff --git a/newsfragments/4074.misc b/newsfragments/4074.misc new file mode 100644 index 000000000..e69de29bb From c941146f43dc91d566a02a9de815d33ef10a317c Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 17 Nov 2023 11:09:35 -0500 Subject: [PATCH 2198/2309] Fix lints --- src/allmydata/web/root.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/allmydata/web/root.py b/src/allmydata/web/root.py index d8ef360c9..5ebac3f0a 100644 --- a/src/allmydata/web/root.py +++ b/src/allmydata/web/root.py @@ -1,7 +1,6 @@ """ Ported to Python 3. """ -import os import time from urllib.parse import quote as urlquote @@ -9,7 +8,6 @@ from hyperlink import DecodedURL, URL from twisted.web import ( http, resource, - static, ) from twisted.web.util import redirectTo, Redirect from twisted.python.filepath import FilePath From ccc35bf7cd8eb3dce53ac791dff6176c82083d49 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 17 Nov 2023 11:10:34 -0500 Subject: [PATCH 2199/2309] Rename --- newsfragments/{4074.misc => 4074.minor} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename newsfragments/{4074.misc => 4074.minor} (100%) diff --git a/newsfragments/4074.misc b/newsfragments/4074.minor similarity index 100% rename from newsfragments/4074.misc rename to newsfragments/4074.minor From e6630b59f7272d448796ab814672b363c52f2807 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 17 Nov 2023 11:18:28 -0500 Subject: [PATCH 2200/2309] Fix mypy complaints, simplifying code while we're at it --- src/allmydata/storage/http_server.py | 2 +- src/allmydata/web/common.py | 29 +++++++++++----------------- src/allmydata/web/introweb.py | 4 ++-- src/allmydata/web/root.py | 4 ++-- 4 files changed, 16 insertions(+), 23 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 5b4e02288..78c5ac5d2 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -695,7 +695,7 @@ class HTTPServer(BaseApp): if accept.best == CBOR_MIME_TYPE: request.setHeader("Content-Type", CBOR_MIME_TYPE) f = TemporaryFile() - cbor2.dump(data, f) + cbor2.dump(data, f) # type: ignore def read_data(offset: int, length: int) -> bytes: f.seek(offset) diff --git a/src/allmydata/web/common.py b/src/allmydata/web/common.py index 2a3a9ea1c..982be491a 100644 --- a/src/allmydata/web/common.py +++ b/src/allmydata/web/common.py @@ -6,7 +6,7 @@ from __future__ import annotations from six import ensure_str from importlib.resources import files as resource_files, as_file from contextlib import ExitStack - +import weakref from typing import Optional, Union, TypeVar, overload from typing_extensions import Literal @@ -857,26 +857,19 @@ def get_keypair(request: IRequest) -> tuple[PublicKey, PrivateKey] | None: return pubkey, privkey -class StaticFiles: +def add_static_children(root: IResource): """ - Serve static files includes as resources. + Add static files from C{allmydata.web} to the given resource. Package resources may be on the filesystem, or they may be in a zip or something, so we need to do a bit more work to serve them as static files. """ - - def __init__(self): - self._temporary_file_manager = ExitStack() - - @classmethod - def add_static_children(cls, root: IResource): - """Add static files from C{allmydata.web} to the given resource.""" - self = cls() - static_dir = resource_files("allmydata.web") / "static" - for child in static_dir.iterdir(): - child_path = child.name.encode("utf-8") - root.putChild(child_path, static.File( - self._temporary_file_manager.enter_context(as_file(child)) - )) - root.__static_files_cleanup = self + temporary_file_manager = ExitStack() + static_dir = resource_files("allmydata.web") / "static" + for child in static_dir.iterdir(): + child_path = child.name.encode("utf-8") + root.putChild(child_path, static.File( + temporary_file_manager.enter_context(as_file(child)) + )) + weakref.finalize(root, temporary_file_manager.close) diff --git a/src/allmydata/web/introweb.py b/src/allmydata/web/introweb.py index 880ff66e5..7cb74a1c1 100644 --- a/src/allmydata/web/introweb.py +++ b/src/allmydata/web/introweb.py @@ -10,7 +10,7 @@ from allmydata.web.common import ( render_time, MultiFormatResource, SlotsSequenceElement, - StaticFiles + add_static_children, ) @@ -28,7 +28,7 @@ class IntroducerRoot(MultiFormatResource): self.introducer_service = introducer_node.getServiceNamed("introducer") # necessary as a root Resource self.putChild(b"", self) - StaticFiles.add_static_children(self) + add_static_children(self) def _create_element(self): """ diff --git a/src/allmydata/web/root.py b/src/allmydata/web/root.py index 5ebac3f0a..090f706f5 100644 --- a/src/allmydata/web/root.py +++ b/src/allmydata/web/root.py @@ -42,7 +42,7 @@ from allmydata.web.common import ( render_time_delta, render_time, render_time_attr, - StaticFiles, + add_static_children, ) from allmydata.web.private import ( create_private_tree, @@ -242,7 +242,7 @@ class Root(MultiFormatResource): self.putChild(b"statistics", status.Statistics(client.stats_provider)) self.putChild(b"report_incident", IncidentReporter()) - StaticFiles.add_static_children(self) + add_static_children(self) @exception_to_child def getChild(self, path, request): From 57534facc5791fe25eed591b55be6d946db3687e Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 17 Nov 2023 11:26:52 -0500 Subject: [PATCH 2201/2309] Python 3.8 support for importlib.resources.files --- setup.py | 3 +++ src/allmydata/web/common.py | 7 ++++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 97a32fa56..e093e1f95 100644 --- a/setup.py +++ b/setup.py @@ -32,6 +32,9 @@ VERSION_PY_FILENAME = 'src/allmydata/_version.py' version = read_version_py(VERSION_PY_FILENAME) install_requires = [ + # importlib.resources.files and friends are new in Python 3.9. + "importlib_resources; python_version < '3.9'", + "zfec >= 1.1.0", # zope.interface >= 3.6.0 is required for Twisted >= 12.1.0. diff --git a/src/allmydata/web/common.py b/src/allmydata/web/common.py index 982be491a..93aba55f7 100644 --- a/src/allmydata/web/common.py +++ b/src/allmydata/web/common.py @@ -4,7 +4,12 @@ Ported to Python 3. from __future__ import annotations from six import ensure_str -from importlib.resources import files as resource_files, as_file +try: + from importlib.resources import files as resource_files, as_file +except ImportError: + import sys + assert sys.version_info[:2] < (3, 9) + from importlib_resources import files as resource_files, as_file from contextlib import ExitStack import weakref from typing import Optional, Union, TypeVar, overload From d28444c2ba391987b594e188d88fcb4ea14b940b Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 17 Nov 2023 11:29:18 -0500 Subject: [PATCH 2202/2309] Don't use latest eliot for now --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index e093e1f95..66c2ad7fd 100644 --- a/setup.py +++ b/setup.py @@ -112,7 +112,7 @@ install_requires = [ "magic-wormhole >= 0.10.2", # We want a new enough version to support custom JSON encoders. - "eliot >= 1.13.0", + "eliot < 1.15.0", # temporary cap, to be fixed in PR #1344 "pyrsistent", From 457b4895b5eb8ce1c5082cb992330f02545f9098 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 17 Nov 2023 11:47:00 -0500 Subject: [PATCH 2203/2309] Pacify mypy when running with newer Python --- src/allmydata/web/common.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/allmydata/web/common.py b/src/allmydata/web/common.py index 93aba55f7..73eaeaef3 100644 --- a/src/allmydata/web/common.py +++ b/src/allmydata/web/common.py @@ -4,11 +4,10 @@ Ported to Python 3. from __future__ import annotations from six import ensure_str -try: +import sys +if sys.version_info[:2] >= (3, 9): from importlib.resources import files as resource_files, as_file -except ImportError: - import sys - assert sys.version_info[:2] < (3, 9) +else: from importlib_resources import files as resource_files, as_file from contextlib import ExitStack import weakref From ddd5fd9d81848c2377894e873d5be65ffdf68899 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 20 Nov 2023 08:30:10 -0500 Subject: [PATCH 2204/2309] Not actually used anywhere --- src/allmydata/_auto_deps.py | 88 ------------------------------------- 1 file changed, 88 deletions(-) delete mode 100644 src/allmydata/_auto_deps.py diff --git a/src/allmydata/_auto_deps.py b/src/allmydata/_auto_deps.py deleted file mode 100644 index 424a4bf4c..000000000 --- a/src/allmydata/_auto_deps.py +++ /dev/null @@ -1,88 +0,0 @@ -""" -Ported to Python 3. -""" - -from __future__ import unicode_literals -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - -# Note: please minimize imports in this file. In particular, do not import -# any module from Tahoe-LAFS or its dependencies, and do not import any -# modules at all at global level. That includes setuptools and pkg_resources. -# It is ok to import modules from the Python Standard Library if they are -# always available, or the import is protected by try...except ImportError. - -# Includes some indirect dependencies, but does not include allmydata. -# These are in the order they should be listed by --version, etc. -package_imports = [ - # package name module name - ('foolscap', 'foolscap'), - ('zfec', 'zfec'), - ('Twisted', 'twisted'), - ('zope.interface', 'zope.interface'), - ('python', None), - ('platform', None), - ('pyOpenSSL', 'OpenSSL'), - ('OpenSSL', None), - ('pyasn1', 'pyasn1'), - ('service-identity', 'service_identity'), - ('pyasn1-modules', 'pyasn1_modules'), - ('cryptography', 'cryptography'), - ('cffi', 'cffi'), - ('six', 'six'), - ('enum34', 'enum'), - ('pycparser', 'pycparser'), - ('PyYAML', 'yaml'), - ('magic-wormhole', 'wormhole'), - ('eliot', 'eliot'), - ('attrs', 'attr'), - ('autobahn', 'autobahn'), -] - -# Dependencies for which we don't know how to get a version number at run-time. -not_import_versionable = [ - 'zope.interface', -] - -# Dependencies reported by pkg_resources that we can safely ignore. -ignorable = [ - 'argparse', - 'distribute', - 'twisted-web', - 'twisted-core', - 'twisted-conch', -] - - -# These are suppressed globally: - -global_deprecation_messages = [ - "BaseException.message has been deprecated as of Python 2.6", - "twisted.internet.interfaces.IFinishableConsumer was deprecated in Twisted 11.1.0: Please use IConsumer (and IConsumer.unregisterProducer) instead.", - "twisted.internet.interfaces.IStreamClientEndpointStringParser was deprecated in Twisted 14.0.0: This interface has been superseded by IStreamClientEndpointStringParserWithReactor.", -] - -# These are suppressed while importing dependencies: - -deprecation_messages = [ - "the sha module is deprecated; use the hashlib module instead", - "object.__new__\(\) takes no parameters", - "The popen2 module is deprecated. Use the subprocess module.", - "the md5 module is deprecated; use hashlib instead", - "twisted.web.error.NoResource is deprecated since Twisted 9.0. See twisted.web.resource.NoResource.", - "the sets module is deprecated", -] - -runtime_warning_messages = [ - "Not using mpz_powm_sec. You should rebuild using libgmp >= 5 to avoid timing attack vulnerability.", -] - -warning_imports = [ - 'twisted.persisted.sob', - 'twisted.python.filepath', -] From 964aba978816cd17eafbbe4d22859a4fee3537fb Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 20 Nov 2023 09:02:43 -0500 Subject: [PATCH 2205/2309] Generate messages that work with stricter Eliot --- src/allmydata/immutable/upload.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/allmydata/immutable/upload.py b/src/allmydata/immutable/upload.py index 36bd86fa6..22210ad0a 100644 --- a/src/allmydata/immutable/upload.py +++ b/src/allmydata/immutable/upload.py @@ -78,7 +78,7 @@ _READONLY_PEERS = Field( def _serialize_existing_shares(existing_shares): return { - server: list(shares) + ensure_str(server): list(shares) for (server, shares) in existing_shares.items() } @@ -91,7 +91,7 @@ _EXISTING_SHARES = Field( def _serialize_happiness_mappings(happiness_mappings): return { - sharenum: base32.b2a(serverid) + str(sharenum): ensure_str(base32.b2a(serverid)) for (sharenum, serverid) in happiness_mappings.items() } @@ -112,7 +112,7 @@ _UPLOAD_TRACKERS = Field( u"upload_trackers", lambda trackers: list( dict( - server=tracker.get_name(), + server=ensure_str(tracker.get_name()), shareids=sorted(tracker.buckets.keys()), ) for tracker @@ -123,7 +123,7 @@ _UPLOAD_TRACKERS = Field( _ALREADY_SERVERIDS = Field( u"already_serverids", - lambda d: d, + lambda d: {str(k): v for k, v in d.items()}, u"Some servers which are already holding some shares that we were interested in uploading.", ) From e5e9cb2b49812d4b1f15f27d8781e950b804af70 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 20 Nov 2023 10:42:55 -0500 Subject: [PATCH 2206/2309] Disable deadlines on 3.12 --- src/allmydata/test/test_storage_http.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index b866f027a..dba7ee5d2 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -26,7 +26,7 @@ from typing import Union, Callable, Tuple, Iterable from queue import Queue from cbor2 import dumps from pycddl import ValidationError as CDDLValidationError -from hypothesis import assume, given, strategies as st +from hypothesis import assume, given, strategies as st, settings as hypothesis_settings from fixtures import Fixture, TempDir, MonkeyPatch from treq.testing import StubTreq from klein import Klein @@ -442,6 +442,9 @@ class CustomHTTPServerTests(SyncTestCase): result_of(client.get_version()) @given(length=st.integers(min_value=1, max_value=1_000_000)) + # On Python 3.12 we're getting weird deadline issues in CI, so disabling + # for now. + @hypothesis_settings(deadline=None) def test_limited_content_fits(self, length): """ ``http_client.limited_content()`` returns the body if it is less than From 6831dfe2c03c8e0b5508c70514f96147ec2378ee Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 20 Nov 2023 10:50:06 -0500 Subject: [PATCH 2207/2309] Run 3.12 Windows tests --- .github/workflows/ci.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e3f1e83df..cadf03040 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -45,11 +45,8 @@ jobs: fail-fast: false matrix: include: - # On macOS don't bother with 3.8, just to get faster builds. - os: macos-12 python-version: "3.12" - - os: macos-12 - python-version: "3.11" # We only support PyPy on Linux at the moment. - os: ubuntu-latest python-version: "pypy-3.8" @@ -57,6 +54,8 @@ jobs: python-version: "pypy-3.9" - os: ubuntu-latest python-version: "3.12" + - os: windows-latest + python-version: "3.12" steps: # See https://github.com/actions/checkout. A fetch-depth of 0 From 8b2b0c7e5b64f62a94494e678d090ba2120f2a84 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 20 Nov 2023 11:13:24 -0500 Subject: [PATCH 2208/2309] Sticking to tox 3 for now --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 1c4604091..a23fc1a6a 100644 --- a/tox.ini +++ b/tox.ini @@ -23,7 +23,7 @@ envlist = typechecks,codechecks,py{38,39,310,311,312}-{coverage},pypy27,pypy38,p minversion = 2.4 [testenv] -passenv = TAHOE_LAFS_*, PIP_*, SUBUNITREPORTER_*, USERPROFILE, HOMEDRIVE, HOMEPATH +passenv = TAHOE_LAFS_* PIP_* SUBUNITREPORTER_* USERPROFILE HOMEDRIVE HOMEPATH deps = # We pull in certify *here* to avoid bug #2913. Basically if a # `setup_requires=...` causes a package to be installed (with setuptools) From 304164ddf8ff9ea2618bfd072d71dc497cefa712 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 20 Nov 2023 16:05:52 -0500 Subject: [PATCH 2209/2309] Try to avoid calling setup.py directly. --- tox.ini | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tox.ini b/tox.ini index a23fc1a6a..202884bb7 100644 --- a/tox.ini +++ b/tox.ini @@ -23,6 +23,11 @@ envlist = typechecks,codechecks,py{38,39,310,311,312}-{coverage},pypy27,pypy38,p minversion = 2.4 [testenv] +# Using setup.py directly is deprecated, so disable options that require that. +# This might be fixed in tox 4, but for now we're using 3.x. +skipsdist = True +usedevelop = False + passenv = TAHOE_LAFS_* PIP_* SUBUNITREPORTER_* USERPROFILE HOMEDRIVE HOMEPATH deps = # We pull in certify *here* to avoid bug #2913. Basically if a @@ -41,10 +46,6 @@ deps = # with the above pins. certifi -# We add usedevelop=False because testing against a true installation gives -# more useful results. -usedevelop = False - extras = # Get general testing environment dependencies so we can run the tests # how we like. From bb3546c9654a9bcc0a7e7da31cf17e1e483abb18 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 20 Nov 2023 16:19:17 -0500 Subject: [PATCH 2210/2309] Try tox 4 again --- .github/workflows/ci.yml | 6 +++--- tox.ini | 13 ++++++------- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cadf03040..845d49e63 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -73,7 +73,7 @@ jobs: - name: Install Python packages run: | - pip install --upgrade "tox<4" tox-gh-actions setuptools + pip install --upgrade tox tox-gh-actions setuptools pip list - name: Display tool versions @@ -205,7 +205,7 @@ jobs: - name: Install Python packages run: | - pip install --upgrade "tox<4" + pip install --upgrade tox pip list - name: Display tool versions @@ -265,7 +265,7 @@ jobs: - name: Install Python packages run: | - pip install --upgrade "tox<4" + pip install --upgrade tox pip list - name: Display tool versions diff --git a/tox.ini b/tox.ini index 202884bb7..00eea4f3b 100644 --- a/tox.ini +++ b/tox.ini @@ -20,15 +20,13 @@ twisted = 1 [tox] envlist = typechecks,codechecks,py{38,39,310,311,312}-{coverage},pypy27,pypy38,pypy39,integration -minversion = 2.4 +minversion = 4 [testenv] -# Using setup.py directly is deprecated, so disable options that require that. -# This might be fixed in tox 4, but for now we're using 3.x. -skipsdist = True +# Install code the real way, for maximum realism. usedevelop = False -passenv = TAHOE_LAFS_* PIP_* SUBUNITREPORTER_* USERPROFILE HOMEDRIVE HOMEPATH +passenv = TAHOE_LAFS_*,PIP_*,SUBUNITREPORTER_*,USERPROFILE,HOMEDRIVE,HOMEPATH deps = # We pull in certify *here* to avoid bug #2913. Basically if a # `setup_requires=...` causes a package to be installed (with setuptools) @@ -83,6 +81,7 @@ commands = coverage: coverage xml [testenv:integration] +usedevelop = False basepython = python3 platform = mylinux: linux mymacos: darwin @@ -142,7 +141,7 @@ commands = mypy src [testenv:draftnews] -passenv = TAHOE_LAFS_* PIP_* SUBUNITREPORTER_* USERPROFILE HOMEDRIVE HOMEPATH +passenv = TAHOE_LAFS_*,PIP_*,SUBUNITREPORTER_*,USERPROFILE,HOMEDRIVE,HOMEPATH deps = # see comment in [testenv] about "certifi" certifi @@ -152,7 +151,7 @@ commands = [testenv:news] # On macOS, git invoked from Tox needs $HOME. -passenv = TAHOE_LAFS_* PIP_* SUBUNITREPORTER_* USERPROFILE HOMEDRIVE HOMEPATH HOME +passenv = TAHOE_LAFS_*,PIP_*,SUBUNITREPORTER_*,USERPROFILE,HOMEDRIVE,HOMEPATH whitelist_externals = git deps = From 4f8269db09c07c409ced901ade0896763c633467 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 20 Nov 2023 16:29:01 -0500 Subject: [PATCH 2211/2309] Use modern versions --- .circleci/create-virtualenv.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/create-virtualenv.sh b/.circleci/create-virtualenv.sh index 7327d0859..05ac64490 100755 --- a/.circleci/create-virtualenv.sh +++ b/.circleci/create-virtualenv.sh @@ -46,8 +46,8 @@ export PIP_FIND_LINKS="file://${WHEELHOUSE_PATH}" # setuptools 45 requires Python 3.5 or newer. Even though we upgraded pip # above, it may still not be able to get us a compatible version unless we # explicitly ask for one. -"${PIP}" install --upgrade setuptools==44.0.0 wheel +"${PIP}" install --upgrade setuptools wheel # Just about every user of this image wants to use tox from the bootstrap # virtualenv so go ahead and install it now. -"${PIP}" install "tox~=3.0" +"${PIP}" install "tox~=4.0" From 4116bfc7152ae03c0c275478d542349d97c7d87d Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 20 Nov 2023 16:55:41 -0500 Subject: [PATCH 2212/2309] Pass COLUMNS down to workaround what may be a bug in twisted.python.usage --- tox.ini | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tox.ini b/tox.ini index 00eea4f3b..9417aef7d 100644 --- a/tox.ini +++ b/tox.ini @@ -26,7 +26,7 @@ minversion = 4 # Install code the real way, for maximum realism. usedevelop = False -passenv = TAHOE_LAFS_*,PIP_*,SUBUNITREPORTER_*,USERPROFILE,HOMEDRIVE,HOMEPATH +passenv = TAHOE_LAFS_*,PIP_*,SUBUNITREPORTER_*,USERPROFILE,HOMEDRIVE,HOMEPATH,COLUMNS deps = # We pull in certify *here* to avoid bug #2913. Basically if a # `setup_requires=...` causes a package to be installed (with setuptools) @@ -141,7 +141,7 @@ commands = mypy src [testenv:draftnews] -passenv = TAHOE_LAFS_*,PIP_*,SUBUNITREPORTER_*,USERPROFILE,HOMEDRIVE,HOMEPATH +passenv = TAHOE_LAFS_*,PIP_*,SUBUNITREPORTER_*,USERPROFILE,HOMEDRIVE,HOMEPATH,COLUMNS deps = # see comment in [testenv] about "certifi" certifi @@ -151,7 +151,7 @@ commands = [testenv:news] # On macOS, git invoked from Tox needs $HOME. -passenv = TAHOE_LAFS_*,PIP_*,SUBUNITREPORTER_*,USERPROFILE,HOMEDRIVE,HOMEPATH +passenv = TAHOE_LAFS_*,PIP_*,SUBUNITREPORTER_*,USERPROFILE,HOMEDRIVE,HOMEPATH,COLUMNS whitelist_externals = git deps = From c105e0da69f0ccebde01f9b853ef14f3099e69e3 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 20 Nov 2023 17:11:37 -0500 Subject: [PATCH 2213/2309] Force setting a value for columns --- tox.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/tox.ini b/tox.ini index 9417aef7d..bf69c097f 100644 --- a/tox.ini +++ b/tox.ini @@ -56,6 +56,7 @@ setenv = # Define TEST_SUITE in the environment as an aid to constructing the # correct test command below. TEST_SUITE = allmydata + COLUMNS = 80 commands = # As an aid to debugging, dump all of the Python packages and their From 44b0b2841e6471a5cb53af3d7ffebf280e5507a4 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 20 Nov 2023 17:31:34 -0500 Subject: [PATCH 2214/2309] Try passing in all environment variables, to try to make integration tests pass --- tox.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/tox.ini b/tox.ini index bf69c097f..9b7b194e9 100644 --- a/tox.ini +++ b/tox.ini @@ -87,6 +87,7 @@ basepython = python3 platform = mylinux: linux mymacos: darwin mywindows: win32 +passenv = * setenv = COVERAGE_PROCESS_START=.coveragerc commands = From 83f3404dbc4b9f7dac3b5b312385edf691fd4fc8 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 20 Nov 2023 17:43:43 -0500 Subject: [PATCH 2215/2309] Unnecessay --- tox.ini | 1 - 1 file changed, 1 deletion(-) diff --git a/tox.ini b/tox.ini index 9b7b194e9..bf69c097f 100644 --- a/tox.ini +++ b/tox.ini @@ -87,7 +87,6 @@ basepython = python3 platform = mylinux: linux mymacos: darwin mywindows: win32 -passenv = * setenv = COVERAGE_PROCESS_START=.coveragerc commands = From 5dfc39cb206ef70863cf0a4a1d56ccd95a8cebd4 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 20 Nov 2023 17:44:56 -0500 Subject: [PATCH 2216/2309] Skip on 3.12 --- integration/test_tor.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/integration/test_tor.py b/integration/test_tor.py index d7fed5790..d114b763a 100644 --- a/integration/test_tor.py +++ b/integration/test_tor.py @@ -30,6 +30,7 @@ from allmydata.util.deferredutil import async_to_deferred if sys.platform.startswith('win'): pytest.skip('Skipping Tor tests on Windows', allow_module_level=True) +@pytest.mark.skipif(sys.version_info[:2] > (3, 11), reason='Chutney still does not support 3.12') @pytest_twisted.inlineCallbacks def test_onion_service_storage(reactor, request, temp_dir, flog_gatherer, tor_network, tor_introducer_furl): """ @@ -140,6 +141,7 @@ def _create_anonymous_node(reactor, name, web_port, request, temp_dir, flog_gath print("okay, launched") return result +@pytest.mark.skipif(sys.version_info[:2] > (3, 11), reason='Chutney still does not support 3.12') @pytest.mark.skipif(sys.platform.startswith('darwin'), reason='This test has issues on macOS') @pytest_twisted.inlineCallbacks def test_anonymous_client(reactor, request, temp_dir, flog_gatherer, tor_network, introducer_furl): From 37176f3df9638fa4f883a659f68c880c708fa935 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 21 Nov 2023 08:16:24 -0500 Subject: [PATCH 2217/2309] New Twisted release --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index bf69c097f..bb57e8bcf 100644 --- a/tox.ini +++ b/tox.ini @@ -137,7 +137,7 @@ deps = types-pyOpenSSL foolscap # Upgrade when new releases come out: - Twisted==23.8.0 + Twisted==23.10.0 commands = mypy src From 17271d2fac6884bfb53a034413c7c8a7a1f890ff Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 21 Nov 2023 08:16:49 -0500 Subject: [PATCH 2218/2309] New towncrier is out with the fix --- tox.ini | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tox.ini b/tox.ini index bb57e8bcf..8ba69db12 100644 --- a/tox.ini +++ b/tox.ini @@ -102,10 +102,6 @@ deps = # Pin a specific version so we get consistent outcomes; update this # occasionally: ruff == 0.0.287 - # towncrier doesn't work with importlib_resources 6.0.0 - # https://github.com/twisted/towncrier/issues/528 - # Will be fixed in first version of Towncrier that is larger than 2023.6. - importlib_resources < 6.0.0 towncrier # On macOS, git inside of towncrier needs $HOME. passenv = HOME From e4bf0a346ba53047209c596bdaa77bf0cbb38731 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 21 Nov 2023 08:17:05 -0500 Subject: [PATCH 2219/2309] Upgrade ruff --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 8ba69db12..300680b1c 100644 --- a/tox.ini +++ b/tox.ini @@ -101,7 +101,7 @@ skip_install = true deps = # Pin a specific version so we get consistent outcomes; update this # occasionally: - ruff == 0.0.287 + ruff == 0.1.6 towncrier # On macOS, git inside of towncrier needs $HOME. passenv = HOME From fe8a94089c7d8063fef9e0525456233c313498e1 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 21 Nov 2023 08:19:32 -0500 Subject: [PATCH 2220/2309] More thorough mypy checks --- tox.ini | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 300680b1c..e78519145 100644 --- a/tox.ini +++ b/tox.ini @@ -134,7 +134,12 @@ deps = foolscap # Upgrade when new releases come out: Twisted==23.10.0 -commands = mypy src +commands = + # Different versions of Python have a different standard library, and we + # want to be compatible with all the variations. For speed's sake we only do + # the earliest and latest versions. + mypy --python-version=3.8 src + mypy --python-version=3.12 src [testenv:draftnews] From 0e512f22be6ac88151a702f32e1341e5f1185e1b Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 21 Nov 2023 08:21:20 -0500 Subject: [PATCH 2221/2309] Update towncrier version --- tox.ini | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index e78519145..913f5523b 100644 --- a/tox.ini +++ b/tox.ini @@ -147,7 +147,7 @@ passenv = TAHOE_LAFS_*,PIP_*,SUBUNITREPORTER_*,USERPROFILE,HOMEDRIVE,HOMEPATH,CO deps = # see comment in [testenv] about "certifi" certifi - towncrier==21.3.0 + towncrier==23.11.0 commands = python -m towncrier --draft --config towncrier.toml @@ -159,7 +159,7 @@ whitelist_externals = deps = # see comment in [testenv] about "certifi" certifi - towncrier==21.3.0 + towncrier==23.11.0 commands = python -m towncrier --yes --config towncrier.toml # commit the changes From c99362f54d9bfb9c3cd4b93270016f256d9d18ea Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 21 Nov 2023 08:22:56 -0500 Subject: [PATCH 2222/2309] News fragment --- newsfragments/4075.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/4075.minor diff --git a/newsfragments/4075.minor b/newsfragments/4075.minor new file mode 100644 index 000000000..e69de29bb From 774b4f2861c2d63f57e6e89ae0d97f949319bde8 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 21 Nov 2023 08:31:34 -0500 Subject: [PATCH 2223/2309] Fix mypy errors --- src/allmydata/grid_manager.py | 2 ++ src/allmydata/node.py | 4 ++-- src/allmydata/util/cputhreadpool.py | 7 ++++--- src/allmydata/web/common.py | 2 +- 4 files changed, 9 insertions(+), 6 deletions(-) diff --git a/src/allmydata/grid_manager.py b/src/allmydata/grid_manager.py index f366391fc..662f402d8 100644 --- a/src/allmydata/grid_manager.py +++ b/src/allmydata/grid_manager.py @@ -11,6 +11,7 @@ from typing import ( Optional, Union, List, + IO ) from twisted.python.filepath import FilePath @@ -178,6 +179,7 @@ def load_grid_manager(config_path: Optional[FilePath]): :raises: ValueError if the confguration is invalid or IOError if expected files can't be opened. """ + config_file: Union[IO[bytes], IO[str]] if config_path is None: config_file = sys.stdin else: diff --git a/src/allmydata/node.py b/src/allmydata/node.py index 33e8fd260..fdb89e13f 100644 --- a/src/allmydata/node.py +++ b/src/allmydata/node.py @@ -200,14 +200,14 @@ def read_config(basedir, portnumfile, generated_files: Iterable = (), _valid_con config_path = FilePath(basedir).child("tahoe.cfg") try: - config_str = config_path.getContent() + config_bytes = config_path.getContent() except EnvironmentError as e: if e.errno != errno.ENOENT: raise # The file is missing, just create empty ConfigParser. config_str = u"" else: - config_str = config_str.decode("utf-8-sig") + config_str = config_bytes.decode("utf-8-sig") return config_from_string( basedir, diff --git a/src/allmydata/util/cputhreadpool.py b/src/allmydata/util/cputhreadpool.py index 032a3a823..3835701fa 100644 --- a/src/allmydata/util/cputhreadpool.py +++ b/src/allmydata/util/cputhreadpool.py @@ -15,7 +15,7 @@ scheduler affinity or cgroups, but that's not the end of the world. """ import os -from typing import TypeVar, Callable +from typing import TypeVar, Callable, cast from functools import partial import threading from typing_extensions import ParamSpec @@ -24,8 +24,9 @@ from unittest import TestCase from twisted.python.threadpool import ThreadPool from twisted.internet.threads import deferToThreadPool from twisted.internet import reactor +from twisted.internet.interfaces import IReactorFromThreads -_CPU_THREAD_POOL = ThreadPool(minthreads=0, maxthreads=os.cpu_count(), name="TahoeCPU") +_CPU_THREAD_POOL = ThreadPool(minthreads=0, maxthreads=os.cpu_count() or 1, name="TahoeCPU") if hasattr(threading, "_register_atexit"): # This is a private API present in Python 3.8 or later, specifically # designed for thread pool shutdown. Since it's private, it might go away @@ -64,7 +65,7 @@ async def defer_to_thread(f: Callable[P, R], *args: P.args, **kwargs: P.kwargs) return f(*args, **kwargs) # deferToThreadPool has no type annotations... - result = await deferToThreadPool(reactor, _CPU_THREAD_POOL, f, *args, **kwargs) + result = await deferToThreadPool(cast(IReactorFromThreads, reactor), _CPU_THREAD_POOL, f, *args, **kwargs) return result diff --git a/src/allmydata/web/common.py b/src/allmydata/web/common.py index 73eaeaef3..cf6eaecff 100644 --- a/src/allmydata/web/common.py +++ b/src/allmydata/web/common.py @@ -874,6 +874,6 @@ def add_static_children(root: IResource): for child in static_dir.iterdir(): child_path = child.name.encode("utf-8") root.putChild(child_path, static.File( - temporary_file_manager.enter_context(as_file(child)) + str(temporary_file_manager.enter_context(as_file(child))) )) weakref.finalize(root, temporary_file_manager.close) From 0c2db2d5a87ffe2bd0693d239fad9d2b27057e2c Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 15 Nov 2023 15:53:25 -0500 Subject: [PATCH 2224/2309] Make sure FEC does some work --- benchmarks/conftest.py | 2 +- newsfragments/4072.feature | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 newsfragments/4072.feature diff --git a/benchmarks/conftest.py b/benchmarks/conftest.py index 926978a29..972d89b48 100644 --- a/benchmarks/conftest.py +++ b/benchmarks/conftest.py @@ -101,7 +101,7 @@ def client_node(request, grid, storage_nodes, number_of_nodes) -> Client: "client_node", needed=number_of_nodes, happy=number_of_nodes, - total=number_of_nodes, + total=number_of_nodes + 3, # Make sure FEC does some work ) ) print(f"Client node pid: {client_node.process.transport.pid}") diff --git a/newsfragments/4072.feature b/newsfragments/4072.feature new file mode 100644 index 000000000..3b0db7a02 --- /dev/null +++ b/newsfragments/4072.feature @@ -0,0 +1 @@ +Continued work to make Tahoe-LAFS take advantage of multiple CPUs. \ No newline at end of file From 4c03d931bd9f3fa0defb57dcdd5d7f41b4ae3a61 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 6 Dec 2023 15:56:58 -0500 Subject: [PATCH 2225/2309] Accept memoryview --- src/allmydata/crypto/aes.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/allmydata/crypto/aes.py b/src/allmydata/crypto/aes.py index ad7cfcba4..a67501eb0 100644 --- a/src/allmydata/crypto/aes.py +++ b/src/allmydata/crypto/aes.py @@ -87,8 +87,8 @@ def encrypt_data(encryptor, plaintext): """ _validate_cryptor(encryptor, encrypt=True) - if not isinstance(plaintext, six.binary_type): - raise ValueError('Plaintext must be bytes') + if not isinstance(plaintext, (six.binary_type, memoryview)): + raise ValueError(f'Plaintext must be bytes or memoryview: {type(plaintext)}') return encryptor.update(plaintext) @@ -126,8 +126,8 @@ def decrypt_data(decryptor, plaintext): """ _validate_cryptor(decryptor, encrypt=False) - if not isinstance(plaintext, six.binary_type): - raise ValueError('Plaintext must be bytes') + if not isinstance(plaintext, (six.binary_type, memoryview)): + raise ValueError(f'Plaintext must be bytes or memoryview: {type(plaintext)}') return decryptor.update(plaintext) From 9a1e73892e1b58cb6da34b205cf2438811ea9a82 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 6 Dec 2023 15:58:30 -0500 Subject: [PATCH 2226/2309] Run joining in a thread --- src/allmydata/mutable/retrieve.py | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/src/allmydata/mutable/retrieve.py b/src/allmydata/mutable/retrieve.py index 54ada2ca9..b4db6a092 100644 --- a/src/allmydata/mutable/retrieve.py +++ b/src/allmydata/mutable/retrieve.py @@ -4,8 +4,9 @@ Ported to Python 3. from __future__ import annotations import time - +from io import BytesIO from itertools import count + from zope.interface import implementer from twisted.internet import defer from twisted.python import failure @@ -873,11 +874,26 @@ class Retrieve(object): shares = shares[:self._required_shares] self.log("decoding segment %d" % segnum) if segnum == self._num_segments - 1: - d = defer.maybeDeferred(self._tail_decoder.decode, shares, shareids) + d = self._tail_decoder.decode(shares, shareids) else: - d = defer.maybeDeferred(self._segment_decoder.decode, shares, shareids) - def _process(buffers): - segment = b"".join(buffers) + d = self._segment_decoder.decode(shares, shareids) + + # For larger shares, this can take a few milliseconds. As such, we want + # to unblock the event loop. Even if it doesn't release the GIL, if it + # really takes too long it will implicitly release it. + def _join(buffers): + f = BytesIO() + for b in buffers: + f.write(b) + return f.getbuffer() + + @deferredutil.async_to_deferred + async def _got_buffers(buffers): + return await defer_to_thread(_join, buffers) + + d.addCallback(_got_buffers) + + def _process(segment): self.log(format="now decoding segment %(segnum)s of %(numsegs)s", segnum=segnum, numsegs=self._num_segments, From 81a5ae6f461cb615bdc2ae118b6b5223fda6b3ba Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 6 Dec 2023 16:01:14 -0500 Subject: [PATCH 2227/2309] Simplify --- src/allmydata/mutable/retrieve.py | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/src/allmydata/mutable/retrieve.py b/src/allmydata/mutable/retrieve.py index b4db6a092..45d7766ee 100644 --- a/src/allmydata/mutable/retrieve.py +++ b/src/allmydata/mutable/retrieve.py @@ -4,7 +4,6 @@ Ported to Python 3. from __future__ import annotations import time -from io import BytesIO from itertools import count from zope.interface import implementer @@ -879,17 +878,11 @@ class Retrieve(object): d = self._segment_decoder.decode(shares, shareids) # For larger shares, this can take a few milliseconds. As such, we want - # to unblock the event loop. Even if it doesn't release the GIL, if it - # really takes too long it will implicitly release it. - def _join(buffers): - f = BytesIO() - for b in buffers: - f.write(b) - return f.getbuffer() - + # to unblock the event loop. In newer Python b"".join() will release + # the GIL: https://github.com/python/cpython/issues/80232 @deferredutil.async_to_deferred async def _got_buffers(buffers): - return await defer_to_thread(_join, buffers) + return await defer_to_thread(lambda: b"".join(buffers)) d.addCallback(_got_buffers) From 2783bd8b7799f6c57a7100e8920bc5adf88c0a52 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 7 Dec 2023 16:39:30 -0500 Subject: [PATCH 2228/2309] Unnecessary maybeDeferred --- src/allmydata/immutable/downloader/node.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/immutable/downloader/node.py b/src/allmydata/immutable/downloader/node.py index a1ef4b485..efa3e09eb 100644 --- a/src/allmydata/immutable/downloader/node.py +++ b/src/allmydata/immutable/downloader/node.py @@ -419,7 +419,7 @@ class DownloadNode(object): def process_blocks(self, segnum, blocks): start = now() - d = defer.maybeDeferred(self._decode_blocks, segnum, blocks) + d = self._decode_blocks(segnum, blocks) d.addCallback(self._check_ciphertext_hash, segnum) def _deliver(result): log.msg(format="delivering segment(%(segnum)d)", From e230a65fa6d2b1d53807f714b4e2554ae57408b6 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 11 Dec 2023 10:09:43 -0500 Subject: [PATCH 2229/2309] Upgrade mypy --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 913f5523b..f64439065 100644 --- a/tox.ini +++ b/tox.ini @@ -124,7 +124,7 @@ commands = [testenv:typechecks] basepython = python3 deps = - mypy==1.5.1 + mypy==1.7.1 mypy-zope types-mock types-six From 7ab0483d07ae9c795ca93a2cdf3bbe867e1326e8 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 11 Dec 2023 10:09:50 -0500 Subject: [PATCH 2230/2309] Pacify newer mypy --- src/allmydata/storage_client.py | 21 ++++++++++++--------- src/allmydata/testing/web.py | 3 ++- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index b714d7757..9e6f94f47 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -32,7 +32,7 @@ Ported to Python 3. from __future__ import annotations -from typing import Union, Callable, Any, Optional, cast, Dict +from typing import Union, Callable, Any, Optional, cast, Dict, Iterable from os import urandom import re import time @@ -139,9 +139,9 @@ class StorageClientConfig(object): only upload to a storage-server that has a valid certificate signed by at least one of these keys. """ - preferred_peers = attr.ib(default=()) - storage_plugins = attr.ib(default=attr.Factory(dict)) - grid_manager_keys = attr.ib(default=attr.Factory(list)) + preferred_peers : Iterable[bytes] = attr.ib(default=()) + storage_plugins : dict[str, dict[str, str]] = attr.ib(default=attr.Factory(dict)) + grid_manager_keys : list[ed25519.Ed25519PublicKey] = attr.ib(default=attr.Factory(list)) @classmethod def from_node_config(cls, config): @@ -1107,18 +1107,21 @@ class NativeStorageServer(service.MultiService): async def _pick_a_http_server( reactor, nurls: list[DecodedURL], - request: Callable[[Any, DecodedURL], defer.Deferred[Any]] + request: Callable[[object, DecodedURL], defer.Deferred[object]] ) -> DecodedURL: """Pick the first server we successfully send a request to. Fires with ``None`` if no server was found, or with the ``DecodedURL`` of the first successfully-connected server. """ - queries = race([ - request(reactor, nurl).addCallback(lambda _, nurl=nurl: nurl) - for nurl in nurls - ]) + requests = [] + for nurl in nurls: + def to_nurl(_: object, nurl: DecodedURL=nurl) -> DecodedURL: + return nurl + requests.append(request(reactor, nurl).addCallback(to_nurl)) + + queries: defer.Deferred[tuple[int, DecodedURL]] = race(requests) _, nurl = await queries return nurl diff --git a/src/allmydata/testing/web.py b/src/allmydata/testing/web.py index 95e92825b..f7c8a4e1e 100644 --- a/src/allmydata/testing/web.py +++ b/src/allmydata/testing/web.py @@ -14,6 +14,7 @@ Test-helpers for clients that use the WebUI. from __future__ import annotations import hashlib +from typing import Iterable import attr @@ -141,7 +142,7 @@ class _FakeTahoeUriHandler(Resource, object): isLeaf = True data: BytesKeyDict = attr.ib(default=attr.Factory(BytesKeyDict)) - capability_generators = attr.ib(default=attr.Factory(dict)) + capability_generators: dict[bytes,Iterable[bytes]] = attr.ib(default=attr.Factory(dict)) def _generate_capability(self, kind): """ From e8066f6e3d10a68c14ec6dc61dc0f135e96541f3 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 11 Dec 2023 10:10:07 -0500 Subject: [PATCH 2231/2309] News file --- newsfragments/4082.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/4082.minor diff --git a/newsfragments/4082.minor b/newsfragments/4082.minor new file mode 100644 index 000000000..e69de29bb From 3a1e07982884c49a39709551c52f68116514da6e Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 11 Dec 2023 10:14:31 -0500 Subject: [PATCH 2232/2309] Pacify newer Mypy --- src/allmydata/test/common.py | 6 +++--- src/allmydata/test/test_storage_client.py | 3 ++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/allmydata/test/common.py b/src/allmydata/test/common.py index bd0feda10..485da9254 100644 --- a/src/allmydata/test/common.py +++ b/src/allmydata/test/common.py @@ -177,8 +177,8 @@ class MemoryIntroducerClient(object): sequencer = attr.ib() cache_filepath = attr.ib() - subscribed_to = attr.ib(default=attr.Factory(list)) - published_announcements = attr.ib(default=attr.Factory(list)) + subscribed_to : list[Subscription] = attr.ib(default=attr.Factory(list)) + published_announcements : list[Announcement] = attr.ib(default=attr.Factory(list)) def setServiceParent(self, parent): @@ -288,7 +288,7 @@ class UseNode(object): basedir = attr.ib(validator=attr.validators.instance_of(FilePath)) introducer_furl = attr.ib(validator=attr.validators.instance_of(str), converter=six.ensure_str) - node_config = attr.ib(default=attr.Factory(dict)) + node_config : dict[bytes,bytes] = attr.ib(default=attr.Factory(dict)) config = attr.ib(default=None) reactor = attr.ib(default=None) diff --git a/src/allmydata/test/test_storage_client.py b/src/allmydata/test/test_storage_client.py index 13c6ccaea..1b2d31bb2 100644 --- a/src/allmydata/test/test_storage_client.py +++ b/src/allmydata/test/test_storage_client.py @@ -41,6 +41,7 @@ import attr from twisted.internet.interfaces import ( IStreamClientEndpoint, + IProtocolFactory, ) from twisted.application.service import ( Service, @@ -604,7 +605,7 @@ class SpyHandler(object): ``Deferred`` that was returned from ``connect`` and the factory that was passed to ``connect``. """ - _connects = attr.ib(default=attr.Factory(list)) + _connects : list[tuple[Deferred[object], IProtocolFactory]]= attr.ib(default=attr.Factory(list)) def hint_to_endpoint(self, hint, reactor, update_status): return (SpyEndpoint(self._connects.append), hint) From 3d4945a26d2b592dc29c01e52d4dfacc1c44b3fd Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 12 Dec 2023 10:33:25 -0500 Subject: [PATCH 2233/2309] Write out Eliot messages in a thread. --- src/allmydata/test/test_eliotutil.py | 53 +++++++++------------------- src/allmydata/util/eliotutil.py | 20 +++++------ 2 files changed, 26 insertions(+), 47 deletions(-) diff --git a/src/allmydata/test/test_eliotutil.py b/src/allmydata/test/test_eliotutil.py index 52d709e4c..5b191cd92 100644 --- a/src/allmydata/test/test_eliotutil.py +++ b/src/allmydata/test/test_eliotutil.py @@ -1,20 +1,7 @@ """ Tests for ``allmydata.util.eliotutil``. - -Ported to Python 3. """ -from __future__ import ( - unicode_literals, - print_function, - absolute_import, - division, -) - -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - from sys import stdout import logging @@ -67,6 +54,7 @@ from ..util.eliotutil import ( _parse_destination_description, _EliotLogging, ) +from ..util.deferredutil import async_to_deferred from .common import ( SyncTestCase, @@ -214,13 +202,14 @@ class ParseDestinationDescriptionTests(SyncTestCase): ) -# Opt out of the great features of common.SyncTestCase because we're -# interacting with Eliot in a very obscure, particular, fragile way. :/ -class EliotLoggingTests(TestCase): +# We need AsyncTestCase because logging happens in a thread tied to the +# reactor. +class EliotLoggingTests(AsyncTestCase): """ Tests for ``_EliotLogging``. """ - def test_stdlib_event_relayed(self): + @async_to_deferred + async def test_stdlib_event_relayed(self): """ An event logged using the stdlib logging module is delivered to the Eliot destination. @@ -228,23 +217,16 @@ class EliotLoggingTests(TestCase): collected = [] service = _EliotLogging([collected.append]) service.startService() - self.addCleanup(service.stopService) - - # The first destination added to the global log destinations gets any - # buffered messages delivered to it. We don't care about those. - # Throw them on the floor. Sorry. - del collected[:] logging.critical("oh no") - self.assertThat( - collected, - AfterPreprocessing( - len, - Equals(1), - ), + await service.stopService() + + self.assertTrue( + "oh no" in str(collected[-1]), collected ) - def test_twisted_event_relayed(self): + @async_to_deferred + async def test_twisted_event_relayed(self): """ An event logged with a ``twisted.logger.Logger`` is delivered to the Eliot destination. @@ -252,15 +234,13 @@ class EliotLoggingTests(TestCase): collected = [] service = _EliotLogging([collected.append]) service.startService() - self.addCleanup(service.stopService) from twisted.logger import Logger Logger().critical("oh no") - self.assertThat( - collected, - AfterPreprocessing( - len, Equals(1), - ), + await service.stopService() + + self.assertTrue( + "oh no" in str(collected[-1]), collected ) def test_validation_failure(self): @@ -318,7 +298,6 @@ class EliotLoggingTests(TestCase): ) - class LogCallDeferredTests(TestCase): """ Tests for ``log_call_deferred``. diff --git a/src/allmydata/util/eliotutil.py b/src/allmydata/util/eliotutil.py index 94d34f96f..9e3cf6ae0 100644 --- a/src/allmydata/util/eliotutil.py +++ b/src/allmydata/util/eliotutil.py @@ -36,7 +36,7 @@ from attr.validators import ( optional, provides, ) - +from twisted.internet import reactor from eliot import ( ILogger, Message, @@ -58,6 +58,7 @@ from eliot.twisted import ( DeferredContext, inline_callbacks, ) +from eliot.logwriter import ThreadedWriter from twisted.python.usage import ( UsageError, ) @@ -75,7 +76,7 @@ from twisted.logger import ( from twisted.internet.defer import ( maybeDeferred, ) -from twisted.application.service import Service +from twisted.application.service import MultiService from .jsonbytes import AnyBytesJSONEncoder @@ -144,7 +145,7 @@ def opt_help_eliot_destinations(self): raise SystemExit(0) -class _EliotLogging(Service): +class _EliotLogging(MultiService): """ A service which adds stdout as an Eliot destination while it is running. """ @@ -153,23 +154,22 @@ class _EliotLogging(Service): :param list destinations: The Eliot destinations which will is added by this service. """ - self.destinations = destinations - + MultiService.__init__(self) + for destination in destinations: + service = ThreadedWriter(destination, reactor) + service.setServiceParent(self) def startService(self): self.stdlib_cleanup = _stdlib_logging_to_eliot_configuration(getLogger()) self.twisted_observer = _TwistedLoggerToEliotObserver() globalLogPublisher.addObserver(self.twisted_observer) - add_destinations(*self.destinations) - return Service.startService(self) + return MultiService.startService(self) def stopService(self): - for dest in self.destinations: - remove_destination(dest) globalLogPublisher.removeObserver(self.twisted_observer) self.stdlib_cleanup() - return Service.stopService(self) + return MultiService.stopService(self) @implementer(ILogObserver) From d5db9e800fedda81c077b1b0eef735e72109d52f Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 12 Dec 2023 10:37:04 -0500 Subject: [PATCH 2234/2309] News fragment --- newsfragments/4804.feature | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/4804.feature diff --git a/newsfragments/4804.feature b/newsfragments/4804.feature new file mode 100644 index 000000000..23b3d3c6e --- /dev/null +++ b/newsfragments/4804.feature @@ -0,0 +1 @@ +Logs are now written in a thread, which should make the application more responsive under load. \ No newline at end of file From 451cdd7dcbcea71ba815d1dfbfc2702c42308627 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 12 Dec 2023 10:39:18 -0500 Subject: [PATCH 2235/2309] Fix lint --- src/allmydata/util/eliotutil.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/allmydata/util/eliotutil.py b/src/allmydata/util/eliotutil.py index 9e3cf6ae0..f0d5bffd2 100644 --- a/src/allmydata/util/eliotutil.py +++ b/src/allmydata/util/eliotutil.py @@ -41,8 +41,6 @@ from eliot import ( ILogger, Message, FileDestination, - add_destinations, - remove_destination, write_traceback, start_action, ) From 25ef11efc8d397548ea5aff5894ada63418b7315 Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 29 Nov 2023 16:33:49 -0700 Subject: [PATCH 2236/2309] news --- newsfragments/4078.bugfix | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/4078.bugfix diff --git a/newsfragments/4078.bugfix b/newsfragments/4078.bugfix new file mode 100644 index 000000000..87b92089c --- /dev/null +++ b/newsfragments/4078.bugfix @@ -0,0 +1 @@ +fix race condition \ No newline at end of file From 194011946c504b270a1082dbbe4e89bb3a32bf58 Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 18 Dec 2023 23:01:25 -0700 Subject: [PATCH 2237/2309] for some reason this makes 'the occasional error' happen all the time? --- src/allmydata/immutable/downloader/node.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/allmydata/immutable/downloader/node.py b/src/allmydata/immutable/downloader/node.py index a1ef4b485..a58d82a01 100644 --- a/src/allmydata/immutable/downloader/node.py +++ b/src/allmydata/immutable/downloader/node.py @@ -132,8 +132,8 @@ class DownloadNode(object): def stop(self): # called by the Terminator at shutdown, mostly for tests if self._active_segment: - self._active_segment.stop() - self._active_segment = None + seg, self._active_segment = self._active_segment, None + seg.stop() self._sharefinder.stop() # things called by outside callers, via CiphertextFileNode. get_segment() @@ -410,11 +410,11 @@ class DownloadNode(object): def fetch_failed(self, sf, f): assert sf is self._active_segment + self._active_segment = None # deliver error upwards for (d,c,seg_ev) in self._extract_requests(sf.segnum): seg_ev.error(now()) eventually(self._deliver, d, c, f) - self._active_segment = None self._start_new_segment() def process_blocks(self, segnum, blocks): @@ -434,6 +434,7 @@ class DownloadNode(object): eventually(self._deliver, d, c, result) else: (offset, segment, decodetime) = result + self._active_segment = None for (d,c,seg_ev) in self._extract_requests(segnum): # when we have two requests for the same segment, the # second one will not be "activated" before the data is @@ -446,7 +447,6 @@ class DownloadNode(object): seg_ev.deliver(when, offset, len(segment), decodetime) eventually(self._deliver, d, c, result) self._download_status.add_misc_event("process_block", start, now()) - self._active_segment = None self._start_new_segment() d.addBoth(_deliver) d.addErrback(log.err, "unhandled error during process_blocks", @@ -536,8 +536,8 @@ class DownloadNode(object): # self._active_segment might be None in rare circumstances, so make # sure we tolerate it if self._active_segment and self._active_segment.segnum not in segnums: - self._active_segment.stop() - self._active_segment = None + seg, self._active_segment = self._active_segment, None##True # XXX None for real + seg.stop() self._start_new_segment() # called by ShareFinder to choose hashtree sizes in CommonShares, and by From d70fa461da9961f03c38d58b0098f9c4a0dc485f Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 18 Dec 2023 23:27:16 -0700 Subject: [PATCH 2238/2309] fix already-stopped --- src/allmydata/immutable/downloader/fetcher.py | 15 ++++++++------- src/allmydata/immutable/downloader/node.py | 3 ++- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/src/allmydata/immutable/downloader/fetcher.py b/src/allmydata/immutable/downloader/fetcher.py index 4e8b7f926..130bb93dc 100644 --- a/src/allmydata/immutable/downloader/fetcher.py +++ b/src/allmydata/immutable/downloader/fetcher.py @@ -63,13 +63,14 @@ class SegmentFetcher(object): self._running = True def stop(self): - log.msg("SegmentFetcher(%r).stop" % self._node._si_prefix, - level=log.NOISY, parent=self._lp, umid="LWyqpg") - self._cancel_all_requests() - self._running = False - # help GC ??? XXX - del self._shares, self._shares_from_server, self._active_share_map - del self._share_observers + if self._running: + log.msg("SegmentFetcher(%r).stop" % self._node._si_prefix, + level=log.NOISY, parent=self._lp, umid="LWyqpg") + self._cancel_all_requests() + self._running = False + # help GC ??? + del self._shares, self._shares_from_server, self._active_share_map + del self._share_observers # called by our parent _Node diff --git a/src/allmydata/immutable/downloader/node.py b/src/allmydata/immutable/downloader/node.py index a58d82a01..102d29a0a 100644 --- a/src/allmydata/immutable/downloader/node.py +++ b/src/allmydata/immutable/downloader/node.py @@ -533,10 +533,11 @@ class DownloadNode(object): self._segment_requests = [t for t in self._segment_requests if t[2] != cancel] segnums = [segnum for (segnum,d,c,seg_ev,lp) in self._segment_requests] + # self._active_segment might be None in rare circumstances, so make # sure we tolerate it if self._active_segment and self._active_segment.segnum not in segnums: - seg, self._active_segment = self._active_segment, None##True # XXX None for real + seg, self._active_segment = self._active_segment, None seg.stop() self._start_new_segment() From f3be22e6303f4f981db5e5442d4e8cfd17c2f7fd Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 21 Dec 2023 01:24:14 -0700 Subject: [PATCH 2239/2309] news --- newsfragments/4078.bugfix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/newsfragments/4078.bugfix b/newsfragments/4078.bugfix index 87b92089c..12ca66bcf 100644 --- a/newsfragments/4078.bugfix +++ b/newsfragments/4078.bugfix @@ -1 +1 @@ -fix race condition \ No newline at end of file +Fix a race condition with SegmentFetcher \ No newline at end of file From 56c861449ac3902518a06d777dc4ebb699d1428c Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 21 Dec 2023 15:22:38 -0700 Subject: [PATCH 2240/2309] match better --- src/allmydata/test/cli/test_run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/cli/test_run.py b/src/allmydata/test/cli/test_run.py index ae0f92131..c7ebcd8c0 100644 --- a/src/allmydata/test/cli/test_run.py +++ b/src/allmydata/test/cli/test_run.py @@ -264,7 +264,7 @@ class RunTests(SyncTestCase): self.assertThat(runs, Equals([])) self.assertThat(result_code, Equals(1)) - good_file_content_re = re.compile(r"\s[0-9]*\s[0-9]*\s", re.M) + good_file_content_re = re.compile(r"\s*[0-9]*\s[0-9]*\s*", re.M) @given(text()) def test_pidfile_contents(self, content): From ed193c7e3fdb7bec8964bc202da6f16203f56404 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 4 Jan 2024 09:35:42 -0500 Subject: [PATCH 2241/2309] Mypy 1.8 --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index f64439065..9a2d7d5b2 100644 --- a/tox.ini +++ b/tox.ini @@ -124,7 +124,7 @@ commands = [testenv:typechecks] basepython = python3 deps = - mypy==1.7.1 + mypy==1.8.0 mypy-zope types-mock types-six From ee19fc5a181acbc2e63d4803961a9aa3ab08d3d2 Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 4 Jan 2024 20:04:04 -0700 Subject: [PATCH 2242/2309] update NEWS.txt for release --- NEWS.rst | 81 ++++++++++++++++++++++++++++++++ newsfragments/2916.feature | 1 - newsfragments/3072.feature | 1 - newsfragments/3508.minor | 0 newsfragments/3622.minor | 0 newsfragments/3783.minor | 0 newsfragments/3870.minor | 0 newsfragments/3874.minor | 0 newsfragments/3880.minor | 0 newsfragments/3899.bugfix | 4 -- newsfragments/3902.feature | 1 - newsfragments/3904.minor | 0 newsfragments/3910.minor | 0 newsfragments/3914.minor | 0 newsfragments/3917.minor | 0 newsfragments/3921.feature | 5 -- newsfragments/3922.documentation | 1 - newsfragments/3927.minor | 0 newsfragments/3928.minor | 0 newsfragments/3935.minor | 0 newsfragments/3936.minor | 0 newsfragments/3937.minor | 0 newsfragments/3938.bugfix | 1 - newsfragments/3939.bugfix | 1 - newsfragments/3940.minor | 0 newsfragments/3942.minor | 1 - newsfragments/3944.minor | 0 newsfragments/3946.bugfix | 1 - newsfragments/3947.minor | 0 newsfragments/3950.minor | 0 newsfragments/3952.minor | 0 newsfragments/3953.minor | 0 newsfragments/3954.minor | 0 newsfragments/3956.minor | 0 newsfragments/3958.minor | 0 newsfragments/3959.minor | 0 newsfragments/3960.minor | 0 newsfragments/3961.other | 1 - newsfragments/3962.feature | 1 - newsfragments/3964.removed | 1 - newsfragments/3965.minor | 0 newsfragments/3966.bugfix | 1 - newsfragments/3967.minor | 0 newsfragments/3968.minor | 0 newsfragments/3969.minor | 0 newsfragments/3970.minor | 0 newsfragments/3971.minor | 1 - newsfragments/3974.minor | 0 newsfragments/3975.minor | 1 - newsfragments/3976.minor | 1 - newsfragments/3978.minor | 0 newsfragments/3982.feature | 1 - newsfragments/3987.minor | 0 newsfragments/3988.minor | 0 newsfragments/3989.installation | 1 - newsfragments/3991.minor | 0 newsfragments/3993.minor | 0 newsfragments/3994.minor | 0 newsfragments/3996.minor | 0 newsfragments/3997.installation | 1 - newsfragments/3998.minor | 0 newsfragments/3999.bugfix | 1 - newsfragments/4000.minor | 0 newsfragments/4001.minor | 0 newsfragments/4002.minor | 0 newsfragments/4003.minor | 0 newsfragments/4004.minor | 0 newsfragments/4005.minor | 0 newsfragments/4006.minor | 0 newsfragments/4009.minor | 0 newsfragments/4010.minor | 0 newsfragments/4012.minor | 0 newsfragments/4014.minor | 0 newsfragments/4015.minor | 0 newsfragments/4016.minor | 0 newsfragments/4018.minor | 0 newsfragments/4019.minor | 0 newsfragments/4020.minor | 1 - newsfragments/4022.minor | 0 newsfragments/4023.minor | 0 newsfragments/4024.minor | 0 newsfragments/4026.minor | 0 newsfragments/4027.minor | 0 newsfragments/4028.minor | 0 newsfragments/4029.bugfix | 2 - newsfragments/4035.minor | 0 newsfragments/4036.feature | 1 - newsfragments/4038.minor | 0 newsfragments/4039.documentation | 1 - newsfragments/4040.minor | 0 newsfragments/4041.feature | 1 - newsfragments/4042.minor | 0 newsfragments/4044.minor | 0 newsfragments/4046.minor | 0 newsfragments/4047.minor | 0 newsfragments/4049.minor | 0 newsfragments/4050.minor | 0 newsfragments/4051.minor | 0 newsfragments/4052.minor | 0 newsfragments/4055.minor | 0 newsfragments/4056.bugfix | 3 -- newsfragments/4059.minor | 0 newsfragments/4060.feature | 1 - newsfragments/4061.minor | 0 newsfragments/4062.minor | 0 newsfragments/4063.minor | 0 newsfragments/4065.minor | 0 newsfragments/4066.minor | 0 newsfragments/4068.feature | 1 - newsfragments/4070.minor | 0 newsfragments/4074.minor | 0 newsfragments/4075.minor | 0 newsfragments/4078.bugfix | 1 - newsfragments/4804.feature | 1 - 114 files changed, 81 insertions(+), 41 deletions(-) delete mode 100644 newsfragments/2916.feature delete mode 100644 newsfragments/3072.feature delete mode 100644 newsfragments/3508.minor delete mode 100644 newsfragments/3622.minor delete mode 100644 newsfragments/3783.minor delete mode 100644 newsfragments/3870.minor delete mode 100644 newsfragments/3874.minor delete mode 100644 newsfragments/3880.minor delete mode 100644 newsfragments/3899.bugfix delete mode 100644 newsfragments/3902.feature delete mode 100644 newsfragments/3904.minor delete mode 100644 newsfragments/3910.minor delete mode 100644 newsfragments/3914.minor delete mode 100644 newsfragments/3917.minor delete mode 100644 newsfragments/3921.feature delete mode 100644 newsfragments/3922.documentation delete mode 100644 newsfragments/3927.minor delete mode 100644 newsfragments/3928.minor delete mode 100644 newsfragments/3935.minor delete mode 100644 newsfragments/3936.minor delete mode 100644 newsfragments/3937.minor delete mode 100644 newsfragments/3938.bugfix delete mode 100644 newsfragments/3939.bugfix delete mode 100644 newsfragments/3940.minor delete mode 100644 newsfragments/3942.minor delete mode 100644 newsfragments/3944.minor delete mode 100644 newsfragments/3946.bugfix delete mode 100644 newsfragments/3947.minor delete mode 100644 newsfragments/3950.minor delete mode 100644 newsfragments/3952.minor delete mode 100644 newsfragments/3953.minor delete mode 100644 newsfragments/3954.minor delete mode 100644 newsfragments/3956.minor delete mode 100644 newsfragments/3958.minor delete mode 100644 newsfragments/3959.minor delete mode 100644 newsfragments/3960.minor delete mode 100644 newsfragments/3961.other delete mode 100644 newsfragments/3962.feature delete mode 100644 newsfragments/3964.removed delete mode 100644 newsfragments/3965.minor delete mode 100644 newsfragments/3966.bugfix delete mode 100644 newsfragments/3967.minor delete mode 100644 newsfragments/3968.minor delete mode 100644 newsfragments/3969.minor delete mode 100644 newsfragments/3970.minor delete mode 100644 newsfragments/3971.minor delete mode 100644 newsfragments/3974.minor delete mode 100644 newsfragments/3975.minor delete mode 100644 newsfragments/3976.minor delete mode 100644 newsfragments/3978.minor delete mode 100644 newsfragments/3982.feature delete mode 100644 newsfragments/3987.minor delete mode 100644 newsfragments/3988.minor delete mode 100644 newsfragments/3989.installation delete mode 100644 newsfragments/3991.minor delete mode 100644 newsfragments/3993.minor delete mode 100644 newsfragments/3994.minor delete mode 100644 newsfragments/3996.minor delete mode 100644 newsfragments/3997.installation delete mode 100644 newsfragments/3998.minor delete mode 100644 newsfragments/3999.bugfix delete mode 100644 newsfragments/4000.minor delete mode 100644 newsfragments/4001.minor delete mode 100644 newsfragments/4002.minor delete mode 100644 newsfragments/4003.minor delete mode 100644 newsfragments/4004.minor delete mode 100644 newsfragments/4005.minor delete mode 100644 newsfragments/4006.minor delete mode 100644 newsfragments/4009.minor delete mode 100644 newsfragments/4010.minor delete mode 100644 newsfragments/4012.minor delete mode 100644 newsfragments/4014.minor delete mode 100644 newsfragments/4015.minor delete mode 100644 newsfragments/4016.minor delete mode 100644 newsfragments/4018.minor delete mode 100644 newsfragments/4019.minor delete mode 100644 newsfragments/4020.minor delete mode 100644 newsfragments/4022.minor delete mode 100644 newsfragments/4023.minor delete mode 100644 newsfragments/4024.minor delete mode 100644 newsfragments/4026.minor delete mode 100644 newsfragments/4027.minor delete mode 100644 newsfragments/4028.minor delete mode 100644 newsfragments/4029.bugfix delete mode 100644 newsfragments/4035.minor delete mode 100644 newsfragments/4036.feature delete mode 100644 newsfragments/4038.minor delete mode 100644 newsfragments/4039.documentation delete mode 100644 newsfragments/4040.minor delete mode 100644 newsfragments/4041.feature delete mode 100644 newsfragments/4042.minor delete mode 100644 newsfragments/4044.minor delete mode 100644 newsfragments/4046.minor delete mode 100644 newsfragments/4047.minor delete mode 100644 newsfragments/4049.minor delete mode 100644 newsfragments/4050.minor delete mode 100644 newsfragments/4051.minor delete mode 100644 newsfragments/4052.minor delete mode 100644 newsfragments/4055.minor delete mode 100644 newsfragments/4056.bugfix delete mode 100644 newsfragments/4059.minor delete mode 100644 newsfragments/4060.feature delete mode 100644 newsfragments/4061.minor delete mode 100644 newsfragments/4062.minor delete mode 100644 newsfragments/4063.minor delete mode 100644 newsfragments/4065.minor delete mode 100644 newsfragments/4066.minor delete mode 100644 newsfragments/4068.feature delete mode 100644 newsfragments/4070.minor delete mode 100644 newsfragments/4074.minor delete mode 100644 newsfragments/4075.minor delete mode 100644 newsfragments/4078.bugfix delete mode 100644 newsfragments/4804.feature diff --git a/NEWS.rst b/NEWS.rst index 7b1fadb8a..c0c36adad 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -5,6 +5,87 @@ User-Visible Changes in Tahoe-LAFS ================================== .. towncrier start line +Release 1.18.0.post1720.dev0 (2024-01-04) +''''''''''''''''''''''''''''''''''''''''' + +No significant changes. + + +Release 1.18.0.post1720 (2024-01-04) +'''''''''''''''''''''''''''''''''''' + +Features +-------- + +- Tahoe-LAFS now includes a new "Grid Manager" specification and implementation adding more options to control which storage servers a client will use for uploads. (`#2916 `_) +- Added support for Python 3.12, and work with Eliot 1.15 (`#3072 `_) +- The new HTTPS-based storage server is now enabled transparently on the same port as the Foolscap server. This will not have any user-facing impact until the HTTPS storage protocol is supported in clients as well. (`#3902 `_) +- `tahoe run ...` will now exit when its stdin is closed. + + This facilitates subprocess management, specifically cleanup. + When a parent process is running tahoe and exits without time to do "proper" cleanup at least the stdin descriptor will be closed. + Subsequently "tahoe run" notices this and exits. (`#3921 `_) +- Mutable objects can now be created with a pre-determined "signature key" using the ``tahoe put`` CLI or the HTTP API. This enables deterministic creation of mutable capabilities. This feature must be used with care to preserve the normal security and reliability properties. (`#3962 `_) +- Added support for Python 3.11. (`#3982 `_) +- tahoe run now accepts --allow-stdin-close to mean "keep running if stdin closes" (`#4036 `_) +- The storage server and client now support a new, HTTPS-based protocol. (`#4041 `_) +- Started work on a new end-to-end benchmarking framework. (`#4060 `_) +- Some operations now run in threads, improving the responsiveness of Tahoe nodes. (`#4068 `_) +- Logs are now written in a thread, which should make the application more responsive under load. (`#4804 `_) + + +Bug Fixes +--------- + +- Provide better feedback from plugin configuration errors + + Local errors now print a useful message and exit. + Announcements that only contain invalid / unusable plugins now show a message in the Welcome page. (`#3899 `_) +- Work with (and require) newer versions of pycddl. (`#3938 `_) +- Uploading immutables will now better use available bandwidth, which should allow for faster uploads in many cases. (`#3939 `_) +- Downloads of large immutables should now finish much faster. (`#3946 `_) +- Fix incompatibility with transitive dependency charset_normalizer >= 3 when using PyInstaller. (`#3966 `_) +- A bug where Introducer nodes configured to listen on Tor or I2P would not actually do so has been fixed. (`#3999 `_) +- The (still off-by-default) HTTP storage client will now use Tor when Tor-based client-side anonymity was requested. + Previously it would use normal TCP connections and not be anonymous. (`#4029 `_) +- Provide our own copy of attrs' "provides()" validator + + This validator is deprecated and slated for removal; that project's suggestion is to copy the code to our project. (`#4056 `_) +- Fix a race condition with SegmentFetcher (`#4078 `_) + + +Dependency/Installation Changes +------------------------------- + +- tenacity is no longer a dependency. (`#3989 `_) +- Tahoe-LAFS is incompatible with cryptography >= 40 and now declares a requirement on an older version. (`#3997 `_) + + +Documentation Changes +--------------------- + +- Several minor errors in the Great Black Swamp proposed specification document have been fixed. (`#3922 `_) +- Document the ``force_foolscap`` configuration options for ``[storage]`` and ``[client]``. (`#4039 `_) + + +Removed Features +---------------- + +- Python 3.7 is no longer supported, and Debian 10 and Ubuntu 18.04 are no longer tested. (`#3964 `_) + + +Other Changes +------------- + +- The integration test suite now includes a set of capability test vectors (``integration/vectors/test_vectors.yaml``) which can be used to verify compatibility between Tahoe-LAFS and other implementations. (`#3961 `_) + + +Misc/Other +---------- + +- `#3508 `_, `#3622 `_, `#3783 `_, `#3870 `_, `#3874 `_, `#3880 `_, `#3904 `_, `#3910 `_, `#3914 `_, `#3917 `_, `#3927 `_, `#3928 `_, `#3935 `_, `#3936 `_, `#3937 `_, `#3940 `_, `#3942 `_, `#3944 `_, `#3947 `_, `#3950 `_, `#3952 `_, `#3953 `_, `#3954 `_, `#3956 `_, `#3958 `_, `#3959 `_, `#3960 `_, `#3965 `_, `#3967 `_, `#3968 `_, `#3969 `_, `#3970 `_, `#3971 `_, `#3974 `_, `#3975 `_, `#3976 `_, `#3978 `_, `#3987 `_, `#3988 `_, `#3991 `_, `#3993 `_, `#3994 `_, `#3996 `_, `#3998 `_, `#4000 `_, `#4001 `_, `#4002 `_, `#4003 `_, `#4004 `_, `#4005 `_, `#4006 `_, `#4009 `_, `#4010 `_, `#4012 `_, `#4014 `_, `#4015 `_, `#4016 `_, `#4018 `_, `#4019 `_, `#4020 `_, `#4022 `_, `#4023 `_, `#4024 `_, `#4026 `_, `#4027 `_, `#4028 `_, `#4035 `_, `#4038 `_, `#4040 `_, `#4042 `_, `#4044 `_, `#4046 `_, `#4047 `_, `#4049 `_, `#4050 `_, `#4051 `_, `#4052 `_, `#4055 `_, `#4059 `_, `#4061 `_, `#4062 `_, `#4063 `_, `#4065 `_, `#4066 `_, `#4070 `_, `#4074 `_, `#4075 `_ + + Release 1.18.0 (2022-10-02) ''''''''''''''''''''''''''' diff --git a/newsfragments/2916.feature b/newsfragments/2916.feature deleted file mode 100644 index c65f473a4..000000000 --- a/newsfragments/2916.feature +++ /dev/null @@ -1 +0,0 @@ -Tahoe-LAFS now includes a new "Grid Manager" specification and implementation adding more options to control which storage servers a client will use for uploads. \ No newline at end of file diff --git a/newsfragments/3072.feature b/newsfragments/3072.feature deleted file mode 100644 index 79ce6d56d..000000000 --- a/newsfragments/3072.feature +++ /dev/null @@ -1 +0,0 @@ -Added support for Python 3.12, and work with Eliot 1.15 \ No newline at end of file diff --git a/newsfragments/3508.minor b/newsfragments/3508.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3622.minor b/newsfragments/3622.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3783.minor b/newsfragments/3783.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3870.minor b/newsfragments/3870.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3874.minor b/newsfragments/3874.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3880.minor b/newsfragments/3880.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3899.bugfix b/newsfragments/3899.bugfix deleted file mode 100644 index 55d4fabd4..000000000 --- a/newsfragments/3899.bugfix +++ /dev/null @@ -1,4 +0,0 @@ -Provide better feedback from plugin configuration errors - -Local errors now print a useful message and exit. -Announcements that only contain invalid / unusable plugins now show a message in the Welcome page. diff --git a/newsfragments/3902.feature b/newsfragments/3902.feature deleted file mode 100644 index 2477d0ae6..000000000 --- a/newsfragments/3902.feature +++ /dev/null @@ -1 +0,0 @@ -The new HTTPS-based storage server is now enabled transparently on the same port as the Foolscap server. This will not have any user-facing impact until the HTTPS storage protocol is supported in clients as well. \ No newline at end of file diff --git a/newsfragments/3904.minor b/newsfragments/3904.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3910.minor b/newsfragments/3910.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3914.minor b/newsfragments/3914.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3917.minor b/newsfragments/3917.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3921.feature b/newsfragments/3921.feature deleted file mode 100644 index 798aee817..000000000 --- a/newsfragments/3921.feature +++ /dev/null @@ -1,5 +0,0 @@ -`tahoe run ...` will now exit when its stdin is closed. - -This facilitates subprocess management, specifically cleanup. -When a parent process is running tahoe and exits without time to do "proper" cleanup at least the stdin descriptor will be closed. -Subsequently "tahoe run" notices this and exits. \ No newline at end of file diff --git a/newsfragments/3922.documentation b/newsfragments/3922.documentation deleted file mode 100644 index d0232dd02..000000000 --- a/newsfragments/3922.documentation +++ /dev/null @@ -1 +0,0 @@ -Several minor errors in the Great Black Swamp proposed specification document have been fixed. \ No newline at end of file diff --git a/newsfragments/3927.minor b/newsfragments/3927.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3928.minor b/newsfragments/3928.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3935.minor b/newsfragments/3935.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3936.minor b/newsfragments/3936.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3937.minor b/newsfragments/3937.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3938.bugfix b/newsfragments/3938.bugfix deleted file mode 100644 index c2778cfdf..000000000 --- a/newsfragments/3938.bugfix +++ /dev/null @@ -1 +0,0 @@ -Work with (and require) newer versions of pycddl. \ No newline at end of file diff --git a/newsfragments/3939.bugfix b/newsfragments/3939.bugfix deleted file mode 100644 index 9d2071d32..000000000 --- a/newsfragments/3939.bugfix +++ /dev/null @@ -1 +0,0 @@ -Uploading immutables will now better use available bandwidth, which should allow for faster uploads in many cases. \ No newline at end of file diff --git a/newsfragments/3940.minor b/newsfragments/3940.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3942.minor b/newsfragments/3942.minor deleted file mode 100644 index 8b1378917..000000000 --- a/newsfragments/3942.minor +++ /dev/null @@ -1 +0,0 @@ - diff --git a/newsfragments/3944.minor b/newsfragments/3944.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3946.bugfix b/newsfragments/3946.bugfix deleted file mode 100644 index c17a098e7..000000000 --- a/newsfragments/3946.bugfix +++ /dev/null @@ -1 +0,0 @@ -Downloads of large immutables should now finish much faster. \ No newline at end of file diff --git a/newsfragments/3947.minor b/newsfragments/3947.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3950.minor b/newsfragments/3950.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3952.minor b/newsfragments/3952.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3953.minor b/newsfragments/3953.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3954.minor b/newsfragments/3954.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3956.minor b/newsfragments/3956.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3958.minor b/newsfragments/3958.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3959.minor b/newsfragments/3959.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3960.minor b/newsfragments/3960.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3961.other b/newsfragments/3961.other deleted file mode 100644 index 1b8085b30..000000000 --- a/newsfragments/3961.other +++ /dev/null @@ -1 +0,0 @@ -The integration test suite now includes a set of capability test vectors (``integration/vectors/test_vectors.yaml``) which can be used to verify compatibility between Tahoe-LAFS and other implementations. diff --git a/newsfragments/3962.feature b/newsfragments/3962.feature deleted file mode 100644 index 86cf62781..000000000 --- a/newsfragments/3962.feature +++ /dev/null @@ -1 +0,0 @@ -Mutable objects can now be created with a pre-determined "signature key" using the ``tahoe put`` CLI or the HTTP API. This enables deterministic creation of mutable capabilities. This feature must be used with care to preserve the normal security and reliability properties. \ No newline at end of file diff --git a/newsfragments/3964.removed b/newsfragments/3964.removed deleted file mode 100644 index d022f94af..000000000 --- a/newsfragments/3964.removed +++ /dev/null @@ -1 +0,0 @@ -Python 3.7 is no longer supported, and Debian 10 and Ubuntu 18.04 are no longer tested. \ No newline at end of file diff --git a/newsfragments/3965.minor b/newsfragments/3965.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3966.bugfix b/newsfragments/3966.bugfix deleted file mode 100644 index 384dcf797..000000000 --- a/newsfragments/3966.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix incompatibility with transitive dependency charset_normalizer >= 3 when using PyInstaller. diff --git a/newsfragments/3967.minor b/newsfragments/3967.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3968.minor b/newsfragments/3968.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3969.minor b/newsfragments/3969.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3970.minor b/newsfragments/3970.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3971.minor b/newsfragments/3971.minor deleted file mode 100644 index a6cbb6a89..000000000 --- a/newsfragments/3971.minor +++ /dev/null @@ -1 +0,0 @@ -Changes made to mypy.ini to make mypy more 'strict' and prevent future regressions. \ No newline at end of file diff --git a/newsfragments/3974.minor b/newsfragments/3974.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3975.minor b/newsfragments/3975.minor deleted file mode 100644 index 08fba6dd6..000000000 --- a/newsfragments/3975.minor +++ /dev/null @@ -1 +0,0 @@ -Fixes truthy conditional in status.py \ No newline at end of file diff --git a/newsfragments/3976.minor b/newsfragments/3976.minor deleted file mode 100644 index 4d6245e73..000000000 --- a/newsfragments/3976.minor +++ /dev/null @@ -1 +0,0 @@ -Fixes variable name same as built-in type. \ No newline at end of file diff --git a/newsfragments/3978.minor b/newsfragments/3978.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3982.feature b/newsfragments/3982.feature deleted file mode 100644 index 0d48fa476..000000000 --- a/newsfragments/3982.feature +++ /dev/null @@ -1 +0,0 @@ -Added support for Python 3.11. \ No newline at end of file diff --git a/newsfragments/3987.minor b/newsfragments/3987.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3988.minor b/newsfragments/3988.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3989.installation b/newsfragments/3989.installation deleted file mode 100644 index a2155b65c..000000000 --- a/newsfragments/3989.installation +++ /dev/null @@ -1 +0,0 @@ -tenacity is no longer a dependency. diff --git a/newsfragments/3991.minor b/newsfragments/3991.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3993.minor b/newsfragments/3993.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3994.minor b/newsfragments/3994.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3996.minor b/newsfragments/3996.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3997.installation b/newsfragments/3997.installation deleted file mode 100644 index 186be0fc2..000000000 --- a/newsfragments/3997.installation +++ /dev/null @@ -1 +0,0 @@ -Tahoe-LAFS is incompatible with cryptography >= 40 and now declares a requirement on an older version. diff --git a/newsfragments/3998.minor b/newsfragments/3998.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3999.bugfix b/newsfragments/3999.bugfix deleted file mode 100644 index a8a8396f4..000000000 --- a/newsfragments/3999.bugfix +++ /dev/null @@ -1 +0,0 @@ -A bug where Introducer nodes configured to listen on Tor or I2P would not actually do so has been fixed. \ No newline at end of file diff --git a/newsfragments/4000.minor b/newsfragments/4000.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/4001.minor b/newsfragments/4001.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/4002.minor b/newsfragments/4002.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/4003.minor b/newsfragments/4003.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/4004.minor b/newsfragments/4004.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/4005.minor b/newsfragments/4005.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/4006.minor b/newsfragments/4006.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/4009.minor b/newsfragments/4009.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/4010.minor b/newsfragments/4010.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/4012.minor b/newsfragments/4012.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/4014.minor b/newsfragments/4014.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/4015.minor b/newsfragments/4015.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/4016.minor b/newsfragments/4016.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/4018.minor b/newsfragments/4018.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/4019.minor b/newsfragments/4019.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/4020.minor b/newsfragments/4020.minor deleted file mode 100644 index 8b1378917..000000000 --- a/newsfragments/4020.minor +++ /dev/null @@ -1 +0,0 @@ - diff --git a/newsfragments/4022.minor b/newsfragments/4022.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/4023.minor b/newsfragments/4023.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/4024.minor b/newsfragments/4024.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/4026.minor b/newsfragments/4026.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/4027.minor b/newsfragments/4027.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/4028.minor b/newsfragments/4028.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/4029.bugfix b/newsfragments/4029.bugfix deleted file mode 100644 index 3ce4670ec..000000000 --- a/newsfragments/4029.bugfix +++ /dev/null @@ -1,2 +0,0 @@ -The (still off-by-default) HTTP storage client will now use Tor when Tor-based client-side anonymity was requested. -Previously it would use normal TCP connections and not be anonymous. \ No newline at end of file diff --git a/newsfragments/4035.minor b/newsfragments/4035.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/4036.feature b/newsfragments/4036.feature deleted file mode 100644 index 36c062718..000000000 --- a/newsfragments/4036.feature +++ /dev/null @@ -1 +0,0 @@ -tahoe run now accepts --allow-stdin-close to mean "keep running if stdin closes" \ No newline at end of file diff --git a/newsfragments/4038.minor b/newsfragments/4038.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/4039.documentation b/newsfragments/4039.documentation deleted file mode 100644 index 33257443b..000000000 --- a/newsfragments/4039.documentation +++ /dev/null @@ -1 +0,0 @@ -Document the ``force_foolscap`` configuration options for ``[storage]`` and ``[client]``. \ No newline at end of file diff --git a/newsfragments/4040.minor b/newsfragments/4040.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/4041.feature b/newsfragments/4041.feature deleted file mode 100644 index 7d8df1a23..000000000 --- a/newsfragments/4041.feature +++ /dev/null @@ -1 +0,0 @@ -The storage server and client now support a new, HTTPS-based protocol. \ No newline at end of file diff --git a/newsfragments/4042.minor b/newsfragments/4042.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/4044.minor b/newsfragments/4044.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/4046.minor b/newsfragments/4046.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/4047.minor b/newsfragments/4047.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/4049.minor b/newsfragments/4049.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/4050.minor b/newsfragments/4050.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/4051.minor b/newsfragments/4051.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/4052.minor b/newsfragments/4052.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/4055.minor b/newsfragments/4055.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/4056.bugfix b/newsfragments/4056.bugfix deleted file mode 100644 index 7e637b48c..000000000 --- a/newsfragments/4056.bugfix +++ /dev/null @@ -1,3 +0,0 @@ -Provide our own copy of attrs' "provides()" validator - -This validator is deprecated and slated for removal; that project's suggestion is to copy the code to our project. diff --git a/newsfragments/4059.minor b/newsfragments/4059.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/4060.feature b/newsfragments/4060.feature deleted file mode 100644 index 5eea8134d..000000000 --- a/newsfragments/4060.feature +++ /dev/null @@ -1 +0,0 @@ -Started work on a new end-to-end benchmarking framework. \ No newline at end of file diff --git a/newsfragments/4061.minor b/newsfragments/4061.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/4062.minor b/newsfragments/4062.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/4063.minor b/newsfragments/4063.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/4065.minor b/newsfragments/4065.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/4066.minor b/newsfragments/4066.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/4068.feature b/newsfragments/4068.feature deleted file mode 100644 index 6c5530cfd..000000000 --- a/newsfragments/4068.feature +++ /dev/null @@ -1 +0,0 @@ -Some operations now run in threads, improving the responsiveness of Tahoe nodes. \ No newline at end of file diff --git a/newsfragments/4070.minor b/newsfragments/4070.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/4074.minor b/newsfragments/4074.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/4075.minor b/newsfragments/4075.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/4078.bugfix b/newsfragments/4078.bugfix deleted file mode 100644 index 12ca66bcf..000000000 --- a/newsfragments/4078.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a race condition with SegmentFetcher \ No newline at end of file diff --git a/newsfragments/4804.feature b/newsfragments/4804.feature deleted file mode 100644 index 23b3d3c6e..000000000 --- a/newsfragments/4804.feature +++ /dev/null @@ -1 +0,0 @@ -Logs are now written in a thread, which should make the application more responsive under load. \ No newline at end of file From 9f3a3bc1e86e325a23692a11a6fa05f4d20bae10 Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 4 Jan 2024 20:06:00 -0700 Subject: [PATCH 2243/2309] changelog for 1.19.0 --- newsfragments/4076.minor | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/4076.minor diff --git a/newsfragments/4076.minor b/newsfragments/4076.minor new file mode 100644 index 000000000..2fec812e5 --- /dev/null +++ b/newsfragments/4076.minor @@ -0,0 +1 @@ +Release 1.19.0 From 164ac29bd8a7761635aff641c640ac4e905e9e21 Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 4 Jan 2024 20:08:10 -0700 Subject: [PATCH 2244/2309] fix news --- NEWS.rst | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/NEWS.rst b/NEWS.rst index c0c36adad..b9ad2b7bb 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -5,14 +5,8 @@ User-Visible Changes in Tahoe-LAFS ================================== .. towncrier start line -Release 1.18.0.post1720.dev0 (2024-01-04) -''''''''''''''''''''''''''''''''''''''''' - -No significant changes. - - -Release 1.18.0.post1720 (2024-01-04) -'''''''''''''''''''''''''''''''''''' +Release 1.190 (2024-01-04) +'''''''''''''''''''''''''' Features -------- From bbaa7cb9ea4f092dd9393c32dff5b29a4d492322 Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 4 Jan 2024 20:12:56 -0700 Subject: [PATCH 2245/2309] release notes --- relnotes.txt | 42 +++++++++++++++++++++++++++++++----------- 1 file changed, 31 insertions(+), 11 deletions(-) diff --git a/relnotes.txt b/relnotes.txt index dd7cc9429..3f85e256c 100644 --- a/relnotes.txt +++ b/relnotes.txt @@ -1,6 +1,6 @@ -ANNOUNCING Tahoe, the Least-Authority File Store, v1.18.0 +ANNOUNCING Tahoe, the Least-Authority File Store, v1.19.0 -The Tahoe-LAFS team is pleased to announce version 1.18.0 of +The Tahoe-LAFS team is pleased to announce version 1.19.0 of Tahoe-LAFS, an extremely reliable decentralized storage system. Get it with "pip install tahoe-lafs", or download a tarball here: @@ -15,12 +15,32 @@ unique security and fault-tolerance properties: https://tahoe-lafs.readthedocs.org/en/latest/about.html -The previous stable release of Tahoe-LAFS was v1.17.1, released on -January 7, 2022. +The previous stable release of Tahoe-LAFS was v1.18.0, released on +October 2, 2022. Major new features and changes in this release: -This release drops support for Python 2 and for Python 3.6 and earlier. -twistd.pid is no longer used (in favour of one with pid + process creation time). -A collection of minor bugs and issues were also fixed. +A new "Grid Manager" feature allows clients to specify any number of +parties whom they will use to limit which storage-server that client +talks to. See docs/managed-grid.rst for more. + +The new HTTP-based "Great Black Swamp" protocol is now enabled +(replacing Foolscap). This allows integrators to start with their +favourite HTTP library (instead of implementing Foolscap first). Both +storage-servers and clients support this new protocol. + +`tahoe run` will now exit if its stdin is closed (but accepts --allow-stdin-close now). + +Mutables may be created with a pre-determined signature key; care must +be taken! + +This release drops Python 3.7 support and adds Python 3.11 +support. Several performance improvements have been made. Introducer +correctly listens on Tor or I2P. Debian 10 and Ubuntu 20.04 are no +longer tested. + +Besides all this there have been dozens of other bug-fixes and +improvements. + +Enjoy! Please see ``NEWS.rst`` [1] for a complete list of changes. @@ -145,12 +165,12 @@ October 1, 2022 Planet Earth -[1] https://github.com/tahoe-lafs/tahoe-lafs/blob/tahoe-lafs-1.18.0/NEWS.rst +[1] https://github.com/tahoe-lafs/tahoe-lafs/blob/tahoe-lafs-1.19.0/NEWS.rst [2] https://github.com/tahoe-lafs/tahoe-lafs/blob/master/docs/known_issues.rst [3] https://tahoe-lafs.org/trac/tahoe-lafs/wiki/RelatedProjects -[4] https://github.com/tahoe-lafs/tahoe-lafs/blob/tahoe-lafs-1.18.0/COPYING.GPL -[5] https://github.com/tahoe-lafs/tahoe-lafs/blob/tahoe-lafs-1.18.0/COPYING.TGPPL.rst -[6] https://tahoe-lafs.readthedocs.org/en/tahoe-lafs-1.18.0/INSTALL.html +[4] https://github.com/tahoe-lafs/tahoe-lafs/blob/tahoe-lafs-1.19.0/COPYING.GPL +[5] https://github.com/tahoe-lafs/tahoe-lafs/blob/tahoe-lafs-1.19.0/COPYING.TGPPL.rst +[6] https://tahoe-lafs.readthedocs.org/en/tahoe-lafs-1.19.0/INSTALL.html [7] https://lists.tahoe-lafs.org/mailman/listinfo/tahoe-dev [8] https://tahoe-lafs.org/trac/tahoe-lafs/roadmap [9] https://github.com/tahoe-lafs/tahoe-lafs/blob/master/CREDITS From 9570e4b199af845e4b10bff496f7eec4b1571e37 Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 4 Jan 2024 20:13:35 -0700 Subject: [PATCH 2246/2309] update nix --- nix/tahoe-lafs.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nix/tahoe-lafs.nix b/nix/tahoe-lafs.nix index dbc4f37f9..273fa3a76 100644 --- a/nix/tahoe-lafs.nix +++ b/nix/tahoe-lafs.nix @@ -1,6 +1,6 @@ let pname = "tahoe-lafs"; - version = "1.18.0.post1"; + version = "1.19.0.post1"; in { lib , pythonPackages From 20b4f8958148cf3b08f13dce386cb8744251f93c Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 8 Jan 2024 10:08:57 -0500 Subject: [PATCH 2247/2309] Hopefully fix dirty reactor tests. --- src/allmydata/test/common_system.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/allmydata/test/common_system.py b/src/allmydata/test/common_system.py index 83ae87883..27c11f660 100644 --- a/src/allmydata/test/common_system.py +++ b/src/allmydata/test/common_system.py @@ -700,6 +700,8 @@ class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin): self.sparent.startService() def _got_new_http_connection_pool(self, pool): + # Make sure the pool closes cached connections quickly: + pool.cachedConnectionTimeout = 0.1 # Register the pool for shutdown later: self._http_client_pools.append(pool) # Disable retries: From e49435927cf64ad59a0123de136e25fef4e112ba Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 8 Jan 2024 10:09:21 -0500 Subject: [PATCH 2248/2309] News file --- newsfragments/4085.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/4085.minor diff --git a/newsfragments/4085.minor b/newsfragments/4085.minor new file mode 100644 index 000000000..e69de29bb From 51e9fcab56a3cc9d1cc82bf1df218c189ca274e7 Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 18 Jan 2024 16:03:12 -0700 Subject: [PATCH 2249/2309] fixups from review (thanks itamarst) --- NEWS.rst | 2 -- relnotes.txt | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/NEWS.rst b/NEWS.rst index b9ad2b7bb..d5dadf06d 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -13,7 +13,6 @@ Features - Tahoe-LAFS now includes a new "Grid Manager" specification and implementation adding more options to control which storage servers a client will use for uploads. (`#2916 `_) - Added support for Python 3.12, and work with Eliot 1.15 (`#3072 `_) -- The new HTTPS-based storage server is now enabled transparently on the same port as the Foolscap server. This will not have any user-facing impact until the HTTPS storage protocol is supported in clients as well. (`#3902 `_) - `tahoe run ...` will now exit when its stdin is closed. This facilitates subprocess management, specifically cleanup. @@ -52,7 +51,6 @@ Dependency/Installation Changes ------------------------------- - tenacity is no longer a dependency. (`#3989 `_) -- Tahoe-LAFS is incompatible with cryptography >= 40 and now declares a requirement on an older version. (`#3997 `_) Documentation Changes diff --git a/relnotes.txt b/relnotes.txt index 3f85e256c..d8caa5e6e 100644 --- a/relnotes.txt +++ b/relnotes.txt @@ -32,7 +32,7 @@ storage-servers and clients support this new protocol. Mutables may be created with a pre-determined signature key; care must be taken! -This release drops Python 3.7 support and adds Python 3.11 +This release drops Python 3.7 support and adds Python 3.11 and 3.12 support. Several performance improvements have been made. Introducer correctly listens on Tor or I2P. Debian 10 and Ubuntu 20.04 are no longer tested. From f1833906ff0a7ae23c325f27deddc19c8fc82869 Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 18 Jan 2024 17:32:18 -0700 Subject: [PATCH 2250/2309] details about how to publish --- docs/release-checklist.rst | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/docs/release-checklist.rst b/docs/release-checklist.rst index aa5531b59..e02844d67 100644 --- a/docs/release-checklist.rst +++ b/docs/release-checklist.rst @@ -147,7 +147,7 @@ they will need to evaluate which contributors' signatures they trust. - when satisfied, sign the tarballs: - - gpg --pinentry=loopback --armor --detach-sign dist/tahoe_lafs-1.16.0rc0-py2.py3-none-any.whl + - gpg --pinentry=loopback --armor -u 0xE34E62D06D0E69CFCA4179FFBDE0D31D68666A7A --detach-sign dist/tahoe_lafs-1.16.0rc0-py2.py3-none-any.whl - gpg --pinentry=loopback --armor --detach-sign dist/tahoe_lafs-1.16.0rc0.tar.gz @@ -197,13 +197,15 @@ Push the signed tag to the main repository: For the actual release, the tarball and signature files need to be uploaded to PyPI as well. -- how to do this? -- (original guide says only `twine upload dist/*`) -- the following developers have access to do this: +- ls dist/*1.19.0* +- twine upload --username __token__ --password `cat SECRET-pypi-tahoe-publish-token` dist/*1.19.0* + +The following developers have access to do this: - warner + - meejah - exarkun (partial?) - - meejah (partial?) + Announcing the Release Candidate ```````````````````````````````` From 6b7d1e985c6f84db185d31483fa16c1e76eef829 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 19 Jan 2024 12:09:20 -0700 Subject: [PATCH 2251/2309] test for 4087 --- src/allmydata/test/test_storage_http.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index dba7ee5d2..5dfb7573b 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -41,6 +41,7 @@ from werkzeug import routing from werkzeug.exceptions import NotFound as WNotFound from testtools.matchers import Equals from zope.interface import implementer +import cbor2 from ..util.deferredutil import async_to_deferred from ..util.cputhreadpool import disable_thread_pool_for_test @@ -1835,3 +1836,14 @@ class MutableSharedTests(SharedImmutableMutableTestsMixin, SyncTestCase): A read with no range returns the whole mutable. """ return self._read_with_no_range_test(data_length) + + def test_roundtrip_cbor2_encoding_issue(self): + """ + Some versions of cbor2 (5.6.0) don't correctly encode bytestrings + bigger than 65535 + """ + for size in range(0, 65535*2, 17): + self.assertEqual( + size, + len(cbor2.loads(cbor2.dumps(b"\12" * size))) + ) From 741f6182f8a9494d834ee499e5c9571e93cbfe58 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 19 Jan 2024 12:10:13 -0700 Subject: [PATCH 2252/2309] exclude cbor2 5.6.0 --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index a6b4cb961..34d436a49 100644 --- a/setup.py +++ b/setup.py @@ -143,7 +143,8 @@ install_requires = [ # 2.2.0 has a bug: https://github.com/pallets/werkzeug/issues/2465 "werkzeug != 2.2.0", "treq", - "cbor2", + # 5.6.0 excluded because https://github.com/agronholm/cbor2/issues/208 + "cbor2 != 5.6.0", # 0.4 adds the ability to pass in mmap() values which greatly reduces the # amount of copying involved. From 52a27a6fa14c5cac993f2732d095249b01ca3c33 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 19 Jan 2024 12:10:38 -0700 Subject: [PATCH 2253/2309] news --- newsfragments/4087.bugfix | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/4087.bugfix diff --git a/newsfragments/4087.bugfix b/newsfragments/4087.bugfix new file mode 100644 index 000000000..e69de29bb From 1db76da2e234d742b9206cec92161a583fdbe480 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 24 Jan 2024 07:55:57 -0500 Subject: [PATCH 2254/2309] Centralize cbor dumping and loading, for better control. --- src/allmydata/storage/http_client.py | 3 +-- src/allmydata/storage/http_server.py | 9 ++++----- src/allmydata/test/test_storage_http.py | 5 ++--- src/allmydata/util/cbor.py | 7 +++++++ 4 files changed, 14 insertions(+), 10 deletions(-) create mode 100644 src/allmydata/util/cbor.py diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index a98b097dd..10f943fe1 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -26,8 +26,6 @@ from attrs import define, asdict, frozen, field from eliot import start_action, register_exception_extractor from eliot.twisted import DeferredContext -# TODO Make sure to import Python version? -from cbor2 import loads, dumps from pycddl import Schema from collections_extended import RangeMap from werkzeug.datastructures import Range, ContentRange @@ -65,6 +63,7 @@ from ..util.hashutil import timing_safe_compare from ..util.deferredutil import async_to_deferred from ..util.tor_provider import _Provider as TorProvider from ..util.cputhreadpool import defer_to_thread +from ..util.cbor import dumps, loads try: from txtorcon import Tor # type: ignore diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 78c5ac5d2..324584f01 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -57,8 +57,6 @@ from hyperlink import DecodedURL from cryptography.x509 import load_pem_x509_certificate -# TODO Make sure to use pure Python versions? -import cbor2 from pycddl import Schema, ValidationError as CDDLValidationError from .server import StorageServer from .http_common import ( @@ -75,7 +73,8 @@ from ..util.hashutil import timing_safe_compare from ..util.base32 import rfc3548_alphabet from ..util.deferredutil import async_to_deferred from ..util.cputhreadpool import defer_to_thread -from allmydata.interfaces import BadWriteEnablerError +from ..util import cbor +from ..interfaces import BadWriteEnablerError class ClientSecretsException(Exception): @@ -647,7 +646,7 @@ async def read_encoded( # Typically deserialization to Python will not release the GIL, and # indeed as of Jan 2023 cbor2 didn't have any code to release the GIL # in the decode path. As such, running it in a different thread has no benefit. - return cbor2.load(request.content) + return cbor.load(request.content) class HTTPServer(BaseApp): @@ -695,7 +694,7 @@ class HTTPServer(BaseApp): if accept.best == CBOR_MIME_TYPE: request.setHeader("Content-Type", CBOR_MIME_TYPE) f = TemporaryFile() - cbor2.dump(data, f) # type: ignore + cbor.dump(data, f) # type: ignore def read_data(offset: int, length: int) -> bytes: f.seek(offset) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 5dfb7573b..066ec8e84 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -24,7 +24,6 @@ from contextlib import contextmanager from os import urandom from typing import Union, Callable, Tuple, Iterable from queue import Queue -from cbor2 import dumps from pycddl import ValidationError as CDDLValidationError from hypothesis import assume, given, strategies as st, settings as hypothesis_settings from fixtures import Fixture, TempDir, MonkeyPatch @@ -41,8 +40,8 @@ from werkzeug import routing from werkzeug.exceptions import NotFound as WNotFound from testtools.matchers import Equals from zope.interface import implementer -import cbor2 +from ..util.cbor import dumps, loads from ..util.deferredutil import async_to_deferred from ..util.cputhreadpool import disable_thread_pool_for_test from .common import SyncTestCase @@ -1845,5 +1844,5 @@ class MutableSharedTests(SharedImmutableMutableTestsMixin, SyncTestCase): for size in range(0, 65535*2, 17): self.assertEqual( size, - len(cbor2.loads(cbor2.dumps(b"\12" * size))) + len(loads(dumps(b"\12" * size))) ) diff --git a/src/allmydata/util/cbor.py b/src/allmydata/util/cbor.py new file mode 100644 index 000000000..527639903 --- /dev/null +++ b/src/allmydata/util/cbor.py @@ -0,0 +1,7 @@ +""" +Unified entry point for CBOR encoding and decoding. +""" + +from cbor2 import dumps, loads, dump, load + +__all__ = ["dumps", "loads", "dump", "load"] From f0ce6e9d688d84622cb255255a0db75cb6d57c12 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 24 Jan 2024 08:05:29 -0500 Subject: [PATCH 2255/2309] Use Python version of CBOR loading. --- src/allmydata/util/cbor.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/src/allmydata/util/cbor.py b/src/allmydata/util/cbor.py index 527639903..9ccde0ed6 100644 --- a/src/allmydata/util/cbor.py +++ b/src/allmydata/util/cbor.py @@ -2,6 +2,20 @@ Unified entry point for CBOR encoding and decoding. """ -from cbor2 import dumps, loads, dump, load +import sys + +# We don't want to use the C extension for loading, at least for now, but using +# it for dumping should be fine. +from cbor2 import dumps, dump + +# Now, override the C extension so we can import the Python versions of loading +# functions. +del sys.modules["cbor2"] +sys.modules["_cbor2"] = None +from cbor2 import load, loads + +# Quick validation that we got the Python version, not the C version. +assert type(load) == type(lambda: None), repr(load) +assert type(loads) == type(lambda: None), repr(loads) __all__ = ["dumps", "loads", "dump", "load"] From ff6e3ed56b8a1a23c93fd5833b401ef82effac8b Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 24 Jan 2024 08:06:07 -0500 Subject: [PATCH 2256/2309] News fragment. --- newsfragments/4088.bugfix | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/4088.bugfix diff --git a/newsfragments/4088.bugfix b/newsfragments/4088.bugfix new file mode 100644 index 000000000..765bdc24f --- /dev/null +++ b/newsfragments/4088.bugfix @@ -0,0 +1 @@ +Stop using the C version of the cbor2 decoder. \ No newline at end of file From 68d63fde27ceff0cae19fea21c364ac66623daca Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 24 Jan 2024 08:11:45 -0500 Subject: [PATCH 2257/2309] Pacify mypy. --- src/allmydata/util/cbor.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/allmydata/util/cbor.py b/src/allmydata/util/cbor.py index 9ccde0ed6..aa67bd7e3 100644 --- a/src/allmydata/util/cbor.py +++ b/src/allmydata/util/cbor.py @@ -11,11 +11,11 @@ from cbor2 import dumps, dump # Now, override the C extension so we can import the Python versions of loading # functions. del sys.modules["cbor2"] -sys.modules["_cbor2"] = None +sys.modules["_cbor2"] = None # type: ignore[assignment] from cbor2 import load, loads # Quick validation that we got the Python version, not the C version. -assert type(load) == type(lambda: None), repr(load) -assert type(loads) == type(lambda: None), repr(loads) +assert type(load) == type(lambda: None), repr(load) # type: ignore[comparison-overlap] +assert type(loads) == type(lambda: None), repr(loads) # type: ignore[comparison-overlap] __all__ = ["dumps", "loads", "dump", "load"] From fced1ab01bb55bc58f521e544d8a2e450a3a9ceb Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 24 Jan 2024 13:50:55 -0500 Subject: [PATCH 2258/2309] Switch to using pycddl for CBOR decoding. --- setup.py | 5 ++--- src/allmydata/storage/http_client.py | 5 ++--- src/allmydata/storage/http_server.py | 13 ++----------- src/allmydata/test/test_storage_http.py | 13 +------------ src/allmydata/util/cbor.py | 18 ++++++++---------- 5 files changed, 15 insertions(+), 39 deletions(-) diff --git a/setup.py b/setup.py index 34d436a49..6011b45fb 100644 --- a/setup.py +++ b/setup.py @@ -146,9 +146,8 @@ install_requires = [ # 5.6.0 excluded because https://github.com/agronholm/cbor2/issues/208 "cbor2 != 5.6.0", - # 0.4 adds the ability to pass in mmap() values which greatly reduces the - # amount of copying involved. - "pycddl >= 0.4", + # 0.6 adds the ability to decode CBOR. + "pycddl >= 0.6", # Command-line parsing "click >= 8.1.1", diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py index 10f943fe1..f0570a4d6 100644 --- a/src/allmydata/storage/http_client.py +++ b/src/allmydata/storage/http_client.py @@ -63,7 +63,7 @@ from ..util.hashutil import timing_safe_compare from ..util.deferredutil import async_to_deferred from ..util.tor_provider import _Provider as TorProvider from ..util.cputhreadpool import defer_to_thread -from ..util.cbor import dumps, loads +from ..util.cbor import dumps try: from txtorcon import Tor # type: ignore @@ -560,8 +560,7 @@ class StorageClient(object): data = f.read() def validate_and_decode(): - schema.validate_cbor(data) - return loads(data) + return schema.validate_cbor(data, True) return await defer_to_thread(validate_and_decode) else: diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index 324584f01..2e1a6a413 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -637,17 +637,8 @@ async def read_encoded( # Pycddl will release the GIL when validating larger documents, so # let's take advantage of multiple CPUs: - await defer_to_thread(schema.validate_cbor, message) - - # The CBOR parser will allocate more memory, but at least we can feed - # it the file-like object, so that if it's large it won't be make two - # copies. - request.content.seek(SEEK_SET, 0) - # Typically deserialization to Python will not release the GIL, and - # indeed as of Jan 2023 cbor2 didn't have any code to release the GIL - # in the decode path. As such, running it in a different thread has no benefit. - return cbor.load(request.content) - + decoded = await defer_to_thread(schema.validate_cbor, message, True) + return decoded class HTTPServer(BaseApp): """ diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 066ec8e84..185cfa995 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -41,7 +41,7 @@ from werkzeug.exceptions import NotFound as WNotFound from testtools.matchers import Equals from zope.interface import implementer -from ..util.cbor import dumps, loads +from ..util.cbor import dumps from ..util.deferredutil import async_to_deferred from ..util.cputhreadpool import disable_thread_pool_for_test from .common import SyncTestCase @@ -1835,14 +1835,3 @@ class MutableSharedTests(SharedImmutableMutableTestsMixin, SyncTestCase): A read with no range returns the whole mutable. """ return self._read_with_no_range_test(data_length) - - def test_roundtrip_cbor2_encoding_issue(self): - """ - Some versions of cbor2 (5.6.0) don't correctly encode bytestrings - bigger than 65535 - """ - for size in range(0, 65535*2, 17): - self.assertEqual( - size, - len(loads(dumps(b"\12" * size))) - ) diff --git a/src/allmydata/util/cbor.py b/src/allmydata/util/cbor.py index aa67bd7e3..a4b33ecec 100644 --- a/src/allmydata/util/cbor.py +++ b/src/allmydata/util/cbor.py @@ -1,21 +1,19 @@ """ Unified entry point for CBOR encoding and decoding. -""" -import sys +Makes it less likely to use ``cbor2.loads()`` by mistake, which we want to avoid. +""" # We don't want to use the C extension for loading, at least for now, but using # it for dumping should be fine. from cbor2 import dumps, dump -# Now, override the C extension so we can import the Python versions of loading -# functions. -del sys.modules["cbor2"] -sys.modules["_cbor2"] = None # type: ignore[assignment] -from cbor2 import load, loads +def load(*args, **kwargs): + """ + Don't use this! Here just in case someone uses it by mistake. + """ + raise RuntimeError("Use pycddl for decoding CBOR") -# Quick validation that we got the Python version, not the C version. -assert type(load) == type(lambda: None), repr(load) # type: ignore[comparison-overlap] -assert type(loads) == type(lambda: None), repr(loads) # type: ignore[comparison-overlap] +loads = load __all__ = ["dumps", "loads", "dump", "load"] From b52f0251d5a6bbecdef0fd72f8c8dc1ef6eeeb8e Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 24 Jan 2024 15:03:30 -0500 Subject: [PATCH 2259/2309] Update nix packaging. --- nix/pycddl.nix | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nix/pycddl.nix b/nix/pycddl.nix index 4c68830d4..a8c0799d1 100644 --- a/nix/pycddl.nix +++ b/nix/pycddl.nix @@ -30,12 +30,12 @@ { lib, fetchPypi, python, buildPythonPackage, rustPlatform }: buildPythonPackage rec { pname = "pycddl"; - version = "0.4.0"; + version = "0.6.0"; format = "pyproject"; src = fetchPypi { inherit pname version; - sha256 = "sha256-w0CGbPeiXyS74HqZXyiXhvaAMUaIj5onwjl9gWKAjqY="; + sha256 = "sha256-kmXXJAHkP4Wltp01Os5DPlygEI7nxd0FdaFqdD43X3g="; }; # Without this, when building for PyPy, `maturin build` seems to fail to @@ -52,6 +52,6 @@ buildPythonPackage rec { cargoDeps = rustPlatform.fetchCargoTarball { inherit src; name = "${pname}-${version}"; - hash = "sha256-g96eeaqN9taPED4u+UKUcoitf5aTGFrW2/TOHoHEVHs="; + hash = "sha256-PjAcAf7T03hKmBhDlXJdkwCkiGNfzc1ajukhf+tFpMo="; }; } From f2f7c1dd48bede4e8d699d3a8eea62411bd85f68 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 26 Jan 2024 09:24:59 -0500 Subject: [PATCH 2260/2309] Fix PyPy. --- nix/pycddl.nix | 6 +++--- setup.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/nix/pycddl.nix b/nix/pycddl.nix index a8c0799d1..8b214a91b 100644 --- a/nix/pycddl.nix +++ b/nix/pycddl.nix @@ -30,12 +30,12 @@ { lib, fetchPypi, python, buildPythonPackage, rustPlatform }: buildPythonPackage rec { pname = "pycddl"; - version = "0.6.0"; + version = "0.6.1"; format = "pyproject"; src = fetchPypi { inherit pname version; - sha256 = "sha256-kmXXJAHkP4Wltp01Os5DPlygEI7nxd0FdaFqdD43X3g="; + sha256 = "sha256-63fe8UJXEH6t4l7ujV8JDvlGb7q3kL6fHHATFdklzFc="; }; # Without this, when building for PyPy, `maturin build` seems to fail to @@ -52,6 +52,6 @@ buildPythonPackage rec { cargoDeps = rustPlatform.fetchCargoTarball { inherit src; name = "${pname}-${version}"; - hash = "sha256-PjAcAf7T03hKmBhDlXJdkwCkiGNfzc1ajukhf+tFpMo="; + hash = "sha256-ssDEKRd3Y9/10oXBZHCxvlRkl9KMh3pGYbCkM4rXThQ="; }; } diff --git a/setup.py b/setup.py index 6011b45fb..71be1e2e1 100644 --- a/setup.py +++ b/setup.py @@ -146,8 +146,8 @@ install_requires = [ # 5.6.0 excluded because https://github.com/agronholm/cbor2/issues/208 "cbor2 != 5.6.0", - # 0.6 adds the ability to decode CBOR. - "pycddl >= 0.6", + # 0.6 adds the ability to decode CBOR. 0.6.1 fixes PyPy. + "pycddl >= 0.6.1", # Command-line parsing "click >= 8.1.1", From b856238110c8c0a2b9cfa6d47302b330fa0ab043 Mon Sep 17 00:00:00 2001 From: Alexandre Detiste Date: Thu, 15 Feb 2024 16:53:34 +0100 Subject: [PATCH 2261/2309] remove old Python2 __future__ statements --- integration/test_aaa_aardvark.py | 4 ---- integration/test_sftp.py | 4 ---- integration/test_streaming_logs.py | 6 ------ misc/awesome_weird_stuff/boodlegrid.tac | 1 - misc/build_helpers/check-build.py | 1 - misc/build_helpers/gen-package-table.py | 1 - misc/build_helpers/run-deprecations.py | 1 - misc/build_helpers/test-osx-pkg.py | 1 - misc/checkers/check_grid.py | 1 - misc/coding_tools/check-debugging.py | 1 - misc/coding_tools/check-interfaces.py | 1 - misc/coding_tools/check-umids.py | 1 - misc/coding_tools/graph-deps.py | 1 - misc/coding_tools/make-canary-files.py | 1 - misc/coding_tools/make_umid | 1 - misc/operations_helpers/cpu-watcher-poll.py | 1 - misc/operations_helpers/cpu-watcher-subscribe.py | 1 - misc/operations_helpers/cpu-watcher.tac | 1 - misc/operations_helpers/find-share-anomalies.py | 1 - misc/operations_helpers/munin/tahoe_cpu_watcher | 1 - misc/operations_helpers/munin/tahoe_diskleft | 1 - misc/operations_helpers/munin/tahoe_disktotal | 1 - misc/operations_helpers/munin/tahoe_diskusage | 1 - misc/operations_helpers/munin/tahoe_diskused | 1 - misc/operations_helpers/munin/tahoe_doomsday | 1 - misc/operations_helpers/munin/tahoe_estimate_files | 1 - misc/operations_helpers/munin/tahoe_files | 1 - misc/operations_helpers/munin/tahoe_helperstats_active | 1 - misc/operations_helpers/munin/tahoe_helperstats_fetched | 1 - misc/operations_helpers/munin/tahoe_introstats | 1 - misc/operations_helpers/munin/tahoe_nodememory | 1 - misc/operations_helpers/munin/tahoe_overhead | 1 - misc/operations_helpers/munin/tahoe_rootdir_space | 1 - misc/operations_helpers/munin/tahoe_server_latency_ | 1 - misc/operations_helpers/munin/tahoe_server_operations_ | 1 - misc/operations_helpers/munin/tahoe_spacetime | 1 - misc/operations_helpers/munin/tahoe_stats | 1 - misc/operations_helpers/munin/tahoe_storagespace | 1 - misc/operations_helpers/provisioning/reliability.py | 1 - misc/operations_helpers/provisioning/test_provisioning.py | 1 - misc/operations_helpers/spacetime/diskwatcher.tac | 1 - misc/simulators/bench_spans.py | 1 - misc/simulators/count_dirs.py | 1 - misc/simulators/hashbasedsig.py | 1 - misc/simulators/ringsim.py | 1 - misc/simulators/simulate_load.py | 1 - misc/simulators/simulator.py | 1 - misc/simulators/sizes.py | 1 - misc/simulators/storage-overhead.py | 1 - pyinstaller.spec | 1 - src/allmydata/__main__.py | 4 ---- src/allmydata/_monkeypatch.py | 4 ---- src/allmydata/blacklist.py | 4 ---- src/allmydata/check_results.py | 4 ---- src/allmydata/codec.py | 4 ---- src/allmydata/crypto/__init__.py | 4 ---- src/allmydata/crypto/aes.py | 4 ---- src/allmydata/crypto/error.py | 4 ---- src/allmydata/crypto/util.py | 4 ---- src/allmydata/deep_stats.py | 4 ---- src/allmydata/dirnode.py | 4 ---- src/allmydata/frontends/auth.py | 4 ---- src/allmydata/frontends/sftpd.py | 4 ---- src/allmydata/hashtree.py | 4 ---- src/allmydata/history.py | 4 ---- src/allmydata/immutable/checker.py | 4 ---- src/allmydata/immutable/downloader/__init__.py | 4 ---- src/allmydata/immutable/downloader/common.py | 4 ---- src/allmydata/immutable/downloader/fetcher.py | 4 ---- src/allmydata/immutable/downloader/finder.py | 4 ---- src/allmydata/immutable/downloader/node.py | 4 ---- src/allmydata/immutable/downloader/segmentation.py | 4 ---- src/allmydata/immutable/downloader/share.py | 4 ---- src/allmydata/immutable/downloader/status.py | 4 ---- src/allmydata/immutable/encode.py | 4 ---- src/allmydata/immutable/filenode.py | 4 ---- src/allmydata/immutable/happiness_upload.py | 4 ---- src/allmydata/immutable/literal.py | 4 ---- src/allmydata/immutable/offloaded.py | 4 ---- src/allmydata/immutable/repairer.py | 4 ---- src/allmydata/interfaces.py | 4 ---- src/allmydata/introducer/__init__.py | 4 ---- src/allmydata/introducer/client.py | 4 ---- src/allmydata/introducer/common.py | 4 ---- src/allmydata/introducer/interfaces.py | 4 ---- src/allmydata/monitor.py | 4 ---- src/allmydata/mutable/checker.py | 4 ---- src/allmydata/mutable/layout.py | 4 ---- src/allmydata/mutable/publish.py | 4 ---- src/allmydata/mutable/repairer.py | 4 ---- src/allmydata/scripts/admin.py | 4 ---- src/allmydata/scripts/backupdb.py | 4 ---- src/allmydata/scripts/default_nodedir.py | 4 ---- src/allmydata/scripts/slow_operation.py | 4 ---- src/allmydata/scripts/tahoe_add_alias.py | 4 ---- src/allmydata/scripts/tahoe_backup.py | 4 ---- src/allmydata/scripts/tahoe_check.py | 4 ---- src/allmydata/scripts/tahoe_cp.py | 4 ---- src/allmydata/scripts/tahoe_get.py | 4 ---- src/allmydata/scripts/tahoe_ls.py | 4 ---- src/allmydata/scripts/tahoe_manifest.py | 4 ---- src/allmydata/scripts/tahoe_mkdir.py | 4 ---- src/allmydata/scripts/tahoe_mv.py | 4 ---- src/allmydata/scripts/tahoe_run.py | 4 ---- src/allmydata/scripts/tahoe_status.py | 4 ---- src/allmydata/scripts/tahoe_unlink.py | 4 ---- src/allmydata/scripts/tahoe_webopen.py | 4 ---- src/allmydata/storage/common.py | 4 ---- src/allmydata/storage/crawler.py | 4 ---- src/allmydata/storage/expirer.py | 4 ---- src/allmydata/storage/immutable.py | 4 ---- src/allmydata/storage/immutable_schema.py | 4 ---- src/allmydata/storage/lease.py | 4 ---- src/allmydata/storage/mutable.py | 4 ---- src/allmydata/storage/mutable_schema.py | 4 ---- src/allmydata/storage/shares.py | 4 ---- src/allmydata/test/__init__.py | 4 ---- src/allmydata/test/cli/common.py | 4 ---- src/allmydata/test/cli/test_admin.py | 4 ---- src/allmydata/test/cli/test_alias.py | 4 ---- src/allmydata/test/cli/test_backup.py | 4 ---- src/allmydata/test/cli/test_backupdb.py | 4 ---- src/allmydata/test/cli/test_check.py | 4 ---- src/allmydata/test/cli/test_cli.py | 4 ---- src/allmydata/test/cli/test_cp.py | 4 ---- src/allmydata/test/cli/test_create_alias.py | 4 ---- src/allmydata/test/cli/test_list.py | 4 ---- src/allmydata/test/cli/test_mv.py | 4 ---- src/allmydata/test/cli/test_status.py | 4 ---- src/allmydata/test/cli_node_api.py | 4 ---- src/allmydata/test/common_util.py | 4 ---- src/allmydata/test/common_web.py | 4 ---- src/allmydata/test/matchers.py | 4 ---- src/allmydata/test/mutable/test_checker.py | 4 ---- src/allmydata/test/mutable/test_datahandle.py | 4 ---- src/allmydata/test/mutable/test_different_encoding.py | 4 ---- src/allmydata/test/mutable/test_exceptions.py | 4 ---- src/allmydata/test/mutable/test_filehandle.py | 4 ---- src/allmydata/test/mutable/test_filenode.py | 4 ---- src/allmydata/test/mutable/test_interoperability.py | 4 ---- src/allmydata/test/mutable/test_multiple_encodings.py | 4 ---- src/allmydata/test/mutable/test_multiple_versions.py | 4 ---- src/allmydata/test/mutable/test_problems.py | 4 ---- src/allmydata/test/mutable/test_repair.py | 4 ---- src/allmydata/test/mutable/test_roundtrip.py | 4 ---- src/allmydata/test/mutable/test_servermap.py | 4 ---- src/allmydata/test/mutable/test_update.py | 4 ---- src/allmydata/test/mutable/util.py | 4 ---- src/allmydata/test/storage_plugin.py | 4 ---- src/allmydata/test/strategies.py | 4 ---- src/allmydata/test/test_abbreviate.py | 4 ---- src/allmydata/test/test_base32.py | 4 ---- src/allmydata/test/test_base62.py | 4 ---- src/allmydata/test/test_checker.py | 4 ---- src/allmydata/test/test_codec.py | 4 ---- src/allmydata/test/test_common_util.py | 4 ---- src/allmydata/test/test_configutil.py | 4 ---- src/allmydata/test/test_connections.py | 4 ---- src/allmydata/test/test_consumer.py | 4 ---- src/allmydata/test/test_crawler.py | 4 ---- src/allmydata/test/test_crypto.py | 4 ---- src/allmydata/test/test_deepcheck.py | 4 ---- src/allmydata/test/test_dirnode.py | 4 ---- src/allmydata/test/test_encode.py | 4 ---- src/allmydata/test/test_encodingutil.py | 4 ---- src/allmydata/test/test_filenode.py | 4 ---- src/allmydata/test/test_happiness.py | 4 ---- src/allmydata/test/test_hashutil.py | 4 ---- src/allmydata/test/test_humanreadable.py | 4 ---- src/allmydata/test/test_hung_server.py | 4 ---- src/allmydata/test/test_i2p_provider.py | 4 ---- src/allmydata/test/test_immutable.py | 4 ---- src/allmydata/test/test_introducer.py | 4 ---- src/allmydata/test/test_json_metadata.py | 4 ---- src/allmydata/test/test_log.py | 4 ---- src/allmydata/test/test_monitor.py | 4 ---- src/allmydata/test/test_multi_introducers.py | 4 ---- src/allmydata/test/test_netstring.py | 4 ---- src/allmydata/test/test_no_network.py | 4 ---- src/allmydata/test/test_observer.py | 4 ---- src/allmydata/test/test_openmetrics.py | 4 ---- src/allmydata/test/test_python2_regressions.py | 4 ---- src/allmydata/test/test_repairer.py | 4 ---- src/allmydata/test/test_runner.py | 7 ------- src/allmydata/test/test_sftp.py | 4 ---- src/allmydata/test/test_spans.py | 4 ---- src/allmydata/test/test_statistics.py | 4 ---- src/allmydata/test/test_stats.py | 4 ---- src/allmydata/test/test_storage_web.py | 4 ---- src/allmydata/test/test_time_format.py | 4 ---- src/allmydata/test/test_upload.py | 4 ---- src/allmydata/test/test_uri.py | 4 ---- src/allmydata/test/test_util.py | 4 ---- src/allmydata/test/web/common.py | 4 ---- src/allmydata/test/web/matchers.py | 4 ---- src/allmydata/test/web/test_common.py | 4 ---- src/allmydata/test/web/test_grid.py | 4 ---- src/allmydata/test/web/test_introducer.py | 4 ---- src/allmydata/test/web/test_logs.py | 7 ------- src/allmydata/test/web/test_private.py | 7 ------- src/allmydata/test/web/test_root.py | 4 ---- src/allmydata/test/web/test_status.py | 4 ---- src/allmydata/test/web/test_util.py | 4 ---- src/allmydata/unknown.py | 4 ---- src/allmydata/util/abbreviate.py | 4 ---- src/allmydata/util/assertutil.py | 4 ---- src/allmydata/util/base62.py | 4 ---- src/allmydata/util/configutil.py | 4 ---- src/allmydata/util/consumer.py | 4 ---- src/allmydata/util/dbutil.py | 4 ---- src/allmydata/util/encodingutil.py | 4 ---- src/allmydata/util/fileutil.py | 4 ---- src/allmydata/util/gcutil.py | 4 ---- src/allmydata/util/happinessutil.py | 4 ---- src/allmydata/util/hashutil.py | 4 ---- src/allmydata/util/humanreadable.py | 4 ---- src/allmydata/util/idlib.py | 4 ---- src/allmydata/util/jsonbytes.py | 4 ---- src/allmydata/util/log.py | 4 ---- src/allmydata/util/mathutil.py | 4 ---- src/allmydata/util/netstring.py | 4 ---- src/allmydata/util/observer.py | 4 ---- src/allmydata/util/rrefutil.py | 4 ---- src/allmydata/util/spans.py | 4 ---- src/allmydata/util/statistics.py | 4 ---- src/allmydata/util/time_format.py | 4 ---- src/allmydata/util/yamlutil.py | 4 ---- src/allmydata/web/check_results.py | 4 ---- src/allmydata/web/directory.py | 4 ---- src/allmydata/web/info.py | 4 ---- src/allmydata/web/logs.py | 6 ------ src/allmydata/web/operations.py | 4 ---- src/allmydata/web/private.py | 4 ---- src/allmydata/web/status.py | 4 ---- src/allmydata/web/storage.py | 4 ---- src/allmydata/web/storage_plugins.py | 4 ---- src/allmydata/windows/fixups.py | 1 - ws_client.py | 1 - 238 files changed, 818 deletions(-) diff --git a/integration/test_aaa_aardvark.py b/integration/test_aaa_aardvark.py index 28ac4c412..e92403497 100644 --- a/integration/test_aaa_aardvark.py +++ b/integration/test_aaa_aardvark.py @@ -5,10 +5,6 @@ # You can safely skip any of these tests, it'll just appear to "take # longer" to start the first test as the fixtures get built -from __future__ import unicode_literals -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function from future.utils import PY2 if PY2: diff --git a/integration/test_sftp.py b/integration/test_sftp.py index 01ddfdf8a..2fdc7522e 100644 --- a/integration/test_sftp.py +++ b/integration/test_sftp.py @@ -10,10 +10,6 @@ These tests use Paramiko, rather than Twisted's Conch, because: 2. Its API is much simpler to use. """ -from __future__ import unicode_literals -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function from future.utils import PY2 if PY2: diff --git a/integration/test_streaming_logs.py b/integration/test_streaming_logs.py index 036d30715..f70a42ead 100644 --- a/integration/test_streaming_logs.py +++ b/integration/test_streaming_logs.py @@ -1,12 +1,6 @@ """ Ported to Python 3. """ -from __future__ import ( - print_function, - unicode_literals, - absolute_import, - division, -) from future.utils import PY2 if PY2: diff --git a/misc/awesome_weird_stuff/boodlegrid.tac b/misc/awesome_weird_stuff/boodlegrid.tac index f13427ceb..f03474756 100644 --- a/misc/awesome_weird_stuff/boodlegrid.tac +++ b/misc/awesome_weird_stuff/boodlegrid.tac @@ -1,6 +1,5 @@ # -*- python -*- -from __future__ import print_function """Monitor a Tahoe grid, by playing sounds in response to remote events. diff --git a/misc/build_helpers/check-build.py b/misc/build_helpers/check-build.py index 994ed650a..03fd53392 100644 --- a/misc/build_helpers/check-build.py +++ b/misc/build_helpers/check-build.py @@ -2,7 +2,6 @@ # This helper script is used with the 'test-desert-island' Makefile target. -from __future__ import print_function import sys diff --git a/misc/build_helpers/gen-package-table.py b/misc/build_helpers/gen-package-table.py index ebcfd1ecd..690e95739 100644 --- a/misc/build_helpers/gen-package-table.py +++ b/misc/build_helpers/gen-package-table.py @@ -2,7 +2,6 @@ # This script generates a table of dependencies in HTML format on stdout. # It expects to be run in the tahoe-lafs-dep-eggs directory. -from __future__ import print_function import re, os, sys import pkg_resources diff --git a/misc/build_helpers/run-deprecations.py b/misc/build_helpers/run-deprecations.py index 2ad335bd1..6338b7ccb 100644 --- a/misc/build_helpers/run-deprecations.py +++ b/misc/build_helpers/run-deprecations.py @@ -1,4 +1,3 @@ -from __future__ import print_function import sys, os, io, re from twisted.internet import reactor, protocol, task, defer diff --git a/misc/build_helpers/test-osx-pkg.py b/misc/build_helpers/test-osx-pkg.py index aaf7bb47a..6dc51eeaf 100644 --- a/misc/build_helpers/test-osx-pkg.py +++ b/misc/build_helpers/test-osx-pkg.py @@ -29,7 +29,6 @@ # characteristic: 14.1.0 (/Applications/tahoe.app/support/lib/python2.7/site-packages) # pyasn1-modules: 0.0.5 (/Applications/tahoe.app/support/lib/python2.7/site-packages/pyasn1_modules-0.0.5-py2.7.egg) -from __future__ import print_function import os, re, shutil, subprocess, sys, tempfile diff --git a/misc/checkers/check_grid.py b/misc/checkers/check_grid.py index 0a68ed899..189e5a260 100644 --- a/misc/checkers/check_grid.py +++ b/misc/checkers/check_grid.py @@ -1,4 +1,3 @@ -from __future__ import print_function """ Test an existing Tahoe grid, both to see if the grid is still running and to diff --git a/misc/coding_tools/check-debugging.py b/misc/coding_tools/check-debugging.py index b920f5634..6bd54fee3 100755 --- a/misc/coding_tools/check-debugging.py +++ b/misc/coding_tools/check-debugging.py @@ -8,7 +8,6 @@ Runs on Python 3. Usage: ./check-debugging.py src """ -from __future__ import print_function import sys, re, os diff --git a/misc/coding_tools/check-interfaces.py b/misc/coding_tools/check-interfaces.py index 66bdf808f..d2657877a 100644 --- a/misc/coding_tools/check-interfaces.py +++ b/misc/coding_tools/check-interfaces.py @@ -4,7 +4,6 @@ # # bin/tahoe @misc/coding_tools/check-interfaces.py -from __future__ import print_function import os, sys, re, platform diff --git a/misc/coding_tools/check-umids.py b/misc/coding_tools/check-umids.py index 345610f3e..1ef557cee 100644 --- a/misc/coding_tools/check-umids.py +++ b/misc/coding_tools/check-umids.py @@ -8,7 +8,6 @@ This runs on Python 3. # ./check-umids.py src -from __future__ import print_function import sys, re, os diff --git a/misc/coding_tools/graph-deps.py b/misc/coding_tools/graph-deps.py index ad049093c..400f3912a 100755 --- a/misc/coding_tools/graph-deps.py +++ b/misc/coding_tools/graph-deps.py @@ -21,7 +21,6 @@ # Install 'click' first. I run this with py2, but py3 might work too, if the # wheels can be built with py3. -from __future__ import unicode_literals, print_function import os, sys, subprocess, json, tempfile, zipfile, re, itertools import email.parser from pprint import pprint diff --git a/misc/coding_tools/make-canary-files.py b/misc/coding_tools/make-canary-files.py index 89f274b38..018462892 100644 --- a/misc/coding_tools/make-canary-files.py +++ b/misc/coding_tools/make-canary-files.py @@ -1,6 +1,5 @@ #!/usr/bin/env python -from __future__ import print_function """ Given a list of nodeids and a 'convergence' file, create a bunch of files diff --git a/misc/coding_tools/make_umid b/misc/coding_tools/make_umid index 6b1759681..870ece1c6 100644 --- a/misc/coding_tools/make_umid +++ b/misc/coding_tools/make_umid @@ -1,6 +1,5 @@ #!/usr/bin/env python -from __future__ import print_function """Create a short probably-unique string for use as a umid= argument in a Foolscap log() call, to make it easier to locate the source code that diff --git a/misc/operations_helpers/cpu-watcher-poll.py b/misc/operations_helpers/cpu-watcher-poll.py index 320dd8ad7..0ecf974c6 100644 --- a/misc/operations_helpers/cpu-watcher-poll.py +++ b/misc/operations_helpers/cpu-watcher-poll.py @@ -1,6 +1,5 @@ #!/usr/bin/env python -from __future__ import print_function from foolscap import Tub, eventual from twisted.internet import reactor diff --git a/misc/operations_helpers/cpu-watcher-subscribe.py b/misc/operations_helpers/cpu-watcher-subscribe.py index 36a69cac7..ce486832f 100644 --- a/misc/operations_helpers/cpu-watcher-subscribe.py +++ b/misc/operations_helpers/cpu-watcher-subscribe.py @@ -1,6 +1,5 @@ # -*- python -*- -from __future__ import print_function from twisted.internet import reactor import sys diff --git a/misc/operations_helpers/cpu-watcher.tac b/misc/operations_helpers/cpu-watcher.tac index c50b51c61..140625d58 100644 --- a/misc/operations_helpers/cpu-watcher.tac +++ b/misc/operations_helpers/cpu-watcher.tac @@ -1,6 +1,5 @@ # -*- python -*- -from __future__ import print_function """ # run this tool on a linux box in its own directory, with a file named diff --git a/misc/operations_helpers/find-share-anomalies.py b/misc/operations_helpers/find-share-anomalies.py index d689a8c99..e3826cc69 100644 --- a/misc/operations_helpers/find-share-anomalies.py +++ b/misc/operations_helpers/find-share-anomalies.py @@ -2,7 +2,6 @@ # feed this the results of 'tahoe catalog-shares' for all servers -from __future__ import print_function import sys diff --git a/misc/operations_helpers/munin/tahoe_cpu_watcher b/misc/operations_helpers/munin/tahoe_cpu_watcher index 8f2876792..ca7e40a32 100644 --- a/misc/operations_helpers/munin/tahoe_cpu_watcher +++ b/misc/operations_helpers/munin/tahoe_cpu_watcher @@ -1,6 +1,5 @@ #!/usr/bin/env python -from __future__ import print_function import os, sys, re import urllib diff --git a/misc/operations_helpers/munin/tahoe_diskleft b/misc/operations_helpers/munin/tahoe_diskleft index d5ce04b1a..b08422575 100644 --- a/misc/operations_helpers/munin/tahoe_diskleft +++ b/misc/operations_helpers/munin/tahoe_diskleft @@ -5,7 +5,6 @@ # is left on all disks across the grid. The plugin should be configured with # env_url= pointing at the diskwatcher.tac webport. -from __future__ import print_function import os, sys, urllib, json diff --git a/misc/operations_helpers/munin/tahoe_disktotal b/misc/operations_helpers/munin/tahoe_disktotal index b6d1a99e6..801eac164 100644 --- a/misc/operations_helpers/munin/tahoe_disktotal +++ b/misc/operations_helpers/munin/tahoe_disktotal @@ -6,7 +6,6 @@ # used. The plugin should be configured with env_url= pointing at the # diskwatcher.tac webport. -from __future__ import print_function import os, sys, urllib, json diff --git a/misc/operations_helpers/munin/tahoe_diskusage b/misc/operations_helpers/munin/tahoe_diskusage index cc37af3df..7eadd8eeb 100644 --- a/misc/operations_helpers/munin/tahoe_diskusage +++ b/misc/operations_helpers/munin/tahoe_diskusage @@ -5,7 +5,6 @@ # is being used per unit time. The plugin should be configured with env_url= # pointing at the diskwatcher.tac webport. -from __future__ import print_function import os, sys, urllib, json diff --git a/misc/operations_helpers/munin/tahoe_diskused b/misc/operations_helpers/munin/tahoe_diskused index 26303af86..151dc826e 100644 --- a/misc/operations_helpers/munin/tahoe_diskused +++ b/misc/operations_helpers/munin/tahoe_diskused @@ -5,7 +5,6 @@ # used on all disks across the grid. The plugin should be configured with # env_url= pointing at the diskwatcher.tac webport. -from __future__ import print_function import os, sys, urllib, json diff --git a/misc/operations_helpers/munin/tahoe_doomsday b/misc/operations_helpers/munin/tahoe_doomsday index 5a87489c2..348b244fe 100644 --- a/misc/operations_helpers/munin/tahoe_doomsday +++ b/misc/operations_helpers/munin/tahoe_doomsday @@ -5,7 +5,6 @@ # left before the grid fills up. The plugin should be configured with # env_url= pointing at the diskwatcher.tac webport. -from __future__ import print_function import os, sys, urllib, json diff --git a/misc/operations_helpers/munin/tahoe_estimate_files b/misc/operations_helpers/munin/tahoe_estimate_files index 1dda5affb..e6b6eff5d 100644 --- a/misc/operations_helpers/munin/tahoe_estimate_files +++ b/misc/operations_helpers/munin/tahoe_estimate_files @@ -1,6 +1,5 @@ #!/usr/bin/env python -from __future__ import print_function import sys, os.path diff --git a/misc/operations_helpers/munin/tahoe_files b/misc/operations_helpers/munin/tahoe_files index ec3ee5073..d951985a8 100644 --- a/misc/operations_helpers/munin/tahoe_files +++ b/misc/operations_helpers/munin/tahoe_files @@ -18,7 +18,6 @@ # env.basedir_NODE3 /path/to/node3 # -from __future__ import print_function import os, sys diff --git a/misc/operations_helpers/munin/tahoe_helperstats_active b/misc/operations_helpers/munin/tahoe_helperstats_active index ba1032acb..315a00886 100644 --- a/misc/operations_helpers/munin/tahoe_helperstats_active +++ b/misc/operations_helpers/munin/tahoe_helperstats_active @@ -1,6 +1,5 @@ #!/usr/bin/env python -from __future__ import print_function import os, sys import urllib diff --git a/misc/operations_helpers/munin/tahoe_helperstats_fetched b/misc/operations_helpers/munin/tahoe_helperstats_fetched index 5f53bb82c..f9577427c 100644 --- a/misc/operations_helpers/munin/tahoe_helperstats_fetched +++ b/misc/operations_helpers/munin/tahoe_helperstats_fetched @@ -1,6 +1,5 @@ #!/usr/bin/env python -from __future__ import print_function import os, sys import urllib diff --git a/misc/operations_helpers/munin/tahoe_introstats b/misc/operations_helpers/munin/tahoe_introstats index 0373c70e2..5dd07f62d 100644 --- a/misc/operations_helpers/munin/tahoe_introstats +++ b/misc/operations_helpers/munin/tahoe_introstats @@ -1,6 +1,5 @@ #!/usr/bin/env python -from __future__ import print_function import os, sys import urllib diff --git a/misc/operations_helpers/munin/tahoe_nodememory b/misc/operations_helpers/munin/tahoe_nodememory index 061a50dc2..71463c031 100644 --- a/misc/operations_helpers/munin/tahoe_nodememory +++ b/misc/operations_helpers/munin/tahoe_nodememory @@ -4,7 +4,6 @@ # by 'allmydata start', then extracts the amount of memory they consume (both # VmSize and VmRSS) from /proc -from __future__ import print_function import os, sys, re diff --git a/misc/operations_helpers/munin/tahoe_overhead b/misc/operations_helpers/munin/tahoe_overhead index 40640d189..6a25bcd46 100644 --- a/misc/operations_helpers/munin/tahoe_overhead +++ b/misc/operations_helpers/munin/tahoe_overhead @@ -27,7 +27,6 @@ # This plugin should be configured with env_diskwatcher_url= pointing at the # diskwatcher.tac webport, and env_deepsize_url= pointing at the PHP script. -from __future__ import print_function import os, sys, urllib, json diff --git a/misc/operations_helpers/munin/tahoe_rootdir_space b/misc/operations_helpers/munin/tahoe_rootdir_space index 1f5709206..ca61ddb13 100644 --- a/misc/operations_helpers/munin/tahoe_rootdir_space +++ b/misc/operations_helpers/munin/tahoe_rootdir_space @@ -1,6 +1,5 @@ #!/usr/bin/env python -from __future__ import print_function import os, sys import urllib diff --git a/misc/operations_helpers/munin/tahoe_server_latency_ b/misc/operations_helpers/munin/tahoe_server_latency_ index c8930804c..4c9a79a7a 100644 --- a/misc/operations_helpers/munin/tahoe_server_latency_ +++ b/misc/operations_helpers/munin/tahoe_server_latency_ @@ -42,7 +42,6 @@ # of course, these URLs must match the webports you have configured into the # storage nodes. -from __future__ import print_function import os, sys import urllib diff --git a/misc/operations_helpers/munin/tahoe_server_operations_ b/misc/operations_helpers/munin/tahoe_server_operations_ index 6156a7f48..cdf0409dd 100644 --- a/misc/operations_helpers/munin/tahoe_server_operations_ +++ b/misc/operations_helpers/munin/tahoe_server_operations_ @@ -32,7 +32,6 @@ # of course, these URLs must match the webports you have configured into the # storage nodes. -from __future__ import print_function import os, sys import urllib diff --git a/misc/operations_helpers/munin/tahoe_spacetime b/misc/operations_helpers/munin/tahoe_spacetime index 12b5121bf..e3a058851 100644 --- a/misc/operations_helpers/munin/tahoe_spacetime +++ b/misc/operations_helpers/munin/tahoe_spacetime @@ -5,7 +5,6 @@ # then extrapolate to guess how many weeks/months/years of storage space we # have left, and output it to another munin graph -from __future__ import print_function import sys, os, time import rrdtool diff --git a/misc/operations_helpers/munin/tahoe_stats b/misc/operations_helpers/munin/tahoe_stats index 03bf116f5..f94dd5b36 100644 --- a/misc/operations_helpers/munin/tahoe_stats +++ b/misc/operations_helpers/munin/tahoe_stats @@ -1,6 +1,5 @@ #!/usr/bin/env python -from __future__ import print_function import os import json diff --git a/misc/operations_helpers/munin/tahoe_storagespace b/misc/operations_helpers/munin/tahoe_storagespace index 73443b428..318283244 100644 --- a/misc/operations_helpers/munin/tahoe_storagespace +++ b/misc/operations_helpers/munin/tahoe_storagespace @@ -18,7 +18,6 @@ # Allmydata-tahoe must be installed on the system where this plugin is used, # since it imports a utility module from allmydata.utils . -from __future__ import print_function import os, sys import commands diff --git a/misc/operations_helpers/provisioning/reliability.py b/misc/operations_helpers/provisioning/reliability.py index fe274c875..c31e398e0 100644 --- a/misc/operations_helpers/provisioning/reliability.py +++ b/misc/operations_helpers/provisioning/reliability.py @@ -1,6 +1,5 @@ #! /usr/bin/python -from __future__ import print_function import math from allmydata.util import statistics diff --git a/misc/operations_helpers/provisioning/test_provisioning.py b/misc/operations_helpers/provisioning/test_provisioning.py index 2b71c8566..5d46f704e 100644 --- a/misc/operations_helpers/provisioning/test_provisioning.py +++ b/misc/operations_helpers/provisioning/test_provisioning.py @@ -1,4 +1,3 @@ -from __future__ import print_function import unittest from allmydata import provisioning diff --git a/misc/operations_helpers/spacetime/diskwatcher.tac b/misc/operations_helpers/spacetime/diskwatcher.tac index 0a43a468e..22c1f8747 100644 --- a/misc/operations_helpers/spacetime/diskwatcher.tac +++ b/misc/operations_helpers/spacetime/diskwatcher.tac @@ -1,6 +1,5 @@ # -*- python -*- -from __future__ import print_function """ Run this tool with twistd in its own directory, with a file named 'urls.txt' diff --git a/misc/simulators/bench_spans.py b/misc/simulators/bench_spans.py index c696dac1e..6bc4f045e 100644 --- a/misc/simulators/bench_spans.py +++ b/misc/simulators/bench_spans.py @@ -1,4 +1,3 @@ -from __future__ import print_function """ To use this, get a trace file such as this one: diff --git a/misc/simulators/count_dirs.py b/misc/simulators/count_dirs.py index 22eda8917..62045d1e2 100644 --- a/misc/simulators/count_dirs.py +++ b/misc/simulators/count_dirs.py @@ -1,6 +1,5 @@ #!/usr/bin/env python -from __future__ import print_function """ This tool estimates how much space would be consumed by a filetree into which diff --git a/misc/simulators/hashbasedsig.py b/misc/simulators/hashbasedsig.py index dbb9ca504..fff92b681 100644 --- a/misc/simulators/hashbasedsig.py +++ b/misc/simulators/hashbasedsig.py @@ -1,6 +1,5 @@ #!python -from __future__ import print_function # range of hash output lengths range_L_hash = [128] diff --git a/misc/simulators/ringsim.py b/misc/simulators/ringsim.py index e6616351c..889785bb4 100644 --- a/misc/simulators/ringsim.py +++ b/misc/simulators/ringsim.py @@ -4,7 +4,6 @@ # import time -from __future__ import print_function import math from hashlib import md5 # sha1, sha256 diff --git a/misc/simulators/simulate_load.py b/misc/simulators/simulate_load.py index ed80ab842..989711207 100644 --- a/misc/simulators/simulate_load.py +++ b/misc/simulators/simulate_load.py @@ -2,7 +2,6 @@ # WARNING. There is a bug in this script so that it does not simulate the actual Tahoe Two server selection algorithm that it was intended to simulate. See http://allmydata.org/trac/tahoe-lafs/ticket/302 (stop permuting peerlist, use SI as offset into ring instead?) -from __future__ import print_function from past.builtins import cmp diff --git a/misc/simulators/simulator.py b/misc/simulators/simulator.py index ceeb05edf..b2f51b0e1 100644 --- a/misc/simulators/simulator.py +++ b/misc/simulators/simulator.py @@ -1,6 +1,5 @@ #! /usr/bin/env python -from __future__ import print_function import hashlib import os, random diff --git a/misc/simulators/sizes.py b/misc/simulators/sizes.py index eb5f3adbf..4b4f6d1fd 100644 --- a/misc/simulators/sizes.py +++ b/misc/simulators/sizes.py @@ -1,6 +1,5 @@ #! /usr/bin/env python -from __future__ import print_function from future.utils import PY2 if PY2: diff --git a/misc/simulators/storage-overhead.py b/misc/simulators/storage-overhead.py index 9959f5575..096c18fba 100644 --- a/misc/simulators/storage-overhead.py +++ b/misc/simulators/storage-overhead.py @@ -1,6 +1,5 @@ #!/usr/bin/env python -from __future__ import print_function import sys, math from allmydata import uri, storage diff --git a/pyinstaller.spec b/pyinstaller.spec index 01b1ac4ac..5c24df430 100644 --- a/pyinstaller.spec +++ b/pyinstaller.spec @@ -1,6 +1,5 @@ # -*- mode: python -*- -from __future__ import print_function from distutils.sysconfig import get_python_lib import hashlib diff --git a/src/allmydata/__main__.py b/src/allmydata/__main__.py index 87f1f6522..972167b30 100644 --- a/src/allmydata/__main__.py +++ b/src/allmydata/__main__.py @@ -2,10 +2,6 @@ Ported to Python 3. """ -from __future__ import unicode_literals -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function from future.utils import PY2 if PY2: diff --git a/src/allmydata/_monkeypatch.py b/src/allmydata/_monkeypatch.py index da37fd979..520a4194a 100644 --- a/src/allmydata/_monkeypatch.py +++ b/src/allmydata/_monkeypatch.py @@ -4,10 +4,6 @@ Monkey-patching of third party libraries. Ported to Python 3. """ -from __future__ import unicode_literals -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function from future.utils import PY2 if PY2: diff --git a/src/allmydata/blacklist.py b/src/allmydata/blacklist.py index 43eb36cc6..a019ccb69 100644 --- a/src/allmydata/blacklist.py +++ b/src/allmydata/blacklist.py @@ -1,10 +1,6 @@ """ Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/check_results.py b/src/allmydata/check_results.py index 4f997b614..1dbf2f7d5 100644 --- a/src/allmydata/check_results.py +++ b/src/allmydata/check_results.py @@ -1,9 +1,5 @@ """Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/codec.py b/src/allmydata/codec.py index af375a117..b58ebae57 100644 --- a/src/allmydata/codec.py +++ b/src/allmydata/codec.py @@ -3,10 +3,6 @@ CRS encoding and decoding. Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/crypto/__init__.py b/src/allmydata/crypto/__init__.py index 04b8f0cc3..3ba70e605 100644 --- a/src/allmydata/crypto/__init__.py +++ b/src/allmydata/crypto/__init__.py @@ -9,10 +9,6 @@ objects that `cryptography` documents. Ported to Python 3. """ -from __future__ import unicode_literals -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function from future.utils import PY2 if PY2: diff --git a/src/allmydata/crypto/aes.py b/src/allmydata/crypto/aes.py index ad7cfcba4..6e1b93c03 100644 --- a/src/allmydata/crypto/aes.py +++ b/src/allmydata/crypto/aes.py @@ -9,10 +9,6 @@ objects that `cryptography` documents. Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/crypto/error.py b/src/allmydata/crypto/error.py index 153e48d33..ea3f78162 100644 --- a/src/allmydata/crypto/error.py +++ b/src/allmydata/crypto/error.py @@ -3,10 +3,6 @@ Exceptions raise by allmydata.crypto.* modules Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/crypto/util.py b/src/allmydata/crypto/util.py index 8b8619e47..20f4bca85 100644 --- a/src/allmydata/crypto/util.py +++ b/src/allmydata/crypto/util.py @@ -3,10 +3,6 @@ Utilities used by allmydata.crypto modules Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/deep_stats.py b/src/allmydata/deep_stats.py index bfb43ebae..a80abe508 100644 --- a/src/allmydata/deep_stats.py +++ b/src/allmydata/deep_stats.py @@ -2,10 +2,6 @@ Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/dirnode.py b/src/allmydata/dirnode.py index ccd045b05..67e44faf4 100644 --- a/src/allmydata/dirnode.py +++ b/src/allmydata/dirnode.py @@ -2,10 +2,6 @@ Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/frontends/auth.py b/src/allmydata/frontends/auth.py index b6f9c2b7e..6d58cf567 100644 --- a/src/allmydata/frontends/auth.py +++ b/src/allmydata/frontends/auth.py @@ -1,10 +1,6 @@ """ Authentication for frontends. """ -from __future__ import unicode_literals -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function from future.utils import PY2 if PY2: diff --git a/src/allmydata/frontends/sftpd.py b/src/allmydata/frontends/sftpd.py index 7ef9a8820..e3c0eb674 100644 --- a/src/allmydata/frontends/sftpd.py +++ b/src/allmydata/frontends/sftpd.py @@ -1,10 +1,6 @@ """ Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/hashtree.py b/src/allmydata/hashtree.py index 57bdbd9a1..ca043f234 100644 --- a/src/allmydata/hashtree.py +++ b/src/allmydata/hashtree.py @@ -49,10 +49,6 @@ or eat your children, but it might. Use at your own risk. Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/history.py b/src/allmydata/history.py index 06a22ab5d..5629d7c43 100644 --- a/src/allmydata/history.py +++ b/src/allmydata/history.py @@ -1,9 +1,5 @@ """Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/immutable/checker.py b/src/allmydata/immutable/checker.py index 30abc68c6..0e8ba14d8 100644 --- a/src/allmydata/immutable/checker.py +++ b/src/allmydata/immutable/checker.py @@ -1,10 +1,6 @@ """ Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/immutable/downloader/__init__.py b/src/allmydata/immutable/downloader/__init__.py index 2d3d9e2a4..84d6b71f9 100644 --- a/src/allmydata/immutable/downloader/__init__.py +++ b/src/allmydata/immutable/downloader/__init__.py @@ -2,10 +2,6 @@ Ported to Python 3. """ -from __future__ import unicode_literals -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function from future.utils import PY2 if PY2: diff --git a/src/allmydata/immutable/downloader/common.py b/src/allmydata/immutable/downloader/common.py index 71430b0d7..e0d1fe3af 100644 --- a/src/allmydata/immutable/downloader/common.py +++ b/src/allmydata/immutable/downloader/common.py @@ -2,10 +2,6 @@ Ported to Python 3. """ -from __future__ import unicode_literals -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function from future.utils import PY2 if PY2: diff --git a/src/allmydata/immutable/downloader/fetcher.py b/src/allmydata/immutable/downloader/fetcher.py index 130bb93dc..ccffa4889 100644 --- a/src/allmydata/immutable/downloader/fetcher.py +++ b/src/allmydata/immutable/downloader/fetcher.py @@ -2,10 +2,6 @@ Ported to Python 3. """ -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/immutable/downloader/finder.py b/src/allmydata/immutable/downloader/finder.py index 4f6d1aa14..1992e72c1 100644 --- a/src/allmydata/immutable/downloader/finder.py +++ b/src/allmydata/immutable/downloader/finder.py @@ -1,10 +1,6 @@ """ Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/immutable/downloader/node.py b/src/allmydata/immutable/downloader/node.py index 102d29a0a..3c69b4084 100644 --- a/src/allmydata/immutable/downloader/node.py +++ b/src/allmydata/immutable/downloader/node.py @@ -1,10 +1,6 @@ """ Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/immutable/downloader/segmentation.py b/src/allmydata/immutable/downloader/segmentation.py index 6d7cb7676..4691f2602 100644 --- a/src/allmydata/immutable/downloader/segmentation.py +++ b/src/allmydata/immutable/downloader/segmentation.py @@ -1,10 +1,6 @@ """ Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/immutable/downloader/share.py b/src/allmydata/immutable/downloader/share.py index 016f1c34d..02e8554d4 100644 --- a/src/allmydata/immutable/downloader/share.py +++ b/src/allmydata/immutable/downloader/share.py @@ -1,10 +1,6 @@ """ Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/immutable/downloader/status.py b/src/allmydata/immutable/downloader/status.py index 425f6893c..11b3e88a7 100644 --- a/src/allmydata/immutable/downloader/status.py +++ b/src/allmydata/immutable/downloader/status.py @@ -1,10 +1,6 @@ """ Ported to Python 3. """ -from __future__ import division -from __future__ import absolute_import -from __future__ import print_function -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/immutable/encode.py b/src/allmydata/immutable/encode.py index 2414527ff..b838e71e1 100644 --- a/src/allmydata/immutable/encode.py +++ b/src/allmydata/immutable/encode.py @@ -4,10 +4,6 @@ Ported to Python 3. """ -from __future__ import division -from __future__ import absolute_import -from __future__ import print_function -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/immutable/filenode.py b/src/allmydata/immutable/filenode.py index 6962c31a4..761e663dd 100644 --- a/src/allmydata/immutable/filenode.py +++ b/src/allmydata/immutable/filenode.py @@ -2,10 +2,6 @@ Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/immutable/happiness_upload.py b/src/allmydata/immutable/happiness_upload.py index 3e3eedbc9..2bc0016b0 100644 --- a/src/allmydata/immutable/happiness_upload.py +++ b/src/allmydata/immutable/happiness_upload.py @@ -4,10 +4,6 @@ on. Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/immutable/literal.py b/src/allmydata/immutable/literal.py index 544a205e1..1ae2743f8 100644 --- a/src/allmydata/immutable/literal.py +++ b/src/allmydata/immutable/literal.py @@ -2,10 +2,6 @@ Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/immutable/offloaded.py b/src/allmydata/immutable/offloaded.py index 8ce51782c..a98d3372b 100644 --- a/src/allmydata/immutable/offloaded.py +++ b/src/allmydata/immutable/offloaded.py @@ -1,10 +1,6 @@ """ Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/immutable/repairer.py b/src/allmydata/immutable/repairer.py index bccd8453d..87873369a 100644 --- a/src/allmydata/immutable/repairer.py +++ b/src/allmydata/immutable/repairer.py @@ -1,10 +1,6 @@ """ Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/interfaces.py b/src/allmydata/interfaces.py index 0f00c5417..7bfcc9a13 100644 --- a/src/allmydata/interfaces.py +++ b/src/allmydata/interfaces.py @@ -5,10 +5,6 @@ Ported to Python 3. Note that for RemoteInterfaces, the __remote_name__ needs to be a native string because of https://github.com/warner/foolscap/blob/43f4485a42c9c28e2c79d655b3a9e24d4e6360ca/src/foolscap/remoteinterface.py#L67 """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals from future.utils import PY2, native_str if PY2: diff --git a/src/allmydata/introducer/__init__.py b/src/allmydata/introducer/__init__.py index bfc960e05..d654f6c89 100644 --- a/src/allmydata/introducer/__init__.py +++ b/src/allmydata/introducer/__init__.py @@ -2,10 +2,6 @@ Ported to Python 3. """ -from __future__ import unicode_literals -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function from future.utils import PY2 if PY2: diff --git a/src/allmydata/introducer/client.py b/src/allmydata/introducer/client.py index a64596f0e..4c8abff60 100644 --- a/src/allmydata/introducer/client.py +++ b/src/allmydata/introducer/client.py @@ -1,10 +1,6 @@ """ Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/introducer/common.py b/src/allmydata/introducer/common.py index f6f70d861..21b16a599 100644 --- a/src/allmydata/introducer/common.py +++ b/src/allmydata/introducer/common.py @@ -1,10 +1,6 @@ """ Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/introducer/interfaces.py b/src/allmydata/introducer/interfaces.py index 24fd3945f..ca97b425f 100644 --- a/src/allmydata/introducer/interfaces.py +++ b/src/allmydata/introducer/interfaces.py @@ -2,10 +2,6 @@ Ported to Python 3. """ -from __future__ import unicode_literals -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function from future.utils import PY2, native_str if PY2: diff --git a/src/allmydata/monitor.py b/src/allmydata/monitor.py index 1559a30d9..c2aae826e 100644 --- a/src/allmydata/monitor.py +++ b/src/allmydata/monitor.py @@ -3,10 +3,6 @@ Manage status of long-running operations. Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/mutable/checker.py b/src/allmydata/mutable/checker.py index 0899168c3..ee57e2ed9 100644 --- a/src/allmydata/mutable/checker.py +++ b/src/allmydata/mutable/checker.py @@ -1,10 +1,6 @@ """ Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/mutable/layout.py b/src/allmydata/mutable/layout.py index 8bb2f3083..9057745e7 100644 --- a/src/allmydata/mutable/layout.py +++ b/src/allmydata/mutable/layout.py @@ -1,10 +1,6 @@ """ Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/mutable/publish.py b/src/allmydata/mutable/publish.py index e57652467..9df1d20b2 100644 --- a/src/allmydata/mutable/publish.py +++ b/src/allmydata/mutable/publish.py @@ -1,10 +1,6 @@ """ Ported to Python 3. """ -from __future__ import division -from __future__ import absolute_import -from __future__ import print_function -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/mutable/repairer.py b/src/allmydata/mutable/repairer.py index 23af02203..5c6537769 100644 --- a/src/allmydata/mutable/repairer.py +++ b/src/allmydata/mutable/repairer.py @@ -1,10 +1,6 @@ """ Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/scripts/admin.py b/src/allmydata/scripts/admin.py index 3acd52267..d1bd59600 100644 --- a/src/allmydata/scripts/admin.py +++ b/src/allmydata/scripts/admin.py @@ -1,10 +1,6 @@ """ Ported to Python 3. """ -from __future__ import unicode_literals -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function from future.utils import PY2 if PY2: diff --git a/src/allmydata/scripts/backupdb.py b/src/allmydata/scripts/backupdb.py index c7827e56e..21db88ee2 100644 --- a/src/allmydata/scripts/backupdb.py +++ b/src/allmydata/scripts/backupdb.py @@ -1,10 +1,6 @@ """ Ported to Python 3. """ -from __future__ import unicode_literals -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function from future.utils import PY2 if PY2: diff --git a/src/allmydata/scripts/default_nodedir.py b/src/allmydata/scripts/default_nodedir.py index 00924b8f9..89d13cd1b 100644 --- a/src/allmydata/scripts/default_nodedir.py +++ b/src/allmydata/scripts/default_nodedir.py @@ -2,10 +2,6 @@ Ported to Python 3. """ -from __future__ import unicode_literals -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function from future.utils import PY2 if PY2: diff --git a/src/allmydata/scripts/slow_operation.py b/src/allmydata/scripts/slow_operation.py index 3c23fb533..092e85823 100644 --- a/src/allmydata/scripts/slow_operation.py +++ b/src/allmydata/scripts/slow_operation.py @@ -1,10 +1,6 @@ """ Ported to Python 3. """ -from __future__ import unicode_literals -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function from future.utils import PY2, PY3 if PY2: diff --git a/src/allmydata/scripts/tahoe_add_alias.py b/src/allmydata/scripts/tahoe_add_alias.py index 8476aeb28..e29d974f4 100644 --- a/src/allmydata/scripts/tahoe_add_alias.py +++ b/src/allmydata/scripts/tahoe_add_alias.py @@ -1,10 +1,6 @@ """ Ported to Python 3. """ -from __future__ import unicode_literals -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function from future.utils import PY2 if PY2: diff --git a/src/allmydata/scripts/tahoe_backup.py b/src/allmydata/scripts/tahoe_backup.py index b574f16e8..058febb8c 100644 --- a/src/allmydata/scripts/tahoe_backup.py +++ b/src/allmydata/scripts/tahoe_backup.py @@ -1,10 +1,6 @@ """ Ported to Python 3. """ -from __future__ import unicode_literals -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function from future.utils import PY2 if PY2: diff --git a/src/allmydata/scripts/tahoe_check.py b/src/allmydata/scripts/tahoe_check.py index 6bafe3d1a..d88d3689e 100644 --- a/src/allmydata/scripts/tahoe_check.py +++ b/src/allmydata/scripts/tahoe_check.py @@ -1,10 +1,6 @@ """ Ported to Python 3. """ -from __future__ import unicode_literals -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function from future.utils import PY2 if PY2: diff --git a/src/allmydata/scripts/tahoe_cp.py b/src/allmydata/scripts/tahoe_cp.py index aae03291f..9ccbf2a05 100644 --- a/src/allmydata/scripts/tahoe_cp.py +++ b/src/allmydata/scripts/tahoe_cp.py @@ -1,10 +1,6 @@ """ Ported to Python 3. """ -from __future__ import unicode_literals -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function from future.utils import PY2 if PY2: diff --git a/src/allmydata/scripts/tahoe_get.py b/src/allmydata/scripts/tahoe_get.py index 39f1686ce..cf0dd8afa 100644 --- a/src/allmydata/scripts/tahoe_get.py +++ b/src/allmydata/scripts/tahoe_get.py @@ -1,10 +1,6 @@ """ Ported to Python 3. """ -from __future__ import unicode_literals -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function from future.utils import PY2, PY3 if PY2: diff --git a/src/allmydata/scripts/tahoe_ls.py b/src/allmydata/scripts/tahoe_ls.py index 5a7136d77..b20dec723 100644 --- a/src/allmydata/scripts/tahoe_ls.py +++ b/src/allmydata/scripts/tahoe_ls.py @@ -1,10 +1,6 @@ """ Ported to Python 3. """ -from __future__ import unicode_literals -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function from future.utils import PY2 if PY2: diff --git a/src/allmydata/scripts/tahoe_manifest.py b/src/allmydata/scripts/tahoe_manifest.py index b55075eef..e1ee95ef6 100644 --- a/src/allmydata/scripts/tahoe_manifest.py +++ b/src/allmydata/scripts/tahoe_manifest.py @@ -1,10 +1,6 @@ """ Ported to Python 3. """ -from __future__ import unicode_literals -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function from future.utils import PY2, PY3 if PY2: diff --git a/src/allmydata/scripts/tahoe_mkdir.py b/src/allmydata/scripts/tahoe_mkdir.py index 85fe12554..2bc35c391 100644 --- a/src/allmydata/scripts/tahoe_mkdir.py +++ b/src/allmydata/scripts/tahoe_mkdir.py @@ -1,10 +1,6 @@ """ Ported to Python 3. """ -from __future__ import unicode_literals -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function from future.utils import PY2 if PY2: diff --git a/src/allmydata/scripts/tahoe_mv.py b/src/allmydata/scripts/tahoe_mv.py index d921047a8..bd1e4f810 100644 --- a/src/allmydata/scripts/tahoe_mv.py +++ b/src/allmydata/scripts/tahoe_mv.py @@ -1,10 +1,6 @@ """ Ported to Python 3. """ -from __future__ import unicode_literals -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function from future.utils import PY2 if PY2: diff --git a/src/allmydata/scripts/tahoe_run.py b/src/allmydata/scripts/tahoe_run.py index eba5ae329..345f85815 100644 --- a/src/allmydata/scripts/tahoe_run.py +++ b/src/allmydata/scripts/tahoe_run.py @@ -1,10 +1,6 @@ """ Ported to Python 3. """ -from __future__ import unicode_literals -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function from future.utils import PY2 if PY2: diff --git a/src/allmydata/scripts/tahoe_status.py b/src/allmydata/scripts/tahoe_status.py index 250bfdea3..5c8c6aee0 100644 --- a/src/allmydata/scripts/tahoe_status.py +++ b/src/allmydata/scripts/tahoe_status.py @@ -1,10 +1,6 @@ """ Ported to Python 3. """ -from __future__ import unicode_literals -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function from future.utils import PY2 if PY2: diff --git a/src/allmydata/scripts/tahoe_unlink.py b/src/allmydata/scripts/tahoe_unlink.py index 5bdebb960..29da80811 100644 --- a/src/allmydata/scripts/tahoe_unlink.py +++ b/src/allmydata/scripts/tahoe_unlink.py @@ -1,10 +1,6 @@ """ Ported to Python 3. """ -from __future__ import unicode_literals -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function from future.utils import PY2 if PY2: diff --git a/src/allmydata/scripts/tahoe_webopen.py b/src/allmydata/scripts/tahoe_webopen.py index dbec31e87..2b16f09e1 100644 --- a/src/allmydata/scripts/tahoe_webopen.py +++ b/src/allmydata/scripts/tahoe_webopen.py @@ -1,10 +1,6 @@ """ Ported to Python 3. """ -from __future__ import unicode_literals -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function from future.utils import PY2 if PY2: diff --git a/src/allmydata/storage/common.py b/src/allmydata/storage/common.py index f6d986f85..e9f5e4e5d 100644 --- a/src/allmydata/storage/common.py +++ b/src/allmydata/storage/common.py @@ -1,10 +1,6 @@ """ Ported to Python 3. """ -from __future__ import unicode_literals -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function from future.utils import PY2, PY3 if PY2: diff --git a/src/allmydata/storage/crawler.py b/src/allmydata/storage/crawler.py index 7516bc4e9..8a7304808 100644 --- a/src/allmydata/storage/crawler.py +++ b/src/allmydata/storage/crawler.py @@ -4,10 +4,6 @@ Crawl the storage server shares. Ported to Python 3. """ -from __future__ import unicode_literals -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function from future.utils import PY2, PY3 if PY2: diff --git a/src/allmydata/storage/expirer.py b/src/allmydata/storage/expirer.py index 55ab51843..bcff8aeeb 100644 --- a/src/allmydata/storage/expirer.py +++ b/src/allmydata/storage/expirer.py @@ -1,7 +1,3 @@ -from __future__ import division -from __future__ import absolute_import -from __future__ import print_function -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/storage/immutable.py b/src/allmydata/storage/immutable.py index 0893513ae..fc696ff60 100644 --- a/src/allmydata/storage/immutable.py +++ b/src/allmydata/storage/immutable.py @@ -2,10 +2,6 @@ Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals from future.utils import PY2, bytes_to_native_str if PY2: diff --git a/src/allmydata/storage/immutable_schema.py b/src/allmydata/storage/immutable_schema.py index 40663b935..beb184700 100644 --- a/src/allmydata/storage/immutable_schema.py +++ b/src/allmydata/storage/immutable_schema.py @@ -2,10 +2,6 @@ Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/storage/lease.py b/src/allmydata/storage/lease.py index c0d11abfd..ad21ce598 100644 --- a/src/allmydata/storage/lease.py +++ b/src/allmydata/storage/lease.py @@ -2,10 +2,6 @@ Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/storage/mutable.py b/src/allmydata/storage/mutable.py index 51c3a3c8b..cb9116a21 100644 --- a/src/allmydata/storage/mutable.py +++ b/src/allmydata/storage/mutable.py @@ -2,10 +2,6 @@ Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/storage/mutable_schema.py b/src/allmydata/storage/mutable_schema.py index 4be0d2137..b8530e0aa 100644 --- a/src/allmydata/storage/mutable_schema.py +++ b/src/allmydata/storage/mutable_schema.py @@ -2,10 +2,6 @@ Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/storage/shares.py b/src/allmydata/storage/shares.py index 59e7b1539..5fe2b106c 100644 --- a/src/allmydata/storage/shares.py +++ b/src/allmydata/storage/shares.py @@ -2,10 +2,6 @@ Ported to Python 3. """ -from __future__ import unicode_literals -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function from future.utils import PY2 if PY2: diff --git a/src/allmydata/test/__init__.py b/src/allmydata/test/__init__.py index 893aa15ce..14ec9b054 100644 --- a/src/allmydata/test/__init__.py +++ b/src/allmydata/test/__init__.py @@ -15,10 +15,6 @@ some side-effects which make things better when the test suite runs. Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/test/cli/common.py b/src/allmydata/test/cli/common.py index ed066c6b6..45cafbec6 100644 --- a/src/allmydata/test/cli/common.py +++ b/src/allmydata/test/cli/common.py @@ -1,10 +1,6 @@ """ Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/test/cli/test_admin.py b/src/allmydata/test/cli/test_admin.py index 0570467a5..d1e9263d0 100644 --- a/src/allmydata/test/cli/test_admin.py +++ b/src/allmydata/test/cli/test_admin.py @@ -1,10 +1,6 @@ """ Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/test/cli/test_alias.py b/src/allmydata/test/cli/test_alias.py index a3ee595b8..025a7ae9d 100644 --- a/src/allmydata/test/cli/test_alias.py +++ b/src/allmydata/test/cli/test_alias.py @@ -1,10 +1,6 @@ """ Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/test/cli/test_backup.py b/src/allmydata/test/cli/test_backup.py index df598b811..804817132 100644 --- a/src/allmydata/test/cli/test_backup.py +++ b/src/allmydata/test/cli/test_backup.py @@ -1,10 +1,6 @@ """ Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/test/cli/test_backupdb.py b/src/allmydata/test/cli/test_backupdb.py index 665382dc8..ca18a4a74 100644 --- a/src/allmydata/test/cli/test_backupdb.py +++ b/src/allmydata/test/cli/test_backupdb.py @@ -1,10 +1,6 @@ """ Ported to Python 3. """ -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/test/cli/test_check.py b/src/allmydata/test/cli/test_check.py index d539f3ebe..24fd0d8e8 100644 --- a/src/allmydata/test/cli/test_check.py +++ b/src/allmydata/test/cli/test_check.py @@ -1,7 +1,3 @@ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/test/cli/test_cli.py b/src/allmydata/test/cli/test_cli.py index 72eb011d0..d7fde2742 100644 --- a/src/allmydata/test/cli/test_cli.py +++ b/src/allmydata/test/cli/test_cli.py @@ -1,10 +1,6 @@ """ Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/test/cli/test_cp.py b/src/allmydata/test/cli/test_cp.py index fff50f331..5e8e411ad 100644 --- a/src/allmydata/test/cli/test_cp.py +++ b/src/allmydata/test/cli/test_cp.py @@ -1,10 +1,6 @@ """ Ported to Python 3. """ -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/test/cli/test_create_alias.py b/src/allmydata/test/cli/test_create_alias.py index 176bf7576..cb8526087 100644 --- a/src/allmydata/test/cli/test_create_alias.py +++ b/src/allmydata/test/cli/test_create_alias.py @@ -1,10 +1,6 @@ """ Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/test/cli/test_list.py b/src/allmydata/test/cli/test_list.py index 1206579f1..141908987 100644 --- a/src/allmydata/test/cli/test_list.py +++ b/src/allmydata/test/cli/test_list.py @@ -1,10 +1,6 @@ """ Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals from future.utils import PY2, PY3 if PY2: diff --git a/src/allmydata/test/cli/test_mv.py b/src/allmydata/test/cli/test_mv.py index 0bb9ba369..cb998025d 100644 --- a/src/allmydata/test/cli/test_mv.py +++ b/src/allmydata/test/cli/test_mv.py @@ -1,10 +1,6 @@ """ Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/test/cli/test_status.py b/src/allmydata/test/cli/test_status.py index a015391e2..e8d2ca3c8 100644 --- a/src/allmydata/test/cli/test_status.py +++ b/src/allmydata/test/cli/test_status.py @@ -1,10 +1,6 @@ """ Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/test/cli_node_api.py b/src/allmydata/test/cli_node_api.py index c324d5565..e7d5e6600 100644 --- a/src/allmydata/test/cli_node_api.py +++ b/src/allmydata/test/cli_node_api.py @@ -1,10 +1,6 @@ """ Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/test/common_util.py b/src/allmydata/test/common_util.py index 2bf88ce20..12e5cec72 100644 --- a/src/allmydata/test/common_util.py +++ b/src/allmydata/test/common_util.py @@ -1,10 +1,6 @@ """ Ported to Python 3. """ -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division -from __future__ import unicode_literals from future.utils import PY2, PY3, bchr, binary_type from future.builtins import str as future_str diff --git a/src/allmydata/test/common_web.py b/src/allmydata/test/common_web.py index bd55a9fe9..e43474da1 100644 --- a/src/allmydata/test/common_web.py +++ b/src/allmydata/test/common_web.py @@ -1,10 +1,6 @@ """ Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/test/matchers.py b/src/allmydata/test/matchers.py index cc8bf47be..20edd88c2 100644 --- a/src/allmydata/test/matchers.py +++ b/src/allmydata/test/matchers.py @@ -3,10 +3,6 @@ Testtools-style matchers useful to the Tahoe-LAFS test suite. Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/test/mutable/test_checker.py b/src/allmydata/test/mutable/test_checker.py index 6d9145d68..8eaa6c845 100644 --- a/src/allmydata/test/mutable/test_checker.py +++ b/src/allmydata/test/mutable/test_checker.py @@ -1,10 +1,6 @@ """ Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/test/mutable/test_datahandle.py b/src/allmydata/test/mutable/test_datahandle.py index 7aabcd8e1..79536e7d1 100644 --- a/src/allmydata/test/mutable/test_datahandle.py +++ b/src/allmydata/test/mutable/test_datahandle.py @@ -1,10 +1,6 @@ """ Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/test/mutable/test_different_encoding.py b/src/allmydata/test/mutable/test_different_encoding.py index f1796d373..6bfc7e89a 100644 --- a/src/allmydata/test/mutable/test_different_encoding.py +++ b/src/allmydata/test/mutable/test_different_encoding.py @@ -1,10 +1,6 @@ """ Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/test/mutable/test_exceptions.py b/src/allmydata/test/mutable/test_exceptions.py index 23674d036..fa1c1a884 100644 --- a/src/allmydata/test/mutable/test_exceptions.py +++ b/src/allmydata/test/mutable/test_exceptions.py @@ -2,10 +2,6 @@ Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/test/mutable/test_filehandle.py b/src/allmydata/test/mutable/test_filehandle.py index 795f60654..1e6c61c68 100644 --- a/src/allmydata/test/mutable/test_filehandle.py +++ b/src/allmydata/test/mutable/test_filehandle.py @@ -1,10 +1,6 @@ """ Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/test/mutable/test_filenode.py b/src/allmydata/test/mutable/test_filenode.py index 6c00e4420..ca76be1b0 100644 --- a/src/allmydata/test/mutable/test_filenode.py +++ b/src/allmydata/test/mutable/test_filenode.py @@ -1,10 +1,6 @@ """ Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/test/mutable/test_interoperability.py b/src/allmydata/test/mutable/test_interoperability.py index 496da1d2a..745baca97 100644 --- a/src/allmydata/test/mutable/test_interoperability.py +++ b/src/allmydata/test/mutable/test_interoperability.py @@ -1,10 +1,6 @@ """ Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/test/mutable/test_multiple_encodings.py b/src/allmydata/test/mutable/test_multiple_encodings.py index 2291b60d8..8ccfd387e 100644 --- a/src/allmydata/test/mutable/test_multiple_encodings.py +++ b/src/allmydata/test/mutable/test_multiple_encodings.py @@ -1,10 +1,6 @@ """ Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/test/mutable/test_multiple_versions.py b/src/allmydata/test/mutable/test_multiple_versions.py index c9b7e71df..c9c1f15aa 100644 --- a/src/allmydata/test/mutable/test_multiple_versions.py +++ b/src/allmydata/test/mutable/test_multiple_versions.py @@ -1,10 +1,6 @@ """ Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/test/mutable/test_problems.py b/src/allmydata/test/mutable/test_problems.py index d3a779905..6f22c02c2 100644 --- a/src/allmydata/test/mutable/test_problems.py +++ b/src/allmydata/test/mutable/test_problems.py @@ -1,10 +1,6 @@ """ Ported to Python 3. """ -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/test/mutable/test_repair.py b/src/allmydata/test/mutable/test_repair.py index deddb8d92..f4ae53e13 100644 --- a/src/allmydata/test/mutable/test_repair.py +++ b/src/allmydata/test/mutable/test_repair.py @@ -1,10 +1,6 @@ """ Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/test/mutable/test_roundtrip.py b/src/allmydata/test/mutable/test_roundtrip.py index 96ecdf640..eddd7f3df 100644 --- a/src/allmydata/test/mutable/test_roundtrip.py +++ b/src/allmydata/test/mutable/test_roundtrip.py @@ -1,10 +1,6 @@ """ Ported to Python 3. """ -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/test/mutable/test_servermap.py b/src/allmydata/test/mutable/test_servermap.py index 505d31e73..b3a247015 100644 --- a/src/allmydata/test/mutable/test_servermap.py +++ b/src/allmydata/test/mutable/test_servermap.py @@ -2,10 +2,6 @@ Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/test/mutable/test_update.py b/src/allmydata/test/mutable/test_update.py index 1c91590bd..02fca24b2 100644 --- a/src/allmydata/test/mutable/test_update.py +++ b/src/allmydata/test/mutable/test_update.py @@ -1,10 +1,6 @@ """ Ported to Python 3. """ -from __future__ import print_function -from __future__ import division -from __future__ import absolute_import -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/test/mutable/util.py b/src/allmydata/test/mutable/util.py index bed350652..336b8bfd7 100644 --- a/src/allmydata/test/mutable/util.py +++ b/src/allmydata/test/mutable/util.py @@ -1,10 +1,6 @@ """ Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals from future.utils import PY2, bchr if PY2: diff --git a/src/allmydata/test/storage_plugin.py b/src/allmydata/test/storage_plugin.py index d3f1ec7c9..cb18f4dda 100644 --- a/src/allmydata/test/storage_plugin.py +++ b/src/allmydata/test/storage_plugin.py @@ -4,10 +4,6 @@ functionality. Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/test/strategies.py b/src/allmydata/test/strategies.py index 35b9766b0..92290203e 100644 --- a/src/allmydata/test/strategies.py +++ b/src/allmydata/test/strategies.py @@ -3,10 +3,6 @@ Hypothesis strategies use for testing Tahoe-LAFS. Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/test/test_abbreviate.py b/src/allmydata/test/test_abbreviate.py index 3ef1e96a6..cefa7f23f 100644 --- a/src/allmydata/test/test_abbreviate.py +++ b/src/allmydata/test/test_abbreviate.py @@ -3,10 +3,6 @@ Tests for allmydata.util.abbreviate. Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/test/test_base32.py b/src/allmydata/test/test_base32.py index 0b9a018b9..236560599 100644 --- a/src/allmydata/test/test_base32.py +++ b/src/allmydata/test/test_base32.py @@ -3,10 +3,6 @@ Tests for allmydata.util.base32. Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/test/test_base62.py b/src/allmydata/test/test_base62.py index 8bbb6dfeb..258ca62ec 100644 --- a/src/allmydata/test/test_base62.py +++ b/src/allmydata/test/test_base62.py @@ -4,10 +4,6 @@ Tests for allmydata.util.base62. Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/test/test_checker.py b/src/allmydata/test/test_checker.py index 3d64d4976..a33fc23b5 100644 --- a/src/allmydata/test/test_checker.py +++ b/src/allmydata/test/test_checker.py @@ -2,10 +2,6 @@ Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/test/test_codec.py b/src/allmydata/test/test_codec.py index ee64e2bf2..efb575cd7 100644 --- a/src/allmydata/test/test_codec.py +++ b/src/allmydata/test/test_codec.py @@ -3,10 +3,6 @@ Tests for allmydata.codec. Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/test/test_common_util.py b/src/allmydata/test/test_common_util.py index c141adc8d..b1c698208 100644 --- a/src/allmydata/test/test_common_util.py +++ b/src/allmydata/test/test_common_util.py @@ -1,10 +1,6 @@ """ This module has been ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/test/test_configutil.py b/src/allmydata/test/test_configutil.py index 1b8fb5029..d1931479f 100644 --- a/src/allmydata/test/test_configutil.py +++ b/src/allmydata/test/test_configutil.py @@ -3,10 +3,6 @@ Tests for allmydata.util.configutil. Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/test/test_connections.py b/src/allmydata/test/test_connections.py index 5816afdab..3cf8ed937 100644 --- a/src/allmydata/test/test_connections.py +++ b/src/allmydata/test/test_connections.py @@ -1,10 +1,6 @@ """ Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/test/test_consumer.py b/src/allmydata/test/test_consumer.py index ee1908ba7..9465b42cd 100644 --- a/src/allmydata/test/test_consumer.py +++ b/src/allmydata/test/test_consumer.py @@ -4,10 +4,6 @@ Tests for allmydata.util.consumer. Ported to Python 3. """ -from __future__ import unicode_literals -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function from future.utils import PY2 if PY2: diff --git a/src/allmydata/test/test_crawler.py b/src/allmydata/test/test_crawler.py index 80d732986..02181dd55 100644 --- a/src/allmydata/test/test_crawler.py +++ b/src/allmydata/test/test_crawler.py @@ -4,10 +4,6 @@ Tests for allmydata.storage.crawler. Ported to Python 3. """ -from __future__ import print_function -from __future__ import division -from __future__ import absolute_import -from __future__ import unicode_literals from future.utils import PY2, PY3 if PY2: diff --git a/src/allmydata/test/test_crypto.py b/src/allmydata/test/test_crypto.py index b7c84b447..2114b053b 100644 --- a/src/allmydata/test/test_crypto.py +++ b/src/allmydata/test/test_crypto.py @@ -1,7 +1,3 @@ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/test/test_deepcheck.py b/src/allmydata/test/test_deepcheck.py index 652e51ea5..8af75bcd0 100644 --- a/src/allmydata/test/test_deepcheck.py +++ b/src/allmydata/test/test_deepcheck.py @@ -1,10 +1,6 @@ """ Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals # Python 2 compatibility # Can't use `builtins.str` because something deep in Twisted callbacks ends up repr'ing diff --git a/src/allmydata/test/test_dirnode.py b/src/allmydata/test/test_dirnode.py index 2319e3ce1..2349f3c4e 100644 --- a/src/allmydata/test/test_dirnode.py +++ b/src/allmydata/test/test_dirnode.py @@ -2,10 +2,6 @@ Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals from past.builtins import long diff --git a/src/allmydata/test/test_encode.py b/src/allmydata/test/test_encode.py index 028a988cb..33c946010 100644 --- a/src/allmydata/test/test_encode.py +++ b/src/allmydata/test/test_encode.py @@ -1,10 +1,6 @@ """ Ported to Python 3. """ -from __future__ import division -from __future__ import absolute_import -from __future__ import print_function -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/test/test_encodingutil.py b/src/allmydata/test/test_encodingutil.py index 062c64ba1..b218b0bd9 100644 --- a/src/allmydata/test/test_encodingutil.py +++ b/src/allmydata/test/test_encodingutil.py @@ -1,7 +1,3 @@ -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division -from __future__ import unicode_literals from future.utils import PY2, PY3 if PY2: diff --git a/src/allmydata/test/test_filenode.py b/src/allmydata/test/test_filenode.py index a8f0e2431..2bf1edd6c 100644 --- a/src/allmydata/test/test_filenode.py +++ b/src/allmydata/test/test_filenode.py @@ -1,10 +1,6 @@ """ Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/test/test_happiness.py b/src/allmydata/test/test_happiness.py index 9ff36ef26..8f21b1363 100644 --- a/src/allmydata/test/test_happiness.py +++ b/src/allmydata/test/test_happiness.py @@ -6,10 +6,6 @@ allmydata.util.happinessutil. Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/test/test_hashutil.py b/src/allmydata/test/test_hashutil.py index 1efa2d9db..d64fb8abc 100644 --- a/src/allmydata/test/test_hashutil.py +++ b/src/allmydata/test/test_hashutil.py @@ -3,10 +3,6 @@ Tests for allmydata.util.hashutil. Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/test/test_humanreadable.py b/src/allmydata/test/test_humanreadable.py index 94de8f6be..b74277448 100644 --- a/src/allmydata/test/test_humanreadable.py +++ b/src/allmydata/test/test_humanreadable.py @@ -4,10 +4,6 @@ Tests for allmydata.util.humanreadable. This module has been ported to Python 3. """ -from __future__ import unicode_literals -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function from future.utils import PY2 if PY2: diff --git a/src/allmydata/test/test_hung_server.py b/src/allmydata/test/test_hung_server.py index 162b1d79c..24e2810e0 100644 --- a/src/allmydata/test/test_hung_server.py +++ b/src/allmydata/test/test_hung_server.py @@ -3,10 +3,6 @@ """ Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/test/test_i2p_provider.py b/src/allmydata/test/test_i2p_provider.py index 072b17647..8628aeed6 100644 --- a/src/allmydata/test/test_i2p_provider.py +++ b/src/allmydata/test/test_i2p_provider.py @@ -1,10 +1,6 @@ """ Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/test/test_immutable.py b/src/allmydata/test/test_immutable.py index 12f2012e0..303485831 100644 --- a/src/allmydata/test/test_immutable.py +++ b/src/allmydata/test/test_immutable.py @@ -1,10 +1,6 @@ """ This module has been ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/test/test_introducer.py b/src/allmydata/test/test_introducer.py index 0475d3f6c..2fd6ff69b 100644 --- a/src/allmydata/test/test_introducer.py +++ b/src/allmydata/test/test_introducer.py @@ -1,10 +1,6 @@ """ Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/test/test_json_metadata.py b/src/allmydata/test/test_json_metadata.py index a0cb9c142..822d37925 100644 --- a/src/allmydata/test/test_json_metadata.py +++ b/src/allmydata/test/test_json_metadata.py @@ -1,10 +1,6 @@ """ Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/test/test_log.py b/src/allmydata/test/test_log.py index bf079aaeb..c3472cad8 100644 --- a/src/allmydata/test/test_log.py +++ b/src/allmydata/test/test_log.py @@ -4,10 +4,6 @@ Tests for allmydata.util.log. Ported to Python 3. """ -from __future__ import unicode_literals -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function from future.utils import PY2, native_str if PY2: diff --git a/src/allmydata/test/test_monitor.py b/src/allmydata/test/test_monitor.py index 7010da73a..bede3e001 100644 --- a/src/allmydata/test/test_monitor.py +++ b/src/allmydata/test/test_monitor.py @@ -2,10 +2,6 @@ Tests for allmydata.monitor. """ -from __future__ import unicode_literals -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function from future.utils import PY2 if PY2: diff --git a/src/allmydata/test/test_multi_introducers.py b/src/allmydata/test/test_multi_introducers.py index 2b0879530..7f42177ab 100644 --- a/src/allmydata/test/test_multi_introducers.py +++ b/src/allmydata/test/test_multi_introducers.py @@ -1,10 +1,6 @@ """ Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/test/test_netstring.py b/src/allmydata/test/test_netstring.py index d5ff379cd..54cc16df8 100644 --- a/src/allmydata/test/test_netstring.py +++ b/src/allmydata/test/test_netstring.py @@ -3,10 +3,6 @@ Tests for allmydata.util.netstring. Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/test/test_no_network.py b/src/allmydata/test/test_no_network.py index b1aa1350a..2d6bdb896 100644 --- a/src/allmydata/test/test_no_network.py +++ b/src/allmydata/test/test_no_network.py @@ -3,10 +3,6 @@ Test the NoNetworkGrid test harness. Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/test/test_observer.py b/src/allmydata/test/test_observer.py index 134876be3..ddc58b074 100644 --- a/src/allmydata/test/test_observer.py +++ b/src/allmydata/test/test_observer.py @@ -4,10 +4,6 @@ Tests for allmydata.util.observer. Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/test/test_openmetrics.py b/src/allmydata/test/test_openmetrics.py index 66cbc7dec..2199bcc5f 100644 --- a/src/allmydata/test/test_openmetrics.py +++ b/src/allmydata/test/test_openmetrics.py @@ -4,10 +4,6 @@ Tests for ``/statistics?t=openmetrics``. Ported to Python 3. """ -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division -from __future__ import unicode_literals from future.utils import PY2 diff --git a/src/allmydata/test/test_python2_regressions.py b/src/allmydata/test/test_python2_regressions.py index c641d2dba..9a12f0374 100644 --- a/src/allmydata/test/test_python2_regressions.py +++ b/src/allmydata/test/test_python2_regressions.py @@ -2,10 +2,6 @@ Tests to check for Python2 regressions """ -from __future__ import unicode_literals -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function from future.utils import PY2 if PY2: diff --git a/src/allmydata/test/test_repairer.py b/src/allmydata/test/test_repairer.py index 8545b1cf4..280825775 100644 --- a/src/allmydata/test/test_repairer.py +++ b/src/allmydata/test/test_repairer.py @@ -2,10 +2,6 @@ """ Ported to Python 3. """ -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/test/test_runner.py b/src/allmydata/test/test_runner.py index a84fa28f8..b784b2c61 100644 --- a/src/allmydata/test/test_runner.py +++ b/src/allmydata/test/test_runner.py @@ -2,13 +2,6 @@ Ported to Python 3 """ -from __future__ import ( - absolute_import, -) -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals - from future.utils import PY2 if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 diff --git a/src/allmydata/test/test_sftp.py b/src/allmydata/test/test_sftp.py index 2214e4e5b..d1902b9ee 100644 --- a/src/allmydata/test/test_sftp.py +++ b/src/allmydata/test/test_sftp.py @@ -1,10 +1,6 @@ """ Ported to Python 3. """ -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/test/test_spans.py b/src/allmydata/test/test_spans.py index 281f916c4..e1ac9a229 100644 --- a/src/allmydata/test/test_spans.py +++ b/src/allmydata/test/test_spans.py @@ -2,10 +2,6 @@ Tests for allmydata.util.spans. """ -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/test/test_statistics.py b/src/allmydata/test/test_statistics.py index 476f0a084..ce91e4efb 100644 --- a/src/allmydata/test/test_statistics.py +++ b/src/allmydata/test/test_statistics.py @@ -3,10 +3,6 @@ Tests for allmydata.util.statistics. Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/test/test_stats.py b/src/allmydata/test/test_stats.py index 6fe690f1f..8f02640e0 100644 --- a/src/allmydata/test/test_stats.py +++ b/src/allmydata/test/test_stats.py @@ -1,10 +1,6 @@ """ Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/test/test_storage_web.py b/src/allmydata/test/test_storage_web.py index b47c93849..10249c83a 100644 --- a/src/allmydata/test/test_storage_web.py +++ b/src/allmydata/test/test_storage_web.py @@ -4,10 +4,6 @@ Tests for twisted.storage that uses Web APIs. Partially ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/test/test_time_format.py b/src/allmydata/test/test_time_format.py index f83a6a53c..2f10a3ea9 100644 --- a/src/allmydata/test/test_time_format.py +++ b/src/allmydata/test/test_time_format.py @@ -1,10 +1,6 @@ """ Tests for allmydata.util.time_format. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/test/test_upload.py b/src/allmydata/test/test_upload.py index 18192de6c..f9541e946 100644 --- a/src/allmydata/test/test_upload.py +++ b/src/allmydata/test/test_upload.py @@ -3,10 +3,6 @@ """ Ported to Python 3. """ -from __future__ import unicode_literals -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function from future.utils import PY2 if PY2: diff --git a/src/allmydata/test/test_uri.py b/src/allmydata/test/test_uri.py index 748a0f6ef..7f4908da6 100644 --- a/src/allmydata/test/test_uri.py +++ b/src/allmydata/test/test_uri.py @@ -4,10 +4,6 @@ Tests for allmydata.uri. Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/test/test_util.py b/src/allmydata/test/test_util.py index d3a36a756..0b4917381 100644 --- a/src/allmydata/test/test_util.py +++ b/src/allmydata/test/test_util.py @@ -2,10 +2,6 @@ Ported to Python3. """ -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/test/web/common.py b/src/allmydata/test/web/common.py index fbf7b015f..be7fe0af1 100644 --- a/src/allmydata/test/web/common.py +++ b/src/allmydata/test/web/common.py @@ -1,10 +1,6 @@ """ Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/test/web/matchers.py b/src/allmydata/test/web/matchers.py index f764da79d..f6d52aaad 100644 --- a/src/allmydata/test/web/matchers.py +++ b/src/allmydata/test/web/matchers.py @@ -1,10 +1,6 @@ """ Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/test/web/test_common.py b/src/allmydata/test/web/test_common.py index 84ab5cab2..6c19a5b78 100644 --- a/src/allmydata/test/web/test_common.py +++ b/src/allmydata/test/web/test_common.py @@ -3,10 +3,6 @@ Tests for ``allmydata.web.common``. Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/test/web/test_grid.py b/src/allmydata/test/web/test_grid.py index 1ebe3a90f..0a7c913c0 100644 --- a/src/allmydata/test/web/test_grid.py +++ b/src/allmydata/test/web/test_grid.py @@ -1,10 +1,6 @@ """ Ported to Python 3. """ -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/test/web/test_introducer.py b/src/allmydata/test/web/test_introducer.py index 69309d35b..d21a663f7 100644 --- a/src/allmydata/test/web/test_introducer.py +++ b/src/allmydata/test/web/test_introducer.py @@ -1,10 +1,6 @@ """ Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/test/web/test_logs.py b/src/allmydata/test/web/test_logs.py index a8b479a97..093503265 100644 --- a/src/allmydata/test/web/test_logs.py +++ b/src/allmydata/test/web/test_logs.py @@ -4,13 +4,6 @@ Tests for ``allmydata.web.logs``. Ported to Python 3. """ -from __future__ import ( - print_function, - unicode_literals, - absolute_import, - division, -) - from future.utils import PY2 if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 diff --git a/src/allmydata/test/web/test_private.py b/src/allmydata/test/web/test_private.py index b426b4d93..5652e8008 100644 --- a/src/allmydata/test/web/test_private.py +++ b/src/allmydata/test/web/test_private.py @@ -4,13 +4,6 @@ Tests for ``allmydata.web.private``. Ported to Python 3. """ -from __future__ import ( - print_function, - unicode_literals, - absolute_import, - division, -) - from future.utils import PY2 if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 diff --git a/src/allmydata/test/web/test_root.py b/src/allmydata/test/web/test_root.py index 228b8e449..e810f9a14 100644 --- a/src/allmydata/test/web/test_root.py +++ b/src/allmydata/test/web/test_root.py @@ -1,10 +1,6 @@ """ Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/test/web/test_status.py b/src/allmydata/test/web/test_status.py index 414925446..c46ac4500 100644 --- a/src/allmydata/test/web/test_status.py +++ b/src/allmydata/test/web/test_status.py @@ -3,10 +3,6 @@ Tests for ```allmydata.web.status```. Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/test/web/test_util.py b/src/allmydata/test/web/test_util.py index c536dc9f1..5afe884b0 100644 --- a/src/allmydata/test/web/test_util.py +++ b/src/allmydata/test/web/test_util.py @@ -1,10 +1,6 @@ """ Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/unknown.py b/src/allmydata/unknown.py index 060696293..bfb56496d 100644 --- a/src/allmydata/unknown.py +++ b/src/allmydata/unknown.py @@ -1,9 +1,5 @@ """Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/util/abbreviate.py b/src/allmydata/util/abbreviate.py index f895c3727..e775813bd 100644 --- a/src/allmydata/util/abbreviate.py +++ b/src/allmydata/util/abbreviate.py @@ -3,10 +3,6 @@ Convert timestamps to abbreviated English text. Ported to Python 3. """ -from __future__ import division -from __future__ import absolute_import -from __future__ import print_function -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/util/assertutil.py b/src/allmydata/util/assertutil.py index ed4b8599f..2088f505c 100644 --- a/src/allmydata/util/assertutil.py +++ b/src/allmydata/util/assertutil.py @@ -7,10 +7,6 @@ have tests. Ported to Python 3. """ -from __future__ import division -from __future__ import absolute_import -from __future__ import print_function -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/util/base62.py b/src/allmydata/util/base62.py index 964baff34..dcde36562 100644 --- a/src/allmydata/util/base62.py +++ b/src/allmydata/util/base62.py @@ -3,10 +3,6 @@ Base62 encoding. Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/util/configutil.py b/src/allmydata/util/configutil.py index 74b0c714a..749f6d7d7 100644 --- a/src/allmydata/util/configutil.py +++ b/src/allmydata/util/configutil.py @@ -5,10 +5,6 @@ Configuration is returned as Unicode strings. Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/util/consumer.py b/src/allmydata/util/consumer.py index 3de82974d..cd73119fe 100644 --- a/src/allmydata/util/consumer.py +++ b/src/allmydata/util/consumer.py @@ -4,10 +4,6 @@ a filenode's read() method. See download_to_data() for an example of its use. Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/util/dbutil.py b/src/allmydata/util/dbutil.py index 45e59cf00..ac7ddbfeb 100644 --- a/src/allmydata/util/dbutil.py +++ b/src/allmydata/util/dbutil.py @@ -6,10 +6,6 @@ Test coverage currently provided by test_backupdb.py. Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/util/encodingutil.py b/src/allmydata/util/encodingutil.py index 5e28f59fe..d13011dd2 100644 --- a/src/allmydata/util/encodingutil.py +++ b/src/allmydata/util/encodingutil.py @@ -7,10 +7,6 @@ Ported to Python 3. Once Python 2 support is dropped, most of this module will obsolete, since Unicode is the default everywhere in Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals from future.utils import PY2, PY3, native_str from future.builtins import str as future_str diff --git a/src/allmydata/util/fileutil.py b/src/allmydata/util/fileutil.py index e40e06180..0df3781b5 100644 --- a/src/allmydata/util/fileutil.py +++ b/src/allmydata/util/fileutil.py @@ -4,10 +4,6 @@ Ported to Python3. Futz with files like a pro. """ -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/util/gcutil.py b/src/allmydata/util/gcutil.py index 33f1f64f5..9eed8b1f4 100644 --- a/src/allmydata/util/gcutil.py +++ b/src/allmydata/util/gcutil.py @@ -10,10 +10,6 @@ Helpers for managing garbage collection. Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/util/happinessutil.py b/src/allmydata/util/happinessutil.py index 9f2617a5e..7f5d0f8a7 100644 --- a/src/allmydata/util/happinessutil.py +++ b/src/allmydata/util/happinessutil.py @@ -4,10 +4,6 @@ reporting it in messages. Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/util/hashutil.py b/src/allmydata/util/hashutil.py index 8525dd95e..579f55d88 100644 --- a/src/allmydata/util/hashutil.py +++ b/src/allmydata/util/hashutil.py @@ -3,10 +3,6 @@ Hashing utilities. Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/util/humanreadable.py b/src/allmydata/util/humanreadable.py index 60ac57083..2eaa7c79a 100644 --- a/src/allmydata/util/humanreadable.py +++ b/src/allmydata/util/humanreadable.py @@ -4,10 +4,6 @@ Utilities for turning objects into human-readable strings. This module has been ported to Python 3. """ -from __future__ import division -from __future__ import absolute_import -from __future__ import print_function -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/util/idlib.py b/src/allmydata/util/idlib.py index eafcbc388..c74f11599 100644 --- a/src/allmydata/util/idlib.py +++ b/src/allmydata/util/idlib.py @@ -1,10 +1,6 @@ """ Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/util/jsonbytes.py b/src/allmydata/util/jsonbytes.py index ea95bb5b8..24c8f3311 100644 --- a/src/allmydata/util/jsonbytes.py +++ b/src/allmydata/util/jsonbytes.py @@ -4,10 +4,6 @@ A JSON encoder than can serialize bytes. Ported to Python 3. """ -from __future__ import unicode_literals -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function from future.utils import PY2 if PY2: diff --git a/src/allmydata/util/log.py b/src/allmydata/util/log.py index b442d30bb..afd612d7a 100644 --- a/src/allmydata/util/log.py +++ b/src/allmydata/util/log.py @@ -3,10 +3,6 @@ Logging utilities. Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/util/mathutil.py b/src/allmydata/util/mathutil.py index 42863c30e..a6a58e834 100644 --- a/src/allmydata/util/mathutil.py +++ b/src/allmydata/util/mathutil.py @@ -6,10 +6,6 @@ Backwards compatibility for direct imports. Ported to Python 3. """ -from __future__ import division -from __future__ import absolute_import -from __future__ import print_function -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/util/netstring.py b/src/allmydata/util/netstring.py index 423b8665b..23d8492ab 100644 --- a/src/allmydata/util/netstring.py +++ b/src/allmydata/util/netstring.py @@ -3,10 +3,6 @@ Netstring encoding and decoding. Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/util/observer.py b/src/allmydata/util/observer.py index 4a39fe014..b753d51ad 100644 --- a/src/allmydata/util/observer.py +++ b/src/allmydata/util/observer.py @@ -4,10 +4,6 @@ Observer for Twisted code. Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/util/rrefutil.py b/src/allmydata/util/rrefutil.py index f39890ff1..1fb757f34 100644 --- a/src/allmydata/util/rrefutil.py +++ b/src/allmydata/util/rrefutil.py @@ -1,10 +1,6 @@ """ Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/util/spans.py b/src/allmydata/util/spans.py index 81e14c0fb..4ddc67704 100644 --- a/src/allmydata/util/spans.py +++ b/src/allmydata/util/spans.py @@ -1,7 +1,3 @@ -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/util/statistics.py b/src/allmydata/util/statistics.py index a517751d6..35a7216a2 100644 --- a/src/allmydata/util/statistics.py +++ b/src/allmydata/util/statistics.py @@ -11,10 +11,6 @@ Ported to Python 3. # Transitive Grace Period Public License, version 1 or later. -from __future__ import unicode_literals -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function from future.utils import PY2 if PY2: diff --git a/src/allmydata/util/time_format.py b/src/allmydata/util/time_format.py index ff267485e..af98bf1d2 100644 --- a/src/allmydata/util/time_format.py +++ b/src/allmydata/util/time_format.py @@ -4,10 +4,6 @@ Time formatting utilities. ISO-8601: http://www.cl.cam.ac.uk/~mgk25/iso-time.html """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/util/yamlutil.py b/src/allmydata/util/yamlutil.py index fd9fc73e2..eae1f8a6e 100644 --- a/src/allmydata/util/yamlutil.py +++ b/src/allmydata/util/yamlutil.py @@ -1,10 +1,6 @@ """ Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/web/check_results.py b/src/allmydata/web/check_results.py index 6c8810f2b..4bb34a2a8 100644 --- a/src/allmydata/web/check_results.py +++ b/src/allmydata/web/check_results.py @@ -1,10 +1,6 @@ """ Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/web/directory.py b/src/allmydata/web/directory.py index 240fdd90c..4012f93fd 100644 --- a/src/allmydata/web/directory.py +++ b/src/allmydata/web/directory.py @@ -1,10 +1,6 @@ """ Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/web/info.py b/src/allmydata/web/info.py index 2d45f9994..ff126a2a1 100644 --- a/src/allmydata/web/info.py +++ b/src/allmydata/web/info.py @@ -1,10 +1,6 @@ """ Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/web/logs.py b/src/allmydata/web/logs.py index a79440eb9..7c4d285f5 100644 --- a/src/allmydata/web/logs.py +++ b/src/allmydata/web/logs.py @@ -1,12 +1,6 @@ """ Ported to Python 3. """ -from __future__ import ( - print_function, - unicode_literals, - absolute_import, - division, -) from autobahn.twisted.resource import WebSocketResource from autobahn.twisted.websocket import ( diff --git a/src/allmydata/web/operations.py b/src/allmydata/web/operations.py index a564f8484..4319a4d9c 100644 --- a/src/allmydata/web/operations.py +++ b/src/allmydata/web/operations.py @@ -1,10 +1,6 @@ """ Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/web/private.py b/src/allmydata/web/private.py index 4f59be33a..55cace5a3 100644 --- a/src/allmydata/web/private.py +++ b/src/allmydata/web/private.py @@ -1,10 +1,6 @@ """ Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/web/status.py b/src/allmydata/web/status.py index 4a902a98b..96d68c87f 100644 --- a/src/allmydata/web/status.py +++ b/src/allmydata/web/status.py @@ -2,10 +2,6 @@ Ported to Python 3. """ -from __future__ import division -from __future__ import absolute_import -from __future__ import print_function -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/web/storage.py b/src/allmydata/web/storage.py index e568d5ed5..ebbef4fa3 100644 --- a/src/allmydata/web/storage.py +++ b/src/allmydata/web/storage.py @@ -1,10 +1,6 @@ """ Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/web/storage_plugins.py b/src/allmydata/web/storage_plugins.py index 41bed9d81..9be4e84af 100644 --- a/src/allmydata/web/storage_plugins.py +++ b/src/allmydata/web/storage_plugins.py @@ -4,10 +4,6 @@ of all enabled storage client plugins. Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals from future.utils import PY2 if PY2: diff --git a/src/allmydata/windows/fixups.py b/src/allmydata/windows/fixups.py index 53eb14d53..87e32a10f 100644 --- a/src/allmydata/windows/fixups.py +++ b/src/allmydata/windows/fixups.py @@ -1,4 +1,3 @@ -from __future__ import print_function from future.utils import PY3 from past.builtins import unicode diff --git a/ws_client.py b/ws_client.py index 15d717d42..b444a878a 100644 --- a/ws_client.py +++ b/ws_client.py @@ -1,4 +1,3 @@ -from __future__ import print_function import sys import json From 09d78b129b38afa1f94c6a0fc407086fc36720c0 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 15 Feb 2024 12:56:00 -0500 Subject: [PATCH 2262/2309] Update line number --- src/allmydata/test/test_humanreadable.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/test_humanreadable.py b/src/allmydata/test/test_humanreadable.py index b74277448..9cdc9678c 100644 --- a/src/allmydata/test/test_humanreadable.py +++ b/src/allmydata/test/test_humanreadable.py @@ -27,7 +27,7 @@ class NoArgumentException(Exception): class HumanReadable(unittest.TestCase): def test_repr(self): hr = humanreadable.hr - self.failUnlessEqual(hr(foo), "") + self.failUnlessEqual(hr(foo), "") self.failUnlessEqual(hr(self.test_repr), ">") self.failUnlessEqual(hr(long(1)), "1") From 43c9d04695671fe6686f8395e48151a986a75552 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 15 Feb 2024 13:56:18 -0500 Subject: [PATCH 2263/2309] News fragment. --- newsfragments/4090.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/4090.minor diff --git a/newsfragments/4090.minor b/newsfragments/4090.minor new file mode 100644 index 000000000..e69de29bb From 1cfe843d23c901ff9dac1e81aef8dda9a73ec7fe Mon Sep 17 00:00:00 2001 From: Alexandre Detiste Date: Fri, 23 Feb 2024 00:40:25 +0100 Subject: [PATCH 2264/2309] more python2 removal --- docs/conf.py | 3 - integration/test_aaa_aardvark.py | 5 -- integration/test_sftp.py | 5 -- integration/test_streaming_logs.py | 4 - misc/simulators/sizes.py | 6 -- src/allmydata/__main__.py | 5 -- src/allmydata/_monkeypatch.py | 27 ------- src/allmydata/blacklist.py | 4 - src/allmydata/check_results.py | 4 - src/allmydata/codec.py | 4 - src/allmydata/crypto/__init__.py | 4 - src/allmydata/crypto/aes.py | 4 - src/allmydata/crypto/error.py | 5 -- src/allmydata/crypto/util.py | 4 - src/allmydata/deep_stats.py | 4 - src/allmydata/dirnode.py | 4 - src/allmydata/frontends/auth.py | 4 - src/allmydata/frontends/sftpd.py | 4 - src/allmydata/hashtree.py | 5 -- src/allmydata/history.py | 5 -- src/allmydata/immutable/checker.py | 4 - .../immutable/downloader/__init__.py | 6 -- src/allmydata/immutable/downloader/common.py | 5 -- src/allmydata/immutable/downloader/fetcher.py | 5 -- src/allmydata/immutable/downloader/finder.py | 3 - src/allmydata/immutable/downloader/node.py | 4 - .../immutable/downloader/segmentation.py | 4 - src/allmydata/immutable/downloader/share.py | 4 - src/allmydata/immutable/downloader/status.py | 4 - src/allmydata/immutable/encode.py | 5 -- src/allmydata/immutable/filenode.py | 5 -- src/allmydata/immutable/happiness_upload.py | 5 -- src/allmydata/immutable/literal.py | 5 -- src/allmydata/immutable/offloaded.py | 4 - src/allmydata/immutable/repairer.py | 4 - src/allmydata/interfaces.py | 4 - src/allmydata/introducer/__init__.py | 6 -- src/allmydata/introducer/client.py | 3 - src/allmydata/introducer/common.py | 4 - src/allmydata/introducer/interfaces.py | 3 - src/allmydata/monitor.py | 4 - src/allmydata/mutable/checker.py | 3 - src/allmydata/mutable/layout.py | 4 - src/allmydata/mutable/publish.py | 4 - src/allmydata/mutable/repairer.py | 4 - src/allmydata/scripts/admin.py | 4 - src/allmydata/scripts/backupdb.py | 4 - src/allmydata/scripts/default_nodedir.py | 5 -- src/allmydata/scripts/runner.py | 21 ----- src/allmydata/scripts/slow_operation.py | 2 - src/allmydata/scripts/tahoe_add_alias.py | 4 - src/allmydata/scripts/tahoe_backup.py | 4 - src/allmydata/scripts/tahoe_check.py | 4 - src/allmydata/scripts/tahoe_cp.py | 4 - src/allmydata/scripts/tahoe_get.py | 2 - src/allmydata/scripts/tahoe_ls.py | 4 - src/allmydata/scripts/tahoe_manifest.py | 2 - src/allmydata/scripts/tahoe_mkdir.py | 4 - src/allmydata/scripts/tahoe_mv.py | 4 - src/allmydata/scripts/tahoe_run.py | 4 - src/allmydata/scripts/tahoe_status.py | 4 - src/allmydata/scripts/tahoe_unlink.py | 4 - src/allmydata/scripts/tahoe_webopen.py | 4 - src/allmydata/storage/common.py | 2 - src/allmydata/storage/crawler.py | 2 - src/allmydata/storage/expirer.py | 4 - src/allmydata/storage/immutable.py | 2 - src/allmydata/storage/immutable_schema.py | 5 -- src/allmydata/storage/lease.py | 5 -- src/allmydata/storage/mutable.py | 5 -- src/allmydata/storage/mutable_schema.py | 5 -- src/allmydata/storage/shares.py | 5 -- src/allmydata/test/__init__.py | 4 - src/allmydata/test/cli/common.py | 4 - src/allmydata/test/cli/test_admin.py | 4 - src/allmydata/test/cli/test_alias.py | 4 - src/allmydata/test/cli/test_backup.py | 4 - src/allmydata/test/cli/test_backupdb.py | 5 -- src/allmydata/test/cli/test_check.py | 4 - src/allmydata/test/cli/test_cli.py | 4 - src/allmydata/test/cli/test_cp.py | 8 -- src/allmydata/test/cli/test_create_alias.py | 4 - src/allmydata/test/cli/test_list.py | 2 - src/allmydata/test/cli/test_mv.py | 4 - src/allmydata/test/cli/test_status.py | 3 - src/allmydata/test/cli_node_api.py | 4 - src/allmydata/test/common.py | 3 - src/allmydata/test/common_util.py | 2 - src/allmydata/test/common_web.py | 4 - src/allmydata/test/matchers.py | 4 - src/allmydata/test/mutable/test_checker.py | 4 - src/allmydata/test/mutable/test_datahandle.py | 4 - .../test/mutable/test_different_encoding.py | 4 - src/allmydata/test/mutable/test_exceptions.py | 5 -- src/allmydata/test/mutable/test_filehandle.py | 4 - src/allmydata/test/mutable/test_filenode.py | 4 - .../test/mutable/test_interoperability.py | 4 - .../test/mutable/test_multiple_encodings.py | 4 - .../test/mutable/test_multiple_versions.py | 4 - src/allmydata/test/mutable/test_problems.py | 4 - src/allmydata/test/mutable/test_repair.py | 4 - src/allmydata/test/mutable/test_roundtrip.py | 4 - src/allmydata/test/mutable/test_servermap.py | 5 -- src/allmydata/test/mutable/test_update.py | 4 - src/allmydata/test/mutable/util.py | 2 - src/allmydata/test/storage_plugin.py | 3 - src/allmydata/test/strategies.py | 4 - src/allmydata/test/test_abbreviate.py | 4 - src/allmydata/test/test_base32.py | 4 - src/allmydata/test/test_base62.py | 5 -- src/allmydata/test/test_checker.py | 6 -- src/allmydata/test/test_codec.py | 4 - src/allmydata/test/test_common_util.py | 2 - src/allmydata/test/test_configutil.py | 5 -- src/allmydata/test/test_connections.py | 4 - src/allmydata/test/test_consumer.py | 5 -- src/allmydata/test/test_crawler.py | 5 -- src/allmydata/test/test_crypto.py | 4 - src/allmydata/test/test_deepcheck.py | 14 ---- src/allmydata/test/test_dictutil.py | 26 ------- src/allmydata/test/test_dirnode.py | 5 -- src/allmydata/test/test_encode.py | 3 - src/allmydata/test/test_encodingutil.py | 33 -------- src/allmydata/test/test_filenode.py | 4 - src/allmydata/test/test_happiness.py | 6 -- src/allmydata/test/test_hashutil.py | 4 - src/allmydata/test/test_humanreadable.py | 5 -- src/allmydata/test/test_hung_server.py | 4 - src/allmydata/test/test_i2p_provider.py | 4 - src/allmydata/test/test_immutable.py | 4 - src/allmydata/test/test_introducer.py | 4 - src/allmydata/test/test_json_metadata.py | 4 - src/allmydata/test/test_log.py | 2 - src/allmydata/test/test_monitor.py | 5 -- src/allmydata/test/test_multi_introducers.py | 3 - src/allmydata/test/test_netstring.py | 4 - src/allmydata/test/test_no_network.py | 4 - src/allmydata/test/test_observer.py | 5 -- src/allmydata/test/test_openmetrics.py | 8 -- .../test/test_python2_regressions.py | 77 ------------------- src/allmydata/test/test_repairer.py | 4 - src/allmydata/test/test_runner.py | 28 ------- src/allmydata/test/test_sftp.py | 4 - src/allmydata/test/test_spans.py | 5 -- src/allmydata/test/test_statistics.py | 4 - src/allmydata/test/test_stats.py | 4 - src/allmydata/test/test_storage_web.py | 7 -- src/allmydata/test/test_time_format.py | 4 - src/allmydata/test/test_upload.py | 4 - src/allmydata/test/test_uri.py | 5 -- src/allmydata/test/test_util.py | 7 -- src/allmydata/test/web/common.py | 4 - src/allmydata/test/web/matchers.py | 4 - src/allmydata/test/web/test_common.py | 4 - src/allmydata/test/web/test_grid.py | 4 - src/allmydata/test/web/test_introducer.py | 4 - src/allmydata/test/web/test_logs.py | 4 - src/allmydata/test/web/test_private.py | 4 - src/allmydata/test/web/test_root.py | 4 - src/allmydata/test/web/test_status.py | 4 - src/allmydata/test/web/test_util.py | 4 - src/allmydata/unknown.py | 4 - src/allmydata/util/abbreviate.py | 4 - src/allmydata/util/assertutil.py | 6 -- src/allmydata/util/base62.py | 2 - src/allmydata/util/configutil.py | 6 -- src/allmydata/util/consumer.py | 4 - src/allmydata/util/dbutil.py | 5 -- src/allmydata/util/encodingutil.py | 3 - src/allmydata/util/fileutil.py | 8 -- src/allmydata/util/gcutil.py | 4 - src/allmydata/util/happinessutil.py | 5 -- src/allmydata/util/humanreadable.py | 5 -- src/allmydata/util/idlib.py | 4 - src/allmydata/util/jsonbytes.py | 18 ----- src/allmydata/util/log.py | 2 - src/allmydata/util/mathutil.py | 6 -- src/allmydata/util/netstring.py | 4 - src/allmydata/util/observer.py | 5 -- src/allmydata/util/rrefutil.py | 4 - src/allmydata/util/spans.py | 4 - src/allmydata/util/statistics.py | 5 -- src/allmydata/util/time_format.py | 3 - src/allmydata/util/yamlutil.py | 27 ------- src/allmydata/web/check_results.py | 4 - src/allmydata/web/directory.py | 6 -- src/allmydata/web/info.py | 4 - src/allmydata/web/operations.py | 4 - src/allmydata/web/private.py | 4 - src/allmydata/web/status.py | 4 - src/allmydata/web/storage.py | 4 - src/allmydata/web/storage_plugins.py | 4 - 192 files changed, 1037 deletions(-) delete mode 100644 src/allmydata/test/test_python2_regressions.py diff --git a/docs/conf.py b/docs/conf.py index cc9a11166..79a57e48c 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -12,9 +12,6 @@ # All configuration values have a default; values that are commented out # serve to show the default. -import sys -import os - # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. diff --git a/integration/test_aaa_aardvark.py b/integration/test_aaa_aardvark.py index e92403497..4a2ef71a6 100644 --- a/integration/test_aaa_aardvark.py +++ b/integration/test_aaa_aardvark.py @@ -6,11 +6,6 @@ # longer" to start the first test as the fixtures get built -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - - def test_create_flogger(flog_gatherer): print("Created flog_gatherer") diff --git a/integration/test_sftp.py b/integration/test_sftp.py index 2fdc7522e..8202245ce 100644 --- a/integration/test_sftp.py +++ b/integration/test_sftp.py @@ -10,11 +10,6 @@ These tests use Paramiko, rather than Twisted's Conch, because: 2. Its API is much simpler to use. """ - -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - import os.path from posixpath import join from stat import S_ISDIR diff --git a/integration/test_streaming_logs.py b/integration/test_streaming_logs.py index f70a42ead..efdb23df8 100644 --- a/integration/test_streaming_logs.py +++ b/integration/test_streaming_logs.py @@ -2,10 +2,6 @@ Ported to Python 3. """ -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - from six import ensure_text import json diff --git a/misc/simulators/sizes.py b/misc/simulators/sizes.py index 4b4f6d1fd..d9f861c2f 100644 --- a/misc/simulators/sizes.py +++ b/misc/simulators/sizes.py @@ -1,11 +1,5 @@ #! /usr/bin/env python - -from future.utils import PY2 -if PY2: - from future.builtins import input - - import random, math, re from twisted.python import usage diff --git a/src/allmydata/__main__.py b/src/allmydata/__main__.py index 972167b30..c6b200991 100644 --- a/src/allmydata/__main__.py +++ b/src/allmydata/__main__.py @@ -2,11 +2,6 @@ Ported to Python 3. """ - -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - import sys from allmydata.scripts.runner import run diff --git a/src/allmydata/_monkeypatch.py b/src/allmydata/_monkeypatch.py index 520a4194a..b5f171224 100644 --- a/src/allmydata/_monkeypatch.py +++ b/src/allmydata/_monkeypatch.py @@ -4,41 +4,14 @@ Monkey-patching of third party libraries. Ported to Python 3. """ - from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from warnings import catch_warnings def patch(): """Path third-party libraries to make Tahoe-LAFS work.""" - # Make sure Foolscap always get native strings passed to method names in callRemote. - # This can be removed when any one of the following happens: - # - # 1. Tahoe-LAFS on Python 2 switches to version of Foolscap that fixes - # https://github.com/warner/foolscap/issues/72 - # 2. Foolscap is dropped as a dependency. - # 3. Tahoe-LAFS drops Python 2 support. if not PY2: # Python 3 doesn't need to monkey patch Foolscap return - - # We need to suppress warnings so as to prevent unexpected output from - # breaking some integration tests. - with catch_warnings(record=True): - # Only tested with this version; ensure correctness with new releases, - # and then either update the assert or hopefully drop the monkeypatch. - from foolscap import __version__ - assert __version__ == "0.13.1", "Wrong version %s of Foolscap" % (__version__,) - - from foolscap.referenceable import RemoteReference - original_getMethodInfo = RemoteReference._getMethodInfo - - def _getMethodInfo(self, name): - if isinstance(name, str): - name = name.encode("utf-8") - return original_getMethodInfo(self, name) - RemoteReference._getMethodInfo = _getMethodInfo diff --git a/src/allmydata/blacklist.py b/src/allmydata/blacklist.py index a019ccb69..db499315a 100644 --- a/src/allmydata/blacklist.py +++ b/src/allmydata/blacklist.py @@ -2,10 +2,6 @@ Ported to Python 3. """ -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - import os from zope.interface import implementer diff --git a/src/allmydata/check_results.py b/src/allmydata/check_results.py index 1dbf2f7d5..8c44fd71e 100644 --- a/src/allmydata/check_results.py +++ b/src/allmydata/check_results.py @@ -1,10 +1,6 @@ """Ported to Python 3. """ -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - from past.builtins import unicode from zope.interface import implementer diff --git a/src/allmydata/codec.py b/src/allmydata/codec.py index b58ebae57..d8a3527c1 100644 --- a/src/allmydata/codec.py +++ b/src/allmydata/codec.py @@ -4,10 +4,6 @@ CRS encoding and decoding. Ported to Python 3. """ -from future.utils import PY2 -if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - from zope.interface import implementer from allmydata.util import mathutil from allmydata.util.assertutil import precondition diff --git a/src/allmydata/crypto/__init__.py b/src/allmydata/crypto/__init__.py index 3ba70e605..ec50d070d 100644 --- a/src/allmydata/crypto/__init__.py +++ b/src/allmydata/crypto/__init__.py @@ -9,7 +9,3 @@ objects that `cryptography` documents. Ported to Python 3. """ - -from future.utils import PY2 -if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 diff --git a/src/allmydata/crypto/aes.py b/src/allmydata/crypto/aes.py index 6e1b93c03..2fa5f8684 100644 --- a/src/allmydata/crypto/aes.py +++ b/src/allmydata/crypto/aes.py @@ -10,10 +10,6 @@ objects that `cryptography` documents. Ported to Python 3. """ -from future.utils import PY2 -if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - import six from cryptography.hazmat.backends import default_backend diff --git a/src/allmydata/crypto/error.py b/src/allmydata/crypto/error.py index ea3f78162..f860cf57b 100644 --- a/src/allmydata/crypto/error.py +++ b/src/allmydata/crypto/error.py @@ -4,11 +4,6 @@ Exceptions raise by allmydata.crypto.* modules Ported to Python 3. """ -from future.utils import PY2 -if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - - class BadSignature(Exception): """ An alleged signature did not match diff --git a/src/allmydata/crypto/util.py b/src/allmydata/crypto/util.py index 20f4bca85..09836533f 100644 --- a/src/allmydata/crypto/util.py +++ b/src/allmydata/crypto/util.py @@ -4,10 +4,6 @@ Utilities used by allmydata.crypto modules Ported to Python 3. """ -from future.utils import PY2 -if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - from allmydata.crypto.error import BadPrefixError diff --git a/src/allmydata/deep_stats.py b/src/allmydata/deep_stats.py index a80abe508..b3671718b 100644 --- a/src/allmydata/deep_stats.py +++ b/src/allmydata/deep_stats.py @@ -3,10 +3,6 @@ Ported to Python 3. """ -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - import math from allmydata.interfaces import IImmutableFileNode diff --git a/src/allmydata/dirnode.py b/src/allmydata/dirnode.py index 67e44faf4..5ba75f6c0 100644 --- a/src/allmydata/dirnode.py +++ b/src/allmydata/dirnode.py @@ -3,10 +3,6 @@ Ported to Python 3. """ -from future.utils import PY2 -if PY2: - # Skip dict so it doesn't break things. - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, list, object, range, str, max, min # noqa: F401 from past.builtins import unicode import time diff --git a/src/allmydata/frontends/auth.py b/src/allmydata/frontends/auth.py index 6d58cf567..973dd2301 100644 --- a/src/allmydata/frontends/auth.py +++ b/src/allmydata/frontends/auth.py @@ -2,10 +2,6 @@ Authentication for frontends. """ -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - from zope.interface import implementer from twisted.internet import defer from twisted.cred import checkers, credentials diff --git a/src/allmydata/frontends/sftpd.py b/src/allmydata/frontends/sftpd.py index e3c0eb674..eaa89160c 100644 --- a/src/allmydata/frontends/sftpd.py +++ b/src/allmydata/frontends/sftpd.py @@ -2,10 +2,6 @@ Ported to Python 3. """ -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - import six import heapq, traceback, stat, struct from stat import S_IFREG, S_IFDIR diff --git a/src/allmydata/hashtree.py b/src/allmydata/hashtree.py index ca043f234..6c4436958 100644 --- a/src/allmydata/hashtree.py +++ b/src/allmydata/hashtree.py @@ -49,11 +49,6 @@ or eat your children, but it might. Use at your own risk. Ported to Python 3. """ - -from future.utils import PY2 -if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - from allmydata.util import mathutil # from the pyutil library from allmydata.util import base32 diff --git a/src/allmydata/history.py b/src/allmydata/history.py index 5629d7c43..befc8cf3d 100644 --- a/src/allmydata/history.py +++ b/src/allmydata/history.py @@ -1,11 +1,6 @@ """Ported to Python 3. """ -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - - import weakref class History(object): diff --git a/src/allmydata/immutable/checker.py b/src/allmydata/immutable/checker.py index 0e8ba14d8..483ddb2a2 100644 --- a/src/allmydata/immutable/checker.py +++ b/src/allmydata/immutable/checker.py @@ -2,10 +2,6 @@ Ported to Python 3. """ -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - from zope.interface import implementer from twisted.internet import defer from foolscap.api import DeadReferenceError, RemoteException diff --git a/src/allmydata/immutable/downloader/__init__.py b/src/allmydata/immutable/downloader/__init__.py index 84d6b71f9..d4f3fe345 100644 --- a/src/allmydata/immutable/downloader/__init__.py +++ b/src/allmydata/immutable/downloader/__init__.py @@ -1,9 +1,3 @@ """ Ported to Python 3. """ - - -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - diff --git a/src/allmydata/immutable/downloader/common.py b/src/allmydata/immutable/downloader/common.py index e0d1fe3af..30f5bcf4b 100644 --- a/src/allmydata/immutable/downloader/common.py +++ b/src/allmydata/immutable/downloader/common.py @@ -2,11 +2,6 @@ Ported to Python 3. """ - -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - (AVAILABLE, PENDING, OVERDUE, COMPLETE, CORRUPT, DEAD, BADSEGNUM) = \ ("AVAILABLE", "PENDING", "OVERDUE", "COMPLETE", "CORRUPT", "DEAD", "BADSEGNUM") diff --git a/src/allmydata/immutable/downloader/fetcher.py b/src/allmydata/immutable/downloader/fetcher.py index ccffa4889..e8e4eefbc 100644 --- a/src/allmydata/immutable/downloader/fetcher.py +++ b/src/allmydata/immutable/downloader/fetcher.py @@ -2,11 +2,6 @@ Ported to Python 3. """ - -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - from twisted.python.failure import Failure from foolscap.api import eventually from allmydata.interfaces import NotEnoughSharesError, NoSharesError diff --git a/src/allmydata/immutable/downloader/finder.py b/src/allmydata/immutable/downloader/finder.py index 1992e72c1..886859e6e 100644 --- a/src/allmydata/immutable/downloader/finder.py +++ b/src/allmydata/immutable/downloader/finder.py @@ -2,9 +2,6 @@ Ported to Python 3. """ -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from six import ensure_str import time diff --git a/src/allmydata/immutable/downloader/node.py b/src/allmydata/immutable/downloader/node.py index 3c69b4084..dcb336f95 100644 --- a/src/allmydata/immutable/downloader/node.py +++ b/src/allmydata/immutable/downloader/node.py @@ -2,10 +2,6 @@ Ported to Python 3. """ -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - import time now = time.time from zope.interface import Interface diff --git a/src/allmydata/immutable/downloader/segmentation.py b/src/allmydata/immutable/downloader/segmentation.py index 4691f2602..80166c965 100644 --- a/src/allmydata/immutable/downloader/segmentation.py +++ b/src/allmydata/immutable/downloader/segmentation.py @@ -2,10 +2,6 @@ Ported to Python 3. """ -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - import time now = time.time from zope.interface import implementer diff --git a/src/allmydata/immutable/downloader/share.py b/src/allmydata/immutable/downloader/share.py index 02e8554d4..7bbf2b900 100644 --- a/src/allmydata/immutable/downloader/share.py +++ b/src/allmydata/immutable/downloader/share.py @@ -2,10 +2,6 @@ Ported to Python 3. """ -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - import struct import time now = time.time diff --git a/src/allmydata/immutable/downloader/status.py b/src/allmydata/immutable/downloader/status.py index 11b3e88a7..4136db3c5 100644 --- a/src/allmydata/immutable/downloader/status.py +++ b/src/allmydata/immutable/downloader/status.py @@ -2,10 +2,6 @@ Ported to Python 3. """ -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - import itertools from zope.interface import implementer from allmydata.interfaces import IDownloadStatus diff --git a/src/allmydata/immutable/encode.py b/src/allmydata/immutable/encode.py index b838e71e1..9d7af2650 100644 --- a/src/allmydata/immutable/encode.py +++ b/src/allmydata/immutable/encode.py @@ -4,11 +4,6 @@ Ported to Python 3. """ - -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - import time from zope.interface import implementer from twisted.internet import defer diff --git a/src/allmydata/immutable/filenode.py b/src/allmydata/immutable/filenode.py index 761e663dd..8dda50bee 100644 --- a/src/allmydata/immutable/filenode.py +++ b/src/allmydata/immutable/filenode.py @@ -2,11 +2,6 @@ Ported to Python 3. """ - -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - from functools import reduce import binascii from time import time as now diff --git a/src/allmydata/immutable/happiness_upload.py b/src/allmydata/immutable/happiness_upload.py index 2bc0016b0..a0af17891 100644 --- a/src/allmydata/immutable/happiness_upload.py +++ b/src/allmydata/immutable/happiness_upload.py @@ -5,11 +5,6 @@ on. Ported to Python 3. """ -from future.utils import PY2 -if PY2: - # We omit dict, just in case newdict breaks things for external Python 2 code. - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, list, object, range, str, max, min # noqa: F401 - from queue import PriorityQueue diff --git a/src/allmydata/immutable/literal.py b/src/allmydata/immutable/literal.py index 1ae2743f8..05f0ed1bc 100644 --- a/src/allmydata/immutable/literal.py +++ b/src/allmydata/immutable/literal.py @@ -2,11 +2,6 @@ Ported to Python 3. """ - -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - from io import BytesIO from zope.interface import implementer diff --git a/src/allmydata/immutable/offloaded.py b/src/allmydata/immutable/offloaded.py index a98d3372b..c609f3b8a 100644 --- a/src/allmydata/immutable/offloaded.py +++ b/src/allmydata/immutable/offloaded.py @@ -2,10 +2,6 @@ Ported to Python 3. """ -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - import os, stat, time, weakref from zope.interface import implementer from twisted.internet import defer diff --git a/src/allmydata/immutable/repairer.py b/src/allmydata/immutable/repairer.py index 87873369a..d12220810 100644 --- a/src/allmydata/immutable/repairer.py +++ b/src/allmydata/immutable/repairer.py @@ -2,10 +2,6 @@ Ported to Python 3. """ -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - from zope.interface import implementer from twisted.internet import defer from allmydata.storage.server import si_b2a diff --git a/src/allmydata/interfaces.py b/src/allmydata/interfaces.py index 7bfcc9a13..168a7b8e4 100644 --- a/src/allmydata/interfaces.py +++ b/src/allmydata/interfaces.py @@ -7,10 +7,6 @@ Note that for RemoteInterfaces, the __remote_name__ needs to be a native string """ from future.utils import PY2, native_str -if PY2: - # Don't import object/str/dict/etc. types, so we don't break any - # interfaces. Not importing open() because it triggers bogus flake8 error. - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, pow, round, super, range, max, min # noqa: F401 from past.builtins import long from typing import Dict diff --git a/src/allmydata/introducer/__init__.py b/src/allmydata/introducer/__init__.py index d654f6c89..52aa56597 100644 --- a/src/allmydata/introducer/__init__.py +++ b/src/allmydata/introducer/__init__.py @@ -2,12 +2,6 @@ Ported to Python 3. """ - -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - - from allmydata.introducer.server import create_introducer # apparently need to support "old .tac files" that may have diff --git a/src/allmydata/introducer/client.py b/src/allmydata/introducer/client.py index 4c8abff60..e6eab3b9f 100644 --- a/src/allmydata/introducer/client.py +++ b/src/allmydata/introducer/client.py @@ -2,9 +2,6 @@ Ported to Python 3. """ -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from past.builtins import long from six import ensure_text, ensure_str diff --git a/src/allmydata/introducer/common.py b/src/allmydata/introducer/common.py index 21b16a599..3b85318ce 100644 --- a/src/allmydata/introducer/common.py +++ b/src/allmydata/introducer/common.py @@ -2,10 +2,6 @@ Ported to Python 3. """ -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - import re from foolscap.furl import decode_furl diff --git a/src/allmydata/introducer/interfaces.py b/src/allmydata/introducer/interfaces.py index ca97b425f..4667c7be0 100644 --- a/src/allmydata/introducer/interfaces.py +++ b/src/allmydata/introducer/interfaces.py @@ -4,9 +4,6 @@ Ported to Python 3. from future.utils import PY2, native_str -if PY2: - # Omitted types (bytes etc.) so future variants don't confuse Foolscap. - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, object, range, max, min # noqa: F401 from zope.interface import Interface from foolscap.api import StringConstraint, SetOf, DictOf, Any, \ diff --git a/src/allmydata/monitor.py b/src/allmydata/monitor.py index c2aae826e..0a213635b 100644 --- a/src/allmydata/monitor.py +++ b/src/allmydata/monitor.py @@ -4,10 +4,6 @@ Manage status of long-running operations. Ported to Python 3. """ -from future.utils import PY2 -if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - from zope.interface import Interface, implementer from allmydata.util import observer diff --git a/src/allmydata/mutable/checker.py b/src/allmydata/mutable/checker.py index ee57e2ed9..14120c476 100644 --- a/src/allmydata/mutable/checker.py +++ b/src/allmydata/mutable/checker.py @@ -2,9 +2,6 @@ Ported to Python 3. """ -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from six import ensure_str from allmydata.uri import from_string diff --git a/src/allmydata/mutable/layout.py b/src/allmydata/mutable/layout.py index 9057745e7..6a5993993 100644 --- a/src/allmydata/mutable/layout.py +++ b/src/allmydata/mutable/layout.py @@ -2,10 +2,6 @@ Ported to Python 3. """ -from future.utils import PY2 -if PY2: - # Omit dict so Python 3 changes don't leak into API callers on Python 2. - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, list, object, range, str, max, min # noqa: F401 from past.utils import old_div import struct diff --git a/src/allmydata/mutable/publish.py b/src/allmydata/mutable/publish.py index 9df1d20b2..ee9faeb2b 100644 --- a/src/allmydata/mutable/publish.py +++ b/src/allmydata/mutable/publish.py @@ -2,10 +2,6 @@ Ported to Python 3. """ -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - import os, time from io import BytesIO from itertools import count diff --git a/src/allmydata/mutable/repairer.py b/src/allmydata/mutable/repairer.py index 5c6537769..4d1df410b 100644 --- a/src/allmydata/mutable/repairer.py +++ b/src/allmydata/mutable/repairer.py @@ -2,10 +2,6 @@ Ported to Python 3. """ -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - from zope.interface import implementer from twisted.internet import defer from allmydata.interfaces import IRepairResults, ICheckResults diff --git a/src/allmydata/scripts/admin.py b/src/allmydata/scripts/admin.py index d1bd59600..34f6e2eaf 100644 --- a/src/allmydata/scripts/admin.py +++ b/src/allmydata/scripts/admin.py @@ -2,10 +2,6 @@ Ported to Python 3. """ -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - from six import ensure_binary from twisted.python import usage diff --git a/src/allmydata/scripts/backupdb.py b/src/allmydata/scripts/backupdb.py index 21db88ee2..45c2bc026 100644 --- a/src/allmydata/scripts/backupdb.py +++ b/src/allmydata/scripts/backupdb.py @@ -2,10 +2,6 @@ Ported to Python 3. """ -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - import os.path, sys, time, random, stat from allmydata.util.netstring import netstring diff --git a/src/allmydata/scripts/default_nodedir.py b/src/allmydata/scripts/default_nodedir.py index 89d13cd1b..c1711b55c 100644 --- a/src/allmydata/scripts/default_nodedir.py +++ b/src/allmydata/scripts/default_nodedir.py @@ -2,11 +2,6 @@ Ported to Python 3. """ - -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - import sys import six from allmydata.util.assertutil import precondition diff --git a/src/allmydata/scripts/runner.py b/src/allmydata/scripts/runner.py index 18387cea5..32c68ee57 100644 --- a/src/allmydata/scripts/runner.py +++ b/src/allmydata/scripts/runner.py @@ -107,27 +107,6 @@ def parse_options(argv, config=None): try: config.parseOptions(argv) except usage.error as e: - if six.PY2: - # On Python 2 the exception may hold non-ascii in a byte string. - # This makes it impossible to convert the exception to any kind of - # string using str() or unicode(). It could also hold non-ascii - # in a unicode string which still makes it difficult to convert it - # to a byte string later. - # - # So, reach inside and turn it into some entirely safe ascii byte - # strings that will survive being written to stdout without - # causing too much damage in the process. - # - # As a result, non-ascii will not be rendered correctly but - # instead as escape sequences. At least this can go away when - # we're done with Python 2 support. - raise usage.error(*( - arg.encode("ascii", errors="backslashreplace") - if isinstance(arg, unicode) - else arg.decode("utf-8").encode("ascii", errors="backslashreplace") - for arg - in e.args - )) raise return config diff --git a/src/allmydata/scripts/slow_operation.py b/src/allmydata/scripts/slow_operation.py index 092e85823..e1a944ba7 100644 --- a/src/allmydata/scripts/slow_operation.py +++ b/src/allmydata/scripts/slow_operation.py @@ -3,8 +3,6 @@ Ported to Python 3. """ from future.utils import PY2, PY3 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from six import ensure_str diff --git a/src/allmydata/scripts/tahoe_add_alias.py b/src/allmydata/scripts/tahoe_add_alias.py index e29d974f4..ac57879b0 100644 --- a/src/allmydata/scripts/tahoe_add_alias.py +++ b/src/allmydata/scripts/tahoe_add_alias.py @@ -2,10 +2,6 @@ Ported to Python 3. """ -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - import os.path import codecs diff --git a/src/allmydata/scripts/tahoe_backup.py b/src/allmydata/scripts/tahoe_backup.py index 058febb8c..7ca79d393 100644 --- a/src/allmydata/scripts/tahoe_backup.py +++ b/src/allmydata/scripts/tahoe_backup.py @@ -2,10 +2,6 @@ Ported to Python 3. """ -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - import os.path import time from urllib.parse import quote as url_quote diff --git a/src/allmydata/scripts/tahoe_check.py b/src/allmydata/scripts/tahoe_check.py index d88d3689e..91125a3a3 100644 --- a/src/allmydata/scripts/tahoe_check.py +++ b/src/allmydata/scripts/tahoe_check.py @@ -2,10 +2,6 @@ Ported to Python 3. """ -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - from six import ensure_str, ensure_text from urllib.parse import quote as url_quote diff --git a/src/allmydata/scripts/tahoe_cp.py b/src/allmydata/scripts/tahoe_cp.py index 9ccbf2a05..1e9726605 100644 --- a/src/allmydata/scripts/tahoe_cp.py +++ b/src/allmydata/scripts/tahoe_cp.py @@ -2,10 +2,6 @@ Ported to Python 3. """ -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - import os.path from urllib.parse import quote as url_quote from collections import defaultdict diff --git a/src/allmydata/scripts/tahoe_get.py b/src/allmydata/scripts/tahoe_get.py index cf0dd8afa..332529d04 100644 --- a/src/allmydata/scripts/tahoe_get.py +++ b/src/allmydata/scripts/tahoe_get.py @@ -3,8 +3,6 @@ Ported to Python 3. """ from future.utils import PY2, PY3 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from urllib.parse import quote as url_quote from allmydata.scripts.common import get_alias, DEFAULT_ALIAS, escape_path, \ diff --git a/src/allmydata/scripts/tahoe_ls.py b/src/allmydata/scripts/tahoe_ls.py index b20dec723..d38fe060c 100644 --- a/src/allmydata/scripts/tahoe_ls.py +++ b/src/allmydata/scripts/tahoe_ls.py @@ -2,10 +2,6 @@ Ported to Python 3. """ -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - from six import ensure_text import time diff --git a/src/allmydata/scripts/tahoe_manifest.py b/src/allmydata/scripts/tahoe_manifest.py index e1ee95ef6..1cc5ba591 100644 --- a/src/allmydata/scripts/tahoe_manifest.py +++ b/src/allmydata/scripts/tahoe_manifest.py @@ -3,8 +3,6 @@ Ported to Python 3. """ from future.utils import PY2, PY3 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from six import ensure_str diff --git a/src/allmydata/scripts/tahoe_mkdir.py b/src/allmydata/scripts/tahoe_mkdir.py index 2bc35c391..8a9dc6262 100644 --- a/src/allmydata/scripts/tahoe_mkdir.py +++ b/src/allmydata/scripts/tahoe_mkdir.py @@ -2,10 +2,6 @@ Ported to Python 3. """ -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - from urllib.parse import quote as url_quote from allmydata.scripts.common_http import do_http, check_http_error from allmydata.scripts.common import get_alias, DEFAULT_ALIAS, UnknownAliasError diff --git a/src/allmydata/scripts/tahoe_mv.py b/src/allmydata/scripts/tahoe_mv.py index bd1e4f810..016c0e725 100644 --- a/src/allmydata/scripts/tahoe_mv.py +++ b/src/allmydata/scripts/tahoe_mv.py @@ -2,10 +2,6 @@ Ported to Python 3. """ -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - import re from urllib.parse import quote as url_quote import json diff --git a/src/allmydata/scripts/tahoe_run.py b/src/allmydata/scripts/tahoe_run.py index 345f85815..d7b570faa 100644 --- a/src/allmydata/scripts/tahoe_run.py +++ b/src/allmydata/scripts/tahoe_run.py @@ -2,10 +2,6 @@ Ported to Python 3. """ -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - __all__ = [ "RunOptions", "run", diff --git a/src/allmydata/scripts/tahoe_status.py b/src/allmydata/scripts/tahoe_status.py index 5c8c6aee0..c7f19910b 100644 --- a/src/allmydata/scripts/tahoe_status.py +++ b/src/allmydata/scripts/tahoe_status.py @@ -2,10 +2,6 @@ Ported to Python 3. """ -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - import os from sys import stdout as _sys_stdout from urllib.parse import urlencode diff --git a/src/allmydata/scripts/tahoe_unlink.py b/src/allmydata/scripts/tahoe_unlink.py index 29da80811..8531ce059 100644 --- a/src/allmydata/scripts/tahoe_unlink.py +++ b/src/allmydata/scripts/tahoe_unlink.py @@ -2,10 +2,6 @@ Ported to Python 3. """ -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - from urllib.parse import quote as url_quote from allmydata.scripts.common_http import do_http, format_http_success, format_http_error from allmydata.scripts.common import get_alias, DEFAULT_ALIAS, escape_path, \ diff --git a/src/allmydata/scripts/tahoe_webopen.py b/src/allmydata/scripts/tahoe_webopen.py index 2b16f09e1..011677b4e 100644 --- a/src/allmydata/scripts/tahoe_webopen.py +++ b/src/allmydata/scripts/tahoe_webopen.py @@ -2,10 +2,6 @@ Ported to Python 3. """ -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - from urllib.parse import quote as url_quote from allmydata.scripts.common import get_alias, DEFAULT_ALIAS, escape_path, \ diff --git a/src/allmydata/storage/common.py b/src/allmydata/storage/common.py index e9f5e4e5d..a7b45ca0f 100644 --- a/src/allmydata/storage/common.py +++ b/src/allmydata/storage/common.py @@ -3,8 +3,6 @@ Ported to Python 3. """ from future.utils import PY2, PY3 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import os.path from allmydata.util import base32 diff --git a/src/allmydata/storage/crawler.py b/src/allmydata/storage/crawler.py index 8a7304808..a464449ed 100644 --- a/src/allmydata/storage/crawler.py +++ b/src/allmydata/storage/crawler.py @@ -6,8 +6,6 @@ Ported to Python 3. from future.utils import PY2, PY3 -if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import os import time diff --git a/src/allmydata/storage/expirer.py b/src/allmydata/storage/expirer.py index bcff8aeeb..c0968fd39 100644 --- a/src/allmydata/storage/expirer.py +++ b/src/allmydata/storage/expirer.py @@ -1,7 +1,3 @@ - -from future.utils import PY2 -if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import json import time import os diff --git a/src/allmydata/storage/immutable.py b/src/allmydata/storage/immutable.py index fc696ff60..90d6d4d2d 100644 --- a/src/allmydata/storage/immutable.py +++ b/src/allmydata/storage/immutable.py @@ -4,8 +4,6 @@ Ported to Python 3. from future.utils import PY2, bytes_to_native_str -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import os, stat, struct, time diff --git a/src/allmydata/storage/immutable_schema.py b/src/allmydata/storage/immutable_schema.py index beb184700..2798ea0cb 100644 --- a/src/allmydata/storage/immutable_schema.py +++ b/src/allmydata/storage/immutable_schema.py @@ -2,11 +2,6 @@ Ported to Python 3. """ - -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - import struct import attr diff --git a/src/allmydata/storage/lease.py b/src/allmydata/storage/lease.py index ad21ce598..4a8b10d01 100644 --- a/src/allmydata/storage/lease.py +++ b/src/allmydata/storage/lease.py @@ -2,11 +2,6 @@ Ported to Python 3. """ - -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - import struct, time import attr diff --git a/src/allmydata/storage/mutable.py b/src/allmydata/storage/mutable.py index cb9116a21..d13a68020 100644 --- a/src/allmydata/storage/mutable.py +++ b/src/allmydata/storage/mutable.py @@ -2,11 +2,6 @@ Ported to Python 3. """ - -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - import os, stat, struct from allmydata.interfaces import ( diff --git a/src/allmydata/storage/mutable_schema.py b/src/allmydata/storage/mutable_schema.py index b8530e0aa..389d743f4 100644 --- a/src/allmydata/storage/mutable_schema.py +++ b/src/allmydata/storage/mutable_schema.py @@ -2,11 +2,6 @@ Ported to Python 3. """ - -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - import struct import attr diff --git a/src/allmydata/storage/shares.py b/src/allmydata/storage/shares.py index 5fe2b106c..6c9526b47 100644 --- a/src/allmydata/storage/shares.py +++ b/src/allmydata/storage/shares.py @@ -2,11 +2,6 @@ Ported to Python 3. """ - -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - from allmydata.storage.mutable import MutableShareFile from allmydata.storage.immutable import ShareFile diff --git a/src/allmydata/test/__init__.py b/src/allmydata/test/__init__.py index 14ec9b054..6779aa527 100644 --- a/src/allmydata/test/__init__.py +++ b/src/allmydata/test/__init__.py @@ -16,10 +16,6 @@ some side-effects which make things better when the test suite runs. Ported to Python 3. """ -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - from traceback import extract_stack, format_list from foolscap.pb import Listener diff --git a/src/allmydata/test/cli/common.py b/src/allmydata/test/cli/common.py index 45cafbec6..351b48baa 100644 --- a/src/allmydata/test/cli/common.py +++ b/src/allmydata/test/cli/common.py @@ -2,10 +2,6 @@ Ported to Python 3. """ -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - from six import ensure_str, ensure_text from ...scripts import runner diff --git a/src/allmydata/test/cli/test_admin.py b/src/allmydata/test/cli/test_admin.py index d1e9263d0..a6370d6e4 100644 --- a/src/allmydata/test/cli/test_admin.py +++ b/src/allmydata/test/cli/test_admin.py @@ -2,10 +2,6 @@ Ported to Python 3. """ -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - # We're going to override stdin/stderr, so want to match their behavior on respective Python versions. from io import StringIO diff --git a/src/allmydata/test/cli/test_alias.py b/src/allmydata/test/cli/test_alias.py index 025a7ae9d..bbbafcabc 100644 --- a/src/allmydata/test/cli/test_alias.py +++ b/src/allmydata/test/cli/test_alias.py @@ -2,10 +2,6 @@ Ported to Python 3. """ -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - import json from twisted.trial import unittest diff --git a/src/allmydata/test/cli/test_backup.py b/src/allmydata/test/cli/test_backup.py index 804817132..744d40f78 100644 --- a/src/allmydata/test/cli/test_backup.py +++ b/src/allmydata/test/cli/test_backup.py @@ -3,8 +3,6 @@ Ported to Python 3. """ from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import os.path from six.moves import cStringIO as StringIO @@ -368,8 +366,6 @@ class Backup(GridTestMixin, CLITestMixin, StallMixin, unittest.TestCase): nice_doc = u"nice_d\u00F8c.lyx" try: doc_pattern_arg_unicode = doc_pattern_arg = u"*d\u00F8c*" - if PY2: - doc_pattern_arg = doc_pattern_arg.encode(get_io_encoding()) except UnicodeEncodeError: raise unittest.SkipTest("A non-ASCII command argument could not be encoded on this platform.") diff --git a/src/allmydata/test/cli/test_backupdb.py b/src/allmydata/test/cli/test_backupdb.py index ca18a4a74..359b06f4f 100644 --- a/src/allmydata/test/cli/test_backupdb.py +++ b/src/allmydata/test/cli/test_backupdb.py @@ -2,11 +2,6 @@ Ported to Python 3. """ -from future.utils import PY2 -if PY2: - # Don't import future bytes so we don't break a couple of tests - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, dict, list, object, range, str, max, min # noqa: F401 - import sys import os.path, time from six.moves import cStringIO as StringIO diff --git a/src/allmydata/test/cli/test_check.py b/src/allmydata/test/cli/test_check.py index 24fd0d8e8..56b36a8d5 100644 --- a/src/allmydata/test/cli/test_check.py +++ b/src/allmydata/test/cli/test_check.py @@ -1,7 +1,3 @@ - -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from six import ensure_text import os.path diff --git a/src/allmydata/test/cli/test_cli.py b/src/allmydata/test/cli/test_cli.py index d7fde2742..ce51cb4f3 100644 --- a/src/allmydata/test/cli/test_cli.py +++ b/src/allmydata/test/cli/test_cli.py @@ -2,10 +2,6 @@ Ported to Python 3. """ -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - from six.moves import cStringIO as StringIO import re from six import ensure_text diff --git a/src/allmydata/test/cli/test_cp.py b/src/allmydata/test/cli/test_cp.py index 5e8e411ad..2dbc6d37e 100644 --- a/src/allmydata/test/cli/test_cp.py +++ b/src/allmydata/test/cli/test_cp.py @@ -2,10 +2,6 @@ Ported to Python 3. """ -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - import os.path, json from twisted.trial import unittest from twisted.python import usage @@ -73,8 +69,6 @@ class Cp(GridTestMixin, CLITestMixin, unittest.TestCase): self.failUnlessIn("files whose names could not be converted", err) else: self.failUnlessReallyEqual(rc, 0) - if PY2: - out = out.decode(get_io_encoding()) self.failUnlessReallyEqual(out, u"Metallica\n\u00C4rtonwall\n\u00C4rtonwall-2\n") self.assertEqual(len(err), 0, err) d.addCallback(_check) @@ -231,8 +225,6 @@ class Cp(GridTestMixin, CLITestMixin, unittest.TestCase): self.failUnlessIn("files whose names could not be converted", err) else: self.failUnlessReallyEqual(rc, 0) - if PY2: - out = out.decode(get_io_encoding()) self.failUnlessReallyEqual(out, u"\u00C4rtonwall\n") self.assertEqual(len(err), 0, err) d.addCallback(_check) diff --git a/src/allmydata/test/cli/test_create_alias.py b/src/allmydata/test/cli/test_create_alias.py index cb8526087..862b2896a 100644 --- a/src/allmydata/test/cli/test_create_alias.py +++ b/src/allmydata/test/cli/test_create_alias.py @@ -2,10 +2,6 @@ Ported to Python 3. """ -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - from six.moves import StringIO import os.path from twisted.trial import unittest diff --git a/src/allmydata/test/cli/test_list.py b/src/allmydata/test/cli/test_list.py index 141908987..24af04ba9 100644 --- a/src/allmydata/test/cli/test_list.py +++ b/src/allmydata/test/cli/test_list.py @@ -3,8 +3,6 @@ Ported to Python 3. """ from future.utils import PY2, PY3 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from six import ensure_str from twisted.trial import unittest diff --git a/src/allmydata/test/cli/test_mv.py b/src/allmydata/test/cli/test_mv.py index cb998025d..183e94725 100644 --- a/src/allmydata/test/cli/test_mv.py +++ b/src/allmydata/test/cli/test_mv.py @@ -2,10 +2,6 @@ Ported to Python 3. """ -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - import os.path from twisted.trial import unittest from allmydata.util import fileutil diff --git a/src/allmydata/test/cli/test_status.py b/src/allmydata/test/cli/test_status.py index e8d2ca3c8..a3921b442 100644 --- a/src/allmydata/test/cli/test_status.py +++ b/src/allmydata/test/cli/test_status.py @@ -2,9 +2,6 @@ Ported to Python 3. """ -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from six import ensure_text import os diff --git a/src/allmydata/test/cli_node_api.py b/src/allmydata/test/cli_node_api.py index e7d5e6600..bed4cfd55 100644 --- a/src/allmydata/test/cli_node_api.py +++ b/src/allmydata/test/cli_node_api.py @@ -2,10 +2,6 @@ Ported to Python 3. """ -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - __all__ = [ "CLINodeAPI", "Expect", diff --git a/src/allmydata/test/common.py b/src/allmydata/test/common.py index 485da9254..4ec3c2300 100644 --- a/src/allmydata/test/common.py +++ b/src/allmydata/test/common.py @@ -1427,7 +1427,4 @@ class TrialTestCase(_TrialTestCase): you try to turn that Exception instance into a string. """ - if six.PY2: - if isinstance(msg, six.text_type): - return super(TrialTestCase, self).fail(msg.encode("utf8")) return super(TrialTestCase, self).fail(msg) diff --git a/src/allmydata/test/common_util.py b/src/allmydata/test/common_util.py index 12e5cec72..fce2881b0 100644 --- a/src/allmydata/test/common_util.py +++ b/src/allmydata/test/common_util.py @@ -4,8 +4,6 @@ Ported to Python 3. from future.utils import PY2, PY3, bchr, binary_type from future.builtins import str as future_str -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, dict, list, object, range, str, max, min # noqa: F401 import os import sys diff --git a/src/allmydata/test/common_web.py b/src/allmydata/test/common_web.py index e43474da1..1f8a58d96 100644 --- a/src/allmydata/test/common_web.py +++ b/src/allmydata/test/common_web.py @@ -2,10 +2,6 @@ Ported to Python 3. """ -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - from six import ensure_str __all__ = [ diff --git a/src/allmydata/test/matchers.py b/src/allmydata/test/matchers.py index 20edd88c2..fc746aed0 100644 --- a/src/allmydata/test/matchers.py +++ b/src/allmydata/test/matchers.py @@ -4,10 +4,6 @@ Testtools-style matchers useful to the Tahoe-LAFS test suite. Ported to Python 3. """ -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - import attr from hyperlink import DecodedURL diff --git a/src/allmydata/test/mutable/test_checker.py b/src/allmydata/test/mutable/test_checker.py index 8eaa6c845..8018c5d05 100644 --- a/src/allmydata/test/mutable/test_checker.py +++ b/src/allmydata/test/mutable/test_checker.py @@ -2,10 +2,6 @@ Ported to Python 3. """ -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - from ..common import AsyncTestCase from foolscap.api import flushEventualQueue from allmydata.monitor import Monitor diff --git a/src/allmydata/test/mutable/test_datahandle.py b/src/allmydata/test/mutable/test_datahandle.py index 79536e7d1..6ddbb61b3 100644 --- a/src/allmydata/test/mutable/test_datahandle.py +++ b/src/allmydata/test/mutable/test_datahandle.py @@ -2,10 +2,6 @@ Ported to Python 3. """ -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - from ..common import SyncTestCase from allmydata.mutable.publish import MutableData from testtools.matchers import Equals, HasLength diff --git a/src/allmydata/test/mutable/test_different_encoding.py b/src/allmydata/test/mutable/test_different_encoding.py index 6bfc7e89a..8efb0bf82 100644 --- a/src/allmydata/test/mutable/test_different_encoding.py +++ b/src/allmydata/test/mutable/test_different_encoding.py @@ -2,10 +2,6 @@ Ported to Python 3. """ -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - from ..common import AsyncTestCase from .util import FakeStorage, make_nodemaker diff --git a/src/allmydata/test/mutable/test_exceptions.py b/src/allmydata/test/mutable/test_exceptions.py index fa1c1a884..1b83f7eb4 100644 --- a/src/allmydata/test/mutable/test_exceptions.py +++ b/src/allmydata/test/mutable/test_exceptions.py @@ -2,11 +2,6 @@ Ported to Python 3. """ - -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - from ..common import SyncTestCase from allmydata.mutable.common import NeedMoreDataError, UncoordinatedWriteError diff --git a/src/allmydata/test/mutable/test_filehandle.py b/src/allmydata/test/mutable/test_filehandle.py index 1e6c61c68..78597f774 100644 --- a/src/allmydata/test/mutable/test_filehandle.py +++ b/src/allmydata/test/mutable/test_filehandle.py @@ -2,10 +2,6 @@ Ported to Python 3. """ -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - import os from io import BytesIO from ..common import SyncTestCase diff --git a/src/allmydata/test/mutable/test_filenode.py b/src/allmydata/test/mutable/test_filenode.py index ca76be1b0..89881111a 100644 --- a/src/allmydata/test/mutable/test_filenode.py +++ b/src/allmydata/test/mutable/test_filenode.py @@ -2,10 +2,6 @@ Ported to Python 3. """ -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - from six.moves import cStringIO as StringIO from twisted.internet import defer, reactor from ..common import AsyncBrokenTestCase diff --git a/src/allmydata/test/mutable/test_interoperability.py b/src/allmydata/test/mutable/test_interoperability.py index 745baca97..deb20bb17 100644 --- a/src/allmydata/test/mutable/test_interoperability.py +++ b/src/allmydata/test/mutable/test_interoperability.py @@ -2,10 +2,6 @@ Ported to Python 3. """ -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - import os, base64 from ..common import AsyncTestCase from testtools.matchers import HasLength diff --git a/src/allmydata/test/mutable/test_multiple_encodings.py b/src/allmydata/test/mutable/test_multiple_encodings.py index 8ccfd387e..7f9699a07 100644 --- a/src/allmydata/test/mutable/test_multiple_encodings.py +++ b/src/allmydata/test/mutable/test_multiple_encodings.py @@ -2,10 +2,6 @@ Ported to Python 3. """ -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - from ..common import AsyncTestCase from testtools.matchers import Equals from allmydata.interfaces import SDMF_VERSION diff --git a/src/allmydata/test/mutable/test_multiple_versions.py b/src/allmydata/test/mutable/test_multiple_versions.py index c9c1f15aa..2062b01d4 100644 --- a/src/allmydata/test/mutable/test_multiple_versions.py +++ b/src/allmydata/test/mutable/test_multiple_versions.py @@ -2,10 +2,6 @@ Ported to Python 3. """ -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - from ..common import AsyncTestCase from testtools.matchers import Equals, HasLength from allmydata.monitor import Monitor diff --git a/src/allmydata/test/mutable/test_problems.py b/src/allmydata/test/mutable/test_problems.py index 6f22c02c2..d94668ff4 100644 --- a/src/allmydata/test/mutable/test_problems.py +++ b/src/allmydata/test/mutable/test_problems.py @@ -2,10 +2,6 @@ Ported to Python 3. """ -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - import os, base64 from ..common import AsyncTestCase from testtools.matchers import HasLength diff --git a/src/allmydata/test/mutable/test_repair.py b/src/allmydata/test/mutable/test_repair.py index f4ae53e13..dd2b435e5 100644 --- a/src/allmydata/test/mutable/test_repair.py +++ b/src/allmydata/test/mutable/test_repair.py @@ -2,10 +2,6 @@ Ported to Python 3. """ -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - from ..common import AsyncTestCase from testtools.matchers import Equals, HasLength from allmydata.interfaces import IRepairResults, ICheckAndRepairResults diff --git a/src/allmydata/test/mutable/test_roundtrip.py b/src/allmydata/test/mutable/test_roundtrip.py index eddd7f3df..238e69c61 100644 --- a/src/allmydata/test/mutable/test_roundtrip.py +++ b/src/allmydata/test/mutable/test_roundtrip.py @@ -2,10 +2,6 @@ Ported to Python 3. """ -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - from six.moves import cStringIO as StringIO from ..common import AsyncTestCase from testtools.matchers import Equals, HasLength, Contains diff --git a/src/allmydata/test/mutable/test_servermap.py b/src/allmydata/test/mutable/test_servermap.py index b3a247015..eaf2eddbc 100644 --- a/src/allmydata/test/mutable/test_servermap.py +++ b/src/allmydata/test/mutable/test_servermap.py @@ -2,11 +2,6 @@ Ported to Python 3. """ - -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - from ..common import AsyncTestCase from testtools.matchers import Equals, NotEquals, HasLength from twisted.internet import defer diff --git a/src/allmydata/test/mutable/test_update.py b/src/allmydata/test/mutable/test_update.py index 02fca24b2..37a4aa6b7 100644 --- a/src/allmydata/test/mutable/test_update.py +++ b/src/allmydata/test/mutable/test_update.py @@ -2,10 +2,6 @@ Ported to Python 3. """ -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - import re from ..common import AsyncTestCase from testtools.matchers import ( diff --git a/src/allmydata/test/mutable/util.py b/src/allmydata/test/mutable/util.py index 336b8bfd7..11696a83d 100644 --- a/src/allmydata/test/mutable/util.py +++ b/src/allmydata/test/mutable/util.py @@ -3,8 +3,6 @@ Ported to Python 3. """ from future.utils import PY2, bchr -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from past.builtins import long diff --git a/src/allmydata/test/storage_plugin.py b/src/allmydata/test/storage_plugin.py index cb18f4dda..b1950387b 100644 --- a/src/allmydata/test/storage_plugin.py +++ b/src/allmydata/test/storage_plugin.py @@ -5,9 +5,6 @@ functionality. Ported to Python 3. """ -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from future.utils import native_str, native_str_to_bytes from six import ensure_str diff --git a/src/allmydata/test/strategies.py b/src/allmydata/test/strategies.py index 92290203e..a15e40d9a 100644 --- a/src/allmydata/test/strategies.py +++ b/src/allmydata/test/strategies.py @@ -4,10 +4,6 @@ Hypothesis strategies use for testing Tahoe-LAFS. Ported to Python 3. """ -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - from hypothesis.strategies import ( one_of, builds, diff --git a/src/allmydata/test/test_abbreviate.py b/src/allmydata/test/test_abbreviate.py index cefa7f23f..082dadf4f 100644 --- a/src/allmydata/test/test_abbreviate.py +++ b/src/allmydata/test/test_abbreviate.py @@ -4,10 +4,6 @@ Tests for allmydata.util.abbreviate. Ported to Python 3. """ -from future.utils import PY2 -if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - from datetime import timedelta from twisted.trial import unittest diff --git a/src/allmydata/test/test_base32.py b/src/allmydata/test/test_base32.py index 236560599..83625371f 100644 --- a/src/allmydata/test/test_base32.py +++ b/src/allmydata/test/test_base32.py @@ -4,10 +4,6 @@ Tests for allmydata.util.base32. Ported to Python 3. """ -from future.utils import PY2 -if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - import base64 from twisted.trial import unittest diff --git a/src/allmydata/test/test_base62.py b/src/allmydata/test/test_base62.py index 258ca62ec..d77eaef9c 100644 --- a/src/allmydata/test/test_base62.py +++ b/src/allmydata/test/test_base62.py @@ -4,11 +4,6 @@ Tests for allmydata.util.base62. Ported to Python 3. """ - -from future.utils import PY2 -if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - from past.builtins import chr as byteschr import random, unittest diff --git a/src/allmydata/test/test_checker.py b/src/allmydata/test/test_checker.py index a33fc23b5..f116606db 100644 --- a/src/allmydata/test/test_checker.py +++ b/src/allmydata/test/test_checker.py @@ -2,12 +2,6 @@ Ported to Python 3. """ - -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - - import json import os.path, shutil diff --git a/src/allmydata/test/test_codec.py b/src/allmydata/test/test_codec.py index efb575cd7..59595f760 100644 --- a/src/allmydata/test/test_codec.py +++ b/src/allmydata/test/test_codec.py @@ -4,10 +4,6 @@ Tests for allmydata.codec. Ported to Python 3. """ -from future.utils import PY2 -if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - import os from twisted.trial import unittest from twisted.python import log diff --git a/src/allmydata/test/test_common_util.py b/src/allmydata/test/test_common_util.py index b1c698208..7f865d743 100644 --- a/src/allmydata/test/test_common_util.py +++ b/src/allmydata/test/test_common_util.py @@ -3,8 +3,6 @@ This module has been ported to Python 3. """ from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import sys import random diff --git a/src/allmydata/test/test_configutil.py b/src/allmydata/test/test_configutil.py index d1931479f..a4e7f56ea 100644 --- a/src/allmydata/test/test_configutil.py +++ b/src/allmydata/test/test_configutil.py @@ -4,11 +4,6 @@ Tests for allmydata.util.configutil. Ported to Python 3. """ -from future.utils import PY2 -if PY2: - # Omitted dict, cause worried about interactions. - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, list, object, range, str, max, min # noqa: F401 - import os.path from configparser import ( ConfigParser, diff --git a/src/allmydata/test/test_connections.py b/src/allmydata/test/test_connections.py index 3cf8ed937..8cc985816 100644 --- a/src/allmydata/test/test_connections.py +++ b/src/allmydata/test/test_connections.py @@ -2,10 +2,6 @@ Ported to Python 3. """ -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - from twisted.trial import unittest from twisted.internet import reactor diff --git a/src/allmydata/test/test_consumer.py b/src/allmydata/test/test_consumer.py index 9465b42cd..085f6b7ac 100644 --- a/src/allmydata/test/test_consumer.py +++ b/src/allmydata/test/test_consumer.py @@ -4,11 +4,6 @@ Tests for allmydata.util.consumer. Ported to Python 3. """ - -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - from zope.interface import implementer from twisted.internet.interfaces import IPushProducer, IPullProducer diff --git a/src/allmydata/test/test_crawler.py b/src/allmydata/test/test_crawler.py index 02181dd55..7a28c1d35 100644 --- a/src/allmydata/test/test_crawler.py +++ b/src/allmydata/test/test_crawler.py @@ -6,11 +6,6 @@ Ported to Python 3. from future.utils import PY2, PY3 -if PY2: - # Don't use future bytes, since it breaks tests. No further works is - # needed, once we're only on Python 3 we'll be deleting this future imports - # anyway, and tests pass just fine on Python 3. - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, dict, list, object, range, str, max, min # noqa: F401 import time import os.path diff --git a/src/allmydata/test/test_crypto.py b/src/allmydata/test/test_crypto.py index 2114b053b..aee9b4156 100644 --- a/src/allmydata/test/test_crypto.py +++ b/src/allmydata/test/test_crypto.py @@ -1,8 +1,4 @@ -from future.utils import PY2 -if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - from future.utils import native_bytes import unittest diff --git a/src/allmydata/test/test_deepcheck.py b/src/allmydata/test/test_deepcheck.py index 8af75bcd0..7473363c7 100644 --- a/src/allmydata/test/test_deepcheck.py +++ b/src/allmydata/test/test_deepcheck.py @@ -2,20 +2,6 @@ Ported to Python 3. """ -# Python 2 compatibility -# Can't use `builtins.str` because something deep in Twisted callbacks ends up repr'ing -# a `future.types.newstr.newstr` as a *Python 3* byte string representation under -# *Python 2*: -# File "/home/rpatterson/src/work/sfu/tahoe-lafs/.tox/py27/lib/python2.7/site-packages/allmydata/util/netstring.py", line 43, in split_netstring -# assert data[position] == b","[0], position -# exceptions.AssertionError: 15 -# ... -# (Pdb) pp data -# '334:12:b\'mutable-good\',90:URI:SSK-RO:... -from past.builtins import unicode as str -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, max, min # noqa: F401 from six import ensure_text import os, json diff --git a/src/allmydata/test/test_dictutil.py b/src/allmydata/test/test_dictutil.py index ce1c4a74c..8ee119a4f 100644 --- a/src/allmydata/test/test_dictutil.py +++ b/src/allmydata/test/test_dictutil.py @@ -3,10 +3,6 @@ Tests for allmydata.util.dictutil. """ from __future__ import annotations -from future.utils import PY2, PY3 - -from unittest import skipIf - from twisted.trial import unittest from allmydata.util import dictutil @@ -88,7 +84,6 @@ class DictUtil(unittest.TestCase): class TypedKeyDict(unittest.TestCase): """Tests for dictionaries that limit keys.""" - @skipIf(PY2, "Python 2 doesn't have issues mixing bytes and unicode.") def setUp(self): pass @@ -141,27 +136,6 @@ class TypedKeyDict(unittest.TestCase): self.assertEqual(d[u"456"], 300) -class TypedKeyDictPython2(unittest.TestCase): - """Tests for dictionaries that limit keys on Python 2.""" - - @skipIf(PY3, "Testing Python 2 behavior.") - def test_python2(self): - """ - On Python2, BytesKeyDict and UnicodeKeyDict are unnecessary, because - dicts can mix both without problem so you don't get confusing behavior - if you get the type wrong. - - Eventually in a Python 3-only world mixing bytes and unicode will be - bad, thus the existence of these classes, but as we port there will be - situations where it's mixed on Python 2, which again is fine. - """ - self.assertIs(dictutil.UnicodeKeyDict, dict) - self.assertIs(dictutil.BytesKeyDict, dict) - # Demonstration of how bytes and unicode can be mixed: - d = {u"abc": 1} - self.assertEqual(d[b"abc"], 1) - - class FilterTests(unittest.TestCase): """ Tests for ``dictutil.filter``. diff --git a/src/allmydata/test/test_dirnode.py b/src/allmydata/test/test_dirnode.py index 2349f3c4e..2ef57b0af 100644 --- a/src/allmydata/test/test_dirnode.py +++ b/src/allmydata/test/test_dirnode.py @@ -5,11 +5,6 @@ Ported to Python 3. from past.builtins import long -from future.utils import PY2 -if PY2: - # Skip list() since it results in spurious test failures - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, object, range, str, max, min # noqa: F401 - import time import unicodedata from zope.interface import implementer diff --git a/src/allmydata/test/test_encode.py b/src/allmydata/test/test_encode.py index 33c946010..8ce5e757a 100644 --- a/src/allmydata/test/test_encode.py +++ b/src/allmydata/test/test_encode.py @@ -2,9 +2,6 @@ Ported to Python 3. """ -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from past.builtins import chr as byteschr, long from zope.interface import implementer diff --git a/src/allmydata/test/test_encodingutil.py b/src/allmydata/test/test_encodingutil.py index b218b0bd9..64b9a243d 100644 --- a/src/allmydata/test/test_encodingutil.py +++ b/src/allmydata/test/test_encodingutil.py @@ -1,8 +1,5 @@ from future.utils import PY2, PY3 -if PY2: - # We don't import str because omg way too ambiguous in this context. - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, max, min # noqa: F401 from past.builtins import unicode @@ -137,19 +134,6 @@ class EncodingUtil(ReallyEqualMixin): def test_unicode_to_url(self): self.failUnless(unicode_to_url(lumiere_nfc), b"lumi\xc3\xa8re") - @skipIf(PY3, "Python 3 is always Unicode, regardless of OS.") - def test_unicode_to_output_py2(self): - if 'argv' not in dir(self): - return - - mock_stdout = MockStdout() - mock_stdout.encoding = self.io_encoding - self.patch(sys, 'stdout', mock_stdout) - - _reload() - self.failUnlessReallyEqual(unicode_to_output(lumiere_nfc), self.argv) - - @skipIf(PY2, "Python 3 only.") def test_unicode_to_output_py3(self): self.failUnlessReallyEqual(unicode_to_output(lumiere_nfc), lumiere_nfc) @@ -167,20 +151,6 @@ class EncodingUtil(ReallyEqualMixin): self.assertIsInstance(result, type(expected_value)) self.assertEqual(result, expected_value) - @skipIf(PY3, "Python 3 only.") - def test_unicode_platform_py2(self): - matrix = { - 'linux2': False, - 'linux3': False, - 'openbsd4': False, - 'win32': True, - 'darwin': True, - } - - _reload() - self.failUnlessReallyEqual(unicode_platform(), matrix[self.platform]) - - @skipIf(PY2, "Python 3 isn't Python 2.") def test_unicode_platform_py3(self): _reload() self.failUnlessReallyEqual(unicode_platform(), True) @@ -361,9 +331,6 @@ class QuoteOutput(ReallyEqualMixin, unittest.TestCase): def test_quote_output_utf8(self, enc='utf-8'): def check(inp, out, optional_quotes=False, quote_newlines=None): - if PY2: - # On Python 3 output is always Unicode: - out = out.encode('utf-8') self._check(inp, out, enc, optional_quotes, quote_newlines) self._test_quote_output_all(enc) diff --git a/src/allmydata/test/test_filenode.py b/src/allmydata/test/test_filenode.py index 2bf1edd6c..311e6516c 100644 --- a/src/allmydata/test/test_filenode.py +++ b/src/allmydata/test/test_filenode.py @@ -2,10 +2,6 @@ Ported to Python 3. """ -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - from twisted.trial import unittest from allmydata import uri, client from allmydata.monitor import Monitor diff --git a/src/allmydata/test/test_happiness.py b/src/allmydata/test/test_happiness.py index 8f21b1363..190a7c7d4 100644 --- a/src/allmydata/test/test_happiness.py +++ b/src/allmydata/test/test_happiness.py @@ -6,12 +6,6 @@ allmydata.util.happinessutil. Ported to Python 3. """ - -from future.utils import PY2 -if PY2: - # We omit dict, just in case newdict breaks things. - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, list, object, range, str, max, min # noqa: F401 - from twisted.trial import unittest from hypothesis import given from hypothesis.strategies import text, sets diff --git a/src/allmydata/test/test_hashutil.py b/src/allmydata/test/test_hashutil.py index d64fb8abc..9be8f05d6 100644 --- a/src/allmydata/test/test_hashutil.py +++ b/src/allmydata/test/test_hashutil.py @@ -4,10 +4,6 @@ Tests for allmydata.util.hashutil. Ported to Python 3. """ -from future.utils import PY2 -if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - from twisted.trial import unittest from allmydata.util import hashutil, base32 diff --git a/src/allmydata/test/test_humanreadable.py b/src/allmydata/test/test_humanreadable.py index 9cdc9678c..caf85ee6c 100644 --- a/src/allmydata/test/test_humanreadable.py +++ b/src/allmydata/test/test_humanreadable.py @@ -4,11 +4,6 @@ Tests for allmydata.util.humanreadable. This module has been ported to Python 3. """ - -from future.utils import PY2 -if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - from past.builtins import long from twisted.trial import unittest diff --git a/src/allmydata/test/test_hung_server.py b/src/allmydata/test/test_hung_server.py index 24e2810e0..a78f8614e 100644 --- a/src/allmydata/test/test_hung_server.py +++ b/src/allmydata/test/test_hung_server.py @@ -4,10 +4,6 @@ Ported to Python 3. """ -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - import os, shutil from twisted.trial import unittest from twisted.internet import defer diff --git a/src/allmydata/test/test_i2p_provider.py b/src/allmydata/test/test_i2p_provider.py index 8628aeed6..3b99646bf 100644 --- a/src/allmydata/test/test_i2p_provider.py +++ b/src/allmydata/test/test_i2p_provider.py @@ -2,10 +2,6 @@ Ported to Python 3. """ -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - import os from twisted.trial import unittest from twisted.internet import defer, error diff --git a/src/allmydata/test/test_immutable.py b/src/allmydata/test/test_immutable.py index 303485831..39c31623d 100644 --- a/src/allmydata/test/test_immutable.py +++ b/src/allmydata/test/test_immutable.py @@ -2,10 +2,6 @@ This module has been ported to Python 3. """ -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - import random from twisted.trial import unittest diff --git a/src/allmydata/test/test_introducer.py b/src/allmydata/test/test_introducer.py index 2fd6ff69b..d37df48a9 100644 --- a/src/allmydata/test/test_introducer.py +++ b/src/allmydata/test/test_introducer.py @@ -2,10 +2,6 @@ Ported to Python 3. """ -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - from six import ensure_binary, ensure_text import os, re, itertools diff --git a/src/allmydata/test/test_json_metadata.py b/src/allmydata/test/test_json_metadata.py index 822d37925..950a5847c 100644 --- a/src/allmydata/test/test_json_metadata.py +++ b/src/allmydata/test/test_json_metadata.py @@ -2,10 +2,6 @@ Ported to Python 3. """ -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - from twisted.trial.unittest import TestCase from allmydata.web.common import get_filenode_metadata, SDMF_VERSION, MDMF_VERSION diff --git a/src/allmydata/test/test_log.py b/src/allmydata/test/test_log.py index c3472cad8..ea03ba730 100644 --- a/src/allmydata/test/test_log.py +++ b/src/allmydata/test/test_log.py @@ -6,8 +6,6 @@ Ported to Python 3. from future.utils import PY2, native_str -if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from twisted.trial import unittest from twisted.python.failure import Failure diff --git a/src/allmydata/test/test_monitor.py b/src/allmydata/test/test_monitor.py index bede3e001..492597bd9 100644 --- a/src/allmydata/test/test_monitor.py +++ b/src/allmydata/test/test_monitor.py @@ -2,11 +2,6 @@ Tests for allmydata.monitor. """ - -from future.utils import PY2 -if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - from twisted.trial import unittest from allmydata.monitor import Monitor, OperationCancelledError diff --git a/src/allmydata/test/test_multi_introducers.py b/src/allmydata/test/test_multi_introducers.py index 7f42177ab..f5d0ff98c 100644 --- a/src/allmydata/test/test_multi_introducers.py +++ b/src/allmydata/test/test_multi_introducers.py @@ -2,9 +2,6 @@ Ported to Python 3. """ -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from six import ensure_binary import os diff --git a/src/allmydata/test/test_netstring.py b/src/allmydata/test/test_netstring.py index 54cc16df8..6f9a21ee2 100644 --- a/src/allmydata/test/test_netstring.py +++ b/src/allmydata/test/test_netstring.py @@ -4,10 +4,6 @@ Tests for allmydata.util.netstring. Ported to Python 3. """ -from future.utils import PY2 -if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - from twisted.trial import unittest from allmydata.util.netstring import netstring, split_netstring diff --git a/src/allmydata/test/test_no_network.py b/src/allmydata/test/test_no_network.py index 2d6bdb896..88eb27979 100644 --- a/src/allmydata/test/test_no_network.py +++ b/src/allmydata/test/test_no_network.py @@ -4,10 +4,6 @@ Test the NoNetworkGrid test harness. Ported to Python 3. """ -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - from twisted.trial import unittest from twisted.application import service from allmydata.test.no_network import NoNetworkGrid diff --git a/src/allmydata/test/test_observer.py b/src/allmydata/test/test_observer.py index ddc58b074..6d26b2470 100644 --- a/src/allmydata/test/test_observer.py +++ b/src/allmydata/test/test_observer.py @@ -4,11 +4,6 @@ Tests for allmydata.util.observer. Ported to Python 3. """ - -from future.utils import PY2 -if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - from twisted.trial import unittest from twisted.internet import defer, reactor from allmydata.util import observer diff --git a/src/allmydata/test/test_openmetrics.py b/src/allmydata/test/test_openmetrics.py index 2199bcc5f..4987aed11 100644 --- a/src/allmydata/test/test_openmetrics.py +++ b/src/allmydata/test/test_openmetrics.py @@ -4,14 +4,6 @@ Tests for ``/statistics?t=openmetrics``. Ported to Python 3. """ - -from future.utils import PY2 - -if PY2: - # fmt: off - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - # fmt: on - from prometheus_client.openmetrics import parser from treq.testing import RequestTraversalAgent diff --git a/src/allmydata/test/test_python2_regressions.py b/src/allmydata/test/test_python2_regressions.py deleted file mode 100644 index 9a12f0374..000000000 --- a/src/allmydata/test/test_python2_regressions.py +++ /dev/null @@ -1,77 +0,0 @@ -""" -Tests to check for Python2 regressions -""" - - -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - -from unittest import skipUnless -from inspect import isclass - -from twisted.python.modules import getModule - -from testtools import ( - TestCase, -) -from testtools.matchers import ( - Equals, -) - -BLACKLIST = { - "allmydata.scripts.types_", - "allmydata.test._win_subprocess", - "allmydata.windows.registry", - "allmydata.windows.fixups", -} - - -def is_new_style(cls): - """ - :return bool: ``True`` if and only if the given class is "new style". - """ - # All new-style classes are instances of type. By definition. - return isinstance(cls, type) - -def defined_here(cls, where): - """ - :return bool: ``True`` if and only if the given class was defined in a - module with the given name. - - :note: Classes can lie about where they are defined. Try not to do that. - """ - return cls.__module__ == where - - -class PythonTwoRegressions(TestCase): - """ - Regression tests for Python 2 behaviors related to Python 3 porting. - """ - @skipUnless(PY2, "No point in running on Python 3.") - def test_new_style_classes(self): - """ - All classes in Tahoe-LAFS are new-style. - """ - newstyle = set() - classic = set() - for mod in getModule("allmydata").walkModules(): - if mod.name in BLACKLIST: - continue - - # iterAttributes will only work on loaded modules. So, load it. - mod.load() - - for attr in mod.iterAttributes(): - value = attr.load() - if isclass(value) and defined_here(value, mod.name): - if is_new_style(value): - newstyle.add(value) - else: - classic.add(value) - - self.assertThat( - classic, - Equals(set()), - "Expected to find no classic classes.", - ) diff --git a/src/allmydata/test/test_repairer.py b/src/allmydata/test/test_repairer.py index 280825775..cf1cf843b 100644 --- a/src/allmydata/test/test_repairer.py +++ b/src/allmydata/test/test_repairer.py @@ -3,10 +3,6 @@ Ported to Python 3. """ -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - from allmydata.test import common from allmydata.monitor import Monitor from allmydata import check_results diff --git a/src/allmydata/test/test_runner.py b/src/allmydata/test/test_runner.py index b784b2c61..e7cb4b082 100644 --- a/src/allmydata/test/test_runner.py +++ b/src/allmydata/test/test_runner.py @@ -3,8 +3,6 @@ Ported to Python 3 """ from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from six import ensure_text @@ -12,11 +10,6 @@ import os.path, re, sys from os import linesep import locale -import six - -from testtools import ( - skipUnless, -) from testtools.matchers import ( MatchesListwise, MatchesAny, @@ -99,27 +92,6 @@ srcfile = allmydata.__file__ rootdir = get_root_from_file(srcfile) -class ParseOptionsTests(SyncTestCase): - """ - Tests for ``parse_options``. - """ - @skipUnless(six.PY2, "Only Python 2 exceptions must stringify to bytes.") - def test_nonascii_unknown_subcommand_python2(self): - """ - When ``parse_options`` is called with an argv indicating a subcommand that - does not exist and which also contains non-ascii characters, the - exception it raises includes the subcommand encoded as UTF-8. - """ - tricky = u"\u00F6" - try: - parse_options([tricky]) - except usage.error as e: - self.assertEqual( - b"Unknown command: \\xf6", - b"{}".format(e), - ) - - class ParseOrExitTests(SyncTestCase): """ Tests for ``parse_or_exit``. diff --git a/src/allmydata/test/test_sftp.py b/src/allmydata/test/test_sftp.py index d1902b9ee..a7de35320 100644 --- a/src/allmydata/test/test_sftp.py +++ b/src/allmydata/test/test_sftp.py @@ -2,10 +2,6 @@ Ported to Python 3. """ -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - import re, struct, traceback, time, calendar from stat import S_IFREG, S_IFDIR diff --git a/src/allmydata/test/test_spans.py b/src/allmydata/test/test_spans.py index e1ac9a229..e6e510e5d 100644 --- a/src/allmydata/test/test_spans.py +++ b/src/allmydata/test/test_spans.py @@ -2,11 +2,6 @@ Tests for allmydata.util.spans. """ - -from future.utils import PY2 -if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - from past.builtins import long import binascii diff --git a/src/allmydata/test/test_statistics.py b/src/allmydata/test/test_statistics.py index ce91e4efb..2442e14a2 100644 --- a/src/allmydata/test/test_statistics.py +++ b/src/allmydata/test/test_statistics.py @@ -4,10 +4,6 @@ Tests for allmydata.util.statistics. Ported to Python 3. """ -from future.utils import PY2 -if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - from six.moves import StringIO # native string StringIO from twisted.trial import unittest diff --git a/src/allmydata/test/test_stats.py b/src/allmydata/test/test_stats.py index 8f02640e0..aba3a0e9c 100644 --- a/src/allmydata/test/test_stats.py +++ b/src/allmydata/test/test_stats.py @@ -2,10 +2,6 @@ Ported to Python 3. """ -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - from twisted.trial import unittest from twisted.application import service from allmydata.stats import CPUUsageMonitor diff --git a/src/allmydata/test/test_storage_web.py b/src/allmydata/test/test_storage_web.py index 10249c83a..e9bca2183 100644 --- a/src/allmydata/test/test_storage_web.py +++ b/src/allmydata/test/test_storage_web.py @@ -4,13 +4,6 @@ Tests for twisted.storage that uses Web APIs. Partially ported to Python 3. """ - -from future.utils import PY2 -if PY2: - # Omitted list since it broke a test on Python 2. Shouldn't require further - # work, when we switch to Python 3 we'll be dropping this, anyway. - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, object, range, str, max, min # noqa: F401 - import time import os.path import re diff --git a/src/allmydata/test/test_time_format.py b/src/allmydata/test/test_time_format.py index 2f10a3ea9..0b409feed 100644 --- a/src/allmydata/test/test_time_format.py +++ b/src/allmydata/test/test_time_format.py @@ -2,10 +2,6 @@ Tests for allmydata.util.time_format. """ -from future.utils import PY2 -if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - from past.builtins import long import time diff --git a/src/allmydata/test/test_upload.py b/src/allmydata/test/test_upload.py index f9541e946..ecb76ec27 100644 --- a/src/allmydata/test/test_upload.py +++ b/src/allmydata/test/test_upload.py @@ -4,10 +4,6 @@ Ported to Python 3. """ -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - import os, shutil from io import BytesIO from base64 import ( diff --git a/src/allmydata/test/test_uri.py b/src/allmydata/test/test_uri.py index 7f4908da6..ae4bf2002 100644 --- a/src/allmydata/test/test_uri.py +++ b/src/allmydata/test/test_uri.py @@ -4,11 +4,6 @@ Tests for allmydata.uri. Ported to Python 3. """ - -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, dict, hex, input, next, oct, open, pow, round, super, bytes, list, object, range, str, max, min # noqa: F401 - import os from twisted.trial import unittest from allmydata import uri diff --git a/src/allmydata/test/test_util.py b/src/allmydata/test/test_util.py index 0b4917381..b6056b7e4 100644 --- a/src/allmydata/test/test_util.py +++ b/src/allmydata/test/test_util.py @@ -2,11 +2,6 @@ Ported to Python3. """ - -from future.utils import PY2 -if PY2: - # open is not here because we want to use native strings on Py2 - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import six import os, time, sys import yaml @@ -189,8 +184,6 @@ class FileUtil(ReallyEqualMixin, unittest.TestCase): self.failUnlessRaises(AssertionError, fileutil.abspath_expanduser_unicode, b"bytestring") saved_cwd = os.path.normpath(os.getcwd()) - if PY2: - saved_cwd = saved_cwd.decode("utf8") abspath_cwd = fileutil.abspath_expanduser_unicode(u".") abspath_cwd_notlong = fileutil.abspath_expanduser_unicode(u".", long_path=False) self.failUnless(isinstance(saved_cwd, str), saved_cwd) diff --git a/src/allmydata/test/web/common.py b/src/allmydata/test/web/common.py index be7fe0af1..f92548810 100644 --- a/src/allmydata/test/web/common.py +++ b/src/allmydata/test/web/common.py @@ -2,10 +2,6 @@ Ported to Python 3. """ -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - import re unknown_rwcap = u"lafs://from_the_future_rw_\u263A".encode('utf-8') diff --git a/src/allmydata/test/web/matchers.py b/src/allmydata/test/web/matchers.py index f6d52aaad..669e7ddf4 100644 --- a/src/allmydata/test/web/matchers.py +++ b/src/allmydata/test/web/matchers.py @@ -2,10 +2,6 @@ Ported to Python 3. """ -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - import attr from testtools.matchers import Mismatch diff --git a/src/allmydata/test/web/test_common.py b/src/allmydata/test/web/test_common.py index 6c19a5b78..34c9a17a3 100644 --- a/src/allmydata/test/web/test_common.py +++ b/src/allmydata/test/web/test_common.py @@ -4,10 +4,6 @@ Tests for ``allmydata.web.common``. Ported to Python 3. """ -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - import gc from bs4 import ( diff --git a/src/allmydata/test/web/test_grid.py b/src/allmydata/test/web/test_grid.py index 0a7c913c0..86404a7bf 100644 --- a/src/allmydata/test/web/test_grid.py +++ b/src/allmydata/test/web/test_grid.py @@ -2,10 +2,6 @@ Ported to Python 3. """ -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - import os.path, re from urllib.parse import quote as url_quote import json diff --git a/src/allmydata/test/web/test_introducer.py b/src/allmydata/test/web/test_introducer.py index d21a663f7..6741c1a2d 100644 --- a/src/allmydata/test/web/test_introducer.py +++ b/src/allmydata/test/web/test_introducer.py @@ -2,10 +2,6 @@ Ported to Python 3. """ -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - import json from os.path import join diff --git a/src/allmydata/test/web/test_logs.py b/src/allmydata/test/web/test_logs.py index 093503265..34ecccff6 100644 --- a/src/allmydata/test/web/test_logs.py +++ b/src/allmydata/test/web/test_logs.py @@ -4,10 +4,6 @@ Tests for ``allmydata.web.logs``. Ported to Python 3. """ -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - import json from twisted.internet.defer import inlineCallbacks diff --git a/src/allmydata/test/web/test_private.py b/src/allmydata/test/web/test_private.py index 5652e8008..110e31ff1 100644 --- a/src/allmydata/test/web/test_private.py +++ b/src/allmydata/test/web/test_private.py @@ -4,10 +4,6 @@ Tests for ``allmydata.web.private``. Ported to Python 3. """ -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - from testtools.matchers import ( Equals, ) diff --git a/src/allmydata/test/web/test_root.py b/src/allmydata/test/web/test_root.py index e810f9a14..f3b877b2d 100644 --- a/src/allmydata/test/web/test_root.py +++ b/src/allmydata/test/web/test_root.py @@ -2,10 +2,6 @@ Ported to Python 3. """ -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - import time import json diff --git a/src/allmydata/test/web/test_status.py b/src/allmydata/test/web/test_status.py index c46ac4500..81c9568e5 100644 --- a/src/allmydata/test/web/test_status.py +++ b/src/allmydata/test/web/test_status.py @@ -4,10 +4,6 @@ Tests for ```allmydata.web.status```. Ported to Python 3. """ -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - from bs4 import BeautifulSoup from twisted.web.template import flattenString diff --git a/src/allmydata/test/web/test_util.py b/src/allmydata/test/web/test_util.py index 5afe884b0..c21a66e98 100644 --- a/src/allmydata/test/web/test_util.py +++ b/src/allmydata/test/web/test_util.py @@ -2,10 +2,6 @@ Ported to Python 3. """ -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - from twisted.trial import unittest from allmydata.web import status, common from allmydata.dirnode import ONLY_FILES diff --git a/src/allmydata/unknown.py b/src/allmydata/unknown.py index bfb56496d..2a81437f6 100644 --- a/src/allmydata/unknown.py +++ b/src/allmydata/unknown.py @@ -1,10 +1,6 @@ """Ported to Python 3. """ -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - from zope.interface import implementer from twisted.internet import defer from allmydata.interfaces import IFilesystemNode, MustNotBeUnknownRWError, \ diff --git a/src/allmydata/util/abbreviate.py b/src/allmydata/util/abbreviate.py index e775813bd..80abf7b05 100644 --- a/src/allmydata/util/abbreviate.py +++ b/src/allmydata/util/abbreviate.py @@ -4,10 +4,6 @@ Convert timestamps to abbreviated English text. Ported to Python 3. """ -from future.utils import PY2 -if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - import re from datetime import timedelta diff --git a/src/allmydata/util/assertutil.py b/src/allmydata/util/assertutil.py index 2088f505c..776ed7ef7 100644 --- a/src/allmydata/util/assertutil.py +++ b/src/allmydata/util/assertutil.py @@ -7,12 +7,6 @@ have tests. Ported to Python 3. """ - -from future.utils import PY2 -if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - - # The API importers expect: from pyutil.assertutil import _assert, precondition, postcondition diff --git a/src/allmydata/util/base62.py b/src/allmydata/util/base62.py index dcde36562..2c4425562 100644 --- a/src/allmydata/util/base62.py +++ b/src/allmydata/util/base62.py @@ -5,8 +5,6 @@ Ported to Python 3. """ from future.utils import PY2 -if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 if PY2: import string diff --git a/src/allmydata/util/configutil.py b/src/allmydata/util/configutil.py index 749f6d7d7..bdb872132 100644 --- a/src/allmydata/util/configutil.py +++ b/src/allmydata/util/configutil.py @@ -6,12 +6,6 @@ Configuration is returned as Unicode strings. Ported to Python 3. """ -from future.utils import PY2 -if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - -# On Python 2 we use the backport package; that means we always get unicode -# out. from configparser import ConfigParser import attr diff --git a/src/allmydata/util/consumer.py b/src/allmydata/util/consumer.py index cd73119fe..c899fc25e 100644 --- a/src/allmydata/util/consumer.py +++ b/src/allmydata/util/consumer.py @@ -5,10 +5,6 @@ a filenode's read() method. See download_to_data() for an example of its use. Ported to Python 3. """ -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - from zope.interface import implementer from twisted.internet.interfaces import IConsumer diff --git a/src/allmydata/util/dbutil.py b/src/allmydata/util/dbutil.py index ac7ddbfeb..b27b58ab5 100644 --- a/src/allmydata/util/dbutil.py +++ b/src/allmydata/util/dbutil.py @@ -6,11 +6,6 @@ Test coverage currently provided by test_backupdb.py. Ported to Python 3. """ - -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - import os, sys import sqlite3 diff --git a/src/allmydata/util/encodingutil.py b/src/allmydata/util/encodingutil.py index d13011dd2..0625bfdec 100644 --- a/src/allmydata/util/encodingutil.py +++ b/src/allmydata/util/encodingutil.py @@ -10,9 +10,6 @@ Unicode is the default everywhere in Python 3. from future.utils import PY2, PY3, native_str from future.builtins import str as future_str -if PY2: - # We omit str() because that seems too tricky to get right. - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, max, min # noqa: F401 from past.builtins import unicode from six import ensure_str diff --git a/src/allmydata/util/fileutil.py b/src/allmydata/util/fileutil.py index 0df3781b5..0a4eebaba 100644 --- a/src/allmydata/util/fileutil.py +++ b/src/allmydata/util/fileutil.py @@ -4,12 +4,6 @@ Ported to Python3. Futz with files like a pro. """ - -from future.utils import PY2 -if PY2: - # open is not here because we want to use native strings on Py2 - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - import sys, os, stat, tempfile, time, binascii import six from collections import namedtuple @@ -342,8 +336,6 @@ def abspath_expanduser_unicode(path, base=None, long_path=True): if not os.path.isabs(path): if base is None: cwd = os.getcwd() - if PY2: - cwd = cwd.decode('utf8') path = os.path.join(cwd, path) else: path = os.path.join(base, path) diff --git a/src/allmydata/util/gcutil.py b/src/allmydata/util/gcutil.py index 9eed8b1f4..2302ae6b7 100644 --- a/src/allmydata/util/gcutil.py +++ b/src/allmydata/util/gcutil.py @@ -11,10 +11,6 @@ Helpers for managing garbage collection. Ported to Python 3. """ -from future.utils import PY2 -if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - __all__ = [ "fileDescriptorResource", ] diff --git a/src/allmydata/util/happinessutil.py b/src/allmydata/util/happinessutil.py index 7f5d0f8a7..19b602826 100644 --- a/src/allmydata/util/happinessutil.py +++ b/src/allmydata/util/happinessutil.py @@ -5,11 +5,6 @@ reporting it in messages. Ported to Python 3. """ -from future.utils import PY2 -if PY2: - # We omit dict, just in case newdict breaks things. - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, list, object, range, str, max, min # noqa: F401 - from copy import deepcopy from allmydata.immutable.happiness_upload import residual_network from allmydata.immutable.happiness_upload import augmenting_path_for diff --git a/src/allmydata/util/humanreadable.py b/src/allmydata/util/humanreadable.py index 2eaa7c79a..356edb659 100644 --- a/src/allmydata/util/humanreadable.py +++ b/src/allmydata/util/humanreadable.py @@ -4,11 +4,6 @@ Utilities for turning objects into human-readable strings. This module has been ported to Python 3. """ - -from future.utils import PY2 -if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - import os from reprlib import Repr diff --git a/src/allmydata/util/idlib.py b/src/allmydata/util/idlib.py index c74f11599..26dd72445 100644 --- a/src/allmydata/util/idlib.py +++ b/src/allmydata/util/idlib.py @@ -2,10 +2,6 @@ Ported to Python 3. """ -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - from six import ensure_text from foolscap import base32 diff --git a/src/allmydata/util/jsonbytes.py b/src/allmydata/util/jsonbytes.py index 24c8f3311..0762702f9 100644 --- a/src/allmydata/util/jsonbytes.py +++ b/src/allmydata/util/jsonbytes.py @@ -4,25 +4,9 @@ A JSON encoder than can serialize bytes. Ported to Python 3. """ - -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - import json import codecs -if PY2: - def backslashreplace_py2(ex): - """ - On Python 2 'backslashreplace' error handler doesn't work, so write our - own. - """ - return ''.join('\\x{:02x}'.format(ord(c)) - for c in ex.object[ex.start:ex.end]), ex.end - - codecs.register_error("backslashreplace_tahoe_py2", backslashreplace_py2) - def bytes_to_unicode(any_bytes, obj): """Convert bytes to unicode. @@ -31,8 +15,6 @@ def bytes_to_unicode(any_bytes, obj): :param obj: Object to de-byte-ify. """ errors = "backslashreplace" if any_bytes else "strict" - if PY2 and errors == "backslashreplace": - errors = "backslashreplace_tahoe_py2" def doit(obj): """Convert any bytes objects to unicode, recursively.""" diff --git a/src/allmydata/util/log.py b/src/allmydata/util/log.py index afd612d7a..3589bc366 100644 --- a/src/allmydata/util/log.py +++ b/src/allmydata/util/log.py @@ -5,8 +5,6 @@ Ported to Python 3. """ from future.utils import PY2 -if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from six import ensure_str from pyutil import nummedobj diff --git a/src/allmydata/util/mathutil.py b/src/allmydata/util/mathutil.py index a6a58e834..2aeb11b9e 100644 --- a/src/allmydata/util/mathutil.py +++ b/src/allmydata/util/mathutil.py @@ -6,12 +6,6 @@ Backwards compatibility for direct imports. Ported to Python 3. """ - -from future.utils import PY2 -if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - - # The API importers expect: from pyutil.mathutil import div_ceil, next_multiple, pad_size, is_power_of_k, next_power_of_k, ave, log_ceil, log_floor diff --git a/src/allmydata/util/netstring.py b/src/allmydata/util/netstring.py index 23d8492ab..db913172f 100644 --- a/src/allmydata/util/netstring.py +++ b/src/allmydata/util/netstring.py @@ -4,10 +4,6 @@ Netstring encoding and decoding. Ported to Python 3. """ -from future.utils import PY2 -if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - from past.builtins import long try: diff --git a/src/allmydata/util/observer.py b/src/allmydata/util/observer.py index b753d51ad..2fa514a02 100644 --- a/src/allmydata/util/observer.py +++ b/src/allmydata/util/observer.py @@ -4,11 +4,6 @@ Observer for Twisted code. Ported to Python 3. """ - -from future.utils import PY2 -if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - import weakref from twisted.internet import defer from foolscap.api import eventually diff --git a/src/allmydata/util/rrefutil.py b/src/allmydata/util/rrefutil.py index 1fb757f34..15622435d 100644 --- a/src/allmydata/util/rrefutil.py +++ b/src/allmydata/util/rrefutil.py @@ -2,10 +2,6 @@ Ported to Python 3. """ -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - from foolscap.api import Violation, RemoteException diff --git a/src/allmydata/util/spans.py b/src/allmydata/util/spans.py index 4ddc67704..e5b265aaa 100644 --- a/src/allmydata/util/spans.py +++ b/src/allmydata/util/spans.py @@ -1,8 +1,4 @@ -from future.utils import PY2 -if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - class Spans(object): """I represent a compressed list of booleans, one per index (an integer). diff --git a/src/allmydata/util/statistics.py b/src/allmydata/util/statistics.py index 35a7216a2..9881dc13f 100644 --- a/src/allmydata/util/statistics.py +++ b/src/allmydata/util/statistics.py @@ -11,11 +11,6 @@ Ported to Python 3. # Transitive Grace Period Public License, version 1 or later. - -from future.utils import PY2 -if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - from allmydata.util.mathutil import round_sigfigs import math from functools import reduce diff --git a/src/allmydata/util/time_format.py b/src/allmydata/util/time_format.py index af98bf1d2..14cf4688e 100644 --- a/src/allmydata/util/time_format.py +++ b/src/allmydata/util/time_format.py @@ -5,9 +5,6 @@ ISO-8601: http://www.cl.cam.ac.uk/~mgk25/iso-time.html """ -from future.utils import PY2 -if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from future.utils import native_str import calendar, datetime, re, time diff --git a/src/allmydata/util/yamlutil.py b/src/allmydata/util/yamlutil.py index eae1f8a6e..512d5a2a9 100644 --- a/src/allmydata/util/yamlutil.py +++ b/src/allmydata/util/yamlutil.py @@ -2,35 +2,8 @@ Ported to Python 3. """ -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - import yaml - -if PY2: - # On Python 2 the way pyyaml deals with Unicode strings is inconsistent. - # - # >>> yaml.safe_load(yaml.safe_dump(u"hello")) - # 'hello' - # >>> yaml.safe_load(yaml.safe_dump(u"hello\u1234")) - # u'hello\u1234' - # - # In other words, Unicode strings get roundtripped to byte strings, but - # only sometimes. - # - # In order to ensure unicode stays unicode, we add a configuration saying - # that the YAML String Language-Independent Type ("a sequence of zero or - # more Unicode characters") should be the underlying Unicode string object, - # rather than converting to bytes when possible. - # - # Reference: https://yaml.org/type/str.html - def construct_unicode(loader, node): - return node.value - yaml.SafeLoader.add_constructor("tag:yaml.org,2002:str", - construct_unicode) - def safe_load(f): return yaml.safe_load(f) diff --git a/src/allmydata/web/check_results.py b/src/allmydata/web/check_results.py index 4bb34a2a8..1ec835658 100644 --- a/src/allmydata/web/check_results.py +++ b/src/allmydata/web/check_results.py @@ -2,10 +2,6 @@ Ported to Python 3. """ -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - import time from twisted.web import ( diff --git a/src/allmydata/web/directory.py b/src/allmydata/web/directory.py index 4012f93fd..001caf22f 100644 --- a/src/allmydata/web/directory.py +++ b/src/allmydata/web/directory.py @@ -2,12 +2,6 @@ Ported to Python 3. """ -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, max, min # noqa: F401 - # Don't use Future's str so that we don't get leaks into bad byte formatting - from past.builtins import unicode as str - from urllib.parse import quote as url_quote from datetime import timedelta diff --git a/src/allmydata/web/info.py b/src/allmydata/web/info.py index ff126a2a1..e10e59061 100644 --- a/src/allmydata/web/info.py +++ b/src/allmydata/web/info.py @@ -2,10 +2,6 @@ Ported to Python 3. """ -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - import os from urllib.parse import quote as urlquote diff --git a/src/allmydata/web/operations.py b/src/allmydata/web/operations.py index 4319a4d9c..0b71cc404 100644 --- a/src/allmydata/web/operations.py +++ b/src/allmydata/web/operations.py @@ -2,10 +2,6 @@ Ported to Python 3. """ -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - import time from hyperlink import ( DecodedURL, diff --git a/src/allmydata/web/private.py b/src/allmydata/web/private.py index 55cace5a3..4410399b8 100644 --- a/src/allmydata/web/private.py +++ b/src/allmydata/web/private.py @@ -2,10 +2,6 @@ Ported to Python 3. """ -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - import attr from zope.interface import ( diff --git a/src/allmydata/web/status.py b/src/allmydata/web/status.py index 96d68c87f..07d0256e8 100644 --- a/src/allmydata/web/status.py +++ b/src/allmydata/web/status.py @@ -2,10 +2,6 @@ Ported to Python 3. """ - -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from past.builtins import long import itertools diff --git a/src/allmydata/web/storage.py b/src/allmydata/web/storage.py index ebbef4fa3..aeefcf62a 100644 --- a/src/allmydata/web/storage.py +++ b/src/allmydata/web/storage.py @@ -2,10 +2,6 @@ Ported to Python 3. """ -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - import time from twisted.python.filepath import FilePath from twisted.web.template import ( diff --git a/src/allmydata/web/storage_plugins.py b/src/allmydata/web/storage_plugins.py index 9be4e84af..ad448ccdd 100644 --- a/src/allmydata/web/storage_plugins.py +++ b/src/allmydata/web/storage_plugins.py @@ -5,10 +5,6 @@ of all enabled storage client plugins. Ported to Python 3. """ -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - from twisted.web.resource import ( Resource, NoResource, From 825e42cdfbbff597384c2d1795350d5633b137e7 Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 26 Feb 2024 14:49:22 -0700 Subject: [PATCH 2265/2309] Fix fragile unit-test --- newsfragments/4091.minor | 0 src/allmydata/test/test_humanreadable.py | 3 ++- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 newsfragments/4091.minor diff --git a/newsfragments/4091.minor b/newsfragments/4091.minor new file mode 100644 index 000000000..e69de29bb diff --git a/src/allmydata/test/test_humanreadable.py b/src/allmydata/test/test_humanreadable.py index 9cdc9678c..0d50d091f 100644 --- a/src/allmydata/test/test_humanreadable.py +++ b/src/allmydata/test/test_humanreadable.py @@ -27,7 +27,8 @@ class NoArgumentException(Exception): class HumanReadable(unittest.TestCase): def test_repr(self): hr = humanreadable.hr - self.failUnlessEqual(hr(foo), "") + # we match on regex so this test isn't fragile about line-numbers + self.assertRegex(hr(foo), r"") self.failUnlessEqual(hr(self.test_repr), ">") self.failUnlessEqual(hr(long(1)), "1") From b2541be7c6dee914fe71c62e70622bae07f0e555 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 27 Feb 2024 15:32:42 -0500 Subject: [PATCH 2266/2309] News fragment. --- newsfragments/4092.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/4092.minor diff --git a/newsfragments/4092.minor b/newsfragments/4092.minor new file mode 100644 index 000000000..e69de29bb From 3fb0bcfff792df45273e47454b2acaa8a6c80cba Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 27 Feb 2024 15:37:53 -0500 Subject: [PATCH 2267/2309] Remove unnecessary imports. --- src/allmydata/_monkeypatch.py | 1 - src/allmydata/interfaces.py | 2 +- src/allmydata/introducer/interfaces.py | 2 +- src/allmydata/scripts/runner.py | 3 +-- src/allmydata/scripts/slow_operation.py | 2 +- src/allmydata/scripts/tahoe_get.py | 2 +- src/allmydata/scripts/tahoe_manifest.py | 2 +- src/allmydata/storage/common.py | 2 +- src/allmydata/storage/immutable.py | 2 +- src/allmydata/test/cli/test_backup.py | 2 +- src/allmydata/test/cli/test_cp.py | 3 +-- src/allmydata/test/cli/test_list.py | 2 +- src/allmydata/test/mutable/util.py | 2 +- src/allmydata/test/test_crawler.py | 2 +- src/allmydata/test/test_log.py | 2 +- src/allmydata/test/test_runner.py | 3 --- src/allmydata/util/encodingutil.py | 2 +- src/allmydata/util/jsonbytes.py | 1 - 18 files changed, 15 insertions(+), 22 deletions(-) diff --git a/src/allmydata/_monkeypatch.py b/src/allmydata/_monkeypatch.py index b5f171224..cbd3ddd13 100644 --- a/src/allmydata/_monkeypatch.py +++ b/src/allmydata/_monkeypatch.py @@ -6,7 +6,6 @@ Ported to Python 3. from future.utils import PY2 -from warnings import catch_warnings def patch(): diff --git a/src/allmydata/interfaces.py b/src/allmydata/interfaces.py index 168a7b8e4..96fccb958 100644 --- a/src/allmydata/interfaces.py +++ b/src/allmydata/interfaces.py @@ -6,7 +6,7 @@ Ported to Python 3. Note that for RemoteInterfaces, the __remote_name__ needs to be a native string because of https://github.com/warner/foolscap/blob/43f4485a42c9c28e2c79d655b3a9e24d4e6360ca/src/foolscap/remoteinterface.py#L67 """ -from future.utils import PY2, native_str +from future.utils import native_str from past.builtins import long from typing import Dict diff --git a/src/allmydata/introducer/interfaces.py b/src/allmydata/introducer/interfaces.py index 4667c7be0..13cd7c3da 100644 --- a/src/allmydata/introducer/interfaces.py +++ b/src/allmydata/introducer/interfaces.py @@ -3,7 +3,7 @@ Ported to Python 3. """ -from future.utils import PY2, native_str +from future.utils import native_str from zope.interface import Interface from foolscap.api import StringConstraint, SetOf, DictOf, Any, \ diff --git a/src/allmydata/scripts/runner.py b/src/allmydata/scripts/runner.py index 32c68ee57..73e2e4b59 100644 --- a/src/allmydata/scripts/runner.py +++ b/src/allmydata/scripts/runner.py @@ -1,6 +1,5 @@ import os, sys from io import StringIO -from past.builtins import unicode import six from twisted.python import usage @@ -106,7 +105,7 @@ def parse_options(argv, config=None): config = Options() try: config.parseOptions(argv) - except usage.error as e: + except usage.error: raise return config diff --git a/src/allmydata/scripts/slow_operation.py b/src/allmydata/scripts/slow_operation.py index e1a944ba7..236c2bf0b 100644 --- a/src/allmydata/scripts/slow_operation.py +++ b/src/allmydata/scripts/slow_operation.py @@ -2,7 +2,7 @@ Ported to Python 3. """ -from future.utils import PY2, PY3 +from future.utils import PY3 from six import ensure_str diff --git a/src/allmydata/scripts/tahoe_get.py b/src/allmydata/scripts/tahoe_get.py index 332529d04..f22d3d293 100644 --- a/src/allmydata/scripts/tahoe_get.py +++ b/src/allmydata/scripts/tahoe_get.py @@ -2,7 +2,7 @@ Ported to Python 3. """ -from future.utils import PY2, PY3 +from future.utils import PY3 from urllib.parse import quote as url_quote from allmydata.scripts.common import get_alias, DEFAULT_ALIAS, escape_path, \ diff --git a/src/allmydata/scripts/tahoe_manifest.py b/src/allmydata/scripts/tahoe_manifest.py index 1cc5ba591..0cd6100bf 100644 --- a/src/allmydata/scripts/tahoe_manifest.py +++ b/src/allmydata/scripts/tahoe_manifest.py @@ -2,7 +2,7 @@ Ported to Python 3. """ -from future.utils import PY2, PY3 +from future.utils import PY3 from six import ensure_str diff --git a/src/allmydata/storage/common.py b/src/allmydata/storage/common.py index a7b45ca0f..c0d77a6b7 100644 --- a/src/allmydata/storage/common.py +++ b/src/allmydata/storage/common.py @@ -2,7 +2,7 @@ Ported to Python 3. """ -from future.utils import PY2, PY3 +from future.utils import PY3 import os.path from allmydata.util import base32 diff --git a/src/allmydata/storage/immutable.py b/src/allmydata/storage/immutable.py index 90d6d4d2d..7a61a1e62 100644 --- a/src/allmydata/storage/immutable.py +++ b/src/allmydata/storage/immutable.py @@ -3,7 +3,7 @@ Ported to Python 3. """ -from future.utils import PY2, bytes_to_native_str +from future.utils import bytes_to_native_str import os, stat, struct, time diff --git a/src/allmydata/test/cli/test_backup.py b/src/allmydata/test/cli/test_backup.py index 744d40f78..c951e7ee3 100644 --- a/src/allmydata/test/cli/test_backup.py +++ b/src/allmydata/test/cli/test_backup.py @@ -14,7 +14,7 @@ from twisted.python.monkey import MonkeyPatcher from allmydata.util import fileutil from allmydata.util.fileutil import abspath_expanduser_unicode -from allmydata.util.encodingutil import get_io_encoding, unicode_to_argv +from allmydata.util.encodingutil import unicode_to_argv from allmydata.util.namespace import Namespace from allmydata.scripts import cli, backupdb from ..common_util import StallMixin diff --git a/src/allmydata/test/cli/test_cp.py b/src/allmydata/test/cli/test_cp.py index 2dbc6d37e..2751dc055 100644 --- a/src/allmydata/test/cli/test_cp.py +++ b/src/allmydata/test/cli/test_cp.py @@ -9,8 +9,7 @@ from twisted.internet import defer from allmydata.scripts import cli from allmydata.util import fileutil -from allmydata.util.encodingutil import (quote_output, get_io_encoding, - unicode_to_output, to_bytes) +from allmydata.util.encodingutil import (quote_output, unicode_to_output, to_bytes) from allmydata.util.assertutil import _assert from ..no_network import GridTestMixin from .common import CLITestMixin diff --git a/src/allmydata/test/cli/test_list.py b/src/allmydata/test/cli/test_list.py index 24af04ba9..55e1d7cc1 100644 --- a/src/allmydata/test/cli/test_list.py +++ b/src/allmydata/test/cli/test_list.py @@ -2,7 +2,7 @@ Ported to Python 3. """ -from future.utils import PY2, PY3 +from future.utils import PY3 from six import ensure_str from twisted.trial import unittest diff --git a/src/allmydata/test/mutable/util.py b/src/allmydata/test/mutable/util.py index 11696a83d..5a2a6a8f8 100644 --- a/src/allmydata/test/mutable/util.py +++ b/src/allmydata/test/mutable/util.py @@ -2,7 +2,7 @@ Ported to Python 3. """ -from future.utils import PY2, bchr +from future.utils import bchr from past.builtins import long diff --git a/src/allmydata/test/test_crawler.py b/src/allmydata/test/test_crawler.py index 7a28c1d35..71b0f3dee 100644 --- a/src/allmydata/test/test_crawler.py +++ b/src/allmydata/test/test_crawler.py @@ -5,7 +5,7 @@ Ported to Python 3. """ -from future.utils import PY2, PY3 +from future.utils import PY3 import time import os.path diff --git a/src/allmydata/test/test_log.py b/src/allmydata/test/test_log.py index ea03ba730..c3671f9b9 100644 --- a/src/allmydata/test/test_log.py +++ b/src/allmydata/test/test_log.py @@ -5,7 +5,7 @@ Ported to Python 3. """ -from future.utils import PY2, native_str +from future.utils import native_str from twisted.trial import unittest from twisted.python.failure import Failure diff --git a/src/allmydata/test/test_runner.py b/src/allmydata/test/test_runner.py index e7cb4b082..f62afd0b0 100644 --- a/src/allmydata/test/test_runner.py +++ b/src/allmydata/test/test_runner.py @@ -46,9 +46,6 @@ from allmydata.util.pid import ( ) from allmydata.test import common_util import allmydata -from allmydata.scripts.runner import ( - parse_options, -) from allmydata.scripts.tahoe_run import ( on_stdin_close, ) diff --git a/src/allmydata/util/encodingutil.py b/src/allmydata/util/encodingutil.py index 0625bfdec..cf8d83a42 100644 --- a/src/allmydata/util/encodingutil.py +++ b/src/allmydata/util/encodingutil.py @@ -8,7 +8,7 @@ Once Python 2 support is dropped, most of this module will obsolete, since Unicode is the default everywhere in Python 3. """ -from future.utils import PY2, PY3, native_str +from future.utils import PY3, native_str from future.builtins import str as future_str from past.builtins import unicode diff --git a/src/allmydata/util/jsonbytes.py b/src/allmydata/util/jsonbytes.py index 0762702f9..7415b4f02 100644 --- a/src/allmydata/util/jsonbytes.py +++ b/src/allmydata/util/jsonbytes.py @@ -5,7 +5,6 @@ Ported to Python 3. """ import json -import codecs def bytes_to_unicode(any_bytes, obj): From 53084f76ced048e5516ce1b79d8a3f055eadd389 Mon Sep 17 00:00:00 2001 From: Alexandre Detiste Date: Wed, 28 Feb 2024 00:49:07 +0100 Subject: [PATCH 2268/2309] remove more Python2 compatibility --- misc/coding_tools/graph-deps.py | 2 +- src/allmydata/_monkeypatch.py | 8 -- src/allmydata/crypto/aes.py | 10 +-- src/allmydata/frontends/sftpd.py | 3 - src/allmydata/node.py | 4 +- src/allmydata/scripts/default_nodedir.py | 7 +- src/allmydata/scripts/runner.py | 6 +- src/allmydata/scripts/slow_operation.py | 6 +- src/allmydata/scripts/tahoe_check.py | 12 ++- src/allmydata/scripts/tahoe_get.py | 4 +- src/allmydata/scripts/tahoe_manifest.py | 10 +-- src/allmydata/storage/common.py | 6 +- src/allmydata/storage/crawler.py | 12 +-- src/allmydata/test/cli/test_backup.py | 9 +-- src/allmydata/test/cli/test_backupdb.py | 2 +- src/allmydata/test/cli/test_check.py | 2 +- src/allmydata/test/cli/test_cli.py | 2 +- src/allmydata/test/cli/test_create_alias.py | 2 +- src/allmydata/test/cli/test_list.py | 13 +-- src/allmydata/test/cli/test_run.py | 2 +- src/allmydata/test/common_util.py | 33 ++------ src/allmydata/test/mutable/test_filenode.py | 2 +- src/allmydata/test/mutable/test_roundtrip.py | 2 +- src/allmydata/test/test_common_util.py | 4 +- src/allmydata/test/test_crawler.py | 16 ++-- src/allmydata/test/test_crypto.py | 15 ++-- src/allmydata/test/test_download.py | 4 - src/allmydata/test/test_encodingutil.py | 53 +++--------- src/allmydata/test/test_i2p_provider.py | 4 +- src/allmydata/test/test_runner.py | 17 +--- src/allmydata/test/test_statistics.py | 2 +- src/allmydata/test/test_storage_web.py | 2 +- src/allmydata/test/test_tor_provider.py | 7 +- src/allmydata/test/test_util.py | 6 +- src/allmydata/test/web/test_grid.py | 2 +- src/allmydata/util/base62.py | 11 +-- src/allmydata/util/encodingutil.py | 59 ++------------ src/allmydata/util/hashutil.py | 9 +-- src/allmydata/util/log.py | 13 +-- src/allmydata/windows/fixups.py | 84 +------------------- 40 files changed, 96 insertions(+), 371 deletions(-) diff --git a/misc/coding_tools/graph-deps.py b/misc/coding_tools/graph-deps.py index 400f3912a..faa94450a 100755 --- a/misc/coding_tools/graph-deps.py +++ b/misc/coding_tools/graph-deps.py @@ -24,7 +24,7 @@ import os, sys, subprocess, json, tempfile, zipfile, re, itertools import email.parser from pprint import pprint -from six.moves import StringIO +from io import StringIO import click all_packages = {} # name -> version diff --git a/src/allmydata/_monkeypatch.py b/src/allmydata/_monkeypatch.py index cbd3ddd13..61f12750f 100644 --- a/src/allmydata/_monkeypatch.py +++ b/src/allmydata/_monkeypatch.py @@ -4,13 +4,5 @@ Monkey-patching of third party libraries. Ported to Python 3. """ -from future.utils import PY2 - - - def patch(): """Path third-party libraries to make Tahoe-LAFS work.""" - - if not PY2: - # Python 3 doesn't need to monkey patch Foolscap - return diff --git a/src/allmydata/crypto/aes.py b/src/allmydata/crypto/aes.py index 2fa5f8684..fd5617b1d 100644 --- a/src/allmydata/crypto/aes.py +++ b/src/allmydata/crypto/aes.py @@ -10,8 +10,6 @@ objects that `cryptography` documents. Ported to Python 3. """ -import six - from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives.ciphers import ( Cipher, @@ -79,7 +77,7 @@ def encrypt_data(encryptor, plaintext): """ _validate_cryptor(encryptor, encrypt=True) - if not isinstance(plaintext, six.binary_type): + if not isinstance(plaintext, bytes): raise ValueError('Plaintext must be bytes') return encryptor.update(plaintext) @@ -118,7 +116,7 @@ def decrypt_data(decryptor, plaintext): """ _validate_cryptor(decryptor, encrypt=False) - if not isinstance(plaintext, six.binary_type): + if not isinstance(plaintext, bytes): raise ValueError('Plaintext must be bytes') return decryptor.update(plaintext) @@ -160,7 +158,7 @@ def _validate_key(key): """ confirm `key` is suitable for AES encryption, or raise ValueError """ - if not isinstance(key, six.binary_type): + if not isinstance(key, bytes): raise TypeError('Key must be bytes') if len(key) not in (16, 32): raise ValueError('Key must be 16 or 32 bytes long') @@ -175,7 +173,7 @@ def _validate_iv(iv): """ if iv is None: return DEFAULT_IV - if not isinstance(iv, six.binary_type): + if not isinstance(iv, bytes): raise TypeError('IV must be bytes') if len(iv) != 16: raise ValueError('IV must be 16 bytes long') diff --git a/src/allmydata/frontends/sftpd.py b/src/allmydata/frontends/sftpd.py index eaa89160c..b775fa49d 100644 --- a/src/allmydata/frontends/sftpd.py +++ b/src/allmydata/frontends/sftpd.py @@ -45,9 +45,6 @@ noisy = True from allmydata.util.log import NOISY, OPERATIONAL, WEIRD, \ msg as logmsg, PrefixingLogMixin -if six.PY3: - long = int - def createSFTPError(errorCode, errorMessage): """ diff --git a/src/allmydata/node.py b/src/allmydata/node.py index fdb89e13f..601f64b93 100644 --- a/src/allmydata/node.py +++ b/src/allmydata/node.py @@ -112,8 +112,8 @@ def formatTimeTahoeStyle(self, when): """ d = datetime.datetime.utcfromtimestamp(when) if d.microsecond: - return d.isoformat(ensure_str(" "))[:-3]+"Z" - return d.isoformat(ensure_str(" ")) + ".000Z" + return d.isoformat(" ")[:-3]+"Z" + return d.isoformat(" ") + ".000Z" PRIV_README = """ This directory contains files which contain private data for the Tahoe node, diff --git a/src/allmydata/scripts/default_nodedir.py b/src/allmydata/scripts/default_nodedir.py index c1711b55c..fff120140 100644 --- a/src/allmydata/scripts/default_nodedir.py +++ b/src/allmydata/scripts/default_nodedir.py @@ -3,7 +3,6 @@ Ported to Python 3. """ import sys -import six from allmydata.util.assertutil import precondition from allmydata.util.fileutil import abspath_expanduser_unicode @@ -13,10 +12,10 @@ if sys.platform == 'win32': from allmydata.windows import registry path = registry.get_base_dir_path() if path: - precondition(isinstance(path, six.text_type), path) + precondition(isinstance(path, str), path) _default_nodedir = abspath_expanduser_unicode(path) if _default_nodedir is None: - path = abspath_expanduser_unicode(u"~/.tahoe") - precondition(isinstance(path, six.text_type), path) + path = abspath_expanduser_unicode("~/.tahoe") + precondition(isinstance(path, str), path) _default_nodedir = path diff --git a/src/allmydata/scripts/runner.py b/src/allmydata/scripts/runner.py index 73e2e4b59..16f43e9d8 100644 --- a/src/allmydata/scripts/runner.py +++ b/src/allmydata/scripts/runner.py @@ -65,8 +65,8 @@ class Options(usage.Options): ] optParameters = [ ["node-directory", "d", None, NODEDIR_HELP], - ["wormhole-server", None, u"ws://wormhole.tahoe-lafs.org:4000/v1", "The magic wormhole server to use.", six.text_type], - ["wormhole-invite-appid", None, u"tahoe-lafs.org/invite", "The appid to use on the wormhole server.", six.text_type], + ["wormhole-server", None, u"ws://wormhole.tahoe-lafs.org:4000/v1", "The magic wormhole server to use.", str], + ["wormhole-invite-appid", None, u"tahoe-lafs.org/invite", "The appid to use on the wormhole server.", str], ] def opt_version(self): @@ -262,7 +262,7 @@ def _setup_coverage(reactor, argv): # can we put this _setup_coverage call after we hit # argument-parsing? # ensure_str() only necessary on Python 2. - if six.ensure_str('--coverage') not in sys.argv: + if '--coverage' not in sys.argv: return argv.remove('--coverage') diff --git a/src/allmydata/scripts/slow_operation.py b/src/allmydata/scripts/slow_operation.py index 236c2bf0b..9596fd805 100644 --- a/src/allmydata/scripts/slow_operation.py +++ b/src/allmydata/scripts/slow_operation.py @@ -2,8 +2,6 @@ Ported to Python 3. """ -from future.utils import PY3 - from six import ensure_str import os, time @@ -81,9 +79,7 @@ class SlowOperationRunner(object): if not data["finished"]: return False if self.options.get("raw"): - if PY3: - # need to write bytes! - stdout = stdout.buffer + stdout = stdout.buffer if is_printable_ascii(jdata): stdout.write(jdata) stdout.write(b"\n") diff --git a/src/allmydata/scripts/tahoe_check.py b/src/allmydata/scripts/tahoe_check.py index 91125a3a3..c5ba07db9 100644 --- a/src/allmydata/scripts/tahoe_check.py +++ b/src/allmydata/scripts/tahoe_check.py @@ -2,7 +2,7 @@ Ported to Python 3. """ -from six import ensure_str, ensure_text +from six import ensure_text from urllib.parse import quote as url_quote import json @@ -168,7 +168,7 @@ class DeepCheckOutput(LineOnlyReceiver, object): # LIT files and directories do not have a "summary" field. summary = cr.get("summary", "Healthy (LIT)") # When Python 2 is dropped the ensure_text()/ensure_str() will be unnecessary. - print(ensure_text(ensure_str("%s: %s") % (quote_path(path), quote_output(summary, quotemarks=False)), + print(ensure_text("%s: %s" % (quote_path(path), quote_output(summary, quotemarks=False)), encoding=get_io_encoding()), file=stdout) # always print out corrupt shares @@ -246,13 +246,11 @@ class DeepCheckAndRepairOutput(LineOnlyReceiver, object): if not path: path = [""] # we don't seem to have a summary available, so build one - # When Python 2 is dropped the ensure_text/ensure_str crap can be - # dropped. if was_healthy: - summary = ensure_str("healthy") + summary = "healthy" else: - summary = ensure_str("not healthy") - print(ensure_text(ensure_str("%s: %s") % (quote_path(path), summary), + summary = "not healthy" + print(ensure_text("%s: %s" % (quote_path(path), summary), encoding=get_io_encoding()), file=stdout) # always print out corrupt shares diff --git a/src/allmydata/scripts/tahoe_get.py b/src/allmydata/scripts/tahoe_get.py index f22d3d293..8e688e432 100644 --- a/src/allmydata/scripts/tahoe_get.py +++ b/src/allmydata/scripts/tahoe_get.py @@ -2,8 +2,6 @@ Ported to Python 3. """ -from future.utils import PY3 - from urllib.parse import quote as url_quote from allmydata.scripts.common import get_alias, DEFAULT_ALIAS, escape_path, \ UnknownAliasError @@ -36,7 +34,7 @@ def get(options): outf = stdout # Make sure we can write bytes; on Python 3 stdout is Unicode by # default. - if PY3 and getattr(outf, "encoding", None) is not None: + if getattr(outf, "encoding", None) is not None: outf = outf.buffer while True: data = resp.read(4096) diff --git a/src/allmydata/scripts/tahoe_manifest.py b/src/allmydata/scripts/tahoe_manifest.py index 0cd6100bf..ebff2e893 100644 --- a/src/allmydata/scripts/tahoe_manifest.py +++ b/src/allmydata/scripts/tahoe_manifest.py @@ -2,10 +2,6 @@ Ported to Python 3. """ -from future.utils import PY3 - -from six import ensure_str - from urllib.parse import quote as url_quote import json from twisted.protocols.basic import LineOnlyReceiver @@ -56,8 +52,7 @@ class ManifestStreamer(LineOnlyReceiver, object): # use Twisted to split this into lines self.in_error = False # Writing bytes, so need binary stdout. - if PY3: - stdout = stdout.buffer + stdout = stdout.buffer while True: chunk = resp.read(100) if not chunk: @@ -99,8 +94,7 @@ class ManifestStreamer(LineOnlyReceiver, object): if vc: print(quote_output(vc, quotemarks=False), file=stdout) else: - # ensure_str() only necessary for Python 2. - print(ensure_str("%s %s") % ( + print("%s %s" % ( quote_output(d["cap"], quotemarks=False), quote_path(d["path"], quotemarks=False)), file=stdout) diff --git a/src/allmydata/storage/common.py b/src/allmydata/storage/common.py index c0d77a6b7..c76e01052 100644 --- a/src/allmydata/storage/common.py +++ b/src/allmydata/storage/common.py @@ -2,8 +2,6 @@ Ported to Python 3. """ -from future.utils import PY3 - import os.path from allmydata.util import base32 @@ -43,7 +41,5 @@ def storage_index_to_dir(storageindex): Returns native string. """ sia = si_b2a(storageindex) - if PY3: - # On Python 3 we expect paths to be unicode. - sia = sia.decode("ascii") + sia = sia.decode("ascii") return os.path.join(sia[:2], sia) diff --git a/src/allmydata/storage/crawler.py b/src/allmydata/storage/crawler.py index a464449ed..613f04bfb 100644 --- a/src/allmydata/storage/crawler.py +++ b/src/allmydata/storage/crawler.py @@ -4,9 +4,6 @@ Crawl the storage server shares. Ported to Python 3. """ - -from future.utils import PY2, PY3 - import os import time import json @@ -150,10 +147,7 @@ def _dump_json_to_file(js, afile): """ with afile.open("wb") as f: data = json.dumps(js) - if PY2: - f.write(data) - else: - f.write(data.encode("utf8")) + f.write(data.encode("utf8")) class _LeaseStateSerializer(object): @@ -249,9 +243,7 @@ class ShareCrawler(service.MultiService): self._state_serializer = _LeaseStateSerializer(statefile) self.prefixes = [si_b2a(struct.pack(">H", i << (16-10)))[:2] for i in range(2**10)] - if PY3: - # On Python 3 we expect the paths to be unicode, not bytes. - self.prefixes = [p.decode("ascii") for p in self.prefixes] + self.prefixes = [p.decode("ascii") for p in self.prefixes] self.prefixes.sort() self.timer = None self.bucket_cache = (None, []) diff --git a/src/allmydata/test/cli/test_backup.py b/src/allmydata/test/cli/test_backup.py index c951e7ee3..7ff1a14d0 100644 --- a/src/allmydata/test/cli/test_backup.py +++ b/src/allmydata/test/cli/test_backup.py @@ -2,10 +2,8 @@ Ported to Python 3. """ -from future.utils import PY2 - import os.path -from six.moves import cStringIO as StringIO +from io import StringIO from datetime import timedelta import re @@ -421,10 +419,7 @@ class Backup(GridTestMixin, CLITestMixin, StallMixin, unittest.TestCase): else: return original_open(name, *args, **kwargs) - if PY2: - from allmydata.scripts import cli as module_to_patch - else: - import builtins as module_to_patch + import builtins as module_to_patch patcher = MonkeyPatcher((module_to_patch, 'open', call_file)) patcher.runWithPatches(parse_options, basedir, "backup", ['--exclude-from-utf-8', unicode_to_argv(exclude_file), 'from', 'to']) self.failUnless(ns.called) diff --git a/src/allmydata/test/cli/test_backupdb.py b/src/allmydata/test/cli/test_backupdb.py index 359b06f4f..53cc3225a 100644 --- a/src/allmydata/test/cli/test_backupdb.py +++ b/src/allmydata/test/cli/test_backupdb.py @@ -4,7 +4,7 @@ Ported to Python 3. import sys import os.path, time -from six.moves import cStringIO as StringIO +from io import StringIO from twisted.trial import unittest from allmydata.util import fileutil diff --git a/src/allmydata/test/cli/test_check.py b/src/allmydata/test/cli/test_check.py index 56b36a8d5..c895451ea 100644 --- a/src/allmydata/test/cli/test_check.py +++ b/src/allmydata/test/cli/test_check.py @@ -3,7 +3,7 @@ from six import ensure_text import os.path import json from twisted.trial import unittest -from six.moves import cStringIO as StringIO +from io import StringIO from allmydata import uri from allmydata.util import base32 diff --git a/src/allmydata/test/cli/test_cli.py b/src/allmydata/test/cli/test_cli.py index ce51cb4f3..432437b61 100644 --- a/src/allmydata/test/cli/test_cli.py +++ b/src/allmydata/test/cli/test_cli.py @@ -2,7 +2,7 @@ Ported to Python 3. """ -from six.moves import cStringIO as StringIO +from io import StringIO import re from six import ensure_text diff --git a/src/allmydata/test/cli/test_create_alias.py b/src/allmydata/test/cli/test_create_alias.py index 862b2896a..02978deca 100644 --- a/src/allmydata/test/cli/test_create_alias.py +++ b/src/allmydata/test/cli/test_create_alias.py @@ -2,7 +2,7 @@ Ported to Python 3. """ -from six.moves import StringIO +from io import StringIO import os.path from twisted.trial import unittest from urllib.parse import quote as url_quote diff --git a/src/allmydata/test/cli/test_list.py b/src/allmydata/test/cli/test_list.py index 55e1d7cc1..f9ab52c53 100644 --- a/src/allmydata/test/cli/test_list.py +++ b/src/allmydata/test/cli/test_list.py @@ -2,9 +2,6 @@ Ported to Python 3. """ -from future.utils import PY3 -from six import ensure_str - from twisted.trial import unittest from twisted.internet import defer @@ -26,10 +23,6 @@ class List(GridTestMixin, CLITestMixin, unittest.TestCase): good_arg = u"g\u00F6\u00F6d" good_out = u"g\u00F6\u00F6d" - # On Python 2 we get bytes, so we need encoded version. On Python 3 - # stdio is unicode so can leave unchanged. - good_out_encoded = good_out if PY3 else good_out.encode(get_io_encoding()) - d = c0.create_dirnode() def _stash_root_and_create_file(n): self.rootnode = n @@ -52,7 +45,7 @@ class List(GridTestMixin, CLITestMixin, unittest.TestCase): (rc, out, err) = args self.failUnlessReallyEqual(rc, 0) self.assertEqual(len(err), 0, err) - expected = sorted([ensure_str("0share"), ensure_str("1share"), good_out_encoded]) + expected = sorted(["0share", "1share", good_out]) self.assertEqual(sorted(out.splitlines()), expected) d.addCallback(_check1) d.addCallback(lambda ign: self.do_cli("ls", "missing")) @@ -85,8 +78,8 @@ class List(GridTestMixin, CLITestMixin, unittest.TestCase): # listing a file (as dir/filename) should have the edge metadata, # including the filename self.failUnlessReallyEqual(rc, 0) - self.failUnlessIn(good_out_encoded, out) - self.failIfIn(ensure_str("-r-- %d -" % len(small)), out, + self.failUnlessIn(good_out, out) + self.failIfIn("-r-- %d -" % len(small), out, "trailing hyphen means unknown date") if good_arg is not None: diff --git a/src/allmydata/test/cli/test_run.py b/src/allmydata/test/cli/test_run.py index c7ebcd8c0..18ee8f67d 100644 --- a/src/allmydata/test/cli/test_run.py +++ b/src/allmydata/test/cli/test_run.py @@ -5,7 +5,7 @@ Tests for ``allmydata.scripts.tahoe_run``. from __future__ import annotations import re -from six.moves import ( +from io import ( StringIO, ) diff --git a/src/allmydata/test/common_util.py b/src/allmydata/test/common_util.py index fce2881b0..9b194f657 100644 --- a/src/allmydata/test/common_util.py +++ b/src/allmydata/test/common_util.py @@ -2,7 +2,7 @@ Ported to Python 3. """ -from future.utils import PY2, PY3, bchr, binary_type +from future.utils import bchr from future.builtins import str as future_str import os @@ -13,8 +13,6 @@ from functools import ( partial, ) from random import randrange -if PY2: - from StringIO import StringIO from io import ( TextIOWrapper, BytesIO, @@ -101,22 +99,7 @@ def run_cli_native(verb, *args, **kwargs): ) argv = ["tahoe"] + nodeargs + [verb] + list(args) stdin = kwargs.get("stdin", "") - if PY2: - # The original behavior, the Python 2 behavior, is to accept either - # bytes or unicode and try to automatically encode or decode as - # necessary. This works okay for ASCII and if LANG is set - # appropriately. These aren't great constraints so we should move - # away from this behavior. - # - # The encoding attribute doesn't change StringIO behavior on Python 2, - # but it's there for realism of the emulation. - stdin = StringIO(stdin) - stdin.encoding = encoding - stdout = StringIO() - stdout.encoding = encoding - stderr = StringIO() - stderr.encoding = encoding - else: + if True: # The new behavior, the Python 3 behavior, is to accept unicode and # encode it using a specific encoding. For older versions of Python 3, # the encoding is determined from LANG (bad) but for newer Python 3, @@ -146,13 +129,13 @@ def run_cli_native(verb, *args, **kwargs): stderr=stderr, ) def _done(rc, stdout=stdout, stderr=stderr): - if return_bytes and PY3: + if return_bytes: stdout = stdout.buffer stderr = stderr.buffer return 0, _getvalue(stdout), _getvalue(stderr) def _err(f, stdout=stdout, stderr=stderr): f.trap(SystemExit) - if return_bytes and PY3: + if return_bytes: stdout = stdout.buffer stderr = stderr.buffer return f.value.code, _getvalue(stdout), _getvalue(stderr) @@ -189,11 +172,7 @@ def run_cli_unicode(verb, argv, nodeargs=None, stdin=None, encoding=None): argv=argv, ) codec = encoding or "ascii" - if PY2: - encode = lambda t: None if t is None else t.encode(codec) - else: - # On Python 3 command-line parsing expects Unicode! - encode = lambda t: t + encode = lambda t: t d = run_cli_native( encode(verb), nodeargs=list(encode(arg) for arg in nodeargs), @@ -238,7 +217,7 @@ def flip_bit(good, which): def flip_one_bit(s, offset=0, size=None): """ flip one random bit of the string s, in a byte greater than or equal to offset and less than offset+size. """ - precondition(isinstance(s, binary_type)) + precondition(isinstance(s, bytes)) if size is None: size=len(s)-offset i = randrange(offset, offset+size) diff --git a/src/allmydata/test/mutable/test_filenode.py b/src/allmydata/test/mutable/test_filenode.py index 89881111a..82f1e5072 100644 --- a/src/allmydata/test/mutable/test_filenode.py +++ b/src/allmydata/test/mutable/test_filenode.py @@ -2,7 +2,7 @@ Ported to Python 3. """ -from six.moves import cStringIO as StringIO +from io import StringIO from twisted.internet import defer, reactor from ..common import AsyncBrokenTestCase from testtools.matchers import ( diff --git a/src/allmydata/test/mutable/test_roundtrip.py b/src/allmydata/test/mutable/test_roundtrip.py index 238e69c61..405219347 100644 --- a/src/allmydata/test/mutable/test_roundtrip.py +++ b/src/allmydata/test/mutable/test_roundtrip.py @@ -2,7 +2,7 @@ Ported to Python 3. """ -from six.moves import cStringIO as StringIO +from io import StringIO from ..common import AsyncTestCase from testtools.matchers import Equals, HasLength, Contains from twisted.internet import defer diff --git a/src/allmydata/test/test_common_util.py b/src/allmydata/test/test_common_util.py index 7f865d743..01982473a 100644 --- a/src/allmydata/test/test_common_util.py +++ b/src/allmydata/test/test_common_util.py @@ -2,8 +2,6 @@ This module has been ported to Python 3. """ -from future.utils import PY2 - import sys import random @@ -31,7 +29,7 @@ class TestFlipOneBit(SyncTestCase): def test_accepts_byte_string(self): actual = flip_one_bit(b'foo') - self.assertEqual(actual, b'fno' if PY2 else b'fom') + self.assertEqual(actual, b'fom') def test_rejects_unicode_string(self): self.assertRaises(AssertionError, flip_one_bit, u'foo') diff --git a/src/allmydata/test/test_crawler.py b/src/allmydata/test/test_crawler.py index 71b0f3dee..bf08828bd 100644 --- a/src/allmydata/test/test_crawler.py +++ b/src/allmydata/test/test_crawler.py @@ -5,8 +5,6 @@ Ported to Python 3. """ -from future.utils import PY3 - import time import os.path from twisted.trial import unittest @@ -28,10 +26,9 @@ class BucketEnumeratingCrawler(ShareCrawler): self.all_buckets = [] self.finished_d = defer.Deferred() def process_bucket(self, cycle, prefix, prefixdir, storage_index_b32): - if PY3: - # Bucket _inputs_ are bytes, and that's what we will compare this - # to: - storage_index_b32 = storage_index_b32.encode("ascii") + # Bucket _inputs_ are bytes, and that's what we will compare this + # to: + storage_index_b32 = storage_index_b32.encode("ascii") self.all_buckets.append(storage_index_b32) def finished_cycle(self, cycle): eventually(self.finished_d.callback, None) @@ -46,10 +43,9 @@ class PacedCrawler(ShareCrawler): self.finished_d = defer.Deferred() self.yield_cb = None def process_bucket(self, cycle, prefix, prefixdir, storage_index_b32): - if PY3: - # Bucket _inputs_ are bytes, and that's what we will compare this - # to: - storage_index_b32 = storage_index_b32.encode("ascii") + # Bucket _inputs_ are bytes, and that's what we will compare this + # to: + storage_index_b32 = storage_index_b32.encode("ascii") self.all_buckets.append(storage_index_b32) self.countdown -= 1 if self.countdown == 0: diff --git a/src/allmydata/test/test_crypto.py b/src/allmydata/test/test_crypto.py index aee9b4156..dd5b53e5f 100644 --- a/src/allmydata/test/test_crypto.py +++ b/src/allmydata/test/test_crypto.py @@ -1,6 +1,3 @@ - -from future.utils import native_bytes - import unittest from base64 import b64decode @@ -40,7 +37,7 @@ class TestRegression(unittest.TestCase): # priv_str = b64encode(priv.serialize()) # pub_str = b64encode(priv.get_verifying_key().serialize()) RSA_2048_PRIV_KEY = b64decode(f.read().strip()) - assert isinstance(RSA_2048_PRIV_KEY, native_bytes) + assert isinstance(RSA_2048_PRIV_KEY, bytes) with RESOURCE_DIR.child('pycryptopp-rsa-2048-sig.txt').open('r') as f: # Signature created using `RSA_2048_PRIV_KEY` via: @@ -61,7 +58,7 @@ class TestRegression(unittest.TestCase): # priv_str = b64encode(priv.serialize()) # pub_str = b64encode(priv.get_verifying_key().serialize()) RSA_TINY_PRIV_KEY = b64decode(f.read().strip()) - assert isinstance(RSA_TINY_PRIV_KEY, native_bytes) + assert isinstance(RSA_TINY_PRIV_KEY, bytes) with RESOURCE_DIR.child('pycryptopp-rsa-32768-priv.txt').open('r') as f: # Created using `pycryptopp`: @@ -72,7 +69,7 @@ class TestRegression(unittest.TestCase): # priv_str = b64encode(priv.serialize()) # pub_str = b64encode(priv.get_verifying_key().serialize()) RSA_HUGE_PRIV_KEY = b64decode(f.read().strip()) - assert isinstance(RSA_HUGE_PRIV_KEY, native_bytes) + assert isinstance(RSA_HUGE_PRIV_KEY, bytes) def test_old_start_up_test(self): """ @@ -324,7 +321,7 @@ class TestEd25519(unittest.TestCase): private_key, public_key = ed25519.create_signing_keypair() private_key_str = ed25519.string_from_signing_key(private_key) - self.assertIsInstance(private_key_str, native_bytes) + self.assertIsInstance(private_key_str, bytes) private_key2, public_key2 = ed25519.signing_keypair_from_string(private_key_str) @@ -340,7 +337,7 @@ class TestEd25519(unittest.TestCase): # ditto, but for the verifying keys public_key_str = ed25519.string_from_verifying_key(public_key) - self.assertIsInstance(public_key_str, native_bytes) + self.assertIsInstance(public_key_str, bytes) public_key2 = ed25519.verifying_key_from_string(public_key_str) self.assertEqual( @@ -444,7 +441,7 @@ class TestRsa(unittest.TestCase): priv_key, pub_key = rsa.create_signing_keypair(2048) priv_key_str = rsa.der_string_from_signing_key(priv_key) - self.assertIsInstance(priv_key_str, native_bytes) + self.assertIsInstance(priv_key_str, bytes) priv_key2, pub_key2 = rsa.create_signing_keypair_from_string(priv_key_str) diff --git a/src/allmydata/test/test_download.py b/src/allmydata/test/test_download.py index 4d57fa828..709786f0e 100644 --- a/src/allmydata/test/test_download.py +++ b/src/allmydata/test/test_download.py @@ -10,7 +10,6 @@ from future.utils import bchr from typing import Any -import six import os from twisted.trial import unittest from twisted.internet import defer, reactor @@ -30,9 +29,6 @@ from allmydata.immutable.downloader.fetcher import SegmentFetcher from allmydata.codec import CRSDecoder from foolscap.eventual import eventually, fireEventually, flushEventualQueue -if six.PY3: - long = int - plaintext = b"This is a moderate-sized file.\n" * 10 mutable_plaintext = b"This is a moderate-sized mutable file.\n" * 10 diff --git a/src/allmydata/test/test_encodingutil.py b/src/allmydata/test/test_encodingutil.py index 64b9a243d..4e16ef2b7 100644 --- a/src/allmydata/test/test_encodingutil.py +++ b/src/allmydata/test/test_encodingutil.py @@ -1,8 +1,4 @@ -from future.utils import PY2, PY3 - -from past.builtins import unicode - lumiere_nfc = u"lumi\u00E8re" Artonwall_nfc = u"\u00C4rtonwall.mp3" Artonwall_nfd = u"A\u0308rtonwall.mp3" @@ -46,13 +42,7 @@ if __name__ == "__main__": for fname in TEST_FILENAMES: open(os.path.join(tmpdir, fname), 'w').close() - # On Python 2, listing directories returns unicode under Windows or - # MacOS X if the input is unicode. On Python 3, it always returns - # Unicode. - if PY2 and sys.platform in ('win32', 'darwin'): - dirlist = os.listdir(unicode(tmpdir)) - else: - dirlist = os.listdir(tmpdir) + dirlist = os.listdir(tmpdir) print(" dirlist = %s" % repr(dirlist)) except: @@ -64,7 +54,6 @@ if __name__ == "__main__": import os, sys -from unittest import skipIf from twisted.trial import unittest @@ -87,15 +76,6 @@ class MockStdout(object): # The following tests apply only to platforms that don't store filenames as # Unicode entities on the filesystem. class EncodingUtilNonUnicodePlatform(unittest.TestCase): - @skipIf(PY3, "Python 3 is always Unicode, regardless of OS.") - def setUp(self): - # Make sure everything goes back to the way it was at the end of the - # test. - self.addCleanup(_reload) - - # Mock sys.platform because unicode_platform() uses it. Cleanups run - # in reverse order so we do this second so it gets undone first. - self.patch(sys, "platform", "linux") def test_listdir_unicode(self): # What happens if latin1-encoded filenames are encountered on an UTF-8 @@ -143,10 +123,7 @@ class EncodingUtil(ReallyEqualMixin): converts to bytes using UTF-8 elsewhere. """ result = unicode_to_argv(lumiere_nfc) - if PY3 or self.platform == "win32": - expected_value = lumiere_nfc - else: - expected_value = lumiere_nfc.encode(self.io_encoding) + expected_value = lumiere_nfc self.assertIsInstance(result, type(expected_value)) self.assertEqual(result, expected_value) @@ -167,13 +144,10 @@ class EncodingUtil(ReallyEqualMixin): % (self.filesystem_encoding,)) def call_os_listdir(path): - if PY2: - return self.dirlist - else: - # Python 3 always lists unicode filenames: - return [d.decode(self.filesystem_encoding) if isinstance(d, bytes) - else d - for d in self.dirlist] + # Python 3 always lists unicode filenames: + return [d.decode(self.filesystem_encoding) if isinstance(d, bytes) + else d + for d in self.dirlist] self.patch(os, 'listdir', call_os_listdir) @@ -204,10 +178,7 @@ class StdlibUnicode(unittest.TestCase): fn = lumiere_nfc + u'/' + lumiere_nfc + u'.txt' open(fn, 'wb').close() self.failUnless(os.path.exists(fn)) - if PY2: - getcwdu = os.getcwdu - else: - getcwdu = os.getcwd + getcwdu = os.getcwd self.failUnless(os.path.exists(os.path.join(getcwdu(), fn))) filenames = listdir_unicode(lumiere_nfc) @@ -237,7 +208,7 @@ class QuoteOutput(ReallyEqualMixin, unittest.TestCase): _reload() def _check(self, inp, out, enc, optional_quotes, quote_newlines): - if PY3 and isinstance(out, bytes): + if isinstance(out, bytes): out = out.decode(enc or encodingutil.io_encoding) out2 = out if optional_quotes: @@ -266,9 +237,7 @@ class QuoteOutput(ReallyEqualMixin, unittest.TestCase): def _test_quote_output_all(self, enc): def check(inp, out, optional_quotes=False, quote_newlines=None): - if PY3: - # Result is always Unicode on Python 3 - out = out.decode("ascii") + out = out.decode("ascii") self._check(inp, out, enc, optional_quotes, quote_newlines) # optional single quotes @@ -354,9 +323,7 @@ def win32_other(win32, other): class QuotePaths(ReallyEqualMixin, unittest.TestCase): def assertPathsEqual(self, actual, expected): - if PY3: - # On Python 3, results should be unicode: - expected = expected.decode("ascii") + expected = expected.decode("ascii") self.failUnlessReallyEqual(actual, expected) def test_quote_path(self): diff --git a/src/allmydata/test/test_i2p_provider.py b/src/allmydata/test/test_i2p_provider.py index 3b99646bf..f470e77af 100644 --- a/src/allmydata/test/test_i2p_provider.py +++ b/src/allmydata/test/test_i2p_provider.py @@ -6,8 +6,8 @@ import os from twisted.trial import unittest from twisted.internet import defer, error from twisted.python.usage import UsageError -from six.moves import StringIO -import mock +from io import StringIO +from unittest import mock from ..util import i2p_provider from ..scripts import create_node, runner diff --git a/src/allmydata/test/test_runner.py b/src/allmydata/test/test_runner.py index f62afd0b0..bc55d507d 100644 --- a/src/allmydata/test/test_runner.py +++ b/src/allmydata/test/test_runner.py @@ -2,10 +2,6 @@ Ported to Python 3 """ -from future.utils import PY2 - -from six import ensure_text - import os.path, re, sys from os import linesep import locale @@ -129,18 +125,14 @@ def run_bintahoe(extra_argv, python_options=None): :return: A three-tuple of stdout (unicode), stderr (unicode), and the child process "returncode" (int). """ - executable = ensure_text(sys.executable) - argv = [executable] + argv = [sys.executable] if python_options is not None: argv.extend(python_options) argv.extend([u"-b", u"-m", u"allmydata.scripts.runner"]) argv.extend(extra_argv) argv = list(unicode_to_argv(arg) for arg in argv) p = Popen(argv, stdout=PIPE, stderr=PIPE) - if PY2: - encoding = "utf-8" - else: - encoding = locale.getpreferredencoding(False) + encoding = locale.getpreferredencoding(False) out = p.stdout.read().decode(encoding) err = p.stderr.read().decode(encoding) returncode = p.wait() @@ -154,10 +146,7 @@ class BinTahoe(common_util.SignalMixin, unittest.TestCase): """ tricky = u"\u00F6" out, err, returncode = run_bintahoe([tricky]) - if PY2: - expected = u"Unknown command: \\xf6" - else: - expected = u"Unknown command: \xf6" + expected = u"Unknown command: \xf6" self.assertEqual(returncode, 1) self.assertIn( expected, diff --git a/src/allmydata/test/test_statistics.py b/src/allmydata/test/test_statistics.py index 2442e14a2..5a382e686 100644 --- a/src/allmydata/test/test_statistics.py +++ b/src/allmydata/test/test_statistics.py @@ -4,7 +4,7 @@ Tests for allmydata.util.statistics. Ported to Python 3. """ -from six.moves import StringIO # native string StringIO +from io import StringIO from twisted.trial import unittest diff --git a/src/allmydata/test/test_storage_web.py b/src/allmydata/test/test_storage_web.py index e9bca2183..71d26af54 100644 --- a/src/allmydata/test/test_storage_web.py +++ b/src/allmydata/test/test_storage_web.py @@ -9,7 +9,7 @@ import os.path import re import json from unittest import skipIf -from six.moves import StringIO +from io import StringIO from twisted.trial import unittest from twisted.internet import defer diff --git a/src/allmydata/test/test_tor_provider.py b/src/allmydata/test/test_tor_provider.py index fc29e41e4..e31a8586b 100644 --- a/src/allmydata/test/test_tor_provider.py +++ b/src/allmydata/test/test_tor_provider.py @@ -5,9 +5,8 @@ Ported to Python 3. import os from twisted.trial import unittest from twisted.internet import defer, error -from six.moves import StringIO -from six import ensure_str -import mock +from io import StringIO +from unittest import mock from ..util import tor_provider from ..scripts import create_node, runner from foolscap.eventual import flushEventualQueue @@ -185,7 +184,7 @@ class CreateOnion(unittest.TestCase): txtorcon = mock.Mock() ehs = mock.Mock() # This appears to be a native string in the real txtorcon object... - ehs.private_key = ensure_str("privkey") + ehs.private_key = "privkey" ehs.hostname = "ONION.onion" txtorcon.EphemeralHiddenService = mock.Mock(return_value=ehs) ehs.add_to_tor = mock.Mock(return_value=defer.succeed(None)) diff --git a/src/allmydata/test/test_util.py b/src/allmydata/test/test_util.py index b6056b7e4..07a2bfb59 100644 --- a/src/allmydata/test/test_util.py +++ b/src/allmydata/test/test_util.py @@ -2,7 +2,6 @@ Ported to Python3. """ -import six import os, time, sys import yaml import json @@ -22,8 +21,7 @@ from allmydata.util.cputhreadpool import defer_to_thread, disable_thread_pool_fo from allmydata.test.common_util import ReallyEqualMixin from .no_network import fireNow, LocalWrapper -if six.PY3: - long = int +long = int class IDLib(unittest.TestCase): @@ -477,7 +475,7 @@ class YAML(unittest.TestCase): Unicode and (ASCII) native strings get roundtripped to Unicode strings. """ data = yaml.safe_dump( - [six.ensure_str("str"), u"unicode", u"\u1234nicode"] + ["str", "unicode", "\u1234nicode"] ) back = yamlutil.safe_load(data) self.assertIsInstance(back[0], str) diff --git a/src/allmydata/test/web/test_grid.py b/src/allmydata/test/web/test_grid.py index 86404a7bf..c782733f9 100644 --- a/src/allmydata/test/web/test_grid.py +++ b/src/allmydata/test/web/test_grid.py @@ -5,7 +5,7 @@ Ported to Python 3. import os.path, re from urllib.parse import quote as url_quote import json -from six.moves import StringIO +from io import StringIO from bs4 import BeautifulSoup diff --git a/src/allmydata/util/base62.py b/src/allmydata/util/base62.py index 2c4425562..3602ef0ef 100644 --- a/src/allmydata/util/base62.py +++ b/src/allmydata/util/base62.py @@ -4,15 +4,8 @@ Base62 encoding. Ported to Python 3. """ -from future.utils import PY2 - -if PY2: - import string - maketrans = string.maketrans - translate = string.translate -else: - maketrans = bytes.maketrans - translate = bytes.translate +maketrans = bytes.maketrans +translate = bytes.translate from past.builtins import chr as byteschr diff --git a/src/allmydata/util/encodingutil.py b/src/allmydata/util/encodingutil.py index cf8d83a42..9bf906ad7 100644 --- a/src/allmydata/util/encodingutil.py +++ b/src/allmydata/util/encodingutil.py @@ -8,7 +8,7 @@ Once Python 2 support is dropped, most of this module will obsolete, since Unicode is the default everywhere in Python 3. """ -from future.utils import PY3, native_str +from future.utils import native_str from future.builtins import str as future_str from past.builtins import unicode @@ -56,25 +56,13 @@ def check_encoding(encoding): io_encoding = "utf-8" filesystem_encoding = None -is_unicode_platform = False -use_unicode_filepath = False +is_unicode_platform = True +use_unicode_filepath = True def _reload(): - global filesystem_encoding, is_unicode_platform, use_unicode_filepath - + global filesystem_encoding filesystem_encoding = canonical_encoding(sys.getfilesystemencoding()) check_encoding(filesystem_encoding) - is_unicode_platform = PY3 or sys.platform in ["win32", "darwin"] - - # Despite the Unicode-mode FilePath support added to Twisted in - # , we can't yet use - # Unicode-mode FilePaths with INotify on non-Windows platforms due to - # . Supposedly 7928 is fixed, - # though... and Tahoe-LAFS doesn't use inotify anymore! - # - # In the interest of not breaking anything, this logic is unchanged for - # Python 2, but on Python 3 the paths are always unicode, like it or not. - use_unicode_filepath = PY3 or sys.platform == "win32" _reload() @@ -128,9 +116,7 @@ def unicode_to_argv(s): Windows, this returns the input unmodified. """ precondition(isinstance(s, unicode), s) - if PY3: - warnings.warn("This will be unnecessary once Python 2 is dropped.", - DeprecationWarning) + warnings.warn("This is unnecessary.", DeprecationWarning) if sys.platform == "win32": return s return ensure_str(s) @@ -184,24 +170,8 @@ def unicode_to_output(s): the responsibility of stdout/stderr, they expect Unicode by default. """ precondition(isinstance(s, unicode), s) - if PY3: - warnings.warn("This will be unnecessary once Python 2 is dropped.", - DeprecationWarning) - return s - - try: - out = s.encode(io_encoding) - except (UnicodeEncodeError, UnicodeDecodeError): - raise UnicodeEncodeError(native_str(io_encoding), s, 0, 0, - native_str("A string could not be encoded as %s for output to the terminal:\n%r" % - (io_encoding, repr(s)))) - - if PRINTABLE_8BIT.search(out) is None: - raise UnicodeEncodeError(native_str(io_encoding), s, 0, 0, - native_str("A string encoded as %s for output to the terminal contained unsafe bytes:\n%r" % - (io_encoding, repr(s)))) - return out - + warnings.warn("This is unnecessary.", DeprecationWarning) + return s def _unicode_escape(m, quote_newlines): u = m.group(0) @@ -303,20 +273,7 @@ def quote_output(s, quotemarks=True, quote_newlines=None, encoding=None): return b'"%s"' % (escaped.encode(encoding, 'backslashreplace'),) result = _encode(s) - if PY3: - # On Python 3 half of what this function does is unnecessary, since - # sys.stdout typically expects Unicode. To ensure no encode errors, one - # can do: - # - # sys.stdout.reconfigure(encoding=sys.stdout.encoding, errors="backslashreplace") - # - # Although the problem is that doesn't work in Python 3.6, only 3.7 or - # later... For now not thinking about it, just returning unicode since - # that is the right thing to do on Python 3. - # - # Now that Python 3.7 is the minimum, this can in theory be done: - # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3866 - result = result.decode(encoding) + result = result.decode(encoding) return result diff --git a/src/allmydata/util/hashutil.py b/src/allmydata/util/hashutil.py index 579f55d88..7217a2d93 100644 --- a/src/allmydata/util/hashutil.py +++ b/src/allmydata/util/hashutil.py @@ -4,13 +4,6 @@ Hashing utilities. Ported to Python 3. """ -from future.utils import PY2 -if PY2: - # Don't import bytes to prevent leaking future's bytes. - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, dict, list, object, range, str, max, min, bytes as future_bytes # noqa: F401 -else: - future_bytes = bytes - from past.builtins import chr as byteschr import os @@ -246,7 +239,7 @@ def bucket_cancel_secret_hash(file_cancel_secret, peerid): def _xor(a, b): - return b"".join([byteschr(c ^ b) for c in future_bytes(a)]) + return b"".join([byteschr(c ^ b) for c in bytes(a)]) def hmac(tag, data): diff --git a/src/allmydata/util/log.py b/src/allmydata/util/log.py index 3589bc366..65df01dfc 100644 --- a/src/allmydata/util/log.py +++ b/src/allmydata/util/log.py @@ -4,7 +4,6 @@ Logging utilities. Ported to Python 3. """ -from future.utils import PY2 from six import ensure_str from pyutil import nummedobj @@ -12,14 +11,10 @@ from pyutil import nummedobj from foolscap.logging import log from twisted.python import log as tw_log -if PY2: - def bytes_to_unicode(ign, obj): - return obj -else: - # We want to convert bytes keys to Unicode, otherwise JSON serialization - # inside foolscap will fail (for details see - # https://github.com/warner/foolscap/issues/88) - from .jsonbytes import bytes_to_unicode +# We want to convert bytes keys to Unicode, otherwise JSON serialization +# inside foolscap will fail (for details see +# https://github.com/warner/foolscap/issues/88) +from .jsonbytes import bytes_to_unicode NOISY = log.NOISY # 10 diff --git a/src/allmydata/windows/fixups.py b/src/allmydata/windows/fixups.py index 87e32a10f..c48db2f82 100644 --- a/src/allmydata/windows/fixups.py +++ b/src/allmydata/windows/fixups.py @@ -1,7 +1,3 @@ - -from future.utils import PY3 -from past.builtins import unicode - # This code isn't loadable or sensible except on Windows. Importers all know # this and are careful. Normally I would just let an import error from ctypes # explain any mistakes but Mypy also needs some help here. This assert @@ -123,82 +119,6 @@ def initialize(): SetErrorMode(SEM_FAILCRITICALERRORS | SEM_NOOPENFILEERRORBOX) - if PY3: - # The rest of this appears to be Python 2-specific - return - - original_stderr = sys.stderr - - # If any exception occurs in this code, we'll probably try to print it on stderr, - # which makes for frustrating debugging if stderr is directed to our wrapper. - # So be paranoid about catching errors and reporting them to original_stderr, - # so that we can at least see them. - def _complain(output_file, message): - print(isinstance(message, str) and message or repr(message), file=output_file) - log.msg(message, level=log.WEIRD) - - _complain = partial(_complain, original_stderr) - - # Work around . - codecs.register(lambda name: name == 'cp65001' and codecs.lookup('utf-8') or None) - - # Make Unicode console output work independently of the current code page. - # This also fixes . - # Credit to Michael Kaplan - # and TZOmegaTZIOY - # . - try: - old_stdout_fileno = None - old_stderr_fileno = None - if hasattr(sys.stdout, 'fileno'): - old_stdout_fileno = sys.stdout.fileno() - if hasattr(sys.stderr, 'fileno'): - old_stderr_fileno = sys.stderr.fileno() - - real_stdout = (old_stdout_fileno == STDOUT_FILENO) - real_stderr = (old_stderr_fileno == STDERR_FILENO) - - if real_stdout: - hStdout = GetStdHandle(STD_OUTPUT_HANDLE) - if not a_console(hStdout): - real_stdout = False - - if real_stderr: - hStderr = GetStdHandle(STD_ERROR_HANDLE) - if not a_console(hStderr): - real_stderr = False - - if real_stdout: - sys.stdout = UnicodeOutput(hStdout, None, STDOUT_FILENO, '', _complain) - else: - sys.stdout = UnicodeOutput(None, sys.stdout, old_stdout_fileno, '', _complain) - - if real_stderr: - sys.stderr = UnicodeOutput(hStderr, None, STDERR_FILENO, '', _complain) - else: - sys.stderr = UnicodeOutput(None, sys.stderr, old_stderr_fileno, '', _complain) - except Exception as e: - _complain("exception %r while fixing up sys.stdout and sys.stderr" % (e,)) - - argv = list(arg.encode("utf-8") for arg in get_argv()) - - # Take only the suffix with the same number of arguments as sys.argv. - # This accounts for anything that can cause initial arguments to be stripped, - # for example, the Python interpreter or any options passed to it, or runner - # scripts such as 'coverage run'. It works even if there are no such arguments, - # as in the case of a frozen executable created by bb-freeze or similar. - # - # Also, modify sys.argv in place. If any code has already taken a - # reference to the original argument list object then this ensures that - # code sees the new values. This reliance on mutation of shared state is, - # of course, awful. Why does this function even modify sys.argv? Why not - # have a function that *returns* the properly initialized argv as a new - # list? I don't know. - # - # At least Python 3 gets sys.argv correct so before very much longer we - # should be able to fix this bad design by deleting it. - sys.argv[:] = argv[-len(sys.argv):] - def a_console(handle): """ @@ -274,13 +194,13 @@ class UnicodeOutput(object): # There is no Windows console available. That means we are # responsible for encoding the unicode to a byte string to # write it to a Python file object. - if isinstance(text, unicode): + if isinstance(text, str): text = text.encode('utf-8') self._stream.write(text) else: # There is a Windows console available. That means Windows is # responsible for dealing with the unicode itself. - if not isinstance(text, unicode): + if not isinstance(text, str): text = str(text).decode('utf-8') remaining = len(text) while remaining > 0: From 0e5b6daa3890978f67043e9555f3b3b21946c59c Mon Sep 17 00:00:00 2001 From: Alexandre Detiste Date: Wed, 28 Feb 2024 00:53:49 +0100 Subject: [PATCH 2269/2309] remove more Python2: unicode -> str, long -> int --- src/allmydata/check_results.py | 4 +--- src/allmydata/immutable/upload.py | 11 +++++------ src/allmydata/scripts/tahoe_status.py | 3 +-- src/allmydata/uri.py | 16 +++++++--------- 4 files changed, 14 insertions(+), 20 deletions(-) diff --git a/src/allmydata/check_results.py b/src/allmydata/check_results.py index 8c44fd71e..e9ef59bdb 100644 --- a/src/allmydata/check_results.py +++ b/src/allmydata/check_results.py @@ -1,8 +1,6 @@ """Ported to Python 3. """ -from past.builtins import unicode - from zope.interface import implementer from allmydata.interfaces import ICheckResults, ICheckAndRepairResults, \ IDeepCheckResults, IDeepCheckAndRepairResults, IURI, IDisplayableServer @@ -64,7 +62,7 @@ class CheckResults(object): # unicode. if isinstance(summary, bytes): summary = unicode(summary, "utf-8") - assert isinstance(summary, unicode) # should be a single string + assert isinstance(summary, str) # should be a single string self._summary = summary assert not isinstance(report, str) # should be list of strings self._report = report diff --git a/src/allmydata/immutable/upload.py b/src/allmydata/immutable/upload.py index 22210ad0a..c13970033 100644 --- a/src/allmydata/immutable/upload.py +++ b/src/allmydata/immutable/upload.py @@ -5,7 +5,6 @@ Ported to Python 3. from __future__ import annotations from future.utils import native_str -from past.builtins import long, unicode from six import ensure_str import os, time, weakref, itertools @@ -57,7 +56,7 @@ from eliot import ( _TOTAL_SHARES = Field.for_types( u"total_shares", - [int, long], + [int, int], u"The total number of shares desired.", ) @@ -104,7 +103,7 @@ _HAPPINESS_MAPPINGS = Field( _HAPPINESS = Field.for_types( u"happiness", - [int, long], + [int, int], u"The computed happiness of a certain placement.", ) @@ -142,7 +141,7 @@ GET_SHARE_PLACEMENTS = MessageType( _EFFECTIVE_HAPPINESS = Field.for_types( u"effective_happiness", - [int, long], + [int, int], u"The computed happiness value of a share placement map.", ) @@ -1622,7 +1621,7 @@ class AssistedUploader(object): # abbreviated), so if we detect old results, just clobber them. sharemap = upload_results.sharemap - if any(isinstance(v, (bytes, unicode)) for v in sharemap.values()): + if any(isinstance(v, (bytes, str)) for v in sharemap.values()): upload_results.sharemap = None def _build_verifycap(self, helper_upload_results): @@ -1701,7 +1700,7 @@ class BaseUploadable(object): def set_default_encoding_parameters(self, default_params): assert isinstance(default_params, dict) for k,v in default_params.items(): - precondition(isinstance(k, (bytes, unicode)), k, v) + precondition(isinstance(k, (bytes, str)), k, v) precondition(isinstance(v, int), k, v) if "k" in default_params: self.default_encoding_param_k = default_params["k"] diff --git a/src/allmydata/scripts/tahoe_status.py b/src/allmydata/scripts/tahoe_status.py index c7f19910b..ef8da35c0 100644 --- a/src/allmydata/scripts/tahoe_status.py +++ b/src/allmydata/scripts/tahoe_status.py @@ -24,13 +24,12 @@ def print(*args, **kwargs): encoding error handler and then write the result whereas builtin print uses the "strict" encoding error handler. """ - from past.builtins import unicode out = kwargs.pop("file", None) if out is None: out = _sys_stdout encoding = out.encoding or "ascii" def ensafe(o): - if isinstance(o, unicode): + if isinstance(o, str): return o.encode(encoding, errors="replace").decode(encoding) return o return _print( diff --git a/src/allmydata/uri.py b/src/allmydata/uri.py index fccf05db9..34f245ac7 100644 --- a/src/allmydata/uri.py +++ b/src/allmydata/uri.py @@ -7,8 +7,6 @@ Methods ending in to_string() are actually to_bytes(), possibly should be fixed in follow-up port. """ -from past.builtins import unicode, long - import re from typing import Type @@ -91,7 +89,7 @@ class CHKFileURI(_BaseURI): def to_string(self): assert isinstance(self.needed_shares, int) assert isinstance(self.total_shares, int) - assert isinstance(self.size, (int,long)) + assert isinstance(self.size, int) return (b'URI:CHK:%s:%s:%d:%d:%d' % (base32.b2a(self.key), @@ -147,7 +145,7 @@ class CHKFileVerifierURI(_BaseURI): def to_string(self): assert isinstance(self.needed_shares, int) assert isinstance(self.total_shares, int) - assert isinstance(self.size, (int,long)) + assert isinstance(self.size, int) return (b'URI:CHK-Verifier:%s:%s:%d:%d:%d' % (si_b2a(self.storage_index), @@ -742,7 +740,7 @@ ALLEGED_IMMUTABLE_PREFIX = b'imm.' def from_string(u, deep_immutable=False, name=u""): """Create URI from either unicode or byte string.""" - if isinstance(u, unicode): + if isinstance(u, str): u = u.encode("utf-8") if not isinstance(u, bytes): raise TypeError("URI must be unicode string or bytes: %r" % (u,)) @@ -844,7 +842,7 @@ def is_uri(s): return False def is_literal_file_uri(s): - if isinstance(s, unicode): + if isinstance(s, str): s = s.encode("utf-8") if not isinstance(s, bytes): return False @@ -853,7 +851,7 @@ def is_literal_file_uri(s): s.startswith(ALLEGED_IMMUTABLE_PREFIX + b'URI:LIT:')) def has_uri_prefix(s): - if isinstance(s, unicode): + if isinstance(s, str): s = s.encode("utf-8") if not isinstance(s, bytes): return False @@ -895,9 +893,9 @@ def pack_extension(data): pieces = [] for k in sorted(data.keys()): value = data[k] - if isinstance(value, (int, long)): + if isinstance(value, int): value = b"%d" % value - if isinstance(k, unicode): + if isinstance(k, str): k = k.encode("utf-8") assert isinstance(value, bytes), k assert re.match(br'^[a-zA-Z_\-]+$', k) From 2243ce3187f3519ff2e92a46202e983f1fdcb91d Mon Sep 17 00:00:00 2001 From: Alexandre Detiste Date: Wed, 28 Feb 2024 01:07:08 +0100 Subject: [PATCH 2270/2309] remove "from past.builtins import long" --- src/allmydata/interfaces.py | 3 +-- src/allmydata/introducer/client.py | 4 +--- src/allmydata/introducer/server.py | 3 +-- src/allmydata/test/mutable/util.py | 6 ++---- src/allmydata/test/test_dirnode.py | 4 +--- src/allmydata/test/test_humanreadable.py | 4 +--- src/allmydata/test/test_spans.py | 11 +++-------- src/allmydata/test/test_time_format.py | 4 +--- src/allmydata/util/netstring.py | 4 +--- src/allmydata/web/status.py | 4 +--- 10 files changed, 13 insertions(+), 34 deletions(-) diff --git a/src/allmydata/interfaces.py b/src/allmydata/interfaces.py index 96fccb958..1cfbb577e 100644 --- a/src/allmydata/interfaces.py +++ b/src/allmydata/interfaces.py @@ -8,7 +8,6 @@ Note that for RemoteInterfaces, the __remote_name__ needs to be a native string from future.utils import native_str -from past.builtins import long from typing import Dict from zope.interface import Interface, Attribute @@ -2774,7 +2773,7 @@ class RIEncryptedUploadable(RemoteInterface): return Offset def get_all_encoding_parameters(): - return (int, int, int, long) + return (int, int, int, int) def read_encrypted(offset=Offset, length=ReadSize): return ListOf(bytes) diff --git a/src/allmydata/introducer/client.py b/src/allmydata/introducer/client.py index e6eab3b9f..fd605dd1d 100644 --- a/src/allmydata/introducer/client.py +++ b/src/allmydata/introducer/client.py @@ -2,8 +2,6 @@ Ported to Python 3. """ -from past.builtins import long - from six import ensure_text, ensure_str import time @@ -304,7 +302,7 @@ class IntroducerClient(service.Service, Referenceable): if "seqnum" in old: # must beat previous sequence number to replace if ("seqnum" not in ann - or not isinstance(ann["seqnum"], (int,long))): + or not isinstance(ann["seqnum"], int)): self.log("not replacing old announcement, no valid seqnum: %s" % (ann,), parent=lp2, level=log.NOISY, umid="zFGH3Q") diff --git a/src/allmydata/introducer/server.py b/src/allmydata/introducer/server.py index 157a1b73c..10048c55e 100644 --- a/src/allmydata/introducer/server.py +++ b/src/allmydata/introducer/server.py @@ -4,7 +4,6 @@ Ported to Python 3. from __future__ import annotations -from past.builtins import long from six import ensure_text import time, os.path, textwrap @@ -262,7 +261,7 @@ class IntroducerService(service.MultiService, Referenceable): # type: ignore[mi if "seqnum" in old_ann: # must beat previous sequence number to replace if ("seqnum" not in ann - or not isinstance(ann["seqnum"], (int,long))): + or not isinstance(ann["seqnum"], int)): self.log("not replacing old ann, no valid seqnum", level=log.NOISY, umid="ySbaVw") self._debug_counts["inbound_no_seqnum"] += 1 diff --git a/src/allmydata/test/mutable/util.py b/src/allmydata/test/mutable/util.py index 5a2a6a8f8..fd1fc2970 100644 --- a/src/allmydata/test/mutable/util.py +++ b/src/allmydata/test/mutable/util.py @@ -4,8 +4,6 @@ Ported to Python 3. from future.utils import bchr -from past.builtins import long - from io import BytesIO import attr from twisted.internet import defer, reactor @@ -129,8 +127,8 @@ class FakeStorageServer(object): continue vector = response[shnum] = [] for (offset, length) in readv: - assert isinstance(offset, (int, long)), offset - assert isinstance(length, (int, long)), length + assert isinstance(offset, int), offset + assert isinstance(length, int), length vector.append(shares[shnum][offset:offset+length]) return response d.addCallback(_read) diff --git a/src/allmydata/test/test_dirnode.py b/src/allmydata/test/test_dirnode.py index 2ef57b0af..30fba005f 100644 --- a/src/allmydata/test/test_dirnode.py +++ b/src/allmydata/test/test_dirnode.py @@ -3,8 +3,6 @@ Ported to Python 3. """ -from past.builtins import long - import time import unicodedata from zope.interface import implementer @@ -1854,7 +1852,7 @@ class DeepStats(testutil.ReallyEqualMixin, unittest.TestCase): (101, 316, 216), (317, 1000, 684), (1001, 3162, 99), - (long(3162277660169), long(10000000000000), 1), + (3162277660169, 10000000000000, 1), ]) class UCWEingMutableFileNode(MutableFileNode): diff --git a/src/allmydata/test/test_humanreadable.py b/src/allmydata/test/test_humanreadable.py index 1ae95145e..277abc283 100644 --- a/src/allmydata/test/test_humanreadable.py +++ b/src/allmydata/test/test_humanreadable.py @@ -4,8 +4,6 @@ Tests for allmydata.util.humanreadable. This module has been ported to Python 3. """ -from past.builtins import long - from twisted.trial import unittest from allmydata.util import humanreadable @@ -26,7 +24,7 @@ class HumanReadable(unittest.TestCase): self.assertRegex(hr(foo), r"") self.failUnlessEqual(hr(self.test_repr), ">") - self.failUnlessEqual(hr(long(1)), "1") + self.failUnlessEqual(hr(1), "1") self.assertIn(hr(10**40), ["100000000000000000...000000000000000000", "100000000000000000...0000000000000000000"]) diff --git a/src/allmydata/test/test_spans.py b/src/allmydata/test/test_spans.py index e6e510e5d..578075e8d 100644 --- a/src/allmydata/test/test_spans.py +++ b/src/allmydata/test/test_spans.py @@ -2,8 +2,6 @@ Tests for allmydata.util.spans. """ -from past.builtins import long - import binascii import hashlib @@ -116,9 +114,6 @@ class ByteSpans(unittest.TestCase): s1 = Spans(3, 4) # 3,4,5,6 self._check1(s1) - s1 = Spans(long(3), long(4)) # 3,4,5,6 - self._check1(s1) - s2 = Spans(s1) self._check1(s2) @@ -446,9 +441,9 @@ class StringSpans(unittest.TestCase): self.failUnlessEqual(ds.get(2, 4), b"fear") ds = klass() - ds.add(long(2), b"four") - ds.add(long(3), b"ea") - self.failUnlessEqual(ds.get(long(2), long(4)), b"fear") + ds.add(2, b"four") + ds.add(3, b"ea") + self.failUnlessEqual(ds.get(2, 4), b"fear") def do_scan(self, klass): diff --git a/src/allmydata/test/test_time_format.py b/src/allmydata/test/test_time_format.py index 0b409feed..f3b9a8990 100644 --- a/src/allmydata/test/test_time_format.py +++ b/src/allmydata/test/test_time_format.py @@ -2,8 +2,6 @@ Tests for allmydata.util.time_format. """ -from past.builtins import long - import time from twisted.trial import unittest @@ -103,7 +101,7 @@ class TimeFormat(unittest.TestCase, TimezoneMixin): def test_parse_date(self): p = time_format.parse_date self.failUnlessEqual(p("2010-02-21"), 1266710400) - self.failUnless(isinstance(p("2009-03-18"), (int, long)), p("2009-03-18")) + self.failUnless(isinstance(p("2009-03-18"), int), p("2009-03-18")) self.failUnlessEqual(p("2009-03-18"), 1237334400) def test_format_time(self): diff --git a/src/allmydata/util/netstring.py b/src/allmydata/util/netstring.py index db913172f..ee7849b5f 100644 --- a/src/allmydata/util/netstring.py +++ b/src/allmydata/util/netstring.py @@ -4,8 +4,6 @@ Netstring encoding and decoding. Ported to Python 3. """ -from past.builtins import long - try: from typing import Optional, Tuple, List # noqa: F401 except ImportError: @@ -27,7 +25,7 @@ def split_netstring(data, numstrings, data does not exactly equal 'required_trailer'.""" assert isinstance(data, bytes) assert required_trailer is None or isinstance(required_trailer, bytes) - assert isinstance(position, (int, long)), (repr(position), type(position)) + assert isinstance(position, int), (repr(position), type(position)) elements = [] assert numstrings >= 0 while position < len(data): diff --git a/src/allmydata/web/status.py b/src/allmydata/web/status.py index 07d0256e8..1737a4d1b 100644 --- a/src/allmydata/web/status.py +++ b/src/allmydata/web/status.py @@ -2,8 +2,6 @@ Ported to Python 3. """ -from past.builtins import long - import itertools import hashlib import re @@ -1393,7 +1391,7 @@ class StatusElement(Element): size = op.get_size() if size is None: size = "(unknown)" - elif isinstance(size, (int, long, float)): + elif isinstance(size, (int, float)): size = abbreviate_size(size) result["total_size"] = size From f47b45ac1a5bf599b9393eb99384f733de550e73 Mon Sep 17 00:00:00 2001 From: Alexandre Detiste Date: Wed, 28 Feb 2024 01:25:06 +0100 Subject: [PATCH 2271/2309] remove static native_str() --- src/allmydata/immutable/upload.py | 3 +-- src/allmydata/interfaces.py | 10 ++++------ src/allmydata/introducer/interfaces.py | 10 +++++----- src/allmydata/test/common_util.py | 17 ++++++----------- src/allmydata/test/storage_plugin.py | 4 ++-- src/allmydata/util/encodingutil.py | 5 +---- src/allmydata/util/iputil.py | 6 +++--- 7 files changed, 22 insertions(+), 33 deletions(-) diff --git a/src/allmydata/immutable/upload.py b/src/allmydata/immutable/upload.py index c13970033..e62f7e0c1 100644 --- a/src/allmydata/immutable/upload.py +++ b/src/allmydata/immutable/upload.py @@ -4,7 +4,6 @@ Ported to Python 3. from __future__ import annotations -from future.utils import native_str from six import ensure_str import os, time, weakref, itertools @@ -165,7 +164,7 @@ class HelperUploadResults(Copyable, RemoteCopy): # package/module/class name # # Needs to be native string to make Foolscap happy. - typeToCopy = native_str("allmydata.upload.UploadResults.tahoe.allmydata.com") + typeToCopy = "allmydata.upload.UploadResults.tahoe.allmydata.com" copytype = typeToCopy # also, think twice about changing the shape of any existing attribute, diff --git a/src/allmydata/interfaces.py b/src/allmydata/interfaces.py index 1cfbb577e..e44a0e8bb 100644 --- a/src/allmydata/interfaces.py +++ b/src/allmydata/interfaces.py @@ -6,8 +6,6 @@ Ported to Python 3. Note that for RemoteInterfaces, the __remote_name__ needs to be a native string because of https://github.com/warner/foolscap/blob/43f4485a42c9c28e2c79d655b3a9e24d4e6360ca/src/foolscap/remoteinterface.py#L67 """ -from future.utils import native_str - from typing import Dict from zope.interface import Interface, Attribute @@ -111,7 +109,7 @@ ReadData = ListOf(ShareData) class RIStorageServer(RemoteInterface): - __remote_name__ = native_str("RIStorageServer.tahoe.allmydata.com") + __remote_name__ = "RIStorageServer.tahoe.allmydata.com" def get_version(): """ @@ -2767,7 +2765,7 @@ UploadResults = Any() #DictOf(bytes, bytes) class RIEncryptedUploadable(RemoteInterface): - __remote_name__ = native_str("RIEncryptedUploadable.tahoe.allmydata.com") + __remote_name__ = "RIEncryptedUploadable.tahoe.allmydata.com" def get_size(): return Offset @@ -2783,7 +2781,7 @@ class RIEncryptedUploadable(RemoteInterface): class RICHKUploadHelper(RemoteInterface): - __remote_name__ = native_str("RIUploadHelper.tahoe.allmydata.com") + __remote_name__ = "RIUploadHelper.tahoe.allmydata.com" def get_version(): """ @@ -2796,7 +2794,7 @@ class RICHKUploadHelper(RemoteInterface): class RIHelper(RemoteInterface): - __remote_name__ = native_str("RIHelper.tahoe.allmydata.com") + __remote_name__ = "RIHelper.tahoe.allmydata.com" def get_version(): """ diff --git a/src/allmydata/introducer/interfaces.py b/src/allmydata/introducer/interfaces.py index 13cd7c3da..e714d7340 100644 --- a/src/allmydata/introducer/interfaces.py +++ b/src/allmydata/introducer/interfaces.py @@ -2,9 +2,6 @@ Ported to Python 3. """ - -from future.utils import native_str - from zope.interface import Interface from foolscap.api import StringConstraint, SetOf, DictOf, Any, \ RemoteInterface, Referenceable @@ -34,7 +31,7 @@ FURL = StringConstraint(1000) Announcement_v2 = Any() class RIIntroducerSubscriberClient_v2(RemoteInterface): - __remote_name__ = native_str("RIIntroducerSubscriberClient_v2.tahoe.allmydata.com") + __remote_name__ = "RIIntroducerSubscriberClient_v2.tahoe.allmydata.com" def announce_v2(announcements=SetOf(Announcement_v2)): """I accept announcements from the publisher.""" @@ -47,11 +44,14 @@ class RIIntroducerPublisherAndSubscriberService_v2(RemoteInterface): announcement message. I will deliver a copy to all connected subscribers. To hear about services, connect to me and subscribe to a specific service_name.""" - __remote_name__ = native_str("RIIntroducerPublisherAndSubscriberService_v2.tahoe.allmydata.com") + __remote_name__ = "RIIntroducerPublisherAndSubscriberService_v2.tahoe.allmydata.com" + def get_version(): return DictOf(bytes, Any()) + def publish_v2(announcement=Announcement_v2, canary=Referenceable): return None + def subscribe_v2(subscriber=RIIntroducerSubscriberClient_v2, service_name=bytes, subscriber_info=SubscriberInfo): """Give me a subscriber reference, and I will call its announce_v2() diff --git a/src/allmydata/test/common_util.py b/src/allmydata/test/common_util.py index 9b194f657..70713a995 100644 --- a/src/allmydata/test/common_util.py +++ b/src/allmydata/test/common_util.py @@ -3,7 +3,6 @@ Ported to Python 3. """ from future.utils import bchr -from future.builtins import str as future_str import os import sys @@ -64,13 +63,13 @@ def run_cli_native(verb, *args, **kwargs): :param runner.Options options: The options instance to use to parse the given arguments. - :param native_str verb: The command to run. For example, + :param str verb: The command to run. For example, ``"create-node"``. - :param [native_str] args: The arguments to pass to the command. For + :param [str] args: The arguments to pass to the command. For example, ``("--hostname=localhost",)``. - :param [native_str] nodeargs: Extra arguments to pass to the Tahoe + :param [str] nodeargs: Extra arguments to pass to the Tahoe executable before ``verb``. :param bytes|unicode stdin: Text or bytes to pass to the command via stdin. @@ -165,7 +164,7 @@ def run_cli_unicode(verb, argv, nodeargs=None, stdin=None, encoding=None): if nodeargs is None: nodeargs = [] precondition( - all(isinstance(arg, future_str) for arg in [verb] + nodeargs + argv), + all(isinstance(arg, str) for arg in [verb] + nodeargs + argv), "arguments to run_cli_unicode must be unicode", verb=verb, nodeargs=nodeargs, @@ -229,13 +228,9 @@ def flip_one_bit(s, offset=0, size=None): class ReallyEqualMixin(object): def failUnlessReallyEqual(self, a, b, msg=None): self.assertEqual(a, b, msg) - # Make sure unicode strings are a consistent type. Specifically there's - # Future newstr (backported Unicode type) vs. Python 2 native unicode - # type. They're equal, and _logically_ the same type, but have - # different types in practice. - if a.__class__ == future_str: + if a.__class__ == str: a = str(a) - if b.__class__ == future_str: + if b.__class__ == str: b = str(b) self.assertEqual(type(a), type(b), "a :: %r (%s), b :: %r (%s), %r" % (a, type(a), b, type(b), msg)) diff --git a/src/allmydata/test/storage_plugin.py b/src/allmydata/test/storage_plugin.py index b1950387b..f638d298f 100644 --- a/src/allmydata/test/storage_plugin.py +++ b/src/allmydata/test/storage_plugin.py @@ -5,7 +5,7 @@ functionality. Ported to Python 3. """ -from future.utils import native_str, native_str_to_bytes +from future.utils import native_str_to_bytes from six import ensure_str import attr @@ -40,7 +40,7 @@ from allmydata.util.jsonbytes import ( class RIDummy(RemoteInterface): - __remote_name__ = native_str("RIDummy.tahoe.allmydata.com") + __remote_name__ = "RIDummy.tahoe.allmydata.com" def just_some_method(): """ diff --git a/src/allmydata/util/encodingutil.py b/src/allmydata/util/encodingutil.py index 9bf906ad7..e2131eb0d 100644 --- a/src/allmydata/util/encodingutil.py +++ b/src/allmydata/util/encodingutil.py @@ -8,9 +8,6 @@ Once Python 2 support is dropped, most of this module will obsolete, since Unicode is the default everywhere in Python 3. """ -from future.utils import native_str -from future.builtins import str as future_str - from past.builtins import unicode from six import ensure_str @@ -124,7 +121,7 @@ def unicode_to_argv(s): # According to unicode_to_argv above, the expected type for # cli args depends on the platform, so capture that expectation. -argv_type = (future_str, native_str) if sys.platform == "win32" else native_str +argv_type = (str,) """ The expected type for args to a subprocess """ diff --git a/src/allmydata/util/iputil.py b/src/allmydata/util/iputil.py index e71e514e8..61fcfc8a0 100644 --- a/src/allmydata/util/iputil.py +++ b/src/allmydata/util/iputil.py @@ -104,7 +104,7 @@ def get_local_addresses_sync(): on the local system. """ return list( - native_str(address[native_str("addr")]) + native_str(address["addr"]) for iface_name in interfaces() for address @@ -161,7 +161,7 @@ def _foolscapEndpointForPortNumber(portnum): # approach is error prone for the reasons described on # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/2787 portnum = allocate_tcp_port() - return (portnum, native_str("tcp:%d" % (portnum,))) + return (portnum, "tcp:%d" % portnum) @implementer(IStreamServerEndpoint) @@ -210,7 +210,7 @@ def listenOnUnused(tub, portnum=None): """ portnum, endpoint = _foolscapEndpointForPortNumber(portnum) tub.listenOn(endpoint) - tub.setLocation(native_str("localhost:%d" % (portnum,))) + tub.setLocation("localhost:%d" % portnum) return portnum From 8e5e886598085bedb5fe4aad7a9875fc443cc842 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 1 Mar 2024 12:09:40 -0500 Subject: [PATCH 2272/2309] News fragment. --- newsfragments/4093.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/4093.minor diff --git a/newsfragments/4093.minor b/newsfragments/4093.minor new file mode 100644 index 000000000..e69de29bb From ddc8e64272c87af6cb31b4bdcd002818a6b8df3f Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 1 Mar 2024 12:11:54 -0500 Subject: [PATCH 2273/2309] We're just going to assume that in 2024 this is not an issue. --- src/allmydata/test/test_encodingutil.py | 32 ------------------------- 1 file changed, 32 deletions(-) diff --git a/src/allmydata/test/test_encodingutil.py b/src/allmydata/test/test_encodingutil.py index 4e16ef2b7..210d46203 100644 --- a/src/allmydata/test/test_encodingutil.py +++ b/src/allmydata/test/test_encodingutil.py @@ -73,38 +73,6 @@ from allmydata.util.encodingutil import unicode_to_url, \ class MockStdout(object): pass -# The following tests apply only to platforms that don't store filenames as -# Unicode entities on the filesystem. -class EncodingUtilNonUnicodePlatform(unittest.TestCase): - - def test_listdir_unicode(self): - # What happens if latin1-encoded filenames are encountered on an UTF-8 - # filesystem? - def call_os_listdir(path): - return [ - lumiere_nfc.encode('utf-8'), - lumiere_nfc.encode('latin1') - ] - self.patch(os, 'listdir', call_os_listdir) - - sys_filesystemencoding = 'utf-8' - def call_sys_getfilesystemencoding(): - return sys_filesystemencoding - self.patch(sys, 'getfilesystemencoding', call_sys_getfilesystemencoding) - - _reload() - self.failUnlessRaises(FilenameEncodingError, - listdir_unicode, - u'/dummy') - - # We're trying to list a directory whose name cannot be represented in - # the filesystem encoding. This should fail. - sys_filesystemencoding = 'ascii' - _reload() - self.failUnlessRaises(FilenameEncodingError, - listdir_unicode, - u'/' + lumiere_nfc) - class EncodingUtil(ReallyEqualMixin): def setUp(self): From 0bbc8f6b5f2aaa626bc39fa6ef619d31043f2c36 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 1 Mar 2024 12:16:40 -0500 Subject: [PATCH 2274/2309] Delete Python 2 specific code. --- src/allmydata/windows/fixups.py | 194 -------------------------------- 1 file changed, 194 deletions(-) diff --git a/src/allmydata/windows/fixups.py b/src/allmydata/windows/fixups.py index c48db2f82..1b204ccf4 100644 --- a/src/allmydata/windows/fixups.py +++ b/src/allmydata/windows/fixups.py @@ -11,104 +11,19 @@ import sys assert sys.platform == "win32" -import codecs -from functools import partial - -from ctypes import WINFUNCTYPE, windll, POINTER, c_int, WinError, byref, get_last_error -from ctypes.wintypes import BOOL, HANDLE, DWORD, LPWSTR, LPCWSTR, LPVOID - # from win32api import ( - STD_OUTPUT_HANDLE, - STD_ERROR_HANDLE, SetErrorMode, - - # - # HANDLE WINAPI GetStdHandle(DWORD nStdHandle); - # returns INVALID_HANDLE_VALUE, NULL, or a valid handle - GetStdHandle, ) from win32con import ( SEM_FAILCRITICALERRORS, SEM_NOOPENFILEERRORBOX, ) -from win32file import ( - INVALID_HANDLE_VALUE, - FILE_TYPE_CHAR, - - # - # DWORD WINAPI GetFileType(DWORD hFile); - GetFileType, -) - -from allmydata.util import ( - log, -) - # Keep track of whether `initialize` has run so we don't do any of the # initialization more than once. _done = False -# -# pywin32 for Python 2.7 does not bind any of these *W variants so we do it -# ourselves. -# - -# -# BOOL WINAPI WriteConsoleW(HANDLE hOutput, LPWSTR lpBuffer, DWORD nChars, -# LPDWORD lpCharsWritten, LPVOID lpReserved); -WriteConsoleW = WINFUNCTYPE( - BOOL, HANDLE, LPWSTR, DWORD, POINTER(DWORD), LPVOID, - use_last_error=True -)(("WriteConsoleW", windll.kernel32)) - -# -GetCommandLineW = WINFUNCTYPE( - LPWSTR, - use_last_error=True -)(("GetCommandLineW", windll.kernel32)) - -# -CommandLineToArgvW = WINFUNCTYPE( - POINTER(LPWSTR), LPCWSTR, POINTER(c_int), - use_last_error=True -)(("CommandLineToArgvW", windll.shell32)) - -# -# BOOL WINAPI GetConsoleMode(HANDLE hConsole, LPDWORD lpMode); -GetConsoleMode = WINFUNCTYPE( - BOOL, HANDLE, POINTER(DWORD), - use_last_error=True -)(("GetConsoleMode", windll.kernel32)) - - -STDOUT_FILENO = 1 -STDERR_FILENO = 2 - -def get_argv(): - """ - :return [unicode]: The argument list this process was invoked with, as - unicode. - - Python 2 does not do a good job exposing this information in - ``sys.argv`` on Windows so this code re-retrieves the underlying - information using Windows API calls and massages it into the right - shape. - """ - command_line = GetCommandLineW() - argc = c_int(0) - argv_unicode = CommandLineToArgvW(command_line, byref(argc)) - if argv_unicode is None: - raise WinError(get_last_error()) - - # Convert it to a normal Python list - return list( - argv_unicode[i] - for i - in range(argc.value) - ) - def initialize(): global _done @@ -118,112 +33,3 @@ def initialize(): _done = True SetErrorMode(SEM_FAILCRITICALERRORS | SEM_NOOPENFILEERRORBOX) - - -def a_console(handle): - """ - :return: ``True`` if ``handle`` refers to a console, ``False`` otherwise. - """ - if handle == INVALID_HANDLE_VALUE: - return False - return ( - # It's a character file (eg a printer or a console) - GetFileType(handle) == FILE_TYPE_CHAR and - # Checking the console mode doesn't fail (thus it's a console) - GetConsoleMode(handle, byref(DWORD())) != 0 - ) - - -class UnicodeOutput(object): - """ - ``UnicodeOutput`` is a file-like object that encodes unicode to UTF-8 and - writes it to another file or writes unicode natively to the Windows - console. - """ - def __init__(self, hConsole, stream, fileno, name, _complain): - """ - :param hConsole: ``None`` or a handle on the console to which to write - unicode. Mutually exclusive with ``stream``. - - :param stream: ``None`` or a file-like object to which to write bytes. - - :param fileno: A result to hand back from method of the same name. - - :param name: A human-friendly identifier for this output object. - - :param _complain: A one-argument callable which accepts bytes to be - written when there's a problem. Care should be taken to not make - this do a write on this object. - """ - self._hConsole = hConsole - self._stream = stream - self._fileno = fileno - self.closed = False - self.softspace = False - self.mode = 'w' - self.encoding = 'utf-8' - self.name = name - - self._complain = _complain - - from allmydata.util.encodingutil import canonical_encoding - from allmydata.util import log - if hasattr(stream, 'encoding') and canonical_encoding(stream.encoding) != 'utf-8': - log.msg("%s: %r had encoding %r, but we're going to write UTF-8 to it" % - (name, stream, stream.encoding), level=log.CURIOUS) - self.flush() - - def isatty(self): - return False - def close(self): - # don't really close the handle, that would only cause problems - self.closed = True - def fileno(self): - return self._fileno - def flush(self): - if self._hConsole is None: - try: - self._stream.flush() - except Exception as e: - self._complain("%s.flush: %r from %r" % (self.name, e, self._stream)) - raise - - def write(self, text): - try: - if self._hConsole is None: - # There is no Windows console available. That means we are - # responsible for encoding the unicode to a byte string to - # write it to a Python file object. - if isinstance(text, str): - text = text.encode('utf-8') - self._stream.write(text) - else: - # There is a Windows console available. That means Windows is - # responsible for dealing with the unicode itself. - if not isinstance(text, str): - text = str(text).decode('utf-8') - remaining = len(text) - while remaining > 0: - n = DWORD(0) - # There is a shorter-than-documented limitation on the - # length of the string passed to WriteConsoleW (see - # #1232). - retval = WriteConsoleW(self._hConsole, text, min(remaining, 10000), byref(n), None) - if retval == 0: - raise IOError("WriteConsoleW failed with WinError: %s" % (WinError(get_last_error()),)) - if n.value == 0: - raise IOError("WriteConsoleW returned %r, n.value = 0" % (retval,)) - remaining -= n.value - if remaining == 0: break - text = text[n.value:] - except Exception as e: - self._complain("%s.write: %r" % (self.name, e)) - raise - - def writelines(self, lines): - try: - for line in lines: - self.write(line) - except Exception as e: - self._complain("%s.writelines: %r" % (self.name, e)) - raise From dfdb6c60d0e0f1fc24ca7409069db72dd4a6cf1c Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 1 Mar 2024 12:17:11 -0500 Subject: [PATCH 2275/2309] Fix lints. --- src/allmydata/check_results.py | 2 +- src/allmydata/test/cli/test_list.py | 2 +- src/allmydata/test/test_encodingutil.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/allmydata/check_results.py b/src/allmydata/check_results.py index e9ef59bdb..44a4a1db8 100644 --- a/src/allmydata/check_results.py +++ b/src/allmydata/check_results.py @@ -61,7 +61,7 @@ class CheckResults(object): # On Python 2, we can mix bytes and Unicode. On Python 3, we want # unicode. if isinstance(summary, bytes): - summary = unicode(summary, "utf-8") + summary = str(summary, "utf-8") assert isinstance(summary, str) # should be a single string self._summary = summary assert not isinstance(report, str) # should be list of strings diff --git a/src/allmydata/test/cli/test_list.py b/src/allmydata/test/cli/test_list.py index f9ab52c53..55f0952fe 100644 --- a/src/allmydata/test/cli/test_list.py +++ b/src/allmydata/test/cli/test_list.py @@ -9,7 +9,7 @@ from allmydata.immutable import upload from allmydata.interfaces import MDMF_VERSION, SDMF_VERSION from allmydata.mutable.publish import MutableData from ..no_network import GridTestMixin -from allmydata.util.encodingutil import quote_output, get_io_encoding +from allmydata.util.encodingutil import quote_output from .common import CLITestMixin diff --git a/src/allmydata/test/test_encodingutil.py b/src/allmydata/test/test_encodingutil.py index 210d46203..c98ef9e40 100644 --- a/src/allmydata/test/test_encodingutil.py +++ b/src/allmydata/test/test_encodingutil.py @@ -65,7 +65,7 @@ from allmydata.test.common_util import ( from allmydata.util import encodingutil, fileutil from allmydata.util.encodingutil import unicode_to_url, \ unicode_to_output, quote_output, quote_path, quote_local_unicode_path, \ - quote_filepath, unicode_platform, listdir_unicode, FilenameEncodingError, \ + quote_filepath, unicode_platform, listdir_unicode, \ get_filesystem_encoding, to_bytes, from_utf8_or_none, _reload, \ to_filepath, extend_filepath, unicode_from_filepath, unicode_segments_from, \ unicode_to_argv From 6c48698f28fae226ff7f872a49fb5810302ec9d5 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 1 Mar 2024 13:00:24 -0500 Subject: [PATCH 2276/2309] No need for repetition. --- src/allmydata/immutable/upload.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/allmydata/immutable/upload.py b/src/allmydata/immutable/upload.py index e62f7e0c1..de59d3dc9 100644 --- a/src/allmydata/immutable/upload.py +++ b/src/allmydata/immutable/upload.py @@ -55,7 +55,7 @@ from eliot import ( _TOTAL_SHARES = Field.for_types( u"total_shares", - [int, int], + [int], u"The total number of shares desired.", ) @@ -102,7 +102,7 @@ _HAPPINESS_MAPPINGS = Field( _HAPPINESS = Field.for_types( u"happiness", - [int, int], + [int], u"The computed happiness of a certain placement.", ) @@ -140,7 +140,7 @@ GET_SHARE_PLACEMENTS = MessageType( _EFFECTIVE_HAPPINESS = Field.for_types( u"effective_happiness", - [int, int], + [int], u"The computed happiness value of a share placement map.", ) From a9015cdb5baa271f70fa1aa3a4fbc04cb7e9bf91 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 1 Mar 2024 13:06:32 -0500 Subject: [PATCH 2277/2309] Remove another future import. --- src/allmydata/test/common_util.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/common_util.py b/src/allmydata/test/common_util.py index 70713a995..d52cb8afa 100644 --- a/src/allmydata/test/common_util.py +++ b/src/allmydata/test/common_util.py @@ -2,8 +2,6 @@ Ported to Python 3. """ -from future.utils import bchr - import os import sys import time @@ -25,6 +23,9 @@ from ..util.assertutil import precondition from ..scripts import runner from allmydata.util.encodingutil import unicode_platform, get_filesystem_encoding, argv_type, unicode_to_argv +def bchr(s): + return bytes([s]) + def skip_if_cannot_represent_filename(u): precondition(isinstance(u, str)) From 09854684e6a3c47e23c7aae8cfcfd676f5d8ffdd Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 1 Mar 2024 13:08:09 -0500 Subject: [PATCH 2278/2309] Remove another future import. --- src/allmydata/test/storage_plugin.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/allmydata/test/storage_plugin.py b/src/allmydata/test/storage_plugin.py index f638d298f..46088903f 100644 --- a/src/allmydata/test/storage_plugin.py +++ b/src/allmydata/test/storage_plugin.py @@ -5,7 +5,6 @@ functionality. Ported to Python 3. """ -from future.utils import native_str_to_bytes from six import ensure_str import attr @@ -87,7 +86,7 @@ class DummyStorage(object): """ items = configuration.items(self._client_section_name, []) resource = Data( - native_str_to_bytes(dumps(dict(items))), + dumps(dict(items)).encode("utf-8"), ensure_str("text/json"), ) # Give it some dynamic stuff too. @@ -105,7 +104,7 @@ class GetCounter(Resource, object): value = 0 def render_GET(self, request): self.value += 1 - return native_str_to_bytes(dumps({"value": self.value})) + return dumps({"value": self.value}).encode("utf-8") @implementer(RIDummy) From 4da491aeb00f75b1e14338f57939f3b4b8ed52c5 Mon Sep 17 00:00:00 2001 From: Alexandre Detiste Date: Mon, 11 Mar 2024 21:37:27 +0100 Subject: [PATCH 2279/2309] remove more usage of "future" --- src/allmydata/dirnode.py | 24 +++++------- src/allmydata/mutable/layout.py | 12 +++--- src/allmydata/storage/immutable.py | 7 +--- src/allmydata/storage/server.py | 21 ++++++++--- src/allmydata/test/test_encode.py | 4 +- src/allmydata/test/test_encodingutil.py | 6 +-- src/allmydata/test/test_log.py | 5 +-- src/allmydata/test/test_storage.py | 10 ++--- src/allmydata/test/test_system.py | 4 +- src/allmydata/util/encodingutil.py | 49 ++++++++----------------- src/allmydata/util/iputil.py | 4 +- src/allmydata/util/time_format.py | 17 ++++++--- 12 files changed, 72 insertions(+), 91 deletions(-) diff --git a/src/allmydata/dirnode.py b/src/allmydata/dirnode.py index 5ba75f6c0..16be8b9ef 100644 --- a/src/allmydata/dirnode.py +++ b/src/allmydata/dirnode.py @@ -3,8 +3,6 @@ Ported to Python 3. """ -from past.builtins import unicode - import time from zope.interface import implementer @@ -39,31 +37,29 @@ from eliot.twisted import ( ) NAME = Field.for_types( - u"name", - # Make sure this works on Python 2; with str, it gets Future str which - # breaks Eliot. - [unicode], - u"The name linking the parent to this node.", + "name", + [str], + "The name linking the parent to this node.", ) METADATA = Field.for_types( - u"metadata", + "metadata", [dict], - u"Data about a node.", + "Data about a node.", ) OVERWRITE = Field.for_types( - u"overwrite", + "overwrite", [bool], - u"True to replace an existing file of the same name, " - u"false to fail with a collision error.", + "True to replace an existing file of the same name, " + "false to fail with a collision error.", ) ADD_FILE = ActionType( - u"dirnode:add-file", + "dirnode:add-file", [NAME, METADATA, OVERWRITE], [], - u"Add a new file as a child of a directory.", + "Add a new file as a child of a directory.", ) diff --git a/src/allmydata/mutable/layout.py b/src/allmydata/mutable/layout.py index 6a5993993..b0a799f5d 100644 --- a/src/allmydata/mutable/layout.py +++ b/src/allmydata/mutable/layout.py @@ -2,8 +2,6 @@ Ported to Python 3. """ -from past.utils import old_div - import struct from allmydata.mutable.common import NeedMoreDataError, UnknownVersionError, \ BadShareError @@ -260,7 +258,7 @@ class SDMFSlotWriteProxy(object): self._required_shares) assert expected_segment_size == segment_size - self._block_size = old_div(self._segment_size, self._required_shares) + self._block_size = self._segment_size // self._required_shares # This is meant to mimic how SDMF files were built before MDMF # entered the picture: we generate each share in its entirety, @@ -793,7 +791,7 @@ class MDMFSlotWriteProxy(object): # and also because it provides a useful amount of bounds checking. self._num_segments = mathutil.div_ceil(self._data_length, self._segment_size) - self._block_size = old_div(self._segment_size, self._required_shares) + self._block_size = self._segment_size // self._required_shares # We also calculate the share size, to help us with block # constraints later. tail_size = self._data_length % self._segment_size @@ -802,7 +800,7 @@ class MDMFSlotWriteProxy(object): else: self._tail_block_size = mathutil.next_multiple(tail_size, self._required_shares) - self._tail_block_size = old_div(self._tail_block_size, self._required_shares) + self._tail_block_size = self._tail_block_size // self._required_shares # We already know where the sharedata starts; right after the end # of the header (which is defined as the signable part + the offsets) @@ -1324,7 +1322,7 @@ class MDMFSlotReadProxy(object): self._segment_size = segsize self._data_length = datalen - self._block_size = old_div(self._segment_size, self._required_shares) + self._block_size = self._segment_size // self._required_shares # We can upload empty files, and need to account for this fact # so as to avoid zero-division and zero-modulo errors. if datalen > 0: @@ -1336,7 +1334,7 @@ class MDMFSlotReadProxy(object): else: self._tail_block_size = mathutil.next_multiple(tail_size, self._required_shares) - self._tail_block_size = old_div(self._tail_block_size, self._required_shares) + self._tail_block_size = self._tail_block_size // self._required_shares return encoding_parameters diff --git a/src/allmydata/storage/immutable.py b/src/allmydata/storage/immutable.py index 7a61a1e62..9cb6cc6ee 100644 --- a/src/allmydata/storage/immutable.py +++ b/src/allmydata/storage/immutable.py @@ -2,9 +2,6 @@ Ported to Python 3. """ - -from future.utils import bytes_to_native_str - import os, stat, struct, time from collections_extended import RangeMap @@ -534,9 +531,7 @@ class BucketReader(object): def __repr__(self): return "<%s %s %s>" % (self.__class__.__name__, - bytes_to_native_str( - base32.b2a(self.storage_index[:8])[:12] - ), + base32.b2a(self.storage_index[:8])[:12].decode(), self.shnum) def read(self, offset, length): diff --git a/src/allmydata/storage/server.py b/src/allmydata/storage/server.py index 858b87b1f..e734f9d74 100644 --- a/src/allmydata/storage/server.py +++ b/src/allmydata/storage/server.py @@ -3,7 +3,6 @@ Ported to Python 3. """ from __future__ import annotations -from future.utils import bytes_to_native_str from typing import Iterable, Any import os, re @@ -905,7 +904,12 @@ share_number: {share_number} """ -def render_corruption_report(share_type, si_s, shnum, reason): +def render_corruption_report( + share_type: bytes, + si_s: bytes, + shnum: int, + reason: bytes +) -> str: """ Create a string that explains a corruption report using freeform text. @@ -920,13 +924,18 @@ def render_corruption_report(share_type, si_s, shnum, reason): report. """ return CORRUPTION_REPORT_FORMAT.format( - type=bytes_to_native_str(share_type), - storage_index=bytes_to_native_str(si_s), + type=share_type.decode(), + storage_index=si_s.decode(), share_number=shnum, - reason=bytes_to_native_str(reason), + reason=reason.decode(), ) -def get_corruption_report_path(base_dir, now, si_s, shnum): +def get_corruption_report_path( + base_dir: str, + now: str, + si_s: str, + shnum: int +) -> str: """ Determine the path to which a certain corruption report should be written. diff --git a/src/allmydata/test/test_encode.py b/src/allmydata/test/test_encode.py index 8ce5e757a..ba11605ab 100644 --- a/src/allmydata/test/test_encode.py +++ b/src/allmydata/test/test_encode.py @@ -2,7 +2,7 @@ Ported to Python 3. """ -from past.builtins import chr as byteschr, long +from past.builtins import chr as byteschr from zope.interface import implementer from twisted.trial import unittest @@ -99,7 +99,7 @@ class FakeBucketReaderWriterProxy(object): def get_block_data(self, blocknum, blocksize, size): d = self._start() def _try(unused=None): - assert isinstance(blocknum, (int, long)) + assert isinstance(blocknum, int) if self.mode == "bad block": return flip_bit(self.blocks[blocknum]) return self.blocks[blocknum] diff --git a/src/allmydata/test/test_encodingutil.py b/src/allmydata/test/test_encodingutil.py index c98ef9e40..fef9e6d57 100644 --- a/src/allmydata/test/test_encodingutil.py +++ b/src/allmydata/test/test_encodingutil.py @@ -343,8 +343,7 @@ class FilePaths(ReallyEqualMixin, unittest.TestCase): for fp in (nosep_fp, sep_fp): self.failUnlessReallyEqual(fp, FilePath(foo_u)) - if encodingutil.use_unicode_filepath: - self.failUnlessReallyEqual(fp.path, foo_u) + self.failUnlessReallyEqual(fp.path, foo_u) if sys.platform == "win32": long_u = u'\\\\?\\C:\\foo' @@ -360,8 +359,7 @@ class FilePaths(ReallyEqualMixin, unittest.TestCase): for foo_fp in (foo_bfp, foo_ufp): fp = extend_filepath(foo_fp, [u'bar', u'baz']) self.failUnlessReallyEqual(fp, FilePath(foo_bar_baz_u)) - if encodingutil.use_unicode_filepath: - self.failUnlessReallyEqual(fp.path, foo_bar_baz_u) + self.failUnlessReallyEqual(fp.path, foo_bar_baz_u) def test_unicode_from_filepath(self): foo_bfp = FilePath(win32_other(b'C:\\foo', b'/foo')) diff --git a/src/allmydata/test/test_log.py b/src/allmydata/test/test_log.py index c3671f9b9..0d3361b36 100644 --- a/src/allmydata/test/test_log.py +++ b/src/allmydata/test/test_log.py @@ -4,9 +4,6 @@ Tests for allmydata.util.log. Ported to Python 3. """ - -from future.utils import native_str - from twisted.trial import unittest from twisted.python.failure import Failure @@ -161,4 +158,4 @@ class Log(unittest.TestCase): obj.log(**{"my": "message"}) for message in self.messages: for k in message[-1].keys(): - self.assertIsInstance(k, native_str) + self.assertIsInstance(k, str) diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index c1d6004e8..2964206c7 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -5,7 +5,7 @@ Ported to Python 3. """ from __future__ import annotations -from future.utils import native_str, bytes_to_native_str, bchr +from future.utils import bchr from six import ensure_str from io import ( @@ -109,7 +109,7 @@ class UtilTests(SyncTestCase): path = storage_index_to_dir(s) parts = os.path.split(path) self.assertThat(parts[0], Equals(parts[1][:2])) - self.assertThat(path, IsInstance(native_str)) + self.assertThat(path, IsInstance(str)) def test_get_share_file_mutable(self): """A mutable share is identified by get_share_file().""" @@ -1242,7 +1242,7 @@ class Server(AsyncTestCase): reports = os.listdir(reportdir) self.assertThat(reports, HasLength(2)) - report_si1 = [r for r in reports if bytes_to_native_str(si1_s) in r][0] + report_si1 = [r for r in reports if si1_s.decode() in r][0] f = open(os.path.join(reportdir, report_si1), "rb") report = f.read() f.close() @@ -1809,10 +1809,10 @@ class MutableServer(SyncTestCase): self.assertThat(readv(b"si1", [], [(0,10)]), Equals({})) # and the bucket directory should now be gone - si = base32.b2a(b"si1") + si = base32.b2a(b"si1").decode() # note: this is a detail of the storage server implementation, and # may change in the future - si = bytes_to_native_str(si) # filesystem paths are native strings + # filesystem paths are native strings prefix = si[:2] prefixdir = os.path.join(self.workdir("test_remove"), "shares", prefix) bucketdir = os.path.join(prefixdir, si) diff --git a/src/allmydata/test/test_system.py b/src/allmydata/test/test_system.py index 18ac6c6e6..b37d6923c 100644 --- a/src/allmydata/test/test_system.py +++ b/src/allmydata/test/test_system.py @@ -3,7 +3,7 @@ Ported to Python 3. """ from __future__ import annotations -from past.builtins import chr as byteschr, long +from past.builtins import chr as byteschr from six import ensure_text import os, re, sys, time, json @@ -395,7 +395,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase): # this is really bytes received rather than sent, but it's # convenient and basically measures the same thing bytes_sent = results.get_ciphertext_fetched() - self.failUnless(isinstance(bytes_sent, (int, long)), bytes_sent) + self.failUnless(isinstance(bytes_sent, int), bytes_sent) # We currently don't support resumption of upload if the data is # encrypted with a random key. (Because that would require us diff --git a/src/allmydata/util/encodingutil.py b/src/allmydata/util/encodingutil.py index e2131eb0d..da6c8d632 100644 --- a/src/allmydata/util/encodingutil.py +++ b/src/allmydata/util/encodingutil.py @@ -8,7 +8,6 @@ Once Python 2 support is dropped, most of this module will obsolete, since Unicode is the default everywhere in Python 3. """ -from past.builtins import unicode from six import ensure_str import sys, os, re @@ -53,8 +52,6 @@ def check_encoding(encoding): io_encoding = "utf-8" filesystem_encoding = None -is_unicode_platform = True -use_unicode_filepath = True def _reload(): global filesystem_encoding @@ -82,13 +79,13 @@ def argv_to_unicode(s): This is the inverse of ``unicode_to_argv``. """ - if isinstance(s, unicode): + if isinstance(s, str): return s precondition(isinstance(s, bytes), s) try: - return unicode(s, io_encoding) + return str(s, io_encoding) except UnicodeDecodeError: raise usage.UsageError("Argument %s cannot be decoded as %s." % (quote_output(s), io_encoding)) @@ -112,7 +109,7 @@ def unicode_to_argv(s): On Python 2 on POSIX, this encodes using UTF-8. On Python 3 and on Windows, this returns the input unmodified. """ - precondition(isinstance(s, unicode), s) + precondition(isinstance(s, str), s) warnings.warn("This is unnecessary.", DeprecationWarning) if sys.platform == "win32": return s @@ -166,7 +163,7 @@ def unicode_to_output(s): On Python 3 just returns the unicode string unchanged, since encoding is the responsibility of stdout/stderr, they expect Unicode by default. """ - precondition(isinstance(s, unicode), s) + precondition(isinstance(s, str), s) warnings.warn("This is unnecessary.", DeprecationWarning) return s @@ -214,7 +211,7 @@ def quote_output_u(*args, **kwargs): Like ``quote_output`` but always return ``unicode``. """ result = quote_output(*args, **kwargs) - if isinstance(result, unicode): + if isinstance(result, str): return result # Since we're quoting, the assumption is this will be read by a human, and # therefore printed, so stdout's encoding is the plausible one. io_encoding @@ -239,7 +236,7 @@ def quote_output(s, quotemarks=True, quote_newlines=None, encoding=None): On Python 3, returns Unicode strings. """ - precondition(isinstance(s, (bytes, unicode)), s) + precondition(isinstance(s, (bytes, str)), s) # Since we're quoting, the assumption is this will be read by a human, and # therefore printed, so stdout's encoding is the plausible one. io_encoding # is now always utf-8. @@ -278,7 +275,7 @@ def quote_path(path, quotemarks=True): return quote_output(b"/".join(map(to_bytes, path)), quotemarks=quotemarks, quote_newlines=True) def quote_local_unicode_path(path, quotemarks=True): - precondition(isinstance(path, unicode), path) + precondition(isinstance(path, str), path) if sys.platform == "win32" and path.startswith(u"\\\\?\\"): path = path[4 :] @@ -298,20 +295,13 @@ def extend_filepath(fp, segments): for segment in segments: fp = fp.child(segment) - if isinstance(fp.path, unicode) and not use_unicode_filepath: - return FilePath(fp.path.encode(filesystem_encoding)) - else: - return fp + return fp def to_filepath(path): - precondition(isinstance(path, unicode if use_unicode_filepath else (bytes, unicode)), - path=path) - - if isinstance(path, unicode) and not use_unicode_filepath: - path = path.encode(filesystem_encoding) + precondition(isinstance(path, str), path=path) if sys.platform == "win32": - _assert(isinstance(path, unicode), path=path) + _assert(isinstance(path, str), path=path) if path.startswith(u"\\\\?\\") and len(path) > 4: # FilePath normally strips trailing path separators, but not in this case. path = path.rstrip(u"\\") @@ -319,7 +309,7 @@ def to_filepath(path): return FilePath(path) def _decode(s): - precondition(isinstance(s, (bytes, unicode)), s=s) + precondition(isinstance(s, (bytes, str)), s=s) if isinstance(s, bytes): return s.decode(filesystem_encoding) @@ -340,7 +330,7 @@ def unicode_platform(): """ Does the current platform handle Unicode filenames natively? """ - return is_unicode_platform + return True class FilenameEncodingError(Exception): """ @@ -356,7 +346,7 @@ def listdir_unicode_fallback(path): If badly encoded filenames are encountered, an exception is raised. """ - precondition(isinstance(path, unicode), path) + precondition(isinstance(path, str), path) try: byte_path = path.encode(filesystem_encoding) @@ -364,7 +354,7 @@ def listdir_unicode_fallback(path): raise FilenameEncodingError(path) try: - return [unicode(fn, filesystem_encoding) for fn in os.listdir(byte_path)] + return [str(fn, filesystem_encoding) for fn in os.listdir(byte_path)] except UnicodeDecodeError as e: raise FilenameEncodingError(e.object) @@ -373,15 +363,8 @@ def listdir_unicode(path): Wrapper around listdir() which provides safe access to the convenient Unicode API even under platforms that don't provide one natively. """ - precondition(isinstance(path, unicode), path) - - # On Windows and MacOS X, the Unicode API is used - # On other platforms (ie. Unix systems), the byte-level API is used - - if is_unicode_platform: - return os.listdir(path) - else: - return listdir_unicode_fallback(path) + precondition(isinstance(path, str), path) + return os.listdir(path) def listdir_filepath(fp): return listdir_unicode(unicode_from_filepath(fp)) diff --git a/src/allmydata/util/iputil.py b/src/allmydata/util/iputil.py index 61fcfc8a0..0666c37d4 100644 --- a/src/allmydata/util/iputil.py +++ b/src/allmydata/util/iputil.py @@ -2,8 +2,6 @@ Utilities for getting IP addresses. """ -from future.utils import native_str - from typing import Callable import os, socket @@ -104,7 +102,7 @@ def get_local_addresses_sync(): on the local system. """ return list( - native_str(address["addr"]) + str(address["addr"]) for iface_name in interfaces() for address diff --git a/src/allmydata/util/time_format.py b/src/allmydata/util/time_format.py index 14cf4688e..fb4d735ab 100644 --- a/src/allmydata/util/time_format.py +++ b/src/allmydata/util/time_format.py @@ -5,22 +5,29 @@ ISO-8601: http://www.cl.cam.ac.uk/~mgk25/iso-time.html """ -from future.utils import native_str - import calendar, datetime, re, time +from typing import Optional + def format_time(t): return time.strftime("%Y-%m-%d %H:%M:%S", t) -def iso_utc_date(now=None, t=time.time): +def iso_utc_date( + now: Optional[float] = None, + t=time.time +) -> str: if now is None: now = t() return datetime.datetime.utcfromtimestamp(now).isoformat()[:10] -def iso_utc(now=None, sep='_', t=time.time): +def iso_utc( + now: Optional[float] = None, + sep: str = '_', + t=time.time +) -> str: if now is None: now = t() - sep = native_str(sep) # Python 2 doesn't allow unicode input to isoformat + sep = str(sep) # should already be a str return datetime.datetime.utcfromtimestamp(now).isoformat(sep) def iso_utc_time_to_seconds(isotime, _conversion_re=re.compile(r"(?P\d{4})-(?P\d{2})-(?P\d{2})[T_ ](?P\d{2}):(?P\d{2}):(?P\d{2})(?P\.\d+)?")): From 1504bec5f9b7f5e313365cd028b904f487a1d771 Mon Sep 17 00:00:00 2001 From: Alexandre Detiste Date: Mon, 11 Mar 2024 21:57:36 +0100 Subject: [PATCH 2280/2309] drop dead code --- src/allmydata/util/encodingutil.py | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/src/allmydata/util/encodingutil.py b/src/allmydata/util/encodingutil.py index da6c8d632..4f0910102 100644 --- a/src/allmydata/util/encodingutil.py +++ b/src/allmydata/util/encodingutil.py @@ -339,25 +339,6 @@ class FilenameEncodingError(Exception): """ pass -def listdir_unicode_fallback(path): - """ - This function emulates a fallback Unicode API similar to one available - under Windows or MacOS X. - - If badly encoded filenames are encountered, an exception is raised. - """ - precondition(isinstance(path, str), path) - - try: - byte_path = path.encode(filesystem_encoding) - except (UnicodeEncodeError, UnicodeDecodeError): - raise FilenameEncodingError(path) - - try: - return [str(fn, filesystem_encoding) for fn in os.listdir(byte_path)] - except UnicodeDecodeError as e: - raise FilenameEncodingError(e.object) - def listdir_unicode(path): """ Wrapper around listdir() which provides safe access to the convenient From dd568ab6f4ebd0979320b17bb38c71b457d323f9 Mon Sep 17 00:00:00 2001 From: "Christopher R. Wood" Date: Fri, 3 May 2024 16:40:01 -0400 Subject: [PATCH 2281/2309] Add tests for supplying RSA private keys to mkdir --- integration/test_web.py | 122 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 122 insertions(+) diff --git a/integration/test_web.py b/integration/test_web.py index 08c6e6217..94120be92 100644 --- a/integration/test_web.py +++ b/integration/test_web.py @@ -12,11 +12,18 @@ exists anywhere, however. from __future__ import annotations import time +from base64 import urlsafe_b64encode from urllib.parse import unquote as url_unquote, quote as url_quote +from cryptography.hazmat.primitives.serialization import load_pem_private_key from twisted.internet.threads import deferToThread import allmydata.uri +from allmydata.crypto.rsa import ( + create_signing_keypair, + der_string_from_signing_key, +) +from allmydata.mutable.common import derive_mutable_keys from allmydata.util import jsonbytes as json from . import util @@ -541,3 +548,118 @@ def test_mkdir_with_children(alice): assert resp.startswith(b"URI:DIR2") cap = allmydata.uri.from_string(resp) assert isinstance(cap, allmydata.uri.DirectoryURI) + + +@run_in_thread +def test_mkdir_with_random_private_key(alice): + """ + Create a new directory with ?t=mkdir&private-key=... using a + randomly-generated RSA private key. + + The writekey and fingerprint derived from the provided RSA key + should match those of the newly-created directory capability. + """ + + privkey, pubkey = create_signing_keypair(2048) + + writekey, _, fingerprint = derive_mutable_keys((pubkey, privkey)) + + # The "private-key" parameter takes a DER-encoded RSA private key + # encoded in URL-safe base64; PEM blocks are not supported. + privkey_der = der_string_from_signing_key(privkey) + privkey_encoded = urlsafe_b64encode(privkey_der).decode("ascii") + + resp = util.web_post( + alice.process, u"uri", + params={ + u"t": "mkdir", + u"private-key": privkey_encoded, + }, + ) + assert resp.startswith(b"URI:DIR2") + + dircap = allmydata.uri.from_string(resp) + assert isinstance(dircap, allmydata.uri.DirectoryURI) + + # DirectoryURI objects lack 'writekey' and 'fingerprint' attributes + # so extract them from the enclosed WriteableSSKFileURI object. + filecap = dircap.get_filenode_cap() + assert isinstance(filecap, allmydata.uri.WriteableSSKFileURI) + + assert (writekey, fingerprint) == (filecap.writekey, filecap.fingerprint) + + +@run_in_thread +def test_mkdir_with_known_private_key(alice): + """ + Create a new directory with ?t=mkdir&private-key=... using a + known-in-advance RSA private key. + + The writekey and fingerprint derived from the provided RSA key + should match those of the newly-created directory capability. + In addition, because the writekey and fingerprint are derived + deterministically, given the same RSA private key, the resultant + directory capability should always be the same. + """ + # Randomly generated with `openssl genrsa -out privkey.pem 2048` + privkey_pem = """-----BEGIN RSA PRIVATE KEY----- +MIIEowIBAAKCAQEAoa9i8v9YIzb+3yRHyXLm4j1eWK9lQc6lFwoQhik8y+joD+5A +v73OlDZAcn6vzlU72vwrJ1f4o54nEVm0rhNrhwCsiHCdxxEDEoqZ8w/19vc4hWj4 +SYwGirhcnyb2ysZSV8v9Lm5HiFe5zZM4jzCzf2rzt0YRlZZj9nhSglaiHZ9BE2e0 +vzOl6GePDz6yS4jbh2RsPsDQtqXNOqZwfGUd+iTsbSxXcm8+rNrT1VAbx6+1Sr0r +aDyc/jp8S1JwJ0ofJLsU3Pb6DYazFf12CNTsrKF1L0hAsbN8v2DSunZIQqQLQGfp +0hnNO9V8q9FjvVu8XY/HhgoTvtESU3vuq+BnIwIDAQABAoIBAGpWDP+/y9mtK8bZ +95SXyx10Ov6crD2xiIY0ilWR/XgmP6lqio8QaDK104D5rOpIyErnmgIQK2iAdTVG +CDyMbSWm3dIGLt5jY9/n5AQltSCtyzCCrvi/7PWC9vd9Csal1DYF5QeKY+VZvMtl +Tcduwj7EunEI1jvJYwkQbUNncsuDi+88/JNwa8DJp1IrR4goxNflGl7mNzfq49re +lhSyezfLSTZKDa3A6sYnNFAAOy82iXZuLXCqKuwRuaiFFilB0R0/egzBSUeBwMJk +sS+SvHHXwv9HsYt4pYiiZFm8HxB4NKYtdpHpvJVJcG9vOXjewnA5YHWVDJsrBfu6 +0kPgbcECgYEA0bqfX2Vc6DizwjWVn9yVlckjQNGTnwf/B9eGW2MgTn6YADe0yjFm +KCtr34hEZc/hv3kBnoLOqSvZJiser8ve3SmwxfmpjEfJdIgA5J5DbCEGBiDm9PMy +0lYsfjykzYykehdasb8f4xd+SPMuTC/CFb1MCTlohex7qn7Xt9IskBECgYEAxVtF +iXwFJPQUil2bSFGnxtaI/8ijypLOkP3CyuVnEcbMt74jDt1hdooRxjQ9VVlg7r7i +EvebPKMukWxdVcQ/38i97oB/oN7MIH0QBCDWTdTQokuNQSEknGLouj6YtLAWRcyJ +9DDENSaGtP42le5dD60hZc732jN09fGxNa6gN/MCgYB5ux98CGJ3q0mzBNUW17q/ +GOLsYXiUitidHZyveIas6M+i+LJn1WpdEG7pbLd+fL2kHEEzVutKx9efTtHd6bAu +oF8pWfLuKFCm4bXa/H1XyocrkXdcX7h0222xy9NAN0zUTK/okW2Zqu4yu2t47xNw ++NGkXPztFsjkugDNgiE5cQKBgQDDy/BqHPORnOIAACw9jF1SpKcYdPsiz5FGQawO +1ZbzCPMzW9y2M6YtD3/gzxUGZv0G/7OUs7h8aTybJBJZM7FXGHZud2ent0J2/Px1 +zAow/3DZgvEp63LCAFL5635ezM/cAbff3r3aKVW9nPOUvf3vvokC01oMTb68/kMc +ihoERwKBgFsoRUrgGPSfG1UZt8BpIXbG/8qfoy/Vy77BRqvJ6ZpdM9RPqdAl7Sih +cdqfxs8w0NVvj+gvM/1CGO0J9lZW2f1J81haIoyUpiITFdoyzLKXLhMSbaF4Y7Hn +yC/N5w3cCLa2LLKoLG8hagFDlXBGSmpT1zgKBk4YxNn6CLdMSzPR +-----END RSA PRIVATE KEY----- +""" + + privkey = load_pem_private_key( + privkey_pem.encode("ascii"), password=None + ) + pubkey = privkey.public_key() + + writekey, _, fingerprint = derive_mutable_keys((pubkey, privkey)) + + # The "private-key" parameter takes a DER-encoded RSA private key + # encoded in URL-safe base64; PEM blocks are not supported. + privkey_der = der_string_from_signing_key(privkey) + privkey_encoded = urlsafe_b64encode(privkey_der).decode("ascii") + + resp = util.web_post( + alice.process, u"uri", + params={ + u"t": "mkdir", + u"private-key": privkey_encoded, + }, + ) + assert resp.startswith(b"URI:DIR2") + + dircap = allmydata.uri.from_string(resp) + assert isinstance(dircap, allmydata.uri.DirectoryURI) + + # DirectoryURI objects lack 'writekey' and 'fingerprint' attributes + # so extract them from the enclosed WriteableSSKFileURI object. + filecap = dircap.get_filenode_cap() + assert isinstance(filecap, allmydata.uri.WriteableSSKFileURI) + + assert (writekey, fingerprint) == (filecap.writekey, filecap.fingerprint) + + assert resp == b"URI:DIR2:3oo7j7f7qqxnet2z2lf57ucup4:cpktmsxlqnd5yeekytxjxvff5e6d6fv7py6rftugcndvss7tzd2a" From 9c2362853db0cbb9dee8529977f70a6c9713d5f3 Mon Sep 17 00:00:00 2001 From: "Christopher R. Wood" Date: Fri, 3 May 2024 16:55:38 -0400 Subject: [PATCH 2282/2309] Allow supplying keypair when creating mutable dirs --- src/allmydata/client.py | 14 ++++++++++++-- src/allmydata/nodemaker.py | 11 +++++++++-- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/src/allmydata/client.py b/src/allmydata/client.py index 03bf609e9..6edbf7eeb 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -1125,8 +1125,18 @@ class _Client(node.Node, pollmixin.PollMixin): # may get an opaque node if there were any problems. return self.nodemaker.create_from_cap(write_uri, read_uri, deep_immutable=deep_immutable, name=name) - def create_dirnode(self, initial_children=None, version=None): - d = self.nodemaker.create_new_mutable_directory(initial_children, version=version) + def create_dirnode( + self, + initial_children=None, + version=None, + *, + unique_keypair: tuple[rsa.PublicKey, rsa.PrivateKey] | None = None + ): + d = self.nodemaker.create_new_mutable_directory( + initial_children, + version=version, + keypair=unique_keypair, + ) return d def create_immutable_dirnode(self, children, convergence=None): diff --git a/src/allmydata/nodemaker.py b/src/allmydata/nodemaker.py index 39663bda9..6e8700cff 100644 --- a/src/allmydata/nodemaker.py +++ b/src/allmydata/nodemaker.py @@ -135,7 +135,13 @@ class NodeMaker(object): d.addCallback(lambda res: n) return d - def create_new_mutable_directory(self, initial_children=None, version=None): + def create_new_mutable_directory( + self, + initial_children=None, + version=None, + *, + keypair: tuple[PublicKey, PrivateKey] | None = None, + ): if initial_children is None: initial_children = {} for (name, (node, metadata)) in initial_children.items(): @@ -145,7 +151,8 @@ class NodeMaker(object): d = self.create_mutable_file(lambda n: MutableData(pack_children(initial_children, n.get_writekey())), - version=version) + version=version, + keypair=keypair) d.addCallback(self._create_dirnode) return d From b93a39fdc2b6353eeb5c5f704159731fd318a371 Mon Sep 17 00:00:00 2001 From: "Christopher R. Wood" Date: Sat, 4 May 2024 09:34:43 -0400 Subject: [PATCH 2283/2309] Allow POST /uri?t=mkdir to accept `private-key` --- src/allmydata/web/unlinked.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/web/unlinked.py b/src/allmydata/web/unlinked.py index 2c7be6f30..a44e7fcb5 100644 --- a/src/allmydata/web/unlinked.py +++ b/src/allmydata/web/unlinked.py @@ -160,7 +160,7 @@ def POSTUnlinkedCreateDirectory(req, client): mt = None if file_format: mt = get_mutable_type(file_format) - d = client.create_dirnode(version=mt) + d = client.create_dirnode(version=mt, unique_keypair=get_keypair(req)) redirect = get_arg(req, "redirect_to_result", "false") if boolean_of_arg(redirect): def _then_redir(res): From 6c7ffbe30d458b0bf2029b922bacb27555b09bbe Mon Sep 17 00:00:00 2001 From: "Christopher R. Wood" Date: Tue, 7 May 2024 14:37:42 -0400 Subject: [PATCH 2284/2309] Allow mkdir-with-children to accept `private-key` --- src/allmydata/web/unlinked.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/web/unlinked.py b/src/allmydata/web/unlinked.py index a44e7fcb5..26c41c7be 100644 --- a/src/allmydata/web/unlinked.py +++ b/src/allmydata/web/unlinked.py @@ -178,7 +178,7 @@ def POSTUnlinkedCreateDirectoryWithChildren(req, client): req.content.seek(0) kids_json = req.content.read() kids = convert_children_json(client.nodemaker, kids_json) - d = client.create_dirnode(initial_children=kids) + d = client.create_dirnode(initial_children=kids, unique_keypair=get_keypair(req)) redirect = get_arg(req, "redirect_to_result", "false") if boolean_of_arg(redirect): def _then_redir(res): From 31b8f195db99b9b344b2bcbc9ab8ddf937ca4c6c Mon Sep 17 00:00:00 2001 From: "Christopher R. Wood" Date: Tue, 7 May 2024 14:49:40 -0400 Subject: [PATCH 2285/2309] Add test for mkdir-with-children with `private-key` --- integration/test_web.py | 93 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 93 insertions(+) diff --git a/integration/test_web.py b/integration/test_web.py index 94120be92..7e715122d 100644 --- a/integration/test_web.py +++ b/integration/test_web.py @@ -663,3 +663,96 @@ yC/N5w3cCLa2LLKoLG8hagFDlXBGSmpT1zgKBk4YxNn6CLdMSzPR assert (writekey, fingerprint) == (filecap.writekey, filecap.fingerprint) assert resp == b"URI:DIR2:3oo7j7f7qqxnet2z2lf57ucup4:cpktmsxlqnd5yeekytxjxvff5e6d6fv7py6rftugcndvss7tzd2a" + + +@run_in_thread +def test_mkdir_with_children_and_random_private_key(alice): + """ + Create a new directory with ?t=mkdir-with-children&private-key=... + using a randomly-generated RSA private key. + + The writekey and fingerprint derived from the provided RSA key + should match those of the newly-created directory capability. + """ + + # create a file to put in our directory + FILE_CONTENTS = u"some file contents\n" * 500 + resp = requests.put( + util.node_url(alice.process.node_dir, u"uri"), + data=FILE_CONTENTS, + ) + filecap = resp.content.strip() + + # create a (sub) directory to put in our directory + resp = requests.post( + util.node_url(alice.process.node_dir, u"uri"), + params={ + u"t": u"mkdir", + } + ) + # (we need both the read-write and read-only URIs I guess) + dircap = resp.content + dircap_obj = allmydata.uri.from_string(dircap) + dircap_ro = dircap_obj.get_readonly().to_string() + + # create json information about our directory + meta = { + "a_file": [ + "filenode", { + "ro_uri": filecap, + "metadata": { + "ctime": 1202777696.7564139, + "mtime": 1202777696.7564139, + "tahoe": { + "linkcrtime": 1202777696.7564139, + "linkmotime": 1202777696.7564139 + } + } + } + ], + "some_subdir": [ + "dirnode", { + "rw_uri": dircap, + "ro_uri": dircap_ro, + "metadata": { + "ctime": 1202778102.7589991, + "mtime": 1202778111.2160511, + "tahoe": { + "linkcrtime": 1202777696.7564139, + "linkmotime": 1202777696.7564139 + } + } + } + ] + } + + privkey, pubkey = create_signing_keypair(2048) + + writekey, _, fingerprint = derive_mutable_keys((pubkey, privkey)) + + # The "private-key" parameter takes a DER-encoded RSA private key + # encoded in URL-safe base64; PEM blocks are not supported. + privkey_der = der_string_from_signing_key(privkey) + privkey_encoded = urlsafe_b64encode(privkey_der).decode("ascii") + + # create a new directory with one file and one sub-dir (all-at-once) + # with the supplied RSA private key + resp = util.web_post( + alice.process, u"uri", + params={ + u"t": "mkdir-with-children", + u"private-key": privkey_encoded, + }, + data=json.dumps(meta), + ) + assert resp.startswith(b"URI:DIR2") + + dircap = allmydata.uri.from_string(resp) + assert isinstance(dircap, allmydata.uri.DirectoryURI) + + # DirectoryURI objects lack 'writekey' and 'fingerprint' attributes + # so extract them from the enclosed WriteableSSKFileURI object. + filecap = dircap.get_filenode_cap() + assert isinstance(filecap, allmydata.uri.WriteableSSKFileURI) + + assert (writekey, fingerprint) == (filecap.writekey, filecap.fingerprint) From 2ef6da5c4e1c04d50869e140447775ec94893ac2 Mon Sep 17 00:00:00 2001 From: "Christopher R. Wood" Date: Tue, 7 May 2024 15:15:01 -0400 Subject: [PATCH 2286/2309] Add test for mkdir-with-children with known `private-key` --- integration/test_web.py | 132 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 132 insertions(+) diff --git a/integration/test_web.py b/integration/test_web.py index 7e715122d..06fc36f1c 100644 --- a/integration/test_web.py +++ b/integration/test_web.py @@ -756,3 +756,135 @@ def test_mkdir_with_children_and_random_private_key(alice): assert isinstance(filecap, allmydata.uri.WriteableSSKFileURI) assert (writekey, fingerprint) == (filecap.writekey, filecap.fingerprint) + + +@run_in_thread +def test_mkdir_with_children_and_known_private_key(alice): + """ + Create a new directory with ?t=mkdir-with-children&private-key=... + using a known-in-advance RSA private key. + + + The writekey and fingerprint derived from the provided RSA key + should match those of the newly-created directory capability. + In addition, because the writekey and fingerprint are derived + deterministically, given the same RSA private key, the resultant + directory capability should always be the same. + """ + + # create a file to put in our directory + FILE_CONTENTS = u"some file contents\n" * 500 + resp = requests.put( + util.node_url(alice.process.node_dir, u"uri"), + data=FILE_CONTENTS, + ) + filecap = resp.content.strip() + + # create a (sub) directory to put in our directory + resp = requests.post( + util.node_url(alice.process.node_dir, u"uri"), + params={ + u"t": u"mkdir", + } + ) + # (we need both the read-write and read-only URIs I guess) + dircap = resp.content + dircap_obj = allmydata.uri.from_string(dircap) + dircap_ro = dircap_obj.get_readonly().to_string() + + # create json information about our directory + meta = { + "a_file": [ + "filenode", { + "ro_uri": filecap, + "metadata": { + "ctime": 1202777696.7564139, + "mtime": 1202777696.7564139, + "tahoe": { + "linkcrtime": 1202777696.7564139, + "linkmotime": 1202777696.7564139 + } + } + } + ], + "some_subdir": [ + "dirnode", { + "rw_uri": dircap, + "ro_uri": dircap_ro, + "metadata": { + "ctime": 1202778102.7589991, + "mtime": 1202778111.2160511, + "tahoe": { + "linkcrtime": 1202777696.7564139, + "linkmotime": 1202777696.7564139 + } + } + } + ] + } + + # Randomly generated with `openssl genrsa -out privkey.pem 2048` + privkey_pem = """-----BEGIN RSA PRIVATE KEY----- +MIIEowIBAAKCAQEA2PL5Ry2BGuuUtRJa20WS0fwBOqVIVSXDVuSvZFYTT1Xji19J +q+ohHcFnIIYHAq0zQG+NgNjK5rogY/5TfbwIhfwLufleeAdL9jXTfxan0o/wwFA1 +DAIHcYsTEYI2dfQe4acOLFY6/Hh6iXCbHvSzzUnEmYkgwCAZvc0v/lD8pMnz/6gQ +2nJnAASfFovcAvfr1T+MZzLJGQem3f2IFp1frurQyFmzFRtZMO5B9PDSsFG4yJVf +cz0iSP8wlc9QydImmJGRvu4xEOkx/55B/XaUdb6CIGpCTkLsDOlImvZt9UHDSgXq +qcE/T7SYMIXqbep64tJw9enjomH+n1KVh9UA2wIDAQABAoIBABCSTrQ/J5N010EV +i9cf810S0M03/tRyM/+ZLESPxp3Sw7TLrIbzNWBee5AibLqpnDaZzsc+yBDjusGo +lZwPFt+VJxgnki288PJ3nhYhFuSglhU6izLFnOfxZZ16wsozwYAfEJgWZh8O3N1O +uqqcqndN4TSRIu1KBm1XFQlqCkJT/stzYjO4k1vhgZT4pqhYRdx7q7FAap4v+sNs +Svhm1blvOXlyeumAbFBdGFttpTxIOGRzI1bp00jcLK4rgssTTxNyEiVu4oJhQY/k +0CptSUzpGio8DZ0/8bNnKCkw8YATUWJZQgSmKraRwAYMMR/SZa7WqjEc2KRTj6xQ +pHmYwZECgYEA700a/7ur8+EwTSulLgDveAOtTV0xEbhuq6cJQgNrEp2rbFqie6FX +g/YJKzEpEnUvj/yOzhEcw3CdQDUaxndlqY87QIhUWMcsnfMPsM1FjhmfksR8s3TF +WZNqa0RAKmcRoLohGclSvRV2OVU8+10mLUwJfR86Nl5+auR3LxWLyB8CgYEA6BaR +r+Z7oTlgkdEDVhnQ58Msktv58y28N+VIbYS79bV01jqUUlogm5uTvdvq5nyENXHx +gnK88mVzWYBMk83D01HlOC5DhpspTVEQQG2V/If6KZa56mxiHP3Mab9jLew9w/kA +g6l/04ATSA8g4i2H/Bz0eEyPEBt6o/+SO0Xv38UCgYEAyTTLvrrNmgF922UXPdcL +gp2U2bfBymSIqUuJPTgij0SDHlgWxlyieRImI2ryXdKqayav7BP3W10U2yfLm5RI +pokICPqX8Q2HNkdoqf/uu8xPn9gWAc3tIaQRlp+MVBrVd48IxeXA67tf7FT/MVrg +/rUwRUQ8bfqF0NrIW46COYECgYAYDJamGoT/DNoD4hutZVlvWpsY0LCS0U9qn1ik ++Jcde+MSe9l4uxwb48AocUxi+84bV6ZF9Su9FmQghxnoSu8ay6ar7qdSoGtkNp0v +f+uF0nVKr/Kt5vM3u9jdsFZPoOY5k2jJO9wiB2h4FBE9PqiTqFBw0sYUTjSkH8yA +VdvoXQKBgFqCC8Y82eVf0/ORGTgG/KhZ72WFQKHyAeryvoLuadZ6JAI6qW9U1l9P +18SMnCO+opGN5GH2Qx7gdg17KzWzTW1gnbv0QUPNnnYEJU8VYMelNuKa8tmNgFH7 +inAwsxbbWoR08ai4exzbJrNrLpDRg5ih2wMtknN6D8m+EAvBC/Gj +-----END RSA PRIVATE KEY----- +""" + + privkey = load_pem_private_key( + privkey_pem.encode("ascii"), password=None + ) + pubkey = privkey.public_key() + + writekey, _, fingerprint = derive_mutable_keys((pubkey, privkey)) + + # The "private-key" parameter takes a DER-encoded RSA private key + # encoded in URL-safe base64; PEM blocks are not supported. + privkey_der = der_string_from_signing_key(privkey) + privkey_encoded = urlsafe_b64encode(privkey_der).decode("ascii") + + # create a new directory with one file and one sub-dir (all-at-once) + # with the supplied RSA private key + resp = util.web_post( + alice.process, u"uri", + params={ + u"t": "mkdir-with-children", + u"private-key": privkey_encoded, + }, + data=json.dumps(meta), + ) + assert resp.startswith(b"URI:DIR2") + + dircap = allmydata.uri.from_string(resp) + assert isinstance(dircap, allmydata.uri.DirectoryURI) + + # DirectoryURI objects lack 'writekey' and 'fingerprint' attributes + # so extract them from the enclosed WriteableSSKFileURI object. + filecap = dircap.get_filenode_cap() + assert isinstance(filecap, allmydata.uri.WriteableSSKFileURI) + + assert (writekey, fingerprint) == (filecap.writekey, filecap.fingerprint) + + assert resp == b"URI:DIR2:ppwzpwrd37xi7tpribxyaa25uy:imdws47wwpzfkc5vfllo4ugspb36iit4cqps6ttuhaouc66jb2da" From 9d10bda2a04e472adb16082e677ef1c3f231a130 Mon Sep 17 00:00:00 2001 From: "Christopher R. Wood" Date: Mon, 13 May 2024 21:37:00 -0400 Subject: [PATCH 2287/2309] Document "private-key=" argument for mkdir --- docs/frontends/webapi.rst | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/docs/frontends/webapi.rst b/docs/frontends/webapi.rst index 77ce11974..b581d7aeb 100644 --- a/docs/frontends/webapi.rst +++ b/docs/frontends/webapi.rst @@ -446,6 +446,16 @@ Creating a New Directory given, the directory's format is determined by the default mutable file format, as configured on the Tahoe-LAFS node responding to the request. + In addition, an optional "private-key=" argument is supported which, if given, + specifies the underlying signing key to be used when creating the directory. + This value must be a DER-encoded 2048-bit RSA private key in urlsafe base64 + encoding. Because this key can be used to derive the write capability for the + associated directory, additional care should be taken to ensure that the key is + unique, that it is kept confidential, and that it was derived from an + appropriate (high-entropy) source of randomness. If this argument is omitted + (the default behavior), Tahoe-LAFS will generate an appropriate signing key + using the underlying operating system's source of entropy. + ``POST /uri?t=mkdir-with-children`` Create a new directory, populated with a set of child nodes, and return its @@ -453,7 +463,8 @@ Creating a New Directory any other directory: the returned write-cap is the only reference to it. The format of the directory can be controlled with the format= argument in - the query string, as described above. + the query string and a signing key can be specified with the private-key= + argument, as described above. Initial children are provided as the body of the POST form (this is more efficient than doing separate mkdir and set_children operations). If the From 5a485545f631334e60f6d6c1c747f0469f70f7d8 Mon Sep 17 00:00:00 2001 From: "Christopher R. Wood" Date: Mon, 13 May 2024 21:47:48 -0400 Subject: [PATCH 2288/2309] Add news fragment --- newsfragments/4094.feature | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/4094.feature diff --git a/newsfragments/4094.feature b/newsfragments/4094.feature new file mode 100644 index 000000000..85c98f3d5 --- /dev/null +++ b/newsfragments/4094.feature @@ -0,0 +1 @@ +Mutable directories can now be created with a pre-determined "signature key" via the web API using the "private-key=..." parameter. The "private-key" value must be a DER-encoded 2048-bit RSA private key in urlsafe base64 encoding. From 23af93cff74540cc9a487bf7190b54a2ff40155e Mon Sep 17 00:00:00 2001 From: "Christopher R. Wood" Date: Mon, 13 May 2024 22:30:04 -0400 Subject: [PATCH 2289/2309] Assert/test types of `privkey` and `pubkey` vars And appease type-checkers. --- integration/test_web.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/integration/test_web.py b/integration/test_web.py index 06fc36f1c..9f0c20c6b 100644 --- a/integration/test_web.py +++ b/integration/test_web.py @@ -22,6 +22,8 @@ import allmydata.uri from allmydata.crypto.rsa import ( create_signing_keypair, der_string_from_signing_key, + PrivateKey, + PublicKey, ) from allmydata.mutable.common import derive_mutable_keys from allmydata.util import jsonbytes as json @@ -634,7 +636,9 @@ yC/N5w3cCLa2LLKoLG8hagFDlXBGSmpT1zgKBk4YxNn6CLdMSzPR privkey = load_pem_private_key( privkey_pem.encode("ascii"), password=None ) + assert isinstance(privkey, PrivateKey) pubkey = privkey.public_key() + assert isinstance(pubkey, PublicKey) writekey, _, fingerprint = derive_mutable_keys((pubkey, privkey)) @@ -856,7 +860,9 @@ inAwsxbbWoR08ai4exzbJrNrLpDRg5ih2wMtknN6D8m+EAvBC/Gj privkey = load_pem_private_key( privkey_pem.encode("ascii"), password=None ) + assert isinstance(privkey, PrivateKey) pubkey = privkey.public_key() + assert isinstance(pubkey, PublicKey) writekey, _, fingerprint = derive_mutable_keys((pubkey, privkey)) From ed2e93582c591b275ad1a9b91f260da9a3c77b66 Mon Sep 17 00:00:00 2001 From: "Christopher R. Wood" Date: Fri, 17 May 2024 10:29:32 -0400 Subject: [PATCH 2290/2309] Add tests for creating dirnodes with given keypair --- src/allmydata/test/test_dirnode.py | 113 ++++++++++++++++++++++++++++- 1 file changed, 112 insertions(+), 1 deletion(-) diff --git a/src/allmydata/test/test_dirnode.py b/src/allmydata/test/test_dirnode.py index 30fba005f..8cfc02a9a 100644 --- a/src/allmydata/test/test_dirnode.py +++ b/src/allmydata/test/test_dirnode.py @@ -11,6 +11,7 @@ from twisted.internet import defer from twisted.internet.interfaces import IConsumer from allmydata import uri, dirnode from allmydata.client import _Client +from allmydata.crypto.rsa import create_signing_keypair from allmydata.immutable import upload from allmydata.immutable.literal import LiteralFileNode from allmydata.interfaces import IImmutableFileNode, IMutableFileNode, \ @@ -19,16 +20,25 @@ from allmydata.interfaces import IImmutableFileNode, IMutableFileNode, \ IDeepCheckResults, IDeepCheckAndRepairResults, \ MDMF_VERSION, SDMF_VERSION from allmydata.mutable.filenode import MutableFileNode -from allmydata.mutable.common import UncoordinatedWriteError +from allmydata.mutable.common import ( + UncoordinatedWriteError, + derive_mutable_keys, +) from allmydata.util import hashutil, base32 from allmydata.util.netstring import split_netstring from allmydata.monitor import Monitor from allmydata.test.common import make_chk_file_uri, make_mutable_file_uri, \ ErrorMixin +from allmydata.test.mutable.util import ( + FakeStorage, + make_nodemaker_with_peers, + make_peer, +) from allmydata.test.no_network import GridTestMixin from allmydata.unknown import UnknownNode, strip_prefix_for_ro from allmydata.nodemaker import NodeMaker from base64 import b32decode +from cryptography.hazmat.primitives.serialization import load_pem_private_key import allmydata.test.common_util as testutil from hypothesis import given @@ -1978,3 +1988,104 @@ class Adder(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin): d.addCallback(_test_adder) return d + + +class DeterministicDirnode(testutil.ReallyEqualMixin, testutil.ShouldFailMixin, unittest.TestCase): + def setUp(self): + # Copied from allmydata.test.mutable.test_filenode + super(DeterministicDirnode, self).setUp() + self._storage = FakeStorage() + self._peers = list( + make_peer(self._storage, n) + for n + in range(10) + ) + self.nodemaker = make_nodemaker_with_peers(self._peers) + + async def test_create_with_random_keypair(self): + """ + Create a dirnode using a random RSA keypair. + + The writekey and fingerprint of the enclosed mutable filecap + should match those derived from the given keypair. + """ + privkey, pubkey = create_signing_keypair(2048) + writekey, _, fingerprint = derive_mutable_keys((pubkey, privkey)) + + node = await self.nodemaker.create_new_mutable_directory( + keypair=(pubkey, privkey) + ) + self.failUnless(isinstance(node, dirnode.DirectoryNode)) + + dircap = uri.from_string(node.get_uri()) + self.failUnless(isinstance(dircap, uri.DirectoryURI)) + + filecap = dircap.get_filenode_cap() + self.failUnless(isinstance(filecap, uri.WriteableSSKFileURI)) + + self.failUnlessReallyEqual(filecap.writekey, writekey) + self.failUnlessReallyEqual(filecap.fingerprint, fingerprint) + + async def test_create_with_known_keypair(self): + """ + Create a dirnode using a known RSA keypair. + + The writekey and fingerprint of the enclosed mutable filecap + should match those derived from the given keypair. Because + these values are derived deterministically, given the same + keypair, the resulting filecap should also always be the same. + """ + # Randomly generated with `openssl genrsa -out privkey.pem 2048` + privkey_pem = """-----BEGIN RSA PRIVATE KEY----- +MIIEowIBAAKCAQEAygMjLBKayDEioOZap2syJhUlqI7Dkk4zV5TfVxlQFO7bR410 +eJRJY1rHGIeZxQPjytsSJvqlYEJrvvVNdhi6XN/6NA3RFL6pDTHkYyM3qbrXqlYC +HUlkS2JAZzIFRizl6nG11yIbHjPsoG+vGSjGSzVIiOP4NeIssYLpoASTIppdZxy+ +syZ6zSmPhZu7W9X73aupLjFrIZpjeKfO2+GfUwEzAH0HckLIgJpQ+vK3sqbSik/2 +1oZK33M8uvtdmba7D3uJXmxWMTJ7oyFLDpDOMl7HSUv1lZY2O2qiDPYfGDUM1BRp +6blxE+BA2INr9NO4A4H8pzhikFnaFnkpH/AxowIDAQABAoIBABprXJ8386w42NmI +JtT8bPuUCm/H9AXfWlGa87aVZebG8kCiXFgktJBc3+ryWQbuIk12ZyJX52b2aNb5 +h97pDv50gGlsYSrAYKWMH91jTrVQ7UGmq/IelhJR0DBu10e9OXh21JxFJpzFl63H +zXOR5JUTa+ATSHPrl4LDp0A5OPDuWbBWa64yx7gUI9/tljbndplCrPjmIE6+h10M +sqxW5oJpLnZpWc73QQUTuPIr+A7fLgGJYHnyCFUu9OW4ZnxNEI3/wNHPvoxkYuHN +2qVonFESiAx9mBv7JzQ7X2KIB8doY3KL6S7sAKi/i/aP7EDJ9QEtl3BR3M8/XP8E +KJVORWECgYEA8Vbw75+aVMxHUl9BJc1zESxqVvr+R0NBqMO47CBj39sTJkXY37O3 +A7j4dzCorI0NaB7Jr+AI2ZZu9CaR31Y2mhAGbNLBPK8yn0Z7iWyDIqOW1OpMDs35 +h2CI1pFLjx1a3PzhsQdzZ68izWKYBdTs2scaFz/ntaPwwPEwORaMDZECgYEA1kie +YfMRJ2GwzvbR35WvEMhVxhnmA6yuRL15Pkb1WDR3iWGM0ld/u3N4sRVCx1nU4wk/ +MMqCRdm4JaxqzR/hl8+/sp3Aai15ecqR+F+ecwbbB2XKVHfi1nqClivYnB+GgCh1 +bQYUd9LT80sIQdBEW5MBdbMFnOkt+1sSpjf1wfMCgYBAavlyrIJQQhqDdSN5iKY/ +HkDgKKy4rs4W0u9IL7kY5mvtGlWyGFEwcC35+oX7UMcUVKt3A3C5S3sgNi9XkraO +VtqwL20e2pDDjNeqrcku9MVs3YEhrn79UJoV08B8WdSICgPf8eIu+cNrWPbFD7mN +B/oB3K/nfvPjPD2n70nA0QKBgGWJN3NWR9SPV8ZZ8gyt0qxzISGjd/hZxKHR3jeC +TBMlmVbBoIay61WZW6EdX+0yRcvmv8iQzLXoendvgZP8/VqAGGe8lEY7kgoB0LUO +Kfh7USHqO7tWq2fR2TrrP9KKpaLoiOvGK8CzZ7cq4Ji+5QU3XUO2NnypiR5Hg0i7 +z3m9AoGBAIEXtoSR9OTwdmrdIQn3vsaFOkN5pyYfvAvdeZ+7wwMg/ZOwhStwctbI +Um7XqocXU+8f/gjczgLgMJj+zqr+QDH5n4vSTUMPeN0gIugI9UwWnc2rhbRCgDdY +W6SwPQGDuGoUa5PxjggkyevUUmtXvGG9jnkt9kozQOA0lOF1vbw/ +-----END RSA PRIVATE KEY----- +""" + privkey = load_pem_private_key( + privkey_pem.encode("ascii"), password=None + ) + pubkey = privkey.public_key() + writekey, _, fingerprint = derive_mutable_keys((pubkey, privkey)) + + node = await self.nodemaker.create_new_mutable_directory( + keypair=(pubkey, privkey) + ) + self.failUnless(isinstance(node, dirnode.DirectoryNode)) + + dircap = uri.from_string(node.get_uri()) + self.failUnless(isinstance(dircap, uri.DirectoryURI)) + + filecap = dircap.get_filenode_cap() + self.failUnless(isinstance(filecap, uri.WriteableSSKFileURI)) + + self.failUnlessReallyEqual(filecap.writekey, writekey) + self.failUnlessReallyEqual(filecap.fingerprint, fingerprint) + + self.failUnlessReallyEqual( + # Despite being named "to_string", this actually returns bytes.. + dircap.to_string(), + b'URI:DIR2:n4opqgewgcn4mddu4oiippaxru:ukpe4z6xdlujdpguoabergyih3bj7iaafukdqzwthy2ytdd5bs2a' + ) From 692be000a8b467fd50bb38727e92d07db249d481 Mon Sep 17 00:00:00 2001 From: "Christopher R. Wood" Date: Thu, 30 May 2024 15:44:21 -0400 Subject: [PATCH 2291/2309] Document converting key to DER-encoded urlsafe b64 --- docs/frontends/webapi.rst | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/docs/frontends/webapi.rst b/docs/frontends/webapi.rst index b581d7aeb..baffa412d 100644 --- a/docs/frontends/webapi.rst +++ b/docs/frontends/webapi.rst @@ -449,7 +449,12 @@ Creating a New Directory In addition, an optional "private-key=" argument is supported which, if given, specifies the underlying signing key to be used when creating the directory. This value must be a DER-encoded 2048-bit RSA private key in urlsafe base64 - encoding. Because this key can be used to derive the write capability for the + encoding. (To convert an existing PEM-encoded RSA key file into the format + required, the following commands may be used -- assuming a modern UNIX-like + environment with common tools already installed: + ``openssl rsa -in key.pem -outform der | base64 -w 0 -i - | tr '+/' '-_'``) + + Because this key can be used to derive the write capability for the associated directory, additional care should be taken to ensure that the key is unique, that it is kept confidential, and that it was derived from an appropriate (high-entropy) source of randomness. If this argument is omitted From a30a7cb4e63415e0736e0a243215843cd58248f8 Mon Sep 17 00:00:00 2001 From: "Christopher R. Wood" Date: Thu, 30 May 2024 15:48:43 -0400 Subject: [PATCH 2292/2309] Factor out inline test keys into "data" directory --- integration/test_web.py | 77 +++---------------- .../test/data/openssl-rsa-2048-2.txt | 27 +++++++ .../test/data/openssl-rsa-2048-3.txt | 27 +++++++ .../test/data/openssl-rsa-2048-4.txt | 27 +++++++ src/allmydata/test/test_dirnode.py | 36 +-------- 5 files changed, 96 insertions(+), 98 deletions(-) create mode 100644 src/allmydata/test/data/openssl-rsa-2048-2.txt create mode 100644 src/allmydata/test/data/openssl-rsa-2048-3.txt create mode 100644 src/allmydata/test/data/openssl-rsa-2048-4.txt diff --git a/integration/test_web.py b/integration/test_web.py index 9f0c20c6b..6ea365017 100644 --- a/integration/test_web.py +++ b/integration/test_web.py @@ -17,6 +17,7 @@ from urllib.parse import unquote as url_unquote, quote as url_quote from cryptography.hazmat.primitives.serialization import load_pem_private_key from twisted.internet.threads import deferToThread +from twisted.python.filepath import FilePath import allmydata.uri from allmydata.crypto.rsa import ( @@ -37,6 +38,10 @@ from bs4 import BeautifulSoup import pytest_twisted + +DATA_PATH = FilePath(__file__).parent().sibling("src").child("allmydata").child("test").child("data") + + @run_in_thread def test_index(alice): """ @@ -603,39 +608,9 @@ def test_mkdir_with_known_private_key(alice): deterministically, given the same RSA private key, the resultant directory capability should always be the same. """ - # Randomly generated with `openssl genrsa -out privkey.pem 2048` - privkey_pem = """-----BEGIN RSA PRIVATE KEY----- -MIIEowIBAAKCAQEAoa9i8v9YIzb+3yRHyXLm4j1eWK9lQc6lFwoQhik8y+joD+5A -v73OlDZAcn6vzlU72vwrJ1f4o54nEVm0rhNrhwCsiHCdxxEDEoqZ8w/19vc4hWj4 -SYwGirhcnyb2ysZSV8v9Lm5HiFe5zZM4jzCzf2rzt0YRlZZj9nhSglaiHZ9BE2e0 -vzOl6GePDz6yS4jbh2RsPsDQtqXNOqZwfGUd+iTsbSxXcm8+rNrT1VAbx6+1Sr0r -aDyc/jp8S1JwJ0ofJLsU3Pb6DYazFf12CNTsrKF1L0hAsbN8v2DSunZIQqQLQGfp -0hnNO9V8q9FjvVu8XY/HhgoTvtESU3vuq+BnIwIDAQABAoIBAGpWDP+/y9mtK8bZ -95SXyx10Ov6crD2xiIY0ilWR/XgmP6lqio8QaDK104D5rOpIyErnmgIQK2iAdTVG -CDyMbSWm3dIGLt5jY9/n5AQltSCtyzCCrvi/7PWC9vd9Csal1DYF5QeKY+VZvMtl -Tcduwj7EunEI1jvJYwkQbUNncsuDi+88/JNwa8DJp1IrR4goxNflGl7mNzfq49re -lhSyezfLSTZKDa3A6sYnNFAAOy82iXZuLXCqKuwRuaiFFilB0R0/egzBSUeBwMJk -sS+SvHHXwv9HsYt4pYiiZFm8HxB4NKYtdpHpvJVJcG9vOXjewnA5YHWVDJsrBfu6 -0kPgbcECgYEA0bqfX2Vc6DizwjWVn9yVlckjQNGTnwf/B9eGW2MgTn6YADe0yjFm -KCtr34hEZc/hv3kBnoLOqSvZJiser8ve3SmwxfmpjEfJdIgA5J5DbCEGBiDm9PMy -0lYsfjykzYykehdasb8f4xd+SPMuTC/CFb1MCTlohex7qn7Xt9IskBECgYEAxVtF -iXwFJPQUil2bSFGnxtaI/8ijypLOkP3CyuVnEcbMt74jDt1hdooRxjQ9VVlg7r7i -EvebPKMukWxdVcQ/38i97oB/oN7MIH0QBCDWTdTQokuNQSEknGLouj6YtLAWRcyJ -9DDENSaGtP42le5dD60hZc732jN09fGxNa6gN/MCgYB5ux98CGJ3q0mzBNUW17q/ -GOLsYXiUitidHZyveIas6M+i+LJn1WpdEG7pbLd+fL2kHEEzVutKx9efTtHd6bAu -oF8pWfLuKFCm4bXa/H1XyocrkXdcX7h0222xy9NAN0zUTK/okW2Zqu4yu2t47xNw -+NGkXPztFsjkugDNgiE5cQKBgQDDy/BqHPORnOIAACw9jF1SpKcYdPsiz5FGQawO -1ZbzCPMzW9y2M6YtD3/gzxUGZv0G/7OUs7h8aTybJBJZM7FXGHZud2ent0J2/Px1 -zAow/3DZgvEp63LCAFL5635ezM/cAbff3r3aKVW9nPOUvf3vvokC01oMTb68/kMc -ihoERwKBgFsoRUrgGPSfG1UZt8BpIXbG/8qfoy/Vy77BRqvJ6ZpdM9RPqdAl7Sih -cdqfxs8w0NVvj+gvM/1CGO0J9lZW2f1J81haIoyUpiITFdoyzLKXLhMSbaF4Y7Hn -yC/N5w3cCLa2LLKoLG8hagFDlXBGSmpT1zgKBk4YxNn6CLdMSzPR ------END RSA PRIVATE KEY----- -""" - - privkey = load_pem_private_key( - privkey_pem.encode("ascii"), password=None - ) + # Generated with `openssl genrsa -out openssl-rsa-2048-3.txt 2048` + pempath = DATA_PATH.child("openssl-rsa-2048-3.txt") + privkey = load_pem_private_key(pempath.getContent(), password=None) assert isinstance(privkey, PrivateKey) pubkey = privkey.public_key() assert isinstance(pubkey, PublicKey) @@ -827,39 +802,9 @@ def test_mkdir_with_children_and_known_private_key(alice): ] } - # Randomly generated with `openssl genrsa -out privkey.pem 2048` - privkey_pem = """-----BEGIN RSA PRIVATE KEY----- -MIIEowIBAAKCAQEA2PL5Ry2BGuuUtRJa20WS0fwBOqVIVSXDVuSvZFYTT1Xji19J -q+ohHcFnIIYHAq0zQG+NgNjK5rogY/5TfbwIhfwLufleeAdL9jXTfxan0o/wwFA1 -DAIHcYsTEYI2dfQe4acOLFY6/Hh6iXCbHvSzzUnEmYkgwCAZvc0v/lD8pMnz/6gQ -2nJnAASfFovcAvfr1T+MZzLJGQem3f2IFp1frurQyFmzFRtZMO5B9PDSsFG4yJVf -cz0iSP8wlc9QydImmJGRvu4xEOkx/55B/XaUdb6CIGpCTkLsDOlImvZt9UHDSgXq -qcE/T7SYMIXqbep64tJw9enjomH+n1KVh9UA2wIDAQABAoIBABCSTrQ/J5N010EV -i9cf810S0M03/tRyM/+ZLESPxp3Sw7TLrIbzNWBee5AibLqpnDaZzsc+yBDjusGo -lZwPFt+VJxgnki288PJ3nhYhFuSglhU6izLFnOfxZZ16wsozwYAfEJgWZh8O3N1O -uqqcqndN4TSRIu1KBm1XFQlqCkJT/stzYjO4k1vhgZT4pqhYRdx7q7FAap4v+sNs -Svhm1blvOXlyeumAbFBdGFttpTxIOGRzI1bp00jcLK4rgssTTxNyEiVu4oJhQY/k -0CptSUzpGio8DZ0/8bNnKCkw8YATUWJZQgSmKraRwAYMMR/SZa7WqjEc2KRTj6xQ -pHmYwZECgYEA700a/7ur8+EwTSulLgDveAOtTV0xEbhuq6cJQgNrEp2rbFqie6FX -g/YJKzEpEnUvj/yOzhEcw3CdQDUaxndlqY87QIhUWMcsnfMPsM1FjhmfksR8s3TF -WZNqa0RAKmcRoLohGclSvRV2OVU8+10mLUwJfR86Nl5+auR3LxWLyB8CgYEA6BaR -r+Z7oTlgkdEDVhnQ58Msktv58y28N+VIbYS79bV01jqUUlogm5uTvdvq5nyENXHx -gnK88mVzWYBMk83D01HlOC5DhpspTVEQQG2V/If6KZa56mxiHP3Mab9jLew9w/kA -g6l/04ATSA8g4i2H/Bz0eEyPEBt6o/+SO0Xv38UCgYEAyTTLvrrNmgF922UXPdcL -gp2U2bfBymSIqUuJPTgij0SDHlgWxlyieRImI2ryXdKqayav7BP3W10U2yfLm5RI -pokICPqX8Q2HNkdoqf/uu8xPn9gWAc3tIaQRlp+MVBrVd48IxeXA67tf7FT/MVrg -/rUwRUQ8bfqF0NrIW46COYECgYAYDJamGoT/DNoD4hutZVlvWpsY0LCS0U9qn1ik -+Jcde+MSe9l4uxwb48AocUxi+84bV6ZF9Su9FmQghxnoSu8ay6ar7qdSoGtkNp0v -f+uF0nVKr/Kt5vM3u9jdsFZPoOY5k2jJO9wiB2h4FBE9PqiTqFBw0sYUTjSkH8yA -VdvoXQKBgFqCC8Y82eVf0/ORGTgG/KhZ72WFQKHyAeryvoLuadZ6JAI6qW9U1l9P -18SMnCO+opGN5GH2Qx7gdg17KzWzTW1gnbv0QUPNnnYEJU8VYMelNuKa8tmNgFH7 -inAwsxbbWoR08ai4exzbJrNrLpDRg5ih2wMtknN6D8m+EAvBC/Gj ------END RSA PRIVATE KEY----- -""" - - privkey = load_pem_private_key( - privkey_pem.encode("ascii"), password=None - ) + # Generated with `openssl genrsa -out openssl-rsa-2048-4.txt 2048` + pempath = DATA_PATH.child("openssl-rsa-2048-4.txt") + privkey = load_pem_private_key(pempath.getContent(), password=None) assert isinstance(privkey, PrivateKey) pubkey = privkey.public_key() assert isinstance(pubkey, PublicKey) diff --git a/src/allmydata/test/data/openssl-rsa-2048-2.txt b/src/allmydata/test/data/openssl-rsa-2048-2.txt new file mode 100644 index 000000000..dd3174209 --- /dev/null +++ b/src/allmydata/test/data/openssl-rsa-2048-2.txt @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEowIBAAKCAQEAygMjLBKayDEioOZap2syJhUlqI7Dkk4zV5TfVxlQFO7bR410 +eJRJY1rHGIeZxQPjytsSJvqlYEJrvvVNdhi6XN/6NA3RFL6pDTHkYyM3qbrXqlYC +HUlkS2JAZzIFRizl6nG11yIbHjPsoG+vGSjGSzVIiOP4NeIssYLpoASTIppdZxy+ +syZ6zSmPhZu7W9X73aupLjFrIZpjeKfO2+GfUwEzAH0HckLIgJpQ+vK3sqbSik/2 +1oZK33M8uvtdmba7D3uJXmxWMTJ7oyFLDpDOMl7HSUv1lZY2O2qiDPYfGDUM1BRp +6blxE+BA2INr9NO4A4H8pzhikFnaFnkpH/AxowIDAQABAoIBABprXJ8386w42NmI +JtT8bPuUCm/H9AXfWlGa87aVZebG8kCiXFgktJBc3+ryWQbuIk12ZyJX52b2aNb5 +h97pDv50gGlsYSrAYKWMH91jTrVQ7UGmq/IelhJR0DBu10e9OXh21JxFJpzFl63H +zXOR5JUTa+ATSHPrl4LDp0A5OPDuWbBWa64yx7gUI9/tljbndplCrPjmIE6+h10M +sqxW5oJpLnZpWc73QQUTuPIr+A7fLgGJYHnyCFUu9OW4ZnxNEI3/wNHPvoxkYuHN +2qVonFESiAx9mBv7JzQ7X2KIB8doY3KL6S7sAKi/i/aP7EDJ9QEtl3BR3M8/XP8E +KJVORWECgYEA8Vbw75+aVMxHUl9BJc1zESxqVvr+R0NBqMO47CBj39sTJkXY37O3 +A7j4dzCorI0NaB7Jr+AI2ZZu9CaR31Y2mhAGbNLBPK8yn0Z7iWyDIqOW1OpMDs35 +h2CI1pFLjx1a3PzhsQdzZ68izWKYBdTs2scaFz/ntaPwwPEwORaMDZECgYEA1kie +YfMRJ2GwzvbR35WvEMhVxhnmA6yuRL15Pkb1WDR3iWGM0ld/u3N4sRVCx1nU4wk/ +MMqCRdm4JaxqzR/hl8+/sp3Aai15ecqR+F+ecwbbB2XKVHfi1nqClivYnB+GgCh1 +bQYUd9LT80sIQdBEW5MBdbMFnOkt+1sSpjf1wfMCgYBAavlyrIJQQhqDdSN5iKY/ +HkDgKKy4rs4W0u9IL7kY5mvtGlWyGFEwcC35+oX7UMcUVKt3A3C5S3sgNi9XkraO +VtqwL20e2pDDjNeqrcku9MVs3YEhrn79UJoV08B8WdSICgPf8eIu+cNrWPbFD7mN +B/oB3K/nfvPjPD2n70nA0QKBgGWJN3NWR9SPV8ZZ8gyt0qxzISGjd/hZxKHR3jeC +TBMlmVbBoIay61WZW6EdX+0yRcvmv8iQzLXoendvgZP8/VqAGGe8lEY7kgoB0LUO +Kfh7USHqO7tWq2fR2TrrP9KKpaLoiOvGK8CzZ7cq4Ji+5QU3XUO2NnypiR5Hg0i7 +z3m9AoGBAIEXtoSR9OTwdmrdIQn3vsaFOkN5pyYfvAvdeZ+7wwMg/ZOwhStwctbI +Um7XqocXU+8f/gjczgLgMJj+zqr+QDH5n4vSTUMPeN0gIugI9UwWnc2rhbRCgDdY +W6SwPQGDuGoUa5PxjggkyevUUmtXvGG9jnkt9kozQOA0lOF1vbw/ +-----END RSA PRIVATE KEY----- diff --git a/src/allmydata/test/data/openssl-rsa-2048-3.txt b/src/allmydata/test/data/openssl-rsa-2048-3.txt new file mode 100644 index 000000000..2c423dc1f --- /dev/null +++ b/src/allmydata/test/data/openssl-rsa-2048-3.txt @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEowIBAAKCAQEAoa9i8v9YIzb+3yRHyXLm4j1eWK9lQc6lFwoQhik8y+joD+5A +v73OlDZAcn6vzlU72vwrJ1f4o54nEVm0rhNrhwCsiHCdxxEDEoqZ8w/19vc4hWj4 +SYwGirhcnyb2ysZSV8v9Lm5HiFe5zZM4jzCzf2rzt0YRlZZj9nhSglaiHZ9BE2e0 +vzOl6GePDz6yS4jbh2RsPsDQtqXNOqZwfGUd+iTsbSxXcm8+rNrT1VAbx6+1Sr0r +aDyc/jp8S1JwJ0ofJLsU3Pb6DYazFf12CNTsrKF1L0hAsbN8v2DSunZIQqQLQGfp +0hnNO9V8q9FjvVu8XY/HhgoTvtESU3vuq+BnIwIDAQABAoIBAGpWDP+/y9mtK8bZ +95SXyx10Ov6crD2xiIY0ilWR/XgmP6lqio8QaDK104D5rOpIyErnmgIQK2iAdTVG +CDyMbSWm3dIGLt5jY9/n5AQltSCtyzCCrvi/7PWC9vd9Csal1DYF5QeKY+VZvMtl +Tcduwj7EunEI1jvJYwkQbUNncsuDi+88/JNwa8DJp1IrR4goxNflGl7mNzfq49re +lhSyezfLSTZKDa3A6sYnNFAAOy82iXZuLXCqKuwRuaiFFilB0R0/egzBSUeBwMJk +sS+SvHHXwv9HsYt4pYiiZFm8HxB4NKYtdpHpvJVJcG9vOXjewnA5YHWVDJsrBfu6 +0kPgbcECgYEA0bqfX2Vc6DizwjWVn9yVlckjQNGTnwf/B9eGW2MgTn6YADe0yjFm +KCtr34hEZc/hv3kBnoLOqSvZJiser8ve3SmwxfmpjEfJdIgA5J5DbCEGBiDm9PMy +0lYsfjykzYykehdasb8f4xd+SPMuTC/CFb1MCTlohex7qn7Xt9IskBECgYEAxVtF +iXwFJPQUil2bSFGnxtaI/8ijypLOkP3CyuVnEcbMt74jDt1hdooRxjQ9VVlg7r7i +EvebPKMukWxdVcQ/38i97oB/oN7MIH0QBCDWTdTQokuNQSEknGLouj6YtLAWRcyJ +9DDENSaGtP42le5dD60hZc732jN09fGxNa6gN/MCgYB5ux98CGJ3q0mzBNUW17q/ +GOLsYXiUitidHZyveIas6M+i+LJn1WpdEG7pbLd+fL2kHEEzVutKx9efTtHd6bAu +oF8pWfLuKFCm4bXa/H1XyocrkXdcX7h0222xy9NAN0zUTK/okW2Zqu4yu2t47xNw ++NGkXPztFsjkugDNgiE5cQKBgQDDy/BqHPORnOIAACw9jF1SpKcYdPsiz5FGQawO +1ZbzCPMzW9y2M6YtD3/gzxUGZv0G/7OUs7h8aTybJBJZM7FXGHZud2ent0J2/Px1 +zAow/3DZgvEp63LCAFL5635ezM/cAbff3r3aKVW9nPOUvf3vvokC01oMTb68/kMc +ihoERwKBgFsoRUrgGPSfG1UZt8BpIXbG/8qfoy/Vy77BRqvJ6ZpdM9RPqdAl7Sih +cdqfxs8w0NVvj+gvM/1CGO0J9lZW2f1J81haIoyUpiITFdoyzLKXLhMSbaF4Y7Hn +yC/N5w3cCLa2LLKoLG8hagFDlXBGSmpT1zgKBk4YxNn6CLdMSzPR +-----END RSA PRIVATE KEY----- diff --git a/src/allmydata/test/data/openssl-rsa-2048-4.txt b/src/allmydata/test/data/openssl-rsa-2048-4.txt new file mode 100644 index 000000000..534ae30bc --- /dev/null +++ b/src/allmydata/test/data/openssl-rsa-2048-4.txt @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEowIBAAKCAQEA2PL5Ry2BGuuUtRJa20WS0fwBOqVIVSXDVuSvZFYTT1Xji19J +q+ohHcFnIIYHAq0zQG+NgNjK5rogY/5TfbwIhfwLufleeAdL9jXTfxan0o/wwFA1 +DAIHcYsTEYI2dfQe4acOLFY6/Hh6iXCbHvSzzUnEmYkgwCAZvc0v/lD8pMnz/6gQ +2nJnAASfFovcAvfr1T+MZzLJGQem3f2IFp1frurQyFmzFRtZMO5B9PDSsFG4yJVf +cz0iSP8wlc9QydImmJGRvu4xEOkx/55B/XaUdb6CIGpCTkLsDOlImvZt9UHDSgXq +qcE/T7SYMIXqbep64tJw9enjomH+n1KVh9UA2wIDAQABAoIBABCSTrQ/J5N010EV +i9cf810S0M03/tRyM/+ZLESPxp3Sw7TLrIbzNWBee5AibLqpnDaZzsc+yBDjusGo +lZwPFt+VJxgnki288PJ3nhYhFuSglhU6izLFnOfxZZ16wsozwYAfEJgWZh8O3N1O +uqqcqndN4TSRIu1KBm1XFQlqCkJT/stzYjO4k1vhgZT4pqhYRdx7q7FAap4v+sNs +Svhm1blvOXlyeumAbFBdGFttpTxIOGRzI1bp00jcLK4rgssTTxNyEiVu4oJhQY/k +0CptSUzpGio8DZ0/8bNnKCkw8YATUWJZQgSmKraRwAYMMR/SZa7WqjEc2KRTj6xQ +pHmYwZECgYEA700a/7ur8+EwTSulLgDveAOtTV0xEbhuq6cJQgNrEp2rbFqie6FX +g/YJKzEpEnUvj/yOzhEcw3CdQDUaxndlqY87QIhUWMcsnfMPsM1FjhmfksR8s3TF +WZNqa0RAKmcRoLohGclSvRV2OVU8+10mLUwJfR86Nl5+auR3LxWLyB8CgYEA6BaR +r+Z7oTlgkdEDVhnQ58Msktv58y28N+VIbYS79bV01jqUUlogm5uTvdvq5nyENXHx +gnK88mVzWYBMk83D01HlOC5DhpspTVEQQG2V/If6KZa56mxiHP3Mab9jLew9w/kA +g6l/04ATSA8g4i2H/Bz0eEyPEBt6o/+SO0Xv38UCgYEAyTTLvrrNmgF922UXPdcL +gp2U2bfBymSIqUuJPTgij0SDHlgWxlyieRImI2ryXdKqayav7BP3W10U2yfLm5RI +pokICPqX8Q2HNkdoqf/uu8xPn9gWAc3tIaQRlp+MVBrVd48IxeXA67tf7FT/MVrg +/rUwRUQ8bfqF0NrIW46COYECgYAYDJamGoT/DNoD4hutZVlvWpsY0LCS0U9qn1ik ++Jcde+MSe9l4uxwb48AocUxi+84bV6ZF9Su9FmQghxnoSu8ay6ar7qdSoGtkNp0v +f+uF0nVKr/Kt5vM3u9jdsFZPoOY5k2jJO9wiB2h4FBE9PqiTqFBw0sYUTjSkH8yA +VdvoXQKBgFqCC8Y82eVf0/ORGTgG/KhZ72WFQKHyAeryvoLuadZ6JAI6qW9U1l9P +18SMnCO+opGN5GH2Qx7gdg17KzWzTW1gnbv0QUPNnnYEJU8VYMelNuKa8tmNgFH7 +inAwsxbbWoR08ai4exzbJrNrLpDRg5ih2wMtknN6D8m+EAvBC/Gj +-----END RSA PRIVATE KEY----- diff --git a/src/allmydata/test/test_dirnode.py b/src/allmydata/test/test_dirnode.py index 8cfc02a9a..93122ba19 100644 --- a/src/allmydata/test/test_dirnode.py +++ b/src/allmydata/test/test_dirnode.py @@ -9,6 +9,7 @@ from zope.interface import implementer from twisted.trial import unittest from twisted.internet import defer from twisted.internet.interfaces import IConsumer +from twisted.python.filepath import FilePath from allmydata import uri, dirnode from allmydata.client import _Client from allmydata.crypto.rsa import create_signing_keypair @@ -2035,38 +2036,9 @@ class DeterministicDirnode(testutil.ReallyEqualMixin, testutil.ShouldFailMixin, these values are derived deterministically, given the same keypair, the resulting filecap should also always be the same. """ - # Randomly generated with `openssl genrsa -out privkey.pem 2048` - privkey_pem = """-----BEGIN RSA PRIVATE KEY----- -MIIEowIBAAKCAQEAygMjLBKayDEioOZap2syJhUlqI7Dkk4zV5TfVxlQFO7bR410 -eJRJY1rHGIeZxQPjytsSJvqlYEJrvvVNdhi6XN/6NA3RFL6pDTHkYyM3qbrXqlYC -HUlkS2JAZzIFRizl6nG11yIbHjPsoG+vGSjGSzVIiOP4NeIssYLpoASTIppdZxy+ -syZ6zSmPhZu7W9X73aupLjFrIZpjeKfO2+GfUwEzAH0HckLIgJpQ+vK3sqbSik/2 -1oZK33M8uvtdmba7D3uJXmxWMTJ7oyFLDpDOMl7HSUv1lZY2O2qiDPYfGDUM1BRp -6blxE+BA2INr9NO4A4H8pzhikFnaFnkpH/AxowIDAQABAoIBABprXJ8386w42NmI -JtT8bPuUCm/H9AXfWlGa87aVZebG8kCiXFgktJBc3+ryWQbuIk12ZyJX52b2aNb5 -h97pDv50gGlsYSrAYKWMH91jTrVQ7UGmq/IelhJR0DBu10e9OXh21JxFJpzFl63H -zXOR5JUTa+ATSHPrl4LDp0A5OPDuWbBWa64yx7gUI9/tljbndplCrPjmIE6+h10M -sqxW5oJpLnZpWc73QQUTuPIr+A7fLgGJYHnyCFUu9OW4ZnxNEI3/wNHPvoxkYuHN -2qVonFESiAx9mBv7JzQ7X2KIB8doY3KL6S7sAKi/i/aP7EDJ9QEtl3BR3M8/XP8E -KJVORWECgYEA8Vbw75+aVMxHUl9BJc1zESxqVvr+R0NBqMO47CBj39sTJkXY37O3 -A7j4dzCorI0NaB7Jr+AI2ZZu9CaR31Y2mhAGbNLBPK8yn0Z7iWyDIqOW1OpMDs35 -h2CI1pFLjx1a3PzhsQdzZ68izWKYBdTs2scaFz/ntaPwwPEwORaMDZECgYEA1kie -YfMRJ2GwzvbR35WvEMhVxhnmA6yuRL15Pkb1WDR3iWGM0ld/u3N4sRVCx1nU4wk/ -MMqCRdm4JaxqzR/hl8+/sp3Aai15ecqR+F+ecwbbB2XKVHfi1nqClivYnB+GgCh1 -bQYUd9LT80sIQdBEW5MBdbMFnOkt+1sSpjf1wfMCgYBAavlyrIJQQhqDdSN5iKY/ -HkDgKKy4rs4W0u9IL7kY5mvtGlWyGFEwcC35+oX7UMcUVKt3A3C5S3sgNi9XkraO -VtqwL20e2pDDjNeqrcku9MVs3YEhrn79UJoV08B8WdSICgPf8eIu+cNrWPbFD7mN -B/oB3K/nfvPjPD2n70nA0QKBgGWJN3NWR9SPV8ZZ8gyt0qxzISGjd/hZxKHR3jeC -TBMlmVbBoIay61WZW6EdX+0yRcvmv8iQzLXoendvgZP8/VqAGGe8lEY7kgoB0LUO -Kfh7USHqO7tWq2fR2TrrP9KKpaLoiOvGK8CzZ7cq4Ji+5QU3XUO2NnypiR5Hg0i7 -z3m9AoGBAIEXtoSR9OTwdmrdIQn3vsaFOkN5pyYfvAvdeZ+7wwMg/ZOwhStwctbI -Um7XqocXU+8f/gjczgLgMJj+zqr+QDH5n4vSTUMPeN0gIugI9UwWnc2rhbRCgDdY -W6SwPQGDuGoUa5PxjggkyevUUmtXvGG9jnkt9kozQOA0lOF1vbw/ ------END RSA PRIVATE KEY----- -""" - privkey = load_pem_private_key( - privkey_pem.encode("ascii"), password=None - ) + # Generated with `openssl genrsa -out openssl-rsa-2048-2.txt 2048` + pempath = FilePath(__file__).sibling("data").child("openssl-rsa-2048-2.txt") + privkey = load_pem_private_key(pempath.getContent(), password=None) pubkey = privkey.public_key() writekey, _, fingerprint = derive_mutable_keys((pubkey, privkey)) From f694224aba0adec125786097d744a694fc72ec0f Mon Sep 17 00:00:00 2001 From: "Christopher R. Wood" Date: Thu, 30 May 2024 15:50:40 -0400 Subject: [PATCH 2293/2309] Add docstring for `create_dirnode` --- src/allmydata/client.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/src/allmydata/client.py b/src/allmydata/client.py index 6edbf7eeb..e942efcc0 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -1132,6 +1132,32 @@ class _Client(node.Node, pollmixin.PollMixin): *, unique_keypair: tuple[rsa.PublicKey, rsa.PrivateKey] | None = None ): + """ + Create a new directory. + + :param initial_children: If given, a structured dict representing the + initial content of the created directory. See + `docs/frontends/webapi.rst` for examples. + + :param version: If given, an int representing the mutable file format + of the new object. Acceptable values are currently `SDMF_VERSION` + or `MDMF_VERSION` (corresponding to 0 or 1, respectively, as + defined in `allmydata.interfaces`). If no such value is provided, + the default mutable format will be used (currently SDMF). + + :param unique_keypair: an optional tuple containing the RSA public + and private key to be used for the new directory. Typically, this + value is omitted (in which case a new random keypair will be + generated at creation time). + + **Warning** This value independently determines the identity of + the mutable object to create. There cannot be two different + mutable objects that share a keypair. They will merge into one + object (with undefined contents). + + :return: A Deferred which will fire with a representation of the new + directory after it has been created. + """ d = self.nodemaker.create_new_mutable_directory( initial_children, version=version, From 00f82a46ae5fae5a706c75fc204a1f002dc13bce Mon Sep 17 00:00:00 2001 From: "Christopher R. Wood" Date: Thu, 30 May 2024 15:51:24 -0400 Subject: [PATCH 2294/2309] Expand type hints for `create_dirnode` --- src/allmydata/client.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/allmydata/client.py b/src/allmydata/client.py index e942efcc0..48f372b05 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -32,6 +32,7 @@ import allmydata from allmydata import node from allmydata.crypto import rsa, ed25519 from allmydata.crypto.util import remove_prefix +from allmydata.dirnode import DirectoryNode from allmydata.storage.server import StorageServer, FoolscapStorageServer from allmydata import storage_client from allmydata.immutable.upload import Uploader @@ -1127,11 +1128,11 @@ class _Client(node.Node, pollmixin.PollMixin): def create_dirnode( self, - initial_children=None, - version=None, + initial_children: dict | None = None, + version: int | None = None, *, unique_keypair: tuple[rsa.PublicKey, rsa.PrivateKey] | None = None - ): + ) -> DirectoryNode: """ Create a new directory. From 5f4d5de739f2524a21dc23511536092f148c4dd1 Mon Sep 17 00:00:00 2001 From: Florian Sesser Date: Thu, 20 Jun 2024 13:30:50 +0000 Subject: [PATCH 2295/2309] Move GBS specification ... from 'docs/proposed' to 'docs/specification'. Fixes ticket:4019 --- docs/proposed/index.rst | 1 - docs/{proposed => specifications}/http-storage-node-protocol.rst | 0 docs/specifications/index.rst | 1 + 3 files changed, 1 insertion(+), 1 deletion(-) rename docs/{proposed => specifications}/http-storage-node-protocol.rst (100%) diff --git a/docs/proposed/index.rst b/docs/proposed/index.rst index d01d92d2d..f0bb2f344 100644 --- a/docs/proposed/index.rst +++ b/docs/proposed/index.rst @@ -14,4 +14,3 @@ index only lists the files that are in .rst format. :maxdepth: 2 leasedb - http-storage-node-protocol diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/specifications/http-storage-node-protocol.rst similarity index 100% rename from docs/proposed/http-storage-node-protocol.rst rename to docs/specifications/http-storage-node-protocol.rst diff --git a/docs/specifications/index.rst b/docs/specifications/index.rst index e813acf07..4f71dc0dc 100644 --- a/docs/specifications/index.rst +++ b/docs/specifications/index.rst @@ -17,3 +17,4 @@ the data formats used by Tahoe. lease servers-of-happiness backends/raic + http-storage-node-protocol From 8fb4de0bd582abe6f4b807e2f338ef50f4867b7b Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 8 Aug 2024 15:47:06 -0600 Subject: [PATCH 2296/2309] re-use 4093 --- newsfragments/4093.minor | 1 + 1 file changed, 1 insertion(+) diff --git a/newsfragments/4093.minor b/newsfragments/4093.minor index e69de29bb..8b1378917 100644 --- a/newsfragments/4093.minor +++ b/newsfragments/4093.minor @@ -0,0 +1 @@ + From 4d6fcc8ad1e89f6fb28205cc87634a91d15168a5 Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 8 Aug 2024 15:52:17 -0600 Subject: [PATCH 2297/2309] import internal attr.provides --- src/allmydata/util/eliotutil.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/allmydata/util/eliotutil.py b/src/allmydata/util/eliotutil.py index f0d5bffd2..387dcaefe 100644 --- a/src/allmydata/util/eliotutil.py +++ b/src/allmydata/util/eliotutil.py @@ -32,10 +32,8 @@ from zope.interface import ( ) import attr -from attr.validators import ( - optional, - provides, -) +from attr.validators import optional +from allmydata.util.attrs_provides import provides from twisted.internet import reactor from eliot import ( ILogger, From 18ba8ef6571ec0b59dbd560143934d789a17badf Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 8 Aug 2024 16:26:14 -0600 Subject: [PATCH 2298/2309] workaround; pin cryptography below 43 --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 71be1e2e1..e2c3ce6a6 100644 --- a/setup.py +++ b/setup.py @@ -62,7 +62,8 @@ install_requires = [ # Twisted[conch] also depends on cryptography and Twisted[tls] # transitively depends on cryptography. So it's anyone's guess what # version of cryptography will *really* be installed. - "cryptography >= 2.6", + # * cryptography 43.0.0 makes __provides__ read-only; see ticket 4300 + "cryptography >= 2.6, < 43.0.0", # * Used for custom HTTPS validation "pyOpenSSL >= 23.2.0", From 659014d4d3a14c6c95241d8be149942b325ad263 Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 8 Aug 2024 16:37:42 -0600 Subject: [PATCH 2299/2309] types-pkg_resources yanked; use types-setuptools instead --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 9a2d7d5b2..5748928fe 100644 --- a/tox.ini +++ b/tox.ini @@ -129,7 +129,7 @@ deps = types-mock types-six types-PyYAML - types-pkg_resources + types-setuptools types-pyOpenSSL foolscap # Upgrade when new releases come out: From 4d34c775de03ef47b9af9903f26f6a6147fd3fa0 Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 8 Aug 2024 16:39:05 -0600 Subject: [PATCH 2300/2309] someone missed moving this link --- docs/specifications/url.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/specifications/url.rst b/docs/specifications/url.rst index 12e2b8642..14c58201c 100644 --- a/docs/specifications/url.rst +++ b/docs/specifications/url.rst @@ -40,7 +40,7 @@ NURLs The authentication and authorization properties of fURLs are a good fit for Tahoe-LAFS' requirements. These are not inherently tied to the Foolscap protocol itself. -In particular they are beneficial to :doc:`../proposed/http-storage-node-protocol` which uses HTTP instead of Foolscap. +In particular they are beneficial to :doc:`http-storage-node-protocol` which uses HTTP instead of Foolscap. It is conceivable they will also be used with WebSockets at some point as well. Continuing to refer to these URLs as fURLs when they are being used for other protocols may cause confusion. From 38fb0894229bf51b86aeeef3d7322a72a9b3ac4f Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 8 Aug 2024 16:58:53 -0600 Subject: [PATCH 2301/2309] Do what 'typing' says I guess? --- src/allmydata/storage/server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/storage/server.py b/src/allmydata/storage/server.py index e734f9d74..5162faf0d 100644 --- a/src/allmydata/storage/server.py +++ b/src/allmydata/storage/server.py @@ -953,5 +953,5 @@ def get_corruption_report_path( # windows can't handle colons in the filename return os.path.join( base_dir, - ("%s--%s-%d" % (now, str(si_s, "utf-8"), shnum)).replace(":","") + ("%s--%s-%d" % (now, str(si_s), shnum)).replace(":","") ) From 7042442c97a34886f5c2fa518f2d3bd0ee83c7c3 Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 8 Aug 2024 17:10:30 -0600 Subject: [PATCH 2302/2309] should always be bytes --- src/allmydata/storage/server.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/storage/server.py b/src/allmydata/storage/server.py index 5162faf0d..9e9fb4b47 100644 --- a/src/allmydata/storage/server.py +++ b/src/allmydata/storage/server.py @@ -788,7 +788,7 @@ class StorageServer(service.MultiService): report_path = get_corruption_report_path( self.corruption_advisory_dir, now, - si_s, + si_s.decode("utf8"), shnum, ) with open(report_path, "w", encoding="utf-8") as f: @@ -953,5 +953,5 @@ def get_corruption_report_path( # windows can't handle colons in the filename return os.path.join( base_dir, - ("%s--%s-%d" % (now, str(si_s), shnum)).replace(":","") + ("%s--%s-%d" % (now, si_s, shnum)).replace(":","") ) From 1539c8dd31b12dada6379ef65f7cac0f520ae59d Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 13 Aug 2024 11:23:14 -0400 Subject: [PATCH 2303/2309] Remove usage of no longer available attrs API --- src/allmydata/util/eliotutil.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/allmydata/util/eliotutil.py b/src/allmydata/util/eliotutil.py index f0d5bffd2..6a43a7b74 100644 --- a/src/allmydata/util/eliotutil.py +++ b/src/allmydata/util/eliotutil.py @@ -32,10 +32,7 @@ from zope.interface import ( ) import attr -from attr.validators import ( - optional, - provides, -) +from attr.validators import optional from twisted.internet import reactor from eliot import ( ILogger, @@ -76,6 +73,7 @@ from twisted.internet.defer import ( ) from twisted.application.service import MultiService +from .attrs_provides import provides from .jsonbytes import AnyBytesJSONEncoder From b90c397649bd5f05441f4dac2f32594986947d92 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 13 Aug 2024 11:25:08 -0400 Subject: [PATCH 2304/2309] News fragment. --- newsfragments/4101.bugfix | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/4101.bugfix diff --git a/newsfragments/4101.bugfix b/newsfragments/4101.bugfix new file mode 100644 index 000000000..b03ca46d6 --- /dev/null +++ b/newsfragments/4101.bugfix @@ -0,0 +1 @@ +Fix incompatibility with attrs 24.1. \ No newline at end of file From bb8c60278f29cd3c6503008f3e48f31feff4e9f9 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 13 Aug 2024 11:25:43 -0400 Subject: [PATCH 2305/2309] News fragment. --- newsfragments/4100.bugfix | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/4100.bugfix diff --git a/newsfragments/4100.bugfix b/newsfragments/4100.bugfix new file mode 100644 index 000000000..d580108ca --- /dev/null +++ b/newsfragments/4100.bugfix @@ -0,0 +1 @@ +Fix incompatibility with cryptography 43. \ No newline at end of file From 43626bf46d46d8963a9048ea8e7a5c85fe04e755 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 13 Aug 2024 12:11:20 -0400 Subject: [PATCH 2306/2309] Stop using directlyProvides() --- src/allmydata/crypto/aes.py | 65 +++++++++++++------------------------ 1 file changed, 23 insertions(+), 42 deletions(-) diff --git a/src/allmydata/crypto/aes.py b/src/allmydata/crypto/aes.py index db9064ca8..5c512c980 100644 --- a/src/allmydata/crypto/aes.py +++ b/src/allmydata/crypto/aes.py @@ -10,6 +10,9 @@ objects that `cryptography` documents. Ported to Python 3. """ +from dataclasses import dataclass +from typing import Optional + from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives.ciphers import ( Cipher, @@ -17,34 +20,34 @@ from cryptography.hazmat.primitives.ciphers import ( modes, CipherContext, ) -from zope.interface import ( - Interface, - directlyProvides, -) DEFAULT_IV = b'\x00' * 16 -class IEncryptor(Interface): +@dataclass +class Encryptor: """ An object which can encrypt data. Create one using :func:`create_encryptor` and use it with :func:`encrypt_data` """ + encrypt_context: CipherContext -class IDecryptor(Interface): +@dataclass +class Decryptor: """ An object which can decrypt data. Create one using :func:`create_decryptor` and use it with :func:`decrypt_data` """ + decrypt_context: CipherContext -def create_encryptor(key, iv=None): +def create_encryptor(key: bytes, iv: Optional[bytes]=None) -> Encryptor: """ Create and return a new object which can do AES encryptions with the given key and initialization vector (IV). The default IV is 16 @@ -57,33 +60,30 @@ def create_encryptor(key, iv=None): or None for the default (which is 16 zero bytes) :returns: an object suitable for use with :func:`encrypt_data` (an - :class:`IEncryptor`) + :class:`Encryptor`) """ cryptor = _create_cryptor(key, iv) - directlyProvides(cryptor, IEncryptor) - return cryptor + return Encryptor(cryptor) -def encrypt_data(encryptor, plaintext): +def encrypt_data(encryptor: Encryptor, plaintext: bytes) -> bytes: """ AES-encrypt `plaintext` with the given `encryptor`. - :param encryptor: an instance of :class:`IEncryptor` previously + :param encryptor: an instance of :class:`Encryptor` previously returned from `create_encryptor` :param bytes plaintext: the data to encrypt :returns: bytes of ciphertext """ - - _validate_cryptor(encryptor, encrypt=True) if not isinstance(plaintext, (bytes, memoryview)): raise ValueError(f'Plaintext must be bytes or memoryview: {type(plaintext)}') - return encryptor.update(plaintext) + return encryptor.encrypt_context.update(plaintext) -def create_decryptor(key, iv=None): +def create_decryptor(key: bytes, iv: Optional[bytes]=None) -> Encryptor: """ Create and return a new object which can do AES decryptions with the given key and initialization vector (IV). The default IV is 16 @@ -96,33 +96,30 @@ def create_decryptor(key, iv=None): or None for the default (which is 16 zero bytes) :returns: an object suitable for use with :func:`decrypt_data` (an - :class:`IDecryptor` instance) + :class:`Decryptor` instance) """ cryptor = _create_cryptor(key, iv) - directlyProvides(cryptor, IDecryptor) - return cryptor + return Decryptor(cryptor) -def decrypt_data(decryptor, plaintext): +def decrypt_data(decryptor: Decryptor, plaintext: bytes) -> bytes: """ AES-decrypt `plaintext` with the given `decryptor`. - :param decryptor: an instance of :class:`IDecryptor` previously + :param decryptor: an instance of :class:`Decryptor` previously returned from `create_decryptor` :param bytes plaintext: the data to decrypt :returns: bytes of ciphertext """ - - _validate_cryptor(decryptor, encrypt=False) if not isinstance(plaintext, (bytes, memoryview)): raise ValueError(f'Plaintext must be bytes or memoryview: {type(plaintext)}') - return decryptor.update(plaintext) + return decryptor.decrypt_context.update(plaintext) -def _create_cryptor(key, iv): +def _create_cryptor(key: bytes, iv: Optional[bytes]) -> CipherContext: """ Internal helper. @@ -135,23 +132,7 @@ def _create_cryptor(key, iv): modes.CTR(iv), backend=default_backend() ) - return cipher.encryptor() - - -def _validate_cryptor(cryptor, encrypt=True): - """ - raise ValueError if `cryptor` is not a valid object - """ - klass = IEncryptor if encrypt else IDecryptor - name = "encryptor" if encrypt else "decryptor" - if not isinstance(cryptor, CipherContext): - raise ValueError( - "'{}' must be a CipherContext".format(name) - ) - if not klass.providedBy(cryptor): - raise ValueError( - "'{}' must be created with create_{}()".format(name, name) - ) + return cipher.encryptor() # type: ignore[return-type] def _validate_key(key): From 50fe6b324976234d7b62ce83602a0ba59bb1cdb9 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 13 Aug 2024 12:13:01 -0400 Subject: [PATCH 2307/2309] Not needed --- tox.ini | 1 - 1 file changed, 1 deletion(-) diff --git a/tox.ini b/tox.ini index 9a2d7d5b2..0e50d6f6d 100644 --- a/tox.ini +++ b/tox.ini @@ -129,7 +129,6 @@ deps = types-mock types-six types-PyYAML - types-pkg_resources types-pyOpenSSL foolscap # Upgrade when new releases come out: From ee2b932c6a492eae2d1fde1c2ffcb0cd9e5071eb Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 13 Aug 2024 12:16:30 -0400 Subject: [PATCH 2308/2309] Correct type annotations --- src/allmydata/crypto/aes.py | 6 +++--- src/allmydata/mutable/common.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/allmydata/crypto/aes.py b/src/allmydata/crypto/aes.py index 5c512c980..4119f080b 100644 --- a/src/allmydata/crypto/aes.py +++ b/src/allmydata/crypto/aes.py @@ -83,7 +83,7 @@ def encrypt_data(encryptor: Encryptor, plaintext: bytes) -> bytes: return encryptor.encrypt_context.update(plaintext) -def create_decryptor(key: bytes, iv: Optional[bytes]=None) -> Encryptor: +def create_decryptor(key: bytes, iv: Optional[bytes]=None) -> Decryptor: """ Create and return a new object which can do AES decryptions with the given key and initialization vector (IV). The default IV is 16 @@ -135,7 +135,7 @@ def _create_cryptor(key: bytes, iv: Optional[bytes]) -> CipherContext: return cipher.encryptor() # type: ignore[return-type] -def _validate_key(key): +def _validate_key(key: bytes) -> bytes: """ confirm `key` is suitable for AES encryption, or raise ValueError """ @@ -146,7 +146,7 @@ def _validate_key(key): return key -def _validate_iv(iv): +def _validate_iv(iv: Optional[bytes]) -> bytes: """ Returns a suitable initialiation vector. If `iv` is `None`, a default is returned. If `iv` is not a suitable initialization diff --git a/src/allmydata/mutable/common.py b/src/allmydata/mutable/common.py index a498ab02a..d663638e7 100644 --- a/src/allmydata/mutable/common.py +++ b/src/allmydata/mutable/common.py @@ -74,7 +74,7 @@ def encrypt_privkey(writekey: bytes, privkey: bytes) -> bytes: crypttext = aes.encrypt_data(encryptor, privkey) return crypttext -def decrypt_privkey(writekey: bytes, enc_privkey: bytes) -> rsa.PrivateKey: +def decrypt_privkey(writekey: bytes, enc_privkey: bytes) -> bytes: """ The inverse of ``encrypt_privkey``. """ From 94c15d851055affd7a3ea21d67ca1fb5b01654e7 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 14 Aug 2024 10:28:56 -0400 Subject: [PATCH 2309/2309] No longer limited to cryptography. --- setup.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/setup.py b/setup.py index e2c3ce6a6..71be1e2e1 100644 --- a/setup.py +++ b/setup.py @@ -62,8 +62,7 @@ install_requires = [ # Twisted[conch] also depends on cryptography and Twisted[tls] # transitively depends on cryptography. So it's anyone's guess what # version of cryptography will *really* be installed. - # * cryptography 43.0.0 makes __provides__ read-only; see ticket 4300 - "cryptography >= 2.6, < 43.0.0", + "cryptography >= 2.6", # * Used for custom HTTPS validation "pyOpenSSL >= 23.2.0",